Various updates

Adventech · Jun 29, 2024 · a370e1d · a370e1d
1 parent 509e58b
commit a370e1d
Show file tree

Hide file tree

Showing 40 changed files with 582 additions and 44 deletions.
diff --git a/ops/deploy/deploy-documents.js b/ops/deploy/deploy-documents.js
@@ -16,7 +16,7 @@ import {
     SEGMENT_TYPES,
     SECTION_DEFAULT_NAME,
     FIREBASE_DATABASE_BLOCKS,
-    FIREBASE_DATABASE_SEGMENTS
+    FIREBASE_DATABASE_SEGMENTS, FIREBASE_DATABASE_DOCUMENTS
 } from "../helpers/constants.js"
 import { SEGMENT_DEFAULT_BLOCK_STYLES } from "../helpers/styles.js"
 
@@ -43,13 +43,21 @@ let getSegmentInfo = async function (segment, processBlocks = false) {
         ...segmentInfoFrontMatter.attributes,
     }
 
-    if (!segmentInfo.type) {
-        if (processBlocks) {
-            segmentInfo.blocks = await parseSegment(segmentInfoFrontMatter.body, segmentPathInfo, "root", 1)
+    let segmentFilterForType = {
+        "block": (b) => b.type !== "space",
+        "story": (b) => {
+            return b.type === "story"
         }
+    }
+
+    if (!segmentInfo.type) {
         segmentInfo.type = SEGMENT_TYPES.BLOCK
     }
 
+    if (processBlocks) {
+        segmentInfo.blocks = await parseSegment(segmentInfoFrontMatter.body, segmentPathInfo, "root", 1, segmentFilterForType[segmentInfo.type])
+    }
+
     segmentInfo.id = `${segmentPathInfo.language}-${segmentPathInfo.type}-${segmentPathInfo.title}-${DOCUMENT_CONTENT_DIRNAME}-${segmentPathInfo.section || SECTION_DEFAULT_NAME}-${segmentPathInfo.document}-segments-${segmentPathInfo.segment}`
     segmentInfo.resourceId = `${segmentPathInfo.language}-${segmentPathInfo.type}-${segmentPathInfo.title}`
     segmentInfo.index = `${segmentPathInfo.language}/${segmentPathInfo.type}/${segmentPathInfo.title}/${DOCUMENT_CONTENT_DIRNAME}/${segmentPathInfo.section || SECTION_DEFAULT_NAME}/${segmentPathInfo.document}/segments/${segmentPathInfo.segment}`
@@ -110,13 +118,15 @@ let processDocuments = async function (resourceType) {
             fs.outputFileSync(`${API_DIST}/${segmentPathInfo.language}/${resourceType}/${segmentPathInfo.title}/${DOCUMENT_CONTENT_DIRNAME}/${segmentPathInfo.section ? `${segmentPathInfo.section}/` : "root/"}${segmentPathInfo.document}/segments/${segmentPathInfo.segment}/index.json`, JSON.stringify(segmentInfo))
         }
 
+        await database.collection(FIREBASE_DATABASE_DOCUMENTS).doc(documentInfo.id).set(documentInfo);
         fs.outputFileSync(`${API_DIST}/${documentPathInfo.language}/${resourceType}/${documentPathInfo.title}/${DOCUMENT_CONTENT_DIRNAME}/${documentPathInfo.section ? `${documentPathInfo.section}/` : "root/"}${documentPathInfo.document}/index.json`, JSON.stringify(documentInfo))
     }
 }
 
 if (isMainModule(import.meta)) {
     await processDocuments(RESOURCE_TYPE.DEVO)
     await processDocuments(RESOURCE_TYPE.PM)
+    await processDocuments(RESOURCE_TYPE.AIJ)
 }
 
 export {

diff --git a/ops/deploy/deploy-resources.js b/ops/deploy/deploy-resources.js
@@ -73,10 +73,10 @@ let getResourceInfo = async function (resource, depth = 0) {
     // TODO: make seamless for local testing
     if (!resourceInfo.covers) {
         resourceInfo.covers = {
-            landscape: `http://localhost:3002/api/v2/en/devo/${resourceInfo.name}/assets/cover-landscape.png`,
-            square: `http://localhost:3002/api/v2/en/devo/${resourceInfo.name}/assets/cover-square.png`,
-            portrait: `http://localhost:3002/api/v2/en/devo/${resourceInfo.name}/assets/cover.png`,
-            splash: `http://localhost:3002/api/v2/en/devo/${resourceInfo.name}/assets/splash.png`,
+            landscape: `http://localhost:3002/api/v2/en/${resourceInfo.type}/${resourceInfo.name}/assets/cover-landscape.png`,
+            square: `http://localhost:3002/api/v2/en/${resourceInfo.type}/${resourceInfo.name}/assets/cover-square.png`,
+            portrait: `http://localhost:3002/api/v2/en/${resourceInfo.type}/${resourceInfo.name}/assets/cover.png`,
+            splash: `http://localhost:3002/api/v2/en/${resourceInfo.type}/${resourceInfo.name}/assets/splash.png`,
         }
     }
 
@@ -193,7 +193,7 @@ let processResources = async function (resourceType) {
     const languages = await getLanguages()
 
     for (let language of languages) {
-        const resources = new fdir()
+        let resources = new fdir()
             .withBasePath()
             .withRelativePaths()
             .withMaxDepth(3)
@@ -308,6 +308,7 @@ let processResources = async function (resourceType) {
 if (isMainModule(import.meta)) {
     await processResources(RESOURCE_TYPE.DEVO)
     await processResources(RESOURCE_TYPE.PM)
+    await processResources(RESOURCE_TYPE.AIJ)
 }
 
 export {

diff --git a/ops/deploy/deploy-sections.js b/ops/deploy/deploy-sections.js
@@ -115,6 +115,7 @@ let processSections = async function (resourceType) {
 if (isMainModule(import.meta)) {
     await processSections(RESOURCE_TYPE.DEVO)
     await processSections(RESOURCE_TYPE.PM)
+    await processSections(RESOURCE_TYPE.AIJ)
 }
 
 export {

diff --git a/ops/helpers/bible.js b/ops/helpers/bible.js
@@ -5,6 +5,7 @@ import { getLanguageInfoSync } from "../deploy/deploy-languages.js"
 let languageInfoGlobal = {}
 
 export const getBibleData = function (resourcePath, text) {
+    let outputText
     let bibleData = []
     let bibleVersionsArray = []
     if (languageInfoGlobal[resourcePath.language]) {
@@ -28,8 +29,9 @@ export const getBibleData = function (resourcePath, text) {
 
         try {
             bibleSearchResult = bibleSearchBCV.search(resourcePath.language, bibleVersionName, text, true)
-            if (!bibleData.length) {
-                text = bibleSearchResult.output
+
+            if (!outputText) {
+                outputText = bibleSearchResult.output
             }
         } catch (e) {
             bibleSearchResult = null
@@ -54,6 +56,6 @@ export const getBibleData = function (resourcePath, text) {
         }
     }
 
-    return {text, bibleData}
+    return {text: outputText ?? text, bibleData}
 }
 
diff --git a/ops/helpers/blocks.js b/ops/helpers/blocks.js
@@ -1,9 +1,11 @@
 import crypto from "crypto"
 import { marked } from "marked"
-import { carousel, slide, image, collapse, audio, video, reference, question, blockquote, hr, heading, list, paragraph, poll, style, superscript, excerpt } from "./blocks/index.js"
+import { story, storySlide, carousel, slide, image, collapse, audio, video, reference, question, blockquote, hr, heading, list, paragraph, poll, style, superscript, excerpt } from "./blocks/index.js"
 
 marked.use({
     extensions: [
+        story.extension,
+        storySlide.extension,
         carousel.extension,
         slide.extension,
         question.extension,
@@ -23,13 +25,15 @@ let parseBlock = async function (block, resourcePath, index, parentId, depth) {
 
     block = superscript(blockStyleReturn.block)
 
-    let documentIndex = `${resourcePath.language}/${resourcePath.type}/${resourcePath.name}/content/${resourcePath.section ? resourcePath.section + "/" : ""}${resourcePath.document}`
+    let documentIndex = `${resourcePath.language}/${resourcePath.type}/${resourcePath.name}/content/${resourcePath.section ? resourcePath.section + "/" : ""}${resourcePath.document}/${resourcePath.segment}`
 
     block.id = crypto.createHash("sha256").update(
         `${documentIndex}-${parentId}-${block.type}-${index}`
     ).digest("hex")
 
     const supportedBlockTypes = {
+        story,
+        storySlide,
         blockquote,
         audio,
         video,

diff --git a/ops/helpers/blocks/image.js b/ops/helpers/blocks/image.js
@@ -1,5 +1,5 @@
 import fs from "fs-extra"
-import { RESOURCE_ASSETS_DIRNAME, SOURCE_DIR } from "../constants.js"
+import { RESOURCE_ASSETS_DIRNAME, RESOURCE_CONTENT_DIRNAME, SECTION_DEFAULT_NAME, SOURCE_DIR } from "../constants.js"
 
 let imageExists = function (src) {
     return fs.pathExistsSync(src)
@@ -28,10 +28,11 @@ export const image = {
         }
     },
     process: async function (block, resourcePath) {
-        const imagePath = `${SOURCE_DIR}/${resourcePath.language}/${resourcePath.type}/${resourcePath.title}/${RESOURCE_ASSETS_DIRNAME}/${block.src}`
+        const imagePathAssets = `${SOURCE_DIR}/${resourcePath.language}/${resourcePath.type}/${resourcePath.title}/${RESOURCE_ASSETS_DIRNAME}/${block.src}`
+        const imagePathDocument = `${SOURCE_DIR}/${resourcePath.language}/${resourcePath.type}/${resourcePath.title}/${RESOURCE_CONTENT_DIRNAME}/${resourcePath.section === SECTION_DEFAULT_NAME ? "" : "/"+resourcePath.section}${resourcePath.document}/${block.src}`
 
         if (!/^http/.test(block.src.trim())) {
-            if (!imageExists(imagePath)) {
+            if (!imageExists(imagePathAssets) && !imageExists(imagePathDocument)) {
                 return null
             }
         }

diff --git a/ops/helpers/blocks/index.js b/ops/helpers/blocks/index.js
@@ -1,6 +1,7 @@
 import { carousel, slide } from "./carousel.js"
 import { image } from "./image.js"
 import { collapse } from "./collapse.js"
+import { story, storySlide } from "./story.js"
 import { audio } from "./audio.js"
 import { video } from "./video.js"
 import { reference } from "./reference.js"
@@ -17,6 +18,8 @@ import { style } from "./style.js"
 import { superscript } from "./superscript.js"
 
 export {
+    story,
+    storySlide,
     carousel,
     slide,
     image,

diff --git a/ops/helpers/blocks/paragraph.js b/ops/helpers/blocks/paragraph.js
@@ -12,34 +12,37 @@ export const paragraph = {
 
         let r =  { id: block.id, type: block.type, markdown: text }
 
+        let documentIndex = `${resourcePath.language}/${resourcePath.type}/${resourcePath.name}/content/${resourcePath.section ? resourcePath.section + "/" : ""}${resourcePath.document}/${resourcePath.segment}`
         if (bibleData && bibleData.bibleData.length) {
             r["markdown"] = bibleData.text
-            let documentIndex = `${resourcePath.language}/${resourcePath.type}/${resourcePath.name}/content/${resourcePath.section ? resourcePath.section + "/" : ""}${resourcePath.document}`
 
-            r.data = {
-                bible: {
-                    id: crypto.createHash("sha256").update(`${documentIndex}-${r.id}-excerpt-0`).digest("hex"),
+            r.data = { bible: {} }
+            let index = 0
+
+            let detectedVerses = [...new Set(bibleData.bibleData.flatMap(item => Object.keys(item.verses)))]
+
+            for (let verse of detectedVerses) {
+                let excerpt = {
+                    id: crypto.createHash("sha256").update(`${documentIndex}-${r.id}-excerpt-${index}`).digest("hex"),
                     type: "excerpt",
+                    items: [],
+                    options: [...new Set(bibleData.bibleData.map(obj => obj.name))]
                 }
-            }
-            r.data.bible.options = [...new Set(bibleData.bibleData.map(obj => obj.name))];
-            r.data.bible.items = []
-
-            for (const [index, passageArray] of bibleData.bibleData.entries()) {
-                let item = {
-                    id: crypto.createHash("sha256").update(`${documentIndex}-${r.data.bible.id}-${block.type}-${index}`).digest("hex"),
-                    option: passageArray.name,
-                    type: "excerpt-item"
-                }
-                let markdown = ""
-                for (let verse of Object.keys(passageArray.verses)) {
-                    markdown += passageArray.verses[verse] + "\n\n"
-                }
-                markdown = markdown.replace(/\n\n$/mg, '')
 
-                item.items = await parseSegment(markdown, resourcePath, item.id)
+                for (let passageArray of bibleData.bibleData) {
+                    let item = {
+                        id: crypto.createHash("sha256").update(`${documentIndex}-${r.id}-excerpt-${index}-${r.type}-${index}`).digest("hex"),
+                        option: passageArray.name,
+                        type: "excerpt-item"
+                    }
+                    let markdown = passageArray.verses[verse]
+
+                    item.items = await parseSegment(markdown, resourcePath, item.id)
+                    excerpt.items.push(item)
+                    index++
 
-                r.data.bible.items.push(item)
+                }
+                r.data.bible[verse] = excerpt
             }
         }
 
@@ -48,7 +51,31 @@ export const paragraph = {
         if (Object.keys(egwData.data.egw).length) {
             if (!r["data"]) { r["data"] = {}}
 
-            r["data"]["egw"] = egwData.data.egw
+            r["data"]["egw"] = {}
+
+            let index = 0
+            for (let egwReferenceKey of Object.keys(egwData.data.egw)) {
+                r.data.egw[egwReferenceKey] = [{
+                    id: crypto.createHash("sha256").update(`${documentIndex}-${r.id}-egw-${index}-${r.type}-${index}-${egwReferenceKey}-reference`).digest("hex"),
+                    type: "heading",
+                    markdown: egwData.data.reference[egwReferenceKey],
+                    depth: 3,
+                }]
+
+                let paragraphs = egwData.data.egw[egwReferenceKey].split("\n\n")
+
+                for (const [indexParagraph, paragraph] of paragraphs.entries()) {
+                    let paragraphBlock = {
+                        id: crypto.createHash("sha256").update(`${documentIndex}-${r.id}-egw-${index}-${r.type}-${index}-${indexParagraph}`).digest("hex"),
+                        type: "paragraph",
+                        markdown: paragraph
+                    }
+                    r.data.egw[egwReferenceKey].push(paragraphBlock)
+                }
+
+                index++
+            }
+
             r["markdown"] = egwData.output
         }
 

diff --git a/ops/helpers/blocks/question.js b/ops/helpers/blocks/question.js
@@ -1,3 +1,5 @@
+import { parseSegment } from "../blocks.js"
+
 export const question = {
     extension: {
         name: "question",
@@ -20,6 +22,10 @@ export const question = {
         }
     },
     process: async function (block, resourcePath) {
-        return { id: block.id, type: block.type, markdown: `**${block.text}**` }
+        let questionText = await parseSegment(block.text, resourcePath, block.id)
+        if (!questionText.length) {
+            questionText = [{markdown: block.text}]
+        }
+        return { id: block.id, type: block.type, markdown: `**${questionText[0].markdown}**`, data: questionText[0].data }
     },
 }
diff --git a/ops/helpers/blocks/story.js b/ops/helpers/blocks/story.js
@@ -0,0 +1,62 @@
+import { parseSegment } from "../blocks.js"
+
+export const story = {
+    extension: {
+        name: "story",
+        level: "block",
+        tokenizer(src, tokens) {
+            const rule = /^;{3}\n([\s\S]*?)\n;{3}/
+            const match = rule.exec(src);
+            if (match) {
+                return {
+                    type: "story",
+                    raw: match[0],
+                    text: match[0].replace(/(^;{3}|;{3})/g, "").trim()
+                }
+            }
+        },
+
+        renderer(token) {
+            // TODO: for web rendering
+            return `TODO: collapse`;
+        }
+    },
+    process: async function (block, resourcePath) {
+        return { id: block.id, type: block.type, items: await parseSegment(block.text, resourcePath, block.id)}
+    },
+}
+export const storySlide = {
+    extension: {
+        name: "storySlide",
+        level: "block",
+        tokenizer(src, tokens) {
+            const rule = /^\^{3}\n([\s\S]*?)\n\^{3}/
+            const match = rule.exec(src);
+            if (match) {
+                return {
+                    type: "storySlide",
+                    raw: match[0],
+                    text: match[0].replace(/(^\^{3}|\^{3}$)/g, "").trim()
+                }
+            }
+        },
+
+        renderer(token) {
+            // TODO: for web rendering
+            return `TODO: collapse`;
+        }
+    },
+    process: async function (block, resourcePath) {
+        let items = await parseSegment(block.text, resourcePath, block.id)
+
+        const image = items.find(b => b.type === "image")
+        const paragraph = items.find(b => b.type === "paragraph")
+
+        if (!image || !paragraph) { return null }
+
+        const  alignment = image?.style?.image?.storyTextAlign ?? "bottom"
+
+        return { id: block.id, type: block.type, image: image.src, markdown: paragraph.markdown, alignment: alignment, ...(paragraph.style && { style: paragraph.style}) }
+    },
+
+}
diff --git a/ops/helpers/blocks/style.js b/ops/helpers/blocks/style.js
@@ -81,6 +81,7 @@ const styleSchema = {
             "properties": {
                 "aspectRatio": { "type": "number" },
                 "expandable": { "type": "boolean" },
+                "storyTextAlign": { "type": "string", "enum": ["top", "bottom"] },
             }
         },
 

diff --git a/ops/helpers/constants.js b/ops/helpers/constants.js
@@ -21,6 +21,7 @@ export const LANGUAGE_INFO_FILENAME = "info.yml"
 export const RESOURCE_TYPE = {
     DEVO: "devo",
     PM: "pm",
+    AIJ: "aij",
 }
 export const RESOURCE_COLOR_PRIMARY = "#d8d8d8"
 export const RESOURCE_COLOR_PRIMARY_DARK = "#949494"

diff --git a/ops/sync/transfer-resource-assets.js b/ops/sync/transfer-resource-assets.js
@@ -21,7 +21,7 @@ import {
     RESOURCE_ASSETS_DIRNAME, DIST_DIR, API_DIST,
 } from "../helpers/constants.js"
 
-let mode = "remote"
+let mode = "local"
 
 if (process && process.env && process.env.GITHUB_TOKEN) {
     mode = "remote"
@@ -194,6 +194,7 @@ let transferResourcesFonts = async function () {
                         let weight = await determineFontWeight(postScriptName)
                         if (weight) {
                             // weird bug of the ttfInfo library, replacing \x00 with nothing
+                            console.log(postScriptName)
                             resourceInfo.fonts.push({
                                 name: postScriptName,
                                 weight: parseInt(weight),