From ba1e0d43ed932debea73cb82b207102b3f3876fc Mon Sep 17 00:00:00 2001 From: Manav Aggarwal Date: Sat, 15 Jul 2023 21:17:44 +0530 Subject: [PATCH 1/4] - extract metadata from IMS cp - Create topic and resouce node for the metadata --- .../channelEdit/components/edit/EditModal.vue | 801 ++++++++++-------- .../views/files/ContentRenderer.vue | 6 +- .../frontend/shared/vuex/file/utils.js | 134 ++- 3 files changed, 579 insertions(+), 362 deletions(-) diff --git a/contentcuration/contentcuration/frontend/channelEdit/components/edit/EditModal.vue b/contentcuration/contentcuration/frontend/channelEdit/components/edit/EditModal.vue index 995072d190..6562538e32 100644 --- a/contentcuration/contentcuration/frontend/channelEdit/components/edit/EditModal.vue +++ b/contentcuration/contentcuration/frontend/channelEdit/components/edit/EditModal.vue @@ -177,331 +177,351 @@ diff --git a/contentcuration/contentcuration/frontend/channelEdit/views/files/ContentRenderer.vue b/contentcuration/contentcuration/frontend/channelEdit/views/files/ContentRenderer.vue index 58faf53ff0..f086de146d 100644 --- a/contentcuration/contentcuration/frontend/channelEdit/views/files/ContentRenderer.vue +++ b/contentcuration/contentcuration/frontend/channelEdit/views/files/ContentRenderer.vue @@ -99,6 +99,10 @@ }, computed: { ...mapGetters('file', ['getContentNodeFileById', 'getContentNodeFiles']), + ...mapGetters('contentNode', ['getContentNode', 'getContentNodeChildren']), + node() { + return this.getContentNode(this.nodeId); + }, file() { return this.getContentNodeFileById(this.nodeId, this.fileId); }, @@ -129,7 +133,7 @@ return this.file.file_format === 'epub'; }, htmlPath() { - return `/zipcontent/${this.file.checksum}.${this.file.file_format}`; + return `/zipcontent/${this.file.checksum}.${this.file.file_format}/${this.node.options && this.node.options.entry || ''}`; }, src() { return this.file && this.file.url; diff --git a/contentcuration/contentcuration/frontend/shared/vuex/file/utils.js b/contentcuration/contentcuration/frontend/shared/vuex/file/utils.js index 12e59b943c..69a67abfbb 100644 --- a/contentcuration/contentcuration/frontend/shared/vuex/file/utils.js +++ b/contentcuration/contentcuration/frontend/shared/vuex/file/utils.js @@ -3,6 +3,7 @@ import JSZip from 'jszip'; import { FormatPresetsList, FormatPresetsNames } from 'shared/leUtils/FormatPresets'; import { LicensesList } from 'shared/leUtils/Licenses'; import LanguagesMap from 'shared/leUtils/Languages'; +import JSZip from 'jszip'; const BLOB_SLICE = File.prototype.slice || File.prototype.mozSlice || File.prototype.webkitSlice; const CHUNK_SIZE = 2097152; @@ -10,10 +11,12 @@ const MEDIA_PRESETS = [ FormatPresetsNames.AUDIO, FormatPresetsNames.HIGH_RES_VIDEO, FormatPresetsNames.LOW_RES_VIDEO, - FormatPresetsNames.H5P, + FormatPresetsNames.QTI, + FormatPresetsNames.HTML5_DEPENDENCY, + FormatPresetsNames.HTML5_ZIP ]; const VIDEO_PRESETS = [FormatPresetsNames.HIGH_RES_VIDEO, FormatPresetsNames.LOW_RES_VIDEO]; -const H5P_PRESETS = [FormatPresetsNames.H5P]; +const IMS_PRESETS = [FormatPresetsNames.QTI, FormatPresetsNames.HTML5_DEPENDENCY, FormatPresetsNames.HTML5_ZIP] export function getHash(file) { return new Promise((resolve, reject) => { @@ -44,6 +47,128 @@ export function getHash(file) { }); } +const getLeafNodes = (node) => { + if (!node.children || node.children.length === 0) { + return [node]; + } + + // Recursive case: Traverse all children and collect leaf nodes + let leafNodes = []; + for (let child of node.children) { + const childLeafNodes = getLeafNodes(child); + leafNodes = leafNodes.concat(childLeafNodes); + } + + return leafNodes; +} + +export async function extractIMSMetadata(fileInput, metadata){ + const zip = new JSZip(); + zip + .loadAsync(fileInput) + .then(function(zip) { + const manifestFile = zip.file("imsmanifest.xml"); + const metadataFile = zip.file("imsmetadata.xml"); + if (!manifestFile) { + reject(new Error("imsmanifest.xml not found in the zip file.")); + return; + } else if(manifestFile && metadataFile){ + manifestFile + .async("text") + .then(content => { + // Parse the XML content + const parser = new DOMParser(); + const xmlDoc = parser.parseFromString(content, "application/xml"); + const data = xmlDoc.getElementsByTagName('organizations') + for( let i = 0 ; i < data.length; i++){ + let orgs = []; + let index = 0; + for( let j = 0 ; j < data[i].childNodes.length; j++){ + let org = [] + if(data[i].childNodes[j].nodeType === 1){ + const orgNode = data[i].childNodes[j]; + const title = orgNode.getElementsByTagName('title') + org.title = title[0].textContent + const items = orgNode.getElementsByTagName('item'); + for(let k = 0; k < items.length; k++){ + const file = {}; + const item = items[k]; + file.title = title[1+k].textContent; + file.identifierref = item.getAttribute('identifierref'); + file.resourceHref = xmlDoc.querySelectorAll(`[identifier=${file.identifierref}]`)[0].getAttribute('href') + org[`file${index}`] = file; + index++; + } + orgs[`org${i}`] = org + } + } + metadata.orgs = orgs + } + }) + metadataFile + .async("text") + .then(content=> { + const parser = new DOMParser(); + const xmlDoc = parser.parseFromString(content, "application/xml"); + metadata.title = xmlDoc.getElementsByTagName('lomes:title').length === 0? undefined : xmlDoc.getElementsByTagName('lomes:title')[0].children[0].textContent; + metadata.language = xmlDoc.getElementsByTagName("lomes:idiom").length === 0 ? undefined : xmlDoc.getElementsByTagName("lomes:idiom")[0].children[0].textContent; + metadata.description = xmlDoc.getElementsByTagName("lomes:description").length === 0 ? undefined: xmlDoc.getElementsByTagName("lomes:description")[0].children[0].textContent; + }) + } else { + manifestFile + .async("text") + .then(content => { + // Parse the XML content + const parser = new DOMParser(); + const xmlDoc = parser.parseFromString(content, "application/xml"); + metadata.title = xmlDoc.getElementsByTagName('title').length === 0? undefined : xmlDoc.getElementsByTagName('title')[0].textContent; + metadata.language = xmlDoc.getElementsByTagName("imsmd:language").length === 0 ? undefined : xmlDoc.getElementsByTagName("imsmd:language")[0].textContent; + metadata.description = xmlDoc.getElementsByTagName("imsmd:description").length === 0 ? undefined: xmlDoc.getElementsByTagName("imsmd:description")[0].textContent; + const data = xmlDoc.getElementsByTagName('organizations') + for( let i = 0 ; i < data.length; i++){ + let orgs = []; + let index = 0; + for( let j = 0 ; j < data[i].childNodes.length; j++){ + let org = [] + if(data[i].childNodes[j].nodeType === 1){ + const orgNode = data[i].childNodes[j]; + const title = orgNode.getElementsByTagName('title') + org.title = title[0].textContent + const items = orgNode.getElementsByTagName('item'); + for(let k = 0; k < items.length; k++){ + const file = {}; + const item = items[k]; + file.title = title[1+k].textContent; + file.identifierref = item.getAttribute('identifierref'); + file.resourceHref = xmlDoc.querySelectorAll(`[identifier=${file.identifierref}]`)[0].getAttribute('href') + const metadataNodes = orgNode.getElementsByTagName('metadata'); + if(metadataNodes && metadataNodes.length != 0){ + for( let l = 0 ; l < metadataNodes.length ; l++){ + let nodeValue = getLeafNodes(metadataNodes[l]); + file[`${nodeValue[0].nodeName}`] = nodeValue[0].textContent + } + } + org[`file${index}`] = file; + index++; + } + orgs[`org${i}`] = org + } + } + metadata.orgs = orgs + } + return metadata + }) + .catch(error => { + reject(error); + }); + } + }) + .catch(error => { + reject(error); // Failed to load the zip file + }); + return metadata +} + const extensionPresetMap = FormatPresetsList.reduce((map, value) => { if (value.display) { value.allowed_formats.forEach(format => { @@ -147,7 +272,7 @@ export function extractMetadata(file, preset = null) { } const isH5P = H5P_PRESETS.includes(metadata.preset); - + const isIMSCP = IMS_PRESETS.includes(metadata.preset); // Extract additional media metadata const isVideo = VIDEO_PRESETS.includes(metadata.preset); @@ -157,6 +282,9 @@ export function extractMetadata(file, preset = null) { Object.assign(metadata, data); }); resolve(metadata); + } else if (isIMSCP) { + extractIMSMetadata(file, metadata) + resolve(metadata); } else { const mediaElement = document.createElement(isVideo ? 'video' : 'audio'); // Add a listener to read the metadata once it has loaded. From 90102de2b48288cdee6604b077d7c591a1d58a16 Mon Sep 17 00:00:00 2001 From: Manav Aggarwal Date: Tue, 22 Aug 2023 03:06:09 +0530 Subject: [PATCH 2/4] - Enhance IMS Content Package Import UI --- .../channelEdit/components/edit/EditList.vue | 53 +- .../components/edit/EditListItem.vue | 7 +- .../components/edit/EditListItems.vue | 92 ++ .../channelEdit/components/edit/EditModal.vue | 843 +++++++++--------- .../views/files/ContentRenderer.vue | 6 +- .../frontend/shared/vuex/file/utils.js | 197 ++-- 6 files changed, 641 insertions(+), 557 deletions(-) create mode 100644 contentcuration/contentcuration/frontend/channelEdit/components/edit/EditListItems.vue diff --git a/contentcuration/contentcuration/frontend/channelEdit/components/edit/EditList.vue b/contentcuration/contentcuration/frontend/channelEdit/components/edit/EditList.vue index dc55fe2485..9648d6b7e8 100644 --- a/contentcuration/contentcuration/frontend/channelEdit/components/edit/EditList.vue +++ b/contentcuration/contentcuration/frontend/channelEdit/components/edit/EditList.vue @@ -7,11 +7,11 @@ class="mb-2 ml-1 mt-0 px-3 py-2" :label="$tr('selectAllLabel')" /> - @@ -22,13 +22,14 @@ + + diff --git a/contentcuration/contentcuration/frontend/channelEdit/components/edit/EditModal.vue b/contentcuration/contentcuration/frontend/channelEdit/components/edit/EditModal.vue index 6562538e32..ad48b52820 100644 --- a/contentcuration/contentcuration/frontend/channelEdit/components/edit/EditModal.vue +++ b/contentcuration/contentcuration/frontend/channelEdit/components/edit/EditModal.vue @@ -71,7 +71,8 @@ @@ -177,390 +178,358 @@ + \ No newline at end of file diff --git a/contentcuration/contentcuration/frontend/channelEdit/views/files/ContentRenderer.vue b/contentcuration/contentcuration/frontend/channelEdit/views/files/ContentRenderer.vue index f086de146d..56b6c5c703 100644 --- a/contentcuration/contentcuration/frontend/channelEdit/views/files/ContentRenderer.vue +++ b/contentcuration/contentcuration/frontend/channelEdit/views/files/ContentRenderer.vue @@ -99,7 +99,7 @@ }, computed: { ...mapGetters('file', ['getContentNodeFileById', 'getContentNodeFiles']), - ...mapGetters('contentNode', ['getContentNode', 'getContentNodeChildren']), + ...mapGetters('contentNode', ['getContentNode']), node() { return this.getContentNode(this.nodeId); }, @@ -133,7 +133,9 @@ return this.file.file_format === 'epub'; }, htmlPath() { - return `/zipcontent/${this.file.checksum}.${this.file.file_format}/${this.node.options && this.node.options.entry || ''}`; + return `/zipcontent/${this.file.checksum}.${this.file.file_format}/${(this.node.options && + this.node.options.entry) || + ''}`; }, src() { return this.file && this.file.url; diff --git a/contentcuration/contentcuration/frontend/shared/vuex/file/utils.js b/contentcuration/contentcuration/frontend/shared/vuex/file/utils.js index 69a67abfbb..1049d8805f 100644 --- a/contentcuration/contentcuration/frontend/shared/vuex/file/utils.js +++ b/contentcuration/contentcuration/frontend/shared/vuex/file/utils.js @@ -3,7 +3,6 @@ import JSZip from 'jszip'; import { FormatPresetsList, FormatPresetsNames } from 'shared/leUtils/FormatPresets'; import { LicensesList } from 'shared/leUtils/Licenses'; import LanguagesMap from 'shared/leUtils/Languages'; -import JSZip from 'jszip'; const BLOB_SLICE = File.prototype.slice || File.prototype.mozSlice || File.prototype.webkitSlice; const CHUNK_SIZE = 2097152; @@ -13,10 +12,15 @@ const MEDIA_PRESETS = [ FormatPresetsNames.LOW_RES_VIDEO, FormatPresetsNames.QTI, FormatPresetsNames.HTML5_DEPENDENCY, - FormatPresetsNames.HTML5_ZIP + FormatPresetsNames.HTML5_ZIP, ]; const VIDEO_PRESETS = [FormatPresetsNames.HIGH_RES_VIDEO, FormatPresetsNames.LOW_RES_VIDEO]; -const IMS_PRESETS = [FormatPresetsNames.QTI, FormatPresetsNames.HTML5_DEPENDENCY, FormatPresetsNames.HTML5_ZIP] +const H5P_PRESETS = [FormatPresetsNames.H5P]; +const IMS_PRESETS = [ + FormatPresetsNames.QTI, + FormatPresetsNames.HTML5_DEPENDENCY, + FormatPresetsNames.HTML5_ZIP, +]; export function getHash(file) { return new Promise((resolve, reject) => { @@ -47,126 +51,109 @@ export function getHash(file) { }); } -const getLeafNodes = (node) => { +const getLeafNodes = node => { if (!node.children || node.children.length === 0) { return [node]; } - + // Recursive case: Traverse all children and collect leaf nodes let leafNodes = []; - for (let child of node.children) { + for (const child of node.children) { const childLeafNodes = getLeafNodes(child); leafNodes = leafNodes.concat(childLeafNodes); } - + return leafNodes; +}; + +function extractManifestFile(data, xmlDoc) { + const orgs = {}; + for (let i = 0; i < data.length; i++) { + let index = 0; + for (let j = 0; j < data[i].childNodes.length; j++) { + const org = {}; + if (data[i].childNodes[j].nodeType === 1) { + const orgNode = data[i].childNodes[j]; + const title = orgNode.getElementsByTagName('title'); + org.title = title[0].textContent; + const items = orgNode.getElementsByTagName('item'); + for (let k = 0; k < items.length; k++) { + const file = {}; + const item = items[k]; + file.title = title[1 + k].textContent; + file.identifierref = item.getAttribute('identifierref'); + file.resourceHref = xmlDoc + .querySelectorAll(`[identifier=${file.identifierref}]`)[0] + .getAttribute('href'); + const metadataNodes = orgNode.getElementsByTagName('metadata'); + if (metadataNodes && metadataNodes.length != 0) { + for (let l = 0; l < metadataNodes.length; l++) { + const nodeValue = getLeafNodes(metadataNodes[l]); + file[`${nodeValue[0].nodeName}`] = nodeValue[0].textContent; + } + } + org[`file${index}`] = file; + index++; + } + orgs[`org${i}`] = org; + } + } + } + return orgs; } -export async function extractIMSMetadata(fileInput, metadata){ +export async function extractIMSMetadata(fileInput) { const zip = new JSZip(); - zip + const metadata = {}; + return zip .loadAsync(fileInput) .then(function(zip) { - const manifestFile = zip.file("imsmanifest.xml"); - const metadataFile = zip.file("imsmetadata.xml"); + const manifestFile = zip.file('imsmanifest.xml'); if (!manifestFile) { - reject(new Error("imsmanifest.xml not found in the zip file.")); - return; - } else if(manifestFile && metadataFile){ - manifestFile - .async("text") - .then(content => { - // Parse the XML content - const parser = new DOMParser(); - const xmlDoc = parser.parseFromString(content, "application/xml"); - const data = xmlDoc.getElementsByTagName('organizations') - for( let i = 0 ; i < data.length; i++){ - let orgs = []; - let index = 0; - for( let j = 0 ; j < data[i].childNodes.length; j++){ - let org = [] - if(data[i].childNodes[j].nodeType === 1){ - const orgNode = data[i].childNodes[j]; - const title = orgNode.getElementsByTagName('title') - org.title = title[0].textContent - const items = orgNode.getElementsByTagName('item'); - for(let k = 0; k < items.length; k++){ - const file = {}; - const item = items[k]; - file.title = title[1+k].textContent; - file.identifierref = item.getAttribute('identifierref'); - file.resourceHref = xmlDoc.querySelectorAll(`[identifier=${file.identifierref}]`)[0].getAttribute('href') - org[`file${index}`] = file; - index++; - } - orgs[`org${i}`] = org - } - } - metadata.orgs = orgs - } - }) - metadataFile - .async("text") - .then(content=> { - const parser = new DOMParser(); - const xmlDoc = parser.parseFromString(content, "application/xml"); - metadata.title = xmlDoc.getElementsByTagName('lomes:title').length === 0? undefined : xmlDoc.getElementsByTagName('lomes:title')[0].children[0].textContent; - metadata.language = xmlDoc.getElementsByTagName("lomes:idiom").length === 0 ? undefined : xmlDoc.getElementsByTagName("lomes:idiom")[0].children[0].textContent; - metadata.description = xmlDoc.getElementsByTagName("lomes:description").length === 0 ? undefined: xmlDoc.getElementsByTagName("lomes:description")[0].children[0].textContent; - }) + throw new Error('imsmanifest.xml not found in the zip file.'); } else { - manifestFile - .async("text") - .then(content => { - // Parse the XML content + return manifestFile.async('text'); + } + }) + .then(manifestFile => { + const parser = new DOMParser(); + const xmlDoc = parser.parseFromString(manifestFile, 'application/xml'); + const data = xmlDoc.getElementsByTagName('organizations'); + metadata.orgs = extractManifestFile(data, xmlDoc); + const metadataFile = zip.file('imsmetadata.xml'); + if (metadataFile) { + metadataFile.async('text').then(content => { const parser = new DOMParser(); - const xmlDoc = parser.parseFromString(content, "application/xml"); - metadata.title = xmlDoc.getElementsByTagName('title').length === 0? undefined : xmlDoc.getElementsByTagName('title')[0].textContent; - metadata.language = xmlDoc.getElementsByTagName("imsmd:language").length === 0 ? undefined : xmlDoc.getElementsByTagName("imsmd:language")[0].textContent; - metadata.description = xmlDoc.getElementsByTagName("imsmd:description").length === 0 ? undefined: xmlDoc.getElementsByTagName("imsmd:description")[0].textContent; - const data = xmlDoc.getElementsByTagName('organizations') - for( let i = 0 ; i < data.length; i++){ - let orgs = []; - let index = 0; - for( let j = 0 ; j < data[i].childNodes.length; j++){ - let org = [] - if(data[i].childNodes[j].nodeType === 1){ - const orgNode = data[i].childNodes[j]; - const title = orgNode.getElementsByTagName('title') - org.title = title[0].textContent - const items = orgNode.getElementsByTagName('item'); - for(let k = 0; k < items.length; k++){ - const file = {}; - const item = items[k]; - file.title = title[1+k].textContent; - file.identifierref = item.getAttribute('identifierref'); - file.resourceHref = xmlDoc.querySelectorAll(`[identifier=${file.identifierref}]`)[0].getAttribute('href') - const metadataNodes = orgNode.getElementsByTagName('metadata'); - if(metadataNodes && metadataNodes.length != 0){ - for( let l = 0 ; l < metadataNodes.length ; l++){ - let nodeValue = getLeafNodes(metadataNodes[l]); - file[`${nodeValue[0].nodeName}`] = nodeValue[0].textContent - } - } - org[`file${index}`] = file; - index++; - } - orgs[`org${i}`] = org - } - } - metadata.orgs = orgs - } - return metadata - }) - .catch(error => { - reject(error); + const xmlDoc = parser.parseFromString(content, 'application/xml'); + metadata.title = + xmlDoc.getElementsByTagName('lomes:title').length === 0 + ? undefined + : xmlDoc.getElementsByTagName('lomes:title')[0].children[0].textContent; + metadata.language = + xmlDoc.getElementsByTagName('lomes:idiom').length === 0 + ? undefined + : xmlDoc.getElementsByTagName('lomes:idiom')[0].children[0].textContent; + metadata.description = + xmlDoc.getElementsByTagName('lomes:description').length === 0 + ? undefined + : xmlDoc.getElementsByTagName('lomes:description')[0].children[0].textContent; }); + } else { + metadata.title = + xmlDoc.getElementsByTagName('title').length === 0 + ? undefined + : xmlDoc.getElementsByTagName('title')[0].textContent; + metadata.language = + xmlDoc.getElementsByTagName('imsmd:language').length === 0 + ? undefined + : xmlDoc.getElementsByTagName('imsmd:language')[0].textContent; + metadata.description = + xmlDoc.getElementsByTagName('imsmd:description').length === 0 + ? undefined + : xmlDoc.getElementsByTagName('imsmd:description')[0].textContent; } - }) - .catch(error => { - reject(error); // Failed to load the zip file + return metadata; }); - return metadata } const extensionPresetMap = FormatPresetsList.reduce((map, value) => { @@ -283,7 +270,9 @@ export function extractMetadata(file, preset = null) { }); resolve(metadata); } else if (isIMSCP) { - extractIMSMetadata(file, metadata) + extractIMSMetadata(file).then(data => { + Object.assign(metadata, data); + }); resolve(metadata); } else { const mediaElement = document.createElement(isVideo ? 'video' : 'audio'); From 606b300d6dec33aabb331ce7a4fffe12538ae258 Mon Sep 17 00:00:00 2001 From: Manav Aggarwal Date: Fri, 1 Sep 2023 01:13:01 +0530 Subject: [PATCH 3/4] - adding code to extract metadata from submanifest file - adding test cases for extractIMSMetadata --- .../channelEdit/components/edit/EditList.vue | 7 +- .../channelEdit/components/edit/EditModal.vue | 80 +++-- .../shared/vuex/file/__tests__/module.spec.js | 288 +++++++++++++++++- .../frontend/shared/vuex/file/utils.js | 224 ++++++++------ 4 files changed, 466 insertions(+), 133 deletions(-) diff --git a/contentcuration/contentcuration/frontend/channelEdit/components/edit/EditList.vue b/contentcuration/contentcuration/frontend/channelEdit/components/edit/EditList.vue index 9648d6b7e8..4d334b8d35 100644 --- a/contentcuration/contentcuration/frontend/channelEdit/components/edit/EditList.vue +++ b/contentcuration/contentcuration/frontend/channelEdit/components/edit/EditList.vue @@ -54,12 +54,7 @@ return this.value; }, set(items) { - if (this.selected.includes(items[0])) { - this.selected = []; - } else { - this.getChildren(items[0]).forEach(item => items.push(item)); - this.$emit('input', items); - } + this.$emit('input', items); }, }, selectAll: { diff --git a/contentcuration/contentcuration/frontend/channelEdit/components/edit/EditModal.vue b/contentcuration/contentcuration/frontend/channelEdit/components/edit/EditModal.vue index ad48b52820..b67bfdc41c 100644 --- a/contentcuration/contentcuration/frontend/channelEdit/components/edit/EditModal.vue +++ b/contentcuration/contentcuration/frontend/channelEdit/components/edit/EditModal.vue @@ -500,7 +500,7 @@ .slice(0, -1) .join('.'); } - if (file.metadata.orgs === undefined) { + if (file.metadata.folders === undefined) { this.createNode( FormatPresets.has(file.preset) && FormatPresets.get(file.preset).kind_id, { title, ...file.metadata } @@ -513,52 +513,50 @@ contentnode: newNodeId, }); }); - } else if (file.metadata.orgs) { + } else if (file.metadata.folders) { this.createNode('topic', file.metadata).then(newNodeId => { - Object.values(file.metadata.orgs).forEach(org => { + file.metadata.folders.forEach(org => { this.createNode('topic', org, newNodeId).then(topicNodeId => { - Object.values(org).forEach(orgFile => { - if (typeof orgFile === 'object') { - const extra_fields = {}; - extra_fields['options'] = { entry: orgFile.resourceHref }; - extra_fields['title'] = orgFile.title; - let file_kind = null; - FormatPresetsList.forEach(p => { - if (p.id === file.metadata.preset) { - file_kind = p.kind_id; - } - }); + org.files.forEach(orgFile => { + const extra_fields = {}; + extra_fields['options'] = { entry: orgFile.resourceHref }; + extra_fields['title'] = orgFile.title; + let file_kind = null; + FormatPresetsList.forEach(p => { + if (p.id === file.metadata.preset) { + file_kind = p.kind_id; + } + }); - this.createNode(file_kind, extra_fields, topicNodeId).then(resourceNodeId => { - return File.uploadUrl({ - checksum: file.checksum, - size: file.file_size, - type: 'application/zip', - name: file.original_filename, - file_format: file.file_format, - preset: file.metadata.preset, - }).then(data => { - const fileObject = { - ...data.file, - loaded: 0, - total: file.size, - }; - if (index === 0) { - this.selected = [resourceNodeId]; + this.createNode(file_kind, extra_fields, topicNodeId).then(resourceNodeId => { + return File.uploadUrl({ + checksum: file.checksum, + size: file.file_size, + type: 'application/zip', + name: file.original_filename, + file_format: file.file_format, + preset: file.metadata.preset, + }).then(data => { + const fileObject = { + ...data.file, + loaded: 0, + total: file.size, + }; + if (index === 0) { + this.selected = [resourceNodeId]; + } + this.updateFile({ + ...fileObject, + contentnode: resourceNodeId, + }).catch(error => { + let errorType = fileErrors.UPLOAD_FAILED; + if (error.response && error.response.status === 412) { + errorType = fileErrors.NO_STORAGE; } - this.updateFile({ - ...fileObject, - contentnode: resourceNodeId, - }).catch(error => { - let errorType = fileErrors.UPLOAD_FAILED; - if (error.response && error.response.status === 412) { - errorType = fileErrors.NO_STORAGE; - } - return Promise.reject(errorType); - }); + return Promise.reject(errorType); }); }); - } + }); }); }); }); diff --git a/contentcuration/contentcuration/frontend/shared/vuex/file/__tests__/module.spec.js b/contentcuration/contentcuration/frontend/shared/vuex/file/__tests__/module.spec.js index 3f38ceabd8..b1da778cb5 100644 --- a/contentcuration/contentcuration/frontend/shared/vuex/file/__tests__/module.spec.js +++ b/contentcuration/contentcuration/frontend/shared/vuex/file/__tests__/module.spec.js @@ -1,5 +1,5 @@ import JSZip from 'jszip'; -import { getH5PMetadata } from '../utils'; +import { getH5PMetadata, extractIMSMetadata } from '../utils'; import storeFactory from 'shared/vuex/baseStore'; import { File, injectVuexStore } from 'shared/data/resources'; import client from 'shared/client'; @@ -237,5 +237,291 @@ describe('file store', () => { }); }); }); + describe('IMS content file extract metadata', () => { + it('extractIMSMetadata should check for imsmanifest.xml file', () => { + const zip = new JSZip(); + return zip.generateAsync({ type: 'blob' }).then(async function(IMSBlob) { + await expect(extractIMSMetadata(IMSBlob)).rejects.toThrow( + 'imsmanifest.xml not found in the zip file.' + ); + }); + }); + it('extractIMSMetadata should extract metadata from imsmanifest.xml', async () => { + // const manifestFile = get_imsmanifest_file({ title: 'Test file' }); + const manifestContent = ` + + + + + + Test File + + en + + Example of test file + + + + + + + Folder 1 + + Test File1 + + + Test File2 + + + Folder 1 + + Test File1 + + + Test File2 + + + + Folder 2 + + Test File3 + + + Test File4 + + + + + + + Folder 2 + + Test File3 + + + Test File4 + + + + + + + + + + + + + + `; + + const zip = new JSZip(); + zip.file('imsmanifest.xml', manifestContent); + // zip.folder('ims_package.zip'); + await zip.generateAsync({ type: 'blob' }).then(async function(imsBlob) { + await expect(extractIMSMetadata(imsBlob)).resolves.toEqual({ + title: 'Test File', + folders: [ + { + files: [ + { + identifierref: 'file1Ref', + resourceHref: 'file1.html', + title: 'Test File1', + }, + { + identifierref: 'file2Ref', + resourceHref: 'file2.html', + title: 'Test File2', + folders: [ + { + files: [ + { + identifierref: 'file1Ref', + resourceHref: 'file1.html', + title: 'Test File1', + }, + { + identifierref: 'file2Ref', + resourceHref: 'file2.html', + title: 'Test File2', + }, + ], + title: 'Folder 1', + }, + { + files: [ + { + identifierref: 'file3Ref', + resourceHref: 'file3.html', + title: 'Test File3', + }, + { + identifierref: 'file4Ref', + resourceHref: 'file4.html', + title: 'Test File4', + }, + ], + title: 'Folder 2', + }, + ], + }, + ], + title: 'Folder 1', + }, + { + files: [ + { + identifierref: 'file3Ref', + resourceHref: 'file3.html', + title: 'Test File3', + }, + { + identifierref: 'file4Ref', + resourceHref: 'file4.html', + title: 'Test File4', + }, + ], + title: 'Folder 2', + }, + ], + description: 'Example of test file', + language: 'en', + }); + }); + }); + it('extractIMSMetadata should extract metadata from multiple manifest file', async () => { + const manifestContent = ` + + + + Folder 1 + + Test File1 + + + Test File2 + + + + + + + + + + `; + + const subManifestContent = ` + + + + Folder 1 + + Test File3 + + + Test File4 + + + + + + + + + + `; + const zip = new JSZip(); + zip.file('imsmanifest.xml', manifestContent); + zip.file('file/imsmanifest.xml', subManifestContent); + + await zip.generateAsync({ type: 'blob' }).then(async function(imsBlob) { + await expect(extractIMSMetadata(imsBlob)).resolves.toEqual({ + folders: [ + { + files: [ + { + identifierref: 'file1Ref', + resourceHref: 'file/file1.html', + title: 'Test File1', + }, + { + identifierref: 'file2Ref', + resourceHref: 'file/file2.html', + title: 'Test File2', + }, + { + identifierref: 'file3Ref', + resourceHref: 'file3.html', + title: 'Test File3', + }, + { + identifierref: 'file4Ref', + resourceHref: 'file4.html', + title: 'Test File4', + }, + ], + title: 'Folder 1', + }, + ], + }); + }); + }); + it('extractIMSMetadata should extract metadata from imsmanifest and imsmetadata files', async () => { + const manifestContent = ``; + + const metadataContent = ` + + + + + Test File + + + en + + + Example of test file + + + + `; + const zip = new JSZip(); + zip.file('imsmanifest.xml', manifestContent); + zip.file('imsmetadata.xml', metadataContent); + + await zip.generateAsync({ type: 'blob' }).then(async function(imsBlob) { + await expect(extractIMSMetadata(imsBlob)).resolves.toEqual({ + title: 'Test File', + description: 'Example of test file', + language: 'en', + }); + }); + }); + it('extractIMSMetadata should not extract und language', async () => { + const manifestContent = ` + + + + + + Test File + + und + + + + `; + const zip = new JSZip(); + zip.file('imsmanifest.xml', manifestContent); + + await zip.generateAsync({ type: 'blob' }).then(async function(imsBlob) { + await expect(extractIMSMetadata(imsBlob)).resolves.toEqual({ + title: 'Test File', + }); + }); + }); + }); }); }); diff --git a/contentcuration/contentcuration/frontend/shared/vuex/file/utils.js b/contentcuration/contentcuration/frontend/shared/vuex/file/utils.js index 1049d8805f..814cb31409 100644 --- a/contentcuration/contentcuration/frontend/shared/vuex/file/utils.js +++ b/contentcuration/contentcuration/frontend/shared/vuex/file/utils.js @@ -51,60 +51,147 @@ export function getHash(file) { }); } -const getLeafNodes = node => { - if (!node.children || node.children.length === 0) { - return [node]; - } - - // Recursive case: Traverse all children and collect leaf nodes - let leafNodes = []; - for (const child of node.children) { - const childLeafNodes = getLeafNodes(child); - leafNodes = leafNodes.concat(childLeafNodes); - } - - return leafNodes; -}; - -function extractManifestFile(data, xmlDoc) { - const orgs = {}; - for (let i = 0; i < data.length; i++) { - let index = 0; - for (let j = 0; j < data[i].childNodes.length; j++) { - const org = {}; - if (data[i].childNodes[j].nodeType === 1) { - const orgNode = data[i].childNodes[j]; - const title = orgNode.getElementsByTagName('title'); - org.title = title[0].textContent; - const items = orgNode.getElementsByTagName('item'); - for (let k = 0; k < items.length; k++) { - const file = {}; - const item = items[k]; - file.title = title[1 + k].textContent; - file.identifierref = item.getAttribute('identifierref'); - file.resourceHref = xmlDoc - .querySelectorAll(`[identifier=${file.identifierref}]`)[0] - .getAttribute('href'); - const metadataNodes = orgNode.getElementsByTagName('metadata'); - if (metadataNodes && metadataNodes.length != 0) { - for (let l = 0; l < metadataNodes.length; l++) { - const nodeValue = getLeafNodes(metadataNodes[l]); - file[`${nodeValue[0].nodeName}`] = nodeValue[0].textContent; +async function getFolderMetadata(data, xmlDoc, zip, procssedFiles) { + const folders = []; + if (data.length && data[0].children && data[0].children.length) { + await Promise.all( + Object.values(data[0].children).map(async orgNode => { + const org = { + title: '', + files: [], + }; + if (orgNode.nodeType === 1) { + const title = orgNode.getElementsByTagName('title'); + org.title = title[0].textContent.replace(/ {2}|\r\n|\n|\r/gm, ''); + const files = orgNode.getElementsByTagName('item'); + const immediateChildNodes = []; + const childNodes = Object.values(orgNode.children); + Object.values(files).forEach(file => { + if (childNodes.includes(file)) { + immediateChildNodes.push(file); } - } - org[`file${index}`] = file; - index++; + }); + await Promise.all( + immediateChildNodes.map(async (fileNode, k) => { + const file = {}; + file.title = title[1 + k].textContent.replace(/ {2}|\r\n|\n|\r/gm, ''); + file.identifierref = fileNode.getAttribute('identifierref'); + file.resourceHref = xmlDoc + .querySelectorAll(`[identifier=${file.identifierref}]`)[0] + .getAttribute('href'); + if (fileNode.getElementsByTagName('organizations').length) { + getFolderMetadata( + fileNode.getElementsByTagName('organizations'), + xmlDoc, + zip, + procssedFiles + ).then(data => { + file.folders = data; + }); + } + const metadataNodes = orgNode.getElementsByTagName('metadata'); + if (metadataNodes && metadataNodes.length != 0) { + Object.values(metadataNodes).forEach(nodeValue => { + file[`${nodeValue[0].nodeName}`] = nodeValue[0].textContent.replace( + / {2}|\r\n|\n|\r/gm, + '' + ); + }); + } + org.files.push(file); + const manifestPath = + file.resourceHref.slice(0, file.resourceHref.lastIndexOf('/') + 1) + + 'imsmanifest.xml'; + const subManifestContent = zip.files[manifestPath]; + if (subManifestContent && !procssedFiles.includes(manifestPath)) { + procssedFiles.push(manifestPath); + const subManifestFile = await Promise.resolve(subManifestContent.async('text')); + const parser = new DOMParser(); + const xmlDoc = parser.parseFromString(subManifestFile, 'application/xml'); + const subManifestData = await getFolderMetadata( + xmlDoc.getElementsByTagName('organizations'), + xmlDoc, + zip, + procssedFiles + ); + if (subManifestData.title) { + org.title = subManifestData[0].title; + } + subManifestData[0].files.map(file => { + org.files.push(file); + }); + } + }) + ); } - orgs[`org${i}`] = org; - } - } + folders.push(org); + return org; + }) + ); + return folders; } - return orgs; } +async function getManifestMetadata(manifestFile, zip, procssedFiles) { + const metadata = {}; + const parser = new DOMParser(); + const xmlDoc = parser.parseFromString(manifestFile, 'application/xml'); + const data = xmlDoc.getElementsByTagName('organizations'); + return await getFolderMetadata(data, xmlDoc, zip, procssedFiles).then(async data => { + if (data) { + metadata.folders = data; + } + const metadataFile = zip.file('imsmetadata.xml'); + if (metadataFile) { + procssedFiles.push('imsmetadata.xml'); + const content = await Promise.resolve(metadataFile.async('text')); + const xmlDoc = parser.parseFromString(content, 'application/xml'); + if (xmlDoc.getElementsByTagName('lomes:title').length) { + metadata.title = xmlDoc + .getElementsByTagName('lomes:title')[0] + .children[0].textContent.replace(/ {2}|\r\n|\n|\r/gm, ''); + } + if ( + xmlDoc.getElementsByTagName('lomes:idiom').length && + xmlDoc + .getElementsByTagName('lomes:idiom')[0] + .textContent.replace(/ {2}|\r\n|\n|\r/gm, '') !== 'und' + ) { + metadata.language = xmlDoc + .getElementsByTagName('lomes:idiom')[0] + .children[0].textContent.replace(/ {2}|\r\n|\n|\r/gm, ''); + } + if (xmlDoc.getElementsByTagName('lomes:description').length) { + metadata.description = xmlDoc + .getElementsByTagName('lomes:description')[0] + .children[0].textContent.replace(/ {2}|\r\n|\n|\r/gm, ''); + } + } else { + if (xmlDoc.getElementsByTagName('imsmd:title').length) { + metadata.title = xmlDoc + .getElementsByTagName('imsmd:title')[0] + .textContent.replace(/ {2}|\r\n|\n|\r/gm, ''); + } + if ( + xmlDoc.getElementsByTagName('imsmd:language').length && + xmlDoc.getElementsByTagName('imsmd:language')[0].textContent !== 'und' + ) { + metadata.language = xmlDoc + .getElementsByTagName('imsmd:language')[0] + .textContent.replace(/ {2}|\r\n|\n|\r/gm, ''); + } + if (xmlDoc.getElementsByTagName('imsmd:description').length) { + metadata.description = xmlDoc + .getElementsByTagName('imsmd:description')[0] + .textContent.replace(/ {2}|\r\n|\n|\r/gm, ''); + } + } + return metadata; + }); +} export async function extractIMSMetadata(fileInput) { const zip = new JSZip(); - const metadata = {}; + const procssedFiles = []; return zip .loadAsync(fileInput) .then(function(zip) { @@ -112,47 +199,14 @@ export async function extractIMSMetadata(fileInput) { if (!manifestFile) { throw new Error('imsmanifest.xml not found in the zip file.'); } else { + procssedFiles.push('imsmanifest.xml'); return manifestFile.async('text'); } }) - .then(manifestFile => { - const parser = new DOMParser(); - const xmlDoc = parser.parseFromString(manifestFile, 'application/xml'); - const data = xmlDoc.getElementsByTagName('organizations'); - metadata.orgs = extractManifestFile(data, xmlDoc); - const metadataFile = zip.file('imsmetadata.xml'); - if (metadataFile) { - metadataFile.async('text').then(content => { - const parser = new DOMParser(); - const xmlDoc = parser.parseFromString(content, 'application/xml'); - metadata.title = - xmlDoc.getElementsByTagName('lomes:title').length === 0 - ? undefined - : xmlDoc.getElementsByTagName('lomes:title')[0].children[0].textContent; - metadata.language = - xmlDoc.getElementsByTagName('lomes:idiom').length === 0 - ? undefined - : xmlDoc.getElementsByTagName('lomes:idiom')[0].children[0].textContent; - metadata.description = - xmlDoc.getElementsByTagName('lomes:description').length === 0 - ? undefined - : xmlDoc.getElementsByTagName('lomes:description')[0].children[0].textContent; - }); - } else { - metadata.title = - xmlDoc.getElementsByTagName('title').length === 0 - ? undefined - : xmlDoc.getElementsByTagName('title')[0].textContent; - metadata.language = - xmlDoc.getElementsByTagName('imsmd:language').length === 0 - ? undefined - : xmlDoc.getElementsByTagName('imsmd:language')[0].textContent; - metadata.description = - xmlDoc.getElementsByTagName('imsmd:description').length === 0 - ? undefined - : xmlDoc.getElementsByTagName('imsmd:description')[0].textContent; - } - return metadata; + .then(async manifestFile => { + return await getManifestMetadata(manifestFile, zip, procssedFiles).then(metadata => { + return metadata; + }); }); } From 6c946687613c3f3072294d902a119897c59c22d8 Mon Sep 17 00:00:00 2001 From: Manav Aggarwal Date: Tue, 19 Sep 2023 18:11:50 +0530 Subject: [PATCH 4/4] - changing metadata key names, removing comments, replacing replace with trim --- .../components/edit/EditListItems.vue | 2 +- .../channelEdit/components/edit/EditModal.vue | 13 ++++---- .../shared/vuex/file/__tests__/module.spec.js | 4 +-- .../frontend/shared/vuex/file/utils.js | 32 ++++++++----------- 4 files changed, 21 insertions(+), 30 deletions(-) diff --git a/contentcuration/contentcuration/frontend/channelEdit/components/edit/EditListItems.vue b/contentcuration/contentcuration/frontend/channelEdit/components/edit/EditListItems.vue index 44cac4a2f5..fd980dc786 100644 --- a/contentcuration/contentcuration/frontend/channelEdit/components/edit/EditListItems.vue +++ b/contentcuration/contentcuration/frontend/channelEdit/components/edit/EditListItems.vue @@ -8,7 +8,7 @@ @input="trackSelect" @removed="handleRemoved" /> -
+
{ - file.metadata.folders.forEach(org => { - this.createNode('topic', org, newNodeId).then(topicNodeId => { - org.files.forEach(orgFile => { + file.metadata.folders.forEach(folder => { + this.createNode('topic', folder, newNodeId).then(topicNodeId => { + folder.files.forEach(folderFile => { const extra_fields = {}; - extra_fields['options'] = { entry: orgFile.resourceHref }; - extra_fields['title'] = orgFile.title; + extra_fields['options'] = { entry: folderFile.resourceHref }; + extra_fields['title'] = folderFile.title; let file_kind = null; FormatPresetsList.forEach(p => { if (p.id === file.metadata.preset) { @@ -532,7 +532,6 @@ return File.uploadUrl({ checksum: file.checksum, size: file.file_size, - type: 'application/zip', name: file.original_filename, file_format: file.file_format, preset: file.metadata.preset, @@ -542,7 +541,7 @@ loaded: 0, total: file.size, }; - if (index === 0) { + if (!this.selected.length) { this.selected = [resourceNodeId]; } this.updateFile({ diff --git a/contentcuration/contentcuration/frontend/shared/vuex/file/__tests__/module.spec.js b/contentcuration/contentcuration/frontend/shared/vuex/file/__tests__/module.spec.js index b1da778cb5..9af92da859 100644 --- a/contentcuration/contentcuration/frontend/shared/vuex/file/__tests__/module.spec.js +++ b/contentcuration/contentcuration/frontend/shared/vuex/file/__tests__/module.spec.js @@ -247,7 +247,6 @@ describe('file store', () => { }); }); it('extractIMSMetadata should extract metadata from imsmanifest.xml', async () => { - // const manifestFile = get_imsmanifest_file({ title: 'Test file' }); const manifestContent = ` @@ -317,7 +316,6 @@ describe('file store', () => { const zip = new JSZip(); zip.file('imsmanifest.xml', manifestContent); - // zip.folder('ims_package.zip'); await zip.generateAsync({ type: 'blob' }).then(async function(imsBlob) { await expect(extractIMSMetadata(imsBlob)).resolves.toEqual({ title: 'Test File', @@ -506,7 +504,7 @@ describe('file store', () => { - Test File + \t\t\t\n\n\n\nTest File\n und diff --git a/contentcuration/contentcuration/frontend/shared/vuex/file/utils.js b/contentcuration/contentcuration/frontend/shared/vuex/file/utils.js index 814cb31409..8f1f3da6f7 100644 --- a/contentcuration/contentcuration/frontend/shared/vuex/file/utils.js +++ b/contentcuration/contentcuration/frontend/shared/vuex/file/utils.js @@ -62,7 +62,7 @@ async function getFolderMetadata(data, xmlDoc, zip, procssedFiles) { }; if (orgNode.nodeType === 1) { const title = orgNode.getElementsByTagName('title'); - org.title = title[0].textContent.replace(/ {2}|\r\n|\n|\r/gm, ''); + org.title = title[0].textContent.trim(); const files = orgNode.getElementsByTagName('item'); const immediateChildNodes = []; const childNodes = Object.values(orgNode.children); @@ -74,7 +74,7 @@ async function getFolderMetadata(data, xmlDoc, zip, procssedFiles) { await Promise.all( immediateChildNodes.map(async (fileNode, k) => { const file = {}; - file.title = title[1 + k].textContent.replace(/ {2}|\r\n|\n|\r/gm, ''); + file.title = title[1 + k].textContent.trim(); file.identifierref = fileNode.getAttribute('identifierref'); file.resourceHref = xmlDoc .querySelectorAll(`[identifier=${file.identifierref}]`)[0] @@ -149,41 +149,37 @@ async function getManifestMetadata(manifestFile, zip, procssedFiles) { if (xmlDoc.getElementsByTagName('lomes:title').length) { metadata.title = xmlDoc .getElementsByTagName('lomes:title')[0] - .children[0].textContent.replace(/ {2}|\r\n|\n|\r/gm, ''); + .children[0].textContent.trim(); } if ( xmlDoc.getElementsByTagName('lomes:idiom').length && - xmlDoc - .getElementsByTagName('lomes:idiom')[0] - .textContent.replace(/ {2}|\r\n|\n|\r/gm, '') !== 'und' + LanguagesMap.has(xmlDoc.getElementsByTagName('lomes:idiom')[0].textContent.trim()) && + xmlDoc.getElementsByTagName('lomes:idiom')[0].textContent.trim() !== 'und' ) { metadata.language = xmlDoc .getElementsByTagName('lomes:idiom')[0] - .children[0].textContent.replace(/ {2}|\r\n|\n|\r/gm, ''); + .children[0].textContent.trim(); } if (xmlDoc.getElementsByTagName('lomes:description').length) { metadata.description = xmlDoc .getElementsByTagName('lomes:description')[0] - .children[0].textContent.replace(/ {2}|\r\n|\n|\r/gm, ''); + .children[0].textContent.trim(); } } else { if (xmlDoc.getElementsByTagName('imsmd:title').length) { - metadata.title = xmlDoc - .getElementsByTagName('imsmd:title')[0] - .textContent.replace(/ {2}|\r\n|\n|\r/gm, ''); + metadata.title = xmlDoc.getElementsByTagName('imsmd:title')[0].textContent.trim(); } if ( xmlDoc.getElementsByTagName('imsmd:language').length && - xmlDoc.getElementsByTagName('imsmd:language')[0].textContent !== 'und' + LanguagesMap.has(xmlDoc.getElementsByTagName('imsmd:language')[0].textContent.trim()) && + xmlDoc.getElementsByTagName('imsmd:language')[0].textContent.trim() !== 'und' ) { - metadata.language = xmlDoc - .getElementsByTagName('imsmd:language')[0] - .textContent.replace(/ {2}|\r\n|\n|\r/gm, ''); + metadata.language = xmlDoc.getElementsByTagName('imsmd:language')[0].textContent.trim(); } if (xmlDoc.getElementsByTagName('imsmd:description').length) { metadata.description = xmlDoc .getElementsByTagName('imsmd:description')[0] - .textContent.replace(/ {2}|\r\n|\n|\r/gm, ''); + .textContent.trim(); } } return metadata; @@ -204,9 +200,7 @@ export async function extractIMSMetadata(fileInput) { } }) .then(async manifestFile => { - return await getManifestMetadata(manifestFile, zip, procssedFiles).then(metadata => { - return metadata; - }); + return await getManifestMetadata(manifestFile, zip, procssedFiles); }); }