From 7da1e02b3beb03d236a7c56e20ce3b2f833583d4 Mon Sep 17 00:00:00 2001 From: Bart Mesuere Date: Thu, 1 Aug 2024 14:20:48 +0200 Subject: [PATCH 1/5] implement json formatter --- lib/commands/unipept/unipept_subcommand.ts | 5 +++-- lib/formatters/formatter_factory.ts | 3 +++ lib/formatters/json_formatter.ts | 18 ++++++++++++++++++ 3 files changed, 24 insertions(+), 2 deletions(-) create mode 100644 lib/formatters/json_formatter.ts diff --git a/lib/commands/unipept/unipept_subcommand.ts b/lib/commands/unipept/unipept_subcommand.ts index a9e6e593..09a62bde 100644 --- a/lib/commands/unipept/unipept_subcommand.ts +++ b/lib/commands/unipept/unipept_subcommand.ts @@ -84,11 +84,12 @@ export abstract class UnipeptSubcommand { const result = await r.json(); if (this.firstBatch && this.options.header) { - this.firstBatch = false; this.outputStream.write(this.formatter.header(result, this.fasta)); } - this.outputStream.write(this.formatter.format(result, this.fasta)); + this.outputStream.write(this.formatter.format(result, this.fasta, this.firstBatch)); + + if (this.firstBatch) this.firstBatch = false; } private constructRequestBody(slice: string[]): URLSearchParams { diff --git a/lib/formatters/formatter_factory.ts b/lib/formatters/formatter_factory.ts index b350df77..110b76ad 100644 --- a/lib/formatters/formatter_factory.ts +++ b/lib/formatters/formatter_factory.ts @@ -1,10 +1,13 @@ import { CSVFormatter } from "./csv_formatter.js"; import { Formatter } from "./formatter.js"; +import { JSONFormatter } from "./json_formatter.js"; export class FormatterFactory { static getFormatter(name: string): Formatter { if (name === "csv") { return new CSVFormatter(); + } else if (name === "json") { + return new JSONFormatter(); } return new CSVFormatter(); } diff --git a/lib/formatters/json_formatter.ts b/lib/formatters/json_formatter.ts new file mode 100644 index 00000000..0ab5436a --- /dev/null +++ b/lib/formatters/json_formatter.ts @@ -0,0 +1,18 @@ +import { Formatter } from "./formatter.js"; + +export class JSONFormatter extends Formatter { + + // eslint-disable-next-line @typescript-eslint/no-unused-vars + header(sampleData: { [key: string]: string }[], fastaMapper?: boolean | undefined): string { + return "["; + } + + footer(): string { + return "]\n"; + } + + convert(data: object[], first: boolean): string { + const output = data.map(d => JSON.stringify(d)).join(","); + return first ? output : `,${output}`; + } +} From ec28bfe11354e23cc1ec16b757334d1f478f49d7 Mon Sep 17 00:00:00 2001 From: Bart Mesuere Date: Thu, 1 Aug 2024 14:28:26 +0200 Subject: [PATCH 2/5] add tests for json formatter --- tests/formatters/json_formatter.test.ts | 28 +++++++++++++++++++++++++ 1 file changed, 28 insertions(+) create mode 100644 tests/formatters/json_formatter.test.ts diff --git a/tests/formatters/json_formatter.test.ts b/tests/formatters/json_formatter.test.ts new file mode 100644 index 00000000..c4e80648 --- /dev/null +++ b/tests/formatters/json_formatter.test.ts @@ -0,0 +1,28 @@ +import { FormatterFactory } from "../../lib/formatters/formatter_factory"; +import { TestObject } from "./test_object"; + +const formatter = FormatterFactory.getFormatter("json"); + +test('test header', () => { + const object = [TestObject.testObject(), TestObject.testObject()]; + expect(formatter.header(object)).toBe("["); +}); + +test('test footer', () => { + expect(formatter.footer()).toBe("]\n"); +}); + +test('test convert', () => { + const object = [TestObject.testObject()]; + const json = TestObject.asJson(); + + expect(formatter.convert(object, true)).toBe(json); + expect(formatter.convert(object, false)).toBe(`,${json}`); +}); + +test('test format with fasta', () => { + //const fasta = [['>test', '5']]; + //const object = [TestObject.testObject()]; + //const json = '{"fasta_header":">test","integer":5,"string":"string","list":["a",2,false]}'; + //expect(formatter.format(object, fasta, true)).toBe(json); +}); From 0f915f5b92a2bdecdc405498f5d770d363d6b1cb Mon Sep 17 00:00:00 2001 From: Bart Mesuere Date: Thu, 1 Aug 2024 14:46:29 +0200 Subject: [PATCH 3/5] add xml formatter --- eslint.config.js | 7 ++++++- lib/formatters/formatter_factory.ts | 5 ++++- lib/formatters/json_formatter.ts | 3 +-- lib/formatters/xml_formatter.ts | 17 +++++++++++++++++ package.json | 3 ++- yarn.lock | 5 +++++ 6 files changed, 35 insertions(+), 5 deletions(-) create mode 100644 lib/formatters/xml_formatter.ts diff --git a/eslint.config.js b/eslint.config.js index 17c0f257..3f8a7e2e 100644 --- a/eslint.config.js +++ b/eslint.config.js @@ -7,5 +7,10 @@ export default [ { languageOptions: { globals: globals.node } }, pluginJs.configs.recommended, ...tseslint.configs.recommended, - { ignores: ["dist/"] } + { + rules: { + "@typescript-eslint/no-unused-vars": ["error", { argsIgnorePattern: "^_" }], + }, + ignores: ["dist/"] + } ]; diff --git a/lib/formatters/formatter_factory.ts b/lib/formatters/formatter_factory.ts index 110b76ad..2a1f3cf5 100644 --- a/lib/formatters/formatter_factory.ts +++ b/lib/formatters/formatter_factory.ts @@ -1,6 +1,7 @@ -import { CSVFormatter } from "./csv_formatter.js"; import { Formatter } from "./formatter.js"; +import { CSVFormatter } from "./csv_formatter.js"; import { JSONFormatter } from "./json_formatter.js"; +import { XMLFormatter } from "./xml_formatter.js"; export class FormatterFactory { static getFormatter(name: string): Formatter { @@ -8,6 +9,8 @@ export class FormatterFactory { return new CSVFormatter(); } else if (name === "json") { return new JSONFormatter(); + } else if (name === "xml") { + return new XMLFormatter(); } return new CSVFormatter(); } diff --git a/lib/formatters/json_formatter.ts b/lib/formatters/json_formatter.ts index 0ab5436a..a6af5efe 100644 --- a/lib/formatters/json_formatter.ts +++ b/lib/formatters/json_formatter.ts @@ -2,8 +2,7 @@ import { Formatter } from "./formatter.js"; export class JSONFormatter extends Formatter { - // eslint-disable-next-line @typescript-eslint/no-unused-vars - header(sampleData: { [key: string]: string }[], fastaMapper?: boolean | undefined): string { + header(_sampleData: { [key: string]: string }[], _fastaMapper?: boolean | undefined): string { return "["; } diff --git a/lib/formatters/xml_formatter.ts b/lib/formatters/xml_formatter.ts new file mode 100644 index 00000000..ebbd5a35 --- /dev/null +++ b/lib/formatters/xml_formatter.ts @@ -0,0 +1,17 @@ +import { Formatter } from "./formatter.js"; +import { toXML } from "to-xml"; + +export class XMLFormatter extends Formatter { + + header(_sampleData: { [key: string]: string }[], _fastaMapper?: boolean | undefined): string { + return ""; + } + + footer(): string { + return "\n"; + } + + convert(data: object[], _first: boolean): string { + return data.map(d => `${toXML(d)}`).join(""); + } +} diff --git a/package.json b/package.json index fc1ff4c4..24a343a8 100644 --- a/package.json +++ b/package.json @@ -25,7 +25,8 @@ }, "dependencies": { "commander": "^12.1.0", - "csv-stringify": "^6.5.0" + "csv-stringify": "^6.5.0", + "to-xml": "^0.1.11" }, "devDependencies": { "@eslint/js": "^9.5.0", diff --git a/yarn.lock b/yarn.lock index 90d703b4..c6c157f9 100644 --- a/yarn.lock +++ b/yarn.lock @@ -2853,6 +2853,11 @@ to-regex-range@^5.0.1: dependencies: is-number "^7.0.0" +to-xml@^0.1.11: + version "0.1.11" + resolved "https://registry.yarnpkg.com/to-xml/-/to-xml-0.1.11.tgz#fae4dafe89889e5013c86e4ea65933dca97d985b" + integrity sha512-deRSQy7vONsawpyrPdhuV0Lh0yXtKQEhAnvfSv3JMahCq3PF0MZJB/BdV1jJANVbuMdnx96zhqD/X0zMMXx0Pw== + ts-api-utils@^1.3.0: version "1.3.0" resolved "https://registry.yarnpkg.com/ts-api-utils/-/ts-api-utils-1.3.0.tgz#4b490e27129f1e8e686b45cc4ab63714dc60eea1" From 8044011173e94f6998852d98e1e69ff665184788 Mon Sep 17 00:00:00 2001 From: Bart Mesuere Date: Thu, 1 Aug 2024 16:03:13 +0200 Subject: [PATCH 4/5] add tests for xml formatter --- lib/formatters/to_xml.ts | 255 +++++++++++++++++++++++++ lib/formatters/xml_formatter.ts | 2 +- package.json | 3 +- tests/formatters/xml_formatter.test.ts | 28 +++ yarn.lock | 5 - 5 files changed, 285 insertions(+), 8 deletions(-) create mode 100644 lib/formatters/to_xml.ts create mode 100644 tests/formatters/xml_formatter.test.ts diff --git a/lib/formatters/to_xml.ts b/lib/formatters/to_xml.ts new file mode 100644 index 00000000..bcebc152 --- /dev/null +++ b/lib/formatters/to_xml.ts @@ -0,0 +1,255 @@ +// eslint-disable-next-line @typescript-eslint/ban-ts-comment +// @ts-nocheck + +// This file was taken from https://github.com/kawanet/to-xml and modified to have a specific output for arrays. + +/** + * The toXML() method converts a JavaScript value to an XML string. + * + * @function toXML + * @param value {Object} The value to convert to an XML string. + * @param [replacer] {Function} A function that alters the behavior + * of the stringification process. + * @param [space] {Number|String} A String or Number object that's + * used to insert white space into the output XML string for + * readability purposes. If this is a Number, it indicates the number + * of space characters to use as white space. + * If this is a String, the string is used as white space. + * @returns {String} + */ + +const TYPES = { + "boolean": fromString, + "number": fromString, + "object": fromObject, + "string": fromString +}; + +const ESCAPE = { + "\t": " ", + "\n": " ", + "\r": " ", + " ": " ", + "&": "&", + "<": "<", + ">": ">", + '"': """ +}; + +const ATTRIBUTE_KEY = "@"; +const CHILD_NODE_KEY = "#"; +const LF = "\n"; + +const isArray = Array.isArray || _isArray; + +const REPLACE = String.prototype.replace; + +function _toXML(value, replacer, space) { + const job = createJob(replacer, space); + fromAny(job, "", value); + return job.r; +} + +function createJob(replacer, space) { + const job = { + f: replacer, // replacer function + // s: "", // indent string + // i: 0, // indent string length + l: "", // current indent string + r: "" // result string + }; + + if (space) { + let str = ""; + + if (space > 0) { + for (let i = space; i; i--) { + str += " "; + } + } else { + str += space; // stringify + } + job.s = str; + + // indent string length + job.i = str.length; + } + + return job; +} + +function fromAny(job, key, value) { + // child node synonym + if (key === CHILD_NODE_KEY) key = ""; + + if (_isArray(value)) return fromArray(job, key, value); + + const replacer = job.f; + if (replacer) value = replacer(key, value); + + const f = TYPES[typeof value]; + if (f) f(job, key, value); +} + +function fromString(job, key, value) { + if (key === "?") { + // XML declaration + value = ""; + } else if (key === "!") { + // comment, CDATA section + value = ""; + } else { + value = escapeTextNode(value); + if (key) { + // text element without attributes + value = "<" + key + ">" + value + ""; + } + } + + if (key && job.i && job.r) { + job.r += LF + job.l; // indent + } + + job.r += value; +} + +function fromArray(job, key, value) { + if (key !== "item") { + fromObject(job, key, { item: value }); + } else { + Array.prototype.forEach.call(value, function (value) { + fromAny(job, key, value); + }); + } +} + +function fromObject(job, key, value) { + // empty tag + const hasTag = !!key; + const closeTag = (value === null); + if (closeTag) { + if (!hasTag) return; + value = {}; + } + + const keys = Object.keys(value); + const keyLength = keys.length; + const attrs = keys.filter(isAttribute); + const attrLength = attrs.length; + const hasIndent = job.i; + const curIndent = job.l; + let willIndent = hasTag && hasIndent; + let didIndent; + + // open tag + if (hasTag) { + if (hasIndent && job.r) { + job.r += LF + curIndent; + } + + job.r += '<' + key; + + // attributes + attrs.forEach(function (name) { + writeAttributes(job, name.substr(1), value[name]); + }); + + // empty element + const isEmpty = closeTag || (attrLength && keyLength === attrLength); + if (isEmpty) { + const firstChar = key[0]; + if (firstChar !== "!" && firstChar !== "?") { + job.r += "/"; + } + } + + job.r += '>'; + + if (isEmpty) return; + } + + keys.forEach(function (name) { + // skip attribute + if (isAttribute(name)) return; + + // indent when it has child node but not fragment + if (willIndent && ((name && name !== CHILD_NODE_KEY) || isArray(value[name]))) { + job.l += job.s; // increase indent level + willIndent = 0; + didIndent = 1; + } + + // child node or text node + fromAny(job, name, value[name]); + }); + + if (didIndent) { + // decrease indent level + job.l = job.l.substr(job.i); + + job.r += LF + job.l; + } + + // close tag + if (hasTag) { + job.r += ''; + } +} + +function writeAttributes(job, key, val) { + if (isArray(val)) { + val.forEach(function (child) { + writeAttributes(job, key, child); + }); + } else if (!key && "object" === typeof val) { + Object.keys(val).forEach(function (name) { + writeAttributes(job, name, val[name]); + }); + } else { + writeAttribute(job, key, val); + } +} + +function writeAttribute(job, key, val) { + const replacer = job.f; + if (replacer) val = replacer(ATTRIBUTE_KEY + key, val); + if ("undefined" === typeof val) return; + + // empty attribute name + if (!key) { + job.r += ' ' + val; + return; + } + + // attribute name + job.r += ' ' + key; + + // property attribute + if (val === null) return; + + job.r += '="' + escapeAttribute(val) + '"'; +} + +function isAttribute(name) { + return name && name[0] === ATTRIBUTE_KEY; +} + +function escapeTextNode(str) { + return REPLACE.call(str, /(^\s|[&<>]|\s$)/g, escapeRef); +} + +function escapeAttribute(str) { + return REPLACE.call(str, /([&"])/g, escapeRef); +} + +function escapeRef(str) { + return ESCAPE[str] || str; +} + +function _isArray(array) { + return array instanceof Array; +} + +export function toXML(value: Object, replacer?: Function, space?: Number | String): String { + return _toXML(value, replacer, space); +} diff --git a/lib/formatters/xml_formatter.ts b/lib/formatters/xml_formatter.ts index ebbd5a35..e30c54e5 100644 --- a/lib/formatters/xml_formatter.ts +++ b/lib/formatters/xml_formatter.ts @@ -1,5 +1,5 @@ import { Formatter } from "./formatter.js"; -import { toXML } from "to-xml"; +import { toXML } from "./to_xml.js"; export class XMLFormatter extends Formatter { diff --git a/package.json b/package.json index 24a343a8..fc1ff4c4 100644 --- a/package.json +++ b/package.json @@ -25,8 +25,7 @@ }, "dependencies": { "commander": "^12.1.0", - "csv-stringify": "^6.5.0", - "to-xml": "^0.1.11" + "csv-stringify": "^6.5.0" }, "devDependencies": { "@eslint/js": "^9.5.0", diff --git a/tests/formatters/xml_formatter.test.ts b/tests/formatters/xml_formatter.test.ts new file mode 100644 index 00000000..c327d034 --- /dev/null +++ b/tests/formatters/xml_formatter.test.ts @@ -0,0 +1,28 @@ +import { FormatterFactory } from "../../lib/formatters/formatter_factory"; +import { TestObject } from "./test_object"; + +const formatter = FormatterFactory.getFormatter("xml"); + +test('test header', () => { + const object = [TestObject.testObject(), TestObject.testObject()]; + expect(formatter.header(object)).toBe(""); +}); + +test('test footer', () => { + expect(formatter.footer()).toBe("\n"); +}); + +test('test convert', () => { + const object = [TestObject.testObject()]; + const xml = `${TestObject.asXml()}`; + + expect(formatter.convert(object, true)).toBe(xml); + expect(formatter.convert(object, false)).toBe(xml); +}); + +test('test format with fasta', () => { + //const fasta = [['>test', '5']]; + //const object = [TestObject.testObject()]; + //const json = '{"fasta_header":">test","integer":5,"string":"string","list":["a",2,false]}'; + //expect(formatter.format(object, fasta, true)).toBe(json); +}); diff --git a/yarn.lock b/yarn.lock index c6c157f9..90d703b4 100644 --- a/yarn.lock +++ b/yarn.lock @@ -2853,11 +2853,6 @@ to-regex-range@^5.0.1: dependencies: is-number "^7.0.0" -to-xml@^0.1.11: - version "0.1.11" - resolved "https://registry.yarnpkg.com/to-xml/-/to-xml-0.1.11.tgz#fae4dafe89889e5013c86e4ea65933dca97d985b" - integrity sha512-deRSQy7vONsawpyrPdhuV0Lh0yXtKQEhAnvfSv3JMahCq3PF0MZJB/BdV1jJANVbuMdnx96zhqD/X0zMMXx0Pw== - ts-api-utils@^1.3.0: version "1.3.0" resolved "https://registry.yarnpkg.com/ts-api-utils/-/ts-api-utils-1.3.0.tgz#4b490e27129f1e8e686b45cc4ab63714dc60eea1" From ed97baba271f9e47ff753f7bb0626f2f2473a437 Mon Sep 17 00:00:00 2001 From: Bart Mesuere Date: Thu, 1 Aug 2024 16:05:44 +0200 Subject: [PATCH 5/5] fix linter --- lib/formatters/to_xml.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/formatters/to_xml.ts b/lib/formatters/to_xml.ts index bcebc152..c40593b6 100644 --- a/lib/formatters/to_xml.ts +++ b/lib/formatters/to_xml.ts @@ -250,6 +250,6 @@ function _isArray(array) { return array instanceof Array; } -export function toXML(value: Object, replacer?: Function, space?: Number | String): String { +export function toXML(value: object, replacer?: function, space?: number | string): string { return _toXML(value, replacer, space); }