From fd9798fd75318b2b89e3abc0bdea52b19142f252 Mon Sep 17 00:00:00 2001 From: "Daniel W. Hieber" Date: Mon, 22 Mar 2021 18:16:31 -0500 Subject: [PATCH] NEW: option: postprocessor (#32) closes #32 --- README.md | 29 ++++++++++++++++------------- bin/toolbox2json.js | 23 +++++++++++++++++++---- src/toolbox2json.js | 32 ++++++++++++++++++++++---------- src/toolbox2json.test.js | 13 +++++++++++++ 4 files changed, 70 insertions(+), 27 deletions(-) diff --git a/README.md b/README.md index a802f30..addaf12 100644 --- a/README.md +++ b/README.md @@ -47,7 +47,7 @@ import convert from '@digitallinguistics/toolbox2json'; convert(`./my-data.db`, { out: `my-data.json` }); ``` -To run the library from the command line, use `toolbox2json `. This will print the results to the console by default. To save the JSON output to a file, use the `‑‑out` option or `‑o` flag: `toolbox2json ‑‑out `. To see the full list of command line options, run `toolbox2json ‑‑help`. +To run the library from the command line, use `toolbox2json `. This will print the results to the console by default. To save the JSON output to a file, use the `--out` option or `-o` flag: `toolbox2json --out `. To see the full list of command line options, run `toolbox2json --help`. ## Field Mappings @@ -57,11 +57,11 @@ If the Toolbox entry contains multiple instances of the same line marker, they w If you would like to customize the property names, use the `mappings` option. This should be an object mapping line markers (not including the initial backslash `\`) to property names. -On the command line, you can specify field mappings by providing the path to a mappings config file using the `‑m, ‑‑mappings` option. This file can be either a JSON or YAML document. +On the command line, you can specify field mappings by providing the path to a mappings config file using the `-m, --mappings` option. This file can be either a JSON or YAML document. ## Transforming Data -By default, the library copies data from each line into its corresponding JSON property unchanged. If you would like to transform the data, you can do so using the `transforms` option (when using the library as an ES module) or the `‑t, ‑‑transforms` flags (when using the library from the command line). When using transforms on the command line, the value passed to `‑t` or `‑‑transforms` should be the path to a JavaScript file that exports a single object containing the transformation methods. +By default, the library copies data from each line into its corresponding JSON property unchanged. If you would like to transform the data, you can do so using the `transforms` option (when using the library as an ES module) or the `-t, --transforms` flags (when using the library from the command line). When using transforms on the command line, the value passed to `-t` or `--transforms` should be the path to a JavaScript file that exports a single object containing the transformation methods. The object passed to the `transforms` option or exported by the transforms file should have methods for each property whose data you would like to transform. These methods will be passed the data for that line as an argument. Your method should transform the data and return it in the desired format. For example, if you would like to transform data in the `\txn` field to lowercase, your `transforms` object might look like this: @@ -85,18 +85,21 @@ const transforms = { } ``` +You can also apply a transform to the entire data object as a last step in the conversion process by providing a function to the `postprocessor` option. The function should accept the data object as input, and return the finalized version of the data object. When using the library as a module, you can provide this function as the `postprocessor` option; when using the library from the command line, the `-p, --postprocessor` option should be the path to a file which exports the postprocessor function as its default export. + ## Options -| Module | Command Line | Flag | Type | Default | Description | -|--------------|-----------------|------|----------|---------|----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| -| | `‑‑help` | `-h` | | | Display help. | -| `parseError` | `‑‑parse‑error` | `-e` | `"warn"` | | How to handle errors when parsing records. `"error"`: Stop and throw an error. `"none"`: Fail silently and continue. No object is created for that entry. `"object"`: Return a ParseError object for that entry. `"warn"`: Throw a warning and continue (_default_). | -| `mappings` | `‑‑mappings` | `-m` | Object | | An object mapping line markers to property names (if using as an ES module), or the path to a JSON or YAML file where the mappings live (if using on the command line). | -| `ndjson` | `‑‑ndjson` | `-n` | Boolean | `false` | Outputs newline-delimited JSON. | -| `out` | `‑‑out` | `-o` | String | | The path where the JSON file should be saved. If this option is provided, the module will return a Promise that resolves when the operation is complete, and no JSON data will be displayed on the command line. Otherwise, the module returns a readable stream of JavaScript objects (one for each entry in the Toolbox file). | -| `silent` | `‑‑silent` | `-s` | Boolean | `false` | Silences console output (except for the converted JSON). | -| | `‑‑version` | `-v` | | | Output the version number. | -| `transform` | `‑‑transform` | `-t` | Object | `{}` | An object containing methods for transforming field data. See [Transforming Data](#transforming-data) | +| Module | Command Line | Flag | Type | Default | Description | +|-----------------|-------------------|------|----------|---------|----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| +| | `--help` | `-h` | | | Display help. | +| `mappings` | `--mappings` | `-m` | Object | | An object mapping line markers to property names (if using as an ES module), or the path to a JSON or YAML file where the mappings live (if using on the command line). | +| `parseError` | `--parse-error` | `-e` | `"warn"` | | How to handle errors when parsing records. `"error"`: Stop and throw an error. `"none"`: Fail silently and continue. No object is created for that entry. `"object"`: Return a ParseError object for that entry. `"warn"`: Throw a warning and continue (_default_). | +| `postprocessor` | `--postprocessor` | `-p` | Function | | A function that accepts the final version of the data, applies any final transforms, and returns the final version of the data. See [Transforming Data](#transforming-data). | +| `ndjson` | `--ndjson` | `-n` | Boolean | `false` | Outputs newline-delimited JSON. | +| `out` | `--out` | `-o` | String | | The path where the JSON file should be saved. If this option is provided, the module will return a Promise that resolves when the operation is complete, and no JSON data will be displayed on the command line. Otherwise, the module returns a readable stream of JavaScript objects (one for each entry in the Toolbox file). | +| `silent` | `--silent` | `-s` | Boolean | `false` | Silences console output (except for the converted JSON). | +| `transforms` | `--transforms` | `-t` | Object | `{}` | An object containing methods for transforming field data. See [Transforming Data](#transforming-data). | +| | `--version` | `-v` | | | Output the version number. | ## Streaming Data diff --git a/bin/toolbox2json.js b/bin/toolbox2json.js index 3a44ab2..9b1fdd2 100644 --- a/bin/toolbox2json.js +++ b/bin/toolbox2json.js @@ -26,14 +26,16 @@ program .option(`-m, --mappings `, `path to file specifying line marker > property name mappings`) .option(`-n, --ndjson`, `output newline-delimited JSON`, false) .option(`-o, --out `, `path for the output JSON file`) +.option(`-p, --postprocessor `, `path to the postprocessor file`) .option(`-s, --silent`, `silences console output`, false) .option(`-t, --transforms `, `path to transformations file`) .action(async (filePath, options) => { const { - mappings: mappingsPath, - out: outPath, - transforms: transformsPath, + mappings: mappingsPath, + out: outPath, + postprocessor: postprocessorPath, + transforms: transformsPath, } = options; // get mappings from file @@ -53,6 +55,15 @@ program } + // get postprocessor function from file + + let postprocessor = entry => entry; + + if (postprocessorPath) { + const importPath = pathToFileURL(path.join(process.cwd(), postprocessorPath)); + ({ default: postprocessor } = await import(importPath)); + } + // get transforms from file let transforms = {}; @@ -64,7 +75,11 @@ program // run module - Object.assign(options, { mappings, transforms }); + Object.assign(options, { + mappings, + postprocessor, + transforms, + }); const readStream = toolbox2json(filePath, options); diff --git a/src/toolbox2json.js b/src/toolbox2json.js index c14eb1c..4959bec 100644 --- a/src/toolbox2json.js +++ b/src/toolbox2json.js @@ -39,19 +39,26 @@ class JS2JSONStream extends Transform { */ class Lines2JSStream extends Transform { - constructor({ mappings, parseError, silent, transforms }) { + constructor({ + mappings, + parseError, + postprocessor, + silent, + transforms, + }) { super({ readableObjectMode: true, writableObjectMode: true, }); - this.fileHeader = true; // the first set of lines is the file header - this.lines = []; - this.mappings = mappings; - this.parseError = parseError; - this.silent = silent; - this.transforms = transforms; + this.fileHeader = true; // the first set of lines is the file header + this.lines = []; + this.mappings = mappings; + this.parseError = parseError; + this.postprocess = postprocessor; + this.silent = silent; + this.transforms = transforms; } @@ -95,7 +102,10 @@ class Lines2JSStream extends Transform { this.transforms, ); - this.push(entry); // write entry to stream + // apply postprocessing function + const finalEntry = this.postprocess(entry); + + this.push(finalEntry); // write entry to stream } catch (e) { @@ -137,8 +147,9 @@ export default function toolbox2json(filePath, { mappings = {}, ndjson = false, out, - silent = false, - transforms = {}, + postprocessor = entry => entry, + silent = false, + transforms = {}, } = {}) { // validation @@ -167,6 +178,7 @@ export default function toolbox2json(filePath, { const lines2js = new Lines2JSStream({ mappings, parseError, + postprocessor, silent, transforms, }); diff --git a/src/toolbox2json.test.js b/src/toolbox2json.test.js index afd747e..96bb336 100644 --- a/src/toolbox2json.test.js +++ b/src/toolbox2json.test.js @@ -213,6 +213,19 @@ describe(`toolbox2json`, () => { expect(exists).to.be(true); }); + specify(`option: postprocessor`, async () => { + + const postprocessor = () => ({ postprocessed: true }); + + await convert(crkPath, { out: outPath, postprocessor, silent: true }); + + const json = await readFile(outPath, `utf8`); + const [entry] = JSON.parse(json); + + expect(entry.postprocessed).to.be(true); + + }); + specify(`option: transforms`, async () => { const { default: transforms } = await import(`../test/transforms.js`);