feat(bom): strip byte-order-mark from CSV files (#93)

pelias · Nov 30, 2021 · b8edafe · b8edafe
1 parent 3f9716c
commit b8edafe
Show file tree

Hide file tree

Showing 2 changed files with 9 additions and 10 deletions.
diff --git a/lib/streams/recordStream.js b/lib/streams/recordStream.js
@@ -1,12 +1,10 @@
-var fs = require( 'fs' );
-var path = require( 'path' );
-
-var csvParse = require( 'csv-parse' );
-var combinedStream = require( 'combined-stream' );
-var _ = require( 'lodash' );
-
-var logger = require( 'pelias-logger' ).get( 'csv-importer' );
-var DocumentStream = require('./documentStream');
+const _ = require('lodash');
+const fs = require('fs');
+const path = require('path');
+const csvParse = require('csv-parse').parse;
+const combinedStream = require('combined-stream');
+const logger = require('pelias-logger').get('csv-importer');
+const DocumentStream = require('./documentStream');
 
 /*
  * Construct a suitable id prefix for a CSV file given
@@ -45,6 +43,7 @@ function createRecordStream( filePath, dirPath ){
 
   var csvParser = csvParse({
     trim: true,
+    bom: true,
     skip_empty_lines: true,
     relax_column_count: true,
     relax: true,

diff --git a/package.json b/package.json
@@ -10,7 +10,7 @@
     "@hapi/joi": "^16.0.0",
     "async": "^3.0.1",
     "combined-stream": "^1.0.7",
-    "csv-parse": "^4.0.1",
+    "csv-parse": "^5.0.3",
     "fs-extra": "^8.0.0",
     "glob": "^7.0.0",
     "lodash": "^4.16.0",