Merge pull request #80 from pelias/blackmad/add-popularity-to-pelias-…

…csv-importer

add optional popularity column to pelias csv importer

README.md

@@ -124,6 +124,10 @@ The contents of the `name_json` field must be a JSON array. As a reminder, in CS
 
 Aliases and languages can _both_ be specified. For example, the `name_json_es` field allows setting multiple aliases in Spanish.
 
+## Popularity
+
+Popularity values can be specified to mark records as more important than others. This value should be an integer greater than zero, in the `popularity` column.
+
 ## Categories
 
 Category values can be added to a record. For a single category, use the `category` field. For multiple categories, use `category_json`, with the same formatting as for alias names.

lib/streams/documentStream.js

@@ -5,6 +5,16 @@ const through = require( 'through2' );
 const peliasModel = require( 'pelias-model' );
 const NAME_REGEX = /^name(_json)?_([a-z]{2})$/i;
 
+function getPopularity(record) {
+  const popularityString = getCaseInsensitive('popularity', record);
+  const popularity = parseInt(popularityString);
+  if (popularityString && (isNaN(popularity) || !/^\d+$/.test(popularityString))) {
+    throw new Error("Popularity must be an int, got " + popularityString)
+  } else {
+    return popularity;
+  }
+}
+
 function getPostalCode(record) {
   return getCaseInsensitive('zipcode', record)
     || getCaseInsensitive('postcode', record)
@@ -155,12 +165,16 @@ function processRecord(record, next_uid, stats) {
       pelias_document.setAddress('number', housenumber);
     }
 
-
     const postcode = getPostalCode(record);
     if (postcode) {
       pelias_document.setAddress('zip', postcode);
     }
 
+    const popularity = getPopularity(record);
+    if (popularity) {
+      pelias_document.setPopularity(popularity);
+    }
+
     const addendumFields = getPrefixedFields('addendum_json_', record);
     Object.keys(addendumFields).forEach(function(namespace) {
       pelias_document.setAddendum(namespace, JSON.parse(addendumFields[namespace]));

test/streams/documentStream.js

@@ -324,4 +324,44 @@ tape('documentStream parses JSON from category_json field', function(test) {
     test.equal(stats.badRecordCount, 0, 'bad record count unchanged');
     test.end();
   });
+});
+
+tape( 'documentStream accepts popularity', function(test) {
+  const input = {
+    NUMBER: '5',
+    STREET: '101st Avenue',
+    LAT: 5,
+    LON: 6,
+    postalcode: '10010',
+    popularity: '5000'
+  };
+  const stats = { badRecordCount: 0 };
+  const documentStream = DocumentStream.create('prefix', stats);
+
+  test_stream([input], documentStream, function(err, actual) {
+    test.equal(actual.length, 1, 'the document should be pushed' );
+    test.equal(stats.badRecordCount, 0, 'bad record count unchanged');
+    test.equal(actual[0].getPopularity(), 5000);
+    test.end();
+  });
+});
+
+
+tape( 'documentStream rejects invalid popularity', function(test) {
+  const input = {
+    NUMBER: '5',
+    STREET: '101st Avenue',
+    LAT: 5,
+    LON: 6,
+    postalcode: '10010',
+    popularity: '500a0'
+  };
+  const stats = { badRecordCount: 0 };
+  const documentStream = DocumentStream.create('prefix', stats);
+
+  test_stream([input], documentStream, function(err, actual) {
+    test.equal(actual.length, 0, 'the document should be skipped' );
+    test.equal(stats.badRecordCount, 1, 'bad record count went up by 1');
+    test.end();
+  });
 });