From 0773f4775d560e8c6c1ed7f9171393ffeed3df01 Mon Sep 17 00:00:00 2001 From: Loki Coyote Date: Sat, 23 Sep 2023 09:54:49 +0000 Subject: [PATCH 1/2] adding support for extracting geometry from wof and importing it into the pelias shape field --- src/components/extractFields.js | 7 ++ src/peliasDocGenerators.js | 5 ++ test/components/extractFieldsTest.js | 10 ++- test/functional.js | 4 + test/peliasDocGeneratorsTest.js | 108 +++++++++++++++++++++++---- test/readStreamTest.js | 9 ++- test/resources/france.json | 2 +- 7 files changed, 126 insertions(+), 19 deletions(-) diff --git a/src/components/extractFields.js b/src/components/extractFields.js index 4865d8b2..6a1085eb 100644 --- a/src/components/extractFields.js +++ b/src/components/extractFields.js @@ -69,6 +69,12 @@ function getBoundingBox(properties) { } } +function getGeometry(json_object) { + if (json_object.hasOwnProperty('geometry')) { + return json_object['geometry'] + } +} + // get an array of language codes spoken at this location function getLanguages(properties) { if (!Array.isArray(properties['wof:lang_x_official'])) { @@ -231,6 +237,7 @@ module.exports.create = function map_fields_stream() { lat: getLat(json_object.properties), lon: getLon(json_object.properties), bounding_box: getBoundingBox(json_object.properties), + geometry: getGeometry(json_object), population: getPopulation(json_object.properties), popularity: json_object.properties['misc:photo_sum'], hierarchies: getHierarchies(json_object.id, json_object.properties), diff --git a/src/peliasDocGenerators.js b/src/peliasDocGenerators.js index 51f63cef..6a1e0fb7 100644 --- a/src/peliasDocGenerators.js +++ b/src/peliasDocGenerators.js @@ -100,6 +100,11 @@ function setupDocument(record, hierarchy) { } wofDoc.setCentroid({ lat: record.lat, lon: record.lon }); + // only set shape if available + // TODO: test to ensure it is a JSON string, try/catch + if (record.geometry) { + wofDoc.setShape(record.geometry) + } // only set population if available if (record.population) { wofDoc.setPopulation(record.population); diff --git a/test/components/extractFieldsTest.js b/test/components/extractFieldsTest.js index 3ca6aea1..9dd9f71b 100644 --- a/test/components/extractFieldsTest.js +++ b/test/components/extractFieldsTest.js @@ -45,7 +45,12 @@ tape('readStreamComponents', function(test) { 'misc:photo_sum': 87654, ignoreField3: 'ignoreField3', ignoreField4: 'ignoreField4', - } + }, + geometry: { + coordinates: [[[-72.122,42.428],[-72.122,42.409],[-72.091,42.409],[-72.091,42.428],[-72.122,42.428]]], + type: 'Polygon' + }, + } ]; @@ -60,6 +65,7 @@ tape('readStreamComponents', function(test) { lon: 21.212121, popularity: 87654, population: undefined, + geometry: {'coordinates':[[[-72.122,42.428],[-72.122,42.409],[-72.091,42.409],[-72.091,42.428],[-72.122,42.428]]],'type':'Polygon'}, abbreviation: 'XY', bounding_box: '-13.691314,49.909613,1.771169,60.847886', hierarchies: [ @@ -102,6 +108,7 @@ tape('readStreamComponents', function(test) { popularity: undefined, abbreviation: undefined, bounding_box: undefined, + geometry: undefined, hierarchies: [], concordances: {} } @@ -141,6 +148,7 @@ tape('readStreamComponents', function(test) { population: undefined, popularity: undefined, abbreviation: undefined, + geometry: undefined, bounding_box: '-13.691314,49.909613,1.771169,60.847886', hierarchies: [ { diff --git a/test/functional.js b/test/functional.js index e2cdde77..2db1adb5 100644 --- a/test/functional.js +++ b/test/functional.js @@ -25,6 +25,10 @@ tape('functional', function(test) { importStream(readStream, documentGenerator, dbClientStream, () => { t.ok(dbRecords[101772677]); t.deepEqual(recordOrder, ['85633147', '1108826387', '85683431', '404227861', '102068587', '404414453', '101772677']); + t.deepEqual(dbRecords[101772677].shape, { + coordinates:[[[4.725057,43.993083],[4.724003,43.993288],[4.722264,43.993505],[4.722074,43.993418],[4.721708,43.992028],[4.721318,43.991503],[4.720796,43.991151],[4.720609,43.991154],[4.719958,43.991894],[4.719771,43.991897],[4.719264,43.991634],[4.716917,43.99069],[4.716227,43.990566],[4.715231,43.990591],[4.71399,43.990799],[4.70856,43.992252],[4.707696,43.992545],[4.702638,43.993956],[4.696921,43.995836],[4.695008,43.996505],[4.691377,43.997705],[4.689834,43.998233],[4.688643,43.998035],[4.686647,43.997985],[4.684845,43.998201],[4.683678,43.998399],[4.681814,43.998617],[4.677906,43.999289],[4.67665,43.999426],[4.673878,43.999729],[4.671699,43.999817],[4.670635,43.999707],[4.669055,43.99938],[4.668244,43.999392],[4.665823,43.999762],[4.664979,44.000324],[4.663961,44.00133],[4.663292,44.001889],[4.663111,44.002117],[4.662806,44.002347],[4.662313,44.002534],[4.660767,44.002972],[4.659794,44.003392],[4.658683,44.003813],[4.653872,44.005183],[4.6485,44.006596],[4.643763,44.006685],[4.638028,44.006753],[4.636845,44.006825],[4.636898,44.008229],[4.636934,44.009038],[4.637026,44.011377],[4.63694,44.014025],[4.636841,44.016241],[4.636722,44.016423],[4.636357,44.016752],[4.636111,44.016891],[4.635235,44.01676],[4.634607,44.016644],[4.634132,44.016606],[4.633688,44.016783],[4.633382,44.016977],[4.632738,44.017122],[4.634274,44.017639],[4.635797,44.018111],[4.63688,44.018896],[4.637322,44.019069],[4.638263,44.019271],[4.642354,44.020092],[4.643425,44.020472],[4.643944,44.020735],[4.6452,44.021022],[4.646701,44.02117],[4.649925,44.020933],[4.650806,44.020775],[4.651978,44.020307],[4.656076,44.020083],[4.65726,44.020057],[4.658355,44.019536],[4.659106,44.019182],[4.662836,44.018766],[4.663153,44.01895],[4.663478,44.019395],[4.66418,44.019916],[4.664387,44.020182],[4.665148,44.020612],[4.665652,44.020784],[4.667087,44.020798],[4.667769,44.020653],[4.668745,44.020323],[4.669972,44.01962],[4.670781,44.019563],[4.671107,44.020044],[4.671424,44.020219],[4.671736,44.020214],[4.672179,44.020028],[4.672835,44.019423],[4.673147,44.019419],[4.675376,44.020537],[4.675817,44.02071],[4.676424,44.020116],[4.676649,44.019707],[4.677832,44.019635],[4.678795,44.02016],[4.68065,44.021293],[4.68084,44.02138],[4.681711,44.021313],[4.682622,44.020894],[4.683782,44.020057],[4.684685,44.019359],[4.68539,44.019573],[4.686525,44.019996],[4.688098,44.020467],[4.688746,44.020853],[4.689204,44.021567],[4.690618,44.022571],[4.69156,44.022772],[4.693503,44.023102],[4.697731,44.024315],[4.69905,44.024601],[4.701818,44.024954],[4.710607,44.026392],[4.71098,44.026341],[4.711287,44.026193],[4.716219,44.024648],[4.716464,44.024509],[4.716957,44.024321],[4.717234,44.023552],[4.718892,44.021041],[4.720255,44.01948],[4.723507,44.016485],[4.724713,44.015512],[4.725845,44.014189],[4.726033,44.014222],[4.726603,44.014483],[4.728755,44.015115],[4.730582,44.015708],[4.730831,44.015704],[4.73214,44.014054],[4.734508,44.013962],[4.738156,44.013337],[4.738476,44.013197],[4.741723,44.007186],[4.735342,43.999167],[4.735273,43.998952],[4.735017,43.998731],[4.727735,43.994652],[4.726975,43.994268],[4.726073,43.993697],[4.725187,43.993261],[4.725057,43.993083]]], + type:"Polygon" + }, 'correctly imported geometry as shape'); t.deepEqual(dbRecords[101772677].parent, { country: ['France'], country_id: ['85633147'], diff --git a/test/peliasDocGeneratorsTest.js b/test/peliasDocGeneratorsTest.js index 199e4807..06c72a79 100644 --- a/test/peliasDocGeneratorsTest.js +++ b/test/peliasDocGeneratorsTest.js @@ -257,9 +257,9 @@ tape('create', function(test) { var expected = [ new Document( 'whosonfirst', 'continent', '1' ) - .setName('default', 'name 1') - .setCentroid({ lat: 12.121212, lon: 21.212121 }) - .setBoundingBox({ upperLeft: { lat:31.313131, lon:12.121212 }, lowerRight: { lat:21.212121 , lon:13.131313 }}) + .setName('default', 'name 1') + .setCentroid({ lat: 12.121212, lon: 21.212121 }) + .setBoundingBox({ upperLeft: { lat:31.313131, lon:12.121212 }, lowerRight: { lat:21.212121 , lon:13.131313 }}) ]; // don't care about hierarchies in this test @@ -278,29 +278,105 @@ tape('create', function(test) { test.test('wofRecord without bounding_box should have undefined bounding box in output', function(t) { var wofRecords = { - 1: { - id: 1, - name: 'name 1', - name_aliases: [], - lat: 12.121212, - lon: 21.212121, - place_type: 'continent' - } + 1: { + id: 1, + name: 'name 1', + name_aliases: [], + lat: 12.121212, + lon: 21.212121, + place_type: 'continent' + } + }; + + var input = [ + wofRecords['1'] + ]; + + var expected = [ + new Document( 'whosonfirst', 'continent', '1' ) + .setName('default', 'name 1') + .setCentroid({ lat: 12.121212, lon: 21.212121 }) + ]; + + // don't care about hierarchies in this test + var hierarchies_finder = function() { + return []; + }; + + var docGenerator = peliasDocGenerators.create(hierarchies_finder); + + test_stream(input, docGenerator, function(err, actual) { + t.deepEqual(actual, expected, 'should have returned true'); + t.end(); + }); + + }); + + test.test('wofRecord with geometry should have shape in Document', function(t) { + var wofRecords = { + 1: { + id: 1, + name: 'name 1', + name_aliases: [], + lat: 12.121212, + lon: 21.212121, + bounding_box: '12.121212,21.212121,13.131313,31.313131', + geometry: {'coordinates':[[[-72.122,42.428],[-72.122,42.409],[-72.091,42.409],[-72.091,42.428],[-72.122,42.428]]],'type':'Polygon'}, + place_type: 'continent' + } + }; + + var input = [ + wofRecords['1'] + ]; + + var expected = [ + new Document( 'whosonfirst', 'continent', '1' ) + .setName('default', 'name 1') + .setCentroid({ lat: 12.121212, lon: 21.212121 }) + .setBoundingBox({ upperLeft: { lat:31.313131, lon:12.121212 }, lowerRight: { lat:21.212121 , lon:13.131313 }}) + .setShape({coordinates:[[[-72.122,42.428],[-72.122,42.409],[-72.091,42.409],[-72.091,42.428],[-72.122,42.428]]],type:"Polygon"}) + ]; + + // don't care about hierarchies in this test + var hierarchies_finder = function() { + return []; + }; + + var docGenerator = peliasDocGenerators.create(hierarchies_finder); + + test_stream(input, docGenerator, function(err, actual) { + t.deepEqual(actual, expected, 'should have returned true'); + t.end(); + }); + + }); + + test.test('wofRecord without geometry should have undefined shape in output', function(t) { + var wofRecords = { + 1: { + id: 1, + name: 'name 1', + name_aliases: [], + lat: 12.121212, + lon: 21.212121, + place_type: 'continent' + } }; var input = [ - wofRecords['1'] + wofRecords['1'] ]; var expected = [ - new Document( 'whosonfirst', 'continent', '1' ) - .setName('default', 'name 1') - .setCentroid({ lat: 12.121212, lon: 21.212121 }) + new Document( 'whosonfirst', 'continent', '1' ) + .setName('default', 'name 1') + .setCentroid({ lat: 12.121212, lon: 21.212121 }) ]; // don't care about hierarchies in this test var hierarchies_finder = function() { - return []; + return []; }; var docGenerator = peliasDocGenerators.create(hierarchies_finder); diff --git a/test/readStreamTest.js b/test/readStreamTest.js index 8aec9a84..2604df9d 100644 --- a/test/readStreamTest.js +++ b/test/readStreamTest.js @@ -31,7 +31,7 @@ tape('readStream', (test) => { ]); generateWOFDB(path.join(temp_dir, 'sqlite', 'whosonfirst-data-admin-xx-latest.db'), [ - { + { id: 123, properties: { 'wof:name': 'name 1', @@ -42,6 +42,10 @@ tape('readStream', (test) => { 'geom:bbox': '-13.691314,49.909613,1.771169,60.847886', 'gn:population': 98765, 'misc:photo_sum': 87654 + }, + geometry: { + coordinates: [[[-72.122, 42.428], [-72.122, 42.409], [-72.091, 42.409], [-72.091, 42.428], [-72.122, 42.428]]], + type: 'Polygon' } }, { @@ -90,6 +94,7 @@ tape('readStream', (test) => { lon: 21.212121, abbreviation: 'XY', bounding_box: '-13.691314,49.909613,1.771169,60.847886', + geometry: {'coordinates':[[[-72.122,42.428],[-72.122,42.409],[-72.091,42.409],[-72.091,42.428],[-72.122,42.428]]],type:'Polygon'}, population: 98765, popularity: 87654, hierarchies: [ @@ -107,6 +112,7 @@ tape('readStream', (test) => { lon: 31.313131, abbreviation: 'XY', bounding_box: '-24.539906,34.815009,69.033946,81.85871', + geometry: undefined, population: undefined, popularity: undefined, hierarchies: [ @@ -192,6 +198,7 @@ tape('readStream', (test) => { lat: 45.240295, lon: 3.916216, bounding_box: undefined, + geometry: undefined, population: undefined, popularity: undefined, hierarchies: [ { 'region_id': 421302191 } ], diff --git a/test/resources/france.json b/test/resources/france.json index 76f636af..4e24c665 100644 --- a/test/resources/france.json +++ b/test/resources/france.json @@ -5,6 +5,6 @@ {"id":404227861,"type":"Feature","properties":{"edtf:cessation":"uuuu","edtf:inception":"uuuu","geom:area":0.35206,"geom:area_square_m":3136211027.53961,"geom:bbox":"4.00134078086,43.4601854327,4.84534415757,44.341674883","geom:latitude":43.910218,"geom:longitude":4.423829,"iso:country":"FR","label:eng_x_preferred_longname":["Arrondissement of Nimes"],"label:eng_x_preferred_placetype":["arrondissement"],"label:fra_x_preferred_longname":["arrondissement de Nîmes"],"label:fra_x_preferred_placetype":["arrondissement"],"name:deu_x_preferred":["Arrondissement Nîmes"],"name:eng_x_preferred":["Nimes"],"name:eng_x_variant":["arrondissement of Nîmes"],"name:fra_x_preferred":["arrondissement de Nîmes"],"name:ita_x_preferred":["arrondissement di Nîmes"],"name:nld_x_preferred":["Arrondissement Nîmes"],"name:spa_x_preferred":["Distrito de Nîmes"],"wd:wordcount":2377,"wof:belongsto":[85683431,102191581,85633147,1108826387,136253037],"wof:breaches":[],"wof:concordances":{"wd:id":"Q702009"},"wof:country":"FR","wof:geomhash":"22cb8cfcff37fcc059691d958c618668","wof:hierarchy":[{"continent_id":102191581,"country_id":85633147,"empire_id":136253037,"macrocounty_id":404227861,"macroregion_id":1108826387,"region_id":85683431}],"wof:id":404227861,"wof:lastmodified":1563296151,"wof:name":"Nimes","wof:parent_id":85683431,"wof:placetype":"macrocounty","wof:repo":"whosonfirst-data-admin-fr","wof:superseded_by":[],"wof:supersedes":[],"wof:tags":[]}} , {"id":102068587,"type":"Feature","properties":{"edtf:cessation":"uuuu","edtf:inception":"uuuu","geom:area":0.017864,"geom:area_square_m":158756574.026902,"geom:bbox":"4.60483917319,43.9916304709,4.84311842532,44.1321854902","geom:latitude":44.053675,"geom:longitude":4.700463,"iso:country":"FR","lbl:latitude":44.050429,"lbl:longitude":4.680607,"mps:latitude":44.050429,"mps:longitude":4.680607,"name:deu_x_preferred":["Roquemaure"],"name:eng_x_preferred":["Roquemaure"],"name:fra_x_preferred":["Roquemaure"],"name:ita_x_preferred":["Roquemaure"],"name:nld_x_preferred":["Roquemaure"],"name:spa_x_preferred":["Roquemaure"],"wd:wordcount":24,"wof:belongsto":[85683431,102191581,85633147,404227861,1108826387,136253037],"wof:breaches":[],"wof:concordances":{"gp:id":12597193,"wd:id":"Q293958"},"wof:country":"FR","wof:geomhash":"218f43551cd3ca214069fe9cda361ea2","wof:hierarchy":[{"continent_id":102191581,"country_id":85633147,"county_id":102068587,"empire_id":136253037,"macrocounty_id":404227861,"macroregion_id":1108826387,"region_id":85683431}],"wof:id":102068587,"wof:lastmodified":1563275379,"wof:name":"Roquemaure","wof:parent_id":404227861,"wof:placetype":"county","wof:repo":"whosonfirst-data-admin-fr","wof:superseded_by":[],"wof:supersedes":[],"wof:tags":[]}} , {"id":404414453,"type":"Feature","properties":{"edtf:cessation":"uuuu","edtf:inception":"uuuu","geom:area":0.002249,"geom:area_square_m":20005440.423096,"geom:bbox":"4.63273757889,43.9905659369,4.74172258101,44.0263921976","geom:latitude":44.009261,"geom:longitude":4.692728,"iso:country":"FR","lbl:latitude":44.00898,"lbl:longitude":4.707933,"name:deu_x_preferred":["Tavel"],"name:fra_x_preferred":["Tavel"],"name:nld_x_preferred":["Tavel"],"wof:belongsto":[85683431,102191581,85633147,404227861,1108826387,136253037,102068587],"wof:breaches":[],"wof:concordances":{"gn:id":6431462,"gp:id":12671550,"qs_pg:id":945760},"wof:coterminous":[101772677],"wof:country":"FR","wof:geomhash":"533be5788201eceb1e49923f366f0421","wof:hierarchy":[{"continent_id":102191581,"country_id":85633147,"county_id":102068587,"empire_id":136253037,"localadmin_id":404414453,"macrocounty_id":404227861,"macroregion_id":1108826387,"region_id":85683431}],"wof:id":404414453,"wof:lastmodified":1564130533,"wof:name":"Tavel","wof:parent_id":102068587,"wof:placetype":"localadmin","wof:placetype_local":"Commune simple","wof:repo":"whosonfirst-data-admin-fr","wof:superseded_by":[],"wof:supersedes":[],"wof:tags":[]}} , - {"id":101772677,"type":"Feature","properties":{"edtf:cessation":"uuuu","edtf:inception":"uuuu","geom:area":0.002249,"geom:area_square_m":20005440.423096,"geom:bbox":"4.63273757889,43.9905659369,4.74172258101,44.0263921976","geom:latitude":44.009261,"geom:longitude":4.692728,"gn:population":1668,"iso:country":"FR","lbl:latitude":44.01144,"lbl:longitude":4.698562,"mps:latitude":44.01144,"mps:longitude":4.698562,"name:deu_x_preferred":["Tavel"],"name:eng_x_preferred":["Tavel"],"name:fra_x_preferred":["Tavel"],"name:ita_x_preferred":["Tavel"],"name:nld_x_preferred":["Tavel"],"name:spa_x_preferred":["Tavel"],"wd:wordcount":48,"wof:belongsto":[85683431,102191581,404227861,404414453,85633147,1108826387,136253037,102068587],"wof:breaches":[],"wof:concordances":{"fct:id":"0a5b4d46-8f76-11e1-848f-cfd5bf3ef515","gn:id":2973264,"gp:id":628145,"qs:id":1040667,"qs_pg:id":1040667,"wd:id":"Q334324","wk:page":"Priego"},"wof:coterminous":[404414453],"wof:country":"FR","wof:geomhash":"533be5788201eceb1e49923f366f0421","wof:hierarchy":[{"continent_id":102191581,"country_id":85633147,"county_id":102068587,"empire_id":136253037,"localadmin_id":404414453,"locality_id":101772677,"macrocounty_id":404227861,"macroregion_id":1108826387,"region_id":85683431}],"wof:id":101772677,"wof:lang":["fre"],"wof:lastmodified":1563293671,"wof:name":"Tavel","wof:parent_id":404414453,"wof:placetype":"locality","wof:population":1668,"wof:population_rank":3,"wof:repo":"whosonfirst-data-admin-fr","wof:superseded_by":[],"wof:supersedes":[],"wof:tags":[]}} + {"id":101772677,"type":"Feature","properties":{"edtf:cessation":"uuuu","edtf:inception":"uuuu","geom:area":0.002249,"geom:area_square_m":20005440.423096,"geom:bbox":"4.63273757889,43.9905659369,4.74172258101,44.0263921976","geom:latitude":44.009261,"geom:longitude":4.692728,"gn:population":1668,"iso:country":"FR","lbl:latitude":44.01144,"lbl:longitude":4.698562,"mps:latitude":44.01144,"mps:longitude":4.698562,"name:deu_x_preferred":["Tavel"],"name:eng_x_preferred":["Tavel"],"name:fra_x_preferred":["Tavel"],"name:ita_x_preferred":["Tavel"],"name:nld_x_preferred":["Tavel"],"name:spa_x_preferred":["Tavel"],"wd:wordcount":48,"wof:belongsto":[85683431,102191581,404227861,404414453,85633147,1108826387,136253037,102068587],"wof:breaches":[],"wof:concordances":{"fct:id":"0a5b4d46-8f76-11e1-848f-cfd5bf3ef515","gn:id":2973264,"gp:id":628145,"qs:id":1040667,"qs_pg:id":1040667,"wd:id":"Q334324","wk:page":"Priego"},"wof:coterminous":[404414453],"wof:country":"FR","wof:geomhash":"533be5788201eceb1e49923f366f0421","wof:hierarchy":[{"continent_id":102191581,"country_id":85633147,"county_id":102068587,"empire_id":136253037,"localadmin_id":404414453,"locality_id":101772677,"macrocounty_id":404227861,"macroregion_id":1108826387,"region_id":85683431}],"wof:id":101772677,"wof:lang":["fre"],"wof:lastmodified":1563293671,"wof:name":"Tavel","wof:parent_id":404414453,"wof:placetype":"locality","wof:population":1668,"wof:population_rank":3,"wof:repo":"whosonfirst-data-admin-fr","wof:superseded_by":[],"wof:supersedes":[],"wof:tags":[]},"geometry": {"coordinates":[[[4.725057,43.993083],[4.724003,43.993288],[4.722264,43.993505],[4.722074,43.993418],[4.721708,43.992028],[4.721318,43.991503],[4.720796,43.991151],[4.720609,43.991154],[4.719958,43.991894],[4.719771,43.991897],[4.719264,43.991634],[4.716917,43.99069],[4.716227,43.990566],[4.715231,43.990591],[4.71399,43.990799],[4.70856,43.992252],[4.707696,43.992545],[4.702638,43.993956],[4.696921,43.995836],[4.695008,43.996505],[4.691377,43.997705],[4.689834,43.998233],[4.688643,43.998035],[4.686647,43.997985],[4.684845,43.998201],[4.683678,43.998399],[4.681814,43.998617],[4.677906,43.999289],[4.67665,43.999426],[4.673878,43.999729],[4.671699,43.999817],[4.670635,43.999707],[4.669055,43.99938],[4.668244,43.999392],[4.665823,43.999762],[4.664979,44.000324],[4.663961,44.00133],[4.663292,44.001889],[4.663111,44.002117],[4.662806,44.002347],[4.662313,44.002534],[4.660767,44.002972],[4.659794,44.003392],[4.658683,44.003813],[4.653872,44.005183],[4.6485,44.006596],[4.643763,44.006685],[4.638028,44.006753],[4.636845,44.006825],[4.636898,44.008229],[4.636934,44.009038],[4.637026,44.011377],[4.63694,44.014025],[4.636841,44.016241],[4.636722,44.016423],[4.636357,44.016752],[4.636111,44.016891],[4.635235,44.01676],[4.634607,44.016644],[4.634132,44.016606],[4.633688,44.016783],[4.633382,44.016977],[4.632738,44.017122],[4.634274,44.017639],[4.635797,44.018111],[4.63688,44.018896],[4.637322,44.019069],[4.638263,44.019271],[4.642354,44.020092],[4.643425,44.020472],[4.643944,44.020735],[4.6452,44.021022],[4.646701,44.02117],[4.649925,44.020933],[4.650806,44.020775],[4.651978,44.020307],[4.656076,44.020083],[4.65726,44.020057],[4.658355,44.019536],[4.659106,44.019182],[4.662836,44.018766],[4.663153,44.01895],[4.663478,44.019395],[4.66418,44.019916],[4.664387,44.020182],[4.665148,44.020612],[4.665652,44.020784],[4.667087,44.020798],[4.667769,44.020653],[4.668745,44.020323],[4.669972,44.01962],[4.670781,44.019563],[4.671107,44.020044],[4.671424,44.020219],[4.671736,44.020214],[4.672179,44.020028],[4.672835,44.019423],[4.673147,44.019419],[4.675376,44.020537],[4.675817,44.02071],[4.676424,44.020116],[4.676649,44.019707],[4.677832,44.019635],[4.678795,44.02016],[4.68065,44.021293],[4.68084,44.02138],[4.681711,44.021313],[4.682622,44.020894],[4.683782,44.020057],[4.684685,44.019359],[4.68539,44.019573],[4.686525,44.019996],[4.688098,44.020467],[4.688746,44.020853],[4.689204,44.021567],[4.690618,44.022571],[4.69156,44.022772],[4.693503,44.023102],[4.697731,44.024315],[4.69905,44.024601],[4.701818,44.024954],[4.710607,44.026392],[4.71098,44.026341],[4.711287,44.026193],[4.716219,44.024648],[4.716464,44.024509],[4.716957,44.024321],[4.717234,44.023552],[4.718892,44.021041],[4.720255,44.01948],[4.723507,44.016485],[4.724713,44.015512],[4.725845,44.014189],[4.726033,44.014222],[4.726603,44.014483],[4.728755,44.015115],[4.730582,44.015708],[4.730831,44.015704],[4.73214,44.014054],[4.734508,44.013962],[4.738156,44.013337],[4.738476,44.013197],[4.741723,44.007186],[4.735342,43.999167],[4.735273,43.998952],[4.735017,43.998731],[4.727735,43.994652],[4.726975,43.994268],[4.726073,43.993697],[4.725187,43.993261],[4.725057,43.993083]]],"type":"Polygon"}} ] } \ No newline at end of file From c611c859228ecc0847108994e09d1b96e3626958 Mon Sep 17 00:00:00 2001 From: Loki Coyote Date: Sat, 23 Sep 2023 16:29:21 +0000 Subject: [PATCH 2/2] adding config value to control importing shapes --- schema.js | 2 ++ src/components/extractFields.js | 4 ++-- src/readStream.js | 2 +- test/components/extractFieldsTest.js | 2 +- test/functional.js | 2 +- test/readStreamTest.js | 1 + 6 files changed, 8 insertions(+), 5 deletions(-) diff --git a/schema.js b/schema.js index 4585d3fb..24502cd0 100644 --- a/schema.js +++ b/schema.js @@ -9,6 +9,7 @@ const Joi = require('@hapi/joi'); // * imports.whosonfirst.importPostalcodes (boolean) (default: false) // * imports.whosonfirst.importConstituencies (boolean) (default: false) // * imports.whosonfirst.importIntersections (boolean) (default: false) +// * imports.whosonfirst.importShapes (boolean) (default: false) // * imports.whosonfirst.importPlace (integer OR array[integer]) (default: none) // * imports.whosonfirst.sqlite (boolean) (default: true) @@ -27,6 +28,7 @@ module.exports = Joi.object().keys({ ], importPostalcodes: Joi.boolean().default(false).truthy('yes').falsy('no'), importConstituencies: Joi.boolean().default(false).truthy('yes').falsy('no'), + importShapes: Joi.boolean().default(false).truthy('yes').falsy('no'), maxDownloads: Joi.number().integer(), sqlite: Joi.boolean().default(true).truthy('yes').falsy('no') }).unknown(false) diff --git a/src/components/extractFields.js b/src/components/extractFields.js index 6a1085eb..70f4b7c0 100644 --- a/src/components/extractFields.js +++ b/src/components/extractFields.js @@ -224,7 +224,7 @@ function getConcordances(properties) { hierarchy-less object is added. If there are multiple hierarchies for the record then a record for each hierarchy is pushed onto the stream. */ -module.exports.create = function map_fields_stream() { +module.exports.create = function map_fields_stream(wofConfig) { return through2.obj(function(json_object, enc, callback) { const default_names = getName(json_object.properties); var record = { @@ -237,7 +237,7 @@ module.exports.create = function map_fields_stream() { lat: getLat(json_object.properties), lon: getLon(json_object.properties), bounding_box: getBoundingBox(json_object.properties), - geometry: getGeometry(json_object), + geometry: _.has(wofConfig, 'importShapes') && wofConfig.importShapes ? getGeometry(json_object) : undefined, population: getPopulation(json_object.properties), popularity: json_object.properties['misc:photo_sum'], hierarchies: getHierarchies(json_object.id, json_object.properties), diff --git a/src/readStream.js b/src/readStream.js index 6c78965e..46e6e943 100644 --- a/src/readStream.js +++ b/src/readStream.js @@ -53,7 +53,7 @@ function createReadStream(wofConfig, types, wofAdminRecords) { .pipe(toJSONStream.create()) .pipe(recordHasIdAndProperties.create()) .pipe(isActiveRecord.create()) - .pipe(extractFields.create()) + .pipe(extractFields.create(wofConfig)) .pipe(recordHasName.create()) .pipe(through2.obj(function(wofRecord, enc, callback) { // store admin records in memory to traverse the heirarchy diff --git a/test/components/extractFieldsTest.js b/test/components/extractFieldsTest.js index 9dd9f71b..fc38eac1 100644 --- a/test/components/extractFieldsTest.js +++ b/test/components/extractFieldsTest.js @@ -80,7 +80,7 @@ tape('readStreamComponents', function(test) { } ]; - test_stream(input, extractFields.create(), function(err, actual) { + test_stream(input, extractFields.create({importShapes: true}), function(err, actual) { t.deepEqual(actual, expected, 'stream should contain only objects with id and properties'); t.end(); }); diff --git a/test/functional.js b/test/functional.js index 2db1adb5..119cc1f4 100644 --- a/test/functional.js +++ b/test/functional.js @@ -15,7 +15,7 @@ tape('functional', function(test) { const wofAdminRecords = {}; const recordOrder = []; const dbRecords = {}; - const readStream = readStreamModule.create({ datapath: temp_dir, sqlite: true }, ['whosonfirst-data-latest.db'], wofAdminRecords); + const readStream = readStreamModule.create({ datapath: temp_dir, sqlite: true, importShapes: true }, ['whosonfirst-data-latest.db'], wofAdminRecords); const documentGenerator = peliasDocGenerators.create(hierarchyFinder(wofAdminRecords)); const dbClientStream = sink.obj(record => { recordOrder.push(record.data.source_id); diff --git a/test/readStreamTest.js b/test/readStreamTest.js index 2604df9d..8c7acbb9 100644 --- a/test/readStreamTest.js +++ b/test/readStreamTest.js @@ -73,6 +73,7 @@ tape('readStream', (test) => { const wofConfig = { datapath: temp_dir, + importShapes: true, sqlite: true };