Luxembourg TRP-VC Streets Dataset (#107)

* Initial commit for Luxembourg

* Conversion now considers residential highways

* README for Luxembourg

data/luxembourg/README.md

@@ -0,0 +1,37 @@
+# Road Completion - 🇱🇺 Luxembourg
+
+Source: <https://data.public.lu/fr/datasets/transport-et-voies-de-communication/>
+
+Complete description of all attributes: <https://data.public.lu/fr/datasets/r/d2d92fc6-1cfa-4c97-8e41-31e78f443099>
+
+## Filters
+
+```sql
+"NOM_RUE" <> '' AND ("TRP_CLAS_L" <> 'Chemin de fer' OR ("TRP_CLAS_L" <> 'Non codé' AND "TRP_CLAS_L" <> 'autre'))
+```
+
+Filtering out features without a name, and features which aren't streets (train tracks, forest tracks, and unknown classification).
+
+| Value | Description (original) | Description (translated) |
+|-------|------------------------|--------------------------|
+| NOM_RUE | Nom de la rue  | Street name |
+| TRP_CLAS_L | Classification | Street Classification |
+
+## Tags
+The tagging scheme follows the [convention used in Luxembourg](https://wiki.openstreetmap.org/wiki/WikiProject_Luxembourg/Roads#Overview).
+
+### `TRP_CLAS_L` (*TRP_CLASS_LOOKUP*)
+| Value | Description | OSM Tags |
+|-------|-------------|----------|
+| Autoroute | Highway | `highway=motorway` |
+| Nationale | National road | `highway=primary` |
+| Chemin repris | State-managed road | `highway=secondary` |
+| Chemin vicinal | Residential road | `highway=residential` |
+| Chemin de fer | Train tracks | |
+| Non codé | Tracks | |
+| autre | Other | |
+
+### `TRP_NIVE_L` (*TRP_NIVEAU_LOOKUP*)
+| Value | Description | OSM Tags |
+|-------|-------------|----------|
+|passage supérieur 1|Bridge|`bridge=yes` and `layer=1`|

data/luxembourg/build-extended-conversion.js

@@ -0,0 +1,39 @@
+'use strict'
+
+const fs = require('fs');
+
+var args = process.argv.slice(2);
+const sourceA = args[0];
+const sourceB = args[1];
+const append = args[2];
+const target = args[3];
+
+function convert_csv_tuples_to_json(filePath) {
+    const data = fs.readFileSync(filePath, 'utf8');
+    let inputRows = data.split(/\r?\n/);
+
+    let result = {}
+
+    inputRows.forEach(element => {
+        if(element.charAt(0) === "#") {
+            return;
+        }
+        var [id, name] = element.split(/\t/);
+        result[id] = {"name": name};
+    });
+
+    return result;
+}
+
+let normalizeNames = { 
+    NOM_RUE: convert_csv_tuples_to_json(sourceA),
+    ID_RUE_CAC: convert_csv_tuples_to_json(sourceB)
+};
+
+const convertTags = JSON.parse(fs.readFileSync(append, 'utf8'));
+let data = JSON.stringify({
+    ...convertTags,
+    ...normalizeNames
+});
+
+fs.writeFileSync(target, data);

data/luxembourg/convert.json

@@ -0,0 +1,22 @@
+{
+  "TRP_CLAS_L": {
+    "Autoroute": {
+      "highway": "motorway"
+    },
+    "Nationale": {
+      "highway": "primary"
+    },
+    "Chemin repris": {
+      "highway": "secondary"
+    },
+    "Chemin vicinal": {
+      "highway": "residential"
+    }
+  },
+  "TRP_NIVE_L": {
+    "passage supérieur 1": {
+      "bridge": "yes",
+      "layer": "1"
+    }
+  }
+}

data/luxembourg/create-uid.js

@@ -0,0 +1,26 @@
+'use strict'
+
+const fs = require('fs');
+const crypto = require('crypto');
+
+var args = process.argv.slice(2);
+const source = args[0];
+const target = args[1];
+
+const data = JSON.parse(fs.readFileSync(source, 'utf8'));
+
+data['features'].forEach(element => {
+    let uid = hashProperties(element['geometry']);
+    element['properties']['original:uid'] = uid;
+});
+
+function hashProperties(properties) {
+    let mergedCoordinates = "";
+    properties.coordinates.forEach(element => {
+      mergedCoordinates += element[0] + element[1];
+    });
+    let hash = crypto.createHash('md5').update(mergedCoordinates).digest('hex');
+    return hash;
+}
+
+fs.writeFileSync(target, JSON.stringify(data));

data/luxembourg/filter.sql

@@ -0,0 +1,2 @@
+SELECT * FROM "TRP_VC" WHERE
+  "NOM_RUE" <> '' AND ("TRP_CLAS_L" <> 'Chemin de fer' OR ("TRP_CLAS_L" <> 'Non codé' AND "TRP_CLAS_L" <> 'autre'))

data/luxembourg/luxembourg.sh

@@ -0,0 +1,37 @@
+#!/bin/sh
+
+# Make script directory working directory
+
+cd `dirname "$(realpath $0)"`
+
+# Download Luxembourg extract
+
+if [ -f "luxembourg-latest.osm.pbf" ]; then rm "luxembourg-latest.osm.pbf"; fi
+
+wget https://download.geofabrik.de/europe/luxembourg-latest.osm.pbf
+
+# Convert to GeoJSON
+
+if [ -f "./luxembourg-lines.geojson" ]; then rm "./luxembourg-lines.geojson"; fi
+if [ -f "./luxembourg-polygons.geojson" ]; then rm "./luxembourg-polygons.geojson"; fi
+
+ogr2ogr -f "GeoJSON" -progress \
+  -sql "SELECT name, highway FROM lines WHERE highway IS NOT NULL" \
+  "./luxembourg-lines.geojson" \
+  "./luxembourg-latest.osm.pbf"
+ogr2ogr -f "GeoJSON" -progress \
+  -sql "SELECT name, hstore_get_value(other_tags, 'highway') AS highway, place FROM multipolygons WHERE (hstore_get_value(other_tags, 'highway') is not null) OR (place = 'square')" \
+  "./luxembourg-polygons.geojson" \
+  "./luxembourg-latest.osm.pbf"
+
+# Generate buffer
+
+node "../../script/buffer.js" --radius=20 "./luxembourg-lines.geojson" "luxembourg-lines-buffers.geojson"
+node "../../script/buffer.js" --radius=5 "./luxembourg-polygons.geojson" "luxembourg-polygons-buffers.geojson"
+
+# Generate vector tiles
+
+tippecanoe --force --no-feature-limit --no-tile-size-limit \
+  --maximum-zoom=14 --minimum-zoom=14 \
+  --layer="buffers" \
+  --output="./luxembourg-buffers.mbtiles" "./luxembourg-lines-buffers.geojson" "./luxembourg-polygons-buffers.geojson"

data/luxembourg/process.sh

@@ -0,0 +1,80 @@
+#!/bin/sh
+
+FILENAME="transport-et-voies-de-communication-shape"
+MAPROULETTE_CHALLENGE=17749
+
+# Make script directory working directory
+
+cd `dirname "$(realpath $0)"`
+
+# Download & unzip data
+
+mkdir -p "./source/"
+
+if [ ! -d "./source/$FILENAME" ]; then
+  wget -O "./source/$FILENAME.zip" "https://data.public.lu/fr/datasets/r/e74aadad-77c2-441e-98fe-e08a441484a2"
+  unzip -j "./source/$FILENAME.zip" -d "./source/$FILENAME/" "TRP_VC.*"
+fi
+
+# Convert to GeoJSON
+
+if [ -d "./temp" ]; then rm -r "./temp/"; fi
+
+mkdir -p "./temp/"
+
+ogr2ogr -f "GeoJSON" -progress \
+  --config SHAPE_ENCODING "ISO-8859-1" \
+  -s_srs "EPSG:2169" -t_srs "EPSG:4326" \
+  -sql "@filter.sql" \
+  -lco COORDINATE_PRECISION=6 \
+  -fieldTypeToString "All" \
+  "./temp/trpvc.geojson" \
+  "./source/$FILENAME/TRP_VC.shp"
+
+
+# Before conferting the fields into OpenStreetMap tags, we extend convert.json with our street name standardization
+# Download our community-driven street name standardization tuples
+
+wget -O "./temp/csventrifuge_enhance_name.csv" "https://raw.githubusercontent.com/osmlu/csventrifuge/master/rules/luxembourg_addresses/rue.csv"
+wget -O "./temp/csventrifuge_enhance_id.csv" "https://raw.githubusercontent.com/osmlu/csventrifuge/master/enhance/luxembourg_addresses/id_caclr_rue/rue.csv"
+
+#  Convert the tuples into JSON and merge them with convert.json
+
+node "./build-extended-conversion.js" "./temp/csventrifuge_enhance_name.csv" "./temp/csventrifuge_enhance_id.csv" "convert.json" "./temp/convert-merged.json"
+
+# Finally, convert fields to OpenStreetMap tags
+
+node "../../script/convert-tags.js" -c "./temp/convert-merged.json" "./temp/trpvc.geojson" "trpvcTagged.geojson"
+
+# Generate vector tiles
+
+tippecanoe --force --no-feature-limit --no-tile-size-limit \
+  --buffer=0 \
+  --maximum-zoom=14 --minimum-zoom=14 \
+  --layer="roads" \
+  --output="./temp/trpvcTagged.mbtiles" "./temp/trpvcTagged.geojson"
+
+# Generate MapRoulette NotAnIssue buffers vector tiles
+
+wget -O "./temp/maproulette.geojson" "https://maproulette.org/api/v2/challenge/view/$MAPROULETTE_CHALLENGE?status=2"
+
+node "../../script/buffer.js" --radius=20 "./temp/maproulette.geojson" "maproulette-buffers.geojson"
+
+# Merge MapRoulette buffers to OpenStreetMap buffers
+
+tippecanoe --force --no-feature-limit --no-tile-size-limit \
+  --maximum-zoom=14 --minimum-zoom=14 \
+  --layer="buffers" \
+  --output="./temp/luxembourg-buffers.mbtiles" \
+  "./luxembourg-lines-buffers.geojson" "./luxembourg-polygons-buffers.geojson" "./temp/maproulette-buffers.geojson"
+
+# Difference
+
+if [ -d "./difference" ]; then rm -r "./difference/"; fi
+
+mkdir -p "./difference"
+
+node "../../script/difference.js" --output-dir="./difference" "./temp/trpvcTagged.mbtiles" "./temp/luxembourg-buffers.mbtiles"
+
+# Our dataset doesn't have unique identifiers for segments of the same road, so we create one based on the geometry
+node "./create-uid.js" "./difference/diff.geojson" "./difference/diff.geojson"