add primary key to bcgw download definitions to enable ogr fid preser…

…vation
smnorris · Dec 18, 2024 · 2109f70 · 2109f70
1 parent 4bc97f4
commit 2109f70
Show file tree

Hide file tree

Showing 2 changed files with 52 additions and 22 deletions.
diff --git a/jobs/bcgw_sources.json b/jobs/bcgw_sources.json
@@ -2,111 +2,139 @@
     {
         "source": "whse_fish.fiss_fish_obsrvtn_pnt_sp",
         "query": "POINT_TYPE_CODE = 'Observation'",
-        "schedule": "W"
+        "schedule": "W",
+        "primary_key": ""
+    },
+    {
+        "source": "whse_fish.pscis_assessment_svw",
+        "query": "",
+        "schedule": "W",
+        "primary_key": "stream_crossing_id"
     },
     {
         "source": "whse_fish.fiss_obstacles_pnt_sp",
         "query": "",
-        "schedule": "W"
+        "schedule": "W",
+        "primary_key": ""
     },
     {
         "source": "whse_fish.pscis_design_proposal_svw",
         "query": "",
-        "schedule": "W"
+        "schedule": "W",
+        "primary_key": "stream_crossing_id"
     },
     {
         "source": "whse_fish.pscis_habitat_confirmation_svw",
         "query": "",
-        "schedule": "W"
+        "schedule": "W",
+        "primary_key": "stream_crossing_id"
     },
     {
         "source": "whse_fish.pscis_remediation_svw",
         "query": "",
-        "schedule": "W"
+        "schedule": "W",
+        "primary_key": "stream_crossing_id"
     },
     {
         "source": "whse_admin_boundaries.clab_indian_reserves",
         "query": "",
-        "schedule": "M"
+        "schedule": "M",
+        "primary_key": "clab_id"
     },
     {
         "source": "whse_admin_boundaries.clab_national_parks",
         "query": "",
-        "schedule": "M"
+        "schedule": "M",
+        "primary_key": "national_park_id"
     },
     {
         "source": "whse_basemapping.gba_local_reg_greenspaces_sp",
         "query": "",
-        "schedule": "M"
+        "schedule": "M",
+        "primary_key": "local_reg_greenspace_id"
     },
     {
         "source": "whse_basemapping.gba_railway_structure_lines_sp",
         "query": "",
-        "schedule": "M"
+        "schedule": "M",
+        "primary_key": "railway_structure_line_id"
     },
     {
         "source": "whse_basemapping.gba_railway_tracks_sp",
         "query": "",
-        "schedule": "M"
+        "schedule": "M",
+        "primary_key": "railway_track_id"
     },
     {
         "source": "whse_basemapping.gba_transmission_lines_sp",
         "query": "",
-        "schedule": "M"
+        "schedule": "M",
+        "primary_key": "transmission_line_id"
     },
     {
         "source": "whse_basemapping.gns_geographical_names_sp",
         "query": "",
-        "schedule": "M"
+        "schedule": "M",
+        "primary_key": "geographical_names_id"
     },
     {
         "source": "whse_environmental_monitoring.envcan_hydrometric_stn_sp",
         "query": "",
-        "schedule": "M"
+        "schedule": "M",
+        "primary_key": "hydrometric_station_id"
     },
     {
         "source": "whse_fish.fiss_stream_sample_sites_sp",
         "query": "",
-        "schedule": "M"
+        "schedule": "M",
+        "primary_key": "stream_sample_site_id"
     },
     {
         "source": "whse_forest_tenure.ften_range_poly_svw",
         "query": "",
-        "schedule": "M"
+        "schedule": "M",
+        "primary_key": "objectid"
     },
     {
         "source": "whse_imagery_and_base_maps.mot_road_structure_sp",
         "query": "",
-        "schedule": "M"
+        "schedule": "M",
+        "primary_key": "hwy_structure_class_id"
     },
     {
         "source": "whse_legal_admin_boundaries.abms_municipalities_sp",
         "query": "",
-        "schedule": "M"
+        "schedule": "M",
+        "primary_key": "lgl_admin_area_id"
     },
     {
         "source": "whse_mineral_tenure.og_petrlm_dev_rds_pre06_pub_sp",
         "query": "",
-        "schedule": "M"
+        "schedule": "M",
+        "primary_key": "og_petrlm_dev_rd_pre06_pub_id"
     },
     {
         "source": "whse_tantalis.ta_conservancy_areas_svw",
         "query": "",
-        "schedule": "M"
+        "schedule": "M",
+        "primary_key": "admin_area_sid"
     },
     {
         "source": "whse_tantalis.ta_park_ecores_pa_svw",
         "query": "",
-        "schedule": "M"
+        "schedule": "M",
+        "primary_key": "admin_area_sid"
     },
     {
         "source": "whse_forest_tenure.ften_road_section_lines_svw",
         "query": "",
-        "schedule": "M"
+        "schedule": "M",
+        "primary_key": ""
     },
     {
         "source": "whse_mineral_tenure.og_road_segment_permit_sp",
         "query": "",
-        "schedule": "M"
+        "schedule": "M",
+        "primary_key": "og_road_segment_permit_id"
     }
 ]
diff --git a/jobs/replicate_bcgw b/jobs/replicate_bcgw
@@ -6,12 +6,14 @@ jq -c '.[]' bcgw_sources.json | while read item; do
     schedule=$(jq -r '.schedule' <<< "$item")
     source=$(jq -r '.source' <<< "$item")
     query=$(jq -r '.query' <<< "$item")
+    pk=$(jq -r '.primary_key' <<< "$item")
     if [ "$schedule" == $1 ] ; then
 
       echo "Replicating $source to object storage"
 
       bcdata dump --promote-to-multi -l --query "$query" -v $source |
         ogr2ogr -f Parquet \
+          -lco FID=$pk \
           /vsis3/bchamp/bcdata/$source.parquet \
           /vsistdin/