diff --git a/jobs/bcgw_sources.json b/jobs/bcgw_sources.json index 7f1ec7d8..4304209a 100644 --- a/jobs/bcgw_sources.json +++ b/jobs/bcgw_sources.json @@ -2,111 +2,139 @@ { "source": "whse_fish.fiss_fish_obsrvtn_pnt_sp", "query": "POINT_TYPE_CODE = 'Observation'", - "schedule": "W" + "schedule": "W", + "primary_key": "" + }, + { + "source": "whse_fish.pscis_assessment_svw", + "query": "", + "schedule": "W", + "primary_key": "stream_crossing_id" }, { "source": "whse_fish.fiss_obstacles_pnt_sp", "query": "", - "schedule": "W" + "schedule": "W", + "primary_key": "" }, { "source": "whse_fish.pscis_design_proposal_svw", "query": "", - "schedule": "W" + "schedule": "W", + "primary_key": "stream_crossing_id" }, { "source": "whse_fish.pscis_habitat_confirmation_svw", "query": "", - "schedule": "W" + "schedule": "W", + "primary_key": "stream_crossing_id" }, { "source": "whse_fish.pscis_remediation_svw", "query": "", - "schedule": "W" + "schedule": "W", + "primary_key": "stream_crossing_id" }, { "source": "whse_admin_boundaries.clab_indian_reserves", "query": "", - "schedule": "M" + "schedule": "M", + "primary_key": "clab_id" }, { "source": "whse_admin_boundaries.clab_national_parks", "query": "", - "schedule": "M" + "schedule": "M", + "primary_key": "national_park_id" }, { "source": "whse_basemapping.gba_local_reg_greenspaces_sp", "query": "", - "schedule": "M" + "schedule": "M", + "primary_key": "local_reg_greenspace_id" }, { "source": "whse_basemapping.gba_railway_structure_lines_sp", "query": "", - "schedule": "M" + "schedule": "M", + "primary_key": "railway_structure_line_id" }, { "source": "whse_basemapping.gba_railway_tracks_sp", "query": "", - "schedule": "M" + "schedule": "M", + "primary_key": "railway_track_id" }, { "source": "whse_basemapping.gba_transmission_lines_sp", "query": "", - "schedule": "M" + "schedule": "M", + "primary_key": "transmission_line_id" }, { "source": "whse_basemapping.gns_geographical_names_sp", "query": "", - "schedule": "M" + "schedule": "M", + "primary_key": "geographical_names_id" }, { "source": "whse_environmental_monitoring.envcan_hydrometric_stn_sp", "query": "", - "schedule": "M" + "schedule": "M", + "primary_key": "hydrometric_station_id" }, { "source": "whse_fish.fiss_stream_sample_sites_sp", "query": "", - "schedule": "M" + "schedule": "M", + "primary_key": "stream_sample_site_id" }, { "source": "whse_forest_tenure.ften_range_poly_svw", "query": "", - "schedule": "M" + "schedule": "M", + "primary_key": "objectid" }, { "source": "whse_imagery_and_base_maps.mot_road_structure_sp", "query": "", - "schedule": "M" + "schedule": "M", + "primary_key": "hwy_structure_class_id" }, { "source": "whse_legal_admin_boundaries.abms_municipalities_sp", "query": "", - "schedule": "M" + "schedule": "M", + "primary_key": "lgl_admin_area_id" }, { "source": "whse_mineral_tenure.og_petrlm_dev_rds_pre06_pub_sp", "query": "", - "schedule": "M" + "schedule": "M", + "primary_key": "og_petrlm_dev_rd_pre06_pub_id" }, { "source": "whse_tantalis.ta_conservancy_areas_svw", "query": "", - "schedule": "M" + "schedule": "M", + "primary_key": "admin_area_sid" }, { "source": "whse_tantalis.ta_park_ecores_pa_svw", "query": "", - "schedule": "M" + "schedule": "M", + "primary_key": "admin_area_sid" }, { "source": "whse_forest_tenure.ften_road_section_lines_svw", "query": "", - "schedule": "M" + "schedule": "M", + "primary_key": "" }, { "source": "whse_mineral_tenure.og_road_segment_permit_sp", "query": "", - "schedule": "M" + "schedule": "M", + "primary_key": "og_road_segment_permit_id" } ] \ No newline at end of file diff --git a/jobs/replicate_bcgw b/jobs/replicate_bcgw index 02f89c11..ce3233da 100755 --- a/jobs/replicate_bcgw +++ b/jobs/replicate_bcgw @@ -6,12 +6,14 @@ jq -c '.[]' bcgw_sources.json | while read item; do schedule=$(jq -r '.schedule' <<< "$item") source=$(jq -r '.source' <<< "$item") query=$(jq -r '.query' <<< "$item") + pk=$(jq -r '.primary_key' <<< "$item") if [ "$schedule" == $1 ] ; then echo "Replicating $source to object storage" bcdata dump --promote-to-multi -l --query "$query" -v $source | ogr2ogr -f Parquet \ + -lco FID=$pk \ /vsis3/bchamp/bcdata/$source.parquet \ /vsistdin/