diff --git a/.github/workflows/production-bcfishpass.yaml b/.github/workflows/production-bcfishpass.yaml index 0ff93c68..5e088ed4 100644 --- a/.github/workflows/production-bcfishpass.yaml +++ b/.github/workflows/production-bcfishpass.yaml @@ -24,8 +24,10 @@ jobs: steps: - name: Check out repository code uses: actions/checkout@v4 - - name: Run the jobs - run: jobs/bcfishpass01_modelledcrossings + - name: Build modelled crossings + run: jobs/bcfishpass01_modelled_stream_crossings + - name: Release modelled crossings + run: jobs/release_modelled_stream_crossings prep: runs-on: ubuntu-latest environment: staging @@ -60,4 +62,4 @@ jobs: - name: Check out repository code uses: actions/checkout@v4 - name: Run the jobs - run: jobs/freshwater_fish_habitat_accessibility_model \ No newline at end of file + run: jobs/release_freshwater_fish_habitat_accessibility_model \ No newline at end of file diff --git a/jobs/bcfishpass01_modelledcrossings b/jobs/bcfishpass01_modelled_stream_crossings similarity index 100% rename from jobs/bcfishpass01_modelledcrossings rename to jobs/bcfishpass01_modelled_stream_crossings diff --git a/jobs/cabd b/jobs/cabd deleted file mode 100755 index c87bbb75..00000000 --- a/jobs/cabd +++ /dev/null @@ -1,26 +0,0 @@ -#!/bin/bash -set -euxo pipefail - -#------- -# download CABD dams and falls -#------- - -PSQL="psql $DATABASE_URL -v ON_ERROR_STOP=1" - -$PSQL -c "truncate cabd.dams" -ogr2ogr -f PostgreSQL \ - "PG:$DATABASE_URL" \ - -append \ - -nln cabd.dams \ - "https://cabd-web.azurewebsites.net/cabd-api/features/dams?filter=province_territory_code:eq:bc&filter=use_analysis:eq:true" \ - OGRGeoJSON - -$PSQL -c "truncate cabd.waterfalls" -ogr2ogr -f PostgreSQL \ - "PG:$DATABASE_URL" \ - -append \ - -nln cabd.waterfalls \ - "https://cabd-web.azurewebsites.net/cabd-api/features/waterfalls?filter=province_territory_code:eq:bc&filter=use_analysis:eq:true" \ - OGRGeoJSON - -# refresh views linking CABD features to FWA streams here? \ No newline at end of file diff --git a/jobs/load_monthly b/jobs/load_monthly index 2f6a5e34..1708a145 100755 --- a/jobs/load_monthly +++ b/jobs/load_monthly @@ -8,6 +8,7 @@ set -euxo pipefail PSQL="psql $DATABASE_URL -v ON_ERROR_STOP=1" # bcdata loads +# ---- bcdata bc2pg -r whse_admin_boundaries.clab_indian_reserves bcdata bc2pg -r whse_admin_boundaries.clab_national_parks bcdata bc2pg -r whse_basemapping.gba_local_reg_greenspaces_sp @@ -24,10 +25,61 @@ bcdata bc2pg -r whse_mineral_tenure.og_petrlm_dev_rds_pre06_pub_sp bcdata bc2pg -r whse_tantalis.ta_conservancy_areas_svw bcdata bc2pg -r whse_tantalis.ta_park_ecores_pa_svw -# other loads -jobs/whse_cadastre.pmbc_parcel_fabric_poly_svw +# parcel fabric, renaming download so it can be read without unzipping +# ---- +curl \ + -o /tmp/pmbc_parcel_fabric_poly_svw.gdb.zip \ + https://pub.data.gov.bc.ca/datasets/4cf233c2-f020-4f7a-9b87-1923252fbc24/pmbc_parcel_fabric_poly_svw.zip + +ogr2ogr \ + -f PostgreSQL \ + PG:$DATABASE_URL \ + --config PG_USE_COPY YES \ + -overwrite \ + -t_srs EPSG:3005 \ + -dim XY \ + -lco GEOMETRY_NAME=geom \ + -lco FID=PARCEL_FABRIC_POLY_ID \ + -nln bcdata.pmbc_parcel_fabric_poly_svw \ + /tmp/pmbc_parcel_fabric_poly_svw.gdb.zip \ + pmbc_parcel_fabric_poly_svw + +# load to target & drop temp (note that schemas do not quite match) +$PSQL -c "truncate whse_cadastre.pmbc_parcel_fabric_poly_svw; + insert into whse_cadastre.pmbc_parcel_fabric_poly_svw ( + parcel_fabric_poly_id , + parcel_name , + plan_number , + pin , + pid_number , + parcel_status , + parcel_class , + owner_type , + parcel_start_date , + municipality , + regional_district , + when_updated , + geom + ) + select + parcel_fabric_poly_id , + parcel_name , + plan_number , + pin , + pid , + parcel_status , + parcel_class , + owner_type , + parcel_start_date , + municipality , + regional_district , + when_updated , + geom + from bcdata.pmbc_parcel_fabric_poly_svw" +$PSQL -c "drop table bcdata.pmbc_parcel_fabric_poly_svw" # additional processing +# ---- $PSQL -c "refresh materialized view bcdata.parks" # range view is too resource intensive for some clients - schedule this job elsewhere #$PSQL -c "refresh materialized view bcdata.ften_range_poly_carto_vw" diff --git a/jobs/load_weekly b/jobs/load_weekly index dc38f5ae..457603fd 100755 --- a/jobs/load_weekly +++ b/jobs/load_weekly @@ -7,6 +7,9 @@ set -euxo pipefail PSQL="psql $DATABASE_URL -v ON_ERROR_STOP=1" +# *note* +# weekly bcfishobs processing is scheduled via workflow bcfishobs repository + # bcdata loads bcdata bc2pg -r whse_fish.fiss_fish_obsrvtn_pnt_sp --query "POINT_TYPE_CODE = 'Observation'" bcdata bc2pg -r whse_fish.fiss_obstacles_pnt_sp @@ -17,8 +20,33 @@ bcdata bc2pg -r whse_fish.pscis_remediation_svw bcdata bc2pg -r whse_forest_tenure.ften_road_section_lines_svw bcdata bc2pg -r whse_mineral_tenure.og_road_segment_permit_sp -# other loads -jobs/cabd -jobs/whse_basemapping.transport_line +# cabd +$PSQL -c "truncate cabd.dams" +ogr2ogr -f PostgreSQL \ + "PG:$DATABASE_URL" \ + -append \ + -nln cabd.dams \ + "https://cabd-web.azurewebsites.net/cabd-api/features/dams?filter=province_territory_code:eq:bc&filter=use_analysis:eq:true" \ + OGRGeoJSON + +$PSQL -c "truncate cabd.waterfalls" +ogr2ogr -f PostgreSQL \ + "PG:$DATABASE_URL" \ + -append \ + -nln cabd.waterfalls \ + "https://cabd-web.azurewebsites.net/cabd-api/features/waterfalls?filter=province_territory_code:eq:bc&filter=use_analysis:eq:true" \ + OGRGeoJSON -# note that weekly bcfishobs processing is scheduled on bcfishpass db using bcfishobs repository \ No newline at end of file +# DRA +ogr2ogr \ + -f PostgreSQL \ + "PG:$DATABASE_URL" \ + --config PG_USE_COPY YES \ + -lco GEOMETRY_NAME=geom \ + -lco FID=transport_line_id \ + -nln bcdata.transport_line \ + /vsizip//vsicurl/ftp://ftp.geobc.gov.bc.ca/sections/outgoing/bmgs/DRA_Public/dgtl_road_atlas.gdb.zip \ + TRANSPORT_LINE +$PSQL -c "truncate whse_basemapping.transport_line; + insert into whse_basemapping.transport_line select * from bcdata.transport_line" +$PSQL -c "drop table bcdata.transport_line" diff --git a/jobs/freshwater_fish_habitat_accessibility_model b/jobs/release_freshwater_fish_habitat_accessibility_model similarity index 100% rename from jobs/freshwater_fish_habitat_accessibility_model rename to jobs/release_freshwater_fish_habitat_accessibility_model diff --git a/jobs/release_modelled_stream_crossings b/jobs/release_modelled_stream_crossings new file mode 100644 index 00000000..f2ac1b09 --- /dev/null +++ b/jobs/release_modelled_stream_crossings @@ -0,0 +1,34 @@ +#!/bin/bash +set -euxo pipefail + +#------- +# release modelled crossings +#------- + +ogr2ogr \ + -f GPKG \ + modelled_stream_crossings.gpkg.zip \ + PG:$DATABASE_URL \ + -nlt PointZM \ + -nln modelled_stream_crossings \ + -FID modelled_crossing_id \ + -sql "select \ + modelled_crossing_id, \ + modelled_crossing_type, \ + array_to_string(modelled_crossing_type_source, '; ') as modelled_crossing_type_source, \ + transport_line_id, \ + ften_road_section_lines_id, \ + og_road_segment_permit_id, \ + og_petrlm_dev_rd_pre06_pub_id, \ + railway_track_id, \ + linear_feature_id, \ + blue_line_key, \ + downstream_route_measure, \ + wscode_ltree, \ + localcode_ltree, \ + watershed_group_code, \ + geom \ + from bcfishpass.modelled_stream_crossings" +aws s3 cp modelled_stream_crossings.gpkg.zip s3://bcfishpass/modelled_stream_crossings.gpkg.zip +echo "Latest modelled crossings data now available at:" +echo "https://bcfishpass.s3.us-west-2.amazonaws.com/modelled_stream_crossings.gpkg.zip" \ No newline at end of file diff --git a/jobs/whse_basemapping.transport_line b/jobs/whse_basemapping.transport_line deleted file mode 100755 index 46808db3..00000000 --- a/jobs/whse_basemapping.transport_line +++ /dev/null @@ -1,21 +0,0 @@ -#!/bin/bash -set -euxo pipefail - -#------- -# DRA - (comes from FTP for download efficiency, plus data is a bit richer than in BCGW) -#------- - -PSQL="psql $DATABASE_URL -v ON_ERROR_STOP=1" - -ogr2ogr \ - -f PostgreSQL \ - "PG:$DATABASE_URL" \ - --config PG_USE_COPY YES \ - -lco GEOMETRY_NAME=geom \ - -lco FID=transport_line_id \ - -nln bcdata.transport_line \ - /vsizip//vsicurl/ftp://ftp.geobc.gov.bc.ca/sections/outgoing/bmgs/DRA_Public/dgtl_road_atlas.gdb.zip \ - TRANSPORT_LINE -$PSQL -c "truncate whse_basemapping.transport_line; - insert into whse_basemapping.transport_line select * from bcdata.transport_line" -$PSQL -c "drop table bcdata.transport_line" \ No newline at end of file diff --git a/jobs/whse_cadastre.pmbc_parcel_fabric_poly_svw b/jobs/whse_cadastre.pmbc_parcel_fabric_poly_svw deleted file mode 100755 index c08cb404..00000000 --- a/jobs/whse_cadastre.pmbc_parcel_fabric_poly_svw +++ /dev/null @@ -1,61 +0,0 @@ -#!/bin/bash -set -euxo pipefail - -#------- -# Parcel fabric -#------- - -PSQL="psql $DATABASE_URL -v ON_ERROR_STOP=1" - -# rename downloaded file so ogr can read without unzipping -curl \ - -o /tmp/pmbc_parcel_fabric_poly_svw.gdb.zip \ - https://pub.data.gov.bc.ca/datasets/4cf233c2-f020-4f7a-9b87-1923252fbc24/pmbc_parcel_fabric_poly_svw.zip - -ogr2ogr \ - -f PostgreSQL \ - PG:$DATABASE_URL \ - --config PG_USE_COPY YES \ - -overwrite \ - -t_srs EPSG:3005 \ - -dim XY \ - -lco GEOMETRY_NAME=geom \ - -lco FID=PARCEL_FABRIC_POLY_ID \ - -nln bcdata.pmbc_parcel_fabric_poly_svw \ - /tmp/pmbc_parcel_fabric_poly_svw.gdb.zip \ - pmbc_parcel_fabric_poly_svw - -# load to target & drop temp -# note that schemas do not quite match -$PSQL -c "truncate whse_cadastre.pmbc_parcel_fabric_poly_svw; - insert into whse_cadastre.pmbc_parcel_fabric_poly_svw ( - parcel_fabric_poly_id , - parcel_name , - plan_number , - pin , - pid_number , - parcel_status , - parcel_class , - owner_type , - parcel_start_date , - municipality , - regional_district , - when_updated , - geom - ) - select - parcel_fabric_poly_id , - parcel_name , - plan_number , - pin , - pid , - parcel_status , - parcel_class , - owner_type , - parcel_start_date , - municipality , - regional_district , - when_updated , - geom - from bcdata.pmbc_parcel_fabric_poly_svw" -$PSQL -c "drop table bcdata.pmbc_parcel_fabric_poly_svw" \ No newline at end of file diff --git a/model/01_access/modelled_stream_crossings/Makefile b/model/01_access/modelled_stream_crossings/Makefile index 818d0c30..700a59b0 100644 --- a/model/01_access/modelled_stream_crossings/Makefile +++ b/model/01_access/modelled_stream_crossings/Makefile @@ -40,34 +40,4 @@ WSGS = $(shell $(PSQL) -AtX -c "SELECT watershed_group_code FROM whse_basemappin # assign modelled_crossing_id from previous version to ensure consistency $(PSQL) -f sql/09_match_archived_crossings.sql - touch $@ - - # dump to file - ogr2ogr \ - -f GPKG \ - modelled_stream_crossings.gpkg.zip \ - PG:$(DATABASE_URL) \ - -nlt PointZM \ - -nln modelled_stream_crossings \ - -FID modelled_crossing_id \ - -sql "select \ - modelled_crossing_id, \ - modelled_crossing_type, \ - array_to_string(modelled_crossing_type_source, '; ') as modelled_crossing_type_source, \ - transport_line_id, \ - ften_road_section_lines_id, \ - og_road_segment_permit_id, \ - og_petrlm_dev_rd_pre06_pub_id, \ - railway_track_id, \ - linear_feature_id, \ - blue_line_key, \ - downstream_route_measure, \ - wscode_ltree, \ - localcode_ltree, \ - watershed_group_code, \ - geom \ - from bcfishpass.modelled_stream_crossings" - - #aws s3 cp modelled_stream_crossings.gpkg.zip s3://bcfishpass/modelled_stream_crossings.gpkg.zip - echo "Latest modelled crossings data now available at:" - echo "https://bcfishpass.s3.us-west-2.amazonaws.com/modelled_stream_crossings.gpkg.zip" \ No newline at end of file + touch $@ \ No newline at end of file