move dump of modelled xings to a release job, standardize naming

smnorris · Mar 14, 2024 · 586c99b · 586c99b
1 parent b626bd1
commit 586c99b
Show file tree

Hide file tree

Showing 10 changed files with 126 additions and 148 deletions.
diff --git a/.github/workflows/production-bcfishpass.yaml b/.github/workflows/production-bcfishpass.yaml
@@ -24,8 +24,10 @@ jobs:
     steps:
       - name: Check out repository code
         uses: actions/checkout@v4
-      - name: Run the jobs
-        run:  jobs/bcfishpass01_modelledcrossings
+      - name: Build modelled crossings
+        run:  jobs/bcfishpass01_modelled_stream_crossings
+      - name: Release modelled crossings
+        run:  jobs/release_modelled_stream_crossings
   prep:
     runs-on: ubuntu-latest
     environment: staging
@@ -60,4 +62,4 @@ jobs:
       - name: Check out repository code
         uses: actions/checkout@v4
       - name: Run the jobs
-        run:  jobs/freshwater_fish_habitat_accessibility_model
+        run:  jobs/release_freshwater_fish_habitat_accessibility_model
diff --git a/jobs/bcfishpass01_modelledcrossings → jobs/bcfishpass01_modelled_stream_crossings b/jobs/bcfishpass01_modelledcrossings → jobs/bcfishpass01_modelled_stream_crossings
diff --git a/jobs/cabd b/jobs/cabd
diff --git a/jobs/load_monthly b/jobs/load_monthly
@@ -8,6 +8,7 @@ set -euxo pipefail
 PSQL="psql $DATABASE_URL -v ON_ERROR_STOP=1"
 
 # bcdata loads
+# ----
 bcdata bc2pg -r whse_admin_boundaries.clab_indian_reserves
 bcdata bc2pg -r whse_admin_boundaries.clab_national_parks
 bcdata bc2pg -r whse_basemapping.gba_local_reg_greenspaces_sp
@@ -24,10 +25,61 @@ bcdata bc2pg -r whse_mineral_tenure.og_petrlm_dev_rds_pre06_pub_sp
 bcdata bc2pg -r whse_tantalis.ta_conservancy_areas_svw
 bcdata bc2pg -r whse_tantalis.ta_park_ecores_pa_svw
 
-# other loads
-jobs/whse_cadastre.pmbc_parcel_fabric_poly_svw
+# parcel fabric, renaming download so it can be read without unzipping
+# ----
+curl \
+  -o /tmp/pmbc_parcel_fabric_poly_svw.gdb.zip \
+  https://pub.data.gov.bc.ca/datasets/4cf233c2-f020-4f7a-9b87-1923252fbc24/pmbc_parcel_fabric_poly_svw.zip
+
+ogr2ogr \
+   -f PostgreSQL \
+   PG:$DATABASE_URL \
+   --config PG_USE_COPY YES \
+   -overwrite \
+   -t_srs EPSG:3005 \
+   -dim XY \
+   -lco GEOMETRY_NAME=geom \
+   -lco FID=PARCEL_FABRIC_POLY_ID \
+   -nln bcdata.pmbc_parcel_fabric_poly_svw \
+   /tmp/pmbc_parcel_fabric_poly_svw.gdb.zip \
+   pmbc_parcel_fabric_poly_svw
+
+# load to target & drop temp (note that schemas do not quite match)
+$PSQL -c "truncate whse_cadastre.pmbc_parcel_fabric_poly_svw;
+          insert into whse_cadastre.pmbc_parcel_fabric_poly_svw (
+            parcel_fabric_poly_id ,
+            parcel_name           ,
+            plan_number           ,
+            pin                   ,
+            pid_number            ,
+            parcel_status         ,
+            parcel_class          ,
+            owner_type            ,
+            parcel_start_date     ,
+            municipality          ,
+            regional_district     ,
+            when_updated          ,
+            geom
+          )
+          select
+            parcel_fabric_poly_id ,
+            parcel_name           ,
+            plan_number           ,
+            pin                   ,
+            pid                   ,
+            parcel_status         ,
+            parcel_class          ,
+            owner_type            ,
+            parcel_start_date     ,
+            municipality          ,
+            regional_district     ,
+            when_updated          ,
+            geom
+          from bcdata.pmbc_parcel_fabric_poly_svw"
+$PSQL -c "drop table bcdata.pmbc_parcel_fabric_poly_svw"
 
 # additional processing
+# ----
 $PSQL -c "refresh materialized view bcdata.parks"
 # range view is too resource intensive for some clients - schedule this job elsewhere
 #$PSQL -c "refresh materialized view bcdata.ften_range_poly_carto_vw"

diff --git a/jobs/load_weekly b/jobs/load_weekly
@@ -7,6 +7,9 @@ set -euxo pipefail
 
 PSQL="psql $DATABASE_URL -v ON_ERROR_STOP=1"
 
+# *note*
+# weekly bcfishobs processing is scheduled via workflow bcfishobs repository
+
 # bcdata loads
 bcdata bc2pg -r whse_fish.fiss_fish_obsrvtn_pnt_sp --query "POINT_TYPE_CODE = 'Observation'"
 bcdata bc2pg -r whse_fish.fiss_obstacles_pnt_sp
@@ -17,8 +20,33 @@ bcdata bc2pg -r whse_fish.pscis_remediation_svw
 bcdata bc2pg -r whse_forest_tenure.ften_road_section_lines_svw
 bcdata bc2pg -r whse_mineral_tenure.og_road_segment_permit_sp
 
-# other loads
-jobs/cabd
-jobs/whse_basemapping.transport_line
+# cabd
+$PSQL -c "truncate cabd.dams"
+ogr2ogr -f PostgreSQL \
+  "PG:$DATABASE_URL" \
+  -append \
+  -nln cabd.dams \
+  "https://cabd-web.azurewebsites.net/cabd-api/features/dams?filter=province_territory_code:eq:bc&filter=use_analysis:eq:true" \
+  OGRGeoJSON
+
+$PSQL -c "truncate cabd.waterfalls"
+ogr2ogr -f PostgreSQL \
+  "PG:$DATABASE_URL" \
+  -append \
+  -nln cabd.waterfalls \
+  "https://cabd-web.azurewebsites.net/cabd-api/features/waterfalls?filter=province_territory_code:eq:bc&filter=use_analysis:eq:true" \
+  OGRGeoJSON
 
-# note that weekly bcfishobs processing is scheduled on bcfishpass db using bcfishobs repository
+# DRA
+ogr2ogr \
+  -f PostgreSQL \
+  "PG:$DATABASE_URL" \
+  --config PG_USE_COPY YES \
+  -lco GEOMETRY_NAME=geom \
+  -lco FID=transport_line_id \
+  -nln bcdata.transport_line \
+  /vsizip//vsicurl/ftp://ftp.geobc.gov.bc.ca/sections/outgoing/bmgs/DRA_Public/dgtl_road_atlas.gdb.zip \
+  TRANSPORT_LINE
+$PSQL -c "truncate whse_basemapping.transport_line;
+          insert into whse_basemapping.transport_line select * from bcdata.transport_line"
+$PSQL -c "drop table bcdata.transport_line"
diff --git a/...eshwater_fish_habitat_accessibility_model → ...eshwater_fish_habitat_accessibility_model b/...eshwater_fish_habitat_accessibility_model → ...eshwater_fish_habitat_accessibility_model
diff --git a/jobs/release_modelled_stream_crossings b/jobs/release_modelled_stream_crossings
@@ -0,0 +1,34 @@
+#!/bin/bash
+set -euxo pipefail
+
+#-------
+# release modelled crossings
+#-------
+
+ogr2ogr \
+	-f GPKG \
+	modelled_stream_crossings.gpkg.zip \
+	PG:$DATABASE_URL \
+	-nlt PointZM \
+	-nln modelled_stream_crossings \
+	-FID modelled_crossing_id \
+	-sql "select \
+		 modelled_crossing_id, \
+		 modelled_crossing_type, \
+		 array_to_string(modelled_crossing_type_source, '; ') as modelled_crossing_type_source, \
+		 transport_line_id, \
+		 ften_road_section_lines_id, \
+		 og_road_segment_permit_id, \
+		 og_petrlm_dev_rd_pre06_pub_id, \
+		 railway_track_id, \
+		 linear_feature_id, \
+		 blue_line_key, \
+		 downstream_route_measure, \
+		 wscode_ltree, \
+		 localcode_ltree, \
+		 watershed_group_code, \
+		 geom \
+		from bcfishpass.modelled_stream_crossings"
+aws s3 cp modelled_stream_crossings.gpkg.zip s3://bcfishpass/modelled_stream_crossings.gpkg.zip
+echo "Latest modelled crossings data now available at:"
+echo "https://bcfishpass.s3.us-west-2.amazonaws.com/modelled_stream_crossings.gpkg.zip"
diff --git a/jobs/whse_basemapping.transport_line b/jobs/whse_basemapping.transport_line
diff --git a/jobs/whse_cadastre.pmbc_parcel_fabric_poly_svw b/jobs/whse_cadastre.pmbc_parcel_fabric_poly_svw
diff --git a/model/01_access/modelled_stream_crossings/Makefile b/model/01_access/modelled_stream_crossings/Makefile
@@ -40,34 +40,4 @@ WSGS = $(shell $(PSQL) -AtX -c "SELECT watershed_group_code FROM whse_basemappin
 
 	# assign modelled_crossing_id from previous version to ensure consistency
 	$(PSQL) -f sql/09_match_archived_crossings.sql
-	touch $@
-
-	# dump to file
-	ogr2ogr \
-		-f GPKG \
-		modelled_stream_crossings.gpkg.zip \
-		PG:$(DATABASE_URL) \
-		-nlt PointZM \
-		-nln modelled_stream_crossings \
-		-FID modelled_crossing_id \
-		-sql "select \
-			 modelled_crossing_id, \
-			 modelled_crossing_type, \
-			 array_to_string(modelled_crossing_type_source, '; ') as modelled_crossing_type_source, \
-			 transport_line_id, \
-			 ften_road_section_lines_id, \
-			 og_road_segment_permit_id, \
-			 og_petrlm_dev_rd_pre06_pub_id, \
-			 railway_track_id, \
-			 linear_feature_id, \
-			 blue_line_key, \
-			 downstream_route_measure, \
-			 wscode_ltree, \
-			 localcode_ltree, \
-			 watershed_group_code, \
-			 geom \
-			from bcfishpass.modelled_stream_crossings"
-
-	#aws s3 cp modelled_stream_crossings.gpkg.zip s3://bcfishpass/modelled_stream_crossings.gpkg.zip
-	echo "Latest modelled crossings data now available at:"
-	echo "https://bcfishpass.s3.us-west-2.amazonaws.com/modelled_stream_crossings.gpkg.zip"
+	touch $@