merge main

.github/workflows/testing-bcfishpass.yaml

@@ -23,7 +23,7 @@ jobs:
     runs-on: ubuntu-latest
     environment: testing
     container: ghcr.io/smnorris/bcfishpass:main
-    needs: load-parameters
+    needs: load-csv
     steps:
       - name: Check out repository code
         uses: actions/checkout@v4

data/user_barriers_definite.csv

@@ -224,3 +224,4 @@ MISC,,356364423,7707,QUES,CWF,2023-04-25,CWF WCRP,"Beaver Falls are a hard barri
 MISC,,356342526,110,BOWR,CWF,2023-04-25,CWF WCRP,"Alluvial fan based on information provided by participants at LDN workshop (Chris Elden, West Fraser)."
 GRADIENT_25,,360874126,463.62,LRDO,SN,2023-09-05,https://a100.gov.bc.ca/pub/acat/documents/r585/whalen_report_1068061634267_b3539f3bfa6440d4af43a65105c6e978.pdf,no anadromous spp in Whalen Lake CO; observations upstream are stocked
 MISC,Impassable cascade barrier on Bassett Creek,356363278,1000,HORS,MC,2024-02-23,PSCIS assessement comments,Bedrock channel w/ cascade sections below impassable
+MISC,,356363594,115843,BONP,MC,2024-07-18,Local Knowledge - Trib review,"ST make is up close to Young Lake, but not upstream. CH, CO will not make it up this far."

data/user_habitat_classification.csv

@@ -14872,7 +14872,7 @@
 360887120,19988,20088,TSAY,CH,spawning,t,SN,2023-08-17,FISS spawning points,
 360887120,22207,22307,TSAY,CO,spawning,t,SN,2023-08-17,FISS spawning points,
 360887120,22867,22967,TSAY,CO,spawning,t,SN,2023-08-17,FISS spawning points,
-360887178,2228.49,8202.77,BABL,SK,spawning,t,SN,2023-08-17,Pacific Salmon Foundation,
+360887178,2228.49,8100,BABL,SK,spawning,t,SN,2023-08-17,Pacific Salmon Foundation,adjusted upstream measure down ~100m to fall downstream of 25pct gradient barrier
 360887178,4109,4209,BABL,CH,spawning,t,SN,2023-08-17,FISS spawning points,
 360887178,4109,4209,BABL,CO,spawning,t,SN,2023-08-17,FISS spawning points,
 360887278,45966,46066,LSKE,PK,spawning,t,SN,2023-08-17,FISS spawning points,
@@ -15301,5 +15301,5 @@
 356348431,1481,1781,SHUL,SK,Spawning,t,MC,2024-06-27,Local Knowledge,Local Knowledge - Some SK seen 300m upstream
 356137288,265.6052138,408,SHUL,SK,Spawning,t,MC,2024-06-27,Local Knowledge,Local Knowledge - Adult SK observed up to rail line
 356347377,0,558,BONP,CH,Spawning,t,MC,2024-06-27,Local Knowledge,Local Knowledge - major spawning for CH
-356311064,6630,8353.604837,QUES,CO,rearing,f,MC,2024-07-04,Field assessments,No channel or water
-356287539,1403.205103,1444.490069,QUES,CO,rearing,f,MC,2024-07-04,Field assessments,No wetland/water
+356311064,6630,8353.604837,QUES,CO,rearing,f,MC,2024-07-04,Lhtako Dene WCRP Field Assessments,No channel or water
+356287539,1403.205103,1444.490069,QUES,CO,rearing,f,MC,2024-07-04,Lhtako Dene WCRP Field Assessments,No wetland/water

data/user_modelled_crossing_fixes.csv

@@ -21684,8 +21684,6 @@ modelled_crossing_id,structure,watershed_group_code,reviewer_name,review_date,so
 18300020,CBS,SHUL,MC,2024-06-27,Local Knowledge,Local Knowledge - Replaced with new CBS in summer 2023
 18303484,CBS,SHUL,MC,2024-06-27,Local Knowledge,Local Knowledge - Barrier confirmed
 18303506,OBS,SHUL,MC,2024-06-27,Local Knowledge,Local Knowledge - Fish getting past this structure  (Bridge)
-1400589,CBS,BONP,MC,2024-06-27,Local Knowledge,Local Knowledge - Culvert is Barrier
-1400042,CBS,BONP,MC,2024-06-27,Local Knowledge,"Local Knowledge - Two poorly installed culverts, Barrier. Entire Creek is groundwater"
 4603156,NONE,ELKR,MC,2024-06-27,imagery,South Line Creek runs parallel to road in satellite imagery. Crossing does not exist. Remove.
 4603391,NONE,ELKR,MC,2024-06-27,imagery,no sign of crossing on satellite imagery. Site of a landslide. Remove?
 4603392,NONE,ELKR,MC,2024-06-27,imagery,no sign of crossing on satellite imagery. Remove?
@@ -21718,16 +21716,32 @@ modelled_crossing_id,structure,watershed_group_code,reviewer_name,review_date,so
 23800103,NONE,USHU,LS,2024-06-26,Bing/Google/ESRI imagery,duplicate crossing - removed in favour of d/s location
 18300160,NONE,SHUL,LS,2024-06-26,Bing/Google/ESRI imagery,redundant mulit-lane highway crossing - removed in favour of most d/s location
 23800079,OBS,USHU,LS,2025-06-28,Bing/Google/ESRI imagery,bridge visible
+17104186,NONE,QUES,MC,2024-07-08,Lhtako Dene WCRP Field Assessments,No channel or water. Culvert on side road connecting manmade ditch line along road.
+1501277,NONE,QUES,MC,2024-07-08,Lhtako Dene WCRP Field Assessments,"No culvert, Low marshy ground on either side of road. No Channel. Small lake to south, wetland to north. Depression in road possibly from culvert removal."
+1501283,NONE,BOWR,MC,2024-07-08,Lhtako Dene WCRP Field Assessments,"No sign of a crossing, but old grown in road nearby. Wetland downstream. "
+1501281,NONE,BOWR,MC,2024-07-08,Lhtako Dene WCRP Field Assessments,"No culvert. Old road crossing pulled 10+ years ago. Alder, large cobbles. "
+1501282,NONE,BOWR,MC,2024-07-08,Lhtako Dene WCRP Field Assessments,"No crossing, no passage issues. Fish in stream."
+17103908,NONE,QUES,MC,2024-07-08,Lhtako Dene WCRP Field Assessments,No Barrier or culvert. No water/wetland.
+17100487,NONE,QUES,MC,2024-07-08,Lhtako Dene WCRP Field Assessments,"In bush off Horsefly/Likely FSR. Very old road, overgrown. No water or channel seen, slightly wet corridor."
+17100488,NONE,QUES,MC,2024-07-08,Lhtako Dene WCRP Field Assessments,"In bush off Horsefly/Likely FSR. Very old road, overgrown. No water or channel seen, slightly wet corridor."
+17103896,NONE,QUES,MC,2024-07-08,Lhtako Dene WCRP Field Assessments,"Old clear area, may be part of mine. Cleared in the last 1-3 years. Small (2x2m) wetland with cattails seen, but no water or streams. No culverts. "
+17102208,NONE,QUES,MC,2024-07-08,Lhtako Dene WCRP Field Assessments,"Very old road, overgrown. No crossing found. Slightly wet area but no water seen. "
+17102191,NONE,QUES,MC,2024-07-08,Lhtako Dene WCRP Field Assessments,"No water or wetland. Possible culvert removed at old road, shallow overgrown ditch. No culvert seen. "
+17100098,NONE,QUES,MC,2024-07-08,Lhtako Dene WCRP Field Assessments,No wetland/water. No culvert seen on road. Private property on either side of road.
+17102374,NONE,QUES,MC,2024-07-08,Lhtako Dene WCRP Field Assessments,Cannot find inlet. Bottom of culvert disintegrated where it touches ground. No water.
 17104186,NONE,QUES,MC,2024-07-08,WCRP partner field assessments,No channel or water. Culvert on side road connecting manmade ditch line along road.
-1501277,NONE,QUES,MC,2024-07-08,WCRP partner field assessments,"No culvert, Low marshy ground on either side of road. No Channel. Small lake to south, wetland to north. Depression in road possibly from culvert removal."
-1501283,NONE,BOWR,MC,2024-07-08,WCRP partner field assessments,"No sign of a crossing, but old grown in road nearby. Wetland downstream."
-1501281,NONE,BOWR,MC,2024-07-08,WCRP partner field assessments,"No culvert. Old road crossing pulled 10+ years ago. Alder, large cobbles."
-1501282,NONE,BOWR,MC,2024-07-08,WCRP partner field assessments,"No crossing, no passage issues. Fish in stream."
-17103908,NONE,QUES,MC,2024-07-08,WCRP partner field assessments,No Barrier or culvert. No water/wetland.
-17100487,NONE,QUES,MC,2024-07-08,WCRP partner field assessments,"In bush off Horsefly/Likely FSR. Very old road, overgrown. No water or channel seen, slightly wet corridor."
-17100488,NONE,QUES,MC,2024-07-08,WCRP partner field assessments,"In bush off Horsefly/Likely FSR. Very old road, overgrown. No water or channel seen, slightly wet corridor."
-17103896,NONE,QUES,MC,2024-07-08,WCRP partner field assessments,"Old clear area, may be part of mine. Cleared in the last 1-3 years. Small (2x2m) wetland with cattails seen, but no water or streams. No culverts."
-17102208,NONE,QUES,MC,2024-07-08,WCRP partner field assessments,"Very old road, overgrown. No crossing found. Slightly wet area but no water seen."
-17102191,NONE,QUES,MC,2024-07-08,WCRP partner field assessments,"No water or wetland. Possible culvert removed at old road, shallow overgrown ditch. No culvert seen."
-17100098,NONE,QUES,MC,2024-07-08,WCRP partner field assessments,No wetland/water. No culvert seen on road. Private property on either side of road.
-17102374,NONE,QUES,MC,2024-07-08,WCRP partner field assessments,Cannot find inlet. Bottom of culvert disintegrated where it touches ground. No water.
+1400691,OBS,BONP,LB,2024-07-11,imagery,
+1404128,OBS,BONP,LB,2024-07-11,imagery,
+1406051,NONE,BONP,LB,2024-07-11,imagery,Misinterpretation of tranportation line/stream line
+1406050,NONE,BONP,LB,2024-07-11,imagery,Misinterpretation of tranportation line/stream line
+1406094,NONE,BONP,LB,2024-07-11,imagery,Misinterpretation of tranportation line/stream line. Leaving most downstream structure 1406093
+1406095,NONE,BONP,LB,2024-07-11,imagery,Misinterpretation of tranportation line/stream line. Leaving most downstream structure 1406093
+24739453,OBS,BONP,LB,2024-07-11,imagery,
+24743296,NONE,BONP,LB,2024-07-11,imagery,Structures are really close together and there likely isn't two. Leaving most downstream structure 24743298
+1402061,FORD,BONP,LB,2024-07-11,imagery,
+1402301,FORD,BONP,LB,2024-07-11,imagery,
+1402720,NONE,BONP,LB,2024-07-11,imagery,Misinterpretation of transportation line.  There is no road here meaning no crossing.
+1403172,FORD,BONP,LB,2024-07-11,imagery,
+1405747,FORD,BONP,LB,2024-07-11,imagery,
+11302064,NONE,LNIC,LB,2024-07-16,imagery,Ortho image indicates the stream has likely migrated/been redirected to the north and no longer crosses this road.
+11301333,OBS,LNIC,LB,2024-07-16,imagery,Appears to be a bridge on sat imagery- stream goes to north of where it's mapped

db/v0.5.1/migrate.sh

@@ -0,0 +1,7 @@
+#!/bin/bash
+set -euxo pipefail
+
+
+psql $DATABASE_URL -f sql/wcrp_crossings_vw.sql
+
+psql $DATABASE_URL -c "update bcfishpass.db_version set tag = '${PWD##*/}'"

db/v0.5.1/sql/wcrp_crossings_vw.sql

@@ -0,0 +1,137 @@
+-- wcrp version of the output crossings view -
+BEGIN;
+
+drop materialized view bcfishpass.crossings_wcrp_vw;
+
+create materialized view bcfishpass.crossings_wcrp_vw as
+
+-- find upstream crossings with wcrp 'all spawning rearing habitat' upstream
+with upstr_wcrp_barriers as materialized (
+  select distinct
+   ba.aggregated_crossings_id,
+   h.aggregated_crossings_id as upstr_barriers,
+   h.all_spawningrearing_km
+  from bcfishpass.crossings_upstr_barriers_anthropogenic ba
+  inner join bcfishpass.crossings_upstream_habitat_wcrp h on h.aggregated_crossings_id = any(ba.features_upstr)
+  where h.all_spawningrearing_km > 0
+  order by ba.aggregated_crossings_id, h.aggregated_crossings_id
+),
+
+-- aggregate the upstream wcrp crossings into a list and count
+upstr_wcrp_barriers_list as (
+  select
+    aggregated_crossings_id,
+    array_to_string(array_agg(upstr_barriers), ';') as barriers_anthropogenic_habitat_wcrp_upstr,
+    coalesce(array_length(array_agg(upstr_barriers), 1), 0) as barriers_anthropogenic_habitat_wcrp_upstr_count
+  from upstr_wcrp_barriers
+  group by aggregated_crossings_id
+  order by aggregated_crossings_id
+)
+
+select
+  -- joining to streams based on measure can be error prone due to precision.
+  -- Join to streams on linear_feature_id and keep the first result
+  -- (since streams are segmented there is often >1 match)
+  distinct on (c.aggregated_crossings_id)
+  c.aggregated_crossings_id,
+  c.modelled_crossing_id,
+  c.crossing_source,
+  cft.crossing_feature_type,
+  c.pscis_status,
+  c.crossing_type_code,
+  c.crossing_subtype_code,
+  c.barrier_status,
+  c.pscis_road_name,
+  c.pscis_stream_name,
+  c.pscis_assessment_comment,
+  c.pscis_assessment_date,
+  c.transport_line_structured_name_1,
+  c.rail_track_name,
+  c.dam_name,
+  c.dam_height,
+  c.dam_owner,
+  c.dam_use,
+  c.dam_operating_status,
+  c.utm_zone,
+  c.utm_easting,
+  c.utm_northing,
+  c.blue_line_key,
+  c.downstream_route_measure,
+  c.wscode_ltree as wscode,
+  c.localcode_ltree as localcode,
+  c.watershed_group_code,
+  c.gnis_stream_name,
+  array_to_string(ad.features_dnstr, ';') as barriers_anthropogenic_dnstr,
+  coalesce(array_length(ad.features_dnstr, 1), 0) as barriers_anthropogenic_dnstr_count,
+  uwbl.barriers_anthropogenic_habitat_wcrp_upstr,
+  uwbl.barriers_anthropogenic_habitat_wcrp_upstr_count,
+
+  -- access models
+  array_to_string(a.barriers_ch_cm_co_pk_sk_dnstr, ';') as barriers_ch_cm_co_pk_sk_dnstr,
+  array_to_string(a.barriers_st_dnstr, ';') as barriers_st_dnstr,
+  array_to_string(a.barriers_wct_dnstr, ';') as barriers_wct_dnstr,
+
+  -- habitat models
+  h.ch_spawning_km,
+  h.ch_rearing_km,
+  h.ch_spawning_belowupstrbarriers_km,
+  h.ch_rearing_belowupstrbarriers_km,
+  h.cm_spawning_km,
+  h.cm_spawning_belowupstrbarriers_km,
+  h.co_spawning_km,
+  h_wcrp.co_rearing_km,
+  h.co_rearing_ha,
+  h.co_spawning_belowupstrbarriers_km,
+  h_wcrp.co_rearing_belowupstrbarriers_km,
+  h.co_rearing_belowupstrbarriers_ha,
+  h.pk_spawning_km,
+  h.pk_spawning_belowupstrbarriers_km,
+  h.sk_spawning_km,
+  h_wcrp.sk_rearing_km,
+  h.sk_rearing_ha,
+  h.sk_spawning_belowupstrbarriers_km,
+  h_wcrp.sk_rearing_belowupstrbarriers_km,
+  h.sk_rearing_belowupstrbarriers_ha,
+  h.st_spawning_km,
+  h.st_rearing_km,
+  h.st_spawning_belowupstrbarriers_km,
+  h.st_rearing_belowupstrbarriers_km,
+  h.wct_spawning_km,
+  h.wct_rearing_km,
+  h.wct_spawning_belowupstrbarriers_km,
+  h.wct_rearing_belowupstrbarriers_km,
+  h_wcrp.all_spawning_km,
+  h_wcrp.all_spawning_belowupstrbarriers_km,
+  h_wcrp.all_rearing_km,
+  h_wcrp.all_rearing_belowupstrbarriers_km,
+  h_wcrp.all_spawningrearing_km,
+  h_wcrp.all_spawningrearing_belowupstrbarriers_km,
+  r.set_id,
+  r.total_hab_gain_set,
+  r.num_barriers_set,
+  r.avg_gain_per_barrier,
+  r.dnstr_set_ids,
+  r.rank_avg_gain_per_barrier,
+  r.rank_avg_gain_tiered,
+  r.rank_total_upstr_hab,
+  r.rank_combined,
+  r.tier_combined,
+  c.geom
+from bcfishpass.crossings c
+inner join bcfishpass.wcrp_watersheds w on c.watershed_group_code = w.watershed_group_code  -- only include crossings in WCRP watersheds
+inner join bcfishpass.crossings_feature_type_vw cft on c.aggregated_crossings_id = cft.aggregated_crossings_id
+left outer join bcfishpass.crossings_dnstr_observations_vw cdo on c.aggregated_crossings_id = cdo.aggregated_crossings_id
+left outer join bcfishpass.crossings_upstr_observations_vw cuo on c.aggregated_crossings_id = cuo.aggregated_crossings_id
+left outer join bcfishpass.crossings_dnstr_crossings cd on c.aggregated_crossings_id = cd.aggregated_crossings_id
+left outer join bcfishpass.crossings_dnstr_barriers_anthropogenic ad on c.aggregated_crossings_id = ad.aggregated_crossings_id
+left outer join bcfishpass.crossings_upstr_barriers_anthropogenic au on c.aggregated_crossings_id = au.aggregated_crossings_id
+left outer join upstr_wcrp_barriers_list uwbl on c.aggregated_crossings_id = uwbl.aggregated_crossings_id
+left outer join bcfishpass.crossings_upstream_access a on c.aggregated_crossings_id = a.aggregated_crossings_id
+left outer join bcfishpass.crossings_upstream_habitat h on c.aggregated_crossings_id = h.aggregated_crossings_id
+left outer join bcfishpass.crossings_upstream_habitat_wcrp h_wcrp on c.aggregated_crossings_id = h_wcrp.aggregated_crossings_id
+left outer join bcfishpass.streams s on c.linear_feature_id = s.linear_feature_id
+left outer join whse_basemapping.dbm_mof_50k_grid t ON ST_Intersects(c.geom, t.geom)
+left outer join bcfishpass.wcrp_ranked_barriers r ON c.aggregated_crossings_id = r.aggregated_crossings_id
+order by c.aggregated_crossings_id, s.downstream_route_measure;
+
+COMMIT;

model/01_access/gradient_barriers/sql/gradient_barriers_load.sql

@@ -3,31 +3,35 @@ WITH grade100m AS
   SELECT
     sv.blue_line_key,
     ROUND(sv.downstream_route_measure::numeric, 2) as downstream_route_measure,
+    -- elevation at vertex (from subquery below)
     ROUND(sv.elevation::numeric, 2) as elevation_a,
+    -- interpolation of elevation 100m upstream of the vertex
     ROUND((ST_Z((ST_Dump((ST_LocateAlong(s2.geom, sv.downstream_route_measure + 100)))).geom))::numeric, 2) as elevation_b,
+    -- gradient between the two points on the stream
     ROUND(((ST_Z((ST_Dump((ST_LocateAlong(s2.geom, sv.downstream_route_measure + 100)))).geom) - sv.elevation) / 100)::numeric, 4) as gradient
-  FROM
-  (SELECT
-    linear_feature_id,
-    blue_line_key,
-    edge_type,
-    ((ST_LineLocatePoint(geom, ST_PointN(geom, generate_series(1, ST_NPoints(geom) - 1))) * length_metre) + downstream_route_measure) as downstream_route_measure,
-    ST_Z(ST_PointN(geom, generate_series(1, ST_NPoints(geom) - 1))) AS elevation
-  FROM whse_basemapping.fwa_stream_networks_sp s
-  WHERE watershed_group_code = :'wsg'
-  AND blue_line_key = watershed_key
-  AND edge_type != 6010
-  ORDER BY blue_line_key, downstream_route_measure
+  FROM (
+    -- get elevation of every vertex on the given stream
+    SELECT
+      linear_feature_id,
+      blue_line_key,
+      edge_type,
+      ((ST_LineLocatePoint(geom, ST_PointN(geom, generate_series(1, ST_NPoints(geom) - 1))) * length_metre) + downstream_route_measure) as downstream_route_measure,
+      ST_Z(ST_PointN(geom, generate_series(1, ST_NPoints(geom) - 1))) AS elevation
+    FROM whse_basemapping.fwa_stream_networks_sp s
+    WHERE watershed_group_code = :'wsg'
+    AND blue_line_key = watershed_key
+    AND edge_type != 6010
+    ORDER BY blue_line_key, downstream_route_measure
   ) as sv
   INNER JOIN whse_basemapping.fwa_stream_networks_sp s2
   ON sv.blue_line_key = s2.blue_line_key AND
-     sv.downstream_route_measure + 100 >= s2.downstream_route_measure AND -- find stream segment 100m up
+     sv.downstream_route_measure + 100 >= s2.downstream_route_measure AND -- find stream segment 100m upstream of given segment
      sv.downstream_route_measure + 100 < s2.upstream_route_measure
   WHERE sv.edge_type IN (1000,1050,1100,1150,1250,1350,1410,2000,2300)
   AND s2.edge_type != 6010
 ),
 
--- note the slope classes
+-- note the slope classes of the vertices
 gradeclass AS
 (
   SELECT
@@ -52,9 +56,12 @@ gradeclass AS
   ORDER BY blue_line_key, downstream_route_measure
 ),
 
--- Group the continuous same-slope class segments (islands) together (thank you Erwin B)
+-- Although we are calculating slope at each vertex, we don't need to retain all this information -
+-- for a given continuous set of vertices with the same slope (say 15@2400m 15@2420m 15@2500m 15@2540m),
+-- only the slope at the most downstream point/minimum measure (2400m) needs to be retained.
+-- Grouping continuous classes (islands) together is done via count/lag window functions (thank you Erwin B)
 -- https://dba.stackexchange.com/questions/166374/grouping-or-window/166397#166397
--- NOTE - this could also be done as a recursive query, potentially much easier to read
+-- NOTE - this could also be done as a recursive query, (potentially easier to read)
 
 islands AS
 (