From af605da43ae4925f18c0d021140249bd873cd1ef Mon Sep 17 00:00:00 2001 From: Simon Norris Date: Wed, 10 Jul 2024 14:49:31 -0700 Subject: [PATCH 1/2] update BT query and add PARS to testing db --- model/01_access/sql/model_access_bt.sql | 149 ++++++++++++++---- .../parameters_habitat_method.csv | 1 + test/build.sh | 2 +- 3 files changed, 116 insertions(+), 36 deletions(-) diff --git a/model/01_access/sql/model_access_bt.sql b/model/01_access/sql/model_access_bt.sql index a3c73a77..78fe1fa3 100644 --- a/model/01_access/sql/model_access_bt.sql +++ b/model/01_access/sql/model_access_bt.sql @@ -45,29 +45,21 @@ with all_barriers as where watershed_group_code = :'wsg' ), --- BT observations, plus salmon/steelhead as any features passable by salmon/steehead --- should also be passable by BT -obs as +-- BT observations, plus salmon/steelhead +-- (any features passable by salmon/steehead should also be passable by BT) +obs_upstr as ( - select * - from bcfishpass.observations_vw - where species_code in ('BT','CH','CM','CO','PK','SK','ST') -), - -barriers as -( - select distinct + select + b.barrier_id, b.barrier_type, - b.barrier_name, - b.linear_feature_id, b.blue_line_key, b.downstream_route_measure, - b.wscode_ltree, - b.localcode_ltree, b.watershed_group_code, - b.geom - from all_barriers b - left outer join obs o + o.species_code as spp, + o.fish_observation_point_id as obs, + o.observation_date as obs_dt + from barriers b + inner join bcfishpass.observations_vw o on fwa_upstream( b.blue_line_key, b.downstream_route_measure, @@ -77,26 +69,98 @@ barriers as o.downstream_route_measure, o.wscode_ltree, o.localcode_ltree, - False, - 1 + false, + 20 -- a large tolerance to discard observations at more or less the same location as the barrier (within 20m) ) - where o.species_code is null + -- do not bother counting observations upstream of barriers that have been noted as barriers in the user control table + left outer join bcfishpass.user_barriers_definite_control bc + on b.blue_line_key = bc.blue_line_key and abs(b.downstream_route_measure - bc.downstream_route_measure) < 1 + where o.species_code in ('BT','CH','CM','CO','PK','SK','ST') + and bc.barrier_ind is null +), - union all +obs_upstr_n as +( + select + o.barrier_id, + count(o.obs) as n_obs + from obs_upstr o + where o.spp in ('BT','CH','CM','CO','PK','SK','ST') + group by o.barrier_id +), - -- include *all* user added features, even those below observations +-- exclude barriers belown known spawning/rearing habitat +habitat as ( select - barrier_type, - barrier_name, - linear_feature_id, - blue_line_key, - downstream_route_measure, - wscode_ltree, - localcode_ltree, - watershed_group_code, - geom - from bcfishpass.barriers_user_definite - where watershed_group_code = :'wsg' + h.blue_line_key, + h.upstream_route_measure, + s.wscode_ltree, + s.localcode_ltree, + h.watershed_group_code, + h.species_code + from bcfishpass.user_habitat_classification h + inner join whse_basemapping.fwa_stream_networks_sp s + ON s.blue_line_key = h.blue_line_key + and round(h.upstream_route_measure::numeric) >= round(s.downstream_route_measure::numeric) + and round(h.upstream_route_measure::numeric) <= round(s.upstream_route_measure::numeric) + where h.habitat_ind is true + and h.species_code in ('BT','CH','CM','CO','PK','SK','ST') + and s.watershed_group_code = :'wsg' +), + +hab_upstr as +( + select + b.barrier_id, + array_agg(species_code) as species_codes + from barriers b + inner join habitat h + on fwa_upstream( + b.blue_line_key, + b.downstream_route_measure, + b.wscode_ltree, + b.localcode_ltree, + h.blue_line_key, + h.upstream_route_measure, + h.wscode_ltree, + h.localcode_ltree, + false, + 20 -- a large tolerance to discard habitat that ends at more or less the same location as the barrier (within 20m) + ) + group by b.barrier_id +), + + +barriers_filtered as ( + select + b.barrier_id as barrier_load_id, + b.barrier_type, + b.barrier_name, + b.linear_feature_id, + b.blue_line_key, + b.downstream_route_measure, + b.wscode_ltree, + b.localcode_ltree, + b.watershed_group_code, + b.geom + from barriers b + left outer join obs_upstr_n as o on b.barrier_id = o.barrier_id + left outer join hab_upstr h on b.barrier_id = h.barrier_id + where watershed_group_code = any( + array( + select watershed_group_code + from bcfishpass.wsg_species_presence + where bt is true + ) + ) + -- do not include barriers with + -- - any BT (or salmon/steelhead) observation upstream + -- - confirmed BT (or salmon/steelhead) habitat upstream + and + ( + o.n_obs is null and + h.species_codes is null + ) ) insert into bcfishpass.barriers_bt @@ -114,7 +178,7 @@ insert into bcfishpass.barriers_bt ) -- add a primary key guaranteed to be unique provincially (presuming unique blkey/measure values within 1m) select - (((blue_line_key::bigint + 1) - 354087611) * 10000000) + round(downstream_route_measure::bigint) as barrier_load_id, + barrier_load_id, barrier_type, barrier_name, linear_feature_id, @@ -132,4 +196,19 @@ where watershed_group_code = any( where bt is true ) ) +union all +-- include *all* user added features, even those below observations + select + (((blue_line_key::bigint + 1) - 354087611) * 10000000) + round(downstream_route_measure::bigint) as barrier_load_id, + barrier_type, + barrier_name, + linear_feature_id, + blue_line_key, + downstream_route_measure, + wscode_ltree, + localcode_ltree, + watershed_group_code, + geom + from bcfishpass.barriers_user_definite + where watershed_group_code = :'wsg' on conflict do nothing; \ No newline at end of file diff --git a/parameters/example_testing/parameters_habitat_method.csv b/parameters/example_testing/parameters_habitat_method.csv index 53ad6315..9a9a00ff 100644 --- a/parameters/example_testing/parameters_habitat_method.csv +++ b/parameters/example_testing/parameters_habitat_method.csv @@ -5,5 +5,6 @@ COWN,cw ELKR,cw HORS,mad LNIC,mad +PARS,cw SANJ,cw VICT,cw \ No newline at end of file diff --git a/test/build.sh b/test/build.sh index e9829f71..e7f73fc1 100755 --- a/test/build.sh +++ b/test/build.sh @@ -7,7 +7,7 @@ set -euxo pipefail DATABASE_URL=postgresql://postgres@localhost:5432/bcfishpass_test PSQL="psql $DATABASE_URL -v ON_ERROR_STOP=1" -WSGS="BELA\nBULK\nCOWN\nELKR\nHORS\nLNIC\nSANJ\nVICT" # edit here to adjust testing watersheds (could pull from parameters/example_testing/parameters_habitat_method) +WSGS="BELA\nBULK\nCOWN\nELKR\nHORS\nLNIC\nPARS\nSANJ\nVICT" # edit here to adjust testing watersheds (could pull from parameters/example_testing/parameters_habitat_method) createdb bcfishpass_test From c52cebaba556b5f164da7373d4809bd07e762e09 Mon Sep 17 00:00:00 2001 From: Simon Norris Date: Wed, 10 Jul 2024 16:26:33 -0700 Subject: [PATCH 2/2] fix #540 --- model/01_access/sql/model_access_bt.sql | 54 +++++++++---------------- 1 file changed, 19 insertions(+), 35 deletions(-) diff --git a/model/01_access/sql/model_access_bt.sql b/model/01_access/sql/model_access_bt.sql index 78fe1fa3..667d1dd9 100644 --- a/model/01_access/sql/model_access_bt.sql +++ b/model/01_access/sql/model_access_bt.sql @@ -1,6 +1,7 @@ -with all_barriers as +with barriers as ( select + barriers_gradient_id as barrier_id, barrier_type, barrier_name, linear_feature_id, @@ -17,6 +18,7 @@ with all_barriers as union all select + barriers_falls_id as barrier_id, barrier_type, barrier_name, linear_feature_id, @@ -32,6 +34,7 @@ with all_barriers as union all select + barriers_subsurfaceflow_id as barrier_id, barrier_type, barrier_name, linear_feature_id, @@ -133,7 +136,7 @@ hab_upstr as barriers_filtered as ( select - b.barrier_id as barrier_load_id, + b.barrier_id, b.barrier_type, b.barrier_name, b.linear_feature_id, @@ -158,7 +161,7 @@ barriers_filtered as ( -- - confirmed BT (or salmon/steelhead) habitat upstream and ( - o.n_obs is null and + (o.n_obs is null or o.n_obs < 1) and h.species_codes is null ) ) @@ -176,39 +179,20 @@ insert into bcfishpass.barriers_bt watershed_group_code, geom ) --- add a primary key guaranteed to be unique provincially (presuming unique blkey/measure values within 1m) -select - barrier_load_id, - barrier_type, - barrier_name, - linear_feature_id, - blue_line_key, - downstream_route_measure, - wscode_ltree, - localcode_ltree, - watershed_group_code, - geom -from barriers b -where watershed_group_code = any( - array( - select watershed_group_code - from bcfishpass.wsg_species_presence - where bt is true - ) - ) +select * from barriers_filtered union all -- include *all* user added features, even those below observations select - (((blue_line_key::bigint + 1) - 354087611) * 10000000) + round(downstream_route_measure::bigint) as barrier_load_id, - barrier_type, - barrier_name, - linear_feature_id, - blue_line_key, - downstream_route_measure, - wscode_ltree, - localcode_ltree, - watershed_group_code, - geom - from bcfishpass.barriers_user_definite - where watershed_group_code = :'wsg' + barriers_user_definite_id as barrier_load_id, + barrier_type, + barrier_name, + linear_feature_id, + blue_line_key, + downstream_route_measure, + wscode_ltree, + localcode_ltree, + watershed_group_code, + geom +from bcfishpass.barriers_user_definite +where watershed_group_code = :'wsg' on conflict do nothing; \ No newline at end of file