From d648f78de8393eda2ddb50ce0a5aca19dd0d1c6d Mon Sep 17 00:00:00 2001 From: Simon Norris Date: Thu, 4 Jan 2024 20:31:43 -0800 Subject: [PATCH] Issue437 (#438) * fix #437 * rename view files so they are called in required order * update doc for fptwg access models, now with boolean observed column * commit missed edit * all views should have simplified wscode column names, remove refresh calls, add draft combined all streams fptwg output --- db/views/01_streams_access_vw.sql | 12 ++ db/views/03_crossings_vw.sql | 4 +- db/views/06_streams_vw.sql | 20 +- ...streams_bt_vw.sql => 11_streams_bt_vw.sql} | 24 +-- ...streams_ch_vw.sql => 12_streams_ch_vw.sql} | 24 +-- ...streams_cm_vw.sql => 13_streams_cm_vw.sql} | 24 +-- ...streams_co_vw.sql => 14_streams_co_vw.sql} | 23 +-- ...streams_pk_vw.sql => 15_streams_pk_vw.sql} | 23 +-- ...salmon_vw.sql => 16_streams_salmon_vw.sql} | 24 +-- ...streams_sk_vw.sql => 17_streams_sk_vw.sql} | 23 +-- ...streams_st_vw.sql => 18_streams_st_vw.sql} | 23 +-- ...reams_wct_vw.sql => 19_streams_wct_vw.sql} | 24 +-- ...ater_fish_habitat_accessibility_model.sql} | 189 +++++++----------- .../model_access_salmon.csv | 3 +- .../model_access_steelhead.csv | 3 +- model/01_access/sql/load_observations.sql | 2 +- model/02_habitat_linear/habitat_linear.sh | 13 -- 17 files changed, 133 insertions(+), 325 deletions(-) rename db/views/{streams_bt_vw.sql => 11_streams_bt_vw.sql} (70%) rename db/views/{streams_ch_vw.sql => 12_streams_ch_vw.sql} (70%) rename db/views/{streams_cm_vw.sql => 13_streams_cm_vw.sql} (70%) rename db/views/{streams_co_vw.sql => 14_streams_co_vw.sql} (70%) rename db/views/{streams_pk_vw.sql => 15_streams_pk_vw.sql} (70%) rename db/views/{streams_salmon_vw.sql => 16_streams_salmon_vw.sql} (73%) rename db/views/{streams_sk_vw.sql => 17_streams_sk_vw.sql} (70%) rename db/views/{streams_st_vw.sql => 18_streams_st_vw.sql} (69%) rename db/views/{streams_wct_vw.sql => 19_streams_wct_vw.sql} (69%) rename db/views/{freshwater_fish_habitat_accessibility_model.sql => 20_freshwater_fish_habitat_accessibility_model.sql} (54%) diff --git a/db/views/01_streams_access_vw.sql b/db/views/01_streams_access_vw.sql index fd4086e5..40eb6d91 100644 --- a/db/views/01_streams_access_vw.sql +++ b/db/views/01_streams_access_vw.sql @@ -39,6 +39,18 @@ select coalesce(ou.obsrvtn_event_upstr, array[]::bigint[]) as obsrvtn_event_upstr, coalesce(ou.obsrvtn_species_codes_upstr, array[]::text[]) as obsrvtn_species_codes_upstr, coalesce(od.species_codes_dnstr, array[]::text[]) as species_codes_dnstr, + + -- include per-species/species groupings boolean 'observed' accessible columns + case when 'BT' = any(ou.obsrvtn_species_codes_upstr) then true else false end as obsrvtn_upstr_bt, + case when 'CH' = any(ou.obsrvtn_species_codes_upstr) then true else false end as obsrvtn_upstr_ch, + case when 'CM' = any(ou.obsrvtn_species_codes_upstr) then true else false end as obsrvtn_upstr_cm, + case when 'CO' = any(ou.obsrvtn_species_codes_upstr) then true else false end as obsrvtn_upstr_co, + case when 'PK' = any(ou.obsrvtn_species_codes_upstr) then true else false end as obsrvtn_upstr_pk, + case when 'SK' = any(ou.obsrvtn_species_codes_upstr) then true else false end as obsrvtn_upstr_sk, + case when 'ST' = any(ou.obsrvtn_species_codes_upstr) then true else false end as obsrvtn_upstr_st, + case when 'WCT' = any(ou.obsrvtn_species_codes_upstr) then true else false end as obsrvtn_upstr_wct, + case when ou.obsrvtn_species_codes_upstr && array['CH','CM','CO','PK','SK'] then true else false end as obsrvtn_upstr_salmon, + cd.crossings_dnstr, case when x.aggregated_crossings_id is not null then true diff --git a/db/views/03_crossings_vw.sql b/db/views/03_crossings_vw.sql index 7e71ba83..e8fe9a02 100644 --- a/db/views/03_crossings_vw.sql +++ b/db/views/03_crossings_vw.sql @@ -50,8 +50,8 @@ select c.blue_line_key, c.watershed_key, c.downstream_route_measure, - c.wscode_ltree, - c.localcode_ltree, + c.wscode_ltree as wscode, + c.localcode_ltree as localcode, c.watershed_group_code, c.gnis_stream_name, c.stream_order, diff --git a/db/views/06_streams_vw.sql b/db/views/06_streams_vw.sql index 82f47355..889945d5 100644 --- a/db/views/06_streams_vw.sql +++ b/db/views/06_streams_vw.sql @@ -1,5 +1,5 @@ -drop materialized view if exists bcfishpass.streams_vw; -create materialized view bcfishpass.streams_vw as +drop view if exists bcfishpass.streams_vw; +create view bcfishpass.streams_vw as select s.segmented_stream_id, s.linear_feature_id, @@ -10,8 +10,8 @@ select s.downstream_route_measure, s.length_metre, s.waterbody_key, - s.wscode_ltree, - s.localcode_ltree, + s.wscode_ltree as wscode, + s.localcode_ltree as localcode, s.gnis_name, s.stream_order, s.stream_magnitude, @@ -39,6 +39,15 @@ select a.remediated_dnstr_ind, array_to_string(a.obsrvtn_event_upstr, ';') as obsrvtn_event_upstr, array_to_string(a.obsrvtn_species_codes_upstr, ';') as obsrvtn_species_codes_upstr, + a.obsrvtn_upstr_bt, + a.obsrvtn_upstr_ch, + a.obsrvtn_upstr_cm, + a.obsrvtn_upstr_co, + a.obsrvtn_upstr_pk, + a.obsrvtn_upstr_sk, + a.obsrvtn_upstr_st, + a.obsrvtn_upstr_wct, + a.obsrvtn_upstr_salmon, array_to_string(a.species_codes_dnstr, ';') as species_codes_dnstr, h.spawning_bt as model_spawning_bt, h.spawning_ch as model_spawning_ch, @@ -68,6 +77,3 @@ from bcfishpass.streams s left outer join bcfishpass.streams_access_vw a on s.segmented_stream_id = a.segmented_stream_id left outer join bcfishpass.streams_habitat_linear_vw h on s.segmented_stream_id = h.segmented_stream_id left outer join bcfishpass.streams_mapping_code_vw m on s.segmented_stream_id = m.segmented_stream_id; - -create unique index on bcfishpass.streams_vw (segmented_stream_id); -create index on bcfishpass.streams_vw using gist (geom); \ No newline at end of file diff --git a/db/views/streams_bt_vw.sql b/db/views/11_streams_bt_vw.sql similarity index 70% rename from db/views/streams_bt_vw.sql rename to db/views/11_streams_bt_vw.sql index 2bb39fdc..760cd764 100644 --- a/db/views/streams_bt_vw.sql +++ b/db/views/11_streams_bt_vw.sql @@ -1,21 +1,6 @@ -drop materialized view if exists bcfishpass.streams_bt_vw; -create materialized view bcfishpass.streams_bt_vw as -with obs as -( - select - segmented_stream_id, - array_to_string(array_agg(spp), ';') as obsrvtn_species_codes_upstr - from ( - select - s.segmented_stream_id, - unnest(o.obsrvtn_species_codes_upstr) as spp - from bcfishpass.streams s - left outer join bcfishpass.streams_upstr_observations o - on s.segmented_stream_id = o.segmented_stream_id - ) as f - where spp = 'BT' - group by segmented_stream_id -) +drop view if exists bcfishpass.streams_bt_vw; + +create view bcfishpass.streams_bt_vw as select s.segmented_stream_id, s.linear_feature_id, @@ -46,7 +31,7 @@ select array_to_string(a.barriers_dams_dnstr, ';') as barriers_dams_dnstr, array_to_string(a.barriers_dams_hydro_dnstr, ';') as barriers_dams_hydro_dnstr, array_to_string(a.crossings_dnstr, ';') as crossings_dnstr, - o.obsrvtn_species_codes_upstr, + a.obsrvtn_upstr_bt, a.dam_dnstr_ind, a.dam_hydro_dnstr_ind, a.remediated_dnstr_ind, @@ -55,7 +40,6 @@ select m.mapping_code_bt as mapping_code, s.geom from bcfishpass.streams s -left outer join obs o on s.segmented_stream_id = o.segmented_stream_id left outer join bcfishpass.streams_access_vw a on s.segmented_stream_id = a.segmented_stream_id left outer join bcfishpass.streams_habitat_linear_vw h on s.segmented_stream_id = h.segmented_stream_id left outer join bcfishpass.streams_mapping_code_vw m on s.segmented_stream_id = m.segmented_stream_id diff --git a/db/views/streams_ch_vw.sql b/db/views/12_streams_ch_vw.sql similarity index 70% rename from db/views/streams_ch_vw.sql rename to db/views/12_streams_ch_vw.sql index bb14c6b6..aaf40fb6 100644 --- a/db/views/streams_ch_vw.sql +++ b/db/views/12_streams_ch_vw.sql @@ -1,21 +1,6 @@ -drop materialized view if exists bcfishpass.streams_ch_vw; -create materialized view bcfishpass.streams_ch_vw as -with obs as -( - select - segmented_stream_id, - array_to_string(array_agg(spp), ';') as obsrvtn_species_codes_upstr - from ( - select - s.segmented_stream_id, - unnest(o.obsrvtn_species_codes_upstr) as spp - from bcfishpass.streams s - left outer join bcfishpass.streams_upstr_observations o - on s.segmented_stream_id = o.segmented_stream_id - ) as f - where spp = 'CH' - group by segmented_stream_id -) +drop view if exists bcfishpass.streams_ch_vw; + +create view bcfishpass.streams_ch_vw as select s.segmented_stream_id, s.linear_feature_id, @@ -46,7 +31,7 @@ select array_to_string(a.barriers_dams_dnstr, ';') as barriers_dams_dnstr, array_to_string(a.barriers_dams_hydro_dnstr, ';') as barriers_dams_hydro_dnstr, array_to_string(a.crossings_dnstr, ';') as crossings_dnstr, - o.obsrvtn_species_codes_upstr, + a.obsrvtn_upstr_ch, a.dam_dnstr_ind, a.dam_hydro_dnstr_ind, a.remediated_dnstr_ind, @@ -55,7 +40,6 @@ select m.mapping_code_ch as mapping_code, s.geom from bcfishpass.streams s -left outer join obs o on s.segmented_stream_id = o.segmented_stream_id left outer join bcfishpass.streams_access_vw a on s.segmented_stream_id = a.segmented_stream_id left outer join bcfishpass.streams_habitat_linear_vw h on s.segmented_stream_id = h.segmented_stream_id left outer join bcfishpass.streams_mapping_code_vw m on s.segmented_stream_id = m.segmented_stream_id diff --git a/db/views/streams_cm_vw.sql b/db/views/13_streams_cm_vw.sql similarity index 70% rename from db/views/streams_cm_vw.sql rename to db/views/13_streams_cm_vw.sql index b8cdbf30..2dee1a74 100644 --- a/db/views/streams_cm_vw.sql +++ b/db/views/13_streams_cm_vw.sql @@ -1,23 +1,6 @@ -drop materialized view if exists bcfishpass.streams_cm_vw; -create materialized view bcfishpass.streams_cm_vw as - -with obs as -( - select - segmented_stream_id, - array_to_string(array_agg(spp), ';') as obsrvtn_species_codes_upstr - from ( - select - s.segmented_stream_id, - unnest(o.obsrvtn_species_codes_upstr) as spp - from bcfishpass.streams s - left outer join bcfishpass.streams_upstr_observations o - on s.segmented_stream_id = o.segmented_stream_id - ) as f - where spp = 'CM' - group by segmented_stream_id -) +drop view if exists bcfishpass.streams_cm_vw; +create view bcfishpass.streams_cm_vw as select s.segmented_stream_id, s.linear_feature_id, @@ -48,7 +31,7 @@ select array_to_string(a.barriers_dams_dnstr, ';') as barriers_dams_dnstr, array_to_string(a.barriers_dams_hydro_dnstr, ';') as barriers_dams_hydro_dnstr, array_to_string(a.crossings_dnstr, ';') as crossings_dnstr, - o.obsrvtn_species_codes_upstr, + a.obsrvtn_upstr_cm, a.dam_dnstr_ind, a.dam_hydro_dnstr_ind, a.remediated_dnstr_ind, @@ -57,7 +40,6 @@ select m.mapping_code_cm as mapping_code, s.geom from bcfishpass.streams s -left outer join obs o on s.segmented_stream_id = o.segmented_stream_id left outer join bcfishpass.streams_access_vw a on s.segmented_stream_id = a.segmented_stream_id left outer join bcfishpass.streams_habitat_linear_vw h on s.segmented_stream_id = h.segmented_stream_id left outer join bcfishpass.streams_mapping_code_vw m on s.segmented_stream_id = m.segmented_stream_id diff --git a/db/views/streams_co_vw.sql b/db/views/14_streams_co_vw.sql similarity index 70% rename from db/views/streams_co_vw.sql rename to db/views/14_streams_co_vw.sql index 9bc98452..31d57e01 100644 --- a/db/views/streams_co_vw.sql +++ b/db/views/14_streams_co_vw.sql @@ -1,22 +1,6 @@ -drop materialized view if exists bcfishpass.streams_co_vw; +drop view if exists bcfishpass.streams_co_vw; -create materialized view bcfishpass.streams_co_vw as -with obs as -( - select - segmented_stream_id, - array_to_string(array_agg(spp), ';') as obsrvtn_species_codes_upstr - from ( - select - s.segmented_stream_id, - unnest(o.obsrvtn_species_codes_upstr) as spp - from bcfishpass.streams s - left outer join bcfishpass.streams_upstr_observations o - on s.segmented_stream_id = o.segmented_stream_id - ) as f - where spp = 'CO' - group by segmented_stream_id -) +create view bcfishpass.streams_co_vw as select s.segmented_stream_id, s.linear_feature_id, @@ -47,7 +31,7 @@ select array_to_string(a.barriers_dams_dnstr, ';') as barriers_dams_dnstr, array_to_string(a.barriers_dams_hydro_dnstr, ';') as barriers_dams_hydro_dnstr, array_to_string(a.crossings_dnstr, ';') as crossings_dnstr, - o.obsrvtn_species_codes_upstr, + a.obsrvtn_upstr_co, a.dam_dnstr_ind, a.dam_hydro_dnstr_ind, a.remediated_dnstr_ind, @@ -56,7 +40,6 @@ select m.mapping_code_co as mapping_code, geom from bcfishpass.streams s -left outer join obs o on s.segmented_stream_id = o.segmented_stream_id left outer join bcfishpass.streams_access_vw a on s.segmented_stream_id = a.segmented_stream_id left outer join bcfishpass.streams_habitat_linear_vw h on s.segmented_stream_id = h.segmented_stream_id left outer join bcfishpass.streams_mapping_code_vw m on s.segmented_stream_id = m.segmented_stream_id diff --git a/db/views/streams_pk_vw.sql b/db/views/15_streams_pk_vw.sql similarity index 70% rename from db/views/streams_pk_vw.sql rename to db/views/15_streams_pk_vw.sql index b10533c9..1b13389d 100644 --- a/db/views/streams_pk_vw.sql +++ b/db/views/15_streams_pk_vw.sql @@ -1,22 +1,6 @@ -drop materialized view if exists bcfishpass.streams_pk_vw; +drop view if exists bcfishpass.streams_pk_vw; -create materialized view bcfishpass.streams_pk_vw as -with obs as -( - select - segmented_stream_id, - array_to_string(array_agg(spp), ';') as obsrvtn_species_codes_upstr - from ( - select - s.segmented_stream_id, - unnest(o.obsrvtn_species_codes_upstr) as spp - from bcfishpass.streams s - left outer join bcfishpass.streams_upstr_observations o - on s.segmented_stream_id = o.segmented_stream_id - ) as f - where spp = 'PK' - group by segmented_stream_id -) +create view bcfishpass.streams_pk_vw as select s.segmented_stream_id, s.linear_feature_id, @@ -47,7 +31,7 @@ select array_to_string(a.barriers_dams_dnstr, ';') as barriers_dams_dnstr, array_to_string(a.barriers_dams_hydro_dnstr, ';') as barriers_dams_hydro_dnstr, array_to_string(a.crossings_dnstr, ';') as crossings_dnstr, - o.obsrvtn_species_codes_upstr, + a.obsrvtn_upstr_pk, a.dam_dnstr_ind, a.dam_hydro_dnstr_ind, a.remediated_dnstr_ind, @@ -56,7 +40,6 @@ select m.mapping_code_pk as mapping_code, s.geom from bcfishpass.streams s -left outer join obs o on s.segmented_stream_id = o.segmented_stream_id left outer join bcfishpass.streams_access_vw a on s.segmented_stream_id = a.segmented_stream_id left outer join bcfishpass.streams_habitat_linear_vw h on s.segmented_stream_id = h.segmented_stream_id left outer join bcfishpass.streams_mapping_code_vw m on s.segmented_stream_id = m.segmented_stream_id diff --git a/db/views/streams_salmon_vw.sql b/db/views/16_streams_salmon_vw.sql similarity index 73% rename from db/views/streams_salmon_vw.sql rename to db/views/16_streams_salmon_vw.sql index 9c8192a5..3b927fbc 100644 --- a/db/views/streams_salmon_vw.sql +++ b/db/views/16_streams_salmon_vw.sql @@ -1,23 +1,6 @@ -drop materialized view if exists bcfishpass.streams_salmon_vw; - -create materialized view bcfishpass.streams_salmon_vw as -with obs as -( - select - segmented_stream_id, - array_to_string(array_agg(spp), ';') as obsrvtn_species_codes_upstr - from ( - select - s.segmented_stream_id, - unnest(o.obsrvtn_species_codes_upstr) as spp - from bcfishpass.streams s - left outer join bcfishpass.streams_upstr_observations o - on s.segmented_stream_id = o.segmented_stream_id - ) as f - where spp in ('CH','CM','CO','PK','SK') - group by segmented_stream_id -) +drop view if exists bcfishpass.streams_salmon_vw; +create view bcfishpass.streams_salmon_vw as select s.segmented_stream_id, s.linear_feature_id, @@ -48,7 +31,7 @@ select array_to_string(a.barriers_dams_dnstr, ';') as barriers_dams_dnstr, array_to_string(a.barriers_dams_hydro_dnstr, ';') as barriers_dams_hydro_dnstr, array_to_string(a.crossings_dnstr, ';') as crossings_dnstr, - o.obsrvtn_species_codes_upstr, + a.obsrvtn_upstr_salmon, a.dam_dnstr_ind, a.dam_hydro_dnstr_ind, a.remediated_dnstr_ind, @@ -73,7 +56,6 @@ select m.mapping_code_salmon as mapping_code, s.geom from bcfishpass.streams s -left outer join obs o on s.segmented_stream_id = o.segmented_stream_id left outer join bcfishpass.streams_access_vw a on s.segmented_stream_id = a.segmented_stream_id left outer join bcfishpass.streams_habitat_linear_vw h on s.segmented_stream_id = h.segmented_stream_id left outer join bcfishpass.streams_mapping_code_vw m on s.segmented_stream_id = m.segmented_stream_id diff --git a/db/views/streams_sk_vw.sql b/db/views/17_streams_sk_vw.sql similarity index 70% rename from db/views/streams_sk_vw.sql rename to db/views/17_streams_sk_vw.sql index d41a49d2..9179dc89 100644 --- a/db/views/streams_sk_vw.sql +++ b/db/views/17_streams_sk_vw.sql @@ -1,22 +1,6 @@ -drop materialized view if exists bcfishpass.streams_sk_vw; +drop view if exists bcfishpass.streams_sk_vw; -create materialized view bcfishpass.streams_sk_vw as -with obs as -( - select - segmented_stream_id, - array_to_string(array_agg(spp), ';') as obsrvtn_species_codes_upstr - from ( - select - s.segmented_stream_id, - unnest(o.obsrvtn_species_codes_upstr) as spp - from bcfishpass.streams s - left outer join bcfishpass.streams_upstr_observations o - on s.segmented_stream_id = o.segmented_stream_id - ) as f - where spp = 'SK' - group by segmented_stream_id -) +create view bcfishpass.streams_sk_vw as select s.segmented_stream_id, s.linear_feature_id, @@ -47,7 +31,7 @@ select array_to_string(a.barriers_dams_dnstr, ';') as barriers_dams_dnstr, array_to_string(a.barriers_dams_hydro_dnstr, ';') as barriers_dams_hydro_dnstr, array_to_string(a.crossings_dnstr, ';') as crossings_dnstr, - o.obsrvtn_species_codes_upstr, + a.obsrvtn_upstr_sk, a.dam_dnstr_ind, a.dam_hydro_dnstr_ind, a.remediated_dnstr_ind, @@ -56,7 +40,6 @@ select m.mapping_code_sk as mapping_code, s.geom from bcfishpass.streams s -left outer join obs o on s.segmented_stream_id = o.segmented_stream_id left outer join bcfishpass.streams_access_vw a on s.segmented_stream_id = a.segmented_stream_id left outer join bcfishpass.streams_habitat_linear_vw h on s.segmented_stream_id = h.segmented_stream_id left outer join bcfishpass.streams_mapping_code_vw m on s.segmented_stream_id = m.segmented_stream_id diff --git a/db/views/streams_st_vw.sql b/db/views/18_streams_st_vw.sql similarity index 69% rename from db/views/streams_st_vw.sql rename to db/views/18_streams_st_vw.sql index 41f30501..9f63e7f8 100644 --- a/db/views/streams_st_vw.sql +++ b/db/views/18_streams_st_vw.sql @@ -1,22 +1,6 @@ -drop materialized view if exists bcfishpass.streams_st_vw; +drop view if exists bcfishpass.streams_st_vw; -create materialized view bcfishpass.streams_st_vw as - with obs as -( - select - segmented_stream_id, - array_to_string(array_agg(spp), ';') as obsrvtn_species_codes_upstr - from ( - select - s.segmented_stream_id, - unnest(o.obsrvtn_species_codes_upstr) as spp - from bcfishpass.streams s - left outer join bcfishpass.streams_upstr_observations o - on s.segmented_stream_id = o.segmented_stream_id - ) as f - where spp = 'ST' - group by segmented_stream_id -) +create view bcfishpass.streams_st_vw as select s.segmented_stream_id, s.linear_feature_id, @@ -47,7 +31,7 @@ select array_to_string(a.barriers_dams_dnstr, ';') as barriers_dams_dnstr, array_to_string(a.barriers_dams_hydro_dnstr, ';') as barriers_dams_hydro_dnstr, array_to_string(a.crossings_dnstr, ';') as crossings_dnstr, - o.obsrvtn_species_codes_upstr, + a.obsrvtn_upstr_st, a.dam_dnstr_ind, a.dam_hydro_dnstr_ind, a.remediated_dnstr_ind, @@ -56,7 +40,6 @@ select m.mapping_code_st as mapping_code, s.geom from bcfishpass.streams s -left outer join obs o on s.segmented_stream_id = o.segmented_stream_id left outer join bcfishpass.streams_access_vw a on s.segmented_stream_id = a.segmented_stream_id left outer join bcfishpass.streams_habitat_linear_vw h on s.segmented_stream_id = h.segmented_stream_id left outer join bcfishpass.streams_mapping_code_vw m on s.segmented_stream_id = m.segmented_stream_id diff --git a/db/views/streams_wct_vw.sql b/db/views/19_streams_wct_vw.sql similarity index 69% rename from db/views/streams_wct_vw.sql rename to db/views/19_streams_wct_vw.sql index 46ef6700..e6d00146 100644 --- a/db/views/streams_wct_vw.sql +++ b/db/views/19_streams_wct_vw.sql @@ -1,23 +1,6 @@ -drop materialized view if exists bcfishpass.streams_wct_vw; - -create materialized view bcfishpass.streams_wct_vw as - with obs as -( - select - segmented_stream_id, - array_to_string(array_agg(spp), ';') as obsrvtn_species_codes_upstr - from ( - select - s.segmented_stream_id, - unnest(o.obsrvtn_species_codes_upstr) as spp - from bcfishpass.streams s - left outer join bcfishpass.streams_upstr_observations o - on s.segmented_stream_id = o.segmented_stream_id - ) as f - where spp = 'WCT' - group by segmented_stream_id -) +drop view if exists bcfishpass.streams_wct_vw; +create view bcfishpass.streams_wct_vw as select s.segmented_stream_id, s.linear_feature_id, @@ -48,7 +31,7 @@ select array_to_string(a.barriers_dams_dnstr, ';') as barriers_dams_dnstr, array_to_string(a.barriers_dams_hydro_dnstr, ';') as barriers_dams_hydro_dnstr, array_to_string(a.crossings_dnstr, ';') as crossings_dnstr, - o.obsrvtn_species_codes_upstr, + a.obsrvtn_upstr_wct, a.dam_dnstr_ind, a.dam_hydro_dnstr_ind, a.remediated_dnstr_ind, @@ -57,7 +40,6 @@ select m.mapping_code_wct as mapping_code, s.geom from bcfishpass.streams s -left outer join obs o on s.segmented_stream_id = o.segmented_stream_id left outer join bcfishpass.streams_access_vw a on s.segmented_stream_id = a.segmented_stream_id left outer join bcfishpass.streams_habitat_linear_vw h on s.segmented_stream_id = h.segmented_stream_id left outer join bcfishpass.streams_mapping_code_vw m on s.segmented_stream_id = m.segmented_stream_id diff --git a/db/views/freshwater_fish_habitat_accessibility_model.sql b/db/views/20_freshwater_fish_habitat_accessibility_model.sql similarity index 54% rename from db/views/freshwater_fish_habitat_accessibility_model.sql rename to db/views/20_freshwater_fish_habitat_accessibility_model.sql index dba9b62f..942469f3 100644 --- a/db/views/freshwater_fish_habitat_accessibility_model.sql +++ b/db/views/20_freshwater_fish_habitat_accessibility_model.sql @@ -1,46 +1,65 @@ -- for publication to DataBC Catalouge -- freshwater_fish_habitat_accessibility_MODEL.gpkg.zip -drop materialized view if exists bcfishpass.freshwater_fish_habitat_accessibility_model_salmon_vw; +drop view if exists bcfishpass.freshwater_fish_habitat_accessibility_model_salmon_vw; +create view bcfishpass.freshwater_fish_habitat_accessibility_model_salmon_vw as +select + segmented_stream_id, + linear_feature_id, + edge_type, + blue_line_key, + downstream_route_measure, + upstream_route_measure, + watershed_group_code, + gnis_name, + stream_order, + stream_magnitude, + gradient, + wscode, + localcode, + feature_code, + obsrvtn_upstr_salmon, + barriers_anthropogenic_dnstr, + barriers_pscis_dnstr, + barriers_dams_dnstr, + dam_dnstr_ind, + dam_hydro_dnstr_ind, + remediated_dnstr_ind, + geom +from bcfishpass.streams_salmon_vw; -create materialized view bcfishpass.freshwater_fish_habitat_accessibility_model_salmon_vw as -with observations as -( - select - segmented_stream_id, - array_agg(fish_observation_point_id) FILTER (WHERE fish_observation_point_id IS NOT NULL) AS obsrvtn_ids_upstr, - array_agg(observation_date) FILTER (WHERE fish_observation_point_id IS NOT NULL) AS obsrvtn_dates_upstr, - array_agg(DISTINCT (species_code)) FILTER (WHERE fish_observation_point_id IS NOT NULL) as obsrvtn_species_codes_upstr - from ( - select distinct - s.segmented_stream_id, - o.species_code, - o.fish_observation_point_id, - o.observation_date - from bcfishpass.streams s - inner join bcfishpass.streams_access_vw a on s.segmented_stream_id = a.segmented_stream_id - left outer join bcfishpass.observations_vw o - on FWA_Upstream( - s.blue_line_key, - s.downstream_route_measure, - s.wscode_ltree, - s.localcode_ltree, - o.blue_line_key, - o.downstream_route_measure, - o.wscode_ltree, - o.localcode_ltree, - False, - 1 - ) - and s.watershed_group_code = o.watershed_group_code - where o.species_code in ('CH','CM','CO','PK','SK') - and a.barriers_ch_cm_co_pk_sk_dnstr = array[]::text[] - ) as f - group by segmented_stream_id -) +drop view if exists bcfishpass.freshwater_fish_habitat_accessibility_model_steelhead_vw; +create view bcfishpass.freshwater_fish_habitat_accessibility_model_steelhead_vw as +select + segmented_stream_id, + linear_feature_id, + edge_type, + blue_line_key, + downstream_route_measure, + upstream_route_measure, + watershed_group_code, + gnis_name, + stream_order, + stream_magnitude, + gradient, + wscode, + localcode, + feature_code, + obsrvtn_upstr_st, + barriers_anthropogenic_dnstr, + barriers_pscis_dnstr, + barriers_dams_dnstr, + dam_dnstr_ind, + dam_hydro_dnstr_ind, + remediated_dnstr_ind, + geom +from bcfishpass.streams_st_vw; -select +-- all streams/combined salmon/steelhead product +drop view if exists bcfishpass.freshwater_fish_habitat_accessibility_model_vw; +create view bcfishpass.freshwater_fish_habitat_accessibility_model_vw as +select s.segmented_stream_id, s.linear_feature_id, s.edge_type, @@ -55,76 +74,18 @@ select s.wscode_ltree as wscode, s.localcode_ltree as localcode, s.feature_code, - array_to_string(o.obsrvtn_ids_upstr, ';') as obsrvtn_ids_upstr, - array_to_string(o.obsrvtn_species_codes_upstr, ';') as obsrvtn_species_codes_upstr, - array_to_string(a.barriers_anthropogenic_dnstr, ';') as barriers_anthropogenic_dnstr, - array_to_string(a.barriers_pscis_dnstr, ';') as barriers_pscis_dnstr, - array_to_string(a.barriers_dams_dnstr, ';') as barriers_dams_dnstr, - a.dam_dnstr_ind, - a.dam_hydro_dnstr_ind, - a.remediated_dnstr_ind, - s.geom -from bcfishpass.streams s -inner join bcfishpass.streams_access_vw a on s.segmented_stream_id = a.segmented_stream_id -left outer join observations o on s.segmented_stream_id = o.segmented_stream_id -where a.barriers_ch_cm_co_pk_sk_dnstr = array[]::text[]; - - -drop materialized view if exists bcfishpass.freshwater_fish_habitat_accessibility_model_steelhead_vw; - -create materialized view bcfishpass.freshwater_fish_habitat_accessibility_model_steelhead_vw as -with observations as -( - select - segmented_stream_id, - array_agg(fish_observation_point_id) FILTER (WHERE fish_observation_point_id IS NOT NULL) AS obsrvtn_ids_upstr, - array_agg(observation_date) FILTER (WHERE fish_observation_point_id IS NOT NULL) AS obsrvtn_dates_upstr, - array_agg(DISTINCT (species_code)) FILTER (WHERE fish_observation_point_id IS NOT NULL) as obsrvtn_species_codes_upstr - from ( - select distinct - s.segmented_stream_id, - o.species_code, - o.fish_observation_point_id, - o.observation_date - from bcfishpass.streams s - inner join bcfishpass.streams_access_vw a on s.segmented_stream_id = a.segmented_stream_id - left outer join bcfishpass.observations_vw o - on FWA_Upstream( - s.blue_line_key, - s.downstream_route_measure, - s.wscode_ltree, - s.localcode_ltree, - o.blue_line_key, - o.downstream_route_measure, - o.wscode_ltree, - o.localcode_ltree, - False, - 1 - ) - and s.watershed_group_code = o.watershed_group_code - where o.species_code = 'ST' - and a.barriers_st_dnstr = array[]::text[] - ) as f - group by segmented_stream_id -) - -select - s.segmented_stream_id, - s.linear_feature_id, - s.edge_type, - s.blue_line_key, - s.watershed_group_code, - s.downstream_route_measure, - s.upstream_route_measure, - s.gnis_name, - s.stream_order, - s.stream_magnitude, - s.gradient, - s.wscode_ltree as wscode, - s.localcode_ltree as localcode, - s.feature_code, - array_to_string(o.obsrvtn_ids_upstr, ';') as obsrvtn_ids_upstr, - array_to_string(o.obsrvtn_species_codes_upstr, ';') as obsrvtn_species_codes_upstr, + case + when a.barriers_ch_cm_co_pk_sk_dnstr = array[]::text[] and a.obsrvtn_upstr_salmon is true then 'OBSERVED' + when a.barriers_ch_cm_co_pk_sk_dnstr = array[]::text[] and a.obsrvtn_upstr_salmon is false then 'INFERRED' + when a.barriers_ch_cm_co_pk_sk_dnstr != array[]::text[] then 'NATURAL_BARRIER' + end as model_access_salmon, + case + when a.barriers_st_dnstr = array[]::text[] and a.obsrvtn_upstr_st is true then 'OBSERVED' + when a.barriers_st_dnstr = array[]::text[] and a.obsrvtn_upstr_st is false then 'INFERRED' + when a.barriers_st_dnstr != array[]::text[] then 'NATURAL_BARRIER' + end as model_access_steelhead, + array_to_string(a.barriers_ch_cm_co_pk_sk_dnstr, ';') as barriers_ch_cm_co_pk_sk_dnstr, + array_to_string(a.barriers_st_dnstr, ';') as barriers_st_dnstr, array_to_string(a.barriers_anthropogenic_dnstr, ';') as barriers_anthropogenic_dnstr, array_to_string(a.barriers_pscis_dnstr, ';') as barriers_pscis_dnstr, array_to_string(a.barriers_dams_dnstr, ';') as barriers_dams_dnstr, @@ -133,22 +94,20 @@ select a.remediated_dnstr_ind, s.geom from bcfishpass.streams s -inner join bcfishpass.streams_access_vw a on s.segmented_stream_id = a.segmented_stream_id -left outer join observations o on s.segmented_stream_id = o.segmented_stream_id -where a.barriers_st_dnstr = array[]::text[]; +left outer join bcfishpass.streams_access_vw a on s.segmented_stream_id = a.segmented_stream_id +left outer join bcfishpass.streams_habitat_linear_vw h on s.segmented_stream_id = h.segmented_stream_id; -- no views required for barrier tables, they can be used directly (only change would be renaming wscode/localcode) -- dump observations for salmon and steelhead used in this analysis -drop materialized view if exists bcfishpass.freshwater_fish_habitat_accessibility_model_observations_vw; -create materialized view bcfishpass.freshwater_fish_habitat_accessibility_model_observations_vw as +drop view if exists bcfishpass.freshwater_fish_habitat_accessibility_model_observations_vw; +create view bcfishpass.freshwater_fish_habitat_accessibility_model_observations_vw as select * from bcfishpass.observations_vw where species_code in ('CH','CM','CO','PK','SK','ST'); -- create view of crossings with just salmon/steelhead related columns - drop view if exists bcfishpass.freshwater_fish_habitat_accessibility_model_crossings_vw; create view bcfishpass.freshwater_fish_habitat_accessibility_model_crossings_vw as select @@ -194,8 +153,8 @@ select c.blue_line_key, c.watershed_key, c.downstream_route_measure, - c.wscode_ltree as wscode, - c.localcode_ltree as localcode, + c.wscode, + c.localcode, c.watershed_group_code, c.gnis_stream_name, c.stream_order, diff --git a/docs/tables/freshwater_fish_habitat_accessibility_model/model_access_salmon.csv b/docs/tables/freshwater_fish_habitat_accessibility_model/model_access_salmon.csv index 7cc29ccc..3e083d12 100644 --- a/docs/tables/freshwater_fish_habitat_accessibility_model/model_access_salmon.csv +++ b/docs/tables/freshwater_fish_habitat_accessibility_model/model_access_salmon.csv @@ -13,8 +13,7 @@ column,description ``wscode``,A truncated version of the BC FWA fwa_watershed_code (trailing zeros removed and '-' replaced with '.' ``localcode``,A truncated version of the BC FWA local_watershed_code (trailing zeros removed and '-' replaced with '.' ``feature_code``,"A value based on the Canadian Council of Surveys and Mapping's (CCSM) system for classification of geographic features." -``obsrvtn_ids_upstr``,IDs of known observations of target fish species upstream -``obsrvtn_species_codes_upstr``,Species codes of known fish observations upstream +``obsrvtn_upstr_salmon``,True if known salmon observations are present upstream ``barriers_anthropogenic_dnstr``,IDs of anthropogenic barriers downstream ``barriers_pscis_dnstr``,IDs of assessed PSCIS barriers downstream ``barriers_dams_dnstr``,IDs of CABD dams downstream noted as barriers diff --git a/docs/tables/freshwater_fish_habitat_accessibility_model/model_access_steelhead.csv b/docs/tables/freshwater_fish_habitat_accessibility_model/model_access_steelhead.csv index 7cc29ccc..c3798010 100644 --- a/docs/tables/freshwater_fish_habitat_accessibility_model/model_access_steelhead.csv +++ b/docs/tables/freshwater_fish_habitat_accessibility_model/model_access_steelhead.csv @@ -13,8 +13,7 @@ column,description ``wscode``,A truncated version of the BC FWA fwa_watershed_code (trailing zeros removed and '-' replaced with '.' ``localcode``,A truncated version of the BC FWA local_watershed_code (trailing zeros removed and '-' replaced with '.' ``feature_code``,"A value based on the Canadian Council of Surveys and Mapping's (CCSM) system for classification of geographic features." -``obsrvtn_ids_upstr``,IDs of known observations of target fish species upstream -``obsrvtn_species_codes_upstr``,Species codes of known fish observations upstream +``obsrvtn_upstr_st``,True if known steelhead observations are present upstream ``barriers_anthropogenic_dnstr``,IDs of anthropogenic barriers downstream ``barriers_pscis_dnstr``,IDs of assessed PSCIS barriers downstream ``barriers_dams_dnstr``,IDs of CABD dams downstream noted as barriers diff --git a/model/01_access/sql/load_observations.sql b/model/01_access/sql/load_observations.sql index 8a95b31c..5f9100aa 100644 --- a/model/01_access/sql/load_observations.sql +++ b/model/01_access/sql/load_observations.sql @@ -63,7 +63,7 @@ species_code_remap as ( -- extract observations of species of interest, -- within watershed groups where they are noted to occur -- - discarding observations outside of those groups --- - TODO - discard observations noted to be of suspect quality in data/user_observations_qa.csv +-- - TODO - discard observations noted to be of suspect quality obs as ( SELECT e.fish_observation_point_id, diff --git a/model/02_habitat_linear/habitat_linear.sh b/model/02_habitat_linear/habitat_linear.sh index cbd620f7..541a32f4 100755 --- a/model/02_habitat_linear/habitat_linear.sh +++ b/model/02_habitat_linear/habitat_linear.sh @@ -29,18 +29,5 @@ do done # with linear model processing complete, refresh materialized views -$PSQL -c "refresh materialized view bcfishpass.streams_vw" $PSQL -c "refresh materialized view bcfishpass.crossings_upstr_barriers_per_model_vw" $PSQL -c "refresh materialized view bcfishpass.crossings_vw" -$PSQL -c "refresh materialized view bcfishpass.streams_bt_vw" -$PSQL -c "refresh materialized view bcfishpass.streams_ch_vw" -$PSQL -c "refresh materialized view bcfishpass.streams_cm_vw" -$PSQL -c "refresh materialized view bcfishpass.streams_co_vw" -$PSQL -c "refresh materialized view bcfishpass.streams_pk_vw" -$PSQL -c "refresh materialized view bcfishpass.streams_salmon_vw" -$PSQL -c "refresh materialized view bcfishpass.streams_sk_vw" -$PSQL -c "refresh materialized view bcfishpass.streams_st_vw" -$PSQL -c "refresh materialized view bcfishpass.streams_wct_vw" -$PSQL -c "refresh materialized view bcfishpass.freshwater_fish_habitat_accessibility_model_salmon_vw" -$PSQL -c "refresh materialized view bcfishpass.freshwater_fish_habitat_accessibility_model_steelhead_vw" -$PSQL -c "refresh materialized view bcfishpass.freshwater_fish_habitat_accessibility_model_observations_vw"