fix conflicts

.github/workflows/sphinx-build.yml

@@ -1,12 +1,11 @@
-# from https://github.com/adempiere/docs/blob/master/.github/workflows/main.yml
-# This is a basic workflow to help you get started with Actions
 name: docs-deploy
-# Controls when the action will run. Triggers the workflow on push or pull request
-# events but only for the main branch
 on:
   push:
-    branches: [ main ]
-# A workflow run is made up of one or more jobs that can run sequentially or in parallel
+    paths:
+      - 'docs/**'
+  create:
+    tags:
+      - '*'
 jobs:
   docs:
     runs-on: ubuntu-latest

CHANGES.txt

@@ -3,29 +3,10 @@ Changes
 
 All issue numbers are relative to https://github.com/smnorris/bcfishpass/issues
 
-v0.1.dev2 (2023-04-24)
-------------------
-- better display of remediated streams (#326)
-- reorganization of model folders/components, with required components as subfolders
-- store parameters files in /data and include only default parameters (open data)
-- control watersheds to process only via parameters file
-- fix BT model to use anadromous spp observations to cancel barriers (#321)
-- ensure data displays in QGIS when accessed via FDW (#318)
-- minor fixes to valley confinement script
-- data fixes related to CARR species
-- add confirmed habitat data (PSE) within CCIRA study (area where it informs access model)
-- add confirmed habitat locations from New Graph BULK field reviews
-- use confirmed habitat to inform access model (#227)
-- note downstream observations in access model output (#290)
+v0.1.1 (2024-01-09)
+-------------------
+- remove typo creating cross-join in stream views (#446)
 
-v0.1.dev1 (2023-02-10)
-------------------
-- dev pre-release
-- precip/channel width/discharge models confirmed to build on fresh database
-- linear habitat model confirmed to build on fresh database
-
-v0.1.dev0 (2023-02-09)
-------------------
-- dev pre-release
-- access model build is successful on clean database
-- habitat models builds are untested on clean database, likely non-functional
+v0.1.0 (2024-01-08)
+-------------------
+- start regularly tagging stable commits to help ensure stability with scheduled processing

Makefile

@@ -7,14 +7,14 @@ QA_ACCESS_OUTPUTS = $(patsubst reports/access/sql/%.sql,reports/access/%.csv,$(Q
 
 all: model/03_habitat_lateral/data/habitat_lateral.tif
 
-.make/db: db/setup.sh db/sql/tables.sql db/sql/functions/*sql
+.make/db: db/setup.sh db/functions/*sql db/tables/*sql db/schemas/*sql db/views/*sql
 	mkdir -p .make
 	cd db; ./setup.sh
 	touch $@
 
 .make/parameters: .make/db parameters/*.csv
-	$(PSQL) -c "DELETE FROM bcfishpass.parameters_habitat_method";
-	$(PSQL) -c "DELETE FROM bcfishpass.parameters_habitat_thresholds";
+	$(PSQL) -c "truncate bcfishpass.parameters_habitat_method";
+	$(PSQL) -c "truncate bcfishpass.parameters_habitat_thresholds";
 	$(PSQL) -c "\copy bcfishpass.parameters_habitat_method FROM parameters/parameters_habitat_method.csv delimiter ',' csv header";
 	$(PSQL) -c "\copy bcfishpass.parameters_habitat_thresholds FROM parameters/parameters_habitat_thresholds.csv delimiter ',' csv header";
 	touch $@

README.md

@@ -37,13 +37,6 @@ An `environment.yml` file is provided to install the required tools:
     conda env create -f environment.yml
     conda activate bcfishpass
 
-Note that `cdo` is required for processing NetCDF discharge files, but this tool is not currently included in the conda environment.
-If processing discharge based habitat models, install `cdo` separately:
-
-- `conda install cdo` (if your system supports this, this may not work on ARM based Macs) 
-- `brew install cdo` 
-- install from [from source](https://code.mpimet.mpg.de/projects/cdo/wiki/Cdo#Download-Compile-Install)
-
 If the database you are working with does not already exist, create it:
 
     createdb bcfishpass
@@ -57,7 +50,12 @@ Once the database is created, load requirements `fwapg` and `bcfishobs` as per i
 
 ## Usage
 
-With `fwapg` and `bcfishobs` loaded to your database, build `bcfishpass`:
+With `fwapg` and `bcfishobs` completed, load railways and map tiles (currently required for output `crossings_vw`):
+
+    bcdata bc2pg whse_basemapping.gba_railway_tracks_sp
+    bcdata bc2pg whse_basemapping.dbm_mof_50k_grid
+
+Finally, build `bcfishpass`:
 
     make
 

data/README.md

@@ -2,6 +2,12 @@
 
 User editable data inputs - mostly fix tables. 
 
+## dfo_known_sockeye_lakes.csv
+
+`waterbody_poly_id` of FWA lakes known to potentially support Sockeye
+Source is `Conservation Units for Pacific Salmon under the Wild Salmon Policy, L. Blair Holtby1 and Kristine A. Ciruna2, 2007`
+Data were provided by PSF as 50k lakes shapefile then referenced to FWA.
+
 
 ## pscis_modelledcrossings_streams_xref.csv
 
@@ -27,6 +33,10 @@ Currently, controls the barrier status of natural barriers (gradient, falls, sub
 NOTE -  this table will only be used to identify modelled barriers known to be passable 
 (gradient, subsurface flow) once bcfishpass incorporates CABD falls
 
+## user_cabd_dams_exclusions.csv
+
+List of CABD dams to exclude from analysis (generally due to incorrect location or incorrect passability status).
+Use as temporary fix and submit any location or passability status to CWF for fix in source.
 
 ## user_falls.csv
 
@@ -66,10 +76,3 @@ Watersheds were assigned presence/absence of given species based on review of:
 This list is by no means a definitive list of watersheds that may support salmon.
 The list was generated as an input for prioritizing watersheds for further salmon/steelhead fish passage assessments.
 As such, watersheds in the Columbia (upstream of the Chief Joseph dam) and Mackenzie Basins were defined as out of scope.
-
-
-## dfo_known_sockeye_lakes.csv
-
-`waterbody_poly_id` of FWA lakes known to potentially support Sockeye
-Source is `Conservation Units for Pacific Salmon under the Wild Salmon Policy, L. Blair Holtby1 and Kristine A. Ciruna2, 2007`
-Data were provided by PSF as 50k lakes shapefile then referenced to FWA by SN.

data/load.sh

@@ -2,6 +2,6 @@
 set -euxo pipefail
 
 for table in ./*.csv; do
-	psql $DATABASE_URL -v ON_ERROR_STOP=1 -c "DELETE FROM bcfishpass.$(basename -- $table .csv)";
+	psql $DATABASE_URL -v ON_ERROR_STOP=1 -c "TRUNCATE bcfishpass.$(basename -- $table .csv)";
 	psql $DATABASE_URL -v ON_ERROR_STOP=1 -c "\copy bcfishpass.$(basename -- $table .csv) FROM $table delimiter ',' csv header";
 done

data/pscis_modelledcrossings_streams_xref.csv

@@ -3555,3 +3555,5 @@ stream_crossing_id,modelled_crossing_id,linear_feature_id,watershed_group_code,r
 198503,,,LNTH,SN,Appears to be on unmapped stream but certainly not on North Thomposon
 8210,,,MORK,SN,Crossing appears to be on an unmapped stream
 6116,,,MCGR,SN,Crossing is on unmapped trib to East Seebach Creek
+198680,3700003,,CRKD,AI,Confirmed in field that this is actual location of Neilson Creek
+198719,3702074,,CRKD,AI,Confirmed in field that this is actual location of Balsam Creek

data/user_cabd_dams_exclusions.csv

@@ -0,0 +1,8 @@
+cabd_id,reviewer_name,review_date,source,notes
+30b88f1b-dc21-4b42-8daa-d4cebae24142,SN,2023-01-01,not noted,
+3ca692b8-37cf-44e8-a783-2a315ec83102,SN,2023-01-01,not noted,
+ba5fe3eb-7bbe-45c1-b301-555872387c16,SN,2023-01-01,not noted,
+8a6b10fa-0d4f-4c45-857c-764d7e8028f8,SN,2023-01-01,not noted,
+48478e95-e063-4df6-a047-6aaf6087011b,SN,2023-01-01,not noted,
+e8e4bd88-c3c9-407c-a7a0-15c6c51704fd,SN,2023-01-01,not noted,dam may or may not be a barrier but location was incorrect at time of review
+6a792d8f-b9c5-44a4-a260-0f06c3b20821,SN,2023-01-01,not noted,dam may or may not be a barrier but location gets matched to Salmon River

data/user_modelled_crossing_fixes.csv

@@ -6160,3 +6160,4 @@ modelled_crossing_id,structure,watershed_group_code,reviewer_name,review_date,so
 3703577,NONE,PARS,MW,2023-11-23,imagery,No visible crossing or road.
 22200074,NONE,UFRA,MW,2023-11-23,field,No visible crossing or road.
 22202127,NONE,UFRA,MW,2023-11-23,field,Downstream railway crossing runs under both sets of tracks.
+3700192,NONE,CRKD,AI,2024-01-02,Bing/Google/ESRI, no structure

db/README.md

@@ -1,6 +1,9 @@
 # db setup
 
-Create functions and user editable tables in the bcfishpass database.
+Set up the bcfishpass database.
+
+Generally speaking, functions/schemas/tables/views are defined here for sources and outputs. 
+Additional internal tables are created by model scripts.
 
 
 ## Usage

db/sql/functions/load_dnstr.sql → db/functions/load_dnstr.sql

@@ -51,7 +51,10 @@ BEGIN
           b.localcode_ltree desc,
           b.downstream_route_measure desc
         ) as d
-    group by %2$s;',
+    group by %2$s
+
+    on conflict ( %2$s )
+    do update set %6$s = EXCLUDED.%6$s;',
 table_a,
 table_a_id,
 table_b,

db/functions/wcrp_barrier_count.sql

@@ -0,0 +1,91 @@
+-- function to query the view (so it is visible in pgfs)
+DROP FUNCTION IF EXISTS postgisftw.wcrp_barrier_count;
+CREATE FUNCTION postgisftw.wcrp_barrier_count(
+  watershed_group_code TEXT,
+  model_status TEXT default 'ALL'
+)
+-- watershed_group_code: BULK, LNIC, HORS, BOWR, QUES, CARR, ELKR
+-- model_status        : HABITAT, ACCESSIBLE, ALL (default)
+
+RETURNS TABLE (
+ crossing_feature_type TEXT,
+ n_passable bigint,
+ n_barrier bigint,
+ n_potential bigint,
+ n_unknown bigint,
+ total bigint
+ )
+LANGUAGE 'plpgsql'
+IMMUTABLE PARALLEL SAFE
+
+AS $$
+
+DECLARE
+   v_wsg   text := watershed_group_code;
+   v_model_status   text := model_status;
+
+BEGIN
+
+IF (v_model_status = 'ALL')
+then return query
+
+SELECT
+  v.crossing_feature_type,
+  sum(v.n_passable)::bigint as n_passable,
+  sum(v.n_barrier)::bigint as n_barrier,
+  sum(v.n_potential)::bigint as n_potential,
+  sum(v.n_unknown)::bigint as n_unknown,
+  (sum(v.n_passable) + sum(v.n_barrier) + sum(v.n_potential) + sum(v.n_unknown))::bigint as total
+FROM bcfishpass.wcrp_barrier_count_vw v
+WHERE v.watershed_group_code = v_wsg
+GROUP BY v.crossing_feature_type;
+
+ELSIF (v_model_status = 'ACCESSIBLE')
+then return query
+SELECT
+  v.crossing_feature_type,
+  sum(v.n_passable)::bigint as n_passable,
+  sum(v.n_barrier)::bigint as n_barrier,
+  sum(v.n_potential)::bigint as n_potential,
+  sum(v.n_unknown)::bigint as n_unknown,
+  (sum(v.n_passable) + sum(v.n_barrier) + sum(v.n_potential) + sum(v.n_unknown))::bigint as total
+FROM bcfishpass.wcrp_barrier_count_vw v
+WHERE
+  v.watershed_group_code = v_wsg and
+  v.model_status in ('ACCESSIBLE', 'HABITAT')
+group by v.crossing_feature_type;
+
+ELSIF (v_model_status = 'HABITAT')
+then return query
+SELECT
+  v.crossing_feature_type,
+  v.n_passable::bigint as n_passable,
+  v.n_barrier::bigint as n_barrier,
+  v.n_potential::bigint as n_potential,
+  v.n_unknown::bigint as n_unknown,
+  (v.n_passable + v.n_barrier + v.n_potential + v.n_unknown)::bigint as total
+FROM bcfishpass.wcrp_barrier_count_vw v
+WHERE
+  v.watershed_group_code = v_wsg and
+  v.model_status = 'HABITAT';
+
+END IF;
+
+END
+
+
+$$;
+
+COMMENT ON FUNCTION postgisftw.wcrp_barrier_count IS
+'Return count of crossings per crossing_feature_type within specified watershed group.
+Returns count of crossings accessible to target species if model_status=ACCESSIBLE is specified,
+Returns count of crossings below modelled habitat if model_status=HABITAT is specified
+Returns count of all crossings if model_status=ALL is specified (default)';
+
+REVOKE EXECUTE ON FUNCTION postgisftw.wcrp_barrier_count FROM public;
+
+-- test
+--select * from postgisftw.wcrp_barrier_count('QUES');
+--select * from postgisftw.wcrp_barrier_count('QUES','ALL');
+--select * from postgisftw.wcrp_barrier_count('QUES','ACCESSIBLE');
+--select * from postgisftw.wcrp_barrier_count('QUES','HABITAT');

db/functions/wcrp_barrier_extent.sql

@@ -0,0 +1,69 @@
+DROP FUNCTION IF EXISTS postgisftw.wcrp_barrier_extent(TEXT);
+
+CREATE FUNCTION postgisftw.wcrp_barrier_extent(watershed_group_code TEXT)
+
+-- watershed_group_code: BULK, LNIC, HORS, BOWR, QUES, CARR, ELKR
+
+  RETURNS TABLE(
+    crossing_feature_type TEXT,
+    all_habitat_blocked_km numeric,
+    total_habitat_km numeric,
+    extent_pct numeric
+  )
+
+  LANGUAGE 'plpgsql'
+  IMMUTABLE PARALLEL SAFE 
+
+AS $$
+
+DECLARE
+   v_wsg   text := watershed_group_code;
+
+BEGIN
+
+RETURN query
+
+with barriers as (
+  select
+    c.watershed_group_code,
+    ft.crossing_feature_type,
+    ROUND(SUM(h_wcrp.all_spawningrearing_belowupstrbarriers_km)::numeric, 2) as all_spawningrearing_blocked_km
+  FROM bcfishpass.crossings c
+  inner join bcfishpass.crossings_upstream_habitat uh using (aggregated_crossings_id)
+  inner join bcfishpass.crossings_feature_type_vw ft using (aggregated_crossings_id)
+  inner join bcfishpass.crossings_upstream_habitat_wcrp h_wcrp using (aggregated_crossings_id)
+  WHERE c.barrier_status IN ('POTENTIAL', 'BARRIER')
+  AND c.aggregated_crossings_id != '1100002536' -- don't count the Elko Dam in ELKR
+  AND c.watershed_group_code = v_wsg
+  GROUP BY c.watershed_group_code, ft.crossing_feature_type
+  ORDER BY c.watershed_group_code, ft.crossing_feature_type
+),
+
+total AS (
+  SELECT
+    b.crossing_feature_type,
+    b.all_spawningrearing_blocked_km,
+    (SELECT all_habitat FROM postgisftw.wcrp_habitat_connectivity_status(v_wsg)) as total_habitat_km
+  FROM barriers b
+)
+
+SELECT
+  t.crossing_feature_type,
+  t.all_spawningrearing_blocked_km,
+  t.total_habitat_km,
+  round((t.all_spawningrearing_blocked_km / t.total_habitat_km) * 100, 2) as extent_pct
+FROM total t;
+
+END
+
+
+$$;
+
+COMMENT ON FUNCTION postgisftw.wcrp_barrier_extent IS
+'Return km of all blocked spawning and rearing by barrier type, and the percentage of
+total spawning and rearing within given watershed group that this blocked habitat represents.';
+
+
+REVOKE EXECUTE ON FUNCTION postgisftw.wcrp_barrier_extent FROM public;
+
+-- select * from postgisftw.wcrp_barrier_extent('QUES');