diff --git a/.github/workflows/production-bcfishpass.yaml b/.github/workflows/production-bcfishpass.yaml index 9236abc2..c6b276d2 100644 --- a/.github/workflows/production-bcfishpass.yaml +++ b/.github/workflows/production-bcfishpass.yaml @@ -10,29 +10,27 @@ env: AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }} AWS_DEFAULT_REGION: ${{ secrets.AWS_DEFAULT_REGION }} jobs: - load-parameters: + load-csv: runs-on: ubuntu-latest environment: production container: ghcr.io/smnorris/bcfishpass:main steps: - name: Check out repository code uses: actions/checkout@v4 - - name: Run the jobs + - name: Load csv data, parameters run: | cp parameters/example_newgraph/*csv parameters/ - make --debug=basic .make/parameters + jobs/load_csv build-xings: runs-on: ubuntu-latest environment: production container: ghcr.io/smnorris/bcfishpass:main - needs: load-parameters + needs: load-csv steps: - name: Check out repository code uses: actions/checkout@v4 - name: Build modelled crossings - run: jobs/bcfishpass01_modelled_stream_crossings - - name: Release modelled crossings - run: jobs/release_modelled_stream_crossings + run: jobs/model_stream_crossings prep: runs-on: ubuntu-latest environment: production @@ -42,7 +40,7 @@ jobs: - name: Check out repository code uses: actions/checkout@v4 - name: Run the jobs - run: jobs/bcfishpass02_prep + run: jobs/model_prep model: runs-on: ubuntu-latest environment: production @@ -57,7 +55,7 @@ jobs: - name: Run the jobs run: | git config --global --add safe.directory /__w/bcfishpass/bcfishpass - jobs/bcfishpass03_model + jobs/model_run release: runs-on: ubuntu-latest environment: production diff --git a/.github/workflows/testing-bcfishpass.yaml b/.github/workflows/testing-bcfishpass.yaml index 98d18e5b..4befa087 100644 --- a/.github/workflows/testing-bcfishpass.yaml +++ b/.github/workflows/testing-bcfishpass.yaml @@ -8,7 +8,7 @@ on: env: DATABASE_URL: ${{ secrets.DATABASE_URL }} jobs: - load-parameters: + load-csv: runs-on: ubuntu-latest environment: testing container: ghcr.io/smnorris/bcfishpass:main @@ -18,19 +18,17 @@ jobs: - name: Run the jobs run: | cp parameters/example_newgraph/*csv parameters/ - make --debug=basic .make/parameters + jobs/load_csv load-xings: runs-on: ubuntu-latest environment: testing container: ghcr.io/smnorris/bcfishpass:main - needs: load-parameters + needs: load-csv steps: - name: Check out repository code uses: actions/checkout@v4 - name: Run the jobs - run: | - cd model/01_access/modelled_stream_crossings - make .make/download_archive + run: jobs/load_modelled_stream_crossings prep: runs-on: ubuntu-latest environment: testing @@ -40,7 +38,7 @@ jobs: - name: Check out repository code uses: actions/checkout@v4 - name: Run the jobs - run: jobs/bcfishpass02_prep + run: jobs/model_prep model: runs-on: ubuntu-latest environment: testing @@ -55,4 +53,4 @@ jobs: - name: Run the jobs run: | git config --global --add safe.directory /__w/bcfishpass/bcfishpass - jobs/bcfishpass03_model \ No newline at end of file + jobs/model_run \ No newline at end of file diff --git a/.gitignore b/.gitignore index d5116579..40c86b44 100644 --- a/.gitignore +++ b/.gitignore @@ -35,4 +35,7 @@ model/01_access/qa/*csv # only include example parameters, # not whatever is being used for a given project/test -parameters/parameters_habitat*.csv \ No newline at end of file +parameters/parameters_habitat*.csv + +# postgres dump files +*.dump \ No newline at end of file diff --git a/CHANGES.txt b/CHANGES.txt deleted file mode 100644 index c481b430..00000000 --- a/CHANGES.txt +++ /dev/null @@ -1,23 +0,0 @@ -Changes -======= - -All issue numbers are relative to https://github.com/smnorris/bcfishpass/issues - -v0.4.0 (2024-03-13) -------------------- -- add draft FPTWG per assessment watershed summary view (#471) -- include 2023 forest cover - -v0.3.0 (2024-03-05) -------------------- -- setup schema before loading data (do not drop source/target tables on model refresh, #443) -- add WCRP queries/functions (#288) -- fix symbolization of salmon/steelhead above dams that are barriers (#396) - -v0.1.1 (2024-01-09) -------------------- -- remove typo creating cross-join in stream views (#446) - -v0.1.0 (2024-01-08) -------------------- -- start regularly tagging stable commits to help ensure stability with scheduled processing \ No newline at end of file diff --git a/Makefile b/Makefile deleted file mode 100644 index 1c62b2e7..00000000 --- a/Makefile +++ /dev/null @@ -1,88 +0,0 @@ -PSQL=psql $(DATABASE_URL) -v ON_ERROR_STOP=1 # point psql to db and stop on errors - -WSG = $(shell $(PSQL) -AtX -c "SELECT watershed_group_code FROM bcfishpass.parameters_habitat_method") - -QA_ACCESS_SCRIPTS = $(wildcard reports/access/sql/*.sql) -QA_ACCESS_OUTPUTS = $(patsubst reports/access/sql/%.sql,reports/access/%.csv,$(QA_SCRIPTS)) - -all: model/03_habitat_lateral/data/habitat_lateral.tif - -# NOTE - db must exist and be set up -# (run /jobs/setup, load fwa and bcfishobs) - -.make/parameters: parameters/*.csv - mkdir -p .make - $(PSQL) -c "truncate bcfishpass.parameters_habitat_method"; - $(PSQL) -c "truncate bcfishpass.parameters_habitat_thresholds"; - $(PSQL) -c "\copy bcfishpass.parameters_habitat_method FROM parameters/parameters_habitat_method.csv delimiter ',' csv header"; - $(PSQL) -c "\copy bcfishpass.parameters_habitat_thresholds FROM parameters/parameters_habitat_thresholds.csv delimiter ',' csv header"; - touch $@ - -.make/data: data/*.csv - mkdir -p .make - cd data; ./load.sh - touch $@ - -.make/model_access: .make/data .make/parameters - cd model/01_access; make - touch $@ - -.make/habitat_linear: .make/model_access - cd model/02_habitat_linear; make - touch $@ - -# ----- -# CROSSING STATS -# add various columns holding upstream/downstream metrics to crossings table and barriers_anthropogenic -# ----- -.make/crossing_stats: .make/habitat_linear \ - reports/crossings/sql/point_report_columns.sql \ - reports/crossings/sql/point_report.sql \ - reports/crossings/sql/point_report_obs_belowupstrbarriers.sql \ - reports/crossings/sql/all_spawningrearing_per_barrier.sql - # todo - optimize below to write to temp tables rather than applying updates - - # run report per watershed group on barriers_anthropogenic - $(PSQL) -f reports/crossings/sql/point_report_columns.sql \ - -v point_table=barriers_anthropogenic - for wsg in $(WSG) ; do \ - set -e ; $(PSQL) -f reports/crossings/sql/point_report.sql \ - -v point_table=barriers_anthropogenic \ - -v point_id=barriers_anthropogenic_id \ - -v barriers_table=barriers_anthropogenic \ - -v dnstr_barriers_id=barriers_anthropogenic_dnstr \ - -v wsg=$$wsg ; \ - done - - ## run report per watershed group on crossings - $(PSQL) -f reports/crossings/sql/point_report_columns.sql \ - -v point_table=crossings - for wsg in $(WSG) ; do \ - set -e ; $(PSQL) -f reports/crossings/sql/point_report.sql \ - -v point_table=crossings \ - -v point_id=aggregated_crossings_id \ - -v barriers_table=barriers_anthropogenic \ - -v dnstr_barriers_id=barriers_anthropogenic_dnstr \ - -v wsg=$$wsg ; \ - done - - # For OBS in the crossings table, report on belowupstrbarriers columns. - # This requires a separate query - # (because the dnstr_barriers_anthropogenic is used in above report, - # and that misses the OBS of interest) - $(PSQL) -f reports/crossings/sql/point_report_obs_belowupstrbarriers.sql - - # add habitat per barrier column to crossings table - for wsg in $(WSG) ; do \ - psql -f reports/crossings/sql/all_spawningrearing_per_barrier.sql -v wsg=$$wsg ; \ - done - - touch $@ - -# ----- -# LATERAL HABITAT MODEL -# ----- -model/03_habitat_lateral/data/habitat_lateral.tif: .make/habitat_linear \ - .make/crossing_stats - cd model/habitat_lateral; make .make/habitat_lateral - diff --git a/README.md b/README.md index 2f8f907c..946ef7c6 100644 --- a/README.md +++ b/README.md @@ -23,7 +23,7 @@ See the [Documentation](https://smnorris.github.io/bcfishpass/) for details. - [fwapg](https://github.com/smnorris/fwapg) - [bcfishobs](https://github.com/smnorris/bcfishobs) -## Setup +## Setup / Usage `bcfishpass` is a collection of shell/sql/Python scripts. To download and use the latest: @@ -47,37 +47,41 @@ All scripts presume that the `DATABASE_URL` environment variable points to your export DATABASE_URL=postgresql://postgres@localhost:5432/bcfishpass -Set up the database schema: - - jobs/setup - -Load FWA data: +Load FWA: git clone https://github.com/smnorris/fwapg cd fwapg - mkdir -p .make; touch .make/db # just reload data - mkdir -p data - make --debug=basicjobs/load_fwa + make --debug=basic + +Load/run `bcfishobs`: + + git clone git@github.com:smnorris/bcfishobs.git + cd bcfishobs + make --debug=basic -Load all additional data: +Create db schema: - jobs/load_static + jobs/db_setup + +Load source data: + + jobs/load_static jobs/load_monthly jobs/load_weekly -Run `bcfishobs`: +Load cached bcfishpass data: - git clone git@github.com:smnorris/bcfishobs.git - cd bcfishobs - mkdir -p .make - make -t .make/setup - make -t .make/load_static - make -t .make/fiss_fish_obsrvtn_pnt_sp - make --debug=basic + jobs/load_gradient_barriers + jobs/load_modelled_stream_crossings + +Run the model: + + jobs/model_prep + jobs/model_run -Finally, navigate back to the root bcfishpass folder and build `bcfishpass`: +# Backups - make +Backup strategies will vary but it can be useful to dump the entire database to file. +This appends the date and commit tag date to the file name: -Note that it is possible (and often preferred) to build components of the modelling separately. -Refer to the various README files in the subfolders within the `model` folder for more info. \ No newline at end of file + pg_dump -Fc $DATABASE_URL > bcfishpass.$(git describe --tags --abbrev=0).$(date +%F).dump \ No newline at end of file diff --git a/data/README.md b/data/README.md index 29eab23a..61db0a70 100644 --- a/data/README.md +++ b/data/README.md @@ -33,11 +33,13 @@ Currently, controls the barrier status of natural barriers (gradient, falls, sub NOTE - this table will only be used to identify modelled barriers known to be passable (gradient, subsurface flow) once bcfishpass incorporates CABD falls + ## user_cabd_dams_exclusions.csv List of CABD dams to exclude from analysis (generally due to incorrect location or incorrect passability status). To be used as temporary fix only - submit any location or passability status to CWF [here](https://forms.office.com/Pages/ResponsePage.aspx?id=A8N2i-wB1UCNmmwvbXjc15ERVmcC4dFPn5j4q5-aulRURE1TSjBJNEtDNlY0WTlXVFY5MkFOMzRVUS4u) + ## user_falls.csv Falls not present in FWA/FISS. Both barriers and non-barriers may be included. @@ -62,6 +64,11 @@ Update the barrier status of PSCIS crossings (for bcfishpass modelling only). PS support planning by adding barrier status of a crossing before a submission is made. +## wcrp.csv + +A list of watershed groups and target species for CWF WCRP reporting. + + ## wsg_species_presence.csv A list of all BC watershed groups and presence/absence of target species (Coho, Chinook, Sockeye, Steelhead, West Slope Cutthroat Trout). diff --git a/data/load.sh b/data/load.sh deleted file mode 100755 index 4a4c7c93..00000000 --- a/data/load.sh +++ /dev/null @@ -1,7 +0,0 @@ -#!/bin/bash -set -euxo pipefail - -for table in ./*.csv; do - psql $DATABASE_URL -v ON_ERROR_STOP=1 -c "TRUNCATE bcfishpass.$(basename -- $table .csv)"; - psql $DATABASE_URL -v ON_ERROR_STOP=1 -c "\copy bcfishpass.$(basename -- $table .csv) FROM $table delimiter ',' csv header"; -done diff --git a/data/user_cabd_blkey_xref.csv b/data/user_cabd_blkey_xref.csv new file mode 100644 index 00000000..e3ca903f --- /dev/null +++ b/data/user_cabd_blkey_xref.csv @@ -0,0 +1,3 @@ +cabd_id,blue_line_key,reviewer_name,review_date,notes +30115bb6-9a75-4c33-bb37-4243ce33e43d,356352213,SN,2024-05-28,matched to stream named in CABD +c3721358-4334-41c0-a4c1-1fa729803d47,354154385,SN,2024-05-28,matched to stream named in CABD (but it is unclear where cascade is actually located) \ No newline at end of file diff --git a/data/user_cabd_dams_exclusions.csv b/data/user_cabd_dams_exclusions.csv index 4f0c9fb4..adfc45c5 100644 --- a/data/user_cabd_dams_exclusions.csv +++ b/data/user_cabd_dams_exclusions.csv @@ -1,8 +1 @@ -cabd_id,reviewer_name,review_date,source,notes -30b88f1b-dc21-4b42-8daa-d4cebae24142,SN,2023-01-01,not noted, -3ca692b8-37cf-44e8-a783-2a315ec83102,SN,2023-01-01,not noted, -ba5fe3eb-7bbe-45c1-b301-555872387c16,SN,2023-01-01,not noted, -8a6b10fa-0d4f-4c45-857c-764d7e8028f8,SN,2023-01-01,not noted, -48478e95-e063-4df6-a047-6aaf6087011b,SN,2023-01-01,not noted, -e8e4bd88-c3c9-407c-a7a0-15c6c51704fd,SN,2023-01-01,not noted,dam may or may not be a barrier but location was incorrect at time of review -6a792d8f-b9c5-44a4-a260-0f06c3b20821,SN,2023-01-01,not noted,dam may or may not be a barrier but location gets matched to Salmon River \ No newline at end of file +cabd_id,reviewer_name,review_date,source,notes \ No newline at end of file diff --git a/data/user_falls.csv b/data/user_falls.csv index ca102bf0..3c695536 100644 --- a/data/user_falls.csv +++ b/data/user_falls.csv @@ -1,75 +1,2 @@ falls_name,height,barrier_ind,blue_line_key,downstream_route_measure,watershed_group_code,reviewer_name,review_date,source,notes -Porphyry Creek Falls,,f,360874977,13421,BULK,CWF,2021-01-01,WWD, -Mudflat Falls,,f,360855611,10172,BULK,CWF,2021-01-01,WWD, -Twin Falls,,f,360821214,6764,BULK,CWF,2021-01-01,WWD, -Moricetown Falls,,f,360873822,55275,BULK,CWF,2021-01-01,WWD, -Buck Falls,,f,360886221,46656,BULK,CWF,2021-01-01,WWD, -Dungate Creek Falls,,f,360760898,13621,BULK,CWF,2021-01-01,WWD, -,,f,356424866,477,ELKR,CWF,2021-01-01,CANVEC, -Fairy Creek Falls,,f,356570035,2011,ELKR,CWF,2021-01-01,WWD, -Lower Josephine Falls,,f,356568903,19098,ELKR,CWF,2021-01-01,WWD, -Middle Josephine Falls,,f,356568903,19718,ELKR,CWF,2021-01-01,WWD, -Nivelle Creek Falls,,f,356546150,17,ELKR,CWF,2021-01-01,WWD, -Josephine Falls,,f,356568903,20200,ELKR,CWF,2021-01-01,WWD, -Morrissey Falls,,t,356562350,10858,ELKR,SN,2022-03-08,WWD,Noted as 15m barrier by New Graph -Upper Fairy Creek Falls,,f,356570035,5615,ELKR,CWF,2021-01-01,WWD, -Elk River Falls,,f,356570562,21797,ELKR,CWF,2021-01-01,WWD, -Waterfall Creek Falls,,f,356266320,184,LNIC,CWF,2021-01-01,WWD, -,,f,356363343,15924,LNIC,CWF,2021-01-01,CANVEC, -,,f,356363343,15287,LNIC,CWF,2021-01-01,CANVEC, -,,f,356363343,11919,LNIC,CWF,2021-01-01,CANVEC, -,,f,356363343,15223,LNIC,CWF,2021-01-01,CANVEC, -,,f,356363343,11263,LNIC,CWF,2021-01-01,CANVEC, -,,f,356363343,6000,LNIC,CWF,2021-01-01,CANVEC, -,,f,356363343,11380,LNIC,CWF,2021-01-01,CANVEC, -,,f,356363343,6811,LNIC,CWF,2021-01-01,CANVEC, -,,f,356363343,14725,LNIC,CWF,2021-01-01,CANVEC, -,,f,356363343,16715,LNIC,CWF,2021-01-01,CANVEC, -,,f,356363343,10079,LNIC,CWF,2021-01-01,CANVEC, -,,f,356363343,8866,LNIC,CWF,2021-01-01,CANVEC, -,,f,356228196,0,LNIC,CWF,2021-01-01,CANVEC, -,,f,356363343,6749,LNIC,CWF,2021-01-01,CANVEC, -,,f,356363343,8621,LNIC,CWF,2021-01-01,CANVEC, -,,f,356363343,15761,LNIC,CWF,2021-01-01,CANVEC, -,,f,356363343,16668,LNIC,CWF,2021-01-01,CANVEC, -,,f,356363343,9821,LNIC,CWF,2021-01-01,CANVEC, -,,f,356363343,9171,LNIC,CWF,2021-01-01,CANVEC, -,,f,356363343,5923,LNIC,CWF,2021-01-01,CANVEC, -,,f,356363343,12221,LNIC,CWF,2021-01-01,CANVEC, -,,f,356363343,14889,LNIC,CWF,2021-01-01,CANVEC, -Canyon Creek Falls,,f,360873164,15825,BULK,CWF,2021-01-01,Irvine 2020, -Deep Creek Falls,,f,360880400,11184,BULK,CWF,2021-01-01,Irvine 2020, -Deep Creek Falls 2,,f,360880400,12418,BULK,CWF,2021-01-01,Irvine 2020, -,,f,360615745,1780,BULK,CWF,2021-01-01,Irvine 2020, -,,f,360817924,4781,BULK,CWF,2021-01-01,Irvine 2020, -,,f,360817924,2958,BULK,CWF,2021-01-01,Irvine 2020, -Rasberry Falls,,t,360817924,2880,BULK,AI,2021-01-01,Irvine 2020, -ThreeFalls,,t,360856488,6732,BULK,AI,2021-01-01,Irvine 2020, -,,f,360856488,9963,BULK,CWF,2021-01-01,Irvine 2020, -,,f,360856488,10086,BULK,CWF,2021-01-01,Irvine 2020, -RichfieldFalls,,t,360788426,3461,BULK,AI,2021-01-01,Irvine 2020, -Lower Findlay Falls,,f,360788426,7274,BULK,CWF,2021-01-01,Irvine 2020, -Redtop Falls,,f,360883143,1441,BULK,CWF,2021-01-01,Irvine 2020, -Tacheck Falls,,f,360788426,13765,BULK,CWF,2021-01-01,Irvine 2020, -Bulkley Falls,,f,360873822,233558,BULK,CWF,2021-01-01,Irvine 2020, -,,f,360872135,5262,BULK,CWF,2021-01-01,Irvine 2020, -,,f,360872135,4962,BULK,CWF,2021-01-01,Irvine 2020, -,,f,360827197,1482,BULK,CWF,2021-01-01,Irvine 2020, -,,f,360827197,3740,BULK,CWF,2021-01-01,Irvine 2020, -,,f,360827197,3826,BULK,CWF,2021-01-01,Irvine 2020, -,,f,360827197,3886,BULK,CWF,2021-01-01,Irvine 2020, -,,f,360827197,4677,BULK,CWF,2021-01-01,Irvine 2020, -Lower Dungate,,f,360760898,3168,BULK,CWF,2021-01-01,Irvine 2020, -Lower Dungate,,f,360760898,2829,BULK,CWF,2021-01-01,Irvine 2020, -,,f,360760898,1530,BULK,CWF,2021-01-01,Irvine 2020, -,,f,360657545,773,BULK,CWF,2021-01-01,Irvine 2020, -Dungate Falls,,f,360760898,13761,BULK,CWF,2021-01-01,Irvine 2020, -Findlay Falls,,f,360788426,9899,BULK,CWF,2021-01-01,Irvine 2020, -,,t,356264650,4,LNIC,NJO,2021-01-01,CWF WCRP,Seasonal stream descends over vertical bedrock wall no headwater lakes to support any isolated fish population -Black Creek Falls,,t,356317730,2087,HORS,SH,2021-01-01,Hocquard 2021, -Wilson Falls,,t,360869846,6617,BULK,AI,2021-01-01,Irvine 2020, -,,t,360875378,3800,BULK,AI,2021-01-01,ECOCAT,McQuarrie Creek 4-5 confirmed impassable falls - http://a100.gov.bc.ca/appsdata/acat/documents/r8931/Mid-BulkleyDetailedFishHabitatRiparian.ChannelAss_1169052197910_e76ab8bf05ee4953b589da961b220f69.pdf -Grave Canyon,,t,356528119,344,ELKR,NMW,2021-01-01,CWF WCRP,Large canyon seperating fish species (WCT are present above the barrier) -Goldstream Falls,,t,354152425,4578,VICT,SN,2022-03-28,Google, -,,t,356318872,2040,HORS,TMK,2023-06-15,CWF barrier prioritization,The waterfall 220 meters above crossing 57430 presents a complete and permanent barrier to all upstream fish passage. -Bonnington Falls,,t,354153515,22292,KOTL,SN,2023-08-11,Wikipedia/CABD,Historic falls restricting salmon from Kootenay Lake and upstream \ No newline at end of file +Bonnington Falls,,t,356570348,22292,KOTL,SN,2023-08-11,Wikipedia/CABD,Historic falls restricting salmon from Kootenay Lake and upstream \ No newline at end of file diff --git a/data/user_habitat_classification.csv b/data/user_habitat_classification.csv index 999f5e58..09b07855 100644 --- a/data/user_habitat_classification.csv +++ b/data/user_habitat_classification.csv @@ -14872,7 +14872,7 @@ 360887120,19988,20088,TSAY,CH,spawning,t,SN,2023-08-17,FISS spawning points, 360887120,22207,22307,TSAY,CO,spawning,t,SN,2023-08-17,FISS spawning points, 360887120,22867,22967,TSAY,CO,spawning,t,SN,2023-08-17,FISS spawning points, -360887178,2228.49,8202.77,BABL,SK,spawning,t,SN,2023-08-17,Pacific Salmon Foundation, +360887178,2228.49,8100,BABL,SK,spawning,t,SN,2023-08-17,Pacific Salmon Foundation,adjusted upstream measure down ~100m to fall downstream of 25pct gradient barrier 360887178,4109,4209,BABL,CH,spawning,t,SN,2023-08-17,FISS spawning points, 360887178,4109,4209,BABL,CO,spawning,t,SN,2023-08-17,FISS spawning points, 360887278,45966,46066,LSKE,PK,spawning,t,SN,2023-08-17,FISS spawning points, @@ -15294,7 +15294,7 @@ 360873822,168869,170329.9685,BULK,SK,spawning,t,LB,2024-06-06,Pacific Salmon Foundation,River sockeye 360881038,0,280.2148356,BULK,SK,spawning,t,LB,2024-06-06,Pacific Salmon Foundation,Lake sockeye 360804115,652.18,852.93,BULK,CO,spawning,t,LB,2024-06-17,Pacific Salmon Foundation, -356209928,25.6202005,486.7184822,SHUL,CO,Rearing,t,MC,2024-06-27,Local Knowledge,Local Knowledge -CO rearing habitati +356209928,25.6202005,486.7184822,SHUL,CO,Rearing,t,MC,2024-06-27,Local Knowledge,Local Knowledge - CO rearing habitat 356209928,486.719176,822,SHUL,CO,Spawning,t,MC,2024-06-27,Local Knowledge,Local Knowledge - CO Spawning 356343800,0,838,SHUL,SK,Spawning,t,MC,2024-06-27,Local Knowledge,Local Knowledge - CO and SK Spawning 356348431,1481,1681,SHUL,SK,Rearing,t,MC,2024-06-27,Local Knowledge,Local Knowledge - Some SK seen 300m upstream @@ -15302,4 +15302,4 @@ 356137288,265.6052138,408,SHUL,SK,Spawning,t,MC,2024-06-27,Local Knowledge,Local Knowledge - Adult SK observed up to rail line 356347377,0,558,BONP,CH,Spawning,t,MC,2024-06-27,Local Knowledge,Local Knowledge - major spawning for CH 356311064,6630,8353.604837,QUES,CO,rearing,f,MC,2024-07-04,Lhtako Dene WCRP Field Assessments,No channel or water -356287539,1403.205103,1444.490069,QUES,CO,rearing,f,MC,2024-07-04,Lhtako Dene WCRP Field Assessments,No wetland/water +356287539,1403.205103,1444.490069,QUES,CO,rearing,f,MC,2024-07-04,Lhtako Dene WCRP Field Assessments,No wetland/water \ No newline at end of file diff --git a/data/user_modelled_crossing_fixes.csv b/data/user_modelled_crossing_fixes.csv index 90752fa5..c8ce077a 100644 --- a/data/user_modelled_crossing_fixes.csv +++ b/data/user_modelled_crossing_fixes.csv @@ -21727,5 +21727,21 @@ modelled_crossing_id,structure,watershed_group_code,reviewer_name,review_date,so 17103896,NONE,QUES,MC,2024-07-08,Lhtako Dene WCRP Field Assessments,"Old clear area, may be part of mine. Cleared in the last 1-3 years. Small (2x2m) wetland with cattails seen, but no water or streams. No culverts. " 17102208,NONE,QUES,MC,2024-07-08,Lhtako Dene WCRP Field Assessments,"Very old road, overgrown. No crossing found. Slightly wet area but no water seen. " 17102191,NONE,QUES,MC,2024-07-08,Lhtako Dene WCRP Field Assessments,"No water or wetland. Possible culvert removed at old road, shallow overgrown ditch. No culvert seen. " -17100098,NONE,QUES,MC,2024-07-08,Lhtako Dene WCRP Field Assessments,No wetland/water. No culvert seen on road. Private property on either side of road. +17100098,NONE,QUES,MC,2024-07-08,Lhtako Dene WCRP Field Assessments,No wetland/water. No culvert seen on road. Private property on either side of road. 17102374,NONE,QUES,MC,2024-07-08,Lhtako Dene WCRP Field Assessments,Cannot find inlet. Bottom of culvert disintegrated where it touches ground. No water. +17104186,NONE,QUES,MC,2024-07-08,WCRP partner field assessments,No channel or water. Culvert on side road connecting manmade ditch line along road. +1400691,OBS,BONP,LB,2024-07-11,imagery, +1404128,OBS,BONP,LB,2024-07-11,imagery, +1406051,NONE,BONP,LB,2024-07-11,imagery,Misinterpretation of tranportation line/stream line +1406050,NONE,BONP,LB,2024-07-11,imagery,Misinterpretation of tranportation line/stream line +1406094,NONE,BONP,LB,2024-07-11,imagery,Misinterpretation of tranportation line/stream line. Leaving most downstream structure 1406093 +1406095,NONE,BONP,LB,2024-07-11,imagery,Misinterpretation of tranportation line/stream line. Leaving most downstream structure 1406093 +24739453,OBS,BONP,LB,2024-07-11,imagery, +24743296,NONE,BONP,LB,2024-07-11,imagery,Structures are really close together and there likely isn't two. Leaving most downstream structure 24743298 +1402061,FORD,BONP,LB,2024-07-11,imagery, +1402301,FORD,BONP,LB,2024-07-11,imagery, +1402720,NONE,BONP,LB,2024-07-11,imagery,Misinterpretation of transportation line. There is no road here meaning no crossing. +1403172,FORD,BONP,LB,2024-07-11,imagery, +1405747,FORD,BONP,LB,2024-07-11,imagery, +11302064,NONE,LNIC,LB,2024-07-16,imagery,Ortho image indicates the stream has likely migrated/been redirected to the north and no longer crosses this road. +11301333,OBS,LNIC,LB,2024-07-16,imagery,Appears to be a bridge on sat imagery- stream goes to north of where it's mapped \ No newline at end of file diff --git a/data/wcrp_watersheds.csv b/data/wcrp_watersheds.csv new file mode 100644 index 00000000..d955b482 --- /dev/null +++ b/data/wcrp_watersheds.csv @@ -0,0 +1,14 @@ +watershed_group_code,ch,cm,co,pk,sk,st,wct,notes +ATNA,t,,t,,t,,, +BELA,t,,t,,t,,, +BONP,t,,t,,t,,, +BOWR,t,,t,,t,,, +BULK,t,,t,,t,t,, +CARR,t,,t,,t,,, +ELKR,,,,,,,t, +HORS,t,,t,,t,,, +LNIC,t,,t,,,t,, +NECL,t,,t,,t,,, +QUES,t,,t,,t,,, +SHUL,t,,t,,t,,, +USHU,t,,t,,t,,, \ No newline at end of file diff --git a/db/README.md b/db/README.md index 73413359..c7478287 100644 --- a/db/README.md +++ b/db/README.md @@ -1,13 +1,26 @@ -# db setup +# db setup / migrations -Set up the bcfishpass database. +Create database schema. Note that some internal bcfishpass tables are created by scripts in /model and not defined here. -Generally speaking, functions/schemas/tables/views are defined here for sources and outputs. -Additional internal tables are created by model scripts. +## Usage +If starting from scratch (an empty database, apart from FWA and bcfishobs), build source data schemas then run all bcfishpass migrations: -## Usage + cd sources; ./migrate.sh; cd .. + for tag in v* ;do + cd "$tag"; ./migrate.sh; cd .. + done + +Or apply db migrations as required: + + cd v; ./migrate.sh + +Note that the /sources folder is only separate from the tagged scripts to make setup of the testing database simple - db schema changes to source data can be applied in subsequent migrations. + +## Creating migrations -From repository root, with $DATABASE_URL connection defined as db superuser: +1. Add a new folder with incremented version tag +2. Within new folder, add sql file(s) as required and a `migrate.sh` script that executes all required sql - jobs/setup \ No newline at end of file +Currently, no migration tooling is used and only one way migrations are supported. +Note that tools like `squitch`, `flyway` or `alembic` could be used to make migrations safer. diff --git a/jobs/setup b/db/sources/migrate.sh similarity index 82% rename from jobs/setup rename to db/sources/migrate.sh index fee132ae..6f0d5c91 100755 --- a/jobs/setup +++ b/db/sources/migrate.sh @@ -1,23 +1,19 @@ #!/bin/bash set -euxo pipefail -#------- -# set up (almost) empty db schema (run as db superuser) -#------- PSQL="psql $DATABASE_URL -v ON_ERROR_STOP=1" -# create fwa schema -git clone https://github.com/smnorris/fwapg.git -cd fwapg; make .make/db; cd .. ; rm -rf fwapg +# create required schemas +$PSQL -f sql/schemas.sql -# create bcfishobs schema (and load selected data) -git clone https://github.com/smnorris/bcfishobs.git -cd bcfishobs; make .make/setup; cd ..; rm -rf bcfishobs +# define cabd +$PSQL -f sql/cabd.sql -$PSQL -f db/schemas.sql +# define dra +$PSQL -f sql/whse_basemapping.transport_line.sql -# create empty whse tables +# create whse tables as specified by bcdc api, using bc2pg for table in whse_admin_boundaries.clab_indian_reserves \ whse_admin_boundaries.clab_national_parks \ whse_admin_boundaries.adm_nr_districts_spg \ @@ -74,15 +70,6 @@ $PSQL -c \ CREATE INDEX ON whse_forest_vegetation.veg_comp_lyr_r1_poly (bclcs_level_5); \ CREATE INDEX ON whse_forest_vegetation.veg_comp_lyr_r1_poly (map_id);" -# create bcfishpass tables/views/functions -for sql in db/tables/*.sql ; do - $PSQL -f "$sql" -done - -for sql in db/views/*.sql ; do - $PSQL -f "$sql" -done - -for sql in db/functions/*.sql ; do - $PSQL -f "$sql" -done +# create additional views on source data +$PSQL -f sql/bcdata.ften_range_poly_carto_vw.sql +$PSQL -f sql/bcdata.parks.sql diff --git a/db/views/bcdata.ften_range_poly_carto_vw.sql b/db/sources/sql/bcdata.ften_range_poly_carto_vw.sql similarity index 100% rename from db/views/bcdata.ften_range_poly_carto_vw.sql rename to db/sources/sql/bcdata.ften_range_poly_carto_vw.sql diff --git a/db/views/bcdata.parks.sql b/db/sources/sql/bcdata.parks.sql similarity index 100% rename from db/views/bcdata.parks.sql rename to db/sources/sql/bcdata.parks.sql diff --git a/db/tables/cabd.sql b/db/sources/sql/cabd.sql similarity index 100% rename from db/tables/cabd.sql rename to db/sources/sql/cabd.sql diff --git a/db/schemas.sql b/db/sources/sql/schemas.sql similarity index 86% rename from db/schemas.sql rename to db/sources/sql/schemas.sql index 7c35d335..1843010e 100644 --- a/db/schemas.sql +++ b/db/sources/sql/schemas.sql @@ -1,5 +1,4 @@ -create schema bcdata; -create schema bcfishpass; +-- create schemas not created by fwapg/bcfishobs scripts create schema cabd; create schema whse_admin_boundaries; create schema whse_cadastre; diff --git a/db/tables/transport_line.sql b/db/sources/sql/whse_basemapping.transport_line.sql similarity index 100% rename from db/tables/transport_line.sql rename to db/sources/sql/whse_basemapping.transport_line.sql diff --git a/db/tables/observations.sql b/db/tables/observations.sql deleted file mode 100644 index 958fd341..00000000 --- a/db/tables/observations.sql +++ /dev/null @@ -1,24 +0,0 @@ -CREATE TABLE bcfishpass.observations -( - fish_obsrvtn_event_id bigint primary key, - linear_feature_id bigint , - blue_line_key integer , - wscode_ltree ltree , - localcode_ltree ltree , - downstream_route_measure double precision , - watershed_group_code character varying(4) , - species_codes text[] , - observation_ids int[] , - observation_dates date[] , - geom geometry(PointZM, 3005) -); - --- index -create index obsrvtn_linear_feature_id_idx on bcfishpass.observations (linear_feature_id); -create index obsrvtn_blue_line_key_idx on bcfishpass.observations (blue_line_key); -create index obsrvtn_watershed_group_code_idx on bcfishpass.observations (watershed_group_code); -create index obsrvtn_wsc_gidx on bcfishpass.observations using gist (wscode_ltree); -create index obsrvtn_wsc_bidx on bcfishpass.observations using btree (wscode_ltree); -create index obsrvtn_lc_gidx on bcfishpass.observations using gist (localcode_ltree); -create index obsrvtn_lc_bidx on bcfishpass.observations using btree (localcode_ltree); -create index obsrvtn_geom_idx on bcfishpass.observations using gist (geom); \ No newline at end of file diff --git a/db/v0.5.0/migrate.sh b/db/v0.5.0/migrate.sh new file mode 100755 index 00000000..0287adad --- /dev/null +++ b/db/v0.5.0/migrate.sh @@ -0,0 +1,31 @@ +#!/bin/bash +set -euxo pipefail + + +PSQL="psql $DATABASE_URL -v ON_ERROR_STOP=1" + + +$PSQL -c "create schema bcfishpass" + +for sql in sql/tables/*.sql ; do + $PSQL -f "$sql" +done + +for sql in sql/views/*.sql ; do + $PSQL -f "$sql" +done + +for sql in sql/functions/*.sql ; do + $PSQL -f "$sql" +done + +# tag the database version +# note - the tag is just a reference to note the latest script applied to a given db, there is no method for downgrades +$PSQL -c "create table bcfishpass.db_version (tag text)" +$PSQL -c "insert into bcfishpass.db_version (tag) values ('${PWD##*/}')" + +# note that reports are not automatically loaded to db as they may have additional setup dependencies +# load/migrate manually as required in a given environment +#for sql in sql/reports/*.sql ; do +# $PSQL -f "$sql" +#done diff --git a/db/functions/break_streams.sql b/db/v0.5.0/sql/functions/break_streams.sql similarity index 100% rename from db/functions/break_streams.sql rename to db/v0.5.0/sql/functions/break_streams.sql diff --git a/db/functions/create_barrier_table.sql b/db/v0.5.0/sql/functions/create_barrier_table.sql similarity index 100% rename from db/functions/create_barrier_table.sql rename to db/v0.5.0/sql/functions/create_barrier_table.sql diff --git a/db/functions/load_dnstr.sql b/db/v0.5.0/sql/functions/load_dnstr.sql similarity index 100% rename from db/functions/load_dnstr.sql rename to db/v0.5.0/sql/functions/load_dnstr.sql diff --git a/db/functions/load_upstr.sql b/db/v0.5.0/sql/functions/load_upstr.sql similarity index 100% rename from db/functions/load_upstr.sql rename to db/v0.5.0/sql/functions/load_upstr.sql diff --git a/db/functions/streamsasmvt.sql b/db/v0.5.0/sql/functions/streamsasmvt.sql similarity index 100% rename from db/functions/streamsasmvt.sql rename to db/v0.5.0/sql/functions/streamsasmvt.sql diff --git a/db/functions/utmzone.sql b/db/v0.5.0/sql/functions/utmzone.sql similarity index 92% rename from db/functions/utmzone.sql rename to db/v0.5.0/sql/functions/utmzone.sql index cec6e8bc..009b6aa8 100755 --- a/db/functions/utmzone.sql +++ b/db/v0.5.0/sql/functions/utmzone.sql @@ -3,7 +3,7 @@ -- DROP FUNCTION utmzone(geometry); -- Usage: SELECT ST_Transform(the_geom, utmzone(ST_Centroid(the_geom)) )FROM sometable; -CREATE FUNCTION public.utmzone(geometry) +CREATE FUNCTION bcfishpass.utmzone(geometry) RETURNS integer AS $BODY$ DECLARE diff --git a/db/functions/wcrp_barrier_count.sql b/db/v0.5.0/sql/functions/wcrp_barrier_count.sql similarity index 62% rename from db/functions/wcrp_barrier_count.sql rename to db/v0.5.0/sql/functions/wcrp_barrier_count.sql index 421d8e28..6a8003b8 100644 --- a/db/functions/wcrp_barrier_count.sql +++ b/db/v0.5.0/sql/functions/wcrp_barrier_count.sql @@ -1,12 +1,14 @@ -- function to query the view (so it is visible in pgfs) +DROP FUNCTION IF EXISTS postgisftw.wcrp_barrier_count; CREATE FUNCTION postgisftw.wcrp_barrier_count( watershed_group_code TEXT, - model_status TEXT default 'ALL' + model_status TEXT default 'OVERVIEW' ) --- watershed_group_code: BULK, LNIC, HORS, BOWR, QUES, CARR, ELKR + -- model_status : HABITAT, ACCESSIBLE, ALL (default) RETURNS TABLE ( + status TEXT, crossing_feature_type TEXT, n_passable bigint, n_barrier bigint, @@ -29,6 +31,7 @@ IF (v_model_status = 'ALL') then return query SELECT + v_model_status as status, v.crossing_feature_type, sum(v.n_passable)::bigint as n_passable, sum(v.n_barrier)::bigint as n_barrier, @@ -42,6 +45,7 @@ GROUP BY v.crossing_feature_type; ELSIF (v_model_status = 'ACCESSIBLE') then return query SELECT + v_model_status as status, v.crossing_feature_type, sum(v.n_passable)::bigint as n_passable, sum(v.n_barrier)::bigint as n_barrier, @@ -57,6 +61,48 @@ group by v.crossing_feature_type; ELSIF (v_model_status = 'HABITAT') then return query SELECT + v_model_status as status, + v.crossing_feature_type, + v.n_passable::bigint as n_passable, + v.n_barrier::bigint as n_barrier, + v.n_potential::bigint as n_potential, + v.n_unknown::bigint as n_unknown, + (v.n_passable + v.n_barrier + v.n_potential + v.n_unknown)::bigint as total +FROM bcfishpass.wcrp_barrier_count_vw v +WHERE + v.watershed_group_code = v_wsg and + v.model_status = 'HABITAT'; + +ELSIF (v_model_status = 'OVERVIEW') +then return query +SELECT + 'ALL' as status, + v.crossing_feature_type, + sum(v.n_passable)::bigint as n_passable, + sum(v.n_barrier)::bigint as n_barrier, + sum(v.n_potential)::bigint as n_potential, + sum(v.n_unknown)::bigint as n_unknown, + (sum(v.n_passable) + sum(v.n_barrier) + sum(v.n_potential) + sum(v.n_unknown))::bigint as total +FROM bcfishpass.wcrp_barrier_count_vw v +WHERE v.watershed_group_code = v_wsg +GROUP BY v.crossing_feature_type +UNION ALL +SELECT + 'ACCESSIBLE' as status, + v.crossing_feature_type, + sum(v.n_passable)::bigint as n_passable, + sum(v.n_barrier)::bigint as n_barrier, + sum(v.n_potential)::bigint as n_potential, + sum(v.n_unknown)::bigint as n_unknown, + (sum(v.n_passable) + sum(v.n_barrier) + sum(v.n_potential) + sum(v.n_unknown))::bigint as total +FROM bcfishpass.wcrp_barrier_count_vw v +WHERE + v.watershed_group_code = v_wsg and + v.model_status in ('ACCESSIBLE', 'HABITAT') +group by v.crossing_feature_type +UNION ALL +SELECT + 'HABITAT' as status, v.crossing_feature_type, v.n_passable::bigint as n_passable, v.n_barrier::bigint as n_barrier, diff --git a/db/functions/wcrp_barrier_extent.sql b/db/v0.5.0/sql/functions/wcrp_barrier_extent.sql similarity index 96% rename from db/functions/wcrp_barrier_extent.sql rename to db/v0.5.0/sql/functions/wcrp_barrier_extent.sql index 7b6daf30..d3168e98 100644 --- a/db/functions/wcrp_barrier_extent.sql +++ b/db/v0.5.0/sql/functions/wcrp_barrier_extent.sql @@ -1,6 +1,7 @@ +DROP FUNCTION IF EXISTS postgisftw.wcrp_barrier_extent; + CREATE FUNCTION postgisftw.wcrp_barrier_extent(watershed_group_code TEXT) --- watershed_group_code: BULK, LNIC, HORS, BOWR, QUES, CARR, ELKR RETURNS TABLE( crossing_feature_type TEXT, diff --git a/db/functions/wcrp_barrier_severity.sql b/db/v0.5.0/sql/functions/wcrp_barrier_severity.sql similarity index 56% rename from db/functions/wcrp_barrier_severity.sql rename to db/v0.5.0/sql/functions/wcrp_barrier_severity.sql index 58f16c1f..b5055857 100644 --- a/db/functions/wcrp_barrier_severity.sql +++ b/db/v0.5.0/sql/functions/wcrp_barrier_severity.sql @@ -1,7 +1,6 @@ +DROP FUNCTION IF EXISTS postgisftw.wcrp_barrier_severity; CREATE FUNCTION postgisftw.wcrp_barrier_severity(watershed_group_code TEXT) --- watershed_group_code: BULK, LNIC, HORS, BOWR, QUES, CARR, ELKR - RETURNS TABLE( structure_type TEXT, n_assessed_barrier bigint, @@ -22,13 +21,26 @@ RETURN query -- dams and assessed crossings on potentially accessible streams (to target spp) -- - n total --- - n barriers +-- - n barriers assessed -- - pct barriers SELECT cft.crossing_feature_type, - count(*) filter (where c.barrier_status in ('BARRIER', 'POTENTIAL')) as n_barrier, + count(*) filter (where c.barrier_status in ('BARRIER', 'POTENTIAL') AND c.pscis_status = 'ASSESSED') as n_barriers_assessed, + count(*) filter (where c.pscis_status NOT IN ('ASSESSED') OR c.pscis_status IS NULL) as n_unassessed, count(*) as n_total, - round((count(*) filter (where c.barrier_status in ('BARRIER', 'POTENTIAL'))::numeric / count(*)::numeric) * 100, 1) as pct_barrier + CASE + WHEN count(*) filter (where c.barrier_status in ('BARRIER', 'POTENTIAL') AND c.pscis_status = 'ASSESSED')::numeric != 0 + THEN + round( + ((count(*) filter (where c.barrier_status in ('BARRIER', 'POTENTIAL') AND c.pscis_status = 'ASSESSED')::numeric -- # assess structures where barrier status IN (BARRIER, POTENTIAL) + + ((count(*) filter (where c.barrier_status in ('BARRIER', 'POTENTIAL') AND c.pscis_status = 'ASSESSED')::numeric + / count(*) filter (where c.pscis_status = 'ASSESSED')::numeric) -- Failure rate (# assessed that are barrier or potential / total # assessed) + * count(*) filter (where c.pscis_status != 'ASSESSED' OR c.pscis_status IS NULL)::numeric) -- # unassessed structures + ) + / count(*)::numeric) * 100 -- total # of structures + , 1 ) + ELSE 0.0 -- 0 if no assessed barriers + END as pct_barrier from bcfishpass.crossings c inner join bcfishpass.crossings_feature_type_vw cft using (aggregated_crossings_id) inner join bcfishpass.crossings_upstream_access a using (aggregated_crossings_id) @@ -53,7 +65,7 @@ $$; COMMENT ON FUNCTION postgisftw.wcrp_barrier_severity IS 'For given watershed group, returns count of dams/pscis crossings on potentially accessible streams - n total -- n barrier +- n barriers assessed - pct that are barrier'; REVOKE EXECUTE ON FUNCTION postgisftw.wcrp_barrier_severity FROM public; diff --git a/db/functions/wcrp_habitat_connectivity_status.sql b/db/v0.5.0/sql/functions/wcrp_habitat_connectivity_status.sql similarity index 95% rename from db/functions/wcrp_habitat_connectivity_status.sql rename to db/v0.5.0/sql/functions/wcrp_habitat_connectivity_status.sql index c845bb51..ade64b15 100644 --- a/db/functions/wcrp_habitat_connectivity_status.sql +++ b/db/v0.5.0/sql/functions/wcrp_habitat_connectivity_status.sql @@ -1,4 +1,6 @@ -- report on total modelled habitat vs accessible modelled habitat +DROP FUNCTION IF EXISTS postgisftw.wcrp_habitat_connectivity_status; + CREATE OR REPLACE FUNCTION postgisftw.wcrp_habitat_connectivity_status( watershed_group_code TEXT, habitat_type TEXT default 'ALL' diff --git a/db/functions/wsg_crossing_summary.sql b/db/v0.5.0/sql/functions/wsg_crossing_summary.sql similarity index 100% rename from db/functions/wsg_crossing_summary.sql rename to db/v0.5.0/sql/functions/wsg_crossing_summary.sql diff --git a/db/functions/wsg_linear_summary.sql b/db/v0.5.0/sql/functions/wsg_linear_summary.sql similarity index 100% rename from db/functions/wsg_linear_summary.sql rename to db/v0.5.0/sql/functions/wsg_linear_summary.sql diff --git a/db/views/fptwg_assmt_wsd_summary_vw.sql b/db/v0.5.0/sql/reports/fptwg_assmt_wsd_summary_vw.sql similarity index 100% rename from db/views/fptwg_assmt_wsd_summary_vw.sql rename to db/v0.5.0/sql/reports/fptwg_assmt_wsd_summary_vw.sql diff --git a/db/v0.5.0/sql/reports/obsrvtn_above_barriers_ch_cm_co_pk_sk.sql b/db/v0.5.0/sql/reports/obsrvtn_above_barriers_ch_cm_co_pk_sk.sql new file mode 100644 index 00000000..98105fd5 --- /dev/null +++ b/db/v0.5.0/sql/reports/obsrvtn_above_barriers_ch_cm_co_pk_sk.sql @@ -0,0 +1,172 @@ +-- report on all potential barriers below salmon/steelhead observations + +-- NOTE - all potential gradient barriers are reported, but only falls tagged as a barrier are included + +drop materialized view if exists bcfishpass.obsrvtn_above_barriers_ch_cm_co_pk_sk_st; + +create materialized view bcfishpass.obsrvtn_above_barriers_ch_cm_co_pk_sk_st as +select * from +(SELECT + a.fish_observation_point_id, + a.species_code, + a.observation_date, + a.activity_code, + a.activity, + a.life_stage_code, + a.life_stage, + a.acat_report_url, + o.agency_name, + o.source, + o.source_ref, + array_to_string(array_agg(DISTINCT g05.barriers_gradient_id), ';') as gradient_05_dnstr, + array_to_string(array_agg(DISTINCT g10.barriers_gradient_id), ';') as gradient_10_dnstr, + array_to_string(array_agg(DISTINCT g15.barriers_gradient_id), ';') as gradient_15_dnstr, + array_to_string(array_agg(DISTINCT g20.barriers_gradient_id), ';') as gradient_20_dnstr, + array_to_string(array_agg(DISTINCT g25.barriers_gradient_id), ';') as gradient_25_dnstr, + array_to_string(array_agg(DISTINCT g30.barriers_gradient_id), ';') as gradient_30_dnstr, + array_to_string(array_agg(DISTINCT f.barriers_falls_id), ';') as falls_dnstr, + array_to_string(array_agg(DISTINCT s.barriers_subsurfaceflow_id), ';') as subsurfaceflow_dnstr, + count(DISTINCT g05.barriers_gradient_id) as gradient_05_dnstr_count, + count(DISTINCT g10.barriers_gradient_id) as gradient_10_dnstr_count, + count(DISTINCT g15.barriers_gradient_id) as gradient_15_dnstr_count, + count(DISTINCT g20.barriers_gradient_id) as gradient_20_dnstr_count, + count(DISTINCT g25.barriers_gradient_id) as gradient_25_dnstr_count, + count(DISTINCT g30.barriers_gradient_id) as gradient_30_dnstr_count, + count(DISTINCT f.barriers_falls_id) as falls_dnstr_count, + count(DISTINCT s.barriers_subsurfaceflow_id) as subsurfaceflow_dnstr_count +FROM bcfishpass.observations_vw a +LEFT OUTER JOIN bcfishpass.barriers_gradient g05 +ON FWA_Downstream( + a.blue_line_key, + a.downstream_route_measure, + a.wscode_ltree, + a.localcode_ltree, + g05.blue_line_key, + g05.downstream_route_measure, + g05.wscode_ltree, + g05.localcode_ltree, + True, + 20 +) +LEFT OUTER JOIN bcfishpass.barriers_gradient g10 +ON FWA_Downstream( + a.blue_line_key, + a.downstream_route_measure, + a.wscode_ltree, + a.localcode_ltree, + g10.blue_line_key, + g10.downstream_route_measure, + g10.wscode_ltree, + g10.localcode_ltree, + True, + 20 +) +LEFT OUTER JOIN bcfishpass.barriers_gradient g15 +ON FWA_Downstream( + a.blue_line_key, + a.downstream_route_measure, + a.wscode_ltree, + a.localcode_ltree, + g15.blue_line_key, + g15.downstream_route_measure, + g15.wscode_ltree, + g15.localcode_ltree, + True, + 20 +) +LEFT OUTER JOIN bcfishpass.barriers_gradient g20 +ON FWA_Downstream( + a.blue_line_key, + a.downstream_route_measure, + a.wscode_ltree, + a.localcode_ltree, + g20.blue_line_key, + g20.downstream_route_measure, + g20.wscode_ltree, + g20.localcode_ltree, + True, + 20 +) +LEFT OUTER JOIN bcfishpass.barriers_gradient g25 +ON FWA_Downstream( + a.blue_line_key, + a.downstream_route_measure, + a.wscode_ltree, + a.localcode_ltree, + g25.blue_line_key, + g25.downstream_route_measure, + g25.wscode_ltree, + g25.localcode_ltree, + True, + 20 +) +LEFT OUTER JOIN bcfishpass.barriers_gradient g30 +ON FWA_Downstream( + a.blue_line_key, + a.downstream_route_measure, + a.wscode_ltree, + a.localcode_ltree, + g30.blue_line_key, + g30.downstream_route_measure, + g30.wscode_ltree, + g30.localcode_ltree, + True, + 20 +) +LEFT OUTER JOIN bcfishpass.barriers_falls f +ON FWA_Downstream( + a.blue_line_key, + a.downstream_route_measure, + a.wscode_ltree, + a.localcode_ltree, + f.blue_line_key, + f.downstream_route_measure, + f.wscode_ltree, + f.localcode_ltree, + True, + 20 +) +LEFT OUTER JOIN bcfishpass.barriers_subsurfaceflow s +ON FWA_Downstream( + a.blue_line_key, + a.downstream_route_measure, + a.wscode_ltree, + a.localcode_ltree, + s.blue_line_key, + s.downstream_route_measure, + s.wscode_ltree, + s.localcode_ltree, + True, + 20 +) +inner join whse_fish.fiss_fish_obsrvtn_pnt_sp o +on a.fish_observation_point_id = o.fish_observation_point_id +WHERE a.species_code in ('CH','CM','CO','PK','SK','ST') +AND g05.barrier_type = 'GRADIENT_05' +AND g10.barrier_type = 'GRADIENT_10' +AND g15.barrier_type = 'GRADIENT_15' +AND g20.barrier_type = 'GRADIENT_20' +AND g25.barrier_type = 'GRADIENT_25' +AND g30.barrier_type = 'GRADIENT_30' +GROUP BY a.species_code, + a.fish_observation_point_id, + a.observation_date, + a.activity_code, + a.activity, + a.life_stage_code, + a.life_stage, + a.acat_report_url, + o.agency_name, + o.source, + o.source_ref +) as f +WHERE + gradient_05_dnstr_count >= 1 or + gradient_10_dnstr_count >= 1 or + gradient_15_dnstr_count >= 1 or + gradient_20_dnstr_count >= 1 or + gradient_25_dnstr_count >= 1 or + gradient_30_dnstr_count >= 1 or + falls_dnstr_count >= 1 or + subsurfaceflow_dnstr_count >= 1 +ORDER BY fish_observation_point_id; \ No newline at end of file diff --git a/db/v0.5.0/sql/reports/qa_barriers_obsrvtn_upstr.sql b/db/v0.5.0/sql/reports/qa_barriers_obsrvtn_upstr.sql new file mode 100644 index 00000000..b9bfd175 --- /dev/null +++ b/db/v0.5.0/sql/reports/qa_barriers_obsrvtn_upstr.sql @@ -0,0 +1,91 @@ +-- list barriers to salmon having salmon observations upstream + + +-- natural barriers to salmon (having <5 salmon observations upstream since 1990) +create materialized view bcfishpass.qa_barriers_natural_obsrvtn_upstr_ch_cm_co_pk_sk as +select + b.barriers_ch_cm_co_pk_sk_id, + b.barrier_type, + b.blue_line_key, + b.downstream_route_measure, + b.watershed_group_code, + count(*) as n_observations, + array_agg(o.fish_observation_point_id) as fish_observation_point_id, + array_agg(o.species_code) as species_codes, + array_agg(o.observation_date) as observation_dates +from bcfishpass.barriers_ch_cm_co_pk_sk b +inner join bcfishpass.observations_vw o on + fwa_upstream( + b.blue_line_key, b.downstream_route_measure, b.wscode_ltree, b.localcode_ltree, + o.blue_line_key, o.downstream_route_measure, o.wscode_ltree, o.localcode_ltree) +where o.species_code in ('CH','CM','CO','PK','SK') +group by + b.barriers_ch_cm_co_pk_sk_id, + b.barrier_type, + b.blue_line_key, + b.downstream_route_measure, + b.watershed_group_code +order by barriers_ch_cm_co_pk_sk_id; + + + +-- natural barriers to steelhead (having <5 salmon or steelhead observations upstream since 1990) +create materialized view bcfishpass.qa_barriers_natural_obsrvtn_upstr_st as +select + b.barriers_st_id, + b.barrier_type, + b.blue_line_key, + b.downstream_route_measure, + b.watershed_group_code, + count(*) as n_observations, + array_agg(o.fish_observation_point_id) as fish_observation_point_id, + array_agg(o.species_code) as species_codes, + array_agg(o.observation_date) as observation_dates +from bcfishpass.barriers_st b +inner join bcfishpass.observations_vw o on + fwa_upstream( + b.blue_line_key, b.downstream_route_measure, b.wscode_ltree, b.localcode_ltree, + o.blue_line_key, o.downstream_route_measure, o.wscode_ltree, o.localcode_ltree) +where o.species_code in ('CH','CM','CO','PK','SK','ST') +group by + b.barriers_st_id, + b.barrier_type, + b.blue_line_key, + b.downstream_route_measure, + b.watershed_group_code +order by barriers_st_id; + + +-- dams, do not aggregate observation info +create materialized view bcfishpass.qa_barriers_dams_obsrvtn_upstr_ch_cm_co_pk_sk_st as +select + d.dam_id, + cabd.dam_name_en, + cabd.owner, + cabd.operating_status, + cabd.construction_year, + cabd.dam_use, + case + when cabd.passability_status_code = 1 THEN 'BARRIER' + when cabd.passability_status_code = 2 THEN 'POTENTIAL' + when cabd.passability_status_code = 3 THEN 'PASSABLE' + when cabd.passability_status_code = 4 THEN 'UNKNOWN' + end AS barrier_status, + o.fish_observation_point_id, + o.species_code, + o.observation_date, + s.agency_name, + s.source, + s.source_ref, + s.activity, + s.life_stage, + s.acat_report_url +from bcfishpass.dams_vw d +inner join bcfishpass.observations_vw o on + fwa_upstream( + d.blue_line_key, d.downstream_route_measure, d.wscode_ltree, d.localcode_ltree, + o.blue_line_key, o.downstream_route_measure, o.wscode_ltree, o.localcode_ltree) +inner join cabd.dams cabd on d.dam_id = cabd.cabd_id::text and cabd.passability_status_code = 1 +inner join whse_fish.fiss_fish_obsrvtn_pnt_sp s on o.fish_observation_point_id = s.fish_observation_point_id +where o.species_code in ('CH','CM','CO','PK','SK','ST') +order by dam_name_en, fish_observation_point_id; diff --git a/db/v0.5.0/sql/reports/summary_diff_prod.sql b/db/v0.5.0/sql/reports/summary_diff_prod.sql new file mode 100644 index 00000000..5d246326 --- /dev/null +++ b/db/v0.5.0/sql/reports/summary_diff_prod.sql @@ -0,0 +1,128 @@ +-- compare current data to current data in foreign database + +create view bcfishpass.wsg_crossing_summary_diff_prod as +select + a.watershed_group_code , + a.crossing_feature_type , + a.n_crossings_total - b.n_crossings_total as n_crossings_total, + a.n_passable_total - b.n_passable_total as n_passable_total, + a.n_barriers_total - b.n_barriers_total as n_barriers_total, + a.n_potential_total - b.n_potential_total as n_potential_total, + a.n_unknown_total - b.n_unknown_total as n_unknown_total, + a.n_barriers_accessible_bt - b.n_barriers_accessible_bt as n_barriers_accessible_bt, + a.n_potential_accessible_bt - b.n_potential_accessible_bt as n_potential_accessible_bt, + a.n_unknown_accessible_bt - b.n_unknown_accessible_bt as n_unknown_accessible_bt, + a.n_barriers_accessible_ch_cm_co_pk_sk - b.n_barriers_accessible_ch_cm_co_pk_sk as n_barriers_accessible_ch_cm_co_pk_sk, + a.n_potential_accessible_ch_cm_co_pk_sk - b.n_potential_accessible_ch_cm_co_pk_sk as n_potential_accessible_ch_cm_co_pk_sk, + a.n_unknown_accessible_ch_cm_co_pk_sk - b.n_unknown_accessible_ch_cm_co_pk_sk as n_unknown_accessible_ch_cm_co_pk_sk, + a.n_barriers_accessible_st - b.n_barriers_accessible_st as n_barriers_accessible_st, + a.n_potential_accessible_st - b.n_potential_accessible_st as n_potential_accessible_st, + a.n_unknown_accessible_st - b.n_unknown_accessible_st as n_unknown_accessible_st, + a.n_barriers_accessible_wct - b.n_barriers_accessible_wct as n_barriers_accessible_wct, + a.n_potential_accessible_wct - b.n_potential_accessible_wct as n_potential_accessible_wct, + a.n_unknown_accessible_wct - b.n_unknown_accessible_wct as n_unknown_accessible_wct, + a.n_barriers_habitat_bt - b.n_barriers_habitat_bt as n_barriers_habitat_bt, + a.n_potential_habitat_bt - b.n_potential_habitat_bt as n_potential_habitat_bt, + a.n_unknown_habitat_bt - b.n_unknown_habitat_bt as n_unknown_habitat_bt, + a.n_barriers_habitat_ch - b.n_barriers_habitat_ch as n_barriers_habitat_ch, + a.n_potential_habitat_ch - b.n_potential_habitat_ch as n_potential_habitat_ch, + a.n_unknown_habitat_ch - b.n_unknown_habitat_ch as n_unknown_habitat_ch, + a.n_barriers_habitat_cm - b.n_barriers_habitat_cm as n_barriers_habitat_cm, + a.n_potential_habitat_cm - b.n_potential_habitat_cm as n_potential_habitat_cm, + a.n_unknown_habitat_cm - b.n_unknown_habitat_cm as n_unknown_habitat_cm, + a.n_barriers_habitat_co - b.n_barriers_habitat_co as n_barriers_habitat_co, + a.n_potential_habitat_co - b.n_potential_habitat_co as n_potential_habitat_co, + a.n_unknown_habitat_co - b.n_unknown_habitat_co as n_unknown_habitat_co, + a.n_barriers_habitat_pk - b.n_barriers_habitat_pk as n_barriers_habitat_pk, + a.n_potential_habitat_pk - b.n_potential_habitat_pk as n_potential_habitat_pk, + a.n_unknown_habitat_pk - b.n_unknown_habitat_pk as n_unknown_habitat_pk, + a.n_barriers_habitat_sk - b.n_barriers_habitat_sk as n_barriers_habitat_sk, + a.n_potential_habitat_sk - b.n_potential_habitat_sk as n_potential_habitat_sk, + a.n_unknown_habitat_sk - b.n_unknown_habitat_sk as n_unknown_habitat_sk, + a.n_barriers_habitat_salmon - b.n_barriers_habitat_salmon as n_barriers_habitat_salmon, + a.n_potential_habitat_salmon - b.n_potential_habitat_salmon as n_potential_habitat_salmon, + a.n_unknown_habitat_salmon - b.n_unknown_habitat_salmon as n_unknown_habitat_salmon, + a.n_barriers_habitat_st - b.n_barriers_habitat_st as n_barriers_habitat_st, + a.n_potential_habitat_st - b.n_potential_habitat_st as n_potential_habitat_st, + a.n_unknown_habitat_st - b.n_unknown_habitat_st as n_unknown_habitat_st, + a.n_barriers_habitat_wct - b.n_barriers_habitat_wct as n_barriers_habitat_wct, + a.n_potential_habitat_wct - b.n_potential_habitat_wct as n_potential_habitat_wct, + a.n_unknown_habitat_wct - b.n_unknown_habitat_wct as n_unknown_habitat_wct +from bcfishpass.wsg_crossing_summary_current a +inner join bcfishpass_prod.wsg_crossing_summary_current b + on a.watershed_group_code = b.watershed_group_code + and a.crossing_feature_type = b.crossing_feature_type +order by a.watershed_group_code; + +create view bcfishpass.wsg_linear_summary_diff_prod as +select + a.watershed_group_code , + a.length_total - b.length_total as length_total, + a.length_potentiallyaccessible_bt - b.length_potentiallyaccessible_bt as length_potentiallyaccessible_bt, + a.length_potentiallyaccessible_bt_observed - b.length_potentiallyaccessible_bt_observed as length_potentiallyaccessible_bt_observed, + a.length_potentiallyaccessible_bt_accessible_a - b.length_potentiallyaccessible_bt_accessible_a as length_potentiallyaccessible_bt_accessible_a, + a.length_potentiallyaccessible_bt_accessible_b - b.length_potentiallyaccessible_bt_accessible_b as length_potentiallyaccessible_bt_accessible_b, + a.length_obsrvd_spawning_rearing_bt - b.length_obsrvd_spawning_rearing_bt as length_obsrvd_spawning_rearing_bt, + a.length_obsrvd_spawning_rearing_bt_accessible_a - b.length_obsrvd_spawning_rearing_bt_accessible_a as length_obsrvd_spawning_rearing_bt_accessible_a, + a.length_obsrvd_spawning_rearing_bt_accessible_b - b.length_obsrvd_spawning_rearing_bt_accessible_b as length_obsrvd_spawning_rearing_bt_accessible_b, + a.length_spawning_rearing_bt - b.length_spawning_rearing_bt as length_spawning_rearing_bt, + a.length_spawning_rearing_bt_accessible_a - b.length_spawning_rearing_bt_accessible_a as length_spawning_rearing_bt_accessible_a, + a.length_spawning_rearing_bt_accessible_b - b.length_spawning_rearing_bt_accessible_b as length_spawning_rearing_bt_accessible_b, + a.length_potentiallyaccessible_ch_cm_co_pk_sk - b.length_potentiallyaccessible_ch_cm_co_pk_sk as length_potentiallyaccessible_ch_cm_co_pk_sk, + a.length_potentiallyaccessible_ch_cm_co_pk_sk_observed - b.length_potentiallyaccessible_ch_cm_co_pk_sk_observed as length_potentiallyaccessible_ch_cm_co_pk_sk_observed, + a.length_potentiallyaccessible_ch_cm_co_pk_sk_accessible_a - b.length_potentiallyaccessible_ch_cm_co_pk_sk_accessible_a as length_potentiallyaccessible_ch_cm_co_pk_sk_accessible_a, + a.length_potentiallyaccessible_ch_cm_co_pk_sk_accessible_b - b.length_potentiallyaccessible_ch_cm_co_pk_sk_accessible_b as length_potentiallyaccessible_ch_cm_co_pk_sk_accessible_b, + a.length_obsrvd_spawning_rearing_ch - b.length_obsrvd_spawning_rearing_ch as length_obsrvd_spawning_rearing_ch, + a.length_obsrvd_spawning_rearing_ch_accessible_a - b.length_obsrvd_spawning_rearing_ch_accessible_a as length_obsrvd_spawning_rearing_ch_accessible_a, + a.length_obsrvd_spawning_rearing_ch_accessible_b - b.length_obsrvd_spawning_rearing_ch_accessible_b as length_obsrvd_spawning_rearing_ch_accessible_b, + a.length_spawning_rearing_ch - b.length_spawning_rearing_ch as length_spawning_rearing_ch, + a.length_spawning_rearing_ch_accessible_a - b.length_spawning_rearing_ch_accessible_a as length_spawning_rearing_ch_accessible_a, + a.length_spawning_rearing_ch_accessible_b - b.length_spawning_rearing_ch_accessible_b as length_spawning_rearing_ch_accessible_b, + a.length_obsrvd_spawning_rearing_cm - b.length_obsrvd_spawning_rearing_cm as length_obsrvd_spawning_rearing_cm, + a.length_obsrvd_spawning_rearing_cm_accessible_a - b.length_obsrvd_spawning_rearing_cm_accessible_a as length_obsrvd_spawning_rearing_cm_accessible_a, + a.length_obsrvd_spawning_rearing_cm_accessible_b - b.length_obsrvd_spawning_rearing_cm_accessible_b as length_obsrvd_spawning_rearing_cm_accessible_b, + a.length_spawning_rearing_cm - b.length_spawning_rearing_cm as length_spawning_rearing_cm, + a.length_spawning_rearing_cm_accessible_a - b.length_spawning_rearing_cm_accessible_a as length_spawning_rearing_cm_accessible_a, + a.length_spawning_rearing_cm_accessible_b - b.length_spawning_rearing_cm_accessible_b as length_spawning_rearing_cm_accessible_b, + a.length_obsrvd_spawning_rearing_co - b.length_obsrvd_spawning_rearing_co as length_obsrvd_spawning_rearing_co, + a.length_obsrvd_spawning_rearing_co_accessible_a - b.length_obsrvd_spawning_rearing_co_accessible_a as length_obsrvd_spawning_rearing_co_accessible_a, + a.length_obsrvd_spawning_rearing_co_accessible_b - b.length_obsrvd_spawning_rearing_co_accessible_b as length_obsrvd_spawning_rearing_co_accessible_b, + a.length_spawning_rearing_co - b.length_spawning_rearing_co as length_spawning_rearing_co, + a.length_spawning_rearing_co_accessible_a - b.length_spawning_rearing_co_accessible_a as length_spawning_rearing_co_accessible_a, + a.length_spawning_rearing_co_accessible_b - b.length_spawning_rearing_co_accessible_b as length_spawning_rearing_co_accessible_b, + a.length_obsrvd_spawning_rearing_pk - b.length_obsrvd_spawning_rearing_pk as length_obsrvd_spawning_rearing_pk, + a.length_obsrvd_spawning_rearing_pk_accessible_a - b.length_obsrvd_spawning_rearing_pk_accessible_a as length_obsrvd_spawning_rearing_pk_accessible_a, + a.length_obsrvd_spawning_rearing_pk_accessible_b - b.length_obsrvd_spawning_rearing_pk_accessible_b as length_obsrvd_spawning_rearing_pk_accessible_b, + a.length_spawning_rearing_pk - b.length_spawning_rearing_pk as length_spawning_rearing_pk, + a.length_spawning_rearing_pk_accessible_a - b.length_spawning_rearing_pk_accessible_a as length_spawning_rearing_pk_accessible_a, + a.length_spawning_rearing_pk_accessible_b - b.length_spawning_rearing_pk_accessible_b as length_spawning_rearing_pk_accessible_b, + a.length_obsrvd_spawning_rearing_sk - b.length_obsrvd_spawning_rearing_sk as length_obsrvd_spawning_rearing_sk, + a.length_obsrvd_spawning_rearing_sk_accessible_a - b.length_obsrvd_spawning_rearing_sk_accessible_a as length_obsrvd_spawning_rearing_sk_accessible_a, + a.length_obsrvd_spawning_rearing_sk_accessible_b - b.length_obsrvd_spawning_rearing_sk_accessible_b as length_obsrvd_spawning_rearing_sk_accessible_b, + a.length_spawning_rearing_sk - b.length_spawning_rearing_sk as length_spawning_rearing_sk, + a.length_spawning_rearing_sk_accessible_a - b.length_spawning_rearing_sk_accessible_a as length_spawning_rearing_sk_accessible_a, + a.length_spawning_rearing_sk_accessible_b - b.length_spawning_rearing_sk_accessible_b as length_spawning_rearing_sk_accessible_b, + a.length_potentiallyaccessible_st - b.length_potentiallyaccessible_st as length_potentiallyaccessible_st, + a.length_potentiallyaccessible_st_observed - b.length_potentiallyaccessible_st_observed as length_potentiallyaccessible_st_observed, + a.length_potentiallyaccessible_st_accessible_a - b.length_potentiallyaccessible_st_accessible_a as length_potentiallyaccessible_st_accessible_a, + a.length_potentiallyaccessible_st_accessible_b - b.length_potentiallyaccessible_st_accessible_b as length_potentiallyaccessible_st_accessible_b, + a.length_obsrvd_spawning_rearing_st - b.length_obsrvd_spawning_rearing_st as length_obsrvd_spawning_rearing_st, + a.length_obsrvd_spawning_rearing_st_accessible_a - b.length_obsrvd_spawning_rearing_st_accessible_a as length_obsrvd_spawning_rearing_st_accessible_a, + a.length_obsrvd_spawning_rearing_st_accessible_b - b.length_obsrvd_spawning_rearing_st_accessible_b as length_obsrvd_spawning_rearing_st_accessible_b, + a.length_spawning_rearing_st - b.length_spawning_rearing_st as length_spawning_rearing_st, + a.length_spawning_rearing_st_accessible_a - b.length_spawning_rearing_st_accessible_a as length_spawning_rearing_st_accessible_a, + a.length_spawning_rearing_st_accessible_b - b.length_spawning_rearing_st_accessible_b as length_spawning_rearing_st_accessible_b, + a.length_potentiallyaccessible_wct - b.length_potentiallyaccessible_wct as length_potentiallyaccessible_wct, + a.length_potentiallyaccessible_wct_observed - b.length_potentiallyaccessible_wct_observed as length_potentiallyaccessible_wct_observed, + a.length_potentiallyaccessible_wct_accessible_a - b.length_potentiallyaccessible_wct_accessible_a as length_potentiallyaccessible_wct_accessible_a, + a.length_potentiallyaccessible_wct_accessible_b - b.length_potentiallyaccessible_wct_accessible_b as length_potentiallyaccessible_wct_accessible_b, + a.length_obsrvd_spawning_rearing_wct - b.length_obsrvd_spawning_rearing_wct as length_obsrvd_spawning_rearing_wct, + a.length_obsrvd_spawning_rearing_wct_accessible_a - b.length_obsrvd_spawning_rearing_wct_accessible_a as length_obsrvd_spawning_rearing_wct_accessible_a, + a.length_obsrvd_spawning_rearing_wct_accessible_b - b.length_obsrvd_spawning_rearing_wct_accessible_b as length_obsrvd_spawning_rearing_wct_accessible_b, + a.length_spawning_rearing_wct - b.length_spawning_rearing_wct as length_spawning_rearing_wct, + a.length_spawning_rearing_wct_accessible_a - b.length_spawning_rearing_wct_accessible_a as length_spawning_rearing_wct_accessible_a, + a.length_spawning_rearing_wct_accessible_b - b.length_spawning_rearing_wct_accessible_b as length_spawning_rearing_wct_accessible_b +from bcfishpass.wsg_linear_summary_current a +inner join bcfishpass_prod.wsg_linear_summary_current b +on a.watershed_group_code = b.watershed_group_code +order by a.watershed_group_code; \ No newline at end of file diff --git a/db/tables/crossings.sql b/db/v0.5.0/sql/tables/crossings.sql similarity index 97% rename from db/tables/crossings.sql rename to db/v0.5.0/sql/tables/crossings.sql index 20719bd1..de7cacf0 100644 --- a/db/tables/crossings.sql +++ b/db/v0.5.0/sql/tables/crossings.sql @@ -47,10 +47,12 @@ create table bcfishpass.crossings -- forest road tenure info ften_forest_file_id text, + ften_road_section_id text, ften_file_type_description text, ften_client_number text, ften_client_name text, ften_life_cycle_status_code text, + ften_map_label text, -- rail info rail_track_name text, @@ -84,7 +86,7 @@ create table bcfishpass.crossings localcode_ltree ltree, watershed_group_code text, gnis_stream_name text, - + stream_order integer, stream_magnitude integer, @@ -94,7 +96,7 @@ create table bcfishpass.crossings -- distinct species upstream/downstream, derived from bcfishobs --observedspp_dnstr text[], --observedspp_upstr text[], - + geom geometry(PointZM, 3005), -- only one crossing per location please @@ -123,10 +125,12 @@ comment on column bcfishpass.crossings.transport_line_structured_name_1 IS 'DRA comment on column bcfishpass.crossings.transport_line_type_description IS 'DRA road type, taken from the nearest DRA road (within 30m)'; comment on column bcfishpass.crossings.transport_line_surface_description IS 'DRA road surface, taken from the nearest DRA road (within 30m)'; comment on column bcfishpass.crossings.ften_forest_file_id IS 'FTEN road forest_file_id value, taken from the nearest FTEN road (within 30m)'; +comment on column bcfishpass.crossings.ften_road_section_id IS 'FTEN road road_section_id value, taken from the nearest FTEN road (within 30m)'; comment on column bcfishpass.crossings.ften_file_type_description IS 'FTEN road tenure type (Forest Service Road, Road Permit, etc), taken from the nearest FTEN road (within 30m)'; comment on column bcfishpass.crossings.ften_client_number IS 'FTEN road client number, taken from the nearest FTEN road (within 30m)'; comment on column bcfishpass.crossings.ften_client_name IS 'FTEN road client name, taken from the nearest FTEN road (within 30m)'; comment on column bcfishpass.crossings.ften_life_cycle_status_code IS 'FTEN road life_cycle_status_code (active or retired, pending roads are not included), taken from the nearest FTEN road (within 30m)'; +comment on column bcfishpass.crossings.ften_map_label IS 'FTEN road map_label value, taken from the nearest FTEN road (within 30m)'; comment on column bcfishpass.crossings.rail_track_name IS 'Railway name, taken from nearest railway (within 25m)'; comment on column bcfishpass.crossings.rail_owner_name IS 'Railway owner name, taken from nearest railway (within 25m)'; comment on column bcfishpass.crossings.rail_operator_english_name IS 'Railway operator name, taken from nearest railway (within 25m)';; diff --git a/db/tables/crossings_upstr_dnstr.sql b/db/v0.5.0/sql/tables/crossings_upstr_dnstr.sql similarity index 100% rename from db/tables/crossings_upstr_dnstr.sql rename to db/v0.5.0/sql/tables/crossings_upstr_dnstr.sql diff --git a/db/tables/crossings_upstream_access.sql b/db/v0.5.0/sql/tables/crossings_upstream_access.sql similarity index 100% rename from db/tables/crossings_upstream_access.sql rename to db/v0.5.0/sql/tables/crossings_upstream_access.sql diff --git a/db/tables/crossings_upstream_habitat.sql b/db/v0.5.0/sql/tables/crossings_upstream_habitat.sql similarity index 67% rename from db/tables/crossings_upstream_habitat.sql rename to db/v0.5.0/sql/tables/crossings_upstream_habitat.sql index 33b556bc..ecfe0c81 100644 --- a/db/tables/crossings_upstream_habitat.sql +++ b/db/v0.5.0/sql/tables/crossings_upstream_habitat.sql @@ -34,24 +34,4 @@ create table bcfishpass.crossings_upstream_habitat ( wct_rearing_km double precision DEFAULT 0, wct_spawning_belowupstrbarriers_km double precision DEFAULT 0, wct_rearing_belowupstrbarriers_km double precision DEFAULT 0 -); - - --- wcrp 'all species' upstream habitat reporting (for columns where it differs from above table) --- - barriers only --- - currently ch/co/sk/st/wct where they exist in watersheds of interest --- - apply 1.5x multiplier to co rearing in wetlands and all sk rearing -create table bcfishpass.crossings_upstream_habitat_wcrp ( - aggregated_crossings_id text primary key, - watershed_group_code character varying (4), - co_rearing_km double precision DEFAULT 0, - co_rearing_belowupstrbarriers_km double precision DEFAULT 0, - sk_rearing_km double precision DEFAULT 0, - sk_rearing_belowupstrbarriers_km double precision DEFAULT 0, - all_spawning_km double precision DEFAULT 0, - all_spawning_belowupstrbarriers_km double precision DEFAULT 0, - all_rearing_km double precision DEFAULT 0, - all_rearing_belowupstrbarriers_km double precision DEFAULT 0, - all_spawningrearing_km double precision DEFAULT 0, - all_spawningrearing_belowupstrbarriers_km double precision DEFAULT 0 ); \ No newline at end of file diff --git a/db/tables/gradient_barriers.sql b/db/v0.5.0/sql/tables/gradient_barriers.sql similarity index 100% rename from db/tables/gradient_barriers.sql rename to db/v0.5.0/sql/tables/gradient_barriers.sql diff --git a/db/tables/habitat_linear.sql b/db/v0.5.0/sql/tables/habitat_linear.sql similarity index 100% rename from db/tables/habitat_linear.sql rename to db/v0.5.0/sql/tables/habitat_linear.sql diff --git a/db/tables/log.sql b/db/v0.5.0/sql/tables/log.sql similarity index 97% rename from db/tables/log.sql rename to db/v0.5.0/sql/tables/log.sql index 6ac77767..cfeaa582 100644 --- a/db/tables/log.sql +++ b/db/v0.5.0/sql/tables/log.sql @@ -20,13 +20,13 @@ create table bcfishpass.log ( -- log parameters used for the given model run -create table bcfishpass.parameters_habitat_method_log ( +create table bcfishpass.log_parameters_habitat_method ( model_run_id integer references bcfishpass.log(model_run_id), watershed_group_code character varying(4), model text ); -create table bcfishpass.parameters_habitat_thresholds_log ( +create table bcfishpass.log_parameters_habitat_thresholds ( model_run_id integer references bcfishpass.log(model_run_id), species_code text , spawn_gradient_max numeric, @@ -42,7 +42,7 @@ create table bcfishpass.parameters_habitat_thresholds_log ( rear_lake_ha_min integer ); -create table bcfishpass.wsg_linear_summary ( +create table bcfishpass.log_wsg_linear_summary ( model_run_id integer references bcfishpass.log(model_run_id), watershed_group_code text, length_total numeric, @@ -112,7 +112,7 @@ create table bcfishpass.wsg_linear_summary ( length_spawning_rearing_wct_accessible_b numeric ); -create table bcfishpass.wsg_crossing_summary ( +create table bcfishpass.log_wsg_crossing_summary ( model_run_id integer references bcfishpass.log(model_run_id), watershed_group_code text, crossing_feature_type text, diff --git a/db/v0.5.0/sql/tables/modelled_stream_crossings.sql b/db/v0.5.0/sql/tables/modelled_stream_crossings.sql new file mode 100644 index 00000000..b22353a4 --- /dev/null +++ b/db/v0.5.0/sql/tables/modelled_stream_crossings.sql @@ -0,0 +1,31 @@ +CREATE TABLE bcfishpass.modelled_stream_crossings +( + modelled_crossing_id serial primary key, + modelled_crossing_type character varying(5), + modelled_crossing_type_source text[], + transport_line_id integer, + ften_road_section_lines_id integer, + og_road_segment_permit_id integer, + og_petrlm_dev_rd_pre06_pub_id integer, + railway_track_id integer, + linear_feature_id bigint, + blue_line_key integer, + downstream_route_measure double precision, + wscode_ltree ltree, + localcode_ltree ltree, + watershed_group_code character varying(4), + geom geometry(PointZM, 3005) +); + +CREATE INDEX ON bcfishpass.modelled_stream_crossings (transport_line_id); +CREATE INDEX ON bcfishpass.modelled_stream_crossings (ften_road_section_lines_id); +CREATE INDEX ON bcfishpass.modelled_stream_crossings (og_road_segment_permit_id); +CREATE INDEX ON bcfishpass.modelled_stream_crossings (og_petrlm_dev_rd_pre06_pub_id); +CREATE INDEX ON bcfishpass.modelled_stream_crossings (railway_track_id); +CREATE INDEX ON bcfishpass.modelled_stream_crossings (blue_line_key); +CREATE INDEX ON bcfishpass.modelled_stream_crossings (linear_feature_id); +CREATE INDEX ON bcfishpass.modelled_stream_crossings USING GIST (geom); +CREATE INDEX ON bcfishpass.modelled_stream_crossings USING GIST (wscode_ltree); +CREATE INDEX ON bcfishpass.modelled_stream_crossings USING BTREE (wscode_ltree); +CREATE INDEX ON bcfishpass.modelled_stream_crossings USING GIST (localcode_ltree); +CREATE INDEX ON bcfishpass.modelled_stream_crossings USING BTREE (localcode_ltree); \ No newline at end of file diff --git a/db/tables/parameters.sql b/db/v0.5.0/sql/tables/parameters.sql similarity index 68% rename from db/tables/parameters.sql rename to db/v0.5.0/sql/tables/parameters.sql index 0f7a6962..f8a77e00 100644 --- a/db/tables/parameters.sql +++ b/db/v0.5.0/sql/tables/parameters.sql @@ -27,14 +27,4 @@ create table bcfishpass.parameters_habitat_thresholds ( rear_mad_min numeric, rear_mad_max numeric, rear_lake_ha_min integer -); - --- -------------- --- list of watershed groups to process in access model/crossings --- -------------- -create table bcfishpass.watershed_groups_access ( - watershed_group_code character varying(4) -); --- and default to all watershed groups -insert into bcfishpass.watershed_groups_access - select watershed_group_code from whse_basemapping.fwa_watershed_groups_poly; \ No newline at end of file +); \ No newline at end of file diff --git a/db/tables/streams.sql b/db/v0.5.0/sql/tables/streams.sql similarity index 100% rename from db/tables/streams.sql rename to db/v0.5.0/sql/tables/streams.sql diff --git a/db/tables/streams_upstr_dnstr.sql b/db/v0.5.0/sql/tables/streams_upstr_dnstr.sql similarity index 100% rename from db/tables/streams_upstr_dnstr.sql rename to db/v0.5.0/sql/tables/streams_upstr_dnstr.sql diff --git a/db/tables/user.sql b/db/v0.5.0/sql/tables/user.sql similarity index 95% rename from db/tables/user.sql rename to db/v0.5.0/sql/tables/user.sql index 84f58434..1f8d45f1 100644 --- a/db/tables/user.sql +++ b/db/v0.5.0/sql/tables/user.sql @@ -210,3 +210,17 @@ create table bcfishpass.wsg_species_presence create table bcfishpass.dfo_known_sockeye_lakes ( waterbody_poly_id integer primary key ); + + +-- -------------- +-- CABD to FWA BLUE LINE KEY LOOKUP +-- +-- link CABD waterfalls and dams to correct FWA stream +-- -------------- +create table bcfishpass.user_cabd_blkey_xref ( + cabd_id text, + blue_line_key integer, + reviewer_name text, + review_date date, + notes text +); \ No newline at end of file diff --git a/db/v0.5.0/sql/tables/wcrp.sql b/db/v0.5.0/sql/tables/wcrp.sql new file mode 100644 index 00000000..7c50e1c6 --- /dev/null +++ b/db/v0.5.0/sql/tables/wcrp.sql @@ -0,0 +1,160 @@ +-- -------------- +-- WCRP WATERSHED GROUPS +-- +-- watersheds and target species for CWF reporting +-- -------------- +create table bcfishpass.wcrp_watersheds +( + watershed_group_code varchar(4), + ch boolean, + cm boolean, + co boolean, + pk boolean, + sk boolean, + st boolean, + wct boolean, + notes text +); + + +-- -------------- +-- WCRP specific upstream habitat reporting (for columns where it differs from bcfishpass.crossings_upstream_habitat) +-- - 'all species' is target species as defined in wcrp_watersheds +-- - apply 1.5x multiplier to co rearing in wetlands and to all sk rearing +-- -------------- +create table bcfishpass.crossings_upstream_habitat_wcrp ( + aggregated_crossings_id text primary key, + watershed_group_code character varying (4), + co_rearing_km double precision DEFAULT 0, + co_rearing_belowupstrbarriers_km double precision DEFAULT 0, + sk_rearing_km double precision DEFAULT 0, + sk_rearing_belowupstrbarriers_km double precision DEFAULT 0, + all_spawning_km double precision DEFAULT 0, + all_spawning_belowupstrbarriers_km double precision DEFAULT 0, + all_rearing_km double precision DEFAULT 0, + all_rearing_belowupstrbarriers_km double precision DEFAULT 0, + all_spawningrearing_km double precision DEFAULT 0, + all_spawningrearing_belowupstrbarriers_km double precision DEFAULT 0 +); + + +-- -------------- +-- WCRP TRACKING TABLES +-- +-- allow tracking of crossings/barriers beyond what PSCIS offers +-- -------------- + +create table bcfishpass.wcrp_confirmed_barriers ( + aggregated_crossings_id text primary key, + internal_name text, + watercourse_name text, + road_name text, + easting integer, + northing integer, + zone integer, + barrier_type text, + barrier_owner text, + assessment_step_completed text CHECK (assessment_step_completed IN ( + 'Informal assessment', + 'Barrier assessment', + 'Habitat confirmation', + 'Detailed habitat investigation' + ) + ), + partial_passability_notes text, + upstream_habitat_quality text, + constructability text, + estimated_cost integer, + cost_benefit_ratio integer, + priority text CHECK (priority IN ('High','Medium','Low')), + next_steps text CHECK (next_steps IN ( + 'Engage with barrier owner', + 'Bring barrier to regulator', + 'Commission engineering designs', + 'Remove', + 'Replace', + 'Leave until end of life cycle', + 'identify barrier owner', + 'Engage in public consultation', + 'Fundraise' + ) + ), + next_steps_timeline text, + next_steps_lead text, + next_steps_others_involved text, + reason text, + comments text +); + +create table bcfishpass.wcrp_data_deficient_structures ( + aggregated_crossings_id text primary key, + internal_name text, + watercourse_name text, + road_name text, + easting integer, + northing integer, + zone integer, + structure_type text, + assessment_step_completed text CHECK (assessment_step_completed IN ( + 'Informal assessment', + 'Barrier assessment', + 'Habitat confirmation', + 'Detailed habitat investigation' + ) + ), + structure_owner text, + next_steps text CHECK (next_steps IN ( + 'Barrier assessment', + 'Habitat confirmation', + 'Detailed habitat investigation', + 'Other', + 'Passage study') + ), + next_steps_lead text, + comments text +); + +create table bcfishpass.wcrp_excluded_structures ( + aggregated_crossings_id text primary key, + internal_name text, + watercourse_name text, + road_name text, + easting integer, + northing integer, + zone integer, + exclusion_reason text CHECK (exclusion_reason IN ('Passable', 'No structure','No key upstream habitat','No structure/key upstream habitat')), + exclusion_method text CHECK (exclusion_method IN ('Imagery review','Field assessment','Local knowledge','Informal assessment')), + comments text, + supporting_links text +); + +create table bcfishpass.wcrp_rehabilitiated_structures ( + aggregated_crossing_id text primary key, + internal_name text, + watercourse_name text, + road_name text, + easting integer, + northing integer, + zone integer, + rehabilitation_type text CHECK (rehabilitation_type IN ('Removal','Replacement - OBS','Replacement - CBS','Decommissioning')), + rehabilitated_by text, + rehabilitation_date date, + rehabilitation_cost_estimate integer, + rehabilitation_cost_actual integer, + comments text, + supporting_links text +); + +create table bcfishpass.wcrp_ranked_barriers ( + aggregated_crossings_id text primary key, + set_id numeric, + total_hab_gain_set numeric, + num_barriers_set integer, + avg_gain_per_barrier numeric, + dnstr_set_ids character varying[], + rank_avg_gain_per_barrier numeric, + rank_avg_gain_tiered numeric, + rank_total_upstr_hab numeric, + rank_combined numeric, + tier_combined character varying +); \ No newline at end of file diff --git a/db/v0.5.0/sql/views/01_observations_vw.sql b/db/v0.5.0/sql/views/01_observations_vw.sql new file mode 100644 index 00000000..0e2ba0cb --- /dev/null +++ b/db/v0.5.0/sql/views/01_observations_vw.sql @@ -0,0 +1,97 @@ +-- -------------- +-- OBSERVATIONS +-- +-- from bcfishobs, extract observations: +-- - of species of interest +-- - within watershed groups where they are confirmed to occur (filtering out bad data) +-- +-- -------------- + +create materialized view bcfishpass.observations_vw as + +-- Convert the boolean species columns in wsg_species_presence into array of species presence for each wsg +-- (We could modify the input file's multiple spp columns into a single column of spp codes but current +-- design works fine and is easy to validate. Down side is that this query must be modified when spp are added) +with wsg_spp as +( +select + watershed_group_code, string_to_array(array_to_string(array[bt, ch, cm, co, ct, dv, gr, pk, rb, sk, st, wct], ','),',') as species_codes +from ( + select + p.watershed_group_code, + case when p.bt is true then 'BT' else NULL end as bt, + case when p.ch is true then 'CH' else NULL end as ch, + case when p.cm is true then 'CM' else NULL end as cm, + case when p.co is true then 'CO' else NULL end as co, + case when p.ct is true then 'CT' else NULL end as ct, + case when p.dv is true then 'DV' else NULL end as dv, + case when p.gr is true then 'GR' else NULL end as gr, + case when p.pk is true then 'PK' else NULL end as pk, + case when p.rb is true then 'RB' else NULL end as rb, + case when p.sk is true then 'SK' else NULL end as sk, + case when p.st is true then 'ST' else NULL end as st, + case when p.wct is true then 'WCT' else NULL end as wct + from bcfishpass.wsg_species_presence p + ) as f +), + +-- simplify CT species codes +species_code_remap as ( + select distinct + species_code, + case + when species_code = 'CCT' then 'CT' + when species_code = 'ACT' then 'CT' + when species_code = 'CT/RB' then 'CT' + else species_code + end as species_code_remap + from bcfishobs.fiss_fish_obsrvtn_events_vw e +), + +-- filter on species code and watershed group +obs as ( + select + e.fish_observation_point_id, + e.fish_obsrvtn_event_id, + e.linear_feature_id, + e.blue_line_key, + e.wscode_ltree, + e.localcode_ltree, + e.downstream_route_measure, + e.watershed_group_code, + r.species_code_remap as species_code, + e.observation_date + from bcfishobs.fiss_fish_obsrvtn_events_vw e + inner join wsg_spp on e.watershed_group_code = wsg_spp.watershed_group_code + inner join species_code_remap r on e.species_code = r.species_code + and array[e.species_code]::text[] && wsg_spp.species_codes +) + +-- add various other attributes from source +select + p.fish_observation_point_id, + p.fish_obsrvtn_event_id, + p.linear_feature_id, + p.blue_line_key, + p.wscode_ltree, + p.localcode_ltree, + p.downstream_route_measure, + p.watershed_group_code, + p.species_code, + p.observation_date, + o.activity_code, + o.activity, + o.life_stage_code, + o.life_stage, + o.acat_report_url, + e.geom +from obs p +inner join bcfishobs.fiss_fish_obsrvtn_events e on p.fish_obsrvtn_event_id = e.fish_obsrvtn_event_id +inner join whse_fish.fiss_fish_obsrvtn_pnt_sp o on p.fish_observation_point_id = o.fish_observation_point_id; + + +create unique index on bcfishpass.observations_vw (fish_observation_point_id); +create index on bcfishpass.observations_vw using gist (wscode_ltree); +create index on bcfishpass.observations_vw using btree (wscode_ltree); +create index on bcfishpass.observations_vw using gist (localcode_ltree); +create index on bcfishpass.observations_vw using btree (localcode_ltree); \ No newline at end of file diff --git a/db/views/01_streams.sql b/db/v0.5.0/sql/views/02_streams.sql similarity index 100% rename from db/views/01_streams.sql rename to db/v0.5.0/sql/views/02_streams.sql diff --git a/db/v0.5.0/sql/views/03_dams_vw.sql b/db/v0.5.0/sql/views/03_dams_vw.sql new file mode 100644 index 00000000..cd06aa7d --- /dev/null +++ b/db/v0.5.0/sql/views/03_dams_vw.sql @@ -0,0 +1,227 @@ +-- reference CABD dams to FWA stream network +create materialized view bcfishpass.dams_vw as +with cabd as ( + select + d.cabd_id as dam_id, + blk.blue_line_key, + st_transform(d.geom, 3005) as geom + from cabd.dams d + -- exclude any dam noted in user exclusion table + left outer join bcfishpass.user_cabd_dams_exclusions x on d.cabd_id = x.cabd_id + left outer join bcfishpass.user_cabd_blkey_xref blk on d.cabd_id = blk.cabd_id + where x.cabd_id is null +), + +matched AS +( + select + pt.dam_id, + str.linear_feature_id, + str.blue_line_key, + str.wscode_ltree, + str.localcode_ltree, + str.watershed_group_code, + str.geom, + st_distance(str.geom, pt.geom) as distance_to_stream, + st_interpolatepoint(str.geom, pt.geom) as downstream_route_measure + from cabd pt + cross join lateral ( + select + linear_feature_id, + blue_line_key, + wscode_ltree, + localcode_ltree, + watershed_group_code, + geom + from whse_basemapping.fwa_stream_networks_sp str + where str.localcode_ltree is not null + and pt.blue_line_key is null + and not str.wscode_ltree <@ '999' + order by str.geom <-> pt.geom + limit 1 + ) as str + where st_distance(str.geom, pt.geom) <= 65 + + union all + + select distinct on (dam_id) -- distinct on in case two segments are equidistant + pt.dam_id, + str.linear_feature_id, + str.blue_line_key, + str.wscode_ltree, + str.localcode_ltree, + str.watershed_group_code, + str.geom, + st_distance(str.geom, pt.geom) as distance_to_stream, + st_interpolatepoint(str.geom, pt.geom) as downstream_route_measure + from cabd pt + cross join lateral ( + select + linear_feature_id, + blue_line_key, + wscode_ltree, + localcode_ltree, + watershed_group_code, + geom + from whse_basemapping.fwa_stream_networks_sp str + where pt.blue_line_key = str.blue_line_key + and pt.blue_line_key is not null + and str.localcode_ltree is not null + and not str.wscode_ltree <@ '999' + order by str.geom <-> pt.geom + limit 1 + ) as str + order by dam_id, distance_to_stream +), + + -- ensure only one feature returned, and interpolate the geom on the stream +cabd_pts as +( + select distinct on (n.dam_id) + n.dam_id, + n.linear_feature_id, + n.blue_line_key, + n.downstream_route_measure, + n.wscode_ltree, + n.localcode_ltree, + n.distance_to_stream, + n.watershed_group_code, + cabd.dam_name_en, + cabd.height_m, + cabd.owner, + cabd.dam_use, + cabd.operating_status, + cabd.passability_status_code, + + ((st_dump(ST_Force2D(st_locatealong(n.geom, n.downstream_route_measure)))).geom)::geometry(Point, 3005) AS geom + FROM matched n + inner join cabd.dams cabd on n.dam_id = cabd.cabd_id + order by dam_id, distance_to_stream +), + +-- placeholders for major USA dams not present in CABD are stored in user_barriers_anthropogenic +usa as +( + select + (a.user_barrier_anthropogenic_id + 1200000000)::text as dam_id, + s.linear_feature_id, + a.blue_line_key, + a.downstream_route_measure, + s.wscode_ltree, + s.localcode_ltree, + 0 as distance_to_stream, + s.watershed_group_code, + a.barrier_name, + st_force2d((st_dump(st_locatealong(s.geom, a.downstream_route_measure))).geom) as geom + from bcfishpass.user_barriers_anthropogenic a + inner join whse_basemapping.fwa_stream_networks_sp s + on a.blue_line_key = s.blue_line_key + AND ROUND(a.downstream_route_measure::numeric) >= ROUND(s.downstream_route_measure::numeric) + AND ROUND(a.downstream_route_measure::numeric) < ROUND(s.upstream_route_measure::numeric) + where a.barrier_type = 'DAM' +), + +dams as ( + select * from cabd_pts + union all + select + dam_id, + linear_feature_id, + blue_line_key, + downstream_route_measure, + wscode_ltree, + localcode_ltree, + distance_to_stream, + watershed_group_code, + barrier_name as dam_name_en, + null as height_m, + null as owner, + null as dam_use, + null as operating_status, + null as passability_status_code, + geom + from usa +), + +upstr_sal as ( + select + d.dam_id, + count(*) as n_sal + from dams d + inner join bcfishpass.observations_vw o + on fwa_upstream( + d.blue_line_key, + d.downstream_route_measure, + d.wscode_ltree, + d.localcode_ltree, + o.blue_line_key, + o.downstream_route_measure, + o.wscode_ltree, + o.localcode_ltree, + false, + 1 + ) + where o.species_code in ('CH','CM','CO','PK','SK') + and o.observation_date > date('1990-01-01') + group by d.dam_id +), + +upstr_st as ( + select + d.dam_id, + count(*) as n_st + from dams d + inner join bcfishpass.observations_vw o + on fwa_upstream( + d.blue_line_key, + d.downstream_route_measure, + d.wscode_ltree, + d.localcode_ltree, + o.blue_line_key, + o.downstream_route_measure, + o.wscode_ltree, + o.localcode_ltree, + false, + 1 + ) + where o.species_code = 'ST' + and o.observation_date > date('1990-01-01') + group by d.dam_id +) + +select + d.dam_id, + d.linear_feature_id, + d.blue_line_key, + d.downstream_route_measure, + d.wscode_ltree, + d.localcode_ltree, + d.distance_to_stream, + d.watershed_group_code, + d.dam_name_en, + d.height_m, + d.owner, + d.dam_use, + d.operating_status, + d.passability_status_code, + s1.n_sal as n_obs_salmon_since_1990, + s2.n_st as n_obs_steelhead_since_1990, + d.geom +from dams d +left outer join upstr_sal s1 on d.dam_id = s1.dam_id +left outer join upstr_st s2 on d.dam_id = s2.dam_id; + +create unique index on bcfishpass.dams_vw (dam_id); +create index on bcfishpass.dams_vw using gist (geom); + + +-- note dams not matched to streams +create view bcfishpass.dams_not_matched_to_streams as +select + a.cabd_id, + a.dam_name_en +from cabd.dams a +left outer join bcfishpass.dams_vw b +on a.cabd_id::text = b.dam_id +where b.dam_id is null +order by a.cabd_id; diff --git a/db/views/02_crossings.sql b/db/v0.5.0/sql/views/04_crossings.sql similarity index 66% rename from db/views/02_crossings.sql rename to db/v0.5.0/sql/views/04_crossings.sql index 15860dc2..53090f72 100644 --- a/db/views/02_crossings.sql +++ b/db/v0.5.0/sql/views/04_crossings.sql @@ -266,10 +266,12 @@ select c.transport_line_type_description, c.transport_line_surface_description, c.ften_forest_file_id, + c.ften_road_section_id, c.ften_file_type_description, c.ften_client_number, c.ften_client_name, c.ften_life_cycle_status_code, + c.ften_map_label, c.rail_track_name, c.rail_owner_name, c.rail_operator_english_name, @@ -487,229 +489,53 @@ create unique index on bcfishpass.crossings_vw (aggregated_crossings_id); create index on bcfishpass.crossings_vw using gist (geom); --- wcrp version of the output crossings view - -create materialized view bcfishpass.crossings_wcrp_vw as -select - -- joining to streams based on measure can be error prone due to precision. - -- Join to streams on linear_feature_id and keep the first result - -- (since streams are segmented there is often >1 match) - distinct on (c.aggregated_crossings_id) - c.aggregated_crossings_id, - c.stream_crossing_id, - c.dam_id, - c.user_barrier_anthropogenic_id, - c.modelled_crossing_id, - c.crossing_source, - cft.crossing_feature_type, - c.pscis_status, - c.crossing_type_code, - c.crossing_subtype_code, - array_to_string(c.modelled_crossing_type_source, ';') as modelled_crossing_type_source, - c.barrier_status, - c.pscis_road_name, - c.pscis_stream_name, - c.pscis_assessment_comment, - c.pscis_assessment_date, - c.pscis_final_score, - c.transport_line_structured_name_1, - c.transport_line_type_description, - c.transport_line_surface_description, - c.ften_forest_file_id, - c.ften_file_type_description, - c.ften_client_number, - c.ften_client_name, - c.ften_life_cycle_status_code, - c.rail_track_name, - c.rail_owner_name, - c.rail_operator_english_name, - c.ogc_proponent, - c.dam_name, - c.dam_height, - c.dam_owner, - c.dam_use, - c.dam_operating_status, - c.utm_zone, - c.utm_easting, - c.utm_northing, - t.map_tile_display_name as dbm_mof_50k_grid, - c.linear_feature_id, - c.blue_line_key, - c.watershed_key, - c.downstream_route_measure, - c.wscode_ltree as wscode, - c.localcode_ltree as localcode, - c.watershed_group_code, - c.gnis_stream_name, - c.stream_order, - c.stream_magnitude, - s.upstream_area_ha, - s.stream_order_parent, - s.stream_order_max, - s.map_upstream, - s.channel_width, - s.mad_m3s, - array_to_string(cdo.observedspp_dnstr, ';') as observedspp_dnstr, - array_to_string(cuo.observedspp_upstr, ';') as observedspp_upstr, - array_to_string(cd.features_dnstr, ';') as crossings_dnstr, - array_to_string(ad.features_dnstr, ';') as barriers_anthropogenic_dnstr, - coalesce(array_length(ad.features_dnstr, 1), 0) as barriers_anthropogenic_dnstr_count, - array_to_string(au.features_upstr, ';') as barriers_anthropogenic_upstr, - coalesce(array_length(au.features_upstr, 1), 0) as barriers_anthropogenic_upstr_count, - array_to_string(aum.barriers_upstr_bt, ';') as barriers_anthropogenic_bt_upstr, - coalesce(array_length(aum.barriers_upstr_bt, 1), 0) as barriers_anthropogenic_upstr_bt_count, - array_to_string(aum.barriers_upstr_ch_cm_co_pk_sk, ';') as barriers_anthropogenic_ch_cm_co_pk_sk_upstr, - coalesce(array_length(aum.barriers_upstr_ch_cm_co_pk_sk, 1), 0) as barriers_anthropogenic_ch_cm_co_pk_sk_upstr_count, - array_to_string(aum.barriers_upstr_st, ';') as barriers_anthropogenic_st_upstr, - coalesce(array_length(aum.barriers_upstr_st, 1), 0) as barriers_anthropogenic_st_upstr_count, - array_to_string(aum.barriers_upstr_wct, ';') as barriers_anthropogenic_wct_upstr, - coalesce(array_length(aum.barriers_upstr_wct, 1), 0) as barriers_anthropogenic_wct_upstr_count, - a.gradient, - a.total_network_km, - a.total_stream_km, - a.total_lakereservoir_ha, - a.total_wetland_ha, - a.total_slopeclass03_waterbodies_km, - a.total_slopeclass03_km, - a.total_slopeclass05_km, - a.total_slopeclass08_km, - a.total_slopeclass15_km, - a.total_slopeclass22_km, - a.total_slopeclass30_km, - a.total_belowupstrbarriers_network_km, - a.total_belowupstrbarriers_stream_km, - a.total_belowupstrbarriers_lakereservoir_ha, - a.total_belowupstrbarriers_wetland_ha, - a.total_belowupstrbarriers_slopeclass03_waterbodies_km, - a.total_belowupstrbarriers_slopeclass03_km, - a.total_belowupstrbarriers_slopeclass05_km, - a.total_belowupstrbarriers_slopeclass08_km, - a.total_belowupstrbarriers_slopeclass15_km, - a.total_belowupstrbarriers_slopeclass22_km, - a.total_belowupstrbarriers_slopeclass30_km, - - -- access models - array_to_string(a.barriers_ch_cm_co_pk_sk_dnstr, ';') as barriers_ch_cm_co_pk_sk_dnstr, - a.ch_cm_co_pk_sk_network_km, - a.ch_cm_co_pk_sk_stream_km, - a.ch_cm_co_pk_sk_lakereservoir_ha, - a.ch_cm_co_pk_sk_wetland_ha, - a.ch_cm_co_pk_sk_slopeclass03_waterbodies_km, - a.ch_cm_co_pk_sk_slopeclass03_km, - a.ch_cm_co_pk_sk_slopeclass05_km, - a.ch_cm_co_pk_sk_slopeclass08_km, - a.ch_cm_co_pk_sk_slopeclass15_km, - a.ch_cm_co_pk_sk_slopeclass22_km, - a.ch_cm_co_pk_sk_slopeclass30_km, - a.ch_cm_co_pk_sk_belowupstrbarriers_network_km, - a.ch_cm_co_pk_sk_belowupstrbarriers_stream_km, - a.ch_cm_co_pk_sk_belowupstrbarriers_lakereservoir_ha, - a.ch_cm_co_pk_sk_belowupstrbarriers_wetland_ha, - a.ch_cm_co_pk_sk_belowupstrbarriers_slopeclass03_waterbodies_km, - a.ch_cm_co_pk_sk_belowupstrbarriers_slopeclass03_km, - a.ch_cm_co_pk_sk_belowupstrbarriers_slopeclass05_km, - a.ch_cm_co_pk_sk_belowupstrbarriers_slopeclass08_km, - a.ch_cm_co_pk_sk_belowupstrbarriers_slopeclass15_km, - a.ch_cm_co_pk_sk_belowupstrbarriers_slopeclass22_km, - a.ch_cm_co_pk_sk_belowupstrbarriers_slopeclass30_km, - - array_to_string(a.barriers_st_dnstr, ';') as barriers_st_dnstr, - a.st_network_km, - a.st_stream_km, - a.st_lakereservoir_ha, - a.st_wetland_ha, - a.st_slopeclass03_waterbodies_km, - a.st_slopeclass03_km, - a.st_slopeclass05_km, - a.st_slopeclass08_km, - a.st_slopeclass15_km, - a.st_slopeclass22_km, - a.st_slopeclass30_km, - a.st_belowupstrbarriers_network_km, - a.st_belowupstrbarriers_stream_km, - a.st_belowupstrbarriers_lakereservoir_ha, - a.st_belowupstrbarriers_wetland_ha, - a.st_belowupstrbarriers_slopeclass03_waterbodies_km, - a.st_belowupstrbarriers_slopeclass03_km, - a.st_belowupstrbarriers_slopeclass05_km, - a.st_belowupstrbarriers_slopeclass08_km, - a.st_belowupstrbarriers_slopeclass15_km, - a.st_belowupstrbarriers_slopeclass22_km, - a.st_belowupstrbarriers_slopeclass30_km, - - array_to_string(a.barriers_wct_dnstr, ';') as barriers_wct_dnstr, - a.wct_network_km, - a.wct_stream_km, - a.wct_lakereservoir_ha, - a.wct_wetland_ha, - a.wct_slopeclass03_waterbodies_km, - a.wct_slopeclass03_km, - a.wct_slopeclass05_km, - a.wct_slopeclass08_km, - a.wct_slopeclass15_km, - a.wct_slopeclass22_km, - a.wct_slopeclass30_km, - a.wct_belowupstrbarriers_network_km, - a.wct_belowupstrbarriers_stream_km, - a.wct_belowupstrbarriers_lakereservoir_ha, - a.wct_belowupstrbarriers_wetland_ha, - a.wct_belowupstrbarriers_slopeclass03_waterbodies_km, - a.wct_belowupstrbarriers_slopeclass03_km, - a.wct_belowupstrbarriers_slopeclass05_km, - a.wct_belowupstrbarriers_slopeclass08_km, - a.wct_belowupstrbarriers_slopeclass15_km, - a.wct_belowupstrbarriers_slopeclass22_km, - a.wct_belowupstrbarriers_slopeclass30_km, - - -- habitat models - h.ch_spawning_km, - h.ch_rearing_km, - h.ch_spawning_belowupstrbarriers_km, - h.ch_rearing_belowupstrbarriers_km, - h.cm_spawning_km, - h.cm_spawning_belowupstrbarriers_km, - h.co_spawning_km, - h_wcrp.co_rearing_km, - h.co_rearing_ha, - h.co_spawning_belowupstrbarriers_km, - h_wcrp.co_rearing_belowupstrbarriers_km, - h.co_rearing_belowupstrbarriers_ha, - h.pk_spawning_km, - h.pk_spawning_belowupstrbarriers_km, - h.sk_spawning_km, - h_wcrp.sk_rearing_km, - h.sk_rearing_ha, - h.sk_spawning_belowupstrbarriers_km, - h_wcrp.sk_rearing_belowupstrbarriers_km, - h.sk_rearing_belowupstrbarriers_ha, - h.st_spawning_km, - h.st_rearing_km, - h.st_spawning_belowupstrbarriers_km, - h.st_rearing_belowupstrbarriers_km, - h.wct_spawning_km, - h.wct_rearing_km, - h.wct_spawning_belowupstrbarriers_km, - h.wct_rearing_belowupstrbarriers_km, - h_wcrp.all_spawning_km, - h_wcrp.all_spawning_belowupstrbarriers_km, - h_wcrp.all_rearing_km, - h_wcrp.all_rearing_belowupstrbarriers_km, - h_wcrp.all_spawningrearing_km, - h_wcrp.all_spawningrearing_belowupstrbarriers_km, - c.geom -from bcfishpass.crossings c -inner join bcfishpass.crossings_feature_type_vw cft on c.aggregated_crossings_id = cft.aggregated_crossings_id -left outer join bcfishpass.crossings_dnstr_observations_vw cdo on c.aggregated_crossings_id = cdo.aggregated_crossings_id -left outer join bcfishpass.crossings_upstr_observations_vw cuo on c.aggregated_crossings_id = cuo.aggregated_crossings_id -left outer join bcfishpass.crossings_dnstr_crossings cd on c.aggregated_crossings_id = cd.aggregated_crossings_id -left outer join bcfishpass.crossings_dnstr_barriers_anthropogenic ad on c.aggregated_crossings_id = ad.aggregated_crossings_id -left outer join bcfishpass.crossings_upstr_barriers_anthropogenic au on c.aggregated_crossings_id = au.aggregated_crossings_id -left outer join bcfishpass.crossings_upstr_barriers_per_model_vw aum on c.aggregated_crossings_id = aum.aggregated_crossings_id -left outer join bcfishpass.crossings_upstream_access a on c.aggregated_crossings_id = a.aggregated_crossings_id -left outer join bcfishpass.crossings_upstream_habitat h on c.aggregated_crossings_id = h.aggregated_crossings_id -left outer join bcfishpass.crossings_upstream_habitat_wcrp h_wcrp on c.aggregated_crossings_id = h_wcrp.aggregated_crossings_id -left outer join bcfishpass.streams s on c.linear_feature_id = s.linear_feature_id -left outer join whse_basemapping.dbm_mof_50k_grid t ON ST_Intersects(c.geom, t.geom) -order by c.aggregated_crossings_id, s.downstream_route_measure; - -create unique index on bcfishpass.crossings_wcrp_vw (aggregated_crossings_id); -create index on bcfishpass.crossings_wcrp_vw using gist (geom); \ No newline at end of file +-- document the columns included +comment on column bcfishpass.crossings_vw.aggregated_crossings_id IS 'unique identifier for crossing, generated from stream_crossing_id, modelled_crossing_id + 1000000000, user_barrier_anthropogenic_id + 1200000000, cabd_id'; +comment on column bcfishpass.crossings_vw.stream_crossing_id IS 'PSCIS stream crossing unique identifier'; +comment on column bcfishpass.crossings_vw.dam_id IS 'BC Dams unique identifier'; +comment on column bcfishpass.crossings_vw.user_barrier_anthropogenic_id IS 'User added misc anthropogenic barriers unique identifier'; +comment on column bcfishpass.crossings_vw.modelled_crossing_id IS 'Modelled crossing unique identifier'; +comment on column bcfishpass.crossings_vw.crossing_source IS 'Data source for the crossing, one of: {PSCIS,MODELLED CROSSINGS,CABD,MISC BARRIERS}'; +comment on column bcfishpass.crossings_vw.pscis_status IS 'From PSCIS, the current_pscis_status of the crossing, one of: {ASSESSED,HABITAT CONFIRMATION,DESIGN,REMEDIATED}'; +comment on column bcfishpass.crossings_vw.crossing_type_code IS 'Defines the type of crossing present at the location of the stream crossing. Acceptable types are: OBS = Open Bottom Structure CBS = Closed Bottom Structure OTHER = Crossing structure does not fit into the above categories. Eg: ford, wier'; +comment on column bcfishpass.crossings_vw.crossing_subtype_code IS 'Further definition of the type of crossing, one of {BRIDGE,CRTBOX,DAM,FORD,OVAL,PIPEARCH,ROUND,WEIR,WOODBOX,NULL}'; +comment on column bcfishpass.crossings_vw.modelled_crossing_type_source IS 'List of sources that indicate if a modelled crossing is open bottom, Acceptable values are: FWA_EDGE_TYPE=double line river, FWA_STREAM_ORDER=stream order >=6, GBA_RAILWAY_STRUCTURE_LINES_SP=railway structure, "MANUAL FIX"=manually identified OBS, MOT_ROAD_STRUCTURE_SP=MoT structure, TRANSPORT_LINE_STRUCTURE_CODE=DRA structure}'; +comment on column bcfishpass.crossings_vw.barrier_status IS 'The evaluation of the crossing as a barrier to the fish passage. From PSCIS, this is based on the FINAL SCORE value. For other data sources this varies. Acceptable Values are: PASSABLE - Passable, POTENTIAL - Potential or partial barrier, BARRIER - Barrier, UNKNOWN - Other'; +comment on column bcfishpass.crossings_vw.pscis_road_name IS 'PSCIS road name, taken from the PSCIS assessment data submission'; +comment on column bcfishpass.crossings_vw.pscis_stream_name IS 'PSCIS stream name, taken from the PSCIS assessment data submission'; +comment on column bcfishpass.crossings_vw.pscis_assessment_comment IS 'PSCIS assessment_comment, taken from the PSCIS assessment data submission'; +comment on column bcfishpass.crossings_vw.pscis_assessment_date IS 'PSCIS assessment_date, taken from the PSCIS assessment data submission'; +comment on column bcfishpass.crossings_vw.pscis_final_score IS 'PSCIS final_score, taken from the PSCIS assessment data submission'; +comment on column bcfishpass.crossings_vw.transport_line_structured_name_1 IS 'DRA road name, taken from the nearest DRA road (within 30m)'; +comment on column bcfishpass.crossings_vw.transport_line_type_description IS 'DRA road type, taken from the nearest DRA road (within 30m)'; +comment on column bcfishpass.crossings_vw.transport_line_surface_description IS 'DRA road surface, taken from the nearest DRA road (within 30m)'; +comment on column bcfishpass.crossings_vw.ften_forest_file_id IS 'FTEN road forest_file_id value, taken from the nearest FTEN road (within 30m)'; +comment on column bcfishpass.crossings_vw.ften_road_section_id IS 'FTEN road road_section_id value, taken from the nearest FTEN road (within 30m)'; +comment on column bcfishpass.crossings_vw.ften_file_type_description IS 'FTEN road tenure type (Forest Service Road, Road Permit, etc), taken from the nearest FTEN road (within 30m)'; +comment on column bcfishpass.crossings_vw.ften_client_number IS 'FTEN road client number, taken from the nearest FTEN road (within 30m)'; +comment on column bcfishpass.crossings_vw.ften_client_name IS 'FTEN road client name, taken from the nearest FTEN road (within 30m)'; +comment on column bcfishpass.crossings_vw.ften_life_cycle_status_code IS 'FTEN road life_cycle_status_code (active or retired, pending roads are not included), taken from the nearest FTEN road (within 30m)'; +comment on column bcfishpass.crossings_vw.ften_map_label IS 'FTEN road map_label value, taken from the nearest FTEN road (within 30m)'; +comment on column bcfishpass.crossings_vw.rail_track_name IS 'Railway name, taken from nearest railway (within 25m)'; +comment on column bcfishpass.crossings_vw.rail_owner_name IS 'Railway owner name, taken from nearest railway (within 25m)'; +comment on column bcfishpass.crossings_vw.rail_operator_english_name IS 'Railway operator name, taken from nearest railway (within 25m)';; +comment on column bcfishpass.crossings_vw.ogc_proponent IS 'OGC road tenure proponent (currently modelled crossings only, taken from OGC road that crosses the stream)'; +comment on column bcfishpass.crossings_vw.dam_name IS 'See CABD dams column: dam_name_en'; +comment on column bcfishpass.crossings_vw.dam_height IS 'See CABD dams column: dam_height'; +comment on column bcfishpass.crossings_vw.dam_owner IS 'See CABD dams column: owner'; +comment on column bcfishpass.crossings_vw.dam_use IS 'See CABD table dam_use_codes'; +comment on column bcfishpass.crossings_vw.dam_operating_status IS 'See CABD dams column dam_operating_status'; +comment on column bcfishpass.crossings_vw.utm_zone IS 'UTM ZONE is a segment of the Earths surface 6 degrees of longitude in width. The zones are numbered eastward starting at the meridian 180 degrees from the prime meridian at Greenwich. There are five zones numbered 7 through 11 that cover British Columbia, e.g., Zone 10 with a central meridian at -123 degrees.'; +comment on column bcfishpass.crossings_vw.utm_easting IS 'UTM EASTING is the distance in meters eastward to or from the central meridian of a UTM zone with a false easting of 500000 meters. e.g., 440698'; +comment on column bcfishpass.crossings_vw.utm_northing IS 'UTM NORTHING is the distance in meters northward from the equator. e.g., 6197826'; +comment on column bcfishpass.crossings_vw.linear_feature_id IS 'From BC FWA, the unique identifier for a stream segment (flow network arc)'; +comment on column bcfishpass.crossings_vw.blue_line_key IS 'From BC FWA, uniquely identifies a single flow line such that a main channel and a secondary channel with the same watershed code would have different blue line keys (the Fraser River and all side channels have different blue line keys).'; +comment on column bcfishpass.crossings_vw.watershed_key IS 'From BC FWA, a key that identifies a stream system. There is a 1:1 match between a watershed key and watershed code. The watershed key will match the blue line key for the mainstem.'; +comment on column bcfishpass.crossings_vw.downstream_route_measure IS 'The distance, in meters, along the blue_line_key from the mouth of the stream/blue_line_key to the feature.'; +comment on column bcfishpass.crossings_vw.wscode IS 'A truncated version of the BC FWA fwa_watershed_code (trailing zeros removed and "-" replaced with ".", stored as postgres type ltree for fast tree based queries'; +comment on column bcfishpass.crossings_vw.localcode IS 'A truncated version of the BC FWA local_watershed_code (trailing zeros removed and "-" replaced with ".", stored as postgres type ltree for fast tree based queries';; +comment on column bcfishpass.crossings_vw.watershed_group_code IS 'The watershed group code associated with the feature.'; +comment on column bcfishpass.crossings_vw.gnis_stream_name IS 'The BCGNIS (BC Geographical Names Information System) name associated with the FWA stream'; +comment on column bcfishpass.crossings_vw.stream_order IS 'Order of FWA stream at point'; +comment on column bcfishpass.crossings_vw.stream_magnitude IS 'Magnitude of FWA stream at point'; +comment on column bcfishpass.crossings_vw.geom IS 'The point geometry associated with the feature'; diff --git a/db/views/04_falls_vw.sql b/db/v0.5.0/sql/views/05_falls_vw.sql similarity index 77% rename from db/views/04_falls_vw.sql rename to db/v0.5.0/sql/views/05_falls_vw.sql index e330d777..cad99416 100644 --- a/db/views/04_falls_vw.sql +++ b/db/v0.5.0/sql/views/05_falls_vw.sql @@ -1,15 +1,16 @@ -- load CABD falls and any misc (temporary) additions from bcfishpass table - create materialized view bcfishpass.falls_vw as with cabd as ( select - cabd_id as falls_id, - st_transform(geom, 3005) as geom - from cabd.waterfalls + w.cabd_id as falls_id, + x.blue_line_key, + st_transform(w.geom, 3005) as geom + from cabd.waterfalls w + left outer join bcfishpass.user_cabd_blkey_xref x on w.cabd_id = x.cabd_id ), -nearest AS +matched AS ( select pt.falls_id, @@ -36,7 +37,40 @@ nearest AS order by str.geom <-> pt.geom limit 1 ) as str - where st_distance(str.geom, pt.geom) <= 65 + where + st_distance(str.geom, pt.geom) <= 65 and + pt.blue_line_key is null + + union all + + select distinct on (falls_id) -- distinct on in case two segments are equidistant + pt.falls_id, + str.linear_feature_id, + str.blue_line_key, + str.wscode_ltree, + str.localcode_ltree, + str.watershed_group_code, + str.geom, + st_distance(str.geom, pt.geom) as distance_to_stream, + st_interpolatepoint(str.geom, pt.geom) as downstream_route_measure + from cabd pt + cross join lateral ( + select + linear_feature_id, + blue_line_key, + wscode_ltree, + localcode_ltree, + watershed_group_code, + geom + from whse_basemapping.fwa_stream_networks_sp str + where pt.blue_line_key = str.blue_line_key + and pt.blue_line_key is not null + and str.localcode_ltree is not null + and not str.wscode_ltree <@ '999' + order by str.geom <-> pt.geom + limit 1 + ) as str + order by falls_id, distance_to_stream ), cabd_pts as ( @@ -56,7 +90,7 @@ cabd_pts as ( else false end as barrier_ind, ((st_dump(ST_Force2D(st_locatealong(n.geom, n.downstream_route_measure)))).geom)::geometry(Point, 3005) AS geom - FROM nearest n + FROM matched n inner join cabd.waterfalls cabd on n.falls_id = cabd.cabd_id order by falls_id, distance_to_stream ) diff --git a/db/views/05_streams_spp.sql b/db/v0.5.0/sql/views/06_streams_spp.sql similarity index 100% rename from db/views/05_streams_spp.sql rename to db/v0.5.0/sql/views/06_streams_spp.sql diff --git a/db/views/20_freshwater_fish_habitat_accessibility_model.sql b/db/v0.5.0/sql/views/fptwg_freshwater_fish_habitat_accessibility_model.sql similarity index 100% rename from db/views/20_freshwater_fish_habitat_accessibility_model.sql rename to db/v0.5.0/sql/views/fptwg_freshwater_fish_habitat_accessibility_model.sql diff --git a/db/views/wcrp_barrier_count_vw.sql b/db/v0.5.0/sql/views/wcrp_barrier_count_vw.sql similarity index 57% rename from db/views/wcrp_barrier_count_vw.sql rename to db/v0.5.0/sql/views/wcrp_barrier_count_vw.sql index d5a60633..7fc67346 100644 --- a/db/views/wcrp_barrier_count_vw.sql +++ b/db/v0.5.0/sql/views/wcrp_barrier_count_vw.sql @@ -4,16 +4,18 @@ select c.aggregated_crossings_id, case when - h.ch_spawning_km > 0 or h.ch_rearing_km > 0 or - h.co_spawning_km > 0 or h.co_rearing_km > 0 or - h.sk_spawning_km > 0 or h.sk_rearing_km > 0 or - h.st_spawning_km > 0 or h.st_rearing_km > 0 or - h.wct_spawning_km > 0 or h.wct_rearing_km > 0 + ((h.ch_spawning_km > 0 or h.ch_rearing_km > 0) and w.ch IS TRUE) or + ((h.co_spawning_km > 0 or h.co_rearing_km > 0) and w.co IS TRUE) or + ((h.sk_spawning_km > 0 or h.sk_rearing_km > 0) and w.sk IS TRUE) or + ((h.st_spawning_km > 0 or h.st_rearing_km > 0) and w.st IS TRUE) or + ((h.wct_spawning_km > 0 or h.wct_rearing_km > 0) and w.wct IS TRUE) then 'HABITAT' when - cardinality(a.barriers_ch_cm_co_pk_sk_dnstr) = 0 or - cardinality(a.barriers_st_dnstr) = 0 or - cardinality(a.barriers_wct_dnstr) = 0 + (w.ch IS TRUE and cardinality(a.barriers_ch_cm_co_pk_sk_dnstr) = 0) or + (w.co IS TRUE and cardinality(a.barriers_ch_cm_co_pk_sk_dnstr) = 0) or + (w.sk IS TRUE and cardinality(a.barriers_ch_cm_co_pk_sk_dnstr) = 0) or + (w.st IS TRUE and cardinality(a.barriers_st_dnstr) = 0) or + (w.wct IS TRUE and cardinality(a.barriers_wct_dnstr) = 0) then 'ACCESSIBLE' else 'NATURAL_BARRIER' end as model_status @@ -22,6 +24,8 @@ left outer join bcfishpass.crossings_upstream_access a on c.aggregated_crossings_id = a.aggregated_crossings_id left outer join bcfishpass.crossings_upstream_habitat h on c.aggregated_crossings_id = h.aggregated_crossings_id +left outer join bcfishpass.wcrp_watersheds w +on c.watershed_group_code = w.watershed_group_code ) SELECT @@ -39,8 +43,8 @@ on c.aggregated_crossings_id = ft.aggregated_crossings_id left outer join model_status ms on c.aggregated_crossings_id = ms.aggregated_crossings_id -- WCRP watersheds only -WHERE c.watershed_group_code in ('BULK','LNIC','HORS','BOWR','QUES','CARR','ELKR') +inner join bcfishpass.wcrp_watersheds w on c.watershed_group_code = w.watershed_group_code -- do not include flathead -AND c.wscode_ltree <@ '300.602565.854327.993941.902282.132363'::ltree IS FALSE +WHERE c.wscode_ltree <@ '300.602565.854327.993941.902282.132363'::ltree IS FALSE GROUP BY c.watershed_group_code, ms.model_status, ft.crossing_feature_type ORDER BY c.watershed_group_code, ms.model_status, ft.crossing_feature_type; \ No newline at end of file diff --git a/db/v0.5.0/sql/views/wcrp_crossings_vw.sql b/db/v0.5.0/sql/views/wcrp_crossings_vw.sql new file mode 100644 index 00000000..902cd352 --- /dev/null +++ b/db/v0.5.0/sql/views/wcrp_crossings_vw.sql @@ -0,0 +1,123 @@ +-- wcrp version of the output crossings view - +create materialized view bcfishpass.crossings_wcrp_vw as + +-- find upstream crossings with wcrp 'all spawning rearing habitat' upstream +with upstr_wcrp_barriers as materialized ( + select distinct + ba.aggregated_crossings_id, + h.aggregated_crossings_id as upstr_barriers, + h.all_spawningrearing_km + from bcfishpass.crossings_upstr_barriers_anthropogenic ba + inner join bcfishpass.crossings_upstream_habitat_wcrp h on h.aggregated_crossings_id = any(ba.features_upstr) + where h.all_spawningrearing_km > 0 + order by ba.aggregated_crossings_id, h.aggregated_crossings_id +), + +-- aggregate the upstream wcrp crossings into a list and count +upstr_wcrp_barriers_list as ( + select + aggregated_crossings_id, + array_to_string(array_agg(upstr_barriers), ';') as barriers_anthropogenic_habitat_wcrp_upstr, + coalesce(array_length(array_agg(upstr_barriers), 1), 0) as barriers_anthropogenic_habitat_wcrp_upstr_count + from upstr_wcrp_barriers + group by aggregated_crossings_id + order by aggregated_crossings_id +) + +select + -- joining to streams based on measure can be error prone due to precision. + -- Join to streams on linear_feature_id and keep the first result + -- (since streams are segmented there is often >1 match) + distinct on (c.aggregated_crossings_id) + c.aggregated_crossings_id, + c.modelled_crossing_id, + c.crossing_source, + cft.crossing_feature_type, + c.pscis_status, + c.crossing_type_code, + c.crossing_subtype_code, + c.barrier_status, + c.pscis_road_name, + c.pscis_stream_name, + c.pscis_assessment_comment, + c.pscis_assessment_date, + c.transport_line_structured_name_1, + c.rail_track_name, + c.dam_name, + c.dam_height, + c.dam_owner, + c.dam_use, + c.dam_operating_status, + c.utm_zone, + c.utm_easting, + c.utm_northing, + c.blue_line_key, + c.watershed_group_code, + c.gnis_stream_name, + array_to_string(ad.features_dnstr, ';') as barriers_anthropogenic_dnstr, + coalesce(array_length(ad.features_dnstr, 1), 0) as barriers_anthropogenic_dnstr_count, + uwbl.barriers_anthropogenic_habitat_wcrp_upstr, + uwbl.barriers_anthropogenic_habitat_wcrp_upstr_count, + + -- habitat models + h.ch_spawning_km, + h.ch_rearing_km, + h.ch_spawning_belowupstrbarriers_km, + h.ch_rearing_belowupstrbarriers_km, + h.cm_spawning_km, + h.cm_spawning_belowupstrbarriers_km, + h.co_spawning_km, + h_wcrp.co_rearing_km, + h.co_rearing_ha, + h.co_spawning_belowupstrbarriers_km, + h_wcrp.co_rearing_belowupstrbarriers_km, + h.co_rearing_belowupstrbarriers_ha, + h.pk_spawning_km, + h.pk_spawning_belowupstrbarriers_km, + h.sk_spawning_km, + h_wcrp.sk_rearing_km, + h.sk_rearing_ha, + h.sk_spawning_belowupstrbarriers_km, + h_wcrp.sk_rearing_belowupstrbarriers_km, + h.sk_rearing_belowupstrbarriers_ha, + h.st_spawning_km, + h.st_rearing_km, + h.st_spawning_belowupstrbarriers_km, + h.st_rearing_belowupstrbarriers_km, + h.wct_spawning_km, + h.wct_rearing_km, + h.wct_spawning_belowupstrbarriers_km, + h.wct_rearing_belowupstrbarriers_km, + h_wcrp.all_spawning_km, + h_wcrp.all_spawning_belowupstrbarriers_km, + h_wcrp.all_rearing_km, + h_wcrp.all_rearing_belowupstrbarriers_km, + h_wcrp.all_spawningrearing_km, + h_wcrp.all_spawningrearing_belowupstrbarriers_km, + r.set_id, + r.total_hab_gain_set, + r.num_barriers_set, + r.avg_gain_per_barrier, + r.dnstr_set_ids, + r.rank_avg_gain_per_barrier, + r.rank_avg_gain_tiered, + r.rank_total_upstr_hab, + r.rank_combined, + r.tier_combined, + c.geom +from bcfishpass.crossings c +inner join bcfishpass.wcrp_watersheds w on c.watershed_group_code = w.watershed_group_code -- only include crossings in WCRP watersheds +inner join bcfishpass.crossings_feature_type_vw cft on c.aggregated_crossings_id = cft.aggregated_crossings_id +left outer join bcfishpass.crossings_dnstr_observations_vw cdo on c.aggregated_crossings_id = cdo.aggregated_crossings_id +left outer join bcfishpass.crossings_upstr_observations_vw cuo on c.aggregated_crossings_id = cuo.aggregated_crossings_id +left outer join bcfishpass.crossings_dnstr_crossings cd on c.aggregated_crossings_id = cd.aggregated_crossings_id +left outer join bcfishpass.crossings_dnstr_barriers_anthropogenic ad on c.aggregated_crossings_id = ad.aggregated_crossings_id +left outer join bcfishpass.crossings_upstr_barriers_anthropogenic au on c.aggregated_crossings_id = au.aggregated_crossings_id +left outer join upstr_wcrp_barriers_list uwbl on c.aggregated_crossings_id = uwbl.aggregated_crossings_id +left outer join bcfishpass.crossings_upstream_access a on c.aggregated_crossings_id = a.aggregated_crossings_id +left outer join bcfishpass.crossings_upstream_habitat h on c.aggregated_crossings_id = h.aggregated_crossings_id +left outer join bcfishpass.crossings_upstream_habitat_wcrp h_wcrp on c.aggregated_crossings_id = h_wcrp.aggregated_crossings_id +left outer join bcfishpass.streams s on c.linear_feature_id = s.linear_feature_id +left outer join whse_basemapping.dbm_mof_50k_grid t ON ST_Intersects(c.geom, t.geom) +left outer join bcfishpass.wcrp_ranked_barriers r ON c.aggregated_crossings_id = r.aggregated_crossings_id +order by c.aggregated_crossings_id, s.downstream_route_measure; \ No newline at end of file diff --git a/db/v0.5.0/sql/views/wcrp_habitat_connectivity_status_vw.sql b/db/v0.5.0/sql/views/wcrp_habitat_connectivity_status_vw.sql new file mode 100644 index 00000000..34c13c6b --- /dev/null +++ b/db/v0.5.0/sql/views/wcrp_habitat_connectivity_status_vw.sql @@ -0,0 +1,242 @@ +-- summarize spawning/rearing/spawning&rearing habitat lengths per group, by accessibility + +create view bcfishpass.wcrp_habitat_connectivity_status_vw as +with length_totals as +( +-- all spawning (ch/co/st/sk/wct) - calculation is simple, just add it up +-- --------------- + SELECT + s.watershed_group_code, + 'SPAWNING' as habitat_type, + coalesce(round((SUM(ST_Length(s.geom)) FILTER ( + WHERE + (h.spawning_ch IS TRUE and w.ch IS TRUE) OR + (h.spawning_co IS TRUE AND w.co IS TRUE) OR + (h.spawning_st IS TRUE AND w.st IS TRUE) OR + (h.spawning_sk IS TRUE AND w.sk IS TRUE) OR + (h.spawning_wct IS TRUE AND w.wct IS TRUE) + ) / 1000)::numeric, 2), 0) as total_km, + + -- spawning accessible + coalesce(round((SUM(ST_Length(s.geom)) FILTER ( + WHERE ( + (h.spawning_ch IS TRUE and w.ch IS TRUE) OR + (h.spawning_co IS TRUE AND w.co IS TRUE) OR + (h.spawning_st IS TRUE AND w.st IS TRUE) OR + (h.spawning_sk IS TRUE AND w.sk IS TRUE) OR + (h.spawning_wct IS TRUE AND w.wct IS TRUE) + ) + AND a.barriers_anthropogenic_dnstr IS NULL + ) / 1000)::numeric, 2), 0) as accessible_km + from bcfishpass.streams s + inner join bcfishpass.streams_habitat_linear_vw h using (segmented_stream_id) + inner join bcfishpass.streams_access_vw a using (segmented_stream_id) + inner join bcfishpass.wcrp_watersheds w on s.watershed_group_code = w.watershed_group_code -- WCRP watersheds only + group by s.watershed_group_code + + UNION ALL + +-- REARING length +-- -------------- +-- rearing is more complex, add an extra .5 for CO/SK rearing in wetlands/lakes respectively + + SELECT + s.watershed_group_code, + 'REARING' as habitat_type, + round( + ( + ( + coalesce(SUM(ST_Length(geom)) FILTER ( + WHERE + (h.rearing_ch IS TRUE AND w.ch IS TRUE) OR + (h.rearing_st IS TRUE AND w.st IS TRUE) OR + (h.rearing_sk IS TRUE AND w.sk IS TRUE) OR + (h.rearing_co IS TRUE AND w.co IS TRUE) OR + (h.rearing_wct IS TRUE AND w.wct IS TRUE) + ), 0) + + -- add .5 coho rearing in wetlands + coalesce(SUM(ST_Length(s.geom) * .5) FILTER (WHERE h.rearing_co IS TRUE AND w.co IS TRUE AND s.edge_type = 1050), 0) + + -- add .5 sockeye rearing in lakes (all of it) + coalesce(SUM(ST_Length(s.geom) * .5) FILTER (WHERE h.spawning_sk IS TRUE AND w.co IS TRUE), 0) + ) / 1000)::numeric, 2 + ) AS total_km, + + -- rearing accessible + round( + ( + ( + coalesce(SUM(ST_Length(geom)) FILTER ( + WHERE ( + (h.rearing_ch IS TRUE AND w.ch IS TRUE) OR + (h.rearing_co IS TRUE AND w.co IS TRUE) OR + (h.rearing_st IS TRUE AND w.st IS TRUE) OR + (h.rearing_sk IS TRUE AND w.sk IS TRUE) OR + (h.rearing_wct IS TRUE AND w.wct IS TRUE) + ) + AND a.barriers_anthropogenic_dnstr IS NULL + ), 0) + + -- add .5 coho rearing in wetlands + coalesce(SUM(ST_Length(geom) * .5) FILTER (WHERE h.rearing_co IS TRUE AND w.co IS TRUE AND edge_type = 1050 AND barriers_anthropogenic_dnstr IS NULL), 0) + + -- add .5 sockeye rearing in lakes (all of it) + coalesce(SUM(ST_Length(geom) * .5) FILTER (WHERE h.spawning_sk IS TRUE AND w.sk IS TRUE AND barriers_anthropogenic_dnstr IS NULL), 0) + ) / 1000)::numeric, 2 + ) AS accessible_km + from bcfishpass.streams s + inner join bcfishpass.streams_habitat_linear_vw h using (segmented_stream_id) + inner join bcfishpass.streams_access_vw a using (segmented_stream_id) + inner join bcfishpass.wcrp_watersheds w on s.watershed_group_code = w.watershed_group_code -- WCRP watersheds only + group by s.watershed_group_code + + UNION ALL + + -- spawning or rearing - total km of habitat + SELECT + s.watershed_group_code, + 'ALL' as habitat_type, + round( + ( + ( + coalesce(SUM(ST_Length(s.geom)) FILTER ( + WHERE + (h.spawning_ch IS TRUE AND w.ch IS TRUE) OR + (h.spawning_co IS TRUE AND w.co IS TRUE) OR + (h.spawning_st IS TRUE AND w.st IS TRUE) OR + (h.spawning_sk IS TRUE AND w.sk IS TRUE) OR + (h.spawning_wct IS TRUE AND w.wct IS TRUE) OR + (h.rearing_ch IS TRUE AND w.ch IS TRUE) OR + (h.rearing_co IS TRUE AND w.co IS TRUE) OR + (h.rearing_st IS TRUE AND w.st IS TRUE) OR + (h.rearing_sk IS TRUE AND w.sk IS TRUE) OR + (h.rearing_wct IS TRUE AND w.wct IS TRUE) + ), 0) + + -- add .5 coho rearing in wetlands + coalesce(SUM(ST_Length(s.geom) * .5) FILTER (WHERE h.rearing_co IS TRUE AND w.co IS TRUE AND s.edge_type = 1050), 0) + + -- add .5 sockeye rearing in lakes (all of it) + coalesce(SUM(ST_Length(geom) * .5) FILTER (WHERE h.spawning_sk IS TRUE AND w.sk IS TRUE), 0) + ) / 1000)::numeric, 2 + ) AS total_km, + + -- total acccessible km + round( + ( + ( + coalesce(SUM(ST_Length(geom)) FILTER ( + WHERE ( + (h.spawning_ch IS TRUE AND w.ch IS TRUE) OR + (h.spawning_co IS TRUE AND w.co IS TRUE) OR + (h.spawning_st IS TRUE AND w.st IS TRUE) OR + (h.spawning_sk IS TRUE AND w.sk IS TRUE) OR + (h.spawning_wct IS TRUE AND w.wct IS TRUE) OR + (h.rearing_ch IS TRUE AND w.ch IS TRUE) OR + (h.rearing_co IS TRUE AND w.co IS TRUE) OR + (h.rearing_st IS TRUE AND w.st IS TRUE) OR + (h.rearing_sk IS TRUE AND w.sk IS TRUE) OR + (h.rearing_wct IS TRUE AND w.wct IS TRUE) + ) + AND a.barriers_anthropogenic_dnstr IS NULL), 0) + + -- add .5 coho rearing in wetlands + coalesce(SUM(ST_Length(geom) * .5) FILTER (WHERE h.rearing_co IS TRUE AND edge_type = 1050 AND a.barriers_anthropogenic_dnstr IS NULL), 0) + + -- add .5 sockeye rearing in lakes (all of it) + coalesce(SUM(ST_Length(geom) * .5) FILTER (WHERE h.spawning_sk IS TRUE AND a.barriers_anthropogenic_dnstr IS NULL), 0) + ) / 1000)::numeric, 2 + ) AS accessible_km + from bcfishpass.streams s + inner join bcfishpass.streams_habitat_linear_vw h using (segmented_stream_id) + inner join bcfishpass.streams_access_vw a using (segmented_stream_id) + inner join bcfishpass.wcrp_watersheds w on s.watershed_group_code = w.watershed_group_code -- WCRP watersheds only + group by s.watershed_group_code + + UNION ALL + + -- Upstream of Elko Dam + SELECT + s.watershed_group_code, + 'UPSTREAM_ELKO' as habitat_type, + round( + ( + ( + coalesce(SUM(ST_Length(s.geom)) FILTER ( + WHERE + (h.spawning_wct IS TRUE AND w.wct IS TRUE) OR + (h.rearing_wct IS TRUE AND w.wct IS TRUE) + ), 0) + ) / 1000)::numeric, 2 + ) AS total_km, + + -- total acccessible km + round( + ( + ( + coalesce(SUM(ST_Length(geom)) FILTER ( + WHERE ( + (h.spawning_wct IS TRUE AND w.wct IS TRUE) OR + (h.rearing_wct IS TRUE AND w.wct IS TRUE) + ) + AND a.barriers_anthropogenic_dnstr = (select barriers_anthropogenic_dnstr + from bcfishpass.streams s + inner join bcfishpass.streams_habitat_linear_vw h using (segmented_stream_id) + inner join bcfishpass.streams_access_vw a using (segmented_stream_id) + where segmented_stream_id like '356570562.22912000')), 0) + ) / 1000)::numeric, 2 + ) AS accessible_km + from bcfishpass.streams s + inner join bcfishpass.streams_habitat_linear_vw h using (segmented_stream_id) + inner join bcfishpass.streams_access_vw a using (segmented_stream_id) + inner join bcfishpass.wcrp_watersheds w on s.watershed_group_code = w.watershed_group_code -- WCRP watersheds only + where FWA_Upstream(356570562, 22910, 22910, '300.625474.584724'::ltree, '300.625474.584724.100997'::ltree, blue_line_key, downstream_route_measure, wscode_ltree, localcode_ltree) -- only above Elko Dam + group by s.watershed_group_code + +UNION ALL + -- Downstream of Elko Dam + SELECT + s.watershed_group_code, + 'DOWNSTREAM_ELKO' as habitat_type, + round( + ( + ( + coalesce(SUM(ST_Length(s.geom)) FILTER ( + WHERE + (h.spawning_wct IS TRUE AND w.wct IS TRUE) OR + (h.rearing_wct IS TRUE AND w.wct IS TRUE) + ), 0) + ) / 1000)::numeric, 2 + ) AS total_km, + + -- total acccessible km + round( + ( + ( + coalesce(SUM(ST_Length(geom)) FILTER ( + WHERE ( + (h.spawning_wct IS TRUE AND w.wct IS TRUE) OR + (h.rearing_wct IS TRUE AND w.wct IS TRUE) + ) + AND a.barriers_anthropogenic_dnstr IS NULL + AND a.barriers_wct_dnstr = array[]::text[] + OR a.barriers_anthropogenic_dnstr = (select distinct barriers_anthropogenic_dnstr + from bcfishpass.streams s + inner join bcfishpass.streams_habitat_linear_vw h using (segmented_stream_id) + inner join bcfishpass.streams_access_vw a using (segmented_stream_id) + where linear_feature_id = 706872063)), 0) + ) / 1000)::numeric, 2 + ) AS accessible_km + from bcfishpass.streams s + inner join bcfishpass.streams_habitat_linear_vw h using (segmented_stream_id) + inner join bcfishpass.streams_access_vw a using (segmented_stream_id) + inner join bcfishpass.wcrp_watersheds w on s.watershed_group_code = w.watershed_group_code -- WCRP watersheds only + where wscode_ltree <@ '300.625474.584724'::ltree -- on the elk system and not above elko dam + AND NOT FWA_Upstream(356570562, 22910, 22910, '300.625474.584724'::ltree, '300.625474.584724.100997'::ltree, blue_line_key, downstream_route_measure, wscode_ltree, localcode_ltree) + group by s.watershed_group_code +) + +select + watershed_group_code, + habitat_type, + total_km, + accessible_km, + round((accessible_km / (total_km + .0001)) * 100, 2) as pct_accessible -- add small amt to avoid division by zero +from length_totals +order by watershed_group_code, habitat_type desc; + + +--select * from bcfishpass.wcrp_habitat_connectivity_status_vw; diff --git a/db/views/wsg_crossing_summary.sql b/db/v0.5.0/sql/views/wsg_crossing_summary.sql similarity index 99% rename from db/views/wsg_crossing_summary.sql rename to db/v0.5.0/sql/views/wsg_crossing_summary.sql index 6f1e84e5..0be4ff5b 100644 --- a/db/views/wsg_crossing_summary.sql +++ b/db/v0.5.0/sql/views/wsg_crossing_summary.sql @@ -47,7 +47,7 @@ select s.n_barriers_habitat_wct , s.n_potential_habitat_wct , s.n_unknown_habitat_wct -from bcfishpass.wsg_crossing_summary s +from bcfishpass.log_wsg_crossing_summary s inner join bcfishpass.log l on s.model_run_id = l.model_run_id where l.model_run_id = (select model_run_id from bcfishpass.log order by model_run_id desc limit 1) @@ -103,7 +103,7 @@ select s.n_barriers_habitat_wct , s.n_potential_habitat_wct , s.n_unknown_habitat_wct -from bcfishpass.wsg_crossing_summary s +from bcfishpass.log_wsg_crossing_summary s inner join bcfishpass.log l on s.model_run_id = l.model_run_id where l.model_run_id = (select model_run_id from bcfishpass.log order by model_run_id desc offset 1 limit 1) diff --git a/db/views/wsg_linear_summary.sql b/db/v0.5.0/sql/views/wsg_linear_summary.sql similarity index 99% rename from db/views/wsg_linear_summary.sql rename to db/v0.5.0/sql/views/wsg_linear_summary.sql index 4a7eea0b..43a00633 100644 --- a/db/views/wsg_linear_summary.sql +++ b/db/v0.5.0/sql/views/wsg_linear_summary.sql @@ -67,7 +67,7 @@ select distinct on (watershed_group_code) s.length_spawning_rearing_wct , s.length_spawning_rearing_wct_accessible_a , s.length_spawning_rearing_wct_accessible_b -from bcfishpass.wsg_linear_summary s +from bcfishpass.log_wsg_linear_summary s inner join bcfishpass.log l on s.model_run_id = l.model_run_id order by s.watershed_group_code, l.date_completed desc; @@ -142,7 +142,7 @@ select s.length_spawning_rearing_wct , s.length_spawning_rearing_wct_accessible_a , s.length_spawning_rearing_wct_accessible_b -from bcfishpass.wsg_linear_summary s +from bcfishpass.log_wsg_linear_summary s inner join bcfishpass.log l on s.model_run_id = l.model_run_id where l.date_completed = (select date_completed from bcfishpass.log order by date_completed desc offset 1 limit 1) diff --git a/db/views/01_dams_vw.sql b/db/views/01_dams_vw.sql deleted file mode 100644 index 25582b9b..00000000 --- a/db/views/01_dams_vw.sql +++ /dev/null @@ -1,121 +0,0 @@ --- reference CABD dams to FWA stream network -create materialized view bcfishpass.dams_vw as -with cabd as ( - select - d.cabd_id as dam_id, - st_transform(d.geom, 3005) as geom - from cabd.dams d - -- exclude any dam noted in user exclusion table - left outer join bcfishpass.user_cabd_dams_exclusions x on d.cabd_id = x.cabd_id - where x.cabd_id is null -), - -nearest AS -( - select - pt.dam_id, - str.linear_feature_id, - str.blue_line_key, - str.wscode_ltree, - str.localcode_ltree, - str.watershed_group_code, - str.geom, - st_distance(str.geom, pt.geom) as distance_to_stream, - st_interpolatepoint(str.geom, pt.geom) as downstream_route_measure - from cabd pt - cross join lateral ( - select - linear_feature_id, - blue_line_key, - wscode_ltree, - localcode_ltree, - watershed_group_code, - geom - from whse_basemapping.fwa_stream_networks_sp str - where str.localcode_ltree is not null - and not str.wscode_ltree <@ '999' - order by str.geom <-> pt.geom - limit 1 - ) as str - where st_distance(str.geom, pt.geom) <= 65 -), - - -- ensure only one feature returned, and interpolate the geom on the stream -cabd_pts as -( - select distinct on (n.dam_id) - n.dam_id, - n.linear_feature_id, - n.blue_line_key, - n.downstream_route_measure, - n.wscode_ltree, - n.localcode_ltree, - n.distance_to_stream, - n.watershed_group_code, - cabd.dam_name_en, - cabd.height_m, - cabd.owner, - cabd.dam_use, - cabd.operating_status, - cabd.passability_status_code, - - ((st_dump(ST_Force2D(st_locatealong(n.geom, n.downstream_route_measure)))).geom)::geometry(Point, 3005) AS geom - FROM nearest n - inner join cabd.dams cabd on n.dam_id = cabd.cabd_id - order by dam_id, distance_to_stream -), - --- placeholders for major USA dams not present in CABD are stored in user_barriers_anthropogenic -usa as -( - select - (a.user_barrier_anthropogenic_id + 1200000000)::text as dam_id, - s.linear_feature_id, - a.blue_line_key, - a.downstream_route_measure, - s.wscode_ltree, - s.localcode_ltree, - 0 as distance_to_stream, - s.watershed_group_code, - a.barrier_name, - st_force2d((st_dump(st_locatealong(s.geom, a.downstream_route_measure))).geom) as geom - from bcfishpass.user_barriers_anthropogenic a - inner join whse_basemapping.fwa_stream_networks_sp s - on a.blue_line_key = s.blue_line_key - AND ROUND(a.downstream_route_measure::numeric) >= ROUND(s.downstream_route_measure::numeric) - AND ROUND(a.downstream_route_measure::numeric) < ROUND(s.upstream_route_measure::numeric) - where a.barrier_type = 'DAM' -) - -select * from cabd_pts -union all -select - dam_id, - linear_feature_id, - blue_line_key, - downstream_route_measure, - wscode_ltree, - localcode_ltree, - distance_to_stream, - watershed_group_code, - barrier_name as dam_name_en, - null as height_m, - null as owner, - null as dam_use, - null as operating_status, - null as passability_status_code, - geom -from usa; - -create unique index on bcfishpass.dams_vw (dam_id); -create index on bcfishpass.dams_vw using gist (geom); - -create view bcfishpass.dams_not_matched_to_streams as -select - a.cabd_id, - a.dam_name_en -from cabd.dams a -left outer join bcfishpass.dams_vw b -on a.cabd_id::text = b.dam_id -where b.dam_id is null -order by a.cabd_id; \ No newline at end of file diff --git a/db/views/03_observations_vw.sql b/db/views/03_observations_vw.sql deleted file mode 100644 index 899d883f..00000000 --- a/db/views/03_observations_vw.sql +++ /dev/null @@ -1,42 +0,0 @@ --- create view of distinct observations for simpler species based queries -create materialized view bcfishpass.observations_vw as -with unnested as -(select - unnest(observation_ids) as fish_observation_point_id, - unnest(species_codes) as species_code, - unnest(observation_dates) as observation_date, - fish_obsrvtn_event_id, - linear_feature_id, - blue_line_key, - wscode_ltree, - localcode_ltree, - downstream_route_measure, - watershed_group_code, - geom -from bcfishpass.observations) -select - u.fish_observation_point_id, - u.fish_obsrvtn_event_id, - u.linear_feature_id, - u.blue_line_key, - u.wscode_ltree, - u.localcode_ltree, - u.downstream_route_measure, - u.watershed_group_code, - u.species_code, - u.observation_date, - o.activity_code, - o.activity, - o.life_stage_code, - o.life_stage, - o.acat_report_url, - u.geom -from unnested u -inner join whse_fish.fiss_fish_obsrvtn_pnt_sp o -on u.fish_observation_point_id = o.fish_observation_point_id; - -create unique index on bcfishpass.observations_vw (fish_observation_point_id); -create index on bcfishpass.observations_vw using gist (wscode_ltree); -create index on bcfishpass.observations_vw using btree (wscode_ltree); -create index on bcfishpass.observations_vw using gist (localcode_ltree); -create index on bcfishpass.observations_vw using btree (localcode_ltree); \ No newline at end of file diff --git a/db/views/wcrp_habitat_connectivity_status_vw.sql b/db/views/wcrp_habitat_connectivity_status_vw.sql deleted file mode 100644 index a7da8d66..00000000 --- a/db/views/wcrp_habitat_connectivity_status_vw.sql +++ /dev/null @@ -1,154 +0,0 @@ --- summarize spawning/rearing/spawning&rearing habitat lengths per group, by accessibility - -create view bcfishpass.wcrp_habitat_connectivity_status_vw as -with length_totals as -( --- all spawning (ch/co/st/sk/wct) - calculation is simple, just add it up --- --------------- - SELECT - s.watershed_group_code, - 'SPAWNING' as habitat_type, - coalesce(round((SUM(ST_Length(s.geom)) FILTER (WHERE - h.spawning_ch IS TRUE OR - h.spawning_co IS TRUE OR - h.spawning_st IS TRUE OR - h.spawning_sk IS TRUE OR - h.spawning_wct IS TRUE - ) / 1000)::numeric, 2), 0) as total_km, - - -- spawning accessible - coalesce(round((SUM(ST_Length(s.geom)) FILTER (WHERE - (h.spawning_ch IS TRUE OR - h.spawning_co IS TRUE OR - h.spawning_st IS TRUE OR - h.spawning_sk IS TRUE OR - h.spawning_wct IS TRUE) - AND a.barriers_anthropogenic_dnstr IS NULL - ) / 1000)::numeric, 2), 0) as accessible_km - from bcfishpass.streams s - inner join bcfishpass.streams_habitat_linear_vw h using (segmented_stream_id) - inner join bcfishpass.streams_access_vw a using (segmented_stream_id) - where s.watershed_group_code in ('BULK','LNIC','HORS','BOWR','QUES','CARR','ELKR') - group by s.watershed_group_code - - UNION ALL - --- REARING length --- -------------- --- rearing is more complex, add an extra .5 for CO/SK rearing in wetlands/lakes respectively - - SELECT - s.watershed_group_code, - 'REARING' as habitat_type, - round( - ( - ( - coalesce(SUM(ST_Length(geom)) FILTER ( - WHERE - h.rearing_ch IS TRUE OR - h.rearing_st IS TRUE OR - h.rearing_sk IS TRUE OR - h.rearing_co IS TRUE OR - h.rearing_wct IS TRUE - ), 0) + - -- add .5 coho rearing in wetlands - coalesce(SUM(ST_Length(s.geom) * .5) FILTER (WHERE h.rearing_co IS TRUE AND s.edge_type = 1050), 0) + - -- add .5 sockeye rearing in lakes (all of it) - coalesce(SUM(ST_Length(s.geom) * .5) FILTER (WHERE h.spawning_sk IS TRUE), 0) - ) / 1000)::numeric, 2 - ) AS total_km, - - -- rearing accessible - round( - ( - ( - coalesce(SUM(ST_Length(geom)) FILTER ( - WHERE ( - h.rearing_ch IS TRUE OR - h.rearing_co IS TRUE OR - h.rearing_st IS TRUE OR - h.rearing_sk IS TRUE OR - h.rearing_wct IS TRUE) - AND a.barriers_anthropogenic_dnstr IS NULL - ), 0) + - -- add .5 coho rearing in wetlands - coalesce(SUM(ST_Length(geom) * .5) FILTER (WHERE h.rearing_co IS TRUE AND edge_type = 1050 AND barriers_anthropogenic_dnstr IS NULL), 0) + - -- add .5 sockeye rearing in lakes (all of it) - coalesce(SUM(ST_Length(geom) * .5) FILTER (WHERE h.spawning_sk IS TRUE AND barriers_anthropogenic_dnstr IS NULL), 0) - ) / 1000)::numeric, 2 - ) AS accessible_km - from bcfishpass.streams s - inner join bcfishpass.streams_habitat_linear_vw h using (segmented_stream_id) - inner join bcfishpass.streams_access_vw a using (segmented_stream_id) - where s.watershed_group_code in ('BULK','LNIC','HORS','BOWR','QUES','CARR','ELKR') - group by s.watershed_group_code - -UNION ALL - -- spawning or rearing - total km of habitat - SELECT - s.watershed_group_code, - 'ALL' as habitat_type, - round( - ( - ( - coalesce(SUM(ST_Length(s.geom)) FILTER (WHERE - h.spawning_ch IS TRUE OR - h.spawning_co IS TRUE OR - h.spawning_st IS TRUE OR - h.spawning_sk IS TRUE OR - h.spawning_wct IS TRUE OR - h.rearing_ch IS TRUE OR - h.rearing_co IS TRUE OR - h.rearing_st IS TRUE OR - h.rearing_sk IS TRUE OR - h.rearing_wct IS TRUE - ), 0) + - -- add .5 coho rearing in wetlands - coalesce(SUM(ST_Length(s.geom) * .5) FILTER (WHERE h.rearing_co IS TRUE AND s.edge_type = 1050), 0) + - -- add .5 sockeye rearing in lakes (all of it) - coalesce(SUM(ST_Length(geom) * .5) FILTER (WHERE h.spawning_sk IS TRUE), 0) - ) / 1000)::numeric, 2 - ) AS total_km, - - -- total acccessible km - round( - ( - ( - coalesce(SUM(ST_Length(geom)) FILTER (WHERE - ( - h.spawning_ch IS TRUE OR - h.spawning_co IS TRUE OR - h.spawning_st IS TRUE OR - h.spawning_sk IS TRUE OR - h.spawning_wct IS TRUE OR - h.rearing_ch IS TRUE OR - h.rearing_co IS TRUE OR - h.rearing_st IS TRUE OR - h.rearing_sk IS TRUE OR - h.rearing_wct IS TRUE - ) - AND a.barriers_anthropogenic_dnstr IS NULL), 0) + - -- add .5 coho rearing in wetlands - coalesce(SUM(ST_Length(geom) * .5) FILTER (WHERE h.rearing_co IS TRUE AND edge_type = 1050 AND a.barriers_anthropogenic_dnstr IS NULL), 0) + - -- add .5 sockeye rearing in lakes (all of it) - coalesce(SUM(ST_Length(geom) * .5) FILTER (WHERE h.spawning_sk IS TRUE AND a.barriers_anthropogenic_dnstr IS NULL), 0) - ) / 1000)::numeric, 2 - ) AS accessible_km - from bcfishpass.streams s - inner join bcfishpass.streams_habitat_linear_vw h using (segmented_stream_id) - inner join bcfishpass.streams_access_vw a using (segmented_stream_id) - where s.watershed_group_code in ('BULK','LNIC','HORS','BOWR','QUES','CARR','ELKR') - group by s.watershed_group_code -) - -select - watershed_group_code, - habitat_type, - total_km, - accessible_km, - round((accessible_km / (total_km + .0001)) * 100, 2) as pct_accessible -- add small amt to avoid division by zero -from length_totals -order by watershed_group_code, habitat_type desc; - - ---select * from bcfishpass.wcrp_habitat_connectivity_status_vw; diff --git a/docs/02_model_access.md b/docs/02_model_access.md index 069cd594..0839a4e1 100644 --- a/docs/02_model_access.md +++ b/docs/02_model_access.md @@ -8,18 +8,20 @@ The processing steps involved are: ## 1. Collect known natural barriers -Collect known natural barriers: waterfalls 5m in height or more, subsurface flow, and miscellaneous known barriers from expert input: +Collect known natural barriers: -| barrier type | source ----------------|----------- -| waterfalls | [FISS obstacles](https://catalogue.data.gov.bc.ca/dataset/provincial-obstacles-to-fish-passage) -| waterfalls | [FISS obstacles, unpublished](https://www.hillcrestgeo.ca/outgoing/public/whse_fish) -| waterfalls | [FWA obstructions](https://catalogue.data.gov.bc.ca/dataset/freshwater-atlas-obstructions) -| waterfalls | [expert/user identified falls](https://github.com/smnorris/bcfishpass/blob/main/data/user_falls.csv) -| subsurface flow | [FWA streams](https://catalogue.data.gov.bc.ca/dataset/freshwater-atlas-stream-network) +| barrier type | source +|------------------------------ |----------- +| waterfalls (of height >=5m) | [Canadian Aquatic Barriers Database (CABD)](https://aquaticbarriers.ca) +| subsurface flow | [FWA streams](https://catalogue.data.gov.bc.ca/dataset/freshwater-atlas-stream-network) | expert/stakeholder identified barriers | [bcfishpass](https://github.com/smnorris/bcfishpass/blob/main/data/user_barriers_definite.csv) -Natural barriers identified by stakeholders include: waterfalls or cascades not identified in provincial inventories, steep gradients not captured by modelling, channels known to be dry year-round, and other similar features. +Natural barriers identified by stakeholders include: + +- waterfalls or cascades not identified in existing inventories / CABD +- steep gradients not captured by modelling +- channels known to be dry year-round +- other similar features (gradient_barriers)= ## 2. Generate gradient barriers diff --git a/docs/conf.py b/docs/conf.py index 0c9bb62f..5d034bd1 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -18,11 +18,11 @@ # -- Project information ----------------------------------------------------- project = 'bcfishpass' -copyright = '2023, Simon Norris, bcfishpass contributors' +copyright = '2024, bcfishpass contributors' author = 'Simon Norris, bcfishpass contributors' # The full version, including alpha/beta/rc tags -release = 'v0.1.dev2' +release = 'v0.5.0' # -- General configuration --------------------------------------------------- diff --git a/jobs/auto_rank.py b/jobs/auto_rank.py new file mode 100644 index 00000000..eb318d71 --- /dev/null +++ b/jobs/auto_rank.py @@ -0,0 +1,470 @@ +""" +@date: June 2024 +@author: Andrew Pozzuoli + +This python script generates sets for barriers in a wcrp +and ranks them by a combination of immediaate and longterm gain. +This populate the wcrp_ranked_barriers table with whichever wcrp is specified. +""" + +import os +import argparse +import psycopg2 as pg2 +from urllib.parse import urlparse + + +def makeParser(): + p = argparse.ArgumentParser(description="Rank barriers in a WCRP") + p.add_argument( + "wcrp", + choices=[ + "hors", + "bulk", + "lnic", + "elkr", + "bonp", + "eagle", + "bessette", + "bela_atna_necl", + ], + nargs=1, + type=str, + ) + return p + + +def buildCondition(wcrp): + """ + Builds the where clause to pass into the + query as the condition + + :wcrp: the wcrp that is having its barriers ranked + """ + + global species + + species = "ch_cm_co_pk_sk" + + if wcrp == "eagle": + return """ + --Eagle River + FWA_Upstream( + 356364536, + 2901, + 2901, + '100.190442.999098.995997.725308'::ltree, + '100.190442.999098.995997.725308'::ltree, + blue_line_key, + downstream_route_measure, + wscode, + localcode + ); + """ + elif wcrp == "bessette": + return """ + -- Bessette Creek + FWA_Upstream( + 356349528, + 0, + 0, + '100.190442.999098.995997.730848.543292'::ltree, + '100.190442.999098.995997.730848.543292'::ltree, + blue_line_key, + downstream_route_measure, + wscode, + localcode + ); + """ + elif wcrp == "bela_atna_necl": + return """ + ("watershed_group_code" IN ('BELA','ATNA') + OR + FWA_Upstream( -- Subset of NECL watershed + 360884575, + 63, + 63, + '910.275583.004276'::ltree, + '910.275583.004276'::ltree, + blue_line_key, + downstream_route_measure, + wscode, + localcode + ) + OR + FWA_Upstream( -- Subset of NECL watershed + 360215254, + 58, + 58, + '910.275583.004276'::ltree, + '910.275583.004276.020918'::ltree, + blue_line_key, + downstream_route_measure, + wscode, + localcode + ) + ) + """ + elif wcrp == "elkr": + species = "wct" + return f""" + "watershed_group_code" IN ('{wcrp}') + """ + else: + # In all other cases, just the watershed group code + return f""" + "watershed_group_code" IN ('{wcrp}') + """ + + +def runQuery(condition, conn): + """ + Runs the set of queries to rank barriers + + :condition: where clause for the wcrp + """ + + with conn.cursor() as cursor: + + q_make_table = f""" + DROP TABLE IF EXISTS bcfishpass.ranked_barriers; + + select + aggregated_crossings_id + ,crossing_source + ,crossing_feature_type + ,pscis_status + ,crossing_type_code + ,crossing_subtype_code + ,barrier_status + ,pscis_road_name + ,pscis_stream_name + ,pscis_assessment_comment + ,pscis_assessment_date + ,utm_zone + ,utm_easting + ,utm_northing + ,blue_line_key + ,watershed_group_code + ,gnis_stream_name + ,barriers_anthropogenic_dnstr + ,barriers_anthropogenic_dnstr_count + ,barriers_anthropogenic_{species}_upstr + ,barriers_anthropogenic_{species}_upstr_count + ,all_spawning_km + ,all_spawning_belowupstrbarriers_km + ,all_rearing_km + ,all_rearing_belowupstrbarriers_km + ,all_spawningrearing_km + ,all_spawningrearing_belowupstrbarriers_km + ,geom + into bcfishpass.ranked_barriers + from bcfishpass.crossings_wcrp_vw + where barrier_status != 'PASSABLE' + AND all_spawningrearing_belowupstrbarriers_km IS NOT NULL + AND all_spawningrearing_belowupstrbarriers_km != 0 + AND {condition} + AND barriers_{species}_dnstr = ''; + + ALTER TABLE IF EXISTS bcfishpass.ranked_barriers + RENAME COLUMN aggregated_crossings_id TO id; + ALTER TABLE IF EXISTS bcfishpass.ranked_barriers + ADD COLUMN set_id numeric; + ALTER TABLE IF EXISTS bcfishpass.ranked_barriers + ALTER COLUMN id SET NOT NULL; + ALTER TABLE IF EXISTS bcfishpass.ranked_barriers + ADD PRIMARY KEY (id); + + """ + cursor.execute(q_make_table) + + q_group_barriers = f""" + -- Index for speeding up queries + DROP INDEX IF EXISTS rank_idx; + DROP INDEX IF EXISTS rank_idx_set_id; + DROP INDEX IF EXISTS rank_idx_id; + + CREATE INDEX rank_idx ON bcfishpass.ranked_barriers (blue_line_key); + CREATE INDEX rank_idx_set_id ON bcfishpass.ranked_barriers (set_id); + CREATE INDEX rank_idx_id ON bcfishpass.ranked_barriers (id); + + + -- Group all barriers by blue line key + WITH mainstems AS ( + SELECT DISTINCT blue_line_key, row_number() OVER () AS set_id + FROM bcfishpass.ranked_barriers + ) + UPDATE bcfishpass.ranked_barriers a SET set_id = m.set_id FROM mainstems m WHERE m.blue_line_key = a.blue_line_key; + + -- Loop to find set that yields maximum average habitat gain per barrier + -- Find average gain for just barrier 1, then 1+2, then 1+2+3, etc. Whichever gives the most average gain becomes a set. + -- For example, if 1+2 become a set then the iteration continues starting at 3. + DO $$ + DECLARE + continue_loop BOOLEAN := TRUE; + i INT := 1; + grp_offset INT := (SELECT COUNT(*)*10 FROM bcfishpass.ranked_barriers); -- Assign set ids and add this large number. The loop will contiue until all barriers have been assigned a set id higher than this offset + BEGIN + WHILE continue_loop LOOP + + i := i + 1; + + perform id, set_id + from bcfishpass.ranked_barriers + where set_id < grp_offset + LIMIT 1; + + IF NOT FOUND THEN + continue_loop := FALSE; + ELSE + with avgVals as ( -- Cumulative average gain per barrier for ungrouped barriers with the same blue line key + select id, blue_line_key, set_id, barriers_anthropogenic_{species}_upstr_count, all_spawningrearing_belowupstrbarriers_km + ,AVG(all_spawningrearing_belowupstrbarriers_km ) OVER(PARTITION BY set_id ORDER BY barriers_anthropogenic_{species}_upstr_count DESC) as average + ,ROW_NUMBER() OVER(PARTITION BY set_id ORDER BY barriers_anthropogenic_{species}_upstr_count DESC) as row_num + from bcfishpass.ranked_barriers + where set_id < grp_offset + ), + max_grp_gain as ( -- get max gain per barrier from cumulative averages + select + set_id + ,max(average) as best_gain + from avgVals + group by set_id + ), + part as ( -- Partition point in avgVals such that all barriers up to that point should be grouped + select mx.*, av.row_num + from max_grp_gain mx + join avgVals av on mx.best_gain = av.average + ), + new_grps as ( -- Assign set ids to barriers that should be grouped + select distinct av.id + ,CASE + WHEN av.row_num <= part.row_num THEN (av.set_id*grp_offset) + i + ELSE av.set_id + END as new_set_id + from avgVals av + join part on av.set_id = part.set_id + ) + -- Update barrier set ids based on new_grps + update bcfishpass.ranked_barriers + set set_id = new_grps.new_set_id + from new_grps + where bcfishpass.ranked_barriers.id = new_grps.id + AND new_grps.new_set_id > grp_offset; + + END IF; + END LOOP; + END $$; + """ + + cursor.execute(q_group_barriers) + + q_calc_group_gains = """ + ----------------- CALCULATE GROUP GAINS ------------------------- + + alter table bcfishpass.ranked_barriers add column total_hab_gain_set numeric; + alter table bcfishpass.ranked_barriers add column num_barriers_set integer; + alter table bcfishpass.ranked_barriers add column avg_gain_per_barrier numeric; + + with temp as ( + SELECT sum(all_spawningrearing_belowupstrbarriers_km ) AS sum, set_id + from bcfishpass.ranked_barriers + group by set_id + ) + + update bcfishpass.ranked_barriers a SET total_hab_gain_set = t.sum FROM temp t WHERE t.set_id = a.set_id; + update bcfishpass.ranked_barriers SET total_hab_gain_set = all_spawningrearing_belowupstrbarriers_km WHERE set_id IS NULL; + + with temp as ( + SELECT count(*) AS cnt, set_id + from bcfishpass.ranked_barriers + group by set_id + ) + + + update bcfishpass.ranked_barriers a SET num_barriers_set = t.cnt FROM temp t WHERE t.set_id = a.set_id; + update bcfishpass.ranked_barriers SET num_barriers_set = 1 WHERE set_id IS NULL; + + update bcfishpass.ranked_barriers SET avg_gain_per_barrier = total_hab_gain_set / num_barriers_set; + """ + cursor.execute(q_calc_group_gains) + + q_downstr_sets = """ + ---------------GET DOWNSTREAM GROUP IDs---------------------------- + + ALTER TABLE bcfishpass.ranked_barriers ADD dnstr_set_ids varchar[]; + + WITH downstr_barriers AS ( + SELECT rb.id, rb.set_id + ,UNNEST(string_to_array(barriers_anthropogenic_dnstr, ';')) AS barriers_anthropogenic_dnstr + FROM bcfishpass.ranked_barriers rb + ), + downstr_group AS ( + SELECT db_.id, db_.set_id as current_group, db_.barriers_anthropogenic_dnstr + ,rb.set_id + FROM downstr_barriers AS db_ + JOIN bcfishpass.ranked_barriers rb + ON rb.id = db_.barriers_anthropogenic_dnstr + WHERE db_.set_id != rb.set_id + ), + dg_arrays AS ( + SELECT dg.id, ARRAY_AGG(DISTINCT dg.set_id)::varchar[] as dnstr_set_ids + FROM downstr_group dg + GROUP BY dg.id + ) + UPDATE bcfishpass.ranked_barriers + SET dnstr_set_ids = dg_arrays.dnstr_set_ids + FROM dg_arrays + WHERE bcfishpass.ranked_barriers.id = dg_arrays.id; + """ + cursor.execute(q_downstr_sets) + + q_assign_rank = """ + ----------------- ASSIGN RANK ID ------------------------- + + -- Rank based on average gain per barrier in a group + ALTER TABLE bcfishpass.ranked_barriers ADD rank_avg_gain_per_barrier numeric; + + WITH ranks AS + ( + select set_id, avg_gain_per_barrier + ,DENSE_RANK() OVER(order by avg_gain_per_barrier DESC) as rank_id + from bcfishpass.ranked_barriers + ) + update bcfishpass.ranked_barriers + SET rank_avg_gain_per_barrier = ranks.rank_id + FROM ranks + WHERE bcfishpass.ranked_barriers.set_id = ranks.set_id; + + -- Rank based on first sorting the barriers into tiers by number of downstream barriers then by avg gain per barrier within those tiers (immediate gain) + -- Also, barrier sets with less than 500 m of average gain per barrier are ranked below all barriers with more than 500 m of average gain + ALTER TABLE bcfishpass.ranked_barriers + ADD rank_avg_gain_tiered numeric; + + WITH sorted AS ( + SELECT id, set_id, barriers_anthropogenic_dnstr_count, total_hab_gain_set, avg_gain_per_barrier + ,ROW_NUMBER() OVER(ORDER BY barriers_anthropogenic_dnstr_count, avg_gain_per_barrier DESC) as row_num + FROM bcfishpass.ranked_barriers + WHERE avg_gain_per_barrier >= 0.5 + UNION ALL + SELECT id, set_id, barriers_anthropogenic_dnstr_count, total_hab_gain_set, avg_gain_per_barrier + ,(SELECT MAX(row_num) FROM ( + SELECT ROW_NUMBER() OVER(ORDER BY barriers_anthropogenic_dnstr_count, avg_gain_per_barrier DESC) as row_num + FROM bcfishpass.ranked_barriers + WHERE avg_gain_per_barrier >= 0.5 + ) AS subquery) + ROW_NUMBER() OVER(ORDER BY barriers_anthropogenic_dnstr_count, avg_gain_per_barrier DESC) as row_num + FROM bcfishpass.ranked_barriers + WHERE avg_gain_per_barrier < 0.5 + ), + ranks AS ( + SELECT id, barriers_anthropogenic_dnstr_count, avg_gain_per_barrier + ,FIRST_VALUE(row_num) OVER(PARTITION BY set_id ORDER BY barriers_anthropogenic_dnstr_count) as ranks + ,FIRST_VALUE(barriers_anthropogenic_dnstr_count) OVER (PARTITION BY set_id ORDER BY barriers_anthropogenic_dnstr_count) as tier + FROM sorted + ORDER BY set_id, barriers_anthropogenic_dnstr_count, avg_gain_per_barrier DESC + ) + UPDATE bcfishpass.ranked_barriers + SET rank_avg_gain_tiered = ranks.ranks + FROM ranks + WHERE bcfishpass.ranked_barriers.id = ranks.id; + + -- Rank based on total habitat upstream (potential gain) + ALTER TABLE bcfishpass.ranked_barriers + ADD rank_total_upstr_hab numeric; + + WITH sorted AS ( + SELECT id, set_id, barriers_anthropogenic_dnstr_count, all_spawningrearing_km, total_hab_gain_set, avg_gain_per_barrier + ,ROW_NUMBER() OVER(ORDER BY all_spawningrearing_km DESC) as row_num + FROM bcfishpass.ranked_barriers + ), + ranks AS ( + SELECT id, set_id, barriers_anthropogenic_dnstr_count, total_hab_gain_set, avg_gain_per_barrier + ,FIRST_VALUE(row_num) OVER(PARTITION BY set_id ORDER BY row_num) as relative_rank + FROM sorted + ORDER BY set_id, barriers_anthropogenic_dnstr_count, avg_gain_per_barrier DESC + ), + densify AS ( + SELECT id, set_id, total_hab_gain_set, avg_gain_per_barrier + ,DENSE_RANK() OVER(ORDER BY relative_rank) as ranks + FROM ranks + ) + UPDATE bcfishpass.ranked_barriers + SET rank_total_upstr_hab = densify.ranks + FROM densify + WHERE bcfishpass.ranked_barriers.id = densify.id; + + -- Composite Rank of potential and immediate gain + ALTER TABLE bcfishpass.ranked_barriers + ADD rank_combined numeric, + ADD tier_combined varchar; + + WITH ranks AS ( + SELECT id, set_id, all_spawningrearing_km, total_hab_gain_set, avg_gain_per_barrier + ,rank_avg_gain_tiered + ,rank_total_upstr_hab + ,rank_avg_gain_per_barrier + ,DENSE_RANK() OVER(ORDER BY rank_avg_gain_tiered + rank_total_upstr_hab, set_id ASC) as rank_composite + FROM bcfishpass.ranked_barriers + ORDER BY rank_composite ASC + ) + UPDATE bcfishpass.ranked_barriers + SET rank_combined = r.rank_composite + ,tier_combined = case + when r.rank_composite <= 10 then 'A' + when r.rank_composite <= 20 then 'B' + when r.rank_composite <= 30 then 'C' + else 'D' + end + FROM ranks r + WHERE bcfishpass.ranked_barriers.id = r.id; + + ALTER TABLE bcfishpass.ranked_barriers + RENAME COLUMN id to aggregated_crossings_id; + """ + cursor.execute(q_assign_rank) + + q_pop_table = f""" + ---- Insert values into wcrp_ranked_barriers table + + DELETE FROM bcfishpass.wcrp_ranked_barriers + WHERE {condition}; + + INSERT INTO bcfishpass.wcrp_ranked_barriers + SELECT + aggregated_crossings_id + ,set_id + ,total_hab_gain_set + ,num_barriers_set + ,avg_gain_per_barrier + ,dnstr_set_ids + ,rank_avg_gain_per_barrier + ,rank_avg_gain_tiered + ,rank_total_upstr_hab + ,rank_combined + ,tier_combined + FROM bcfishpass.ranked_barriers; + + DROP TABLE bcfishpass.ranked_barriers; + """ + cursor.execute(q_pop_table) + conn.commit() + + +def main(): + parser = makeParser() + args = parser.parse_args() + condition = buildCondition(args.wcrp[0]) + p = urlparse(os.environ["DATABASE_URL"]) + pg_conn_dict = { + 'dbname': p.path[1:], + 'user': p.username, + 'password': p.password, + 'port': p.port, + 'host': p.hostname + } + conn = pg2.connect(**pg_conn_dict) + runQuery(condition, conn) + print("Done!") + + +if __name__ == "__main__": + main() diff --git a/jobs/bcfishpass01_modelled_stream_crossings b/jobs/bcfishpass01_modelled_stream_crossings deleted file mode 100755 index fb4819e9..00000000 --- a/jobs/bcfishpass01_modelled_stream_crossings +++ /dev/null @@ -1,16 +0,0 @@ -#!/bin/bash -set -euxo pipefail - -#------- -# Process/refresh modelled crossings -#------- - -# load latest data fixes to the db -mkdir .make -make --debug=basic .make/data - -# build the crossings -cd model/01_access/modelled_stream_crossings -rm -Rf .make -rm -Rf data -make .make/modelled_stream_crossings --debug=basic \ No newline at end of file diff --git a/jobs/bcfishpass02_prep b/jobs/bcfishpass02_prep deleted file mode 100755 index 46947b69..00000000 --- a/jobs/bcfishpass02_prep +++ /dev/null @@ -1,18 +0,0 @@ -#!/bin/bash -set -euxo pipefail - -#------- -# Prep access model -#------- - -# load latest data fixes to the db -make --debug=basic .make/data - -# already built -cd model/01_access -mkdir -p .make -make -t .make/gradient_barriers -make -t .make/modelled_stream_crossings - -# build everything else -make .make/model_access_prep --debug=basic \ No newline at end of file diff --git a/jobs/db_setup b/jobs/db_setup new file mode 100755 index 00000000..daec4abc --- /dev/null +++ b/jobs/db_setup @@ -0,0 +1,18 @@ +#!/bin/bash +set -euxo pipefail + +PSQL="psql $DATABASE_URL -v ON_ERROR_STOP=1" + +#------- +# set up database schema from scratch (apart from fwa/bcfishobs) +#------- + +# source dataset schemas +cd db/sources; ./migrate.sh; cd .. + +# bcfishpass schema and all migrations +for tag in v* ;do + cd "$tag"; ./migrate.sh; cd .. +done + +cd .. diff --git a/jobs/load_csv b/jobs/load_csv new file mode 100755 index 00000000..ba9c91d6 --- /dev/null +++ b/jobs/load_csv @@ -0,0 +1,18 @@ +#!/bin/bash +set -euxo pipefail + +#------- +# Load local bcfishpass data and parameters +#------- + +PSQL="psql $DATABASE_URL -v ON_ERROR_STOP=1" + +$PSQL -c "truncate bcfishpass.parameters_habitat_method"; +$PSQL -c "truncate bcfishpass.parameters_habitat_thresholds"; +$PSQL -c "\copy bcfishpass.parameters_habitat_method FROM parameters/parameters_habitat_method.csv delimiter ',' csv header"; +$PSQL -c "\copy bcfishpass.parameters_habitat_thresholds FROM parameters/parameters_habitat_thresholds.csv delimiter ',' csv header"; + +for table in data/*.csv; do + $PSQL -c "TRUNCATE bcfishpass.$(basename -- $table .csv)"; + $PSQL -c "\copy bcfishpass.$(basename -- $table .csv) FROM $table delimiter ',' csv header"; +done diff --git a/jobs/load_fwa b/jobs/load_fwa index 2b6c3d73..f5d9a554 100755 --- a/jobs/load_fwa +++ b/jobs/load_fwa @@ -7,6 +7,4 @@ set -euxo pipefail #------- git clone https://github.com/smnorris/fwapg cd fwapg -mkdir -p .make; touch .make/db # db schema/functions/etc are presumed to already exist, just reload data -mkdir -p data make --debug=basic \ No newline at end of file diff --git a/jobs/load_modelled_stream_crossings b/jobs/load_modelled_stream_crossings new file mode 100755 index 00000000..fe29e11d --- /dev/null +++ b/jobs/load_modelled_stream_crossings @@ -0,0 +1,65 @@ +#!/bin/bash +set -euxo pipefail + +#------- +# Download modelled crossings from archive, load to db. Run when: +# - initializing the primary bcfishpass database +# - running any bcfishpass db other than the primary db (only the primary db needs to generate the crossings) +#------- + +PSQL="psql $DATABASE_URL -v ON_ERROR_STOP=1" + +mkdir -p tmp + +curl \ + -o tmp/modelled_stream_crossings.gpkg.zip \ + https://bcfishpass.s3.us-west-2.amazonaws.com/modelled_stream_crossings.gpkg.zip + +unzip -qun tmp/modelled_stream_crossings.gpkg.zip -d tmp + +ogr2ogr \ + -f PostgreSQL \ + "PG:$DATABASE_URL" \ + -overwrite \ + -nln bcfishpass.modelled_stream_crossings_archive \ + tmp/modelled_stream_crossings.gpkg \ + modelled_stream_crossings + +$PSQL -c "truncate bcfishpass.modelled_stream_crossings; + insert into bcfishpass.modelled_stream_crossings ( + modelled_crossing_id, + modelled_crossing_type, + modelled_crossing_type_source, + transport_line_id, + ften_road_section_lines_id, + og_road_segment_permit_id, + og_petrlm_dev_rd_pre06_pub_id, + railway_track_id, + linear_feature_id, + blue_line_key, + downstream_route_measure, + wscode_ltree, + localcode_ltree, + watershed_group_code, + geom +) +select + modelled_crossing_id, + modelled_crossing_type, + string_to_array(modelled_crossing_type_source, '; ') as modelled_crossing_type_source, + transport_line_id, + ften_road_section_lines_id, + og_road_segment_permit_id, + og_petrlm_dev_rd_pre06_pub_id, + railway_track_id, + linear_feature_id, + blue_line_key, + downstream_route_measure, + wscode_ltree::ltree, + localcode_ltree::ltree, + watershed_group_code, + geom +from bcfishpass.modelled_stream_crossings_archive; +drop table bcfishpass.modelled_stream_crossings_archive;" + +rm -rf tmp \ No newline at end of file diff --git a/jobs/load_static b/jobs/load_static index 9dda6637..880492c5 100755 --- a/jobs/load_static +++ b/jobs/load_static @@ -8,15 +8,9 @@ set -euxo pipefail PSQL="psql $DATABASE_URL -v ON_ERROR_STOP=1" WSGS=$($PSQL -AXt -c "SELECT watershed_group_code FROM whse_basemapping.fwa_watershed_groups_poly") -#------- -# Gradient barriers -# load *all* gradient barriers as one time job -# (bcfishpass defaults to processesing just groups included in parameters) -#------- -cd model/01_access/gradient_barriers -$PSQL -c "truncate bcfishpass.gradient_barriers" # just in case job failed -parallel $PSQL -f sql/gradient_barriers_load.sql -v wsg={1} ::: $WSGS - +# ** +# ** note temp dra data cache to be replaced with updated source** +# ** #------- # DRA code tables #------- @@ -30,7 +24,7 @@ ogr2ogr \ -update \ -append \ -nln whse_basemapping.transport_line_type_code \ - /vsizip//vsicurl/ftp://ftp.geobc.gov.bc.ca/sections/outgoing/bmgs/DRA_Public/dgtl_road_atlas.gdb.zip \ + /vsizip//vsicurl/https://bcfishpass.s3.us-west-2.amazonaws.com/dgtl_road_atlas.gdb.zip \ TRANSPORT_LINE_TYPE_CODE ogr2ogr \ -f PostgreSQL \ @@ -38,7 +32,7 @@ ogr2ogr \ -update \ -append \ -nln whse_basemapping.transport_line_surface_code \ - /vsizip//vsicurl/ftp://ftp.geobc.gov.bc.ca/sections/outgoing/bmgs/DRA_Public/dgtl_road_atlas.gdb.zip \ + /vsizip//vsicurl/https://bcfishpass.s3.us-west-2.amazonaws.com/dgtl_road_atlas.gdb.zip \ TRANSPORT_LINE_SURFACE_CODE ogr2ogr \ -f PostgreSQL \ @@ -46,7 +40,7 @@ ogr2ogr \ -update \ -append \ -nln whse_basemapping.transport_line_divided_code \ - /vsizip//vsicurl/ftp://ftp.geobc.gov.bc.ca/sections/outgoing/bmgs/DRA_Public/dgtl_road_atlas.gdb.zip \ + /vsizip//vsicurl/https://bcfishpass.s3.us-west-2.amazonaws.com/dgtl_road_atlas.gdb.zip \ TRANSPORT_LINE_DIVIDED_CODE ogr2ogr \ -f PostgreSQL \ @@ -54,7 +48,7 @@ ogr2ogr \ -update \ -append \ -nln whse_basemapping.transport_line_structure_code \ - /vsizip//vsicurl/ftp://ftp.geobc.gov.bc.ca/sections/outgoing/bmgs/DRA_Public/dgtl_road_atlas.gdb.zip \ + /vsizip//vsicurl/https://bcfishpass.s3.us-west-2.amazonaws.com/dgtl_road_atlas.gdb.zip \ TRANSPORT_LINE_STRUCTURE_CODE #------- diff --git a/jobs/model_gradient_barriers b/jobs/model_gradient_barriers new file mode 100755 index 00000000..95f96a34 --- /dev/null +++ b/jobs/model_gradient_barriers @@ -0,0 +1,11 @@ +#!/bin/bash + +set -euxo pipefail + +PSQL="psql $DATABASE_URL -v ON_ERROR_STOP=1" +WSGS=$($PSQL -AXt -c "select watershed_group_code from whse_basemapping.fwa_watershed_groups_poly") + + +cd model/01_access/gradient_barriers +$PSQL -c "truncate bcfishpass.gradient_barriers" # just in case a previous job failed +parallel $PSQL -f sql/gradient_barriers_load.sql -v wsg={1} ::: $WSGS \ No newline at end of file diff --git a/jobs/model_prep b/jobs/model_prep new file mode 100755 index 00000000..643f3c9f --- /dev/null +++ b/jobs/model_prep @@ -0,0 +1,28 @@ +#!/bin/bash +set -euxo pipefail + +#------- +# Prep access model +#------- + +PSQL="psql $DATABASE_URL -v ON_ERROR_STOP=1" + +# refresh observations/dams/falls +$PSQL -c "refresh materialized view bcfishpass.observations_vw" +$PSQL -c "refresh materialized view bcfishpass.dams_vw" +$PSQL -c "refresh materialized view bcfishpass.falls_vw" +$PSQL -c "refresh materialized view bcfishpass.falls_upstr_anadromous_vw" + +# load pscis +cd model/01_access/pscis; ./pscis.sh; cd .. + +# load crossings table +$PSQL -c "truncate bcfishpass.crossings" +for wsg in $($PSQL -AXt -c "select watershed_group_code from whse_basemapping.fwa_watershed_groups_poly"); +do + set -e ; $PSQL -f sql/load_crossings.sql -v wsg=$wsg ; +done + +# build and index barriers +make clean +make --debug=basic \ No newline at end of file diff --git a/jobs/bcfishpass03_model b/jobs/model_run similarity index 100% rename from jobs/bcfishpass03_model rename to jobs/model_run diff --git a/jobs/release_modelled_stream_crossings b/jobs/model_stream_crossings similarity index 79% rename from jobs/release_modelled_stream_crossings rename to jobs/model_stream_crossings index bd94ee7a..a479f836 100755 --- a/jobs/release_modelled_stream_crossings +++ b/jobs/model_stream_crossings @@ -2,9 +2,13 @@ set -euxo pipefail #------- -# release modelled crossings as zipped .gpkg +# Process/refresh modelled crossings #------- +# build the crossings +cd model/01_access/modelled_stream_crossings; ./modelled_stream_crossings.sh + +# dump to s3 ogr2ogr \ -f GPKG \ modelled_stream_crossings.gpkg.zip \ @@ -30,3 +34,4 @@ ogr2ogr \ from bcfishpass.modelled_stream_crossings" aws s3 cp modelled_stream_crossings.gpkg.zip s3://bcfishpass +rm modelled_stream_crossings.gpkg.zip \ No newline at end of file diff --git a/model/01_access/Makefile b/model/01_access/Makefile index 2006c65a..c7406509 100644 --- a/model/01_access/Makefile +++ b/model/01_access/Makefile @@ -3,19 +3,14 @@ PSQL=psql $(DATABASE_URL) -v ON_ERROR_STOP=1 # point psql to db and stop on errors -# watershed groups to process are defined by bcfishpass.watershed_groups_access -WSGS = $(shell $(PSQL) -AtX -c "SELECT watershed_group_code FROM bcfishpass.watershed_groups_access") +# process all watershed groups listed in parameters_method +WSGS = $(shell $(PSQL) -AtX -c "SELECT watershed_group_code FROM bcfishpass.parameters_habitat_method") # define barrier table targets NATURAL_BARRIERS = .make/barriers_falls \ .make/barriers_subsurfaceflow \ .make/barriers_user_definite \ - .make/barriers_gradient_05 \ - .make/barriers_gradient_10 \ - .make/barriers_gradient_15 \ - .make/barriers_gradient_20 \ - .make/barriers_gradient_25 \ - .make/barriers_gradient_30 + .make/barriers_gradient ANTH_BARRIERS = .make/barriers_anthropogenic \ .make/barriers_dams \ .make/barriers_dams_hydro \ @@ -24,127 +19,11 @@ ANTH_BARRIERS = .make/barriers_anthropogenic \ # access models - combine barriers into tables of per-species scenarios ACCESS_MODELS = $(patsubst sql/%.sql, .make/%, $(wildcard sql/model_access_*.sql)) -all: .make/qa +all: .make/access_prep clean: rm -Rf .make -clean_natural: - rm -f $(NATURAL_BARRIERS) - rm -f .make/model_access* - rm -f .make/qa - -clean_access: - rm -f .make/model_access* - rm -f .make/qa - -clean_anth: - rm -f .make/crossings - rm -f .make/dams - rm -f .make/pscis - rm -f $(ANTH_BARRIERS) - rm -f .make/index_crossings - rm -f .make/model_access_prep - rm -f .make/model_access - rm -f .make/qa - -# for access model processing that is per-watershed group, use just watershed groups noted in parameters file -# (for testing only) -test: - $(PSQL) -c "truncate bcfishpass.watershed_groups_access" - $(PSQL) -c "insert into bcfishpass.watershed_groups_access select watershed_group_code from bcfishpass.parameters_habitat_method" - -# revert to processing all watershed groups -full: - $(PSQL) -c "truncate bcfishpass.watershed_groups_access" - $(PSQL) -c "insert into bcfishpass.watershed_groups_access select watershed_group_code from whse_basemapping.fwa_watershed_groups_poly" - -# ------ -# FALLS -# ------ -.make/falls: falls/falls.sh \ - falls/sql/falls.sql - mkdir -p .make - cd falls; ./falls.sh - touch $@ - -# ready to go but not currently used -.make/cabd_waterfalls: - mkdir -p .make - $(PSQL) -c "truncate cabd.waterfalls" - ogr2ogr -f PostgreSQL \ - "PG:$(DATABASE_URL)" \ - -append \ - -nln cabd.waterfalls \ - "https://cabd-web.azurewebsites.net/cabd-api/features/waterfalls?filter=province_territory_code:eq:bc&filter=use_analysis:eq:true" \ - OGRGeoJSON - $(PSQL) -c "refresh materialized view bcfishpass.falls_vw" - $(PSQL) -c "refresh materialized view bcfishpass.falls_upstr_anadromous_vw" - touch $@ - -# ------ -# GRADIENT BARRIERS -# ------ -# Generate all gradient barriers at 5/10/15/20/25/30% thresholds. -.make/gradient_barriers: - cd gradient_barriers; ./gradient_barriers.sh - touch $@ - -# ------ -# DAMS -# ------ -.make/dams: - mkdir -p .make - $(PSQL) -c "truncate cabd.dams" - ogr2ogr -f PostgreSQL \ - "PG:$(DATABASE_URL)" \ - -append \ - -nln cabd.dams \ - "https://cabd-web.azurewebsites.net/cabd-api/features/dams?filter=province_territory_code:eq:bc&filter=use_analysis:eq:true" \ - OGRGeoJSON - $(PSQL) -c "refresh materialized view bcfishpass.dams_vw" - touch $@ - -# ------ -# MODELLED ROAD-STREAM CROSSINGS -# ------ -# Load modelled crossings from archive posted to s3 (this ensures consistent modelled crossing ids for all users) -.make/modelled_stream_crossings: - mkdir -p .make - cd modelled_stream_crossings; make .make/download_archive - touch $@ - -# ------ -# PSCIS STREAM CROSSINGS -# ------ -.make/pscis: .make/modelled_stream_crossings - mkdir -p .make - cd pscis; ./pscis.sh - touch $@ - - -# ----- -# CROSSINGS -# ----- -# combine modelled crossings, pscis, dams into bcfishpass.crossings -.make/crossings: .make/dams \ - .make/pscis \ - sql/load_crossings.sql - $(PSQL) -c "truncate bcfishpass.crossings" - # load crossings in series, parallel inserts can be confounded by locks - for wsg in $(WSGS) ; do \ - set -e ; $(PSQL) -f sql/load_crossings.sql -v wsg=$$wsg ; \ - done - touch $@ - -# ----- -# OBSERVATIONS -# ----- -# load observations of all spp of interest from bcfishobs to bcfishpass.observations -.make/observations: sql/load_observations.sql - $(PSQL) -f $^ - touch $@ - # ----- # LOAD BARRIER TYPE TABLES # ----- @@ -152,14 +31,8 @@ full: $(NATURAL_BARRIERS): sql/barriers_falls.sql \ sql/barriers_subsurfaceflow.sql \ sql/barriers_user_definite.sql \ - sql/barriers_gradient_05.sql \ - sql/barriers_gradient_10.sql \ - sql/barriers_gradient_15.sql \ - sql/barriers_gradient_20.sql \ - sql/barriers_gradient_25.sql \ - sql/barriers_gradient_30.sql \ - .make/falls \ - .make/gradient_barriers + sql/barriers_gradient.sql + mkdir -p .make $(eval BARRIERTYPE=$(subst .make/barriers_,,$@)) # create the table if it does not exist echo "select bcfishpass.create_barrier_table(:'barriertype')" | \ @@ -171,11 +44,11 @@ $(NATURAL_BARRIERS): sql/barriers_falls.sql \ touch $@ # Create standardized barrier tables for anthropogenic barriers -$(ANTH_BARRIERS): .make/crossings \ - sql/barriers_anthropogenic.sql \ +$(ANTH_BARRIERS): sql/barriers_anthropogenic.sql \ sql/barriers_dams.sql \ sql/barriers_dams_hydro.sql \ sql/barriers_pscis.sql + mkdir -p .make $(eval BARRIERTYPE=$(subst .make/barriers_,,$@)) # create the table if it does not exist echo "select bcfishpass.create_barrier_table(:'barriertype')" | \ @@ -193,9 +66,9 @@ $(ANTH_BARRIERS): .make/crossings \ # group being modelled, then retain only the records with no other records downstream (ie, only # the lowest in the system) # Note that observations are a requirement because they can be used to cancel natural barriers +# (but this is presumed to already be present) $(ACCESS_MODELS): .make/model_access_%: sql/model_access_%.sql \ - $(NATURAL_BARRIERS) \ - .make/observations + $(NATURAL_BARRIERS) $(eval BARRIERTYPE=$(subst .make/model_access_,,$@)) # create table if it does not exist echo "select bcfishpass.create_barrier_table(:'barriertype')" | $(PSQL) -v barriertype=$(BARRIERTYPE) @@ -291,17 +164,6 @@ $(ACCESS_MODELS): .make/model_access_%: sql/model_access_%.sql \ touch $@ -# intermediate target for proessing all of above, but not the model_access script -.make/model_access_prep: $(ACCESS_MODELS) .make/index_crossings - touch $@ - -# run the access model -# (breaking streams and recording what is upstream/downstream of a given stream segment) -.make/model_access: .make/model_access_prep - ./model_access_1.sh - ./model_access_2.sh +# target for all +.make/access_prep: $(ACCESS_MODELS) .make/index_crossings touch $@ - -.make/qa: .make/model_access - cd qa; ./qa.sh - touch $@ \ No newline at end of file diff --git a/model/01_access/falls/README.md b/model/01_access/falls/README.md deleted file mode 100644 index 0582033e..00000000 --- a/model/01_access/falls/README.md +++ /dev/null @@ -1,78 +0,0 @@ -# Falls - -Download waterfall data from various sources, match to FWA stream network, identify barriers. - -## Sources - -- FISS obstacles, [BC Data Catalogue](https://catalogue.data.gov.bc.ca/dataset/provincial-obstacles-to-fish-passage) -- FISS obstacles, [unpublished](https://www.hillcrestgeo.ca/outgoing/public/whse_fish) -- FWA obstructions, [BC Data Catalogue](https://catalogue.data.gov.bc.ca/dataset/freshwater-atlas-obstructions) -- Additional user compiled falls, [falls_other.csv](/data/falls/falls_other.csv) -- User modification of fall barrier status, [falls_barrier_ind.csv](/data/falls/falls_barrier_ind.csv) - - -## Barrier status logic - -A fall is automatically identified as a barrier if one of these conditions is met: - -- data source is a FISS obstacles / FISS obstacles unpublished *AND* height >= 5m -- data source is FWA obstructions (this data source does not include a height) -- data source is `falls_other.csv` and `barrier_ind = True` - -See below for manually modifying the barrier status of individual falls with `falls_barrier_ind.csv` - - -## Processing - -1. Add records/edit `falls_other.csv` as required -2. Download input data, match to FWA streams, combine into a single falls table: - - ./falls.sh - -## Output table - -`bcfishpass.falls` contains all known falls features (only the subset of falls with `barrier_ind IS TRUE` are used for connectivity modelling). -Note that there is no ID used as a primary key. The `blue_line_key`/`downstream_route_measure` columns are used as the primary key instead (the source FISS obstacles table does not include a stable primary key so tracking these falls is easiest via their position on the network). -``` - Table "bcfishpass.falls" - Column | Type | Collation | Nullable | Default ---------------------------+----------------------+-----------+----------+--------- - source | text | | | - height | double precision | | | - barrier_ind | boolean | | | - reviewer | text | | | - notes | text | | | - distance_to_stream | double precision | | | - linear_feature_id | bigint | | | - blue_line_key | integer | | not null | - downstream_route_measure | double precision | | not null | - wscode_ltree | ltree | | | - localcode_ltree | ltree | | | - watershed_group_code | text | | | - geom | geometry(Point,3005) | | | -Indexes: - "falls_pkey" PRIMARY KEY, btree (blue_line_key, downstream_route_measure) - "falls_blue_line_key_idx" btree (blue_line_key) - "falls_geom_idx" gist (geom) - "falls_linear_feature_id_idx" btree (linear_feature_id) - "falls_localcode_ltree_idx" gist (localcode_ltree) - "falls_localcode_ltree_idx1" btree (localcode_ltree) - "falls_wscode_ltree_idx" gist (wscode_ltree) - "falls_wscode_ltree_idx1" btree (wscode_ltree) -``` - -## Edit barrrier status - -When the barrier status of a falls neeeds to be updated, add to `falls_barrier_ind.csv` as required. -A falls is identified in this table by its position on the network (`blue_line_key`/`downstream_route_measure` combination). -For example, to remove an invalid falls, add a record like this: - -| blue_line_key | downstream_route_measure | barrier_ind | watershed_group_code | reviewer | notes -| -------- | -------- | -------- | -------- | -------- | -------- | -| 356252974 | 0 | F | HORS | NJO | Does not exist | - -Or, if a falls that defaults to passable has been found to be a barrier: - -| blue_line_key | downstream_route_measure | barrier_ind | watershed_group_code | reviewer | notes -| -------- | -------- | -------- | -------- | -------- | -------- | -|123456789|525|T|AGRP|SN|Barrier confirmed by local source A.Fish.| \ No newline at end of file diff --git a/model/01_access/falls/falls.sh b/model/01_access/falls/falls.sh deleted file mode 100755 index ca6c28e6..00000000 --- a/model/01_access/falls/falls.sh +++ /dev/null @@ -1,55 +0,0 @@ -#!/bin/bash - -# ---------- -# Script to: -# - load falls from various sources -# - match pts to streams -# - combine falls into bcfishpass.falls -# - identify barriers / non-barriers -# ---------- - -set -exo pipefail - -PSQL_CMD="psql $DATABASE_URL -v ON_ERROR_STOP=1" - -# obstacles data refreshed separately (and we no longer archive, as it will soon be replace by cabd) - -# Archive existing FISS obstacles data just in case we want it later, -# the unique IDs aren't stable. Note that the archive is a full copy and -# will be made every time this is run on a different day -# if $PSQL_CMD -c "SELECT to_regclass('whse_fish.fiss_obstacles_pnt_sp')" | grep -q 'whse_fish.fiss_obstacles_pnt_sp'; then -# DOWNLOAD_DATE=$($PSQL_CMD -tc \ -# "SELECT replace(DATE(latest_download)::text, '-', '') \ -# FROM bcdata.log \ -# WHERE table_name = 'whse_fish.fiss_obstacles_pnt_sp'" \ -# | sed -e 's/^[[:space:]]*//') -# # just drop existing archive for given date if it aleady exists and re-archive -# $PSQL_CMD -c "DROP TABLE IF EXISTS whse_fish.fiss_obstacles_pnt_sp_$DOWNLOAD_DATE" -# $PSQL_CMD -c "CREATE TABLE whse_fish.fiss_obstacles_pnt_sp_$DOWNLOAD_DATE AS SELECT * FROM whse_fish.fiss_obstacles_pnt_sp" -# fi -# load the latest obstacles data -# bcdata bc2pg WHSE_FISH.FISS_OBSTACLES_PNT_SP - -# load additional (unpublished) obstacle data (provided by the Province, 2014) -mkdir -p data -curl -o data/whse_fish.fiss_obstacles_unpublished.csv.zip \ - https://hillcrestgeo.ca/outgoing/public/whse_fish/whse_fish.fiss_obstacles_unpublished.csv.zip -unzip -qjun -d data data/whse_fish.fiss_obstacles_unpublished.csv.zip - -$PSQL_CMD -c "DROP TABLE IF EXISTS bcfishpass.fiss_obstacles_unpublished;" -$PSQL_CMD -c "CREATE TABLE bcfishpass.fiss_obstacles_unpublished - (id integer , - featur_typ_code character varying , - point_id_field numeric , - utm_zone numeric , - utm_easting numeric , - utm_northing numeric , - height numeric , - length numeric , - strsrvy_rchsrvy_id numeric , - sitesrvy_id numeric , - comments character varying)" -$PSQL_CMD -c "\copy bcfishpass.fiss_obstacles_unpublished FROM 'data/fiss_obstacles_unpublished.csv' delimiter ',' csv header" - -# match falls to streams, combine sources -$PSQL_CMD -f sql/falls.sql diff --git a/model/01_access/falls/sql/falls.sql b/model/01_access/falls/sql/falls.sql deleted file mode 100644 index 3745f1ee..00000000 --- a/model/01_access/falls/sql/falls.sql +++ /dev/null @@ -1,484 +0,0 @@ -DROP TABLE IF EXISTS bcfishpass.fiss_obstacles_falls; - --- --------------------------------------------- --- Create fiss_obstacles_falls table from fiss obsatacles (from bcgw and unpublished), --- combining data to get only one feature at a given point (using max height at that point) --- --------------------------------------------- -CREATE TABLE bcfishpass.fiss_obstacles_falls -( - fiss_obstacles_falls_id serial primary key , - height double precision , - new_watershed_code text , - watershed_group_code text , - geom geometry(Point, 3005) -); - -WITH distinct_unpublished AS -( - SELECT DISTINCT -- source may include duplicates - featur_typ_code as feature_type_code, - -- tweak codes to match those in existing table - CASE - WHEN featur_typ_code = 'F' THEN 'Falls' - WHEN featur_typ_code = 'D' THEN 'Dam' - END AS obstacle_name, - point_id_field, - utm_zone, - utm_easting, - utm_northing, - height, - length, - sitesrvy_id, - comments, - ST_Transform(ST_PointFromText('POINT (' || utm_easting || ' ' || utm_northing || ')', 32600 + utm_zone::int), 3005) as geom - FROM bcfishpass.fiss_obstacles_unpublished -), - -singlepart_unpublished AS -( - SELECT - feature_type_code, - obstacle_name, - height, - length, - utm_zone, - utm_easting, - utm_northing, - (ST_Dump(geom)).geom - FROM distinct_unpublished - -- not all data has geom (lots of null UTMs), filter those out - WHERE geom is not null -), - -height_cleaned AS -( - SELECT - CASE - -- remove garbage from height values - WHEN height = 999 THEN NULL - WHEN height = 9999 THEN NULL - WHEN height = -1000 THEN NULL - ELSE height - END as height, - new_watershed_code, - geom - FROM whse_fish.fiss_obstacles_pnt_sp o - WHERE o.obstacle_name = 'Falls' - UNION ALL - SELECT - CASE - -- remove garbage from height values - WHEN height = 999 THEN NULL - WHEN height = 9999 THEN NULL - WHEN height = -1000 THEN NULL - ELSE height - END as height, - NULL as new_watershed_code, - geom - FROM singlepart_unpublished o - WHERE o.obstacle_name = 'Falls' -), - -agg AS -( - SELECT - unnest(array_agg(height)) as height, - new_watershed_code, - geom - FROM height_cleaned - GROUP BY new_watershed_code, geom -) - -INSERT INTO bcfishpass.fiss_obstacles_falls -(height, new_watershed_code, watershed_group_code, geom) -SELECT - max(agg.height) as height, - agg.new_watershed_code, - g.watershed_group_code, - (st_dump(agg.geom)).geom -FROM agg -INNER JOin whse_basemapping.fwa_watershed_groups_poly g -ON ST_Intersects(agg.geom, g.geom) -GROUP BY g.watershed_group_code, agg.new_watershed_code, agg.geom; - - --- --------------------------------------------- --- Create table holding all falls, referenced to stream network --- --------------------------------------------- -DROP TABLE IF EXISTS bcfishpass.falls; - -CREATE TABLE bcfishpass.falls - ( - -- source falls data does not have pk, generate a (big) integer pk from blkey and measure - falls_id bigint - GENERATED ALWAYS AS ((((blue_line_key::bigint + 1) - 354087611) * 10000000) + round(downstream_route_measure::bigint)) STORED PRIMARY KEY, - fiss_obstacles_falls_id integer , -- temporary column, for tracking what gets inserted - source text , - height double precision , - barrier_ind boolean , - falls_name text , - reviewer_name text , - notes text , - distance_to_stream double precision , - linear_feature_id bigint , - blue_line_key integer , - downstream_route_measure double precision , - wscode_ltree ltree , - localcode_ltree ltree , - watershed_group_code text, - geom geometry(Point, 3005), - UNIQUE (blue_line_key, downstream_route_measure) -); - - --- ---------------------------------------------- --- Reference FISS falls to the FWA stream network, matching to closest stream --- that has matching watershed code (via 50k/20k lookup) --- ---------------------------------------------- - -WITH candidates AS -- first, find up to 10 streams within 200m of the falls - ( SELECT - pt.fiss_obstacles_falls_id, - nn.linear_feature_id, - nn.wscode_ltree, - nn.localcode_ltree, - nn.blue_line_key, - nn.waterbody_key, - nn.length_metre, - nn.downstream_route_measure, - nn.upstream_route_measure, - nn.distance_to_stream, - pt.height, - pt.watershed_group_code, - ST_LineMerge(nn.geom) AS geom - FROM bcfishpass.fiss_obstacles_falls as pt - CROSS JOIN LATERAL - (SELECT - str.linear_feature_id, - str.wscode_ltree, - str.localcode_ltree, - str.blue_line_key, - str.waterbody_key, - str.length_metre, - str.downstream_route_measure, - str.upstream_route_measure, - str.geom, - ST_Distance(str.geom, pt.geom) as distance_to_stream - FROM whse_basemapping.fwa_stream_networks_sp AS str - WHERE str.localcode_ltree IS NOT NULL - AND NOT str.wscode_ltree <@ '999' - ORDER BY str.geom <-> pt.geom - LIMIT 10) as nn - WHERE nn.distance_to_stream < 200 -), - --- find just the closest point for distinct blue_line_keys - --- we don't want to match to all individual stream segments -bluelines AS -(SELECT * FROM - (SELECT - fiss_obstacles_falls_id, - blue_line_key, - min(distance_to_stream) AS distance_to_stream - FROM candidates - GROUP BY fiss_obstacles_falls_id, blue_line_key) as f - ORDER BY distance_to_stream asc -), - --- from the selected blue lines, generate downstream_route_measure --- and join to the 20k-50k lookup table -events AS -( - SELECT - bluelines.fiss_obstacles_falls_id, - candidates.linear_feature_id, - (REPLACE(REPLACE(lut.fwa_watershed_code_20k, '-000000', ''), '-', '.')::ltree) as wscode_ltree_lookup, - candidates.wscode_ltree, - candidates.localcode_ltree, - candidates.waterbody_key, - bluelines.blue_line_key, - -- reference the point to the stream, making output measure an integer - -- (ensuring point measure is between stream's downtream measure and upstream measure) - CEIL(GREATEST(candidates.downstream_route_measure, FLOOR(LEAST(candidates.upstream_route_measure, - (ST_LineLocatePoint(candidates.geom, ST_ClosestPoint(candidates.geom, pts.geom)) * candidates.length_metre) + candidates.downstream_route_measure - )))) as downstream_route_measure, - candidates.distance_to_stream, - candidates.height, - candidates.watershed_group_code - FROM bluelines - INNER JOIN candidates ON bluelines.fiss_obstacles_falls_id = candidates.fiss_obstacles_falls_id - AND bluelines.blue_line_key = candidates.blue_line_key - AND bluelines.distance_to_stream = candidates.distance_to_stream - INNER JOIN bcfishpass.fiss_obstacles_falls pts - ON bluelines.fiss_obstacles_falls_id = pts.fiss_obstacles_falls_id - LEFT OUTER JOIN whse_basemapping.fwa_streams_20k_50k lut - ON REPLACE(pts.new_watershed_code,'-','') = lut.watershed_code_50k - ORDER BY bluelines.fiss_obstacles_falls_id, candidates.distance_to_stream asc -), - --- grab closest with a matched code -matched AS -( -SELECT DISTINCT ON (fiss_obstacles_falls_id) * -FROM events -WHERE wscode_ltree_lookup = wscode_ltree -ORDER BY fiss_obstacles_falls_id, distance_to_stream asc -) - -INSERT INTO bcfishpass.falls - ( - fiss_obstacles_falls_id, - source, - height, - distance_to_stream, - linear_feature_id, - blue_line_key, - downstream_route_measure, - wscode_ltree, - localcode_ltree, - watershed_group_code, - geom) -SELECT - fiss_obstacles_falls_id, - 'FISS' as source, - e.height, - e.distance_to_stream, - e.linear_feature_id, - e.blue_line_key, - e.downstream_route_measure, - e.wscode_ltree, - e.localcode_ltree, - e.watershed_group_code, - (ST_Dump(ST_Force2D(ST_locateAlong(s.geom, e.downstream_route_measure)))).geom as geom -FROM matched e -INNER JOIN whse_basemapping.fwa_stream_networks_sp s -ON e.linear_feature_id = s.linear_feature_id -ON CONFLICT DO NOTHING; - - --- ---------------------------------------------- --- Above only inserts records with matching watershed codes. --- Now insert records that are not inserted above but are still very close to a stream (50m) --- ---------------------------------------------- -WITH pts AS -( - SELECT * - FROM bcfishpass.fiss_obstacles_falls - WHERE fiss_obstacles_falls_id NOT IN ( - SELECT fiss_obstacles_falls_id from bcfishpass.falls - WHERE fiss_obstacles_falls_id IS NOT NULL - ) -), - -candidates AS -- now find up to 10 streams within 50m of the falls - ( SELECT - pt.fiss_obstacles_falls_id, - nn.linear_feature_id, - nn.wscode_ltree, - nn.localcode_ltree, - nn.blue_line_key, - nn.waterbody_key, - nn.length_metre, - nn.downstream_route_measure, - nn.upstream_route_measure, - nn.distance_to_stream, - pt.height, - pt.watershed_group_code, - ST_LineMerge(nn.geom) AS geom - FROM pts pt - CROSS JOIN LATERAL - (SELECT - str.linear_feature_id, - str.wscode_ltree, - str.localcode_ltree, - str.blue_line_key, - str.waterbody_key, - str.length_metre, - str.downstream_route_measure, - str.upstream_route_measure, - str.geom, - ST_Distance(str.geom, pt.geom) as distance_to_stream - FROM whse_basemapping.fwa_stream_networks_sp AS str - WHERE str.localcode_ltree IS NOT NULL - AND NOT str.wscode_ltree <@ '999' - ORDER BY str.geom <-> pt.geom - LIMIT 10) as nn - WHERE nn.distance_to_stream < 50 -), - --- find just the closest point for distinct blue_line_keys - --- we don't want to match to all individual stream segments -bluelines AS -(SELECT * FROM - (SELECT - fiss_obstacles_falls_id, - blue_line_key, - min(distance_to_stream) AS distance_to_stream - FROM candidates - GROUP BY fiss_obstacles_falls_id, blue_line_key) as f - ORDER BY distance_to_stream asc -), - --- from the selected blue lines, generate downstream_route_measure --- and join to the 20k 50k lookup table -events AS -( - SELECT DISTINCT ON (bluelines.fiss_obstacles_falls_id) - bluelines.fiss_obstacles_falls_id, - candidates.linear_feature_id, - candidates.wscode_ltree, - candidates.localcode_ltree, - candidates.waterbody_key, - bluelines.blue_line_key, - -- reference the point to the stream, making output measure an integer - -- (ensuring point measure is between stream's downtream measure and upstream measure) - CEIL(GREATEST(candidates.downstream_route_measure, FLOOR(LEAST(candidates.upstream_route_measure, - (ST_LineLocatePoint(candidates.geom, ST_ClosestPoint(candidates.geom, pts.geom)) * candidates.length_metre) + candidates.downstream_route_measure - )))) as downstream_route_measure, - candidates.distance_to_stream, - candidates.height, - candidates.watershed_group_code - FROM bluelines - INNER JOIN candidates ON bluelines.fiss_obstacles_falls_id = candidates.fiss_obstacles_falls_id - AND bluelines.blue_line_key = candidates.blue_line_key - AND bluelines.distance_to_stream = candidates.distance_to_stream - INNER JOIN bcfishpass.fiss_obstacles_falls pts - ON bluelines.fiss_obstacles_falls_id = pts.fiss_obstacles_falls_id - ORDER BY bluelines.fiss_obstacles_falls_id, candidates.distance_to_stream asc -) - -INSERT INTO bcfishpass.falls - ( - source, - height, - distance_to_stream, - linear_feature_id, - blue_line_key, - downstream_route_measure, - wscode_ltree, - localcode_ltree, - watershed_group_code, - geom) -SELECT - 'FISS' as source, - e.height, - e.distance_to_stream, - e.linear_feature_id, - e.blue_line_key, - e.downstream_route_measure, - e.wscode_ltree, - e.localcode_ltree, - e.watershed_group_code, - (ST_Dump(ST_Force2D(ST_locateAlong(s.geom, e.downstream_route_measure)))).geom as geom -FROM events e -INNER JOIN whse_basemapping.fwa_stream_networks_sp s -ON e.linear_feature_id = s.linear_feature_id -ON CONFLICT DO NOTHING; - - -ALTER TABLE bcfishpass.falls DROP COLUMN fiss_obstacles_falls_id; -- dump the meaningless column - --- ---------------------------------------------- --- insert falls from FWA obstructions --- ---------------------------------------------- -INSERT INTO bcfishpass.falls -(source, - linear_feature_id, - blue_line_key, - downstream_route_measure, - wscode_ltree, - localcode_ltree, - watershed_group_code, - geom) -SELECT - 'FWA' as source, - linear_feature_id, - blue_line_key, - route_measure, - wscode_ltree, - localcode_ltree, - watershed_group_code, - (ST_Dump(geom)).geom as geom -FROM whse_basemapping.fwa_obstructions_sp -WHERE obstruction_type = 'Falls' -ON CONFLICT DO NOTHING; - --- --------------------------------------------- --- Load manually added falls --- --------------------------------------------- -INSERT INTO bcfishpass.falls - ( - source, - height, - barrier_ind, - falls_name, - reviewer_name, - notes, - linear_feature_id, - blue_line_key, - downstream_route_measure, - wscode_ltree, - localcode_ltree, - watershed_group_code, - geom) -SELECT - p.source, - p.height, - p.barrier_ind, - p.falls_name, - p.reviewer_name, - p.notes, - s.linear_feature_id, - p.blue_line_key, - p.downstream_route_measure, - s.wscode_ltree, - s.localcode_ltree, - s.watershed_group_code, - (ST_Dump(ST_Force2D(ST_locateAlong(s.geom, p.downstream_route_measure)))).geom as geom -FROM bcfishpass.user_falls p -INNER JOIN whse_basemapping.fwa_stream_networks_sp s -ON p.blue_line_key = s.blue_line_key AND -p.downstream_route_measure > s.downstream_route_measure - .001 AND -p.downstream_route_measure + .001 < s.upstream_route_measure -ON CONFLICT DO NOTHING; - --- index -CREATE INDEX ON bcfishpass.falls (linear_feature_id); -CREATE INDEX ON bcfishpass.falls (blue_line_key); -CREATE INDEX ON bcfishpass.falls USING GIST (wscode_ltree); -CREATE INDEX ON bcfishpass.falls USING BTREE (wscode_ltree); -CREATE INDEX ON bcfishpass.falls USING GIST (localcode_ltree); -CREATE INDEX ON bcfishpass.falls USING BTREE (localcode_ltree); -CREATE INDEX ON bcfishpass.falls USING GIST (geom); - --- drop load/intermediate tables (but keeping the barrier_ind table for QA) -DROP TABLE bcfishpass.fiss_obstacles_falls; -DROP TABLE bcfishpass.fiss_obstacles_unpublished; - - --- -------------------------- --- set default barrier status for fiss/fwa records --- -------------------------- - --- fiss falls < 5m or NULL default to passable -UPDATE bcfishpass.falls -SET barrier_ind = False -WHERE source = 'FISS' AND (height < 5 or height is null); - --- fiss falls >= 5m are barriers -UPDATE bcfishpass.falls -SET barrier_ind = True -WHERE source = 'FISS' AND height >= 5; - --- all fwa features default to barriers -UPDATE bcfishpass.falls -SET barrier_ind = True -WHERE source = 'FWA'; - --- -------------------------- --- finalize barrier status from the user control table --- -------------------------- -UPDATE bcfishpass.falls a -SET barrier_ind = b.barrier_ind -FROM bcfishpass.user_barriers_definite_control b -WHERE a.blue_line_key = b.blue_line_key and abs(a.downstream_route_measure - b.downstream_route_measure) < 1; \ No newline at end of file diff --git a/model/01_access/gradient_barriers/sql/gradient_barriers_load.sql b/model/01_access/gradient_barriers/sql/gradient_barriers_load.sql index a17137b1..e25f2af7 100644 --- a/model/01_access/gradient_barriers/sql/gradient_barriers_load.sql +++ b/model/01_access/gradient_barriers/sql/gradient_barriers_load.sql @@ -3,31 +3,35 @@ WITH grade100m AS SELECT sv.blue_line_key, ROUND(sv.downstream_route_measure::numeric, 2) as downstream_route_measure, + -- elevation at vertex (from subquery below) ROUND(sv.elevation::numeric, 2) as elevation_a, + -- interpolation of elevation 100m upstream of the vertex ROUND((ST_Z((ST_Dump((ST_LocateAlong(s2.geom, sv.downstream_route_measure + 100)))).geom))::numeric, 2) as elevation_b, + -- gradient between the two points on the stream ROUND(((ST_Z((ST_Dump((ST_LocateAlong(s2.geom, sv.downstream_route_measure + 100)))).geom) - sv.elevation) / 100)::numeric, 4) as gradient - FROM - (SELECT - linear_feature_id, - blue_line_key, - edge_type, - ((ST_LineLocatePoint(geom, ST_PointN(geom, generate_series(1, ST_NPoints(geom) - 1))) * length_metre) + downstream_route_measure) as downstream_route_measure, - ST_Z(ST_PointN(geom, generate_series(1, ST_NPoints(geom) - 1))) AS elevation - FROM whse_basemapping.fwa_stream_networks_sp s - WHERE watershed_group_code = :'wsg' - AND blue_line_key = watershed_key - AND edge_type != 6010 - ORDER BY blue_line_key, downstream_route_measure + FROM ( + -- get elevation of every vertex on the given stream + SELECT + linear_feature_id, + blue_line_key, + edge_type, + ((ST_LineLocatePoint(geom, ST_PointN(geom, generate_series(1, ST_NPoints(geom) - 1))) * length_metre) + downstream_route_measure) as downstream_route_measure, + ST_Z(ST_PointN(geom, generate_series(1, ST_NPoints(geom) - 1))) AS elevation + FROM whse_basemapping.fwa_stream_networks_sp s + WHERE watershed_group_code = :'wsg' + AND blue_line_key = watershed_key + AND edge_type != 6010 + ORDER BY blue_line_key, downstream_route_measure ) as sv INNER JOIN whse_basemapping.fwa_stream_networks_sp s2 ON sv.blue_line_key = s2.blue_line_key AND - sv.downstream_route_measure + 100 >= s2.downstream_route_measure AND -- find stream segment 100m up + sv.downstream_route_measure + 100 >= s2.downstream_route_measure AND -- find stream segment 100m upstream of given segment sv.downstream_route_measure + 100 < s2.upstream_route_measure WHERE sv.edge_type IN (1000,1050,1100,1150,1250,1350,1410,2000,2300) AND s2.edge_type != 6010 ), --- note the slope classes +-- note the slope classes of the vertices gradeclass AS ( SELECT @@ -52,9 +56,12 @@ gradeclass AS ORDER BY blue_line_key, downstream_route_measure ), --- Group the continuous same-slope class segments (islands) together (thank you Erwin B) +-- Although we are calculating slope at each vertex, we don't need to retain all this information - +-- for a given continuous set of vertices with the same slope (say 15@2400m 15@2420m 15@2500m 15@2540m), +-- only the slope at the most downstream point/minimum measure (2400m) needs to be retained. +-- Grouping continuous classes (islands) together is done via count/lag window functions (thank you Erwin B) -- https://dba.stackexchange.com/questions/166374/grouping-or-window/166397#166397 --- NOTE - this could also be done as a recursive query, potentially much easier to read +-- NOTE - this could also be done as a recursive query, (potentially easier to read) islands AS ( diff --git a/model/01_access/model_access_1.sh b/model/01_access/model_access_1.sh index 36ba907a..2f99a256 100755 --- a/model/01_access/model_access_1.sh +++ b/model/01_access/model_access_1.sh @@ -24,7 +24,7 @@ $PSQL -c "VACUUM ANALYZE bcfishpass.streams" # break at observations $PARALLEL \ "echo \"SELECT bcfishpass.break_streams(:'point_table', :'wsg');\" | \ - $PSQL -v wsg={1} -v point_table=observations" ::: $WSGS + $PSQL -v wsg={1} -v point_table=observations_vw" ::: $WSGS # break at crossings $PARALLEL \ diff --git a/model/01_access/modelled_stream_crossings/Makefile b/model/01_access/modelled_stream_crossings/Makefile deleted file mode 100644 index 700a59b0..00000000 --- a/model/01_access/modelled_stream_crossings/Makefile +++ /dev/null @@ -1,43 +0,0 @@ -PSQL = psql $(DATABASE_URL) -v ON_ERROR_STOP=1 -WSGS = $(shell $(PSQL) -AtX -c "SELECT watershed_group_code FROM whse_basemapping.fwa_watershed_groups_poly") - -# download latest from s3 -.make/download_archive: - mkdir -p .make - mkdir -p data - curl \ - -o data/modelled_stream_crossings.gpkg.zip \ - https://bcfishpass.s3.us-west-2.amazonaws.com/modelled_stream_crossings.gpkg.zip - unzip -qun data/modelled_stream_crossings.gpkg.zip -d data - ogr2ogr \ - -f PostgreSQL \ - "PG:$(DATABASE_URL)" \ - -overwrite \ - -nln bcfishpass.modelled_stream_crossings_archive \ - data/modelled_stream_crossings.gpkg \ - modelled_stream_crossings - rm data/modelled_stream_crossings.gpkg - $(PSQL) -f sql/01_create_output_table.sql - $(PSQL) -f sql/load_from_archive.sql - rm -rf data/modelled_stream_crossings.gpkg - touch $@ - -# run the overlays/analysis -# (this is only done on primary db, other dbs just need to grab the archive) -.make/modelled_stream_crossings: .make/download_archive - $(PSQL) -f sql/01_create_output_table.sql - - # load preliminary crossings, iterating through watershed groups for each data source - parallel $(PSQL) -f sql/02_intersect_dra.sql -v wsg={1} ::: $(WSGS) - parallel $(PSQL) -f sql/03_intersect_ften.sql -v wsg={1} ::: $(WSGS) - parallel $(PSQL) -f sql/04_intersect_ogc.sql -v wsg={1} ::: $(WSGS) - parallel $(PSQL) -f sql/05_intersect_ogcpre06.sql -v wsg={1} ::: $(WSGS) - parallel $(PSQL) -f sql/06_intersect_railway.sql -v wsg={1} ::: $(WSGS) - - # remove duplicate crossings introduced by using multiple sources - $(PSQL) -f sql/07_remove_duplicates.sql - $(PSQL) -f sql/08_identify_open_bottom_structures.sql - - # assign modelled_crossing_id from previous version to ensure consistency - $(PSQL) -f sql/09_match_archived_crossings.sql - touch $@ \ No newline at end of file diff --git a/model/01_access/modelled_stream_crossings/README.md b/model/01_access/modelled_stream_crossings/README.md index 8c9f69ec..3a834b8c 100644 --- a/model/01_access/modelled_stream_crossings/README.md +++ b/model/01_access/modelled_stream_crossings/README.md @@ -9,14 +9,14 @@ In addition to generating the intersection points of roads/railways and streams, - maintain a consistent unique identifier value (`modelled_crossing_id`) that is stable with script re-runs **NOTE** -`bcfishpass` downloads an archive of modelled stream crossings generated weekly by this repository. This ensures that the `modelled_crossing_id` values for new records are consistent for all users as roads are added to the landscape. +To ensure that the `modelled_crossing_id` values are consistent, load existing crossings from s3 before running this job. -## Run scripts +## Run job -To generate a fresh set of crossings using the latest roads data, and load correct ids to the output from the archive: +To generate a fresh set of crossings using the latest roads data, using existing ids from existing table `bcfishpass.modelled_stream_crossings` where applicable: - make .make/modelled_stream_crossings + ./modelled_stream_crossings.sh ## Fixes diff --git a/model/01_access/modelled_stream_crossings/modelled_stream_crossings.sh b/model/01_access/modelled_stream_crossings/modelled_stream_crossings.sh new file mode 100755 index 00000000..90a74136 --- /dev/null +++ b/model/01_access/modelled_stream_crossings/modelled_stream_crossings.sh @@ -0,0 +1,70 @@ +#!/bin/bash +set -euxo pipefail + +#------- +# Download modelled crossings archive, load to db +#------- + +PSQL="psql $DATABASE_URL -v ON_ERROR_STOP=1" +WSGS=$($PSQL -AXt -c "SELECT watershed_group_code FROM whse_basemapping.fwa_watershed_groups_poly") + +# Overlay streams with roads, attempt to identify unique crossings and classify as culvert/bridge +# Note that this is only performed on primary db, other dbs only need to download the archive + +# create output table +$PSQL -f sql/01_create_output_table.sql + +# load preliminary crossings, iterating through watershed groups for each data source +parallel $PSQL -f sql/02_intersect_dra.sql -v wsg={1} ::: $WSGS +parallel $PSQL -f sql/03_intersect_ften.sql -v wsg={1} ::: $WSGS +parallel $PSQL -f sql/04_intersect_ogc.sql -v wsg={1} ::: $WSGS +parallel $PSQL -f sql/05_intersect_ogcpre06.sql -v wsg={1} ::: $WSGS +parallel $PSQL -f sql/06_intersect_railway.sql -v wsg={1} ::: $WSGS + +# remove duplicate crossings introduced by using multiple sources +$PSQL -f sql/07_remove_duplicates.sql +$PSQL -f sql/08_identify_open_bottom_structures.sql + +# assign modelled_crossing_id from previous version to ensure consistency +$PSQL -f sql/09_match_existing_crossings.sql + +# load new data to primary table +$PSQL -c "truncate bcfishpass.modelled_stream_crossings; +INSERT INTO bcfishpass.modelled_stream_crossings ( + modelled_crossing_id, + modelled_crossing_type, + modelled_crossing_type_source, + transport_line_id, + ften_road_section_lines_id, + og_road_segment_permit_id, + og_petrlm_dev_rd_pre06_pub_id, + railway_track_id, + linear_feature_id, + blue_line_key, + downstream_route_measure, + wscode_ltree, + localcode_ltree, + watershed_group_code, + geom +) +select + modelled_crossing_id, + modelled_crossing_type, + modelled_crossing_type_source, + transport_line_id, + ften_road_section_lines_id, + og_road_segment_permit_id, + og_petrlm_dev_rd_pre06_pub_id, + railway_track_id, + linear_feature_id, + blue_line_key, + downstream_route_measure, + wscode_ltree, + localcode_ltree, + watershed_group_code, + geom +from bcfishpass.modelled_stream_crossings_output" + +# cleanup +$PSQL -c "DROP TABLE bcfishpass.modelled_stream_crossings_build" +$PSQL -c "DROP TABLE bcfishpass.modelled_stream_crossings_output" \ No newline at end of file diff --git a/model/01_access/modelled_stream_crossings/sql/01_create_output_table.sql b/model/01_access/modelled_stream_crossings/sql/01_create_output_table.sql index 702d1a9d..64184166 100644 --- a/model/01_access/modelled_stream_crossings/sql/01_create_output_table.sql +++ b/model/01_access/modelled_stream_crossings/sql/01_create_output_table.sql @@ -1,8 +1,43 @@ +DROP TABLE IF EXISTS bcfishpass.modelled_stream_crossings_build; + +CREATE TABLE bcfishpass.modelled_stream_crossings_build +( + modelled_crossing_id serial primary key, + modelled_crossing_type character varying(5), + modelled_crossing_type_source text[], + transport_line_id integer, + ften_road_section_lines_id integer, + og_road_segment_permit_id integer, + og_petrlm_dev_rd_pre06_pub_id integer, + railway_track_id integer, + linear_feature_id bigint, + blue_line_key integer, + downstream_route_measure double precision, + wscode_ltree ltree, + localcode_ltree ltree, + watershed_group_code character varying(4), + geom geometry(PointZM, 3005) +); + +CREATE INDEX ON bcfishpass.modelled_stream_crossings_build (transport_line_id); +CREATE INDEX ON bcfishpass.modelled_stream_crossings_build (ften_road_section_lines_id); +CREATE INDEX ON bcfishpass.modelled_stream_crossings_build (og_road_segment_permit_id); +CREATE INDEX ON bcfishpass.modelled_stream_crossings_build (og_petrlm_dev_rd_pre06_pub_id); +CREATE INDEX ON bcfishpass.modelled_stream_crossings_build (railway_track_id); +CREATE INDEX ON bcfishpass.modelled_stream_crossings_build (blue_line_key); +CREATE INDEX ON bcfishpass.modelled_stream_crossings_build (linear_feature_id); +CREATE INDEX ON bcfishpass.modelled_stream_crossings_build USING GIST (geom); +CREATE INDEX ON bcfishpass.modelled_stream_crossings_build USING GIST (wscode_ltree); +CREATE INDEX ON bcfishpass.modelled_stream_crossings_build USING BTREE (wscode_ltree); +CREATE INDEX ON bcfishpass.modelled_stream_crossings_build USING GIST (localcode_ltree); +CREATE INDEX ON bcfishpass.modelled_stream_crossings_build USING BTREE (localcode_ltree); -DROP TABLE IF EXISTS bcfishpass.modelled_stream_crossings; -CREATE TABLE bcfishpass.modelled_stream_crossings +DROP TABLE IF EXISTS bcfishpass.modelled_stream_crossings_output; + +CREATE TABLE bcfishpass.modelled_stream_crossings_output ( + temp_id integer, modelled_crossing_id serial primary key, modelled_crossing_type character varying(5), modelled_crossing_type_source text[], @@ -20,15 +55,6 @@ CREATE TABLE bcfishpass.modelled_stream_crossings geom geometry(PointZM, 3005) ); -CREATE INDEX ON bcfishpass.modelled_stream_crossings (transport_line_id); -CREATE INDEX ON bcfishpass.modelled_stream_crossings (ften_road_section_lines_id); -CREATE INDEX ON bcfishpass.modelled_stream_crossings (og_road_segment_permit_id); -CREATE INDEX ON bcfishpass.modelled_stream_crossings (og_petrlm_dev_rd_pre06_pub_id); -CREATE INDEX ON bcfishpass.modelled_stream_crossings (railway_track_id); -CREATE INDEX ON bcfishpass.modelled_stream_crossings (blue_line_key); -CREATE INDEX ON bcfishpass.modelled_stream_crossings (linear_feature_id); -CREATE INDEX ON bcfishpass.modelled_stream_crossings USING GIST (geom); -CREATE INDEX ON bcfishpass.modelled_stream_crossings USING GIST (wscode_ltree); -CREATE INDEX ON bcfishpass.modelled_stream_crossings USING BTREE (wscode_ltree); -CREATE INDEX ON bcfishpass.modelled_stream_crossings USING GIST (localcode_ltree); -CREATE INDEX ON bcfishpass.modelled_stream_crossings USING BTREE (localcode_ltree); \ No newline at end of file +SELECT setval('bcfishpass.modelled_stream_crossings_output_modelled_crossing_id_seq', (SELECT max(modelled_crossing_id) FROM bcfishpass.modelled_stream_crossings)); + +CREATE INDEX ON bcfishpass.modelled_stream_crossings_output (temp_id); \ No newline at end of file diff --git a/model/01_access/modelled_stream_crossings/sql/02_intersect_dra.sql b/model/01_access/modelled_stream_crossings/sql/02_intersect_dra.sql index b324ef36..f423a827 100644 --- a/model/01_access/modelled_stream_crossings/sql/02_intersect_dra.sql +++ b/model/01_access/modelled_stream_crossings/sql/02_intersect_dra.sql @@ -164,7 +164,7 @@ crossing_geoms AS ) -- and finally, join back to closest road and insert the result into the crossing table -INSERT INTO bcfishpass.modelled_stream_crossings +INSERT INTO bcfishpass.modelled_stream_crossings_build ( transport_line_id, linear_feature_id, diff --git a/model/01_access/modelled_stream_crossings/sql/03_intersect_ften.sql b/model/01_access/modelled_stream_crossings/sql/03_intersect_ften.sql index 53aaa464..d4dbadb5 100644 --- a/model/01_access/modelled_stream_crossings/sql/03_intersect_ften.sql +++ b/model/01_access/modelled_stream_crossings/sql/03_intersect_ften.sql @@ -74,7 +74,7 @@ intersections_measures AS ) -- finally, generate the point from the measure. -INSERT INTO bcfishpass.modelled_stream_crossings +INSERT INTO bcfishpass.modelled_stream_crossings_build (ften_road_section_lines_id, linear_feature_id, blue_line_key, diff --git a/model/01_access/modelled_stream_crossings/sql/04_intersect_ogc.sql b/model/01_access/modelled_stream_crossings/sql/04_intersect_ogc.sql index 56e414ca..270b7119 100644 --- a/model/01_access/modelled_stream_crossings/sql/04_intersect_ogc.sql +++ b/model/01_access/modelled_stream_crossings/sql/04_intersect_ogc.sql @@ -75,7 +75,7 @@ intersections_measures AS ) -- finally, generate the point from the measure. -INSERT INTO bcfishpass.modelled_stream_crossings +INSERT INTO bcfishpass.modelled_stream_crossings_build (og_road_segment_permit_id, linear_feature_id, blue_line_key, diff --git a/model/01_access/modelled_stream_crossings/sql/05_intersect_ogcpre06.sql b/model/01_access/modelled_stream_crossings/sql/05_intersect_ogcpre06.sql index 39a64ee5..dd752ba6 100644 --- a/model/01_access/modelled_stream_crossings/sql/05_intersect_ogcpre06.sql +++ b/model/01_access/modelled_stream_crossings/sql/05_intersect_ogcpre06.sql @@ -75,7 +75,7 @@ intersections_measures AS ) -- finally, generate the point from the measure. -INSERT INTO bcfishpass.modelled_stream_crossings +INSERT INTO bcfishpass.modelled_stream_crossings_build (og_petrlm_dev_rd_pre06_pub_id, linear_feature_id, blue_line_key, diff --git a/model/01_access/modelled_stream_crossings/sql/06_intersect_railway.sql b/model/01_access/modelled_stream_crossings/sql/06_intersect_railway.sql index 9e0ea897..7f8b8b99 100644 --- a/model/01_access/modelled_stream_crossings/sql/06_intersect_railway.sql +++ b/model/01_access/modelled_stream_crossings/sql/06_intersect_railway.sql @@ -75,7 +75,7 @@ intersections_measures AS ) -- finally, generate output point from the measure -INSERT INTO bcfishpass.modelled_stream_crossings +INSERT INTO bcfishpass.modelled_stream_crossings_build (railway_track_id, linear_feature_id, blue_line_key, diff --git a/model/01_access/modelled_stream_crossings/sql/07_remove_duplicates.sql b/model/01_access/modelled_stream_crossings/sql/07_remove_duplicates.sql index 06a191dc..e38947fa 100644 --- a/model/01_access/modelled_stream_crossings/sql/07_remove_duplicates.sql +++ b/model/01_access/modelled_stream_crossings/sql/07_remove_duplicates.sql @@ -43,7 +43,7 @@ WITH matched_xings AS nn.ften_road_section_lines_id as ften_road_section_lines_id_keep, nn.og_road_segment_permit_id as og_road_segment_permit_id_keep, nn.og_petrlm_dev_rd_pre06_pub_id as og_petrlm_dev_rd_pre06_pub_id_keep - FROM bcfishpass.modelled_stream_crossings t1 + FROM bcfishpass.modelled_stream_crossings_build t1 CROSS JOIN LATERAL (SELECT modelled_crossing_id, @@ -52,7 +52,7 @@ WITH matched_xings AS og_road_segment_permit_id, og_petrlm_dev_rd_pre06_pub_id, ST_Distance(t1.geom, t2.geom) as dist - FROM bcfishpass.modelled_stream_crossings t2 + FROM bcfishpass.modelled_stream_crossings_build t2 WHERE t2.transport_line_id IS NOT NULL ORDER BY t1.geom <-> t2.geom LIMIT 1) as nn @@ -60,7 +60,7 @@ WITH matched_xings AS AND nn.dist < 20 ORDER BY t1.modelled_crossing_id ) -UPDATE bcfishpass.modelled_stream_crossings x +UPDATE bcfishpass.modelled_stream_crossings_build x SET ften_road_section_lines_id = m.ften_road_section_lines_id_del FROM matched_xings m WHERE x.modelled_crossing_id = m.modelled_crossing_id_keep; @@ -80,7 +80,7 @@ WITH matched_xings AS nn.ften_road_section_lines_id as ften_road_section_lines_id_keep, nn.og_road_segment_permit_id as og_road_segment_permit_id_keep, nn.og_petrlm_dev_rd_pre06_pub_id as og_petrlm_dev_rd_pre06_pub_id_keep - FROM bcfishpass.modelled_stream_crossings t1 + FROM bcfishpass.modelled_stream_crossings_build t1 CROSS JOIN LATERAL (SELECT modelled_crossing_id, @@ -89,7 +89,7 @@ WITH matched_xings AS og_road_segment_permit_id, og_petrlm_dev_rd_pre06_pub_id, ST_Distance(t1.geom, t2.geom) as dist - FROM bcfishpass.modelled_stream_crossings t2 + FROM bcfishpass.modelled_stream_crossings_build t2 WHERE t2.transport_line_id IS NOT NULL ORDER BY t1.geom <-> t2.geom LIMIT 1) as nn @@ -98,7 +98,7 @@ WITH matched_xings AS AND nn.dist < 20 ORDER BY t1.modelled_crossing_id ) -DELETE FROM bcfishpass.modelled_stream_crossings x +DELETE FROM bcfishpass.modelled_stream_crossings_build x WHERE modelled_crossing_id IN (SELECT modelled_crossing_id_del FROM matched_xings); @@ -119,7 +119,7 @@ SELECT nn.ften_road_section_lines_id as ften_road_section_lines_id_keep, nn.og_road_segment_permit_id as og_road_segment_permit_id_keep, nn.og_petrlm_dev_rd_pre06_pub_id as og_petrlm_dev_rd_pre06_pub_id_keep - FROM bcfishpass.modelled_stream_crossings t1 + FROM bcfishpass.modelled_stream_crossings_build t1 CROSS JOIN LATERAL (SELECT modelled_crossing_id, @@ -128,7 +128,7 @@ SELECT og_road_segment_permit_id, og_petrlm_dev_rd_pre06_pub_id, ST_Distance(t1.geom, t2.geom) as dist - FROM bcfishpass.modelled_stream_crossings t2 + FROM bcfishpass.modelled_stream_crossings_build t2 WHERE t2.transport_line_id IS NOT NULL OR t2.ften_road_section_lines_id IS NOT NULL ORDER BY t1.geom <-> t2.geom LIMIT 1) as nn @@ -136,7 +136,7 @@ SELECT AND nn.dist < 20 ORDER BY t1.modelled_crossing_id ) -UPDATE bcfishpass.modelled_stream_crossings x +UPDATE bcfishpass.modelled_stream_crossings_build x SET og_road_segment_permit_id = m.og_road_segment_permit_id_del FROM matched_xings m WHERE x.modelled_crossing_id = m.modelled_crossing_id_keep; @@ -155,7 +155,7 @@ SELECT nn.ften_road_section_lines_id as ften_road_section_lines_id_keep, nn.og_road_segment_permit_id as og_road_segment_permit_id_keep, nn.og_petrlm_dev_rd_pre06_pub_id as og_petrlm_dev_rd_pre06_pub_id_keep - FROM bcfishpass.modelled_stream_crossings t1 + FROM bcfishpass.modelled_stream_crossings_build t1 CROSS JOIN LATERAL (SELECT modelled_crossing_id, @@ -164,7 +164,7 @@ SELECT og_road_segment_permit_id, og_petrlm_dev_rd_pre06_pub_id, ST_Distance(t1.geom, t2.geom) as dist - FROM bcfishpass.modelled_stream_crossings t2 + FROM bcfishpass.modelled_stream_crossings_build t2 WHERE t2.transport_line_id IS NOT NULL OR t2.ften_road_section_lines_id IS NOT NULL ORDER BY t1.geom <-> t2.geom LIMIT 1) as nn @@ -174,7 +174,7 @@ SELECT AND nn.dist < 20 ORDER BY t1.modelled_crossing_id ) -DELETE FROM bcfishpass.modelled_stream_crossings +DELETE FROM bcfishpass.modelled_stream_crossings_build WHERE modelled_crossing_id IN (SELECT modelled_crossing_id_del FROM matched_xings); -- ----------------------------------------------------------- @@ -194,7 +194,7 @@ SELECT nn.ften_road_section_lines_id as ften_road_section_lines_id_keep, nn.og_road_segment_permit_id as og_road_segment_permit_id_keep, nn.og_petrlm_dev_rd_pre06_pub_id as og_petrlm_dev_rd_pre06_pub_id_keep - FROM bcfishpass.modelled_stream_crossings t1 + FROM bcfishpass.modelled_stream_crossings_build t1 CROSS JOIN LATERAL (SELECT modelled_crossing_id, @@ -203,7 +203,7 @@ SELECT og_road_segment_permit_id, og_petrlm_dev_rd_pre06_pub_id, ST_Distance(t1.geom, t2.geom) as dist - FROM bcfishpass.modelled_stream_crossings t2 + FROM bcfishpass.modelled_stream_crossings_build t2 WHERE t2.transport_line_id IS NOT NULL OR t2.ften_road_section_lines_id IS NOT NULL @@ -214,7 +214,7 @@ SELECT AND nn.dist < 20 ORDER BY t1.modelled_crossing_id ) -UPDATE bcfishpass.modelled_stream_crossings x +UPDATE bcfishpass.modelled_stream_crossings_build x SET og_petrlm_dev_rd_pre06_pub_id = m.og_petrlm_dev_rd_pre06_pub_id_del FROM matched_xings m WHERE x.modelled_crossing_id = m.modelled_crossing_id_keep; @@ -233,7 +233,7 @@ SELECT nn.ften_road_section_lines_id as ften_road_section_lines_id_keep, nn.og_road_segment_permit_id as og_road_segment_permit_id_keep, nn.og_petrlm_dev_rd_pre06_pub_id as og_petrlm_dev_rd_pre06_pub_id_keep - FROM bcfishpass.modelled_stream_crossings t1 + FROM bcfishpass.modelled_stream_crossings_build t1 CROSS JOIN LATERAL (SELECT modelled_crossing_id, @@ -242,7 +242,7 @@ SELECT og_road_segment_permit_id, og_petrlm_dev_rd_pre06_pub_id, ST_Distance(t1.geom, t2.geom) as dist - FROM bcfishpass.modelled_stream_crossings t2 + FROM bcfishpass.modelled_stream_crossings_build t2 WHERE t2.transport_line_id IS NOT NULL OR t2.ften_road_section_lines_id IS NOT NULL @@ -256,5 +256,5 @@ SELECT AND nn.dist < 20 ORDER BY t1.modelled_crossing_id ) -DELETE FROM bcfishpass.modelled_stream_crossings +DELETE FROM bcfishpass.modelled_stream_crossings_build WHERE modelled_crossing_id IN (SELECT modelled_crossing_id_del FROM matched_xings); \ No newline at end of file diff --git a/model/01_access/modelled_stream_crossings/sql/08_identify_open_bottom_structures.sql b/model/01_access/modelled_stream_crossings/sql/08_identify_open_bottom_structures.sql index a2a7ebe0..382440d1 100644 --- a/model/01_access/modelled_stream_crossings/sql/08_identify_open_bottom_structures.sql +++ b/model/01_access/modelled_stream_crossings/sql/08_identify_open_bottom_structures.sql @@ -3,7 +3,7 @@ -- ----------------------------------------------- -- Start with stream order -UPDATE bcfishpass.modelled_stream_crossings x +UPDATE bcfishpass.modelled_stream_crossings_build x SET modelled_crossing_type = 'OBS', modelled_crossing_type_source = ARRAY['FWA_STREAM_ORDER'] @@ -16,7 +16,7 @@ AND s.stream_order >= 6 AND s.linear_feature_id != 701296585; -- double line streams/waterbodies -UPDATE bcfishpass.modelled_stream_crossings x +UPDATE bcfishpass.modelled_stream_crossings_build x SET modelled_crossing_type = 'OBS', modelled_crossing_type_source = modelled_crossing_type_source||ARRAY['FWA_EDGE_TYPE'] @@ -35,21 +35,21 @@ WITH mot_bridges AS (SELECT modelled_crossing_id, ST_Distance(a.geom, b.geom) as dist - FROM bcfishpass.modelled_stream_crossings b + FROM bcfishpass.modelled_stream_crossings_build b ORDER BY a.geom <-> b.geom LIMIT 1) as nn WHERE UPPER(a.bmis_structure_type) = 'BRIDGE' AND UPPER(a.bmis_struct_status_type_desc) = 'OPEN/IN USE' AND nn.dist < 15 ) -UPDATE bcfishpass.modelled_stream_crossings x +UPDATE bcfishpass.modelled_stream_crossings_build x SET modelled_crossing_type = 'OBS', modelled_crossing_type_source = modelled_crossing_type_source||ARRAY['MOT_ROAD_STRUCTURE_SP'] FROM mot_bridges y WHERE x.modelled_crossing_id = y.modelled_crossing_id; -- DRA structures, simply join on id -UPDATE bcfishpass.modelled_stream_crossings x +UPDATE bcfishpass.modelled_stream_crossings_build x SET modelled_crossing_type = 'OBS', modelled_crossing_type_source = modelled_crossing_type_source||ARRAY['TRANSPORT_LINE_STRUCTURE_CODE'] @@ -59,7 +59,7 @@ AND r.transport_line_structure_code IN ('B','C','E','F','O','R','V'); -- Railway structures don't join back to railway tracks 1:1, find crossings -- within 10.5m of the bridges (because crossings were clustered to 20m) -UPDATE bcfishpass.modelled_stream_crossings x +UPDATE bcfishpass.modelled_stream_crossings_build x SET modelled_crossing_type = 'OBS', modelled_crossing_type_source = modelled_crossing_type_source||ARRAY['GBA_RAILWAY_STRUCTURE_LINES_SP'] @@ -70,7 +70,7 @@ AND x.railway_track_id IS NOT NULL; -- default everything else to CBS -- (but note that PSCIS will replace many of these with OBS) -UPDATE bcfishpass.modelled_stream_crossings x +UPDATE bcfishpass.modelled_stream_crossings_build x SET modelled_crossing_type = 'CBS' WHERE modelled_crossing_type IS NULL; @@ -84,13 +84,13 @@ WITH de_duped AS ( SELECT DISTINCT modelled_crossing_id, unnest(modelled_crossing_type_source) as modelled_crossing_type_source - FROM bcfishpass.modelled_stream_crossings + FROM bcfishpass.modelled_stream_crossings_build ORDER BY modelled_crossing_id, modelled_crossing_type_source ) as f GROUP BY modelled_crossing_id ) -UPDATE bcfishpass.modelled_stream_crossings x +UPDATE bcfishpass.modelled_stream_crossings_build x SET modelled_crossing_type_source = d.modelled_crossing_type_source FROM de_duped d WHERE x.modelled_crossing_id = d.modelled_crossing_id; diff --git a/model/01_access/modelled_stream_crossings/sql/09_match_archived_crossings.sql b/model/01_access/modelled_stream_crossings/sql/09_match_existing_crossings.sql similarity index 53% rename from model/01_access/modelled_stream_crossings/sql/09_match_archived_crossings.sql rename to model/01_access/modelled_stream_crossings/sql/09_match_existing_crossings.sql index 4783bc13..9acac7cc 100644 --- a/model/01_access/modelled_stream_crossings/sql/09_match_archived_crossings.sql +++ b/model/01_access/modelled_stream_crossings/sql/09_match_existing_crossings.sql @@ -2,32 +2,6 @@ -- match is done for crossings within 10m distance in any direction -- (rather than matching on blue_line_key and measure, just in case the FWA stream has changed) -DROP TABLE IF EXISTS bcfishpass.modelled_stream_crossings_temp; - -CREATE TABLE bcfishpass.modelled_stream_crossings_temp -( - temp_id integer, - modelled_crossing_id serial primary key, - modelled_crossing_type character varying(5), - modelled_crossing_type_source text[], - transport_line_id integer, - ften_road_section_lines_id integer, - og_road_segment_permit_id integer, - og_petrlm_dev_rd_pre06_pub_id integer, - railway_track_id integer, - linear_feature_id bigint, - blue_line_key integer, - downstream_route_measure double precision, - wscode_ltree ltree, - localcode_ltree ltree, - watershed_group_code character varying(4), - geom geometry(PointZM, 3005) -); - -SELECT setval('bcfishpass.modelled_stream_crossings_temp_modelled_crossing_id_seq', (SELECT max(modelled_crossing_id) FROM bcfishpass.modelled_stream_crossings_archive)); - -CREATE INDEX ON bcfishpass.modelled_stream_crossings_temp (temp_id); - WITH matched AS ( SELECT @@ -48,19 +22,19 @@ WITH matched AS a.geom, nn.modelled_crossing_id as archive_id, nn.dist - FROM bcfishpass.modelled_stream_crossings a + FROM bcfishpass.modelled_stream_crossings_build a CROSS JOIN LATERAL (SELECT modelled_crossing_id, ST_Distance(a.geom, b.geom) as dist - FROM bcfishpass.modelled_stream_crossings_archive b + FROM bcfishpass.modelled_stream_crossings b ORDER BY a.geom <-> b.geom LIMIT 1) as nn WHERE nn.dist < 10 ) -- be sure to only return one match -INSERT INTO bcfishpass.modelled_stream_crossings_temp +INSERT INTO bcfishpass.modelled_stream_crossings_output ( temp_id, modelled_crossing_id, @@ -100,7 +74,7 @@ FROM matched m ORDER BY archive_id, dist; -- now insert records that did not get matched -INSERT INTO bcfishpass.modelled_stream_crossings_temp +INSERT INTO bcfishpass.modelled_stream_crossings_output ( temp_id, modelled_crossing_type, @@ -134,24 +108,7 @@ SELECT localcode_ltree, watershed_group_code, geom -FROM bcfishpass.modelled_stream_crossings -WHERE modelled_crossing_id NOT IN (SELECT temp_id FROM bcfishpass.modelled_stream_crossings_temp); - -ALTER TABLE bcfishpass.modelled_stream_crossings_temp DROP COLUMN temp_id; - -DROP TABLE bcfishpass.modelled_stream_crossings; -ALTER TABLE bcfishpass.modelled_stream_crossings_temp RENAME TO modelled_stream_crossings; +FROM bcfishpass.modelled_stream_crossings_build +WHERE modelled_crossing_id NOT IN (SELECT temp_id FROM bcfishpass.modelled_stream_crossings_output); --- recreate indexes -CREATE INDEX ON bcfishpass.modelled_stream_crossings (transport_line_id); -CREATE INDEX ON bcfishpass.modelled_stream_crossings (ften_road_section_lines_id); -CREATE INDEX ON bcfishpass.modelled_stream_crossings (og_road_segment_permit_id); -CREATE INDEX ON bcfishpass.modelled_stream_crossings (og_petrlm_dev_rd_pre06_pub_id); -CREATE INDEX ON bcfishpass.modelled_stream_crossings (railway_track_id); -CREATE INDEX ON bcfishpass.modelled_stream_crossings (blue_line_key); -CREATE INDEX ON bcfishpass.modelled_stream_crossings (linear_feature_id); -CREATE INDEX ON bcfishpass.modelled_stream_crossings USING GIST (geom); -CREATE INDEX ON bcfishpass.modelled_stream_crossings USING GIST (wscode_ltree); -CREATE INDEX ON bcfishpass.modelled_stream_crossings USING BTREE (wscode_ltree); -CREATE INDEX ON bcfishpass.modelled_stream_crossings USING GIST (localcode_ltree); -CREATE INDEX ON bcfishpass.modelled_stream_crossings USING BTREE (localcode_ltree); \ No newline at end of file +ALTER TABLE bcfishpass.modelled_stream_crossings_output DROP COLUMN temp_id; diff --git a/model/01_access/modelled_stream_crossings/sql/load_from_archive.sql b/model/01_access/modelled_stream_crossings/sql/load_from_archive.sql deleted file mode 100644 index 4319e9bd..00000000 --- a/model/01_access/modelled_stream_crossings/sql/load_from_archive.sql +++ /dev/null @@ -1,34 +0,0 @@ -insert into bcfishpass.modelled_stream_crossings ( - modelled_crossing_id, - modelled_crossing_type, - modelled_crossing_type_source, - transport_line_id, - ften_road_section_lines_id, - og_road_segment_permit_id, - og_petrlm_dev_rd_pre06_pub_id, - railway_track_id, - linear_feature_id, - blue_line_key, - downstream_route_measure, - wscode_ltree, - localcode_ltree, - watershed_group_code, - geom -) -select - modelled_crossing_id, - modelled_crossing_type, - string_to_array(modelled_crossing_type_source, '; ') as modelled_crossing_type_source, - transport_line_id, - ften_road_section_lines_id, - og_road_segment_permit_id, - og_petrlm_dev_rd_pre06_pub_id, - railway_track_id, - linear_feature_id, - blue_line_key, - downstream_route_measure, - wscode_ltree::ltree, - localcode_ltree::ltree, - watershed_group_code, - geom -from bcfishpass.modelled_stream_crossings_archive; \ No newline at end of file diff --git a/model/01_access/qa/README.md b/model/01_access/qa/README.md deleted file mode 100644 index e69de29b..00000000 diff --git a/model/01_access/qa/qa.sh b/model/01_access/qa/qa.sh deleted file mode 100755 index d3dc78b0..00000000 --- a/model/01_access/qa/qa.sh +++ /dev/null @@ -1,12 +0,0 @@ -#!/bin/bash - -set -euxo pipefail - -PSQL="psql $DATABASE_URL -v ON_ERROR_STOP=1" - - -# create views displaying access/habitat per species / species group -for query in $(ls sql/*.sql) -do - $PSQL --csv -f $query > $(basename $query .sql).csv -done \ No newline at end of file diff --git a/model/01_access/qa/sql/barriers_ch_cm_co_pk_sk.sql b/model/01_access/qa/sql/barriers_ch_cm_co_pk_sk.sql deleted file mode 100644 index 434cbcaa..00000000 --- a/model/01_access/qa/sql/barriers_ch_cm_co_pk_sk.sql +++ /dev/null @@ -1,23 +0,0 @@ --- list salmon barriers with salmon observations upstream -select - b.barriers_ch_cm_co_pk_sk_id, - b.barrier_type, - b.blue_line_key, - b.downstream_route_measure, - b.watershed_group_code, - count(*) as n_observations, - array_agg(o.fish_observation_point_id) as fish_observation_point_id, - array_agg(o.species_code) as species_codes, - array_agg(o.observation_date) as observation_dates -from bcfishpass.barriers_ch_cm_co_pk_sk b -inner join bcfishpass.observations_vw o on - fwa_upstream( - b.blue_line_key, b.downstream_route_measure, b.wscode_ltree, b.localcode_ltree, - o.blue_line_key, o.downstream_route_measure, o.wscode_ltree, o.localcode_ltree) -where o.species_code in ('CH','CM','CO','PK','SK') -group by b.barriers_ch_cm_co_pk_sk_id, - b.barrier_type, - b.blue_line_key, - b.downstream_route_measure, - b.watershed_group_code -order by barriers_ch_cm_co_pk_sk_id; \ No newline at end of file diff --git a/model/01_access/qa/sql/barriers_st.sql b/model/01_access/qa/sql/barriers_st.sql deleted file mode 100644 index fe1b535b..00000000 --- a/model/01_access/qa/sql/barriers_st.sql +++ /dev/null @@ -1,23 +0,0 @@ --- list steelhead barriers with steelhead or salmon observations upstream -select - b.barriers_st_id, - b.barrier_type, - b.blue_line_key, - b.downstream_route_measure, - b.watershed_group_code, - count(*) as n_observations, - array_agg(o.fish_observation_point_id) as fish_observation_point_id, - array_agg(o.species_code) as species_codes, - array_agg(o.observation_date) as observation_dates -from bcfishpass.barriers_st b -inner join bcfishpass.observations_vw o on - fwa_upstream( - b.blue_line_key, b.downstream_route_measure, b.wscode_ltree, b.localcode_ltree, - o.blue_line_key, o.downstream_route_measure, o.wscode_ltree, o.localcode_ltree) -where o.species_code in ('CH','CM','CO','PK','SK','ST') -group by b.barriers_st_id, - b.barrier_type, - b.blue_line_key, - b.downstream_route_measure, - b.watershed_group_code -order by barriers_st_id; \ No newline at end of file diff --git a/model/01_access/qa/sql/dams.sql b/model/01_access/qa/sql/dams.sql deleted file mode 100644 index 8c246725..00000000 --- a/model/01_access/qa/sql/dams.sql +++ /dev/null @@ -1,35 +0,0 @@ -select - d.dam_id, - cabd.dam_name_en, - cabd.owner, - cabd.operating_status, - cabd.construction_year, - cabd.dam_use, - case - when cabd.passability_status_code = 1 THEN 'BARRIER' - when cabd.passability_status_code = 2 THEN 'POTENTIAL' - when cabd.passability_status_code = 3 THEN 'PASSABLE' - when cabd.passability_status_code = 4 THEN 'UNKNOWN' - end AS barrier_status, - o.fish_observation_point_id, - o.species_code, - o.observation_date, - s.agency_name, - s.source, - s.source_ref, - s.activity, - s.life_stage, - s.acat_report_url -from bcfishpass.dams d -inner join bcfishpass.observations_vw o on - fwa_upstream( - d.blue_line_key, d.downstream_route_measure, d.wscode_ltree, d.localcode_ltree, - o.blue_line_key, o.downstream_route_measure, o.wscode_ltree, o.localcode_ltree) -inner join cabd.dams cabd on d.dam_id = cabd.cabd_id::text and cabd.passability_status_code = 1 -inner join whse_fish.fiss_fish_obsrvtn_pnt_sp s on o.fish_observation_point_id = s.fish_observation_point_id -where o.species_code in ('CH','CM','CO','PK','SK','ST') -and dam_id not in ('7130db52-4978-42b8-99ad-11d8d65e87ae','Terzaghi Dam') -order by dam_name_en, fish_observation_point_id; - - - diff --git a/model/01_access/qa/sql/observations_ch_cm_co_pk_sk.sql b/model/01_access/qa/sql/observations_ch_cm_co_pk_sk.sql deleted file mode 100644 index 587462a8..00000000 --- a/model/01_access/qa/sql/observations_ch_cm_co_pk_sk.sql +++ /dev/null @@ -1,101 +0,0 @@ -select * from -(SELECT - a.species_code, - a.fish_observation_point_id, - a.observation_date, - o.agency_name, - o.source, - o.source_ref, - --array_agg(DISTINCT d.barriers_gradient_15_id) as barriers_gradient_15_dnstr, - count(DISTINCT d.barriers_gradient_15_id) as n_barriers_gradient_15_dnstr, - --array_agg(DISTINCT e.barriers_gradient_20_id) as barriers_gradient_20_dnstr, - count(DISTINCT e.barriers_gradient_20_id) as n_barriers_gradient_20_dnstr, - --array_agg(DISTINCT f.barriers_gradient_25_id) as barriers_gradient_25_dnstr, - count(DISTINCT f.barriers_gradient_25_id) as n_barriers_gradient_25_dnstr, - --array_agg(DISTINCT g.barriers_gradient_30_id) as barriers_gradient_30_dnstr, - count(DISTINCT g.barriers_gradient_30_id) as n_barriers_gradient_30_dnstr, - --array_agg(DISTINCT h.barriers_falls_id) as barriers_falls_dnstr, - count(DISTINCT h.barriers_falls_id) as n_barriers_falls_dnstr -FROM bcfishpass.observations_vw a -LEFT OUTER JOIN bcfishpass.barriers_gradient_15 d -ON FWA_Downstream( - a.blue_line_key, - a.downstream_route_measure, - a.wscode_ltree, - a.localcode_ltree, - d.blue_line_key, - d.downstream_route_measure, - d.wscode_ltree, - d.localcode_ltree, - True, - 1 -) -LEFT OUTER JOIN bcfishpass.barriers_gradient_20 e -ON FWA_Downstream( - a.blue_line_key, - a.downstream_route_measure, - a.wscode_ltree, - a.localcode_ltree, - e.blue_line_key, - e.downstream_route_measure, - e.wscode_ltree, - e.localcode_ltree, - True, - 1 -) -LEFT OUTER JOIN bcfishpass.barriers_gradient_25 f -ON FWA_Downstream( - a.blue_line_key, - a.downstream_route_measure, - a.wscode_ltree, - a.localcode_ltree, - f.blue_line_key, - f.downstream_route_measure, - f.wscode_ltree, - f.localcode_ltree, - True, - 1 -) -LEFT OUTER JOIN bcfishpass.barriers_gradient_30 g -ON FWA_Downstream( - a.blue_line_key, - a.downstream_route_measure, - a.wscode_ltree, - a.localcode_ltree, - g.blue_line_key, - g.downstream_route_measure, - g.wscode_ltree, - g.localcode_ltree, - True, - 1 -) -LEFT OUTER JOIN bcfishpass.barriers_falls h -ON FWA_Downstream( - a.blue_line_key, - a.downstream_route_measure, - a.wscode_ltree, - a.localcode_ltree, - h.blue_line_key, - h.downstream_route_measure, - h.wscode_ltree, - h.localcode_ltree, - True, - 1 -) -inner join whse_fish.fiss_fish_obsrvtn_pnt_sp o -on a.fish_observation_point_id = o.fish_observation_point_id -WHERE a.species_code in ('CO','CM','CH','PK','SK') -GROUP BY a.species_code, - a.fish_observation_point_id, - a.observation_date, - o.agency_name, - o.source, - o.source_ref -) as f -WHERE - n_barriers_gradient_15_dnstr >= 1 or - n_barriers_gradient_20_dnstr >= 1 or - n_barriers_gradient_25_dnstr >= 1 or - n_barriers_gradient_30_dnstr >= 1 or - n_barriers_falls_dnstr >= 1 -ORDER BY fish_observation_point_id; \ No newline at end of file diff --git a/model/01_access/qa/sql/observations_st.sql b/model/01_access/qa/sql/observations_st.sql deleted file mode 100644 index db4c7b01..00000000 --- a/model/01_access/qa/sql/observations_st.sql +++ /dev/null @@ -1,85 +0,0 @@ -select * from -(SELECT - a.species_code, - a.fish_observation_point_id, - a.observation_date, - o.agency_name, - o.source, - o.source_ref, - --array_agg(DISTINCT e.barriers_gradient_20_id) as barriers_gradient_20_dnstr, - count(DISTINCT e.barriers_gradient_20_id) as n_barriers_gradient_20_dnstr, - --array_agg(DISTINCT f.barriers_gradient_25_id) as barriers_gradient_25_dnstr, - count(DISTINCT f.barriers_gradient_25_id) as n_barriers_gradient_25_dnstr, - --array_agg(DISTINCT g.barriers_gradient_30_id) as barriers_gradient_30_dnstr, - count(DISTINCT g.barriers_gradient_30_id) as n_barriers_gradient_30_dnstr, - --array_agg(DISTINCT h.barriers_falls_id) as barriers_falls_dnstr, - count(DISTINCT h.barriers_falls_id) as n_barriers_falls_dnstr -FROM bcfishpass.observations_vw a -LEFT OUTER JOIN bcfishpass.barriers_gradient_20 e -ON FWA_Downstream( - a.blue_line_key, - a.downstream_route_measure, - a.wscode_ltree, - a.localcode_ltree, - e.blue_line_key, - e.downstream_route_measure, - e.wscode_ltree, - e.localcode_ltree, - True, - 1 -) -LEFT OUTER JOIN bcfishpass.barriers_gradient_25 f -ON FWA_Downstream( - a.blue_line_key, - a.downstream_route_measure, - a.wscode_ltree, - a.localcode_ltree, - f.blue_line_key, - f.downstream_route_measure, - f.wscode_ltree, - f.localcode_ltree, - True, - 1 -) -LEFT OUTER JOIN bcfishpass.barriers_gradient_30 g -ON FWA_Downstream( - a.blue_line_key, - a.downstream_route_measure, - a.wscode_ltree, - a.localcode_ltree, - g.blue_line_key, - g.downstream_route_measure, - g.wscode_ltree, - g.localcode_ltree, - True, - 1 -) -LEFT OUTER JOIN bcfishpass.barriers_falls h -ON FWA_Downstream( - a.blue_line_key, - a.downstream_route_measure, - a.wscode_ltree, - a.localcode_ltree, - h.blue_line_key, - h.downstream_route_measure, - h.wscode_ltree, - h.localcode_ltree, - True, - 1 -) -inner join whse_fish.fiss_fish_obsrvtn_pnt_sp o -on a.fish_observation_point_id = o.fish_observation_point_id -WHERE a.species_code = 'ST' -GROUP BY a.species_code, - a.fish_observation_point_id, - a.observation_date, - o.agency_name, - o.source, - o.source_ref -) as f -WHERE - n_barriers_gradient_20_dnstr >= 1 or - n_barriers_gradient_25_dnstr >= 1 or - n_barriers_gradient_30_dnstr >= 1 or - n_barriers_falls_dnstr >= 1 -ORDER BY fish_observation_point_id; \ No newline at end of file diff --git a/model/01_access/sql/barriers_falls.sql b/model/01_access/sql/barriers_falls.sql index e44be891..cee42000 100644 --- a/model/01_access/sql/barriers_falls.sql +++ b/model/01_access/sql/barriers_falls.sql @@ -24,7 +24,7 @@ SELECT f.localcode_ltree, f.watershed_group_code, f.geom -FROM bcfishpass.falls f +FROM bcfishpass.falls_vw f INNER JOIN whse_basemapping.fwa_stream_networks_sp s ON f.linear_feature_id = s.linear_feature_id WHERE diff --git a/model/01_access/sql/barriers_gradient_05.sql b/model/01_access/sql/barriers_gradient.sql similarity index 74% rename from model/01_access/sql/barriers_gradient_05.sql rename to model/01_access/sql/barriers_gradient.sql index 59ea293e..84d145ac 100644 --- a/model/01_access/sql/barriers_gradient_05.sql +++ b/model/01_access/sql/barriers_gradient.sql @@ -1,6 +1,6 @@ -INSERT INTO bcfishpass.barriers_gradient_05 +INSERT INTO bcfishpass.barriers_gradient ( - barriers_gradient_05_id, + barriers_gradient_id, barrier_type, barrier_name, linear_feature_id, @@ -14,7 +14,14 @@ INSERT INTO bcfishpass.barriers_gradient_05 ) SELECT gradient_barrier_id as barrier_load_id, - 'GRADIENT_05' as barrier_type, + case + when b.gradient_class = 5 then'GRADIENT_05' + when b.gradient_class = 10 then'GRADIENT_10' + when b.gradient_class = 15 then'GRADIENT_15' + when b.gradient_class = 20 then'GRADIENT_20' + when b.gradient_class = 25 then'GRADIENT_25' + when b.gradient_class = 30 then'GRADIENT_30' + end as barrier_type, NULL as barrier_name, s.linear_feature_id, b.blue_line_key, @@ -33,7 +40,6 @@ LEFT OUTER JOIN bcfishpass.user_barriers_definite_control p ON b.blue_line_key = p.blue_line_key AND abs(b.downstream_route_measure - p.downstream_route_measure) < 1 WHERE - b.gradient_class = 5 AND (p.barrier_ind IS NULL or p.barrier_ind is true) AND -- do not include records forced to be passable in control table b.watershed_group_code = :'wsg' ORDER BY b.blue_line_key, b.downstream_route_measure diff --git a/model/01_access/sql/barriers_gradient_10.sql b/model/01_access/sql/barriers_gradient_10.sql deleted file mode 100644 index eb22e669..00000000 --- a/model/01_access/sql/barriers_gradient_10.sql +++ /dev/null @@ -1,40 +0,0 @@ -INSERT INTO bcfishpass.barriers_gradient_10 -( - barriers_gradient_10_id, - barrier_type, - barrier_name, - linear_feature_id, - blue_line_key, - watershed_key, - downstream_route_measure, - wscode_ltree, - localcode_ltree, - watershed_group_code, - geom -) -SELECT - gradient_barrier_id as barrier_load_id, - 'GRADIENT_10' as barrier_type, - NULL as barrier_name, - s.linear_feature_id, - b.blue_line_key, - s.watershed_key, - b.downstream_route_measure, - s.wscode_ltree, - s.localcode_ltree, - s.watershed_group_code, - ST_Force2D((ST_Dump(ST_Locatealong(s.geom, b.downstream_route_measure))).geom)::geometry(Point,3005) as geom -FROM bcfishpass.gradient_barriers b -INNER JOIN whse_basemapping.fwa_stream_networks_sp s -ON b.blue_line_key = s.blue_line_key -AND s.downstream_route_measure <= b.downstream_route_measure -AND s.upstream_route_measure + .01 > b.downstream_route_measure -LEFT OUTER JOIN bcfishpass.user_barriers_definite_control p - ON b.blue_line_key = p.blue_line_key - AND abs(b.downstream_route_measure - p.downstream_route_measure) < 1 -WHERE - b.gradient_class = 10 AND - (p.barrier_ind IS NULL or p.barrier_ind is true) AND -- do not include records forced to be passable in control table - b.watershed_group_code = :'wsg' -ORDER BY b.blue_line_key, b.downstream_route_measure -ON CONFLICT DO NOTHING; \ No newline at end of file diff --git a/model/01_access/sql/barriers_gradient_15.sql b/model/01_access/sql/barriers_gradient_15.sql deleted file mode 100644 index ef2bbc97..00000000 --- a/model/01_access/sql/barriers_gradient_15.sql +++ /dev/null @@ -1,40 +0,0 @@ -INSERT INTO bcfishpass.barriers_gradient_15 -( - barriers_gradient_15_id, - barrier_type, - barrier_name, - linear_feature_id, - blue_line_key, - watershed_key, - downstream_route_measure, - wscode_ltree, - localcode_ltree, - watershed_group_code, - geom -) -SELECT - gradient_barrier_id as barrier_load_id, - 'GRADIENT_15' as barrier_type, - NULL as barrier_name, - s.linear_feature_id, - b.blue_line_key, - s.watershed_key, - b.downstream_route_measure, - s.wscode_ltree, - s.localcode_ltree, - s.watershed_group_code, - ST_Force2D((ST_Dump(ST_Locatealong(s.geom, b.downstream_route_measure))).geom)::geometry(Point,3005) as geom -FROM bcfishpass.gradient_barriers b -INNER JOIN whse_basemapping.fwa_stream_networks_sp s -ON b.blue_line_key = s.blue_line_key -AND s.downstream_route_measure <= b.downstream_route_measure -AND s.upstream_route_measure + .01 > b.downstream_route_measure -LEFT OUTER JOIN bcfishpass.user_barriers_definite_control p - ON b.blue_line_key = p.blue_line_key - AND abs(b.downstream_route_measure - p.downstream_route_measure) < 1 -WHERE - b.gradient_class = 15 AND - (p.barrier_ind IS NULL or p.barrier_ind is true) AND -- do not include records forced to be passable in control table - b.watershed_group_code = :'wsg' -ORDER BY b.blue_line_key, b.downstream_route_measure -ON CONFLICT DO NOTHING; diff --git a/model/01_access/sql/barriers_gradient_20.sql b/model/01_access/sql/barriers_gradient_20.sql deleted file mode 100644 index 86540a4a..00000000 --- a/model/01_access/sql/barriers_gradient_20.sql +++ /dev/null @@ -1,40 +0,0 @@ -INSERT INTO bcfishpass.barriers_gradient_20 -( - barriers_gradient_20_id, - barrier_type, - barrier_name, - linear_feature_id, - blue_line_key, - watershed_key, - downstream_route_measure, - wscode_ltree, - localcode_ltree, - watershed_group_code, - geom -) -SELECT - gradient_barrier_id as barrier_load_id, - 'GRADIENT_20' as barrier_type, - NULL as barrier_name, - s.linear_feature_id, - b.blue_line_key, - s.watershed_key, - b.downstream_route_measure, - s.wscode_ltree, - s.localcode_ltree, - s.watershed_group_code, - ST_Force2D((ST_Dump(ST_Locatealong(s.geom, b.downstream_route_measure))).geom)::geometry(Point,3005) as geom -FROM bcfishpass.gradient_barriers b -INNER JOIN whse_basemapping.fwa_stream_networks_sp s -ON b.blue_line_key = s.blue_line_key -AND s.downstream_route_measure <= b.downstream_route_measure -AND s.upstream_route_measure + .01 > b.downstream_route_measure -LEFT OUTER JOIN bcfishpass.user_barriers_definite_control p - ON b.blue_line_key = p.blue_line_key - AND abs(b.downstream_route_measure - p.downstream_route_measure) < 1 -WHERE - b.gradient_class = 20 AND - (p.barrier_ind IS NULL or p.barrier_ind is true) AND -- do not include records forced to be passable in control table - b.watershed_group_code = :'wsg' -ORDER BY b.blue_line_key, b.downstream_route_measure -ON CONFLICT DO NOTHING; \ No newline at end of file diff --git a/model/01_access/sql/barriers_gradient_25.sql b/model/01_access/sql/barriers_gradient_25.sql deleted file mode 100644 index b9e93629..00000000 --- a/model/01_access/sql/barriers_gradient_25.sql +++ /dev/null @@ -1,40 +0,0 @@ -INSERT INTO bcfishpass.barriers_gradient_25 -( - barriers_gradient_25_id, - barrier_type, - barrier_name, - linear_feature_id, - blue_line_key, - watershed_key, - downstream_route_measure, - wscode_ltree, - localcode_ltree, - watershed_group_code, - geom -) -SELECT - gradient_barrier_id as barrier_load_id, - 'GRADIENT_25' as barrier_type, - NULL as barrier_name, - s.linear_feature_id, - b.blue_line_key, - s.watershed_key, - b.downstream_route_measure, - s.wscode_ltree, - s.localcode_ltree, - s.watershed_group_code, - ST_Force2D((ST_Dump(ST_Locatealong(s.geom, b.downstream_route_measure))).geom)::geometry(Point,3005) as geom -FROM bcfishpass.gradient_barriers b -INNER JOIN whse_basemapping.fwa_stream_networks_sp s - ON b.blue_line_key = s.blue_line_key - AND s.downstream_route_measure <= b.downstream_route_measure - AND s.upstream_route_measure + .01 > b.downstream_route_measure -LEFT OUTER JOIN bcfishpass.user_barriers_definite_control p - ON b.blue_line_key = p.blue_line_key - AND abs(b.downstream_route_measure - p.downstream_route_measure) < 1 -WHERE - b.gradient_class = 25 AND - (p.barrier_ind IS NULL or p.barrier_ind is true) AND -- do not include records forced to be passable in control table - b.watershed_group_code = :'wsg' -ORDER BY b.blue_line_key, b.downstream_route_measure -ON CONFLICT DO NOTHING; \ No newline at end of file diff --git a/model/01_access/sql/barriers_gradient_30.sql b/model/01_access/sql/barriers_gradient_30.sql deleted file mode 100644 index 60394b4c..00000000 --- a/model/01_access/sql/barriers_gradient_30.sql +++ /dev/null @@ -1,40 +0,0 @@ -INSERT INTO bcfishpass.barriers_gradient_30 -( - barriers_gradient_30_id, - barrier_type, - barrier_name, - linear_feature_id, - blue_line_key, - watershed_key, - downstream_route_measure, - wscode_ltree, - localcode_ltree, - watershed_group_code, - geom -) -SELECT - gradient_barrier_id as barrier_load_id, - 'GRADIENT_30' as barrier_type, - NULL as barrier_name, - s.linear_feature_id, - b.blue_line_key, - s.watershed_key, - b.downstream_route_measure, - s.wscode_ltree, - s.localcode_ltree, - s.watershed_group_code, - ST_Force2D((ST_Dump(ST_Locatealong(s.geom, b.downstream_route_measure))).geom)::geometry(Point,3005) as geom -FROM bcfishpass.gradient_barriers b -INNER JOIN whse_basemapping.fwa_stream_networks_sp s - ON b.blue_line_key = s.blue_line_key - AND s.downstream_route_measure <= b.downstream_route_measure - AND s.upstream_route_measure + .01 > b.downstream_route_measure -LEFT OUTER JOIN bcfishpass.user_barriers_definite_control p - ON b.blue_line_key = p.blue_line_key - AND abs(b.downstream_route_measure - p.downstream_route_measure) < 1 -WHERE - b.gradient_class = 30 AND - (p.barrier_ind IS NULL or p.barrier_ind is true) AND -- do not include records forced to be passable in control table - b.watershed_group_code = :'wsg' -ORDER BY b.blue_line_key, b.downstream_route_measure -ON CONFLICT DO NOTHING; \ No newline at end of file diff --git a/model/01_access/sql/load_crossings.sql b/model/01_access/sql/load_crossings.sql index 95056802..06676fbf 100644 --- a/model/01_access/sql/load_crossings.sql +++ b/model/01_access/sql/load_crossings.sql @@ -23,10 +23,12 @@ insert into bcfishpass.crossings transport_line_type_description, transport_line_surface_description, ften_forest_file_id, + ften_road_section_id, ften_file_type_description, ften_client_number, ften_client_name, ften_life_cycle_status_code, + ften_map_label, rail_track_name, rail_owner_name, rail_operator_english_name, @@ -56,7 +58,9 @@ select e.current_crossing_type_code as crossing_type_code, e.current_crossing_subtype_code as crossing_subtype_code, case - when mf.structure = 'OBS' THEN array['MANUAL FIX'] -- note modelled crossings that have been manually identified as OBS + when mf.structure = 'OBS' THEN array['MANUAL FIX'] -- tag modelled crossings that have been manually identified as OBS + -- NOTE - why do we do this? modelled crossing type is not retained (replaced by PSCIS crossing type) + -- Why note where it comes from? else m.modelled_crossing_type_source end AS modelled_crossing_type_source, case @@ -75,10 +79,12 @@ select drasurface.description as transport_line_surface_description, ften.forest_file_id as ften_forest_file_id, + ften.road_section_id as ften_road_section_id, ften.file_type_description as ften_file_type_description, ften.client_number as ften_client_number, ften.client_name as ften_client_name, ften.life_cycle_status_code as ften_life_cycle_status_code, + ften.map_label as ften_map_label, rail.track_name as rail_track_name, rail.owner_name AS rail_owner_name, @@ -86,9 +92,9 @@ select coalesce(ogc1.proponent, ogc2.proponent) as ogc_proponent, - substring(to_char(utmzone(e.geom),'999999') from 6 for 2)::int as utm_zone, - st_x(st_transform(e.geom, utmzone(e.geom)))::int as utm_easting, - st_y(st_transform(e.geom, utmzone(e.geom)))::int as utm_northing, + substring(to_char(bcfishpass.utmzone(e.geom),'999999') from 6 for 2)::int as utm_zone, + st_x(st_transform(e.geom, bcfishpass.utmzone(e.geom)))::int as utm_easting, + st_y(st_transform(e.geom, bcfishpass.utmzone(e.geom)))::int as utm_northing, e.linear_feature_id, e.blue_line_key, s.watershed_key, @@ -149,10 +155,12 @@ WITH rail AS NULL as transport_line_type_description, NULL as transport_line_surface_description, NULL as ften_forest_file_id, + NULL as ften_road_section_id, NULL as ften_file_type_description, NULL AS ften_client_number, NULL AS ften_client_name, NULL AS ften_life_cycle_status_code, + NULL AS ften_map_label, track_name as rail_track_name, owner_name as rail_owner_name, operator_english_name as rail_operator_english_name, @@ -198,10 +206,12 @@ ften as ( cross join lateral (select forest_file_id, + road_section_id, file_type_description, client_number, client_name, life_cycle_status_code, + map_label, ST_Distance(rd.geom, pt.geom) as distance_to_road from whse_forest_tenure.ften_road_section_lines_svw AS rd where life_cycle_status_code NOT IN ('PENDING') @@ -223,10 +233,12 @@ roads AS dratype.description AS transport_line_type_description, drasurface.description AS transport_line_surface_description, b.forest_file_id AS ften_forest_file_id, + b.road_section_id as ften_road_section_id, b.file_type_description AS ften_file_type_description, b.client_number AS ften_client_number, b.client_name AS ften_client_name, b.life_cycle_status_code AS ften_life_cycle_status_code, + b.map_label as ften_map_label, NULL as rail_owner_name, NULL as rail_track_name, NULL as rail_operator_english_name, @@ -264,10 +276,12 @@ insert into bcfishpass.crossings transport_line_surface_description, ften_forest_file_id, + ften_road_section_id, ften_file_type_description, ften_client_number, ften_client_name, ften_life_cycle_status_code, + ften_map_label, rail_track_name, rail_owner_name, @@ -309,16 +323,18 @@ select distinct ON (e.stream_crossing_id) r.transport_line_type_description, r.transport_line_surface_description, r.ften_forest_file_id, + r.ften_road_section_id, r.ften_file_type_description, r.ften_client_number, r.ften_client_name, r.ften_life_cycle_status_code, + r.ften_map_label, r.rail_track_name, r.rail_owner_name, r.rail_operator_english_name, - substring(to_char(utmzone(e.geom),'999999') from 6 for 2)::int as utm_zone, - st_x(st_transform(e.geom, utmzone(e.geom)))::int as utm_easting, - st_y(st_transform(e.geom, utmzone(e.geom)))::int as utm_northing, + substring(to_char(bcfishpass.utmzone(e.geom),'999999') from 6 for 2)::int as utm_zone, + st_x(st_transform(e.geom, bcfishpass.utmzone(e.geom)))::int as utm_easting, + st_y(st_transform(e.geom, bcfishpass.utmzone(e.geom)))::int as utm_northing, e.linear_feature_id, e.blue_line_key, s.watershed_key, @@ -396,9 +412,9 @@ select cabd.dam_use, cabd.operating_status as dam_operating_status, - substring(to_char(utmzone(d.geom),'999999') from 6 for 2)::int as utm_zone, - st_x(ST_transform(d.geom, utmzone(d.geom)))::int as utm_easting, - st_y(st_transform(d.geom, utmzone(d.geom)))::int as utm_northing, + substring(to_char(bcfishpass.utmzone(d.geom),'999999') from 6 for 2)::int as utm_zone, + st_x(ST_transform(d.geom, bcfishpass.utmzone(d.geom)))::int as utm_easting, + st_y(st_transform(d.geom, bcfishpass.utmzone(d.geom)))::int as utm_northing, d.linear_feature_id, d.blue_line_key, s.watershed_key, @@ -453,9 +469,9 @@ select 'DAM' AS crossing_subtype_code, 'BARRIER' AS barrier_status, 'USA DAM PLACEHOLDER' as dam_name, - substring(to_char(utmzone(d.geom),'999999') from 6 for 2)::int as utm_zone, - st_x(ST_transform(d.geom, utmzone(d.geom)))::int as utm_easting, - st_y(st_transform(d.geom, utmzone(d.geom)))::int as utm_northing, + substring(to_char(bcfishpass.utmzone(d.geom),'999999') from 6 for 2)::int as utm_zone, + st_x(ST_transform(d.geom, bcfishpass.utmzone(d.geom)))::int as utm_easting, + st_y(st_transform(d.geom, bcfishpass.utmzone(d.geom)))::int as utm_northing, d.linear_feature_id, d.blue_line_key, s.watershed_key, @@ -537,9 +553,9 @@ select 'OTHER' AS crossing_type_code, -- to match up with PSCIS crossing_type_code b.barrier_type AS crossing_subtype_code, 'BARRIER' AS barrier_status, - substring(to_char(utmzone(b.geom),'999999') from 6 for 2)::int as utm_zone, - st_x(st_transform(b.geom, utmzone(b.geom)))::int as utm_easting, - st_y(st_transform(b.geom, utmzone(b.geom)))::int as utm_northing, + substring(to_char(bcfishpass.utmzone(b.geom),'999999') from 6 for 2)::int as utm_zone, + st_x(st_transform(b.geom, bcfishpass.utmzone(b.geom)))::int as utm_easting, + st_y(st_transform(b.geom, bcfishpass.utmzone(b.geom)))::int as utm_northing, b.linear_feature_id, b.blue_line_key, b.watershed_key, @@ -574,10 +590,12 @@ insert into bcfishpass.crossings transport_line_type_description, transport_line_surface_description, ften_forest_file_id, + ften_road_section_id, ften_file_type_description, ften_client_number, ften_client_name, ften_life_cycle_status_code, + ften_map_label, rail_track_name, rail_owner_name, rail_operator_english_name, @@ -619,10 +637,12 @@ select drasurface.description AS transport_line_surface_description, ften.forest_file_id AS ften_forest_file_id, + ften.road_section_id as ften_road_section_id, ften.file_type_description AS ften_file_type_description, ften.client_number AS ften_client_number, ften.client_name AS ften_client_name, ften.life_cycle_status_code AS ften_life_cycle_status_code, + ften.map_label as ften_map_label, rail.track_name AS rail_track_name, rail.owner_name AS rail_owner_name, @@ -630,9 +650,9 @@ select coalesce(ogc1.proponent, ogc2.proponent) as ogc_proponent, - substring(to_char(utmzone(b.geom),'999999') from 6 for 2)::int as utm_zone, - st_x(ST_transform(b.geom, utmzone(b.geom)))::int as utm_easting, - st_y(st_transform(b.geom, utmzone(b.geom)))::int as utm_northing, + substring(to_char(bcfishpass.utmzone(b.geom),'999999') from 6 for 2)::int as utm_zone, + st_x(ST_transform(b.geom, bcfishpass.utmzone(b.geom)))::int as utm_easting, + st_y(st_transform(b.geom, bcfishpass.utmzone(b.geom)))::int as utm_northing, b.linear_feature_id, b.blue_line_key, s.watershed_key, diff --git a/model/01_access/sql/load_observations.sql b/model/01_access/sql/load_observations.sql deleted file mode 100644 index d7cdc18e..00000000 --- a/model/01_access/sql/load_observations.sql +++ /dev/null @@ -1,111 +0,0 @@ --- -------------- --- OBSERVATIONS --- --- extract observations for species of interest from bcfishobs --- -------------- - -truncate bcfishpass.observations; - --- insert records for watersheds of interest / spp of interest -INSERT INTO bcfishpass.observations -( - fish_obsrvtn_event_id, - linear_feature_id, - blue_line_key, - wscode_ltree, - localcode_ltree, - downstream_route_measure, - watershed_group_code, - species_codes, - observation_ids, - observation_dates, - geom -) - --- Convert the boolean species columns in wsg_species_presence into array of species presence for each wsg --- (We could modify the input file's multiple spp columns into a single column of spp codes but current --- design works fine and is easy to validate. Down side is that this query must be modified when spp are added) -WITH wsg_spp AS -( -SELECT - watershed_group_code, string_to_array(array_to_string(ARRAY[bt, ch, cm, co, ct, dv, gr, pk, rb, sk, st, wct], ','),',') as species_codes -FROM ( - SELECT - p.watershed_group_code, - CASE WHEN p.bt is true THEN 'BT' ELSE NULL END as bt, - CASE WHEN p.ch is true THEN 'CH' ELSE NULL END as ch, - CASE WHEN p.cm is true THEN 'CM' ELSE NULL END as cm, - CASE WHEN p.co is true THEN 'CO' ELSE NULL END as co, - CASE WHEN p.ct is true THEN 'CT' ELSE NULL END as ct, - CASE WHEN p.dv is true THEN 'DV' ELSE NULL END as dv, - CASE WHEN p.gr is true THEN 'GR' ELSE NULL END as gr, - CASE WHEN p.pk is true THEN 'PK' ELSE NULL END as pk, - CASE WHEN p.rb is true THEN 'RB' ELSE NULL END as rb, - CASE WHEN p.sk is true THEN 'SK' ELSE NULL END as sk, - CASE WHEN p.st is true THEN 'ST' ELSE NULL END as st, - CASE WHEN p.wct is true THEN 'WCT' ELSE NULL END as wct - FROM bcfishpass.wsg_species_presence p - ) as f -), - --- simplify CT species codes -species_code_remap as ( - select distinct - species_code, - case - when species_code = 'CCT' then 'CT' - when species_code = 'ACT' then 'CT' - when species_code = 'CT/RB' then 'CT' - else species_code - end as species_code_remap - from bcfishobs.fiss_fish_obsrvtn_events_vw e -), - --- extract observations of species of interest, --- within watershed groups where they are noted to occur --- - discarding observations outside of those groups --- - TODO - discard observations noted to be of suspect quality -obs as ( - SELECT - e.fish_observation_point_id, - e.fish_obsrvtn_event_id, - e.linear_feature_id, - e.blue_line_key, - e.wscode_ltree, - e.localcode_ltree, - e.downstream_route_measure, - e.watershed_group_code, - r.species_code_remap as species_code, - e.observation_date - FROM bcfishobs.fiss_fish_obsrvtn_events_vw e - INNER JOIN wsg_spp ON e.watershed_group_code = wsg_spp.watershed_group_code - inner join species_code_remap r on e.species_code = r.species_code - AND array[e.species_code]::text[] && wsg_spp.species_codes -) - -SELECT - p.fish_obsrvtn_event_id, - p.linear_feature_id, - p.blue_line_key, - p.wscode_ltree, - p.localcode_ltree, - p.downstream_route_measure, - p.watershed_group_code, - array_agg(p.species_code) as species_codes, - array_agg(p.fish_observation_point_id) as observation_ids, - array_agg(p.observation_date) as observation_dates, - e.geom -FROM obs p -INNER JOIN bcfishobs.fiss_fish_obsrvtn_events e -ON p.fish_obsrvtn_event_id = e.fish_obsrvtn_event_id -GROUP BY p.fish_obsrvtn_event_id, - p.linear_feature_id, - p.blue_line_key, - p.wscode_ltree, - p.localcode_ltree, - p.downstream_route_measure, - p.watershed_group_code, - e.geom; - --- refresh the view with the new data -refresh materialized view bcfishpass.observations_vw; \ No newline at end of file diff --git a/model/01_access/sql/load_streams_dnstr_species.sql b/model/01_access/sql/load_streams_dnstr_species.sql index 962da8ea..0bc4e384 100644 --- a/model/01_access/sql/load_streams_dnstr_species.sql +++ b/model/01_access/sql/load_streams_dnstr_species.sql @@ -5,10 +5,10 @@ with obsrvtn as ( b.wscode_ltree, b.localcode_ltree, b.downstream_route_measure as meas_b, - unnest(species_codes) as species_code + b.species_code from bcfishpass.streams a - inner join bcfishpass.observations b on + inner join bcfishpass.observations_vw b on fwa_downstream( a.blue_line_key, a.downstream_route_measure, diff --git a/model/01_access/sql/load_streams_upstr_observations.sql b/model/01_access/sql/load_streams_upstr_observations.sql index 827585cd..013f2c2b 100644 --- a/model/01_access/sql/load_streams_upstr_observations.sql +++ b/model/01_access/sql/load_streams_upstr_observations.sql @@ -16,10 +16,10 @@ SELECT b.localcode_ltree, b.downstream_route_measure as meas_b, b.fish_obsrvtn_event_id as upstr_id, - unnest(species_codes) as species_code + b.species_code FROM bcfishpass.streams a - INNER JOIN bcfishpass.observations b ON + INNER JOIN bcfishpass.observations_vw b ON FWA_Upstream( a.blue_line_key, a.downstream_route_measure, diff --git a/model/01_access/sql/model_access_bt.sql b/model/01_access/sql/model_access_bt.sql index 305a9dec..d6215a07 100644 --- a/model/01_access/sql/model_access_bt.sql +++ b/model/01_access/sql/model_access_bt.sql @@ -1,6 +1,7 @@ -with all_barriers as +with barriers as ( select + barriers_gradient_id as barrier_id, barrier_type, barrier_name, linear_feature_id, @@ -10,27 +11,14 @@ with all_barriers as localcode_ltree, watershed_group_code, geom - from bcfishpass.barriers_gradient_25 - where watershed_group_code = :'wsg' - - union all - - select - barrier_type, - barrier_name, - linear_feature_id, - blue_line_key, - downstream_route_measure, - wscode_ltree, - localcode_ltree, - watershed_group_code, - geom - from bcfishpass.barriers_gradient_30 + from bcfishpass.barriers_gradient where watershed_group_code = :'wsg' + and barrier_type in ('GRADIENT_25', 'GRADIENT_30') union all select + barriers_falls_id as barrier_id, barrier_type, barrier_name, linear_feature_id, @@ -46,6 +34,7 @@ with all_barriers as union all select + barriers_subsurfaceflow_id as barrier_id, barrier_type, barrier_name, linear_feature_id, @@ -59,29 +48,21 @@ with all_barriers as where watershed_group_code = :'wsg' ), --- BT observations, plus salmon/steelhead as any features passable by salmon/steehead --- should also be passable by BT -obs as -( - select * - from bcfishpass.observations - where species_codes && array['BT','CH','CM','CO','PK','SK','ST'] is true -), - -barriers as +-- BT observations, plus salmon/steelhead +-- (any features passable by salmon/steehead should also be passable by BT) +obs_upstr as ( - select distinct + select + b.barrier_id, b.barrier_type, - b.barrier_name, - b.linear_feature_id, b.blue_line_key, b.downstream_route_measure, - b.wscode_ltree, - b.localcode_ltree, b.watershed_group_code, - b.geom - from all_barriers b - left outer join obs o + o.species_code as spp, + o.fish_observation_point_id as obs, + o.observation_date as obs_dt + from barriers b + inner join bcfishpass.observations_vw o on fwa_upstream( b.blue_line_key, b.downstream_route_measure, @@ -91,26 +72,94 @@ barriers as o.downstream_route_measure, o.wscode_ltree, o.localcode_ltree, - False, - 1 + false, + 20 -- a large tolerance to discard observations at more or less the same location as the barrier (within 20m) ) - where o.species_codes is null + where o.species_code in ('BT','CH','CM','CO','PK','SK','ST') +), - union all +obs_upstr_n as +( + select + o.barrier_id, + count(o.obs) as n_obs + from obs_upstr o + where o.spp in ('BT','CH','CM','CO','PK','SK','ST') + group by o.barrier_id +), - -- include *all* user added features, even those below observations +-- exclude barriers belown known spawning/rearing habitat +habitat as ( select - barrier_type, - barrier_name, - linear_feature_id, - blue_line_key, - downstream_route_measure, - wscode_ltree, - localcode_ltree, - watershed_group_code, - geom - from bcfishpass.barriers_user_definite - where watershed_group_code = :'wsg' + h.blue_line_key, + h.upstream_route_measure, + s.wscode_ltree, + s.localcode_ltree, + h.watershed_group_code, + h.species_code + from bcfishpass.user_habitat_classification h + inner join whse_basemapping.fwa_stream_networks_sp s + ON s.blue_line_key = h.blue_line_key + and round(h.upstream_route_measure::numeric) >= round(s.downstream_route_measure::numeric) + and round(h.upstream_route_measure::numeric) <= round(s.upstream_route_measure::numeric) + where h.habitat_ind is true + and h.species_code in ('BT','CH','CM','CO','PK','SK','ST') + and s.watershed_group_code = :'wsg' +), + +hab_upstr as +( + select + b.barrier_id, + array_agg(species_code) as species_codes + from barriers b + inner join habitat h + on fwa_upstream( + b.blue_line_key, + b.downstream_route_measure, + b.wscode_ltree, + b.localcode_ltree, + h.blue_line_key, + h.upstream_route_measure, + h.wscode_ltree, + h.localcode_ltree, + false, + 20 -- a large tolerance to discard habitat that ends at more or less the same location as the barrier (within 20m) + ) + group by b.barrier_id +), + + +barriers_filtered as ( + select + b.barrier_id, + b.barrier_type, + b.barrier_name, + b.linear_feature_id, + b.blue_line_key, + b.downstream_route_measure, + b.wscode_ltree, + b.localcode_ltree, + b.watershed_group_code, + b.geom + from barriers b + left outer join obs_upstr_n as o on b.barrier_id = o.barrier_id + left outer join hab_upstr h on b.barrier_id = h.barrier_id + where watershed_group_code = any( + array( + select watershed_group_code + from bcfishpass.wsg_species_presence + where bt is true + ) + ) + -- do not include barriers with + -- - any BT (or salmon/steelhead) observation upstream + -- - confirmed BT (or salmon/steelhead) habitat upstream + and + ( + (o.n_obs is null or o.n_obs < 1) and + h.species_codes is null + ) ) insert into bcfishpass.barriers_bt @@ -126,24 +175,20 @@ insert into bcfishpass.barriers_bt watershed_group_code, geom ) --- add a primary key guaranteed to be unique provincially (presuming unique blkey/measure values within 1m) -select - (((blue_line_key::bigint + 1) - 354087611) * 10000000) + round(downstream_route_measure::bigint) as barrier_load_id, - barrier_type, - barrier_name, - linear_feature_id, - blue_line_key, - downstream_route_measure, - wscode_ltree, - localcode_ltree, - watershed_group_code, - geom -from barriers b -where watershed_group_code = any( - array( - select watershed_group_code - from bcfishpass.wsg_species_presence - where bt is true - ) - ) +select * from barriers_filtered +union all +-- include *all* user added features, even those below observations + select + barriers_user_definite_id as barrier_load_id, + barrier_type, + barrier_name, + linear_feature_id, + blue_line_key, + downstream_route_measure, + wscode_ltree, + localcode_ltree, + watershed_group_code, + geom +from bcfishpass.barriers_user_definite +where watershed_group_code = :'wsg' on conflict do nothing; \ No newline at end of file diff --git a/model/01_access/sql/model_access_ch_cm_co_pk_sk.sql b/model/01_access/sql/model_access_ch_cm_co_pk_sk.sql index d48f08d7..50023770 100644 --- a/model/01_access/sql/model_access_ch_cm_co_pk_sk.sql +++ b/model/01_access/sql/model_access_ch_cm_co_pk_sk.sql @@ -1,7 +1,7 @@ with barriers as ( select - barriers_gradient_15_id as barrier_id, + barriers_gradient_id as barrier_id, barrier_type, barrier_name, linear_feature_id, @@ -11,50 +11,9 @@ with barriers as localcode_ltree, watershed_group_code, geom - from bcfishpass.barriers_gradient_15 - where watershed_group_code = :'wsg' - union all - select - barriers_gradient_20_id as barrier_id, - barrier_type, - barrier_name, - linear_feature_id, - blue_line_key, - downstream_route_measure, - wscode_ltree, - localcode_ltree, - watershed_group_code, - geom - from bcfishpass.barriers_gradient_20 - where watershed_group_code = :'wsg' - union all - select - barriers_gradient_25_id as barrier_id, - barrier_type, - barrier_name, - linear_feature_id, - blue_line_key, - downstream_route_measure, - wscode_ltree, - localcode_ltree, - watershed_group_code, - geom - from bcfishpass.barriers_gradient_25 - where watershed_group_code = :'wsg' - union all - select - barriers_gradient_30_id as barrier_id, - barrier_type, - barrier_name, - linear_feature_id, - blue_line_key, - downstream_route_measure, - wscode_ltree, - localcode_ltree, - watershed_group_code, - geom - from bcfishpass.barriers_gradient_30 + from bcfishpass.barriers_gradient where watershed_group_code = :'wsg' + and barrier_type in ('GRADIENT_15', 'GRADIENT_20', 'GRADIENT_25', 'GRADIENT_30') union all select barriers_falls_id as barrier_id, @@ -93,11 +52,11 @@ obs_upstr as b.blue_line_key, b.downstream_route_measure, b.watershed_group_code, - unnest(o.species_codes) as spp, - unnest(o.observation_ids) as obs, - unnest(o.observation_dates) as obs_dt + o.species_code as spp, + o.fish_observation_point_id as obs, + o.observation_date as obs_dt from barriers b - inner join bcfishpass.observations o + inner join bcfishpass.observations_vw o on fwa_upstream( b.blue_line_key, b.downstream_route_measure, @@ -108,12 +67,12 @@ obs_upstr as o.wscode_ltree, o.localcode_ltree, false, - 1 + 20 -- a large tolerance to discard observations at more or less the same location as the barrier (within 20m) ) -- do not bother counting observations upstream of barriers that have been noted as barriers in the user control table left outer join bcfishpass.user_barriers_definite_control bc on b.blue_line_key = bc.blue_line_key and abs(b.downstream_route_measure - bc.downstream_route_measure) < 1 - where o.species_codes && array['CH','CM','CO','PK','SK'] + where o.species_code in ('CH','CM','CO','PK','SK') and bc.barrier_ind is null ), @@ -164,7 +123,7 @@ hab_upstr as h.wscode_ltree, h.localcode_ltree, false, - 1 + 20 -- a large tolerance to discard habitat that ends at more or less the same location as the barrier (within 20m) ) group by b.barrier_id ), diff --git a/model/01_access/sql/model_access_ct_dv_rb.sql b/model/01_access/sql/model_access_ct_dv_rb.sql index ea6f0ec9..34a210f5 100644 --- a/model/01_access/sql/model_access_ct_dv_rb.sql +++ b/model/01_access/sql/model_access_ct_dv_rb.sql @@ -3,7 +3,7 @@ with all_barriers as ( select - barriers_gradient_25_id as barrier_id, + barriers_gradient_id as barrier_id, barrier_type, barrier_name, linear_feature_id, @@ -13,24 +13,9 @@ with all_barriers as localcode_ltree, watershed_group_code, geom - from bcfishpass.barriers_gradient_25 - where watershed_group_code = :'wsg' - - union all - - select - barriers_gradient_30_id as barrier_id, - barrier_type, - barrier_name, - linear_feature_id, - blue_line_key, - downstream_route_measure, - wscode_ltree, - localcode_ltree, - watershed_group_code, - geom - from bcfishpass.barriers_gradient_30 + from bcfishpass.barriers_gradient where watershed_group_code = :'wsg' + and barrier_type in ('GRADIENT_25', 'GRADIENT_30') union all @@ -69,9 +54,9 @@ with all_barriers as obs as ( select * - from bcfishpass.observations + from bcfishpass.observations_vw -- include bt as equivalent to dv for this model - where species_codes && array['BT','DV','CT','RB'] is true + where species_code in ('BT','DV','CT','RB') ), -- known habitat to any species will be accessible to these species. @@ -142,7 +127,7 @@ barriers as 1 ) left outer join hab_upstr h on b.barrier_id = h.barrier_id - where o.species_codes is null + where o.species_code is null and h.species_codes is null union all diff --git a/model/01_access/sql/model_access_st.sql b/model/01_access/sql/model_access_st.sql index 53eb45c0..903c7281 100644 --- a/model/01_access/sql/model_access_st.sql +++ b/model/01_access/sql/model_access_st.sql @@ -1,46 +1,19 @@ with barriers as ( select - barriers_gradient_20_id as barrier_id, - barrier_type, - barrier_name, - linear_feature_id, - blue_line_key, - downstream_route_measure, - wscode_ltree, - localcode_ltree, - watershed_group_code, - geom - from bcfishpass.barriers_gradient_20 - where watershed_group_code = :'wsg' - union all - select - barriers_gradient_25_id as barrier_id, - barrier_type, - barrier_name, - linear_feature_id, - blue_line_key, - downstream_route_measure, - wscode_ltree, - localcode_ltree, - watershed_group_code, - geom - from bcfishpass.barriers_gradient_25 - where watershed_group_code = :'wsg' - union all - select - barriers_gradient_30_id as barrier_id, - barrier_type, - barrier_name, - linear_feature_id, - blue_line_key, - downstream_route_measure, - wscode_ltree, - localcode_ltree, - watershed_group_code, - geom - from bcfishpass.barriers_gradient_30 + barriers_gradient_id as barrier_id, + barrier_type, + barrier_name, + linear_feature_id, + blue_line_key, + downstream_route_measure, + wscode_ltree, + localcode_ltree, + watershed_group_code, + geom + from bcfishpass.barriers_gradient where watershed_group_code = :'wsg' + and barrier_type in ('GRADIENT_20', 'GRADIENT_25', 'GRADIENT_30') union all select barriers_falls_id as barrier_id, @@ -79,11 +52,11 @@ obs_upstr as b.blue_line_key, b.downstream_route_measure, b.watershed_group_code, - unnest(o.species_codes) as spp, - unnest(o.observation_ids) as obs, - unnest(o.observation_dates) as obs_dt + o.species_code as spp, + o.fish_observation_point_id as obs, + o.observation_date as obs_dt from barriers b - inner join bcfishpass.observations o + inner join bcfishpass.observations_vw o on fwa_upstream( b.blue_line_key, b.downstream_route_measure, @@ -94,12 +67,12 @@ obs_upstr as o.wscode_ltree, o.localcode_ltree, false, - 1 + 20 -- a large tolerance to discard observations at more or less the same location as the barrier (within 20m) ) -- do not bother counting observations upstream of barriers that have been noted as barriers in the user control table left outer join bcfishpass.user_barriers_definite_control bc on b.blue_line_key = bc.blue_line_key and abs(b.downstream_route_measure - bc.downstream_route_measure) < 1 - where o.species_codes && array['CH','CM','CO','PK','SK','ST'] + where o.species_code in ('CH','CM','CO','PK','SK','ST') and bc.barrier_ind is null ), @@ -150,7 +123,7 @@ hab_upstr as h.wscode_ltree, h.localcode_ltree, false, - 1 + 20 -- a large tolerance to discard habitat that ends at more or less the same location as the barrier (within 20m) ) group by b.barrier_id ), diff --git a/model/01_access/sql/model_access_wct.sql b/model/01_access/sql/model_access_wct.sql index d99b976f..2f65540a 100644 --- a/model/01_access/sql/model_access_wct.sql +++ b/model/01_access/sql/model_access_wct.sql @@ -10,38 +10,9 @@ with all_barriers as localcode_ltree, watershed_group_code, geom - from bcfishpass.barriers_gradient_20 - where watershed_group_code = :'wsg' - - union all - - select - barrier_type, - barrier_name, - linear_feature_id, - blue_line_key, - downstream_route_measure, - wscode_ltree, - localcode_ltree, - watershed_group_code, - geom - from bcfishpass.barriers_gradient_25 - where watershed_group_code = :'wsg' - - union all - - select - barrier_type, - barrier_name, - linear_feature_id, - blue_line_key, - downstream_route_measure, - wscode_ltree, - localcode_ltree, - watershed_group_code, - geom - from bcfishpass.barriers_gradient_30 + from bcfishpass.barriers_gradient where watershed_group_code = :'wsg' + and barrier_type in ('GRADIENT_20', 'GRADIENT_25', 'GRADIENT_30') union all @@ -77,8 +48,8 @@ with all_barriers as obs as ( select * - from bcfishpass.observations - where species_codes && array['WCT'] IS TRUE + from bcfishpass.observations_vw + where species_code = 'WCT' ), barriers as @@ -105,9 +76,9 @@ barriers as o.wscode_ltree, o.localcode_ltree, False, - 1 + 20 -- a large tolerance to discard observations at more or less the same location as the barrier (within 20m) ) - where o.species_codes is null + where o.species_code is null union all diff --git a/model/02_habitat_linear/habitat_linear.sh b/model/02_habitat_linear/habitat_linear.sh index 5731ddbf..2490c907 100755 --- a/model/02_habitat_linear/habitat_linear.sh +++ b/model/02_habitat_linear/habitat_linear.sh @@ -51,11 +51,11 @@ model_version=$(git describe) model_run_id=$($PSQL -qtAX -c "insert into bcfishpass.log (model_type, model_version) VALUES ('LINEAR', '$model_version') returning model_run_id") # log parameters -$PSQL -c "insert into bcfishpass.parameters_habitat_method_log +$PSQL -c "insert into bcfishpass.log_parameters_habitat_method (model_run_id, watershed_group_code, model) select $model_run_id, watershed_group_code, model from bcfishpass.parameters_habitat_method;" -$PSQL -c "insert into bcfishpass.parameters_habitat_thresholds_log ( +$PSQL -c "insert into bcfishpass.log_parameters_habitat_thresholds ( model_run_id , species_code , spawn_gradient_max , @@ -87,5 +87,5 @@ $PSQL -c "insert into bcfishpass.parameters_habitat_thresholds_log ( from bcfishpass.parameters_habitat_thresholds;" # log summaries -$PSQL -c "insert into bcfishpass.wsg_linear_summary select $model_run_id as model_run_id, * from bcfishpass.wsg_linear_summary()" -$PSQL -c "insert into bcfishpass.wsg_crossing_summary select $model_run_id as model_run_id, * from bcfishpass.wsg_crossing_summary()" +$PSQL -c "insert into bcfishpass.log_wsg_linear_summary select $model_run_id as model_run_id, * from bcfishpass.wsg_linear_summary()" +$PSQL -c "insert into bcfishpass.log_wsg_crossing_summary select $model_run_id as model_run_id, * from bcfishpass.wsg_crossing_summary()" diff --git a/model/02_habitat_linear/sql/load_crossings_upstream_habitat_wcrp.sql b/model/02_habitat_linear/sql/load_crossings_upstream_habitat_wcrp.sql index 3be94be7..1ce5128f 100644 --- a/model/02_habitat_linear/sql/load_crossings_upstream_habitat_wcrp.sql +++ b/model/02_habitat_linear/sql/load_crossings_upstream_habitat_wcrp.sql @@ -1,10 +1,8 @@ --- report on all ch/co/sk/st/wct habitat present in WCRP watersheds --- note that this query presumes that if any of these species are present --- in a given group, they are all of interest in the reporting - --- ** if adding a new watershed group or target species, additional per-watershed group logic may be required ** --- (for example, SHUL contains co/sk/wct - if WCT were not a target species in this watershed group, logic would be --- required to exclude it from the 'all' computations) +-- report on target species habitat present in WCRP watersheds +-- NOTE +-- If a barrier has another WCRP watershed upstream, the WCRP target species for that upstream watershed will be used +-- to derive the 'all species' spawning/rearing summaries within that watershed. +-- This should not generally be an issue, cross-watershed barriers are generally major dams that are out of scope for WCRP reporting truncate bcfishpass.crossings_upstream_habitat_wcrp; @@ -26,6 +24,11 @@ with upstr as materialized h.rearing_st, h.spawning_wct, h.rearing_wct, + w.ch, + w.co, + w.sk, + w.st, + w.wct, s.edge_type, st_length(s.geom) as length_metre from bcfishpass.crossings a @@ -43,10 +46,8 @@ with upstr as materialized 1 ) inner join bcfishpass.streams_habitat_linear_vw h on s.segmented_stream_id = h.segmented_stream_id - where a.watershed_group_code in ('BULK','LNIC','HORS','BOWR','QUES','CARR','ELKR') - and a.blue_line_key = a.watershed_key -- do not report on crossings on side channels - -- barriers only - and a.barrier_status in ('BARRIER', 'POTENTIAL') + inner join bcfishpass.wcrp_watersheds w on a.watershed_group_code = w.watershed_group_code + where a.blue_line_key = a.watershed_key -- do not report on crossings on side channels ) insert into bcfishpass.crossings_upstream_habitat_wcrp @@ -66,8 +67,8 @@ select round( ( ( - coalesce(sum(length_metre) FILTER (WHERE s.rearing_co IS TRUE), 0) + - coalesce(sum(length_metre * .5) FILTER (WHERE s.rearing_co IS TRUE AND edge_type = 1050), 0) + coalesce(sum(length_metre) FILTER (WHERE s.rearing_co IS TRUE AND s.co IS TRUE), 0) + + coalesce(sum(length_metre * .5) FILTER (WHERE s.rearing_co IS TRUE AND s.co IS TRUE AND edge_type = 1050), 0) ) / 1000 )::numeric, 2 ) AS co_rearing_km, @@ -76,18 +77,20 @@ select round( ( ( - coalesce(sum(length_metre * 1.5) FILTER (WHERE s.rearing_sk IS TRUE), 0) + coalesce(sum(length_metre * 1.5) FILTER (WHERE s.rearing_sk IS TRUE AND s.sk IS TRUE), 0) ) / 1000 )::numeric, 2 ) as sk_rearing_km, -- all spawning - coalesce(round(((sum(length_metre) filter (where - s.spawning_ch is true or - s.spawning_co is true or - s.spawning_sk is true or - s.spawning_st is true or - s.spawning_wct is true) / 1000))::numeric, 2), 0) as all_spawning_km, + coalesce(round(((sum(length_metre) filter ( + where + (s.spawning_ch is true and s.ch is true) or + (s.spawning_co is true and s.co is true) or + (s.spawning_sk is true and s.sk is true) or + (s.spawning_st is true and s.st is true) or + (s.spawning_wct is true and s.wct is true) + ) / 1000))::numeric, 2), 0) as all_spawning_km, -- all rearing round( @@ -95,16 +98,16 @@ select ( coalesce(sum(length_metre) FILTER ( WHERE - s.rearing_ch IS TRUE OR - s.rearing_st IS TRUE OR - s.rearing_sk IS TRUE OR - s.rearing_co IS TRUE OR - s.rearing_wct IS TRUE + (s.rearing_ch IS TRUE AND s.ch IS TRUE) OR + (s.rearing_st IS TRUE AND s.st IS TRUE) OR + (s.rearing_sk IS TRUE AND s.sk IS TRUE) OR + (s.rearing_co IS TRUE AND s.co IS TRUE) OR + (s.rearing_wct IS TRUE AND s.wct IS TRUE) ), 0) + -- add .5 coho rearing in wetlands - coalesce(sum(length_metre * .5) FILTER (WHERE s.rearing_co IS TRUE AND s.edge_type = 1050), 0) + + coalesce(sum(length_metre * .5) FILTER (WHERE s.rearing_co IS TRUE AND s.co IS TRUE AND s.edge_type = 1050), 0) + -- add .5 sockeye rearing in lakes (all of it) - coalesce(sum(length_metre * .5) FILTER (WHERE s.spawning_sk IS TRUE), 0) + coalesce(sum(length_metre * .5) FILTER (WHERE s.spawning_sk IS TRUE AND s.sk IS TRUE), 0) ) / 1000)::numeric, 2 ) as all_rearing_km, @@ -114,21 +117,21 @@ select ( coalesce(sum(length_metre) FILTER ( WHERE - s.spawning_ch is true or - s.spawning_co is true or - s.spawning_sk is true or - s.spawning_st is true or - s.spawning_wct is true or - s.rearing_ch is true or - s.rearing_st is true or - s.rearing_sk is true or - s.rearing_co is true or - s.rearing_wct is true + (s.spawning_ch is true and s.ch is true) or + (s.spawning_co is true and s.co is true) or + (s.spawning_sk is true and s.sk is true) or + (s.spawning_st is true and s.st is true) or + (s.spawning_wct is true and s.wct is true) or + (s.rearing_ch is true and s.ch is true) or + (s.rearing_st is true and s.st is true) or + (s.rearing_sk is true and s.sk is true) or + (s.rearing_co is true and s.co is true) or + (s.rearing_wct is true and s.wct is true) ), 0) + -- add .5 coho rearing in wetlands - coalesce(sum(length_metre * .5) FILTER (WHERE s.rearing_co IS TRUE AND s.edge_type = 1050), 0) + + coalesce(sum(length_metre * .5) FILTER (WHERE s.rearing_co IS TRUE AND s.co IS TRUE AND s.edge_type = 1050), 0) + -- add .5 sockeye rearing in lakes (all of it) - coalesce(sum(length_metre * .5) FILTER (WHERE s.spawning_sk IS TRUE), 0) + coalesce(sum(length_metre * .5) FILTER (WHERE s.spawning_sk IS TRUE AND s.sk IS TRUE), 0) ) / 1000)::numeric, 2 ) as all_spawningrearing_km from upstr s diff --git a/model/02_habitat_linear/sql/load_habitat_linear_bt.sql b/model/02_habitat_linear/sql/load_habitat_linear_bt.sql index e523d94d..a1928799 100644 --- a/model/02_habitat_linear/sql/load_habitat_linear_bt.sql +++ b/model/02_habitat_linear/sql/load_habitat_linear_bt.sql @@ -21,6 +21,8 @@ model AS cw.channel_width, s.gradient, CASE + WHEN hk.spawning_bt is true -- observed/known habitat + THEN true WHEN wsg.model = 'cw' AND s.gradient <= t.spawn_gradient_max AND @@ -45,6 +47,7 @@ model AS LEFT OUTER JOIN bcfishpass.parameters_habitat_thresholds t ON t.species_code = 'BT' INNER JOIN bcfishpass.wsg_species_presence p ON s.watershed_group_code = p.watershed_group_code LEFT OUTER JOIN rivers r ON s.waterbody_key = r.waterbody_key + left outer join bcfishpass.streams_habitat_known_vw hk on s.segmented_stream_id = hk.segmented_stream_id WHERE p.bt is true AND s.watershed_group_code = :'wsg' AND @@ -337,4 +340,26 @@ FROM rearing_clusters a INNER JOIN valid_rearing b ON a.cid = b.cid on conflict (segmented_stream_id) +do update set rearing = EXCLUDED.rearing; + + +-- finally, add any known/observed rearing +with observed_rearing as ( + select + hk.segmented_stream_id + from bcfishpass.streams_habitat_known_vw hk + left outer join bcfishpass.streams s + on hk.segmented_stream_id = s.segmented_stream_id + where rearing_bt is true + and s.watershed_group_code = :'wsg' +) +INSERT INTO bcfishpass.habitat_linear_bt ( + segmented_stream_id, + rearing +) +SELECT + a.segmented_stream_id, + true as rearing +FROM observed_rearing a +on conflict (segmented_stream_id) do update set rearing = EXCLUDED.rearing; \ No newline at end of file diff --git a/model/02_habitat_linear/sql/load_habitat_linear_ch.sql b/model/02_habitat_linear/sql/load_habitat_linear_ch.sql index b4a62062..ee666a04 100644 --- a/model/02_habitat_linear/sql/load_habitat_linear_ch.sql +++ b/model/02_habitat_linear/sql/load_habitat_linear_ch.sql @@ -20,6 +20,8 @@ model AS cw.channel_width, s.gradient, CASE + WHEN hk.spawning_ch is true -- observed/known habitat + THEN true WHEN wsg.model = 'cw' AND s.gradient <= t.spawn_gradient_max AND @@ -40,12 +42,13 @@ model AS inner join bcfishpass.streams_access_vw av on s.segmented_stream_id = av.segmented_stream_id left outer join whse_basemapping.fwa_stream_networks_channel_width cw on s.linear_feature_id = cw.linear_feature_id left outer join whse_basemapping.fwa_stream_networks_discharge mad on s.linear_feature_id = mad.linear_feature_id - INNER JOIN bcfishpass.parameters_habitat_method wsg ON s.watershed_group_code = wsg.watershed_group_code - LEFT OUTER JOIN whse_basemapping.fwa_waterbodies wb ON s.waterbody_key = wb.waterbody_key - LEFT OUTER JOIN bcfishpass.parameters_habitat_thresholds t ON t.species_code = 'CH' - INNER JOIN bcfishpass.wsg_species_presence p ON s.watershed_group_code = p.watershed_group_code - LEFT OUTER JOIN rivers r ON s.waterbody_key = r.waterbody_key - WHERE + inner join bcfishpass.parameters_habitat_method wsg on s.watershed_group_code = wsg.watershed_group_code + left outer join whse_basemapping.fwa_waterbodies wb on s.waterbody_key = wb.waterbody_key + left outer join bcfishpass.parameters_habitat_thresholds t on t.species_code = 'CH' + inner join bcfishpass.wsg_species_presence p on s.watershed_group_code = p.watershed_group_code + left outer join rivers r on s.waterbody_key = r.waterbody_key + left outer join bcfishpass.streams_habitat_known_vw hk on s.segmented_stream_id = hk.segmented_stream_id + where p.ch is true AND s.watershed_group_code = :'wsg' AND -- streams and rivers only @@ -354,4 +357,25 @@ FROM rearing_clusters a INNER JOIN valid_rearing b ON a.cid = b.cid on conflict (segmented_stream_id) +do update set rearing = EXCLUDED.rearing; + +-- finally, add any known/observed rearing +with observed_rearing as ( + select + hk.segmented_stream_id + from bcfishpass.streams_habitat_known_vw hk + left outer join bcfishpass.streams s + on hk.segmented_stream_id = s.segmented_stream_id + where rearing_ch is true + and s.watershed_group_code = :'wsg' +) +INSERT INTO bcfishpass.habitat_linear_ch ( + segmented_stream_id, + rearing +) +SELECT + a.segmented_stream_id, + true as rearing +FROM observed_rearing a +on conflict (segmented_stream_id) do update set rearing = EXCLUDED.rearing; \ No newline at end of file diff --git a/model/02_habitat_linear/sql/load_habitat_linear_cm.sql b/model/02_habitat_linear/sql/load_habitat_linear_cm.sql index ad7fce47..34f32566 100644 --- a/model/02_habitat_linear/sql/load_habitat_linear_cm.sql +++ b/model/02_habitat_linear/sql/load_habitat_linear_cm.sql @@ -21,6 +21,8 @@ model AS cw.channel_width, s.gradient, CASE + WHEN hk.spawning_cm is true -- observed/known habitat + THEN true WHEN wsg.model = 'cw' AND s.gradient <= t.spawn_gradient_max AND @@ -45,6 +47,7 @@ model AS LEFT OUTER JOIN bcfishpass.parameters_habitat_thresholds t ON t.species_code = 'CM' INNER JOIN bcfishpass.wsg_species_presence p ON s.watershed_group_code = p.watershed_group_code LEFT OUTER JOIN rivers r ON s.waterbody_key = r.waterbody_key + left outer join bcfishpass.streams_habitat_known_vw hk on s.segmented_stream_id = hk.segmented_stream_id WHERE p.cm is true AND s.watershed_group_code = :'wsg' AND diff --git a/model/02_habitat_linear/sql/load_habitat_linear_co.sql b/model/02_habitat_linear/sql/load_habitat_linear_co.sql index fa9bed4e..7bb272a9 100644 --- a/model/02_habitat_linear/sql/load_habitat_linear_co.sql +++ b/model/02_habitat_linear/sql/load_habitat_linear_co.sql @@ -20,6 +20,8 @@ model AS cw.channel_width, s.gradient, CASE + WHEN hk.spawning_co is true -- observed/known habitat + THEN true WHEN wsg.model = 'cw' AND s.gradient <= t.spawn_gradient_max AND @@ -45,6 +47,7 @@ model AS LEFT OUTER JOIN bcfishpass.parameters_habitat_thresholds t ON t.species_code = 'CO' INNER JOIN bcfishpass.wsg_species_presence p ON s.watershed_group_code = p.watershed_group_code LEFT OUTER JOIN rivers r ON s.waterbody_key = r.waterbody_key + left outer join bcfishpass.streams_habitat_known_vw hk on s.segmented_stream_id = hk.segmented_stream_id WHERE p.co is true AND s.watershed_group_code = :'wsg' AND @@ -358,4 +361,25 @@ FROM rearing_clusters a INNER JOIN valid_rearing b ON a.cid = b.cid on conflict (segmented_stream_id) +do update set rearing = EXCLUDED.rearing; + +-- finally, add any known/observed rearing +with observed_rearing as ( + select + hk.segmented_stream_id + from bcfishpass.streams_habitat_known_vw hk + left outer join bcfishpass.streams s + on hk.segmented_stream_id = s.segmented_stream_id + where rearing_co is true + and s.watershed_group_code = :'wsg' +) +INSERT INTO bcfishpass.habitat_linear_co ( + segmented_stream_id, + rearing +) +SELECT + a.segmented_stream_id, + true as rearing +FROM observed_rearing a +on conflict (segmented_stream_id) do update set rearing = EXCLUDED.rearing; \ No newline at end of file diff --git a/model/02_habitat_linear/sql/load_habitat_linear_pk.sql b/model/02_habitat_linear/sql/load_habitat_linear_pk.sql index 200401a4..6d72f3de 100644 --- a/model/02_habitat_linear/sql/load_habitat_linear_pk.sql +++ b/model/02_habitat_linear/sql/load_habitat_linear_pk.sql @@ -21,6 +21,8 @@ model AS cw.channel_width, s.gradient, CASE + WHEN hk.spawning_pk is true -- observed/known habitat + THEN true WHEN wsg.model = 'cw' AND s.gradient <= t.spawn_gradient_max AND @@ -45,6 +47,7 @@ model AS LEFT OUTER JOIN bcfishpass.parameters_habitat_thresholds t ON t.species_code = 'PK' INNER JOIN bcfishpass.wsg_species_presence p ON s.watershed_group_code = p.watershed_group_code LEFT OUTER JOIN rivers r ON s.waterbody_key = r.waterbody_key + left outer join bcfishpass.streams_habitat_known_vw hk on s.segmented_stream_id = hk.segmented_stream_id WHERE p.pk is true AND s.watershed_group_code = :'wsg' AND diff --git a/model/02_habitat_linear/sql/load_habitat_linear_sk.sql b/model/02_habitat_linear/sql/load_habitat_linear_sk.sql index a1dd365e..5610b24f 100644 --- a/model/02_habitat_linear/sql/load_habitat_linear_sk.sql +++ b/model/02_habitat_linear/sql/load_habitat_linear_sk.sql @@ -251,3 +251,25 @@ ON a.cid = b.cid WHERE a.wb is true on conflict (segmented_stream_id) do update set spawning = EXCLUDED.spawning; + + +-- finally, add any known/observed spawning +with observed_spawning as ( + select + hk.segmented_stream_id + from bcfishpass.streams_habitat_known_vw hk + left outer join bcfishpass.streams s + on hk.segmented_stream_id = s.segmented_stream_id + where spawning_sk is true + and s.watershed_group_code = :'wsg' +) +INSERT INTO bcfishpass.habitat_linear_sk ( + segmented_stream_id, + spawning +) +SELECT + a.segmented_stream_id, + true as spawning +FROM observed_spawning a +on conflict (segmented_stream_id) +do update set spawning = EXCLUDED.spawning; diff --git a/model/02_habitat_linear/sql/load_habitat_linear_st.sql b/model/02_habitat_linear/sql/load_habitat_linear_st.sql index f0902653..43c9ae4f 100644 --- a/model/02_habitat_linear/sql/load_habitat_linear_st.sql +++ b/model/02_habitat_linear/sql/load_habitat_linear_st.sql @@ -10,47 +10,50 @@ WITH rivers AS -- get unique river waterbodies, there are some duplicates FROM whse_basemapping.fwa_rivers_poly ), -model AS -(SELECT - s.segmented_stream_id, - s.blue_line_key, - s.wscode_ltree, - s.localcode_ltree, - cw.channel_width, - s.gradient, - av.barriers_st_dnstr, - CASE - WHEN - wsg.model = 'cw' AND - s.gradient <= t.spawn_gradient_max AND - (cw.channel_width > t.spawn_channel_width_min OR r.waterbody_key IS NOT NULL) AND - cw.channel_width <= t.spawn_channel_width_max AND - av.barriers_st_dnstr = array[]::text[] - THEN true - WHEN - wsg.model = 'mad' AND +model AS ( + SELECT + s.segmented_stream_id, + s.blue_line_key, + s.wscode_ltree, + s.localcode_ltree, + cw.channel_width, + s.gradient, + av.barriers_st_dnstr, + CASE + WHEN hk.spawning_st is true -- observed/known habitat + THEN true + WHEN + wsg.model = 'cw' AND s.gradient <= t.spawn_gradient_max AND - (mad.mad_m3s > t.spawn_mad_min OR - s.stream_order >= 8) AND + (cw.channel_width > t.spawn_channel_width_min OR r.waterbody_key IS NOT NULL) AND + cw.channel_width <= t.spawn_channel_width_max AND av.barriers_st_dnstr = array[]::text[] THEN true - END AS spawning -FROM bcfishpass.streams s -inner join bcfishpass.streams_access_vw av on s.segmented_stream_id = av.segmented_stream_id -left outer join whse_basemapping.fwa_stream_networks_channel_width cw on s.linear_feature_id = cw.linear_feature_id -left outer join whse_basemapping.fwa_stream_networks_discharge mad on s.linear_feature_id = mad.linear_feature_id -INNER JOIN bcfishpass.parameters_habitat_method wsg ON s.watershed_group_code = wsg.watershed_group_code -LEFT OUTER JOIN whse_basemapping.fwa_waterbodies wb ON s.waterbody_key = wb.waterbody_key -LEFT OUTER JOIN bcfishpass.parameters_habitat_thresholds t ON t.species_code = 'ST' -INNER JOIN bcfishpass.wsg_species_presence p ON s.watershed_group_code = p.watershed_group_code -LEFT OUTER JOIN rivers r ON s.waterbody_key = r.waterbody_key -WHERE - p.st is true AND - s.watershed_group_code = :'wsg' AND - -- streams and rivers only - ( - wb.waterbody_type = 'R' OR (wb.waterbody_type IS NULL AND s.edge_type IN (1000,1100,2000,2300)) - ) + WHEN + wsg.model = 'mad' AND + s.gradient <= t.spawn_gradient_max AND + (mad.mad_m3s > t.spawn_mad_min OR + s.stream_order >= 8) AND + av.barriers_st_dnstr = array[]::text[] + THEN true + END AS spawning + FROM bcfishpass.streams s + inner join bcfishpass.streams_access_vw av on s.segmented_stream_id = av.segmented_stream_id + left outer join whse_basemapping.fwa_stream_networks_channel_width cw on s.linear_feature_id = cw.linear_feature_id + left outer join whse_basemapping.fwa_stream_networks_discharge mad on s.linear_feature_id = mad.linear_feature_id + INNER JOIN bcfishpass.parameters_habitat_method wsg ON s.watershed_group_code = wsg.watershed_group_code + LEFT OUTER JOIN whse_basemapping.fwa_waterbodies wb ON s.waterbody_key = wb.waterbody_key + LEFT OUTER JOIN bcfishpass.parameters_habitat_thresholds t ON t.species_code = 'ST' + INNER JOIN bcfishpass.wsg_species_presence p ON s.watershed_group_code = p.watershed_group_code + LEFT OUTER JOIN rivers r ON s.waterbody_key = r.waterbody_key + left outer join bcfishpass.streams_habitat_known_vw hk on s.segmented_stream_id = hk.segmented_stream_id + WHERE + p.st is true AND + s.watershed_group_code = :'wsg' AND + -- streams and rivers only + ( + wb.waterbody_type = 'R' OR (wb.waterbody_type IS NULL AND s.edge_type IN (1000,1100,2000,2300)) + ) ) insert into bcfishpass.habitat_linear_st @@ -351,4 +354,25 @@ FROM rearing_clusters a INNER JOIN valid_rearing b ON a.cid = b.cid on conflict (segmented_stream_id) +do update set rearing = EXCLUDED.rearing; + +-- finally, add any known/observed rearing +with observed_rearing as ( + select + hk.segmented_stream_id + from bcfishpass.streams_habitat_known_vw hk + left outer join bcfishpass.streams s + on hk.segmented_stream_id = s.segmented_stream_id + where rearing_st is true + and s.watershed_group_code = :'wsg' +) +INSERT INTO bcfishpass.habitat_linear_st ( + segmented_stream_id, + rearing +) +SELECT + a.segmented_stream_id, + true as rearing +FROM observed_rearing a +on conflict (segmented_stream_id) do update set rearing = EXCLUDED.rearing; \ No newline at end of file diff --git a/model/02_habitat_linear/sql/load_habitat_linear_wct.sql b/model/02_habitat_linear/sql/load_habitat_linear_wct.sql index 226b36c1..5fbbe75b 100644 --- a/model/02_habitat_linear/sql/load_habitat_linear_wct.sql +++ b/model/02_habitat_linear/sql/load_habitat_linear_wct.sql @@ -20,6 +20,8 @@ model AS ( cw.channel_width, s.gradient, CASE + WHEN hk.spawning_wct is true -- observed/known habitat + THEN true WHEN wsg.model = 'cw' AND s.gradient <= t.spawn_gradient_max AND @@ -44,6 +46,7 @@ model AS ( LEFT OUTER JOIN bcfishpass.parameters_habitat_thresholds t ON t.species_code = 'WCT' INNER JOIN bcfishpass.wsg_species_presence p ON s.watershed_group_code = p.watershed_group_code LEFT OUTER JOIN rivers r ON s.waterbody_key = r.waterbody_key + left outer join bcfishpass.streams_habitat_known_vw hk on s.segmented_stream_id = hk.segmented_stream_id WHERE p.wct is true AND s.watershed_group_code = :'wsg' AND @@ -353,4 +356,26 @@ FROM rearing_clusters a INNER JOIN valid_rearing b ON a.cid = b.cid on conflict (segmented_stream_id) +do update set rearing = EXCLUDED.rearing; + + +-- finally, add any known/observed rearing +with observed_rearing as ( + select + hk.segmented_stream_id + from bcfishpass.streams_habitat_known_vw hk + left outer join bcfishpass.streams s + on hk.segmented_stream_id = s.segmented_stream_id + where rearing_wct is true + and s.watershed_group_code = :'wsg' +) +INSERT INTO bcfishpass.habitat_linear_wct ( + segmented_stream_id, + rearing +) +SELECT + a.segmented_stream_id, + true as rearing +FROM observed_rearing a +on conflict (segmented_stream_id) do update set rearing = EXCLUDED.rearing; \ No newline at end of file diff --git a/model/02_habitat_linear/sql/streams_model_habitat.sql b/model/02_habitat_linear/sql/streams_model_habitat.sql deleted file mode 100644 index 1248d0f0..00000000 --- a/model/02_habitat_linear/sql/streams_model_habitat.sql +++ /dev/null @@ -1,390 +0,0 @@ ---- load habitat model data into stream table -with habitat as ( - select - s.segmented_stream_id, - case - when s.remediated_dnstr_ind is true then 'REMEDIATED' -- remediated crossing is downstream (with no additional barriers in between) - when s.dam_dnstr_ind is true then 'DAM' -- a dam is the next barrier downstream - when - s.barriers_anthropogenic_dnstr is not null and -- a pscis barrier is downstream - s.barriers_pscis_dnstr is not null and - s.dam_dnstr_ind is false - then 'ASSESSED' - when - s.barriers_anthropogenic_dnstr is not null and -- a modelled barrier is downstream - s.barriers_pscis_dnstr is null and - s.dam_dnstr_ind is false - then 'MODELLED' - when s.barriers_anthropogenic_dnstr is null then 'NONE' -- no barriers exist downstream - - end as mapping_code_barrier, - case - when s.feature_code = 'GA24850150' then 'INTERMITTENT' - else NULL - end as mapping_code_intermittent, - coalesce(u.spawning_bt, bt.spawning, false) as model_spawning_bt, - coalesce(u.spawning_ch, ch.spawning, false) as model_spawning_ch, - coalesce(u.spawning_cm, cm.spawning, false) as model_spawning_cm, - coalesce(u.spawning_co, co.spawning, false) as model_spawning_co, - coalesce(u.spawning_pk, pk.spawning, false) as model_spawning_pk, - coalesce(u.spawning_sk, sk.spawning, false) as model_spawning_sk, - coalesce(u.spawning_st, st.spawning, false) as model_spawning_st, - coalesce(u.spawning_wct, wct.spawning, false) as model_spawning_wct, - coalesce(u.rearing_bt, bt.rearing, false) as model_rearing_bt, - coalesce(u.rearing_ch, ch.rearing, false) as model_rearing_ch, - coalesce(u.rearing_co, co.rearing, false) as model_rearing_co, - coalesce(u.rearing_sk, sk.rearing, false) as model_rearing_sk, - coalesce(u.rearing_st, st.rearing, false) as model_rearing_st, - coalesce(u.rearing_wct, wct.rearing, false) as model_rearing_wct - from bcfishpass.streams s - left outer join bcfishpass.habitat_bt bt on s.segmented_stream_id = bt.segmented_stream_id - left outer join bcfishpass.habitat_ch ch on s.segmented_stream_id = ch.segmented_stream_id - left outer join bcfishpass.habitat_cm cm on s.segmented_stream_id = cm.segmented_stream_id - left outer join bcfishpass.habitat_co co on s.segmented_stream_id = co.segmented_stream_id - left outer join bcfishpass.habitat_pk pk on s.segmented_stream_id = pk.segmented_stream_id - left outer join bcfishpass.habitat_sk sk on s.segmented_stream_id = sk.segmented_stream_id - left outer join bcfishpass.habitat_st st on s.segmented_stream_id = st.segmented_stream_id - left outer join bcfishpass.habitat_wct wct on s.segmented_stream_id = wct.segmented_stream_id - left outer join bcfishpass.streams_habitat_known_vw u on s.segmented_stream_id = u.segmented_stream_id - where s.watershed_group_code = :'wsg' -) - -insert into bcfishpass.streams_model_habitat ( - linear_feature_id, - edge_type, - blue_line_key, - watershed_key, - watershed_group_code, - waterbody_key, - wscode_ltree, - localcode_ltree, - gnis_name, - stream_order, - stream_magnitude, - feature_code, - upstream_area_ha, - stream_order_parent, - stream_order_max, - map_upstream, - channel_width, - mad_m3s, - barriers_anthropogenic_dnstr, - barriers_pscis_dnstr, - barriers_dams_dnstr, - barriers_dams_hydro_dnstr, - barriers_bt_dnstr, - barriers_ch_cm_co_pk_sk_dnstr, - barriers_ct_dv_rb_dnstr, - barriers_st_dnstr, - barriers_wct_dnstr, - obsrvtn_event_upstr, - obsrvtn_species_codes_upstr, - species_codes_dnstr, - crossings_dnstr, - dam_dnstr_ind, - dam_hydro_dnstr_ind, - remediated_dnstr_ind, - model_spawning_bt, - model_spawning_ch, - model_spawning_cm, - model_spawning_co, - model_spawning_pk, - model_spawning_sk, - model_spawning_st, - model_spawning_wct, - model_rearing_bt, - model_rearing_ch, - model_rearing_co, - model_rearing_sk, - model_rearing_st, - model_rearing_wct, - mapping_code_bt, - mapping_code_ch, - mapping_code_cm, - mapping_code_co, - mapping_code_pk, - mapping_code_sk, - mapping_code_st, - mapping_code_wct, - mapping_code_salmon, - geom -) -select - s.linear_feature_id, - s.edge_type, - s.blue_line_key, - s.watershed_key, - s.watershed_group_code, - s.waterbody_key, - s.wscode_ltree, - s.localcode_ltree, - s.gnis_name, - s.stream_order, - s.stream_magnitude, - s.feature_code, - s.upstream_area_ha, - s.stream_order_parent, - s.stream_order_max, - s.map_upstream, - s.channel_width, - s.mad_m3s, - s.barriers_anthropogenic_dnstr, - s.barriers_pscis_dnstr, - s.barriers_dams_dnstr, - s.barriers_dams_hydro_dnstr, - s.barriers_bt_dnstr, - s.barriers_ch_cm_co_pk_sk_dnstr, - s.barriers_ct_dv_rb_dnstr, - s.barriers_st_dnstr, - s.barriers_wct_dnstr, - s.obsrvtn_event_upstr, - s.obsrvtn_species_codes_upstr, - s.species_codes_dnstr, - s.crossings_dnstr, - s.dam_dnstr_ind, - s.dam_hydro_dnstr_ind, - s.remediated_dnstr_ind, - h.model_spawning_bt, - h.model_spawning_ch, - h.model_spawning_cm, - h.model_spawning_co, - h.model_spawning_pk, - h.model_spawning_sk, - h.model_spawning_st, - h.model_spawning_wct, - h.model_rearing_bt, - h.model_rearing_ch, - h.model_rearing_co, - h.model_rearing_sk, - h.model_rearing_st, - h.model_rearing_wct, - -- per-species mapping codes for easy symbolization - array_to_string(array[ - case - when - s.barriers_bt_dnstr = array[]::text[] and - h.model_spawning_bt is false and - h.model_rearing_bt is false - then 'ACCESS' - when h.model_spawning_bt is true - then 'SPAWN' - when - h.model_spawning_bt is false and - h.model_rearing_bt is true - then 'REAR' - end, - case - when s.barriers_bt_dnstr = array[]::text[] - then h.mapping_code_barrier - else null end, - case - when s.barriers_bt_dnstr = array[]::text[] - then h.mapping_code_intermittent - else null end - ], ';') as mapping_code_bt, - - array_to_string(array[ - case - when - s.barriers_ch_cm_co_pk_sk_dnstr = array[]::text[] and - h.model_spawning_ch is false and - h.model_rearing_ch is false - then 'ACCESS' - when h.model_spawning_ch is true - then 'SPAWN' - when - h.model_spawning_ch is false and - h.model_rearing_ch is true - then 'REAR' - end, - case - when s.barriers_ch_cm_co_pk_sk_dnstr = array[]::text[] - then h.mapping_code_barrier - else null end, - case - when s.barriers_ch_cm_co_pk_sk_dnstr = array[]::text[] - then h.mapping_code_intermittent - else null end - ], ';') as mapping_code_ch, - - array_to_string(array[ - case - when - s.barriers_ch_cm_co_pk_sk_dnstr = array[]::text[] and - h.model_spawning_cm is false - then 'ACCESS' - when h.model_spawning_cm is true - then 'SPAWN' - end, - case - when s.barriers_ch_cm_co_pk_sk_dnstr = array[]::text[] - then h.mapping_code_barrier - else null end, - case - when s.barriers_ch_cm_co_pk_sk_dnstr = array[]::text[] - then h.mapping_code_intermittent - else null end - ], ';') as mapping_code_cm, - - array_to_string(array[ - case - when - s.barriers_ch_cm_co_pk_sk_dnstr = array[]::text[] and - h.model_spawning_co is false and - h.model_rearing_co is false - then 'ACCESS' - when h.model_spawning_co is true - then 'SPAWN' - when - h.model_spawning_co is false and - h.model_rearing_co is true - then 'REAR' - end, - case - when s.barriers_ch_cm_co_pk_sk_dnstr = array[]::text[] - then h.mapping_code_barrier - else null end, - case - when s.barriers_ch_cm_co_pk_sk_dnstr = array[]::text[] - then h.mapping_code_intermittent - else null end - ], ';') as mapping_code_co, - - array_to_string(array[ - case - when - s.barriers_ch_cm_co_pk_sk_dnstr = array[]::text[] and - h.model_spawning_pk is false - then 'ACCESS' - when h.model_spawning_pk is true - then 'SPAWN' - end, - case - when s.barriers_ch_cm_co_pk_sk_dnstr = array[]::text[] - then h.mapping_code_barrier - else null end, - case - when s.barriers_ch_cm_co_pk_sk_dnstr = array[]::text[] - then h.mapping_code_intermittent - else null end - ], ';') as mapping_code_pk, - - array_to_string(array[ - case - when - s.barriers_ch_cm_co_pk_sk_dnstr = array[]::text[] and - h.model_spawning_sk is false and - h.model_rearing_sk is false - then 'ACCESS' - when h.model_spawning_sk is true - then 'SPAWN' - when - h.model_spawning_sk is false and - h.model_rearing_sk is true - then 'REAR' - end, - case - when s.barriers_ch_cm_co_pk_sk_dnstr = array[]::text[] - then h.mapping_code_barrier - else null end, - case - when s.barriers_ch_cm_co_pk_sk_dnstr = array[]::text[] - then h.mapping_code_intermittent - else null end - ], ';') as mapping_code_sk, - - array_to_string(array[ - case - when - s.barriers_st_dnstr = array[]::text[] and - h.model_spawning_st is false and - h.model_rearing_st is false - then 'ACCESS' - when h.model_spawning_st is true - then 'SPAWN' - when - h.model_spawning_st is false and - h.model_rearing_st is true - then 'REAR' - end, - case - when s.barriers_st_dnstr = array[]::text[] - then h.mapping_code_barrier - else null end, - case - when s.barriers_st_dnstr = array[]::text[] - then h.mapping_code_intermittent - else null end - ], ';') as mapping_code_st, - - array_to_string(array[ - case - when - s.barriers_wct_dnstr = array[]::text[] and - h.model_spawning_wct is false and - h.model_rearing_wct is false - then 'ACCESS' - when h.model_spawning_wct is true - then 'SPAWN' - when - h.model_spawning_wct is false and - h.model_rearing_wct is true - then 'REAR' - end, - case - when s.barriers_wct_dnstr = array[]::text[] - then h.mapping_code_barrier - else null end, - case - when s.barriers_wct_dnstr = array[]::text[] - then h.mapping_code_intermittent - else null end - ], ';') as mapping_code_wct, - array_to_string(array[ - -- combined salmon mapping code - case - when - barriers_ch_cm_co_pk_sk_dnstr = array[]::text[] and - h.model_spawning_ch is false and - h.model_spawning_cm is false and - h.model_spawning_co is false and - h.model_spawning_pk is false and - h.model_spawning_sk is false and - h.model_rearing_ch is false and - h.model_rearing_co is false and - h.model_rearing_sk is false - then 'ACCESS' - -- potential spawning - when - h.model_spawning_ch is true or - h.model_spawning_cm is true or - h.model_spawning_co is true or - h.model_spawning_pk is true or - h.model_spawning_sk is true - then 'SPAWN' - -- potential rearing (and not spawning) - when - h.model_spawning_ch is false and - h.model_spawning_cm is false and - h.model_spawning_co is false and - h.model_spawning_pk is false and - h.model_spawning_sk is false and - ( - h.model_rearing_ch is true or - h.model_rearing_co is true or - h.model_rearing_sk is true - ) - then 'REAR' - end, - case - when s.barriers_ch_cm_co_pk_sk_dnstr = array[]::text[] - then h.mapping_code_barrier - else null end, - case - when s.barriers_ch_cm_co_pk_sk_dnstr = array[]::text[] - then h.mapping_code_intermittent - else null end - ], ';') as mapping_code_salmon, - geom -from bcfishpass.streams s -inner join habitat h on s.segmented_stream_id = h.segmented_stream_id diff --git a/parameters/example_testing/parameters_habitat_method.csv b/parameters/example_testing/parameters_habitat_method.csv index 293807ec..9a9a00ff 100644 --- a/parameters/example_testing/parameters_habitat_method.csv +++ b/parameters/example_testing/parameters_habitat_method.csv @@ -1,8 +1,10 @@ -watershed_group_code,model -BULK,cw -ELKR,cw -HORS,mad -KETL,cw -LNIC,mad -SANJ,cw +watershed_group_code,model +BELA,cw +BULK,cw +COWN,cw +ELKR,cw +HORS,mad +LNIC,mad +PARS,cw +SANJ,cw VICT,cw \ No newline at end of file diff --git a/test/README.md b/test/README.md new file mode 100644 index 00000000..049eb9fa --- /dev/null +++ b/test/README.md @@ -0,0 +1,21 @@ +# A minimal db for development and testing + +## Create dump file + +Bootstrap database `bcfishpass_test` with fwapg/bcfishobs, load selected data, dump to file, drop db: + + ./build.sh + +## Testing usage + +Restore fwapg/bcfishobs db from dump, load data, run model: + + ./test.sh + +## Validation + +1. obviously, did all jobs complete? +2. is modelled habitat / barrier count / etc reasonably equivalent to previous output? + +For item 2, how do we define/record previous outputs to make a comparison? +Multiple runs in the same testing db works for local dev, and for current GHA workflows - but for a GHA workflow where a temp db is created for the build, compare file based outputs/summaries? \ No newline at end of file diff --git a/test/build.sh b/test/build.sh new file mode 100755 index 00000000..e7f73fc1 --- /dev/null +++ b/test/build.sh @@ -0,0 +1,112 @@ +#!/bin/bash +set -euxo pipefail + +# ------ +# Build a minimal fwapg/bcfishobs database for testing +# ------ + +DATABASE_URL=postgresql://postgres@localhost:5432/bcfishpass_test +PSQL="psql $DATABASE_URL -v ON_ERROR_STOP=1" +WSGS="BELA\nBULK\nCOWN\nELKR\nHORS\nLNIC\nPARS\nSANJ\nVICT" # edit here to adjust testing watersheds (could pull from parameters/example_testing/parameters_habitat_method) + +createdb bcfishpass_test + +# ------ +# load_fwa +# ------ +# load only watershed groups of interest +# wsg.txt controls what watersheds are loaded (for chunked data) + +git clone https://github.com/smnorris/fwapg +cd fwapg +echo -e $WSGS > wsg.txt +make --debug=basic .make/fwa_stream_networks_sp +make --debug=basic .make/fwa_watershed_groups_poly +make --debug=basic .make/fwa_assessment_watersheds_poly +make --debug=basic .make/extras + +# keep the data slim - retain only in testing groups (where wsg defined) +for table in fwa_watershed_groups_poly \ + fwa_assessment_watersheds_poly \ + fwa_stream_networks_discharge \ + fwa_stream_networks_mean_annual_precip; + do + $PSQL -c "delete from $table where watershed_group_code not in (select distinct watershed_group_code from whse_basemapping.fwa_stream_networks_sp)"; +done + +$PSQL -c "delete from whse_basemapping.fwa_stream_networks_channel_width where linear_feature_id not in (select linear_feature_id from whse_basemapping.fwa_stream_networks_sp)" +$PSQL -c "delete from whse_basemapping.fwa_assessment_watersheds_lut where watershed_feature_id not in (select watershed_feature_id from whse_basemapping.fwa_assessment_watersheds_poly)" +$PSQL -c "delete from whse_basemapping.fwa_assessment_watersheds_streams_lut where assmnt_watershed_id not in (select watershed_feature_id from whse_basemapping.fwa_assessment_watersheds_poly)" +$PSQL -c "delete from whse_basemapping.fwa_waterbodies_upstream_area where linear_feature_id not in (select linear_feature_id from whse_basemapping.fwa_stream_networks_sp)" +$PSQL -c "delete from whse_basemapping.fwa_watersheds_upstream_area" # just delete all of this + +cd .. ; rm -rf fwapg + +# run bcfishobs +git clone git@github.com:smnorris/bcfishobs.git +cd bcfishobs +make --debug=basic +cd .. ; rm -rf bcfishobs + +# set up the source data schema +cd ../db/sources; ./migrate.sh; cd .. + +# run all migration scripts present in /db +for tag in v* ;do + cd "$tag"; ./migrate.sh; cd .. +done +cd .. + +# load source data +jobs/load_static +jobs/load_monthly +jobs/load_weekly +cd test + +# delete data not needed for basic model (rather than editing the job files) +$PSQL -c "delete from whse_admin_boundaries.adm_indian_reserves_bands_sp; +delete from whse_admin_boundaries.adm_nr_districts_spg; +delete from whse_basemapping.bcgs_20k_grid; +delete from whse_basemapping.dbm_mof_50k_grid; +delete from whse_basemapping.nts_250k_grid; +delete from whse_basemapping.trim_cultural_lines; +delete from whse_basemapping.trim_cultural_points; +delete from whse_basemapping.trim_ebm_airfields; +delete from whse_basemapping.trim_ebm_ocean; +delete from whse_basemapping.utmg_utm_zones_sp; +delete from whse_legal_admin_boundaries.abms_municipalities_sp; +delete from whse_legal_admin_boundaries.abms_regional_districts_sp; +delete from whse_admin_boundaries.clab_indian_reserves; +delete from whse_admin_boundaries.clab_national_parks; +delete from whse_basemapping.gba_local_reg_greenspaces_sp; +delete from whse_basemapping.gns_geographical_names_sp; +delete from whse_environmental_monitoring.envcan_hydrometric_stn_sp; +delete from whse_fish.fiss_stream_sample_sites_sp; +delete from whse_forest_tenure.ften_range_poly_svw; +delete from whse_legal_admin_boundaries.abms_municipalities_sp; +delete from whse_tantalis.ta_conservancy_areas_svw; +delete from whse_tantalis.ta_park_ecores_pa_svw; +delete from whse_cadastre.pmbc_parcel_fabric_poly_svw; +" + +# delete dra/ften roads/observations not in test wsg +$PSQL -c "create temporary table roads as select transport_line_id from whse_basemapping.transport_line a inner join whse_basemapping.fwa_watershed_groups_poly b on st_intersects(a.geom, b.geom); + delete from whse_basemapping.transport_line where transport_line_id not in (select transport_line_id from roads);" +$PSQL -c "create temporary table roads as select objectid from whse_forest_tenure.ften_road_section_lines_svw a inner join whse_basemapping.fwa_watershed_groups_poly b on st_intersects(a.geom, b.geom); + delete from whse_forest_tenure.ften_road_section_lines_svw where objectid not in (select objectid from roads);" +$PSQL -c "create temporary table roads as select og_road_segment_permit_id from whse_mineral_tenure.og_road_segment_permit_sp a inner join whse_basemapping.fwa_watershed_groups_poly b on st_intersects(a.geom, b.geom); + delete from whse_mineral_tenure.og_road_segment_permit_sp where og_road_segment_permit_id not in (select objectid from roads);" +$PSQL -c "create temporary table obs as select fish_observation_point_id from whse_fish.fiss_fish_obsrvtn_pnt_sp a inner join whse_basemapping.fwa_watershed_groups_poly b on st_intersects(a.geom, b.geom); + delete from whse_fish.fiss_fish_obsrvtn_pnt_sp where fish_observation_point_id not in (select fish_observation_point_id from obs);" + +# vaccum/analyze +$PSQL -c "vacuum full analyze" + +# what are the biggest relations? +#https://stackoverflow.com/questions/21738408/postgresql-list-and-order-tables-by-size + +# dump the database (~200MB) +pg_dump -Fc bcfishpass_test > bcfishpass_test.dump + +# cleanup +psql postgres -c "drop database bcfishpass_test" \ No newline at end of file diff --git a/test/test.sh b/test/test.sh new file mode 100755 index 00000000..7ad194de --- /dev/null +++ b/test/test.sh @@ -0,0 +1,35 @@ +#!/bin/bash +set -euxo pipefail + +DATABASE_URL=postgresql://postgres@localhost:5432/bcfishpass_test +PSQL="psql $DATABASE_URL -v ON_ERROR_STOP=1" + +createdb bcfishpass_test +$PSQL -c "ALTER DATABASE bcfishpass_test SET search_path TO public,whse_basemapping,usgs,hydrosheds" +pg_restore -d bcfishpass_test bcfishpass_test.dump + +# drop the existing schema and postgisftw.wcrp functions from test db +$PSQL -c "drop schema bcfishpass cascade" +$PSQL -c "drop function postgisftw.wcrp_barrier_count" +$PSQL -c "drop function postgisftw.wcrp_barrier_extent" +$PSQL -c "drop function postgisftw.wcrp_barrier_severity" +$PSQL -c "drop function postgisftw.wcrp_habitat_connectivity_status" + + +# run all migration scripts present in /db +cd ../db +for tag in v* ;do + cd "$tag"; ./migrate.sh; cd .. +done +cd .. + +# load parameters and run jobs +cp parameters/example_testing/*csv parameters +jobs/load_csv +jobs/load_modelled_stream_crossings +$PSQL -c "delete from bcfishpass.modelled_stream_crossings where watershed_group_code not in (select watershed_group_code from whse_basemapping.fwa_watershed_groups_poly)" +jobs/model_gradient_barriers +jobs/model_prep +jobs/model_run + +cd test \ No newline at end of file