resolve merge conflict with two concurrent data updates

.github/workflows/production-bcfishpass.yaml

@@ -10,29 +10,27 @@ env:
   AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
   AWS_DEFAULT_REGION: ${{ secrets.AWS_DEFAULT_REGION }}
 jobs:
-  load-parameters:
+  load-csv:
     runs-on: ubuntu-latest
     environment: production
     container: ghcr.io/smnorris/bcfishpass:main
     steps:
       - name: Check out repository code
         uses: actions/checkout@v4
-      - name: Run the jobs
+      - name: Load csv data, parameters
         run:  |
           cp parameters/example_newgraph/*csv parameters/
-          make --debug=basic .make/parameters
+          jobs/load_csv
   build-xings:
     runs-on: ubuntu-latest
     environment: production
     container: ghcr.io/smnorris/bcfishpass:main
-    needs: load-parameters
+    needs: load-csv
     steps:
       - name: Check out repository code
         uses: actions/checkout@v4
       - name: Build modelled crossings
-        run:  jobs/bcfishpass01_modelled_stream_crossings
-      - name: Release modelled crossings
-        run:  jobs/release_modelled_stream_crossings
+        run:  jobs/model_stream_crossings
   prep:
     runs-on: ubuntu-latest
     environment: production
@@ -42,7 +40,7 @@ jobs:
       - name: Check out repository code
         uses: actions/checkout@v4
       - name: Run the jobs
-        run:  jobs/bcfishpass02_prep
+        run:  jobs/model_prep
   model:
     runs-on: ubuntu-latest
     environment: production
@@ -57,7 +55,7 @@ jobs:
       - name: Run the jobs
         run:  |
           git config --global --add safe.directory /__w/bcfishpass/bcfishpass
-          jobs/bcfishpass03_model
+          jobs/model_run
   release:
     runs-on: ubuntu-latest
     environment: production

.github/workflows/testing-bcfishpass.yaml

@@ -8,7 +8,7 @@ on:
 env:
   DATABASE_URL: ${{ secrets.DATABASE_URL }}
 jobs:
-  load-parameters:
+  load-csv:
     runs-on: ubuntu-latest
     environment: testing
     container: ghcr.io/smnorris/bcfishpass:main
@@ -18,19 +18,17 @@ jobs:
       - name: Run the jobs
         run:  |
           cp parameters/example_newgraph/*csv parameters/
-          make --debug=basic .make/parameters
+          jobs/load_csv
   load-xings:
     runs-on: ubuntu-latest
     environment: testing
     container: ghcr.io/smnorris/bcfishpass:main
-    needs: load-parameters
+    needs: load-csv
     steps:
       - name: Check out repository code
         uses: actions/checkout@v4
       - name: Run the jobs
-        run:  |
-          cd model/01_access/modelled_stream_crossings
-          make .make/download_archive
+        run:  jobs/load_modelled_stream_crossings
   prep:
     runs-on: ubuntu-latest
     environment: testing
@@ -40,7 +38,7 @@ jobs:
       - name: Check out repository code
         uses: actions/checkout@v4
       - name: Run the jobs
-        run:  jobs/bcfishpass02_prep
+        run:  jobs/model_prep
   model:
     runs-on: ubuntu-latest
     environment: testing
@@ -55,4 +53,4 @@ jobs:
       - name: Run the jobs
         run:  |
           git config --global --add safe.directory /__w/bcfishpass/bcfishpass
-          jobs/bcfishpass03_model
+          jobs/model_run

.gitignore

@@ -35,4 +35,7 @@ model/01_access/qa/*csv
 
 # only include example parameters,
 # not whatever is being used for a given project/test
-parameters/parameters_habitat*.csv
+parameters/parameters_habitat*.csv
+
+# postgres dump files
+*.dump

README.md

@@ -23,7 +23,7 @@ See the [Documentation](https://smnorris.github.io/bcfishpass/) for details.
 - [fwapg](https://github.com/smnorris/fwapg)
 - [bcfishobs](https://github.com/smnorris/bcfishobs)
 
-## Setup
+## Setup / Usage
 
 `bcfishpass` is a collection of shell/sql/Python scripts. To download and use the latest:
 
@@ -47,37 +47,41 @@ All scripts presume that the `DATABASE_URL` environment variable points to your
 
     export DATABASE_URL=postgresql://postgres@localhost:5432/bcfishpass
 
-Set up the database schema:
-
-    jobs/setup
-
-Load FWA data:
+Load FWA:
 
     git clone https://github.com/smnorris/fwapg
     cd fwapg
-    mkdir -p .make; touch .make/db  # just reload data
-    mkdir -p data
-    make --debug=basicjobs/load_fwa
+    make --debug=basic
+
+Load/run `bcfishobs`:
+
+    git clone [email protected]:smnorris/bcfishobs.git
+    cd bcfishobs
+    make --debug=basic
 
-Load all additional data:
+Create db schema:
 
-    jobs/load_static
+    jobs/db_setup
+
+Load source data:
+
+    jobs/load_static                     
     jobs/load_monthly
     jobs/load_weekly
 
-Run `bcfishobs`:
+Load cached bcfishpass data:
 
-    git clone [email protected]:smnorris/bcfishobs.git
-    cd bcfishobs
-    mkdir -p .make
-    make -t .make/setup
-    make -t .make/load_static
-    make -t .make/fiss_fish_obsrvtn_pnt_sp
-    make --debug=basic
+    jobs/load_gradient_barriers
+    jobs/load_modelled_stream_crossings
+
+Run the model:
+
+    jobs/model_prep
+    jobs/model_run
 
-Finally, navigate back to the root bcfishpass folder and build `bcfishpass`:
+# Backups
 
-    make
+Backup strategies will vary but it can be useful to dump the entire database to file.  
+This appends the date and commit tag date to the file name:
 
-Note that it is possible (and often preferred) to build components of the modelling separately. 
-Refer to the various README files in the subfolders within the `model` folder for more info.
+        pg_dump -Fc $DATABASE_URL > bcfishpass.$(git describe --tags --abbrev=0).$(date +%F).dump    

data/README.md

@@ -33,11 +33,13 @@ Currently, controls the barrier status of natural barriers (gradient, falls, sub
 NOTE -  this table will only be used to identify modelled barriers known to be passable 
 (gradient, subsurface flow) once bcfishpass incorporates CABD falls
 
+
 ## user_cabd_dams_exclusions.csv
 
 List of CABD dams to exclude from analysis (generally due to incorrect location or incorrect passability status).
 To be used as temporary fix only - submit any location or passability status to CWF [here](https://forms.office.com/Pages/ResponsePage.aspx?id=A8N2i-wB1UCNmmwvbXjc15ERVmcC4dFPn5j4q5-aulRURE1TSjBJNEtDNlY0WTlXVFY5MkFOMzRVUS4u)
 
+
 ## user_falls.csv
 
 Falls not present in FWA/FISS. Both barriers and non-barriers may be included.
@@ -62,6 +64,11 @@ Update the barrier status of PSCIS crossings (for bcfishpass modelling only). PS
 support planning by adding barrier status of a crossing before a submission is made.
 
 
+## wcrp.csv
+
+A list of watershed groups and target species for CWF WCRP reporting.
+
+
 ## wsg_species_presence.csv
 
 A list of all BC watershed groups and presence/absence of target species (Coho, Chinook, Sockeye, Steelhead, West Slope Cutthroat Trout).

data/user_cabd_blkey_xref.csv

@@ -0,0 +1,3 @@
+cabd_id,blue_line_key,reviewer_name,review_date,notes
+30115bb6-9a75-4c33-bb37-4243ce33e43d,356352213,SN,2024-05-28,matched to stream named in CABD
+c3721358-4334-41c0-a4c1-1fa729803d47,354154385,SN,2024-05-28,matched to stream named in CABD (but it is unclear where cascade is actually located)

data/user_cabd_dams_exclusions.csv

@@ -1,8 +1 @@
-cabd_id,reviewer_name,review_date,source,notes
-30b88f1b-dc21-4b42-8daa-d4cebae24142,SN,2023-01-01,not noted,
-3ca692b8-37cf-44e8-a783-2a315ec83102,SN,2023-01-01,not noted,
-ba5fe3eb-7bbe-45c1-b301-555872387c16,SN,2023-01-01,not noted,
-8a6b10fa-0d4f-4c45-857c-764d7e8028f8,SN,2023-01-01,not noted,
-48478e95-e063-4df6-a047-6aaf6087011b,SN,2023-01-01,not noted,
-e8e4bd88-c3c9-407c-a7a0-15c6c51704fd,SN,2023-01-01,not noted,dam may or may not be a barrier but location was incorrect at time of review
-6a792d8f-b9c5-44a4-a260-0f06c3b20821,SN,2023-01-01,not noted,dam may or may not be a barrier but location gets matched to Salmon River
+cabd_id,reviewer_name,review_date,source,notes