fix query loading USA dam placeholders to crossing table (was loading…

… all dams...)

db/tables/log.sql

@@ -1,16 +1,46 @@
---do not generally drop or truncate this table - we want to retain this data
+--do not generally drop or truncate these tables - we want to retain this data
+-- but note that if dropping the main log table, all other tables must also be dropped
+-- (if they are not, model_id values get orphaned/duplicated, views will be incorrect)
 --drop table bcfishpass.log cascade;
---truncate bcfishpass.log cascade;
+--drop table bcfishpass.parameters_habitat_method_log;
+--drop table bcfishpass.parameters_habitat_thresholds_log;
+--drop table bcfishpass.wsg_linear_summary;
+--drop table bcfishpass.wsg_crossing_summary;
 create table if not exists bcfishpass.log (
   model_run_id serial primary key,
   model_type text not null,
   date_completed timestamp not null default CURRENT_TIMESTAMP,
-  git_id bytea not null,
+  model_version text not null,
   check (model_type in ('LINEAR','LATERAL'))
 );
 
 -- usage:
--- insert into bcfishpass.log (model_type, git_id) values ('LINEAR', decode('feac3689cef93cc02a4cb4ac6a0fdadebe980f4d', 'hex')) RETURNING model_run_id;
+-- insert into bcfishpass.log (model_type, model_version)
+-- values ('LINEAR', 'v0.1.dev5-71-gec2db00') RETURNING model_run_id;
+
+
+-- log parameters used for the given model run
+create table if not exists bcfishpass.parameters_habitat_method_log (
+  model_run_id integer references bcfishpass.log(model_run_id),
+  watershed_group_code character varying(4),
+  model text
+);
+
+create table if not exists bcfishpass.parameters_habitat_thresholds_log (
+ model_run_id integer references bcfishpass.log(model_run_id),
+ species_code             text   ,
+ spawn_gradient_max       numeric,
+ spawn_channel_width_min  numeric,
+ spawn_channel_width_max  numeric,
+ spawn_mad_min            numeric,
+ spawn_mad_max            numeric,
+ rear_gradient_max        numeric,
+ rear_channel_width_min   numeric,
+ rear_channel_width_max   numeric,
+ rear_mad_min             numeric,
+ rear_mad_max             numeric,
+ rear_lake_ha_min         integer
+);
 
 create table if not exists bcfishpass.wsg_linear_summary (
  model_run_id                                             integer references bcfishpass.log(model_run_id),

db/views/wsg_crossing_summary.sql

@@ -52,7 +52,7 @@ select
 from bcfishpass.wsg_crossing_summary s
 inner join bcfishpass.log l 
 on s.model_run_id = l.model_run_id
-where l.date_completed = (select date_completed from bcfishpass.log order by date_completed desc limit 1)
+where l.model_run_id = (select model_run_id from bcfishpass.log order by model_run_id desc limit 1)
 order by s.watershed_group_code, s.crossing_feature_type;
 
 
@@ -110,10 +110,10 @@ select
 from bcfishpass.wsg_crossing_summary s
 inner join bcfishpass.log l
 on s.model_run_id = l.model_run_id
-where l.date_completed = (select date_completed from bcfishpass.log order by date_completed desc offset 1 limit 1)
+where l.model_run_id = (select model_run_id from bcfishpass.log order by model_run_id desc limit 1)
 order by watershed_group_code;
 
-
+drop view if exists bcfishpass.wsg_crossing_summary_diff;
 create view bcfishpass.wsg_crossing_summary_diff as
 select
   a.watershed_group_code                  ,

model/01_access/Makefile

@@ -4,9 +4,9 @@
 PSQL=psql $(DATABASE_URL) -v ON_ERROR_STOP=1          # point psql to db and stop on errors
 
 # process all watershed groups
-WSGS = $(shell $(PSQL) -AtX -c "SELECT watershed_group_code FROM whse_basemapping.fwa_watershed_groups_poly")
+#WSGS = $(shell $(PSQL) -AtX -c "SELECT watershed_group_code FROM whse_basemapping.fwa_watershed_groups_poly")
 # test with just watershed groups noted in param (misses cross-wsg barreriers)
-#WSGS = $(shell $(PSQL) -AtX -c "SELECT watershed_group_code FROM bcfishpass.parameters_habitat_method")
+WSGS = $(shell $(PSQL) -AtX -c "SELECT watershed_group_code FROM bcfishpass.parameters_habitat_method")
 
 # define barrier table targets
 NATURAL_BARRIERS = .make/barriers_falls \
@@ -95,8 +95,10 @@ clean:
 	.make/dbm_mof_50k_grid \
 	sql/load_crossings.sql
 	$(PSQL) -c "truncate bcfishpass.crossings"
-	parallel $(PSQL) -f sql/load_crossings.sql -v wsg={1} ::: $(WSGS)
-	# compute upstr/downstr observations here (as previous)?
+	# load crossings in series, parallel inserts can be confounded by locks
+	for wsg in $(WSGS) ; do \
+		$(PSQL) -f sql/load_crossings.sql -v wsg=$$wsg ; \
+	done
 	touch $@
 
 # -----

model/01_access/sql/load_crossings.sql

@@ -416,7 +416,7 @@ order by dam_id
 on conflict do nothing;
 
 -- --------------------------------
--- placeholders for USA dams, from user_barriers_anthropogenic
+-- non-cabd dams (placeholders for USA dams, from user_barriers_anthropogenic)
 -----------------------------------
 insert into bcfishpass.crossings
 (
@@ -466,8 +466,10 @@ select
     (st_dump(st_locatealong(s.geom, d.downstream_route_measure))).geom as geom
 from bcfishpass.dams d
 inner join whse_basemapping.fwa_stream_networks_sp s on d.linear_feature_id = s.linear_feature_id
--- note lack of watershed group code query (as these are outside of bc)
--- just run this insert every time, the conflicts are ignored
+inner join bcfishpass.user_barriers_anthropogenic ba
+  on d.blue_line_key = ba.blue_line_key
+  and d.downstream_route_measure = ba.downstream_route_measure
+where d.watershed_group_code = :'wsg'
 order by dam_id
 on conflict do nothing;
 

model/02_habitat_linear/habitat_linear.sh

@@ -36,10 +36,47 @@ $PSQL -c "refresh materialized view bcfishpass.crossings_vw"
 
 # Finished processing!
 # Now add model run to log, returning the id
-git_id=$(git rev-parse HEAD)
-model_run_id=$($PSQL -qtAX -c "insert into bcfishpass.log (model_type, git_id) VALUES ('LINEAR', decode('$git_id', 'hex')) returning model_run_id")
+# note that below logging could be done in db with triggers/functions but this works fine for now
+model_version=$(git describe)
+model_run_id=$($PSQL -qtAX -c "insert into bcfishpass.log (model_type, model_version) VALUES ('LINEAR', '$model_version') returning model_run_id")
 
-# log summaries (todo - call these functions as a trigger on adding row to bcfishpass.log table rather than calling here)
+# log parameters
+$PSQL -c "insert into bcfishpass.parameters_habitat_method_log
+          (model_run_id, watershed_group_code, model)
+          select $model_run_id, watershed_group_code, model from bcfishpass.parameters_habitat_method;"
+
+$PSQL -c "insert into bcfishpass.parameters_habitat_thresholds_log (
+          model_run_id            ,
+          species_code            ,
+          spawn_gradient_max      ,
+          spawn_channel_width_min ,
+          spawn_channel_width_max ,
+          spawn_mad_min           ,
+          spawn_mad_max           ,
+          rear_gradient_max       ,
+          rear_channel_width_min  ,
+          rear_channel_width_max  ,
+          rear_mad_min            ,
+          rear_mad_max            ,
+          rear_lake_ha_min
+        )
+        select
+         $model_run_id,
+         species_code,
+         spawn_gradient_max,
+         spawn_channel_width_min,
+         spawn_channel_width_max,
+         spawn_mad_min,
+         spawn_mad_max,
+         rear_gradient_max,
+         rear_channel_width_min,
+         rear_channel_width_max,
+         rear_mad_min,
+         rear_mad_max,
+         rear_lake_ha_min
+        from bcfishpass.parameters_habitat_thresholds;"
+
+# log summaries
 $PSQL -c "insert into bcfishpass.wsg_linear_summary select $model_run_id as model_run_id, * from bcfishpass.wsg_linear_summary()"
 $PSQL -c "insert into bcfishpass.wsg_crossing_summary select $model_run_id as model_run_id, * from bcfishpass.wsg_crossing_summary()"