From 4b2d4683aaeb6bacbae8f389d0d888ab612feb95 Mon Sep 17 00:00:00 2001 From: Simon Norris Date: Fri, 5 Jan 2024 10:07:16 -0800 Subject: [PATCH] Log model runs and results summaries (#439) * add model log table and write to it when linear habitat processing is complete * get model_run_id when log record inserted * update log * add prelim crossings summary * move summaries from views to funcs * create tables if not exist --- .../wsg_crossing_summary.sql} | 54 ++++++- .../wsg_linear_summary.sql} | 78 +++++++++- db/tables/log.sql | 133 ++++++++++++++++++ model/02_habitat_linear/habitat_linear.sh | 10 ++ 4 files changed, 268 insertions(+), 7 deletions(-) rename db/{views/09_wsg_crossing_summary_vw.sql => functions/wsg_crossing_summary.sql} (72%) rename db/{views/08_wsg_linear_summary_vw.sql => functions/wsg_linear_summary.sql} (81%) create mode 100644 db/tables/log.sql diff --git a/db/views/09_wsg_crossing_summary_vw.sql b/db/functions/wsg_crossing_summary.sql similarity index 72% rename from db/views/09_wsg_crossing_summary_vw.sql rename to db/functions/wsg_crossing_summary.sql index 64c5d8e8..e7b811a8 100644 --- a/db/views/09_wsg_crossing_summary_vw.sql +++ b/db/functions/wsg_crossing_summary.sql @@ -3,8 +3,57 @@ -- - accessible (per model) -- - with spawning rearing habitat upstream (per model) -drop view if exists bcfishpass.crossings_summary_vw; -create view bcfishpass.crossings_summary_vw as +CREATE FUNCTION bcfishpass.wsg_crossings_summary() +RETURNS table ( +watershed_group_code text, + crossing_feature_type text, + n_crossings_total integer, + n_passable_total integer, + n_barriers_total integer, + n_potential_total integer, + n_unknown_total integer, + n_barriers_accessible_bt integer, + n_potential_accessible_bt integer, + n_unknown_accessible_bt integer, + n_barriers_accessible_ch_cm_co_pk_sk integer, + n_potential_accessible_ch_cm_co_pk_sk integer, + n_unknown_accessible_ch_cm_co_pk_sk integer, + n_barriers_accessible_st integer, + n_potential_accessible_st integer, + n_unknown_accessible_st integer, + n_barriers_accessible_wct integer, + n_potential_accessible_wct integer, + n_unknown_accessible_wct integer, + n_barriers_habitat_bt integer, + n_potential_habitat_bt integer, + n_unknown_habitat_bt integer, + n_barriers_habitat_ch integer, + n_potential_habitat_ch integer, + n_unknown_habitat_ch integer, + n_barriers_habitat_cm integer, + n_potential_habitat_cm integer, + n_unknown_habitat_cm integer, + n_barriers_habitat_co integer, + n_potential_habitat_co integer, + n_unknown_habitat_co integer, + n_barriers_habitat_pk integer, + n_potential_habitat_pk integer, + n_unknown_habitat_pk integer, + n_barriers_habitat_sk integer, + n_potential_habitat_sk integer, + n_unknown_habitat_sk integer, + n_barriers_habitat_salmon integer, + n_potential_habitat_salmon integer, + n_unknown_habitat_salmon integer, + n_barriers_habitat_st integer, + n_potential_habitat_st integer, + n_unknown_habitat_st integer, + n_barriers_habitat_wct integer, + n_potential_habitat_wct integer, + n_unknown_habitat_wct integer +) +AS $$ + select watershed_group_code, crossing_feature_type, @@ -152,3 +201,4 @@ select from bcfishpass.crossings_vw group by watershed_group_code, crossing_feature_type; +$$ LANGUAGE SQL; \ No newline at end of file diff --git a/db/views/08_wsg_linear_summary_vw.sql b/db/functions/wsg_linear_summary.sql similarity index 81% rename from db/views/08_wsg_linear_summary_vw.sql rename to db/functions/wsg_linear_summary.sql index 9213c0ab..bc1fe1ad 100644 --- a/db/views/08_wsg_linear_summary_vw.sql +++ b/db/functions/wsg_linear_summary.sql @@ -10,10 +10,76 @@ -- - spawning or rearing habitat not isolated by dam or a known pscis barrier (accessible_a) -- - spawning or rearing habitat not isolated by all anthropogenic barriers (observed and modelled) (accessible_b) - -drop view if exists bcfishpass.wsg_linear_summary_vw; - -create view bcfishpass.wsg_linear_summary_vw as +CREATE FUNCTION bcfishpass.wsg_linear_summary() +RETURNS table ( + watershed_group_code text, + length_total numeric, + length_potentiallyaccessible_bt numeric, + length_potentiallyaccessible_bt_observed numeric, + length_potentiallyaccessible_bt_accessible_a numeric, + length_potentiallyaccessible_bt_accessible_b numeric, + length_obsrvd_spawning_rearing_bt numeric, + length_obsrvd_spawning_rearing_bt_accessible_a numeric, + length_obsrvd_spawning_rearing_bt_accessible_b numeric, + length_spawning_rearing_bt numeric, + length_spawning_rearing_bt_accessible_a numeric, + length_spawning_rearing_bt_accessible_b numeric, + length_potentiallyaccessible_ch_cm_co_pk_sk numeric, + length_potentiallyaccessible_ch_cm_co_pk_sk_observed numeric, + length_potentiallyaccessible_ch_cm_co_pk_sk_accessible_a numeric, + length_potentiallyaccessible_ch_cm_co_pk_sk_accessible_b numeric, + length_obsrvd_spawning_rearing_ch numeric, + length_obsrvd_spawning_rearing_ch_accessible_a numeric, + length_obsrvd_spawning_rearing_ch_accessible_b numeric, + length_spawning_rearing_ch numeric, + length_spawning_rearing_ch_accessible_a numeric, + length_spawning_rearing_ch_accessible_b numeric, + length_obsrvd_spawning_rearing_cm numeric, + length_obsrvd_spawning_rearing_cm_accessible_a numeric, + length_obsrvd_spawning_rearing_cm_accessible_b numeric, + length_spawning_rearing_cm numeric, + length_spawning_rearing_cm_accessible_a numeric, + length_spawning_rearing_cm_accessible_b numeric, + length_obsrvd_spawning_rearing_co numeric, + length_obsrvd_spawning_rearing_co_accessible_a numeric, + length_obsrvd_spawning_rearing_co_accessible_b numeric, + length_spawning_rearing_co numeric, + length_spawning_rearing_co_accessible_a numeric, + length_spawning_rearing_co_accessible_b numeric, + length_obsrvd_spawning_rearing_pk numeric, + length_obsrvd_spawning_rearing_pk_accessible_a numeric, + length_obsrvd_spawning_rearing_pk_accessible_b numeric, + length_spawning_rearing_pk numeric, + length_spawning_rearing_pk_accessible_a numeric, + length_spawning_rearing_pk_accessible_b numeric, + length_obsrvd_spawning_rearing_sk numeric, + length_obsrvd_spawning_rearing_sk_accessible_a numeric, + length_obsrvd_spawning_rearing_sk_accessible_b numeric, + length_spawning_rearing_sk numeric, + length_spawning_rearing_sk_accessible_a numeric, + length_spawning_rearing_sk_accessible_b numeric, + length_potentiallyaccessible_st numeric, + length_potentiallyaccessible_st_observed numeric, + length_potentiallyaccessible_st_accessible_a numeric, + length_potentiallyaccessible_st_accessible_b numeric, + length_obsrvd_spawning_rearing_st numeric, + length_obsrvd_spawning_rearing_st_accessible_a numeric, + length_obsrvd_spawning_rearing_st_accessible_b numeric, + length_spawning_rearing_st numeric, + length_spawning_rearing_st_accessible_a numeric, + length_spawning_rearing_st_accessible_b numeric, + length_potentiallyaccessible_wct numeric, + length_potentiallyaccessible_wct_observed numeric, + length_potentiallyaccessible_wct_accessible_a numeric, + length_potentiallyaccessible_wct_accessible_b numeric, + length_obsrvd_spawning_rearing_wct numeric, + length_obsrvd_spawning_rearing_wct_accessible_a numeric, + length_obsrvd_spawning_rearing_wct_accessible_b numeric, + length_spawning_rearing_wct numeric, + length_spawning_rearing_wct_accessible_a numeric, + length_spawning_rearing_wct_accessible_b numeric + ) +AS $$ with accessible as ( @@ -209,4 +275,6 @@ select from accessible a left outer join spawning_rearing_observed o on a.watershed_group_code = o.watershed_group_code -left outer join spawning_rearing_modelled m on a.watershed_group_code = m.watershed_group_code +left outer join spawning_rearing_modelled m on a.watershed_group_code = m.watershed_group_code; + +$$ LANGUAGE SQL; \ No newline at end of file diff --git a/db/tables/log.sql b/db/tables/log.sql new file mode 100644 index 00000000..5b7d4b31 --- /dev/null +++ b/db/tables/log.sql @@ -0,0 +1,133 @@ +--do not generally drop or truncate this table - we want to retain this data +--drop table bcfishpass.log cascade; +--truncate bcfishpass.log cascade; +create table if not exists bcfishpass.log ( + model_run_id serial primary key, + model_type text not null, + date_completed timestamp not null default CURRENT_TIMESTAMP, + git_id bytea not null, + check (model_type in ('LINEAR','LATERAL')) +); + +-- usage: +-- insert into bcfishpass.log (model_type, git_id) values ('LINEAR', decode('feac3689cef93cc02a4cb4ac6a0fdadebe980f4d', 'hex')) RETURNING model_run_id; + +create table if not exists bcfishpass.wsg_linear_summary ( + model_run_id integer references bcfishpass.log(model_run_id), + watershed_group_code text, + length_total numeric, + length_potentiallyaccessible_bt numeric, + length_potentiallyaccessible_bt_observed numeric, + length_potentiallyaccessible_bt_accessible_a numeric, + length_potentiallyaccessible_bt_accessible_b numeric, + length_obsrvd_spawning_rearing_bt numeric, + length_obsrvd_spawning_rearing_bt_accessible_a numeric, + length_obsrvd_spawning_rearing_bt_accessible_b numeric, + length_spawning_rearing_bt numeric, + length_spawning_rearing_bt_accessible_a numeric, + length_spawning_rearing_bt_accessible_b numeric, + length_potentiallyaccessible_ch_cm_co_pk_sk numeric, + length_potentiallyaccessible_ch_cm_co_pk_sk_observed numeric, + length_potentiallyaccessible_ch_cm_co_pk_sk_accessible_a numeric, + length_potentiallyaccessible_ch_cm_co_pk_sk_accessible_b numeric, + length_obsrvd_spawning_rearing_ch numeric, + length_obsrvd_spawning_rearing_ch_accessible_a numeric, + length_obsrvd_spawning_rearing_ch_accessible_b numeric, + length_spawning_rearing_ch numeric, + length_spawning_rearing_ch_accessible_a numeric, + length_spawning_rearing_ch_accessible_b numeric, + length_obsrvd_spawning_rearing_cm numeric, + length_obsrvd_spawning_rearing_cm_accessible_a numeric, + length_obsrvd_spawning_rearing_cm_accessible_b numeric, + length_spawning_rearing_cm numeric, + length_spawning_rearing_cm_accessible_a numeric, + length_spawning_rearing_cm_accessible_b numeric, + length_obsrvd_spawning_rearing_co numeric, + length_obsrvd_spawning_rearing_co_accessible_a numeric, + length_obsrvd_spawning_rearing_co_accessible_b numeric, + length_spawning_rearing_co numeric, + length_spawning_rearing_co_accessible_a numeric, + length_spawning_rearing_co_accessible_b numeric, + length_obsrvd_spawning_rearing_pk numeric, + length_obsrvd_spawning_rearing_pk_accessible_a numeric, + length_obsrvd_spawning_rearing_pk_accessible_b numeric, + length_spawning_rearing_pk numeric, + length_spawning_rearing_pk_accessible_a numeric, + length_spawning_rearing_pk_accessible_b numeric, + length_obsrvd_spawning_rearing_sk numeric, + length_obsrvd_spawning_rearing_sk_accessible_a numeric, + length_obsrvd_spawning_rearing_sk_accessible_b numeric, + length_spawning_rearing_sk numeric, + length_spawning_rearing_sk_accessible_a numeric, + length_spawning_rearing_sk_accessible_b numeric, + length_potentiallyaccessible_st numeric, + length_potentiallyaccessible_st_observed numeric, + length_potentiallyaccessible_st_accessible_a numeric, + length_potentiallyaccessible_st_accessible_b numeric, + length_obsrvd_spawning_rearing_st numeric, + length_obsrvd_spawning_rearing_st_accessible_a numeric, + length_obsrvd_spawning_rearing_st_accessible_b numeric, + length_spawning_rearing_st numeric, + length_spawning_rearing_st_accessible_a numeric, + length_spawning_rearing_st_accessible_b numeric, + length_potentiallyaccessible_wct numeric, + length_potentiallyaccessible_wct_observed numeric, + length_potentiallyaccessible_wct_accessible_a numeric, + length_potentiallyaccessible_wct_accessible_b numeric, + length_obsrvd_spawning_rearing_wct numeric, + length_obsrvd_spawning_rearing_wct_accessible_a numeric, + length_obsrvd_spawning_rearing_wct_accessible_b numeric, + length_spawning_rearing_wct numeric, + length_spawning_rearing_wct_accessible_a numeric, + length_spawning_rearing_wct_accessible_b numeric +); + +create table if not exists bcfishpass.wsg_crossing_summary ( + model_run_id integer references bcfishpass.log(model_run_id), + watershed_group_code text, + crossing_feature_type text, + n_crossings_total integer, + n_passable_total integer, + n_barriers_total integer, + n_potential_total integer, + n_unknown_total integer, + n_barriers_accessible_bt integer, + n_potential_accessible_bt integer, + n_unknown_accessible_bt integer, + n_barriers_accessible_ch_cm_co_pk_sk integer, + n_potential_accessible_ch_cm_co_pk_sk integer, + n_unknown_accessible_ch_cm_co_pk_sk integer, + n_barriers_accessible_st integer, + n_potential_accessible_st integer, + n_unknown_accessible_st integer, + n_barriers_accessible_wct integer, + n_potential_accessible_wct integer, + n_unknown_accessible_wct integer, + n_barriers_habitat_bt integer, + n_potential_habitat_bt integer, + n_unknown_habitat_bt integer, + n_barriers_habitat_ch integer, + n_potential_habitat_ch integer, + n_unknown_habitat_ch integer, + n_barriers_habitat_cm integer, + n_potential_habitat_cm integer, + n_unknown_habitat_cm integer, + n_barriers_habitat_co integer, + n_potential_habitat_co integer, + n_unknown_habitat_co integer, + n_barriers_habitat_pk integer, + n_potential_habitat_pk integer, + n_unknown_habitat_pk integer, + n_barriers_habitat_sk integer, + n_potential_habitat_sk integer, + n_unknown_habitat_sk integer, + n_barriers_habitat_salmon integer, + n_potential_habitat_salmon integer, + n_unknown_habitat_salmon integer, + n_barriers_habitat_st integer, + n_potential_habitat_st integer, + n_unknown_habitat_st integer, + n_barriers_habitat_wct integer, + n_potential_habitat_wct integer, + n_unknown_habitat_wct integer +); \ No newline at end of file diff --git a/model/02_habitat_linear/habitat_linear.sh b/model/02_habitat_linear/habitat_linear.sh index 541a32f4..185f34de 100755 --- a/model/02_habitat_linear/habitat_linear.sh +++ b/model/02_habitat_linear/habitat_linear.sh @@ -31,3 +31,13 @@ done # with linear model processing complete, refresh materialized views $PSQL -c "refresh materialized view bcfishpass.crossings_upstr_barriers_per_model_vw" $PSQL -c "refresh materialized view bcfishpass.crossings_vw" + +# Finished processing! +# Now add model run to log, returning the id +git_id=$(git rev-parse HEAD) +model_run_id=$($PSQL -qtAX -c "insert into bcfishpass.log (model_type, git_id) VALUES ('LINEAR', decode('$git_id', 'hex')) returning model_run_id") + +# log summaries (todo - call these functions as a trigger on adding row to bcfishpass.log table rather than calling here) +$PSQL -c "insert into bcfishpass.wsg_linear_summary select $model_run_id as model_run_id, * from bcfishpass.wsg_linear_summary()" +$PSQL -c "insert into bcfishpass.wsg_crossing_summary select $model_run_id as model_run_id, * from bcfishpass.wsg_crossing_summary()" +