Skip to content

Commit

Permalink
Issue440 (#441)
Browse files Browse the repository at this point in the history
* tidy generation of crossing table

* fix crossings summary views and restore upstr/dnstr observation columns to output views

* move crossings admin query to materialized view but do not run until source data are loaded
  • Loading branch information
smnorris authored Jan 6, 2024
1 parent decdf95 commit ec2db00
Show file tree
Hide file tree
Showing 14 changed files with 314 additions and 325 deletions.
16 changes: 8 additions & 8 deletions db/tables/crossings.sql
Original file line number Diff line number Diff line change
Expand Up @@ -27,7 +27,7 @@ create table bcfishpass.crossings
user_barrier_anthropogenic_id bigint unique,
modelled_crossing_id integer unique,
crossing_source text, -- pscis/dam/model, can be inferred from above ids
crossing_feature_type text, -- general type of crossing (rail/road/trail/dam/weir)
--crossing_feature_type text, -- general type of crossing (rail/road/trail/dam/weir)

-- basic crossing status/info
pscis_status text, -- ASSESSED/HABITAT CONFIRMATION etc
Expand Down Expand Up @@ -76,7 +76,7 @@ create table bcfishpass.crossings
utm_northing integer,

-- map tile for pdfs
dbm_mof_50k_grid text,
--dbm_mof_50k_grid text,

-- basic FWA info
linear_feature_id integer,
Expand All @@ -95,8 +95,8 @@ create table bcfishpass.crossings
-- watershed_upstr_ha double precision DEFAULT 0,

-- distinct species upstream/downstream, derived from bcfishobs
observedspp_dnstr text[],
observedspp_upstr text[],
--observedspp_dnstr text[],
--observedspp_upstr text[],

geom geometry(PointZM, 3005),

Expand All @@ -111,7 +111,7 @@ comment on column bcfishpass.crossings.dam_id IS 'BC Dams unique identifier';
comment on column bcfishpass.crossings.user_barrier_anthropogenic_id IS 'User added misc anthropogenic barriers unique identifier';
comment on column bcfishpass.crossings.modelled_crossing_id IS 'Modelled crossing unique identifier';
comment on column bcfishpass.crossings.crossing_source IS 'Data source for the crossing, one of: {PSCIS,MODELLED CROSSINGS,CABD,MISC BARRIERS}';
comment on column bcfishpass.crossings.crossing_feature_type IS 'The general type of feature crossing the stream, valid feature types are {DAM,RAIL,"ROAD, DEMOGRAPHIC","ROAD, RESOURCE/OTHER",TRAIL,WEIR}';
--comment on column bcfishpass.crossings.crossing_feature_type IS 'The general type of feature crossing the stream, valid feature types are {DAM,RAIL,"ROAD, DEMOGRAPHIC","ROAD, RESOURCE/OTHER",TRAIL,WEIR}';
comment on column bcfishpass.crossings.pscis_status IS 'From PSCIS, the current_pscis_status of the crossing, one of: {ASSESSED,HABITAT CONFIRMATION,DESIGN,REMEDIATED}';
comment on column bcfishpass.crossings.crossing_type_code IS 'Defines the type of crossing present at the location of the stream crossing. Acceptable types are: OBS = Open Bottom Structure CBS = Closed Bottom Structure OTHER = Crossing structure does not fit into the above categories. Eg: ford, wier';
comment on column bcfishpass.crossings.crossing_subtype_code IS 'Further definition of the type of crossing, one of {BRIDGE,CRTBOX,DAM,FORD,OVAL,PIPEARCH,ROUND,WEIR,WOODBOX,NULL}';
Expand Down Expand Up @@ -143,7 +143,7 @@ comment on column bcfishpass.crossings.dam_operating_status IS 'See CABD dams co
comment on column bcfishpass.crossings.utm_zone IS 'UTM ZONE is a segment of the Earths surface 6 degrees of longitude in width. The zones are numbered eastward starting at the meridian 180 degrees from the prime meridian at Greenwich. There are five zones numbered 7 through 11 that cover British Columbia, e.g., Zone 10 with a central meridian at -123 degrees.';
comment on column bcfishpass.crossings.utm_easting IS 'UTM EASTING is the distance in meters eastward to or from the central meridian of a UTM zone with a false easting of 500000 meters. e.g., 440698';
comment on column bcfishpass.crossings.utm_northing IS 'UTM NORTHING is the distance in meters northward from the equator. e.g., 6197826';
comment on column bcfishpass.crossings.dbm_mof_50k_grid IS 'WHSE_BASEMAPPING.DBM_MOF_50K_GRID map_tile_display_name, used for generating planning map pdfs';
--comment on column bcfishpass.crossings.dbm_mof_50k_grid IS 'WHSE_BASEMAPPING.DBM_MOF_50K_GRID map_tile_display_name, used for generating planning map pdfs';
comment on column bcfishpass.crossings.linear_feature_id IS 'From BC FWA, the unique identifier for a stream segment (flow network arc)';
comment on column bcfishpass.crossings.blue_line_key IS 'From BC FWA, uniquely identifies a single flow line such that a main channel and a secondary channel with the same watershed code would have different blue line keys (the Fraser River and all side channels have different blue line keys).';
comment on column bcfishpass.crossings.watershed_key IS 'From BC FWA, a key that identifies a stream system. There is a 1:1 match between a watershed key and watershed code. The watershed key will match the blue line key for the mainstem.';
Expand All @@ -155,8 +155,8 @@ comment on column bcfishpass.crossings.gnis_stream_name IS 'The BCGNIS (BC Geogr
comment on column bcfishpass.crossings.stream_order IS 'Order of FWA stream at point';
comment on column bcfishpass.crossings.stream_magnitude IS 'Magnitude of FWA stream at point';
--comment on column bcfishpass.crossings.watershed_upstr_ha IS 'Total watershed area upstream of point (approximate, does not include area of the fundamental watershed in which the point lies)';
comment on column bcfishpass.crossings.observedspp_dnstr IS 'Fish species observed downstream of point *within the same watershed group*';
comment on column bcfishpass.crossings.observedspp_upstr IS 'Fish species observed upstream of point *within the same watershed group*';
--comment on column bcfishpass.crossings.observedspp_dnstr IS 'Fish species observed downstream of point *within the same watershed group*';
--comment on column bcfishpass.crossings.observedspp_upstr IS 'Fish species observed upstream of point *within the same watershed group*';
comment on column bcfishpass.crossings.geom IS 'The point geometry associated with the feature';

-- index for speed
Expand Down
81 changes: 81 additions & 0 deletions db/views/01_crossings_feature_type_vw.sql
Original file line number Diff line number Diff line change
@@ -0,0 +1,81 @@
-- crossing feature type
drop view if exists bcfishpass.crossings_feature_type_vw;
create view bcfishpass.crossings_feature_type_vw as
select
aggregated_crossings_id,
case
when crossing_subtype_code = 'WEIR' then 'WEIR'
when crossing_subtype_code = 'DAM' then 'DAM'
-- RAIL, pscis crossings within 50m of rail line and with RAIL in the road name
when
rail_owner_name is not null
and aggregated_crossings_id in (
select
aggregated_crossings_id
from bcfishpass.crossings c
inner join whse_basemapping.gba_railway_tracks_sp r
on st_dwithin(c.geom, r.geom, 50)
where crossing_source = 'PSCIS'
and upper(pscis_road_name) like '%RAIL%' and upper(pscis_road_name) not like '%TRAIL%'
)
then 'RAIL'
-- RAIL, pscis crossings within 10m of rail line
when
rail_owner_name is not null
and aggregated_crossings_id in (
select
aggregated_crossings_id
from bcfishpass.crossings c
inner join whse_basemapping.gba_railway_tracks_sp r
on st_dwithin(c.geom, r.geom, 10)
where crossing_source = 'PSCIS'
)
then 'RAIL'
-- modelled rail crossings
when
rail_owner_name is not null
and crossing_source = 'MODELLED CROSSINGS'
then 'RAIL'
-- tenured roads
when
ften_forest_file_id is not NULL OR
ogc_proponent is not NULL
then 'ROAD, RESOURCE/OTHER'
-- demographic roads
when
transport_line_type_description IN (
'Road alleyway',
'Road arterial major',
'Road arterial minor',
'Road collector major',
'Road collector minor',
'Road freeway',
'Road highway major',
'Road highway minor',
'Road lane',
'Road local',
'Private driveway demographic',
'Road pedestrian mall',
'Road runway non-demographic',
'Road recreation demographic',
'Road ramp',
'Road restricted',
'Road strata',
'Road service',
'Road yield lane'
)
then 'ROAD, DEMOGRAPHIC'
-- trails
when
upper(transport_line_type_description) LIKE 'TRAIL%'
then 'TRAIL'
-- everything else (DRA)
when
transport_line_type_description is not NULL
then 'ROAD, RESOURCE/OTHER'
-- in the absence of any of the above info, assume a PSCIS crossing is on a resource/other road
when
stream_crossing_id is not NULL
then 'ROAD, RESOURCE/OTHER'
end as crossing_feature_type
from bcfishpass.crossings;
31 changes: 31 additions & 0 deletions db/views/02_crossings_admin.sql
Original file line number Diff line number Diff line change
@@ -0,0 +1,31 @@
-- below datasets are not guaranteed to exist

--drop materialized view if exists bcfishpass.crossings_admin;
--create materialized view bcfishpass.crossings_admin;
--SELECT DISTINCT ON (c.aggregated_crossings_id) -- some of the admin areas are not clean/distinct, make sure to select just one
-- c.aggregated_crossings_id,
-- rd.admin_area_abbreviation as abms_regional_district,
-- muni.admin_area_abbreviation as abms_municipality,
-- ir.english_name as clab_indian_reserve_name,
-- ir.band_name as clab_indian_reserve_band_name,
-- np.english_name as clab_national_park_name,
-- pp.protected_lands_name as bc_protected_lands_name,
-- pmbc.owner_type as pmbc_owner_type,
-- nr.region_org_unit_name as adm_nr_region,
-- nr.district_name as adm_nr_district
--FROM bcfishpass.crossings c
--LEFT OUTER JOIN whse_legal_admin_boundaries.abms_regional_districts_sp rd
--ON ST_Intersects(c.geom, rd.geom)
--LEFT OUTER JOIN whse_legal_admin_boundaries.abms_municipalities_sp muni
--ON ST_Intersects(c.geom, muni.geom)
--LEFT OUTER JOIN whse_admin_boundaries.adm_indian_reserves_bands_sp ir
--ON ST_Intersects(c.geom, ir.geom)
--LEFT OUTER JOIN whse_admin_boundaries.clab_national_parks np
--ON ST_Intersects(c.geom, np.geom)
--LEFT OUTER JOIN whse_tantalis.ta_park_ecores_pa_svw pp
--ON ST_Intersects(c.geom, pp.geom)
--LEFT OUTER JOIN whse_cadastre.pmbc_parcel_fabric_poly_svw pmbc
--ON ST_Intersects(c.geom, pmbc.geom)
--LEFT OUTER JOIN whse_admin_boundaries.adm_nr_districts_sp nr
--ON ST_Intersects(c.geom, pmbc.geom)
--ORDER BY c.aggregated_crossings_id, rd.admin_area_abbreviation, muni.admin_area_abbreviation, ir.english_name, pp.protected_lands_name, pmbc.owner_type, nr.district_name;
62 changes: 62 additions & 0 deletions db/views/02_crossings_upstr_dnstr_observations.sql
Original file line number Diff line number Diff line change
@@ -0,0 +1,62 @@
-- downstream observations ***within the same watershed group***
drop materialized view if exists bcfishpass.crossings_dnstr_observations;

create materialized view bcfishpass.crossings_dnstr_observations as
select
aggregated_crossings_id,
array_agg(species_code) as observedspp_dnstr
FROM
(
select distinct
a.aggregated_crossings_id,
unnest(species_codes) as species_code
from bcfishpass.crossings a
left outer join bcfishobs.fiss_fish_obsrvtn_events fo
on FWA_Downstream(
a.blue_line_key,
a.downstream_route_measure,
a.wscode_ltree,
a.localcode_ltree,
fo.blue_line_key,
fo.downstream_route_measure,
fo.wscode_ltree,
fo.localcode_ltree
)
and a.watershed_group_code = fo.watershed_group_code
order by a.aggregated_crossings_id, species_code
) AS f
group by aggregated_crossings_id;

create index on bcfishpass.crossings_dnstr_observations (aggregated_crossings_id);


-- upstream observations ***within the same watershed group***
drop materialized view if exists bcfishpass.crossings_upstr_observations;

create materialized view bcfishpass.crossings_upstr_observations as
select
aggregated_crossings_id,
array_agg(species_code) as observedspp_upstr
FROM
(
select distinct
a.aggregated_crossings_id,
unnest(species_codes) as species_code
from bcfishpass.crossings a
left outer join bcfishobs.fiss_fish_obsrvtn_events fo
on FWA_Upstream(
a.blue_line_key,
a.downstream_route_measure,
a.wscode_ltree,
a.localcode_ltree,
fo.blue_line_key,
fo.downstream_route_measure,
fo.wscode_ltree,
fo.localcode_ltree
)
and a.watershed_group_code = fo.watershed_group_code
order by a.aggregated_crossings_id, species_code
) AS f
group by aggregated_crossings_id;

create index on bcfishpass.crossings_upstr_observations (aggregated_crossings_id);
12 changes: 8 additions & 4 deletions db/views/03_crossings_vw.sql
Original file line number Diff line number Diff line change
Expand Up @@ -14,7 +14,7 @@ select
c.user_barrier_anthropogenic_id,
c.modelled_crossing_id,
c.crossing_source,
c.crossing_feature_type,
cft.crossing_feature_type,
c.pscis_status,
c.crossing_type_code,
c.crossing_subtype_code,
Expand Down Expand Up @@ -45,7 +45,7 @@ select
c.utm_zone,
c.utm_easting,
c.utm_northing,
c.dbm_mof_50k_grid,
t.map_tile_display_name as dbm_mof_50k_grid,
c.linear_feature_id,
c.blue_line_key,
c.watershed_key,
Expand All @@ -62,8 +62,8 @@ select
s.map_upstream,
s.channel_width,
s.mad_m3s,
array_to_string(c.observedspp_dnstr, ';') as observedspp_dnstr,
array_to_string(c.observedspp_upstr, ';') as observedspp_upstr,
array_to_string(cdo.observedspp_dnstr, ';') as observedspp_dnstr,
array_to_string(cuo.observedspp_upstr, ';') as observedspp_upstr,
array_to_string(cd.features_dnstr, ';') as crossings_dnstr,
array_to_string(ad.features_dnstr, ';') as barriers_anthropogenic_dnstr,
coalesce(array_length(ad.features_dnstr, 1), 0) as barriers_anthropogenic_dnstr_count,
Expand Down Expand Up @@ -233,13 +233,17 @@ select
h.wct_rearing_belowupstrbarriers_km,
c.geom
from bcfishpass.crossings c
inner join bcfishpass.crossings_feature_type_vw cft on c.aggregated_crossings_id = cft.aggregated_crossings_id
left outer join bcfishpass.crossings_dnstr_observations cdo on c.aggregated_crossings_id = cdo.aggregated_crossings_id
left outer join bcfishpass.crossings_upstr_observations cuo on c.aggregated_crossings_id = cuo.aggregated_crossings_id
left outer join bcfishpass.crossings_dnstr_crossings cd on c.aggregated_crossings_id = cd.aggregated_crossings_id
left outer join bcfishpass.crossings_dnstr_barriers_anthropogenic ad on c.aggregated_crossings_id = ad.aggregated_crossings_id
left outer join bcfishpass.crossings_upstr_barriers_anthropogenic au on c.aggregated_crossings_id = au.aggregated_crossings_id
left outer join bcfishpass.crossings_upstr_barriers_per_model_vw aum on c.aggregated_crossings_id = aum.aggregated_crossings_id
left outer join bcfishpass.crossings_upstream_access a on c.aggregated_crossings_id = a.aggregated_crossings_id
left outer join bcfishpass.crossings_upstream_habitat h on c.aggregated_crossings_id = h.aggregated_crossings_id
left outer join bcfishpass.streams s on c.linear_feature_id = s.linear_feature_id
left outer join whse_basemapping.dbm_mof_50k_grid t ON ST_Intersects(c.geom, t.geom)
order by c.aggregated_crossings_id, s.downstream_route_measure;

create unique index on bcfishpass.crossings_vw (aggregated_crossings_id);
Expand Down
Loading

0 comments on commit ec2db00

Please sign in to comment.