From e6fae771a1afe1b2a458242dd3783ad92b7eca4e Mon Sep 17 00:00:00 2001 From: Simon Norris Date: Thu, 3 Oct 2024 11:11:51 -0700 Subject: [PATCH] add fwapg prefix to s3 files, cache hydrosheds on s3 --- Makefile | 6 +++--- extras/README.md | 14 +++++++------- extras/lookups/README.md | 2 +- 3 files changed, 11 insertions(+), 11 deletions(-) diff --git a/Makefile b/Makefile index 77ab139..cfd0d66 100644 --- a/Makefile +++ b/Makefile @@ -233,7 +233,7 @@ data/WBD_National_GDB.zip: touch $@ # For YT, NWT, AB watersheds, use hydrosheds https://www.hydrosheds.org/ -# Source hydrosheds shapefiles must be manually downloaded, so I've cached them at hillcrestgeo +# Source hydrosheds shapefiles must be manually downloaded, this downloads a cached copy .make/hydrosheds: .make/db # clear any existing data $(PSQL) -c "truncate hydrosheds.hybas_lev12_v1c" @@ -249,7 +249,7 @@ data/WBD_National_GDB.zip: -preserve_fid \ -where "hybas_id is not null" \ -nlt PROMOTE_TO_MULTI \ - /vsizip/vsicurl/https://www.hillcrestgeo.ca/outgoing/public/fwapg/hydrosheds.gpkg.zip + /vsizip/vsicurl/https://nrs.objectstore.gov.bc.ca/bchamp/fwapg/hydrosheds.gpkg.zip touch $@ # rather than generating these lookups/datasets (scripts in /extras), download pre-generated data @@ -257,6 +257,6 @@ data/WBD_National_GDB.zip: for table in $(EXTRAS) ; do \ echo $$table ;\ set -e; $(PSQL) -c "truncate whse_basemapping.$$table" ; \ - set -e; $(PSQL) -c "\copy whse_basemapping.$$table FROM PROGRAM 'curl -s https://nrs.objectstore.gov.bc.ca/bchamp/$$table.csv.gz | gunzip' delimiter ',' csv header" ; \ + set -e; $(PSQL) -c "\copy whse_basemapping.$$table FROM PROGRAM 'curl -s https://nrs.objectstore.gov.bc.ca/bchamp/fwapg/$$table.csv.gz | gunzip' delimiter ',' csv header" ; \ done touch $@ \ No newline at end of file diff --git a/extras/README.md b/extras/README.md index df6e55f..3f77c82 100644 --- a/extras/README.md +++ b/extras/README.md @@ -11,10 +11,10 @@ If updates to the lookups are required, regenerate the csv files using these scr To upload data to BC object storage on completion of scripts: -aws s3 cp channel_width/fwa_stream_networks_channel_width.csv.gz s3://$BUCKET --acl public-read -aws s3 cp discharge/fwa_stream_networks_discharge.csv.gz s3://$BUCKET --acl public-read -aws s3 cp precipitation/fwa_stream_networks_mean_annual_precip.csv.gz s3://$BUCKET --acl public-read -aws s3 cp lookups/fwa_assessment_watersheds_lut.csv.gz s3://$BUCKET --acl public-read -aws s3 cp lookups/fwa_assessment_watersheds_streams_lut.csv.gz s3://$BUCKET --acl public-read -aws s3 cp lookups/fwa_waterbodies_upstream_area.csv.gz s3://$BUCKET --acl public-read -aws s3 cp lookups/fwa_watersheds_upstream_area.csv.gz s3://$BUCKET --acl public-read \ No newline at end of file +aws s3 cp channel_width/fwa_stream_networks_channel_width.csv.gz s3://bchamp/fwapg/fwa_stream_networks_channel_width.csv.gz --acl public-read +aws s3 cp discharge/fwa_stream_networks_discharge.csv.gz s3://bchamp/fwapg/fwa_stream_networks_discharge.csv.gz --acl public-read +aws s3 cp precipitation/fwa_stream_networks_mean_annual_precip.csv.gz s3://bchamp/fwapg/fwa_stream_networks_mean_annual_precip.csv.gz --acl public-read +aws s3 cp lookups/fwa_assessment_watersheds_lut.csv.gz s3://bchamp/fwapg/fwa_assessment_watersheds_lut.csv.gz --acl public-read +aws s3 cp lookups/fwa_assessment_watersheds_streams_lut.csv.gz s3://bchamp/fwapg/fwa_assessment_watersheds_streams_lut.csv.gz --acl public-read +aws s3 cp lookups/fwa_waterbodies_upstream_area.csv.gz s3://bchamp/fwapg/fwa_waterbodies_upstream_area.csv.gz --acl public-read +aws s3 cp lookups/fwa_watersheds_upstream_area.csv.gz s3://bchamp/fwapg/fwa_watersheds_upstream_area.csv.gz --acl public-read \ No newline at end of file diff --git a/extras/lookups/README.md b/extras/lookups/README.md index 5dcbfd0..3a6a8ff 100644 --- a/extras/lookups/README.md +++ b/extras/lookups/README.md @@ -29,6 +29,6 @@ As with upstream watershed area, it is often useful to know how much lake/reserv ## Assessment watersheds lookups -Some workflows require relating `fwa_assessment_watersheds_poly` to stream segments and fundamental watersheds. There are no existing keys in the data that maintain this link - the query requires a resource intensive spatial function. Rather than running a spatial query every time, we can create lookups. The lookups are provided at `https://hillcrestgeo.ca/outgoing/public/fwapg/` and loaded by `01_load.sh`, but they can be created from scratch with this script: +Some workflows require relating `fwa_assessment_watersheds_poly` to stream segments and fundamental watersheds. There are no existing keys in the data that maintain this link - the query requires a resource intensive spatial function. Rather than running a spatial query every time, we can create lookups. The lookups are provided at `https://nrs.objectstore.gov.bc.ca/bchamp/fwapg` and loaded by `01_load.sh`, but they can be created from scratch with this script: ./assessment_watersheds_lookups.sh