Skip to content

Commit

Permalink
cache data on s3
Browse files Browse the repository at this point in the history
  • Loading branch information
@smnorris
smnorris committed Nov 20, 2024
1 parent b96a9ec commit 87c37f7
Show file tree
Hide file tree
Showing 2 changed files with 19 additions and 4 deletions.
8 changes: 4 additions & 4 deletions Makefile
View file
Original file line number Diff line number Diff line change
Expand Up @@ -75,16 +75,16 @@ clean_db:

# download and rename (so we do not have to unzip)
data/FWA_BC.gdb.zip: .make/db
curl -o $@ ftp://ftp.geobc.gov.bc.ca/sections/outgoing/bmgs/FWA_Public/FWA_BC.zip
curl -o $@ https://nrs.objectstore.gov.bc.ca/bchamp/fwapg/FWA_BC.zip

data/FWA_LINEAR_BOUNDARIES_SP.gdb.zip: .make/db
curl -o $@ ftp://ftp.geobc.gov.bc.ca/sections/outgoing/bmgs/FWA_Public/FWA_LINEAR_BOUNDARIES_SP.zip
curl -o $@ https://nrs.objectstore.gov.bc.ca/bchamp/fwapg/FWA_LINEAR_BOUNDARIES_SP.zip

data/FWA_WATERSHEDS_POLY.gdb.zip: .make/db
curl -o $@ ftp://ftp.geobc.gov.bc.ca/sections/outgoing/bmgs/FWA_Public/FWA_WATERSHEDS_POLY.zip
curl -o $@ https://nrs.objectstore.gov.bc.ca/bchamp/fwapg/FWA_WATERSHEDS_POLY.zip

data/FWA_STREAM_NETWORKS_SP.gdb.zip: .make/db
curl -o $@ ftp://ftp.geobc.gov.bc.ca/sections/outgoing/bmgs/FWA_Public/FWA_STREAM_NETWORKS_SP.zip
curl -o $@ https://nrs.objectstore.gov.bc.ca/bchamp/fwapg/FWA_STREAM_NETWORKS_SP.zip

# load the larger tables per watershed group
.make/fwa_stream_networks_sp: data/FWA_STREAM_NETWORKS_SP.gdb.zip .make/db
Expand Down
15 changes: 15 additions & 0 deletions cache.sh
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,15 @@
#!/bin/bash
set -euxo pipefail

# cache to s3 for (much) faster downloads
# also, add .gdb to filename so files can be accessed directly via /vsizip//vsicurl

curl -o /tmp/FWA_BC.gdb.zip ftp://ftp.geobc.gov.bc.ca/sections/outgoing/bmgs/FWA_Public/FWA_BC.zip
curl -o /tmp/FWA_LINEAR_BOUNDARIES_SP.gdb.zip ftp://ftp.geobc.gov.bc.ca/sections/outgoing/bmgs/FWA_Public/FWA_LINEAR_BOUNDARIES_SP.zip
curl -o /tmp/FWA_WATERSHEDS_POLY.gdb.zip ftp://ftp.geobc.gov.bc.ca/sections/outgoing/bmgs/FWA_Public/FWA_WATERSHEDS_POLY.zip
curl -o /tmp/FWA_STREAM_NETWORKS_SP.gdb.zip ftp://ftp.geobc.gov.bc.ca/sections/outgoing/bmgs/FWA_Public/FWA_STREAM_NETWORKS_SP.zip

aws s3 cp /tmp/FWA_BC.gdb.zip s3://bchamp/fwapg/FWA_BC.gdb.zip --acl public-read
aws s3 cp /tmp/FWA_LINEAR_BOUNDARIES_SP.gdb.zip s3://bchamp/fwapg/FWA_LINEAR_BOUNDARIES_SP.gdb.zip --acl public-read
aws s3 cp /tmp/FWA_WATERSHEDS_POLY.gdb.zip s3://bchamp/fwapg/FWA_WATERSHEDS_POLY.gdb.zip --acl public-read
aws s3 cp /tmp/FWA_STREAM_NETWORKS_SP.gdb.zip s3://bchamp/fwapg/FWA_STREAM_NETWORKS_SP.gdb.zip --acl public-read

0 comments on commit 87c37f7

Please sign in to comment.