update dra code table replication

smnorris · Dec 24, 2024 · 8f11642 · 8f11642
1 parent b7a8493
commit 8f11642
Show file tree

Hide file tree

Showing 3 changed files with 36 additions and 41 deletions.
diff --git a/jobs/load_monthly b/jobs/load_monthly
@@ -34,7 +34,7 @@ ogr2ogr -f PostgreSQL \
   -nln whse_basemapping.transport_line \
   --debug ON \
   --config PG_USE_COPY=YES \
-  /vsicurl/https://nrs.objectstore.gov.bc.ca/bchamp/bcdata/transport_line.parquet \
+  /vsicurl/https://nrs.objectstore.gov.bc.ca/bchamp/bcdata/whse_basemapping.transport_line.parquet \
   transport_line
 
 # parcel fabric, renaming download so it can be read without unzipping

diff --git a/jobs/load_static b/jobs/load_static
@@ -8,48 +8,20 @@ set -euxo pipefail
 PSQL="psql $DATABASE_URL -v ON_ERROR_STOP=1"
 WSGS=$($PSQL -AXt -c "SELECT watershed_group_code FROM whse_basemapping.fwa_watershed_groups_poly")
 
-# **
-# ** note temp dra data cache to be replaced with updated source**
-# **
+
 #-------
 # DRA code tables
 #-------
-$PSQL -c "truncate whse_basemapping.transport_line_type_code;
-          truncate whse_basemapping.transport_line_surface_code;
-          truncate whse_basemapping.transport_line_divided_code;
-          truncate whse_basemapping.transport_line_structure_code;"
-ogr2ogr \
-  -f PostgreSQL \
-  "PG:$DATABASE_URL" \
-  -update \
-  -append \
-  -nln whse_basemapping.transport_line_type_code \
-  /vsizip//vsicurl/https://bcfishpass.s3.us-west-2.amazonaws.com/dgtl_road_atlas.gdb.zip \
-  TRANSPORT_LINE_TYPE_CODE
-ogr2ogr \
-  -f PostgreSQL \
-  "PG:$DATABASE_URL" \
-  -update \
-  -append \
-  -nln whse_basemapping.transport_line_surface_code \
-  /vsizip//vsicurl/https://bcfishpass.s3.us-west-2.amazonaws.com/dgtl_road_atlas.gdb.zip \
-  TRANSPORT_LINE_SURFACE_CODE
-ogr2ogr \
-  -f PostgreSQL \
-  "PG:$DATABASE_URL" \
-  -update \
-  -append \
-  -nln whse_basemapping.transport_line_divided_code \
-  /vsizip//vsicurl/https://bcfishpass.s3.us-west-2.amazonaws.com/dgtl_road_atlas.gdb.zip \
-  TRANSPORT_LINE_DIVIDED_CODE
-ogr2ogr \
-  -f PostgreSQL \
-  "PG:$DATABASE_URL" \
-  -update \
-  -append \
-  -nln whse_basemapping.transport_line_structure_code \
-  /vsizip//vsicurl/https://bcfishpass.s3.us-west-2.amazonaws.com/dgtl_road_atlas.gdb.zip \
-  TRANSPORT_LINE_STRUCTURE_CODE
+aws s3 cp s3://bchamp/bcdata/whse_basemapping.transport_line_type_code.csv .
+aws s3 cp s3://bchamp/bcdata/whse_basemapping.transport_line_surface_code.csv .
+aws s3 cp s3://bchamp/bcdata/whse_basemapping.transport_line_divided_code.csv .
+aws s3 cp s3://bchamp/bcdata/whse_basemapping.transport_line_structure_code.csv .
+
+$PSQL -c "\copy bcfishpass.parameters_habitat_method FROM parameters/parameters_habitat_method.csv delimiter ',' csv header"
+$PSQL -c "\copy whse_basemapping.transport_line_type_code FROM whse_basemapping.transport_line_type_code.csv delimiter ',' csv header"
+$PSQL -c "\copy whse_basemapping.transport_line_surface_code FROM whse_basemapping.transport_line_surface_code.csv delimiter ',' csv header"
+$PSQL -c "\copy whse_basemapping.transport_line_divided_code FROM whse_basemapping.transport_line_divided_code.csv delimiter ',' csv header"
+$PSQL -c "\copy whse_basemapping.transport_line_structure_code FROM whse_basemapping.transport_line_structure_code.csv delimiter ',' csv header"
 
 #-------
 # Admin areas/tiles/grids/cartographic

diff --git a/jobs/replicate_dra b/jobs/replicate_dra
@@ -42,4 +42,27 @@ ogr2ogr \
      GEOMETRY as geometry
   from TRANSPORT_LINE"
 
-aws s3 cp /tmp/transport_line.parquet s3://bchamp/bcdata/transport_line.parquet --acl public-read
+# dump dra code tables to csv and send to s3 vis awscli
+ogr2ogr \
+  -f CSV \
+  /tmp/whse_basemapping.transport_line_structure_code.csv \
+  /tmp/transport_line.gdb \
+  TRANSPORT_LINE_STRUCTURE_CODE
+
+ogr2ogr \
+  -f CSV \
+  /tmp/whse_basemapping.transport_line_type_code.csv \
+  /tmp/transport_line.gdb \
+  TRANSPORT_LINE_TYPE_CODE
+
+ogr2ogr \
+  -f CSV \
+  /tmp/whse_basemapping.transport_line_surface_code.csv \
+  /tmp/transport_line.gdb \
+  TRANSPORT_LINE_SURFACE_CODE
+
+aws s3 cp /tmp/transport_line.parquet s3://bchamp/bcdata/whse_basemapping.transport_line.parquet --acl public-read
+aws s3 cp /tmp/whse_basemapping.transport_line_structure_code.csv s3://bchamp/bcdata/whse_basemapping.transport_line_structure_code.csv
+aws s3 cp /tmp/whse_basemapping.transport_line_type_code.csv s3://bchamp/bcdata/whse_basemapping.transport_line_type_code.csv
+aws s3 cp /tmp/whse_basemapping.transport_line_surface_code.csv s3://bchamp/bcdata/whse_basemapping.transport_line_surface_code.csv
+