save a lot of load time using s3 and materialized view based caching …

…for geojson, env based s3 files to come
bcgov · Oct 31, 2023 · 249d3bd · 249d3bd
1 parent 8306286
commit 249d3bd
Show file tree

Hide file tree

Showing 3 changed files with 58 additions and 34 deletions.
diff --git a/api/src/queries/activity-queries.ts b/api/src/queries/activity-queries.ts
@@ -207,7 +207,7 @@ export const getActivitiesSQL = (
         cpo.activity_incoming_data_id,
         string_agg(cpo.invasive_plant, ', ') AS current_positive_species
     FROM
-        current_positive_observations cpo
+        current_positive_observations_materialized cpo
     GROUP BY
         cpo.activity_incoming_data_id
 ),
@@ -216,7 +216,7 @@ CurrentNegativeObservations AS (
         cno.activity_incoming_data_id,
         string_agg(cno.invasive_plant, ', ') AS current_negative_species
     FROM
-        current_negative_observations cno
+        current_negative_observations_materialized cno
     GROUP BY
         cno.activity_incoming_data_id
 )  `);
@@ -437,6 +437,10 @@ LEFT JOIN
 
   sqlStatement.append(SQL` where 1 = 1`);
 
+  if(lean) {
+    sqlStatement.append(SQL` and a.activity_incoming_data_id > 36345`)
+  }
+
   if (searchCriteria.activity_type && searchCriteria.activity_type.length) {
     sqlStatement.append(SQL` AND activity_type IN (`);
 

diff --git a/app/src/state/sagas/map/online.ts b/app/src/state/sagas/map/online.ts
@@ -23,47 +23,43 @@ const checkForErrors = (response: any, status?: any, url?: any) => {
 
 //
 export function* handle_ACTIVITIES_GEOJSON_GET_ONLINE(action) {
+  const networkReturnForSignedURLToCachedData = yield InvasivesAPI_Call('POST', `/api/activities-lean/`, {
+    ...action.payload.activitiesFilterCriteria,
+    s3SignedUrlRequest: true
+  });
 
-  //get a signed url to the zipped file on s3:
-
-
-
-  const networkReturn = yield InvasivesAPI_Call(
-    'POST',
-    `/api/activities-lean/`,
-    {...action.payload.activitiesFilterCriteria, s3SignedUrlRequest: true}
-  );
-
-  console.dir(networkReturn)
-  const signedURL = networkReturn.data.signedURL
-
-  //get the zipped file from the signed url:
+  const signedURL = networkReturnForSignedURLToCachedData.data.signedURL;
 
-  let networkReturn2
+  let networkReturnS3;
 
   try {
+    networkReturnS3 = yield Http.request({
+      method: 'GET',
+      headers: {
+        'Content-Type': 'application/json',
+        'Accept-Encoding': 'gzip'
+      },
+      url: signedURL
+    });
+  } catch (e) {
+    console.dir(e);
+  }
 
-  networkReturn2 = yield Http.request({
-    method: 'GET',
-    headers: {
-      'Content-Type': 'application/json',
-      'Accept-Encoding': 'gzip'
-    },
-    url: signedURL
+  const networkReturn3 = yield InvasivesAPI_Call('POST', `/api/activities-lean/`, {
+    ...action.payload.activitiesFilterCriteria,
+    s3SignedUrlRequest: false
   });
 
-}
-catch(e) {
-  console.dir(e)
-}
-
-  console.dir(networkReturn2)
-
   let featureCollection = {
     type: 'FeatureCollection',
-    features: networkReturn2.data.result.rows.map((row) => {
-      return row.geojson ? row.geojson : row;
-    })
+    features: [
+      ...networkReturnS3.data.result.rows.map((row) => {
+        return row.geojson ? row.geojson : row;
+      }),
+      ...networkReturn3.data.result.rows.map((row) => {
+        return row.geojson ? row.geojson : row;
+      })
+    ]
   };
 
   yield put({

diff --git a/database/src/migrations/0072_materialize_current_and_negative_views.ts b/database/src/migrations/0072_materialize_current_and_negative_views.ts
@@ -0,0 +1,24 @@
+import { Knex } from 'knex';
+
+export async function up(knex: Knex): Promise<void> {
+  await knex.raw(
+    `
+    set search_path=invasivesbc,public;
+
+   
+   
+   
+   create materialized view if not exists invasivesbc.current_positive_observations_materialized as (select * from invasivesbc.current_positive_observations );
+   create materialized view if not exists invasivesbc.current_negative_observations_materialized as (select * from invasivesbc.current_negative_observations ); 
+  `
+  );
+}
+
+export async function down(knex: Knex): Promise<void> {
+  await knex.raw(`
+
+  drop materialized view invasivesbc.current_positive_observations_materialized;
+  drop materialized view invasivesbc.current_negative_observations_materialized;
+
+  `);
+}