csv stream s3

api/src/utils/iapp-json-utils.ts

@@ -355,6 +355,38 @@ const getIAPPjson = (row: any, extract: any, searchCriteria: any) => {
   }
 };
 
+var AWS = require('aws-sdk');
+const { Readable, PassThrough } = require('stream');
+import { v4 as uuidv4 } from 'uuid';
+import { getS3SignedURL } from './file-utils';
+const OBJECT_STORE_URL = process.env.OBJECT_STORE_URL || 'nrs.objectstore.gov.bc.ca';
+const AWS_ENDPOINT = new AWS.Endpoint(OBJECT_STORE_URL);
+
+const S3 = new AWS.S3({
+  endpoint: AWS_ENDPOINT.href,
+  accessKeyId: process.env.OBJECT_STORE_ACCESS_KEY_ID,
+  secretAccessKey: process.env.OBJECT_STORE_SECRET_KEY_ID,
+  signatureVersion: 'v4',
+  s3ForcePathStyle: true
+});
+
+function upload(S3, key) {
+  let pass = new PassThrough();
+
+  let params = {
+    Bucket: process.env.OBJECT_STORE_BUCKET_NAME,
+    Key: key,
+    Body: pass
+  };
+
+  S3.upload(params, function (error, data) {
+    console.error(error);
+    console.info(data);
+  });
+
+  return pass;
+}
+
 export async function streamActivitiesResult(searchCriteria: any, res: any) {
   const connection = await getDBConnection();
 
@@ -369,97 +401,103 @@ export async function streamActivitiesResult(searchCriteria: any, res: any) {
   }
 
   try {
-    res.contentType('text/csv')
-      .setHeader('Content-Disposition', 'attachment; filename="export.csv"')
-      .setHeader('transfer-encoding', 'chunked');
-
-
     const cursor = await connection.query(new Cursor(sqlStatement.text, sqlStatement.values));
 
     const generatedRows = generateSitesCSV(cursor, searchCriteria.CSVType);
-    for await (const row of generatedRows) {
-      res.write(row);
-    }
-  } finally {
-    res.end();
-    connection.release();
+    const readable = Readable.from(generatedRows);
+    const key = `CSV-${searchCriteria.CSVType}-${uuidv4()}.csv`;
+
+    readable.pipe(upload(S3, key)).on('end', async () => {
+      // get signed url
+      const url = await getS3SignedURL(key);
+      res.status(200).send(url);
+      res.end();
+      connection.release();
+    });
+  } catch (e) {
+    console.error(e);
+    res.status(500);
   }
 }
 
 export const streamIAPPResult = async (searchCriteria: any, res: any) => {
-    let connection;
+  let connection;
+  try {
+    connection = await getDBConnection();
+  } catch (e) {
+    throw {
+      message: 'Error connecting to database',
+      code: 500,
+      namespace: 'iapp-json-utils'
+    };
+  }
+
+  if (!connection) {
+    throw {
+      code: 503,
+      message: 'Failed to establish database connection',
+      namespace: 'iapp-json-utils'
+    };
+  }
+
+  const sqlStatement: SQLStatement = getSitesBasedOnSearchCriteriaSQL(searchCriteria);
+
+  if (!sqlStatement) {
+    throw {
+      code: 400,
+      message: 'Failed to build SQL statement',
+      namespace: 'iapp-json-utils'
+    };
+  }
+
+  if (searchCriteria.isCSV) {
     try {
-      connection = await getDBConnection();
+      const cursor = await connection.query(new Cursor(sqlStatement.text, sqlStatement.values));
+
+      const generatedRows = generateSitesCSV(cursor, searchCriteria.CSVType);
+      const readable = Readable.from(generatedRows);
+      const key = `CSV-${searchCriteria.CSVType}-${uuidv4()}.csv`;
+      readable.pipe(upload(S3, key)).on('end', async () => {
+        // get signed url
+        const url = await getS3SignedURL(key);
+        res.status(200).send(url);
+        res.end();
+        connection.release();
+        return;
+      });
     } catch (e) {
-      throw {
-        message: 'Error connecting to database',
-        code: 500,
-        namespace: 'iapp-json-utils'
-      };
-    }
-
-    if (!connection) {
-      throw {
-        code: 503,
-        message: 'Failed to establish database connection',
-        namespace: 'iapp-json-utils'
-      };
+      console.error(e);
+      res.status(500);
     }
+  } else {
+    try {
+      const response = await connection.query(sqlStatement.text, sqlStatement.values);
+      var returnVal2 = response.rowCount > 0 ? await mapSitesRowsToJSON(response, searchCriteria) : [];
 
-    const sqlStatement: SQLStatement = getSitesBasedOnSearchCriteriaSQL(searchCriteria);
+      res.setHeader('Content-Type', 'application/json; charset=utf-8');
 
-    if (!sqlStatement) {
+      res.write(
+        JSON.stringify({
+          message: 'Got points of interest by search filter criteria',
+          request: searchCriteria,
+          result: {
+            rows: returnVal2
+          },
+          count: returnVal2.length,
+          namespace: 'points-of-interest',
+          code: 200
+        })
+      );
+    } catch (error) {
+      defaultLog.debug({ label: 'getIAPPjson', message: 'error', error });
       throw {
-        code: 400,
-        message: 'Failed to build SQL statement',
+        code: 500,
+        message: 'Failed to get IAPP sites',
         namespace: 'iapp-json-utils'
       };
-    }
-
-    try {
-      if (searchCriteria.isCSV) {
-        res.contentType('text/csv')
-          .setHeader('Content-Disposition', 'attachment; filename="export.csv"')
-          .setHeader('transfer-encoding', 'chunked');
-
-        const cursor = await connection.query(new Cursor(sqlStatement.text, sqlStatement.values));
-
-        const generatedRows = generateSitesCSV(cursor, searchCriteria.CSVType);
-        for await (const row of generatedRows) {
-          res.write(row);
-        }
-
-      } else {
-        try {
-          const response = await connection.query(sqlStatement.text, sqlStatement.values);
-          var returnVal2 = response.rowCount > 0 ? await mapSitesRowsToJSON(response, searchCriteria) : [];
-
-          res.setHeader('Content-Type', 'application/json; charset=utf-8');
-
-          res.write(
-            JSON.stringify({
-              message: 'Got points of interest by search filter criteria',
-              request: searchCriteria,
-              result: {
-                rows: returnVal2
-              },
-              count: returnVal2.length,
-              namespace: 'points-of-interest',
-              code: 200
-            })
-          );
-        } catch (error) {
-          defaultLog.debug({ label: 'getIAPPjson', message: 'error', error });
-          throw {
-            code: 500,
-            message: 'Failed to get IAPP sites',
-            namespace: 'iapp-json-utils'
-          };
-        }
-      }
     } finally {
       res.end();
       connection.release();
     }
   }
-;
+};

app/src/components/activities-list/Tables/ExcelExporter.css

@@ -0,0 +1,6 @@
+.CSV-spinner > * {
+  position: relative;
+  top: 0;
+  left: 0;
+  margin: 0 1rem;
+}

app/src/components/activities-list/Tables/ExcelExporter.tsx

@@ -1,10 +1,12 @@
 import { Button, MenuItem, Select, Tooltip } from '@mui/material';
 import React, { useEffect, useState } from 'react';
 import { useDispatch, useSelector } from 'react-redux';
-import { RECORD_SET_TO_EXCEL_REQUEST } from 'state/actions';
+import { CSV_LINK_CLICKED, RECORD_SET_TO_EXCEL_REQUEST } from 'state/actions';
 import { selectUserSettings } from 'state/reducers/userSettings';
 import DownloadIcon from '@mui/icons-material/Download';
 import { selectMap } from 'state/reducers/map';
+import Spinner from 'components/spinner/Spinner';
+import "./ExcelExporter.css";
 
 const ExcelExporter = (props) => {
   const dispatch = useDispatch();
@@ -47,23 +49,48 @@ const ExcelExporter = (props) => {
   return (
     <>
       <Tooltip title="CSV Export">
-        <Button
-          disabled={!mapState?.CanTriggerCSV}
-          onClick={() =>
-            dispatch({
-              type: RECORD_SET_TO_EXCEL_REQUEST,
-              payload: {
-                id: props.setName,
-                CSVType: selection
+        {mapState?.linkToCSV && props.setName === mapState?.recordSetForCSV ?
+          <a href={mapState?.linkToCSV} download>
+            <Button
+              onClick={() =>
+                dispatch({
+                  type: CSV_LINK_CLICKED
+                })
               }
-            })
-          }
-          sx={{ mr: 1, ml: 'auto' }}
-          size={'small'}
-          variant="contained">
-          CSV
-          <DownloadIcon />
-        </Button>
+              disabled={mapState?.linkToCSV.length < 1}
+              sx={{ mr: 1, ml: 'auto' }}
+              size={'small'}
+              variant="contained">
+              Download CSV
+              <DownloadIcon />
+            </Button>
+          </a>
+          :
+          <div className='CSV-spinner'>
+            {(
+              mapState?.CanTriggerCSV ? 
+              <Button
+              disabled={!mapState?.CanTriggerCSV}
+              onClick={() =>
+                dispatch({
+                  type: RECORD_SET_TO_EXCEL_REQUEST,
+                  payload: {
+                      id: props.setName,
+                      CSVType: selection
+                    }
+                  })
+                }
+                sx={{ mr: 1, ml: 'auto' }}
+                size={'small'}
+                variant="contained">
+                Generate CSV link
+                <DownloadIcon />
+              </Button>
+              : 
+              <Spinner></Spinner>
+            )}
+          </div>
+        }
       </Tooltip>
       <Tooltip title="Choose report type" placement="right">
         <Select

app/src/state/actions/index.ts

@@ -239,6 +239,7 @@ export const TOGGLE_BASIC_PICKER_LAYER = 'TOGGLE_BASIC_PICKER_LAYER';
 export const RECORD_SET_TO_EXCEL_REQUEST = 'RECORD_SET_TO_EXCEL_REQUEST';
 export const RECORD_SET_TO_EXCEL_SUCCESS = 'RECORD_SET_TO_EXCEL_SUCCESS';
 export const RECORD_SET_TO_EXCEL_FAILURE = 'RECORD_SET_TO_EXCEL_FAILURE';
+export const CSV_LINK_CLICKED = 'CSV_LINK_CLICKED';
 export const MAP_LABEL_EXTENT_FILTER_REQUEST = 'MAP_LABEL_EXTENT_FILTER_REQUEST';
 export const MAP_LABEL_EXTENT_FILTER_SUCCESS = 'MAP_LABEL_EXTENT_FILTER_SUCCESS';
 export const MAP_LABEL_EXTENT_FILTER_FAILURE = 'MAP_LABEL_EXTENT_FILTER_FAILURE';

app/src/state/reducers/map.ts

@@ -40,7 +40,8 @@ import {
   IAPP_EXTENT_FILTER_SUCCESS,
   RECORD_SET_TO_EXCEL_REQUEST,
   RECORD_SET_TO_EXCEL_FAILURE,
-  RECORD_SET_TO_EXCEL_SUCCESS
+  RECORD_SET_TO_EXCEL_SUCCESS,
+  CSV_LINK_CLICKED
 } from '../actions';
 
 import { AppConfig } from '../config';
@@ -78,7 +79,9 @@ class MapState {
   labelBoundsPolygon: any;
   IAPPBoundsPolygon: any;
   tooManyLabelsDialog: IGeneralDialog;
-  CanTriggerCSV: true;
+  CanTriggerCSV: boolean;
+  linkToCSV: string;
+  recordSetForCSV: number;
 
   constructor() {
     this.initialized = false;
@@ -121,6 +124,8 @@ class MapState {
       dialogContentText: null
     };
     this.CanTriggerCSV = true;
+    this.linkToCSV = null;
+    this.recordSetForCSV = null;
     this.whatsHere = {
       toggle: false,
       feature: null,
@@ -551,7 +556,16 @@ function createMapReducer(configuration: AppConfig): (MapState, AnyAction) => Ma
       case RECORD_SET_TO_EXCEL_SUCCESS: {
         return {
           ...state,
-          CanTriggerCSV: true
+          CanTriggerCSV: true,
+          linkToCSV: action.payload.link,
+          recordSetForCSV: action.payload.id
+        };
+      }
+      case CSV_LINK_CLICKED: {
+        return {
+          ...state,
+          linkToCSV: null,
+          recordSetForCSV: null
         };
       }
       case RECORD_SET_TO_EXCEL_FAILURE: {