From 2f6427665456026a40d300e753d2312a30752185 Mon Sep 17 00:00:00 2001 From: Micheal Wells Date: Fri, 3 Nov 2023 16:14:18 -0700 Subject: [PATCH 1/4] streaming + s3 for v2 csv exports api side --- api/src/paths/v2/activities.ts | 85 ++++++++++++++++++++++++++++++-- api/src/utils/iapp-json-utils.ts | 5 +- 2 files changed, 84 insertions(+), 6 deletions(-) diff --git a/api/src/paths/v2/activities.ts b/api/src/paths/v2/activities.ts index d9433b1d2..8f0feac37 100644 --- a/api/src/paths/v2/activities.ts +++ b/api/src/paths/v2/activities.ts @@ -7,6 +7,7 @@ import { getuid } from 'process'; import SQL, { SQLStatement } from 'sql-template-strings'; import { InvasivesRequest } from 'utils/auth-utils'; import { getLogger } from '../../utils/logger'; +import { streamActivitiesResult } from '../../utils/iapp-json-utils'; const defaultLog = getLogger('activity'); const CACHENAME = 'Activities v2 - Fat'; @@ -201,6 +202,9 @@ function sanitizeActivityFilterObject(filterObject: any, req: any) { sanitizedSearchCriteria.serverFilterGeometries = serverFilterGeometries; sanitizedSearchCriteria.clientFilterGeometries = clientFilterGeometries; sanitizedSearchCriteria.clientReqTableFilters = sanitizedTableFilters; + //todo actually validate: + sanitizedSearchCriteria.isCSV = filterObject?.isCSV; + sanitizedSearchCriteria.CSVType = filterObject?.CSVType; defaultLog.debug({ label: 'getActivitiesBySearchFilterCriteria', message: 'sanitizedObject', @@ -237,6 +241,10 @@ function getActivitiesBySearchFilterCriteria(): RequestHandler { } sql = getActivitiesSQLv2(filterObject); + + if (filterObject.isCSV && filterObject.CSVType) { + await streamActivitiesResult(filterObject, res, sql); + } const response = await connection.query(sql.text, sql.values); return res.status(200).json({ @@ -420,10 +428,15 @@ activities as ( function selectStatement(sqlStatement: SQLStatement, filterObject: any) { if (filterObject.selectColumns) { - const select = sqlStatement.append( - `select ${filterObject.selectColumns.map((column) => `activities.${column}`).join(',')} ` - ); - return select; + if (filterObject.isCSV) { + const select = sqlStatement.append(SQL` select extract.* `); + return select; + } else { + const select = sqlStatement.append( + `select ${filterObject.selectColumns.map((column) => `activities.${column}`).join(',')} ` + ); + return select; + } } else { const select = sqlStatement.append(`select * `); return select; @@ -432,6 +445,70 @@ function selectStatement(sqlStatement: SQLStatement, filterObject: any) { function fromStatement(sqlStatement: SQLStatement, filterObject: any) { const from = sqlStatement.append(`from activities `); + if (filterObject.isCSV) { + switch (filterObject.CSVType) { + case 'terrestrial_plant_observation': + sqlStatement.append( + 'join observation_terrestrial_plant_summary extract ON extract.activity_id = b.activity_id ' + ); + break; + case 'aquatic_plant_observation': + sqlStatement.append('join observation_aquatic_plant_summary extract ON extract.activity_id = b.activity_id '); + break; + case 'terrestrial_chemical_treatment': + sqlStatement.append( + 'join treatment_chemical_terrestrial_plant_summary extract ON extract.activity_id = b.activity_id ' + ); + break; + case 'aquatic_chemical_treatment': + sqlStatement.append( + 'join treatment_chemical_aquatic_plant_summary extract ON extract.activity_id = b.activity_id ' + ); + break; + case 'terrestrial_mechanical_treatment': + sqlStatement.append( + 'join treatment_mechanical_terrestrial_plant_summary extract ON extract.activity_id = b.activity_id ' + ); + break; + case 'aquatic_mechanical_treatment': + sqlStatement.append( + 'join treatment_mechanical_aquatic_plant_summary extract ON extract.activity_id = b.activity_id ' + ); + break; + case 'biocontrol_release': + sqlStatement.append('join biocontrol_release_summary extract ON extract.activity_id = b.activity_id '); + break; + case 'biocontrol_collection': + sqlStatement.append('join biocontrol_collection_summary extract ON extract.activity_id = b.activity_id '); + break; + case 'biocontrol_dispersal': + sqlStatement.append( + 'join biocontrol_dispersal_monitoring_summary extract ON extract.activity_id = b.activity_id ' + ); + break; + case 'chemical_treatment_monitoring': + sqlStatement.append( + 'join chemical_treatment_monitoring_summary extract ON extract.activity_id = b.activity_id ' + ); + break; + case 'mechanical_treatment_monitoring': + sqlStatement.append( + 'join mechanical_treatment_monitoring_summary extract ON extract.activity_id = b.activity_id ' + ); + break; + case 'biocontrol_release_monitoring': + sqlStatement.append( + 'join biocontrol_release_monitoring_summary extract ON extract.activity_id = b.activity_id ' + ); + break; + + default: + sqlStatement.append( + 'join observation_terrestrial_plant_summary extract ON extract.activity_id = b.activity_id ' + ); + break; + } + } return from; } diff --git a/api/src/utils/iapp-json-utils.ts b/api/src/utils/iapp-json-utils.ts index 6ef10636e..12180843f 100644 --- a/api/src/utils/iapp-json-utils.ts +++ b/api/src/utils/iapp-json-utils.ts @@ -387,10 +387,11 @@ function upload(S3, key) { return pass; } -export async function streamActivitiesResult(searchCriteria: any, res: any) { +export async function streamActivitiesResult(searchCriteria: any, res: any, sqlStatementOverride?: SQLStatement) { const connection = await getDBConnection(); - const sqlStatement: SQLStatement = getActivitiesSQL(searchCriteria, false, true); + + const sqlStatement: SQLStatement = (sqlStatementOverride)? sqlStatementOverride: getActivitiesSQL(searchCriteria, false, true); if (!sqlStatement) { throw { From 0b7071d6040716d3df176e74fb06c8c67fcc9b34 Mon Sep 17 00:00:00 2001 From: Micheal Wells Date: Fri, 3 Nov 2023 16:37:08 -0700 Subject: [PATCH 2/4] grab excel exporter code from v1 and make state tweaks for v2 --- .../src/UI/Overlay/Records/ExcelExporter.css | 6 + .../src/UI/Overlay/Records/ExcelExporter.tsx | 107 ++++++++++++++++++ appv2/src/UI/Overlay/Records/RecordSet.tsx | 2 + appv2/src/state/actions.tsx | 1 + appv2/src/state/reducers/map.ts | 40 ++++++- appv2/src/state/sagas/map.ts | 38 ++----- 6 files changed, 167 insertions(+), 27 deletions(-) create mode 100644 appv2/src/UI/Overlay/Records/ExcelExporter.css create mode 100644 appv2/src/UI/Overlay/Records/ExcelExporter.tsx diff --git a/appv2/src/UI/Overlay/Records/ExcelExporter.css b/appv2/src/UI/Overlay/Records/ExcelExporter.css new file mode 100644 index 000000000..ac8821380 --- /dev/null +++ b/appv2/src/UI/Overlay/Records/ExcelExporter.css @@ -0,0 +1,6 @@ +.CSV-spinner > * { + position: relative; + top: 0; + left: 0; + margin: 0 1rem; +} \ No newline at end of file diff --git a/appv2/src/UI/Overlay/Records/ExcelExporter.tsx b/appv2/src/UI/Overlay/Records/ExcelExporter.tsx new file mode 100644 index 000000000..e727a023c --- /dev/null +++ b/appv2/src/UI/Overlay/Records/ExcelExporter.tsx @@ -0,0 +1,107 @@ +import { Button, MenuItem, Select, Tooltip } from '@mui/material'; +import React, { useEffect, useState } from 'react'; +import { useDispatch, useSelector } from 'react-redux'; +import { CSV_LINK_CLICKED, RECORD_SET_TO_EXCEL_REQUEST } from 'state/actions'; +import DownloadIcon from '@mui/icons-material/Download'; +import Spinner from 'UI/Spinner/Spinner' +import "./ExcelExporter.css"; + +const ExcelExporter = (props) => { + const dispatch = useDispatch(); + const linkToCSV = useSelector((state: any) => state.Map.linkToCSV) + const recordSetForCSV = useSelector((state: any) => state.Map.recordSetForCSV) + const CanTriggerCSV = useSelector((state: any) => state.Map.CanTriggerCSV) + const setType = useSelector((state: any) => state.UserSettings.recordSets[props.setName]?.recordSetType); + const [selection, setSelection] = useState( + setType === 'POI' ? 'site_selection_extract' : 'terrestrial_plant_observation' + ); + + let items; + if (setType === 'POI') { + items = [ + Site Selection Extract, + Survey Extract, + Chemical Treatment Extract, + Mechanical Treatment Extract, + Chemical Monitoring Extract, + Mechanical Monitoring Extract, + Biological Treatment Extract, + Biological Monitoring Extract, + Biological Dispersal Extract + ]; + } else { + items = [ + Terrestrial Plant Observation Summary, + Aquatic Plant Observation Summary, + Terrestrial Chemical Treatment Summary, + Aquatic Chemical Treatment Summary, + Terrestrial Mechanical Treatment Summary, + Biocontrol Release Summary, + Biocontrol Collection Summary, + Biocontrol Dispersal Summary, + Chemical Treatment Monitoring Summary, + Mechanical Treatment Monitoring Summary, + Biocontrol Release Monitoring Summary + ]; + } + + return ( + <> + + {linkToCSV && props.setName === recordSetForCSV ? + + + + : +
+ {( + CanTriggerCSV ? + + : + + )} +
+ } + + + + + + ); +}; + +export default ExcelExporter; diff --git a/appv2/src/UI/Overlay/Records/RecordSet.tsx b/appv2/src/UI/Overlay/Records/RecordSet.tsx index 093c23456..ed0b24134 100644 --- a/appv2/src/UI/Overlay/Records/RecordSet.tsx +++ b/appv2/src/UI/Overlay/Records/RecordSet.tsx @@ -31,6 +31,7 @@ import FilterAltIcon from '@mui/icons-material/FilterAlt'; import FilterAltOffIcon from '@mui/icons-material/FilterAltOff'; import { TouchHoldHandler } from '../TouchHoldHandler/TouchHoldHandler'; import { detectTouchDevice } from 'util/detectTouch'; +import ExcelExporter from './ExcelExporter'; export const RecordSet = (props) => { const userSettingsState = useSelector(selectUserSettings); @@ -119,6 +120,7 @@ export const RecordSet = (props) => { + +
diff --git a/appv2/src/state/actions.tsx b/appv2/src/state/actions.tsx index 18cc67e04..553844e66 100644 --- a/appv2/src/state/actions.tsx +++ b/appv2/src/state/actions.tsx @@ -346,6 +346,7 @@ export const BATCH_TEMPLATE_DOWNLOAD_REQUEST = 'BATCH_TEMPLATE_DOWNLOAD_REQUEST' export const BATCH_TEMPLATE_DOWNLOAD_SUCCESS = 'BATCH_TEMPLATE_DOWNLOAD_SUCCESS'; export const BATCH_TEMPLATE_DOWNLOAD_ERROR = 'BATCH_TEMPLATE_DOWNLOAD_ERROR'; export const BATCH_TEMPLATE_DOWNLOAD_CSV_REQUEST = 'BATCH_TEMPLATE_DOWNLOAD_CSV_REQUEST'; +export const CSV_LINK_CLICKED = 'CSV_LINK_CLICKED' export const TRAINING_VIDEOS_LIST_REQUEST = 'TRAINING_VIDEOS_LIST_REQUEST'; export const TRAINING_VIDEOS_LIST_REQUEST_COMPLETE = 'TRAINING_VIDEOS_LIST_REQUEST_COMPLETE'; diff --git a/appv2/src/state/reducers/map.ts b/appv2/src/state/reducers/map.ts index 23d136c3c..4ac1792ee 100644 --- a/appv2/src/state/reducers/map.ts +++ b/appv2/src/state/reducers/map.ts @@ -48,7 +48,11 @@ import { INIT_SERVER_BOUNDARIES_GET, TOGGLE_QUICK_PAN_TO_RECORD, USER_TOUCHED_RECORD, - TOGGLE_CUSTOMIZE_LAYERS + TOGGLE_CUSTOMIZE_LAYERS, + RECORD_SET_TO_EXCEL_REQUEST, + RECORD_SET_TO_EXCEL_SUCCESS, + CSV_LINK_CLICKED, + RECORD_SET_TO_EXCEL_FAILURE } from '../actions'; import { AppConfig } from '../config'; @@ -96,9 +100,16 @@ class MapState { userRecordOnHoverRecordID: any; userRecordOnHoverRecordRow: any; customizeLayersToggle: boolean; + linkToCSV: string; + CanTriggerCSV: boolean; + recordSetForCSV: number; + constructor() { this.initialized = false; + this.linkToCSV = null; + this.CanTriggerCSV = true; + this.recordSetForCSV = null; this.viewFilters = true; this.map_center = [53, -127]; this.map_zoom = 5; @@ -180,6 +191,33 @@ function createMapReducer(configuration: AppConfig): (MapState, AnyAction) => Ma draftState.viewFilters = !draftState.viewFilters; }); return nextState; + } + case RECORD_SET_TO_EXCEL_REQUEST: { + return { + ...state, + CanTriggerCSV: false + }; + } + case RECORD_SET_TO_EXCEL_SUCCESS: { + return { + ...state, + CanTriggerCSV: true, + linkToCSV: action.payload.link, + recordSetForCSV: action.payload.id + }; + } + case CSV_LINK_CLICKED: { + return { + ...state, + linkToCSV: null, + recordSetForCSV: null + }; + } + case RECORD_SET_TO_EXCEL_FAILURE: { + return { + ...state, + CanTriggerCSV: true + }; } case USER_CLICKED_RECORD: { const nextState = createNextState(state, (draftState) => { diff --git a/appv2/src/state/sagas/map.ts b/appv2/src/state/sagas/map.ts index ed9f3aeb8..c745a569b 100644 --- a/appv2/src/state/sagas/map.ts +++ b/appv2/src/state/sagas/map.ts @@ -487,6 +487,7 @@ function* handle_WHATS_HERE_PAGE_ACTIVITY(action) { yield put({ type: WHATS_HERE_ACTIVITY_ROWS_REQUEST }); } + function* handle_RECORD_SET_TO_EXCEL_REQUEST(action) { const authState = yield select(selectAuth); const mapState = yield select(selectMap); @@ -495,6 +496,7 @@ function* handle_RECORD_SET_TO_EXCEL_REQUEST(action) { try { let rows = []; let networkReturn; + let conditionallyUnnestedURL; if (set.recordSetType === 'POI') { let filters = getSearchCriteriaFromFilters( set?.advancedFilters ? set?.advancedFilters : null, @@ -511,19 +513,9 @@ function* handle_RECORD_SET_TO_EXCEL_REQUEST(action) { filters.isCSV = true; filters.CSVType = action.payload.CSVType; - networkReturn = yield InvasivesAPI_Call('GET', `/api/points-of-interest/`, filters, { - 'Content-Type': 'text/csv' - }); - const daBlob = new Blob([networkReturn.data]); - const url = window.URL.createObjectURL(daBlob); - const link = document.createElement('a'); - link.href = url; - const time = new Date().getTime(); - const timestamp = format(time, 'yyyy-MM-dd HH:mm:ss'); - link.setAttribute('download', `${filters.CSVType}_` + timestamp + `.csv`); - document.body.appendChild(link); - link.click(); - link.remove(); + networkReturn = yield InvasivesAPI_Call('GET', `/api/points-of-interest/`, filters, {}); + + conditionallyUnnestedURL = networkReturn?.data?.result ? networkReturn.data.result : networkReturn?.data; } else { const filters = getSearchCriteriaFromFilters( set.advancedFilters, @@ -538,21 +530,15 @@ function* handle_RECORD_SET_TO_EXCEL_REQUEST(action) { filters.isCSV = true; filters.CSVType = action.payload.CSVType; - networkReturn = yield InvasivesAPI_Call('GET', `/api/activities/`, filters, { 'Content-Type': 'text/csv' }); - const daBlob = new Blob([networkReturn.data]); - - const url = window.URL.createObjectURL(daBlob); - const link = document.createElement('a'); - link.href = url; - const time = new Date().getTime(); - const timestamp = format(time, 'yyyy-MM-dd HH:mm:ss'); - link.setAttribute('download', `${filters.CSVType}_` + timestamp + `.csv`); - document.body.appendChild(link); - link.click(); - link.remove(); + networkReturn = yield InvasivesAPI_Call('GET', `/api/activities/`, filters, {}); + conditionallyUnnestedURL = networkReturn?.data?.result ? networkReturn.data.result : networkReturn?.data; } yield put({ - type: RECORD_SET_TO_EXCEL_SUCCESS + type: RECORD_SET_TO_EXCEL_SUCCESS, + payload: { + link: conditionallyUnnestedURL, + id: action.payload.id + } }); } catch (e) { console.error(e); From 69ee3aeae24bec36f1bef74095d1c4403c7cc299 Mon Sep 17 00:00:00 2001 From: Micheal Wells Date: Fri, 3 Nov 2023 17:04:17 -0700 Subject: [PATCH 3/4] where Gabe and I left off --- appv2/src/state/sagas/map.ts | 26 +++++++++++++------------- 1 file changed, 13 insertions(+), 13 deletions(-) diff --git a/appv2/src/state/sagas/map.ts b/appv2/src/state/sagas/map.ts index c745a569b..83c786b6f 100644 --- a/appv2/src/state/sagas/map.ts +++ b/appv2/src/state/sagas/map.ts @@ -57,6 +57,7 @@ import { WHATS_HERE_SORT_FILTER_UPDATE } from '../actions'; import { + getRecordFilterObjectFromStateForAPI, handle_ACTIVITIES_GEOJSON_GET_REQUEST, handle_ACTIVITIES_GET_IDS_FOR_RECORDSET_REQUEST, handle_ACTIVITIES_TABLE_ROWS_GET_REQUEST, @@ -517,20 +518,19 @@ function* handle_RECORD_SET_TO_EXCEL_REQUEST(action) { conditionallyUnnestedURL = networkReturn?.data?.result ? networkReturn.data.result : networkReturn?.data; } else { - const filters = getSearchCriteriaFromFilters( - set.advancedFilters, - userSettings?.recordSets, - action.payload.id, - false, - set?.gridFilters, - 0, - null, - mapState?.layers?.[action.payload.id]?.filters?.sortColumns - ); - filters.isCSV = true; - filters.CSVType = action.payload.CSVType; + const currentState = yield select(selectUserSettings); + const mapState = yield select(selectMap); + + let filterObject = getRecordFilterObjectFromStateForAPI(action.payload.id, currentState); + //filterObject.page = action.payload.page ? action.payload.page : mapState.recordTables?.[action.payload.recordSetID]?.page; + filterObject.limit = 200000; + filterObject.isCSV = true; + filterObject.CSVType = action.payload.CSVType; + + const networkReturn = yield InvasivesAPI_Call('POST', `/api/v2/activities/`, { + filterObjects: [filterObject] + }); - networkReturn = yield InvasivesAPI_Call('GET', `/api/activities/`, filters, {}); conditionallyUnnestedURL = networkReturn?.data?.result ? networkReturn.data.result : networkReturn?.data; } yield put({ From b3e46c3d81a06066fac553d873e0a2f1a4d0e79a Mon Sep 17 00:00:00 2001 From: Micheal Wells Date: Tue, 7 Nov 2023 12:04:43 -0800 Subject: [PATCH 4/4] iapp csv stream v2 --- api/src/paths/v2/activities.ts | 25 ++++++++------- api/src/paths/v2/iapp.ts | 54 ++++++++++++++++++++++++++++++-- api/src/utils/iapp-json-utils.ts | 9 ++++-- 3 files changed, 72 insertions(+), 16 deletions(-) diff --git a/api/src/paths/v2/activities.ts b/api/src/paths/v2/activities.ts index 8f0feac37..97c69ac11 100644 --- a/api/src/paths/v2/activities.ts +++ b/api/src/paths/v2/activities.ts @@ -243,18 +243,20 @@ function getActivitiesBySearchFilterCriteria(): RequestHandler { sql = getActivitiesSQLv2(filterObject); if (filterObject.isCSV && filterObject.CSVType) { + res.status(200); await streamActivitiesResult(filterObject, res, sql); + } else { + const response = await connection.query(sql.text, sql.values); + + return res.status(200).json({ + message: 'fetched activities by criteria', + request: req.body, + result: response.rows, + count: response.rowCount, + namespace: 'activities', + code: 200 + }); } - const response = await connection.query(sql.text, sql.values); - - return res.status(200).json({ - message: 'fetched activities by criteria', - request: req.body, - result: response.rows, - count: response.rowCount, - namespace: 'activities', - code: 200 - }); } catch (error) { defaultLog.debug({ label: 'getActivitiesBySearchFilterCriteria', message: 'error', error }); return res.status(500).json({ @@ -444,8 +446,9 @@ function selectStatement(sqlStatement: SQLStatement, filterObject: any) { } function fromStatement(sqlStatement: SQLStatement, filterObject: any) { - const from = sqlStatement.append(`from activities `); + let from = sqlStatement.append(`from activities `); if (filterObject.isCSV) { + from = sqlStatement.append(` b `); switch (filterObject.CSVType) { case 'terrestrial_plant_observation': sqlStatement.append( diff --git a/api/src/paths/v2/iapp.ts b/api/src/paths/v2/iapp.ts index 2d618016d..f3f4a220b 100644 --- a/api/src/paths/v2/iapp.ts +++ b/api/src/paths/v2/iapp.ts @@ -7,6 +7,8 @@ import { getuid } from 'process'; import SQL, { SQLStatement } from 'sql-template-strings'; import { InvasivesRequest } from 'utils/auth-utils'; import { getLogger } from '../../utils/logger'; +import { streamIAPPResult } from '../../utils/iapp-json-utils'; +import { filter } from 'lodash'; const defaultLog = getLogger('IAPP'); const CACHENAME = 'IAPPv2 - Fat'; @@ -248,6 +250,13 @@ function getIAPPSitesBySearchFilterCriteria(): RequestHandler { } sql = getIAPPSQLv2(filterObject); + + if(filterObject.isCSV) + { + await streamIAPPResult(filterObject, res, sql); + } + else + { const response = await connection.query(sql.text, sql.values); return res.status(200).json({ @@ -258,7 +267,7 @@ function getIAPPSitesBySearchFilterCriteria(): RequestHandler { namespace: 'IAPP', code: 200 }); - } catch (error) { + }} catch (error) { defaultLog.debug({ label: 'getIAPPBySearchFilterCriteria', message: 'error', error }); return res.status(500).json({ message: 'Error getting sites by search filter criteria', @@ -406,8 +415,8 @@ sites as ( sqlStatement.append(` from iapp_sites a - join invasivesbc.iapp_site_summary_and_geojson b on a.site_id = b.site_id - `); + join invasivesbc.iapp_site_summary_and_geojson b on a.site_id = b.site_id`); + if (filterObject?.serverFilterGeometries?.length > 0) { sqlStatement.append(` @@ -430,6 +439,11 @@ sites as ( } function selectStatement(sqlStatement: SQLStatement, filterObject: any) { + if(filterObject.isCSV) + { + const select = sqlStatement.append(`select pe.* `); + return select; + } if (filterObject.selectColumns) { const select = sqlStatement.append( `select ${filterObject.selectColumns.map((column) => `sites.${column}`).join(',')} ` @@ -443,6 +457,40 @@ function selectStatement(sqlStatement: SQLStatement, filterObject: any) { function fromStatement(sqlStatement: SQLStatement, filterObject: any) { const from = sqlStatement.append(`from sites `); + if (filterObject.isCSV) { + sqlStatement.append(` i `) + switch (filterObject.CSVType) { + case 'site_selection_extract': + sqlStatement.append(SQL` INNER JOIN site_selection_extract pe ON i.site_id = pe.site_id `); + break; + case 'survey_extract': + sqlStatement.append(SQL` INNER JOIN survey_extract pe ON i.site_id = pe.site_id`); + break; + case 'chemical_treatment_extract': + sqlStatement.append(SQL` INNER JOIN chemical_treatment_extract pe ON i.site_id = pe.site_id`); + break; + case 'mechanical_treatment_extract': + sqlStatement.append(SQL` INNER JOIN mechanical_treatment_extract pe ON i.site_id = pe.site_id`); + break; + case 'chemical_monitoring_extract': + sqlStatement.append(SQL` INNER JOIN chemical_monitoring_extract pe ON i.site_id = pe.site_id`); + break; + case 'mechanical_monitoring_extract': + sqlStatement.append(SQL` INNER JOIN mechanical_monitoring_extract pe ON i.site_id = pe.site_id`); + break; + case 'biological_treatment_extract': + sqlStatement.append(SQL` INNER JOIN biological_treatment_extract pe ON i.site_id = pe.site_id`); + break; + case 'biological_monitoring_extract': + sqlStatement.append(SQL` INNER JOIN biological_monitoring_extract pe ON i.site_id = pe.site_id`); + break; + case 'biological_dispersal_extract': + sqlStatement.append(SQL` INNER JOIN biological_dispersal_extract pe ON i.site_id = pe.site_id`); + break; + default: + sqlStatement.append(SQL` INNER JOIN site_selection_extract pe ON i.site_id = pe.site_id `); + break; + }} return from; } diff --git a/api/src/utils/iapp-json-utils.ts b/api/src/utils/iapp-json-utils.ts index 12180843f..2332ebc0e 100644 --- a/api/src/utils/iapp-json-utils.ts +++ b/api/src/utils/iapp-json-utils.ts @@ -410,6 +410,8 @@ export async function streamActivitiesResult(searchCriteria: any, res: any, sqlS readable.pipe(upload(S3, key)).on('end', async () => { // get signed url + + await new Promise(r => setTimeout(r, 2000)); const url = await getS3SignedURL(key); res.status(200).send(url); res.end(); @@ -421,7 +423,7 @@ export async function streamActivitiesResult(searchCriteria: any, res: any, sqlS } } -export const streamIAPPResult = async (searchCriteria: any, res: any) => { +export const streamIAPPResult = async (searchCriteria: any, res: any, sqlStatementOverride?: any) => { let connection; try { connection = await getDBConnection(); @@ -441,7 +443,7 @@ export const streamIAPPResult = async (searchCriteria: any, res: any) => { }; } - const sqlStatement: SQLStatement = getSitesBasedOnSearchCriteriaSQL(searchCriteria); + const sqlStatement: SQLStatement = sqlStatementOverride? sqlStatementOverride: getSitesBasedOnSearchCriteriaSQL(searchCriteria); if (!sqlStatement) { throw { @@ -461,6 +463,9 @@ export const streamIAPPResult = async (searchCriteria: any, res: any) => { readable.pipe(upload(S3, key)).on('end', async () => { // get signed url const url = await getS3SignedURL(key); + + await new Promise(r => setTimeout(r, 2000)); + res.status(200).send(url); res.end(); connection.release();