Skip to content

Commit

Permalink
iapp csv stream v2
Browse files Browse the repository at this point in the history
  • Loading branch information
@micheal-w-wells
micheal-w-wells committed Nov 7, 2023
1 parent ec83da7 commit 441f4fd
Show file tree
Hide file tree
Showing 3 changed files with 72 additions and 16 deletions.
25 changes: 14 additions & 11 deletions api/src/paths/v2/activities.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -243,18 +243,20 @@ function getActivitiesBySearchFilterCriteria(): RequestHandler {
sql = getActivitiesSQLv2(filterObject);

if (filterObject.isCSV && filterObject.CSVType) {
res.status(200);
await streamActivitiesResult(filterObject, res, sql);
} else {
const response = await connection.query(sql.text, sql.values);

return res.status(200).json({
message: 'fetched activities by criteria',
request: req.body,
result: response.rows,
count: response.rowCount,
namespace: 'activities',
code: 200
});
}
const response = await connection.query(sql.text, sql.values);

return res.status(200).json({
message: 'fetched activities by criteria',
request: req.body,
result: response.rows,
count: response.rowCount,
namespace: 'activities',
code: 200
});
} catch (error) {
defaultLog.debug({ label: 'getActivitiesBySearchFilterCriteria', message: 'error', error });
return res.status(500).json({
Expand Down Expand Up @@ -444,8 +446,9 @@ function selectStatement(sqlStatement: SQLStatement, filterObject: any) {
}

function fromStatement(sqlStatement: SQLStatement, filterObject: any) {
const from = sqlStatement.append(`from activities `);
let from = sqlStatement.append(`from activities `);
if (filterObject.isCSV) {
from = sqlStatement.append(` b `);
switch (filterObject.CSVType) {
case 'terrestrial_plant_observation':
sqlStatement.append(
Expand Down
54 changes: 51 additions & 3 deletions api/src/paths/v2/iapp.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -7,6 +7,8 @@ import { getuid } from 'process';
import SQL, { SQLStatement } from 'sql-template-strings';
import { InvasivesRequest } from 'utils/auth-utils';
import { getLogger } from '../../utils/logger';
import { streamIAPPResult } from '../../utils/iapp-json-utils';
import { filter } from 'lodash';

const defaultLog = getLogger('IAPP');
const CACHENAME = 'IAPPv2 - Fat';
Expand Down Expand Up @@ -248,6 +250,13 @@ function getIAPPSitesBySearchFilterCriteria(): RequestHandler {
}

sql = getIAPPSQLv2(filterObject);

if(filterObject.isCSV)
{
await streamIAPPResult(filterObject, res, sql);
}
else
{
const response = await connection.query(sql.text, sql.values);

return res.status(200).json({
Expand All @@ -258,7 +267,7 @@ function getIAPPSitesBySearchFilterCriteria(): RequestHandler {
namespace: 'IAPP',
code: 200
});
} catch (error) {
}} catch (error) {
defaultLog.debug({ label: 'getIAPPBySearchFilterCriteria', message: 'error', error });
return res.status(500).json({
message: 'Error getting sites by search filter criteria',
Expand Down Expand Up @@ -406,8 +415,8 @@ sites as (

sqlStatement.append(`
from iapp_sites a
join invasivesbc.iapp_site_summary_and_geojson b on a.site_id = b.site_id
`);
join invasivesbc.iapp_site_summary_and_geojson b on a.site_id = b.site_id`);


if (filterObject?.serverFilterGeometries?.length > 0) {
sqlStatement.append(`
Expand All @@ -430,6 +439,11 @@ sites as (
}

function selectStatement(sqlStatement: SQLStatement, filterObject: any) {
if(filterObject.isCSV)
{
const select = sqlStatement.append(`select pe.* `);
return select;
}
if (filterObject.selectColumns) {
const select = sqlStatement.append(
`select ${filterObject.selectColumns.map((column) => `sites.${column}`).join(',')} `
Expand All @@ -443,6 +457,40 @@ function selectStatement(sqlStatement: SQLStatement, filterObject: any) {

function fromStatement(sqlStatement: SQLStatement, filterObject: any) {
const from = sqlStatement.append(`from sites `);
if (filterObject.isCSV) {
sqlStatement.append(` i `)
switch (filterObject.CSVType) {
case 'site_selection_extract':
sqlStatement.append(SQL` INNER JOIN site_selection_extract pe ON i.site_id = pe.site_id `);
break;
case 'survey_extract':
sqlStatement.append(SQL` INNER JOIN survey_extract pe ON i.site_id = pe.site_id`);
break;
case 'chemical_treatment_extract':
sqlStatement.append(SQL` INNER JOIN chemical_treatment_extract pe ON i.site_id = pe.site_id`);
break;
case 'mechanical_treatment_extract':
sqlStatement.append(SQL` INNER JOIN mechanical_treatment_extract pe ON i.site_id = pe.site_id`);
break;
case 'chemical_monitoring_extract':
sqlStatement.append(SQL` INNER JOIN chemical_monitoring_extract pe ON i.site_id = pe.site_id`);
break;
case 'mechanical_monitoring_extract':
sqlStatement.append(SQL` INNER JOIN mechanical_monitoring_extract pe ON i.site_id = pe.site_id`);
break;
case 'biological_treatment_extract':
sqlStatement.append(SQL` INNER JOIN biological_treatment_extract pe ON i.site_id = pe.site_id`);
break;
case 'biological_monitoring_extract':
sqlStatement.append(SQL` INNER JOIN biological_monitoring_extract pe ON i.site_id = pe.site_id`);
break;
case 'biological_dispersal_extract':
sqlStatement.append(SQL` INNER JOIN biological_dispersal_extract pe ON i.site_id = pe.site_id`);
break;
default:
sqlStatement.append(SQL` INNER JOIN site_selection_extract pe ON i.site_id = pe.site_id `);
break;
}}
return from;
}

Expand Down
9 changes: 7 additions & 2 deletions api/src/utils/iapp-json-utils.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -410,6 +410,8 @@ export async function streamActivitiesResult(searchCriteria: any, res: any, sqlS

readable.pipe(upload(S3, key)).on('end', async () => {
// get signed url

await new Promise(r => setTimeout(r, 2000));
const url = await getS3SignedURL(key);
res.status(200).send(url);
res.end();
Expand All @@ -421,7 +423,7 @@ export async function streamActivitiesResult(searchCriteria: any, res: any, sqlS
}
}

export const streamIAPPResult = async (searchCriteria: any, res: any) => {
export const streamIAPPResult = async (searchCriteria: any, res: any, sqlStatementOverride?: any) => {
let connection;
try {
connection = await getDBConnection();
Expand All @@ -441,7 +443,7 @@ export const streamIAPPResult = async (searchCriteria: any, res: any) => {
};
}

const sqlStatement: SQLStatement = getSitesBasedOnSearchCriteriaSQL(searchCriteria);
const sqlStatement: SQLStatement = sqlStatementOverride? sqlStatementOverride: getSitesBasedOnSearchCriteriaSQL(searchCriteria);

if (!sqlStatement) {
throw {
Expand All @@ -461,6 +463,9 @@ export const streamIAPPResult = async (searchCriteria: any, res: any) => {
readable.pipe(upload(S3, key)).on('end', async () => {
// get signed url
const url = await getS3SignedURL(key);

await new Promise(r => setTimeout(r, 2000));

res.status(200).send(url);
res.end();
connection.release();
Expand Down

0 comments on commit 441f4fd

Please sign in to comment.