Merge pull request #2799 from bcgov/public-map-v1

Public map v1

api/package.json

@@ -10,6 +10,7 @@
   },
   "scripts": {
     "start": "ts-node src/server",
+    "export-map": "ts-node src/map-exporter",
     "clean": "gulp clean",
     "build": "gulp build",
     "start:reload": "./node_modules/.bin/nodemon src/server.ts --exec ts-node",

api/src/constants/misc.ts

@@ -96,7 +96,8 @@ export enum ActivitySubType {
  * @enum {number}
  */
 export enum S3ACLRole {
-  AUTH_READ = 'authenticated-read'
+  AUTH_READ = 'authenticated-read',
+  PUBLIC_READ = 'public-read'
 }
 
 /**

api/src/map-exporter.ts

@@ -0,0 +1,16 @@
+import { getLogger } from './utils/logger';
+import { getDBConnection } from './database/db';
+import { buildPublicMapExport } from './utils/public-map';
+
+const defaultLog = getLogger('map-exporter');
+
+async function run() {
+  const connection = await getDBConnection();
+  await buildPublicMapExport(connection);
+  await connection.release();
+  defaultLog.info({ message: 'run complete' });
+}
+
+run().then(() => {
+  defaultLog.info({ message: 'shutting down' });
+});

api/src/models/point-of-interest.ts

@@ -83,9 +83,11 @@ export class PointOfInterestSearchCriteria {
 
 
   //for use in csv endpoint
-  isCSV?: boolean
+  isCSV?: boolean;
   CSVType?: string;
-
+
+  isGeoJSON: boolean;
+
   pointOfInterest_type: string;
   pointOfInterest_subtype: string;
   iappType: string;
@@ -151,6 +153,8 @@ export class PointOfInterestSearchCriteria {
     this.jurisdiction = obj.jurisdiction || [];
     this.species_positive = obj?.species_positive || [];
     this.species_negative = obj?.species_negative || [];
+
+    this.isGeoJSON = false;
   }
 
   setPage(page: number): number {

api/src/paths/public-map/activities.ts

@@ -0,0 +1,113 @@
+'use strict';
+
+import { RequestHandler } from 'express';
+import { Operation } from 'express-openapi';
+import { SQLStatement } from 'sql-template-strings';
+import { InvasivesRequest } from 'utils/auth-utils';
+import { getDBConnection } from '../../database/db';
+import { getLogger } from '../../utils/logger';
+import { getPublicActivitiesSQL } from '../../queries/public-queries';
+
+const defaultLog = getLogger('activity');
+
+export const GET: Operation = [getPublicActivities()];
+
+GET.apiDoc = {
+  description: 'Fetches all activities based on search criteria.',
+  tags: ['activity'],
+  security: [],
+  responses: {
+    200: {
+      description: 'Activities lean get response object array.',
+      content: {
+        'application/json': {
+          schema: {
+            type: 'array',
+            items: {
+              type: 'object',
+              properties: {
+                rows: {
+                  type: 'array',
+                  items: {
+                    type: 'object',
+                    properties: {
+                      // Don't specify exact object properties, as it will vary, and is not currently enforced anyways
+                      // Eventually this could be updated to be a oneOf list, similar to the Post request below.
+                    }
+                  }
+                },
+                count: {
+                  type: 'number'
+                }
+              }
+            }
+          }
+        }
+      }
+    },
+    503: {
+      $ref: '#/components/responses/503'
+    },
+    default: {
+      $ref: '#/components/responses/default'
+    }
+  }
+};
+
+/**
+ * Fetches all activity records based on request search filter criteria.
+ *
+ * @return {RequestHandler}
+ */
+function getPublicActivities(): RequestHandler {
+  return async (req: InvasivesRequest, res) => {
+    const connection = await getDBConnection();
+
+    if (!connection) {
+      return res.status(503).json({
+        message: 'Database connection unavailable',
+        request: req.body,
+        namespace: 'public',
+        code: 503
+      });
+    }
+
+    try {
+      const sqlStatement: SQLStatement = getPublicActivitiesSQL();
+
+      const response = await connection.query(sqlStatement.text, sqlStatement.values);
+
+      // parse the rows from the response
+      const rows = { rows: (response && response.rows) || [] };
+
+      // parse the count from the response
+      const count = { count: rows.rows.length && parseInt(rows.rows[0]['total_rows_count']) } || {};
+
+      defaultLog.info({
+        label: 'activities-lean',
+        message: 'response',
+        body: count
+      });
+
+      return res.status(200).json({
+        message: 'Got activities by search filter criteria',
+        request: req.body,
+        result: rows,
+        count: count,
+        namespace: 'activities-lean',
+        code: 200
+      });
+    } catch (error) {
+      defaultLog.debug({ label: 'getActivitiesBySearchFilterCriteria', message: 'error', error });
+      return res.status(500).json({
+        message: 'Error getting activities by search filter criteria',
+        error: error,
+        request: req.body,
+        namespace: 'activities-lean',
+        code: 500
+      });
+    } finally {
+      connection.release();
+    }
+  };
+}

api/src/queries/iapp-queries.ts

@@ -42,7 +42,9 @@ export const getSitesBasedOnSearchCriteriaSQL = (searchCriteria: PointOfInterest
     );
   }
 
-  if (searchCriteria.site_id_only) {
+  if (searchCriteria.isGeoJSON) {
+    sqlStatement.append(SQL`SELECT i.geojson `);
+  } else if (searchCriteria.site_id_only) {
     sqlStatement.append(SQL`SELECT i.site_id `);
   } else if (searchCriteria.isCSV && searchCriteria.isIAPP) {
     sqlStatement.append(SQL`SELECT pe.* `);

api/src/queries/public-map.ts

@@ -0,0 +1,23 @@
+import {SQL} from "sql-template-strings";
+
+export const PUBLIC_IAPP_SQL = SQL`SELECT i.geojson as feature
+                                   from iapp_site_summary_and_geojson i`;
+
+export const PUBLIC_ACTIVITY_SQL = SQL`select jsonb_build_object(
+                                                'type', 'Feature',
+                                                'id', a.short_id,
+                                                'geometry', st_asgeojson(a.geog)::jsonb,
+                                                'properties', jsonb_build_object(
+                                                  'id', a.short_id,
+                                                  'activityType', a.activity_type,
+                                                  'speciesPositive', a.species_positive_full,
+                                                  'speciesNegative', a.species_negative_full,
+                                                  'jurisdiction', a.jurisdiction_display,
+                                                  'RISO', a.regional_invasive_species_organization_areas,
+                                                  'agency', a.agency,
+                                                  'regionalDistricts', a.regional_districts,
+                                                  'MOTIDistricts', a.moti_districts,
+                                                  'FLNRODistricts', a.flnro_districts)) as feature
+                                       from activity_incoming_data as a
+                                       where a.form_status = 'Submitted'
+                                         and a.activity_type = 'Observation'`;

api/src/queries/public-queries.ts

@@ -0,0 +1,23 @@
+import SQL, { SQLStatement } from 'sql-template-strings';
+
+export function getPublicActivitiesSQL(): SQLStatement {
+  const f: SQLStatement = SQL`select a.activity_type,
+                                     a.activity_subtype,
+                                     a.flnro_districts,
+                                     a.moti_districts,
+                                     a.invasive_plant_management_areas,
+                                     a.jurisdiction,
+                                     a.regional_invasive_species_organization_areas,
+                                     a.well_proximity,
+                                     a.species_positive,
+                                     a.species_negative,
+                                     a.regional_districts,
+                                     a.received_timestamp,
+                                     a.short_id,
+                                     st_asgeojson(a.geog) as geo
+                              from activity_incoming_data a
+                              where a.form_status = 'Submitted'
+                                and a.activity_type = 'Observation'`;
+
+  return f;
+}

api/src/utils/public-map/index.ts

@@ -0,0 +1,93 @@
+import {tmpdir} from 'os';
+import Crypto from 'crypto';
+import {exec} from 'child_process';
+
+import * as Path from 'path';
+import * as fs from 'fs';
+import {getLogger} from '../logger';
+import {S3ACLRole} from '../../constants/misc';
+import AWS from 'aws-sdk';
+import {PUBLIC_ACTIVITY_SQL, PUBLIC_IAPP_SQL} from '../../queries/public-map';
+
+const defaultLog = getLogger('tile_processor');
+
+const OBJECT_STORE_BUCKET_NAME = process.env.OBJECT_STORE_BUCKET_NAME;
+const OBJECT_STORE_URL = process.env.OBJECT_STORE_URL || 'nrs.objectstore.gov.bc.ca';
+const AWS_ENDPOINT = new AWS.Endpoint(OBJECT_STORE_URL);
+
+const S3 = new AWS.S3({
+  endpoint: AWS_ENDPOINT.href,
+  accessKeyId: process.env.OBJECT_STORE_ACCESS_KEY_ID,
+  secretAccessKey: process.env.OBJECT_STORE_SECRET_KEY_ID,
+  signatureVersion: 'v4',
+  s3ForcePathStyle: true
+});
+
+async function dumpGeoJSONToFile(connection, filename, query) {
+  const response = await connection.query(query.text, query.values);
+
+  defaultLog.debug({message: 'Writing query result to tempfile', filename});
+  fs.writeFileSync(filename, JSON.stringify(response.rows.map(r => r['feature']), null, 2));
+}
+
+export async function buildPublicMapExport(connection) {
+  const randomBytes = Crypto.randomBytes(6).readUIntLE(0, 6).toString(36);
+  const filePrefix = Path.join(tmpdir(), `map.${randomBytes}`);
+
+  await dumpGeoJSONToFile(connection, `${filePrefix}-iapp.json`, PUBLIC_IAPP_SQL);
+  await dumpGeoJSONToFile(connection, `${filePrefix}-activities.json`, PUBLIC_ACTIVITY_SQL);
+
+  try {
+    await new Promise<void>((resolve, reject) => {
+      exec(
+        `tippecanoe -o ${filePrefix}.mbtiles -n IAPP -zg --extend-zooms-if-still-dropping -r1 --cluster-distance=3 -S3 -ah -U 3 -ae -Liapp:${filePrefix}-iapp.json -Linvasives:${filePrefix}-activities.json`,
+        (error, stdout, stderr) => {
+          if (error) {
+            defaultLog.error({message: 'Error in tippecanoe', stdout, stderr});
+            reject('Subprocess returned error code');
+          }
+          resolve();
+        }
+      );
+    });
+  } catch (e) {
+    defaultLog.error({message: 'error in tippecanoe', error: e});
+  } finally {
+    fs.unlinkSync(`${filePrefix}-iapp.json`);
+    fs.unlinkSync(`${filePrefix}-activities.json`);
+  }
+
+  defaultLog.info({message: 'tippecanoe pass complete, now optimizing with pmtiles'});
+
+  try {
+    await new Promise<void>((resolve, reject) => {
+      exec(`pmtiles convert ${filePrefix}.mbtiles ${filePrefix}.pmtiles`, (error, stdout, stderr) => {
+        if (error) {
+          defaultLog.error({message: 'Error in pmtiles', stdout, stderr});
+          reject('Subprocess returned error code');
+        }
+        resolve();
+      });
+    });
+  } catch (e) {
+    defaultLog.error({message: 'error in pmtiles', error: e});
+  } finally {
+    fs.unlinkSync(`${filePrefix}.mbtiles`);
+  }
+
+  defaultLog.info({message: `processing complete, starting upload`});
+
+  const s3key = `invasives.pmtiles`;
+
+  try {
+    await S3.upload({
+      Bucket: OBJECT_STORE_BUCKET_NAME,
+      Body: fs.readFileSync(`${filePrefix}.pmtiles`),
+      Key: s3key,
+      ACL: S3ACLRole.PUBLIC_READ,
+      Metadata: {}
+    }).promise();
+  } finally {
+    fs.unlinkSync(`${filePrefix}.pmtiles`);
+  }
+}