Merge pull request #161 from jujaga/chore/maintenance

General Maintenance updates
bcgov · Apr 21, 2021 · 38e5df1 · 38e5df1
2 parents 4bed47f + b1058a3
commit 38e5df1
Show file tree

Hide file tree

Showing 15 changed files with 2,923 additions and 3,095 deletions.
diff --git a/app/app.js b/app/app.js
@@ -1,11 +1,13 @@
 const compression = require('compression');
 const config = require('config');
 const express = require('express');
+const fs = require('fs');
 const log = require('npmlog');
 const morgan = require('morgan');
 const path = require('path');
 const Problem = require('api-problem');
 const querystring = require('querystring');
+const Writable = require('stream').Writable;
 
 const keycloak = require('./src/components/keycloak');
 const v1Router = require('./src/routes/v1');
@@ -18,8 +20,10 @@ const state = {
   connections: {
     data: false
   },
+  ready: false,
   shutdown: false
 };
+let probeId;
 
 const app = express();
 app.use(compression());
@@ -30,20 +34,52 @@ app.use(express.urlencoded({ extended: true }));
 log.level = config.get('server.logLevel');
 log.addLevel('debug', 1500, { fg: 'cyan' });
 
+let logFileStream;
+let teeStream;
+if (config.has('server.logFile')) {
+  // Write to logFile in append mode
+  logFileStream = fs.createWriteStream(config.get('server.logFile'), { flags: 'a' });
+  teeStream = new Writable({
+    objectMode: true,
+    write: (data, _, done) => {
+      process.stdout.write(data);
+      logFileStream.write(data);
+      done();
+    }
+  });
+  log.disableColor();
+  log.stream = teeStream;
+}
+
 // Skip if running tests
 if (process.env.NODE_ENV !== 'test') {
   const morganOpts = {
     // Skip logging kube-probe requests
     skip: (req) => req.headers['user-agent'] && req.headers['user-agent'].includes('kube-probe')
   };
+  if (config.has('server.logFile')) {
+    morganOpts.stream = teeStream;
+  }
   // Add Morgan endpoint logging
   app.use(morgan(config.get('server.morganFormat'), morganOpts));
+  // Initialize connections and exit if unsuccessful
   initializeConnections();
 }
 
 // Use Keycloak OIDC Middleware
 app.use(keycloak.middleware());
 
+// Block requests until service is ready
+app.use((_req, res, next) => {
+  if (state.shutdown) {
+    new Problem(503, { details: 'Server is shutting down' }).send(res);
+  } else if (!state.ready) {
+    new Problem(503, { details: 'Server is not ready' }).send(res);
+  } else {
+    next();
+  }
+});
+
 // Frontend configuration endpoint
 apiRouter.use('/config', (_req, res, next) => {
   try {
@@ -78,6 +114,11 @@ app.use(staticFilesPath, express.static(path.join(__dirname, 'frontend/dist')));
 // Handle 500
 // eslint-disable-next-line no-unused-vars
 app.use((err, _req, res, _next) => {
+  // Attempt to reset DB connection
+  if (!state.shutdown) {
+    dataConnection.resetConnection();
+  }
+
   if (err.stack) {
     log.error(err.stack);
   }
@@ -112,81 +153,100 @@ process.on('unhandledRejection', err => {
   }
 });
 
+// Graceful shutdown support
+process.on('SIGTERM', shutdown);
+process.on('SIGINT', shutdown);
+process.on('SIGUSR1', shutdown);
+process.on('SIGUSR2', shutdown);
+process.on('exit', () => {
+  log.info('Exiting...');
+});
+
 /**
  * @function shutdown
- * Begins shutting down this application. It will hard exit after 3 seconds.
+ * Shuts down this application after at least 3 seconds.
  */
 function shutdown() {
-  if (!state.shutdown) {
-    log.info('Received kill signal. Shutting down...');
-    state.shutdown = true;
-    // Wait 3 seconds before hard exiting
-    setTimeout(() => process.exit(), 3000);
-  }
+  log.info('Received kill signal. Shutting down...');
+  // Wait 3 seconds before starting cleanup
+  if (!state.shutdown) setTimeout(cleanup, 3000);
 }
 
-process.on('SIGTERM', shutdown);
-process.on('SIGINT', shutdown);
+/**
+ * @function cleanup
+ * Cleans up connections in this application.
+ */
+function cleanup() {
+  log.info('Service no longer accepting traffic');
+  state.shutdown = true;
+
+  log.info('Cleaning up...');
+  clearInterval(probeId);
+
+  dataConnection.close(() => process.exit());
+
+  // Wait 10 seconds max before hard exiting
+  setTimeout(() => process.exit(), 10000);
+}
 
 /**
  *  @function initializeConnections
- *  Initializes the database, queue and email connections
+ *  Initializes the database connections
  *  This will force the application to exit if it fails
  */
 function initializeConnections() {
   // Initialize connections and exit if unsuccessful
-  try {
-    const tasks = [
-      dataConnection.checkAll()
-    ];
-
-    Promise.all(tasks)
-      .then(results => {
-        state.connections.data = results[0];
-
-        if (state.connections.data) log.info('DataConnection', 'Connected');
-      })
-      .catch(error => {
-        log.error(error);
-        log.error('initializeConnections', `Initialization failed: Database OK = ${state.connections.data}`);
-      })
-      .finally(() => {
-        state.ready = Object.values(state.connections).every(x => x);
-        if (!state.ready) shutdown();
-      });
-
-  } catch (error) {
-    log.error('initializeConnections', 'Connection initialization failure', error.message);
-    if (!state.ready) shutdown();
-  }
-
-  // Start asynchronous connection probe
-  connectionProbe();
+  const tasks = [
+    dataConnection.checkAll()
+  ];
+
+  Promise.all(tasks)
+    .then(results => {
+      state.connections.data = results[0];
+
+      if (state.connections.data) log.info('DataConnection', 'Reachable');
+    })
+    .catch(error => {
+      log.error('initializeConnections', `Initialization failed: Database OK = ${state.connections.data}`);
+      log.error('initializeConnections', 'Connection initialization failure', error.message);
+      if (!state.ready) {
+        process.exitCode = 1;
+        shutdown();
+      }
+    })
+    .finally(() => {
+      state.ready = Object.values(state.connections).every(x => x);
+      if (state.ready) {
+        log.info('Service ready to accept traffic');
+        // Start periodic 10 second connection probe check
+        probeId = setInterval(checkConnections, 10000);
+      }
+    });
 }
 
 /**
- *  @function connectionProbe
- *  Periodically checks the status of the connections at a specific interval
- *  This will force the application to exit a connection fails
- *  @param {integer} [interval=10000] Number of milliseconds to wait before
+ * @function checkConnections
+ * Checks Database connectivity
+ * This will force the application to exit if a connection fails
  */
-function connectionProbe(interval = 10000) {
-  const checkConnections = () => {
-    if (!state.shutdown) {
-      const tasks = [
-        dataConnection.checkConnection()
-      ];
+function checkConnections() {
+  const wasReady = state.ready;
+  if (!state.shutdown) {
+    const tasks = [
+      dataConnection.checkConnection()
+    ];
 
+    Promise.all(tasks).then(results => {
+      state.connections.data = results[0];
+      state.ready = Object.values(state.connections).every(x => x);
+      if (!wasReady && state.ready) log.info('Service ready to accept traffic');
       log.verbose(JSON.stringify(state));
-      Promise.all(tasks).then(results => {
-        state.connections.data = results[0];
-        state.ready = Object.values(state.connections).every(x => x);
-        if (!state.ready) shutdown();
-      });
-    }
-  };
-
-  setInterval(checkConnections, interval);
+      if (!state.ready) {
+        process.exitCode = 1;
+        shutdown();
+      }
+    });
+  }
 }
 
 module.exports = app;
diff --git a/app/config/custom-environment-variables.json b/app/config/custom-environment-variables.json
@@ -48,6 +48,7 @@
       "realm": "SERVER_KC_REALM",
       "serverUrl": "SERVER_KC_SERVERURL"
     },
+    "logFile": "SERVER_LOGFILE",
     "logLevel": "SERVER_LOGLEVEL",
     "morganFormat": "SERVER_MORGANFORMAT",
     "port": "SERVER_PORT"

diff --git a/app/config/database.js b/app/config/database.js