chore(#9585): teach Sentinel sign language (#9658)

No longer start and stop sentinel to stop/start transitions and run scheduled tasks. Instead make it listen to two kill signals.
Kill the test process if API is down in the before hook. This way we might get logs when the process ends up hanging.

#9585

sentinel/server.js

@@ -35,20 +35,26 @@ const waitForApi = () => new Promise(resolve => {
 });
 
 logger.info('Running server checks...');
-serverChecks
-  .check(environment.couchUrl)
-  .then(waitForApi)
-  .then(() => {
-    // Even requiring this boots translations, so has to be required after
-    // api has booted
+
+(async () => {
+  try {
+    await serverChecks.check(environment.couchUrl);
+    await waitForApi();
+
     const config = require('./src/config');
-    return config.init().then(() => {
-      require('./src/schedule').init();
-      logger.info('startup complete.');
-    });
-  })
-  .catch(err => {
-    logger.error('Fatal error intialising sentinel');
+    await config.init();
+
+    const schedule = require('./src/schedule');
+    schedule.init();
+
+    logger.info('startup complete.');
+
+    const processHooks = require('./src/process-hooks');
+    processHooks.init();
+
+  } catch (err) {
+    logger.error('Fatal error initialising sentinel');
     logger.error('%o', err);
     process.exit(1);
-  });
+  }
+})();

sentinel/src/lib/feed.js

@@ -15,6 +15,7 @@ const MAX_QUEUE_SIZE = 100;
 
 let request;
 let processed = 0;
+let processing;
 
 const enqueue = change => changeQueue.push(change);
 
@@ -100,33 +101,34 @@ changeQueue.drain(() => {
   resumeProcessing();
 });
 
+/**
+ * Start listening from the last processed seq. Will restart
+ * automatically on error.
+ */
 const listen = () => {
+  logger.info('Starting transition processing');
+  processing = true;
   changeQueue.resume();
   return resumeProcessing();
 };
 
-module.exports = {
+/**
+ * Stops listening for changes. Must be restarted manually
+ * by calling listen.
+ */
+const cancel =  () => {
+  logger.info('Suspending transition processing');
+  processing = false;
+  changeQueue.pause();
+  if (request) {
+    request.cancel && request.cancel();
+    request = null;
+  }
+};
 
-  /**
-   * Start listening from the last processed seq. Will restart
-   * automatically on error.
-   */
+module.exports = {
   listen,
+  cancel,
 
-  /**
-   * Stops listening for changes. Must be restarted manually
-   * by calling listen.
-   */
-  cancel: () => {
-    changeQueue.pause();
-    if (request) {
-      request.cancel && request.cancel();
-      request = null;
-    }
-  },
-
-  // exposed for testing
-  _changeQueue: changeQueue,
-  _transitionsLib: transitionsLib,
-  _enqueue: enqueue,
+  toggle: () => processing ? cancel() : listen()
 };

sentinel/src/process-hooks.js

@@ -0,0 +1,13 @@
+const feed = require('./lib/feed');
+const schedule = require('./schedule');
+
+module.exports = {
+  init: () => {
+    process.on('SIGUSR1', () => {
+      feed.toggle();
+    });
+    process.on('SIGUSR2', () => {
+      schedule.runTasks();
+    });
+  }
+};

sentinel/src/schedule/index.js

@@ -72,3 +72,4 @@ exports.init = () => {
   runTasks();
   interval = setInterval(runTasks, RUN_EVERY_MS);
 };
+exports.runTasks = runTasks;

sentinel/src/transitions.js

@@ -20,8 +20,5 @@ module.exports = {
   /**
    * Loads the transitions and starts watching for db changes.
    */
-  loadTransitions: loadTransitions,
-
-  // exposed for testing
-  _transitionsLib: transitionsLib,
+  loadTransitions,
 };