feat(#575): retry failed commands

package.json

@@ -39,6 +39,7 @@
     "@medic/translation-checker": "^1.0.1",
     "@parcel/watcher": "^2.0.5",
     "@xmldom/xmldom": "^0.8.2",
+    "async-retry": "^1.3.3",
     "canonical-json": "0.0.4",
     "csv-parse": "^4.16.0",
     "dom-compare": "^0.6.0",

src/lib/main.js

@@ -1,4 +1,6 @@
 const open = require('open');
+const retry = require('async-retry');
+
 const checkForUpdates = require('../lib/check-for-updates');
 const checkChtConfDependencyVersion = require('../lib/check-cht-conf-dependency-version');
 const environment = require('./environment');
@@ -189,8 +191,7 @@ module.exports = async (argv, env) => {
   }
 
   for (let action of actions) {
-    info(`Starting action: ${action.name}…`);
-    await executeAction(action);
+    await executeActionWithRetry(action);
     info(`${action.name} complete.`);
   }
 
@@ -254,5 +255,22 @@ function buildActions(cmdArgs, skipValidate) {
   });
 }
 
+function executeActionWithRetry(action) {
+  return retry(
+    (bail, attempt) => {
+      info(`Starting action: ${action.name}, attempt #${attempt}…`);
+      return executeAction(action);
+    },
+    {
+      retries: 5,
+      randomize: false,
+      factor: 2,
+      onRetry(error, attempt) {
+        info(`Attempt #${attempt} of action ${action.name} failed with message: "${error.message}". Retrying…`);
+      }
+    },
+  );
+}
+
 // Exists for generic mocking purposes
 const executeAction = action => action.execute();

test/lib/main.spec.js

@@ -275,4 +275,64 @@ describe('main', () => {
     expect(apiAvailable.callCount).to.eq(1);
     expect(mocks.error.callCount).to.eq(0);
   });
+
+  describe('retry', () => {
+    it('should throw after the command failed 6 times', async function () {
+      this.timeout(35000);
+      apiAvailable.resolves(false);
+      const expectedError = new Error('Error: connect ECONNREFUSED');
+      mocks.executeAction.rejects(expectedError);
+      try {
+        await main([...normalArgv, 'upload-app-settings']);
+      } catch (error) {
+        expect(error.message).to.equal('Error: connect ECONNREFUSED');
+        const actionLogs = mocks.info.args.filter(args => args[0].startsWith('Starting action') || args[0].endsWith('Retrying…'));
+        expect(actionLogs).to.deep.equal([
+          ['Starting action: upload-app-settings, attempt #1…'],
+          ['Attempt #1 of action upload-app-settings failed with message: "Error: connect ECONNREFUSED". Retrying…'],
+          ['Starting action: upload-app-settings, attempt #2…'],
+          ['Attempt #2 of action upload-app-settings failed with message: "Error: connect ECONNREFUSED". Retrying…'],
+          ['Starting action: upload-app-settings, attempt #3…'],
+          ['Attempt #3 of action upload-app-settings failed with message: "Error: connect ECONNREFUSED". Retrying…'],
+          ['Starting action: upload-app-settings, attempt #4…'],
+          ['Attempt #4 of action upload-app-settings failed with message: "Error: connect ECONNREFUSED". Retrying…'],
+          ['Starting action: upload-app-settings, attempt #5…'],
+          ['Attempt #5 of action upload-app-settings failed with message: "Error: connect ECONNREFUSED". Retrying…'],
+          ['Starting action: upload-app-settings, attempt #6…']
+        ]);
+      }
+    });
+
+    it('should successfully run the command despite it failing 3 times', async function () {
+      this.timeout(10000);
+      apiAvailable.resolves(false);
+      const expectedError = new Error('Error: connect ECONNREFUSED');
+      mocks.executeAction.onCall(0).rejects(expectedError);
+      mocks.executeAction.onCall(1).rejects(expectedError);
+      mocks.executeAction.onCall(2).rejects(expectedError);
+      mocks.executeAction.resolves();
+      await main([...normalArgv, 'upload-app-settings']);
+      const actionLogs = mocks.info.args.filter(args => args[0].startsWith('Starting action') || args[0].endsWith('Retrying…'));
+      expect(actionLogs).to.deep.equal([
+        ['Starting action: upload-app-settings, attempt #1…'],
+        ['Attempt #1 of action upload-app-settings failed with message: "Error: connect ECONNREFUSED". Retrying…'],
+        ['Starting action: upload-app-settings, attempt #2…'],
+        ['Attempt #2 of action upload-app-settings failed with message: "Error: connect ECONNREFUSED". Retrying…'],
+        ['Starting action: upload-app-settings, attempt #3…'],
+        ['Attempt #3 of action upload-app-settings failed with message: "Error: connect ECONNREFUSED". Retrying…'],
+        ['Starting action: upload-app-settings, attempt #4…'],
+      ]);
+    });
+
+    it('should immediately throw and not retry when the command doesnt exist', async () => {
+      try {
+        await main([...normalArgv, '--local', 'not-an-action'], {});
+        expect.fail('Expected error to be thrown');
+      } catch(e) {
+        expect(mocks.executeAction.called).to.be.false;
+        expect(mocks.usage.calledOnce).to.be.true;
+        expect(e.message).to.equal('Unsupported action(s): not-an-action');
+      }
+    });
+  });
 });