Okta: Report throttling errors (#58)

* Report throttling errors * Report throllted HTTP responces * Bump package version.
alertlogic · Feb 7, 2020 · 7234652 · 7234652
1 parent ac1124e
commit 7234652
Show file tree

Hide file tree

Showing 3 changed files with 76 additions and 5 deletions.
diff --git a/collectors/okta/okta_collector.js b/collectors/okta/okta_collector.js
@@ -14,6 +14,7 @@ const okta = require('@okta/okta-sdk-nodejs');
 const parse = require('@alertlogic/al-collector-js').Parse;
 const PawsCollector = require('@alertlogic/paws-collector').PawsCollector;
 
+const THROTTLING_ERROR_REGEXP = /rateLimit/g;
 
 const typeIdPaths = [
     { path: ['eventType'] },
@@ -43,7 +44,7 @@ class OktaCollector extends PawsCollector {
         let collector = this;
         const oktaClient = new okta.Client({
             orgUrl: process.env.paws_endpoint,
-            token: collector._pawsCreds.secret
+            token: collector.secret
         });
         console.info(`OKTA000001 Collecting data from ${state.since} till ${state.until}`);
         const collection = oktaClient.getLogs({
@@ -60,10 +61,21 @@ class OktaCollector extends PawsCollector {
             return callback(null, logAcc, newState, newState.poll_interval_sec);
         })
         .catch((error) => {
-            return callback(error);
+            if (this._isThrottlingError(error)) {
+                collector.reportApiThrottling(function() {
+                    return callback(error);
+                });
+            } else {
+                return callback(error);
+            }
         });
     }
 
+    _isThrottlingError(error) {
+        return (error.status === 429) ||
+             (error.message && error.message.match(THROTTLING_ERROR_REGEXP));
+    }
+
     _getNextCollectionState(curState) {
         const nowMoment = moment();
         const curUntilMoment = moment(curState.until);

diff --git a/collectors/okta/package.json b/collectors/okta/package.json
@@ -1,6 +1,6 @@
 {
   "name": "okta-collector",
-  "version": "0.1.0",
+  "version": "1.0.0",
   "description": "Alert Logic AWS based Okta Log Collector",
   "repository": {},
   "private": true,
@@ -14,14 +14,15 @@
     "jshint": "^2.9.5",
     "mocha": "^6.2.2",
     "mocha-jenkins-reporter": "^0.4.2",
+    "nock": "^10.0.6",
     "nyc": "^14.1.1",
     "rewire": "^4.0.1",
     "sinon": "^7.5.0"
   },
   "dependencies": {
     "@okta/okta-sdk-nodejs": "3.1.0",
     "@alertlogic/al-collector-js": "1.4.2",
-    "@alertlogic/paws-collector": "1.0.12",
+    "@alertlogic/paws-collector": "1.0.14",
     "async": "3.1.0",
     "debug": "4.1.1",
     "moment": "2.24.0"

diff --git a/collectors/okta/test/okta_test.js b/collectors/okta/test/okta_test.js
@@ -1,6 +1,7 @@
 const assert = require('assert');
 const sinon = require('sinon');
 const moment = require('moment');
+const nock = require('nock');
 var AWS = require('aws-sdk-mock');
 const m_response = require('cfn-response');
 const okta = require('@okta/okta-sdk-nodejs');
@@ -75,7 +76,12 @@ function mockSetEnvStub() {
 }
 
 describe('Unit Tests', function() {
+
     beforeEach(function(){
+        if (!nock.isActive()) {
+            nock.activate();
+        }
+
         AWS.mock('SSM', 'getParameter', function (params, callback) {
             const data = new Buffer('test-secret');
             return callback(null, {Parameter : { Value: data.toString('base64')}});
@@ -189,7 +195,7 @@ describe('Unit Tests', function() {
                 };
             });
             OktaCollector.load().then(function(creds) {
-                var collector = new OktaCollector(ctx, creds, 'okta');
+                var collector = new OktaCollector(ctx, creds);
                 const startDate = moment().subtract(1, 'days').toISOString();
                 const mockState = {
                     since: startDate,
@@ -204,6 +210,58 @@ describe('Unit Tests', function() {
                 });
             });
         });
+
+        it('gets logs throttling', function(done) {
+            const {Client} = okta;
+            const oktaSdkMock = sinon.stub(Client.prototype, 'getLogs').callsFake(() => {
+                return {
+                    each: () => {
+                        return new Promise((res, rej) => {
+                            rej(new Error('HTTP request time exceeded okta.client.rateLimit.requestTimeout'));
+                        });
+                    }
+                };
+            });
+            OktaCollector.load().then(function(creds) {
+                var collector = new OktaCollector(ctx, creds);
+                const startDate = moment().subtract(1, 'days').toISOString();
+                const mockState = {
+                    since: startDate,
+                    until: moment().toISOString()
+                };
+                var reportSpy = sinon.spy(collector, 'reportApiThrottling');
+
+                collector.pawsGetLogs(mockState, (err) => {
+                    assert.equal(true, reportSpy.calledOnce);
+                    oktaSdkMock.restore();
+                    done();
+                });
+            });
+        });
+
+        it('gets logs throttling response', function(done) {
+
+            // Okta endpoints mock
+            nock('https://test.alertlogic.com:443', {'encodedQueryParams':true})
+            .get('/api/v1/logs')
+            .query(true)
+            .times(1)
+            .reply(429);
+            OktaCollector.load().then(function(creds) {
+                var collector = new OktaCollector(ctx, creds);
+                const startDate = moment().subtract(1, 'days').toISOString();
+                const mockState = {
+                    since: startDate,
+                    until: moment().toISOString()
+                };
+                var reportSpy = sinon.spy(collector, 'reportApiThrottling');
+
+                collector.pawsGetLogs(mockState, (err) => {
+                    assert.equal(true, reportSpy.calledOnce);
+                    done();
+                });
+            });
+        });
     });
 
     describe('_getNextCollectionState', function() {