From c8a2ba0e44341954137e0c8d9e3adb4cdd6e06bd Mon Sep 17 00:00:00 2001 From: JDragovichAlertLogic <47750771+JDragovichAlertLogic@users.noreply.github.com> Date: Thu, 23 Jan 2020 12:51:39 +0000 Subject: [PATCH] handle creds stored in s3 buckets (#48) * rought out s3 cred storeage * action comments and make implemtatio simpler * clean some errant cfn stragglers * address las few comments * a few fixes from running unit tests * remove copy past error * clean up and add some tests * action comments * one more typo * add rewire and check content of cred * fix linting error --- cfn/paws-collector.template | 35 ++++++++++++-- package.json | 6 +-- paws_collector.js | 80 ++++++++++++++++++++++++++----- test/paws_mock.js | 4 +- test/paws_test.js | 95 ++++++++++++++++++++++++++++++++++++- 5 files changed, 198 insertions(+), 22 deletions(-) diff --git a/cfn/paws-collector.template b/cfn/paws-collector.template index 392df369..28f35ab6 100644 --- a/cfn/paws-collector.template +++ b/cfn/paws-collector.template @@ -62,8 +62,7 @@ }, "PawsEndpoint": { "Description": "URL to poll", - "Type": "String", - "Default": "https://alertlogic-admin.okta.com/" + "Type": "String" }, "PawsAuthType": { "Description": "Target API authentication type. Supported types: ssws, oauth2", @@ -85,6 +84,12 @@ "Type": "Number", "Default": 10 }, + "PawsCredsS3Uri": { + "Description": "The S3 uri where object where credentials are stored. in the for s3:/// Only used when PawsAuthType is 's3object'", + "Type": "String", + "AllowedPattern" : "^s3:\/\/[a-z,\-,0-9]*\/[a-z,\-,\.,\_,0-9]*", + "Default": "" + }, "CollectionStartTs": { "Description": "Optional. Timestamp when log collection starts. For example, 2020-01-13T16:00:00Z.", "Type": "String", @@ -363,10 +368,11 @@ "\n", "exports.handler = (event, context, callback) => {\n", " if (event.ResourceType == 'AWS::CloudFormation::CustomResource' &&\n", - " event.RequestType == 'Create') {\n", + " event.RequestType == 'Create' &&\n", + " event.ResourceProperties.Plaintext.length > 0) {\n", " return encrypt(event, context);\n", " }\n", - " return response.send(event, context, response.SUCCESS);\n", + " return response.send(event, context, response.SUCCESS, {EncryptedText: \"\"});\n", "}" ] ] @@ -545,9 +551,15 @@ "paws_api_client_id":{ "Ref":"PawsClientId" }, + "paws_kms_key_arn":{ + "Fn::GetAtt": [ "LambdaKmsKey", "Arn" ] + }, "paws_api_secret":{ "Fn::GetAtt": ["EncryptPawsSecret", "EncryptedText"] }, + "paws_s3_object_path":{ + "Ref": "PawsCredsS3Uri" + }, "paws_collection_start_ts":{ "Ref":"CollectionStartTs" }, @@ -619,6 +631,21 @@ ] }] }, + { + "Effect":"Allow", + "Action":"s3:Get*", + "Resource":[ + { + "Fn::Join":["", + [ + "arn:aws:s3:::", + { "Fn::Select": ["1", {"Fn::Split": ["//", {"Ref": "PawsCredsS3Uri"}] } ] } + ] + ] + + } + ] + }, { "Effect":"Allow", "Action":[ diff --git a/package.json b/package.json index 460f11e5..f17bab47 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "@alertlogic/paws-collector", - "version": "1.0.10", + "version": "1.0.11", "license": "MIT", "description": "Alert Logic AWS based API Poll Log Collector Library", "repository": { @@ -28,8 +28,8 @@ "mocha-jenkins-reporter": "^0.4.2", "nyc": "^14.1.1", "rewire": "^4.0.1", - "yargs": "^15.0.2", - "sinon": "^7.5.0" + "sinon": "^7.5.0", + "yargs": "^15.0.2" }, "dependencies": { "@alertlogic/al-aws-collector-js": "2.0.4", diff --git a/paws_collector.js b/paws_collector.js index cee600c0..673f7ddf 100644 --- a/paws_collector.js +++ b/paws_collector.js @@ -13,19 +13,59 @@ const async = require('async'); const debug = require('debug')('index'); const AWS = require('aws-sdk'); +const fs = require('fs'); const AlAwsCollector = require('@alertlogic/al-aws-collector-js').AlAwsCollector; const m_packageJson = require('./package.json'); +const CREDS_FILE_PATH = '/tmp/paws_creds.json'; var PAWS_DECRYPTED_CREDS = null; -function getDecryptedPawsCredentials(callback) { +function getPawsCredsFile(){ + return new Promise((resolve, reject) => { + // check if the creds file is cached and retrieve it if it is not + if(!fs.existsSync(CREDS_FILE_PATH)){ + const s3 = new AWS.S3({apiVersion: '2006-03-01'}); + const kms = new AWS.KMS(); + + // doing the string manipulation here because doing it here is way less groos than doing it in the cfn + const s3Path = process.env.paws_s3_object_path.split('//')[1]; + const s3PathParts = s3Path.split('/'); + + // retrive the object from S3 + var params = { + Bucket: s3PathParts.shift(), + Key: s3PathParts.join('/') + }; + s3.getObject(params, (err, data) => { + if (err) return reject(new Error(err, err.stack)); + + // encrypt the file contents and cache on the lambda container file system + const encryptParams ={ + Plaintext: data.Body, + KeyId: process.env.paws_kms_key_arn + }; + kms.encrypt(encryptParams, (encryptError, encryptResponse) => { + if (encryptError) return reject(new Error(encryptError, encryptError.stack)); + + fs.writeFileSync(CREDS_FILE_PATH, encryptResponse.CiphertextBlob); + return resolve(encryptResponse.CiphertextBlob); + }) + }); + } + else { + return resolve(fs.readFileSync(CREDS_FILE_PATH)); + } + }); +}; + +function getDecryptedPawsCredentials(credsBuffer, callback) { if (PAWS_DECRYPTED_CREDS) { return callback(null, PAWS_DECRYPTED_CREDS); } else { const kms = new AWS.KMS(); kms.decrypt( - {CiphertextBlob: Buffer.from(process.env.paws_api_secret, 'base64')}, + {CiphertextBlob: credsBuffer}, (err, data) => { if (err) { return callback(err); @@ -35,7 +75,7 @@ function getDecryptedPawsCredentials(callback) { client_id: process.env.paws_api_client_id, secret: data.Plaintext.toString('ascii') }; - + return callback(null, PAWS_DECRYPTED_CREDS); } }); @@ -47,17 +87,30 @@ class PawsCollector extends AlAwsCollector { static load() { return new Promise(function(resolve, reject){ AlAwsCollector.load().then(function(aimsCreds) { - getDecryptedPawsCredentials(function(err, pawsCreds) { - if (err){ - reject(err); - } else { - resolve({aimsCreds : aimsCreds, pawsCreds: pawsCreds}); - } - }); - }) - }) + let credsPromise; + + switch(process.env.paws_auth_type){ + case 's3object': + credsPromise = getPawsCredsFile(); + break; + default: + const enVarCreds = Buffer.from(process.env.paws_api_secret, 'base64'); + credsPromise = new Promise(res => res(enVarCreds)); + } + + return credsPromise.then(credsBuffer => { + getDecryptedPawsCredentials(credsBuffer, function(err, pawsCreds) { + if (err){ + reject(err); + } else { + resolve({aimsCreds : aimsCreds, pawsCreds: pawsCreds}); + } + }); + }).catch(err => console.error(`PAWS000400 Error getting Paws Credentials ${err}`)); + }); + }); } - + constructor(context, {aimsCreds, pawsCreds}) { super(context, 'paws', AlAwsCollector.IngestTypes.LOGMSGS, @@ -254,6 +307,7 @@ class PawsCollector extends AlAwsCollector { } module.exports = { + getPawsCredsFile, PawsCollector: PawsCollector } diff --git a/test/paws_mock.js b/test/paws_mock.js index 0802f30f..17e550a9 100644 --- a/test/paws_mock.js +++ b/test/paws_mock.js @@ -7,8 +7,10 @@ process.env.aims_secret_key = 'aims-secret-key-encrypted'; process.env.log_group = 'logGroupName'; process.env.paws_state_queue_arn = 'arn:aws:sqs:us-east-1:352283894008:test-queue'; process.env.paws_state_queue_url = 'https://sqs.us-east-1.amazonaws.com/352283894008/test-queue'; +process.env.paws_s3_object_path = "s3://joe-creds-test/paws_creds.json"; +process.env.paws_kms_key_arn = "arn:aws:kms:us-east-1:352283894008:key/cdda86d5-615b-4dcc-9319-77ab34510473"; process.env.paws_type_name = 'okta'; -process.env.paws_api_auth_type = 'auth'; +process.env.paws_auth_type = 'auth'; process.env.paws_poll_interval = 900; process.env.paws_endpoint = 'https://test.alertlogic.com/'; process.env.paws_api_secret = 'api-token'; diff --git a/test/paws_test.js b/test/paws_test.js index df638f1a..f27a6d75 100644 --- a/test/paws_test.js +++ b/test/paws_test.js @@ -1,6 +1,8 @@ const assert = require('assert'); +const rewire = require('rewire'); const sinon = require('sinon'); var AWS = require('aws-sdk-mock'); +const fs = require('fs'); const m_response = require('cfn-response'); const pawsMock = require('./paws_mock'); @@ -154,8 +156,32 @@ describe('Unit Tests', function() { beforeEach(function(){ AWS.mock('KMS', 'decrypt', function (params, callback) { + let data; + if(params.CiphertextBlob.toString() === 'creds-from-file'){ + console.log('dcrypting file'); + data = { + Plaintext : 'decrypted-secret-key-from-file' + }; + } + else{ + console.log('decrypting somthing else'); + data = { + Plaintext : 'decrypted-sercret-key' + }; + } + return callback(null, data); + }); + + AWS.mock('KMS', 'encrypt', function (params, callback) { + const data = { + CiphertextBlob : Buffer.from('creds-from-file') + }; + return callback(null, data); + }); + + AWS.mock('S3', 'getObject', function (params, callback) { const data = { - Plaintext : 'decrypted-sercret-key' + Body: Buffer.from('creds-from-file') }; return callback(null, data); }); @@ -177,6 +203,73 @@ describe('Unit Tests', function() { setEnvStub.restore(); responseStub.restore(); }); + + describe('Load function', function() { + const rewiredTestModule = rewire('../paws_collector'); + const {PawsCollector: LoadTestCollector} = rewiredTestModule; + let fileWriteStub; + let fileReadStub; + let fileExistsStub; + const CREDS_FILE_PATH = '/tmp/paws_creds.json'; + + beforeEach(() => { + rewiredTestModule.__set__("PAWS_DECRYPTED_CREDS", null); + fileWriteStub = sinon.spy(fs, 'writeFileSync'); + fileReadStub = sinon.spy(fs, 'readFileSync'); + }); + + afterEach(() => { + fileWriteStub.restore(); + fileReadStub.restore(); + }); + + it('gets creds from s3 if present and writes them to cache', function(done){ + const oldAuthType = process.env.paws_auth_type; + process.env.paws_auth_type = 's3object'; + fileExistsStub = sinon.stub(fs, 'existsSync').callsFake(() => { + return false; + }); + + LoadTestCollector.load().then(function({pawsCreds}){ + assert.equal(pawsCreds.secret, 'decrypted-secret-key-from-file'); + assert.ok(fileReadStub.called); + assert.ok(fileWriteStub.calledWith(CREDS_FILE_PATH)); + process.env.paws_auth_type = oldAuthType; + fileExistsStub.restore(); + done(); + }); + }); + + it('gets creds from cache', function(done){ + const oldAuthType = process.env.paws_auth_type; + process.env.paws_auth_type = 's3object'; + fs.writeFileSync(CREDS_FILE_PATH, 'creds-from-file'); + fileWriteStub.resetHistory(); + fileExistsStub = sinon.stub(fs, 'existsSync').callsFake(() => { + return true; + }); + + LoadTestCollector.load().then(function({pawsCreds}){ + assert.equal(pawsCreds.secret, 'decrypted-secret-key-from-file'); + assert.equal(fileReadStub.calledWith(CREDS_FILE_PATH), true); + assert.equal(fileWriteStub.calledWith(CREDS_FILE_PATH), false); + process.env.paws_auth_type = oldAuthType; + fs.unlinkSync(CREDS_FILE_PATH); + fileExistsStub.restore(); + done(); + }); + }); + + it('does not get the creds from a file when the auth type is not s3object', function(done){ + LoadTestCollector.load().then(function({pawsCreds}){ + assert.notEqual(pawsCreds.secret, 'decrypted-secret-key-from-file'); + assert.equal(fileWriteStub.called, false); + assert.equal(fileReadStub.called, false); + fileExistsStub.restore(); + done(); + }); + }); + }); describe('Poll Request Tests', function() { it('poll request success, single state', function(done) {