Skip to content

Commit

Permalink
handle creds stored in s3 buckets (#48)
Browse files Browse the repository at this point in the history 
* rought out s3 cred storeage

*  action comments and make implemtatio simpler

* clean some errant cfn stragglers

* address las few comments

* a few fixes from running unit tests

* remove copy past error

* clean up and add some tests

* action comments

* one more typo

* add rewire and check content of cred

* fix linting error
  • Loading branch information
@JDragovichAlertLogic
JDragovichAlertLogic authored and kkuzmin committed Jan 23, 2020
1 parent 5fd3117 commit c8a2ba0
Show file tree
Hide file tree
Showing 5 changed files with 198 additions and 22 deletions.
35 changes: 31 additions & 4 deletions cfn/paws-collector.template
View file
Original file line number Diff line number Diff line change
Expand Up @@ -62,8 +62,7 @@
},
"PawsEndpoint": {
"Description": "URL to poll",
"Type": "String",
"Default": "https://alertlogic-admin.okta.com/"
"Type": "String"
},
"PawsAuthType": {
"Description": "Target API authentication type. Supported types: ssws, oauth2",
Expand All @@ -85,6 +84,12 @@
"Type": "Number",
"Default": 10
},
"PawsCredsS3Uri": {
"Description": "The S3 uri where object where credentials are stored. in the for s3://<bucket>/<object-path> Only used when PawsAuthType is 's3object'",
"Type": "String",
"AllowedPattern" : "^s3:\/\/[a-z,\-,0-9]*\/[a-z,\-,\.,\_,0-9]*",
"Default": ""
},
"CollectionStartTs": {
"Description": "Optional. Timestamp when log collection starts. For example, 2020-01-13T16:00:00Z.",
"Type": "String",
Expand Down Expand Up @@ -363,10 +368,11 @@
"\n",
"exports.handler = (event, context, callback) => {\n",
" if (event.ResourceType == 'AWS::CloudFormation::CustomResource' &&\n",
" event.RequestType == 'Create') {\n",
" event.RequestType == 'Create' &&\n",
" event.ResourceProperties.Plaintext.length > 0) {\n",
" return encrypt(event, context);\n",
" }\n",
" return response.send(event, context, response.SUCCESS);\n",
" return response.send(event, context, response.SUCCESS, {EncryptedText: \"\"});\n",
"}"
]
]
Expand Down Expand Up @@ -545,9 +551,15 @@
"paws_api_client_id":{
"Ref":"PawsClientId"
},
"paws_kms_key_arn":{
"Fn::GetAtt": [ "LambdaKmsKey", "Arn" ]
},
"paws_api_secret":{
"Fn::GetAtt": ["EncryptPawsSecret", "EncryptedText"]
},
"paws_s3_object_path":{
"Ref": "PawsCredsS3Uri"
},
"paws_collection_start_ts":{
"Ref":"CollectionStartTs"
},
Expand Down Expand Up @@ -619,6 +631,21 @@
]
}]
},
{
"Effect":"Allow",
"Action":"s3:Get*",
"Resource":[
{
"Fn::Join":["",
[
"arn:aws:s3:::",
{ "Fn::Select": ["1", {"Fn::Split": ["//", {"Ref": "PawsCredsS3Uri"}] } ] }
]
]

}
]
},
{
"Effect":"Allow",
"Action":[
Expand Down
6 changes: 3 additions & 3 deletions package.json
View file
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
{
"name": "@alertlogic/paws-collector",
"version": "1.0.10",
"version": "1.0.11",
"license": "MIT",
"description": "Alert Logic AWS based API Poll Log Collector Library",
"repository": {
Expand Down Expand Up @@ -28,8 +28,8 @@
"mocha-jenkins-reporter": "^0.4.2",
"nyc": "^14.1.1",
"rewire": "^4.0.1",
"yargs": "^15.0.2",
"sinon": "^7.5.0"
"sinon": "^7.5.0",
"yargs": "^15.0.2"
},
"dependencies": {
"@alertlogic/al-aws-collector-js": "2.0.4",
Expand Down
80 changes: 67 additions & 13 deletions paws_collector.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -13,19 +13,59 @@
const async = require('async');
const debug = require('debug')('index');
const AWS = require('aws-sdk');
const fs = require('fs');

const AlAwsCollector = require('@alertlogic/al-aws-collector-js').AlAwsCollector;
const m_packageJson = require('./package.json');

const CREDS_FILE_PATH = '/tmp/paws_creds.json';
var PAWS_DECRYPTED_CREDS = null;

function getDecryptedPawsCredentials(callback) {
function getPawsCredsFile(){
return new Promise((resolve, reject) => {
// check if the creds file is cached and retrieve it if it is not
if(!fs.existsSync(CREDS_FILE_PATH)){
const s3 = new AWS.S3({apiVersion: '2006-03-01'});
const kms = new AWS.KMS();

// doing the string manipulation here because doing it here is way less groos than doing it in the cfn
const s3Path = process.env.paws_s3_object_path.split('//')[1];
const s3PathParts = s3Path.split('/');

// retrive the object from S3
var params = {
Bucket: s3PathParts.shift(),
Key: s3PathParts.join('/')
};
s3.getObject(params, (err, data) => {
if (err) return reject(new Error(err, err.stack));

// encrypt the file contents and cache on the lambda container file system
const encryptParams ={
Plaintext: data.Body,
KeyId: process.env.paws_kms_key_arn
};
kms.encrypt(encryptParams, (encryptError, encryptResponse) => {
if (encryptError) return reject(new Error(encryptError, encryptError.stack));

fs.writeFileSync(CREDS_FILE_PATH, encryptResponse.CiphertextBlob);
return resolve(encryptResponse.CiphertextBlob);
})
});
}
else {
return resolve(fs.readFileSync(CREDS_FILE_PATH));
}
});
};

function getDecryptedPawsCredentials(credsBuffer, callback) {
if (PAWS_DECRYPTED_CREDS) {
return callback(null, PAWS_DECRYPTED_CREDS);
} else {
const kms = new AWS.KMS();
kms.decrypt(
{CiphertextBlob: Buffer.from(process.env.paws_api_secret, 'base64')},
{CiphertextBlob: credsBuffer},
(err, data) => {
if (err) {
return callback(err);
Expand All @@ -35,7 +75,7 @@ function getDecryptedPawsCredentials(callback) {
client_id: process.env.paws_api_client_id,
secret: data.Plaintext.toString('ascii')
};

return callback(null, PAWS_DECRYPTED_CREDS);
}
});
Expand All @@ -47,17 +87,30 @@ class PawsCollector extends AlAwsCollector {
static load() {
return new Promise(function(resolve, reject){
AlAwsCollector.load().then(function(aimsCreds) {
getDecryptedPawsCredentials(function(err, pawsCreds) {
if (err){
reject(err);
} else {
resolve({aimsCreds : aimsCreds, pawsCreds: pawsCreds});
}
});
})
})
let credsPromise;

switch(process.env.paws_auth_type){
case 's3object':
credsPromise = getPawsCredsFile();
break;
default:
const enVarCreds = Buffer.from(process.env.paws_api_secret, 'base64');
credsPromise = new Promise(res => res(enVarCreds));
}

return credsPromise.then(credsBuffer => {
getDecryptedPawsCredentials(credsBuffer, function(err, pawsCreds) {
if (err){
reject(err);
} else {
resolve({aimsCreds : aimsCreds, pawsCreds: pawsCreds});
}
});
}).catch(err => console.error(`PAWS000400 Error getting Paws Credentials ${err}`));
});
});
}

constructor(context, {aimsCreds, pawsCreds}) {
super(context, 'paws',
AlAwsCollector.IngestTypes.LOGMSGS,
Expand Down Expand Up @@ -254,6 +307,7 @@ class PawsCollector extends AlAwsCollector {
}

module.exports = {
getPawsCredsFile,
PawsCollector: PawsCollector
}

4 changes: 3 additions & 1 deletion test/paws_mock.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -7,8 +7,10 @@ process.env.aims_secret_key = 'aims-secret-key-encrypted';
process.env.log_group = 'logGroupName';
process.env.paws_state_queue_arn = 'arn:aws:sqs:us-east-1:352283894008:test-queue';
process.env.paws_state_queue_url = 'https://sqs.us-east-1.amazonaws.com/352283894008/test-queue';
process.env.paws_s3_object_path = "s3://joe-creds-test/paws_creds.json";
process.env.paws_kms_key_arn = "arn:aws:kms:us-east-1:352283894008:key/cdda86d5-615b-4dcc-9319-77ab34510473";
process.env.paws_type_name = 'okta';
process.env.paws_api_auth_type = 'auth';
process.env.paws_auth_type = 'auth';
process.env.paws_poll_interval = 900;
process.env.paws_endpoint = 'https://test.alertlogic.com/';
process.env.paws_api_secret = 'api-token';
Expand Down
95 changes: 94 additions & 1 deletion test/paws_test.js
View file
Original file line number Diff line number Diff line change
@@ -1,6 +1,8 @@
const assert = require('assert');
const rewire = require('rewire');
const sinon = require('sinon');
var AWS = require('aws-sdk-mock');
const fs = require('fs');
const m_response = require('cfn-response');

const pawsMock = require('./paws_mock');
Expand Down Expand Up @@ -154,8 +156,32 @@ describe('Unit Tests', function() {

beforeEach(function(){
AWS.mock('KMS', 'decrypt', function (params, callback) {
let data;
if(params.CiphertextBlob.toString() === 'creds-from-file'){
console.log('dcrypting file');
data = {
Plaintext : 'decrypted-secret-key-from-file'
};
}
else{
console.log('decrypting somthing else');
data = {
Plaintext : 'decrypted-sercret-key'
};
}
return callback(null, data);
});

AWS.mock('KMS', 'encrypt', function (params, callback) {
const data = {
CiphertextBlob : Buffer.from('creds-from-file')
};
return callback(null, data);
});

AWS.mock('S3', 'getObject', function (params, callback) {
const data = {
Plaintext : 'decrypted-sercret-key'
Body: Buffer.from('creds-from-file')
};
return callback(null, data);
});
Expand All @@ -177,6 +203,73 @@ describe('Unit Tests', function() {
setEnvStub.restore();
responseStub.restore();
});

describe('Load function', function() {
const rewiredTestModule = rewire('../paws_collector');
const {PawsCollector: LoadTestCollector} = rewiredTestModule;
let fileWriteStub;
let fileReadStub;
let fileExistsStub;
const CREDS_FILE_PATH = '/tmp/paws_creds.json';

beforeEach(() => {
rewiredTestModule.__set__("PAWS_DECRYPTED_CREDS", null);
fileWriteStub = sinon.spy(fs, 'writeFileSync');
fileReadStub = sinon.spy(fs, 'readFileSync');
});

afterEach(() => {
fileWriteStub.restore();
fileReadStub.restore();
});

it('gets creds from s3 if present and writes them to cache', function(done){
const oldAuthType = process.env.paws_auth_type;
process.env.paws_auth_type = 's3object';
fileExistsStub = sinon.stub(fs, 'existsSync').callsFake(() => {
return false;
});

LoadTestCollector.load().then(function({pawsCreds}){
assert.equal(pawsCreds.secret, 'decrypted-secret-key-from-file');
assert.ok(fileReadStub.called);
assert.ok(fileWriteStub.calledWith(CREDS_FILE_PATH));
process.env.paws_auth_type = oldAuthType;
fileExistsStub.restore();
done();
});
});

it('gets creds from cache', function(done){
const oldAuthType = process.env.paws_auth_type;
process.env.paws_auth_type = 's3object';
fs.writeFileSync(CREDS_FILE_PATH, 'creds-from-file');
fileWriteStub.resetHistory();
fileExistsStub = sinon.stub(fs, 'existsSync').callsFake(() => {
return true;
});

LoadTestCollector.load().then(function({pawsCreds}){
assert.equal(pawsCreds.secret, 'decrypted-secret-key-from-file');
assert.equal(fileReadStub.calledWith(CREDS_FILE_PATH), true);
assert.equal(fileWriteStub.calledWith(CREDS_FILE_PATH), false);
process.env.paws_auth_type = oldAuthType;
fs.unlinkSync(CREDS_FILE_PATH);
fileExistsStub.restore();
done();
});
});

it('does not get the creds from a file when the auth type is not s3object', function(done){
LoadTestCollector.load().then(function({pawsCreds}){
assert.notEqual(pawsCreds.secret, 'decrypted-secret-key-from-file');
assert.equal(fileWriteStub.called, false);
assert.equal(fileReadStub.called, false);
fileExistsStub.restore();
done();
});
});
});

describe('Poll Request Tests', function() {
it('poll request success, single state', function(done) {
Expand Down

0 comments on commit c8a2ba0

Please sign in to comment.