diff --git a/build.gradle b/build.gradle index c0636fe7..4a567cd1 100644 --- a/build.gradle +++ b/build.gradle @@ -9,6 +9,9 @@ allprojects { apply plugin: "com.github.johnrengelman.shadow" } +sourceCompatibility = 1.8 +targetCompatibility = 1.8 + apply from: file('gradle/dependencies.gradle') apply from: file('gradle/check.gradle') apply from: file('gradle/integration.gradle') diff --git a/gradle.properties b/gradle.properties index 00730170..0209d420 100644 --- a/gradle.properties +++ b/gradle.properties @@ -16,4 +16,4 @@ group=com.nike artifactId=cerberus-lifecycle-cli -version=4.0.0 +version=4.1.0 diff --git a/gradle/dependencies.gradle b/gradle/dependencies.gradle index 91492b8e..0669ece8 100644 --- a/gradle/dependencies.gradle +++ b/gradle/dependencies.gradle @@ -19,7 +19,7 @@ allprojects { jcenter() } - def awsSDKVersion = '1.11.229' + def awsSDKVersion = '1.11.269' //noinspection GroovyAssignabilityCheck dependencies { @@ -36,9 +36,8 @@ allprojects { compile group: 'com.amazonaws', name: 'aws-java-sdk-route53', version: awsSDKVersion compile group: 'com.amazonaws', name: 'aws-java-sdk-elasticloadbalancingv2', version: awsSDKVersion compile group: 'com.amazonaws', name: 'aws-java-sdk-rds', version: awsSDKVersion - - // https://mvnrepository.com/artifact/com.amazonaws/aws-encryption-sdk-java compile group: 'com.amazonaws', name: 'aws-encryption-sdk-java', version: '1.3.1' + compile group: 'com.amazonaws', name: 'aws-java-sdk-athena', version: awsSDKVersion compile 'com.nike:vault-client:1.4.1' compile 'com.squareup.okhttp3:okhttp:3.3.1' diff --git a/src/main/java/com/nike/cerberus/ConfigConstants.java b/src/main/java/com/nike/cerberus/ConfigConstants.java index 7bf8741c..61075801 100644 --- a/src/main/java/com/nike/cerberus/ConfigConstants.java +++ b/src/main/java/com/nike/cerberus/ConfigConstants.java @@ -21,54 +21,33 @@ public class ConfigConstants { public static final String ENV_PREFIX = "cerberus-"; - public static final String DEFAULT_ENCODING = "UTF-8"; - public static final int MINIMUM_AZS = 3; - public static final String CONFIG_BUCKET_KEY = "cerberusconfigbucket"; - public static final String DEFAULT_CMS_DB_NAME = "cms"; - public static final String ENVIRONMENT_DATA_FILE = "environment.json"; - public static final String CERT_PART_CA = "ca.pem"; - public static final String CERT_PART_CERT = "cert.pem"; - public static final String CERT_PART_PKCS8_KEY = "pkcs8-key.pem"; - public static final String CERT_PART_KEY = "key.pem"; - public static final String CERT_PART_PUBKEY = "pubkey.pem"; - public static final String CERT_ACME_ACCOUNT_PRIVATE_KEY = "certificates/acme/account-private-key-pkcs1.pem"; - public static final String CMS_ENV_CONFIG_PATH = "cms/environment.properties"; - public static final String VERSION_PROPERTY = "cli.version"; - public static final String CMS_ADMIN_GROUP_KEY = "cms.admin.group"; - public static final String ROOT_USER_ARN_KEY = "root.user.arn"; - public static final String ADMIN_ROLE_ARN_KEY = "admin.role.arn"; - public static final String CMS_ROLE_ARN_KEY = "cms.role.arn"; - public static final String JDBC_URL_KEY = "JDBC.url"; - public static final String JDBC_USERNAME_KEY = "JDBC.username"; - public static final String JDBC_PASSWORD_KEY = "JDBC.password"; - public static final String CMK_ARNS_KEY = "cms.encryption.cmk.arns"; - public static final String HASH_SALT = "cms.auth.token.hash.salt"; - public static final String CMS_ENV_NAME = "cms.env.name"; - public static final String CMS_CERTIFICATE_TO_USE = "cms.ssl.certificateName"; + public static final String AUDIT_LOG_PROCESSOR = "cms.event.processors.com.nike.cerberus.event.processor.AuditLogProcessor"; + public static final String AUDIT_LOG_BUCKET = "cms.audit.bucket"; + public static final String AUDIT_LOG_BUCKET_REGION = "cms.audit.bucket_region"; public static final ImmutableSet SYSTEM_CONFIGURED_CMS_PROPERTIES = ImmutableSet.of( ROOT_USER_ARN_KEY, @@ -80,7 +59,10 @@ public class ConfigConstants { CMK_ARNS_KEY, HASH_SALT, CMS_ENV_NAME, - CMS_CERTIFICATE_TO_USE); + CMS_CERTIFICATE_TO_USE, + AUDIT_LOG_PROCESSOR, + AUDIT_LOG_BUCKET, + AUDIT_LOG_BUCKET_REGION); public static final String CERBERUS_AMI_TAG_NAME = "tag:cerberus_component"; diff --git a/src/main/java/com/nike/cerberus/cli/CerberusRunner.java b/src/main/java/com/nike/cerberus/cli/CerberusRunner.java index 9d182bf4..50761ea7 100644 --- a/src/main/java/com/nike/cerberus/cli/CerberusRunner.java +++ b/src/main/java/com/nike/cerberus/cli/CerberusRunner.java @@ -27,6 +27,10 @@ import com.nike.cerberus.ConfigConstants; import com.nike.cerberus.command.CerberusCommand; import com.nike.cerberus.command.Command; +import com.nike.cerberus.command.audit.CreateAuditAthenaDbAndTableCommand; +import com.nike.cerberus.command.audit.CreateAuditLoggingStackCommand; +import com.nike.cerberus.command.audit.DisableAuditLoggingCommand; +import com.nike.cerberus.command.audit.EnableAuditLoggingForExistingEnvironmentCommand; import com.nike.cerberus.command.certificates.RotateAcmeAccountPrivateKeyCommand; import com.nike.cerberus.command.cms.CreateCmsClusterCommand; import com.nike.cerberus.command.cms.CreateCmsConfigCommand; @@ -208,6 +212,10 @@ private void registerAllCommands() { registerCommand(new GenerateAndRotateCertificatesCommand()); registerCommand(new RotateAcmeAccountPrivateKeyCommand()); registerCommand(new CleanUpRdsSnapshotsCommand()); + registerCommand(new CreateAuditLoggingStackCommand()); + registerCommand(new CreateAuditAthenaDbAndTableCommand()); + registerCommand(new DisableAuditLoggingCommand()); + registerCommand(new EnableAuditLoggingForExistingEnvironmentCommand()); } /** diff --git a/src/main/java/com/nike/cerberus/cli/EnvironmentConfigToArgsMapper.java b/src/main/java/com/nike/cerberus/cli/EnvironmentConfigToArgsMapper.java index 26912796..fda23e07 100644 --- a/src/main/java/com/nike/cerberus/cli/EnvironmentConfigToArgsMapper.java +++ b/src/main/java/com/nike/cerberus/cli/EnvironmentConfigToArgsMapper.java @@ -18,6 +18,7 @@ import com.google.common.collect.Lists; import com.nike.cerberus.command.StackDelegate; +import com.nike.cerberus.command.audit.CreateAuditLoggingStackCommand; import com.nike.cerberus.command.cms.CreateCmsClusterCommand; import com.nike.cerberus.command.cms.CreateCmsConfigCommand; import com.nike.cerberus.command.cms.UpdateCmsConfigCommand; @@ -142,6 +143,9 @@ public static List getArgsForCommand(EnvironmentConfig environmentConfig case RotateCertificatesCommand.COMMAND_NAME: args = getUploadCertFilesCommandArgs(environmentConfig, passedArgs); break; + case CreateAuditLoggingStackCommand.COMMAND_NAME: + args = getCreateAuditLoggingStackCommandArgs(environmentConfig); + break; default: break; } @@ -152,6 +156,12 @@ public static List getArgsForCommand(EnvironmentConfig environmentConfig return args; } + private static List getCreateAuditLoggingStackCommandArgs(EnvironmentConfig environmentConfig) { + return ArgsBuilder.create() + .addOption(CreateAuditLoggingStackCommand.ADMIN_ROLE_ARN_LONG_ARG, environmentConfig.getAdminRoleArn()) + .build(); + } + private static List getCreateEdgeDomainRecordCommandArgs(EnvironmentConfig environmentConfig) { ArgsBuilder args = ArgsBuilder.create() .addOption(CreateEdgeDomainRecordCommand.BASE_DOMAIN_NAME_LONG_ARG, environmentConfig.getBaseDomainName()) diff --git a/src/main/java/com/nike/cerberus/client/aws/AthenaAwsClientFactory.java b/src/main/java/com/nike/cerberus/client/aws/AthenaAwsClientFactory.java new file mode 100644 index 00000000..cdf52803 --- /dev/null +++ b/src/main/java/com/nike/cerberus/client/aws/AthenaAwsClientFactory.java @@ -0,0 +1,24 @@ +package com.nike.cerberus.client.aws; + +import com.amazonaws.regions.Regions; +import com.amazonaws.services.athena.AmazonAthenaClient; +import com.nike.cerberus.service.AwsClientFactory; + +public class AthenaAwsClientFactory extends AwsClientFactory { + + @Override + public AmazonAthenaClient getClient(Regions region) { + if (!clients.containsKey(region)) { + clients.put(region, createClient(region)); + } + return clients.get(region); + } + + private AmazonAthenaClient createClient(Regions region) { + return (AmazonAthenaClient) AmazonAthenaClient.builder() + .withRegion(region) + .withCredentials(getAWSCredentialsProviderChain()) + .build(); + } + +} diff --git a/src/main/java/com/nike/cerberus/command/audit/CreateAuditAthenaDbAndTableCommand.java b/src/main/java/com/nike/cerberus/command/audit/CreateAuditAthenaDbAndTableCommand.java new file mode 100644 index 00000000..2b872f72 --- /dev/null +++ b/src/main/java/com/nike/cerberus/command/audit/CreateAuditAthenaDbAndTableCommand.java @@ -0,0 +1,27 @@ +package com.nike.cerberus.command.audit; + +import com.beust.jcommander.Parameters; +import com.nike.cerberus.command.Command; +import com.nike.cerberus.operation.Operation; +import com.nike.cerberus.operation.audit.CreateAuditAthenaDbAndTableOperation; + +import static com.nike.cerberus.command.audit.CreateAuditAthenaDbAndTableCommand.COMMAND_NAME; + +@Parameters( + commandNames = COMMAND_NAME, + commandDescription = "Creates the db and table needed in athena to enable interacting with the audit data via athena" +) +public class CreateAuditAthenaDbAndTableCommand implements Command { + + public static final String COMMAND_NAME = "create-audit-log-athena-db-and-table"; + + @Override + public String getCommandName() { + return COMMAND_NAME; + } + + @Override + public Class> getOperationClass() { + return CreateAuditAthenaDbAndTableOperation.class; + } +} diff --git a/src/main/java/com/nike/cerberus/command/audit/CreateAuditLoggingStackCommand.java b/src/main/java/com/nike/cerberus/command/audit/CreateAuditLoggingStackCommand.java new file mode 100644 index 00000000..6a34b7ed --- /dev/null +++ b/src/main/java/com/nike/cerberus/command/audit/CreateAuditLoggingStackCommand.java @@ -0,0 +1,51 @@ +package com.nike.cerberus.command.audit; + +import com.beust.jcommander.Parameter; +import com.beust.jcommander.Parameters; +import com.beust.jcommander.ParametersDelegate; +import com.nike.cerberus.command.Command; +import com.nike.cerberus.domain.cloudformation.TagParametersDelegate; +import com.nike.cerberus.operation.Operation; +import com.nike.cerberus.operation.audit.CreateAuditStackOperation; + +import static com.nike.cerberus.command.audit.CreateAuditLoggingStackCommand.COMMAND_NAME; + +@Parameters( + commandNames = COMMAND_NAME, + commandDescription = "Creates an S3 bucket and IAM roles configured to allow CMS to write audit log data and " + + "IAM role that allows AWS Athena/Glue queries" +) +public class CreateAuditLoggingStackCommand implements Command { + + public static final String COMMAND_NAME = "create-audit-logging-stack"; + + public static final String ADMIN_ROLE_ARN_LONG_ARG = "--admin-role-arn"; + + @Parameter( + names = ADMIN_ROLE_ARN_LONG_ARG, + description = "An IAM role ARN that will be given elevated privileges for the KMS CMKs created.", + required = true + ) + private String adminRoleArn; + + public String getAdminRoleArn() { + return adminRoleArn; + } + + @ParametersDelegate + private TagParametersDelegate tagsDelegate = new TagParametersDelegate(); + + public TagParametersDelegate getTagsDelegate() { + return tagsDelegate; + } + + @Override + public String getCommandName() { + return COMMAND_NAME; + } + + @Override + public Class> getOperationClass() { + return CreateAuditStackOperation.class; + } +} diff --git a/src/main/java/com/nike/cerberus/command/audit/DisableAuditLoggingCommand.java b/src/main/java/com/nike/cerberus/command/audit/DisableAuditLoggingCommand.java new file mode 100644 index 00000000..60ad0766 --- /dev/null +++ b/src/main/java/com/nike/cerberus/command/audit/DisableAuditLoggingCommand.java @@ -0,0 +1,27 @@ +package com.nike.cerberus.command.audit; + +import com.beust.jcommander.Parameters; +import com.nike.cerberus.command.Command; +import com.nike.cerberus.operation.Operation; +import com.nike.cerberus.operation.audit.DisableAuditLoggingOperation; + +import static com.nike.cerberus.command.audit.DisableAuditLoggingCommand.COMMAND_NAME; + +@Parameters( + commandNames = COMMAND_NAME, + commandDescription = "Disables the CLI to set the required CMS properties to enable audit logging, when creating or updating CMS config" +) +public class DisableAuditLoggingCommand implements Command { + + public static final String COMMAND_NAME = "disable-audit-logging"; + + @Override + public String getCommandName() { + return COMMAND_NAME; + } + + @Override + public Class> getOperationClass() { + return DisableAuditLoggingOperation.class; + } +} diff --git a/src/main/java/com/nike/cerberus/command/audit/EnableAuditLoggingCommand.java b/src/main/java/com/nike/cerberus/command/audit/EnableAuditLoggingCommand.java new file mode 100644 index 00000000..b70842e8 --- /dev/null +++ b/src/main/java/com/nike/cerberus/command/audit/EnableAuditLoggingCommand.java @@ -0,0 +1,27 @@ +package com.nike.cerberus.command.audit; + +import com.beust.jcommander.Parameters; +import com.nike.cerberus.command.Command; +import com.nike.cerberus.operation.Operation; +import com.nike.cerberus.operation.audit.EnableAuditLoggingOperation; + +import static com.nike.cerberus.command.audit.EnableAuditLoggingCommand.COMMAND_NAME; + +@Parameters( + commandNames = COMMAND_NAME, + commandDescription = "Enables the CLI to set the required CMS properties to enable audit logging, when creating or updating CMS config" +) +public class EnableAuditLoggingCommand implements Command { + + public static final String COMMAND_NAME = "enable-audit-logging"; + + @Override + public String getCommandName() { + return COMMAND_NAME; + } + + @Override + public Class> getOperationClass() { + return EnableAuditLoggingOperation.class; + } +} diff --git a/src/main/java/com/nike/cerberus/command/audit/EnableAuditLoggingForExistingEnvironmentCommand.java b/src/main/java/com/nike/cerberus/command/audit/EnableAuditLoggingForExistingEnvironmentCommand.java new file mode 100644 index 00000000..3e83f7ad --- /dev/null +++ b/src/main/java/com/nike/cerberus/command/audit/EnableAuditLoggingForExistingEnvironmentCommand.java @@ -0,0 +1,36 @@ +package com.nike.cerberus.command.audit; + +import com.beust.jcommander.Parameters; +import com.nike.cerberus.command.Command; +import com.nike.cerberus.operation.Operation; +import com.nike.cerberus.operation.audit.EnableAuditLoggingForExistingEnvironmentOperation; + +import static com.nike.cerberus.command.audit.EnableAuditLoggingForExistingEnvironmentCommand.COMMAND_DESCRIPTION; +import static com.nike.cerberus.command.audit.EnableAuditLoggingForExistingEnvironmentCommand.COMMAND_NAME; + +@Parameters( + commandNames = COMMAND_NAME, + commandDescription = COMMAND_DESCRIPTION +) +public class EnableAuditLoggingForExistingEnvironmentCommand implements Command { + + public static final String COMMAND_NAME = "enable-audit-logging-for-existing-environment"; + public static final String COMMAND_DESCRIPTION = + "A Composite command that will will execute the following commands in order: " + + "create-audit-logging-stack, " + + "create-audit-log-athena-db-and-table, " + + "enable-audit-logging, " + + "update-cms-config, " + + "reboot-cms. " + + "This will do everything required to enable audit logging for an existing environment."; + + @Override + public String getCommandName() { + return COMMAND_NAME; + } + + @Override + public Class> getOperationClass() { + return EnableAuditLoggingForExistingEnvironmentOperation.class; + } +} diff --git a/src/main/java/com/nike/cerberus/domain/cloudformation/AuditOutputs.java b/src/main/java/com/nike/cerberus/domain/cloudformation/AuditOutputs.java new file mode 100644 index 00000000..b5f74228 --- /dev/null +++ b/src/main/java/com/nike/cerberus/domain/cloudformation/AuditOutputs.java @@ -0,0 +1,14 @@ +package com.nike.cerberus.domain.cloudformation; + +public class AuditOutputs { + String auditBucketName; + + public String getAuditBucketName() { + return auditBucketName; + } + + public AuditOutputs setAuditBucketName(String auditBucketName) { + this.auditBucketName = auditBucketName; + return this; + } +} diff --git a/src/main/java/com/nike/cerberus/domain/cloudformation/AuditParameters.java b/src/main/java/com/nike/cerberus/domain/cloudformation/AuditParameters.java new file mode 100644 index 00000000..07680209 --- /dev/null +++ b/src/main/java/com/nike/cerberus/domain/cloudformation/AuditParameters.java @@ -0,0 +1,35 @@ +package com.nike.cerberus.domain.cloudformation; + +public class AuditParameters { + + private String cmsIamRoleArn; + private String accountAdminArn; + private String environmentName; + + public String getCmsIamRoleArn() { + return cmsIamRoleArn; + } + + public AuditParameters setCmsIamRoleArn(String cmsIamRoleArn) { + this.cmsIamRoleArn = cmsIamRoleArn; + return this; + } + + public String getAccountAdminArn() { + return accountAdminArn; + } + + public AuditParameters setAccountAdminArn(String accountAdminArn) { + this.accountAdminArn = accountAdminArn; + return this; + } + + public String getEnvironmentName() { + return environmentName; + } + + public AuditParameters setEnvironmentName(String environmentName) { + this.environmentName = environmentName; + return this; + } +} diff --git a/src/main/java/com/nike/cerberus/domain/environment/EnvironmentData.java b/src/main/java/com/nike/cerberus/domain/environment/EnvironmentData.java index 11c876d4..5d2b6b07 100644 --- a/src/main/java/com/nike/cerberus/domain/environment/EnvironmentData.java +++ b/src/main/java/com/nike/cerberus/domain/environment/EnvironmentData.java @@ -46,6 +46,8 @@ public class EnvironmentData { private Map regionData = new HashMap<>();; + private boolean auditLoggingEnabled; + public String getEnvironmentName() { return environmentName; } @@ -90,6 +92,15 @@ public LinkedList getCertificateData() { return certificateInfoList; } + public boolean isAuditLoggingEnabled() { + return auditLoggingEnabled; + } + + public EnvironmentData setAuditLoggingEnabled(boolean auditLoggingEnabled) { + this.auditLoggingEnabled = auditLoggingEnabled; + return this; + } + public void addNewCertificateData(CertificateInformation certificateData) { getCertificateData().add(certificateData); } diff --git a/src/main/java/com/nike/cerberus/domain/environment/Stack.java b/src/main/java/com/nike/cerberus/domain/environment/Stack.java index d86b5674..56de0c24 100644 --- a/src/main/java/com/nike/cerberus/domain/environment/Stack.java +++ b/src/main/java/com/nike/cerberus/domain/environment/Stack.java @@ -41,6 +41,7 @@ public class Stack implements Comparable { public static final Stack CMS = new Stack("cms", "cms-cluster.yaml", true); public static final Stack WAF = new Stack("web-app-firewall", "web-app-firewall.yaml", false); public static final Stack ROUTE53 = new Stack("route53", "route53.yaml", false); + public static final Stack AUDIT = new Stack("audit", "audit.yaml", false); public static final ImmutableList ALL_STACKS = ImmutableList.of( IAM_ROLES, @@ -51,7 +52,8 @@ public class Stack implements Comparable { LOAD_BALANCER, CMS, WAF, - ROUTE53 + ROUTE53, + AUDIT ); public static final ImmutableList ALL_STACK_NAMES = ImmutableList.copyOf(ALL_STACKS.stream().map(Stack::getName).collect(Collectors.toList())); diff --git a/src/main/java/com/nike/cerberus/domain/input/EnvironmentConfig.java b/src/main/java/com/nike/cerberus/domain/input/EnvironmentConfig.java index a731be6a..4c4368d6 100644 --- a/src/main/java/com/nike/cerberus/domain/input/EnvironmentConfig.java +++ b/src/main/java/com/nike/cerberus/domain/input/EnvironmentConfig.java @@ -34,6 +34,7 @@ public class EnvironmentConfig { private List additionalSubjectNames; private String loadBalancerSslPolicyOverride; private String hostedZoneId; + private boolean enableAuditLogs; private boolean generateKeysAndCerts; private String acmeApiUrl; private boolean enableLeCertFix; @@ -115,6 +116,15 @@ public void setHostedZoneId(String hostedZoneId) { this.hostedZoneId = hostedZoneId; } + public boolean isEnableAuditLogs() { + return enableAuditLogs; + } + + public EnvironmentConfig setEnableAuditLogs(boolean enableAuditLogs) { + this.enableAuditLogs = enableAuditLogs; + return this; + } + public boolean isGenerateKeysAndCerts() { return generateKeysAndCerts; } diff --git a/src/main/java/com/nike/cerberus/module/CerberusModule.java b/src/main/java/com/nike/cerberus/module/CerberusModule.java index 8ee35715..1e84ecb9 100644 --- a/src/main/java/com/nike/cerberus/module/CerberusModule.java +++ b/src/main/java/com/nike/cerberus/module/CerberusModule.java @@ -17,6 +17,7 @@ package com.nike.cerberus.module; import com.amazonaws.regions.Regions; +import com.amazonaws.services.athena.AmazonAthenaClient; import com.amazonaws.services.autoscaling.AmazonAutoScalingClient; import com.amazonaws.services.cloudformation.AmazonCloudFormationClient; import com.amazonaws.services.ec2.AmazonEC2Client; @@ -44,6 +45,7 @@ import com.google.inject.multibindings.OptionalBinder; import com.google.inject.name.Names; import com.google.inject.util.Providers; +import com.nike.cerberus.client.aws.AthenaAwsClientFactory; import com.nike.cerberus.command.CerberusCommand; import com.nike.cerberus.command.ProxyDelegate; import com.nike.cerberus.domain.environment.RegionDeserializer; @@ -122,6 +124,7 @@ private void bindAwsClientFactories() { bind(new TypeLiteral>() {}).toInstance(new AwsClientFactory() {}); bind(new TypeLiteral>() {}).toInstance(new AwsClientFactory() {}); bind(new TypeLiteral>() {}).toInstance(new AwsClientFactory() {}); + bind(new TypeLiteral>() {}).toInstance(new AthenaAwsClientFactory()); } /** diff --git a/src/main/java/com/nike/cerberus/operation/audit/CreateAuditAthenaDbAndTableOperation.java b/src/main/java/com/nike/cerberus/operation/audit/CreateAuditAthenaDbAndTableOperation.java new file mode 100644 index 00000000..67827d9d --- /dev/null +++ b/src/main/java/com/nike/cerberus/operation/audit/CreateAuditAthenaDbAndTableOperation.java @@ -0,0 +1,131 @@ +package com.nike.cerberus.operation.audit; + +import com.amazonaws.services.athena.AmazonAthenaClient; +import com.amazonaws.services.athena.model.GetQueryExecutionRequest; +import com.amazonaws.services.athena.model.GetQueryResultsRequest; +import com.amazonaws.services.athena.model.GetQueryResultsResult; +import com.amazonaws.services.athena.model.ResultConfiguration; +import com.amazonaws.services.athena.model.StartQueryExecutionRequest; +import com.amazonaws.services.athena.model.StartQueryExecutionResult; +import com.github.tomaslanger.chalk.Chalk; +import com.nike.cerberus.ConfigConstants; +import com.nike.cerberus.command.audit.CreateAuditAthenaDbAndTableCommand; +import com.nike.cerberus.domain.cloudformation.AuditOutputs; +import com.nike.cerberus.domain.environment.Stack; +import com.nike.cerberus.operation.Operation; +import com.nike.cerberus.service.AwsClientFactory; +import com.nike.cerberus.service.CloudFormationService; +import com.nike.cerberus.store.ConfigStore; +import org.apache.commons.io.IOUtils; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import javax.inject.Inject; +import javax.inject.Named; + +import java.io.IOException; + +import static com.nike.cerberus.module.CerberusModule.ENV_NAME; + +public class CreateAuditAthenaDbAndTableOperation implements Operation { + + private final Logger log = LoggerFactory.getLogger(getClass()); + + private final CloudFormationService cloudFormationService; + private final ConfigStore configStore; + private final AwsClientFactory athenaClientFactory; + private final String databaseName; + private final String tableName; + private final String environmentName; + + @Inject + public CreateAuditAthenaDbAndTableOperation(CloudFormationService cloudFormationService, + ConfigStore configStore, + @Named(ENV_NAME) String environmentName, + AwsClientFactory athenaClientFactory) { + + this.cloudFormationService = cloudFormationService; + this.configStore = configStore; + this.athenaClientFactory = athenaClientFactory; + + databaseName = environmentName + "_audit_db"; + tableName = databaseName + ".audit_data"; + this.environmentName = environmentName; + } + + @Override + public void run(CreateAuditAthenaDbAndTableCommand command) { + AuditOutputs outputs = + configStore.getStackOutputs(configStore.getPrimaryRegion(), + Stack.AUDIT.getFullName(environmentName), AuditOutputs.class); + + String bucketName = outputs.getAuditBucketName(); + + log.info("Creating Athena DB"); + String createDb = "CREATE DATABASE IF NOT EXISTS " + databaseName + ";"; + log.info(executeAthenaQuery(createDb, bucketName).toString()); + log.info("Creating table"); + String createAuditTable; + try { + String template = "/com/nike/cerberus/operation/audit/create_audit_table.ddl"; + createAuditTable = IOUtils.toString(getClass().getResourceAsStream(template), ConfigConstants.DEFAULT_ENCODING); + createAuditTable = createAuditTable.replace("@@TABLE_NAME@@", tableName); + createAuditTable = createAuditTable.replace("@@BUCKET_NAME@@", bucketName); + } catch (IOException e) { + throw new RuntimeException("failed to load create athena table template", e); + } + log.info(executeAthenaQuery(createAuditTable, bucketName).toString()); + + String msg = Chalk.on("ATTENTION: ").red().bold().toString() + + "Table creation complete, please note that before you execute queries against '" + tableName + "'\n" + + "You will have to run the following query '" + Chalk.on("MSCK REPAIR TABLE " + tableName).green().bold().toString() + "'\n" + + "CMS will uploads logs every 5 minutes and creates partition folders for every hour.\n" + + "You can automate that query to run every hour or run it before you query audit data.\n" + + "That query is free and scans the S3 folders in the audit bucket and add the new partitions (The hour folders)"; + + log.info(msg); + } + + @Override + public boolean isRunnable(CreateAuditAthenaDbAndTableCommand command) { + boolean isRunnable = true; + + if (! cloudFormationService.isStackPresent(configStore.getPrimaryRegion(), Stack.AUDIT.getFullName(environmentName))) { + log.error("You must create the audit stack using create-audit-logging-stack command"); + isRunnable = false; + } + + return isRunnable; + } + + /** + * Executes an Athena query and waits for it to finish returning the results + */ + private GetQueryResultsResult executeAthenaQuery(String query, String bucketName) { + AmazonAthenaClient athena = athenaClientFactory.getClient(configStore.getPrimaryRegion()); + + StartQueryExecutionResult result = athena + .startQueryExecution(new StartQueryExecutionRequest() + .withQueryString(query) + .withResultConfiguration(new ResultConfiguration().withOutputLocation(String.format("s3://%s/results/", bucketName))) + ); + + String id = result.getQueryExecutionId(); + + String state; + do { + state = athena.getQueryExecution(new GetQueryExecutionRequest().withQueryExecutionId(id)).getQueryExecution().getStatus().getState(); + log.info("polling for query to finish: current status: {}", state); + try { + Thread.sleep(1000); + } catch (InterruptedException e) { + log.info("Failed to sleep", e); + Thread.currentThread().interrupt(); + } + } while (state.equals("RUNNING")); + + log.info("The query: {} is in state: {}, fetching results", id, state); + + return athena.getQueryResults(new GetQueryResultsRequest().withQueryExecutionId(id)); + } +} diff --git a/src/main/java/com/nike/cerberus/operation/audit/CreateAuditStackOperation.java b/src/main/java/com/nike/cerberus/operation/audit/CreateAuditStackOperation.java new file mode 100644 index 00000000..afa80684 --- /dev/null +++ b/src/main/java/com/nike/cerberus/operation/audit/CreateAuditStackOperation.java @@ -0,0 +1,66 @@ +package com.nike.cerberus.operation.audit; + +import com.amazonaws.regions.Regions; +import com.nike.cerberus.command.audit.CreateAuditLoggingStackCommand; +import com.nike.cerberus.domain.cloudformation.AuditParameters; +import com.nike.cerberus.domain.environment.Stack; +import com.nike.cerberus.operation.Operation; +import com.nike.cerberus.service.CloudFormationService; +import com.nike.cerberus.store.ConfigStore; +import com.nike.cerberus.util.CloudFormationObjectMapper; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import javax.inject.Inject; +import javax.inject.Named; + +import java.util.Map; + +import static com.nike.cerberus.module.CerberusModule.ENV_NAME; + + +public class CreateAuditStackOperation implements Operation { + + private final Logger log = LoggerFactory.getLogger(getClass()); + + private final CloudFormationService cloudFormationService; + private final ConfigStore configStore; + private final String environmentName; + private final CloudFormationObjectMapper cloudFormationObjectMapper; + + @Inject + public CreateAuditStackOperation(CloudFormationService cloudFormationService, + ConfigStore configStore, + @Named(ENV_NAME) String environmentName, + CloudFormationObjectMapper cloudFormationObjectMapper) { + + this.cloudFormationService = cloudFormationService; + this.configStore = configStore; + this.environmentName = environmentName; + this.cloudFormationObjectMapper = cloudFormationObjectMapper; + } + + @Override + public void run(CreateAuditLoggingStackCommand command) { + Regions primaryRegion = configStore.getPrimaryRegion(); + AuditParameters auditParameters = new AuditParameters() + .setAccountAdminArn(command.getAdminRoleArn()) + .setCmsIamRoleArn(configStore.getCmsIamRoleOutputs().getCmsIamRoleArn()) + .setEnvironmentName(environmentName); + Map parameters = cloudFormationObjectMapper.convertValue(auditParameters); + cloudFormationService.createStackAndWait(primaryRegion, Stack.AUDIT, parameters, true, command.getTagsDelegate().getTags()); + } + + + @Override + public boolean isRunnable(CreateAuditLoggingStackCommand command) { + boolean isRunnable = true; + + if (cloudFormationService.isStackPresent(configStore.getPrimaryRegion(), Stack.AUDIT.getFullName(environmentName))) { + log.error("The audit stack already exists use the update-stack command"); + isRunnable = false; + } + + return isRunnable; + } +} diff --git a/src/main/java/com/nike/cerberus/operation/audit/DisableAuditLoggingOperation.java b/src/main/java/com/nike/cerberus/operation/audit/DisableAuditLoggingOperation.java new file mode 100644 index 00000000..459b5889 --- /dev/null +++ b/src/main/java/com/nike/cerberus/operation/audit/DisableAuditLoggingOperation.java @@ -0,0 +1,37 @@ +package com.nike.cerberus.operation.audit; + +import com.nike.cerberus.command.audit.DisableAuditLoggingCommand; +import com.nike.cerberus.operation.Operation; +import com.nike.cerberus.store.ConfigStore; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +public class DisableAuditLoggingOperation implements Operation { + + private final Logger log = LoggerFactory.getLogger(getClass()); + + private final ConfigStore configStore; + + public DisableAuditLoggingOperation(ConfigStore configStore) { + + this.configStore = configStore; + } + + @Override + public void run(DisableAuditLoggingCommand command) { + configStore.setAuditLoggingEnabled(false); + log.info("Audit Logging disabled for CMS, please make sure to run update-cms-config and reboot-cms"); + } + + @Override + public boolean isRunnable(DisableAuditLoggingCommand command) { + boolean isRunnable = true; + + if (configStore.isAuditLoggingEnabled()) { + log.info("Audit logging is not enabled, nothing to do"); + isRunnable = false; + } + + return isRunnable; + } +} diff --git a/src/main/java/com/nike/cerberus/operation/audit/EnableAuditLoggingForExistingEnvironmentOperation.java b/src/main/java/com/nike/cerberus/operation/audit/EnableAuditLoggingForExistingEnvironmentOperation.java new file mode 100644 index 00000000..53626ca6 --- /dev/null +++ b/src/main/java/com/nike/cerberus/operation/audit/EnableAuditLoggingForExistingEnvironmentOperation.java @@ -0,0 +1,79 @@ +package com.nike.cerberus.operation.audit; + +import com.google.common.collect.ImmutableList; +import com.google.common.collect.Lists; +import com.nike.cerberus.command.audit.CreateAuditAthenaDbAndTableCommand; +import com.nike.cerberus.command.audit.CreateAuditLoggingStackCommand; +import com.nike.cerberus.command.audit.EnableAuditLoggingCommand; +import com.nike.cerberus.command.audit.EnableAuditLoggingForExistingEnvironmentCommand; +import com.nike.cerberus.command.cms.UpdateCmsConfigCommand; +import com.nike.cerberus.command.core.RebootCmsCommand; +import com.nike.cerberus.domain.cloudformation.ConfigParameters; +import com.nike.cerberus.domain.environment.Stack; +import com.nike.cerberus.operation.composite.ChainableCommand; +import com.nike.cerberus.operation.composite.CompositeOperation; +import com.nike.cerberus.service.CloudFormationService; +import com.nike.cerberus.store.ConfigStore; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import javax.inject.Inject; +import javax.inject.Named; +import java.util.List; + +import static com.nike.cerberus.module.CerberusModule.ENV_NAME; + +public class EnableAuditLoggingForExistingEnvironmentOperation extends CompositeOperation { + + private final Logger log = LoggerFactory.getLogger(getClass()); + + private final CloudFormationService cloudFormationService; + private final ConfigStore configStore; + private final String environmentName; + + @Inject + public EnableAuditLoggingForExistingEnvironmentOperation(CloudFormationService cloudFormationService, + ConfigStore configStore, + @Named(ENV_NAME) String environmentName) { + + this.cloudFormationService = cloudFormationService; + this.configStore = configStore; + this.environmentName = environmentName; + } + + @Override + protected List getCompositeCommandChain(EnableAuditLoggingForExistingEnvironmentCommand compositeCommand) { + String adminArn = configStore.getStackParameters( + configStore.getPrimaryRegion(), + Stack.CONFIG.getFullName(environmentName), + ConfigParameters.class).getAccountAdminArn(); + + return ImmutableList.of( + ChainableCommand.Builder.create() + .withCommand(new CreateAuditLoggingStackCommand()) + .withOption(CreateAuditLoggingStackCommand.ADMIN_ROLE_ARN_LONG_ARG, adminArn) + .build(), + new ChainableCommand(new CreateAuditAthenaDbAndTableCommand()), + new ChainableCommand(new EnableAuditLoggingCommand()), + new ChainableCommand(new UpdateCmsConfigCommand()), + new ChainableCommand(new RebootCmsCommand()) + ); + } + + @Override + public boolean isRunnable(EnableAuditLoggingForExistingEnvironmentCommand command) { + boolean isRunnable = true; + + if (! cloudFormationService.isStackPresent(configStore.getPrimaryRegion(), Stack.CMS.getFullName(environmentName))) { + log.info("The CMS stack does not exist, this command is intended to be ran on and environment that already exists"); + isRunnable = false; + } + + return isRunnable; + } + + @Override + public boolean isEnvironmentConfigRequired() { + return false; + } +} diff --git a/src/main/java/com/nike/cerberus/operation/audit/EnableAuditLoggingOperation.java b/src/main/java/com/nike/cerberus/operation/audit/EnableAuditLoggingOperation.java new file mode 100644 index 00000000..b47c725e --- /dev/null +++ b/src/main/java/com/nike/cerberus/operation/audit/EnableAuditLoggingOperation.java @@ -0,0 +1,51 @@ +package com.nike.cerberus.operation.audit; + +import com.nike.cerberus.command.audit.EnableAuditLoggingCommand; +import com.nike.cerberus.domain.environment.Stack; +import com.nike.cerberus.operation.Operation; +import com.nike.cerberus.service.CloudFormationService; +import com.nike.cerberus.store.ConfigStore; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import javax.inject.Inject; +import javax.inject.Named; + +import static com.nike.cerberus.module.CerberusModule.ENV_NAME; + +public class EnableAuditLoggingOperation implements Operation { + + private final Logger log = LoggerFactory.getLogger(getClass()); + + private final CloudFormationService cloudFormationService; + private final ConfigStore configStore; + private final String environmentName; + + @Inject + public EnableAuditLoggingOperation(CloudFormationService cloudFormationService, + ConfigStore configStore, + @Named(ENV_NAME) String environmentName) { + + this.cloudFormationService = cloudFormationService; + this.configStore = configStore; + this.environmentName = environmentName; + } + + @Override + public void run(EnableAuditLoggingCommand command) { + configStore.setAuditLoggingEnabled(true); + log.info("Audit Logging enabled for CMS, please make sure to run update-cms-config and reboot-cms"); + } + + @Override + public boolean isRunnable(EnableAuditLoggingCommand command) { + boolean isRunnable = true; + + if (! cloudFormationService.isStackPresent(configStore.getPrimaryRegion(), Stack.AUDIT.getFullName(environmentName))) { + log.info("The audit stack has not been created, please make sure to run create-audit-logging-stack and create-audit-log-athena-db-and-table before enabling the audit log in CMS"); + isRunnable = false; + } + + return isRunnable; + } +} diff --git a/src/main/java/com/nike/cerberus/operation/composite/ChainableCommand.java b/src/main/java/com/nike/cerberus/operation/composite/ChainableCommand.java index 23482d94..d888ed1a 100644 --- a/src/main/java/com/nike/cerberus/operation/composite/ChainableCommand.java +++ b/src/main/java/com/nike/cerberus/operation/composite/ChainableCommand.java @@ -35,7 +35,7 @@ public class ChainableCommand { private Command command; - private String[] additionalArgs; + private String[] additionalArgs = new String[]{}; public ChainableCommand() { diff --git a/src/main/java/com/nike/cerberus/operation/composite/CreateEnvironmentOperation.java b/src/main/java/com/nike/cerberus/operation/composite/CreateEnvironmentOperation.java index 629cef3b..bddccaf2 100644 --- a/src/main/java/com/nike/cerberus/operation/composite/CreateEnvironmentOperation.java +++ b/src/main/java/com/nike/cerberus/operation/composite/CreateEnvironmentOperation.java @@ -17,6 +17,9 @@ package com.nike.cerberus.operation.composite; import com.google.common.collect.Lists; +import com.nike.cerberus.command.audit.CreateAuditAthenaDbAndTableCommand; +import com.nike.cerberus.command.audit.CreateAuditLoggingStackCommand; +import com.nike.cerberus.command.audit.EnableAuditLoggingCommand; import com.nike.cerberus.command.cms.CreateCmsClusterCommand; import com.nike.cerberus.command.cms.CreateCmsConfigCommand; import com.nike.cerberus.command.composite.CreateEnvironmentCommand; @@ -46,48 +49,54 @@ public class CreateEnvironmentOperation extends CompositeOperation getCompositeCommandChain(CreateEnvironmentCommand compositeCommand) { List list = Lists.newArrayList( - // Step 1 Create the Base Cloud Formation Stack that creates S3 Buckets, Iam Roles and KMS keys needed for config + // Create the Base Cloud Formation Stack that creates S3 Buckets, Iam Roles and KMS keys needed for config new ChainableCommand(new InitializeEnvironmentCommand()), - // Step 2 Create the VPC Cloud Formation Stack that Cerberus will use + // Create the VPC Cloud Formation Stack that Cerberus will use new ChainableCommand(new CreateVpcCommand()), - // Step 3 Create the Security Group Cloud Formation Stack + // Create the Security Group Cloud Formation Stack new ChainableCommand(new CreateSecurityGroupsCommand()), - // Step 3.5 Add the vpc whitelist CIDRs + // Add the vpc whitelist CIDRs new ChainableCommand(new WhitelistCidrForVpcAccessCommand()), - // Step 4 Create the RDS Database Cloud Formation Stack + // Create the RDS Database Cloud Formation Stack new ChainableCommand(new CreateDatabaseCommand()) ); - // Step 5 Generate the PKCS private and public keys as well as the x509 certificates needed to enable https + // Generate the PKCS private and public keys as well as the x509 certificates needed to enable https if (environmentConfig.isGenerateKeysAndCerts()) { list.add(new ChainableCommand(new GenerateCertificateFilesCommand())); } + + if (environmentConfig.isEnableAuditLogs()) { + list.add(new ChainableCommand(new CreateAuditLoggingStackCommand())); + list.add(new ChainableCommand(new CreateAuditAthenaDbAndTableCommand())); + list.add(new ChainableCommand(new EnableAuditLoggingCommand())); + } list.addAll(Lists.newArrayList( - // Step 6 Upload the certs and keys to S3 and the IAM Cert Management service so that the ALB and CMS can use the certs + // Upload the certs and keys to S3 and the IAM Cert Management service so that the ALB and CMS can use the certs new ChainableCommand(new UploadCertificateFilesCommand()), - // Step 7 Create the Application Load Balancer Cloud Formation Stack + // Create the Application Load Balancer Cloud Formation Stack new ChainableCommand(new CreateLoadBalancerCommand()), - // Step 8 Generate the CMS config with org specific setting and first secrets encrypt, + // Generate the CMS config with org specific setting and first secrets encrypt, // Upload to S3 for CMS to download at service start new ChainableCommand(new CreateCmsConfigCommand()), - // Step 9 Create the CMS Cluster Stack + // Create the CMS Cluster Stack new ChainableCommand(new CreateCmsClusterCommand()), - // Step 10 Create the Web Application Fire wall stack + // Create the Web Application Fire wall stack new ChainableCommand(new CreateWafCommand()), - // Step 11 Create the Route 53 DNS Record Stack for origin and the load balancer + // Create the Route 53 DNS Record Stack for origin and the load balancer new ChainableCommand(new CreateRoute53Command()), - // Step 12 Create the outer most domain name record that will point to the origin record + // Create the outer most domain name record that will point to the origin record new ChainableCommand(new CreateEdgeDomainRecordCommand()) )); diff --git a/src/main/java/com/nike/cerberus/service/AwsClientFactory.java b/src/main/java/com/nike/cerberus/service/AwsClientFactory.java index ec545b0d..16f0969c 100644 --- a/src/main/java/com/nike/cerberus/service/AwsClientFactory.java +++ b/src/main/java/com/nike/cerberus/service/AwsClientFactory.java @@ -46,7 +46,7 @@ abstract public class AwsClientFactory { /** * Cache of clients by region */ - private Map clients = Maps.newHashMap(); + protected Map clients = Maps.newHashMap(); /** * Factory that creates and caches Aws clients by region for re-use; @@ -59,21 +59,21 @@ public T getClient(Regions region) { } @SuppressWarnings("unchecked") - private Class getGenericTypeClass() { + protected Class getGenericTypeClass() { TypeToken typeToken = new TypeToken(getClass()) {}; return (Class) typeToken.getRawType(); } - private M createAmazonClientInstance(Class clientClass, Regions region) { + protected M createAmazonClientInstance(Class clientClass, Regions region) { return Region.getRegion(region) .createClient(clientClass, getAWSCredentialsProviderChain(), getClientConfiguration()); } - private ClientConfiguration getClientConfiguration() { + protected ClientConfiguration getClientConfiguration() { return new ClientConfiguration(); } - private AWSCredentialsProviderChain getAWSCredentialsProviderChain() { + protected AWSCredentialsProviderChain getAWSCredentialsProviderChain() { String cerberusRoleToAssume = System.getenv(CERBERUS_ASSUME_ROLE_ARN) != null ? System.getenv(CERBERUS_ASSUME_ROLE_ARN) : ""; String cerberusRoleToAssumeExternalId = System.getenv(CERBERUS_ASSUME_ROLE_EXTERNAL_ID) != null ? diff --git a/src/main/java/com/nike/cerberus/store/ConfigStore.java b/src/main/java/com/nike/cerberus/store/ConfigStore.java index 6f34cf83..f81b06f5 100644 --- a/src/main/java/com/nike/cerberus/store/ConfigStore.java +++ b/src/main/java/com/nike/cerberus/store/ConfigStore.java @@ -32,6 +32,7 @@ import com.fasterxml.jackson.databind.ObjectMapper; import com.github.tomaslanger.chalk.Chalk; import com.nike.cerberus.ConfigConstants; +import com.nike.cerberus.domain.cloudformation.AuditOutputs; import com.nike.cerberus.domain.cloudformation.IamRolesOutputs; import com.nike.cerberus.domain.cloudformation.DatabaseOutputs; import com.nike.cerberus.domain.cloudformation.ConfigOutputs; @@ -308,6 +309,14 @@ private Properties generateBaseCmsSystemProperties() { properties.put(CMS_CERTIFICATE_TO_USE, getCertificationInformationList().getLast().getCertificateName()); properties.put(CMK_ARNS_KEY, StringUtils.join(data.getManagementServiceCmkArns(), ",")); + if (data.isAuditLoggingEnabled()) { + properties.put(AUDIT_LOG_PROCESSOR, String.valueOf(true)); + properties.put(AUDIT_LOG_BUCKET, getAuditStackOutputs(getPrimaryRegion()).getAuditBucketName()); + properties.put(AUDIT_LOG_BUCKET_REGION, getPrimaryRegion().getName()); + } else { + properties.put(AUDIT_LOG_PROCESSOR, String.valueOf(false)); + } + return properties; } @@ -478,6 +487,10 @@ public DatabaseOutputs getDatabaseStackOutputs(Regions region) { return getStackOutputs(region, getCloudFormationStackName(Stack.DATABASE), DatabaseOutputs.class); } + public AuditOutputs getAuditStackOutputs(Regions region) { + return getStackOutputs(region, getCloudFormationStackName(Stack.AUDIT), AuditOutputs.class); + } + /** * Get the stack outputs for a specific stack name. * @@ -716,4 +729,14 @@ public String getEnvironmentDataSecureDataKmsCmkRegion(Regions region) { public EnvironmentData getEnvironmentData() { return getDecryptedEnvironmentData(); } + + public void setAuditLoggingEnabled(boolean auditLoggingEnabled) { + EnvironmentData environmentData = getDecryptedEnvironmentData(); + environmentData.setAuditLoggingEnabled(auditLoggingEnabled); + saveEnvironmentData(environmentData); + } + + public boolean isAuditLoggingEnabled() { + return getEnvironmentData().isAuditLoggingEnabled(); + } } diff --git a/src/main/resources/cloudformation/audit.yaml b/src/main/resources/cloudformation/audit.yaml new file mode 100644 index 00000000..acbeac36 --- /dev/null +++ b/src/main/resources/cloudformation/audit.yaml @@ -0,0 +1,126 @@ +AWSTemplateFormatVersion: '2010-09-09' +Description: Creates the S3 config bucket for the audit logs +Outputs: + auditBucketName: + Value: !Ref 'CerberusAuditBucket' +Parameters: + cmsIamRoleArn: + Description: The ARN for for the management service IAM Role. + Type: String + accountAdminArn: + Description: The ARN for a IAM user, group or role that can create this stack. + Type: String + environmentName: + Description: The Cerberus environment name. + Type: String +Resources: + CerberusAuditBucket: + Properties: + AccessControl: Private + BucketEncryption: + ServerSideEncryptionConfiguration: + - ServerSideEncryptionByDefault: + KMSMasterKeyID: !Ref 'AuditLogS3Cmk' + SSEAlgorithm: 'aws:kms' + Type: AWS::S3::Bucket + CerberusConfigBucketAccessPolicy: + Type: AWS::S3::BucketPolicy + Properties: + Bucket: !Ref 'CerberusAuditBucket' + PolicyDocument: + Statement: + - Action: + - s3:* + Effect: Allow + Principal: + AWS: + - !Ref 'cmsIamRoleArn' + Resource: + - !Join ['', ['arn:aws:s3:::', !Ref 'CerberusAuditBucket']] + - !Join ['', ['arn:aws:s3:::', !Ref 'CerberusAuditBucket', /*]] + Sid: Allow-Bucket-Access-For-CMS + - Action: + - s3:* + Effect: Allow + Principal: + AWS: + - !Ref 'accountAdminArn' + Resource: + - !Join ['', ['arn:aws:s3:::', !Ref 'CerberusAuditBucket']] + - !Join ['', ['arn:aws:s3:::', !Ref 'CerberusAuditBucket', /*]] + Sid: Allow-Bucket-Access-For-AuditLogAthenaIamRole + - Action: + - s3:* + Effect: Allow + Principal: + AWS: + - !GetAtt 'AuditLogAthenaIamRole.Arn' + Resource: + - !Join ['', ['arn:aws:s3:::', !Ref 'CerberusAuditBucket']] + - !Join ['', ['arn:aws:s3:::', !Ref 'CerberusAuditBucket', /*]] + Sid: Allow-Bucket-Access-For-AuditLogAthenaIamRole + Version: '2012-10-17' + AuditLogAthenaIamRole: + Type: AWS::IAM::Role + Properties: + AssumeRolePolicyDocument: + Statement: + - Action: + - sts:AssumeRole + Effect: Allow + Principal: + Service: + - glue.amazonaws.com + - ec2.amazonaws.com + - lambda.amazonaws.com + Version: '2012-10-17' + Path: / + ManagedPolicyArns: + - arn:aws:iam::aws:policy/AmazonAthenaFullAccess + - arn:aws:iam::aws:policy/service-role/AWSGlueServiceRole + AuditLogS3Cmk: + Type: AWS::KMS::Key + Properties: + Description: !Join ['', ['Environment: ', !Ref 'environmentName', ' CMK for S3 to use to enable server-side of audit log data.']] + Enabled: 'true' + EnableKeyRotation: 'true' + KeyPolicy: + Statement: + - Sid: Allow-Root-User + Action: + - kms:* + Effect: Allow + Principal: + AWS: + - !Join ['', ['arn:aws:iam::', !Ref 'AWS::AccountId', ':root']] + Resource: '*' + - Sid: Allow-All-From-Cms-Instances + Action: + - kms:* + Effect: Allow + Principal: + AWS: + - !Ref 'cmsIamRoleArn' + Resource: '*' + - Sid: Allow-Account-Admin + Action: + - kms:* + Effect: Allow + Principal: + AWS: + !Ref 'accountAdminArn' + Resource: '*' + - Sid: Allow-AuditLogAthenaIamRole + Action: + - kms:* + Effect: Allow + Principal: + AWS: + !GetAtt 'AuditLogAthenaIamRole.Arn' + Resource: '*' + Version: '2012-10-17' + Tags: + - Key: created_by + Value: cerberus_cli + - Key: created_for + Value: cerberus_cms diff --git a/src/main/resources/com/nike/cerberus/operation/audit/create_audit_table.ddl b/src/main/resources/com/nike/cerberus/operation/audit/create_audit_table.ddl new file mode 100644 index 00000000..d5e5a359 --- /dev/null +++ b/src/main/resources/com/nike/cerberus/operation/audit/create_audit_table.ddl @@ -0,0 +1,24 @@ +CREATE EXTERNAL TABLE IF NOT EXISTS @@TABLE_NAME@@ ( + event_timestamp TIMESTAMP, + principal_name string, + principal_type string, + principal_token_created TIMESTAMP, + principal_token_expires TIMESTAMP, + principal_is_admin string, + ip_address string, + x_forwarded_for string, + client_version string, + http_method string, + path string, + action string, + was_success string, + name string, + sdb_name_slug string, + originating_class string, + trace_id string +) PARTITIONED BY (year INT, month INT, day INT, hour INT) +ROW FORMAT serde 'org.apache.hive.hcatalog.data.JsonSerDe' +with serdeproperties ( + "ignore.malformed.json"="true" +) +LOCATION 's3://@@BUCKET_NAME@@/audit-logs/partitioned/'; diff --git a/src/test/java/com/nike/cerberus/cli/EnvironmentConfigToArgsMapperTest.java b/src/test/java/com/nike/cerberus/cli/EnvironmentConfigToArgsMapperTest.java index f0aa3b0c..1e6d8743 100644 --- a/src/test/java/com/nike/cerberus/cli/EnvironmentConfigToArgsMapperTest.java +++ b/src/test/java/com/nike/cerberus/cli/EnvironmentConfigToArgsMapperTest.java @@ -20,6 +20,7 @@ import com.fasterxml.jackson.databind.PropertyNamingStrategy; import com.fasterxml.jackson.dataformat.yaml.YAMLFactory; import com.nike.cerberus.command.StackDelegate; +import com.nike.cerberus.command.audit.CreateAuditLoggingStackCommand; import com.nike.cerberus.command.cms.CreateCmsClusterCommand; import com.nike.cerberus.command.cms.CreateCmsConfigCommand; import com.nike.cerberus.command.core.InitializeEnvironmentCommand; @@ -186,6 +187,23 @@ public void test_vpc_access_whitelist() { assertArgsAreEqual(expected, actual, commandName); } + @Test + public void test_audit() { + String commandName = CreateAuditLoggingStackCommand.COMMAND_NAME; + + String[] userInput = {"-f", "/path/to/environment.yaml", commandName}; + + String[] expected = { + "-f", "/path/to/environment.yaml", + commandName, + CreateAuditLoggingStackCommand.ADMIN_ROLE_ARN_LONG_ARG, "arn:aws:iam::111111111:role/admin" + }; + + String[] actual = EnvironmentConfigToArgsMapper.getArgs(environmentConfig, userInput); + + assertArgsAreEqual(expected, actual, commandName); + } + @Test public void test_create_cms_config() { String commandName = CreateCmsConfigCommand.COMMAND_NAME; diff --git a/src/test/resources/environment.yaml b/src/test/resources/environment.yaml index f68fd295..30927e56 100644 --- a/src/test/resources/environment.yaml +++ b/src/test/resources/environment.yaml @@ -103,6 +103,12 @@ vpc-access-whitelist: cidrs: - 50.39.106.150/32 +# Enable audit logs +# This can be turned on latter with the enable-audit-logging-for-existing-environment command +# If enabled the CLI will create the Audit stack (S3 Bucket and IAM role) for CMS to upload logs to +# The cli will also create an Athena table for querying the data and configure CMS to send audit logs to the S3 bucket +enable-audit-logs: true + # Global Cerberus Management Service config management-service: # Group that has admin privileges in CMS.