From 2bb36c6c3b8034e78d0ae9f0fab36b0b4bcc3622 Mon Sep 17 00:00:00 2001 From: Justin Field Date: Thu, 16 Mar 2017 15:00:27 -0700 Subject: [PATCH] Feature/restore complete (#21) Add restore complete command. --- gradle.properties | 2 +- gradle/dependencies.gradle | 2 +- .../com/nike/cerberus/cli/CerberusRunner.java | 8 +- .../cerberus/client/CerberusAdminClient.java | 89 +++++ ...mpleteCerberusDataFromS3BackupCommand.java | 88 +++++ ...leteCerberusDataFromS3BackupOperation.java | 347 ++++++++++++++++++ .../nike/cerberus/service/ConsoleService.java | 48 +++ .../nike/cerberus/service/S3StoreService.java | 48 ++- .../nike/cerberus/service/StoreService.java | 4 +- .../vault/VaultAdminClientFactory.java | 2 +- .../cerberus/service/S3StoreServiceTest.java | 8 +- 11 files changed, 622 insertions(+), 24 deletions(-) create mode 100644 src/main/java/com/nike/cerberus/client/CerberusAdminClient.java create mode 100644 src/main/java/com/nike/cerberus/command/core/RestoreCompleteCerberusDataFromS3BackupCommand.java create mode 100644 src/main/java/com/nike/cerberus/operation/core/RestoreCompleteCerberusDataFromS3BackupOperation.java create mode 100644 src/main/java/com/nike/cerberus/service/ConsoleService.java diff --git a/gradle.properties b/gradle.properties index b779d343..ea5122b1 100644 --- a/gradle.properties +++ b/gradle.properties @@ -16,4 +16,4 @@ group=com.nike artifactId=cerberus-lifecycle-cli -version=0.14.2 +version=0.15.0 diff --git a/gradle/dependencies.gradle b/gradle/dependencies.gradle index 43988419..3e4c11fe 100644 --- a/gradle/dependencies.gradle +++ b/gradle/dependencies.gradle @@ -34,7 +34,7 @@ allprojects { compile group: 'com.amazonaws', name: 'aws-java-sdk-sns', version: awsSDKVersion compile group: 'com.amazonaws', name: 'aws-java-sdk-lambda', version: awsSDKVersion - compile 'com.nike:vault-client:1.0.0' + compile 'com.nike:vault-client:1.2.1' compile 'com.squareup.okhttp3:okhttp:3.3.1' compile 'com.beust:jcommander:1.55' compile 'com.fasterxml.jackson.core:jackson-core:2.7.+' diff --git a/src/main/java/com/nike/cerberus/cli/CerberusRunner.java b/src/main/java/com/nike/cerberus/cli/CerberusRunner.java index 05efd48c..c7906314 100644 --- a/src/main/java/com/nike/cerberus/cli/CerberusRunner.java +++ b/src/main/java/com/nike/cerberus/cli/CerberusRunner.java @@ -37,6 +37,7 @@ import com.nike.cerberus.command.consul.CreateVaultAclCommand; import com.nike.cerberus.command.core.CreateBaseCommand; import com.nike.cerberus.command.core.PrintStackInfoCommand; +import com.nike.cerberus.command.core.RestoreCompleteCerberusDataFromS3BackupCommand; import com.nike.cerberus.command.core.UpdateStackCommand; import com.nike.cerberus.command.core.UploadCertFilesCommand; import com.nike.cerberus.command.core.WhitelistCidrForVpcAccessCommand; @@ -138,7 +139,11 @@ public void run(String[] args) { if (StringUtils.isNotBlank(commandName)) { commander.usage(commandName); } else { - printCustomUsage(); + if (StringUtils.isNotBlank(commandName)) { + commander.usage(commandName); + } else { + printCustomUsage(); + } } } } @@ -285,6 +290,7 @@ private void registerAllCommands() { registerCommand(new CreateCloudFrontLogProcessingLambdaConfigCommand()); registerCommand(new CreateCloudFrontSecurityGroupUpdaterLambdaCommand()); registerCommand(new WhitelistCidrForVpcAccessCommand()); + registerCommand(new RestoreCompleteCerberusDataFromS3BackupCommand()); } /** diff --git a/src/main/java/com/nike/cerberus/client/CerberusAdminClient.java b/src/main/java/com/nike/cerberus/client/CerberusAdminClient.java new file mode 100644 index 00000000..29d36c3f --- /dev/null +++ b/src/main/java/com/nike/cerberus/client/CerberusAdminClient.java @@ -0,0 +1,89 @@ +/* + * Copyright (c) 2017 Nike, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.nike.cerberus.client; + +import com.nike.vault.client.UrlResolver; +import com.nike.vault.client.VaultAdminClient; +import com.nike.vault.client.VaultClientException; +import com.nike.vault.client.auth.VaultCredentialsProvider; +import com.nike.vault.client.http.HttpHeader; +import com.nike.vault.client.http.HttpMethod; +import okhttp3.HttpUrl; +import okhttp3.OkHttpClient; +import okhttp3.Request; +import okhttp3.RequestBody; +import okhttp3.Response; + +import javax.net.ssl.SSLException; +import java.io.IOException; + +/** + * A Cerberus admin client with the ability to restore metadata + */ +public class CerberusAdminClient extends VaultAdminClient { + + protected OkHttpClient httpClient; + protected VaultCredentialsProvider credentialsProvider; + + /** + * Explicit constructor that allows for full control over construction of the Vault client. + * + * @param vaultUrlResolver URL resolver for Vault + * @param credentialsProvider Credential provider for acquiring a token for interacting with Vault + * @param httpClient HTTP client for calling Vault + */ + public CerberusAdminClient(UrlResolver vaultUrlResolver, + VaultCredentialsProvider credentialsProvider, + OkHttpClient httpClient) { + + super(vaultUrlResolver, credentialsProvider, httpClient); + this.httpClient = httpClient; + this.credentialsProvider = credentialsProvider; + } + + public void restoreMetadata(String jsonPayload) { + HttpUrl url = buildUrl("v1/", "metadata"); + Response response = execute(url, HttpMethod.PUT, jsonPayload); + if (! response.isSuccessful()) { + throw new RuntimeException("Failed to restore metadata with cms body: " + response.message()); + } + } + + protected Response execute(final HttpUrl url, final String method, final String json) { + try { + Request.Builder requestBuilder = new Request.Builder() + .url(url) + .addHeader(HttpHeader.VAULT_TOKEN, credentialsProvider.getCredentials().getToken()) + .addHeader(HttpHeader.ACCEPT, DEFAULT_MEDIA_TYPE.toString()); + + requestBuilder.addHeader(HttpHeader.CONTENT_TYPE, DEFAULT_MEDIA_TYPE.toString()) + .method(method, RequestBody.create(DEFAULT_MEDIA_TYPE, json)); + + return httpClient.newCall(requestBuilder.build()).execute(); + } catch (IOException e) { + if (e instanceof SSLException + && e.getMessage() != null + && e.getMessage().contains("Unrecognized SSL message, plaintext connection?")) { + // AnyConnect web security proxy can be disabled with: + // `sudo /opt/cisco/anyconnect/bin/acwebsecagent -disablesvc -websecurity` + throw new VaultClientException("I/O error while communicating with vault. Unrecognized SSL message may be due to a web proxy e.g. AnyConnect", e); + } else { + throw new VaultClientException("I/O error while communicating with vault.", e); + } + } + } +} diff --git a/src/main/java/com/nike/cerberus/command/core/RestoreCompleteCerberusDataFromS3BackupCommand.java b/src/main/java/com/nike/cerberus/command/core/RestoreCompleteCerberusDataFromS3BackupCommand.java new file mode 100644 index 00000000..21bcdb8a --- /dev/null +++ b/src/main/java/com/nike/cerberus/command/core/RestoreCompleteCerberusDataFromS3BackupCommand.java @@ -0,0 +1,88 @@ +/* + * Copyright (c) 2017 Nike, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.nike.cerberus.command.core; + +import com.beust.jcommander.Parameter; +import com.beust.jcommander.Parameters; +import com.nike.cerberus.command.Command; +import com.nike.cerberus.operation.Operation; +import com.nike.cerberus.operation.core.RestoreCompleteCerberusDataFromS3BackupOperation; + +import static com.nike.cerberus.command.core.WhitelistCidrForVpcAccessCommand.COMMAND_NAME; + +/** + * Command for restoring Safe Deposit Box Metadata and Vault secret data for SDBs from backups that are in S3 from + * the cross region backup lambda. + */ +@Parameters( + commandNames = COMMAND_NAME, + commandDescription = "Allows Cerberus operators to restore a complete backup from S3 that was created using the cross region backup lambda." +) +public class RestoreCompleteCerberusDataFromS3BackupCommand implements Command { + + public static final String COMMAND_NAME = "restore-complete"; + + @Parameter(names = "-s3-region", + description = "The region for the bucket that contains the backups", + required = true + ) + private String s3Region; + + @Parameter(names = "-s3-bucket", + description = "The bucket that contains the backups", + required = true + ) + private String s3Bucket; + + @Parameter(names = "-s3-prefix", + description = "the folder that contains the json backup files", + required = true + ) + private String s3Prefix; + + @Parameter(names = "-url", + description = "The cerberus api, to restore to", + required = true + ) + private String cerberusUrl; + + public String getS3Region() { + return s3Region; + } + + public String getS3Bucket() { + return s3Bucket; + } + + public String getS3Prefix() { + return s3Prefix; + } + + public String getCerberusUrl() { + return cerberusUrl; + } + + @Override + public String getCommandName() { + return COMMAND_NAME; + } + + @Override + public Class> getOperationClass() { + return RestoreCompleteCerberusDataFromS3BackupOperation.class; + } +} diff --git a/src/main/java/com/nike/cerberus/operation/core/RestoreCompleteCerberusDataFromS3BackupOperation.java b/src/main/java/com/nike/cerberus/operation/core/RestoreCompleteCerberusDataFromS3BackupOperation.java new file mode 100644 index 00000000..61da7c4a --- /dev/null +++ b/src/main/java/com/nike/cerberus/operation/core/RestoreCompleteCerberusDataFromS3BackupOperation.java @@ -0,0 +1,347 @@ +/* + * Copyright (c) 2017 Nike, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.nike.cerberus.operation.core; + +import com.amazonaws.auth.DefaultAWSCredentialsProviderChain; +import com.amazonaws.regions.Region; +import com.amazonaws.regions.Regions; +import com.amazonaws.services.s3.AmazonS3; +import com.amazonaws.services.s3.AmazonS3Client; +import com.amazonaws.services.s3.AmazonS3EncryptionClient; +import com.amazonaws.services.s3.model.CryptoConfiguration; +import com.amazonaws.services.s3.model.KMSEncryptionMaterialsProvider; +import com.fasterxml.jackson.core.type.TypeReference; +import com.fasterxml.jackson.databind.JsonNode; +import com.fasterxml.jackson.databind.ObjectMapper; +import com.github.tomaslanger.chalk.Chalk; +import com.google.inject.Inject; +import com.nike.cerberus.client.CerberusAdminClient; +import com.nike.cerberus.command.core.RestoreCompleteCerberusDataFromS3BackupCommand; +import com.nike.cerberus.module.CerberusModule; +import com.nike.cerberus.operation.Operation; +import com.nike.cerberus.service.ConsoleService; +import com.nike.cerberus.service.S3StoreService; +import com.nike.cerberus.vault.VaultAdminClientFactory; +import com.nike.vault.client.StaticVaultUrlResolver; +import com.nike.vault.client.VaultAdminClient; +import com.nike.vault.client.VaultClientException; +import com.nike.vault.client.model.VaultAuthResponse; +import com.nike.vault.client.model.VaultListResponse; +import com.nike.vault.client.model.VaultTokenAuthRequest; +import okhttp3.OkHttpClient; +import org.apache.commons.lang3.StringUtils; +import org.apache.http.conn.ssl.NoopHostnameVerifier; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import javax.inject.Named; +import java.io.IOException; +import java.text.DecimalFormat; +import java.util.HashMap; +import java.util.HashSet; +import java.util.List; +import java.util.Map; +import java.util.Optional; +import java.util.Set; +import java.util.concurrent.TimeUnit; + +/** + * Operation for restoring Safe Deposit Box Metadata and Vault secret data for SDBs from backups that are in S3 from + * the cross region backup lambda. + */ +public class RestoreCompleteCerberusDataFromS3BackupOperation implements Operation { + + private final Logger logger = LoggerFactory.getLogger(getClass()); + + private static final int DEFAULT_TIMEOUT = 60; + private static final TimeUnit DEFAULT_TIMEOUT_UNIT = TimeUnit.SECONDS; + private static final String CERBERUS_BACKUP_METADATA_JSON_FILE_KEY = "cerberus-backup-metadata.json"; + private static final String CERBERUS_BACKUP_API_URL = "cerberusUrl"; + private static final String CERBERUS_BACKUP_DATE = "backupDate"; + private static final String CERBERUS_BACKUP_LAMBDA_ACCOUNT_ID = "lambdaBackupAccountId"; + private static final String CERBERUS_BACKUP_LAMBDA_REGION = "lambdaBackupRegion"; + private static final String CERBERUS_BACKUP_SDB_COUNT = "numberOfSdbs"; + + private final VaultAdminClientFactory vaultAdminClientFactory; + private final ObjectMapper objectMapper; + private final ConsoleService console; + + @Inject + public RestoreCompleteCerberusDataFromS3BackupOperation(VaultAdminClientFactory vaultAdminClientFactory, + @Named(CerberusModule.CONFIG_OBJECT_MAPPER) + ObjectMapper objectMapper, + ConsoleService console) { + + this.vaultAdminClientFactory = vaultAdminClientFactory; + this.objectMapper = objectMapper; + this.console = console; + } + + @Override + public void run(RestoreCompleteCerberusDataFromS3BackupCommand command) { + String backup = Chalk.on(String.format("s3://%s/%s", + command.getS3Bucket(), + command.getS3Prefix()) + ).yellow().bold().toString(); + + logger.info(Chalk.on( + String.format("Starting restore with backup located at %s", backup) + ).green().toString()); + Region region = Region.getRegion(Regions.fromName(command.getS3Region())); + + AmazonS3 s3 = new AmazonS3Client(); + s3.setRegion(region); + S3StoreService s3StoreService = new S3StoreService(s3, command.getS3Bucket(), command.getS3Prefix()); + + Set keys = s3StoreService.getKeysInPartialPath(""); + if (keys.isEmpty()) { + logger.error("There where no keys in {}/{}", command.getS3Bucket(), command.getS3Prefix()); + } + + if (! keys.contains(CERBERUS_BACKUP_METADATA_JSON_FILE_KEY)) { + throw new RuntimeException( + String.format("cerberus-backup-metadata.json was not found in s3://%s/%s/ is this a complete backup?", + command.getS3Bucket(), command.getS3Prefix())); + } + + String kmsCustomerMasterKeyId = getKmsCmkId(CERBERUS_BACKUP_METADATA_JSON_FILE_KEY, s3StoreService); + S3StoreService s3EncryptionStoreService = getS3EncryptionStoreService(kmsCustomerMasterKeyId, command); + CerberusAdminClient cerberusAdminClient = new CerberusAdminClient( + new StaticVaultUrlResolver(command.getCerberusUrl()), + new VaultAdminClientFactory.RootCredentialsProvider(generateAdminToken()), + new OkHttpClient.Builder() + .hostnameVerifier(new NoopHostnameVerifier()) + .connectTimeout(DEFAULT_TIMEOUT, DEFAULT_TIMEOUT_UNIT) + .writeTimeout(DEFAULT_TIMEOUT, DEFAULT_TIMEOUT_UNIT) + .readTimeout(DEFAULT_TIMEOUT, DEFAULT_TIMEOUT_UNIT) + .build() + ); + + validateRestore(s3EncryptionStoreService, command); + + keys.remove(CERBERUS_BACKUP_METADATA_JSON_FILE_KEY); + Double i = 0d; + DecimalFormat df = new DecimalFormat("#.##"); + for (String sdbBackupKey : keys) { + try { + Double percent = i / keys.size() * 100; + System.out.print(String.format("Restoring backups %s%% complete\r", df.format(percent))); + String json = getDecryptedJson(sdbBackupKey, s3EncryptionStoreService); + processBackup(json, cerberusAdminClient); + } catch (Throwable t) { + logger.error("Failed to process backup json for {}", Chalk.on(sdbBackupKey).red().toString(), t); + } + i++; + } + System.out.print("Restoring backups 100% complete\n"); + logger.info("Restore complete"); + } + + /** + * Use the metadata from the backup and ensure that the user wants to proceed + * @param s3StoreService - The encrypted S3 store service + */ + private void validateRestore(S3StoreService s3StoreService, RestoreCompleteCerberusDataFromS3BackupCommand command) { + String backupMetadataJsonString = getDecryptedJson(CERBERUS_BACKUP_METADATA_JSON_FILE_KEY, s3StoreService); + Map backupMetadata; + try { + backupMetadata = objectMapper.readValue(backupMetadataJsonString, new TypeReference>() {}); + } catch (IOException e) { + throw new RuntimeException("Failed to deserialize backup metadata", e); + } + + String backupApiUrl = backupMetadata.containsKey(CERBERUS_BACKUP_API_URL) ? backupMetadata.get(CERBERUS_BACKUP_API_URL) : "unknown"; + String backupDate = backupMetadata.containsKey(CERBERUS_BACKUP_DATE) ? backupMetadata.get(CERBERUS_BACKUP_DATE) : "unknown"; + String backupLambdaAccountId = backupMetadata.containsKey(CERBERUS_BACKUP_LAMBDA_ACCOUNT_ID) ? backupMetadata.get(CERBERUS_BACKUP_LAMBDA_ACCOUNT_ID) : "unknown"; + String backupLambdaRegion = backupMetadata.containsKey(CERBERUS_BACKUP_LAMBDA_REGION) ? backupMetadata.get(CERBERUS_BACKUP_LAMBDA_REGION) : "unknown"; + String backupSdbCount = backupMetadata.containsKey(CERBERUS_BACKUP_SDB_COUNT) ? backupMetadata.get(CERBERUS_BACKUP_SDB_COUNT) : "unknown"; + + StringBuilder msg = new StringBuilder() + .append("\nThe backup you are attempting to restore was created from ").append(Chalk.on(backupApiUrl).green().toString()).append(" on ").append(Chalk.on(backupDate).green().bold().toString()) + .append("\nFrom the backup lambda in account ") + .append(Chalk.on(backupLambdaAccountId).green().toString()) + .append(" in region ") + .append(Chalk.on(backupLambdaRegion).green().toString()) + .append("\nThis backup contains ") + .append(Chalk.on(backupSdbCount).green().toString()) + .append(" SDB records. ") + .append("\nYou are attempting to restore this backup to ") + .append(Chalk.on(command.getCerberusUrl()).green().toString()); + + if (! backupApiUrl.equalsIgnoreCase(command.getCerberusUrl())) { + msg.append("\n\n") + .append(Chalk.on("Warning: ").red().toString()) + .append(Chalk.on("The backup was created for ").red().toString()) + .append(Chalk.on(backupApiUrl).green().toString()) + .append(Chalk.on("\nYou are attempting to restore to ").red().toString()) + .append(Chalk.on(command.getCerberusUrl()).green().toString()) + .append(Chalk.on("\nThese urls do not match, proceed with extreme caution!\n").red().toString()); + } + + msg.append("\nType \"proceed\" to restore this backup, anything else will exit"); + + logger.info(""); + logger.info(Chalk.on("##########################################################################################").red().toString()); + logger.info(Chalk.on("# DANGER ZONE #").red().toString()); + logger.info(Chalk.on("##########################################################################################").red().toString()); + logger.info(msg.toString()); + + String proceed; + try { + proceed = console.readLine(""); + } catch (IOException e) { + throw new RuntimeException("Failed to validate that the user wanted to proceed with backup", e); + } + + if (! proceed.equalsIgnoreCase("proceed")) { + throw new RuntimeException("User did not confirm to proceed with backup restore"); + } + } + + private S3StoreService getS3EncryptionStoreService(String cmkId, + RestoreCompleteCerberusDataFromS3BackupCommand command) { + + Region region = Region.getRegion(Regions.fromName(command.getS3Region())); + KMSEncryptionMaterialsProvider materialProvider = new KMSEncryptionMaterialsProvider(cmkId); + AmazonS3EncryptionClient encryptionClient = + new AmazonS3EncryptionClient( + new DefaultAWSCredentialsProviderChain(), + materialProvider, + new CryptoConfiguration() + .withAwsKmsRegion(region)) + .withRegion(region); + + return new S3StoreService(encryptionClient, command.getS3Bucket(), command.getS3Prefix()); + } + + private String getKmsCmkId(String path, S3StoreService s3StoreService) { + Map metadata = s3StoreService.getS3ObjectUserMetaData(path); + if (! metadata.containsKey("x-amz-matdesc")) { + throw new RuntimeException("Failed to get Customer Master Key ID from object user metadata. " + + "'x-amz-matdesc' not found in metadata for object at path: " + path); + } + + String serializedEncryptionContext = metadata.get("x-amz-matdesc"); + + Map encryptionContextMap; + try { + encryptionContextMap = objectMapper.readValue(serializedEncryptionContext, + new TypeReference>() {}); + } catch (IOException e) { + throw new RuntimeException("Failed to convert encryption context metadata value into Map"); + } + return encryptionContextMap.get("kms_cmk_id"); + } + + private String getDecryptedJson(String sdbBackupKey, S3StoreService s3StoreService) { + Optional json = s3StoreService.get(sdbBackupKey); + if (!json.isPresent()) { + logger.error("Failed to get json from S3 for {}", sdbBackupKey); + } + return json.get(); + } + + /** + * Process the stored backup json + * Step 1: Restores the metadata to CMS + * Step 2: Restores the secrete data to Vault + * @param sdbBackupJson the json string from s3 + */ + private void processBackup(String sdbBackupJson, CerberusAdminClient cerberusAdminClient) throws IOException { + + JsonNode sdb = objectMapper.readTree(sdbBackupJson); + + // restore metadata to cms + cerberusAdminClient.restoreMetadata(sdbBackupJson); + deleteAllSecrets(sdb.get("path").asText(), cerberusAdminClient); + // restore secret vault data + JsonNode data = sdb.get("data"); + Map> kvPairs = objectMapper.convertValue(data, + new TypeReference>>() {}); + + kvPairs.forEach(cerberusAdminClient::write); + } + + /** + * Deletes all of the secrets from Vault stored at the safe deposit box's path. + * + * @param path path to start deleting at. + */ + private void deleteAllSecrets(final String path, VaultAdminClient vaultAdminClient) { + try { + String fixedPath = path; + + if (StringUtils.endsWith(path, "/")) { + fixedPath = StringUtils.substring(path, 0, StringUtils.lastIndexOf(path, "/")); + } + + final VaultListResponse listResponse = vaultAdminClient.list(fixedPath); + final List keys = listResponse.getKeys(); + + if (keys == null || keys.isEmpty()) { + return; + } + + for (final String key : keys) { + if (StringUtils.endsWith(key, "/")) { + final String fixedKey = StringUtils.substring(key, 0, key.lastIndexOf("/")); + deleteAllSecrets(fixedPath + "/" + fixedKey, vaultAdminClient); + } else { + vaultAdminClient.delete(fixedPath + "/" + key); + } + } + } catch (VaultClientException vce) { + throw new RuntimeException("Failed to delete secrets from Vault. for path: " + path); + } + } + + /** + * Generates and admin token that CMS will reconize as an Admin so we can us the restore endpoint + */ + private String generateAdminToken() { + VaultAuthResponse vaultAuthResponse; + try { + logger.info("Attempting to generate an admin token with the root token"); + VaultAdminClient adminClient = vaultAdminClientFactory.getClientForLeader().get(); + + Map metadata = new HashMap<>(); + metadata.put("is_admin", "true"); + metadata.put("username", "admin-cli"); + + Set policies = new HashSet<>(); + policies.add("root"); + + vaultAuthResponse = adminClient.createOrphanToken(new VaultTokenAuthRequest() + .setDisplayName("admin-cli") + .setPolicies(policies) + .setMeta(metadata) + .setTtl("10m") + .setNoDefaultPolicy(true)); + } catch (VaultClientException e) { + throw new RuntimeException("There was an error while trying to create an admin token, this command " + + "requires proxy access or direct a connect to the vault leader, is your ip white listed?", e); + } + + return vaultAuthResponse.getClientToken(); + } + + @Override + public boolean isRunnable(RestoreCompleteCerberusDataFromS3BackupCommand command) { + return true; + } + +} diff --git a/src/main/java/com/nike/cerberus/service/ConsoleService.java b/src/main/java/com/nike/cerberus/service/ConsoleService.java new file mode 100644 index 00000000..31f7bac2 --- /dev/null +++ b/src/main/java/com/nike/cerberus/service/ConsoleService.java @@ -0,0 +1,48 @@ +/* + * Copyright (c) 2017 Nike, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.nike.cerberus.service; + +import com.google.inject.Singleton; + +import java.io.BufferedReader; +import java.io.IOException; +import java.io.InputStreamReader; + +/** + * To allow us to run this code base in IDE's and have access to civilized dev tools we will have this util + * that will handle System.console returning null in dev envs, as a result in Dev modes secrets will be entered + * in plain text on the console + */ +@Singleton +public class ConsoleService { + + public String readLine(String format, Object... args) throws IOException { + if (System.console() != null) { + return System.console().readLine(format, args); + } + System.out.print(String.format(format, args)); + BufferedReader reader = new BufferedReader(new InputStreamReader(System.in, "UTF-8")); + return reader.readLine(); + } + + public char[] readPassword(String format, Object... args) throws IOException { + if (System.console() != null) { + return System.console().readPassword(format, args); + } + return readLine(format, args).toCharArray(); + } +} diff --git a/src/main/java/com/nike/cerberus/service/S3StoreService.java b/src/main/java/com/nike/cerberus/service/S3StoreService.java index 9be17efa..eed6ca0f 100644 --- a/src/main/java/com/nike/cerberus/service/S3StoreService.java +++ b/src/main/java/com/nike/cerberus/service/S3StoreService.java @@ -35,7 +35,9 @@ import java.io.UnsupportedEncodingException; import java.util.Collections; import java.util.List; +import java.util.Map; import java.util.Optional; +import java.util.Set; import java.util.stream.Collectors; /** @@ -72,13 +74,29 @@ public void put(String path, String value) { public Optional get(String path) { - GetObjectRequest request = new GetObjectRequest(s3Bucket, getFullPath(path)); + Optional s3ObjectOptional = getS3Object(path); + if (! s3ObjectOptional.isPresent()) { + return Optional.empty(); + } try { - S3Object s3Object = s3Client.getObject(request); - InputStream object = s3Object.getObjectContent(); + InputStream object = s3ObjectOptional.get().getObjectContent(); return Optional.of(IOUtils.toString(object, ConfigConstants.DEFAULT_ENCODING)); - } catch (AmazonServiceException ase) { + } catch (IOException e) { + String errorMessage = + String.format("Unable to read contents of S3 object. Bucket: %s, Key: %s, Expected Encoding: %s", + s3Bucket, getFullPath(path), ConfigConstants.DEFAULT_ENCODING); + logger.error(errorMessage); + throw new UnexpectedDataEncodingException(errorMessage, e); + } + } + + protected Optional getS3Object(String path) { + GetObjectRequest request = new GetObjectRequest(s3Bucket, getFullPath(path)); + try { + return Optional.of(s3Client.getObject(request)); + } + catch (AmazonServiceException ase) { if (StringUtils.equalsIgnoreCase(ase.getErrorCode(), "NoSuchKey")) { logger.debug(String.format("The S3 object doesn't exist. Bucket: %s, Key: %s", s3Bucket, request.getKey())); return Optional.empty(); @@ -86,24 +104,26 @@ public Optional get(String path) { logger.error("Unexpected error communicating with AWS.", ase); throw ase; } - } catch (IOException e) { - String errorMessage = - String.format("Unable to read contents of S3 object. Bucket: %s, Key: %s, Expected Encoding: %s", - s3Bucket, request.getKey(), ConfigConstants.DEFAULT_ENCODING); - logger.error(errorMessage); - throw new UnexpectedDataEncodingException(errorMessage, e); } } - public List listKeysInPartialPath(String path) { + public Map getS3ObjectUserMetaData(String path) { + Optional encryptedObjectOptional = getS3Object(path); + if (! encryptedObjectOptional.isPresent()) { + throw new RuntimeException("Cannot get metadata for an S3 Object that is not present"); + } + S3Object s3Object = encryptedObjectOptional.get(); + return s3Object.getObjectMetadata().getUserMetadata(); + } + + public Set getKeysInPartialPath(String path) { ObjectListing objectListing = s3Client.listObjects(s3Bucket, getFullPath(path)); - List s3PathKeys = objectListing + return objectListing .getObjectSummaries() .stream() .map(objectSummary -> StringUtils.stripStart(objectSummary.getKey(), s3Prefix + "/")) - .collect(Collectors.toList()); - return Collections.unmodifiableList(s3PathKeys); + .collect(Collectors.toSet()); } public void deleteAllKeysOnPartialPath(String path) { diff --git a/src/main/java/com/nike/cerberus/service/StoreService.java b/src/main/java/com/nike/cerberus/service/StoreService.java index baec4563..88314c9b 100644 --- a/src/main/java/com/nike/cerberus/service/StoreService.java +++ b/src/main/java/com/nike/cerberus/service/StoreService.java @@ -16,8 +16,8 @@ package com.nike.cerberus.service; -import java.util.List; import java.util.Optional; +import java.util.Set; /** * Interface for common operations on storage services. @@ -28,7 +28,7 @@ public interface StoreService { Optional get(String path); - List listKeysInPartialPath(String path); + Set getKeysInPartialPath(String path); void deleteAllKeysOnPartialPath(String path); } diff --git a/src/main/java/com/nike/cerberus/vault/VaultAdminClientFactory.java b/src/main/java/com/nike/cerberus/vault/VaultAdminClientFactory.java index 601e8069..82ba4426 100644 --- a/src/main/java/com/nike/cerberus/vault/VaultAdminClientFactory.java +++ b/src/main/java/com/nike/cerberus/vault/VaultAdminClientFactory.java @@ -151,7 +151,7 @@ private String toVaultUrl(final String hostname) { } - private static class RootCredentialsProvider implements VaultCredentialsProvider { + public static class RootCredentialsProvider implements VaultCredentialsProvider { private final String rootToken; diff --git a/src/test/java/com/nike/cerberus/service/S3StoreServiceTest.java b/src/test/java/com/nike/cerberus/service/S3StoreServiceTest.java index f3be9e01..b24bc045 100644 --- a/src/test/java/com/nike/cerberus/service/S3StoreServiceTest.java +++ b/src/test/java/com/nike/cerberus/service/S3StoreServiceTest.java @@ -33,8 +33,8 @@ import java.io.IOException; import java.io.InputStream; -import java.util.List; import java.util.Optional; +import java.util.Set; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertFalse; @@ -173,7 +173,7 @@ public void testGetIOException() throws IOException { } @Test - public void testListKeysInPartialPath() { + public void testGetKeysInPartialPath() { AmazonS3 client = mock(AmazonS3.class); S3StoreService service = new S3StoreService(client, S3_BUCKET, S3_PREFIX); @@ -189,10 +189,10 @@ public void testListKeysInPartialPath() { when(client.listObjects(S3_BUCKET, S3_PREFIX + "/" + path)).thenReturn(listing); // invoke method under test - List results = service.listKeysInPartialPath(path); + Set results = service.getKeysInPartialPath(path); assertEquals(1, results.size()); - assertEquals(key, results.get(0)); + assertEquals(key, results.iterator().next()); } @Test