diff --git a/.gitignore b/.gitignore index a205fd53d2..d3c5ae7f08 100644 --- a/.gitignore +++ b/.gitignore @@ -19,3 +19,4 @@ node_modules package-lock.json cdk.out .env +**/*.dylib diff --git a/validation_testing/main.py b/validation_testing/main.py index 68c6504e1d..bf03f13670 100644 --- a/validation_testing/main.py +++ b/validation_testing/main.py @@ -25,6 +25,16 @@ def assert_required_env_vars_set(): if not all([os.environ.get(env_var) for env_var in required_env_vars]): raise RuntimeError("not all expected environment variables were set!") +def pre_test_infra_check(): + # Responsible for making sure that all stacks from previous runs are cleaned + # This is needed as sometimes we have resources that are stuck around after tests + # That makes the stack deletion impossible, such as ENIs. + # This will ensure that everything is cleaned up before bringing up a new stack + for connector in TESTABLE_CONNECTORS: + shell_command = f'sh run_cleanup_infra.sh {connector}' + subprocess.run(shell_command, shell=True, check=True) + if __name__ == '__main__': assert_required_env_vars_set() + pre_test_infra_check() run_all_connector_release_tests() diff --git a/validation_testing/run_cleanup_infra.sh b/validation_testing/run_cleanup_infra.sh new file mode 100644 index 0000000000..6d025a29fe --- /dev/null +++ b/validation_testing/run_cleanup_infra.sh @@ -0,0 +1,75 @@ +CONNECTOR_NAME=$1 +VALIDATION_TESTING_ROOT=$REPOSITORY_ROOT/validation_testing + +# go to cdk dir, build/synth/deploy +cd $(dirname $(find . -name ATHENA_INFRA_SPINUP_ROOT))/app; + +cat < .env +DATABASE_PASSWORD=$DATABASE_PASSWORD +S3_DATA_PATH=$S3_PATH +REPOSITORY_ROOT=$REPOSITORY_ROOT +EOF + +# cd back to validation root +cd $VALIDATION_TESTING_ROOT + +### There are few objects that need to be cleaned up manually before attempting to destroy the stack: +# 1- ENIs (Affects Postgresql Stack mostly) +# 2- Redhisft Spill bucket can't be deleted because its not empty (Redshift) +# 3- Redshift Parameter Group is required to be cleaned up manually as well + +#### STEP 1: +# Glue attaches ENIs to the VPC subnet, if there is one, in order to connect to the VPC, and then will issue a delete of them later, but if they aren't finished deleting by the time we invoke cdk destroy, +# CFN will not know about the resources and not be able to delete the stack. So, we proactively look for and delete ENIs attached to glue ourselves first +echo "STEP 1: CLEANING UP ANY ENIs" +aws ec2 describe-subnets --filters "Name=tag:Name,Values=${CONNECTOR_NAME}CdkStack*" | jq ".Subnets[].SubnetId" | tr -d '"' \ +| xargs -n 1 -I {} aws ec2 describe-network-interfaces --filters "Name=subnet-id,Values={}" | jq ".NetworkInterfaces[] | select(.Description | startswith(\"Attached to Glue\")) | .NetworkInterfaceId" | tr -d '"' \ +| xargs -I {} aws ec2 delete-network-interface --network-interface-id {} + + +#### STEP 2: +# Cleanup any stack specific S3 bucket: + +# List and filter buckets +echo "STEP 2: CLEANING UP ANY S3 RESDUE BUCKETS" +buckets=$(aws s3 ls | grep ${CONNECTOR_NAME}cdkstack | awk '{print $3}') + +# Loop through each bucket +for bucket in $buckets; do + echo "Emptying bucket $bucket" + aws s3 rm s3://$bucket --recursive --quiet + + echo "Deleting bucket $bucket" + aws s3 rb s3://$bucket +done + +#### STEP3: +# Cleanup Redshift Parameter group +# #todo: this is results in latest changes of redshift construct +# We need to move back to L2 contruct, once its fixed +echo "STEP 3: CLEANING UP ANY REDSHIFT PARAMETER GROUP" +if [ "${CONNECTOR_NAME}" = "redshift" ]; then + aws redshift delete-cluster-parameter-group -- delete-cluster-parameter-group caseinsensitiveparametername +fi + +# once that is done, we can delete our CDK stack if it exists. +cd $(dirname $(find . -name ATHENA_INFRA_SPINUP_ROOT))/app; +prefix="${CONNECTOR_NAME}CdkStack" + +npm install; +npm run build; +npm run cdk synth; +yes | npm run cdk destroy ${CONNECTOR_NAME}CdkStack; + +# Check if the stack is clean up -- Otherwise fail so that On-Call can manually clean up the stack. +stacks=$(aws cloudformation list-stacks --stack-status-filter CREATE_FAILED ROLLBACK_FAILED DELETE_IN_PROGRESS DELETE_FAILED | jq -r --arg prefix "$prefix" '.StackSummaries[] | select(.StackName | startswith($prefix)) | .StackName') +if [ -z "$stacks" ]; then + echo "FINISHED CLEANING UP RESOURCES FOR ${CONNECTOR_NAME}." + # cd back to validation root + cd $VALIDATION_TESTING_ROOT + exit 0 +else + echo "FINISHED CLEANING UP RESOURCES FOR ${CONNECTOR_NAME}." + echo "PLEASE CLEAN UP RESOURCES FOR THE STACK MANUALLY." + exit 1 +fi