Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Address ghost instance #69

Open
wants to merge 10 commits into
base: main
Choose a base branch
from
121 changes: 121 additions & 0 deletions src/aws/dependencies/detectIncompleteState.sh
Original file line number Diff line number Diff line change
@@ -0,0 +1,121 @@
#!/bin/bash

# Function to describe an EC2 instance by its Name tag
# Accepts one argument: instanceName
describeInstanceByName() {
local instanceName=$1

# Check if instanceName was provided
if [ -z "$instanceName" ]; then
echo "Error: Please provide an instance name."
return 1
fi

# Run the AWS CLI command with the provided instanceName and capture the result in JSON format
local result=$(aws ec2 describe-instances \
--filters "Name=tag:Name,Values=$instanceName" \
--query 'Reservations[*].Instances[*].{Instance:InstanceId,State:State.Name,PublicIP:PublicIpAddress}' \
--output json)

# Check if result is empty, meaning no instances found
if [ -z "$result" ] || [ "$result" = "[]" ]; then
echo "No instances found with Name tag: $instanceName"
return 1
fi

# Use jq to parse and display InstanceId, State, and PublicIP
echo "$result" | jq -r '.[] | .[] | "\nInstanceId: \(.Instance)\nState: \(.State)\nPublicIP: \(.PublicIP // "N/A")\n"'
return 0
}

# Function to check if a security group exists by its name
# Accepts one argument: groupName
checkSecurityGroupExists() {
local groupName=$1

# Describe security groups filtered by group name
local result=$(aws ec2 describe-security-groups --group-names "$groupName" 2>&1)

# Check if the result contains an error (security group not found)
if echo "$result" | grep -q "InvalidGroup.NotFound"; then
echo "Security group '$groupName' does not exist."
return 1
else
echo "Security group '$groupName' exists."
return 0
fi
}

# Function to check if a key pair exists by its name
# Accepts one argument: keyName
checkKeyPairExists() {
local keyName=$1

# Describe key pairs filtered by key name
local result=$(aws ec2 describe-key-pairs --key-name "$keyName" 2>&1)

# Check if the result contains an error (key pair not found)
if echo "$result" | grep -q "InvalidKeyPair.NotFound"; then
echo "Key pair '$keyName' does not exist."
return 1
else
echo "Key pair '$keyName' exists."
return 0
fi
}

instanceExists() {
local binaryState=$1
! ((binaryState & 4))
}

securityGroupExists() {
local binaryState=$1
! ((binaryState & 2))
}

keyPairExists() {
local binaryState=$1
! ((binaryState & 1))
}

# Parent function to detect the state of the instance, security group, and key pair
# Accepts three arguments:
# $1: instanceName (for describeInstanceByName)
# $2: groupName (for checkSecurityGroupExists)
# $3: keyName (for checkKeyPairExists)
# Returns a binary value representing the state:
# - Bit 2 (100): Instance exists
# - Bit 1 (010): Security group exists
# - Bit 0 (001): Key pair exists
detectIncompleteState() {
local instanceName=$1
local groupName=$2
local keyName=$3

# Initialize existence status variables (0 = exists, 1 = doesn't exist)
local keyPairExists=0
local securityGroupExists=0
local instanceExists=0

# Check if the instance exists
describeInstanceByName "$instanceName"
instanceExists=$? # 0 if exists, 1 if doesn't exist

# Check if the security group exists
checkSecurityGroupExists "$groupName"
securityGroupExists=$? # 0 if exists, 1 if doesn't exist

# Check if the key pair exists
checkKeyPairExists "$keyName"
keyPairExists=$? # 0 if exists, 1 if doesn't exist

# Construct a binary value based on the resource existence statuses
# Bit 2 -> instanceExists (shift left by 2 positions)
# Bit 1 -> securityGroupExists (shift left by 1 position)
# Bit 0 -> keyPairExists
local binaryState=$(((instanceExists << 2) | (securityGroupExists << 1) | keyPairExists))

#echo "Binary state (in binary): $(echo "obase=2; $binaryState" | bc)"
return $binaryState
}
130 changes: 102 additions & 28 deletions src/aws/down.sh
Original file line number Diff line number Diff line change
Expand Up @@ -3,58 +3,132 @@
#BASE=$HOME
BASE=/home/pi

function down(){
# Fetch the instance ID based on balloonName, then terminate the EC2 instance if found.
# If the instance does not exist, the function exits early.
fetchInstanceIdAndTerminateEc2() {
local balloonName=$1

balloonName=$(setBalloonName "$1")
instanceId=$(getValueByAttribute "$balloonName" instanceId)

if ! isBalloonNameValid "$balloonName"; then
echo "Please provide a valid balloon name"
exit 1
if [ "$instanceId" = "null" ]; then
echo "$balloonName does not exist"
exit 0
fi

instanceId=$(getValueByAttribute $balloonName instanceId)
echo $instanceId
aws ec2 terminate-instances --instance-ids $instanceId
echo "EC2 instance terminated"
}

if [ "$instanceId" = "null" ]; then
echo "$balloonName does not exist"
exit 0
# Store TCP/UDP port information for a group, then attempt to delete the security group.
# If a dependency prevents deletion, it prints a dependency violation message.
storePortAndDeleteSecurityGroup() {
local groupName=$1
local balloonName=$2

storePortArrayString "$groupName" tcp "$balloonName"
storePortArrayString "$groupName" udp "$balloonName"

echo "$groupName"
output=$(aws ec2 delete-security-group --group-name "$groupName" 2>&1)

if [[ $? -eq 0 ]]; then
echo "Security group '$groupName' successfully deleted."
elif [[ $output == *"DependencyViolation"* ]]; then
echo "Dependency violation. Security group could not be deleted."
else
echo "An error occurred: $output"
fi
}

keyName=$(getValueByAttribute $balloonName key)
groupName=$(getValueByAttribute $balloonName groupName)
# Delete an EC2 key pair by its name.
deleteEc2KeyPair() {
local keyName=$1

echo "$keyName"
aws ec2 delete-key-pair --key-name "$keyName"
echo "EC2 key pair deleted"
}

storePortArrayString $groupName tcp $balloonName
storePortArrayString $groupName udp $balloonName
updateSshtunnelConfig $balloonName
# Store TCP/UDP port information for a group, sleep for a given duration, and then attempt
# to delete the security group. If a dependency violation occurs, it retries after sleeping.
storePortAndDeleteSecurityGroupWithSleepAndRetry() {
local groupName=$1
local balloonName=$2
local sleepDuration=$3 # Third argument for sleep duration

echo $instanceId
aws ec2 terminate-instances --instance-ids $instanceId
echo "ec2 instance delete"
storePortArrayString "$groupName" tcp "$balloonName"
storePortArrayString "$groupName" udp "$balloonName"

echo $keyName
aws ec2 delete-key-pair --key-name $keyName
echo "security key delete"
echo "Sleeping for $sleepDuration seconds before attempting to delete security group..."
sleep "$sleepDuration"

treehouses sshtunnel remove all
echo "remove all sshtunnel"
echo "$groupName"

sleep 30
echo $groupName
while true; do
output=$(aws ec2 delete-security-group --group-name "$groupName" 2>&1)

if [[ $? -eq 0 ]]; then
echo "Security group '$groupName' successfully deleted."
break
elif [[ $output == *"DependencyViolation"* ]]; then
echo "Dependency violation. Retrying in 5 seconds..."
sleep 10
echo "Dependency violation. Retrying in $sleepDuration seconds..."
sleep "$sleepDuration"
else
echo "An error occurred: $output"
break
fi
done
}

treehousesConfigHas() {
local keyName=$1
local groupName=$2

if [ "$keyName" == "null" ] || [ "$groupName" == "null" ]; then
return 1
fi

return 0
}

function down() {

balloonName=$(setBalloonName "$1")
keyName=$(getValueByAttribute "$balloonName" key)
groupName=$(getValueByAttribute "$balloonName" groupName)

treehousesConfigHas "$keyName" "$groupName"
checkResult=$?

if [ $checkResult -eq 1 ]; then
echo "Treehouses config is corrupted, manual cleanup required"
exit 1
fi

detectIncompleteState "$balloonName" "$groupName" "$keyName"
binaryState=$?

if instanceExists "$binaryState" && securityGroupExists "$binaryState"; then
fetchInstanceIdAndTerminateEc2 "$balloonName"
storePortAndDeleteSecurityGroupWithSleepAndRetry "$groupName" "$balloonName" 30
else
if instanceExists $binaryState; then
fetchInstanceIdAndTerminateEc2 "$balloonName"
fi
if securityGroupExists $binaryState; then
storePortAndDeleteSecurityGroup "$groupName" "$balloonName"
fi
fi
if keyPairExists $binaryState; then
deleteEc2KeyPair "$keyName"
fi

updateSshtunnelConfig "$balloonName"
treehouses sshtunnel remove all
echo "remove all sshtunnel"

deleteSshConfig $balloonName
deleteObsoleteKeyValue $balloonName
deleteSshConfig "$balloonName"
deleteObsoleteKeyValue "$balloonName"

}
1 change: 1 addition & 0 deletions src/aws/load.sh
Original file line number Diff line number Diff line change
Expand Up @@ -12,6 +12,7 @@ source $manageConfigPath/src/aws/dependencies/sshtunnelFunction.sh
source $manageConfigPath/src/aws/dependencies/reverseShell.sh
source $manageConfigPath/src/aws/dependencies/updateOrAppend.sh
source $manageConfigPath/src/aws/dependencies/getProcessNumber.sh
source $manageConfigPath/src/aws/dependencies/detectIncompleteState.sh

source $manageConfigPath/src/utils/dependencies/config.sh
source $manageConfigPath/src/utils/dependencies/array.sh
Expand Down
34 changes: 18 additions & 16 deletions src/aws/up.sh
Original file line number Diff line number Diff line change
Expand Up @@ -158,6 +158,7 @@ function up {
echo "Success to add ssh key: $importedKeyName"
else
echo "The key pair $keyname already exists. Please use another key name."
importedKeyName=$keyname
fi

if ! checkSecurityGroup; then
Expand Down Expand Up @@ -196,21 +197,22 @@ function up {
echo "Wait for starting on start command until instance is stopped."
exit 1
fi

case "$instanceState" in
"running")
echo "EC2 instance is already running."
;;
"stopped")
echo "Starting stopped EC2 instance..."
start $instanceName
;;
"terminated")
createAndTagInstance
;;
*)
echo "EC2 instance is in state: $instanceState."
;;
esac
echo "Success to add ssh key: $importedKeyName"
fi

case "$instanceState" in
"running")
echo "EC2 instance is already running."
;;
"stopped")
echo "Starting stopped EC2 instance..."
start $instanceName
;;
"terminated")
createAndTagInstance
;;
*)
echo "EC2 instance is in state: $instanceState."
;;
esac
}
Loading