Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

chore: enable ctrl-c only after changeset is created and executed #5112

Merged
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Original file line number Diff line number Diff line change
Expand Up @@ -33,6 +33,7 @@ type Double struct {
ErrorEventsFn func(stackName string) ([]cfn.StackEvent, error)
ListStacksWithTagsFn func(tags map[string]string) ([]cfn.StackDescription, error)
DescribeStackEventsFn func(input *sdk.DescribeStackEventsInput) (*sdk.DescribeStackEventsOutput, error)
CancelUpdateStackFn func(stackName string) error
}

// Create calls the stubbed function.
Expand Down Expand Up @@ -139,3 +140,8 @@ func (d *Double) ListStacksWithTags(tags map[string]string) ([]cfn.StackDescript
func (d *Double) DescribeStackEvents(input *sdk.DescribeStackEventsInput) (*sdk.DescribeStackEventsOutput, error) {
return d.DescribeStackEventsFn(input)
}

// CancelUpdateStack calls the stubbed function.
func (d *Double) CancelUpdateStack(stackName string) error {
return d.CancelUpdateStackFn(stackName)
}
7 changes: 4 additions & 3 deletions internal/pkg/cli/env_delete_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -6,9 +6,10 @@ package cli
import (
"errors"
"fmt"
"github.com/aws/copilot-cli/internal/pkg/deploy/cloudformation"
"testing"

"github.com/aws/copilot-cli/internal/pkg/deploy/cloudformation"

"github.com/aws/aws-sdk-go/aws"
"github.com/aws/aws-sdk-go/service/resourcegroupstaggingapi"
"github.com/aws/copilot-cli/internal/pkg/cli/mocks"
Expand Down Expand Up @@ -353,7 +354,7 @@ Resources:
store := mocks.NewMockenvironmentStore(ctrl)
store.EXPECT().ListEnvironments("phonetool").Return([]*config.Environment{
&mockEnv,
&config.Environment{
{
KollaAdithya marked this conversation as resolved.
Show resolved Hide resolved
Name: "prod",
Region: "us-west-2",
AccountID: "5678",
Expand Down Expand Up @@ -431,7 +432,7 @@ Resources:
store := mocks.NewMockenvironmentStore(ctrl)
store.EXPECT().ListEnvironments("phonetool").Return([]*config.Environment{
&mockEnv,
&config.Environment{
{
Name: "prod",
Region: "us-west-2",
AccountID: "5678",
Expand Down
10 changes: 10 additions & 0 deletions internal/pkg/cli/job_deploy.go
Original file line number Diff line number Diff line change
Expand Up @@ -16,6 +16,7 @@ import (
"github.com/aws/copilot-cli/internal/pkg/version"
"github.com/spf13/afero"

deploycfn "github.com/aws/copilot-cli/internal/pkg/deploy/cloudformation"
"github.com/aws/copilot-cli/internal/pkg/deploy/cloudformation/stack"
"github.com/aws/copilot-cli/internal/pkg/exec"
"github.com/aws/copilot-cli/internal/pkg/term/log"
Expand Down Expand Up @@ -236,6 +237,8 @@ func (o *deployJobOpts) Execute() error {
return nil
}
}
var errStackDeletedOnInterrupt *deploycfn.ErrStackDeletedOnInterrupt
var errStackUpdateCanceledOnInterrupt *deploycfn.ErrStackUpdateCanceledOnInterrupt
if _, err = deployer.DeployWorkload(&deploy.DeployWorkloadInput{
StackRuntimeConfiguration: deploy.StackRuntimeConfiguration{
ImageDigests: uploadOut.ImageDigests,
Expand All @@ -250,6 +253,13 @@ func (o *deployJobOpts) Execute() error {
DisableRollback: o.disableRollback,
},
}); err != nil {
if errors.As(err, &errStackDeletedOnInterrupt) {
return nil
}
if errors.As(err, &errStackUpdateCanceledOnInterrupt) {
log.Successf("Successfully rolled back service %s to the previous configuration.\n", color.HighlightUserInput(o.name))
return nil
}
if o.disableRollback {
stackName := stack.NameForWorkload(o.targetApp.Name, o.targetEnv.Name, o.name)
rollbackCmd := fmt.Sprintf("aws cloudformation rollback-stack --stack-name %s --role-arn %s", stackName, o.targetEnv.ExecutionRoleARN)
Expand Down
14 changes: 13 additions & 1 deletion internal/pkg/cli/svc_deploy.go
Original file line number Diff line number Diff line change
Expand Up @@ -17,6 +17,7 @@ import (
"github.com/aws/copilot-cli/internal/pkg/aws/cloudformation"
"github.com/aws/copilot-cli/internal/pkg/aws/identity"
"github.com/aws/copilot-cli/internal/pkg/aws/tags"
deploycfn "github.com/aws/copilot-cli/internal/pkg/deploy/cloudformation"
"github.com/aws/copilot-cli/internal/pkg/deploy/cloudformation/stack"
"github.com/aws/copilot-cli/internal/pkg/manifest/manifestinfo"
"github.com/aws/copilot-cli/internal/pkg/template"
Expand Down Expand Up @@ -291,6 +292,8 @@ func (o *deploySvcOpts) Execute() error {
return nil
}
}
var errStackDeletedOnInterrupt *deploycfn.ErrStackDeletedOnInterrupt
var errStackUpdateCanceledOnInterrupt *deploycfn.ErrStackUpdateCanceledOnInterrupt
deployRecs, err := deployer.DeployWorkload(&clideploy.DeployWorkloadInput{
StackRuntimeConfiguration: clideploy.StackRuntimeConfiguration{
ImageDigests: uploadOut.ImageDigests,
Expand All @@ -308,6 +311,15 @@ func (o *deploySvcOpts) Execute() error {
},
})
if err != nil {
if errors.As(err, &errStackDeletedOnInterrupt) {
o.noDeploy = true
dannyrandall marked this conversation as resolved.
Show resolved Hide resolved
return nil
}
if errors.As(err, &errStackUpdateCanceledOnInterrupt) {
log.Successf("Successfully rolled back service %s to the previous configuration.\n", color.HighlightUserInput(o.name))
o.noDeploy = true
return nil
}
if o.disableRollback {
stackName := stack.NameForWorkload(o.targetApp.Name, o.targetEnv.Name, o.name)
rollbackCmd := fmt.Sprintf("aws cloudformation rollback-stack --stack-name %s --role-arn %s", stackName, o.targetEnv.ExecutionRoleARN)
Expand All @@ -321,8 +333,8 @@ After fixing the deployment, you can:
}
return fmt.Errorf("deploy service %s to environment %s: %w", o.name, o.envName, err)
}
o.deployRecs = deployRecs
log.Successf("Deployed service %s.\n", color.HighlightUserInput(o.name))
o.deployRecs = deployRecs
return nil
}

Expand Down
170 changes: 166 additions & 4 deletions internal/pkg/deploy/cloudformation/cloudformation.go
Original file line number Diff line number Diff line change
Expand Up @@ -8,13 +8,16 @@ import (
"context"
"errors"
"fmt"
"github.com/aws/copilot-cli/internal/pkg/aws/ecr"
"github.com/aws/copilot-cli/internal/pkg/deploy/cloudformation/stack"
"io"
"os"
"os/signal"
"strings"
"syscall"
"time"

"github.com/aws/copilot-cli/internal/pkg/aws/ecr"
"github.com/aws/copilot-cli/internal/pkg/deploy/cloudformation/stack"

"github.com/aws/aws-sdk-go/aws"
"github.com/aws/aws-sdk-go/aws/session"
sdkcloudformation "github.com/aws/aws-sdk-go/service/cloudformation"
Expand Down Expand Up @@ -117,6 +120,7 @@ type cfnClient interface {
Outputs(stack *cloudformation.Stack) (map[string]string, error)
StackResources(name string) ([]*cloudformation.StackResource, error)
Metadata(opts cloudformation.MetadataOpts) (string, error)
CancelUpdateStack(stackName string) error

// Methods vended by the aws sdk struct.
DescribeStackEvents(*sdkcloudformation.DescribeStackEventsInput) (*sdkcloudformation.DescribeStackEventsOutput, error)
Expand Down Expand Up @@ -198,6 +202,7 @@ type CloudFormation struct {
// Overridden in tests.
renderStackSet func(input renderStackSetInput) error
dnsDelegatedAccountsForStack func(stack *sdkcloudformation.Stack) []string
notifySignals func() chan os.Signal
}

// New returns a configured CloudFormation client.
Expand Down Expand Up @@ -233,6 +238,7 @@ func New(sess *session.Session, opts ...OptFn) CloudFormation {
}
client.renderStackSet = client.renderStackSetImpl
client.dnsDelegatedAccountsForStack = stack.DNSDelegatedAccountsForStack
client.notifySignals = notifySignals
return client
}

Expand Down Expand Up @@ -269,6 +275,15 @@ type executeAndRenderChangeSetInput struct {
stackName string
stackDescription string
createChangeSet func() (string, error)
enableInterrupt bool
}

type executeAndRenderChangeSetOption func(in *executeAndRenderChangeSetInput)

func withEnableInterrupt() executeAndRenderChangeSetOption {
return func(in *executeAndRenderChangeSetInput) {
in.enableInterrupt = true
}
}

func (cf CloudFormation) newCreateChangeSetInput(w progress.FileWriter, stack *cloudformation.Stack) *executeAndRenderChangeSetInput {
Expand All @@ -293,7 +308,7 @@ func (cf CloudFormation) newCreateChangeSetInput(w progress.FileWriter, stack *c
return in
}

func (cf CloudFormation) newUpsertChangeSetInput(w progress.FileWriter, stack *cloudformation.Stack) *executeAndRenderChangeSetInput {
func (cf CloudFormation) newUpsertChangeSetInput(w progress.FileWriter, stack *cloudformation.Stack, opts ...executeAndRenderChangeSetOption) *executeAndRenderChangeSetInput {
in := &executeAndRenderChangeSetInput{
stackName: stack.Name,
stackDescription: fmt.Sprintf("Creating the infrastructure for stack %s", stack.Name),
Expand Down Expand Up @@ -331,6 +346,9 @@ func (cf CloudFormation) newUpsertChangeSetInput(w progress.FileWriter, stack *c
spinner.Stop(log.Ssuccessf("%s\n", label))
return changeSetID, nil
}
for _, opt := range opts {
opt(in)
}
return in
}

Expand All @@ -339,10 +357,35 @@ func (cf CloudFormation) executeAndRenderChangeSet(in *executeAndRenderChangeSet
if err != nil {
return err
}
var sigChannel chan os.Signal
if in.enableInterrupt {
sigChannel = cf.notifySignals()
}
g, ctx := errgroup.WithContext(context.Background())
ctx, cancel := context.WithCancel(ctx)
defer cancel()
g.Go(func() error {
defer cancel()
if err := cf.renderChangeSet(ctx, changeSetID, in); err != nil {
if !errors.Is(err, context.Canceled) {
return err
}
}
return nil
})
if in.enableInterrupt {
g.Go(func() error {
return cf.waitForSignalAndHandleInterrupt(ctx, cancel, sigChannel, in.stackName)
})
}
return g.Wait()
}

func (cf CloudFormation) renderChangeSet(ctx context.Context, changeSetID string, in *executeAndRenderChangeSetInput) error {
if _, ok := cf.console.(*discardFile); ok { // If we don't have to render skip the additional network calls.
return nil
}
waitCtx, cancelWait := context.WithTimeout(context.Background(), waitForStackTimeout)
waitCtx, cancelWait := context.WithTimeout(ctx, waitForStackTimeout)
defer cancelWait()
g, ctx := errgroup.WithContext(waitCtx)

Expand All @@ -363,6 +406,125 @@ func (cf CloudFormation) executeAndRenderChangeSet(in *executeAndRenderChangeSet
return nil
}

func (cf CloudFormation) waitForSignalAndHandleInterrupt(ctx context.Context, cancelFn context.CancelFunc, sigCh chan os.Signal, stackName string) error {
for {
select {
case <-sigCh:
cancelFn()
stopCatchSignals(sigCh)
stackDescr, err := cf.cfnClient.Describe(stackName)
if err != nil {
return fmt.Errorf("describe stack %s: %w", stackName, err)
}
switch aws.StringValue(stackDescr.StackStatus) {
case sdkcloudformation.StackStatusCreateInProgress:
log.Infof(`Received Interrupt for Ctrl-C.
Pressing Ctrl-C again will exit immediately but the deletion of stack %s will continue
`, stackName)
description := fmt.Sprintf("Delete stack %s", stackName)
if err := cf.deleteAndRenderStack(stackName, description, func() error {
return cf.cfnClient.DeleteAndWait(stackName)
}); err != nil {
return err
}
return &ErrStackDeletedOnInterrupt{stackName: stackName}
case sdkcloudformation.StackStatusUpdateInProgress:
log.Infof(`Received Interrupt for Ctrl-C.
Pressing Ctrl-C again will exit immediately but stack %s rollback will continue
`, stackName)
description := fmt.Sprintf("Canceling stack update %s", stackName)
if err := cf.cancelUpdateAndRender(&cancelUpdateAndRenderInput{
stackName: stackName,
description: description,
cancelUpdateFn: func() error {
return cf.cfnClient.CancelUpdateStack(stackName)
},
}); err != nil {
return err
}
return &ErrStackUpdateCanceledOnInterrupt{stackName: stackName}
}
return nil
case <-ctx.Done():
stopCatchSignals(sigCh)
return nil
}
}
}

type cancelUpdateAndRenderInput struct {
stackName string
description string
cancelUpdateFn func() error
}

func (cf CloudFormation) cancelUpdateAndRender(in *cancelUpdateAndRenderInput) error {
stackDescr, err := cf.cfnClient.Describe(in.stackName)
if err != nil {
return fmt.Errorf("describe stack %s: %w", in.stackName, err)
}
if stackDescr.ChangeSetId == nil {
return fmt.Errorf("ChangeSetID not found for stack %s", in.stackName)

}
ctx, cancel := context.WithTimeout(context.Background(), waitForStackTimeout)
defer cancel()
g, ctx := errgroup.WithContext(ctx)
renderer, err := cf.createChangeSetRenderer(g, ctx, aws.StringValue(stackDescr.ChangeSetId), in.stackName, in.description, progress.RenderOptions{})
if err != nil {
return err
}
KollaAdithya marked this conversation as resolved.
Show resolved Hide resolved
g.Go(in.cancelUpdateFn)
g.Go(func() error {
_, err := progress.Render(ctx, progress.NewTabbedFileWriter(cf.console), renderer)
return err
})
if err := g.Wait(); err != nil {
return err
}
return cf.errOnFailedCancelUpdate(in.stackName)
}
func (cf CloudFormation) errOnFailedCancelUpdate(stackName string) error {
stack, err := cf.cfnClient.Describe(stackName)
if err != nil {
return fmt.Errorf("describe stack %s: %w", stackName, err)
}
status := aws.StringValue(stack.StackStatus)
if status != sdkcloudformation.StackStatusUpdateRollbackComplete {
return fmt.Errorf("stack %s did not rollback successfully and exited with status %s", stackName, status)
}
return nil
}

func notifySignals() chan os.Signal {
sigCh := make(chan os.Signal, 1)
signal.Notify(sigCh, syscall.SIGINT)
return sigCh
}

func stopCatchSignals(sigCh chan os.Signal) {
signal.Stop(sigCh)
close(sigCh)
}

// ErrStackDeletedOnInterrupt means stack is deleted on interrupt.
type ErrStackDeletedOnInterrupt struct {
stackName string
}

func (e *ErrStackDeletedOnInterrupt) Error() string {
return fmt.Sprintf("stack %s was deleted on interrupt signal", e.stackName)
}

// ErrStackUpdateCanceledOnInterrupt means stack update is canceled on interrupt.
type ErrStackUpdateCanceledOnInterrupt struct {
stackName string
}

func (e *ErrStackUpdateCanceledOnInterrupt) Error() string {
return fmt.Sprintf("update for stack %s was canceled on interrupt signal", e.stackName)
}

func (cf CloudFormation) createChangeSetRenderer(group *errgroup.Group, ctx context.Context, changeSetID, stackName, description string, opts progress.RenderOptions) (progress.DynamicRenderer, error) {
changeSet, err := cf.cfnClient.DescribeChangeSet(changeSetID, stackName)
if err != nil {
Expand Down
Loading
Loading