Refactored and added cache

README.md

@@ -55,7 +55,8 @@ folders up to `$HOME`
 
 `migration = "./migration"` refers to the directory where the migration files are located, this path is relative to the
 config file. </br>
-`history` is where you can configure where to store the history, currently its only `storage "s3"` that's supported.</br>
+`history` is where you can configure where to store the history, currently its only `storage "s3"` that's
+supported.</br>
 `state` is where you configure where your state is located, also here its only `state "s3"` that's supported.
 
 A nice feature in the config file is the support for variables using the common `${variable_name}` syntax, How to
@@ -106,8 +107,8 @@ optional.
 The `migrate` blocks have 2 supported types: `move` and `remove`.</br>
 On `move` you need to specify a `from` and `to` block, while on `remove` you can provide the `state` and `resource`
 variable. </br>
-In both cases `state` refers to the location within the s3_bucket. `resource` refers to the resource type + name in
-the state file.
+In both cases `state` refers to the location within the s3_bucket. `resource` refers to the resource type + name in the
+state file.
 
 ### Integrate with terragrunt:
 
@@ -143,6 +144,7 @@ before_hook "run_migrations" {
 }
 ```
 
-Notice the `--cv`(config-variables) flag, here we specify the `ACCOUNT_ID` and the `ASSUME_ROLE` variable that we use in the config file.</br>
-Also notice the `prod` sub-command, this is telling tgmigrate to only apply migrations for the prod environment. </br> 
+Notice the `--cv`(config-variables) flag, here we specify the `ACCOUNT_ID` and the `ASSUME_ROLE` variable that we use in
+the config file.</br>
+Also notice the `prod` sub-command, this is telling tgmigrate to only apply migrations for the prod environment. </br>
 This means that the migration file above would be applied in this case, because it has `prod` in the environments list.

command/apply.go

@@ -1,15 +1,19 @@
 package command
 
 import (
+	"github.com/aleksanderaleksic/tgmigrate/common"
+	"github.com/aleksanderaleksic/tgmigrate/config"
+	"github.com/aleksanderaleksic/tgmigrate/history"
 	"github.com/aleksanderaleksic/tgmigrate/migration"
+	"github.com/aleksanderaleksic/tgmigrate/state"
 	"github.com/urfave/cli/v2"
 )
 
 type ApplyCommand struct {
 	Runner *migration.Runner
 }
 
-func (command ApplyCommand) GetCLICommand() *cli.Command {
+func (command *ApplyCommand) GetCLICommand() *cli.Command {
 	cmd := cli.Command{
 		Name:         "apply",
 		Aliases:      nil,
@@ -19,15 +23,18 @@ func (command ApplyCommand) GetCLICommand() *cli.Command {
 		ArgsUsage:    "",
 		Category:     "",
 		BashComplete: nil,
-		Before:       nil,
-		After: func(context *cli.Context) error {
-			command.Runner.Cleanup()
-			return nil
+		Before:       command.initialize,
+		After:        nil,
+		Action:       command.runAll,
+		OnUsageError: nil,
+		Subcommands:  nil,
+		Flags: []cli.Flag{
+			&cli.BoolFlag{
+				Name:    "dryrun",
+				Aliases: []string{"d"},
+				Usage:   "Dont do any permanent changes, like actually uploading the state changes",
+			},
 		},
-		Action:                 command.runAll,
-		OnUsageError:           nil,
-		Subcommands:            nil,
-		Flags:                  nil,
 		SkipFlagParsing:        false,
 		HideHelp:               false,
 		HideHelpCommand:        false,
@@ -39,10 +46,52 @@ func (command ApplyCommand) GetCLICommand() *cli.Command {
 	return &cmd
 }
 
-func (command ApplyCommand) runAll(c *cli.Context) error {
+func (command *ApplyCommand) runAll(c *cli.Context) error {
 	environment := c.Args().First()
 	if environment == "" {
 		return command.Runner.Apply(nil)
 	}
 	return command.Runner.Apply(&environment)
 }
+
+func (command *ApplyCommand) initialize(c *cli.Context) error {
+	cfg, err := config.GetConfigFile(c)
+	if err != nil {
+		return err
+	}
+
+	chc := common.Cache{
+		ConfigFilePath: cfg.Path,
+	}
+
+	ctx := common.Context{
+		SkipUserInteraction: c.Bool("y"),
+		DryRun:              c.Bool("d"),
+	}
+
+	migrationFiles, err := migration.GetMigrationFiles(*cfg)
+	if err != nil {
+		return err
+	}
+
+	stateInterface, err := state.GetStateInterface(*cfg, ctx, chc)
+	if err != nil {
+		return err
+	}
+
+	historyInterface, err := history.GetHistoryInterface(*cfg, ctx, chc)
+	if err != nil {
+		return err
+	}
+
+	runner := migration.Runner{
+		Context:          &ctx,
+		HistoryInterface: historyInterface,
+		StateInterface:   stateInterface,
+		MigrationFiles:   *migrationFiles,
+	}
+
+	command.Runner = &runner
+
+	return nil
+}

common/cache.go

@@ -0,0 +1,17 @@
+package common
+
+import (
+	"path/filepath"
+)
+
+type Cache struct {
+	ConfigFilePath string
+}
+
+func (c Cache) GetCacheDirectoryPath() string {
+	return filepath.Join(filepath.Dir(c.ConfigFilePath), ".tgmigrate_cache")
+}
+
+func (c Cache) filepath(name string) string {
+	return filepath.Join(c.GetCacheDirectoryPath(), name)
+}

config/config.go

@@ -9,6 +9,7 @@ import (
 	"io/ioutil"
 	"os"
 	"path/filepath"
+	"strings"
 )
 
 var defaultConfigFile = ".tgmigrate.hcl"
@@ -30,14 +31,16 @@ type File struct {
 }
 
 type Config struct {
+	Path                 string
 	MigrationDir         string
 	AbsoluteMigrationDir string
 	History              History
 	State                State
 }
 
-func GetConfigFile(ctx *cli.Context, configVariables map[string]cty.Value) (*Config, error) {
-	confFilePath := getConfigFilePathFromFlags(ctx)
+func GetConfigFile(ctx *cli.Context) (*Config, error) {
+	configVariables := getConfigVariables(ctx)
+	confFilePath := GetConfigFilePathFromFlags(ctx)
 	source, err := ioutil.ReadFile(confFilePath)
 	if err != nil {
 		return nil, fmt.Errorf("unable to find %s in current or parrent directories, a config file is required", defaultConfigFile)
@@ -56,11 +59,11 @@ func GetConfigFile(ctx *cli.Context, configVariables map[string]cty.Value) (*Con
 	return conf, nil
 }
 
-func parseConfigFile(filename string, source []byte, ctx *hcl.EvalContext) (*Config, error) {
+func parseConfigFile(filePath string, source []byte, ctx *hcl.EvalContext) (*Config, error) {
 	var f File
-	err := hclsimple.Decode(filename, source, nil, &f)
+	err := hclsimple.Decode(filePath, source, nil, &f)
 	if err != nil {
-		return nil, fmt.Errorf("failed to decode config file: %s, err: %s", filename, err)
+		return nil, fmt.Errorf("failed to decode config file: %s, err: %s", filePath, err)
 	}
 
 	historyStorageConfig, err := getHistoryStorageConfig(f, ctx)
@@ -73,12 +76,13 @@ func parseConfigFile(filename string, source []byte, ctx *hcl.EvalContext) (*Con
 		return nil, err
 	}
 
-	path, err := getAbsoluteMigrationDirPath(filename, f.Migration.MigrationDir)
+	path, err := getAbsoluteMigrationDirPath(filePath, f.Migration.MigrationDir)
 	if err != nil {
 		return nil, err
 	}
 
 	cfg := Config{
+		Path:                 filePath,
 		MigrationDir:         f.Migration.MigrationDir,
 		AbsoluteMigrationDir: *path,
 		History: History{
@@ -129,7 +133,7 @@ func getAbsoluteMigrationDirPath(configFilePath string, migrationDir string) (*s
 	return &migrationDirectory, nil
 }
 
-func getConfigFilePathFromFlags(c *cli.Context) string {
+func GetConfigFilePathFromFlags(c *cli.Context) string {
 	configFlagValue := c.String("config")
 
 	if configFlagValue != "" {
@@ -173,3 +177,25 @@ func findFirstConfigFileInParentFolders() (*string, error) {
 
 	return &defaultConfigFile, nil
 }
+
+func getConfigVariables(c *cli.Context) map[string]cty.Value {
+	configVariablesFlag := c.String("cv")
+
+	if configVariablesFlag == "" {
+		return map[string]cty.Value{}
+	}
+
+	rawKeyValue := strings.Split(configVariablesFlag, ";")
+
+	var keyValue = map[string]cty.Value{}
+
+	for _, raw := range rawKeyValue {
+		if raw == "" {
+			break
+		}
+		split := strings.Split(raw, "=")
+		keyValue[split[0]] = cty.StringVal(split[1])
+	}
+
+	return keyValue
+}

config/config_history_storage_s3.go

@@ -3,15 +3,13 @@ package config
 import (
 	"github.com/hashicorp/hcl/v2"
 	"github.com/hashicorp/hcl/v2/gohcl"
-	"os"
 )
 
 type S3HistoryStorageConfig struct {
-	Bucket      string  `hcl:"bucket"`
-	Region      string  `hcl:"region"`
-	Key         string  `hcl:"key"`
-	AssumeRole  *string `hcl:"assume_role,optional"`
-	historyPath *string
+	Bucket     string  `hcl:"bucket"`
+	Region     string  `hcl:"region"`
+	Key        string  `hcl:"key"`
+	AssumeRole *string `hcl:"assume_role,optional"`
 }
 
 func ParseHistoryS3StorageConfig(configFile File, ctx *hcl.EvalContext) (*S3HistoryStorageConfig, error) {
@@ -24,13 +22,3 @@ func ParseHistoryS3StorageConfig(configFile File, ctx *hcl.EvalContext) (*S3Hist
 
 	return &config, nil
 }
-
-func (s *S3HistoryStorageConfig) GetLocalHistoryPath() string {
-	if s.historyPath != nil {
-		return *s.historyPath
-	}
-	dirName := s.Bucket + "_history"
-	s.historyPath = &dirName
-	_ = os.Mkdir(dirName, 0777)
-	return *s.historyPath
-}

config/config_state.go

@@ -8,7 +8,5 @@ type State struct {
 }
 
 type StateConfig interface {
-	GetStateDirectory() string
-	GetBackupStateDirectory() string
 	GetStateFileName() string
 }

config/config_state_s3.go

@@ -3,16 +3,13 @@ package config
 import (
 	"github.com/hashicorp/hcl/v2"
 	"github.com/hashicorp/hcl/v2/gohcl"
-	"os"
 )
 
 type S3StateConfig struct {
-	Bucket               string  `hcl:"bucket"`
-	Region               string  `hcl:"region"`
-	StateFileName        *string `hcl:"state_file_name,optional"`
-	AssumeRole           *string `hcl:"assume_role,optional"`
-	stateDirectory       *string
-	backupStateDirectory *string
+	Bucket        string  `hcl:"bucket"`
+	Region        string  `hcl:"region"`
+	StateFileName *string `hcl:"state_file_name,optional"`
+	AssumeRole    *string `hcl:"assume_role,optional"`
 }
 
 func ParseS3StateConfig(configFile File, ctx *hcl.EvalContext) (*S3StateConfig, error) {
@@ -26,20 +23,6 @@ func ParseS3StateConfig(configFile File, ctx *hcl.EvalContext) (*S3StateConfig,
 	return &config, nil
 }
 
-func (s *S3StateConfig) GetStateDirectory() string {
-	if s.stateDirectory != nil {
-		return *s.stateDirectory
-	}
-	dirName := s.Bucket + "_state"
-	s.stateDirectory = &dirName
-	_ = os.Mkdir(dirName, 0777)
-	return *s.stateDirectory
-}
-
-func (s *S3StateConfig) GetBackupStateDirectory() string {
-	return s.GetStateDirectory()
-}
-
 func (s S3StateConfig) GetStateFileName() string {
 	if s.StateFileName != nil {
 		return *s.StateFileName

example/.tgmigrate.hcl

@@ -3,16 +3,16 @@ migration {
 
   history {
     storage "s3" {
-      bucket = "airthings-terraform-states-${ACCOUNT}"
+      bucket = "airthings-terraform-states-${ACCOUNT_ID}"
       region = "us-east-1"
-      assume_role = "arn:aws:iam::${ACCOUNT}:role/admin"
+      assume_role = "arn:aws:iam::${ACCOUNT_ID}:role/admin"
       key = "history.json"
     }
   }
 
   state "s3" {
-    bucket = "airthings-terraform-states-${ACCOUNT}"
+    bucket = "airthings-terraform-states-${ACCOUNT_ID}"
     region = "us-east-1"
-    assume_role = "arn:aws:iam::${ACCOUNT}:role/admin"
+    assume_role = "arn:aws:iam::${ACCOUNT_ID}:role/admin"
   }
 }

history/history.go

@@ -21,7 +21,7 @@ type History interface {
 	Cleanup()
 }
 
-func GetHistoryInterface(c config.Config, ctx common.Context) (History, error) {
+func GetHistoryInterface(c config.Config, ctx common.Context, cache common.Cache) (History, error) {
 	switch c.History.Storage.Type {
 	case "s3":
 		conf := c.History.Storage.Config.(*config.S3HistoryStorageConfig)
@@ -48,13 +48,19 @@ func GetHistoryInterface(c config.Config, ctx common.Context) (History, error) {
 			context:         ctx,
 			S3StorageConfig: *conf,
 			session:         *sess,
+			Cache:           cache,
 		}, nil
 	default:
 		return nil, fmt.Errorf("unknown history storage type: %s", c.History.Storage.Type)
 	}
 }
 
 func writeStorageHistory(path string, storageHistory StorageHistory) error {
+	err := os.MkdirAll(filepath.Dir(path), os.ModePerm)
+	if err != nil {
+		return err
+	}
+
 	encodedStorageHistory, err := EncodeStorageHistory(storageHistory)
 	if err != nil {
 		return err