From 54eea4638d42d1d60391716a3a9d84aa3b539361 Mon Sep 17 00:00:00 2001 From: Shlomi Noach <2607934+shlomi-noach@users.noreply.github.com> Date: Mon, 27 Jul 2020 09:52:05 +0300 Subject: [PATCH 1/3] new command line flag: postpone-shutdown-flag-file, new server command; 'state' --- go/base/context.go | 16 ++++++++++++++++ go/cmd/gh-ost/main.go | 1 + go/logic/migrator.go | 12 ++++++++++++ go/logic/server.go | 6 ++++++ 4 files changed, 35 insertions(+) diff --git a/go/base/context.go b/go/base/context.go index 1030463e5..1d29e47ed 100644 --- a/go/base/context.go +++ b/go/base/context.go @@ -36,6 +36,14 @@ const ( type CutOver int +type MigrationState string + +const ( + MigrationStateRunning MigrationState = "Running" + MigrationStateComplete MigrationState = "Complete" + MigrationStateFailed MigrationState = "Failed" +) + const ( CutOverAtomic CutOver = iota CutOverTwoStep @@ -128,6 +136,7 @@ type MigrationContext struct { CriticalLoadIntervalMilliseconds int64 CriticalLoadHibernateSeconds int64 PostponeCutOverFlagFile string + PostponeShutdownFlagFile string CutOverLockTimeoutSeconds int64 CutOverExponentialBackoff bool ExponentialBackoffMaxInterval int64 @@ -728,6 +737,13 @@ func (this *MigrationContext) AddThrottleControlReplicaKey(key mysql.InstanceKey return nil } +func (this *MigrationContext) GetMigrationState() MigrationState { + if atomic.LoadInt64(&this.CutOverCompleteFlag) > 0 { + return MigrationStateComplete + } + return MigrationStateRunning +} + // ApplyCredentials sorts out the credentials between the config file and the CLI flags func (this *MigrationContext) ApplyCredentials() { this.configMutex.Lock() diff --git a/go/cmd/gh-ost/main.go b/go/cmd/gh-ost/main.go index ee0c98686..1b05c78e1 100644 --- a/go/cmd/gh-ost/main.go +++ b/go/cmd/gh-ost/main.go @@ -111,6 +111,7 @@ func main() { flag.StringVar(&migrationContext.ThrottleFlagFile, "throttle-flag-file", "", "operation pauses when this file exists; hint: use a file that is specific to the table being altered") flag.StringVar(&migrationContext.ThrottleAdditionalFlagFile, "throttle-additional-flag-file", "/tmp/gh-ost.throttle", "operation pauses when this file exists; hint: keep default, use for throttling multiple gh-ost operations") flag.StringVar(&migrationContext.PostponeCutOverFlagFile, "postpone-cut-over-flag-file", "", "while this file exists, migration will postpone the final stage of swapping tables, and will keep on syncing the ghost table. Cut-over/swapping would be ready to perform the moment the file is deleted.") + flag.StringVar(&migrationContext.PostponeShutdownFlagFile, "postpone-shutdown-flag-file", "", "while this file exists gh-ost will not terminate. This does not postpone cut-over. Use case: be able to advertise that migration is complete.") flag.StringVar(&migrationContext.PanicFlagFile, "panic-flag-file", "", "when this file is created, gh-ost will immediately terminate, without cleanup") flag.BoolVar(&migrationContext.DropServeSocket, "initially-drop-socket-file", false, "Should gh-ost forcibly delete an existing socket file. Be careful: this might drop the socket file of a running migration!") diff --git a/go/logic/migrator.go b/go/logic/migrator.go index bb5407a8a..f83286212 100644 --- a/go/logic/migrator.go +++ b/go/logic/migrator.go @@ -419,6 +419,18 @@ func (this *Migrator) Migrate() (err error) { if err := this.hooksExecutor.onSuccess(); err != nil { return err } + this.sleepWhileTrue( + func() (bool, error) { + if this.migrationContext.PostponeShutdownFlagFile == "" { + return false, nil + } + if base.FileExists(this.migrationContext.PostponeShutdownFlagFile) { + // Postpone file defined and exists! + return true, nil + } + return false, nil + }, + ) this.migrationContext.Log.Infof("Done migrating %s.%s", sql.EscapeName(this.migrationContext.DatabaseName), sql.EscapeName(this.migrationContext.OriginalTableName)) return nil } diff --git a/go/logic/server.go b/go/logic/server.go index 16068844c..585cf78b6 100644 --- a/go/logic/server.go +++ b/go/logic/server.go @@ -146,6 +146,7 @@ func (this *Server) applyServerCommand(command string, writer *bufio.Writer) (pr fmt.Fprint(writer, `available commands: status # Print a detailed status message sup # Print a short status message +state # Print Running|Complete coordinates # Print the currently inspected coordinates chunk-size= # Set a new chunk-size dml-batch-size= # Set a new dml-batch-size @@ -169,6 +170,11 @@ help # This message return ForcePrintStatusOnlyRule, nil case "info", "status": return ForcePrintStatusAndHintRule, nil + case "state": + { + fmt.Fprintf(writer, "%+v\n", this.migrationContext.GetMigrationState()) + return NoPrintStatusRule, nil + } case "coordinates": { if argIsQuestion || arg == "" { From da5d6d77ad390f383be0a621084539e37df61360 Mon Sep 17 00:00:00 2001 From: Shlomi Noach <2607934+shlomi-noach@users.noreply.github.com> Date: Mon, 27 Jul 2020 10:01:48 +0300 Subject: [PATCH 2/3] document postpone-shutdown-flag-file command line flag --- doc/command-line-flags.md | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/doc/command-line-flags.md b/doc/command-line-flags.md index 629e9f958..1b04fba31 100644 --- a/doc/command-line-flags.md +++ b/doc/command-line-flags.md @@ -169,6 +169,10 @@ Indicate a file name, such that the final [cut-over](cut-over.md) step does not When this flag is set, `gh-ost` expects the file to exist on startup, or else tries to create it. `gh-ost` exits with error if the file does not exist and `gh-ost` is unable to create it. With this flag set, the migration will cut-over upon deletion of the file or upon `cut-over` [interactive command](interactive-commands.md). +### postpone-shutdown-flag-file + +If indicated file exists, and assuming execution is successful, `gh-ost` will wait after cut-over, and after running the success hooks, instead of exiting. This can be useful in state based environments where we may wish to ask `gh-ost` wat the state of the migration is, as opposed to assuming it reports the state via hooks. This gives a controller/operator the chance to communicate to `gh-ost` and confirm that the migration is successful. See also the `state` server command. + ### replica-server-id Defaults to 99999. If you run multiple migrations then you must provide a different, unique `--replica-server-id` for each `gh-ost` process. From 596a6b10e87d50a02adfd93b38deb5b0354cfa72 Mon Sep 17 00:00:00 2001 From: Shlomi Noach <2607934+shlomi-noach@users.noreply.github.com> Date: Mon, 27 Jul 2020 10:21:52 +0300 Subject: [PATCH 3/3] documenting 'state' server command --- doc/interactive-commands.md | 1 + 1 file changed, 1 insertion(+) diff --git a/doc/interactive-commands.md b/doc/interactive-commands.md index 591aa4921..170a5506f 100644 --- a/doc/interactive-commands.md +++ b/doc/interactive-commands.md @@ -17,6 +17,7 @@ Both interfaces may serve at the same time. Both respond to simple text command, - `help`: shows a brief list of available commands - `status`: returns a detailed status summary of migration progress and configuration - `sup`: returns a brief status summary of migration progress +- `state`: returns `Running` or `Complete`. For the latter, see `--postpone-shutdown-flag-file` - `coordinates`: returns recent (though not exactly up to date) binary log coordinates of the inspected server - `chunk-size=`: modify the `chunk-size`; applies on next running copy-iteration - `dml-batch-size=`: modify the `dml-batch-size`; applies on next applying of binary log events