Merge branch 'master' into master

CHANGELOG.md

@@ -1,5 +1,19 @@
 TiUP Changelog
 
+## [1.13.1] 2023-09-25
+
+### Fixes
+
+- Increase timeout when publish package in `tiup`  (#2269, @nexustar)
+- Fix pd microservice component id in `tiup-playground` (#2272, @iosmanthus)
+- Fix grafana for multiple instances using same host in `tiup-cluster` and `tiup-dm` (#2277, @lastincisor)
+- Add cdn workaround (#2285, @nexustar)
+- Mirror: fix progress bar is not accurate (#2284, @nexustar)
+
+### Improvement
+
+- Support ignore version check when upgrade in `tiup-cluster` and `tiup-dm` (#2282, @nexustar)
+
 ## [1.13.0] 2023-08-26
 
 ### New Features

components/cluster/command/upgrade.go

@@ -20,6 +20,7 @@ import (
 
 func newUpgradeCmd() *cobra.Command {
 	offlineMode := false
+	ignoreVersionCheck := false
 
 	cmd := &cobra.Command{
 		Use:   "upgrade <cluster-name> <version>",
@@ -38,7 +39,7 @@ func newUpgradeCmd() *cobra.Command {
 			teleCommand = append(teleCommand, scrubClusterName(clusterName))
 			teleCommand = append(teleCommand, version)
 
-			return cm.Upgrade(clusterName, version, gOpt, skipConfirm, offlineMode)
+			return cm.Upgrade(clusterName, version, gOpt, skipConfirm, offlineMode, ignoreVersionCheck)
 		},
 		ValidArgsFunction: func(cmd *cobra.Command, args []string, toComplete string) ([]string, cobra.ShellCompDirective) {
 			switch len(args) {
@@ -53,6 +54,7 @@ func newUpgradeCmd() *cobra.Command {
 	cmd.Flags().Uint64Var(&gOpt.APITimeout, "transfer-timeout", 600, "Timeout in seconds when transferring PD and TiKV store leaders, also for TiCDC drain one capture")
 	cmd.Flags().BoolVarP(&gOpt.IgnoreConfigCheck, "ignore-config-check", "", false, "Ignore the config check result")
 	cmd.Flags().BoolVarP(&offlineMode, "offline", "", false, "Upgrade a stopped cluster")
+	cmd.Flags().BoolVarP(&ignoreVersionCheck, "ignore-version-check", "", false, "Ignore checking if target version is bigger than current version")
 	cmd.Flags().StringVar(&gOpt.SSHCustomScripts.BeforeRestartInstance.Raw, "pre-upgrade-script", "", "(EXPERIMENTAL) Custom script to be executed on each server before the server is upgraded")
 	cmd.Flags().StringVar(&gOpt.SSHCustomScripts.AfterRestartInstance.Raw, "post-upgrade-script", "", "(EXPERIMENTAL) Custom script to be executed on each server after the server is upgraded")
 

components/dm/command/upgrade.go

@@ -19,7 +19,7 @@ import (
 
 func newUpgradeCmd() *cobra.Command {
 	offlineMode := false
-
+	ignoreVersionCheck := false
 	cmd := &cobra.Command{
 		Use:   "upgrade <cluster-name> <version>",
 		Short: "Upgrade a specified DM cluster",
@@ -28,7 +28,7 @@ func newUpgradeCmd() *cobra.Command {
 				return cmd.Help()
 			}
 
-			return cm.Upgrade(args[0], args[1], gOpt, skipConfirm, offlineMode)
+			return cm.Upgrade(args[0], args[1], gOpt, skipConfirm, offlineMode, ignoreVersionCheck)
 		},
 		ValidArgsFunction: func(cmd *cobra.Command, args []string, toComplete string) ([]string, cobra.ShellCompDirective) {
 			switch len(args) {
@@ -41,6 +41,7 @@ func newUpgradeCmd() *cobra.Command {
 	}
 
 	cmd.Flags().BoolVarP(&offlineMode, "offline", "", false, "Upgrade a stopped cluster")
+	cmd.Flags().BoolVarP(&ignoreVersionCheck, "ignore-version-check", "", false, "Ignore checking if target version is higher than current version")
 
 	return cmd
 }

components/playground/playground.go

@@ -674,7 +674,7 @@ func (p *Playground) addInstance(componentID string, pdRole instance.PDRole, tif
 	dataDir := p.dataDir
 
 	id := p.allocID(componentID)
-	dir := filepath.Join(dataDir, fmt.Sprintf("%s-%d", pdRole, id))
+	dir := filepath.Join(dataDir, fmt.Sprintf("%s-%d", componentID, id))
 	if err = utils.MkdirAll(dir, 0755); err != nil {
 		return nil, err
 	}

pkg/cluster/api/tidbapi.go

@@ -0,0 +1,78 @@
+// Copyright 2022 PingCAP, Inc.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package api
+
+import (
+	"context"
+	"crypto/tls"
+	"fmt"
+	"time"
+
+	"github.com/pingcap/tiup/pkg/utils"
+)
+
+// TiDBClient is client for access TiKVCDC Open API
+type TiDBClient struct {
+	urls   []string
+	client *utils.HTTPClient
+	ctx    context.Context
+}
+
+// NewTiDBClient return a `TiDBClient`
+func NewTiDBClient(ctx context.Context, addresses []string, timeout time.Duration, tlsConfig *tls.Config) *TiDBClient {
+	httpPrefix := "http"
+	if tlsConfig != nil {
+		httpPrefix = "https"
+	}
+	urls := make([]string, 0, len(addresses))
+	for _, addr := range addresses {
+		urls = append(urls, fmt.Sprintf("%s://%s", httpPrefix, addr))
+	}
+
+	return &TiDBClient{
+		urls:   urls,
+		client: utils.NewHTTPClient(timeout, tlsConfig),
+		ctx:    ctx,
+	}
+}
+
+func (c *TiDBClient) getEndpoints(api string) (endpoints []string) {
+	for _, url := range c.urls {
+		endpoints = append(endpoints, fmt.Sprintf("%s%s", url, api))
+	}
+	return endpoints
+}
+
+// StartUpgrade sends the start upgrade message to the TiDB server
+func (c *TiDBClient) StartUpgrade() error {
+	api := "/upgrade/start"
+	endpoints := c.getEndpoints(api)
+	_, err := tryURLs(endpoints, func(endpoint string) ([]byte, error) {
+		return c.client.Post(c.ctx, endpoint, nil)
+	})
+
+	return err
+}
+
+// FinishUpgrade sends the finish upgrade message to the TiDB server
+func (c *TiDBClient) FinishUpgrade() error {
+	api := "/upgrade/finish"
+	endpoints := c.getEndpoints(api)
+	_, err := tryURLs(endpoints, func(endpoint string) ([]byte, error) {
+		return c.client.Post(c.ctx, endpoint, nil)
+	})
+
+	return err
+}
+

pkg/cluster/manager/patch.go

@@ -104,7 +104,8 @@ func (m *Manager) Patch(name string, packagePath string, opt operator.Options, o
 			if offline {
 				return nil
 			}
-			return operator.Upgrade(ctx, topo, opt, tlsCfg, base.Version)
+			// TBD: should patch be treated as an upgrade?
+			return operator.Upgrade(ctx, topo, opt, tlsCfg, base.Version, base.Version)
 		}).
 		Build()
 

pkg/cluster/manager/reload.go

@@ -124,7 +124,7 @@ func (m *Manager) Reload(name string, gOpt operator.Options, skipRestart, skipCo
 			return err
 		}
 		b.Func("Upgrade Cluster", func(ctx context.Context) error {
-			return operator.Upgrade(ctx, topo, gOpt, tlsCfg, base.Version)
+			return operator.Upgrade(ctx, topo, gOpt, tlsCfg, base.Version, base.Version)
 		})
 	}
 

pkg/cluster/manager/upgrade.go

@@ -37,7 +37,7 @@ import (
 )
 
 // Upgrade the cluster.
-func (m *Manager) Upgrade(name string, clusterVersion string, opt operator.Options, skipConfirm, offline bool) error {
+func (m *Manager) Upgrade(name string, clusterVersion string, opt operator.Options, skipConfirm, offline, ignoreVersionCheck bool) error {
 	if err := clusterutil.ValidateClusterNameOrError(name); err != nil {
 		return err
 	}
@@ -68,7 +68,10 @@ func (m *Manager) Upgrade(name string, clusterVersion string, opt operator.Optio
 	)
 
 	if err := versionCompare(base.Version, clusterVersion); err != nil {
-		return err
+		if !ignoreVersionCheck {
+			return err
+		}
+		m.logger.Warnf(color.RedString("There is no guarantee that the cluster can be downgraded. Be careful before you continue."))
 	}
 
 	if !skipConfirm {
@@ -234,7 +237,7 @@ Do you want to continue? [y/N]:`,
 			if offline {
 				return nil
 			}
-			return operator.Upgrade(ctx, topo, opt, tlsCfg, base.Version)
+			return operator.Upgrade(ctx, topo, opt, tlsCfg, base.Version, clusterVersion)
 		}).
 		Build()
 

pkg/cluster/operation/upgrade.go

@@ -39,13 +39,14 @@ var (
 	increaseLimitPoint = checkpoint.Register()
 )
 
-// Upgrade the cluster.
+// Upgrade the cluster. (actually, it's rolling restart)
 func Upgrade(
 	ctx context.Context,
 	topo spec.Topology,
 	options Options,
 	tlsCfg *tls.Config,
 	currentVersion string,
+	targetVersion string,
 ) error {
 	roleFilter := set.NewStringSet(options.Roles...)
 	nodeFilter := set.NewStringSet(options.Nodes...)
@@ -70,6 +71,7 @@ func Upgrade(
 		var origRegionScheduleLimit int
 		var err error
 
+		var tidbClient *api.TiDBClient
 		var pdEndpoints []string
 		forcePDEndpoints := os.Getenv(EnvNamePDEndpointOverwrite) // custom set PD endpoint list
 
@@ -100,6 +102,21 @@ func Upgrade(
 					}
 				}()
 			}
+		case spec.ComponentTiDB:
+			dbs := topo.(*spec.Specification).TiDBServers
+			endpoints := []string{}
+			for _, db := range dbs {
+				endpoints = append(endpoints, utils.JoinHostPort(db.GetManageHost(), db.StatusPort))
+			}
+
+			if currentVersion != targetVersion && tidbver.TiDBSupportUpgradeAPI(currentVersion) && tidbver.TiDBSupportUpgradeAPI(targetVersion) {
+				tidbClient = api.NewTiDBClient(ctx, endpoints, 10*time.Second, tlsCfg)
+				err = tidbClient.StartUpgrade()
+				if err != nil {
+					return err
+				}
+			}
+
 		default:
 			// do nothing, kept for future usage with other components
 		}
@@ -178,6 +195,19 @@ func Upgrade(
 				return err
 			}
 		}
+
+		switch component.Name() {
+		case spec.ComponentTiDB:
+			if currentVersion != targetVersion && tidbver.TiDBSupportUpgradeAPI(currentVersion) && tidbver.TiDBSupportUpgradeAPI(targetVersion) {
+				err = tidbClient.FinishUpgrade()
+				if err != nil {
+					return err
+				}
+			}
+
+		default:
+			// do nothing, kept for future usage with other components
+		}
 	}
 
 	if topo.GetMonitoredOptions() == nil {

pkg/cluster/spec/grafana.go

@@ -181,7 +181,7 @@ func (i *GrafanaInstance) InitConfig(
 
 	// transfer config
 	spec := i.InstanceSpec.(*GrafanaSpec)
-	fp = filepath.Join(paths.Cache, fmt.Sprintf("grafana_%s.ini", i.GetHost()))
+	fp = filepath.Join(paths.Cache, fmt.Sprintf("grafana_%s_%d.ini", i.GetHost(), i.GetPort()))
 	if err := config.NewGrafanaConfig(i.GetHost(), paths.Deploy).
 		WithPort(uint64(i.GetPort())).
 		WithUsername(spec.Username).

pkg/repository/mirror.go

@@ -15,6 +15,7 @@ package repository
 
 import (
 	"bytes"
+	"crypto/tls"
 	"encoding/json"
 	stderrors "errors"
 	"fmt"
@@ -293,6 +294,14 @@ func (l *httpMirror) downloadFile(url string, to string, maxSize int64) (io.Read
 	}(time.Now())
 
 	client := grab.NewClient()
+
+	// workaround to resolve cdn error "tls: protocol version not supported"
+	client.HTTPClient.(*http.Client).Transport = &http.Transport{
+		Proxy: http.ProxyFromEnvironment,
+		// avoid using http/2 by setting non-nil TLSClientConfig
+		TLSClientConfig: &tls.Config{},
+	}
+
 	client.UserAgent = fmt.Sprintf("tiup/%s", version.NewTiUPVersion().SemVer())
 	req, err := grab.NewRequest(to, url)
 	if err != nil {
@@ -326,6 +335,7 @@ L:
 			}
 			progress.SetCurrent(resp.BytesComplete())
 		case <-resp.Done:
+			progress.SetCurrent(resp.BytesComplete())
 			progress.Finish()
 			break L
 		}

pkg/repository/progress.go

@@ -52,6 +52,5 @@ func (p *ProgressBar) SetCurrent(size int64) {
 
 // Finish implement the DownloadProgress interface
 func (p *ProgressBar) Finish() {
-	p.bar.SetCurrent(p.size)
 	p.bar.Finish()
 }

pkg/tidbver/tidbver.go

@@ -26,6 +26,13 @@ func TiDBSupportSecureBoot(version string) bool {
 	return semver.Compare(version, "v5.3.0") >= 0 || strings.Contains(version, "nightly")
 }
 
+// TiDBSupportUpgradeAPI return if given version of TiDB support upgrade API
+func TiDBSupportUpgradeAPI(version string) bool {
+	return semver.Compare(version, "v7.4.0") >= 0 ||
+		(semver.MajorMinor(version) == "v7.1" && semver.Compare(version, "v7.1.2") >= 0) ||
+		strings.Contains(version, "nightly")
+}
+
 // TiKVSupportAdvertiseStatusAddr return if given version of TiKV support --advertise-status-addr
 func TiKVSupportAdvertiseStatusAddr(version string) bool {
 	// TiKV support --advertise-status-addr since v4.0.1

pkg/utils/retry.go

@@ -77,8 +77,9 @@ func Retry(doFunc func() error, opts ...RetryOption) error {
 
 	// call the function
 	var attemptCount int64
+	var err error
 	for attemptCount = 0; attemptCount < cfg.Attempts; attemptCount++ {
-		if err := doFunc(); err == nil {
+		if err = doFunc(); err == nil {
 			return nil
 		}
 
@@ -91,7 +92,7 @@ func Retry(doFunc func() error, opts ...RetryOption) error {
 		}
 	}
 
-	return fmt.Errorf("operation exceeds the max retry attempts of %d", cfg.Attempts)
+	return fmt.Errorf("operation exceeds the max retry attempts of %d. error of last attempt: %s", cfg.Attempts, err)
 }
 
 // IsTimeoutOrMaxRetry return true if it's timeout or reach max retry.

pkg/version/version.go

@@ -23,7 +23,7 @@ var (
 	// TiUPVerMinor is the minor version of TiUP
 	TiUPVerMinor = 13
 	// TiUPVerPatch is the patch version of TiUP
-	TiUPVerPatch = 0
+	TiUPVerPatch = 1
 	// TiUPVerName is an alternative name of the version
 	TiUPVerName = "tiup"
 	// GitHash is the current git commit hash