cherrypick #2065 #2094 and #2133 to release-2.0.7 (#2143)

.golangci.yml

@@ -114,6 +114,13 @@ issues:
     - EXC0002 # golint
     - EXC0003
 
+  exclude-rules:
+    # TODO Remove it.
+    # Exclude some staticcheck messages
+    - linters:
+        - staticcheck
+      text: "SA1019:"
+
   # Fix found issues (if it's supported by the linter)
   fix: false
 

dm/ctl/common/util.go

@@ -25,7 +25,7 @@ import (
 	"sync"
 	"time"
 
-	"github.com/golang/protobuf/jsonpb"
+	"github.com/gogo/protobuf/jsonpb"
 	"github.com/golang/protobuf/proto"
 	"github.com/pingcap/errors"
 	"github.com/pingcap/failpoint"

dumpling/dumpling_test.go

@@ -33,6 +33,11 @@ import (
 
 var _ = Suite(&testDumplingSuite{})
 
+const (
+	testDumplingSchemaName = "INFORMATION_SCHEMA"
+	testDumplingTableName  = "TABLES"
+)
+
 func TestSuite(t *testing.T) {
 	TestingT(t)
 }
@@ -72,7 +77,11 @@ func (d *testDumplingSuite) SetUpSuite(c *C) {
 			Dir: dir,
 		},
 		BAList: &filter.Rules{
-			DoDBs: []string{"information_schema"},
+			DoDBs: []string{testDumplingSchemaName},
+			DoTables: []*filter.Table{{
+				Schema: testDumplingSchemaName,
+				Name:   testDumplingTableName,
+			}},
 		},
 	}
 	c.Assert(log.InitLogger(&log.Config{}), IsNil)

go.mod

@@ -3,28 +3,28 @@ module github.com/pingcap/dm
 require (
 	github.com/BurntSushi/toml v0.3.1
 	github.com/DATA-DOG/go-sqlmock v1.5.0
-	github.com/chaos-mesh/go-sqlsmith v0.0.0-20201120053641-47c50b530c01
+	github.com/chaos-mesh/go-sqlsmith v0.0.0-20210914111832-b3d328d69449
 	github.com/chzyer/readline v0.0.0-20180603132655-2972be24d48e
 	github.com/coreos/go-semver v0.3.0
 	github.com/docker/go-units v0.4.0
 	github.com/dustin/go-humanize v1.0.0
 	github.com/go-mysql-org/go-mysql v1.1.3-0.20210705101833-83965e516929
-	github.com/go-sql-driver/mysql v1.5.0
+	github.com/go-sql-driver/mysql v1.6.0
 	github.com/gogo/gateway v1.1.0
 	github.com/gogo/protobuf v1.3.2
-	github.com/golang/mock v1.4.4
-	github.com/golang/protobuf v1.3.4
-	github.com/google/uuid v1.1.1
-	github.com/grpc-ecosystem/grpc-gateway v1.14.3
+	github.com/golang/mock v1.6.0
+	github.com/golang/protobuf v1.5.2
+	github.com/google/uuid v1.1.2
+	github.com/grpc-ecosystem/grpc-gateway v1.16.0
 	github.com/kami-zh/go-capturer v0.0.0-20171211120116-e492ea43421d
 	github.com/pingcap/check v0.0.0-20200212061837-5e12011dc712
-	github.com/pingcap/dumpling v0.0.0-20210407092432-e1cfe4ce0a53
-	github.com/pingcap/errors v0.11.5-0.20201126102027-b0a155152ca3
+	github.com/pingcap/dumpling v0.0.0-20210914144241-99aca9186bc8
+	github.com/pingcap/errors v0.11.5-0.20210513014640-40f9a1999b3b
 	github.com/pingcap/failpoint v0.0.0-20210316064728-7acb0f0a3dfd
-	github.com/pingcap/log v0.0.0-20210317133921-96f4fcab92a4
-	github.com/pingcap/parser v0.0.0-20210415081931-48e7f467fd74
-	github.com/pingcap/tidb v1.1.0-beta.0.20210330094614-60111e1c4b6f
-	github.com/pingcap/tidb-tools v5.2.0-alpha.0.20210721090336-4921149b5e5c+incompatible
+	github.com/pingcap/log v0.0.0-20210906054005-afc726e70354
+	github.com/pingcap/parser v0.0.0-20210907051057-948434fa20e4
+	github.com/pingcap/tidb v1.1.0-beta.0.20210914112841-6ebfe8aa4257
+	github.com/pingcap/tidb-tools v5.2.0-alpha.0.20210727084616-915b22e4d42c+incompatible
 	github.com/prometheus/client_golang v1.5.1
 	github.com/rakyll/statik v0.1.6
 	github.com/shopspring/decimal v0.0.0-20200105231215-408a2507e114
@@ -34,18 +34,19 @@ require (
 	github.com/syndtr/goleveldb v1.0.1-0.20190625010220-02440ea7a285
 	github.com/tidwall/gjson v1.6.1
 	github.com/tidwall/sjson v1.1.2
-	github.com/tikv/pd v1.1.0-beta.0.20210323121136-78679e5e209d
+	github.com/tikv/pd v1.1.0-beta.0.20210818082359-acba1da0018d
 	github.com/unrolled/render v1.0.1
 	go.etcd.io/etcd v0.5.0-alpha.5.0.20200824191128-ae9734ed278b
-	go.uber.org/atomic v1.7.0
-	go.uber.org/zap v1.16.0
-	golang.org/x/exp v0.0.0-20200513190911-00229845015e // indirect
-	golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9
-	golang.org/x/sys v0.0.0-20210320140829-1e4c9ba3b0c4
-	golang.org/x/time v0.0.0-20191024005414-555d28b269f0
-	google.golang.org/genproto v0.0.0-20200224152610-e50cd9704f63
-	google.golang.org/grpc v1.27.1
+	go.uber.org/atomic v1.9.0
+	go.uber.org/zap v1.19.0
+	golang.org/x/sync v0.0.0-20210220032951-036812b2e83c
+	golang.org/x/sys v0.0.0-20210806184541-e5e7981a1069
+	golang.org/x/time v0.0.0-20210220033141-f8bda1e9f3ba
+	google.golang.org/genproto v0.0.0-20210825212027-de86158e7fda
+	google.golang.org/grpc v1.40.0
 	gopkg.in/yaml.v2 v2.4.0
 )
 
+replace google.golang.org/grpc => google.golang.org/grpc v1.29.1
+
 go 1.16

pkg/log/log.go

@@ -18,9 +18,10 @@ import (
 	"fmt"
 	"strings"
 
+	"github.com/pingcap/tidb/util/logutil"
+
 	"github.com/pingcap/errors"
 	pclog "github.com/pingcap/log"
-	"github.com/pingcap/tidb/util/logutil"
 	"go.uber.org/zap"
 	"go.uber.org/zap/zapcore"
 
@@ -104,11 +105,7 @@ var (
 
 // InitLogger initializes DM's and also the TiDB library's loggers.
 func InitLogger(cfg *Config) error {
-	err := logutil.InitLogger(&logutil.LogConfig{Config: pclog.Config{Level: cfg.Level}})
-	if err != nil {
-		return terror.ErrInitLoggerFail.Delegate(err)
-	}
-
+	// init DM logger
 	logger, props, err := pclog.InitLogger(&pclog.Config{
 		Level:  cfg.Level,
 		Format: cfg.Format,
@@ -128,7 +125,14 @@ func InitLogger(cfg *Config) error {
 	appLogger = Logger{logger.WithOptions(zap.AddStacktrace(zap.DPanicLevel))}
 	appLevel = props.Level
 	appProps = props
-
+	// init and set tidb slow query logger to stdout if log level is debug
+	if cfg.Level == "debug" {
+		slowQueryLogger := zap.NewExample()
+		slowQueryLogger = slowQueryLogger.With(zap.String("component", "slow query logger"))
+		logutil.SlowQueryLogger = slowQueryLogger
+	} else {
+		logutil.SlowQueryLogger = zap.NewNop()
+	}
 	return nil
 }
 

pkg/log/log_test.go

@@ -14,9 +14,15 @@
 package log
 
 import (
+	"bytes"
 	"context"
+	"encoding/json"
+	"io"
+	"os"
+	"strings"
 	"testing"
 
+	"github.com/pingcap/tidb/util/logutil"
 	"go.uber.org/zap"
 	"go.uber.org/zap/zapcore"
 	"go.uber.org/zap/zaptest"
@@ -83,3 +89,64 @@ func (s *testLogSuite) TestLogLevel(c *C) {
 	c.Assert(L().Check(zap.WarnLevel, "This is a warn log"), NotNil)
 	c.Assert(L().Check(zap.DebugLevel, "This is a debug log"), IsNil)
 }
+
+func captureStdout(f func()) ([]string, error) {
+	r, w, _ := os.Pipe()
+	stdout := os.Stdout
+	os.Stdout = w
+
+	f()
+
+	var buf bytes.Buffer
+	output := make(chan string, 1)
+	errs := make(chan error, 1)
+
+	go func() {
+		_, err := io.Copy(&buf, r)
+		output <- buf.String()
+		errs <- err
+		r.Close()
+	}()
+
+	os.Stdout = stdout
+	w.Close()
+	return strings.Split(<-output, "\n"), <-errs
+}
+
+func (s *testLogSuite) TestInitSlowQueryLoggerInDebugLevel(c *C) {
+	// test slow query logger can write debug log
+	logLevel := "debug"
+	cfg := &Config{Level: logLevel, Format: "json"}
+	cfg.Adjust()
+	output, err := captureStdout(func() {
+		c.Assert(InitLogger(cfg), IsNil)
+		logutil.SlowQueryLogger.Debug("this is test info")
+		appLogger.Debug("this is from applogger")
+	})
+	c.Assert(err, IsNil)
+	c.Assert(output[0], Matches, ".*this is test info.*component.*slow query logger.*")
+	c.Assert(output[1], Matches, ".*this is from applogger.*")
+	// test log is json formart
+	type jsonLog struct {
+		Component string `json:"component"`
+	}
+	oneLog := jsonLog{}
+	c.Assert(json.Unmarshal([]byte(output[0]), &oneLog), IsNil)
+	c.Assert(oneLog.Component, Equals, "slow query logger")
+}
+
+func (s *testLogSuite) TestInitSlowQueryLoggerNotInDebugLevel(c *C) {
+	// test slow query logger can not write log in other log level
+	logLevel := "info"
+	cfg := &Config{Level: logLevel, Format: "json"}
+	cfg.Adjust()
+	output, err := captureStdout(func() {
+		c.Assert(InitLogger(cfg), IsNil)
+		logutil.SlowQueryLogger.Info("this is test info")
+		appLogger.Info("this is from applogger")
+	})
+	c.Assert(err, IsNil)
+	c.Assert(output, HasLen, 2)
+	c.Assert(output[0], Matches, ".*this is from applogger.*")
+	c.Assert(output[1], Equals, "") // no output
+}

pkg/schema/tracker.go

@@ -21,6 +21,7 @@ import (
 	"github.com/pingcap/errors"
 	"github.com/pingcap/parser/ast"
 	"github.com/pingcap/parser/model"
+	"github.com/pingcap/parser/terror"
 	"github.com/pingcap/tidb-tools/pkg/dbutil"
 	"github.com/pingcap/tidb-tools/pkg/filter"
 	tidbConfig "github.com/pingcap/tidb/config"
@@ -29,6 +30,7 @@ import (
 	"github.com/pingcap/tidb/infoschema"
 	"github.com/pingcap/tidb/kv"
 	"github.com/pingcap/tidb/session"
+	"github.com/pingcap/tidb/sessionctx/variable"
 	"github.com/pingcap/tidb/store/mockstore"
 	"go.uber.org/zap"
 
@@ -40,17 +42,14 @@ import (
 const (
 	// TiDBClusteredIndex is the variable name for clustered index.
 	TiDBClusteredIndex = "tidb_enable_clustered_index"
-	// TiDBChangeColumnType is the variable name for column type changing.
-	TiDBChangeColumnType = "tidb_enable_change_column_type"
 )
 
 var (
 	// don't read clustered index variable from downstream because it may changed during syncing
 	// we always using OFF tidb_enable_clustered_index unless user set it in config.
 	downstreamVars    = []string{"sql_mode", "tidb_skip_utf8_check"}
 	defaultGlobalVars = map[string]string{
-		TiDBClusteredIndex:   "OFF",
-		TiDBChangeColumnType: "ON", // NOTE The default value of tidb_enable_change_column_type was changed to ON after the release of TiDB 5.1.
+		TiDBClusteredIndex: "OFF",
 	}
 )
 
@@ -67,11 +66,11 @@ type Tracker struct {
 func NewTracker(ctx context.Context, task string, sessionCfg map[string]string, tidbConn *conn.BaseConn) (*Tracker, error) {
 	// NOTE: tidb uses a **global** config so can't isolate tracker's config from each other. If that isolation is needed,
 	// we might SetGlobalConfig before every call to tracker, or use some patch like https://github.com/bouk/monkey
-	toSet := tidbConfig.NewConfig()
-	// bypass wait time of https://github.com/pingcap/tidb/pull/20550
-	toSet.TiKVClient.AsyncCommit.SafeWindow = 0
-	toSet.TiKVClient.AsyncCommit.AllowedClockDrift = 0
-	tidbConfig.StoreGlobalConfig(toSet)
+	tidbConfig.UpdateGlobal(func(conf *tidbConfig.Config) {
+		// bypass wait time of https://github.com/pingcap/tidb/pull/20550
+		conf.TiKVClient.AsyncCommit.SafeWindow = 0
+		conf.TiKVClient.AsyncCommit.AllowedClockDrift = 0
+	})
 
 	if len(sessionCfg) == 0 {
 		sessionCfg = make(map[string]string)
@@ -111,6 +110,7 @@ func NewTracker(ctx context.Context, task string, sessionCfg map[string]string,
 
 	// avoid data race and of course no use in DM
 	domain.RunAutoAnalyze = false
+	session.DisableStats4Test()
 
 	dom, err := session.BootstrapSession(store)
 	if err != nil {
@@ -131,13 +131,18 @@ func NewTracker(ctx context.Context, task string, sessionCfg map[string]string,
 	}
 
 	for k, v := range sessionCfg {
-		err = se.GetSessionVars().SetSystemVar(k, v)
+		err = se.GetSessionVars().SetSystemVarWithRelaxedValidation(k, v)
 		if err != nil {
+			// when user set some unsupported variable, we just ignore it
+			if terror.ErrorEqual(err, variable.ErrUnknownSystemVar) {
+				log.L().Warn("can not set this variable", zap.Error(err))
+				continue
+			}
 			return nil, err
 		}
 	}
 	for k, v := range globalVarsToSet {
-		err = se.GetSessionVars().SetSystemVar(k, v)
+		err = se.GetSessionVars().SetSystemVarWithRelaxedValidation(k, v)
 		if err != nil {
 			return nil, err
 		}

pkg/utils/common.go

@@ -265,10 +265,10 @@ func UnpackTableID(id string) (string, string) {
 
 type session struct {
 	sessionctx.Context
-	vars   *variable.SessionVars
-	values map[fmt.Stringer]interface{}
-
-	mu sync.RWMutex
+	vars                 *variable.SessionVars
+	values               map[fmt.Stringer]interface{}
+	builtinFunctionUsage map[string]uint32
+	mu                   sync.RWMutex
 }
 
 // GetSessionVars implements the sessionctx.Context interface.
@@ -291,6 +291,16 @@ func (se *session) Value(key fmt.Stringer) interface{} {
 	return value
 }
 
+// GetInfoSchema implements the sessionctx.Context interface.
+func (se *session) GetInfoSchema() sessionctx.InfoschemaMetaVersion {
+	return nil
+}
+
+// GetBuiltinFunctionUsage implements the sessionctx.Context interface.
+func (se *session) GetBuiltinFunctionUsage() map[string]uint32 {
+	return se.builtinFunctionUsage
+}
+
 // UTCSession can be used as a sessionctx.Context, with UTC timezone.
 var UTCSession *session
 
@@ -300,6 +310,7 @@ func init() {
 	vars.StmtCtx.TimeZone = time.UTC
 	UTCSession.vars = vars
 	UTCSession.values = make(map[fmt.Stringer]interface{}, 1)
+	UTCSession.builtinFunctionUsage = make(map[string]uint32)
 }
 
 // AdjustBinaryProtocolForDatum converts the data in binlog to TiDB datum.