Merge branch 'master' into fix/keyviz-invalid-border

.github/workflows/manual-create-pd-pr.yaml

@@ -12,8 +12,9 @@ jobs:
     name: Create PD PR
     runs-on: ubuntu-latest
     strategy:
+      fail-fast: false
       matrix:
-        branch: [master, release-5.3, release-5.4]
+        branch: [master, release-5.4, release-6.1]
     steps:
       - name: Check out PD code base
         uses: actions/checkout@master
@@ -22,19 +23,23 @@ jobs:
           ref: ${{ matrix.branch }}
       - uses: actions/setup-go@v2
         with:
-          go-version: "1.13.5"
+          go-version: "1.18.1"
       - name: Update TiDB Dashboard in PD code base
         run: |
           go get -d "github.com/pingcap/tidb-dashboard@${{ github.event.inputs.release_version }}"
           go mod tidy
+          cd tests/client
+          go mod tidy
+          cd ../..
           make pd-server
           go mod tidy
       - name: Commit PD code base change
         id: git_commit
         run: |
+          git diff
           git config user.name "tidb-dashboard-bot"
           git config user.email "[email protected]"
-          git add go.mod go.sum
+          git add .
           if git status | grep -q "Changes to be committed"
           then
             git commit --signoff --message "Update TiDB Dashboard to ${{ github.event.inputs.release_version }}, ref #4257"
@@ -55,6 +60,8 @@ jobs:
           body: |
             ### What problem does this PR solve?
 
+            Issue Number: ref #4257
+
             Update TiDB Dashboard to ${{ github.event.inputs.release_version }}.
 
             Upstream commit: https://github.com/${{ github.repository }}/commit/${{ github.sha }} .

.github/workflows/release.yaml

@@ -88,8 +88,9 @@ jobs:
     runs-on: ubuntu-latest
     needs: release
     strategy:
+      fail-fast: false
       matrix:
-        branch: [master, release-5.3, release-5.4]
+        branch: [master, release-5.4, release-6.1]
     steps:
       - name: Check out PD code base
         uses: actions/checkout@master
@@ -98,19 +99,23 @@ jobs:
           ref: ${{ matrix.branch }}
       - uses: actions/setup-go@v2
         with:
-          go-version: "1.16.5"
+          go-version: "1.18.1"
       - name: Update TiDB Dashboard in PD code base
         run: |
           go get -d "github.com/pingcap/tidb-dashboard@${{ needs.release.outputs.release_version }}"
           go mod tidy
+          cd tests/client
+          go mod tidy
+          cd ../..
           make pd-server
           go mod tidy
       - name: Commit PD code base change
         id: git_commit
         run: |
+          git diff
           git config user.name "tidb-dashboard-bot"
           git config user.email "[email protected]"
-          git add go.mod go.sum
+          git add .
           if git status | grep -q "Changes to be committed"
           then
             git commit --signoff --message "Update TiDB Dashboard to ${{ needs.release.outputs.release_version }}, ref #4257"
@@ -131,6 +136,8 @@ jobs:
           body: |
             ### What problem does this PR solve?
 
+            Issue Number: ref #4257
+
             This is an automatic updating PR for TiDB Dashboard. See #4257 for details.
 
             This PR updates TiDB Dashboard to ${{ needs.release.outputs.release_version }} for upstream commit: https://github.com/${{ github.repository }}/commit/${{ github.sha }} .

.github/workflows/test.yaml

@@ -76,12 +76,21 @@ jobs:
       matrix:
         # test latest features and compatibility of lower version
         include:
+          - feature_version: 6.0.0
+            tidb_version: latest
+            without_ngm: false
+          - feature_version: 6.0.0
+            tidb_version: latest
+            without_ngm: true
           - feature_version: 6.0.0
             tidb_version: nightly
+            without_ngm: false
           - feature_version: 5.4.0
             tidb_version: v5.4.0
+            without_ngm: false
           - feature_version: 5.0.0
             tidb_version: v5.0.0
+            without_ngm: false
     steps:
       - name: Checkout code
         uses: actions/checkout@v2
@@ -117,7 +126,7 @@ jobs:
       - name: Install and run TiUP in the background
         run: |
           chmod u+x scripts/start_tiup.sh
-          scripts/start_tiup.sh ${{ matrix.tidb_version }}
+          scripts/start_tiup.sh ${{ matrix.tidb_version }} ${{ matrix.without_ngm }}
       - name: Build UI
         run: |
           make ui
@@ -150,18 +159,14 @@ jobs:
           UI: 1
           FEATURE_VERSION: ${{ matrix.feature_version }}
       - name: Run E2E Features Test
-        run: make test_e2e
+        run: make e2e_test
         env:
           SERVER_URL: http://127.0.0.1:12333/dashboard/
           CI: true
           FEATURE_VERSION: ${{ matrix.feature_version }}
           TIDB_VERSION: ${{ matrix.tidb_version }}
-      - name: Archive Test Results
-        if: always()
-        run: |
-          cat ui/start_tiup.log
-          echo "==============="
-          cat ui/wait_tiup.log
+          CYPRESS_ALLOW_SCREENSHOT: true
+          WITHOUT_NGM: ${{ matrix.without_ngm }}
       - name: Upload coverage to Codecov
         uses: codecov/codecov-action@v2
         with:

.github/workflows/upload-e2e-snapshots.yaml

@@ -7,6 +7,9 @@ on:
       ref:
         description: "The branch, tag or SHA to create snapshots"
         required: true
+      spec:
+        description: "Specify the spec files to run, example: `topsql/topsql.spec.ts`"
+        required: true
 
 jobs:
   e2e_test_snapshots:
@@ -19,7 +22,7 @@ jobs:
         # test latest features and compatibility of lower version
         include:
           - feature_version: 6.0.0
-            tidb_version: nightly
+            tidb_version: latest
           - feature_version: 5.4.0
             tidb_version: v5.4.0
           - feature_version: 5.0.0
@@ -61,7 +64,7 @@ jobs:
       - name: Install and run TiUP in the background
         run: |
           chmod u+x scripts/start_tiup.sh
-          scripts/start_tiup.sh ${{ matrix.tidb_version }}
+          scripts/start_tiup.sh ${{ matrix.tidb_version }} false
       - name: Build UI
         run: |
           make ui
@@ -92,22 +95,26 @@ jobs:
         env:
           UI: 1
           FEATURE_VERSION: ${{ matrix.feature_version }}
+      - name: Delete Previous Snapshots
+        run: rm -rf ${{ github.workspace }}/ui/cypress/snapshots
       - name: Run E2E Features Test
-        run: make e2e_test
+        run: make e2e_test_specify
         env:
           SERVER_URL: http://127.0.0.1:12333/dashboard/
           CI: true
           FEATURE_VERSION: ${{ matrix.feature_version }}
           TIDB_VERSION: ${{ matrix.tidb_version }}
           CYPRESS_ALLOW_SCREENSHOT: true
-      - name: Archive Test Results
+          E2E_SPEC: cypress/integration/${{ github.event.inputs.spec }}
+      - name: Archive Test Video
         if: always()
-        run: |
-          cat ui/start_tiup.log
-          echo "==============="
-          cat ui/wait_tiup.log
+        uses: actions/upload-artifact@v2
+        with:
+          name: e2e-video-${{ matrix.feature_version }}
+          path: ${{ github.workspace }}/ui/cypress/videos/**/*
       - name: Upload snapshots artifact
         uses: actions/upload-artifact@v2
+        if: always()
         with:
           name: e2e-snapshots-${{ matrix.feature_version }}
           path: ${{ github.workspace }}/ui/cypress/snapshots/**/*

Makefile

@@ -6,6 +6,10 @@ LDFLAGS ?=
 
 FEATURE_VERSION ?= 6.0.0
 
+WITHOUT_NGM ?= false
+
+E2E_SPEC ?=
+
 ifeq ($(UI),1)
 	BUILD_TAGS += ui_server
 endif
@@ -45,6 +49,39 @@ integration_test:
 	@mkdir -p ./coverage
 	@TIDB_VERSION=${TIDB_VERSION} tests/run.sh
 
+.PHONY: e2e_test
+e2e_test:
+	@if $(WITHOUT_NGM); then\
+		make e2e_without_ngm_test;\
+	else\
+		make e2e_compat_features_test;\
+		make e2e_common_features_test;\
+	fi
+
+.PHONY: e2e_compat_features_test
+e2e_compat_features_test:
+	cd ui &&\
+	yarn &&\
+	yarn run:e2e-test:compat-features --env FEATURE_VERSION=$(FEATURE_VERSION) TIDB_VERSION=$(TIDB_VERSION)
+
+.PHONY: e2e_common_features_test
+e2e_common_features_test:
+	cd ui &&\
+	yarn &&\
+	yarn run:e2e-test:common-features --env TIDB_VERSION=$(TIDB_VERSION)
+
+.PHONY: e2e_without_ngm_test
+e2e_without_ngm_test:
+	cd ui &&\
+	yarn &&\
+	yarn run:e2e-test:without-ngm --env TIDB_VERSION=$(TIDB_VERSION) WITHOUT_NGM=$(WITHOUT_NGM)
+
+.PHONY: e2e_test_specify
+e2e_test_specify:
+	cd ui &&\
+	yarn &&\
+	yarn run:e2e-test:specify --env TIDB_VERSION=$(TIDB_VERSION) -- --spec $(E2E_SPEC)
+
 .PHONY: dev
 dev: lint default
 
@@ -74,13 +111,3 @@ endif
 .PHONY: run
 run:
 	bin/tidb-dashboard --debug --experimental --feature-version "$(FEATURE_VERSION)" --host 0.0.0.0
-
-test_e2e_compat_features:
-	cd ui &&\
-	yarn run:e2e-test:compat-features --env FEATURE_VERSION=$(FEATURE_VERSION) TIDB_VERSION=$(TIDB_VERSION)
-
-test_e2e_common_features:
-	cd ui &&\
-	yarn run:e2e-test:common-features --env TIDB_VERSION=$(TIDB_VERSION)
-
-test_e2e: test_e2e_compat_features test_e2e_common_features

pkg/apiserver/debugapi/apis.go

@@ -387,7 +387,7 @@ var apiEndpoints = []endpoint.APIDefinition{
 	{
 		ID:        "pd_pprof",
 		Component: topo.KindPD,
-		Path:      "/pd/api/v1/debug/pprof/{kind}",
+		Path:      "/debug/pprof/{kind}",
 		Method:    resty.MethodGet,
 		PathParams: []endpoint.APIParamDefinition{
 			commomParamPprofKinds,

pkg/apiserver/profiling/fetcher.go

@@ -94,6 +94,6 @@ func (f *pdFetcher) fetch(op *fetchOptions) ([]byte, error) {
 	return f.client.
 		WithTimeout(maxProfilingTimeout).
 		WithBaseURL(baseURL).
-		AddRequestHeader("PD-Allow-follower-handle", "true").
+		WithoutPrefix(). // pprof API does not have /pd/api/v1 prefix
 		SendGetRequest(op.path)
 }

pkg/pd/client.go

@@ -23,11 +23,12 @@ const (
 )
 
 type Client struct {
-	httpScheme   string
-	baseURL      string
-	httpClient   *httpc.Client
-	lifecycleCtx context.Context
-	timeout      time.Duration
+	httpScheme    string
+	baseURL       string
+	withoutPrefix bool
+	httpClient    *httpc.Client
+	lifecycleCtx  context.Context
+	timeout       time.Duration
 }
 
 func NewPDClient(lc fx.Lifecycle, httpClient *httpc.Client, config *config.Config) *Client {
@@ -64,13 +65,25 @@ func (c Client) WithTimeout(timeout time.Duration) *Client {
 	return &c
 }
 
+func (c Client) WithoutPrefix() *Client {
+	c.withoutPrefix = true
+	return &c
+}
+
+func (c Client) getPrefix() string {
+	if c.withoutPrefix {
+		return ""
+	}
+	return "/pd/api/v1"
+}
+
 func (c Client) AddRequestHeader(key, value string) *Client {
 	c.httpClient = c.httpClient.CloneAndAddRequestHeader(key, value)
 	return &c
 }
 
 func (c *Client) Get(relativeURI string) (*httpc.Response, error) {
-	uri := fmt.Sprintf("%s/pd/api/v1%s", c.baseURL, relativeURI)
+	uri := fmt.Sprintf("%s%s%s", c.baseURL, c.getPrefix(), relativeURI)
 	return c.httpClient.WithTimeout(c.timeout).Send(c.lifecycleCtx, uri, http.MethodGet, nil, ErrPDClientRequestFailed, distro.R().PD)
 }
 
@@ -83,6 +96,6 @@ func (c *Client) SendGetRequest(relativeURI string) ([]byte, error) {
 }
 
 func (c *Client) SendPostRequest(relativeURI string, body io.Reader) ([]byte, error) {
-	uri := fmt.Sprintf("%s/pd/api/v1%s", c.baseURL, relativeURI)
+	uri := fmt.Sprintf("%s%s%s", c.baseURL, c.getPrefix(), relativeURI)
 	return c.httpClient.WithTimeout(c.timeout).SendRequest(c.lifecycleCtx, uri, http.MethodPost, body, ErrPDClientRequestFailed, distro.R().PD)
 }

tests/_inc/download_tools.sh → scripts/_inc/download_tools.sh

@@ -10,22 +10,26 @@ BIN="${PROJECT_DIR}/bin"
 download_tools() {
   echo "+ Download tools"
 
-  if ! command -v tiup >/dev/null 2>&1; then
-    echo "  - Downloading tiup..."
-    curl --proto '=https' --tlsv1.2 -sSf https://tiup-mirrors.pingcap.com/install.sh | sh
-  fi
+  download_tiup
 
   mkdir -p $BIN
 
   if [ ! -e "$BIN/toolkit.tar.gz" ]; then
     echo "  - Downloading toolkit..."
-    curl -L -f -o "$BIN/toolkit.tar.gz" "https://download.pingcap.org/tidb-toolkit-nightly-linux-amd64.tar.gz"
+    curl -L -f -o "$BIN/toolkit.tar.gz" "https://download.pingcap.org/tidb-toolkit-v6.0.0-linux-amd64.tar.gz"
   fi
 
   if [ ! -e "$BIN/dumpling" ]; then
-    tar -x -f "$BIN/toolkit.tar.gz" -C "$BIN/" tidb-toolkit-nightly-linux-amd64/bin/dumpling
-    mv "$BIN"/tidb-toolkit-nightly-linux-amd64/bin/dumpling "$BIN/dumpling"
+    tar -x -f "$BIN/toolkit.tar.gz" -C "$BIN/" tidb-toolkit-v6.0.0-linux-amd64/bin/dumpling
+    mv "$BIN"/tidb-toolkit-v6.0.0-linux-amd64/bin/dumpling "$BIN/dumpling"
   fi
 
   echo "+ All binaries are now available."
 }
+
+download_tiup() {
+  if ! command -v tiup >/dev/null 2>&1; then
+    echo "  - Downloading tiup..."
+    curl --proto '=https' --tlsv1.2 -sSf https://tiup-mirrors.pingcap.com/install.sh | sh
+  fi
+}

scripts/start_tiup.sh

@@ -1,7 +1,8 @@
 #!/usr/bin/env bash
 set -ex
 tidb_version=$1
-mode=$2
+without_ngm=${2:-false}
+mode=${3:-"start"}
 
 TIUP_BIN_DIR=$HOME/.tiup/bin/tiup
 DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )"
@@ -26,5 +27,5 @@ else
     $TIUP_BIN_DIR update playground
 
     # Run Tiup
-    $TIUP_BIN_DIR playground ${tidb_version} --tiflash=0 &> start_tiup.log &
+    $TIUP_BIN_DIR playground ${tidb_version} --without-monitor=${without_ngm} --tiflash=0 &> start_tiup.log &
 fi