diff --git a/.github/workflows/install.yaml b/.github/workflows/install.yaml index 3fa960ce8a..78b4a12679 100644 --- a/.github/workflows/install.yaml +++ b/.github/workflows/install.yaml @@ -16,7 +16,7 @@ on: jobs: install: name: Install - runs-on: ubuntu-latest + runs-on: ubuntu-20.04 env: working-directory: ${{ github.workspace }}/go/src/github.com/${{ github.repository }} steps: @@ -41,10 +41,10 @@ jobs: env: working-directory: ${{ github.workspace }}/go/src/github.com/${{ github.repository }} steps: - - name: Set up Go 1.19 - uses: actions/setup-go@v2 + - name: Set up Go 1.21 + uses: actions/setup-go@v4 with: - go-version: 1.19.x + go-version: 1.21.x id: go - name: Check out code into the Go module directory diff --git a/.github/workflows/integrate-cluster-cmd.yaml b/.github/workflows/integrate-cluster-cmd.yaml index 28b4c67435..db14720d69 100644 --- a/.github/workflows/integrate-cluster-cmd.yaml +++ b/.github/workflows/integrate-cluster-cmd.yaml @@ -30,7 +30,7 @@ on: jobs: cluster: - runs-on: ubuntu-latest + runs-on: ubuntu-20.04 strategy: fail-fast: true matrix: @@ -43,10 +43,10 @@ jobs: env: working-directory: ${{ github.workspace }}/go/src/github.com/${{ github.repository }} steps: - - name: Set up Go 1.19 - uses: actions/setup-go@v2 + - name: Set up Go 1.21 + uses: actions/setup-go@v4 with: - go-version: 1.19.x + go-version: 1.21.x id: go - name: Check out code into the Go module directory diff --git a/.github/workflows/integrate-cluster-scale.yaml b/.github/workflows/integrate-cluster-scale.yaml index bdf5d48862..7746b8cbd6 100644 --- a/.github/workflows/integrate-cluster-scale.yaml +++ b/.github/workflows/integrate-cluster-scale.yaml @@ -30,7 +30,7 @@ on: jobs: cluster: - runs-on: ubuntu-latest + runs-on: ubuntu-20.04 strategy: fail-fast: true matrix: @@ -42,10 +42,10 @@ jobs: env: working-directory: ${{ github.workspace }}/go/src/github.com/${{ github.repository }} steps: - - name: Set up Go 1.19 - uses: actions/setup-go@v2 + - name: Set up Go 1.21 + uses: actions/setup-go@v4 with: - go-version: 1.19.x + go-version: 1.21.x id: go - name: Check out code into the Go module directory diff --git a/.github/workflows/integrate-dm.yaml b/.github/workflows/integrate-dm.yaml index b3865fe88a..37ef609075 100644 --- a/.github/workflows/integrate-dm.yaml +++ b/.github/workflows/integrate-dm.yaml @@ -30,7 +30,7 @@ on: jobs: dm: - runs-on: ubuntu-latest + runs-on: ubuntu-20.04 strategy: fail-fast: true matrix: @@ -42,10 +42,10 @@ jobs: env: working-directory: ${{ github.workspace }}/go/src/github.com/${{ github.repository }} steps: - - name: Set up Go 1.19 - uses: actions/setup-go@v2 + - name: Set up Go 1.21 + uses: actions/setup-go@v4 with: - go-version: 1.19.x + go-version: 1.21.x id: go - name: Check out code into the Go module directory diff --git a/.github/workflows/integrate-playground.yaml b/.github/workflows/integrate-playground.yaml index 14ace90df1..c8c407f0d7 100644 --- a/.github/workflows/integrate-playground.yaml +++ b/.github/workflows/integrate-playground.yaml @@ -31,7 +31,7 @@ on: jobs: playground: - runs-on: ubuntu-latest + runs-on: ubuntu-20.04 timeout-minutes: 10 strategy: fail-fast: true @@ -41,10 +41,10 @@ jobs: env: working-directory: ${{ github.workspace }}/go/src/github.com/${{ github.repository }} steps: - - name: Set up Go 1.19 - uses: actions/setup-go@v2 + - name: Set up Go 1.21 + uses: actions/setup-go@v4 with: - go-version: 1.19.x + go-version: 1.21.x id: go - name: Check out code into the Go module directory diff --git a/.github/workflows/integrate-tiup.yaml b/.github/workflows/integrate-tiup.yaml index 6fd5da2626..50b7225a56 100644 --- a/.github/workflows/integrate-tiup.yaml +++ b/.github/workflows/integrate-tiup.yaml @@ -35,7 +35,7 @@ on: jobs: tiup: - runs-on: ubuntu-latest + runs-on: ubuntu-20.04 timeout-minutes: 10 strategy: fail-fast: true @@ -45,10 +45,10 @@ jobs: env: working-directory: ${{ github.workspace }}/go/src/github.com/${{ github.repository }} steps: - - name: Set up Go 1.19 - uses: actions/setup-go@v2 + - name: Set up Go 1.21 + uses: actions/setup-go@v4 with: - go-version: 1.19.x + go-version: 1.21.x id: go - name: Check out code into the Go module directory @@ -90,10 +90,10 @@ jobs: ref: ${{ github.event.pull_request.head.sha }} path: go/src/github.com/${{ github.repository }} - - name: Set up Go 1.19 - uses: actions/setup-go@v2 + - name: Set up Go 1.21 + uses: actions/setup-go@v4 with: - go-version: 1.19.x + go-version: 1.21.x id: go - name: make unit-test diff --git a/.github/workflows/release-tiup.yaml b/.github/workflows/release-tiup.yaml index 5a4122f293..b082e71cb0 100644 --- a/.github/workflows/release-tiup.yaml +++ b/.github/workflows/release-tiup.yaml @@ -6,6 +6,7 @@ on: branches: - ci/gh-* - release-* + - master paths-ignore: - '**.html' - '**.md' @@ -24,7 +25,7 @@ on: jobs: release: - runs-on: ubuntu-latest + runs-on: ubuntu-20.04 timeout-minutes: 30 outputs: REL_VER: ${{ steps.build_tiup.outputs.REL_VER }} @@ -40,10 +41,10 @@ jobs: env: working-directory: ${{ github.workspace }}/go/src/github.com/${{ github.repository }} steps: - - name: Set up Go 1.19 - uses: actions/setup-go@v2 + - name: Set up Go 1.21 + uses: actions/setup-go@v4 with: - go-version: 1.19.x + go-version: 1.21.x id: go - name: Check out code into the Go module directory @@ -58,7 +59,7 @@ jobs: working-directory: ${{ env.working-directory }} if: github.event_name != 'release' run: | - STAGING_VER=`git describe --tags | sed -r "s/([^-]*-g)/r\1/"` + STAGING_VER=`git describe --tags | sed 's/-[^-]*$//' | sed -r 's/(-[^-]*$)/-nightly\1/'` echo ::set-output name=STAGING::$STAGING_VER - name: Get git ref and commit @@ -90,7 +91,7 @@ jobs: if [ ! -z ${{ steps.check_staging.outputs.STAGING }} ]; then export EXTRA_LDFLAGS="-X \"github.com/${{ github.repository }}/pkg/version.TiUPVerName=${{ steps.check_staging.outputs.STAGING }}\"" - export REL_VER=${{ steps.check_staging.outputs.STAGING }}+staging + export REL_VER=${{ steps.check_staging.outputs.STAGING }} else export REL_VER=`git describe --tags --always` fi @@ -147,7 +148,7 @@ jobs: - name: Publish packages working-directory: ${{ env.working-directory }} env: - TIUP_MIRRORS: ${{ secrets.TIUP_SERVER_STAGING }} + TIUP_MIRRORS: ${{ secrets.TIUP_SERVER_PROD }} TIUP_HOME: ${{ steps.packaging.outputs.TIUP_HOME }} TIUP_BIN: ${{ steps.packaging.outputs.TIUP_BIN }} REL_VER: ${{ steps.build_tiup.outputs.REL_VER }} @@ -159,20 +160,10 @@ jobs: client_desc: "Client to connect playground" server_desc: "TiUP publish/cache server" run: | - if [ ! -z ${{ steps.check_staging.outputs.STAGING }} ]; then - TIUP_MIRRORS=${{ secrets.TIUP_SERVER_STAGING }} - else - TIUP_MIRRORS=${{ secrets.TIUP_SERVER_PROD }} - fi ${TIUP_BIN} mirror set ${TIUP_MIRRORS} echo ${{ secrets.TIUP_COMP_KEY_PINGCAP }} | base64 -d > $TIUP_HOME/keys/private.json - # build the latest tiup to use for publish, this is a workaround and should - # be removed after this PR is merged and released - GOOS=linux GOARCH=amd64 make tiup - cp bin/tiup ${TIUP_BIN} - # add a random delay before actually publishing, this can help reduce manifest conflict # when there are multiple jobs running in parallel sleep $[ ( $RANDOM % 10 ) ]s diff --git a/.github/workflows/reprotest.yaml b/.github/workflows/reprotest.yaml index 7d80ee827b..2d7bbddcae 100644 --- a/.github/workflows/reprotest.yaml +++ b/.github/workflows/reprotest.yaml @@ -30,19 +30,19 @@ on: jobs: reprotest: # The type of runner that the job will run on - runs-on: ubuntu-latest + runs-on: ubuntu-20.04 strategy: matrix: go: - - 1.19.x + - 1.21.x # Steps represent a sequence of tasks that will be executed as part of the job steps: - uses: actions/checkout@v3 - name: Set up Go ${{ matrix.go }} - uses: actions/setup-go@v2 + uses: actions/setup-go@v4 with: go-version: ${{ matrix.go }} diff --git a/CHANGELOG.md b/CHANGELOG.md index 2ff005eb00..a27d643f30 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,177 @@ TiUP Changelog +## [1.13.1] 2023-09-25 + +### Fixes + +- Increase timeout when publish package in `tiup` (#2269, @nexustar) +- Fix pd microservice component id in `tiup-playground` (#2272, @iosmanthus) +- Fix grafana for multiple instances using same host in `tiup-cluster` and `tiup-dm` (#2277, @lastincisor) +- Add cdn workaround (#2285, @nexustar) +- Mirror: fix progress bar is not accurate (#2284, @nexustar) + +### Improvement + +- Support ignore version check when upgrade in `tiup-cluster` and `tiup-dm` (#2282, @nexustar) + +## [1.13.0] 2023-08-26 + +### New Features + +- Support run components without tiup in `tiup` (#2265, @nexustar) +- Support tiproxy in `tiup-playground` (#2202, @xhebox) +- Show warning and continue if cannot delete data dir when destroy/scale-in instead of error and exit in `tiup-cluster` (#2256, @nexustar) +- Support specifying package name of components in `tiup-cluster` (#2236, @nexustar) +- playground: support microservices mode in `tiup-playground` (#2226, @rleungx) + +### Fixes + +- Fix the issue meta backup is failed when symbolic links exist in meta dir in `tiup-cluster` (#2255, @@KanShiori) +- Fix node filter on exec in `tiup-cluster` (#2251, @nexustar) + +### Improvements + +- Move to Go 1.21 (#2249, @dveeden) +- Upgrade TiCDC before TiKV and PD when cluster is equal or greater than v5.1.0 in `tiup-cluster` (#2253, @KanShiori) +- progress: Add option to add error detail (#2203, @dveeden) + +## [1.12.5] 2023-7-17 + +### Fix + +- Fix cannot start tiflash above v7.1.0 in `tiup-cluster` (#2230, @zanmato1984) + +## [1.12.4] 2023-7-13 + +### Fix + +- Fix cannot show tiflash uptime in `tiup-cluster` (#2227, @nexustar) + +### Improvement + +- Remove tcp_port for tiflash in `tiup-cluster` and `tiup-playground` (#2220, @zanmato1984) + +## [1.12.3] 2023-6-14 + +### Fixes + +- Fix cannot edit manage_host on an exist cluster in `tiup-cluster` (#2210, @nexustar) +- Fix still use host instead of manage_host in `tiup-cluster` (#2206 #2207, @nexustar) + +### Improvement + +- Check if the compnoent exists when uninstall in `tiup` (#2209, @srstack) + +## [1.12.2] 2023-5-19 + +### Notes + +- Chore: migrate to new prow reviewing (#2160, @wuhuizuo) +- Publish nightly tiup packages based on master branch (#2192, @nexustar) + +### Fixes + +- Fix cannot start alertmanager in `tiup-cluster` (#2200, @nexustar) +- Fix TiFlash config not working in `tiup-playground` (#2190, @breezewish) +- Fix --tiflash 0 not working in `tiup-playground` (#2189, @breezewish) +- Fix port conflict when cluster version >= 5.4 and < 7.1 in `tiup-playground` (#2188, @Lloyd-Pottiger) +- Fix cannot prune/scale-in tikv when use ipv6 in `tiup-cluster` (#2180, @nexustar) +- Fix exec error when use ipv6 in `tiup-cluster` (#2193, @nexustar) + +### Improvements + +- Output PD endpoints in tikv-slim mode in `tiup-playground` (#2196, @breezewish) +- Add --mode=tidb-disagg in `tiup-playground` (#2194, @breezewish) +- Add option to specify TiKV port in `tiup-playground` (#2183, @dveeden) +- include '--comments' in mysql command line arguments in `tiup-playground` (#2187, @mjonss) +- Code refine for TiFlash in `tiup-playground` (#2177, @breezewish) +- Code refine for args in `tiup-playground` (#2178, @breezewish) + +## [1.12.1] 2023-4-13 + +### Fix + +- Fix ngm config in `tiup-cluster` (#2175, @Mystery-cyf) + +### Improvements + +- Remove TiFlash http service in `tiup-cluster` and `tiup-playground` (#2165, Lloyd-Pottige) +- Remove ":" on default backup filename in `tiup-cluster` (#2174, @nexustar) +- Allows setting other storage config for TiFlash when `storage.main` is not defined in `tiup-cluster` (#2161, @breezewish) +- Support run TiFlash via args in `tiup-playground` (#2162, @breezewish) +- Tidy output in `tiup-playground` (#2163, @breezewish) + +## [1.12.0] 2023-3-31 + +### New Features + +- Support ipv6 in `tiup-cluster` and `tiup-dm` (#2068 #2069 #2070 #2075 #2117, @nexustar) +- Add support for displaying the grafana URLs in JSON output in `tiup-cluster` and `tiup-dm` (#2041, @onlyacat) +- Allow download directory when use builtin scp (#2098, @nexustar) +- Support set manage host in `tiup-cluster` and `tiup-dm` (#2147, @srstack) +- User perm from parent dir in `tiup-cluster` (#2143, @nexustar) +- Support display tiflash uptime in `tiup-cluster` (#2094, @srstack) +- Add ticdc port prob config in `tiup-cluster` (#2124, @sdojjy) +- Allow users to run custom shell commands during the cluster rolling upgrade in `tiup-cluster` (#2130, @darkelf21cn) +- Check upgrade offline in `tiup-cluster` (#2116, @srstack) +- Check required CPU flags for TiFlash >= v6.3.0 in `tiup-cluster` (#2054, @solotzg) +- Support connect to external Pushgateway in `tiup-cluster` (#2137, @nexustar) +- Allow setting the TiCDC port in `tiup-playground` (#2140, @dveeden) + +### Improvements + +- Check the cdc capture can be found before drain the capture in `tiup-cluster` (#2059, @3AceShowHand) +- Enable AutoTLS by default in `tiup-playground` (#2077, @dveeden) +- Wait for leader to transfer back during rolling restart in `tiup-cluster` (#2051, @cosven) +- Sync pdapi & typeutil with pd 6.3.0 in `tiup-cluster` (#2063, @AstroProfundis) +- Optimize checks & tests (#2056, @AstroProfundis) +- Mark openEuler as supported in `tiup-cluster` (#2145, @nexustar) +- Merge tiflash config in tiup playground to reduce handwrite config in `tiup-playground` (#2086, @iosmanthus) +- Add TIUP_VERBOSE in doc (#2111, @dveeden) +- Delete storage.remote.cache.dir when prune in `tiup-cluster` (#2142, @hehechen) +- Check process name before killing it in `tiup clean` (#2141, @dveeden) +- Display disaggregated mode of TiFlash in `tiup-cluster` (#2133, @hehechen) +- Remove mark_cache_size default value of tiflash.toml after v5.4.0 (#2138, @hongyunyan) +- Delete !#$%&= from init password in `tiup-cluster` (#2136, @nexustar) + +## [1.11.3] 2023-02-14 + +### Improvement + +- Disable telemetry by default ([#2119](https://github.com/pingcap/tiup/pull/2119), [@nexustar](https://github.com/nexustar)) + +## [1.11.2] 2023-02-02 + +### Fixes + +- base52: Check for invalid chars in `Decode()` ([#2103](https://github.com/pingcap/tiup/pull/2103), [@AstroProfundis](https://github.com/AstroProfundis)) + +### Improvements + +- Update upgrade message in `tiup-cluster` ([#2110](https://github.com/pingcap/tiup/pull/2110), [@nexustar](https://github.com/nexustar)) +- Comment out tidb-dashboard in template in `tiup-cluster` ([#2112](https://github.com/pingcap/tiup/pull/2112), [@nexustar](https://github.com/nexustar)) +- Warn for cleanup failures due to missing datadir in `tiup clean` ([#2105](https://github.com/pingcap/tiup/pull/2105), [@dveeden](https://github.com/dveeden)) +- Improve tiup timeout message in `tiup` ([#2109](https://github.com/pingcap/tiup/pull/2109), [@nexustar](https://github.com/nexustar)) + +## [1.11.1] 2022-11-24 + +## Note + +`tiup bench` has been **moved** to https://github.com/PingCAP-QE/tiup-bench and not release with tiup + +### Fixes + +- Set `madvdontneed=1` for PD to shrink RSS after GC in `tiup-cluster` ([#2019](https://github.com/pingcap/tiup/pull/2019), [@lhy1024](https://github.com/lhy1024)) +- Fix playground may panic when component fail to start in `tiup-playground` ([#2076](https://github.com/pingcap/tiup/pull/2076), [@nexustar](https://github.com/nexustar)) + +### Improvements + +- Support configuring `scrape_interval` and `scrape_timeout` for prometheus in `tiup-cluster` ([#2071](https://github.com/pingcap/tiup/pull/2071), [@nexustar](https://github.com/nexustar)) +- Remove the restriction that tikv num must be greater than 0 in `tiup-playground` ([#2073](https://github.com/pingcap/tiup/pull/2073), [@lilinghai](https://github.com/lilinghai)) +- Mark kylin v10 as supported OS in `tiup-cluster` ([#2079](https://github.com/pingcap/tiup/pull/2079), [@nexustar](https://github.com/nexustar)) +- Set `ETCDCTL_API=v3` to get the v3 API in `tiup-ctl` ([#2081](https://github.com/pingcap/tiup/pull/2081), [@dveeden](https://github.com/dveeden)) +- Avoid redirect when grafana `root_url` is set in `tiup-cluster` ([#2082](https://github.com/pingcap/tiup/pull/2082), [@nexustar](https://github.com/nexustar)) + ## [1.11.0] 2022-09-23 ### New Features diff --git a/Makefile b/Makefile index 2bcc2222d9..9afaddcc6c 100644 --- a/Makefile +++ b/Makefile @@ -69,14 +69,6 @@ dm: @# Target: build the tiup-dm component $(GOBUILD) -ldflags '$(LDFLAGS)' -o bin/tiup-dm ./components/dm -doc: - @# Target: build the tiup-doc component - $(GOBUILD) -ldflags '$(LDFLAGS)' -o bin/tiup-doc ./components/doc - -errdoc: - @# Target: build the tiup-errdoc component - $(GOBUILD) -ldflags '$(LDFLAGS)' -o bin/tiup-errdoc ./components/errdoc - ctl: @# Target: build the tiup-ctl component $(GOBUILD) -ldflags '$(LDFLAGS)' -o bin/tiup-ctl ./components/ctl @@ -158,5 +150,5 @@ tools/bin/revive: tools/check/go.mod $(GO) build -o ../bin/revive github.com/mgechev/revive tools/bin/golangci-lint: - @# Target: pull in specific version of golangci-lint (v1.50.0) - curl -sfL https://raw.githubusercontent.com/golangci/golangci-lint/master/install.sh | sh -s -- -b ./tools/bin v1.50.0 + @# Target: pull in specific version of golangci-lint (v1.54.1) + curl -sfL https://raw.githubusercontent.com/golangci/golangci-lint/master/install.sh | sh -s -- -b ./tools/bin v1.54.1 diff --git a/OWNERS b/OWNERS new file mode 100644 index 0000000000..495d05a59e --- /dev/null +++ b/OWNERS @@ -0,0 +1,15 @@ +# See the OWNERS docs at https://go.k8s.io/owners +approvers: +# tiup-maintainers: +- kaaaaaaang +# tiup-committers: 0 members +emeritus_approvers: +- lonng +- lucklove +- AstroProfundis +reviewers: +# tiup-reviewers: +- nexustar +- breezewish +- srstack +- iguoyr \ No newline at end of file diff --git a/cmd/clean.go b/cmd/clean.go index 8510815269..b5ec18a598 100644 --- a/cmd/clean.go +++ b/cmd/clean.go @@ -67,13 +67,27 @@ func cleanData(env *environment.Environment, names []string, all bool) error { return err } if process == nil { + fmt.Fprintf(os.Stderr, "Can't clean directory due to missing meta file: %s\n", + filepath.Join(dataDir, dir.Name())) continue } if p, err := gops.NewProcess(int32(process.Pid)); err == nil { - fmt.Printf("Kill instance of `%s`, pid: %v\n", process.Component, process.Pid) - if err := p.Kill(); err != nil { - return err + pName, err := p.Name() + if err != nil { + fmt.Fprintf(os.Stderr, "Failed to get process info for `%s`, pid: %v\n", + process.Component, process.Pid) + } else { + if pName != "tiup-playground" { + fmt.Printf("Process name mismatch (`%s` != `tiup-playground`, not killing it.\n", + pName) + } else { + fmt.Printf("Kill instance of `%s`, pid: %v\n", + process.Component, process.Pid) + if err := p.Kill(); err != nil { + return err + } + } } } diff --git a/cmd/link.go b/cmd/link.go new file mode 100644 index 0000000000..f54b462433 --- /dev/null +++ b/cmd/link.go @@ -0,0 +1,37 @@ +// Copyright 2021 PingCAP, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// See the License for the specific language governing permissions and +// limitations under the License. + +package cmd + +import ( + "github.com/pingcap/tiup/pkg/environment" + "github.com/spf13/cobra" +) + +func newLinkCmd() *cobra.Command { + cmd := &cobra.Command{ + Use: "link [:version]", + Short: "Link component binary to $TIUP_HOME/bin/", + Long: `[experimental] Link component binary to $TIUP_HOME/bin/`, + RunE: func(cmd *cobra.Command, args []string) error { + teleCommand = cmd.CommandPath() + env := environment.GlobalEnv() + if len(args) != 1 { + return cmd.Help() + } + component, version := environment.ParseCompVersion(args[0]) + return env.Link(component, version) + }, + } + return cmd +} diff --git a/cmd/mirror.go b/cmd/mirror.go index 2847f6bc8a..980ae44191 100644 --- a/cmd/mirror.go +++ b/cmd/mirror.go @@ -29,7 +29,6 @@ import ( "github.com/fatih/color" perrs "github.com/pingcap/errors" - "github.com/pingcap/tiup/pkg/cluster/spec" "github.com/pingcap/tiup/pkg/environment" "github.com/pingcap/tiup/pkg/localdata" "github.com/pingcap/tiup/pkg/repository" @@ -132,7 +131,7 @@ func newMirrorSignCmd() *cobra.Command { return err } - if err = os.WriteFile(args[0], data, 0664); err != nil { + if err = utils.WriteFile(args[0], data, 0664); err != nil { return perrs.Annotatef(err, "write manifest file %s", args[0]) } @@ -990,10 +989,6 @@ func newMirrorCloneCmd() *cobra.Command { } }() - var versionMapper = func(comp string) string { - return spec.TiDBComponentVersion(comp, "") - } - // format input versions versionList := make([]string, 0) for _, ver := range args[1:] { @@ -1004,7 +999,7 @@ func newMirrorCloneCmd() *cobra.Command { versionList = append(versionList, v) } - return repository.CloneMirror(repo, components, versionMapper, args[0], versionList, options) + return repository.CloneMirror(repo, components, args[0], versionList, options) }, } diff --git a/cmd/root.go b/cmd/root.go index abd7edfd52..173c645c31 100644 --- a/cmd/root.go +++ b/cmd/root.go @@ -225,6 +225,8 @@ the latest stable version will be downloaded from the repository.`, newTelemetryCmd(), newEnvCmd(), newHistoryCmd(), + newLinkCmd(), + newUnlinkCmd(), ) } diff --git a/cmd/uninstall.go b/cmd/uninstall.go index 058727cd4c..0b243bc6cc 100644 --- a/cmd/uninstall.go +++ b/cmd/uninstall.go @@ -51,7 +51,7 @@ which is used to uninstall tiup. teleCommand = cmd.CommandPath() env := environment.GlobalEnv() if self { - deletable := []string{"bin", "manifest", "manifests", "components", "storage/cluster/packages"} + deletable := []string{"storage/cluster/packages", "components", "manifests", "manifest", "bin"} for _, dir := range deletable { if err := os.RemoveAll(env.Profile().Path(dir)); err != nil { return errors.Trace(err) @@ -86,6 +86,9 @@ func removeComponents(env *environment.Environment, specs []string, all bool) er if strings.Contains(spec, ":") { parts := strings.SplitN(spec, ":", 2) // after this version is deleted, component will have no version left. delete the whole component dir directly + if !utils.IsExist(env.LocalPath(localdata.ComponentParentDir, parts[0])) { + return errors.Trace(fmt.Errorf("component `%s` is not installed, please use `tiup list %s` to check", parts[0], parts[0])) + } dir, err := os.ReadDir(env.LocalPath(localdata.ComponentParentDir, parts[0])) if err != nil { return errors.Trace(err) @@ -99,6 +102,7 @@ func removeComponents(env *environment.Environment, specs []string, all bool) er } else { paths = append(paths, env.LocalPath(localdata.ComponentParentDir, parts[0], parts[1])) } + // if no more version left, delete the whole component dir if len(dir)-len(paths) < 1 { paths = append(paths, env.LocalPath(localdata.ComponentParentDir, parts[0])) } @@ -110,6 +114,10 @@ func removeComponents(env *environment.Environment, specs []string, all bool) er paths = append(paths, env.LocalPath(localdata.ComponentParentDir, spec)) } for _, path := range paths { + if !utils.IsExist(path) { + return errors.Trace(fmt.Errorf("component `%s` is not installed, please check", spec)) + } + fmt.Println(path) if err := os.RemoveAll(path); err != nil { return errors.Trace(err) } diff --git a/cmd/unlink.go b/cmd/unlink.go new file mode 100644 index 0000000000..d31c74b4cb --- /dev/null +++ b/cmd/unlink.go @@ -0,0 +1,53 @@ +// Copyright 2021 PingCAP, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// See the License for the specific language governing permissions and +// limitations under the License. + +package cmd + +import ( + "os" + "path/filepath" + + "github.com/pingcap/tiup/pkg/environment" + "github.com/pingcap/tiup/pkg/tui" + "github.com/spf13/cobra" +) + +func newUnlinkCmd() *cobra.Command { + cmd := &cobra.Command{ + Use: "unlink ", + Short: "Unlink component binary to $TIUP_HOME/bin/", + Long: `[experimental] Unlink component binary in $TIUP_HOME/bin/`, + RunE: func(cmd *cobra.Command, args []string) error { + teleCommand = cmd.CommandPath() + env := environment.GlobalEnv() + if len(args) != 1 { + return cmd.Help() + } + component, version := environment.ParseCompVersion(args[0]) + version, err := env.SelectInstalledVersion(component, version) + if err != nil { + return err + } + binPath, err := env.BinaryPath(component, version) + if err != nil { + return err + } + target := env.LocalPath("bin", filepath.Base(binPath)) + if err := tui.PromptForConfirmOrAbortError("%s will be removed.\n Do you want to continue? [y/N]:", target); err != nil { + return err + } + return os.Remove(target) + }, + } + return cmd +} diff --git a/components/client/go.mod b/components/client/go.mod index c72f22d217..8cf40d0b71 100644 --- a/components/client/go.mod +++ b/components/client/go.mod @@ -1,6 +1,6 @@ module github.com/tiup/components/client -go 1.18 +go 1.21 require ( github.com/gizak/termui/v3 v3.1.0 @@ -13,7 +13,7 @@ require ( github.com/alecthomas/chroma v0.9.4 // indirect github.com/dlclark/regexp2 v1.4.0 // indirect github.com/go-ole/go-ole v1.2.6 // indirect - github.com/go-sql-driver/mysql v1.6.0 // indirect + github.com/go-sql-driver/mysql v1.7.1 // indirect github.com/gohxs/readline v0.0.0-20171011095936-a780388e6e7c // indirect github.com/inconshreveable/mousetrap v1.0.0 // indirect github.com/mattn/go-isatty v0.0.14 // indirect diff --git a/components/client/go.sum b/components/client/go.sum index e3af98fcd1..ae9d2750c9 100644 --- a/components/client/go.sum +++ b/components/client/go.sum @@ -35,6 +35,7 @@ cloud.google.com/go/bigquery v1.5.0/go.mod h1:snEHRnqQbz117VIFhE8bmtwIDY80NLUZUM cloud.google.com/go/bigquery v1.7.0/go.mod h1://okPTzCYNXSlb24MZs83e2Do+h+VXtc4gLoIoXIAPc= cloud.google.com/go/bigquery v1.8.0/go.mod h1:J5hqkt3O0uAFnINi6JXValWIb1v0goeZM77hZzJN/fQ= cloud.google.com/go/bigquery v1.24.0 h1:HpSE9zWHkLxEcEglpzGuAOkdMQr8lWxRtWITIjbgplY= +cloud.google.com/go/bigquery v1.24.0/go.mod h1:TuYTJSF39gNCsiXccewKQNjq5K6m3PnRNq42rT49eC8= cloud.google.com/go/datastore v1.0.0/go.mod h1:LXYbyblFSglQ5pkeyhO+Qmw7ukd3C+pD7TKLgZqpHYE= cloud.google.com/go/datastore v1.1.0/go.mod h1:umbIZjpQpHh4hmRpGhH4tLFup+FVzqBi1b3c64qFpCk= cloud.google.com/go/firestore v1.6.1/go.mod h1:asNXNOzBdyVQmEU+ggO8UPodTkEVFW5Qx+rwHnAz+EY= @@ -43,6 +44,7 @@ cloud.google.com/go/pubsub v1.1.0/go.mod h1:EwwdRX2sKPjnvnqCa270oGRyludottCI76h+ cloud.google.com/go/pubsub v1.2.0/go.mod h1:jhfEVHT8odbXTkndysNHCcx0awwzvfOlguIAii9o8iA= cloud.google.com/go/pubsub v1.3.1/go.mod h1:i+ucay31+CNRpDW4Lu78I4xXG+O1r/MAHgjpRVR+TSU= cloud.google.com/go/spanner v1.27.0 h1:F5CmUC0pc83kigWjG3YfXfV02+ip3G394jLYzbDtoEQ= +cloud.google.com/go/spanner v1.27.0/go.mod h1:/YWC/deZAtNGNmUXUJBjzcprotUO8q0q0AwPISFncwg= cloud.google.com/go/storage v1.0.0/go.mod h1:IhtSnM/ZTZV8YYJWCY8RULGVqBDmpoyjwiyrjsg+URw= cloud.google.com/go/storage v1.5.0/go.mod h1:tpKbwo567HUNpVclU5sGELwQWBDZ8gh0ZeosJ0Rtdos= cloud.google.com/go/storage v1.6.0/go.mod h1:N7U0C8pVQ/+NIKOBQyamJIeKQKkZ+mxpohlUTyfDhBk= @@ -50,21 +52,33 @@ cloud.google.com/go/storage v1.8.0/go.mod h1:Wv1Oy7z6Yz3DshWRJFhqM/UCfaWIRTdp0RX cloud.google.com/go/storage v1.10.0/go.mod h1:FLPqc6j+Ki4BU591ie1oL6qBQGu2Bl/tZ9ullr3+Kg0= dmitri.shuralyov.com/gpu/mtl v0.0.0-20190408044501-666a987793e9/go.mod h1:H6x//7gZCb22OMCxBHrMx7a5I7Hp++hsVxbQ4BYO7hU= github.com/Azure/azure-pipeline-go v0.2.3 h1:7U9HBg1JFK3jHl5qmo4CTZKFTVgMwdFHMVtCdfBE21U= +github.com/Azure/azure-pipeline-go v0.2.3/go.mod h1:x841ezTBIMG6O3lAcl8ATHnsOPVl2bqk7S3ta6S6u4k= github.com/Azure/azure-storage-blob-go v0.14.0 h1:1BCg74AmVdYwO3dlKwtFU1V0wU2PZdREkXvAmZJRUlM= +github.com/Azure/azure-storage-blob-go v0.14.0/go.mod h1:SMqIBi+SuiQH32bvyjngEewEeXoPfKMgWlBDaYf6fck= github.com/Azure/go-ansiterm v0.0.0-20170929234023-d6e3b3328b78 h1:w+iIsaOQNcT7OZ575w+acHgRric5iCyQh+xv+KJ4HB8= +github.com/Azure/go-ansiterm v0.0.0-20170929234023-d6e3b3328b78/go.mod h1:LmzpDX56iTiv29bbRTIsUNlaFfuhWRQBWjQdVyAevI8= github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802/go.mod h1:IVnqGOEym/WlBOVXweHU+Q+/VP0lqqI8lqeDx9IjBqo= github.com/ClickHouse/clickhouse-go v1.5.1 h1:I8zVFZTz80crCs0FFEBJooIxsPcV0xfthzK1YrkpJTc= +github.com/ClickHouse/clickhouse-go v1.5.1/go.mod h1:EaI/sW7Azgz9UATzd5ZdZHRUhHgv5+JMS9NSr2smCJI= github.com/DATA-DOG/go-sqlmock v1.5.0 h1:Shsta01QNfFxHCfpW6YH2STWB0MudeXXEWMr20OEh60= +github.com/DATA-DOG/go-sqlmock v1.5.0/go.mod h1:f/Ixk793poVmq4qj/V1dPUg2JEAKC73Q5eFN3EC/SaM= github.com/DataDog/datadog-go v3.2.0+incompatible/go.mod h1:LButxg5PwREeZtORoXG3tL4fMGNddJ+vMq1mwgfaqoQ= github.com/IBM/nzgo v0.0.0-20210406171630-186d127e2795 h1:W9EhSwnh9MrP4MW89gKCoe/t59uZkDL3bGMTirPyvyI= +github.com/IBM/nzgo v0.0.0-20210406171630-186d127e2795/go.mod h1:n1QK6KJjNa8fe+HQPynW+mJpRpkffLXMO8LR9Nja0JU= github.com/Masterminds/semver v1.5.0 h1:H65muMkzWKEuNDnfl9d70GUjFniHKHRbFPGBuZ3QEww= +github.com/Masterminds/semver v1.5.0/go.mod h1:MB6lktGJrhw8PrUyiEoblNEGEQ+RzHPF078ddwwvV3Y= github.com/MichaelS11/go-cql-driver v0.1.1 h1:ntFKov/39Tl36HckP4tzld3XMeyDYHHO00MiZNdoL1A= +github.com/MichaelS11/go-cql-driver v0.1.1/go.mod h1:rMwGk5bMWiYI/If6r6dbqEfZG6nQLvqJHTplv5yTDaw= github.com/Microsoft/go-winio v0.4.17-0.20210211115548-6eac466e5fa3 h1:mw6pDQqv38/WGF1cO/jF5t/jyAJ2yi7CmtFLLO5tGFI= +github.com/Microsoft/go-winio v0.4.17-0.20210211115548-6eac466e5fa3/go.mod h1:JPGBdM1cNvN/6ISo+n8V5iA4v8pBzdOpzfwIujj1a84= github.com/Nvveen/Gotty v0.0.0-20120604004816-cd527374f1e5 h1:TngWCqHvy9oXAN6lEVMRuU21PR1EtLVZJmdB18Gu3Rw= +github.com/Nvveen/Gotty v0.0.0-20120604004816-cd527374f1e5/go.mod h1:lmUJ/7eu/Q8D7ML55dXQrVaamCz2vxCfdQBasLZfHKk= github.com/OneOfOne/xxhash v1.2.2/go.mod h1:HSdplMjZKSmBqAxg5vPj2TmRDmfkzw+cTzAElWljhcU= github.com/SAP/go-hdb v0.105.5 h1:mop9KOZU1ro9PjJLgqseHUZzwoOJh4d7f8Nrvf1a2Uo= +github.com/SAP/go-hdb v0.105.5/go.mod h1:xbtJDvjqm9MQhIAnalynGNAbqxolS9W02qQo/vBqyaU= github.com/VoltDB/voltdb-client-go v1.0.8 h1:Xhf4LREQumXaRSUixMt1eWea6+0dz/Yo9mkqyEULbj8= +github.com/VoltDB/voltdb-client-go v1.0.8/go.mod h1:mMhb5zwkT46Ef3NvkFqt+kX0j+ltQ2Sdqj9+ICq+Yto= github.com/alecthomas/chroma v0.9.4 h1:YL7sOAE3p8HS96T9km7RgvmsZIctqbK1qJ0b7hzed44= github.com/alecthomas/chroma v0.9.4/go.mod h1:jtJATyUxlIORhUOFNA9NZDWGAQ8wpxQQqNSB4rjA/1s= github.com/alecthomas/template v0.0.0-20160405071501-a0175ee3bccc/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc= @@ -72,41 +86,67 @@ github.com/alecthomas/template v0.0.0-20190718012654-fb15b899a751/go.mod h1:LOuy github.com/alecthomas/units v0.0.0-20151022065526-2efee857e7cf/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0= github.com/alecthomas/units v0.0.0-20190717042225-c3de453c63f4/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0= github.com/alexbrainman/odbc v0.0.0-20210605012845-39f8520b0d5f h1:qJp6jWdG+PBNCDtIwRpspahMaZ3hlfde/25ExBORKso= +github.com/alexbrainman/odbc v0.0.0-20210605012845-39f8520b0d5f/go.mod h1:c5eyz5amZqTKvY3ipqerFO/74a/8CYmXOahSr40c+Ww= github.com/amsokol/ignite-go-client v0.12.2 h1:q4Mr+UUiKVnR7ykjR1YARVS5jp+ZU6ekCIs0V4WgFDo= +github.com/amsokol/ignite-go-client v0.12.2/go.mod h1:K3tKJGcLQORFD+ds7f0f9fl88tv0KZcpfuNhzRyuLVE= github.com/antihax/optional v1.0.0/go.mod h1:uupD/76wgC+ih3iEmQUL+0Ugr19nfwCT1kdvxnR2qWY= github.com/apache/arrow/go/arrow v0.0.0-20210901201644-e9eeff1c9297 h1:c4n5hcewl0g/LOJMGz+nvMnru5G2AE0lWwK5cKQ63zo= +github.com/apache/arrow/go/arrow v0.0.0-20210901201644-e9eeff1c9297/go.mod h1:2qMFB56yOP3KzkB3PbYZ4AlUFg3a88F67TIx5lB/WwY= github.com/apache/calcite-avatica-go/v5 v5.0.0 h1:B0l4O6KGR/sRHDaAgFT+AdGFouvp5Y/B9/82ag5rhds= +github.com/apache/calcite-avatica-go/v5 v5.0.0/go.mod h1:mbmgGJJQk6WZfh7U9z0QOa2OjS4L9w5bVMj2LD6wwVM= github.com/apache/thrift v0.14.2 h1:hY4rAyg7Eqbb27GB6gkhUKrRAuc8xRjlNtJq+LseKeY= +github.com/apache/thrift v0.14.2/go.mod h1:cp2SuWMxlEZw2r+iP2GNCdIi4C1qmUzdZFSVb+bacwQ= github.com/armon/circbuf v0.0.0-20150827004946-bbbad097214e/go.mod h1:3U/XgcO3hCbHZ8TKRvWD2dDTCfh9M9ya+I9JpbB7O8o= github.com/armon/go-metrics v0.0.0-20180917152333-f0300d1749da/go.mod h1:Q73ZrmVTwzkszR9V5SSuryQ31EELlFMUz1kKyl939pY= github.com/armon/go-metrics v0.3.10/go.mod h1:4O98XIr/9W0sxpJ8UaYkvjk10Iff7SnFrb4QAOwNTFc= github.com/armon/go-radix v0.0.0-20180808171621-7fddfc383310/go.mod h1:ufUuZ+zHj4x4TnLV4JWEpy2hxWSpsRywHrMgIH9cCH8= github.com/armon/go-radix v1.0.0/go.mod h1:ufUuZ+zHj4x4TnLV4JWEpy2hxWSpsRywHrMgIH9cCH8= github.com/asaskevich/govalidator v0.0.0-20210307081110-f21760c49a8d h1:Byv0BzEl3/e6D5CLfI0j/7hiIEtvGVFPCZ7Ei2oq8iQ= +github.com/asaskevich/govalidator v0.0.0-20210307081110-f21760c49a8d/go.mod h1:WaHUgvxTVq04UNunO+XhnAqY/wQc+bxr74GqbsZ/Jqw= github.com/aws/aws-sdk-go v1.42.9 h1:8ptAGgA+uC2TUbdvUeOVSfBocIZvGE2NKiLxkAcn1GA= +github.com/aws/aws-sdk-go v1.42.9/go.mod h1:585smgzpB/KqRA+K3y/NL/oYRqQvpNJYvLm+LY1U59Q= github.com/aws/aws-sdk-go-v2 v1.9.0 h1:+S+dSqQCN3MSU5vJRu1HqHrq00cJn6heIMU7X9hcsoo= +github.com/aws/aws-sdk-go-v2 v1.9.0/go.mod h1:cK/D0BBs0b/oWPIcX/Z/obahJK1TT7IPVjy53i/mX/4= github.com/aws/aws-sdk-go-v2/credentials v1.4.0 h1:kmvesfjY861FzlCU9mvAfe01D9aeXcG2ZuC+k9F2YLM= +github.com/aws/aws-sdk-go-v2/credentials v1.4.0/go.mod h1:dgGR+Qq7Wjcd4AOAW5Rf5Tnv3+x7ed6kETXyS9WCuAY= github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.5.0 h1:zzH1xd1/PX7bFO4/BQwVQP+UcBfYieI1sMH9DA68xZY= +github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.5.0/go.mod h1:fPU0eFGnS47RyKHHs8BNcCKOm5oOA5xm0BlrZWsQT/A= github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.3.0 h1:gceOysEWNNwLd6cki65IMBZ4WAM0MwgBQq2n7kejoT8= +github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.3.0/go.mod h1:v8ygadNyATSm6elwJ/4gzJwcFhri9RqS8skgHKiwXPU= github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.3.0 h1:VNJ5NLBteVXEwE2F1zEXVmyIH58mZ6kIQGJoC7C+vkg= +github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.3.0/go.mod h1:R1KK+vY8AfalhG1AOu5e35pOD2SdoPKQCFLTvnxiohk= github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.6.0 h1:B/1pIeV/oFnrOwhoMA6ASX+qT4FzMqn1MYsPiIXgMqQ= +github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.6.0/go.mod h1:LKb3cKNQIMh+itGnEpKGcnL/6OIjPZqrtYah1w5f+3o= github.com/aws/aws-sdk-go-v2/service/s3 v1.14.0 h1:nR9j0xMxpXk6orC/C03fbHNrbb1NaXp8LdVV7V1oVLE= +github.com/aws/aws-sdk-go-v2/service/s3 v1.14.0/go.mod h1:Qit9H3zjAmF7CLHOkrepE9b2ndX/2l3scstsM5g2jSk= github.com/aws/smithy-go v1.8.0 h1:AEwwwXQZtUwP5Mz506FeXXrKBe0jA8gVM+1gEcSRooc= +github.com/aws/smithy-go v1.8.0/go.mod h1:SObp3lf9smib00L/v3U2eAKG8FyQ7iLrJnQiAmR5n+E= github.com/beltran/gohive v1.5.1 h1:lsOIka3miPuzz1s39KQZNXwlf90QKekBC6ZsuRUVlcU= +github.com/beltran/gohive v1.5.1/go.mod h1:BJbXAhof7gWk5+kl0y6Ox8TFDJ1xv6gwDksF7l15LFI= github.com/beltran/gosasl v0.0.0-20210629234946-b41ac5bb612a h1:bOmOqN7GrefYKBHR7KIILIQHjyqYB9h5hluINvBUY6I= +github.com/beltran/gosasl v0.0.0-20210629234946-b41ac5bb612a/go.mod h1:Qx8cW6jkI8riyzmklj80kAIkv+iezFUTBiGU0qHhHes= github.com/beltran/gssapi v0.0.0-20200324152954-d86554db4bab h1:ayfcn60tXOSYy5zUN1AMSTQo4nJCf7hrdzAVchpPst4= +github.com/beltran/gssapi v0.0.0-20200324152954-d86554db4bab/go.mod h1:GLe4UoSyvJ3cVG+DVtKen5eAiaD8mAJFuV5PT3Eeg9Q= github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973/go.mod h1:Dwedo/Wpr24TaqPxmxbtue+5NUziq4I4S80YR8gNf3Q= github.com/beorn7/perks v1.0.0/go.mod h1:KWe93zE9D1o94FZ5RNwFwVgaQK1VOXiVxmqh+CedLV8= github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw= github.com/bgentry/speakeasy v0.1.0/go.mod h1:+zsyZBPWlz7T6j88CTgSN5bM796AkVf0kBD4zp0CCIs= github.com/bippio/go-impala v2.1.0+incompatible h1:H/N1Ms5KhVa2IoRZ6NO9ZzBryfTGhNiLG/zmNXS0CHY= +github.com/bippio/go-impala v2.1.0+incompatible/go.mod h1:lcyV/9s/ri5lFj3zdyyneQwDso8/Fd62fELt05Wts8g= github.com/btnguyen2k/consu/gjrc v0.1.1 h1:2ZXT2ySAFt5yJbdR2BAbwRKl5OjcysJeg3pzF4Hw5bE= +github.com/btnguyen2k/consu/gjrc v0.1.1/go.mod h1:jOTI8M8Kkly9fHyeSoBTw3o2Q9ayQEEYTHQfedIEqzE= github.com/btnguyen2k/consu/olaf v0.1.3 h1:0dWWmN5nOB/9pJdo7o1S3wR2+l3kG7pXHv3Vwki8uNM= +github.com/btnguyen2k/consu/olaf v0.1.3/go.mod h1:6ybEnJcdcK/PNiSfkKnMoxYuKyH2vJPBvHRuuZpPvD8= github.com/btnguyen2k/consu/reddo v0.1.7 h1:YhU/UWSzUki3WUi31lsByx+OWNCKs9uKjiy8G7MmwW0= +github.com/btnguyen2k/consu/reddo v0.1.7/go.mod h1:pdY5oIVX3noZIaZu3nvoKZ59+seXL/taXNGWh9xJDbg= github.com/btnguyen2k/consu/semita v0.1.5 h1:fu71xNJTbCV8T+6QPJdJu3bxtmLWvTjCepkvujF74+I= +github.com/btnguyen2k/consu/semita v0.1.5/go.mod h1:fksCe3L4kxiJVnKKhUXKI8mcFdB9974mtedwUVVFu1M= github.com/btnguyen2k/gocosmos v0.1.4 h1:9R12Hvk5/6HV9HYNLr103z0LNzWt9FJTRDaDPSV8lao= +github.com/btnguyen2k/gocosmos v0.1.4/go.mod h1:BYrAnNLG+e5NCz9q0X9dkjhMM1XkYhyRG74bQX4Wb1w= github.com/buger/jsonparser v1.1.1 h1:2PnMjfWD7wBILjqQbt530v576A/cAbQvEW9gGIpYMUs= +github.com/buger/jsonparser v1.1.1/go.mod h1:6RYKKt7H4d4+iWqouImQ9R2FZql3VbhNgx27UK13J/0= github.com/cenkalti/backoff/v3 v3.0.0 h1:ske+9nBpD9qZsTBoF41nW5L+AIuFBKMeze18XQ3eG1c= +github.com/cenkalti/backoff/v3 v3.0.0/go.mod h1:cIeZDE3IrqwwJl6VUwCN6trj1oXrTS4rc0ij+ULvLYs= github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU= github.com/census-instrumentation/opencensus-proto v0.3.0 h1:t/LhUZLVitR1Ow2YOnduCsavhwFUklBMoGVYUCqmCqk= github.com/census-instrumentation/opencensus-proto v0.3.0/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU= @@ -124,7 +164,9 @@ github.com/circonus-labs/circonus-gometrics v2.3.1+incompatible/go.mod h1:nmEj6D github.com/circonus-labs/circonusllhist v0.1.3/go.mod h1:kMXHVDlOchFAehlya5ePtbp5jckzBHf4XRpQvBOLI+I= github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw= github.com/cloudflare/golz4 v0.0.0-20150217214814-ef862a3cdc58 h1:F1EaeKL/ta07PY/k9Os/UFtwERei2/XzGemhpGnBKNg= +github.com/cloudflare/golz4 v0.0.0-20150217214814-ef862a3cdc58/go.mod h1:EOBUe0h4xcZ5GoxqC5SDxFQ8gwyZPKQoEzownBlhI80= github.com/cloudspannerecosystem/go-sql-spanner v0.0.0-20211102174514-f5f9ecd7ca01 h1:rCois7yiTyOqN0+OzK02yba6p4jAzDx7nyzFsAtY7TY= +github.com/cloudspannerecosystem/go-sql-spanner v0.0.0-20211102174514-f5f9ecd7ca01/go.mod h1:TIVV66EXen7aCX7vJg4akbc4BwkYC+kn0RADJk0h73s= github.com/cncf/udpa/go v0.0.0-20191209042840-269d4d468f6f/go.mod h1:M8M6+tZqaGXZJjfX53e64911xZQV5JYwmTeXPW+k8Sc= github.com/cncf/udpa/go v0.0.0-20200629203442-efcf912fb354/go.mod h1:WmhPx2Nbnhtbo57+VJT5O0JRkEi1Wbu0z5j0R8u5Hbk= github.com/cncf/udpa/go v0.0.0-20201120205902-5459f2c99403/go.mod h1:WmhPx2Nbnhtbo57+VJT5O0JRkEi1Wbu0z5j0R8u5Hbk= @@ -138,24 +180,34 @@ github.com/cncf/xds/go v0.0.0-20211011173535-cb28da3451f1/go.mod h1:eXthEFrGJvWH github.com/cncf/xds/go v0.0.0-20211130200136-a8f946100490 h1:KwaoQzs/WeUxxJqiJsZ4euOly1Az/IgZXXSxlD/UBNk= github.com/cncf/xds/go v0.0.0-20211130200136-a8f946100490/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs= github.com/containerd/continuity v0.0.0-20210208174643-50096c924a4e h1:6JKvHHt396/qabvMhnhUZvWaHZzfVfldxE60TK8YLhg= +github.com/containerd/continuity v0.0.0-20210208174643-50096c924a4e/go.mod h1:EXlVlkqNba9rJe3j7w3Xa924itAMLgZH4UD/Q4PExuQ= github.com/coreos/go-semver v0.3.0/go.mod h1:nnelYz7RCh+5ahJtPPxZlU+153eP4D4r3EedlOD2RNk= github.com/coreos/go-systemd/v22 v22.3.2/go.mod h1:Y58oyj3AT4RCenI/lSvhwexgC+NSVTIJ3seZv2GcEnc= github.com/couchbase/go-couchbase v0.1.1 h1:ClFXELcKj/ojyoTYbsY34QUrrYCBi/1G749sXSCkdhk= +github.com/couchbase/go-couchbase v0.1.1/go.mod h1:+/bddYDxXsf9qt0xpDUtRR47A2GjaXmGGAqQ/k3GJ8A= github.com/couchbase/go_n1ql v0.0.0-20160215142504-6cf4e348b127 h1:6Y3osD8rWOIAu/z5lrBns2cZSKSr6s+GEX64Ii0JtuI= +github.com/couchbase/go_n1ql v0.0.0-20160215142504-6cf4e348b127/go.mod h1:Rn19fO9CVfhJkqyIED9ixL5Kh5XuH7hXgDTxyfGY7hM= github.com/couchbase/gomemcached v0.1.4 h1:5n5wmr4dBu+X7XteP8QHP5S9inK9MBjNpN9b7WSQfuA= +github.com/couchbase/gomemcached v0.1.4/go.mod h1:mxliKQxOv84gQ0bJWbI+w9Wxdpt9HjDvgW9MjCym5Vo= github.com/couchbase/goutils v0.1.2 h1:gWr8B6XNWPIhfalHNog3qQKfGiYyh4K4VhO3P2o9BCs= +github.com/couchbase/goutils v0.1.2/go.mod h1:h89Ek/tiOxxqjz30nPPlwZdQbdB8BwgnuBxeoUe/ViE= github.com/cpuguy83/go-md2man/v2 v2.0.1 h1:r/myEWzV9lfsM1tFLgDyu0atFtJ1fXn261LKYj/3DxU= github.com/cpuguy83/go-md2man/v2 v2.0.1/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o= github.com/creack/pty v1.1.17 h1:QeVUsEDNrLBW4tMgZHvxy18sKtr6VI492kBhUfhDJNI= +github.com/creack/pty v1.1.17/go.mod h1:MOBLtS5ELjhRRrroQr9kyvTxUAFNvYEK993ew/Vr4O4= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/denisenkom/go-mssqldb v0.11.0 h1:9rHa233rhdOyrz2GcP9NM+gi2psgJZ4GWDpL/7ND8HI= +github.com/denisenkom/go-mssqldb v0.11.0/go.mod h1:xbL0rPBG9cCiLr28tMa8zpbdarY27NDyej4t/EjAShU= github.com/dlclark/regexp2 v1.4.0 h1:F1rxgk7p4uKjwIQxBs9oAXe5CqrXlCduYEJvrF4u93E= github.com/dlclark/regexp2 v1.4.0/go.mod h1:2pZnwuY/m+8K6iRw6wQdMtk+rH5tNGR1i55kozfMjCc= github.com/docker/go-connections v0.4.0 h1:El9xVISelRB7BuFusrZozjnkIM5YnzCViNKohAFqRJQ= +github.com/docker/go-connections v0.4.0/go.mod h1:Gbd7IOopHjR8Iph03tsViu4nIes5XhDvyHbTtUxmeec= github.com/docker/go-units v0.4.0 h1:3uh0PgVws3nIA0Q+MwDC8yjEPf9zjRfZZWXZYDct3Tw= +github.com/docker/go-units v0.4.0/go.mod h1:fgPhTUdO+D/Jk86RDLlptpiXQzgHJF7gydDDbaIK4Dk= github.com/edsrzf/mmap-go v1.0.0 h1:CEBF7HpRnUCSJgGUb5h1Gm7e3VkmVDrR8lvWVLtrOFw= +github.com/edsrzf/mmap-go v1.0.0/go.mod h1:YO35OhQPt3KJa3ryjFM5Bs14WD66h8eGKpfaBNrHW5M= github.com/envoyproxy/go-control-plane v0.9.0/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= github.com/envoyproxy/go-control-plane v0.9.1-0.20191026205805-5f8ba28d4473/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= github.com/envoyproxy/go-control-plane v0.9.4/go.mod h1:6rpuAdCZL397s3pYoYcLgu1mIlRU8Am5FuJP05cCM98= @@ -170,14 +222,18 @@ github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7 github.com/envoyproxy/protoc-gen-validate v0.6.2 h1:JiO+kJTpmYGjEodY7O1Zk8oZcNz1+f30UtwtXoFUPzE= github.com/envoyproxy/protoc-gen-validate v0.6.2/go.mod h1:2t7qjJNvHPx8IjnBOzl9E9/baC+qXE/TeeyBRzgJDws= github.com/exasol/error-reporting-go v0.1.0 h1:p81VbfknFLJHT1jnAlSa1iz3Ysx8sMSJnVnNlqarVqQ= +github.com/exasol/error-reporting-go v0.1.0/go.mod h1:im4x2ucQTPXFn+/B1EFcya6dnSUNYsuf6wC0DefHLK4= github.com/exasol/exasol-driver-go v0.2.0 h1:ASHugf1q5otYj2uqk3/ezKrQ6/kytqYEuBzw9arH21c= +github.com/exasol/exasol-driver-go v0.2.0/go.mod h1:3plWWycl0TqHpTY6rQxNImIIk5iLPfVfaRhwsPnP8rs= github.com/fatih/color v1.7.0/go.mod h1:Zm6kSWBoL9eyXnKyktHP6abPY2pDugNf5KwzbycvMj4= github.com/fatih/color v1.9.0/go.mod h1:eQcE1qtQxscV5RaZvpXrrb8Drkc3/DdQ+uUYCNjL+zU= github.com/fatih/color v1.13.0 h1:8LOYc1KYPPmyKMuN8QV2DNRWNbLo6LZ0iLs8+mlH53w= github.com/fatih/color v1.13.0/go.mod h1:kLAiJbzzSOZDVNGyDpeOxJ47H46qBXwg5ILebYFFOfk= github.com/form3tech-oss/jwt-go v3.2.5+incompatible h1:/l4kBbb4/vGSsdtB5nUe8L7B9mImVMaBPw9L/0TBHU8= +github.com/form3tech-oss/jwt-go v3.2.5+incompatible/go.mod h1:pbq4aXjuKjdthFRnoDwaVPLA+WlJuPGy+QneDUgJi2k= github.com/fsnotify/fsnotify v1.5.1/go.mod h1:T3375wBYaZdLLcVNkcVbzGHY7f1l/uK5T5Ai1i3InKU= github.com/genjidb/genji v0.13.0 h1:WKjuoy4TRemCxVgcZ1/+POGKhjkDj3ZSTB/zlHmS7l0= +github.com/genjidb/genji v0.13.0/go.mod h1:2QAT8ZovhZiEDQCaeOMI4HW5UiLm6PmXUxlUiKEuSjU= github.com/ghodss/yaml v1.0.0/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04= github.com/gizak/termui/v3 v3.1.0 h1:ZZmVDgwHl7gR7elfKf1xc4IudXZ5qqfDh4wExk4Iajc= github.com/gizak/termui/v3 v3.1.0/go.mod h1:bXQEBkJpzxUAKf0+xq9MSWAvWZlE7c+aidmyFlkYTrY= @@ -189,23 +245,31 @@ github.com/go-kit/kit v0.9.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2 github.com/go-logfmt/logfmt v0.3.0/go.mod h1:Qt1PoO58o5twSAckw1HlFXLmHsOX5/0LbT9GBnD5lWE= github.com/go-logfmt/logfmt v0.4.0/go.mod h1:3RMwSq7FuexP4Kalkev3ejPJsZTpXXBr9+V4qmtdjCk= github.com/go-logfmt/logfmt v0.5.0 h1:TrB8swr/68K7m9CcGut2g3UOihhbcbiMAYiuTXdEih4= +github.com/go-logfmt/logfmt v0.5.0/go.mod h1:wCYkCAKZfumFQihp8CzCvQ3paCTfi41vtzG1KdI/P7A= github.com/go-ole/go-ole v1.2.6 h1:/Fpf6oFPoeFik9ty7siob0G6Ke8QvQEuVcuChpwXzpY= github.com/go-ole/go-ole v1.2.6/go.mod h1:pprOEPIfldk/42T2oK7lQ4v4JSDwmV0As9GaiUsvbm0= github.com/go-openapi/errors v0.20.1 h1:j23mMDtRxMwIobkpId7sWh7Ddcx4ivaoqUbfXx5P+a8= +github.com/go-openapi/errors v0.20.1/go.mod h1:cM//ZKUKyO06HSwqAelJ5NsEMMcpa6VpXe8DOa1Mi1M= github.com/go-openapi/strfmt v0.21.0 h1:hX2qEZKmYks+t0hKeb4VTJpUm2UYsdL3+DCid5swxIs= -github.com/go-sql-driver/mysql v1.6.0 h1:BCTh4TKNUYmOmMUcQ3IipzF5prigylS7XXjEkfCHuOE= -github.com/go-sql-driver/mysql v1.6.0/go.mod h1:DCzpHaOWr8IXmIStZouvnhqoel9Qv2LBy8hT2VhHyBg= +github.com/go-openapi/strfmt v0.21.0/go.mod h1:ZRQ409bWMj+SOgXofQAGTIo2Ebu72Gs+WaRADcS5iNg= +github.com/go-sql-driver/mysql v1.7.1 h1:lUIinVbN1DY0xBg0eMOzmmtGoHwWBbvnWubQUrtU8EI= +github.com/go-sql-driver/mysql v1.7.1/go.mod h1:OXbVy3sEdcQ2Doequ6Z5BW6fXNQTmx+9S1MCJN5yJMI= github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/melR3HDY= github.com/go-stack/stack v1.8.1 h1:ntEHSVwIt7PNXNpgPmVfMrNhLtgjlmnZha2kOpuRiDw= +github.com/go-stack/stack v1.8.1/go.mod h1:dcoOX6HbPZSZptuspn9bctJ+N/CnF5gGygcUP3XYfe4= github.com/go-zookeeper/zk v1.0.2 h1:4mx0EYENAdX/B/rbunjlt5+4RTA/a9SMHBRuSKdGxPM= +github.com/go-zookeeper/zk v1.0.2/go.mod h1:nOB03cncLtlp4t+UAkGSV+9beXP/akpekBwL+UX1Qcw= github.com/gocql/gocql v0.0.0-20211015133455-b225f9b53fa1 h1:px9qUCy/RNJNsfCam4m2IxWGxNuimkrioEF0vrrbPsg= +github.com/gocql/gocql v0.0.0-20211015133455-b225f9b53fa1/go.mod h1:3gM2c4D3AnkISwBxGnMMsS8Oy4y2lhbPRsH4xnJrHG8= github.com/godbus/dbus/v5 v5.0.4/go.mod h1:xhWf0FNVPg57R7Z0UbKHbJfkEywrmjJnf7w5xrFpKfA= github.com/godror/godror v0.25.3 h1:ltL94Ct9otjMfUNTRMqyZh0GpepPd9f9pyFgtUciT9k= +github.com/godror/godror v0.25.3/go.mod h1:JgtdZ1iSaNoioa/B53BVVWji9J9iGPDDj2763T5d1So= github.com/gogo/protobuf v1.1.1/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ= github.com/gogo/protobuf v1.3.2/go.mod h1:P1XiOD3dCwIKUDQYPy72D8LYyHL2YPYrpS2s69NZV8Q= github.com/gohxs/readline v0.0.0-20171011095936-a780388e6e7c h1:yE35fKFwcelIte3q5q1/cPiY7pI7vvf5/j/0ddxNCKs= github.com/gohxs/readline v0.0.0-20171011095936-a780388e6e7c/go.mod h1:9S/fKAutQ6wVHqm1jnp9D9sc5hu689s9AaTWFS92LaU= github.com/golang-sql/civil v0.0.0-20190719163853-cb61b32ac6fe h1:lXe2qZdvpiX5WZkZR4hgp4KJVfY3nMkvmwbVkpv1rVY= +github.com/golang-sql/civil v0.0.0-20190719163853-cb61b32ac6fe/go.mod h1:8vg3r2VgvsThLBIFL93Qb5yWzgyZWhEmBwUJWevAkK0= github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q= github.com/golang/groupcache v0.0.0-20190702054246-869f871628b6/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= github.com/golang/groupcache v0.0.0-20191227052852-215e87163ea7/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= @@ -241,10 +305,13 @@ github.com/golang/protobuf v1.5.2 h1:ROPKBNFfQgOUMifHyP+KYbvpjbdoFNs+aK7DXlji0Tw github.com/golang/protobuf v1.5.2/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY= github.com/golang/snappy v0.0.3/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= github.com/golang/snappy v0.0.4 h1:yAGX7huGHXlcLOEtBnF4w7FQwA26wojNCwOYAEhLjQM= +github.com/golang/snappy v0.0.4/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= github.com/google/btree v0.0.0-20180813153112-4030bb1f1f0c/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= github.com/google/btree v1.0.0/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= github.com/google/btree v1.0.1 h1:gK4Kx5IaGY9CD5sPJ36FHiBJ6ZXl0kilRiiCj+jdYp4= +github.com/google/btree v1.0.1/go.mod h1:xXMiIv4Fb/0kKde4SpL7qlzvu5cMJDRkFDxJfI9uaxA= github.com/google/flatbuffers v2.0.0+incompatible h1:dicJ2oXwypfwUGnB2/TYWYEKiuk9eYQlQO/AnOHl5mI= +github.com/google/flatbuffers v2.0.0+incompatible/go.mod h1:1AeVuKshWv4vARoZatz6mlQ0JxURH0Kv5+zNeJKJCa8= github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M= github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= @@ -280,14 +347,17 @@ github.com/google/pprof v0.0.0-20210720184732-4bb14d4b1be1/go.mod h1:kpwsk12EmLe github.com/google/renameio v0.1.0/go.mod h1:KWCgfxg9yswjAJkECMjeO8J8rahYeXnNhOm40UhjYkI= github.com/google/uuid v1.1.2/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/google/uuid v1.3.0 h1:t6JiXgmwXMjEs8VusXIJk2BXHsn+wx8BZdTaoZ5fu7I= +github.com/google/uuid v1.3.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/googleapis/gax-go/v2 v2.0.4/go.mod h1:0Wqv26UfaUD9n4G6kQubkQ+KchISgw+vpHVxEJEs9eg= github.com/googleapis/gax-go/v2 v2.0.5/go.mod h1:DWXyrwAJ9X0FpwwEdw+IPEYBICEFu5mhpdKc/us6bOk= github.com/googleapis/gax-go/v2 v2.1.0/go.mod h1:Q3nei7sK6ybPYH7twZdmQpAd1MKb7pfu6SK+H1/DsU0= github.com/googleapis/gax-go/v2 v2.1.1 h1:dp3bWCh+PPO1zjRRiCSczJav13sBvG4UhNyVTa1KqdU= github.com/googleapis/gax-go/v2 v2.1.1/go.mod h1:hddJymUZASv3XPyGkUpKj8pPO47Rmb0eJc8R6ouapiM= github.com/gorilla/websocket v1.4.2 h1:+/TMaTYc4QFitKJxsQ7Yye35DkWvkdLcvGKqM+x0Ufc= +github.com/gorilla/websocket v1.4.2/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE= github.com/grpc-ecosystem/grpc-gateway v1.16.0/go.mod h1:BDjrQk3hbvj6Nolgz8mAMFbcEtjT1g+wF4CSlocrBnw= github.com/hailocab/go-hostpool v0.0.0-20160125115350-e80d13ce29ed h1:5upAirOpQc1Q53c0bnx2ufif5kANL7bfZWcc6VJWJd8= +github.com/hailocab/go-hostpool v0.0.0-20160125115350-e80d13ce29ed/go.mod h1:tMWxXQ9wFIaZeTI9F+hmhFiGpFmhOHzyShyFUhRm0H4= github.com/hashicorp/consul/api v1.11.0/go.mod h1:XjsvQN+RJGWI2TWy1/kqaE16HrR2J/FWgkYjdZQsX9M= github.com/hashicorp/consul/sdk v0.8.0/go.mod h1:GBvyrGALthsZObzUGsfgHZQDXjg4lOjagTIwIR1vPms= github.com/hashicorp/errwrap v1.0.0/go.mod h1:YH+1FKiLXxHSkmPseP+kNlulaMuP3n2brvKWEqk/Jc4= @@ -308,6 +378,7 @@ github.com/hashicorp/go-syslog v1.0.0/go.mod h1:qPfqrKkXGihmCqbJM2mZgkZGvKG1dFdv github.com/hashicorp/go-uuid v1.0.0/go.mod h1:6SBZvOh/SIDV7/2o3Jml5SYk/TvGqwFJ/bN7x4byOro= github.com/hashicorp/go-uuid v1.0.1/go.mod h1:6SBZvOh/SIDV7/2o3Jml5SYk/TvGqwFJ/bN7x4byOro= github.com/hashicorp/go-uuid v1.0.2 h1:cfejS+Tpcp13yd5nYHWDI6qVCny6wyX2Mt5SGur2IGE= +github.com/hashicorp/go-uuid v1.0.2/go.mod h1:6SBZvOh/SIDV7/2o3Jml5SYk/TvGqwFJ/bN7x4byOro= github.com/hashicorp/golang-lru v0.5.0/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= github.com/hashicorp/golang-lru v0.5.1/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= github.com/hashicorp/golang-lru v0.5.4/go.mod h1:iADmTwqILo4mZ8BN3D2Q6+9jd8WM5uGBxy+E8yxSoD4= @@ -325,22 +396,39 @@ github.com/ianlancetaylor/demangle v0.0.0-20200824232613-28f6c0f3b639/go.mod h1: github.com/inconshreveable/mousetrap v1.0.0 h1:Z8tu5sraLXCXIcARxBp/8cbvlwVa7Z1NHg9XEKhtSvM= github.com/inconshreveable/mousetrap v1.0.0/go.mod h1:PxqpIevigyE2G7u3NXJIT2ANytuPF1OarO4DADm73n8= github.com/jackc/chunkreader/v2 v2.0.1 h1:i+RDz65UE+mmpjTfyz0MoVTnzeYxroil2G82ki7MGG8= +github.com/jackc/chunkreader/v2 v2.0.1/go.mod h1:odVSm741yZoC3dpHEUXIqA9tQRhFrgOHwnPIn9lDKlk= github.com/jackc/pgconn v1.10.0 h1:4EYhlDVEMsJ30nNj0mmgwIUXoq7e9sMJrVC2ED6QlCU= +github.com/jackc/pgconn v1.10.0/go.mod h1:4z2w8XhRbP1hYxkpTuBjTS3ne3J48K83+u0zoyvg2pI= github.com/jackc/pgio v1.0.0 h1:g12B9UwVnzGhueNavwioyEEpAmqMe1E/BN9ES+8ovkE= +github.com/jackc/pgio v1.0.0/go.mod h1:oP+2QK2wFfUWgr+gxjoBH9KGBb31Eio69xUb0w5bYf8= github.com/jackc/pgpassfile v1.0.0 h1:/6Hmqy13Ss2zCq62VdNG8tM1wchn8zjSGOBJ6icpsIM= +github.com/jackc/pgpassfile v1.0.0/go.mod h1:CEx0iS5ambNFdcRtxPj5JhEz+xB6uRky5eyVu/W2HEg= github.com/jackc/pgproto3/v2 v2.1.1 h1:7PQ/4gLoqnl87ZxL7xjO0DR5gYuviDCZxQJsUlFW1eI= +github.com/jackc/pgproto3/v2 v2.1.1/go.mod h1:WfJCnwN3HIg9Ish/j3sgWXnAfK8A9Y0bwXYU5xKaEdA= github.com/jackc/pgservicefile v0.0.0-20200714003250-2b9c44734f2b h1:C8S2+VttkHFdOOCXJe+YGfa4vHYwlt4Zx+IVXQ97jYg= +github.com/jackc/pgservicefile v0.0.0-20200714003250-2b9c44734f2b/go.mod h1:vsD4gTJCa9TptPL8sPkXrLZ+hDuNrZCnj29CQpr4X1E= github.com/jackc/pgtype v1.8.1 h1:9k0IXtdJXHJbyAWQgbWr1lU+MEhPXZz6RIXxfR5oxXs= +github.com/jackc/pgtype v1.8.1/go.mod h1:LUMuVrfsFfdKGLw+AFFVv6KtHOFMwRgDDzBt76IqCA4= github.com/jackc/pgx/v4 v4.13.0 h1:JCjhT5vmhMAf/YwBHLvrBn4OGdIQBiFG6ym8Zmdx570= +github.com/jackc/pgx/v4 v4.13.0/go.mod h1:9P4X524sErlaxj0XSGZk7s+LD0eOyu1ZDUrrpznYDF0= github.com/jcmturner/aescts/v2 v2.0.0 h1:9YKLH6ey7H4eDBXW8khjYslgyqG2xZikXP0EQFKrle8= +github.com/jcmturner/aescts/v2 v2.0.0/go.mod h1:AiaICIRyfYg35RUkr8yESTqvSy7csK90qZ5xfvvsoNs= github.com/jcmturner/dnsutils/v2 v2.0.0 h1:lltnkeZGL0wILNvrNiVCR6Ro5PGU/SeBvVO/8c/iPbo= +github.com/jcmturner/dnsutils/v2 v2.0.0/go.mod h1:b0TnjGOvI/n42bZa+hmXL+kFJZsFT7G4t3HTlQ184QM= github.com/jcmturner/gofork v1.0.0 h1:J7uCkflzTEhUZ64xqKnkDxq3kzc96ajM1Gli5ktUem8= +github.com/jcmturner/gofork v1.0.0/go.mod h1:MK8+TM0La+2rjBD4jE12Kj1pCCxK7d2LK/UM3ncEo0o= github.com/jcmturner/goidentity/v6 v6.0.1 h1:VKnZd2oEIMorCTsFBnJWbExfNN7yZr3EhJAxwOkZg6o= +github.com/jcmturner/goidentity/v6 v6.0.1/go.mod h1:X1YW3bgtvwAXju7V3LCIMpY0Gbxyjn/mY9zx4tFonSg= github.com/jcmturner/gokrb5/v8 v8.4.2 h1:6ZIM6b/JJN0X8UM43ZOM6Z4SJzla+a/u7scXFJzodkA= +github.com/jcmturner/gokrb5/v8 v8.4.2/go.mod h1:sb+Xq/fTY5yktf/VxLsE3wlfPqQjp0aWNYyvBVK62bc= github.com/jcmturner/rpc/v2 v2.0.3 h1:7FXXj8Ti1IaVFpSAziCZWNzbNuZmnvw/i6CqLNdWfZY= +github.com/jcmturner/rpc/v2 v2.0.3/go.mod h1:VUJYCIDm3PVOEHw8sgt091/20OJjskO/YJki3ELg/Hc= github.com/jedib0t/go-pretty v4.3.0+incompatible h1:CGs8AVhEKg/n9YbUenWmNStRW2PHJzaeDodcfvRAbIo= +github.com/jedib0t/go-pretty v4.3.0+incompatible/go.mod h1:XemHduiw8R651AF9Pt4FwCTKeG3oo7hrHJAoznj9nag= github.com/jmespath/go-jmespath v0.4.0 h1:BEgLn5cpjn8UN1mAw4NjwDrS35OdebyEtFe+9YPoQUg= +github.com/jmespath/go-jmespath v0.4.0/go.mod h1:T8mJZnbsbmF+m6zOOFylbeCJqk5+pHWvzYPziyZiYoo= github.com/jmrobles/h2go v0.5.0 h1:r+V3J1+8z5tExKHcVc8u0tXJfov391zEffJYALWKhA0= +github.com/jmrobles/h2go v0.5.0/go.mod h1:p7Vjfu/9f7g2RI1CkpwXnwqskV+47HviBg4C4FlW8eI= github.com/json-iterator/go v1.1.6/go.mod h1:+SdeFBvtyEkXs7REEP0seUULqWtbJapLOCVDaaPEHmU= github.com/json-iterator/go v1.1.9/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4= github.com/json-iterator/go v1.1.11/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4= @@ -349,10 +437,13 @@ github.com/jstemmer/go-junit-report v0.0.0-20190106144839-af01ea7f8024/go.mod h1 github.com/jstemmer/go-junit-report v0.9.1/go.mod h1:Brl9GWCQeLvo8nXZwPNNblvFj/XSXhF0NWZEnDohbsk= github.com/julienschmidt/httprouter v1.2.0/go.mod h1:SYymIcj16QtmaHHD7aYtjjsJG7VTCxuUUipMqKk8s4w= github.com/kardianos/osext v0.0.0-20190222173326-2bc1f35cddc0 h1:iQTw/8FWTuc7uiaSepXwyf3o52HaUYcV+Tu66S3F5GA= +github.com/kardianos/osext v0.0.0-20190222173326-2bc1f35cddc0/go.mod h1:1NbS8ALrpOvjt0rHPNLyCIeMtbizbir8U//inJ+zuB8= github.com/kballard/go-shellquote v0.0.0-20180428030007-95032a82bc51 h1:Z9n2FFNUXsshfwJMBgNA0RU6/i7WVaAegv3PtuIHPMs= +github.com/kballard/go-shellquote v0.0.0-20180428030007-95032a82bc51/go.mod h1:CzGEWj7cYgsdH8dAjBGEr58BoE7ScuLd+fwFZ44+/x8= github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI2bnpBCr8= github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= github.com/klauspost/compress v1.13.6 h1:P76CopJELS0TiO2mebmnzgWaajssP/EszplttgQxcgc= +github.com/klauspost/compress v1.13.6/go.mod h1:/3/Vjq9QcHkK5uEr5lBEmyoZ1iFhe47etQ6QUkpK6sk= github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= github.com/kr/fs v0.1.0/go.mod h1:FFnZGqtBN9Gxj7eW1uZ42v5BccTP0vu6NEaFoC2HwRg= github.com/kr/logfmt v0.0.0-20140226030751-b84e30acd515/go.mod h1:+0opPa2QZZtGFBFZlji/RkVcI2GknAs/DXo4wKdlNEc= @@ -361,9 +452,11 @@ github.com/kr/pretty v0.2.0/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfn github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= github.com/lib/pq v1.10.4 h1:SO9z7FRPzA03QhHKJrH5BXA6HU1rS4V2nIVrrNC1iYk= +github.com/lib/pq v1.10.4/go.mod h1:AlVN5x4E4T544tWzH6hKfbfQvm3HdbOxrmggDNAPY9o= github.com/lyft/protoc-gen-star v0.5.3/go.mod h1:V0xaHgaf5oCCqmcxYcWiDfTiKsZsRc87/1qhoTACD8w= github.com/magiconair/properties v1.8.5/go.mod h1:y3VJvCyxH9uVvJTWEGAELF3aiYNyPKd5NZ3oSwXrF60= github.com/mattn/go-adodb v0.0.1 h1:g/pk3V8m/WFX2IQRI58wAC24OQUFFXEiNsvs7dQ1WKg= +github.com/mattn/go-adodb v0.0.1/go.mod h1:jaSTRde4bohMuQgYQPxW3xRTPtX/cZKyxPrFVseJULo= github.com/mattn/go-colorable v0.0.9/go.mod h1:9vuHe8Xs5qXnSaW/c/ABM9alt+Vo+STaOChaDxuIBZU= github.com/mattn/go-colorable v0.1.4/go.mod h1:U0ppj6V5qS13XJ6of8GYAs25YV2eR4EVcfRqFIhoBtE= github.com/mattn/go-colorable v0.1.6/go.mod h1:u6P/XSegPjTcexA+o6vUJrdnUu04hMope9wVRipJSqc= @@ -371,6 +464,7 @@ github.com/mattn/go-colorable v0.1.9/go.mod h1:u6P/XSegPjTcexA+o6vUJrdnUu04hMope github.com/mattn/go-colorable v0.1.12 h1:jF+Du6AlPIjs2BiUiQlKOX0rt3SujHxPnksPKZbaA40= github.com/mattn/go-colorable v0.1.12/go.mod h1:u5H1YNBxpqRaxsYJYSkiCWKzEfiAb1Gb520KVy5xxl4= github.com/mattn/go-ieproxy v0.0.1 h1:qiyop7gCflfhwCzGyeT0gro3sF9AIg9HU98JORTkqfI= +github.com/mattn/go-ieproxy v0.0.1/go.mod h1:pYabZ6IHcRpFh7vIaLfK7rdcWgFEb3SFJ6/gNWuh88E= github.com/mattn/go-isatty v0.0.3/go.mod h1:M+lRXTBqGeGNdLjl/ufCoiOlB5xdOkqRJdNxMWT7Zi4= github.com/mattn/go-isatty v0.0.8/go.mod h1:Iq45c/XA43vh69/j3iqttzPXn0bhXyGjM0Hdxcsrc5s= github.com/mattn/go-isatty v0.0.10/go.mod h1:qgIWMr58cqv1PHHyhnkY9lrL7etaEgOFcMEpPG5Rm84= @@ -382,6 +476,7 @@ github.com/mattn/go-runewidth v0.0.2/go.mod h1:LwmH8dsx7+W8Uxz3IHJYH5QSwggIsqBzp github.com/mattn/go-runewidth v0.0.13 h1:lTGmDsbAYt5DmK6OnoV7EuIF1wEIFAcxld6ypU4OSgU= github.com/mattn/go-runewidth v0.0.13/go.mod h1:Jdepj2loyihRzMpdS35Xk/zdY8IAYHsh153qUoGf23w= github.com/mattn/go-sqlite3 v2.0.3+incompatible h1:gXHsfypPkaMZrKbD5209QV9jbUTJKjyR5WD3HYQSd+U= +github.com/mattn/go-sqlite3 v2.0.3+incompatible/go.mod h1:FPy6KqzDD04eiIsT53CuJW3U88zkxoIYsOqkbpncsNc= github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5Ld7szi9bcBfOoFv/3dc6xSMkL2PC0= github.com/miekg/dns v1.0.14/go.mod h1:W1PPwlIAgtquWBMBEV9nkV9Cazfe8ScdGz/Lj7v3Nrg= github.com/miekg/dns v1.1.26/go.mod h1:bPDLeHnStXmXAq1m/Ch/hvfNHr14JKNPMBo3VZKjuso= @@ -397,12 +492,19 @@ github.com/mitchellh/mapstructure v1.1.2/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh github.com/mitchellh/mapstructure v1.4.3 h1:OVowDSCllw/YjdLkam3/sm7wEtOy59d8ndGgCcyj8cs= github.com/mitchellh/mapstructure v1.4.3/go.mod h1:bFUtVrKA4DC2yAKiSyO/QUcy7e+RRV2QTWOzhPopBRo= github.com/mithrandie/csvq v1.15.2 h1:m64b9WFGN7ACxnK3zWHEr+XPu+dvl3MnFzokaoHefPA= +github.com/mithrandie/csvq v1.15.2/go.mod h1:90MXOaH60IGYnudlDViKO3NZr5/HV8L4vqD5bf7re1E= github.com/mithrandie/csvq-driver v1.4.3 h1:BmAprI8cZzXrhldfVj1+XEVJuGbMaeXxWNOGZ1pRrOU= +github.com/mithrandie/csvq-driver v1.4.3/go.mod h1:K9H6vQEQnB/ryujBiUfRbuYAawtrAzadnRV1z/BnFBA= github.com/mithrandie/go-file/v2 v2.0.2 h1:3/yzItlTssDX9wOZrj9MtRyXbr52OZURmXFMuvpJ6Fg= +github.com/mithrandie/go-file/v2 v2.0.2/go.mod h1:98a9loPjYr7ffsfwMDdJ7iH/dO8EXAca8XiI6SZpPV8= github.com/mithrandie/go-text v1.4.2 h1:TPL4rDywSn796sRaE4U73xIW+0AYgwiALnBrP/pRHMs= +github.com/mithrandie/go-text v1.4.2/go.mod h1:Ivtyn3cuJYAs2UpihWmnY1KAixmLonELJ454EOZGzDg= github.com/mithrandie/readline-csvq v1.1.1 h1:kp9W5WPUAB+NOgW5axPdu8mZe1M9CP/D1xpabj39JVY= +github.com/mithrandie/readline-csvq v1.1.1/go.mod h1:eOJt0j6UI9lhwM/KP+v40ugarhXsnPIXStvkfIaq79E= github.com/mithrandie/ternary v1.1.0 h1:BlN8EoTsIYjhuWkfXHrh7+G+/Y0VvvWGVVldyjNH2VU= +github.com/mithrandie/ternary v1.1.0/go.mod h1:0D9Ba3+09K2TdSZO7/bFCC0GjSXetCvYuYq0u8FY/1g= github.com/moby/term v0.0.0-20201216013528-df9cb8a40635 h1:rzf0wL0CHVc8CEsgyygG0Mn9CNCCPZqOPaz8RiiHYQk= +github.com/moby/term v0.0.0-20201216013528-df9cb8a40635/go.mod h1:FBS0z0QWA44HXygs7VXDUOGoN/1TV3RuWkLO04am3wc= github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= github.com/modern-go/reflect2 v0.0.0-20180701023420-4b7aa43c6742/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0= @@ -410,30 +512,40 @@ github.com/modern-go/reflect2 v1.0.1/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3Rllmb github.com/modern-go/reflect2 v1.0.2/go.mod h1:yWuevngMOJpCy52FWWMvUC8ws7m/LJsjYzDa0/r8luk= github.com/mwitkow/go-conntrack v0.0.0-20161129095857-cc309e4a2223/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U= github.com/nakagami/firebirdsql v0.9.3 h1:/uNYufsFSRZF6n1xqzQxXY6b2TRD8o4aiPlb+qYGQH8= +github.com/nakagami/firebirdsql v0.9.3/go.mod h1:yU71hYllTfU4JbEysWLY2XovxsFnaLWkraaegJ7GW3M= github.com/nathan-fiscaletti/consolesize-go v0.0.0-20210105204122-a87d9f614b9d h1:PQW4Aqovdqc9efHl9EVA+bhKmuZ4ME1HvSYYDvaDiK0= github.com/nathan-fiscaletti/consolesize-go v0.0.0-20210105204122-a87d9f614b9d/go.mod h1:cxIIfNMTwff8f/ZvRouvWYF6wOoO7nj99neWSx2q/Es= github.com/nsf/termbox-go v0.0.0-20190121233118-02980233997d h1:x3S6kxmy49zXVVyhcnrFqxvNVCBPb2KZ9hV2RBdS840= github.com/nsf/termbox-go v0.0.0-20190121233118-02980233997d/go.mod h1:IuKpRQcYE1Tfu+oAQqaLisqDeXgjyyltCfsaoYN18NQ= github.com/oklog/ulid v1.3.1 h1:EGfNDEx6MqHz8B3uNV6QAib1UR2Lm97sHi3ocA6ESJ4= +github.com/oklog/ulid v1.3.1/go.mod h1:CirwcVhetQ6Lv90oh/F+FBtV6XMibvdAFo93nm5qn4U= github.com/opencontainers/go-digest v1.0.0 h1:apOUWs51W5PlhuyGyz9FCeeBIOUDA/6nW8Oi/yOhh5U= +github.com/opencontainers/go-digest v1.0.0/go.mod h1:0JzlMkj0TRzQZfJkVvzbP0HBR3IKzErnv2BNG4W4MAM= github.com/opencontainers/image-spec v1.0.1 h1:JMemWkRwHx4Zj+fVxWoMCFm/8sYGGrUVojFA6h/TRcI= +github.com/opencontainers/image-spec v1.0.1/go.mod h1:BtxoFyWECRxE4U/7sNtV5W15zMzWCbyJoFRP3s7yZA0= github.com/opencontainers/runc v1.0.0-rc93 h1:x2UMpOOVf3kQ8arv/EsDGwim8PTNqzL1/EYDr/+scOM= +github.com/opencontainers/runc v1.0.0-rc93/go.mod h1:3NOsor4w32B2tC0Zbl8Knk4Wg84SM2ImC1fxBuqJ/H0= github.com/ory/dockertest/v3 v3.6.3 h1:L8JWiGgR+fnj90AEOkTFIEp4j5uWAK72P3IUsYgn2cs= +github.com/ory/dockertest/v3 v3.6.3/go.mod h1:EFLcVUOl8qCwp9NyDAcCDtq/QviLtYswW/VbWzUnTNE= github.com/pascaldekloe/goe v0.0.0-20180627143212-57f6aae5913c/go.mod h1:lzWF7FIEvWOWxwDKqyGYQf6ZUaNfKdP144TG7ZOy1lc= github.com/pascaldekloe/goe v0.1.0/go.mod h1:lzWF7FIEvWOWxwDKqyGYQf6ZUaNfKdP144TG7ZOy1lc= github.com/pelletier/go-toml v1.9.4/go.mod h1:u1nR/EPcESfeI/szUZKdtJ0xRNbUoANCkoOuaOx1Y+c= github.com/pierrec/lz4 v2.2.6+incompatible h1:6aCX4/YZ9v8q69hTyiR7dNLnTA3fgtKHVVW5BCd5Znw= github.com/pierrec/lz4/v4 v4.1.8 h1:ieHkV+i2BRzngO4Wd/3HGowuZStgq6QkPsD1eolNAO4= +github.com/pierrec/lz4/v4 v4.1.8/go.mod h1:gZWDp/Ze/IJXGXf23ltt2EXimqmTUXEy0GFuRQyBid4= github.com/pkg/browser v0.0.0-20210706143420-7d21f8c997e2 h1:acNfDZXmm28D2Yg/c3ALnZStzNaZMSagpbr96vY6Zjc= +github.com/pkg/browser v0.0.0-20210706143420-7d21f8c997e2/go.mod h1:HKlIX3XHQyzLZPlr7++PzdhaXEj94dEiJgZDTsxEqUI= github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= +github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pkg/sftp v1.10.1/go.mod h1:lYOWFsE0bwd1+KfKJaKeuokY15vzFx25BLbzYYoAxZI= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/posener/complete v1.1.1/go.mod h1:em0nMJCgc9GFtwrmVmEMR/ZL6WyhyjMBndrE9hABlRI= github.com/posener/complete v1.2.3/go.mod h1:WZIdtGGp+qx0sLrYKtIRAruyNpv6hFCicSgv7Sy7s/s= github.com/prestodb/presto-go-client v0.0.0-20201204133205-8958eb37e584 h1:utxLhK2pYK3qDug5474YP2bbm2dbAPXA+dDrTvrkxeU= +github.com/prestodb/presto-go-client v0.0.0-20201204133205-8958eb37e584/go.mod h1:cwaFkElLIrI4vTXo5A1oDobUBFad0aVtZiZvfxJyX6I= github.com/prometheus/client_golang v0.9.1/go.mod h1:7SWBe2y4D6OKWSNQJUaRYU/AaXPKyh/dDVn+NZz0KFw= github.com/prometheus/client_golang v1.0.0/go.mod h1:db9x61etRT2tGnBNRi70OPL5FsnadC4Ky3P0J6CfImo= github.com/prometheus/client_golang v1.4.0/go.mod h1:e9GMxYsXl05ICDXkRhurwBS4Q3OK1iX/F2sw+iXX5zU= @@ -447,6 +559,7 @@ github.com/prometheus/procfs v0.0.0-20181005140218-185b4288413d/go.mod h1:c3At6R github.com/prometheus/procfs v0.0.2/go.mod h1:TjEm7ze935MbeOT/UhFTIMYKhuLP4wbCsTZCD3I8kEA= github.com/prometheus/procfs v0.0.8/go.mod h1:7Qr8sr6344vo1JqZ6HhLceV9o3AJ1Ff+GxbHq6oeK9A= github.com/remyoudompheng/bigfft v0.0.0-20200410134404-eec4a21b6bb0 h1:OdAsTTz6OkFY5QxjkYwrChwuRruF69c169dPK26NUlk= +github.com/remyoudompheng/bigfft v0.0.0-20200410134404-eec4a21b6bb0/go.mod h1:qqbHyh8v60DhA7CoWK5oRCqLrMHRGoxYCSS9EjAz6Eo= github.com/rivo/uniseg v0.2.0 h1:S1pD9weZBuJdFmowNwbpi7BJ8TNftyUImj/0WQi72jY= github.com/rivo/uniseg v0.2.0/go.mod h1:J6wj4VEh+S6ZtnVlnTBMWIodfgj8LQOQFoIToxlJtxc= github.com/rogpeppe/fastuuid v1.2.0/go.mod h1:jVj6XXZzXRy/MSR5jhDC/2q6DgLz+nrA6LYCDYWNEvQ= @@ -459,13 +572,18 @@ github.com/sean-/seed v0.0.0-20170313163322-e2103e2c3529/go.mod h1:DxrIzT+xaE7yg github.com/shirou/gopsutil v3.21.11+incompatible h1:+1+c1VGhc88SSonWP6foOcLhvnKlUeu/erjjvaPEYiI= github.com/shirou/gopsutil v3.21.11+incompatible/go.mod h1:5b4v6he4MtMOwMlS0TUMTu2PcXUg8+E1lC7eC3UO/RA= github.com/shopspring/decimal v1.3.1 h1:2Usl1nmF/WZucqkFZhnfFYxxxu8LG21F6nPQBE5gKV8= +github.com/shopspring/decimal v1.3.1/go.mod h1:DKyhrW/HYNuLGql+MJL6WCR6knT2jwCFRcu2hWCYk4o= github.com/sijms/go-ora/v2 v2.2.15 h1:GJfudyOHT+DUgBPvS2wP/8lgmuer3OKfha8C0xnThW8= +github.com/sijms/go-ora/v2 v2.2.15/go.mod h1:jzfAFD+4CXHE+LjGWFl6cPrtiIpQVxakI2gvrMF2w6Y= github.com/sirupsen/logrus v1.2.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo= github.com/sirupsen/logrus v1.4.2/go.mod h1:tLMulIdttU9McNUspp0xgXVQah82FyeX6MwdIuYE2rE= github.com/sirupsen/logrus v1.8.1 h1:dJKuHgqk1NNQlqoA6BTlM1Wf9DOH3NBjQyu0h9+AZZE= +github.com/sirupsen/logrus v1.8.1/go.mod h1:yWOB1SBYBC5VeMP7gHvWumXLIWorT60ONWic61uBYv0= github.com/snowflakedb/gosnowflake v1.6.1 h1:gaRt3oK7ATFmLgAg6Gw7aKvWhWts3WV33d0YE4Ofh2U= +github.com/snowflakedb/gosnowflake v1.6.1/go.mod h1:1kyg2XEduwti88V11PKRHImhXLK5WpGiayY6lFNYb98= github.com/spaolacci/murmur3 v0.0.0-20180118202830-f09979ecbc72/go.mod h1:JwIasOWyU6f++ZhiEuf87xNszmSA2myDM2Kzu9HwQUA= github.com/spaolacci/murmur3 v1.1.0 h1:7c1g84S4BPRrfL5Xrdp6fOJ206sU9y293DDHaoy0bLI= +github.com/spaolacci/murmur3 v1.1.0/go.mod h1:JwIasOWyU6f++ZhiEuf87xNszmSA2myDM2Kzu9HwQUA= github.com/spf13/afero v1.3.3/go.mod h1:5KUK8ByomD5Ti5Artl0RtHeI5pTF7MIDuXL3yY520V4= github.com/spf13/afero v1.6.0/go.mod h1:Ai8FlHk4v/PARR026UzYexafAt9roJ7LcLMAmO6Z93I= github.com/spf13/cast v1.4.1/go.mod h1:Qx5cxh0v+4UWYiBimWS+eyWzqEqokIECu5etghLkUJE= @@ -486,21 +604,32 @@ github.com/stretchr/testify v1.7.0 h1:nwc3DEeHmmLAfoZucVR881uASk0Mfjw8xYJ99tb5Cc github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/subosito/gotenv v1.2.0/go.mod h1:N0PQaV/YGNqwC0u51sEeR/aUtSLEXKX9iv69rRypqCw= github.com/thda/tds v0.1.7 h1:s29kbnJK0agL3ps85A/sb9XS2uxgKF5UJ6AZjbyqXX4= +github.com/thda/tds v0.1.7/go.mod h1:isLIF1oZdXfkqVMJM8RyNrsjlHPlTKnPlnsBs7ngZcM= github.com/tklauser/go-sysconf v0.3.9 h1:JeUVdAOWhhxVcU6Eqr/ATFHgXk/mmiItdKeJPev3vTo= github.com/tklauser/go-sysconf v0.3.9/go.mod h1:11DU/5sG7UexIrp/O6g35hrWzu0JxlwQ3LSFUzyeuhs= github.com/tklauser/numcpus v0.3.0 h1:ILuRUQBtssgnxw0XXIjKUC56fgnOrFoQQ/4+DeU2biQ= github.com/tklauser/numcpus v0.3.0/go.mod h1:yFGUr7TUHQRAhyqBcEg0Ge34zDBAsIvJJcyE6boqnA8= github.com/trinodb/trino-go-client v0.300.0 h1:DnTskW//HKm9oggYviFsYIMdLyl7TqlWs4yXhW9NlKE= +github.com/trinodb/trino-go-client v0.300.0/go.mod h1:/CMFmXqrFGmn76o/ZGefBLq7X6l0tsTJV4zrgQMVvvM= github.com/tv42/httpunix v0.0.0-20150427012821-b75d8614f926/go.mod h1:9ESjWnEqriFuLhtthL60Sar/7RFoluCcXsuvEwTV5KM= github.com/twmb/murmur3 v1.1.6 h1:mqrRot1BRxm+Yct+vavLMou2/iJt0tNVTTC0QoIjaZg= +github.com/twmb/murmur3 v1.1.6/go.mod h1:Qq/R7NUyOfr65zD+6Q5IHKsJLwP7exErjN6lyyq3OSQ= github.com/uber-go/tally v3.4.2+incompatible h1:wEKPHq3KIjguuHz/M6SXVjDlUTh+39OtnhlLWsfR7z0= +github.com/uber-go/tally v3.4.2+incompatible/go.mod h1:YDTIBxdXyOU/sCWilKB4bgyufu1cEi0jdVnRdxvjnmU= github.com/uber/athenadriver v1.1.13 h1:h67yRqRl0Vj5f30bn6Nsn42bI08rY+yielMn1pdUhV0= +github.com/uber/athenadriver v1.1.13/go.mod h1:mpa8cVqc/JH/xnViFoWFvNkoYyIj5ohn9mxAnBj3fik= github.com/unchartedsoftware/witch v0.0.0-20200617171400-4f405404126f h1:CfRRJzp88GWCAR3+GrJ/slqrbhCqesDRY/FmT6s4uF8= +github.com/unchartedsoftware/witch v0.0.0-20200617171400-4f405404126f/go.mod h1:xN7cr17jYCNJzZO78A4c0BcspGPbSAdFNq1NfDEE2do= github.com/urfave/cli v1.22.5 h1:lNq9sAHXK2qfdI8W+GRItjCEkI+2oR4d+MEHy1CKXoU= +github.com/urfave/cli v1.22.5/go.mod h1:Gos4lmkARVdJ6EkW0WaNv/tZAAMe9V7XWyB60NtXRu0= github.com/vertica/vertica-sql-go v1.2.0 h1:9znZNdweNWHarVf0iUEsUjnEihrtaAXkrVmb3JDnsx0= +github.com/vertica/vertica-sql-go v1.2.0/go.mod h1:fGr44VWdEvL+f+Qt5LkKLOT7GoxaWdoUCnPBU9h6t04= github.com/vmihailenco/msgpack/v5 v5.3.5 h1:5gO0H1iULLWGhs2H5tbAHIZTV8/cYafcFOr9znI5mJU= +github.com/vmihailenco/msgpack/v5 v5.3.5/go.mod h1:7xyJ9e+0+9SaZT0Wt1RGleJXzli6Q/V5KbhBonMG9jc= github.com/vmihailenco/tagparser/v2 v2.0.0 h1:y09buUbR+b5aycVFQs/g70pqKVZNBmxwAhO7/IwNM9g= +github.com/vmihailenco/tagparser/v2 v2.0.0/go.mod h1:Wri+At7QHww0WTrCBeu4J6bNtoV6mEfg5OIWRZA9qds= github.com/xinsnake/go-http-digest-auth-client v0.6.0 h1:nrYFWDrB2F7VwYlNravXZS0nOtg9axlATH3Jns55/F0= +github.com/xinsnake/go-http-digest-auth-client v0.6.0/go.mod h1:QK1t1v7ylyGb363vGWu+6Irh7gyFj+N7+UZzM0L6g8I= github.com/xo/dburl v0.9.0 h1:ME8QfRqZz/YDwf+VVEe9sq4wgEZCAOdYcUTeuAf+wQQ= github.com/xo/dburl v0.9.0/go.mod h1:7Uupe87dIDxNrbKFRrpw6bAf2l3/rqU42iwlpq1nyjY= github.com/xo/tblfmt v0.8.0 h1:/7DE0kyTdhFfdAK1jEUMDBqy1m78jRMSXGtD48Z4aUA= @@ -510,6 +639,7 @@ github.com/xo/terminfo v0.0.0-20210125001918-ca9a967f8778/go.mod h1:2MuV+tbUrU1z github.com/xo/usql v0.9.5 h1:II6gbWtwPQtMNw+wE32zz30wLUk8V9FWW1Km6lt0vgY= github.com/xo/usql v0.9.5/go.mod h1:DORND3Bhs6CglGmbFQdlA5OEw4IINiOKOLCbsiHFbhg= github.com/xwb1989/sqlparser v0.0.0-20180606152119-120387863bf2 h1:zzrxE1FKn5ryBNl9eKOeqQ58Y/Qpo3Q9QNxKHX5uzzQ= +github.com/xwb1989/sqlparser v0.0.0-20180606152119-120387863bf2/go.mod h1:hzfGeIUDq/j97IG+FhNqkowIyEcD88LrW6fyU3K3WqY= github.com/yuin/goldmark v1.1.25/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.1.32/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= @@ -520,12 +650,16 @@ github.com/yusufpapurcu/wmi v1.2.2/go.mod h1:SBZ9tNy3G9/m5Oi98Zks0QjeHVDvuK0qfxQ github.com/zaf/temp v0.0.0-20170209143821-94e385923345 h1:YirhcaVb0RNq54Vh/50S0MPEbr9b4tjZVXvoeeKoYyc= github.com/zaf/temp v0.0.0-20170209143821-94e385923345/go.mod h1:sXsZgXwh6DB0qlskmZVB4HE93e5YrktMrgUDPy9iYmY= github.com/ziutek/mymysql v1.5.4 h1:GB0qdRGsTwQSBVYuVShFBKaXSnSnYYC2d9knnE1LHFs= +github.com/ziutek/mymysql v1.5.4/go.mod h1:LMSpPZ6DbqWFxNCHW77HeMg9I646SAhApZ/wKdgO/C0= gitlab.com/nyarla/go-crypt v0.0.0-20160106005555-d9a5dc2b789b h1:7gd+rd8P3bqcn/96gOZa3F5dpJr/vEiDQYlNb/y2uNs= +gitlab.com/nyarla/go-crypt v0.0.0-20160106005555-d9a5dc2b789b/go.mod h1:T3BPAOm2cqquPa0MKWeNkmOM5RQsRhkrwMWonFMN7fE= go.etcd.io/bbolt v1.3.6 h1:/ecaJf0sk1l4l6V4awd65v2C3ILy7MSj+s/x1ADCIMU= +go.etcd.io/bbolt v1.3.6/go.mod h1:qXsaaIqmgQH0T+OPdb99Bf+PKfBBQVAdyD6TY9G8XM4= go.etcd.io/etcd/api/v3 v3.5.1/go.mod h1:cbVKeC6lCfl7j/8jBhAK6aIYO9XOjdptoxU/nLQcPvs= go.etcd.io/etcd/client/pkg/v3 v3.5.1/go.mod h1:IJHfcCEKxYu1Os13ZdwCwIUTUVGYTSAM3YSwc9/Ac1g= go.etcd.io/etcd/client/v2 v2.305.1/go.mod h1:pMEacxZW7o8pg4CrFE7pquyCJJzZvkvdD2RibOCCCGs= go.mongodb.org/mongo-driver v1.7.4 h1:sllcioag8Mec0LYkftYWq+cKNPIR4Kqq3iv9ZXY0g/E= +go.mongodb.org/mongo-driver v1.7.4/go.mod h1:NqaYOwnXWr5Pm7AOpO5QFxKJ503nbMse/R79oO62zWg= go.opencensus.io v0.21.0/go.mod h1:mSImk1erAIZhrmZN+AvHh14ztQfjbGwt4TtuofqLduU= go.opencensus.io v0.22.0/go.mod h1:+kGneAE2xo2IficOXnaByMWTGM9T73dGwxeWcUqIpI8= go.opencensus.io v0.22.2/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= @@ -537,10 +671,13 @@ go.opencensus.io v0.23.0/go.mod h1:XItmlyltB5F7CS4xOC1DcqMoFqwtC6OG2xF7mCv7P7E= go.opentelemetry.io/proto/otlp v0.7.0/go.mod h1:PqfVotwruBrMGOCsRd/89rSnXhoiJIqeYNgFYFoEGnI= go.uber.org/atomic v1.7.0/go.mod h1:fEN4uk6kAWBTFdckzkM89CLk9XfWZrxpCo0nPH17wJc= go.uber.org/atomic v1.9.0 h1:ECmE8Bn/WFTYwEW/bpKD3M8VtR/zQVbavAoalC1PYyE= +go.uber.org/atomic v1.9.0/go.mod h1:fEN4uk6kAWBTFdckzkM89CLk9XfWZrxpCo0nPH17wJc= go.uber.org/multierr v1.6.0/go.mod h1:cdWPpRnG4AhwMwsgIHip0KRBQjJy5kYEpYjJxpXp9iU= go.uber.org/multierr v1.7.0 h1:zaiO/rmgFjbmCXdSYJWQcdvOCsthmdaHfr3Gm2Kx4Ec= +go.uber.org/multierr v1.7.0/go.mod h1:7EAYxJLBy9rStEaz58O2t4Uvip6FSURkq8/ppBp95ak= go.uber.org/zap v1.17.0/go.mod h1:MXVU+bhUf/A7Xi2HNOnopQOrmycQ5Ih87HtOu4q5SSo= go.uber.org/zap v1.19.1 h1:ue41HOKd1vGURxrmeKIgELGb3jPW9DMUDGtsinblHwI= +go.uber.org/zap v1.19.1/go.mod h1:j3DNczoxDZroyBnOT1L/Q79cfUMGZxlv/9dzN7SM1rI= golang.org/x/crypto v0.0.0-20180904163835-0709b304e793/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= golang.org/x/crypto v0.0.0-20181029021203-45a5f77698d3/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= @@ -552,6 +689,7 @@ golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8U golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/crypto v0.0.0-20210817164053-32db794688a5/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= golang.org/x/crypto v0.0.0-20211117183948-ae814b36b871 h1:/pEO3GD/ABYAjuakUS6xSEmmlyVS4kxBNkeA9tLJiTI= +golang.org/x/crypto v0.0.0-20211117183948-ae814b36b871/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4= golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20190306152737-a1d7652674e8/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20190510132918-efd6b22b2522/go.mod h1:ZjyILWgesfNpC6sMxTJOJm9Kp84zZh5NQWvqDGG3Qr8= @@ -589,6 +727,7 @@ golang.org/x/mod v0.4.1/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.4.2/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.5.0/go.mod h1:5OXOZSfqPIIbmVBIIKWRFfZjPR0E5r58TLhUjH0a2Ro= golang.org/x/mod v0.5.1 h1:OJxoQ/rynoF0dcCdI7cLPktw/hR2cueqYfjm43oqK38= +golang.org/x/mod v0.5.1/go.mod h1:5OXOZSfqPIIbmVBIIKWRFfZjPR0E5r58TLhUjH0a2Ro= golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20181023162649-9b4f9f5ad519/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= @@ -631,6 +770,7 @@ golang.org/x/net v0.0.0-20210410081132-afb366fc7cd1/go.mod h1:9tjilg8BloeKEkVJvy golang.org/x/net v0.0.0-20210503060351-7fd8e65b6420/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/net v0.0.0-20210813160813-60bc85c4be6d/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/net v0.0.0-20211118161319-6a13c67c3ce4 h1:DZshvxDdVoeKIbudAdFEKi+f70l51luSy/7b76ibTY0= +golang.org/x/net v0.0.0-20211118161319-6a13c67c3ce4/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= @@ -730,6 +870,7 @@ golang.org/x/sys v0.0.0-20211205182925-97ca703d548d h1:FjkYO/PPp4Wi0EAUOVLxePm7q golang.org/x/sys v0.0.0-20211205182925-97ca703d548d/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/term v0.0.0-20210927222741-03fcf44c2211 h1:JGgROgKl9N8DuW20oFS5gxc+lE67/N3FcwmBPMe7ArY= +golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= golang.org/x/text v0.0.0-20170915032832-14c0d48ead0c/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= @@ -798,6 +939,7 @@ golang.org/x/tools v0.1.3/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= golang.org/x/tools v0.1.4/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= golang.org/x/tools v0.1.5/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= golang.org/x/tools v0.1.7 h1:6j8CgantCy3yc8JGBqkDLMKWqZ0RDU2g1HVgacojGWQ= +golang.org/x/tools v0.1.7/go.mod h1:LGqMHiF4EqQNHR1JncWGqT5BVaXmza+X+BDGol+dOxo= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= @@ -957,11 +1099,16 @@ gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8 gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/errgo.v2 v2.1.0/go.mod h1:hNsd1EY+bozCKY1Ytp96fpM3vjJbqLJn88ws8XvfDNI= gopkg.in/inf.v0 v0.9.1 h1:73M5CoZyi3ZLMOyDlQh031Cx6N9NDJ2Vvfl76EDAgDc= +gopkg.in/inf.v0 v0.9.1/go.mod h1:cWUDdTG/fYaXco+Dcufb5Vnc6Gp2YChqWtbxRZE0mXw= gopkg.in/ini.v1 v1.66.2/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k= gopkg.in/jcmturner/aescts.v1 v1.0.1 h1:cVVZBK2b1zY26haWB4vbBiZrfFQnfbTVrE3xZq6hrEw= +gopkg.in/jcmturner/aescts.v1 v1.0.1/go.mod h1:nsR8qBOg+OucoIW+WMhB3GspUQXq9XorLnQb9XtvcOo= gopkg.in/jcmturner/dnsutils.v1 v1.0.1 h1:cIuC1OLRGZrld+16ZJvvZxVJeKPsvd5eUIvxfoN5hSM= +gopkg.in/jcmturner/dnsutils.v1 v1.0.1/go.mod h1:m3v+5svpVOhtFAP/wSz+yzh4Mc0Fg7eRhxkJMWSIz9Q= gopkg.in/jcmturner/gokrb5.v6 v6.1.1 h1:n0KFjpbuM5pFMN38/Ay+Br3l91netGSVqHPHEXeWUqk= +gopkg.in/jcmturner/gokrb5.v6 v6.1.1/go.mod h1:NFjHNLrHQiruory+EmqDXCGv6CrjkeYeA+bR9mIfNFk= gopkg.in/jcmturner/rpc.v1 v1.1.0 h1:QHIUxTX1ISuAv9dD2wJ9HWQVuWDX/Zc0PfeC2tjc4rU= +gopkg.in/jcmturner/rpc.v1 v1.1.0/go.mod h1:YIdkC4XfD6GXbzje11McwsDuOlZQSb9W4vfLvuNnlv8= gopkg.in/yaml.v2 v2.2.1/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.3/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= @@ -973,6 +1120,7 @@ gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b h1:h8qDotaEPuJATrMmW04NCwg7v22aHH28wwpauUhK9Oo= gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gorm.io/driver/bigquery v1.0.16 h1:KlAgkctWTlliZHxrzS1gCkTmbm0M4p/wOl/I7sbphL8= +gorm.io/driver/bigquery v1.0.16/go.mod h1:fXwpNNQBJ5qbgTt5594M9/zI6rC1iZ7hlmpv5whAgKA= honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= honnef.co/go/tools v0.0.0-20190106161140-3f1c8253044a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= honnef.co/go/tools v0.0.0-20190418001031-e561f6794a2a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= @@ -981,27 +1129,49 @@ honnef.co/go/tools v0.0.1-2019.2.3/go.mod h1:a3bituU0lyd329TUQxRnasdCoJDkEUEAqEt honnef.co/go/tools v0.0.1-2020.1.3/go.mod h1:X/FiERA/W4tHapMX5mGpAtMSVEeEUOyHaw9vFzvIQ3k= honnef.co/go/tools v0.0.1-2020.1.4/go.mod h1:X/FiERA/W4tHapMX5mGpAtMSVEeEUOyHaw9vFzvIQ3k= lukechampine.com/uint128 v1.1.1 h1:pnxCASz787iMf+02ssImqk6OLt+Z5QHMoZyUXR4z6JU= +lukechampine.com/uint128 v1.1.1/go.mod h1:c4eWIwlEGaxC/+H1VguhU4PHXNWDCDMUlWdIWl2j1gk= modernc.org/b v1.0.2 h1:iPC2u39ebzq12GOC2yXT4mve0HrWcH85cz+midWjzeo= +modernc.org/b v1.0.2/go.mod h1:fVGfCIzkZw5RsuF2A2WHbJmY7FiMIq30nP4s52uWsoY= modernc.org/cc/v3 v3.35.18 h1:rMZhRcWrba0y3nVmdiQ7kxAgOOSq2m2f2VzjHLgEs6U= +modernc.org/cc/v3 v3.35.18/go.mod h1:iPJg1pkwXqAV16SNgFBVYmggfMg6xhs+2oiO0vclK3g= modernc.org/ccgo/v3 v3.12.67 h1:ezyuzzP/vVqzpkn0dqu17xVNDcRtV+hP9sKtNKz8aNA= +modernc.org/ccgo/v3 v3.12.67/go.mod h1:Bll3KwKvGROizP2Xj17GEGOTrlvB1XcVaBrC90ORO84= modernc.org/db v1.0.3 h1:apxOlWU69je04bY22OT6J0RL23mzvUy22EgTAVyw+Yg= +modernc.org/db v1.0.3/go.mod h1:L4ltUg8tu2pkSJk+fKaRrXs/3EdW79ZKYQ5PfVDT53U= modernc.org/file v1.0.3 h1:McYGAMMuqjRp6ptmpcLr3r5yw3gNPsonFCAJ0tNK74U= +modernc.org/file v1.0.3/go.mod h1:CNj/pwOfCtCbqiHcXDUlHBB2vWrzdaDCWdcnjtS1+XY= modernc.org/fileutil v1.0.0 h1:Z1AFLZwl6BO8A5NldQg/xTSjGLetp+1Ubvl4alfGx8w= +modernc.org/fileutil v1.0.0/go.mod h1:JHsWpkrk/CnVV1H/eGlFf85BEpfkrp56ro8nojIq9Q8= modernc.org/golex v1.0.1 h1:EYKY1a3wStt0RzHaH8mdSRNg78Ub0OHxYfCRWw35YtM= +modernc.org/golex v1.0.1/go.mod h1:QCA53QtsT1NdGkaZZkF5ezFwk4IXh4BGNafAARTC254= modernc.org/internal v1.0.2 h1:Sn3+ojjMRnPaOR6jFISs6KAdRHnR4q9KNuwfKINKmZA= +modernc.org/internal v1.0.2/go.mod h1:bycJAcev709ZU/47nil584PeBD+kbu8nv61ozeMso9E= modernc.org/libc v1.11.75 h1:K9HFZtM6TwLsUgjIHEZeZ3JWoexZ1p8FixkKUwOiqSo= +modernc.org/libc v1.11.75/go.mod h1:dGRVugT6edz361wmD9gk6ax1AbDSe0x5vji0dGJiPT0= modernc.org/lldb v1.0.2 h1:LBw58xVFl01OuM5U9++tLy3wmu+PoWok6T3dHuNjcZk= +modernc.org/lldb v1.0.2/go.mod h1:ovbKqyzA9H/iPwHkAOH0qJbIQVT9rlijecenxDwVUi0= modernc.org/mathutil v1.4.1 h1:ij3fYGe8zBF4Vu+g0oT7mB06r8sqGWKuJu1yXeR4by8= +modernc.org/mathutil v1.4.1/go.mod h1:mZW8CKdRPY1v87qxC/wUdX5O1qDzXMP5TH3wjfpga6E= modernc.org/memory v1.0.5 h1:XRch8trV7GgvTec2i7jc33YlUI0RKVDBvZ5eZ5m8y14= +modernc.org/memory v1.0.5/go.mod h1:B7OYswTRnfGg+4tDH1t1OeUNnsy2viGTdME4tzd+IjM= modernc.org/opt v0.1.1 h1:/0RX92k9vwVeDXj+Xn23DKp2VJubL7k8qNffND6qn3A= +modernc.org/opt v0.1.1/go.mod h1:WdSiB5evDcignE70guQKxYUl14mgWtbClRi5wmkkTX0= modernc.org/ql v1.4.0 h1:CqLAho+y4N8JwvqT7NJsYsp7YPwiRv6RE2n0n1ksSCU= +modernc.org/ql v1.4.0/go.mod h1:q4c29Bgdx+iAtxx47ODW5Xo2X0PDkjSCK9NdQl6KFxc= modernc.org/sortutil v1.1.0 h1:oP3U4uM+NT/qBQcbg/K2iqAX0Nx7B1b6YZtq3Gk/PjM= +modernc.org/sortutil v1.1.0/go.mod h1:ZyL98OQHJgH9IEfN71VsamvJgrtRX9Dj2gX+vH86L1k= modernc.org/sqlite v1.14.1 h1:jthfQCbWKfbK/lvZSjFEpBk0QzIBN6pQbFdDqBMR490= +modernc.org/sqlite v1.14.1/go.mod h1:04Lqa+3PuAEUhAPAPWeDMljT4UYA31nb2DHTFG47L1g= modernc.org/strutil v1.1.1 h1:xv+J1BXY3Opl2ALrBwyfEikFAj8pmqcpnfmuwUwcozs= +modernc.org/strutil v1.1.1/go.mod h1:DE+MQQ/hjKBZS2zNInV5hhcipt5rLPWkmpbGeW5mmdw= modernc.org/token v1.0.0 h1:a0jaWiNMDhDUtqOj09wvjWWAqd3q7WpBulmL9H2egsk= +modernc.org/token v1.0.0/go.mod h1:UGzOrNV1mAFSEB63lOFHIpNRUVMvYTc6yu1SMY/XTDM= modernc.org/zappy v1.0.3 h1:Tr+P3kclDSrvC6zYBW2hWmOmu5SjG6PtvCt3RCjRmss= +modernc.org/zappy v1.0.3/go.mod h1:w/Akq8ipfols/xZJdR5IYiQNOqC80qz2mVvsEwEbkiI= rsc.io/binaryregexp v0.2.0/go.mod h1:qTv7/COck+e2FymRvadv62gMdZztPaShugOCi3I+8D8= rsc.io/quote/v3 v3.1.0/go.mod h1:yEA65RcK8LyAZtP9Kv3t0HmxON59tX3rD+tICJqUlj0= rsc.io/sampler v1.3.0/go.mod h1:T1hPZKmBbMNahiBKFy5HrXp6adAjACjK9JXDnKaTXpA= sqlflow.org/gohive v0.0.0-20200521083454-ed52ee669b84 h1:6Q27ES8FgvwwQssYYqyR7rFLisaDA1Iqqk9TSWpTHxw= +sqlflow.org/gohive v0.0.0-20200521083454-ed52ee669b84/go.mod h1:IudT38uGbK5q+Ztx2AsZDzjc04yBsIOUtIDMh/WSJkk= sqlflow.org/gomaxcompute v0.0.0-20210805062559-c14ae028b44c h1:Zo3qlfUn/rlMx9vWHpGE/luEtweuXHwrYbrFZwTG978= +sqlflow.org/gomaxcompute v0.0.0-20210805062559-c14ae028b44c/go.mod h1:MxRFJp6UEk1OfnnVOIL3Jc7ROBH0dOpwF/J14A9LNdM= diff --git a/components/cluster/command/display.go b/components/cluster/command/display.go index 66dd011989..e35a652141 100644 --- a/components/cluster/command/display.go +++ b/components/cluster/command/display.go @@ -96,6 +96,9 @@ func newDisplayCmd() *cobra.Command { cmd.Flags().BoolVar(&showVersionOnly, "version", false, "Only display TiDB cluster version") cmd.Flags().BoolVar(&showTiKVLabels, "labels", false, "Only display labels of specified TiKV role or nodes") cmd.Flags().BoolVar(&dopt.ShowProcess, "process", false, "display cpu and memory usage of nodes") + cmd.Flags().BoolVar(&dopt.ShowManageHost, "manage-host", false, "display manage host of nodes") + cmd.Flags().BoolVar(&dopt.ShowNuma, "numa", false, "display numa information of nodes") + cmd.Flags().BoolVar(&dopt.ShowVersions, "versions", false, "display component version of instances") cmd.Flags().Uint64Var(&statusTimeout, "status-timeout", 10, "Timeout in seconds when getting node status") return cmd diff --git a/components/cluster/command/import.go b/components/cluster/command/import.go index f153c20d6e..09935015a0 100644 --- a/components/cluster/command/import.go +++ b/components/cluster/command/import.go @@ -24,7 +24,7 @@ import ( "github.com/pingcap/tiup/pkg/cluster/ctxt" "github.com/pingcap/tiup/pkg/cluster/spec" "github.com/pingcap/tiup/pkg/tui" - tiuputils "github.com/pingcap/tiup/pkg/utils" + "github.com/pingcap/tiup/pkg/utils" "github.com/spf13/cobra" ) @@ -123,17 +123,17 @@ func newImportCmd() *cobra.Command { } // copy SSH key to TiUP profile directory - if err = tiuputils.CreateDir(spec.ClusterPath(clsName, "ssh")); err != nil { + if err = utils.MkdirAll(spec.ClusterPath(clsName, "ssh"), 0755); err != nil { return err } srcKeyPathPriv := ansible.SSHKeyPath() srcKeyPathPub := srcKeyPathPriv + ".pub" dstKeyPathPriv := spec.ClusterPath(clsName, "ssh", "id_rsa") dstKeyPathPub := dstKeyPathPriv + ".pub" - if err = tiuputils.Copy(srcKeyPathPriv, dstKeyPathPriv); err != nil { + if err = utils.Copy(srcKeyPathPriv, dstKeyPathPriv); err != nil { return err } - if err = tiuputils.Copy(srcKeyPathPub, dstKeyPathPub); err != nil { + if err = utils.Copy(srcKeyPathPub, dstKeyPathPub); err != nil { return err } @@ -159,13 +159,13 @@ func newImportCmd() *cobra.Command { // backup ansible files if noBackup { // rename original TiDB-Ansible inventory file - if err = tiuputils.Move(filepath.Join(ansibleDir, inventoryFileName), backupFile); err != nil { + if err = utils.Move(filepath.Join(ansibleDir, inventoryFileName), backupFile); err != nil { return err } log.Infof("Ansible inventory renamed to %s.", color.HiCyanString(backupFile)) } else { // move original TiDB-Ansible directory to a staged location - if err = tiuputils.Move(ansibleDir, backupDir); err != nil { + if err = utils.Move(ansibleDir, backupDir); err != nil { return err } log.Infof("Ansible inventory saved in %s.", color.HiCyanString(backupDir)) diff --git a/components/cluster/command/meta.go b/components/cluster/command/meta.go index ec1b74046d..182cc84e97 100644 --- a/components/cluster/command/meta.go +++ b/components/cluster/command/meta.go @@ -15,6 +15,7 @@ package command import ( "fmt" + "strings" "time" "github.com/spf13/cobra" @@ -36,7 +37,7 @@ func newMetaCmd() *cobra.Command { return fmt.Errorf("please input cluster-name") } if filePath == "" { - filePath = "tiup-cluster_" + args[0] + "_metabackup_" + time.Now().Format(time.RFC3339) + ".tar.gz" + filePath = "tiup-cluster_" + args[0] + "_metabackup_" + strings.ReplaceAll(time.Now().Format(time.RFC3339), ":", "-") + ".tar.gz" } err := cm.BackupClusterMeta(args[0], filePath) if err == nil { diff --git a/components/cluster/command/reload.go b/components/cluster/command/reload.go index 9fb48aaac1..ab2c388037 100644 --- a/components/cluster/command/reload.go +++ b/components/cluster/command/reload.go @@ -55,6 +55,8 @@ func newReloadCmd() *cobra.Command { cmd.Flags().Uint64Var(&gOpt.APITimeout, "transfer-timeout", 600, "Timeout in seconds when transferring PD and TiKV store leaders, also for TiCDC drain one capture") cmd.Flags().BoolVarP(&gOpt.IgnoreConfigCheck, "ignore-config-check", "", false, "Ignore the config check result") cmd.Flags().BoolVar(&skipRestart, "skip-restart", false, "Only refresh configuration to remote and do not restart services") + cmd.Flags().StringVar(&gOpt.SSHCustomScripts.BeforeRestartInstance.Raw, "pre-restart-script", "", "(EXPERIMENTAL) Custom script to be executed on each server before the service is restarted, does not take effect when --skip-restart is set to true") + cmd.Flags().StringVar(&gOpt.SSHCustomScripts.AfterRestartInstance.Raw, "post-restart-script", "", "(EXPERIMENTAL) Custom script to be executed on each server after the service is restarted, does not take effect when --skip-restart is set to true") return cmd } diff --git a/components/cluster/command/root.go b/components/cluster/command/root.go index d672c1fb90..e2cdf15e2f 100644 --- a/components/cluster/command/root.go +++ b/components/cluster/command/root.go @@ -110,7 +110,7 @@ func init() { } tidbSpec = spec.GetSpecManager() - cm = manager.NewManager("tidb", tidbSpec, spec.TiDBComponentVersion, log) + cm = manager.NewManager("tidb", tidbSpec, log) if cmd.Name() != "__complete" { logger.EnableAuditLog(spec.AuditDir()) } @@ -213,6 +213,7 @@ func init() { newTemplateCmd(), newTLSCmd(), newMetaCmd(), + newRotateSSHCmd(), ) } diff --git a/components/cluster/command/rotate_ssh.go b/components/cluster/command/rotate_ssh.go new file mode 100644 index 0000000000..fc05cfcff0 --- /dev/null +++ b/components/cluster/command/rotate_ssh.go @@ -0,0 +1,36 @@ +// Copyright 2021 PingCAP, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// See the License for the specific language governing permissions and +// limitations under the License. + +package command + +import ( + "github.com/spf13/cobra" +) + +func newRotateSSHCmd() *cobra.Command { + cmd := &cobra.Command{ + Use: "rotatessh ", + Short: "rotate ssh keys on all nodes", + RunE: func(cmd *cobra.Command, args []string) error { + if len(args) != 1 { + return cmd.Help() + } + + clusterName := args[0] + + return cm.RotateSSH(clusterName, gOpt, skipConfirm) + }, + } + + return cmd +} diff --git a/components/cluster/command/upgrade.go b/components/cluster/command/upgrade.go index 592cef0c4b..d9285b136d 100644 --- a/components/cluster/command/upgrade.go +++ b/components/cluster/command/upgrade.go @@ -14,12 +14,15 @@ package command import ( + "github.com/pingcap/tiup/pkg/cluster/spec" "github.com/pingcap/tiup/pkg/utils" "github.com/spf13/cobra" ) func newUpgradeCmd() *cobra.Command { offlineMode := false + ignoreVersionCheck := false + var tidbVer, tikvVer, pdVer, tiflashVer, kvcdcVer, dashboardVer, cdcVer, alertmanagerVer, nodeExporterVer, blackboxExporterVer, tiproxyVer string cmd := &cobra.Command{ Use: "upgrade ", @@ -38,7 +41,21 @@ func newUpgradeCmd() *cobra.Command { teleCommand = append(teleCommand, scrubClusterName(clusterName)) teleCommand = append(teleCommand, version) - return cm.Upgrade(clusterName, version, gOpt, skipConfirm, offlineMode) + componentVersions := map[string]string{ + spec.ComponentDashboard: dashboardVer, + spec.ComponentAlertmanager: alertmanagerVer, + spec.ComponentTiDB: tidbVer, + spec.ComponentTiKV: tikvVer, + spec.ComponentPD: pdVer, + spec.ComponentTiFlash: tiflashVer, + spec.ComponentTiKVCDC: kvcdcVer, + spec.ComponentCDC: cdcVer, + spec.ComponentTiProxy: tiproxyVer, + spec.ComponentBlackboxExporter: blackboxExporterVer, + spec.ComponentNodeExporter: nodeExporterVer, + } + + return cm.Upgrade(clusterName, version, componentVersions, gOpt, skipConfirm, offlineMode, ignoreVersionCheck) }, ValidArgsFunction: func(cmd *cobra.Command, args []string, toComplete string) ([]string, cobra.ShellCompDirective) { switch len(args) { @@ -53,6 +70,20 @@ func newUpgradeCmd() *cobra.Command { cmd.Flags().Uint64Var(&gOpt.APITimeout, "transfer-timeout", 600, "Timeout in seconds when transferring PD and TiKV store leaders, also for TiCDC drain one capture") cmd.Flags().BoolVarP(&gOpt.IgnoreConfigCheck, "ignore-config-check", "", false, "Ignore the config check result") cmd.Flags().BoolVarP(&offlineMode, "offline", "", false, "Upgrade a stopped cluster") + cmd.Flags().BoolVarP(&ignoreVersionCheck, "ignore-version-check", "", false, "Ignore checking if target version is bigger than current version") + cmd.Flags().StringVar(&gOpt.SSHCustomScripts.BeforeRestartInstance.Raw, "pre-upgrade-script", "", "(EXPERIMENTAL) Custom script to be executed on each server before the server is upgraded") + cmd.Flags().StringVar(&gOpt.SSHCustomScripts.AfterRestartInstance.Raw, "post-upgrade-script", "", "(EXPERIMENTAL) Custom script to be executed on each server after the server is upgraded") + // cmd.Flags().StringVar(&tidbVer, "tidb-version", "", "Fix the version of tidb and no longer follows the cluster version.") + cmd.Flags().StringVar(&tikvVer, "tikv-version", "", "Fix the version of tikv and no longer follows the cluster version.") + cmd.Flags().StringVar(&pdVer, "pd-version", "", "Fix the version of pv and no longer follows the cluster version.") + cmd.Flags().StringVar(&tiflashVer, "tiflash-version", "", "Fix the version of tiflash and no longer follows the cluster version.") + cmd.Flags().StringVar(&dashboardVer, "tidb-dashboard-version", "", "Fix the version of tidb-dashboard and no longer follows the cluster version.") + cmd.Flags().StringVar(&cdcVer, "cdc-version", "", "Fix the version of cdc and no longer follows the cluster version.") + cmd.Flags().StringVar(&kvcdcVer, "tikv-cdc-version", "", "Fix the version of tikv-cdc and no longer follows the cluster version.") + cmd.Flags().StringVar(&alertmanagerVer, "alertmanager-version", "", "Fix the version of alertmanager and no longer follows the cluster version.") + cmd.Flags().StringVar(&nodeExporterVer, "node-exporter-version", "", "Fix the version of node-exporter and no longer follows the cluster version.") + cmd.Flags().StringVar(&blackboxExporterVer, "blackbox-exporter-version", "", "Fix the version of blackbox-exporter and no longer follows the cluster version.") + cmd.Flags().StringVar(&tiproxyVer, "tiproxy-version", "", "Fix the version of tiproxy and no longer follows the cluster version.") return cmd } diff --git a/components/ctl/main.go b/components/ctl/main.go index 69fc4da719..fb3f94cc71 100644 --- a/components/ctl/main.go +++ b/components/ctl/main.go @@ -101,6 +101,7 @@ func binaryPath(home, cmd string) (string, error) { } func run(name string, args ...string) error { + os.Setenv("ETCDCTL_API", "3") // Handle `cdc cli` if strings.Contains(name, " ") { xs := strings.Split(name, " ") diff --git a/components/dm/command/prune.go b/components/dm/command/prune.go index c6bb13b97d..6b7c50cbc7 100644 --- a/components/dm/command/prune.go +++ b/components/dm/command/prune.go @@ -76,7 +76,7 @@ func clearOutDatedEtcdInfo(clusterName string, metadata *spec.Metadata, opt oper if err != nil { return err } - dmMasterClient := api.NewDMMasterClient(topo.GetMasterList(), 10*time.Second, tlsCfg) + dmMasterClient := api.NewDMMasterClient(topo.GetMasterListWithManageHost(), 10*time.Second, tlsCfg) registeredMasters, registeredWorkers, err := dmMasterClient.GetRegisteredMembers() if err != nil { return err diff --git a/components/dm/command/root.go b/components/dm/command/root.go index f749fa782b..94c9bb2241 100644 --- a/components/dm/command/root.go +++ b/components/dm/command/root.go @@ -85,7 +85,7 @@ please backup your data before process.`, dmspec = spec.GetSpecManager() logger.EnableAuditLog(cspec.AuditDir()) - cm = manager.NewManager("dm", dmspec, spec.DMComponentVersion, log) + cm = manager.NewManager("dm", dmspec, log) // Running in other OS/ARCH Should be fine we only download manifest file. env, err = tiupmeta.InitEnv(repository.Options{ @@ -170,6 +170,7 @@ please backup your data before process.`, newReplayCmd(), newTemplateCmd(), newMetaCmd(), + newRotateSSHCmd(), ) } diff --git a/components/dm/command/rotate_ssh.go b/components/dm/command/rotate_ssh.go new file mode 100644 index 0000000000..fc05cfcff0 --- /dev/null +++ b/components/dm/command/rotate_ssh.go @@ -0,0 +1,36 @@ +// Copyright 2021 PingCAP, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// See the License for the specific language governing permissions and +// limitations under the License. + +package command + +import ( + "github.com/spf13/cobra" +) + +func newRotateSSHCmd() *cobra.Command { + cmd := &cobra.Command{ + Use: "rotatessh ", + Short: "rotate ssh keys on all nodes", + RunE: func(cmd *cobra.Command, args []string) error { + if len(args) != 1 { + return cmd.Help() + } + + clusterName := args[0] + + return cm.RotateSSH(clusterName, gOpt, skipConfirm) + }, + } + + return cmd +} diff --git a/components/dm/command/scale_in.go b/components/dm/command/scale_in.go index 6ac038daa4..e9bd661ab1 100644 --- a/components/dm/command/scale_in.go +++ b/components/dm/command/scale_in.go @@ -27,6 +27,7 @@ import ( "github.com/pingcap/tiup/pkg/cluster/spec" "github.com/pingcap/tiup/pkg/cluster/task" "github.com/pingcap/tiup/pkg/set" + "github.com/pingcap/tiup/pkg/utils" "github.com/spf13/cobra" ) @@ -86,7 +87,7 @@ func ScaleInDMCluster( for _, component := range topo.ComponentsByStartOrder() { for _, instance := range component.Instances() { instances[instance.ID()] = instance - instCount[instance.GetHost()]++ + instCount[instance.GetManageHost()]++ } } @@ -112,8 +113,8 @@ func ScaleInDMCluster( if !deletedNodes.Exist(instance.ID()) { continue } - instCount[instance.GetHost()]-- - if err := operator.StopAndDestroyInstance(ctx, topo, instance, options, false, instCount[instance.GetHost()] == 0, tlsCfg); err != nil { + instCount[instance.GetManageHost()]-- + if err := operator.StopAndDestroyInstance(ctx, topo, instance, options, false, instCount[instance.GetManageHost()] == 0, tlsCfg); err != nil { log.Warnf("failed to stop/destroy %s: %v", component.Name(), err) } } @@ -126,7 +127,7 @@ func ScaleInDMCluster( var dmMasterEndpoint []string for _, instance := range (&dm.DMMasterComponent{Topology: topo}).Instances() { if !deletedNodes.Exist(instance.ID()) { - dmMasterEndpoint = append(dmMasterEndpoint, operator.Addr(instance)) + dmMasterEndpoint = append(dmMasterEndpoint, utils.JoinHostPort(instance.GetManageHost(), instance.GetPort())) } } @@ -139,7 +140,7 @@ func ScaleInDMCluster( noAgentHosts := set.NewStringSet() topo.IterInstance(func(inst dm.Instance) { if inst.IgnoreMonitorAgent() { - noAgentHosts.Insert(inst.GetHost()) + noAgentHosts.Insert(inst.GetManageHost()) } }) @@ -182,9 +183,9 @@ func ScaleInDMCluster( return errors.Annotatef(err, "failed to destroy %s", component.Name()) } - instCount[instance.GetHost()]-- - if instCount[instance.GetHost()] == 0 { - if err := operator.DeletePublicKey(ctx, instance.GetHost()); err != nil { + instCount[instance.GetManageHost()]-- + if instCount[instance.GetManageHost()] == 0 { + if err := operator.DeletePublicKey(ctx, instance.GetManageHost()); err != nil { return errors.Annotatef(err, "failed to delete public key") } } diff --git a/components/dm/command/upgrade.go b/components/dm/command/upgrade.go index bb6abf0ac7..d902979295 100644 --- a/components/dm/command/upgrade.go +++ b/components/dm/command/upgrade.go @@ -19,7 +19,7 @@ import ( func newUpgradeCmd() *cobra.Command { offlineMode := false - + ignoreVersionCheck := false cmd := &cobra.Command{ Use: "upgrade ", Short: "Upgrade a specified DM cluster", @@ -28,7 +28,7 @@ func newUpgradeCmd() *cobra.Command { return cmd.Help() } - return cm.Upgrade(args[0], args[1], gOpt, skipConfirm, offlineMode) + return cm.Upgrade(args[0], args[1], nil, gOpt, skipConfirm, offlineMode, ignoreVersionCheck) }, ValidArgsFunction: func(cmd *cobra.Command, args []string, toComplete string) ([]string, cobra.ShellCompDirective) { switch len(args) { @@ -41,6 +41,7 @@ func newUpgradeCmd() *cobra.Command { } cmd.Flags().BoolVarP(&offlineMode, "offline", "", false, "Upgrade a stopped cluster") + cmd.Flags().BoolVarP(&ignoreVersionCheck, "ignore-version-check", "", false, "Ignore checking if target version is higher than current version") return cmd } diff --git a/components/dm/spec/bindversion.go b/components/dm/spec/bindversion.go deleted file mode 100644 index cfda8ffa05..0000000000 --- a/components/dm/spec/bindversion.go +++ /dev/null @@ -1,18 +0,0 @@ -package spec - -import "github.com/pingcap/tiup/pkg/cluster/spec" - -// DMComponentVersion maps the dm version to the third components binding version -// Empty version means the latest stable one -func DMComponentVersion(comp, version string) string { - switch comp { - case spec.ComponentAlertmanager, - spec.ComponentGrafana, - spec.ComponentPrometheus, - spec.ComponentBlackboxExporter, - spec.ComponentNodeExporter: - return "" - default: - return version - } -} diff --git a/components/dm/spec/logic.go b/components/dm/spec/logic.go index c354d29335..3f8c48c20a 100644 --- a/components/dm/spec/logic.go +++ b/components/dm/spec/logic.go @@ -26,6 +26,7 @@ import ( "github.com/pingcap/tiup/pkg/cluster/spec" "github.com/pingcap/tiup/pkg/cluster/template/scripts" "github.com/pingcap/tiup/pkg/meta" + "github.com/pingcap/tiup/pkg/utils" ) // Components names supported by TiUP @@ -67,6 +68,16 @@ func (c *DMMasterComponent) Role() string { return ComponentDMMaster } +// CalculateVersion implements the Component interface +func (c *DMMasterComponent) CalculateVersion(clusterVersion string) string { + return clusterVersion +} + +// SetVersion implements Component interface. +func (c *DMMasterComponent) SetVersion(version string) { + // not supported now +} + // Instances implements Component interface. func (c *DMMasterComponent) Instances() []Instance { ins := make([]Instance, 0) @@ -78,8 +89,11 @@ func (c *DMMasterComponent) Instances() []Instance { InstanceSpec: s, Name: c.Name(), Host: s.Host, + ManageHost: s.ManageHost, + ListenHost: c.Topology.BaseTopo().GlobalOptions.ListenHost, Port: s.Port, SSHP: s.SSHPort, + Source: s.GetSource(), Ports: []int{ s.Port, @@ -93,6 +107,7 @@ func (c *DMMasterComponent) Instances() []Instance { UptimeFn: func(_ context.Context, timeout time.Duration, tlsCfg *tls.Config) time.Duration { return spec.UptimeByHost(s.Host, s.Port, timeout, tlsCfg) }, + Component: c, }, topo: c.Topology, }) @@ -122,14 +137,25 @@ func (i *MasterInstance) InitConfig( enableTLS := i.topo.GlobalOptions.TLSEnabled spec := i.InstanceSpec.(*MasterSpec) - cfg := scripts.NewDMMasterScript( - spec.Name, - i.GetHost(), - paths.Deploy, - paths.Data[0], - paths.Log, - enableTLS, - ).WithPort(spec.Port).WithNumaNode(spec.NumaNode).WithPeerPort(spec.PeerPort).AppendEndpoints(i.topo.Endpoints(deployUser)...).WithV1SourcePath(spec.V1SourcePath) + scheme := utils.Ternary(enableTLS, "https", "http").(string) + + initialCluster := []string{} + for _, masterspec := range i.topo.Masters { + initialCluster = append(initialCluster, fmt.Sprintf("%s=%s", masterspec.Name, masterspec.GetAdvertisePeerURL(enableTLS))) + } + cfg := &scripts.DMMasterScript{ + Name: spec.Name, + V1SourcePath: spec.V1SourcePath, + MasterAddr: utils.JoinHostPort(i.GetListenHost(), spec.Port), + AdvertiseAddr: utils.JoinHostPort(spec.Host, spec.Port), + PeerURL: fmt.Sprintf("%s://%s", scheme, utils.JoinHostPort(i.GetListenHost(), spec.PeerPort)), + AdvertisePeerURL: spec.GetAdvertisePeerURL(enableTLS), + InitialCluster: strings.Join(initialCluster, ","), + DeployDir: paths.Deploy, + DataDir: paths.Data[0], + LogDir: paths.Log, + NumaNode: spec.NumaNode, + } fp := filepath.Join(paths.Cache, fmt.Sprintf("run_dm-master_%s_%d.sh", i.GetHost(), i.GetPort())) if err := cfg.ConfigToFile(fp); err != nil { @@ -206,16 +232,27 @@ func (i *MasterInstance) ScaleConfig( } enableTLS := i.topo.GlobalOptions.TLSEnabled - c := topo.(*Specification) spec := i.InstanceSpec.(*MasterSpec) - cfg := scripts.NewDMMasterScaleScript( - spec.Name, - i.GetHost(), - paths.Deploy, - paths.Data[0], - paths.Log, - enableTLS, - ).WithPort(spec.Port).WithNumaNode(spec.NumaNode).WithPeerPort(spec.PeerPort).AppendEndpoints(c.Endpoints(deployUser)...) + scheme := utils.Ternary(enableTLS, "https", "http").(string) + + masters := []string{} + // master list from exist topo file + for _, masterspec := range topo.(*Specification).Masters { + masters = append(masters, utils.JoinHostPort(masterspec.Host, masterspec.Port)) + } + cfg := &scripts.DMMasterScaleScript{ + Name: spec.Name, + V1SourcePath: spec.V1SourcePath, + MasterAddr: utils.JoinHostPort(i.GetListenHost(), spec.Port), + AdvertiseAddr: utils.JoinHostPort(spec.Host, spec.Port), + PeerURL: fmt.Sprintf("%s://%s", scheme, utils.JoinHostPort(i.GetListenHost(), spec.PeerPort)), + AdvertisePeerURL: spec.GetAdvertisePeerURL(enableTLS), + Join: strings.Join(masters, ","), + DeployDir: paths.Deploy, + DataDir: paths.Data[0], + LogDir: paths.Log, + NumaNode: spec.NumaNode, + } fp := filepath.Join(paths.Cache, fmt.Sprintf("run_dm-master_%s_%d.sh", i.GetHost(), i.GetPort())) if err := cfg.ConfigToFile(fp); err != nil { @@ -246,6 +283,16 @@ func (c *DMWorkerComponent) Role() string { return ComponentDMWorker } +// CalculateVersion implements the Component interface +func (c *DMWorkerComponent) CalculateVersion(clusterVersion string) string { + return clusterVersion +} + +// SetVersion implements Component interface. +func (c *DMWorkerComponent) SetVersion(version string) { + // not supported now +} + // Instances implements Component interface. func (c *DMWorkerComponent) Instances() []Instance { ins := make([]Instance, 0) @@ -257,8 +304,11 @@ func (c *DMWorkerComponent) Instances() []Instance { InstanceSpec: s, Name: c.Name(), Host: s.Host, + ManageHost: s.ManageHost, + ListenHost: c.Topology.BaseTopo().GlobalOptions.ListenHost, Port: s.Port, SSHP: s.SSHPort, + Source: s.GetSource(), Ports: []int{ s.Port, @@ -271,6 +321,7 @@ func (c *DMWorkerComponent) Instances() []Instance { UptimeFn: func(_ context.Context, timeout time.Duration, tlsCfg *tls.Config) time.Duration { return spec.UptimeByHost(s.Host, s.Port, timeout, tlsCfg) }, + Component: c, }, topo: c.Topology, }) @@ -301,12 +352,22 @@ func (i *WorkerInstance) InitConfig( enableTLS := i.topo.GlobalOptions.TLSEnabled spec := i.InstanceSpec.(*WorkerSpec) - cfg := scripts.NewDMWorkerScript( - i.Name, - i.GetHost(), - paths.Deploy, - paths.Log, - ).WithPort(spec.Port).WithNumaNode(spec.NumaNode).AppendEndpoints(i.topo.Endpoints(deployUser)...) + + masters := []string{} + for _, masterspec := range i.topo.Masters { + masters = append(masters, utils.JoinHostPort(masterspec.Host, masterspec.Port)) + } + cfg := &scripts.DMWorkerScript{ + Name: i.Name, + WorkerAddr: utils.JoinHostPort(i.GetListenHost(), spec.Port), + AdvertiseAddr: utils.JoinHostPort(spec.Host, spec.Port), + Join: strings.Join(masters, ","), + + DeployDir: paths.Deploy, + LogDir: paths.Log, + NumaNode: spec.NumaNode, + } + fp := filepath.Join(paths.Cache, fmt.Sprintf("run_dm-worker_%s_%d.sh", i.GetHost(), i.GetPort())) if err := cfg.ConfigToFile(fp); err != nil { return err @@ -423,7 +484,7 @@ func (topo *Specification) ComponentsByStartOrder() (comps []Component) { } // ComponentsByUpdateOrder return component in the order need to be updated. -func (topo *Specification) ComponentsByUpdateOrder() (comps []Component) { +func (topo *Specification) ComponentsByUpdateOrder(curVer string) (comps []Component) { // "dm-master", "dm-worker" comps = append(comps, &DMMasterComponent{topo}) comps = append(comps, &DMWorkerComponent{topo}) @@ -479,31 +540,3 @@ func (topo *Specification) IterHost(fn func(instance Instance)) { } } } - -// Endpoints returns the PD endpoints configurations -func (topo *Specification) Endpoints(user string) []*scripts.DMMasterScript { - var ends []*scripts.DMMasterScript - for _, s := range topo.Masters { - deployDir := spec.Abs(user, s.DeployDir) - // data dir would be empty for components which don't need it - dataDir := s.DataDir - // the default data_dir is relative to deploy_dir - if dataDir != "" && !strings.HasPrefix(dataDir, "/") { - dataDir = filepath.Join(deployDir, dataDir) - } - // log dir will always be with values, but might not used by the component - logDir := spec.Abs(user, s.LogDir) - - script := scripts.NewDMMasterScript( - s.Name, - s.Host, - deployDir, - dataDir, - logDir, - topo.GlobalOptions.TLSEnabled). - WithPort(s.Port). - WithPeerPort(s.PeerPort) - ends = append(ends, script) - } - return ends -} diff --git a/components/dm/spec/topology_dm.go b/components/dm/spec/topology_dm.go index e672aa5637..6e073fb7c3 100644 --- a/components/dm/spec/topology_dm.go +++ b/components/dm/spec/topology_dm.go @@ -122,6 +122,7 @@ func AllDMComponentNames() (roles []string) { // MasterSpec represents the Master topology specification in topology.yaml type MasterSpec struct { Host string `yaml:"host"` + ManageHost string `yaml:"manage_host,omitempty" validate:"manage_host:editable"` SSHPort int `yaml:"ssh_port,omitempty" validate:"ssh_port:editable"` Imported bool `yaml:"imported,omitempty"` Patched bool `yaml:"patched,omitempty"` @@ -133,6 +134,7 @@ type MasterSpec struct { DeployDir string `yaml:"deploy_dir,omitempty"` DataDir string `yaml:"data_dir,omitempty"` LogDir string `yaml:"log_dir,omitempty"` + Source string `yaml:"source,omitempty" validate:"source:editable"` NumaNode string `yaml:"numa_node,omitempty" validate:"numa_node:editable"` Config map[string]any `yaml:"config,omitempty" validate:"config:ignore"` ResourceControl ResourceControl `yaml:"resource_control,omitempty" validate:"resource_control:editable"` @@ -173,7 +175,11 @@ func (s *MasterSpec) Role() string { // SSH returns the host and SSH port of the instance func (s *MasterSpec) SSH() (string, int) { - return s.Host, s.SSHPort + host := s.Host + if s.ManageHost != "" { + host = s.ManageHost + } + return host, s.SSHPort } // GetMainPort returns the main port of the instance @@ -191,9 +197,24 @@ func (s *MasterSpec) IgnoreMonitorAgent() bool { return s.IgnoreExporter } +// GetAdvertisePeerURL returns AdvertisePeerURL +func (s *MasterSpec) GetAdvertisePeerURL(enableTLS bool) string { + scheme := utils.Ternary(enableTLS, "https", "http").(string) + return fmt.Sprintf("%s://%s", scheme, utils.JoinHostPort(s.Host, s.PeerPort)) +} + +// GetSource returns source to download the component +func (s *MasterSpec) GetSource() string { + if s.Source == "" { + return ComponentDMMaster + } + return s.Source +} + // WorkerSpec represents the Master topology specification in topology.yaml type WorkerSpec struct { Host string `yaml:"host"` + ManageHost string `yaml:"manage_host,omitempty" validate:"manage_host:editable"` SSHPort int `yaml:"ssh_port,omitempty" validate:"ssh_port:editable"` Imported bool `yaml:"imported,omitempty"` Patched bool `yaml:"patched,omitempty"` @@ -204,6 +225,7 @@ type WorkerSpec struct { DeployDir string `yaml:"deploy_dir,omitempty"` DataDir string `yaml:"data_dir,omitempty"` LogDir string `yaml:"log_dir,omitempty"` + Source string `yaml:"source,omitempty" validate:"source:editable"` NumaNode string `yaml:"numa_node,omitempty" validate:"numa_node:editable"` Config map[string]any `yaml:"config,omitempty" validate:"config:ignore"` ResourceControl ResourceControl `yaml:"resource_control,omitempty" validate:"resource_control:editable"` @@ -238,7 +260,11 @@ func (s *WorkerSpec) Role() string { // SSH returns the host and SSH port of the instance func (s *WorkerSpec) SSH() (string, int) { - return s.Host, s.SSHPort + host := s.Host + if s.ManageHost != "" { + host = s.ManageHost + } + return host, s.SSHPort } // GetMainPort returns the main port of the instance @@ -256,6 +282,14 @@ func (s *WorkerSpec) IgnoreMonitorAgent() bool { return s.IgnoreExporter } +// GetSource returns source to download the component +func (s *WorkerSpec) GetSource() string { + if s.Source == "" { + return ComponentDMWorker + } + return s.Source +} + // UnmarshalYAML sets default values when unmarshaling the topology file func (s *Specification) UnmarshalYAML(unmarshal func(any) error) error { type topology Specification @@ -659,7 +693,7 @@ func (s *Specification) BaseTopo() *spec.BaseTopo { return &spec.BaseTopo{ GlobalOptions: &s.GlobalOptions, MonitoredOptions: s.GetMonitoredOptions(), - MasterList: s.GetMasterList(), + MasterList: s.GetMasterListWithManageHost(), Monitors: s.Monitors, Grafanas: s.Grafanas, Alertmanagers: s.Alertmanagers, @@ -685,12 +719,16 @@ func (s *Specification) MergeTopo(rhs spec.Topology) spec.Topology { return s.Merge(other) } -// GetMasterList returns a list of Master API hosts of the current cluster -func (s *Specification) GetMasterList() []string { +// GetMasterListWithManageHost returns a list of Master API hosts of the current cluster +func (s *Specification) GetMasterListWithManageHost() []string { var masterList []string for _, master := range s.Masters { - masterList = append(masterList, utils.JoinHostPort(master.Host, master.Port)) + host := master.Host + if master.ManageHost != "" { + host = master.ManageHost + } + masterList = append(masterList, utils.JoinHostPort(host, master.Port)) } return masterList diff --git a/components/doc/main.go b/components/doc/main.go deleted file mode 100644 index c671af7484..0000000000 --- a/components/doc/main.go +++ /dev/null @@ -1,53 +0,0 @@ -// Copyright 2020 PingCAP, Inc. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// See the License for the specific language governing permissions and -// limitations under the License. - -package main - -import ( - "errors" - "os" - - "github.com/skratchdot/open-golang/open" - "github.com/spf13/cobra" -) - -func main() { - if err := execute(); err != nil { - os.Exit(1) - } -} - -func execute() error { - lang := "en" - - rootCmd := &cobra.Command{ - Use: "tiup doc", - Short: "TiDB document summary page", - SilenceUsage: true, - RunE: func(cmd *cobra.Command, args []string) error { - var url string - switch lang { - case "en": - url = "https://pingcap.com/docs/stable" - case "cn": - url = "https://pingcap.com/docs-cn/stable" - default: - return errors.New("unrecognized language (only `en` and `cn` supported)") - } - return open.Run(url) - }, - } - - rootCmd.Flags().StringVar(&lang, "lang", lang, "The language of the documentation: en/cn") - return rootCmd.Execute() -} diff --git a/components/errdoc/errdoc-gen/README.md b/components/errdoc/errdoc-gen/README.md deleted file mode 100644 index 8f98afccfa..0000000000 --- a/components/errdoc/errdoc-gen/README.md +++ /dev/null @@ -1,6 +0,0 @@ -## Usage - -```shell script -# eg: ./errdoc-gen --source devel/pingap/tidb --module github.com/pingcap/tidb --output devel/pingap/tidb/errors.toml -./errdoc-gen --source /path/to/source/code --module ${module-name} --output /path/to/errors.toml -``` \ No newline at end of file diff --git a/components/errdoc/errdoc-gen/main.go b/components/errdoc/errdoc-gen/main.go deleted file mode 100644 index a72b8eb22b..0000000000 --- a/components/errdoc/errdoc-gen/main.go +++ /dev/null @@ -1,339 +0,0 @@ -// Copyright 2020 PingCAP, Inc. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// See the License for the specific language governing permissions and -// limitations under the License. - -package main - -import ( - "flag" - "fmt" - "go/ast" - "go/parser" - "go/token" - "os" - "os/exec" - "path/filepath" - "strings" - "text/template" - - "github.com/pingcap/tiup/pkg/utils" -) - -var opt struct { - source string - module string - output string - retainCode bool -} - -func init() { - flag.StringVar(&opt.source, "source", "", "The source directory of error documentation") - flag.StringVar(&opt.module, "module", "", "The module name of target repository") - flag.StringVar(&opt.output, "output", "", "The output path of error documentation file") - flag.BoolVar(&opt.retainCode, "retain-code", false, "Retain the generated code when generator exit") -} - -func log(format string, args ...any) { - fmt.Println(fmt.Sprintf(format, args...)) -} - -func fatal(format string, args ...any) { - log(format, args...) - os.Exit(1) -} - -const autoDirectoryName = "_errdoc-generator" -const entryFileName = "main.go" - -func main() { - flag.Parse() - if opt.source == "" { - fatal("The source directory cannot be empty") - } - - source, err := filepath.EvalSymlinks(opt.source) - if err != nil { - fatal("Evaluate symbol link path %s failed: %v", opt.source, err) - } - opt.source = source - - if !utils.IsExist(filepath.Join(opt.source, "go.mod")) { - fatal("The source directory is not the root path of codebase(go.mod not found)") - } - - if opt.output == "" { - opt.output = filepath.Join(opt.source, "errors.toml") - log("The --output argument is missing, default to %s", opt.output) - } - - errNames, err := errdoc(opt.source, opt.module) - if err != nil { - log("Extract the error documentation failed: %+v", err) - } - - targetDir := filepath.Join(opt.source, autoDirectoryName) - if err := os.MkdirAll(targetDir, 0755); err != nil { - fatal("Cannot create the errdoc: %+v", err) - } - - if !opt.retainCode { - defer os.RemoveAll(targetDir) - } - - tmpl := ` -package main - -import ( - "bytes" - "flag" - "os" - "reflect" - "fmt" - "sort" - "strings" - - "github.com/BurntSushi/toml" - "github.com/pingcap/errors" -{{- range $decl := .}} - {{$decl.PackageName}} "{{- $decl.ImportPath}}" -{{- end}} -) - -func main() { - var outpath string - flag.StringVar(&outpath, "output", "", "Specify the error documentation output file path") - flag.Parse() - if outpath == "" { - println("Usage: ./_errdoc-generator --output /path/to/errors.toml") - os.Exit(1) - } - - // Read-in the exists file and merge the description/workaround from exists file - existDefinition := map[string]spec{} - if file, err := os.ReadFile(outpath); err == nil { - err = toml.Unmarshal(file, &existDefinition) - if err != nil { - println(fmt.Sprintf("Invalid toml file %s when merging exists description/workaround: %v", outpath, err)) - os.Exit(1) - } - } - - var allErrors []error - {{- range $decl := .}} - {{- range $err := $decl.ErrNames}} - allErrors = append(allErrors, {{$decl.PackageName}}.{{- $err}}) - {{- end}} - {{- end}} - - var dedup = map[string]spec{} - for _, e := range allErrors { - terr, ok := e.(*errors.Error) - if !ok { - println("Non-normalized error:", e.Error()) - } else { - val := reflect.ValueOf(terr).Elem() - codeText := val.FieldByName("codeText") - message := val.FieldByName("message") - if previous, found := dedup[codeText.String()]; found { - println("Duplicated error code:", codeText.String()) - if message.String() < previous.Error { - continue - } - } - s := spec{ - Code: codeText.String(), - Error: message.String(), - } - if exist, found := existDefinition[s.Code]; found { - s.Description = strings.TrimSpace(exist.Description) - s.Workaround = strings.TrimSpace(exist.Workaround) - } - dedup[codeText.String()] = s - } - } - - var sorted []spec - for _, item := range dedup { - sorted = append(sorted, item) - } - sort.Slice(sorted, func(i, j int) bool { - // TiDB exits duplicated code - if sorted[i].Code == sorted[j].Code { - return sorted[i].Error < sorted[j].Error - } - return sorted[i].Code < sorted[j].Code - }) - - // We don't use toml library to serialize it due to cannot reserve the order for map[string]spec - buffer := bytes.NewBufferString("# AUTOGENERATED BY github.com/pingcap/tiup/components/errdoc/errdoc-gen\n" + - "# YOU CAN CHANGE THE 'description'/'workaround' FIELDS IF THEM ARE IMPROPER.\n\n") - for _, item := range sorted { - buffer.WriteString(fmt.Sprintf("[\"%s\"]\nerror = '''\n%s\n'''\n", item.Code, item.Error)) - if item.Description != "" { - buffer.WriteString(fmt.Sprintf("description = '''\n%s\n'''\n", item.Description)) - } - if item.Workaround != "" { - buffer.WriteString(fmt.Sprintf("workaround = '''\n%s\n'''\n", item.Workaround)) - } - buffer.WriteString("\n") - } - if err := os.WriteFile(outpath, buffer.Bytes(), os.ModePerm); err != nil { - panic(err) - } -} -` + "type spec struct {\n" + - "Code string\n" + - "Error string `toml:\"error\"`\n" + - "Description string `toml:\"description\"`\n" + - "Workaround string `toml:\"workaround\"`\n" + - "}" - - t, err := template.New("_errdoc-template").Parse(tmpl) - if err != nil { - fatal("Parse template failed: %+v", err) - } - - outFile := filepath.Join(targetDir, entryFileName) - out, err := os.OpenFile(outFile, os.O_TRUNC|os.O_WRONLY|os.O_CREATE, os.ModePerm) - if err != nil { - fatal("Open %s failed: %+v", outFile, err) - } - defer out.Close() - - if err := t.Execute(out, errNames); err != nil { - fatal("Render template failed: %+v", err) - } - - output, err := filepath.Abs(opt.output) - if err != nil { - fatal("Evaluate the absolute path of output failed: %+v", err) - } - - cmd := exec.Command("go", "run", filepath.Join(autoDirectoryName, entryFileName), "--output", output) - cmd.Dir = opt.source - data, err := cmd.CombinedOutput() - if err != nil { - fatal("Generate %s failed: %v, output:\n%s", opt.output, err, string(data)) - } - - log("Generate successfully, output:\n%s", string(data)) -} - -type errDecl struct { - ImportPath string - PackageName string - ErrNames []string -} - -func errdoc(source, module string) ([]*errDecl, error) { - source, err := filepath.Abs(source) - if err != nil { - return nil, err - } - - dedup := map[string]*errDecl{} - - err = filepath.Walk(source, func(path string, info os.FileInfo, err error) error { - if err != nil { - return err - } - if info.IsDir() { - return nil - } - if !strings.HasSuffix(path, ".go") { - return nil - } - fset := token.NewFileSet() - file, err := parser.ParseFile(fset, path, nil, parser.ParseComments) - if err != nil { - // Ignore invalid source file - return nil - } - errNames := export(file) - if len(errNames) < 1 { - return nil - } - dirPath := filepath.Dir(path) - subPath, err := filepath.Rel(source, dirPath) - if err != nil { - return err - } - packageName := strings.ReplaceAll(subPath, "/", "_") - if decl, found := dedup[packageName]; found { - decl.ErrNames = append(decl.ErrNames, errNames...) - } else { - decl := &errDecl{ - ImportPath: filepath.Join(module, subPath), - PackageName: packageName, - ErrNames: errNames, - } - dedup[packageName] = decl - } - return nil - }) - - var errDecls []*errDecl - for _, decl := range dedup { - errDecls = append(errDecls, decl) - } - - return errDecls, err -} - -func export(f *ast.File) []string { - if len(f.Decls) == 0 { - return nil - } - - var errNames []string - for _, decl := range f.Decls { - gen, ok := decl.(*ast.GenDecl) - if !ok || len(gen.Specs) == 0 { - continue - } - for _, spec := range gen.Specs { - switch errSpec := spec.(type) { - case *ast.ValueSpec: - // CASES: - // var ErrXXX = errors.Normalize(string, opts...) - // var ( - // ErrYYY = errors.Normalize(string, opts...) - // ErrZZZ = errors.Normalize(string, opts...) - // A = errors.Normalize(string, opts...) - // ) - // var ErrXXX, ErrYYY = errors.Normalize(string, opts...), errors.Normalize(string, opts...) - // var ( - // ErrYYY = errors.Normalize(string, opts...) - // ErrZZZ, ErrWWW = errors.Normalize(string, opts...), errors.Normalize(string, opts...) - // A = errors.Normalize(string, opts...) - // ) - // - if len(errSpec.Names) != len(errSpec.Values) { - continue - } - for i, name := range errSpec.Names { - if !strings.HasPrefix(name.Name, "Err") { - continue - } - _, ok := errSpec.Values[i].(*ast.CallExpr) - if !ok { - continue - } - errNames = append(errNames, name.Name) - } - default: - continue - } - } - } - return errNames -} diff --git a/components/errdoc/main.go b/components/errdoc/main.go deleted file mode 100644 index c4d5f76c4d..0000000000 --- a/components/errdoc/main.go +++ /dev/null @@ -1,174 +0,0 @@ -// Copyright 2020 PingCAP, Inc. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// See the License for the specific language governing permissions and -// limitations under the License. - -package main - -import ( - "errors" - "fmt" - "os" - "path/filepath" - "sort" - "strings" - - "github.com/BurntSushi/toml" - "github.com/blevesearch/bleve" - _ "github.com/blevesearch/bleve/index/store/goleveldb" - "github.com/blevesearch/bleve/search/query" - "github.com/pingcap/tiup/components/errdoc/spec" - "github.com/pingcap/tiup/pkg/localdata" - "github.com/spf13/cobra" -) - -func init() { - bleve.Config.DefaultKVStore = "goleveldb" -} - -func main() { - rootCmd := &cobra.Command{ - Use: "tiup errdoc", - Short: "Show detailed error message via error code or keyword", - SilenceUsage: true, - RunE: func(cmd *cobra.Command, args []string) error { - if len(args) == 0 { - return cmd.Help() - } - return searchError(args) - }, - } - - if err := rootCmd.Execute(); err != nil { - os.Exit(1) - } -} - -func searchError(args []string) error { - index, errStore, err := loadIndex() - if err != nil { - return err - } - - // Bleve search - queries := []query.Query{ - bleve.NewMatchPhraseQuery(strings.Join(args, " ")), - } - var terms []query.Query - var prefix []query.Query - for _, arg := range args { - terms = append(terms, bleve.NewTermQuery(arg)) - prefix = append(prefix, bleve.NewPrefixQuery(arg)) - } - queries = append(queries, bleve.NewConjunctionQuery(terms...)) - queries = append(queries, bleve.NewConjunctionQuery(prefix...)) - - result, err := index.Search(bleve.NewSearchRequest(bleve.NewDisjunctionQuery(queries...))) - if err != nil { - return err - } - - all := map[string]struct{}{} - for _, match := range result.Hits { - all[match.ID] = struct{}{} - } - - // Error code prefix match - if len(args) == 1 { - c := strings.ToLower(args[0]) - for code := range errStore { - if strings.HasPrefix(strings.ToLower(code), c) { - all[code] = struct{}{} - } - } - } - - var sorted []string - for code := range all { - sorted = append(sorted, code) - } - sort.Slice(sorted, func(i, j int) bool { - return sorted[i] < sorted[j] - }) - for _, code := range sorted { - spec := errStore[code] - fmt.Println(spec) - } - - fmt.Printf("%d matched\n", len(sorted)) - - return nil -} - -func loadIndex() (bleve.Index, map[string]*spec.ErrorSpec, error) { - dir := os.Getenv(localdata.EnvNameComponentInstallDir) - if dir == "" { - return nil, nil, errors.New("component `errdoc` doesn't running in TiUP mode") - } - - type tomlFile map[string]*spec.ErrorSpec - indexPath := filepath.Join(dir, "index") - - index, err := bleve.Open(indexPath) - if err != nil && err != bleve.ErrorIndexPathDoesNotExist { - return nil, nil, err - } - - needIndex := err == bleve.ErrorIndexPathDoesNotExist - - if needIndex { - indexMapping := bleve.NewIndexMapping() - if err := os.MkdirAll(indexPath, 0755); err != nil { - return nil, nil, err - } - index, err = bleve.New(indexPath, indexMapping) - if err != nil { - return nil, nil, err - } - } - - errStore := map[string]*spec.ErrorSpec{} - - err = filepath.Walk(dir, func(path string, info os.FileInfo, err error) error { - if err != nil { - return err - } - if info.IsDir() { - return nil - } - if !strings.HasSuffix(info.Name(), "toml") { - return nil - } - reader, err := os.Open(path) - if err != nil { - return err - } - // Ignore file if cannot be unmarshalled to error specification - file := tomlFile{} - if _, err := toml.NewDecoder(reader).Decode(&file); err != nil { - return nil - } - for code, spec := range file { - spec.Code = code - spec.ExtraCode = strings.ReplaceAll(code, ":", " ") - spec.ExtraError = strings.ReplaceAll(spec.Error, ":", " ") - errStore[code] = spec - if !needIndex { - continue - } - if err := index.Index(code, spec); err != nil { - return err - } - } - return nil - }) - return index, errStore, err -} diff --git a/components/errdoc/spec/spec.go b/components/errdoc/spec/spec.go deleted file mode 100644 index 8f4741c539..0000000000 --- a/components/errdoc/spec/spec.go +++ /dev/null @@ -1,65 +0,0 @@ -// Copyright 2020 PingCAP, Inc. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// See the License for the specific language governing permissions and -// limitations under the License. - -package spec - -import ( - "fmt" - "strings" - - "github.com/tj/go-termd" -) - -// ErrorSpec represents the standard error specification introduced by -// https://github.com/pingcap/tidb/blob/master/docs/design/2020-05-08-standardize-error-codes-and-messages.md -type ErrorSpec struct { - Code string `toml:"code" json:"code"` - Error string `toml:"error" json:"error"` - Description string `toml:"description" json:"description"` - Tags []string `toml:"tags" json:"tags"` - Workaround string `toml:"workaround" json:"workaround"` - // Used for indexes - ExtraCode string `toml:"extracode" json:"extracode"` - ExtraError string `toml:"extraerror" json:"extraerror"` -} - -func newCompiler() *termd.Compiler { - return &termd.Compiler{ - Columns: 80, - } -} - -// String implements the fmt.Stringer interface -func (f ErrorSpec) String() string { - var header string - if len(f.Tags) > 0 { - header = fmt.Sprintf("# Error: **%s**, Tags: %v", f.Code, f.Tags) - } else { - header = fmt.Sprintf("# Error: **%s**", f.Code) - } - - tmpl := header + "\n" + strings.TrimSpace(f.Error) - - description := f.Description - if description != "" { - tmpl += `## Description - ` + strings.TrimSpace(description) - } - workaround := f.Workaround - if workaround != "" { - tmpl += `## Workaround - ` + strings.TrimSpace(workaround) - } - - return newCompiler().Compile(tmpl) -} diff --git a/components/errdoc/testdata/tidb.toml b/components/errdoc/testdata/tidb.toml deleted file mode 100644 index 22c77e1351..0000000000 --- a/components/errdoc/testdata/tidb.toml +++ /dev/null @@ -1,20 +0,0 @@ -[8005] -error = '''Write Conflict, txnStartTS is stale''' -tags = ["tikv", "tidb-txn"] -description = '''In versions before 3.0.8, TiDB implements the optimistic transaction mode by default, in which the transaction commit might fail because of transaction conflict. To make sure that the commit succeeds, you need to modify the application andad an automatic retry mechanism. You can avoid this issue by using the pessimistic transaction mode of TiDB.''' -workaround = '''To apply the pessimistic transaction mode, choose any of the following three methods that suits your needs: -- Execute the `BEGIN PESSIMISTIC;` statement to allow the transaction to apply the pessimistic transaction mode. You can write it in comment style as `BEGIN /*!90000 PESSIMISTIC */;` to make it compatible with the MySQL syntax. - -- Execute the `set @@tidb_txn_mode = 'pessimistic';` statement to allow all the explicit transactions (namely non-autocommit transactions) processed in this session to apply the pessimistic transaction mode. - -- Execute the `set @@global.tidb_txn_mode = 'pessimistic';` statement to allow all newly created sessions of the entire cluster to apply the pessimistic transaction mode to execute explicit transactions. -After you set `global.tidb_txn_mode` to `pessimistic`, the pessimistic transaction mode is applied by default. To apply the optimistic transaction mode to the transaction, you can use any of the following three methods: - -- Execute the `BEGIN OPTIMISTIC;` statement to allow the transaction to apply the optimistic transaction mode. You can write it in comment style as `BEGIN /*!90000 OPTIMISTIC */;` to make it compatible with the MySQL syntax. - -- Execute the `set @@tidb_txn_mode = 'optimistic';` statement to allow all the transactions processed in this session to apply the optimistic transaction mode. - -- Execute the `set @@global.tidb_txn_mode = 'optimistic;'` or `set @@global.tidb_txn_mode = '';` to allow all newly created sessions of the entire cluster to apply the optimistic transaction mode to the transactions. - -The `BEGIN PESSIMISTIC;` and `BEGIN OPTIMISTIC;` statements take precedence over the `tidb_txn_mode` system variable. Transactions started with these two statements ignore the system variable and support using both the pessimistic and optimistic transaction modes. -To disable the pessimistic transaction mode, modify the configuration file and add `enable = false` to the `[pessimistic-txn]` category.''' diff --git a/components/playground/command.go b/components/playground/command.go index 4857178f24..d43fe87e30 100644 --- a/components/playground/command.go +++ b/components/playground/command.go @@ -54,6 +54,7 @@ func buildCommands(tp CommandType, opt *BootOptions) (cmds []Command) { {"tikv", opt.TiKV}, {"pump", opt.Pump}, {"tiflash", opt.TiFlash}, + {"tiproxy", opt.TiProxy}, {"tidb", opt.TiDB}, {"ticdc", opt.TiCDC}, {"tikv-cdc", opt.TiKVCDC}, @@ -98,18 +99,20 @@ func newScaleOut() *cobra.Command { cmd.Flags().IntVarP(&opt.TiKV.Num, "kv", "", opt.TiKV.Num, "TiKV instance number") cmd.Flags().IntVarP(&opt.PD.Num, "pd", "", opt.PD.Num, "PD instance number") cmd.Flags().IntVarP(&opt.TiFlash.Num, "tiflash", "", opt.TiFlash.Num, "TiFlash instance number") + cmd.Flags().IntVarP(&opt.TiProxy.Num, "tiproxy", "", opt.TiProxy.Num, "TiProxy instance number") cmd.Flags().IntVarP(&opt.TiCDC.Num, "ticdc", "", opt.TiCDC.Num, "TiCDC instance number") cmd.Flags().IntVarP(&opt.TiKVCDC.Num, "kvcdc", "", opt.TiKVCDC.Num, "TiKV-CDC instance number") cmd.Flags().IntVarP(&opt.Pump.Num, "pump", "", opt.Pump.Num, "Pump instance number") cmd.Flags().IntVarP(&opt.Drainer.Num, "drainer", "", opt.Pump.Num, "Drainer instance number") - cmd.Flags().StringVarP(&opt.TiDB.Host, "db.host", "", opt.TiDB.Host, "Playground TiDB host. If not provided, TiDB will still use `host` flag as its host") cmd.Flags().StringVarP(&opt.PD.Host, "pd.host", "", opt.PD.Host, "Playground PD host. If not provided, PD will still use `host` flag as its host") + cmd.Flags().StringVarP(&opt.TiProxy.Host, "tiproxy.host", "", opt.PD.Host, "Playground TiProxy host. If not provided, TiProxy will still use `host` flag as its host") cmd.Flags().StringVarP(&opt.TiDB.ConfigPath, "db.config", "", opt.TiDB.ConfigPath, "TiDB instance configuration file") cmd.Flags().StringVarP(&opt.TiKV.ConfigPath, "kv.config", "", opt.TiKV.ConfigPath, "TiKV instance configuration file") cmd.Flags().StringVarP(&opt.PD.ConfigPath, "pd.config", "", opt.PD.ConfigPath, "PD instance configuration file") - cmd.Flags().StringVarP(&opt.TiDB.ConfigPath, "tiflash.config", "", opt.TiDB.ConfigPath, "TiFlash instance configuration file") + cmd.Flags().StringVarP(&opt.TiFlash.ConfigPath, "tiflash.config", "", opt.TiFlash.ConfigPath, "TiFlash instance configuration file") + cmd.Flags().StringVarP(&opt.TiProxy.ConfigPath, "tiproxy.config", "", opt.TiProxy.ConfigPath, "TiProxy instance configuration file") cmd.Flags().StringVarP(&opt.Pump.ConfigPath, "pump.config", "", opt.Pump.ConfigPath, "Pump instance configuration file") cmd.Flags().StringVarP(&opt.Drainer.ConfigPath, "drainer.config", "", opt.Drainer.ConfigPath, "Drainer instance configuration file") @@ -117,6 +120,7 @@ func newScaleOut() *cobra.Command { cmd.Flags().StringVarP(&opt.TiKV.BinPath, "kv.binpath", "", opt.TiKV.BinPath, "TiKV instance binary path") cmd.Flags().StringVarP(&opt.PD.BinPath, "pd.binpath", "", opt.PD.BinPath, "PD instance binary path") cmd.Flags().StringVarP(&opt.TiFlash.BinPath, "tiflash.binpath", "", opt.TiFlash.BinPath, "TiFlash instance binary path") + cmd.Flags().StringVarP(&opt.TiProxy.BinPath, "tiproxy.binpath", "", opt.TiProxy.BinPath, "TiProxy instance binary path") cmd.Flags().StringVarP(&opt.TiCDC.BinPath, "ticdc.binpath", "", opt.TiCDC.BinPath, "TiCDC instance binary path") cmd.Flags().StringVarP(&opt.TiKVCDC.BinPath, "kvcdc.binpath", "", opt.TiKVCDC.BinPath, "TiKVCDC instance binary path") cmd.Flags().StringVarP(&opt.Pump.BinPath, "pump.binpath", "", opt.Pump.BinPath, "Pump instance binary path") diff --git a/components/playground/grafana.go b/components/playground/grafana.go index d56b9e55e6..c921a29ca5 100644 --- a/components/playground/grafana.go +++ b/components/playground/grafana.go @@ -69,7 +69,7 @@ datasources: ` s := fmt.Sprintf(tpl, clusterName, p8sURL) - err = os.WriteFile(fname, []byte(s), 0644) + err = utils.WriteFile(fname, []byte(s), 0644) if err != nil { return errors.AddStack(err) } @@ -104,7 +104,7 @@ func replaceDatasource(dashboardDir string, datasourceName string) error { s = strings.ReplaceAll(s, "${DS_LIGHTNING}", datasourceName) s = re.ReplaceAllLiteralString(s, datasourceName) - return os.WriteFile(path, []byte(s), 0644) + return utils.WriteFile(path, []byte(s), 0644) }) if err != nil { @@ -134,7 +134,7 @@ providers: ` s := fmt.Sprintf(tpl, clusterName, clusterName, dir) - err = os.WriteFile(fname, []byte(s), 0644) + err = utils.WriteFile(fname, []byte(s), 0644) if err != nil { return errors.AddStack(err) } @@ -143,7 +143,7 @@ providers: } func makeSureDir(fname string) error { - return os.MkdirAll(filepath.Dir(fname), 0755) + return utils.MkdirAll(filepath.Dir(fname), 0755) } var clusterName = "Test-Cluster" @@ -176,7 +176,7 @@ http_addr = %s # The http port to use http_port = %d ` - err = os.MkdirAll(filepath.Join(dir, "conf"), 0755) + err = utils.MkdirAll(filepath.Join(dir, "conf"), 0755) if err != nil { return errors.AddStack(err) } @@ -184,7 +184,7 @@ http_port = %d custome := fmt.Sprintf(tpl, g.host, g.port) customeFName := filepath.Join(dir, "conf", "custom.ini") - err = os.WriteFile(customeFName, []byte(custome), 0644) + err = utils.WriteFile(customeFName, []byte(custome), 0644) if err != nil { return errors.AddStack(err) } diff --git a/components/playground/instance/instance.go b/components/playground/instance/instance.go index f084b011b1..620f2a502d 100644 --- a/components/playground/instance/instance.go +++ b/components/playground/instance/instance.go @@ -17,7 +17,12 @@ import ( "context" "fmt" "net" + "os" + "path/filepath" + "github.com/BurntSushi/toml" + "github.com/pingcap/errors" + "github.com/pingcap/tiup/pkg/cluster/spec" "github.com/pingcap/tiup/pkg/utils" ) @@ -42,6 +47,12 @@ type instance struct { BinPath string } +// MetricAddr will be used by prometheus scrape_configs. +type MetricAddr struct { + Targets []string `json:"targets"` + Labels map[string]string `json:"labels"` +} + // Instance represent running component type Instance interface { Pid() int @@ -54,16 +65,16 @@ type Instance interface { LogFile() string // Uptime show uptime. Uptime() string - // StatusAddrs return the address to pull metrics. - StatusAddrs() []string + // MetricAddr return the address to pull metrics. + MetricAddr() MetricAddr // Wait Should only call this if the instance is started successfully. // The implementation should be safe to call Wait multi times. Wait() error } -func (inst *instance) StatusAddrs() (addrs []string) { +func (inst *instance) MetricAddr() (r MetricAddr) { if inst.Host != "" && inst.StatusPort != 0 { - addrs = append(addrs, utils.JoinHostPort(inst.Host, inst.StatusPort)) + r.Targets = append(r.Targets, utils.JoinHostPort(inst.Host, inst.StatusPort)) } return } @@ -104,6 +115,9 @@ func logIfErr(err error) { func pdEndpoints(pds []*PDInstance, isHTTP bool) []string { var endpoints []string for _, pd := range pds { + if pd.Role == PDRoleTSO || pd.Role == PDRoleScheduling || pd.Role == PDRoleResourceManager { + continue + } if isHTTP { endpoints = append(endpoints, "http://"+utils.JoinHostPort(AdvertiseHost(pd.Host), pd.StatusPort)) } else { @@ -112,3 +126,45 @@ func pdEndpoints(pds []*PDInstance, isHTTP bool) []string { } return endpoints } + +// prepareConfig accepts a user specified config and merge user config with a +// pre-defined one. +func prepareConfig(outputConfigPath string, userConfigPath string, preDefinedConfig map[string]any) error { + dir := filepath.Dir(outputConfigPath) + if err := utils.MkdirAll(dir, 0755); err != nil { + return err + } + + userConfig, err := unmarshalConfig(userConfigPath) + if err != nil { + return errors.Trace(err) + } + if userConfig == nil { + userConfig = make(map[string]any) + } + + cf, err := os.Create(outputConfigPath) + if err != nil { + return errors.Trace(err) + } + + enc := toml.NewEncoder(cf) + enc.Indent = "" + return enc.Encode(spec.MergeConfig(preDefinedConfig, userConfig)) +} + +func unmarshalConfig(path string) (map[string]any, error) { + if path == "" { + return nil, nil + } + data, err := os.ReadFile(path) + if err != nil { + return nil, err + } + c := make(map[string]any) + err = toml.Unmarshal(data, &c) + if err != nil { + return nil, err + } + return c, nil +} diff --git a/components/playground/instance/pd.go b/components/playground/instance/pd.go index b37bee57de..5d9a549cac 100644 --- a/components/playground/instance/pd.go +++ b/components/playground/instance/pd.go @@ -24,16 +24,34 @@ import ( "github.com/pingcap/tiup/pkg/utils" ) +// PDRole is the role of PD. +type PDRole string + +const ( + // PDRoleNormal is the default role of PD + PDRoleNormal PDRole = "pd" + // PDRoleAPI is the role of PD API + PDRoleAPI PDRole = "api" + // PDRoleTSO is the role of PD TSO + PDRoleTSO PDRole = "tso" + // PDRoleScheduling is the role of PD scheduling + PDRoleScheduling PDRole = "scheduling" + // PDRoleResourceManager is the role of PD resource manager + PDRoleResourceManager PDRole = "resource_manager" +) + // PDInstance represent a running pd-server type PDInstance struct { instance + Role PDRole initEndpoints []*PDInstance joinEndpoints []*PDInstance + pds []*PDInstance Process } // NewPDInstance return a PDInstance -func NewPDInstance(binPath, dir, host, configPath string, id, port int) *PDInstance { +func NewPDInstance(role PDRole, binPath, dir, host, configPath string, id int, pds []*PDInstance, port int) *PDInstance { if port <= 0 { port = 2379 } @@ -47,6 +65,8 @@ func NewPDInstance(binPath, dir, host, configPath string, id, port int) *PDInsta StatusPort: utils.MustGetFreePort(host, port), ConfigPath: configPath, }, + Role: role, + pds: pds, } } @@ -70,35 +90,80 @@ func (inst *PDInstance) Name() string { // Start calls set inst.cmd and Start func (inst *PDInstance) Start(ctx context.Context, version utils.Version) error { uid := inst.Name() - args := []string{ - "--name=" + uid, - fmt.Sprintf("--data-dir=%s", filepath.Join(inst.Dir, "data")), - fmt.Sprintf("--peer-urls=http://%s", utils.JoinHostPort(inst.Host, inst.Port)), - fmt.Sprintf("--advertise-peer-urls=http://%s", utils.JoinHostPort(AdvertiseHost(inst.Host), inst.Port)), - fmt.Sprintf("--client-urls=http://%s", utils.JoinHostPort(inst.Host, inst.StatusPort)), - fmt.Sprintf("--advertise-client-urls=http://%s", utils.JoinHostPort(AdvertiseHost(inst.Host), inst.StatusPort)), - fmt.Sprintf("--log-file=%s", inst.LogFile()), - } - if inst.ConfigPath != "" { - args = append(args, fmt.Sprintf("--config=%s", inst.ConfigPath)) - } - - switch { - case len(inst.initEndpoints) > 0: - endpoints := make([]string, 0) - for _, pd := range inst.initEndpoints { - uid := fmt.Sprintf("pd-%d", pd.ID) - endpoints = append(endpoints, fmt.Sprintf("%s=http://%s", uid, utils.JoinHostPort(AdvertiseHost(inst.Host), pd.Port))) + var args []string + switch inst.Role { + case PDRoleNormal, PDRoleAPI: + if inst.Role == PDRoleAPI { + args = []string{"services", "api"} + } + args = append(args, []string{ + "--name=" + uid, + fmt.Sprintf("--data-dir=%s", filepath.Join(inst.Dir, "data")), + fmt.Sprintf("--peer-urls=http://%s", utils.JoinHostPort(inst.Host, inst.Port)), + fmt.Sprintf("--advertise-peer-urls=http://%s", utils.JoinHostPort(AdvertiseHost(inst.Host), inst.Port)), + fmt.Sprintf("--client-urls=http://%s", utils.JoinHostPort(inst.Host, inst.StatusPort)), + fmt.Sprintf("--advertise-client-urls=http://%s", utils.JoinHostPort(AdvertiseHost(inst.Host), inst.StatusPort)), + fmt.Sprintf("--log-file=%s", inst.LogFile()), + }...) + if inst.ConfigPath != "" { + args = append(args, fmt.Sprintf("--config=%s", inst.ConfigPath)) } - args = append(args, fmt.Sprintf("--initial-cluster=%s", strings.Join(endpoints, ","))) - case len(inst.joinEndpoints) > 0: - endpoints := make([]string, 0) - for _, pd := range inst.joinEndpoints { - endpoints = append(endpoints, fmt.Sprintf("http://%s", utils.JoinHostPort(AdvertiseHost(inst.Host), pd.Port))) + switch { + case len(inst.initEndpoints) > 0: + endpoints := make([]string, 0) + for _, pd := range inst.initEndpoints { + uid := fmt.Sprintf("pd-%d", pd.ID) + endpoints = append(endpoints, fmt.Sprintf("%s=http://%s", uid, utils.JoinHostPort(AdvertiseHost(inst.Host), pd.Port))) + } + args = append(args, fmt.Sprintf("--initial-cluster=%s", strings.Join(endpoints, ","))) + case len(inst.joinEndpoints) > 0: + endpoints := make([]string, 0) + for _, pd := range inst.joinEndpoints { + endpoints = append(endpoints, fmt.Sprintf("http://%s", utils.JoinHostPort(AdvertiseHost(inst.Host), pd.Port))) + } + args = append(args, fmt.Sprintf("--join=%s", strings.Join(endpoints, ","))) + default: + return errors.Errorf("must set the init or join instances") + } + case PDRoleTSO: + endpoints := pdEndpoints(inst.pds, true) + args = []string{ + "services", + "tso", + fmt.Sprintf("--listen-addr=http://%s", utils.JoinHostPort(inst.Host, inst.StatusPort)), + fmt.Sprintf("--advertise-listen-addr=http://%s", utils.JoinHostPort(AdvertiseHost(inst.Host), inst.StatusPort)), + fmt.Sprintf("--backend-endpoints=%s", strings.Join(endpoints, ",")), + fmt.Sprintf("--log-file=%s", inst.LogFile()), + } + if inst.ConfigPath != "" { + args = append(args, fmt.Sprintf("--config=%s", inst.ConfigPath)) + } + case PDRoleScheduling: + endpoints := pdEndpoints(inst.pds, true) + args = []string{ + "services", + "scheduling", + fmt.Sprintf("--listen-addr=http://%s", utils.JoinHostPort(inst.Host, inst.StatusPort)), + fmt.Sprintf("--advertise-listen-addr=http://%s", utils.JoinHostPort(AdvertiseHost(inst.Host), inst.StatusPort)), + fmt.Sprintf("--backend-endpoints=%s", strings.Join(endpoints, ",")), + fmt.Sprintf("--log-file=%s", inst.LogFile()), + } + if inst.ConfigPath != "" { + args = append(args, fmt.Sprintf("--config=%s", inst.ConfigPath)) + } + case PDRoleResourceManager: + endpoints := pdEndpoints(inst.pds, true) + args = []string{ + "services", + "resource-manager", + fmt.Sprintf("--listen-addr=http://%s", utils.JoinHostPort(inst.Host, inst.StatusPort)), + fmt.Sprintf("--advertise-listen-addr=http://%s", utils.JoinHostPort(AdvertiseHost(inst.Host), inst.StatusPort)), + fmt.Sprintf("--backend-endpoints=%s", strings.Join(endpoints, ",")), + fmt.Sprintf("--log-file=%s", inst.LogFile()), + } + if inst.ConfigPath != "" { + args = append(args, fmt.Sprintf("--config=%s", inst.ConfigPath)) } - args = append(args, fmt.Sprintf("--join=%s", strings.Join(endpoints, ","))) - default: - return errors.Errorf("must set the init or join instances") } var err error @@ -113,12 +178,15 @@ func (inst *PDInstance) Start(ctx context.Context, version utils.Version) error // Component return the component name. func (inst *PDInstance) Component() string { - return "pd" + if inst.Role == PDRoleNormal { + return "pd" + } + return fmt.Sprintf("pd %s", inst.Role) } // LogFile return the log file. func (inst *PDInstance) LogFile() string { - return filepath.Join(inst.Dir, "pd.log") + return filepath.Join(inst.Dir, fmt.Sprintf("%s.log", string(inst.Role))) } // Addr return the listen address of PD diff --git a/components/playground/instance/ticdc.go b/components/playground/instance/ticdc.go index 36a4ab6c55..5c987b2c5a 100644 --- a/components/playground/instance/ticdc.go +++ b/components/playground/instance/ticdc.go @@ -34,14 +34,17 @@ type TiCDC struct { var _ Instance = &TiCDC{} // NewTiCDC create a TiCDC instance. -func NewTiCDC(binPath string, dir, host, configPath string, id int, pds []*PDInstance) *TiCDC { +func NewTiCDC(binPath string, dir, host, configPath string, id int, port int, pds []*PDInstance) *TiCDC { + if port <= 0 { + port = 8300 + } ticdc := &TiCDC{ instance: instance{ BinPath: binPath, ID: id, Dir: dir, Host: host, - Port: utils.MustGetFreePort(host, 8300), + Port: utils.MustGetFreePort(host, port), ConfigPath: configPath, }, pds: pds, diff --git a/components/playground/instance/tidb.go b/components/playground/instance/tidb.go index 2ba35536c0..93acc95e24 100644 --- a/components/playground/instance/tidb.go +++ b/components/playground/instance/tidb.go @@ -30,10 +30,11 @@ type TiDBInstance struct { pds []*PDInstance Process enableBinlog bool + isDisaggMode bool } // NewTiDBInstance return a TiDBInstance -func NewTiDBInstance(binPath string, dir, host, configPath string, id, port int, pds []*PDInstance, enableBinlog bool) *TiDBInstance { +func NewTiDBInstance(binPath string, dir, host, configPath string, id, port int, pds []*PDInstance, enableBinlog bool, isDisaggMode bool) *TiDBInstance { if port <= 0 { port = 4000 } @@ -49,11 +50,21 @@ func NewTiDBInstance(binPath string, dir, host, configPath string, id, port int, }, pds: pds, enableBinlog: enableBinlog, + isDisaggMode: isDisaggMode, } } // Start calls set inst.cmd and Start func (inst *TiDBInstance) Start(ctx context.Context, version utils.Version) error { + configPath := filepath.Join(inst.Dir, "tidb.toml") + if err := prepareConfig( + configPath, + inst.ConfigPath, + inst.getConfig(), + ); err != nil { + return err + } + endpoints := pdEndpoints(inst.pds, false) args := []string{ @@ -63,9 +74,7 @@ func (inst *TiDBInstance) Start(ctx context.Context, version utils.Version) erro fmt.Sprintf("--status=%d", inst.StatusPort), fmt.Sprintf("--path=%s", strings.Join(endpoints, ",")), fmt.Sprintf("--log-file=%s", filepath.Join(inst.Dir, "tidb.log")), - } - if inst.ConfigPath != "" { - args = append(args, fmt.Sprintf("--config=%s", inst.ConfigPath)) + fmt.Sprintf("--config=%s", configPath), } if inst.enableBinlog { args = append(args, "--enable-binlog=true") diff --git a/components/playground/instance/tidb_config.go b/components/playground/instance/tidb_config.go new file mode 100644 index 0000000000..74731b8c39 --- /dev/null +++ b/components/playground/instance/tidb_config.go @@ -0,0 +1,26 @@ +// Copyright 2023 PingCAP, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// See the License for the specific language governing permissions and +// limitations under the License. + +package instance + +func (inst *TiDBInstance) getConfig() map[string]any { + config := make(map[string]any) + config["security.auto-tls"] = true + + if inst.isDisaggMode { + config["use-autoscaler"] = false + config["disaggregated-tiflash"] = true + } + + return config +} diff --git a/components/playground/instance/tiflash.go b/components/playground/instance/tiflash.go index 071b386d00..cb73f2d722 100644 --- a/components/playground/instance/tiflash.go +++ b/components/playground/instance/tiflash.go @@ -14,73 +14,83 @@ package instance import ( - "bytes" "context" - "encoding/json" "fmt" - "os" "os/exec" - "path" "path/filepath" - "time" + "strings" - "github.com/pingcap/errors" - "github.com/pingcap/tiup/pkg/cluster/api" tiupexec "github.com/pingcap/tiup/pkg/exec" + "github.com/pingcap/tiup/pkg/tidbver" "github.com/pingcap/tiup/pkg/utils" ) +// TiFlashRole is the role of TiFlash. +type TiFlashRole string + +const ( + // TiFlashRoleNormal is used when TiFlash is not in disaggregated mode. + TiFlashRoleNormal TiFlashRole = "normal" + + // TiFlashRoleDisaggWrite is used when TiFlash is in disaggregated mode and is the write node. + TiFlashRoleDisaggWrite TiFlashRole = "write" + // TiFlashRoleDisaggCompute is used when TiFlash is in disaggregated mode and is the compute node. + TiFlashRoleDisaggCompute TiFlashRole = "compute" +) + +// DisaggOptions contains configs to run TiFlash in disaggregated mode. +type DisaggOptions struct { + S3Endpoint string `yaml:"s3_endpoint"` + Bucket string `yaml:"bucket"` + AccessKey string `yaml:"access_key"` + SecretKey string `yaml:"secret_key"` +} + // TiFlashInstance represent a running TiFlash type TiFlashInstance struct { instance + Role TiFlashRole + DisaggOpts DisaggOptions TCPPort int ServicePort int ProxyPort int ProxyStatusPort int - ProxyConfigPath string pds []*PDInstance dbs []*TiDBInstance Process } // NewTiFlashInstance return a TiFlashInstance -func NewTiFlashInstance(binPath, dir, host, configPath string, id int, pds []*PDInstance, dbs []*TiDBInstance) *TiFlashInstance { +func NewTiFlashInstance(role TiFlashRole, disaggOptions DisaggOptions, binPath, dir, host, configPath string, id int, pds []*PDInstance, dbs []*TiDBInstance, version string) *TiFlashInstance { + if role != TiFlashRoleNormal && role != TiFlashRoleDisaggWrite && role != TiFlashRoleDisaggCompute { + panic(fmt.Sprintf("Unknown TiFlash role %s", role)) + } + + httpPort := 8123 + if !tidbver.TiFlashNotNeedHTTPPortConfig(version) { + httpPort = utils.MustGetFreePort(host, httpPort) + } return &TiFlashInstance{ instance: instance{ BinPath: binPath, ID: id, Dir: dir, Host: host, - Port: utils.MustGetFreePort(host, 8123), + Port: httpPort, StatusPort: utils.MustGetFreePort(host, 8234), ConfigPath: configPath, }, - TCPPort: utils.MustGetFreePort(host, 9000), + Role: role, + DisaggOpts: disaggOptions, + TCPPort: utils.MustGetFreePort(host, 9100), // 9000 for default object store port ServicePort: utils.MustGetFreePort(host, 3930), ProxyPort: utils.MustGetFreePort(host, 20170), ProxyStatusPort: utils.MustGetFreePort(host, 20292), - ProxyConfigPath: configPath, pds: pds, dbs: dbs, } } -func getFlashClusterPath(dir string) string { - return fmt.Sprintf("%s/flash_cluster_manager", dir) -} - -type scheduleConfig struct { - LowSpaceRatio float64 `json:"low-space-ratio"` -} - -type replicateMaxReplicaConfig struct { - MaxReplicas int `json:"max-replicas"` -} - -type replicateEnablePlacementRulesConfig struct { - EnablePlacementRules string `json:"enable-placement-rules"` -} - // Addr return the address of tiflash func (inst *TiFlashInstance) Addr() string { return utils.JoinHostPort(AdvertiseHost(inst.Host), inst.ServicePort) @@ -95,68 +105,56 @@ func (inst *TiFlashInstance) StatusAddrs() (addrs []string) { // Start calls set inst.cmd and Start func (inst *TiFlashInstance) Start(ctx context.Context, version utils.Version) error { - endpoints := pdEndpoints(inst.pds, false) - - tidbStatusAddrs := make([]string, 0, len(inst.dbs)) - for _, db := range inst.dbs { - tidbStatusAddrs = append(tidbStatusAddrs, utils.JoinHostPort(AdvertiseHost(db.Host), db.StatusPort)) - } - wd, err := filepath.Abs(inst.Dir) - if err != nil { - return err + if !tidbver.TiFlashPlaygroundNewStartMode(version.String()) { + return inst.startOld(ctx, version) } - // Wait for PD - pdClient := api.NewPDClient(ctx, endpoints, 10*time.Second, nil) - // set low-space-ratio to 1 to avoid low disk space - lowSpaceRatio, err := json.Marshal(scheduleConfig{ - LowSpaceRatio: 0.99, - }) - if err != nil { - return err - } - if err = pdClient.UpdateScheduleConfig(bytes.NewBuffer(lowSpaceRatio)); err != nil { - return err - } - // Update maxReplicas before placement rules so that it would not be overwritten - maxReplicas, err := json.Marshal(replicateMaxReplicaConfig{ - MaxReplicas: 1, - }) - if err != nil { - return err - } - if err = pdClient.UpdateReplicateConfig(bytes.NewBuffer(maxReplicas)); err != nil { - return err - } - // Set enable-placement-rules to allow TiFlash work properly - enablePlacementRules, err := json.Marshal(replicateEnablePlacementRulesConfig{ - EnablePlacementRules: "true", - }) - if err != nil { - return err - } - if err = pdClient.UpdateReplicateConfig(bytes.NewBuffer(enablePlacementRules)); err != nil { + proxyConfigPath := filepath.Join(inst.Dir, "tiflash_proxy.toml") + if err := prepareConfig( + proxyConfigPath, + "", + inst.getProxyConfig(), + ); err != nil { return err } - if inst.BinPath, err = tiupexec.PrepareBinary("tiflash", version, inst.BinPath); err != nil { + configPath := filepath.Join(inst.Dir, "tiflash.toml") + if err := prepareConfig( + configPath, + inst.ConfigPath, + inst.getConfig(), + ); err != nil { return err } - dirPath := filepath.Dir(inst.BinPath) - clusterManagerPath := getFlashClusterPath(dirPath) - if err = inst.checkConfig(wd, clusterManagerPath, version, tidbStatusAddrs, endpoints); err != nil { - return err - } + endpoints := pdEndpoints(inst.pds, false) args := []string{ "server", - fmt.Sprintf("--config-file=%s", inst.ConfigPath), } - envs := []string{ - fmt.Sprintf("LD_LIBRARY_PATH=%s:$LD_LIBRARY_PATH", dirPath), + args = append(args, + fmt.Sprintf("--config-file=%s", configPath), + "--", + fmt.Sprintf("--tmp_path=%s", filepath.Join(inst.Dir, "tmp")), + fmt.Sprintf("--path=%s", filepath.Join(inst.Dir, "data")), + fmt.Sprintf("--listen_host=%s", inst.Host), + fmt.Sprintf("--logger.log=%s", inst.LogFile()), + fmt.Sprintf("--logger.errorlog=%s", filepath.Join(inst.Dir, "tiflash_error.log")), + fmt.Sprintf("--status.metrics_port=%d", inst.StatusPort), + fmt.Sprintf("--flash.service_addr=%s", utils.JoinHostPort(AdvertiseHost(inst.Host), inst.ServicePort)), + fmt.Sprintf("--raft.pd_addr=%s", strings.Join(endpoints, ",")), + fmt.Sprintf("--flash.proxy.addr=%s", utils.JoinHostPort(inst.Host, inst.ProxyPort)), + fmt.Sprintf("--flash.proxy.advertise-addr=%s", utils.JoinHostPort(AdvertiseHost(inst.Host), inst.ProxyPort)), + fmt.Sprintf("--flash.proxy.status-addr=%s", utils.JoinHostPort(inst.Host, inst.ProxyStatusPort)), + fmt.Sprintf("--flash.proxy.data-dir=%s", filepath.Join(inst.Dir, "proxy_data")), + fmt.Sprintf("--flash.proxy.log-file=%s", filepath.Join(inst.Dir, "tiflash_tikv.log")), + ) + + var err error + if inst.BinPath, err = tiupexec.PrepareBinary("tiflash", version, inst.BinPath); err != nil { + return err } - inst.Process = &process{cmd: PrepareCommand(ctx, inst.BinPath, args, envs, inst.Dir)} + inst.Process = &process{cmd: PrepareCommand(ctx, inst.BinPath, args, nil, inst.Dir)} logIfErr(inst.Process.SetOutputFile(inst.LogFile())) return inst.Process.Start() @@ -181,50 +179,3 @@ func (inst *TiFlashInstance) Cmd() *exec.Cmd { func (inst *TiFlashInstance) StoreAddr() string { return utils.JoinHostPort(AdvertiseHost(inst.Host), inst.ServicePort) } - -func (inst *TiFlashInstance) checkConfig(deployDir, clusterManagerPath string, version utils.Version, tidbStatusAddrs, endpoints []string) error { - if err := os.MkdirAll(inst.Dir, 0755); err != nil { - return errors.Trace(err) - } - if inst.ConfigPath == "" { - inst.ConfigPath = path.Join(inst.Dir, "tiflash.toml") - } - if inst.ProxyConfigPath == "" { - inst.ProxyConfigPath = path.Join(inst.Dir, "tiflash-learner.toml") - } - - _, err := os.Stat(inst.ConfigPath) - if err == nil || os.IsExist(err) { - return nil - } - if !os.IsNotExist(err) { - return errors.Trace(err) - } - cf, err := os.Create(inst.ConfigPath) - if err != nil { - return errors.Trace(err) - } - defer cf.Close() - - _, err = os.Stat(inst.ProxyConfigPath) - if err == nil || os.IsExist(err) { - return nil - } - if !os.IsNotExist(err) { - return errors.Trace(err) - } - cf2, err := os.Create(inst.ProxyConfigPath) - if err != nil { - return errors.Trace(err) - } - defer cf2.Close() - if err := writeTiFlashConfig(cf, version, inst.TCPPort, inst.Port, inst.ServicePort, inst.StatusPort, - inst.Host, deployDir, clusterManagerPath, tidbStatusAddrs, endpoints); err != nil { - return errors.Trace(err) - } - if err := writeTiFlashProxyConfig(cf2, version, inst.Host, deployDir, inst.ServicePort, inst.ProxyPort, inst.ProxyStatusPort); err != nil { - return errors.Trace(err) - } - - return nil -} diff --git a/components/playground/instance/tiflash_config.go b/components/playground/instance/tiflash_config.go index ddf825c5ef..077d65375b 100644 --- a/components/playground/instance/tiflash_config.go +++ b/components/playground/instance/tiflash_config.go @@ -1,4 +1,4 @@ -// Copyright 2020 PingCAP, Inc. +// Copyright 2023 PingCAP, Inc. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. @@ -13,106 +13,41 @@ package instance -import ( - "fmt" - "io" - "strings" +import "path/filepath" - "github.com/pingcap/tiup/pkg/tidbver" - "github.com/pingcap/tiup/pkg/utils" -) - -const tiflashDaemonConfig = ` - -[application] -runAsDaemon = true - -` - -const tiflashConfig = ` -default_profile = "default" -display_name = "TiFlash" -http_port = %[2]d -listen_host = "0.0.0.0" -mark_cache_size = 5368709120 -path = "%[5]s" -tcp_port = %[3]d -tmp_path = "%[6]s" -%[13]s -[flash] -service_addr = "%[10]s:%[8]d" -tidb_status_addr = "%[11]s" -[flash.flash_cluster] -cluster_manager_path = "%[12]s" -log = "%[7]s/tiflash_cluster_manager.log" -master_ttl = 60 -refresh_interval = 20 -update_rule_interval = 5 -[flash.proxy] -config = "%[4]s/tiflash-learner.toml" - -[logger] -count = 20 -errorlog = "%[7]s/tiflash_error.log" -level = "debug" -log = "%[7]s/tiflash.log" -size = "1000M" - -[profiles] -[profiles.default] -load_balancing = "random" -max_memory_usage = 0 -use_uncompressed_cache = 0 -[profiles.readonly] -readonly = 1 - -[quotas] -[quotas.default] -[quotas.default.interval] -duration = 3600 -errors = 0 -execution_time = 0 -queries = 0 -read_rows = 0 -result_rows = 0 - -[raft] -pd_addr = "%[1]s" - -[status] -metrics_port = %[9]d - -[users] -[users.default] -password = "" -profile = "default" -quota = "default" -[users.default.networks] -ip = "::/0" -[users.readonly] -password = "" -profile = "readonly" -quota = "default" -[users.readonly.networks] -ip = "::/0" -` +func (inst *TiFlashInstance) getProxyConfig() map[string]any { + config := make(map[string]any) + config["rocksdb.max-open-files"] = 256 + config["raftdb.max-open-files"] = 256 + config["storage.reserve-space"] = 0 + config["storage.reserve-raft-space"] = 0 + return config +} -func writeTiFlashConfig(w io.Writer, version utils.Version, tcpPort, httpPort, servicePort, metricsPort int, host, deployDir, clusterManagerPath string, tidbStatusAddrs, endpoints []string) error { - pdAddrs := strings.Join(endpoints, ",") - dataDir := fmt.Sprintf("%s/data", deployDir) - tmpDir := fmt.Sprintf("%s/tmp", deployDir) - logDir := fmt.Sprintf("%s/log", deployDir) - ip := AdvertiseHost(host) - var conf string - if tidbver.TiFlashNotNeedSomeConfig(version.String()) { - conf = fmt.Sprintf(tiflashConfig, pdAddrs, httpPort, tcpPort, - deployDir, dataDir, tmpDir, logDir, servicePort, metricsPort, - ip, strings.Join(tidbStatusAddrs, ","), clusterManagerPath, "") - } else { - conf = fmt.Sprintf(tiflashConfig, pdAddrs, httpPort, tcpPort, - deployDir, dataDir, tmpDir, logDir, servicePort, metricsPort, - ip, strings.Join(tidbStatusAddrs, ","), clusterManagerPath, tiflashDaemonConfig) +func (inst *TiFlashInstance) getConfig() map[string]any { + config := make(map[string]any) + + config["flash.proxy.config"] = filepath.Join(inst.Dir, "tiflash_proxy.toml") + + if inst.Role == TiFlashRoleDisaggWrite { + config["storage.s3.endpoint"] = inst.DisaggOpts.S3Endpoint + config["storage.s3.bucket"] = inst.DisaggOpts.Bucket + config["storage.s3.root"] = "/" + config["storage.s3.access_key_id"] = inst.DisaggOpts.AccessKey + config["storage.s3.secret_access_key"] = inst.DisaggOpts.SecretKey + config["flash.disaggregated_mode"] = "tiflash_write" + config["flash.use_autoscaler"] = false + } else if inst.Role == TiFlashRoleDisaggCompute { + config["storage.s3.endpoint"] = inst.DisaggOpts.S3Endpoint + config["storage.s3.bucket"] = inst.DisaggOpts.Bucket + config["storage.s3.root"] = "/" + config["storage.s3.access_key_id"] = inst.DisaggOpts.AccessKey + config["storage.s3.secret_access_key"] = inst.DisaggOpts.SecretKey + config["storage.remote.cache.dir"] = filepath.Join(inst.Dir, "remote_cache") + config["storage.remote.cache.capacity"] = 1000000000 // 1GB + config["flash.disaggregated_mode"] = "tiflash_compute" + config["flash.use_autoscaler"] = false } - _, err := w.Write([]byte(conf)) - return err + + return config } diff --git a/components/playground/instance/tiflash_pre7.go b/components/playground/instance/tiflash_pre7.go new file mode 100644 index 0000000000..02f59747c4 --- /dev/null +++ b/components/playground/instance/tiflash_pre7.go @@ -0,0 +1,190 @@ +// Copyright 2020 PingCAP, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// See the License for the specific language governing permissions and +// limitations under the License. + +package instance + +import ( + "bytes" + "context" + "encoding/json" + "fmt" + "path" + "path/filepath" + "time" + + "github.com/BurntSushi/toml" + "github.com/pingcap/errors" + "github.com/pingcap/tiup/pkg/cluster/api" + "github.com/pingcap/tiup/pkg/cluster/spec" + tiupexec "github.com/pingcap/tiup/pkg/exec" + "github.com/pingcap/tiup/pkg/utils" +) + +func getFlashClusterPath(dir string) string { + return fmt.Sprintf("%s/flash_cluster_manager", dir) +} + +type scheduleConfig struct { + LowSpaceRatio float64 `json:"low-space-ratio"` +} + +type replicateMaxReplicaConfig struct { + MaxReplicas int `json:"max-replicas"` +} + +type replicateEnablePlacementRulesConfig struct { + EnablePlacementRules string `json:"enable-placement-rules"` +} + +// startOld is for < 7.1.0. Not maintained any more. Do not introduce new features. +func (inst *TiFlashInstance) startOld(ctx context.Context, version utils.Version) error { + endpoints := pdEndpoints(inst.pds, false) + + tidbStatusAddrs := make([]string, 0, len(inst.dbs)) + for _, db := range inst.dbs { + tidbStatusAddrs = append(tidbStatusAddrs, utils.JoinHostPort(AdvertiseHost(db.Host), db.StatusPort)) + } + wd, err := filepath.Abs(inst.Dir) + if err != nil { + return err + } + + // Wait for PD + pdClient := api.NewPDClient(ctx, endpoints, 10*time.Second, nil) + // set low-space-ratio to 1 to avoid low disk space + lowSpaceRatio, err := json.Marshal(scheduleConfig{ + LowSpaceRatio: 0.99, + }) + if err != nil { + return err + } + if err = pdClient.UpdateScheduleConfig(bytes.NewBuffer(lowSpaceRatio)); err != nil { + return err + } + // Update maxReplicas before placement rules so that it would not be overwritten + maxReplicas, err := json.Marshal(replicateMaxReplicaConfig{ + MaxReplicas: 1, + }) + if err != nil { + return err + } + if err = pdClient.UpdateReplicateConfig(bytes.NewBuffer(maxReplicas)); err != nil { + return err + } + // Set enable-placement-rules to allow TiFlash work properly + enablePlacementRules, err := json.Marshal(replicateEnablePlacementRulesConfig{ + EnablePlacementRules: "true", + }) + if err != nil { + return err + } + if err = pdClient.UpdateReplicateConfig(bytes.NewBuffer(enablePlacementRules)); err != nil { + return err + } + + if inst.BinPath, err = tiupexec.PrepareBinary("tiflash", version, inst.BinPath); err != nil { + return err + } + + dirPath := filepath.Dir(inst.BinPath) + clusterManagerPath := getFlashClusterPath(dirPath) + if err = inst.checkConfigOld(wd, clusterManagerPath, version, tidbStatusAddrs, endpoints); err != nil { + return err + } + + args := []string{ + "server", + fmt.Sprintf("--config-file=%s", inst.ConfigPath), + } + envs := []string{ + fmt.Sprintf("LD_LIBRARY_PATH=%s:$LD_LIBRARY_PATH", dirPath), + } + inst.Process = &process{cmd: PrepareCommand(ctx, inst.BinPath, args, envs, inst.Dir)} + + logIfErr(inst.Process.SetOutputFile(inst.LogFile())) + return inst.Process.Start() +} + +// checkConfigOld is for < 7.1.0. Not maintained any more. Do not introduce new features. +func (inst *TiFlashInstance) checkConfigOld(deployDir, clusterManagerPath string, + version utils.Version, tidbStatusAddrs, endpoints []string) (err error) { + if err := utils.MkdirAll(inst.Dir, 0755); err != nil { + return errors.Trace(err) + } + + var ( + flashBuf = new(bytes.Buffer) + proxyBuf = new(bytes.Buffer) + + flashCfgPath = path.Join(inst.Dir, "tiflash.toml") + proxyCfgPath = path.Join(inst.Dir, "tiflash-learner.toml") + ) + + defer func() { + if err != nil { + return + } + if err = utils.WriteFile(flashCfgPath, flashBuf.Bytes(), 0644); err != nil { + return + } + + if err = utils.WriteFile(proxyCfgPath, proxyBuf.Bytes(), 0644); err != nil { + return + } + + inst.ConfigPath = flashCfgPath + }() + + // Write default config to buffer + if err := writeTiFlashConfigOld(flashBuf, version, inst.TCPPort, inst.Port, inst.ServicePort, inst.StatusPort, + inst.Host, deployDir, clusterManagerPath, tidbStatusAddrs, endpoints); err != nil { + return errors.Trace(err) + } + if err := writeTiFlashProxyConfigOld(proxyBuf, version, inst.Host, deployDir, + inst.ServicePort, inst.ProxyPort, inst.ProxyStatusPort); err != nil { + return errors.Trace(err) + } + + if inst.ConfigPath == "" { + return + } + + cfg, err := unmarshalConfig(inst.ConfigPath) + if err != nil { + return errors.Trace(err) + } + proxyPath := getTiFlashProxyConfigPathOld(cfg) + if proxyPath != "" { + proxyCfg, err := unmarshalConfig(proxyPath) + if err != nil { + return errors.Trace(err) + } + err = overwriteBuf(proxyBuf, proxyCfg) + if err != nil { + return errors.Trace(err) + } + } + + // Always use the tiflash proxy config file in the instance directory + setTiFlashProxyConfigPathOld(cfg, proxyCfgPath) + return errors.Trace(overwriteBuf(flashBuf, cfg)) +} + +func overwriteBuf(buf *bytes.Buffer, overwrite map[string]any) (err error) { + cfg := make(map[string]any) + if err = toml.Unmarshal(buf.Bytes(), &cfg); err != nil { + return + } + buf.Reset() + return toml.NewEncoder(buf).Encode(spec.MergeConfig(cfg, overwrite)) +} diff --git a/components/playground/instance/tiflash_pre7_config.go b/components/playground/instance/tiflash_pre7_config.go new file mode 100644 index 0000000000..3481540c18 --- /dev/null +++ b/components/playground/instance/tiflash_pre7_config.go @@ -0,0 +1,135 @@ +// Copyright 2020 PingCAP, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// See the License for the specific language governing permissions and +// limitations under the License. + +package instance + +import ( + "fmt" + "io" + "strings" + + "github.com/pingcap/tiup/pkg/tidbver" + "github.com/pingcap/tiup/pkg/utils" +) + +const tiflashDaemonConfigOld = ` + +[application] +runAsDaemon = true + +` + +const tiflashMarkCacheSizeOld = `mark_cache_size = 5368709120` + +const tiflashConfigOld = ` +default_profile = "default" +display_name = "TiFlash" +http_port = %[2]d +listen_host = "0.0.0.0" +tcp_port = %[3]d +path = "%[5]s" +tmp_path = "%[6]s" +%[14]s +%[13]s +[flash] +service_addr = "%[10]s:%[8]d" +tidb_status_addr = "%[11]s" +[flash.flash_cluster] +cluster_manager_path = "%[12]s" +log = "%[7]s/tiflash_cluster_manager.log" +master_ttl = 60 +refresh_interval = 20 +update_rule_interval = 5 +[flash.proxy] +config = "%[4]s/tiflash-learner.toml" + +[logger] +count = 20 +errorlog = "%[7]s/tiflash_error.log" +level = "debug" +log = "%[7]s/tiflash.log" +size = "1000M" + +[profiles] +[profiles.default] +load_balancing = "random" +max_memory_usage = 0 +use_uncompressed_cache = 0 +[profiles.readonly] +readonly = 1 + +[quotas] +[quotas.default] +[quotas.default.interval] +duration = 3600 +errors = 0 +execution_time = 0 +queries = 0 +read_rows = 0 +result_rows = 0 + +[raft] +pd_addr = "%[1]s" + +[status] +metrics_port = %[9]d + +[users] +[users.default] +password = "" +profile = "default" +quota = "default" +[users.default.networks] +ip = "::/0" +[users.readonly] +password = "" +profile = "readonly" +quota = "default" +[users.readonly.networks] +ip = "::/0" +` + +// writeTiFlashConfigOld is for < 7.1.0. Not maintained any more. Do not introduce new features. +func writeTiFlashConfigOld(w io.Writer, version utils.Version, tcpPort, httpPort, servicePort, metricsPort int, host, deployDir, clusterManagerPath string, tidbStatusAddrs, endpoints []string) error { + pdAddrs := strings.Join(endpoints, ",") + dataDir := fmt.Sprintf("%s/data", deployDir) + tmpDir := fmt.Sprintf("%s/tmp", deployDir) + logDir := fmt.Sprintf("%s/log", deployDir) + ip := AdvertiseHost(host) + var conf string + + if tidbver.TiFlashNotNeedSomeConfig(version.String()) { + conf = fmt.Sprintf(tiflashConfigOld, pdAddrs, httpPort, tcpPort, + deployDir, dataDir, tmpDir, logDir, servicePort, metricsPort, + ip, strings.Join(tidbStatusAddrs, ","), clusterManagerPath, "", "") + } else { + conf = fmt.Sprintf(tiflashConfigOld, pdAddrs, httpPort, tcpPort, + deployDir, dataDir, tmpDir, logDir, servicePort, metricsPort, + ip, strings.Join(tidbStatusAddrs, ","), clusterManagerPath, tiflashDaemonConfigOld, tiflashMarkCacheSizeOld) + } + _, err := w.Write([]byte(conf)) + return err +} + +func getTiFlashProxyConfigPathOld(cfg map[string]any) string { + defer func() { + if r := recover(); r != nil { + return + } + }() + return cfg["flash"].(map[string]any)["proxy"].(map[string]any)["config"].(string) +} + +func setTiFlashProxyConfigPathOld(cfg map[string]any, path string) { + cfg["flash"].(map[string]any)["proxy"].(map[string]any)["config"] = path +} diff --git a/components/playground/instance/tiflash_proxy_config.go b/components/playground/instance/tiflash_pre7_proxy_config.go similarity index 78% rename from components/playground/instance/tiflash_proxy_config.go rename to components/playground/instance/tiflash_pre7_proxy_config.go index 868974711e..b6b8fd815d 100644 --- a/components/playground/instance/tiflash_proxy_config.go +++ b/components/playground/instance/tiflash_pre7_proxy_config.go @@ -21,7 +21,7 @@ import ( "github.com/pingcap/tiup/pkg/utils" ) -const tiflashProxyConfig = ` +const tiflashProxyConfigOld = ` log-file = "%[1]s/tiflash_tikv.log" [rocksdb] @@ -46,7 +46,8 @@ data-dir = "%[6]s" max-open-files = 256 ` -func writeTiFlashProxyConfig(w io.Writer, version utils.Version, host, deployDir string, servicePort, proxyPort, proxyStatusPort int) error { +// writeTiFlashProxyConfigOld is for < 7.1.0. Not maintained any more. Do not introduce new features. +func writeTiFlashProxyConfigOld(w io.Writer, version utils.Version, host, deployDir string, servicePort, proxyPort, proxyStatusPort int) error { // TODO: support multi-dir dataDir := fmt.Sprintf("%s/flash", deployDir) logDir := fmt.Sprintf("%s/log", deployDir) @@ -58,7 +59,7 @@ advertise-status-addr = "%[1]s:%[2]d"`, ip, proxyStatusPort) } else { statusAddr = fmt.Sprintf(`status-addr = "%[1]s:%[2]d"`, ip, proxyStatusPort) } - conf := fmt.Sprintf(tiflashProxyConfig, logDir, ip, servicePort, proxyPort, statusAddr, dataDir) + conf := fmt.Sprintf(tiflashProxyConfigOld, logDir, ip, servicePort, proxyPort, statusAddr, dataDir) _, err := w.Write([]byte(conf)) return err } diff --git a/components/playground/instance/tikv.go b/components/playground/instance/tikv.go index 743a90ba6e..e767ec61a2 100644 --- a/components/playground/instance/tikv.go +++ b/components/playground/instance/tikv.go @@ -16,12 +16,9 @@ package instance import ( "context" "fmt" - "os" - "path" "path/filepath" "strings" - "github.com/pingcap/errors" tiupexec "github.com/pingcap/tiup/pkg/exec" "github.com/pingcap/tiup/pkg/utils" ) @@ -34,14 +31,17 @@ type TiKVInstance struct { } // NewTiKVInstance return a TiKVInstance -func NewTiKVInstance(binPath string, dir, host, configPath string, id int, pds []*PDInstance) *TiKVInstance { +func NewTiKVInstance(binPath string, dir, host, configPath string, id int, port int, pds []*PDInstance) *TiKVInstance { + if port <= 0 { + port = 20160 + } return &TiKVInstance{ instance: instance{ BinPath: binPath, ID: id, Dir: dir, Host: host, - Port: utils.MustGetFreePort(host, 20160), + Port: utils.MustGetFreePort(host, port), StatusPort: utils.MustGetFreePort(host, 20180), ConfigPath: configPath, }, @@ -56,7 +56,12 @@ func (inst *TiKVInstance) Addr() string { // Start calls set inst.cmd and Start func (inst *TiKVInstance) Start(ctx context.Context, version utils.Version) error { - if err := inst.checkConfig(); err != nil { + configPath := filepath.Join(inst.Dir, "tikv.toml") + if err := prepareConfig( + configPath, + inst.ConfigPath, + inst.getConfig(), + ); err != nil { return err } @@ -65,8 +70,8 @@ func (inst *TiKVInstance) Start(ctx context.Context, version utils.Version) erro fmt.Sprintf("--addr=%s", utils.JoinHostPort(inst.Host, inst.Port)), fmt.Sprintf("--advertise-addr=%s", utils.JoinHostPort(AdvertiseHost(inst.Host), inst.Port)), fmt.Sprintf("--status-addr=%s", utils.JoinHostPort(inst.Host, inst.StatusPort)), - fmt.Sprintf("--pd=%s", strings.Join(endpoints, ",")), - fmt.Sprintf("--config=%s", inst.ConfigPath), + fmt.Sprintf("--pd-endpoints=%s", strings.Join(endpoints, ",")), + fmt.Sprintf("--config=%s", configPath), fmt.Sprintf("--data-dir=%s", filepath.Join(inst.Dir, "data")), fmt.Sprintf("--log-file=%s", inst.LogFile()), } @@ -96,32 +101,3 @@ func (inst *TiKVInstance) LogFile() string { func (inst *TiKVInstance) StoreAddr() string { return utils.JoinHostPort(AdvertiseHost(inst.Host), inst.Port) } - -func (inst *TiKVInstance) checkConfig() error { - if err := os.MkdirAll(inst.Dir, 0755); err != nil { - return err - } - if inst.ConfigPath == "" { - inst.ConfigPath = path.Join(inst.Dir, "tikv.toml") - } - - _, err := os.Stat(inst.ConfigPath) - if err == nil || os.IsExist(err) { - return nil - } - if !os.IsNotExist(err) { - return errors.Trace(err) - } - - cf, err := os.Create(inst.ConfigPath) - if err != nil { - return errors.Trace(err) - } - - defer cf.Close() - if err := writeTiKVConfig(cf); err != nil { - return errors.Trace(err) - } - - return nil -} diff --git a/components/playground/instance/tikv_config.go b/components/playground/instance/tikv_config.go index 8ba287e527..197a9492c7 100644 --- a/components/playground/instance/tikv_config.go +++ b/components/playground/instance/tikv_config.go @@ -13,22 +13,11 @@ package instance -import ( - "io" -) - -const tikvConfig = ` -[rocksdb] -max-open-files = 256 - -[raftdb] -max-open-files = 256 - -[storage] -reserve-space = 0 -` - -func writeTiKVConfig(w io.Writer) error { - _, err := w.Write([]byte(tikvConfig)) - return err +func (inst *TiKVInstance) getConfig() map[string]any { + config := make(map[string]any) + config["rocksdb.max-open-files"] = 256 + config["raftdb.max-open-files"] = 256 + config["storage.reserve-space"] = 0 + config["storage.reserve-raft-space"] = 0 + return config } diff --git a/components/playground/instance/tiproxy.go b/components/playground/instance/tiproxy.go new file mode 100644 index 0000000000..95b6b13cd8 --- /dev/null +++ b/components/playground/instance/tiproxy.go @@ -0,0 +1,129 @@ +// Copyright 2020 PingCAP, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// See the License for the specific language governing permissions and +// limitations under the License. + +package instance + +import ( + "context" + "fmt" + "os" + "path/filepath" + "strings" + + "github.com/pelletier/go-toml" + tiupexec "github.com/pingcap/tiup/pkg/exec" + "github.com/pingcap/tiup/pkg/utils" +) + +// TiProxy represent a ticdc instance. +type TiProxy struct { + instance + pds []*PDInstance + Process +} + +var _ Instance = &TiProxy{} + +// NewTiProxy create a TiProxy instance. +func NewTiProxy(binPath string, dir, host, configPath string, id int, port int, pds []*PDInstance) *TiProxy { + if port <= 0 { + port = 6000 + } + tiproxy := &TiProxy{ + instance: instance{ + BinPath: binPath, + ID: id, + Dir: dir, + Host: host, + Port: utils.MustGetFreePort(host, port), + StatusPort: utils.MustGetFreePort(host, 3080), + ConfigPath: configPath, + }, + pds: pds, + } + return tiproxy +} + +// MetricAddr implements Instance interface. +func (c *TiProxy) MetricAddr() (r MetricAddr) { + r.Targets = append(r.Targets, utils.JoinHostPort(c.Host, c.StatusPort)) + r.Labels = map[string]string{ + "__metrics_path__": "/api/metrics", + } + return +} + +// Start implements Instance interface. +func (c *TiProxy) Start(ctx context.Context, version utils.Version) error { + endpoints := pdEndpoints(c.pds, true) + + configPath := filepath.Join(c.Dir, "config", "proxy.toml") + if c.ConfigPath != "" { + configPath = c.ConfigPath + } + + configContent := "" + if b, err := os.ReadFile(configPath); err == nil { + configContent = string(b) + } + + config, err := toml.Load(configContent) + if err != nil { + return err + } + + config.Set("proxy.pd-addrs", strings.Join(endpoints, ",")) + config.Set("proxy.addr", utils.JoinHostPort(c.Host, c.Port)) + config.Set("proxy.require-backend-tls", false) + config.Set("api.addr", utils.JoinHostPort(c.Host, c.StatusPort)) + + b, err := config.Marshal() + if err != nil { + return err + } + + if err := os.MkdirAll(filepath.Dir(configPath), 0755); err != nil { + return err + } + if err := os.WriteFile(configPath, b, 0644); err != nil { + return err + } + + args := []string{ + fmt.Sprintf("--config=%s", configPath), + } + + if c.BinPath, err = tiupexec.PrepareBinary("tiproxy", version, c.BinPath); err != nil { + return err + } + + c.Process = &process{cmd: PrepareCommand(ctx, c.BinPath, args, nil, c.Dir)} + + logIfErr(c.Process.SetOutputFile(c.LogFile())) + return c.Process.Start() +} + +// Addr return addresses that can be connected by MySQL clients. +func (c *TiProxy) Addr() string { + return utils.JoinHostPort(AdvertiseHost(c.Host), c.Port) +} + +// Component return component name. +func (c *TiProxy) Component() string { + return "tiproxy" +} + +// LogFile return the log file. +func (c *TiProxy) LogFile() string { + return filepath.Join(c.Dir, "tiproxy.log") +} diff --git a/components/playground/main.go b/components/playground/main.go index 6a8dec4897..fb94016bab 100644 --- a/components/playground/main.go +++ b/components/playground/main.go @@ -15,9 +15,9 @@ package main import ( "context" - "database/sql" "encoding/json" "fmt" + "net" "net/http" _ "net/http/pprof" "os" @@ -54,17 +54,27 @@ import ( // BootOptions is the topology and options used to start a playground cluster type BootOptions struct { - Version string `yaml:"version"` - PD instance.Config `yaml:"pd"` - TiDB instance.Config `yaml:"tidb"` - TiKV instance.Config `yaml:"tikv"` - TiFlash instance.Config `yaml:"tiflash"` - TiCDC instance.Config `yaml:"ticdc"` - TiKVCDC instance.Config `yaml:"tikv_cdc"` - Pump instance.Config `yaml:"pump"` - Drainer instance.Config `yaml:"drainer"` - Host string `yaml:"host"` - Monitor bool `yaml:"monitor"` + Mode string `yaml:"mode"` + PDMode string `yaml:"pd_mode"` + Version string `yaml:"version"` + PD instance.Config `yaml:"pd"` // ignored when pd_mode == ms + PDAPI instance.Config `yaml:"pd_api"` // Only available when pd_mode == ms + PDTSO instance.Config `yaml:"pd_tso"` // Only available when pd_mode == ms + PDScheduling instance.Config `yaml:"pd_scheduling"` // Only available when pd_mode == ms + PDRM instance.Config `yaml:"pd_rm"` // Only available when pd_mode == ms + TiProxy instance.Config `yaml:"tiproxy"` + TiDB instance.Config `yaml:"tidb"` + TiKV instance.Config `yaml:"tikv"` + TiFlash instance.Config `yaml:"tiflash"` // ignored when mode == tidb-disagg + TiFlashWrite instance.Config `yaml:"tiflash_write"` // Only available when mode == tidb-disagg + TiFlashCompute instance.Config `yaml:"tiflash_compute"` // Only available when mode == tidb-disagg + TiCDC instance.Config `yaml:"ticdc"` + TiKVCDC instance.Config `yaml:"tikv_cdc"` + Pump instance.Config `yaml:"pump"` + Drainer instance.Config `yaml:"drainer"` + Host string `yaml:"host"` + Monitor bool `yaml:"monitor"` + DisaggOpts instance.DisaggOptions `yaml:"disagg"` // Only available when mode == tidb-disagg } var ( @@ -79,56 +89,6 @@ var ( log = logprinter.NewLogger("") ) -const ( - mode = "mode" - withMonitor = "monitor" - withoutMonitor = "without-monitor" - - // instance numbers - db = "db" - kv = "kv" - pd = "pd" - tiflash = "tiflash" - ticdc = "ticdc" - kvcdc = "kvcdc" - pump = "pump" - drainer = "drainer" - - // up timeouts - dbTimeout = "db.timeout" - tiflashTimeout = "tiflash.timeout" - - // hosts - clusterHost = "host" - dbHost = "db.host" - dbPort = "db.port" - pdHost = "pd.host" - pdPort = "pd.port" - - // config paths - dbConfig = "db.config" - kvConfig = "kv.config" - pdConfig = "pd.config" - tiflashConfig = "tiflash.config" - ticdcConfig = "ticdc.config" - kvcdcConfig = "kvcdc.config" - pumpConfig = "pump.config" - drainerConfig = "drainer.config" - - // binary path - dbBinpath = "db.binpath" - kvBinpath = "kv.binpath" - pdBinpath = "pd.binpath" - tiflashBinpath = "tiflash.binpath" - ticdcBinpath = "ticdc.binpath" - kvcdcBinpath = "kvcdc.binpath" - pumpBinpath = "pump.binpath" - drainerBinpath = "drainer.binpath" - - // component version - kvcdcVersion = "kvcdc.version" -) - func installIfMissing(component, version string) error { env := environment.GlobalEnv() @@ -184,7 +144,7 @@ Examples: dataDir = filepath.Join(tiupHome, localdata.DataParentDir, tag) deleteWhenExit = true } - err := os.MkdirAll(dataDir, os.ModePerm) + err := utils.MkdirAll(dataDir, os.ModePerm) if err != nil { return err } @@ -211,7 +171,7 @@ Examples: options.Version = args[0] } - if err := populateOpt(cmd.Flags()); err != nil { + if err := populateDefaultOpt(cmd.Flags()); err != nil { return err } @@ -282,7 +242,7 @@ Examples: version = "nightly" } fmt.Println(color.YellowString(`Using the version %s for version constraint "%s". - + If you'd like to use a TiDB version other than %s, cancel and retry with the following arguments: Specify version manually: tiup playground Specify version range: tiup playground ^5 @@ -312,52 +272,89 @@ If you'd like to use a TiDB version other than %s, cancel and retry with the fol }, } - defaultMode := "tidb" - defaultOptions := &BootOptions{} - - rootCmd.Flags().String(mode, defaultMode, "TiUP playground mode: 'tidb', 'tikv-slim'") + rootCmd.Flags().StringVar(&options.Mode, "mode", "tidb", "TiUP playground mode: 'tidb', 'tidb-disagg', 'tikv-slim'") + rootCmd.Flags().StringVar(&options.PDMode, "pd.mode", "pd", "PD mode: 'pd', 'ms'") rootCmd.PersistentFlags().StringVarP(&tag, "tag", "T", "", "Specify a tag for playground") // Use `PersistentFlags()` to make it available to subcommands. - rootCmd.Flags().Bool(withoutMonitor, false, "Don't start prometheus and grafana component") - rootCmd.Flags().Bool(withMonitor, true, "Start prometheus and grafana component") - _ = rootCmd.Flags().MarkDeprecated(withMonitor, "Please use --without-monitor to control whether to disable monitor.") - - rootCmd.Flags().Int(db, defaultOptions.TiDB.Num, "TiDB instance number") - rootCmd.Flags().Int(kv, defaultOptions.TiKV.Num, "TiKV instance number") - rootCmd.Flags().Int(pd, defaultOptions.PD.Num, "PD instance number") - rootCmd.Flags().Int(tiflash, defaultOptions.TiFlash.Num, "TiFlash instance number") - rootCmd.Flags().Int(ticdc, defaultOptions.TiCDC.Num, "TiCDC instance number") - rootCmd.Flags().Int(kvcdc, defaultOptions.TiKVCDC.Num, "TiKV-CDC instance number") - rootCmd.Flags().Int(pump, defaultOptions.Pump.Num, "Pump instance number") - rootCmd.Flags().Int(drainer, defaultOptions.Drainer.Num, "Drainer instance number") - - rootCmd.Flags().Int(dbTimeout, defaultOptions.TiDB.UpTimeout, "TiDB max wait time in seconds for starting, 0 means no limit") - rootCmd.Flags().Int(tiflashTimeout, defaultOptions.TiFlash.UpTimeout, "TiFlash max wait time in seconds for starting, 0 means no limit") - - rootCmd.Flags().String(clusterHost, defaultOptions.Host, "Playground cluster host") - rootCmd.Flags().String(dbHost, defaultOptions.TiDB.Host, "Playground TiDB host. If not provided, TiDB will still use `host` flag as its host") - rootCmd.Flags().Int(dbPort, defaultOptions.TiDB.Port, "Playground TiDB port. If not provided, TiDB will use 4000 as its port") - rootCmd.Flags().String(pdHost, defaultOptions.PD.Host, "Playground PD host. If not provided, PD will still use `host` flag as its host") - rootCmd.Flags().Int(pdPort, defaultOptions.PD.Port, "Playground PD port. If not provided, PD will use 2379 as its port") - - rootCmd.Flags().String(dbConfig, defaultOptions.TiDB.ConfigPath, "TiDB instance configuration file") - rootCmd.Flags().String(kvConfig, defaultOptions.TiKV.ConfigPath, "TiKV instance configuration file") - rootCmd.Flags().String(pdConfig, defaultOptions.PD.ConfigPath, "PD instance configuration file") - rootCmd.Flags().String(tiflashConfig, defaultOptions.TiDB.ConfigPath, "TiFlash instance configuration file") - rootCmd.Flags().String(pumpConfig, defaultOptions.Pump.ConfigPath, "Pump instance configuration file") - rootCmd.Flags().String(drainerConfig, defaultOptions.Drainer.ConfigPath, "Drainer instance configuration file") - rootCmd.Flags().String(ticdcConfig, defaultOptions.TiCDC.ConfigPath, "TiCDC instance configuration file") - rootCmd.Flags().String(kvcdcConfig, defaultOptions.TiKVCDC.ConfigPath, "TiKV-CDC instance configuration file") - - rootCmd.Flags().String(dbBinpath, defaultOptions.TiDB.BinPath, "TiDB instance binary path") - rootCmd.Flags().String(kvBinpath, defaultOptions.TiKV.BinPath, "TiKV instance binary path") - rootCmd.Flags().String(pdBinpath, defaultOptions.PD.BinPath, "PD instance binary path") - rootCmd.Flags().String(tiflashBinpath, defaultOptions.TiFlash.BinPath, "TiFlash instance binary path") - rootCmd.Flags().String(ticdcBinpath, defaultOptions.TiCDC.BinPath, "TiCDC instance binary path") - rootCmd.Flags().String(kvcdcBinpath, defaultOptions.TiKVCDC.BinPath, "TiKV-CDC instance binary path") - rootCmd.Flags().String(pumpBinpath, defaultOptions.Pump.BinPath, "Pump instance binary path") - rootCmd.Flags().String(drainerBinpath, defaultOptions.Drainer.BinPath, "Drainer instance binary path") - - rootCmd.Flags().String(kvcdcVersion, defaultOptions.TiKVCDC.Version, "TiKV-CDC instance version") + rootCmd.Flags().Bool("without-monitor", false, "Don't start prometheus and grafana component") + rootCmd.Flags().BoolVar(&options.Monitor, "monitor", true, "Start prometheus and grafana component") + _ = rootCmd.Flags().MarkDeprecated("monitor", "Please use --without-monitor to control whether to disable monitor.") + + // NOTE: Do not set default values if they may be changed in different modes. + + rootCmd.Flags().IntVar(&options.TiDB.Num, "db", 0, "TiDB instance number") + rootCmd.Flags().IntVar(&options.TiKV.Num, "kv", 0, "TiKV instance number") + rootCmd.Flags().IntVar(&options.PD.Num, "pd", 0, "PD instance number") + rootCmd.Flags().IntVar(&options.TiProxy.Num, "tiproxy", 0, "TiProxy instance number") + rootCmd.Flags().IntVar(&options.TiFlash.Num, "tiflash", 0, "TiFlash instance number, when --mode=tidb-disagg this will set instance number for both Write Node and Compute Node") + rootCmd.Flags().IntVar(&options.TiFlashWrite.Num, "tiflash.write", 0, "TiFlash Write instance number, available when --mode=tidb-disagg, take precedence over --tiflash") + rootCmd.Flags().IntVar(&options.TiFlashCompute.Num, "tiflash.compute", 0, "TiFlash Compute instance number, available when --mode=tidb-disagg, take precedence over --tiflash") + rootCmd.Flags().IntVar(&options.TiCDC.Num, "ticdc", 0, "TiCDC instance number") + rootCmd.Flags().IntVar(&options.TiKVCDC.Num, "kvcdc", 0, "TiKV-CDC instance number") + rootCmd.Flags().IntVar(&options.Pump.Num, "pump", 0, "Pump instance number") + rootCmd.Flags().IntVar(&options.Drainer.Num, "drainer", 0, "Drainer instance number") + + rootCmd.Flags().IntVar(&options.PDAPI.Num, "pd.api", 0, "PD API instance number") + rootCmd.Flags().IntVar(&options.PDTSO.Num, "pd.tso", 0, "PD TSO instance number") + rootCmd.Flags().IntVar(&options.PDScheduling.Num, "pd.scheduling", 0, "PD scheduling instance number") + rootCmd.Flags().IntVar(&options.PDRM.Num, "pd.rm", 0, "PD resource manager instance number") + + rootCmd.Flags().IntVar(&options.TiDB.UpTimeout, "db.timeout", 60, "TiDB max wait time in seconds for starting, 0 means no limit") + rootCmd.Flags().IntVar(&options.TiFlash.UpTimeout, "tiflash.timeout", 120, "TiFlash max wait time in seconds for starting, 0 means no limit") + rootCmd.Flags().IntVar(&options.TiProxy.UpTimeout, "tiproxy.timeout", 60, "TiProxy max wait time in seconds for starting, 0 means no limit") + + rootCmd.Flags().StringVar(&options.Host, "host", "127.0.0.1", "Playground cluster host") + rootCmd.Flags().StringVar(&options.TiDB.Host, "db.host", "", "Playground TiDB host. If not provided, TiDB will still use `host` flag as its host") + rootCmd.Flags().IntVar(&options.TiDB.Port, "db.port", 0, "Playground TiDB port. If not provided, TiDB will use 4000 as its port. Or 6000 if TiProxy is enabled.") + rootCmd.Flags().StringVar(&options.PD.Host, "pd.host", "", "Playground PD host. If not provided, PD will still use `host` flag as its host") + rootCmd.Flags().IntVar(&options.PD.Port, "pd.port", 0, "Playground PD port. If not provided, PD will use 2379 as its port") + rootCmd.Flags().StringVar(&options.TiKV.Host, "kv.host", "", "Playground TiKV host. If not provided, TiKV will still use `host` flag as its host") + rootCmd.Flags().IntVar(&options.TiKV.Port, "kv.port", 0, "Playground TiKV port. If not provided, TiKV will use 20160 as its port") + rootCmd.Flags().StringVar(&options.TiCDC.Host, "ticdc.host", "", "Playground TiCDC host. If not provided, TiDB will still use `host` flag as its host") + rootCmd.Flags().IntVar(&options.TiCDC.Port, "ticdc.port", 0, "Playground TiCDC port. If not provided, TiCDC will use 8300 as its port") + rootCmd.Flags().StringVar(&options.TiProxy.Host, "tiproxy.host", "", "Playground TiProxy host. If not provided, TiProxy will still use `host` flag as its host") + rootCmd.Flags().IntVar(&options.TiProxy.Port, "tiproxy.port", 0, "Playground TiProxy port. If not provided, TiProxy will use 6000 as its port") + + rootCmd.Flags().StringVar(&options.TiDB.ConfigPath, "db.config", "", "TiDB instance configuration file") + rootCmd.Flags().StringVar(&options.TiKV.ConfigPath, "kv.config", "", "TiKV instance configuration file") + rootCmd.Flags().StringVar(&options.PD.ConfigPath, "pd.config", "", "PD instance configuration file") + rootCmd.Flags().StringVar(&options.TiProxy.ConfigPath, "tiproxy.config", "", "TiProxy instance configuration file") + rootCmd.Flags().StringVar(&options.TiFlash.ConfigPath, "tiflash.config", "", "TiFlash instance configuration file, when --mode=tidb-disagg this will set config file for both Write Node and Compute Node") + rootCmd.Flags().StringVar(&options.TiFlashWrite.ConfigPath, "tiflash.write.config", "", "TiFlash Write instance configuration file, available when --mode=tidb-disagg, take precedence over --tiflash.config") + rootCmd.Flags().StringVar(&options.TiFlashCompute.ConfigPath, "tiflash.compute.config", "", "TiFlash Compute instance configuration file, available when --mode=tidb-disagg, take precedence over --tiflash.config") + rootCmd.Flags().StringVar(&options.Pump.ConfigPath, "pump.config", "", "Pump instance configuration file") + rootCmd.Flags().StringVar(&options.Drainer.ConfigPath, "drainer.config", "", "Drainer instance configuration file") + rootCmd.Flags().StringVar(&options.TiCDC.ConfigPath, "ticdc.config", "", "TiCDC instance configuration file") + rootCmd.Flags().StringVar(&options.TiKVCDC.ConfigPath, "kvcdc.config", "", "TiKV-CDC instance configuration file") + + rootCmd.Flags().StringVar(&options.PDAPI.ConfigPath, "pd.api.config", "", "PD API instance configuration file") + rootCmd.Flags().StringVar(&options.PDTSO.ConfigPath, "pd.tso.config", "", "PD TSO instance configuration file") + rootCmd.Flags().StringVar(&options.PDScheduling.ConfigPath, "pd.scheduling.config", "", "PD scheduling instance configuration file") + rootCmd.Flags().StringVar(&options.PDRM.ConfigPath, "pd.rm.config", "", "PD resource manager instance configuration file") + + rootCmd.Flags().StringVar(&options.TiDB.BinPath, "db.binpath", "", "TiDB instance binary path") + rootCmd.Flags().StringVar(&options.TiKV.BinPath, "kv.binpath", "", "TiKV instance binary path") + rootCmd.Flags().StringVar(&options.PD.BinPath, "pd.binpath", "", "PD instance binary path") + rootCmd.Flags().StringVar(&options.TiProxy.BinPath, "tiproxy.binpath", "", "TiProxy instance binary path") + rootCmd.Flags().StringVar(&options.TiProxy.Version, "tiproxy.version", "", "TiProxy instance version") + rootCmd.Flags().StringVar(&options.TiFlash.BinPath, "tiflash.binpath", "", "TiFlash instance binary path, when --mode=tidb-disagg this will set binary path for both Write Node and Compute Node") + rootCmd.Flags().StringVar(&options.TiFlashWrite.BinPath, "tiflash.write.binpath", "", "TiFlash Write instance binary path, available when --mode=tidb-disagg, take precedence over --tiflash.binpath") + rootCmd.Flags().StringVar(&options.TiFlashCompute.BinPath, "tiflash.compute.binpath", "", "TiFlash Compute instance binary path, available when --mode=tidb-disagg, take precedence over --tiflash.binpath") + rootCmd.Flags().StringVar(&options.TiCDC.BinPath, "ticdc.binpath", "", "TiCDC instance binary path") + rootCmd.Flags().StringVar(&options.TiKVCDC.BinPath, "kvcdc.binpath", "", "TiKV-CDC instance binary path") + rootCmd.Flags().StringVar(&options.Pump.BinPath, "pump.binpath", "", "Pump instance binary path") + rootCmd.Flags().StringVar(&options.Drainer.BinPath, "drainer.binpath", "", "Drainer instance binary path") + + rootCmd.Flags().StringVar(&options.PDAPI.BinPath, "pd.api.binpath", "", "PD API instance binary path") + rootCmd.Flags().StringVar(&options.PDTSO.BinPath, "pd.tso.binpath", "", "PD TSO instance binary path") + rootCmd.Flags().StringVar(&options.PDScheduling.BinPath, "pd.scheduling.binpath", "", "PD scheduling instance binary path") + rootCmd.Flags().StringVar(&options.PDRM.BinPath, "pd.rm.binpath", "", "PD resource manager instance binary path") + + rootCmd.Flags().StringVar(&options.TiKVCDC.Version, "kvcdc.version", "", "TiKV-CDC instance version") + + rootCmd.Flags().StringVar(&options.DisaggOpts.S3Endpoint, "disagg.s3_endpoint", "127.0.0.1:9000", "Object store URL for the disaggregated TiFlash, available when --mode=tidb-disagg") + rootCmd.Flags().StringVar(&options.DisaggOpts.Bucket, "disagg.bucket", "tiflash", "Object store bucket for the disaggregated TiFlash, available when --mode=tidb-disagg") + rootCmd.Flags().StringVar(&options.DisaggOpts.AccessKey, "disagg.access_key", "minioadmin", "Object store access key, available when --mode=tidb-disagg") + rootCmd.Flags().StringVar(&options.DisaggOpts.SecretKey, "disagg.secret_key", "minioadmin", "Object store secret key, available when --mode=tidb-disagg") rootCmd.AddCommand(newDisplay()) rootCmd.AddCommand(newScaleOut()) @@ -366,178 +363,84 @@ If you'd like to use a TiDB version other than %s, cancel and retry with the fol return rootCmd.Execute() } -func populateOpt(flagSet *pflag.FlagSet) (err error) { - var modeVal string - if modeVal, err = flagSet.GetString(mode); err != nil { - return +func populateDefaultOpt(flagSet *pflag.FlagSet) error { + if flagSet.Lookup("without-monitor").Changed { + v, _ := flagSet.GetBool("without-monitor") + options.Monitor = !v + } + + defaultInt := func(variable *int, flagName string, defaultValue int) { + if !flagSet.Lookup(flagName).Changed { + *variable = defaultValue + } } - switch modeVal { + defaultStr := func(variable *string, flagName string, defaultValue string) { + if !flagSet.Lookup(flagName).Changed { + *variable = defaultValue + } + } + + switch options.Mode { case "tidb": - options.TiDB.Num = 1 - options.TiDB.UpTimeout = 60 - options.TiKV.Num = 1 - options.PD.Num = 1 - options.TiFlash.Num = 1 - options.TiFlash.UpTimeout = 120 - options.Host = "127.0.0.1" - options.Monitor = true + defaultInt(&options.TiDB.Num, "db", 1) + defaultInt(&options.TiKV.Num, "kv", 1) + defaultInt(&options.TiFlash.Num, "tiflash", 1) case "tikv-slim": - options.TiKV.Num = 1 - options.PD.Num = 1 - options.Host = "127.0.0.1" - options.Monitor = true + defaultInt(&options.TiKV.Num, "kv", 1) + case "tidb-disagg": + defaultInt(&options.TiDB.Num, "db", 1) + defaultInt(&options.TiKV.Num, "kv", 1) + defaultInt(&options.TiFlash.Num, "tiflash", 1) + defaultInt(&options.TiFlashWrite.Num, "tiflash.write", options.TiFlash.Num) + defaultStr(&options.TiFlashWrite.BinPath, "tiflash.write.binpath", options.TiFlash.BinPath) + defaultStr(&options.TiFlashWrite.ConfigPath, "tiflash.write.config", options.TiFlash.ConfigPath) + options.TiFlashWrite.UpTimeout = options.TiFlash.UpTimeout + defaultInt(&options.TiFlashCompute.Num, "tiflash.compute", options.TiFlash.Num) + defaultStr(&options.TiFlashCompute.BinPath, "tiflash.compute.binpath", options.TiFlash.BinPath) + defaultStr(&options.TiFlashCompute.ConfigPath, "tiflash.compute.config", options.TiFlash.ConfigPath) + options.TiFlashCompute.UpTimeout = options.TiFlash.UpTimeout default: - err = errors.Errorf("unknown playground mode: %s", modeVal) - return + return errors.Errorf("Unknown --mode %s", options.Mode) } - flagSet.Visit(func(flag *pflag.Flag) { - switch flag.Name { - case withMonitor: - options.Monitor, err = strconv.ParseBool(flag.Value.String()) - if err != nil { - return - } - case withoutMonitor: - options.Monitor, err = strconv.ParseBool(flag.Value.String()) - if err != nil { - return - } - options.Monitor = !options.Monitor - case db: - options.TiDB.Num, err = strconv.Atoi(flag.Value.String()) - if err != nil { - return - } - case kv: - options.TiKV.Num, err = strconv.Atoi(flag.Value.String()) - if err != nil { - return - } - case pd: - options.PD.Num, err = strconv.Atoi(flag.Value.String()) - if err != nil { - return - } - case tiflash: - options.TiFlash.Num, err = strconv.Atoi(flag.Value.String()) - if err != nil { - return - } - case ticdc: - options.TiCDC.Num, err = strconv.Atoi(flag.Value.String()) - if err != nil { - return - } - case kvcdc: - options.TiKVCDC.Num, err = strconv.Atoi(flag.Value.String()) - if err != nil { - return - } - case pump: - options.Pump.Num, err = strconv.Atoi(flag.Value.String()) - if err != nil { - return - } - case drainer: - options.Drainer.Num, err = strconv.Atoi(flag.Value.String()) - if err != nil { - return - } - - case dbConfig: - options.TiDB.ConfigPath = flag.Value.String() - case kvConfig: - options.TiKV.ConfigPath = flag.Value.String() - case pdConfig: - options.PD.ConfigPath = flag.Value.String() - case tiflashConfig: - options.TiFlash.ConfigPath = flag.Value.String() - case ticdcConfig: - options.TiCDC.ConfigPath = flag.Value.String() - case kvcdcConfig: - options.TiKVCDC.ConfigPath = flag.Value.String() - case pumpConfig: - options.Pump.ConfigPath = flag.Value.String() - case drainerConfig: - options.Drainer.ConfigPath = flag.Value.String() - - case dbBinpath: - options.TiDB.BinPath = flag.Value.String() - case kvBinpath: - options.TiKV.BinPath = flag.Value.String() - case pdBinpath: - options.PD.BinPath = flag.Value.String() - case tiflashBinpath: - options.TiFlash.BinPath = flag.Value.String() - case ticdcBinpath: - options.TiCDC.BinPath = flag.Value.String() - case kvcdcBinpath: - options.TiKVCDC.BinPath = flag.Value.String() - case pumpBinpath: - options.Pump.BinPath = flag.Value.String() - case drainerBinpath: - options.Drainer.BinPath = flag.Value.String() - - case dbTimeout: - options.TiDB.UpTimeout, err = strconv.Atoi(flag.Value.String()) - if err != nil { - return - } - case tiflashTimeout: - options.TiFlash.UpTimeout, err = strconv.Atoi(flag.Value.String()) - if err != nil { - return - } - - case clusterHost: - options.Host = flag.Value.String() - case dbHost: - options.TiDB.Host = flag.Value.String() - case dbPort: - options.TiDB.Port, err = strconv.Atoi(flag.Value.String()) - if err != nil { - return - } - case pdHost: - options.PD.Host = flag.Value.String() - case pdPort: - options.PD.Port, err = strconv.Atoi(flag.Value.String()) - if err != nil { - return - } - - case kvcdcVersion: - options.TiKVCDC.Version = flag.Value.String() - } - }) + switch options.PDMode { + case "pd": + defaultInt(&options.PD.Num, "pd", 1) + case "ms": + defaultInt(&options.PDAPI.Num, "pd.api", 1) + defaultStr(&options.PDAPI.BinPath, "pd.api.binpath", options.PDAPI.BinPath) + defaultStr(&options.PDAPI.ConfigPath, "pd.api.config", options.PDAPI.ConfigPath) + defaultInt(&options.PDTSO.Num, "pd.tso", 1) + defaultStr(&options.PDTSO.BinPath, "pd.tso.binpath", options.PDTSO.BinPath) + defaultStr(&options.PDTSO.ConfigPath, "pd.tso.config", options.PDTSO.ConfigPath) + defaultInt(&options.PDScheduling.Num, "pd.scheduling", 1) + defaultStr(&options.PDScheduling.BinPath, "pd.scheduling.binpath", options.PDScheduling.BinPath) + defaultStr(&options.PDScheduling.ConfigPath, "pd.scheduling.config", options.PDScheduling.ConfigPath) + defaultInt(&options.PDRM.Num, "pd.rm", 1) + defaultStr(&options.PDRM.BinPath, "pd.rm.binpath", options.PDRM.BinPath) + defaultStr(&options.PDRM.ConfigPath, "pd.rm.config", options.PDRM.ConfigPath) + default: + return errors.Errorf("Unknown --pd.mode %s", options.PDMode) + } - return + return nil } -func tryConnect(dsn string) error { - cli, err := sql.Open("mysql", dsn) - if err != nil { - return err - } - defer cli.Close() - - conn, err := cli.Conn(context.Background()) +func tryConnect(addr string, timeoutSec int) error { + conn, err := net.DialTimeout("tcp", addr, time.Duration(timeoutSec)*time.Second) if err != nil { return err } defer conn.Close() - return nil } // checkDB check if the addr is connectable by getting a connection from sql.DB. timeout <=0 means no timeout func checkDB(dbAddr string, timeout int) bool { - dsn := fmt.Sprintf("root:@tcp(%s)/", dbAddr) if timeout > 0 { for i := 0; i < timeout; i++ { - if tryConnect(dsn) == nil { + if tryConnect(dbAddr, timeout) == nil { return true } time.Sleep(time.Second) @@ -545,7 +448,7 @@ func checkDB(dbAddr string, timeout int) bool { return false } for { - if err := tryConnect(dsn); err == nil { + if err := tryConnect(dbAddr, timeout); err == nil { return true } time.Sleep(time.Second) @@ -615,7 +518,7 @@ func getAbsolutePath(path string) (string, error) { } func dumpPort(fname string, port int) error { - return os.WriteFile(fname, []byte(strconv.Itoa(port)), 0644) + return utils.WriteFile(fname, []byte(strconv.Itoa(port)), 0644) } func loadPort(dir string) (port int, err error) { @@ -633,7 +536,7 @@ func dumpDSN(fname string, dbs []*instance.TiDBInstance) { for _, db := range dbs { dsn = append(dsn, fmt.Sprintf("mysql://root@%s", db.Addr())) } - _ = os.WriteFile(fname, []byte(strings.Join(dsn, "\n")), 0644) + _ = utils.WriteFile(fname, []byte(strings.Join(dsn, "\n")), 0644) } func newEtcdClient(endpoint string) (*clientv3.Client, error) { diff --git a/components/playground/monitor.go b/components/playground/monitor.go index 04254b3392..3d17a7edae 100644 --- a/components/playground/monitor.go +++ b/components/playground/monitor.go @@ -29,24 +29,20 @@ import ( ) // ref: https://prometheus.io/docs/prometheus/latest/configuration/configuration/#file_sd_config -func (m *monitor) renderSDFile(cid2targets map[string][]string) error { - type Item struct { - Targets []string `json:"targets"` - Labels map[string]string `json:"labels"` - } - - cid2targets["prometheus"] = []string{utils.JoinHostPort(m.host, m.port)} +func (m *monitor) renderSDFile(cid2targets map[string]instance.MetricAddr) error { + cid2targets["prometheus"] = instance.MetricAddr{Targets: []string{utils.JoinHostPort(m.host, m.port)}} - var items []Item + var items []instance.MetricAddr - for id, targets := range cid2targets { - item := Item{ - Targets: targets, - Labels: map[string]string{ - "job": id, - }, + for id, t := range cid2targets { + it := instance.MetricAddr{ + Targets: t.Targets, + Labels: map[string]string{"job": id}, + } + for k, v := range t.Labels { + it.Labels[k] = v } - items = append(items, item) + items = append(items, it) } data, err := json.MarshalIndent(&items, "", "\t") @@ -54,7 +50,7 @@ func (m *monitor) renderSDFile(cid2targets map[string][]string) error { return errors.AddStack(err) } - err = os.WriteFile(m.sdFname, data, 0644) + err = utils.WriteFile(m.sdFname, data, 0644) if err != nil { return errors.AddStack(err) } @@ -83,7 +79,7 @@ func (m *monitor) wait() error { // the cmd is not started after return func newMonitor(ctx context.Context, version string, host, dir string) (*monitor, error) { - if err := os.MkdirAll(dir, 0755); err != nil { + if err := utils.MkdirAll(dir, 0755); err != nil { return nil, errors.AddStack(err) } @@ -124,7 +120,7 @@ scrape_configs: m := new(monitor) m.sdFname = filepath.Join(dir, "targets.json") - if err := os.WriteFile(filepath.Join(dir, "prometheus.yml"), []byte(tmpl), os.ModePerm); err != nil { + if err := utils.WriteFile(filepath.Join(dir, "prometheus.yml"), []byte(tmpl), os.ModePerm); err != nil { return nil, errors.AddStack(err) } diff --git a/components/playground/ngmonitoring.go b/components/playground/ngmonitoring.go index d6c19dcb83..beb7a58552 100644 --- a/components/playground/ngmonitoring.go +++ b/components/playground/ngmonitoring.go @@ -16,7 +16,6 @@ package main import ( "context" "fmt" - "os" "os/exec" "path/filepath" "strings" @@ -47,7 +46,7 @@ func (m *ngMonitoring) wait() error { // the cmd is not started after return func newNGMonitoring(ctx context.Context, version string, host, dir string, pds []*instance.PDInstance) (*ngMonitoring, error) { - if err := os.MkdirAll(dir, 0755); err != nil { + if err := utils.MkdirAll(dir, 0755); err != nil { return nil, errors.AddStack(err) } diff --git a/components/playground/playground.go b/components/playground/playground.go index d14985b7f5..142feb5c89 100644 --- a/components/playground/playground.go +++ b/components/playground/playground.go @@ -33,18 +33,19 @@ import ( "github.com/AstroProfundis/tabby" "github.com/fatih/color" "github.com/juju/ansiterm" + "github.com/minio/minio-go/v7" + "github.com/minio/minio-go/v7/pkg/credentials" "github.com/pingcap/errors" "github.com/pingcap/tiup/components/playground/instance" "github.com/pingcap/tiup/pkg/cluster/api" "github.com/pingcap/tiup/pkg/cluster/spec" "github.com/pingcap/tiup/pkg/environment" logprinter "github.com/pingcap/tiup/pkg/logger/printer" + "github.com/pingcap/tiup/pkg/tidbver" "github.com/pingcap/tiup/pkg/tui/progress" "github.com/pingcap/tiup/pkg/utils" "golang.org/x/mod/semver" "golang.org/x/sync/errgroup" - "golang.org/x/text/cases" - "golang.org/x/text/language" ) // The duration process need to quit gracefully, or we kill the process. @@ -60,9 +61,13 @@ type Playground struct { port int pds []*instance.PDInstance + tsos []*instance.PDInstance + schedulings []*instance.PDInstance + rms []*instance.PDInstance tikvs []*instance.TiKVInstance tidbs []*instance.TiDBInstance tiflashs []*instance.TiFlashInstance + tiproxys []*instance.TiProxy ticdcs []*instance.TiCDC tikvCdcs []*instance.TiKVCDC pumps []*instance.Pump @@ -274,6 +279,7 @@ func (p *Playground) handleScaleIn(w io.Writer, pid int) error { switch cid { case spec.ComponentPD: + // microservice not support scale in temporarily for i := 0; i < len(p.pds); i++ { if p.pds[i].Pid() == pid { inst := p.pds[i] @@ -326,7 +332,7 @@ func (p *Playground) handleScaleIn(w io.Writer, pid int) error { } go p.killTiFlashIfTombstone(inst) - fmt.Fprintf(w, "tiflash will be stop when tombstone\n") + fmt.Fprintf(w, "TiFlash will be stop when tombstone\n") return nil } } @@ -428,8 +434,14 @@ func (p *Playground) sanitizeComponentConfig(cid string, cfg *instance.Config) e } func (p *Playground) startInstance(ctx context.Context, inst instance.Instance) error { + var version utils.Version + var err error boundVersion := p.bindVersion(inst.Component(), p.bootOptions.Version) - version, err := environment.GlobalEnv().V1Repository().ResolveComponentVersion(inst.Component(), boundVersion) + component := inst.Component() + if strings.HasPrefix(component, "pd") { + component = string(instance.PDRoleNormal) + } + version, err = environment.GlobalEnv().V1Repository().ResolveComponentVersion(component, boundVersion) if err != nil { return err } @@ -467,7 +479,8 @@ func (p *Playground) handleScaleOut(w io.Writer, cmd *Command) error { if err != nil { return err } - inst, err := p.addInstance(cmd.ComponentID, cmd.Config) + // TODO: Support scale-out in disaggregated mode + inst, err := p.addInstance(cmd.ComponentID, instance.PDRoleNormal, instance.TiFlashRoleNormal, cmd.Config) if err != nil { return err } @@ -489,6 +502,15 @@ func (p *Playground) handleScaleOut(w io.Writer, cmd *Command) error { fmt.Fprintln(w, color.GreenString(connectMsg, ss[0], ss[1])) } } + if cmd.ComponentID == "tiproxy" { + addr := p.tiproxys[len(p.tidbs)-1].Addr() + if checkDB(addr, cmd.UpTimeout) { + ss := strings.Split(addr, ":") + connectMsg := "To connect to the newly added TiProxy: mysql --comments --host %s --port %s -u root -p (no password)" + fmt.Println(color.GreenString(connectMsg, ss[0], ss[1])) + fmt.Fprintln(w, color.GreenString(connectMsg, ss[0], ss[1])) + } + } logIfErr(p.renderSDFile()) @@ -536,7 +558,7 @@ func (p *Playground) commandHandler(w http.ResponseWriter, r *http.Request) { } // Mapping command line component id to internal spec component id. - if cmd.ComponentID == ticdc { + if cmd.ComponentID == "ticdc" { cmd.ComponentID = spec.ComponentCDC } @@ -575,6 +597,24 @@ func (p *Playground) WalkInstances(fn func(componentID string, ins instance.Inst return err } } + for _, ins := range p.tsos { + err := fn(spec.ComponentPD, ins) + if err != nil { + return err + } + } + for _, ins := range p.schedulings { + err := fn(spec.ComponentPD, ins) + if err != nil { + return err + } + } + for _, ins := range p.rms { + err := fn(spec.ComponentPD, ins) + if err != nil { + return err + } + } for _, ins := range p.tikvs { err := fn(spec.ComponentTiKV, ins) if err != nil { @@ -596,6 +636,13 @@ func (p *Playground) WalkInstances(fn func(componentID string, ins instance.Inst } } + for _, ins := range p.tiproxys { + err := fn(spec.ComponentTiProxy, ins) + if err != nil { + return err + } + } + for _, ins := range p.ticdcs { err := fn(spec.ComponentCDC, ins) if err != nil { @@ -623,6 +670,7 @@ func (p *Playground) WalkInstances(fn func(componentID string, ins instance.Inst return err } } + return nil } @@ -630,7 +678,7 @@ func (p *Playground) enableBinlog() bool { return p.bootOptions.Pump.Num > 0 } -func (p *Playground) addInstance(componentID string, cfg instance.Config) (ins instance.Instance, err error) { +func (p *Playground) addInstance(componentID string, pdRole instance.PDRole, tiflashRole instance.TiFlashRole, cfg instance.Config) (ins instance.Instance, err error) { if cfg.BinPath != "" { cfg.BinPath, err = getAbsolutePath(cfg.BinPath) if err != nil { @@ -649,7 +697,11 @@ func (p *Playground) addInstance(componentID string, cfg instance.Config) (ins i id := p.allocID(componentID) dir := filepath.Join(dataDir, fmt.Sprintf("%s-%d", componentID, id)) - if err = os.MkdirAll(dir, 0755); err != nil { + if componentID == string(instance.PDRoleNormal) && pdRole != instance.PDRoleNormal { + id = p.allocID(fmt.Sprintf("%s-%s", componentID, pdRole)) + dir = filepath.Join(dataDir, fmt.Sprintf("%s-%s-%d", componentID, pdRole, id)) + } + if err = utils.MkdirAll(dir, 0755); err != nil { return nil, err } // look more like listen ip? @@ -660,31 +712,43 @@ func (p *Playground) addInstance(componentID string, cfg instance.Config) (ins i switch componentID { case spec.ComponentPD: - inst := instance.NewPDInstance(cfg.BinPath, dir, host, cfg.ConfigPath, id, cfg.Port) + inst := instance.NewPDInstance(pdRole, cfg.BinPath, dir, host, cfg.ConfigPath, id, p.pds, cfg.Port) ins = inst - if p.booted { - inst.Join(p.pds) - p.pds = append(p.pds, inst) - } else { - p.pds = append(p.pds, inst) - for _, pd := range p.pds { - pd.InitCluster(p.pds) + if pdRole == instance.PDRoleNormal || pdRole == instance.PDRoleAPI { + if p.booted { + inst.Join(p.pds) + p.pds = append(p.pds, inst) + } else { + p.pds = append(p.pds, inst) + for _, pd := range p.pds { + pd.InitCluster(p.pds) + } } + } else if pdRole == instance.PDRoleTSO { + p.tsos = append(p.tsos, inst) + } else if pdRole == instance.PDRoleScheduling { + p.schedulings = append(p.schedulings, inst) + } else if pdRole == instance.PDRoleResourceManager { + p.rms = append(p.rms, inst) } case spec.ComponentTiDB: - inst := instance.NewTiDBInstance(cfg.BinPath, dir, host, cfg.ConfigPath, id, cfg.Port, p.pds, p.enableBinlog()) + inst := instance.NewTiDBInstance(cfg.BinPath, dir, host, cfg.ConfigPath, id, cfg.Port, p.pds, p.enableBinlog(), p.bootOptions.Mode == "tidb-disagg") ins = inst p.tidbs = append(p.tidbs, inst) case spec.ComponentTiKV: - inst := instance.NewTiKVInstance(cfg.BinPath, dir, host, cfg.ConfigPath, id, p.pds) + inst := instance.NewTiKVInstance(cfg.BinPath, dir, host, cfg.ConfigPath, id, cfg.Port, p.pds) ins = inst p.tikvs = append(p.tikvs, inst) case spec.ComponentTiFlash: - inst := instance.NewTiFlashInstance(cfg.BinPath, dir, host, cfg.ConfigPath, id, p.pds, p.tidbs) + inst := instance.NewTiFlashInstance(tiflashRole, p.bootOptions.DisaggOpts, cfg.BinPath, dir, host, cfg.ConfigPath, id, p.pds, p.tidbs, cfg.Version) ins = inst p.tiflashs = append(p.tiflashs, inst) + case spec.ComponentTiProxy: + inst := instance.NewTiProxy(cfg.BinPath, dir, host, cfg.ConfigPath, id, cfg.Port, p.pds) + ins = inst + p.tiproxys = append(p.tiproxys, inst) case spec.ComponentCDC: - inst := instance.NewTiCDC(cfg.BinPath, dir, host, cfg.ConfigPath, id, p.pds) + inst := instance.NewTiCDC(cfg.BinPath, dir, host, cfg.ConfigPath, id, cfg.Port, p.pds) ins = inst p.ticdcs = append(p.ticdcs, inst) case spec.ComponentTiKVCDC: @@ -706,12 +770,18 @@ func (p *Playground) addInstance(componentID string, cfg instance.Config) (ins i return } -func (p *Playground) waitAllTidbUp() []string { - var succ []string +func (p *Playground) waitAllDBUp() ([]string, []string) { + var tidbSucc []string + var tiproxySucc []string if len(p.tidbs) > 0 { var wg sync.WaitGroup - var appendMutex sync.Mutex - bars := progress.NewMultiBar(color.YellowString("Waiting for tidb instances ready")) + var tidbMu, tiproxyMu sync.Mutex + var bars *progress.MultiBar + if len(p.tiproxys) > 0 { + bars = progress.NewMultiBar(color.YellowString("Waiting for tidb and tiproxy instances ready")) + } else { + bars = progress.NewMultiBar(color.YellowString("Waiting for tidb instances ready")) + } for _, db := range p.tidbs { wg.Add(1) prefix := color.YellowString(db.Addr()) @@ -720,9 +790,33 @@ func (p *Playground) waitAllTidbUp() []string { defer wg.Done() if s := checkDB(dbInst.Addr(), options.TiDB.UpTimeout); s { { - appendMutex.Lock() - succ = append(succ, dbInst.Addr()) - appendMutex.Unlock() + tidbMu.Lock() + tidbSucc = append(tidbSucc, dbInst.Addr()) + tidbMu.Unlock() + } + bar.UpdateDisplay(&progress.DisplayProps{ + Prefix: prefix, + Mode: progress.ModeDone, + }) + } else { + bar.UpdateDisplay(&progress.DisplayProps{ + Prefix: prefix, + Mode: progress.ModeError, + }) + } + }(db) + } + for _, db := range p.tiproxys { + wg.Add(1) + prefix := color.YellowString(db.Addr()) + bar := bars.AddBar(prefix) + go func(dbInst *instance.TiProxy) { + defer wg.Done() + if s := checkDB(dbInst.Addr(), options.TiProxy.UpTimeout); s { + { + tiproxyMu.Lock() + tiproxySucc = append(tiproxySucc, dbInst.Addr()) + tiproxyMu.Unlock() } bar.UpdateDisplay(&progress.DisplayProps{ Prefix: prefix, @@ -740,7 +834,7 @@ func (p *Playground) waitAllTidbUp() []string { wg.Wait() bars.StopRenderLoop() } - return succ + return tidbSucc, tiproxySucc } func (p *Playground) waitAllTiFlashUp() { @@ -790,6 +884,8 @@ func (p *Playground) bindVersion(comp string, version string) (bindVersion strin switch comp { case spec.ComponentTiKVCDC: return p.bootOptions.TiKVCDC.Version + case spec.ComponentTiProxy: + return p.bootOptions.TiProxy.Version default: return version } @@ -798,9 +894,16 @@ func (p *Playground) bindVersion(comp string, version string) (bindVersion strin func (p *Playground) bootCluster(ctx context.Context, env *environment.Environment, options *BootOptions) error { for _, cfg := range []*instance.Config{ &options.PD, + &options.PDAPI, + &options.PDTSO, + &options.PDScheduling, + &options.PDRM, + &options.TiProxy, &options.TiDB, &options.TiKV, &options.TiFlash, + &options.TiFlashCompute, + &options.TiFlashWrite, &options.Pump, &options.Drainer, &options.TiKVCDC, @@ -815,7 +918,7 @@ func (p *Playground) bootCluster(ctx context.Context, env *environment.Environme p.bootOptions = options // All others components depend on the pd, we just ensure the pd count must be great than 0 - if options.PD.Num < 1 { + if options.PDMode != "ms" && options.PD.Num < 1 { return fmt.Errorf("all components count must be great than 0 (pd=%v)", options.PD.Num) } @@ -830,31 +933,94 @@ func (p *Playground) bootCluster(ctx context.Context, env *environment.Environme } } - instances := []struct { - comp string + type InstancePair struct { + comp string + pdRole instance.PDRole + tiflashRole instance.TiFlashRole instance.Config - }{ - {spec.ComponentPD, options.PD}, - {spec.ComponentTiKV, options.TiKV}, - {spec.ComponentPump, options.Pump}, - {spec.ComponentTiDB, options.TiDB}, - {spec.ComponentCDC, options.TiCDC}, - {spec.ComponentTiKVCDC, options.TiKVCDC}, - {spec.ComponentDrainer, options.Drainer}, - {spec.ComponentTiFlash, options.TiFlash}, + } + + instances := []InstancePair{ + {spec.ComponentTiProxy, "", "", options.TiProxy}, + {spec.ComponentTiKV, "", "", options.TiKV}, + {spec.ComponentPump, "", "", options.Pump}, + {spec.ComponentTiDB, "", "", options.TiDB}, + {spec.ComponentCDC, "", "", options.TiCDC}, + {spec.ComponentTiKVCDC, "", "", options.TiKVCDC}, + {spec.ComponentDrainer, "", "", options.Drainer}, + } + + if options.Mode == "tidb" { + instances = append(instances, + InstancePair{spec.ComponentTiFlash, instance.PDRoleNormal, instance.TiFlashRoleNormal, options.TiFlash}, + ) + } else if options.Mode == "tidb-disagg" { + if !tidbver.TiDBSupportDisagg(options.Version) { + return fmt.Errorf("TiDB cluster doesn't support disaggregated mode in version %s", options.Version) + } + if !tidbver.TiFlashPlaygroundNewStartMode(options.Version) { + // For simplicity, currently we only implemented disagg mode when TiFlash can run without config. + return fmt.Errorf("TiUP playground only supports disaggregated mode for TiDB cluster >= v7.1.0 (or nightly)") + } + + // Preflight check whether specified object storage is available. + s3Client, err := minio.New(options.DisaggOpts.S3Endpoint, &minio.Options{ + Creds: credentials.NewStaticV4(options.DisaggOpts.AccessKey, options.DisaggOpts.SecretKey, ""), + Secure: false, + }) + if err != nil { + return errors.Annotate(err, "Disaggregate mode preflight check failed") + } + + ctxCheck, cancel := context.WithTimeout(ctx, 5*time.Second) + defer cancel() + + bucketExists, err := s3Client.BucketExists(ctxCheck, options.DisaggOpts.Bucket) + if err != nil { + return errors.Annotate(err, "Disaggregate mode preflight check failed") + } + + if !bucketExists { + // Try to create bucket. + err := s3Client.MakeBucket(ctxCheck, options.DisaggOpts.Bucket, minio.MakeBucketOptions{}) + if err != nil { + return fmt.Errorf("Disaggregate mode preflight check failed: Bucket %s doesn't exist", options.DisaggOpts.Bucket) + } + } + + instances = append( + instances, + InstancePair{spec.ComponentTiFlash, instance.PDRoleNormal, instance.TiFlashRoleDisaggWrite, options.TiFlashWrite}, + InstancePair{spec.ComponentTiFlash, instance.PDRoleNormal, instance.TiFlashRoleDisaggCompute, options.TiFlashCompute}, + ) + } + + if options.PDMode == "pd" { + instances = append([]InstancePair{{spec.ComponentPD, instance.PDRoleNormal, instance.TiFlashRoleNormal, options.PD}}, + instances..., + ) + } else if options.PDMode == "ms" { + if !tidbver.PDSupportMicroServices(options.Version) { + return fmt.Errorf("PD cluster doesn't support microservices mode in version %s", options.Version) + } + instances = append([]InstancePair{ + {spec.ComponentPD, instance.PDRoleAPI, instance.TiFlashRoleNormal, options.PDAPI}, + {spec.ComponentPD, instance.PDRoleTSO, instance.TiFlashRoleNormal, options.PDTSO}, + {spec.ComponentPD, instance.PDRoleScheduling, instance.TiFlashRoleNormal, options.PDScheduling}, + {spec.ComponentPD, instance.PDRoleResourceManager, instance.TiFlashRoleNormal, options.PDRM}}, + instances..., + ) } for _, inst := range instances { for i := 0; i < inst.Num; i++ { - _, err := p.addInstance(inst.comp, inst.Config) + _, err := p.addInstance(inst.comp, inst.pdRole, inst.tiflashRole, inst.Config) if err != nil { return err } } } - fmt.Println("Playground Bootstrapping...") - anyPumpReady := false // Start all instance except tiflash. err := p.WalkInstances(func(cid string, ins instance.Instance) error { @@ -886,7 +1052,7 @@ func (p *Playground) bootCluster(ctx context.Context, env *environment.Environme p.booted = true - succ := p.waitAllTidbUp() + tidbSucc, tiproxySucc := p.waitAllDBUp() var monitorInfo *MonitorInfo if options.Monitor { @@ -908,7 +1074,9 @@ func (p *Playground) bootCluster(ctx context.Context, env *environment.Environme } } - if len(succ) > 0 { + colorCmd := color.New(color.FgHiCyan, color.Bold) + + if len(tidbSucc) > 0 { // start TiFlash after at least one TiDB is up. var started []*instance.TiFlashInstance for _, flash := range p.tiflashs { @@ -921,23 +1089,64 @@ func (p *Playground) bootCluster(ctx context.Context, env *environment.Environme p.tiflashs = started p.waitAllTiFlashUp() - fmt.Println(color.GreenString("CLUSTER START SUCCESSFULLY, Enjoy it ^-^")) - for _, dbAddr := range succ { + fmt.Println() + color.New(color.FgGreen, color.Bold).Println("🎉 TiDB Playground Cluster is started, enjoy!") + fmt.Println() + for _, dbAddr := range tidbSucc { ss := strings.Split(dbAddr, ":") - connectMsg := "To connect TiDB: mysql --comments --host %s --port %s -u root -p (no password)" - fmt.Println(color.GreenString(connectMsg, ss[0], ss[1])) + fmt.Printf("Connect TiDB: ") + colorCmd.Printf("mysql --comments --host %s --port %s -u root\n", ss[0], ss[1]) + } + for _, dbAddr := range tiproxySucc { + ss := strings.Split(dbAddr, ":") + fmt.Printf("Connect TiProxy: ") + colorCmd.Printf("mysql --comments --host %s --port %s -u root\n", ss[0], ss[1]) } } if pdAddr := p.pds[0].Addr(); len(p.tidbs) > 0 && hasDashboard(pdAddr) { - fmt.Println(color.GreenString("To view the dashboard: http://%s/dashboard", pdAddr)) - } + fmt.Printf("TiDB Dashboard: ") + colorCmd.Printf("http://%s/dashboard\n", pdAddr) + } + + if p.bootOptions.Mode == "tikv-slim" { + if p.bootOptions.PDMode == "ms" { + var ( + tsoAddr []string + apiAddr []string + rmAddr []string + schedulingAddr []string + ) + for _, api := range p.pds { + apiAddr = append(apiAddr, api.Addr()) + } + for _, tso := range p.tsos { + tsoAddr = append(tsoAddr, tso.Addr()) + } + for _, scheduling := range p.schedulings { + schedulingAddr = append(schedulingAddr, scheduling.Addr()) + } + for _, rm := range p.rms { + rmAddr = append(rmAddr, rm.Addr()) + } - var pdAddrs []string - for _, pd := range p.pds { - pdAddrs = append(pdAddrs, pd.Addr()) + fmt.Printf("PD API Endpoints: ") + colorCmd.Printf("%s\n", strings.Join(apiAddr, ",")) + fmt.Printf("PD TSO Endpoints: ") + colorCmd.Printf("%s\n", strings.Join(tsoAddr, ",")) + fmt.Printf("PD Scheduling Endpoints: ") + colorCmd.Printf("%s\n", strings.Join(schedulingAddr, ",")) + fmt.Printf("PD Resource Manager Endpoints: ") + colorCmd.Printf("%s\n", strings.Join(rmAddr, ",")) + } else { + var pdAddrs []string + for _, pd := range p.pds { + pdAddrs = append(pdAddrs, pd.Addr()) + } + fmt.Printf("PD Endpoints: ") + colorCmd.Printf("%s\n", strings.Join(pdAddrs, ",")) + } } - fmt.Println(color.GreenString("PD client endpoints: %v", pdAddrs)) if monitorInfo != nil { p.updateMonitorTopology(spec.ComponentPrometheus, *monitorInfo) @@ -957,6 +1166,8 @@ func (p *Playground) bootCluster(ctx context.Context, env *environment.Environme if g := p.grafana; g != nil { p.updateMonitorTopology(spec.ComponentGrafana, MonitorInfo{g.host, g.port, g.cmd.Path}) + fmt.Printf("Grafana: ") + colorCmd.Printf("http://%s\n", utils.JoinHostPort(g.host, g.port)) } return nil @@ -964,11 +1175,6 @@ func (p *Playground) bootCluster(ctx context.Context, env *environment.Environme func (p *Playground) updateMonitorTopology(componentID string, info MonitorInfo) { info.IP = instance.AdvertiseHost(info.IP) - fmt.Print(color.GreenString( - "To view the %s: http://%s\n", - cases.Title(language.English).String(componentID), - utils.JoinHostPort(info.IP, info.Port), - )) if len(p.pds) == 0 { return } @@ -1012,55 +1218,75 @@ func (p *Playground) terminate(sig syscall.Signal) { timer.Stop() } - if p.monitor != nil { + if p.monitor != nil && p.monitor.cmd != nil && p.monitor.cmd.Process != nil { go kill("prometheus", p.monitor.cmd.Process.Pid, p.monitor.wait) } - if p.ngmonitoring != nil { + if p.ngmonitoring != nil && p.ngmonitoring.cmd != nil && p.ngmonitoring.cmd.Process != nil { go kill("ng-monitoring", p.ngmonitoring.cmd.Process.Pid, p.ngmonitoring.wait) } - if p.grafana != nil { + if p.grafana != nil && p.grafana.cmd != nil && p.grafana.cmd.Process != nil { go kill("grafana", p.grafana.cmd.Process.Pid, p.grafana.wait) } for _, inst := range p.tiflashs { - if inst.Process != nil { + if inst.Process != nil && inst.Process.Cmd() != nil && inst.Process.Cmd().Process != nil { kill(inst.Component(), inst.Pid(), inst.Wait) } } for _, inst := range p.ticdcs { - if inst.Process != nil { + if inst.Process != nil && inst.Process.Cmd() != nil && inst.Process.Cmd().Process != nil { kill(inst.Component(), inst.Pid(), inst.Wait) } } for _, inst := range p.tikvCdcs { - if inst.Process != nil { + if inst.Process != nil && inst.Process.Cmd() != nil && inst.Process.Cmd().Process != nil { kill(inst.Component(), inst.Pid(), inst.Wait) } } for _, inst := range p.drainers { - if inst.Process != nil { + if inst.Process != nil && inst.Process.Cmd() != nil && inst.Process.Cmd().Process != nil { kill(inst.Component(), inst.Pid(), inst.Wait) } } // tidb must exit earlier then pd for _, inst := range p.tidbs { - if inst.Process != nil { + if inst.Process != nil && inst.Process.Cmd() != nil && inst.Process.Cmd().Process != nil { kill(inst.Component(), inst.Pid(), inst.Wait) } } for _, inst := range p.pumps { - if inst.Process != nil { + if inst.Process != nil && inst.Process.Cmd() != nil && inst.Process.Cmd().Process != nil { kill(inst.Component(), inst.Pid(), inst.Wait) } } for _, inst := range p.tikvs { - if inst.Process != nil { + if inst.Process != nil && inst.Process.Cmd() != nil && inst.Process.Cmd().Process != nil { kill(inst.Component(), inst.Pid(), inst.Wait) } } for _, inst := range p.pds { - if inst.Process != nil { + if inst.Process != nil && inst.Process.Cmd() != nil && inst.Process.Cmd().Process != nil { + kill(inst.Component(), inst.Pid(), inst.Wait) + } + } + for _, inst := range p.tsos { + if inst.Process != nil && inst.Process.Cmd() != nil && inst.Process.Cmd().Process != nil { + kill(inst.Component(), inst.Pid(), inst.Wait) + } + } + for _, inst := range p.schedulings { + if inst.Process != nil && inst.Process.Cmd() != nil && inst.Process.Cmd().Process != nil { + kill(inst.Component(), inst.Pid(), inst.Wait) + } + } + for _, inst := range p.rms { + if inst.Process != nil && inst.Process.Cmd() != nil && inst.Process.Cmd().Process != nil { + kill(inst.Component(), inst.Pid(), inst.Wait) + } + } + for _, inst := range p.tiproxys { + if inst.Process != nil && inst.Process.Cmd() != nil && inst.Process.Cmd().Process != nil { kill(inst.Component(), inst.Pid(), inst.Wait) } } @@ -1072,12 +1298,10 @@ func (p *Playground) renderSDFile() error { return nil } - cid2targets := make(map[string][]string) + cid2targets := make(map[string]instance.MetricAddr) _ = p.WalkInstances(func(cid string, inst instance.Instance) error { - targets := cid2targets[cid] - targets = append(targets, inst.StatusAddrs()...) - cid2targets[cid] = targets + cid2targets[cid] = inst.MetricAddr() return nil }) @@ -1192,7 +1416,7 @@ func (p *Playground) bootGrafana(ctx context.Context, env *environment.Environme } dashboardDir := filepath.Join(grafanaDir, "dashboards") - err = os.MkdirAll(dashboardDir, 0755) + err = utils.MkdirAll(dashboardDir, 0755) if err != nil { return nil, errors.AddStack(err) } diff --git a/doc/dev/README.md b/doc/dev/README.md index 19009db791..8d8c150ee5 100644 --- a/doc/dev/README.md +++ b/doc/dev/README.md @@ -6,7 +6,7 @@ You can build TiUP on any platform that supports Go. Prerequisites: -* Go (minimum version: 1.18; [installation instructions](https://golang.org/doc/install)) +* Go (minimum version: 1.21; [installation instructions](https://golang.org/doc/install)) * golint (`go get -u golang.org/x/lint/golint`) * make @@ -16,6 +16,7 @@ To build TiUP, run `make`. For development, you don't want to use any global directories. You may also want to supply your own metadata. TiUP can be modified using the following environment variables: +* `TIUP_VERBOSE` this enables verbose logging when set to `enabled`. * `TIUP_HOME` the profile directory, where TiUP stores its metadata. If not set, `~/.tiup` will be used. * `TIUP_MIRRORS` set the location of TiUP's registry, can be a directory or URL. If not set, `https://tiup-mirrors.pingcap.com` will be used. diff --git a/doc/rfcs/0002-support-run-component-without-tiup.md b/doc/rfcs/0002-support-run-component-without-tiup.md new file mode 100644 index 0000000000..58443f4d76 --- /dev/null +++ b/doc/rfcs/0002-support-run-component-without-tiup.md @@ -0,0 +1,23 @@ +# Support Run Component without TiUP + +## Summary + +Allow users to only use TiUP during installation and upgrade, without needing to invoke TiUP while using the component. + +## Motivation + +- Some users do not like the mandatory update checks and additional outputs by TiUP before invoking the components. +- For programs that are not only distributed through TiUP, the existing usage patterns have led to documentation fragmentation. + +## Detailed design + +1. Add `tiup link [:version]` command to add soft link to $TIUP_HOME/bin + +2. Add `tiup unlink [:version]` command to delete soft link from $HOME/bin + +3. Add `--link` flag to `tiup install` and `tiup update` command to link while install/update + +4. Mark these command and flag as experimental feature and we keep the old behavior as the default usage method. +Warn: Users may need to manually set environment variables for certain components, such as ctl. + +5. There is an additional benefit that user could use `tiup update tiup v1.13.0` to update tiup itself to specified version.And it makes TiUP unnecessary to handle special upgrades for itself. \ No newline at end of file diff --git a/embed/examples/cluster/minimal.yaml b/embed/examples/cluster/minimal.yaml index 447f7943d9..7d469a3758 100644 --- a/embed/examples/cluster/minimal.yaml +++ b/embed/examples/cluster/minimal.yaml @@ -11,6 +11,8 @@ global: deploy_dir: "/tidb-deploy" # # TiDB Cluster data storage directory data_dir: "/tidb-data" + # # default listen_host for all components + listen_host: 0.0.0.0 # # Supported values: "amd64", "arm64" (default: "amd64") arch: "amd64" # # Resource Control is used to limit the resource of an instance. @@ -42,9 +44,9 @@ monitored: # # Server configs are used to specify the runtime configuration of TiDB components. # # All configuration items can be found in TiDB docs: -# # - TiDB: https://pingcap.com/docs/stable/reference/configuration/tidb-server/configuration-file/ -# # - TiKV: https://pingcap.com/docs/stable/reference/configuration/tikv-server/configuration-file/ -# # - PD: https://pingcap.com/docs/stable/reference/configuration/pd-server/configuration-file/ +# # - TiDB: https://docs.pingcap.com/tidb/stable/tidb-configuration-file +# # - TiKV: https://docs.pingcap.com/tidb/stable/tikv-configuration-file +# # - PD: https://docs.pingcap.com/tidb/stable/pd-configuration-file # # - TiFlash: https://docs.pingcap.com/tidb/stable/tiflash-configuration # # # # All configuration items use points to represent the hierarchy, e.g: @@ -182,9 +184,8 @@ tiflash_servers: # # SSH port of the server. # ssh_port: 22 # # TiFlash TCP Service port. + # # Since 7.1.0, it is not actually listened, and only being used as part of the instance identity. # tcp_port: 9000 - # # TiFlash HTTP Service port. - # http_port: 8123 # # TiFlash raft service and coprocessor service listening address. # flash_service_port: 3930 # # TiFlash Proxy service port. @@ -207,7 +208,6 @@ tiflash_servers: - host: 10.0.1.21 # ssh_port: 22 # tcp_port: 9000 - # http_port: 8123 # flash_service_port: 3930 # flash_proxy_port: 20170 # flash_proxy_status_port: 20292 diff --git a/embed/examples/cluster/multi-dc.yaml b/embed/examples/cluster/multi-dc.yaml index 37e8ded064..3fcb2bb7af 100644 --- a/embed/examples/cluster/multi-dc.yaml +++ b/embed/examples/cluster/multi-dc.yaml @@ -11,6 +11,8 @@ global: deploy_dir: "/tidb-deploy" # # TiDB Cluster data storage directory data_dir: "/tidb-data" + # # default listen_host for all components + listen_host: 0.0.0.0 # # Supported values: "amd64", "arm64" (default: "amd64") arch: "amd64" # # Resource Control is used to limit the resource of an instance. @@ -235,9 +237,8 @@ tiflash_servers: # # SSH port of the server. # ssh_port: 22 # # TiFlash TCP Service port. + # # Since 7.1.0, it is not actually listened, and only being used as part of the instance identity. # tcp_port: 9000 - # # TiFlash HTTP Service port. - # http_port: 8123 # # TiFlash raft service and coprocessor service listening address. # flash_service_port: 3930 # # TiFlash Proxy service port. @@ -259,7 +260,6 @@ tiflash_servers: - host: 10.0.1.25 # ssh_port: 22 # tcp_port: 9000 - # http_port: 8123 # flash_service_port: 3930 # flash_proxy_port: 20170 # flash_proxy_status_port: 20292 diff --git a/embed/examples/cluster/topology.example.yaml b/embed/examples/cluster/topology.example.yaml index 38c477f819..72998cc847 100644 --- a/embed/examples/cluster/topology.example.yaml +++ b/embed/examples/cluster/topology.example.yaml @@ -11,6 +11,8 @@ global: deploy_dir: "/tidb-deploy" # # TiDB Cluster data storage directory data_dir: "/tidb-data" + # # default listen_host for all components + listen_host: 0.0.0.0 # # Supported values: "amd64", "arm64" (default: "amd64") arch: "amd64" # # Resource Control is used to limit the resource of an instance. @@ -42,9 +44,9 @@ monitored: # # Server configs are used to specify the runtime configuration of TiDB components. # # All configuration items can be found in TiDB docs: -# # - TiDB: https://pingcap.com/docs/stable/reference/configuration/tidb-server/configuration-file/ -# # - TiKV: https://pingcap.com/docs/stable/reference/configuration/tikv-server/configuration-file/ -# # - PD: https://pingcap.com/docs/stable/reference/configuration/pd-server/configuration-file/ +# # - TiDB: https://docs.pingcap.com/tidb/stable/tidb-configuration-file +# # - TiKV: https://docs.pingcap.com/tidb/stable/tikv-configuration-file +# # - PD: https://docs.pingcap.com/tidb/stable/pd-configuration-file # # - TiFlash: https://docs.pingcap.com/tidb/stable/tiflash-configuration # # # # All configuration items use points to represent the hierarchy, e.g: @@ -224,9 +226,8 @@ tiflash_servers: # # SSH port of the server. # ssh_port: 22 # # TiFlash TCP Service port. + # # Since 7.1.0, it is not actually listened, and only being used as part of the instance identity. tcp_port: 9000 - # # TiFlash HTTP Service port. - http_port: 8123 # # TiFlash raft service and coprocessor service listening address. flash_service_port: 3930 # # TiFlash Proxy service port. @@ -248,7 +249,6 @@ tiflash_servers: - host: 10.0.1.19 # ssh_port: 22 tcp_port: 9001 - http_port: 8124 flash_service_port: 3931 flash_proxy_port: 20171 flash_proxy_status_port: 20293 @@ -259,7 +259,6 @@ tiflash_servers: - host: 10.0.1.20 # ssh_port: 22 tcp_port: 9000 - http_port: 8123 flash_service_port: 3930 flash_proxy_port: 20170 flash_proxy_status_port: 20292 @@ -270,7 +269,6 @@ tiflash_servers: - host: 10.0.1.20 # ssh_port: 22 tcp_port: 9001 - http_port: 8124 flash_service_port: 3931 flash_proxy_port: 20171 flash_proxy_status_port: 20293 @@ -294,20 +292,20 @@ kvcdc_servers: data_dir: "/data1/tidb-data/tikv-cdc-8600" log_dir: "/data1/tidb-deploy/tikv-cdc-8600/log" -# # Server configs are used to specify the configuration of TiDB Dashboard Servers. -tidb_dashboard_servers: +# # Server configs are used to specify the configuration of TiDB Dashboard Servers. Available from v6.5.0 +# tidb_dashboard_servers: # # The ip address of the PD Server. - - host: 10.0.1.11 +# - host: 10.0.1.11 # # SSH port of the server. # ssh_port: 22 # # port of TiDB Dashboard - # port: 2380 + # port: 12333 # # TiDB Dashboard deployment file, startup script, configuration file storage directory. - # deploy_dir: "/tidb-deploy/tidb-dashboard-23333" + # deploy_dir: "/tidb-deploy/tidb-dashboard-12333" # # PD Server data storage directory. - # data_dir: "/tidb-data/tidb-dashboard-23333" + # data_dir: "/tidb-data/tidb-dashboard-12333" # # PD Server log file storage directory. - # log_dir: "/tidb-deploy/tidb-dashboard-23333/log" + # log_dir: "/tidb-deploy/tidb-dashboard-12333/log" # # numa node bindings. # numa_node: "0,1" diff --git a/embed/templates/config/datasource.yml.tpl b/embed/templates/config/datasource.yml.tpl index 29314dd2fe..b19985b1a1 100644 --- a/embed/templates/config/datasource.yml.tpl +++ b/embed/templates/config/datasource.yml.tpl @@ -3,7 +3,7 @@ datasources: - name: {{.ClusterName}} type: prometheus access: proxy - url: http://{{.IP}}:{{.Port}} + url: {{.URL}} withCredentials: false isDefault: false tlsAuth: false diff --git a/embed/templates/config/ngmonitoring.toml.tpl b/embed/templates/config/ngmonitoring.toml.tpl index eea083f74b..a03804188d 100644 --- a/embed/templates/config/ngmonitoring.toml.tpl +++ b/embed/templates/config/ngmonitoring.toml.tpl @@ -1,8 +1,8 @@ # NG Monitoring Server Configuration. # Server address. -address = "0.0.0.0:{{.Port}}" -advertise-address = "{{.IP}}:{{.Port}}" +address = "{{.Address}}" +advertise-address = "{{.AdvertiseAddress}}" [log] # Log path diff --git a/embed/templates/config/prometheus.yml.tpl b/embed/templates/config/prometheus.yml.tpl index 52ecb35cec..238eb179ab 100644 --- a/embed/templates/config/prometheus.yml.tpl +++ b/embed/templates/config/prometheus.yml.tpl @@ -65,28 +65,15 @@ alerting: {{- end}} scrape_configs: -{{- if .PushgatewayAddr}} +{{- if .PushgatewayAddrs}} - job_name: 'overwritten-cluster' scrape_interval: 15s honor_labels: true # don't overwrite job & instance labels - static_configs: - - targets: ['{{.PushgatewayAddr}}'] - - - job_name: "blackbox_exporter_http" - scrape_interval: 30s - metrics_path: /probe - params: - module: [http_2xx] static_configs: - targets: - - 'http://{{.PushgatewayAddr}}/metrics' - relabel_configs: - - source_labels: [__address__] - target_label: __param_target - - source_labels: [__param_target] - target_label: instance - - target_label: __address__ - replacement: '{{.BlackboxAddr}}' +{{- range .PushgatewayAddrs}} + - '{{.}}' +{{- end}} {{- end}} {{- if .LightningAddrs}} - job_name: "lightning" @@ -122,6 +109,22 @@ scrape_configs: - targets: {{- range .TiDBStatusAddrs}} - '{{.}}' +{{- end}} + - job_name: "tiproxy" + honor_labels: true # don't overwrite job & instance labels + metrics_path: /api/metrics +{{- if .TLSEnabled}} + scheme: https + tls_config: + insecure_skip_verify: false + ca_file: ../tls/ca.crt + cert_file: ../tls/prometheus.crt + key_file: ../tls/prometheus.pem +{{- end}} + static_configs: + - targets: +{{- range .TiProxyStatusAddrs}} + - '{{.}}' {{- end}} - job_name: "tikv" honor_labels: true # don't overwrite job & instance labels @@ -320,6 +323,14 @@ scrape_configs: {{- end}} labels: group: 'tiflash' +{{- end}} +{{- if .CDCAddrs}} + - targets: + {{- range .CDCAddrs}} + - '{{.}}' + {{- end}} + labels: + group: 'ticdc' {{- end}} relabel_configs: - source_labels: [__address__] @@ -334,9 +345,11 @@ scrape_configs: params: module: [tcp_connect] static_configs: -{{- if .PushgatewayAddr}} +{{- if .PushgatewayAddrs}} - targets: - - '{{.PushgatewayAddr}}' +{{- range .PushgatewayAddrs}} + - '{{.}}' +{{- end}} labels: group: 'pushgateway' {{- end}} diff --git a/embed/templates/scripts/run_dm-master.sh.tpl b/embed/templates/scripts/run_dm-master.sh.tpl index 074f1a76df..4d91ee6501 100644 --- a/embed/templates/scripts/run_dm-master.sh.tpl +++ b/embed/templates/scripts/run_dm-master.sh.tpl @@ -6,16 +6,6 @@ set -e DEPLOY_DIR={{.DeployDir}} cd "${DEPLOY_DIR}" || exit 1 -{{- define "MasterList"}} - {{- range $idx, $master := .}} - {{- if eq $idx 0}} - {{- $master.Name}}={{$master.Scheme}}://{{$master.IP}}:{{$master.PeerPort}} - {{- else -}} - ,{{- $master.Name}}={{$master.Scheme}}://{{$master.IP}}:{{$master.PeerPort}} - {{- end}} - {{- end}} -{{- end}} - {{- if .NumaNode}} exec numactl --cpunodebind={{.NumaNode}} --membind={{.NumaNode}} bin/dm-master/dm-master \ {{- else}} @@ -25,11 +15,11 @@ exec bin/dm-master/dm-master \ --v1-sources-path="{{.V1SourcePath}}" \ {{- end}} --name="{{.Name}}" \ - --master-addr="0.0.0.0:{{.Port}}" \ - --advertise-addr="{{.IP}}:{{.Port}}" \ - --peer-urls="{{.Scheme}}://0.0.0.0:{{.PeerPort}}" \ - --advertise-peer-urls="{{.Scheme}}://{{.IP}}:{{.PeerPort}}" \ + --master-addr="{{.MasterAddr}}" \ + --advertise-addr="{{.AdvertiseAddr}}" \ + --peer-urls="{{.PeerURL}}" \ + --advertise-peer-urls="{{.AdvertisePeerURL}}" \ --log-file="{{.LogDir}}/dm-master.log" \ --data-dir="{{.DataDir}}" \ - --initial-cluster="{{template "MasterList" .Endpoints}}" \ + --initial-cluster="{{.InitialCluster}}" \ --config=conf/dm-master.toml >> "{{.LogDir}}/dm-master_stdout.log" 2>> "{{.LogDir}}/dm-master_stderr.log" diff --git a/embed/templates/scripts/run_dm-master_scale.sh.tpl b/embed/templates/scripts/run_dm-master_scale.sh.tpl index 41fd33a678..af8432a819 100644 --- a/embed/templates/scripts/run_dm-master_scale.sh.tpl +++ b/embed/templates/scripts/run_dm-master_scale.sh.tpl @@ -6,27 +6,17 @@ set -e DEPLOY_DIR={{.DeployDir}} cd "${DEPLOY_DIR}" || exit 1 -{{- define "MasterList"}} - {{- range $idx, $master := .}} - {{- if eq $idx 0}} - {{- $master.IP}}:{{$master.Port}} - {{- else -}} - ,{{- $master.IP}}:{{$master.Port}} - {{- end}} - {{- end}} -{{- end}} - {{- if .NumaNode}} exec numactl --cpunodebind={{.NumaNode}} --membind={{.NumaNode}} bin/dm-master/dm-master \ {{- else}} exec bin/dm-master/dm-master \ {{- end}} --name="{{.Name}}" \ - --master-addr="0.0.0.0:{{.Port}}" \ - --advertise-addr="{{.IP}}:{{.Port}}" \ - --peer-urls="{{.Scheme}}://0.0.0.0:{{.PeerPort}}" \ - --advertise-peer-urls="{{.Scheme}}://{{.IP}}:{{.PeerPort}}" \ + --master-addr="{{.MasterAddr}}" \ + --advertise-addr="{{.AdvertiseAddr}}" \ + --peer-urls="{{.PeerURL}}" \ + --advertise-peer-urls="{{.AdvertisePeerURL}}" \ --log-file="{{.LogDir}}/dm-master.log" \ --data-dir="{{.DataDir}}" \ - --join="{{template "MasterList" .Endpoints}}" \ + --join="{{.Join}}" \ --config=conf/dm-master.toml >> "{{.LogDir}}/dm-master_stdout.log" 2>> "{{.LogDir}}/dm-master_stderr.log" diff --git a/embed/templates/scripts/run_dm-worker.sh.tpl b/embed/templates/scripts/run_dm-worker.sh.tpl index f1074b4967..2c5f9acb08 100644 --- a/embed/templates/scripts/run_dm-worker.sh.tpl +++ b/embed/templates/scripts/run_dm-worker.sh.tpl @@ -7,24 +7,14 @@ DEPLOY_DIR={{.DeployDir}} cd "${DEPLOY_DIR}" || exit 1 -{{- define "MasterList"}} - {{- range $idx, $master := .}} - {{- if eq $idx 0}} - {{- $master.IP}}:{{$master.Port}} - {{- else -}} - ,{{$master.IP}}:{{$master.Port}} - {{- end}} - {{- end}} -{{- end}} - {{- if .NumaNode}} exec numactl --cpunodebind={{.NumaNode}} --membind={{.NumaNode}} bin/dm-worker/dm-worker \ {{- else}} exec bin/dm-worker/dm-worker \ {{- end}} --name="{{.Name}}" \ - --worker-addr="0.0.0.0:{{.Port}}" \ - --advertise-addr="{{.IP}}:{{.Port}}" \ + --worker-addr="{{.WorkerAddr}}" \ + --advertise-addr="{{.AdvertiseAddr}}" \ --log-file="{{.LogDir}}/dm-worker.log" \ - --join="{{template "MasterList" .Endpoints}}" \ + --join="{{.Join}}" \ --config=conf/dm-worker.toml >> "{{.LogDir}}/dm-worker_stdout.log" 2>> "{{.LogDir}}/dm-worker_stderr.log" diff --git a/embed/templates/scripts/run_tidb-dashboard.sh.tpl b/embed/templates/scripts/run_tidb-dashboard.sh.tpl index 553b2d713b..41d9868616 100644 --- a/embed/templates/scripts/run_tidb-dashboard.sh.tpl +++ b/embed/templates/scripts/run_tidb-dashboard.sh.tpl @@ -13,7 +13,7 @@ exec numactl --cpunodebind={{.NumaNode}} --membind={{.NumaNode}} bin/tidb-dashbo exec bin/tidb-dashboard \ {{- end}} --feature-version="{{.TidbVersion}}" \ - --host="{{.IP}}" \ + --host="{{.Host}}" \ --port="{{.Port}}" \ --pd="{{.PD}}" \ --data-dir="{{.DataDir}}" \ diff --git a/embed/templates/scripts/run_tiproxy.sh.tpl b/embed/templates/scripts/run_tiproxy.sh.tpl new file mode 100644 index 0000000000..83362c9121 --- /dev/null +++ b/embed/templates/scripts/run_tiproxy.sh.tpl @@ -0,0 +1,14 @@ +#!/bin/bash +set -e + +# WARNING: This file was auto-generated. Do not edit! +# All your edit might be overwritten! +DEPLOY_DIR={{.DeployDir}} +cd "${DEPLOY_DIR}" || exit 1 + +{{- if .NumaNode}} +exec numactl --cpunodebind={{.NumaNode}} --membind={{.NumaNode}} bin/tiproxy \ +{{- else}} +exec bin/tiproxy \ +{{- end}} + --config conf/tiproxy.toml diff --git a/go.mod b/go.mod index edbb7e3464..7e31a583a6 100644 --- a/go.mod +++ b/go.mod @@ -1,68 +1,67 @@ module github.com/pingcap/tiup -go 1.18 +go 1.21 replace gopkg.in/yaml.v2 => github.com/july2993/yaml v0.0.0-20200423062752-adcfa5abe2ed require ( github.com/AstroProfundis/sysinfo v0.0.0-20220902033416-231991f6df3c github.com/AstroProfundis/tabby v1.1.1-color - github.com/BurntSushi/toml v1.2.0 - github.com/ScaleFT/sshkeys v0.0.0-20200327173127-6142f742bca5 + github.com/BurntSushi/toml v1.2.1 + github.com/ScaleFT/sshkeys v1.2.0 github.com/alecthomas/assert v1.0.0 github.com/appleboy/easyssh-proxy v1.3.10-0.20211209134747-6671f69d85f5 github.com/asaskevich/EventBus v0.0.0-20200907212545-49d423059eef - github.com/blevesearch/bleve v1.0.14 github.com/cavaliergopher/grab/v3 v3.0.1 - github.com/cheggaaa/pb/v3 v3.1.0 - github.com/creasty/defaults v1.6.0 + github.com/cheggaaa/pb/v3 v3.1.2 + github.com/creasty/defaults v1.7.0 github.com/docker/go-units v0.5.0 - github.com/fatih/color v1.13.0 + github.com/fatih/color v1.15.0 github.com/gibson042/canonicaljson-go v1.0.3 - github.com/go-sql-driver/mysql v1.6.0 + github.com/go-sql-driver/mysql v1.7.1 github.com/gofrs/flock v0.8.1 github.com/gogo/protobuf v1.3.2 - github.com/golang/protobuf v1.5.2 + github.com/golang/protobuf v1.5.3 github.com/google/uuid v1.3.0 github.com/gorilla/mux v1.8.0 github.com/grpc-ecosystem/grpc-gateway v1.16.0 github.com/jeremywohl/flatten v1.0.1 github.com/joomcode/errorx v1.1.0 github.com/juju/ansiterm v1.0.0 - github.com/mattn/go-runewidth v0.0.13 - github.com/otiai10/copy v1.7.0 + github.com/mattn/go-runewidth v0.0.14 + github.com/minio/minio-go/v7 v7.0.52 + github.com/otiai10/copy v1.9.0 + github.com/pelletier/go-toml v1.9.5 github.com/pingcap/check v0.0.0-20211026125417-57bd13f7b5f0 github.com/pingcap/errors v0.11.5-0.20201126102027-b0a155152ca3 github.com/pingcap/failpoint v0.0.0-20220801062533-2eaa32854a6c github.com/pingcap/fn v1.0.0 - github.com/pingcap/kvproto v0.0.0-20220913050750-f6d05706948a + github.com/pingcap/kvproto v0.0.0-20230331024443-349815129e6d github.com/pingcap/tidb-insight/collector v0.0.0-20220902034607-fb5ae0ddc8c1 github.com/pkg/errors v0.9.1 - github.com/prometheus/client_model v0.2.0 - github.com/prometheus/common v0.37.0 - github.com/prometheus/prom2json v1.3.1 + github.com/prometheus/client_model v0.3.0 + github.com/prometheus/common v0.42.0 + github.com/prometheus/prom2json v1.3.2 github.com/r3labs/diff/v2 v2.15.1 github.com/relex/aini v1.5.0 - github.com/sergi/go-diff v1.2.0 + github.com/sergi/go-diff v1.3.1 github.com/sethvargo/go-password v0.2.0 github.com/shirou/gopsutil v3.21.11+incompatible - github.com/skratchdot/open-golang v0.0.0-20200116055534-eef842397966 - github.com/spf13/cobra v1.5.0 + github.com/spf13/cobra v1.6.1 github.com/spf13/pflag v1.0.5 - github.com/stretchr/testify v1.8.0 - github.com/tj/go-termd v0.0.1 - go.etcd.io/etcd/client/pkg/v3 v3.5.4 - go.etcd.io/etcd/client/v3 v3.5.4 + github.com/stretchr/testify v1.8.2 + go.etcd.io/etcd/client/pkg/v3 v3.5.7 + go.etcd.io/etcd/client/v3 v3.5.7 go.uber.org/atomic v1.10.0 - go.uber.org/zap v1.23.0 - golang.org/x/crypto v0.0.0-20220829220503-c86fa9a7ed90 - golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4 - golang.org/x/sync v0.0.0-20220819030929-7fc1605a5dde - golang.org/x/sys v0.0.0-20220829200755-d48e67d00261 - golang.org/x/term v0.0.0-20220722155259-a9ba230a4035 - golang.org/x/text v0.3.7 - google.golang.org/genproto v0.0.0-20220829175752-36a9c930ecbf - google.golang.org/grpc v1.49.0 + go.uber.org/zap v1.24.0 + golang.org/x/crypto v0.7.0 + golang.org/x/mod v0.9.0 + golang.org/x/sync v0.1.0 + golang.org/x/sys v0.6.0 + golang.org/x/term v0.6.0 + golang.org/x/text v0.8.0 + google.golang.org/genproto v0.0.0-20230330200707-38013875ee22 + google.golang.org/grpc v1.54.0 gopkg.in/ini.v1 v1.67.0 gopkg.in/yaml.v2 v2.4.0 gopkg.in/yaml.v3 v3.0.1 @@ -70,73 +69,51 @@ require ( ) require ( - github.com/RoaringBitmap/roaring v0.4.23 // indirect github.com/VividCortex/ewma v1.2.0 // indirect - github.com/alecthomas/chroma v0.6.8 // indirect github.com/alecthomas/colour v0.1.0 // indirect - github.com/alecthomas/repr v0.0.0-20210801044451-80ca428c5142 // indirect - github.com/aybabtme/rgbterm v0.0.0-20170906152045-cc83f3b3ce59 // indirect + github.com/alecthomas/repr v0.2.0 // indirect github.com/benbjohnson/clock v1.3.0 // indirect - github.com/blevesearch/go-porterstemmer v1.0.3 // indirect - github.com/blevesearch/mmap-go v1.0.2 // indirect - github.com/blevesearch/segment v0.9.0 // indirect - github.com/blevesearch/snowballstem v0.9.0 // indirect - github.com/blevesearch/zap/v11 v11.0.14 // indirect - github.com/blevesearch/zap/v12 v12.0.14 // indirect - github.com/blevesearch/zap/v13 v13.0.6 // indirect - github.com/blevesearch/zap/v14 v14.0.5 // indirect - github.com/blevesearch/zap/v15 v15.0.3 // indirect - github.com/coreos/go-semver v0.3.0 // indirect - github.com/coreos/go-systemd/v22 v22.3.2 // indirect - github.com/couchbase/vellum v1.0.2 // indirect - github.com/danwakefield/fnmatch v0.0.0-20160403171240-cbb64ac3d964 // indirect + github.com/coreos/go-semver v0.3.1 // indirect + github.com/coreos/go-systemd/v22 v22.5.0 // indirect github.com/davecgh/go-spew v1.1.1 // indirect github.com/dchest/bcrypt_pbkdf v0.0.0-20150205184540-83f37f9c154a // indirect - github.com/dlclark/regexp2 v1.1.6 // indirect - github.com/fsnotify/fsnotify v1.5.1 // indirect - github.com/glycerine/go-unsnap-stream v0.0.0-20181221182339-f9677308dec2 // indirect + github.com/dustin/go-humanize v1.0.1 // indirect github.com/go-ole/go-ole v1.2.6 // indirect - github.com/golang/snappy v0.0.4 // indirect github.com/google/shlex v0.0.0-20191202100458-e7afc7fbc510 // indirect - github.com/inconshreveable/mousetrap v1.0.0 // indirect - github.com/kr/pretty v0.3.0 // indirect - github.com/kr/text v0.2.0 // indirect + github.com/inconshreveable/mousetrap v1.1.0 // indirect + github.com/json-iterator/go v1.1.12 // indirect + github.com/klauspost/compress v1.16.0 // indirect + github.com/klauspost/cpuid/v2 v2.2.4 // indirect + github.com/kr/pretty v0.3.1 // indirect github.com/lorenzosaino/go-sysctl v0.3.1 // indirect github.com/lunixbochs/vtclean v1.0.0 // indirect - github.com/mattn/go-colorable v0.1.12 // indirect - github.com/mattn/go-isatty v0.0.14 // indirect - github.com/matttproud/golang_protobuf_extensions v1.0.1 // indirect - github.com/mitchellh/go-wordwrap v1.0.0 // indirect - github.com/mschoch/smat v0.2.0 // indirect - github.com/onsi/ginkgo v1.16.5 // indirect - github.com/onsi/gomega v1.18.1 // indirect - github.com/philhofer/fwd v1.1.1 // indirect + github.com/mattn/go-colorable v0.1.13 // indirect + github.com/mattn/go-isatty v0.0.18 // indirect + github.com/matttproud/golang_protobuf_extensions v1.0.4 // indirect + github.com/minio/md5-simd v1.1.2 // indirect + github.com/minio/sha256-simd v1.0.0 // indirect + github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect + github.com/modern-go/reflect2 v1.0.2 // indirect github.com/pingcap/log v1.1.0 // indirect github.com/pmezard/go-difflib v1.0.0 // indirect - github.com/rivo/uniseg v0.2.0 // indirect - github.com/russross/blackfriday v2.0.0+incompatible // indirect - github.com/shurcooL/sanitized_anchor_name v1.0.0 // indirect - github.com/steveyen/gtreap v0.1.0 // indirect - github.com/syndtr/goleveldb v1.0.1-0.20210305035536-64b5b1c73954 // indirect - github.com/tinylib/msgp v1.1.6 // indirect - github.com/tj/go-css v0.0.0-20191108133013-220a796d1705 // indirect - github.com/tklauser/go-sysconf v0.3.10 // indirect - github.com/tklauser/numcpus v0.5.0 // indirect + github.com/rivo/uniseg v0.4.4 // indirect + github.com/rs/xid v1.4.0 // indirect + github.com/sirupsen/logrus v1.9.0 // indirect + github.com/tklauser/go-sysconf v0.3.11 // indirect + github.com/tklauser/numcpus v0.6.0 // indirect github.com/vishvananda/netlink v0.0.0-20210530105856-14e832ae1e8f // indirect - github.com/vishvananda/netns v0.0.0-20200728191858-db3c7e526aae // indirect + github.com/vishvananda/netns v0.0.4 // indirect github.com/vmihailenco/msgpack v4.0.4+incompatible // indirect - github.com/willf/bitset v1.1.10 // indirect github.com/yusufpapurcu/wmi v1.2.2 // indirect - go.etcd.io/bbolt v1.3.6 // indirect - go.etcd.io/etcd/api/v3 v3.5.4 // indirect - go.uber.org/goleak v1.1.12 // indirect - go.uber.org/multierr v1.8.0 // indirect - golang.org/x/exp/typeparams v0.0.0-20220613132600-b0d781184e0d // indirect + go.etcd.io/etcd/api/v3 v3.5.7 // indirect + go.uber.org/goleak v1.2.1 // indirect + go.uber.org/multierr v1.11.0 // indirect + golang.org/x/exp/typeparams v0.0.0-20230321023759-10a507213a29 // indirect golang.org/x/lint v0.0.0-20210508222113-6edffad5e616 // indirect - golang.org/x/net v0.0.0-20220722155237-a158d28d115b // indirect - golang.org/x/tools v0.1.12 // indirect + golang.org/x/net v0.8.0 // indirect + golang.org/x/tools v0.7.0 // indirect google.golang.org/appengine v1.6.7 // indirect - google.golang.org/protobuf v1.28.1 // indirect - gopkg.in/natefinch/lumberjack.v2 v2.0.0 // indirect - honnef.co/go/tools v0.3.2 // indirect + google.golang.org/protobuf v1.30.0 // indirect + gopkg.in/natefinch/lumberjack.v2 v2.2.1 // indirect + honnef.co/go/tools v0.4.3 // indirect ) diff --git a/go.sum b/go.sum index b0d07b0745..a12f529e22 100644 --- a/go.sum +++ b/go.sum @@ -1,36 +1,5 @@ cloud.google.com/go v0.26.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw= cloud.google.com/go v0.34.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw= -cloud.google.com/go v0.38.0/go.mod h1:990N+gfupTy94rShfmMCWGDn0LpTmnzTp2qbd1dvSRU= -cloud.google.com/go v0.44.1/go.mod h1:iSa0KzasP4Uvy3f1mN/7PiObzGgflwredwwASm/v6AU= -cloud.google.com/go v0.44.2/go.mod h1:60680Gw3Yr4ikxnPRS/oxxkBccT6SA1yMk63TGekxKY= -cloud.google.com/go v0.45.1/go.mod h1:RpBamKRgapWJb87xiFSdk4g1CME7QZg3uwTez+TSTjc= -cloud.google.com/go v0.46.3/go.mod h1:a6bKKbmY7er1mI7TEI4lsAkts/mkhTSZK8w33B4RAg0= -cloud.google.com/go v0.50.0/go.mod h1:r9sluTvynVuxRIOHXQEHMFffphuXHOMZMycpNR5e6To= -cloud.google.com/go v0.52.0/go.mod h1:pXajvRH/6o3+F9jDHZWQ5PbGhn+o8w9qiu/CffaVdO4= -cloud.google.com/go v0.53.0/go.mod h1:fp/UouUEsRkN6ryDKNW/Upv/JBKnv6WDthjR6+vze6M= -cloud.google.com/go v0.54.0/go.mod h1:1rq2OEkV3YMf6n/9ZvGWI3GWw0VoqH/1x2nd8Is/bPc= -cloud.google.com/go v0.56.0/go.mod h1:jr7tqZxxKOVYizybht9+26Z/gUq7tiRzu+ACVAMbKVk= -cloud.google.com/go v0.57.0/go.mod h1:oXiQ6Rzq3RAkkY7N6t3TcE6jE+CIBBbA36lwQ1JyzZs= -cloud.google.com/go v0.62.0/go.mod h1:jmCYTdRCQuc1PHIIJ/maLInMho30T/Y0M4hTdTShOYc= -cloud.google.com/go v0.65.0/go.mod h1:O5N8zS7uWy9vkA9vayVHs65eM1ubvY4h553ofrNHObY= -cloud.google.com/go/bigquery v1.0.1/go.mod h1:i/xbL2UlR5RvWAURpBYZTtm/cXjCha9lbfbpx4poX+o= -cloud.google.com/go/bigquery v1.3.0/go.mod h1:PjpwJnslEMmckchkHFfq+HTD2DmtT67aNFKH1/VBDHE= -cloud.google.com/go/bigquery v1.4.0/go.mod h1:S8dzgnTigyfTmLBfrtrhyYhwRxG72rYxvftPBK2Dvzc= -cloud.google.com/go/bigquery v1.5.0/go.mod h1:snEHRnqQbz117VIFhE8bmtwIDY80NLUZUMb4Nv6dBIg= -cloud.google.com/go/bigquery v1.7.0/go.mod h1://okPTzCYNXSlb24MZs83e2Do+h+VXtc4gLoIoXIAPc= -cloud.google.com/go/bigquery v1.8.0/go.mod h1:J5hqkt3O0uAFnINi6JXValWIb1v0goeZM77hZzJN/fQ= -cloud.google.com/go/datastore v1.0.0/go.mod h1:LXYbyblFSglQ5pkeyhO+Qmw7ukd3C+pD7TKLgZqpHYE= -cloud.google.com/go/datastore v1.1.0/go.mod h1:umbIZjpQpHh4hmRpGhH4tLFup+FVzqBi1b3c64qFpCk= -cloud.google.com/go/pubsub v1.0.1/go.mod h1:R0Gpsv3s54REJCy4fxDixWD93lHJMoZTyQ2kNxGRt3I= -cloud.google.com/go/pubsub v1.1.0/go.mod h1:EwwdRX2sKPjnvnqCa270oGRyludottCI76h+R3AArQw= -cloud.google.com/go/pubsub v1.2.0/go.mod h1:jhfEVHT8odbXTkndysNHCcx0awwzvfOlguIAii9o8iA= -cloud.google.com/go/pubsub v1.3.1/go.mod h1:i+ucay31+CNRpDW4Lu78I4xXG+O1r/MAHgjpRVR+TSU= -cloud.google.com/go/storage v1.0.0/go.mod h1:IhtSnM/ZTZV8YYJWCY8RULGVqBDmpoyjwiyrjsg+URw= -cloud.google.com/go/storage v1.5.0/go.mod h1:tpKbwo567HUNpVclU5sGELwQWBDZ8gh0ZeosJ0Rtdos= -cloud.google.com/go/storage v1.6.0/go.mod h1:N7U0C8pVQ/+NIKOBQyamJIeKQKkZ+mxpohlUTyfDhBk= -cloud.google.com/go/storage v1.8.0/go.mod h1:Wv1Oy7z6Yz3DshWRJFhqM/UCfaWIRTdp0RXyy7KQOVs= -cloud.google.com/go/storage v1.10.0/go.mod h1:FLPqc6j+Ki4BU591ie1oL6qBQGu2Bl/tZ9ullr3+Kg0= -dmitri.shuralyov.com/gpu/mtl v0.0.0-20190408044501-666a987793e9/go.mod h1:H6x//7gZCb22OMCxBHrMx7a5I7Hp++hsVxbQ4BYO7hU= github.com/AstroProfundis/sysinfo v0.0.0-20220902033416-231991f6df3c h1:Uf603egKJU8kRS5SsL8tBuBQb/t/qmKio06pwTeRH24= github.com/AstroProfundis/sysinfo v0.0.0-20220902033416-231991f6df3c/go.mod h1:Zc/DBosH2jLHhf/5Ux6sifoneJ1JSvtVCRExJ84A/q8= github.com/AstroProfundis/tabby v1.1.1-color h1:gi/Fs4W2KVcC1m29lqa39tDnsGfZG/PYvZmwOhAH5m0= @@ -38,284 +7,121 @@ github.com/AstroProfundis/tabby v1.1.1-color/go.mod h1:Wcm+uinH1saEOFGLK2LdY37lA github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= github.com/BurntSushi/toml v0.4.1/go.mod h1:CxXYINrC8qIiEnFrOxCa7Jy5BFHlXnUU2pbicEuybxQ= github.com/BurntSushi/toml v1.1.0/go.mod h1:CxXYINrC8qIiEnFrOxCa7Jy5BFHlXnUU2pbicEuybxQ= -github.com/BurntSushi/toml v1.2.0 h1:Rt8g24XnyGTyglgET/PRUNlrUeu9F5L+7FilkXfZgs0= -github.com/BurntSushi/toml v1.2.0/go.mod h1:CxXYINrC8qIiEnFrOxCa7Jy5BFHlXnUU2pbicEuybxQ= -github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802/go.mod h1:IVnqGOEym/WlBOVXweHU+Q+/VP0lqqI8lqeDx9IjBqo= -github.com/GeertJohan/go.incremental v1.0.0/go.mod h1:6fAjUhbVuX1KcMD3c8TEgVUqmo4seqhv0i0kdATSkM0= -github.com/GeertJohan/go.rice v1.0.0/go.mod h1:eH6gbSOAUv07dQuZVnBmoDP8mgsM1rtixis4Tib9if0= -github.com/RoaringBitmap/roaring v0.4.23 h1:gpyfd12QohbqhFO4NVDUdoPOCXsyahYRQhINmlHxKeo= -github.com/RoaringBitmap/roaring v0.4.23/go.mod h1:D0gp8kJQgE1A4LQ5wFLggQEyvDi06Mq5mKs52e1TwOo= -github.com/ScaleFT/sshkeys v0.0.0-20200327173127-6142f742bca5 h1:VauE2GcJNZFun2Och6tIT2zJZK1v6jxALQDA9BIji/E= +github.com/BurntSushi/toml v1.2.1 h1:9F2/+DoOYIOksmaJFPw1tGFy1eDnIJXg+UHjuD8lTak= +github.com/BurntSushi/toml v1.2.1/go.mod h1:CxXYINrC8qIiEnFrOxCa7Jy5BFHlXnUU2pbicEuybxQ= github.com/ScaleFT/sshkeys v0.0.0-20200327173127-6142f742bca5/go.mod h1:gxOHeajFfvGQh/fxlC8oOKBe23xnnJTif00IFFbiT+o= -github.com/VividCortex/ewma v1.1.1/go.mod h1:2Tkkvm3sRDVXaiyucHiACn4cqf7DpdyLvmxzcbUokwA= +github.com/ScaleFT/sshkeys v1.2.0 h1:5BRp6rTVIhJzXT3VcUQrKgXR8zWA3sOsNeuyW15WUA8= +github.com/ScaleFT/sshkeys v1.2.0/go.mod h1:gxOHeajFfvGQh/fxlC8oOKBe23xnnJTif00IFFbiT+o= github.com/VividCortex/ewma v1.2.0 h1:f58SaIzcDXrSy3kWaHNvuJgJ3Nmz59Zji6XoJR/q1ow= github.com/VividCortex/ewma v1.2.0/go.mod h1:nz4BbCtbLyFDeC9SUHbtcT5644juEuWfUAUnGx7j5l4= -github.com/akavel/rsrc v0.8.0/go.mod h1:uLoCtb9J+EyAqh+26kdrTgmzRBFPGOolLWKpdxkKq+c= -github.com/alecthomas/assert v0.0.0-20170929043011-405dbfeb8e38/go.mod h1:r7bzyVFMNntcxPZXK3/+KdruV1H5KSlyVY0gc+NgInI= github.com/alecthomas/assert v1.0.0 h1:3XmGh/PSuLzDbK3W2gUbRXwgW5lqPkuqvRgeQ30FI5o= github.com/alecthomas/assert v1.0.0/go.mod h1:va/d2JC+M7F6s+80kl/R3G7FUiW6JzUO+hPhLyJ36ZY= -github.com/alecthomas/chroma v0.6.8 h1:TW4JJaIdbAbMyUtGEd6BukFlOKYvVQz3vVhLBEUNwMU= -github.com/alecthomas/chroma v0.6.8/go.mod h1:o9ohftueRi7H5be3+Q2cQCNa/YnLBFUNx40ZJfGVFKA= -github.com/alecthomas/colour v0.0.0-20160524082231-60882d9e2721/go.mod h1:QO9JBoKquHd+jz9nshCh40fOfO+JzsoXy8qTHF68zU0= github.com/alecthomas/colour v0.1.0 h1:nOE9rJm6dsZ66RGWYSFrXw461ZIt9A6+nHgL7FRrDUk= github.com/alecthomas/colour v0.1.0/go.mod h1:QO9JBoKquHd+jz9nshCh40fOfO+JzsoXy8qTHF68zU0= -github.com/alecthomas/kong v0.1.17-0.20190424132513-439c674f7ae0/go.mod h1:+inYUSluD+p4L8KdviBSgzcqEjUQOfC5fQDRFuc36lI= -github.com/alecthomas/kong v0.2.1-0.20190708041108-0548c6b1afae/go.mod h1:+inYUSluD+p4L8KdviBSgzcqEjUQOfC5fQDRFuc36lI= -github.com/alecthomas/kong-hcl v0.1.8-0.20190615233001-b21fea9723c8/go.mod h1:MRgZdU3vrFd05IQ89AxUZ0aYdF39BYoNFa324SodPCA= -github.com/alecthomas/repr v0.0.0-20180818092828-117648cd9897/go.mod h1:xTS7Pm1pD1mvyM075QCDSRqH6qRLXylzS24ZTpRiSzQ= -github.com/alecthomas/repr v0.0.0-20210801044451-80ca428c5142 h1:8Uy0oSf5co/NZXje7U1z8Mpep++QJOldL2hs/sBQf48= -github.com/alecthomas/repr v0.0.0-20210801044451-80ca428c5142/go.mod h1:2kn6fqh/zIyPLmm3ugklbEi5hg5wS435eygvNfaDQL8= -github.com/alecthomas/template v0.0.0-20160405071501-a0175ee3bccc/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc= -github.com/alecthomas/template v0.0.0-20190718012654-fb15b899a751/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc= -github.com/alecthomas/units v0.0.0-20151022065526-2efee857e7cf/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0= -github.com/alecthomas/units v0.0.0-20190717042225-c3de453c63f4/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0= -github.com/alecthomas/units v0.0.0-20190924025748-f65c72e2690d/go.mod h1:rBZYJk541a8SKzHPHnH3zbiI+7dagKZ0cgpgrD7Fyho= -github.com/antihax/optional v0.0.0-20180407024304-ca021399b1a6/go.mod h1:V8iCPQYkqmusNa815XgQio277wI47sdRh1dUOLdyC6Q= +github.com/alecthomas/repr v0.2.0 h1:HAzS41CIzNW5syS8Mf9UwXhNH1J9aix/BvDRf1Ml2Yk= +github.com/alecthomas/repr v0.2.0/go.mod h1:Fr0507jx4eOXV7AlPV6AVZLYrLIuIeSOWtW57eE/O/4= github.com/antihax/optional v1.0.0/go.mod h1:uupD/76wgC+ih3iEmQUL+0Ugr19nfwCT1kdvxnR2qWY= github.com/appleboy/easyssh-proxy v1.3.10-0.20211209134747-6671f69d85f5 h1:4YNuL/4gurMQu1r10vp9kuD3X2z0Nn4VayQpmIjoJlY= github.com/appleboy/easyssh-proxy v1.3.10-0.20211209134747-6671f69d85f5/go.mod h1:SnOlkuIAoaj8FvRIGcWp2BbLOwvHZnNNOva+DBMbHWU= -github.com/armon/consul-api v0.0.0-20180202201655-eb2c6b5be1b6/go.mod h1:grANhF5doyWs3UAsr3K4I6qtAmlQcZDesFNEHPZAzj8= github.com/asaskevich/EventBus v0.0.0-20200907212545-49d423059eef h1:2JGTg6JapxP9/R33ZaagQtAM4EkkSYnIAlOG5EI8gkM= github.com/asaskevich/EventBus v0.0.0-20200907212545-49d423059eef/go.mod h1:JS7hed4L1fj0hXcyEejnW57/7LCetXggd+vwrRnYeII= -github.com/aybabtme/rgbterm v0.0.0-20170906152045-cc83f3b3ce59 h1:WWB576BN5zNSZc/M9d/10pqEx5VHNhaQ/yOVAkmj5Yo= -github.com/aybabtme/rgbterm v0.0.0-20170906152045-cc83f3b3ce59/go.mod h1:q/89r3U2H7sSsE2t6Kca0lfwTK8JdoNGS/yzM/4iH5I= github.com/benbjohnson/clock v1.1.0/go.mod h1:J11/hYXuz8f4ySSvYwY0FKfm+ezbsZBKZxNJlLklBHA= github.com/benbjohnson/clock v1.3.0 h1:ip6w0uFQkncKQ979AypyG0ER7mqUSBdKLOgAle/AT8A= github.com/benbjohnson/clock v1.3.0/go.mod h1:J11/hYXuz8f4ySSvYwY0FKfm+ezbsZBKZxNJlLklBHA= -github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973/go.mod h1:Dwedo/Wpr24TaqPxmxbtue+5NUziq4I4S80YR8gNf3Q= -github.com/beorn7/perks v1.0.0/go.mod h1:KWe93zE9D1o94FZ5RNwFwVgaQK1VOXiVxmqh+CedLV8= -github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw= -github.com/blevesearch/bleve v1.0.14 h1:Q8r+fHTt35jtGXJUM0ULwM3Tzg+MRfyai4ZkWDy2xO4= -github.com/blevesearch/bleve v1.0.14/go.mod h1:e/LJTr+E7EaoVdkQZTfoz7dt4KoDNvDbLb8MSKuNTLQ= -github.com/blevesearch/blevex v1.0.0 h1:pnilj2Qi3YSEGdWgLj1Pn9Io7ukfXPoQcpAI1Bv8n/o= -github.com/blevesearch/blevex v1.0.0/go.mod h1:2rNVqoG2BZI8t1/P1awgTKnGlx5MP9ZbtEciQaNhswc= -github.com/blevesearch/cld2 v0.0.0-20200327141045-8b5f551d37f5/go.mod h1:PN0QNTLs9+j1bKy3d/GB/59wsNBFC4sWLWG3k69lWbc= -github.com/blevesearch/go-porterstemmer v1.0.3 h1:GtmsqID0aZdCSNiY8SkuPJ12pD4jI+DdXTAn4YRcHCo= -github.com/blevesearch/go-porterstemmer v1.0.3/go.mod h1:angGc5Ht+k2xhJdZi511LtmxuEf0OVpvUUNrwmM1P7M= -github.com/blevesearch/mmap-go v1.0.2 h1:JtMHb+FgQCTTYIhtMvimw15dJwu1Y5lrZDMOFXVWPk0= -github.com/blevesearch/mmap-go v1.0.2/go.mod h1:ol2qBqYaOUsGdm7aRMRrYGgPvnwLe6Y+7LMvAB5IbSA= -github.com/blevesearch/segment v0.9.0 h1:5lG7yBCx98or7gK2cHMKPukPZ/31Kag7nONpoBt22Ac= -github.com/blevesearch/segment v0.9.0/go.mod h1:9PfHYUdQCgHktBgvtUOF4x+pc4/l8rdH0u5spnW85UQ= -github.com/blevesearch/snowballstem v0.9.0 h1:lMQ189YspGP6sXvZQ4WZ+MLawfV8wOmPoD/iWeNXm8s= -github.com/blevesearch/snowballstem v0.9.0/go.mod h1:PivSj3JMc8WuaFkTSRDW2SlrulNWPl4ABg1tC/hlgLs= -github.com/blevesearch/zap/v11 v11.0.14 h1:IrDAvtlzDylh6H2QCmS0OGcN9Hpf6mISJlfKjcwJs7k= -github.com/blevesearch/zap/v11 v11.0.14/go.mod h1:MUEZh6VHGXv1PKx3WnCbdP404LGG2IZVa/L66pyFwnY= -github.com/blevesearch/zap/v12 v12.0.14 h1:2o9iRtl1xaRjsJ1xcqTyLX414qPAwykHNV7wNVmbp3w= -github.com/blevesearch/zap/v12 v12.0.14/go.mod h1:rOnuZOiMKPQj18AEKEHJxuI14236tTQ1ZJz4PAnWlUg= -github.com/blevesearch/zap/v13 v13.0.6 h1:r+VNSVImi9cBhTNNR+Kfl5uiGy8kIbb0JMz/h8r6+O4= -github.com/blevesearch/zap/v13 v13.0.6/go.mod h1:L89gsjdRKGyGrRN6nCpIScCvvkyxvmeDCwZRcjjPCrw= -github.com/blevesearch/zap/v14 v14.0.5 h1:NdcT+81Nvmp2zL+NhwSvGSLh7xNgGL8QRVZ67njR0NU= -github.com/blevesearch/zap/v14 v14.0.5/go.mod h1:bWe8S7tRrSBTIaZ6cLRbgNH4TUDaC9LZSpRGs85AsGY= -github.com/blevesearch/zap/v15 v15.0.3 h1:Ylj8Oe+mo0P25tr9iLPp33lN6d4qcztGjaIsP51UxaY= -github.com/blevesearch/zap/v15 v15.0.3/go.mod h1:iuwQrImsh1WjWJ0Ue2kBqY83a0rFtJTqfa9fp1rbVVU= github.com/cavaliergopher/grab/v3 v3.0.1 h1:4z7TkBfmPjmLAAmkkAZNX/6QJ1nNFdv3SdIHXju0Fr4= github.com/cavaliergopher/grab/v3 v3.0.1/go.mod h1:1U/KNnD+Ft6JJiYoYBAimKH2XrYptb8Kl3DFGmsjpq4= github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU= -github.com/cespare/xxhash/v2 v2.1.1/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= -github.com/cespare/xxhash/v2 v2.1.2/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= -github.com/cheggaaa/pb/v3 v3.1.0 h1:3uouEsl32RL7gTiQsuaXD4Bzbfl5tGztXGUvXbs4O04= -github.com/cheggaaa/pb/v3 v3.1.0/go.mod h1:YjrevcBqadFDaGQKRdmZxTY42pXEqda48Ea3lt0K/BE= -github.com/chzyer/logex v1.1.10/go.mod h1:+Ywpsq7O8HXn0nuIou7OrIPyXbp3wmkHB+jjWRnGsAI= -github.com/chzyer/readline v0.0.0-20180603132655-2972be24d48e/go.mod h1:nSuG5e5PlCu98SY8svDHJxuZscDgtXS6KTTbou5AhLI= -github.com/chzyer/test v0.0.0-20180213035817-a1ea475d72b1/go.mod h1:Q3SI9o4m/ZMnBNeIyt5eFwwo7qiLfzFZmjNmxjkiQlU= +github.com/cheggaaa/pb/v3 v3.1.2 h1:FIxT3ZjOj9XJl0U4o2XbEhjFfZl7jCVCDOGq1ZAB7wQ= +github.com/cheggaaa/pb/v3 v3.1.2/go.mod h1:SNjnd0yKcW+kw0brSusraeDd5Bf1zBfxAzTL2ss3yQ4= github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw= github.com/cncf/udpa/go v0.0.0-20191209042840-269d4d468f6f/go.mod h1:M8M6+tZqaGXZJjfX53e64911xZQV5JYwmTeXPW+k8Sc= -github.com/cncf/udpa/go v0.0.0-20201120205902-5459f2c99403/go.mod h1:WmhPx2Nbnhtbo57+VJT5O0JRkEi1Wbu0z5j0R8u5Hbk= -github.com/coreos/etcd v3.3.10+incompatible/go.mod h1:uF7uidLiAD3TWHmW31ZFd/JWoc32PjwdhPthX9715RE= -github.com/coreos/go-etcd v2.0.0+incompatible/go.mod h1:Jez6KQU2B/sWsbdaef3ED8NzMklzPG4d5KIOhIy30Tk= -github.com/coreos/go-semver v0.2.0/go.mod h1:nnelYz7RCh+5ahJtPPxZlU+153eP4D4r3EedlOD2RNk= -github.com/coreos/go-semver v0.3.0 h1:wkHLiw0WNATZnSG7epLsujiMCgPAc9xhjJ4tgnAxmfM= -github.com/coreos/go-semver v0.3.0/go.mod h1:nnelYz7RCh+5ahJtPPxZlU+153eP4D4r3EedlOD2RNk= -github.com/coreos/go-systemd/v22 v22.3.2 h1:D9/bQk5vlXQFZ6Kwuu6zaiXJ9oTPe68++AzAJc1DzSI= -github.com/coreos/go-systemd/v22 v22.3.2/go.mod h1:Y58oyj3AT4RCenI/lSvhwexgC+NSVTIJ3seZv2GcEnc= -github.com/couchbase/ghistogram v0.1.0/go.mod h1:s1Jhy76zqfEecpNWJfWUiKZookAFaiGOEoyzgHt9i7k= -github.com/couchbase/moss v0.1.0/go.mod h1:9MaHIaRuy9pvLPUJxB8sh8OrLfyDczECVL37grCIubs= -github.com/couchbase/vellum v1.0.2 h1:BrbP0NKiyDdndMPec8Jjhy0U47CZ0Lgx3xUC2r9rZqw= -github.com/couchbase/vellum v1.0.2/go.mod h1:FcwrEivFpNi24R3jLOs3n+fs5RnuQnQqCLBJ1uAg1W4= -github.com/cpuguy83/go-md2man v1.0.10/go.mod h1:SmD6nW6nTyfqj6ABTjUi3V3JVMnlJmwcJI5acqYI6dE= +github.com/coreos/go-semver v0.3.1 h1:yi21YpKnrx1gt5R+la8n5WgS0kCrsPp33dmEyHReZr4= +github.com/coreos/go-semver v0.3.1/go.mod h1:irMmmIw/7yzSRPWryHsK7EYSg09caPQL03VsM8rvUec= +github.com/coreos/go-systemd/v22 v22.5.0 h1:RrqgGjYQKalulkV8NGVIfkXQf6YYmOyiJKk8iXXhfZs= +github.com/coreos/go-systemd/v22 v22.5.0/go.mod h1:Y58oyj3AT4RCenI/lSvhwexgC+NSVTIJ3seZv2GcEnc= github.com/cpuguy83/go-md2man/v2 v2.0.2/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o= github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E= -github.com/creasty/defaults v1.6.0 h1:ltuE9cfphUtlrBeomuu8PEyISTXnxqkBIoQfXgv7BSc= -github.com/creasty/defaults v1.6.0/go.mod h1:iGzKe6pbEHnpMPtfDXZEr0NVxWnPTjb1bbDy08fPzYM= -github.com/cznic/b v0.0.0-20181122101859-a26611c4d92d h1:SwD98825d6bdB+pEuTxWOXiSjBrHdOl/UVp75eI7JT8= -github.com/cznic/b v0.0.0-20181122101859-a26611c4d92d/go.mod h1:URriBxXwVq5ijiJ12C7iIZqlA69nTlI+LgI6/pwftG8= -github.com/cznic/mathutil v0.0.0-20181122101859-297441e03548/go.mod h1:e6NPNENfs9mPDVNRekM7lKScauxd5kXTr1Mfyig6TDM= -github.com/cznic/strutil v0.0.0-20181122101858-275e90344537/go.mod h1:AHHPPPXTw0h6pVabbcbyGRK1DckRn7r/STdZEeIDzZc= -github.com/daaku/go.zipexe v1.0.0/go.mod h1:z8IiR6TsVLEYKwXAoE/I+8ys/sDkgTzSL0CLnGVd57E= -github.com/danwakefield/fnmatch v0.0.0-20160403171240-cbb64ac3d964 h1:y5HC9v93H5EPKqaS1UYVg1uYah5Xf51mBfIoWehClUQ= -github.com/danwakefield/fnmatch v0.0.0-20160403171240-cbb64ac3d964/go.mod h1:Xd9hchkHSWYkEqJwUGisez3G1QY8Ryz0sdWrLPMGjLk= +github.com/creasty/defaults v1.7.0 h1:eNdqZvc5B509z18lD8yc212CAqJNvfT1Jq6L8WowdBA= +github.com/creasty/defaults v1.7.0/go.mod h1:iGzKe6pbEHnpMPtfDXZEr0NVxWnPTjb1bbDy08fPzYM= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/dchest/bcrypt_pbkdf v0.0.0-20150205184540-83f37f9c154a h1:saTgr5tMLFnmy/yg3qDTft4rE5DY2uJ/cCxCe3q0XTU= github.com/dchest/bcrypt_pbkdf v0.0.0-20150205184540-83f37f9c154a/go.mod h1:Bw9BbhOJVNR+t0jCqx2GC6zv0TGBsShs56Y3gfSCvl0= -github.com/dlclark/regexp2 v1.1.6 h1:CqB4MjHw0MFCDj+PHHjiESmHX+N7t0tJzKvC6M97BRg= -github.com/dlclark/regexp2 v1.1.6/go.mod h1:2pZnwuY/m+8K6iRw6wQdMtk+rH5tNGR1i55kozfMjCc= github.com/docker/go-units v0.5.0 h1:69rxXcBk27SvSaaxTtLh/8llcHD8vYHT7WSdRZ/jvr4= github.com/docker/go-units v0.5.0/go.mod h1:fgPhTUdO+D/Jk86RDLlptpiXQzgHJF7gydDDbaIK4Dk= -github.com/dustin/go-humanize v1.0.0/go.mod h1:HtrtbFcZ19U5GC7JDqmcUSB87Iq5E25KnS6fMYU6eOk= +github.com/dustin/go-humanize v1.0.1 h1:GzkhY7T5VNhEkwH0PVJgjz+fX1rhBrR7pRT3mDkpeCY= +github.com/dustin/go-humanize v1.0.1/go.mod h1:Mu1zIs6XwVuF/gI1OepvI0qD18qycQx+mFykh5fBlto= github.com/envoyproxy/go-control-plane v0.9.0/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= github.com/envoyproxy/go-control-plane v0.9.1-0.20191026205805-5f8ba28d4473/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= github.com/envoyproxy/go-control-plane v0.9.4/go.mod h1:6rpuAdCZL397s3pYoYcLgu1mIlRU8Am5FuJP05cCM98= -github.com/envoyproxy/go-control-plane v0.9.9-0.20210217033140-668b12f5399d/go.mod h1:cXg6YxExXjJnVBQHBLXeUAgxn2UodCpnH306RInaBQk= github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c= -github.com/facebookgo/ensure v0.0.0-20200202191622-63f1cf65ac4c/go.mod h1:Yg+htXGokKKdzcwhuNDwVvN+uBxDGXJ7G/VN1d8fa64= -github.com/facebookgo/stack v0.0.0-20160209184415-751773369052/go.mod h1:UbMTZqLaRiH3MsBH8va0n7s1pQYcu3uTb8G4tygF4Zg= -github.com/facebookgo/subset v0.0.0-20200203212716-c811ad88dec4/go.mod h1:5tD+neXqOorC30/tWg0LCSkrqj/AR6gu8yY8/fpw1q0= -github.com/fatih/color v1.10.0/go.mod h1:ELkj/draVOlAH/xkhN6mQ50Qd0MPOk5AAr3maGEBuJM= -github.com/fatih/color v1.13.0 h1:8LOYc1KYPPmyKMuN8QV2DNRWNbLo6LZ0iLs8+mlH53w= -github.com/fatih/color v1.13.0/go.mod h1:kLAiJbzzSOZDVNGyDpeOxJ47H46qBXwg5ILebYFFOfk= -github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo= -github.com/fsnotify/fsnotify v1.4.9/go.mod h1:znqG4EE+3YCdAaPaxE2ZRY/06pZUdp0tY4IgpuI1SZQ= -github.com/fsnotify/fsnotify v1.5.1 h1:mZcQUHVQUQWoPXXtuf9yuEXKudkV2sx1E06UadKWpgI= -github.com/fsnotify/fsnotify v1.5.1/go.mod h1:T3375wBYaZdLLcVNkcVbzGHY7f1l/uK5T5Ai1i3InKU= +github.com/fatih/color v1.15.0 h1:kOqh6YHBtK8aywxGerMG2Eq3H6Qgoqeo13Bk2Mv/nBs= +github.com/fatih/color v1.15.0/go.mod h1:0h5ZqXfHYED7Bhv2ZJamyIOUej9KtShiJESRwBDUSsw= github.com/ghodss/yaml v1.0.0/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04= github.com/gibson042/canonicaljson-go v1.0.3 h1:EAyF8L74AWabkyUmrvEFHEt/AGFQeD6RfwbAuf0j1bI= github.com/gibson042/canonicaljson-go v1.0.3/go.mod h1:DsLpJTThXyGNO+KZlI85C1/KDcImpP67k/RKVjcaEqo= -github.com/glycerine/go-unsnap-stream v0.0.0-20181221182339-f9677308dec2 h1:Ujru1hufTHVb++eG6OuNDKMxZnGIvF6o/u8q/8h2+I4= -github.com/glycerine/go-unsnap-stream v0.0.0-20181221182339-f9677308dec2/go.mod h1:/20jfyN9Y5QPEAprSgKAUr+glWDY39ZiUEAYOEv5dsE= -github.com/glycerine/goconvey v0.0.0-20190410193231-58a59202ab31 h1:gclg6gY70GLy3PbkQ1AERPfmLMMagS60DKF78eWwLn8= -github.com/glycerine/goconvey v0.0.0-20190410193231-58a59202ab31/go.mod h1:Ogl1Tioa0aV7gstGFO7KhffUsb9M4ydbEbbxpcEDc24= -github.com/go-gl/glfw v0.0.0-20190409004039-e6da0acd62b1/go.mod h1:vR7hzQXu2zJy9AVAgeJqvqgH9Q5CA+iKCZ2gyEVpxRU= -github.com/go-gl/glfw/v3.3/glfw v0.0.0-20191125211704-12ad95a8df72/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8= -github.com/go-gl/glfw/v3.3/glfw v0.0.0-20200222043503-6f7a984d4dc4/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8= -github.com/go-kit/kit v0.8.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as= -github.com/go-kit/kit v0.9.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as= -github.com/go-kit/log v0.1.0/go.mod h1:zbhenjAZHb184qTLMA9ZjW7ThYL0H2mk7Q6pNt4vbaY= -github.com/go-kit/log v0.2.0/go.mod h1:NwTd00d/i8cPZ3xOwwiv2PO5MOcx78fFErGNcVmBjv0= -github.com/go-logfmt/logfmt v0.3.0/go.mod h1:Qt1PoO58o5twSAckw1HlFXLmHsOX5/0LbT9GBnD5lWE= -github.com/go-logfmt/logfmt v0.4.0/go.mod h1:3RMwSq7FuexP4Kalkev3ejPJsZTpXXBr9+V4qmtdjCk= -github.com/go-logfmt/logfmt v0.5.0/go.mod h1:wCYkCAKZfumFQihp8CzCvQ3paCTfi41vtzG1KdI/P7A= -github.com/go-logfmt/logfmt v0.5.1/go.mod h1:WYhtIu8zTZfxdn5+rREduYbwxfcBr/Vr6KEVveWlfTs= github.com/go-ole/go-ole v1.2.6 h1:/Fpf6oFPoeFik9ty7siob0G6Ke8QvQEuVcuChpwXzpY= github.com/go-ole/go-ole v1.2.6/go.mod h1:pprOEPIfldk/42T2oK7lQ4v4JSDwmV0As9GaiUsvbm0= -github.com/go-sql-driver/mysql v1.6.0 h1:BCTh4TKNUYmOmMUcQ3IipzF5prigylS7XXjEkfCHuOE= -github.com/go-sql-driver/mysql v1.6.0/go.mod h1:DCzpHaOWr8IXmIStZouvnhqoel9Qv2LBy8hT2VhHyBg= -github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/melR3HDY= -github.com/go-task/slim-sprig v0.0.0-20210107165309-348f09dbbbc0/go.mod h1:fyg7847qk6SyHyPtNmDHnmrv/HOrqktSC+C9fM+CJOE= +github.com/go-sql-driver/mysql v1.7.1 h1:lUIinVbN1DY0xBg0eMOzmmtGoHwWBbvnWubQUrtU8EI= +github.com/go-sql-driver/mysql v1.7.1/go.mod h1:OXbVy3sEdcQ2Doequ6Z5BW6fXNQTmx+9S1MCJN5yJMI= github.com/godbus/dbus/v5 v5.0.4/go.mod h1:xhWf0FNVPg57R7Z0UbKHbJfkEywrmjJnf7w5xrFpKfA= github.com/gofrs/flock v0.8.1 h1:+gYjHKf32LDeiEEFhQaotPbLuUXjY5ZqxKgXy7n59aw= github.com/gofrs/flock v0.8.1/go.mod h1:F1TvTiK9OcQqauNUHlbJvyl9Qa1QvF/gOUDKA14jxHU= -github.com/gogo/protobuf v1.1.1/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ= github.com/gogo/protobuf v1.3.2 h1:Ov1cvc58UF3b5XjBnZv7+opcTcQFZebYjWzi34vdm4Q= github.com/gogo/protobuf v1.3.2/go.mod h1:P1XiOD3dCwIKUDQYPy72D8LYyHL2YPYrpS2s69NZV8Q= github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q= -github.com/golang/groupcache v0.0.0-20190702054246-869f871628b6/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= -github.com/golang/groupcache v0.0.0-20191227052852-215e87163ea7/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= -github.com/golang/groupcache v0.0.0-20200121045136-8c9f03a8e57e/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= github.com/golang/mock v1.1.1/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A= -github.com/golang/mock v1.2.0/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A= -github.com/golang/mock v1.3.1/go.mod h1:sBzyDLLjw3U8JLTeZvSv8jJB+tU5PVekmnlKIyFUx0Y= -github.com/golang/mock v1.4.0/go.mod h1:UOMv5ysSaYNkG+OFQykRIcU/QvvxJf3p21QfJ2Bt3cw= -github.com/golang/mock v1.4.1/go.mod h1:UOMv5ysSaYNkG+OFQykRIcU/QvvxJf3p21QfJ2Bt3cw= -github.com/golang/mock v1.4.3/go.mod h1:UOMv5ysSaYNkG+OFQykRIcU/QvvxJf3p21QfJ2Bt3cw= -github.com/golang/mock v1.4.4/go.mod h1:l3mdAwkq5BuhzHwde/uurv3sEJeZMXNpwsxVWU71h+4= github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= github.com/golang/protobuf v1.3.1/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= github.com/golang/protobuf v1.3.2/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= github.com/golang/protobuf v1.3.3/go.mod h1:vzj43D7+SQXF/4pzW/hwtAqwc6iTitCiVSaWz5lYuqw= -github.com/golang/protobuf v1.3.4/go.mod h1:vzj43D7+SQXF/4pzW/hwtAqwc6iTitCiVSaWz5lYuqw= github.com/golang/protobuf v1.3.5/go.mod h1:6O5/vntMXwX2lRkT1hjjk0nAC1IDOTvTlVgjlRvqsdk= -github.com/golang/protobuf v1.4.0-rc.1/go.mod h1:ceaxUfeHdC40wWswd/P6IGgMaK3YpKi5j83Wpe3EHw8= -github.com/golang/protobuf v1.4.0-rc.1.0.20200221234624-67d41d38c208/go.mod h1:xKAWHe0F5eneWXFV3EuXVDTCmh+JuBKY0li0aMyXATA= -github.com/golang/protobuf v1.4.0-rc.2/go.mod h1:LlEzMj4AhA7rCAGe4KMBDvJI+AwstrUpVNzEA03Pprs= -github.com/golang/protobuf v1.4.0-rc.4.0.20200313231945-b860323f09d0/go.mod h1:WU3c8KckQ9AFe+yFwt9sWVRKCVIyN9cPHBJSNnbL67w= -github.com/golang/protobuf v1.4.0/go.mod h1:jodUvKwWbYaEsadDk5Fwe5c77LiNKVO9IDvqG2KuDX0= -github.com/golang/protobuf v1.4.1/go.mod h1:U8fpvMrcmy5pZrNK1lt4xCsGvpyWQ/VVv6QDs8UjoX8= -github.com/golang/protobuf v1.4.2/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI= -github.com/golang/protobuf v1.4.3/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI= github.com/golang/protobuf v1.5.0/go.mod h1:FsONVRAS9T7sI+LIUmWTfcYkHO4aIWwzhcaSAoJOfIk= -github.com/golang/protobuf v1.5.2 h1:ROPKBNFfQgOUMifHyP+KYbvpjbdoFNs+aK7DXlji0Tw= -github.com/golang/protobuf v1.5.2/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY= -github.com/golang/snappy v0.0.0-20180518054509-2e65f85255db/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= -github.com/golang/snappy v0.0.1/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= -github.com/golang/snappy v0.0.4 h1:yAGX7huGHXlcLOEtBnF4w7FQwA26wojNCwOYAEhLjQM= -github.com/golang/snappy v0.0.4/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= -github.com/google/btree v0.0.0-20180813153112-4030bb1f1f0c/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= -github.com/google/btree v1.0.0/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= +github.com/golang/protobuf v1.5.3 h1:KhyjKVUg7Usr/dYsdSqoFveMYd5ko72D+zANwlG1mmg= +github.com/golang/protobuf v1.5.3/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY= github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M= -github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= -github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= github.com/google/go-cmp v0.4.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= -github.com/google/go-cmp v0.4.1/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= -github.com/google/go-cmp v0.5.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= -github.com/google/go-cmp v0.5.1/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= -github.com/google/go-cmp v0.5.4/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= -github.com/google/go-cmp v0.5.8 h1:e6P7q2lk1O+qJJb4BtCQXlK8vWEO8V1ZeuEdJNOqZyg= github.com/google/go-cmp v0.5.8/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= +github.com/google/go-cmp v0.5.9 h1:O2Tfq5qg4qc4AmwVlvv0oLiVAGB7enBSJ2x2DqQFi38= +github.com/google/go-cmp v0.5.9/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= -github.com/google/martian v2.1.0+incompatible/go.mod h1:9I4somxYTbIHy5NJKHRl3wXiIaQGbYVAs8BPL6v8lEs= -github.com/google/martian/v3 v3.0.0/go.mod h1:y5Zk1BBys9G+gd6Jrk0W3cC1+ELVxBWuIGO+w/tUAp0= -github.com/google/pprof v0.0.0-20181206194817-3ea8567a2e57/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc= -github.com/google/pprof v0.0.0-20190515194954-54271f7e092f/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc= -github.com/google/pprof v0.0.0-20191218002539-d4f498aebedc/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM= -github.com/google/pprof v0.0.0-20200212024743-f11f1df84d12/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM= -github.com/google/pprof v0.0.0-20200229191704-1ebb73c60ed3/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM= -github.com/google/pprof v0.0.0-20200430221834-fc25d7d30c6d/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM= -github.com/google/pprof v0.0.0-20200708004538-1a94d8640e99/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM= -github.com/google/pprof v0.0.0-20210407192527-94a9f03dee38/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= github.com/google/renameio v0.1.0/go.mod h1:KWCgfxg9yswjAJkECMjeO8J8rahYeXnNhOm40UhjYkI= github.com/google/shlex v0.0.0-20191202100458-e7afc7fbc510 h1:El6M4kTTCOh6aBiKaUGG7oYTSPP8MxqL4YI3kZKwcP4= github.com/google/shlex v0.0.0-20191202100458-e7afc7fbc510/go.mod h1:pupxD2MaaD3pAXIBCelhxNneeOaAeabZDe5s4K6zSpQ= github.com/google/uuid v1.1.2/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/google/uuid v1.3.0 h1:t6JiXgmwXMjEs8VusXIJk2BXHsn+wx8BZdTaoZ5fu7I= github.com/google/uuid v1.3.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= -github.com/googleapis/gax-go/v2 v2.0.4/go.mod h1:0Wqv26UfaUD9n4G6kQubkQ+KchISgw+vpHVxEJEs9eg= -github.com/googleapis/gax-go/v2 v2.0.5/go.mod h1:DWXyrwAJ9X0FpwwEdw+IPEYBICEFu5mhpdKc/us6bOk= -github.com/gopherjs/gopherjs v0.0.0-20190910122728-9d188e94fb99 h1:twflg0XRTjwKpxb/jFExr4HGq6on2dEOmnL6FV+fgPw= -github.com/gopherjs/gopherjs v0.0.0-20190910122728-9d188e94fb99/go.mod h1:wJfORRmW1u3UXTncJ5qlYoELFm8eSnnEO6hX4iZ3EWY= -github.com/gorilla/csrf v1.6.0/go.mod h1:7tSf8kmjNYr7IWDCYhd3U8Ck34iQ/Yw5CJu7bAkHEGI= -github.com/gorilla/handlers v1.4.1/go.mod h1:Qkdc/uu4tH4g6mTK6auzZ766c4CA0Ng8+o/OAirnOIQ= -github.com/gorilla/mux v1.7.3/go.mod h1:1lud6UwP+6orDFRuTfBEV8e9/aOM/c4fVVCaMa2zaAs= github.com/gorilla/mux v1.8.0 h1:i40aqfkR1h2SlN9hojwV5ZA91wcXFOvkdNIeFDP5koI= github.com/gorilla/mux v1.8.0/go.mod h1:DVbg23sWSpFRCP0SfiEN6jmj59UnW/n46BH5rLB71So= -github.com/gorilla/securecookie v1.1.1/go.mod h1:ra0sb63/xPlUeL+yeDciTfxMRAA+MP+HVt/4epWDjd4= -github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0/go.mod h1:8NvIoxWQoOIhqOTXgfV/d3M/q6VIi02HzZEHgUlZvzk= -github.com/grpc-ecosystem/grpc-gateway v1.12.1/go.mod h1:8XEsbTttt/W+VvjtQhLACqCisSPWTxCZ7sBRjU6iH9c= github.com/grpc-ecosystem/grpc-gateway v1.16.0 h1:gmcG1KaJ57LophUzW0Hy8NmPhnMZb4M0+kPpLofRdBo= github.com/grpc-ecosystem/grpc-gateway v1.16.0/go.mod h1:BDjrQk3hbvj6Nolgz8mAMFbcEtjT1g+wF4CSlocrBnw= -github.com/hashicorp/golang-lru v0.5.0/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= -github.com/hashicorp/golang-lru v0.5.1/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= -github.com/hashicorp/hcl v1.0.0/go.mod h1:E5yfLk+7swimpb2L/Alb/PJmXilQ/rhwaUYs4T20WEQ= -github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU= -github.com/ianlancetaylor/demangle v0.0.0-20181102032728-5e5cf60278f6/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc= -github.com/ianlancetaylor/demangle v0.0.0-20200824232613-28f6c0f3b639/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc= -github.com/ikawaha/kagome.ipadic v1.1.2/go.mod h1:DPSBbU0czaJhAb/5uKQZHMc9MTVRpDugJfX+HddPHHg= -github.com/inconshreveable/mousetrap v1.0.0 h1:Z8tu5sraLXCXIcARxBp/8cbvlwVa7Z1NHg9XEKhtSvM= -github.com/inconshreveable/mousetrap v1.0.0/go.mod h1:PxqpIevigyE2G7u3NXJIT2ANytuPF1OarO4DADm73n8= +github.com/inconshreveable/mousetrap v1.0.1/go.mod h1:vpF70FUmC8bwa3OWnCshd2FqLfsEA9PFc4w1p2J65bw= +github.com/inconshreveable/mousetrap v1.1.0 h1:wN+x4NVGpMsO7ErUn/mUI3vEoE6Jt13X2s0bqwp9tc8= +github.com/inconshreveable/mousetrap v1.1.0/go.mod h1:vpF70FUmC8bwa3OWnCshd2FqLfsEA9PFc4w1p2J65bw= github.com/jeremywohl/flatten v1.0.1 h1:LrsxmB3hfwJuE+ptGOijix1PIfOoKLJ3Uee/mzbgtrs= github.com/jeremywohl/flatten v1.0.1/go.mod h1:4AmD/VxjWcI5SRB0n6szE2A6s2fsNHDLO0nAlMHgfLQ= -github.com/jessevdk/go-flags v1.4.0/go.mod h1:4FA24M0QyGHXBuZZK/XkWh8h0e1EYbRYJSGM75WSRxI= -github.com/jmhodges/levigo v1.0.0 h1:q5EC36kV79HWeTBWsod3mG11EgStG3qArTKcvlksN1U= -github.com/jmhodges/levigo v1.0.0/go.mod h1:Q6Qx+uH3RAqyK4rFQroq9RL7mdkABMcfhEI+nNuzMJQ= github.com/joomcode/errorx v1.1.0 h1:dizuSG6yHzlvXOOGHW00gwsmM4Sb9x/yWEfdtPztqcs= github.com/joomcode/errorx v1.1.0/go.mod h1:eQzdtdlNyN7etw6YCS4W4+lu442waxZYw5yvz0ULrRo= -github.com/jpillora/backoff v1.0.0/go.mod h1:J/6gKK9jxlEcS3zixgDgUAsiuZ7yrSoa/FX5e0EB2j4= -github.com/json-iterator/go v1.1.6/go.mod h1:+SdeFBvtyEkXs7REEP0seUULqWtbJapLOCVDaaPEHmU= -github.com/json-iterator/go v1.1.10/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4= -github.com/json-iterator/go v1.1.11/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4= +github.com/json-iterator/go v1.1.12 h1:PV8peI4a0ysnczrg+LtxykD8LfKY9ML6u2jnxaEnrnM= github.com/json-iterator/go v1.1.12/go.mod h1:e30LSqwooZae/UwlEbR2852Gd8hjQvJoHmT4TnhNGBo= -github.com/jstemmer/go-junit-report v0.0.0-20190106144839-af01ea7f8024/go.mod h1:6v2b51hI/fHJwM22ozAgKL4VKDeJcHhJFhtBdhmNjmU= -github.com/jstemmer/go-junit-report v0.9.1/go.mod h1:Brl9GWCQeLvo8nXZwPNNblvFj/XSXhF0NWZEnDohbsk= -github.com/jtolds/gls v4.20.0+incompatible h1:xdiiI2gbIgH/gLH7ADydsJ1uDOEzR8yvV7C0MuV77Wo= -github.com/jtolds/gls v4.20.0+incompatible/go.mod h1:QJZ7F/aHp+rZTRtaJ1ow/lLfFfVYBRgL+9YlvaHOwJU= github.com/juju/ansiterm v1.0.0 h1:gmMvnZRq7JZJx6jkfSq9/+2LMrVEwGwt7UR6G+lmDEg= github.com/juju/ansiterm v1.0.0/go.mod h1:PyXUpnI3olx3bsPcHt98FGPX/KCFZ1Fi+hw1XLI6384= -github.com/julienschmidt/httprouter v1.2.0/go.mod h1:SYymIcj16QtmaHHD7aYtjjsJG7VTCxuUUipMqKk8s4w= -github.com/julienschmidt/httprouter v1.3.0/go.mod h1:JR6WtHb+2LUe8TCKY3cZOxFyyO8IZAc4RVcycCCAKdM= github.com/july2993/yaml v0.0.0-20200423062752-adcfa5abe2ed h1:4dMu41sJad3+E9AMc2hCimGhll184HfPKdu05R6YJtc= github.com/july2993/yaml v0.0.0-20200423062752-adcfa5abe2ed/go.mod h1:7FtNkxNa9xDptPhcc7mWT/Ov++hkuMU/NKtgkPZPUwo= github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI2bnpBCr8= github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= -github.com/kljensen/snowball v0.6.0/go.mod h1:27N7E8fVU5H68RlUmnWwZCfxgt4POBJfENGMvNRhldw= -github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= -github.com/konsorten/go-windows-terminal-sequences v1.0.3/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= -github.com/kr/logfmt v0.0.0-20140226030751-b84e30acd515/go.mod h1:+0opPa2QZZtGFBFZlji/RkVcI2GknAs/DXo4wKdlNEc= +github.com/klauspost/compress v1.16.0 h1:iULayQNOReoYUe+1qtKOqw9CwJv3aNQu8ivo7lw1HU4= +github.com/klauspost/compress v1.16.0/go.mod h1:ntbaceVETuRiXiv4DpjP66DpAtAGkEQskQzEyD//IeE= +github.com/klauspost/cpuid/v2 v2.0.1/go.mod h1:FInQzS24/EEf25PyTYn52gqo7WaD8xa0213Md/qVLRg= +github.com/klauspost/cpuid/v2 v2.0.4/go.mod h1:FInQzS24/EEf25PyTYn52gqo7WaD8xa0213Md/qVLRg= +github.com/klauspost/cpuid/v2 v2.2.4 h1:acbojRNwl3o09bUq+yDCtZFc1aiwaAAxtcn8YkZXnvk= +github.com/klauspost/cpuid/v2 v2.2.4/go.mod h1:RVVoqg1df56z8g3pUjL/3lE5UfnlrJX8tyFgg4nqhuY= github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= github.com/kr/pretty v0.2.1/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI= -github.com/kr/pretty v0.3.0 h1:WgNl7dwNpEZ6jJ9k1snq4pZsg7DOEN8hP9Xw0Tsjwk0= -github.com/kr/pretty v0.3.0/go.mod h1:640gp4NfQd8pI5XOwp5fnNeVWj67G7CFk/SaSQn7NBk= +github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE= +github.com/kr/pretty v0.3.1/go.mod h1:hoEshYVHaxMs3cyo3Yncou5ZscifuDolrwPKZanG3xk= github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= @@ -324,65 +130,38 @@ github.com/lorenzosaino/go-sysctl v0.3.1 h1:3phX80tdITw2fJjZlwbXQnDWs4S30beNcMbw github.com/lorenzosaino/go-sysctl v0.3.1/go.mod h1:5grcsBRpspKknNS1qzt1eIeRDLrhpKZAtz8Fcuvs1Rc= github.com/lunixbochs/vtclean v1.0.0 h1:xu2sLAri4lGiovBDQKxl5mrXyESr3gUr5m5SM5+LVb8= github.com/lunixbochs/vtclean v1.0.0/go.mod h1:pHhQNgMf3btfWnGBVipUOjRYhoOsdGqdm/+2c2E2WMI= -github.com/magiconair/properties v1.8.0/go.mod h1:PppfXfuXeibc/6YijjN8zIbojt8czPbwD3XqdrwzmxQ= -github.com/mattn/go-colorable v0.0.9/go.mod h1:9vuHe8Xs5qXnSaW/c/ABM9alt+Vo+STaOChaDxuIBZU= -github.com/mattn/go-colorable v0.1.8/go.mod h1:u6P/XSegPjTcexA+o6vUJrdnUu04hMope9wVRipJSqc= -github.com/mattn/go-colorable v0.1.9/go.mod h1:u6P/XSegPjTcexA+o6vUJrdnUu04hMope9wVRipJSqc= github.com/mattn/go-colorable v0.1.10/go.mod h1:u6P/XSegPjTcexA+o6vUJrdnUu04hMope9wVRipJSqc= -github.com/mattn/go-colorable v0.1.12 h1:jF+Du6AlPIjs2BiUiQlKOX0rt3SujHxPnksPKZbaA40= -github.com/mattn/go-colorable v0.1.12/go.mod h1:u5H1YNBxpqRaxsYJYSkiCWKzEfiAb1Gb520KVy5xxl4= -github.com/mattn/go-isatty v0.0.4/go.mod h1:M+lRXTBqGeGNdLjl/ufCoiOlB5xdOkqRJdNxMWT7Zi4= +github.com/mattn/go-colorable v0.1.13 h1:fFA4WZxdEF4tXPZVKMLwD8oUnCTTo08duU7wxecdEvA= +github.com/mattn/go-colorable v0.1.13/go.mod h1:7S9/ev0klgBDR4GtXTXX8a3vIGJpMovkB8vQcUbaXHg= github.com/mattn/go-isatty v0.0.12/go.mod h1:cbi8OIDigv2wuxKPP5vlRcQ1OAZbq2CE4Kysco4FUpU= -github.com/mattn/go-isatty v0.0.14 h1:yVuAays6BHfxijgZPzw+3Zlu5yQgKGP2/hcQbHb7S9Y= github.com/mattn/go-isatty v0.0.14/go.mod h1:7GGIvUiUoEMVVmxf/4nioHXj79iQHKdU27kJ6hsGG94= -github.com/mattn/go-runewidth v0.0.12/go.mod h1:RAqKPSqVFrSLVXbA8x7dzmKdmGzieGRCM46jaSJTDAk= -github.com/mattn/go-runewidth v0.0.13 h1:lTGmDsbAYt5DmK6OnoV7EuIF1wEIFAcxld6ypU4OSgU= -github.com/mattn/go-runewidth v0.0.13/go.mod h1:Jdepj2loyihRzMpdS35Xk/zdY8IAYHsh153qUoGf23w= -github.com/matttproud/golang_protobuf_extensions v1.0.1 h1:4hp9jkHxhMHkqkrB3Ix0jegS5sx/RkqARlsWZ6pIwiU= -github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5Ld7szi9bcBfOoFv/3dc6xSMkL2PC0= -github.com/mitchellh/go-homedir v1.1.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0= -github.com/mitchellh/go-wordwrap v1.0.0 h1:6GlHJ/LTGMrIJbwgdqdl2eEH8o+Exx/0m8ir9Gns0u4= -github.com/mitchellh/go-wordwrap v1.0.0/go.mod h1:ZXFpozHsX6DPmq2I0TCekCxypsnAUbP2oI0UX1GXzOo= -github.com/mitchellh/mapstructure v1.1.2/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh9fWfEaFds41c1Y= +github.com/mattn/go-isatty v0.0.16/go.mod h1:kYGgaQfpe5nmfYZH+SKPsOc2e4SrIfOl2e/yFXSvRLM= +github.com/mattn/go-isatty v0.0.18 h1:DOKFKCQ7FNG2L1rbrmstDN4QVRdS89Nkh85u68Uwp98= +github.com/mattn/go-isatty v0.0.18/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y= +github.com/mattn/go-runewidth v0.0.14 h1:+xnbZSEeDbOIg5/mE6JF0w6n9duR1l3/WmbinWVwUuU= +github.com/mattn/go-runewidth v0.0.14/go.mod h1:Jdepj2loyihRzMpdS35Xk/zdY8IAYHsh153qUoGf23w= +github.com/matttproud/golang_protobuf_extensions v1.0.4 h1:mmDVorXM7PCGKw94cs5zkfA9PSy5pEvNWRP0ET0TIVo= +github.com/matttproud/golang_protobuf_extensions v1.0.4/go.mod h1:BSXmuO+STAnVfrANrmjBb36TMTDstsz7MSK+HVaYKv4= +github.com/minio/md5-simd v1.1.2 h1:Gdi1DZK69+ZVMoNHRXJyNcxrMA4dSxoYHZSQbirFg34= +github.com/minio/md5-simd v1.1.2/go.mod h1:MzdKDxYpY2BT9XQFocsiZf/NKVtR7nkE4RoEpN+20RM= +github.com/minio/minio-go/v7 v7.0.52 h1:8XhG36F6oKQUDDSuz6dY3rioMzovKjW40W6ANuN0Dps= +github.com/minio/minio-go/v7 v7.0.52/go.mod h1:IbbodHyjUAguneyucUaahv+VMNs/EOTV9du7A7/Z3HU= +github.com/minio/sha256-simd v1.0.0 h1:v1ta+49hkWZyvaKwrQB8elexRqm6Y0aMLjCNsrYxo6g= +github.com/minio/sha256-simd v1.0.0/go.mod h1:OuYzVNI5vcoYIAmbIvHPl3N3jUzVedXbKy5RFepssQM= github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= +github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd h1:TRLaZ9cD/w8PVh93nsPXa1VrQ6jlwL5oN8l14QlcNfg= github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= -github.com/modern-go/reflect2 v0.0.0-20180701023420-4b7aa43c6742/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0= -github.com/modern-go/reflect2 v1.0.1/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0= +github.com/modern-go/reflect2 v1.0.2 h1:xBagoLtFs94CBntxluKeaWgTMpvLxC4ur3nMaC9Gz0M= github.com/modern-go/reflect2 v1.0.2/go.mod h1:yWuevngMOJpCy52FWWMvUC8ws7m/LJsjYzDa0/r8luk= -github.com/mschoch/smat v0.0.0-20160514031455-90eadee771ae/go.mod h1:qAyveg+e4CE+eKJXWVjKXM4ck2QobLqTDytGJbLLhJg= -github.com/mschoch/smat v0.2.0 h1:8imxQsjDm8yFEAVBe7azKmKSgzSkZXDuKkSq9374khM= -github.com/mschoch/smat v0.2.0/go.mod h1:kc9mz7DoBKqDyiRL7VZN8KvXQMWeTaVnttLRXOlotKw= -github.com/mwitkow/go-conntrack v0.0.0-20161129095857-cc309e4a2223/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U= -github.com/mwitkow/go-conntrack v0.0.0-20190716064945-2f068394615f/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U= -github.com/nkovacs/streamquote v0.0.0-20170412213628-49af9bddb229/go.mod h1:0aYXnNPJ8l7uZxf45rWW1a/uME32OF0rhiYGNQ2oF2E= -github.com/nxadm/tail v1.4.4/go.mod h1:kenIhsEOeOJmVchQTgglprH7qJGnHDVpk1VPCcaMI8A= -github.com/nxadm/tail v1.4.8 h1:nPr65rt6Y5JFSKQO7qToXr7pePgD6Gwiw05lkbyAQTE= -github.com/nxadm/tail v1.4.8/go.mod h1:+ncqLTQzXmGhMZNUePPaPqPvBxHAIsmXswZKocGu+AU= -github.com/onsi/ginkgo v1.6.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= -github.com/onsi/ginkgo v1.7.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= -github.com/onsi/ginkgo v1.12.1/go.mod h1:zj2OWP4+oCPe1qIXoGWkgMRwljMUYCdkwsT2108oapk= -github.com/onsi/ginkgo v1.14.0/go.mod h1:iSB4RoI2tjJc9BBv4NKIKWKya62Rps+oPG/Lv9klQyY= -github.com/onsi/ginkgo v1.16.4/go.mod h1:dX+/inL/fNMqNlz0e9LfyB9TswhZpCVdJM/Z6Vvnwo0= -github.com/onsi/ginkgo v1.16.5 h1:8xi0RTUf59SOSfEtZMvwTvXYMzG4gV23XVHOZiXNtnE= -github.com/onsi/ginkgo v1.16.5/go.mod h1:+E8gABHa3K6zRBolWtd+ROzc/U5bkGt0FwiG042wbpU= -github.com/onsi/ginkgo/v2 v2.0.0/go.mod h1:vw5CSIxN1JObi/U8gcbwft7ZxR2dgaR70JSE3/PpL4c= -github.com/onsi/gomega v1.4.3/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY= -github.com/onsi/gomega v1.7.1/go.mod h1:XdKZgCCFLUoM/7CFJVPcG8C1xQ1AJ0vpAezJrB7JYyY= -github.com/onsi/gomega v1.10.1/go.mod h1:iN09h71vgCQne3DLsj+A5owkum+a2tYe+TOCB1ybHNo= -github.com/onsi/gomega v1.17.0/go.mod h1:HnhC7FXeEQY45zxNK3PPoIUhzk/80Xly9PcubAlGdZY= -github.com/onsi/gomega v1.18.1 h1:M1GfJqGRrBrrGGsbxzV5dqM2U2ApXefZCQpkukxYRLE= -github.com/onsi/gomega v1.18.1/go.mod h1:0q+aL8jAiMXy9hbwj2mr5GziHiwhAIQpFmmtT5hitRs= -github.com/otiai10/copy v1.7.0 h1:hVoPiN+t+7d2nzzwMiDHPSOogsWAStewq3TwU05+clE= -github.com/otiai10/copy v1.7.0/go.mod h1:rmRl6QPdJj6EiUqXQ/4Nn2lLXoNQjFCQbbNrxgc/t3U= +github.com/otiai10/copy v1.9.0 h1:7KFNiCgZ91Ru4qW4CWPf/7jqtxLagGRmIxWldPP9VY4= +github.com/otiai10/copy v1.9.0/go.mod h1:hsfX19wcn0UWIHUQ3/4fHuehhk2UyArQ9dVFAn3FczI= github.com/otiai10/curr v0.0.0-20150429015615-9b4961190c95/go.mod h1:9qAhocn7zKJG+0mI8eUu6xqkFDYS2kb2saOteoSB3cE= github.com/otiai10/curr v1.0.0/go.mod h1:LskTG5wDwr8Rs+nNQ+1LlxRjAtTZZjtJW4rMXl6j4vs= github.com/otiai10/mint v1.3.0/go.mod h1:F5AjcsTsWUqX+Na9fpHb52P8pcRX2CI6A3ctIT91xUo= -github.com/otiai10/mint v1.3.3 h1:7JgpsBaN0uMkyju4tbYHu0mnM55hNKVYLsXmwr15NQI= -github.com/otiai10/mint v1.3.3/go.mod h1:/yxELlJQ0ufhjUwhshSj+wFjZ78CnZ48/1wtmBH1OTc= -github.com/pelletier/go-toml v1.2.0/go.mod h1:5z9KED0ma1S8pY6P1sdut58dfprrGBbd/94hg7ilaic= -github.com/philhofer/fwd v1.0.0/go.mod h1:gk3iGcWd9+svBvR0sR+KPcfE+RNWozjowpeBVG3ZVNU= -github.com/philhofer/fwd v1.1.1 h1:GdGcTjf5RNAxwS4QLsiMzJYj5KEvPJD3Abr261yRQXQ= -github.com/philhofer/fwd v1.1.1/go.mod h1:gk3iGcWd9+svBvR0sR+KPcfE+RNWozjowpeBVG3ZVNU= +github.com/otiai10/mint v1.4.0 h1:umwcf7gbpEwf7WFzqmWwSv0CzbeMsae2u9ZvpP8j2q4= +github.com/otiai10/mint v1.4.0/go.mod h1:gifjb2MYOoULtKLqUAEILUG/9KONW6f7YsJ6vQLTlFI= +github.com/pelletier/go-toml v1.9.5 h1:4yBQzkHv+7BHq2PQUZF3Mx0IYxG7LsP222s7Agd3ve8= +github.com/pelletier/go-toml v1.9.5/go.mod h1:u1nR/EPcESfeI/szUZKdtJ0xRNbUoANCkoOuaOx1Y+c= github.com/pingcap/check v0.0.0-20190102082844-67f458068fc8/go.mod h1:B1+S9LNcuMyLH/4HMTViQOJevkGiik3wW2AN9zb2fNQ= github.com/pingcap/check v0.0.0-20211026125417-57bd13f7b5f0 h1:HVl5539r48eA+uDuX/ziBmQCxzT1pGrzWbKuXT46Bq0= github.com/pingcap/check v0.0.0-20211026125417-57bd13f7b5f0/go.mod h1:PYMCGwN0JHjoqGr3HrZoD+b8Tgx8bKnArhSq8YVzUMc= @@ -394,92 +173,57 @@ github.com/pingcap/failpoint v0.0.0-20220801062533-2eaa32854a6c h1:CgbKAHto5CQgW github.com/pingcap/failpoint v0.0.0-20220801062533-2eaa32854a6c/go.mod h1:4qGtCB0QK0wBzKtFEGDhxXnSnbQApw1gc9siScUl8ew= github.com/pingcap/fn v1.0.0 h1:CyA6AxcOZkQh52wIqYlAmaVmF6EvrcqFywP463pjA8g= github.com/pingcap/fn v1.0.0/go.mod h1:u9WZ1ZiOD1RpNhcI42RucFh/lBuzTu6rw88a+oF2Z24= -github.com/pingcap/kvproto v0.0.0-20220913050750-f6d05706948a h1:LCtkOPEzjWk86NclzxdZ42nNtbhuIN1p6cpd/FYUqkU= -github.com/pingcap/kvproto v0.0.0-20220913050750-f6d05706948a/go.mod h1:OYtxs0786qojVTmkVeufx93xe+jUgm56GUYRIKnmaGI= +github.com/pingcap/kvproto v0.0.0-20230331024443-349815129e6d h1:QR9Gk/Hi7DU399ec81cG7b3X/Umwv8FIcyx5WwD+O7M= +github.com/pingcap/kvproto v0.0.0-20230331024443-349815129e6d/go.mod h1:RjuuhxITxwATlt5adgTedg3ehKk01M03L1U4jNHdeeQ= github.com/pingcap/log v0.0.0-20191012051959-b742a5d432e9/go.mod h1:4rbK1p9ILyIfb6hU7OG2CiWSqMXnp3JMbiaVJ6mvoY8= github.com/pingcap/log v0.0.0-20200511115504-543df19646ad/go.mod h1:4rbK1p9ILyIfb6hU7OG2CiWSqMXnp3JMbiaVJ6mvoY8= github.com/pingcap/log v1.1.0 h1:ELiPxACz7vdo1qAvvaWJg1NrYFoY6gqAh/+Uo6aXdD8= github.com/pingcap/log v1.1.0/go.mod h1:DWQW5jICDR7UJh4HtxXSM20Churx4CQL0fwL/SoOSA4= github.com/pingcap/tidb-insight/collector v0.0.0-20220902034607-fb5ae0ddc8c1 h1:tqb0pvBBTpalOMC8SJFwZU/jmUJk/97NmkfLUkzHhjs= github.com/pingcap/tidb-insight/collector v0.0.0-20220902034607-fb5ae0ddc8c1/go.mod h1:RzwBK8J9G55iy4zpBqhFTARN8E/qu1ARGdXIsrOmgBM= -github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= +github.com/pkg/diff v0.0.0-20210226163009-20ebb0f2a09e/go.mod h1:pJLUxLENpZxwdsKMEsNbx1VGcRFpLqf3715MtcvvzbA= github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= -github.com/prometheus/client_golang v0.9.1/go.mod h1:7SWBe2y4D6OKWSNQJUaRYU/AaXPKyh/dDVn+NZz0KFw= -github.com/prometheus/client_golang v1.0.0/go.mod h1:db9x61etRT2tGnBNRi70OPL5FsnadC4Ky3P0J6CfImo= -github.com/prometheus/client_golang v1.7.1/go.mod h1:PY5Wy2awLA44sXw4AOSfFBetzPP4j5+D6mVACh+pe2M= -github.com/prometheus/client_golang v1.11.0/go.mod h1:Z6t4BnS23TR94PD6BsDNk8yVqroYurpAkEiz0P2BEV0= -github.com/prometheus/client_golang v1.11.1/go.mod h1:Z6t4BnS23TR94PD6BsDNk8yVqroYurpAkEiz0P2BEV0= -github.com/prometheus/client_golang v1.12.1/go.mod h1:3Z9XVyYiZYEO+YQWt3RD2R3jrbd179Rt297l4aS6nDY= -github.com/prometheus/client_model v0.0.0-20180712105110-5c3871d89910/go.mod h1:MbSGuTsp3dbXC40dX6PRTWyKYBIrTGTE9sqQNg2J8bo= -github.com/prometheus/client_model v0.0.0-20190129233127-fd36f4220a90/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= -github.com/prometheus/client_model v0.2.0 h1:uq5h0d+GuxiXLJLNABMgp2qUWDPiLvgCzz2dUR+/W/M= -github.com/prometheus/client_model v0.2.0/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= -github.com/prometheus/common v0.4.1/go.mod h1:TNfzLD0ON7rHzMJeJkieUDPYmFC7Snx/y86RQel1bk4= -github.com/prometheus/common v0.10.0/go.mod h1:Tlit/dnDKsSWFlCLTWaA1cyBgKHSMdTB80sz/V91rCo= -github.com/prometheus/common v0.26.0/go.mod h1:M7rCNAaPfAosfx8veZJCuw84e35h3Cfd9VFqTh1DIvc= -github.com/prometheus/common v0.32.1/go.mod h1:vu+V0TpY+O6vW9J44gczi3Ap/oXXR10b+M/gUGO4Hls= -github.com/prometheus/common v0.33.0/go.mod h1:gB3sOl7P0TvJabZpLY5uQMpUqRCPPCyRLCZYc7JZTNE= -github.com/prometheus/common v0.37.0 h1:ccBbHCgIiT9uSoFY0vX8H3zsNR5eLt17/RQLUvn8pXE= -github.com/prometheus/common v0.37.0/go.mod h1:phzohg0JFMnBEFGxTDbfu3QyL5GI8gTQJFhYO5B3mfA= -github.com/prometheus/procfs v0.0.0-20181005140218-185b4288413d/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk= -github.com/prometheus/procfs v0.0.2/go.mod h1:TjEm7ze935MbeOT/UhFTIMYKhuLP4wbCsTZCD3I8kEA= -github.com/prometheus/procfs v0.1.3/go.mod h1:lV6e/gmhEcM9IjHGsFOCxxuZ+z1YqCvr4OA4YeYWdaU= -github.com/prometheus/procfs v0.6.0/go.mod h1:cz+aTbrPOrUb4q7XlbU9ygM+/jj0fzG6c1xBZuNvfVA= -github.com/prometheus/procfs v0.7.3/go.mod h1:cz+aTbrPOrUb4q7XlbU9ygM+/jj0fzG6c1xBZuNvfVA= -github.com/prometheus/prom2json v1.3.1 h1:OogL5hsrJpLPz3jZ4LPz4sJRTtADzViCNRQoqrzUQvk= -github.com/prometheus/prom2json v1.3.1/go.mod h1:A8Oy9aiQx4wrJY9ya1i4nHOySGmkVp5EO0aU1iSJR+g= +github.com/prometheus/client_model v0.3.0 h1:UBgGFHqYdG/TPFD1B1ogZywDqEkwp3fBMvqdiQ7Xew4= +github.com/prometheus/client_model v0.3.0/go.mod h1:LDGWKZIo7rky3hgvBe+caln+Dr3dPggB5dvjtD7w9+w= +github.com/prometheus/common v0.42.0 h1:EKsfXEYo4JpWMHH5cg+KOUWeuJSov1Id8zGR8eeI1YM= +github.com/prometheus/common v0.42.0/go.mod h1:xBwqVerjNdUDjgODMpudtOMwlOwf2SaTr1yjz4b7Zbc= +github.com/prometheus/prom2json v1.3.2 h1:heRKAGHWqm8N3IaRDDNBBJNVS6a9mLdsTlFhvOaNGb0= +github.com/prometheus/prom2json v1.3.2/go.mod h1:TQ9o1OxW0eyhl4BBpVpGGsavyJfTDETna4/d0Kib+V0= github.com/r3labs/diff/v2 v2.15.1 h1:EOrVqPUzi+njlumoqJwiS/TgGgmZo83619FNDB9xQUg= github.com/r3labs/diff/v2 v2.15.1/go.mod h1:I8noH9Fc2fjSaMxqF3G2lhDdC0b+JXCfyx85tWFM9kc= -github.com/rcrowley/go-metrics v0.0.0-20190826022208-cac0b30c2563/go.mod h1:bCqnVzQkZxMG4s8nGwiZ5l3QUCyqpo9Y+/ZMZ9VjZe4= github.com/relex/aini v1.5.0 h1:6euW/m6b2Y2hkSY8rsyGzcYGpMUWx2dnTzXgQvunTzQ= github.com/relex/aini v1.5.0/go.mod h1:qUMEteDeWDTMHUP7WsaOTc7gawELU5Gcrn2YHz4EAr0= -github.com/remyoudompheng/bigfft v0.0.0-20200410134404-eec4a21b6bb0/go.mod h1:qqbHyh8v60DhA7CoWK5oRCqLrMHRGoxYCSS9EjAz6Eo= -github.com/rivo/uniseg v0.1.0/go.mod h1:J6wj4VEh+S6ZtnVlnTBMWIodfgj8LQOQFoIToxlJtxc= -github.com/rivo/uniseg v0.2.0 h1:S1pD9weZBuJdFmowNwbpi7BJ8TNftyUImj/0WQi72jY= github.com/rivo/uniseg v0.2.0/go.mod h1:J6wj4VEh+S6ZtnVlnTBMWIodfgj8LQOQFoIToxlJtxc= +github.com/rivo/uniseg v0.4.4 h1:8TfxU8dW6PdqD27gjM8MVNuicgxIjxpm4K7x4jp8sis= +github.com/rivo/uniseg v0.4.4/go.mod h1:FN3SvrM+Zdj16jyLfmOkMNblXMcoc8DfTHruCPUcx88= github.com/rogpeppe/fastuuid v1.2.0/go.mod h1:jVj6XXZzXRy/MSR5jhDC/2q6DgLz+nrA6LYCDYWNEvQ= github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4= -github.com/rogpeppe/go-internal v1.6.1 h1:/FiVV8dS/e+YqF2JvO3yXRFbBLTIuSDkuC7aBOAvL+k= -github.com/rogpeppe/go-internal v1.6.1/go.mod h1:xXDCJY+GAPziupqXw64V24skbSoqbTEfhy4qGm1nDQc= -github.com/russross/blackfriday v1.5.2/go.mod h1:JO/DiYxRf+HjHt06OyowR9PTA263kcR/rfWxYHBV53g= -github.com/russross/blackfriday v2.0.0+incompatible h1:cBXrhZNUf9C+La9/YpS+UHpUT8YD6Td9ZMSU9APFcsk= -github.com/russross/blackfriday v2.0.0+incompatible/go.mod h1:JO/DiYxRf+HjHt06OyowR9PTA263kcR/rfWxYHBV53g= +github.com/rogpeppe/go-internal v1.9.0 h1:73kH8U+JUqXU8lRuOHeVHaa/SZPifC7BkcraZVejAe8= +github.com/rogpeppe/go-internal v1.9.0/go.mod h1:WtVeX8xhTBvf0smdhujwtBcq4Qrzq/fJaraNFVN+nFs= +github.com/rs/xid v1.4.0 h1:qd7wPTDkN6KQx2VmMBLrpHkiyQwgFXRnkOLacUiaSNY= +github.com/rs/xid v1.4.0/go.mod h1:trrq9SKmegXys3aeAKXMUTdJsYXVwGY3RLcfgqegfbg= github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= -github.com/sergi/go-diff v1.0.0/go.mod h1:0CfEIISq7TuYL3j771MWULgwwjU+GofnZX9QAmXWZgo= github.com/sergi/go-diff v1.1.0/go.mod h1:STckp+ISIX8hZLjrqAeVduY0gWCT9IjLuqbuNXdaHfM= -github.com/sergi/go-diff v1.2.0 h1:XU+rvMAioB0UC3q1MFrIQy4Vo5/4VsRDQQXHsEya6xQ= -github.com/sergi/go-diff v1.2.0/go.mod h1:STckp+ISIX8hZLjrqAeVduY0gWCT9IjLuqbuNXdaHfM= +github.com/sergi/go-diff v1.3.1 h1:xkr+Oxo4BOQKmkn/B9eMK0g5Kg/983T9DqqPHwYqD+8= +github.com/sergi/go-diff v1.3.1/go.mod h1:aMJSSKb2lpPvRNec0+w3fl7LP9IOFzdc9Pa4NFbPK1I= github.com/sethvargo/go-password v0.2.0 h1:BTDl4CC/gjf/axHMaDQtw507ogrXLci6XRiLc7i/UHI= github.com/sethvargo/go-password v0.2.0/go.mod h1:Ym4Mr9JXLBycr02MFuVQ/0JHidNetSgbzutTr3zsYXE= github.com/shirou/gopsutil v3.21.11+incompatible h1:+1+c1VGhc88SSonWP6foOcLhvnKlUeu/erjjvaPEYiI= github.com/shirou/gopsutil v3.21.11+incompatible/go.mod h1:5b4v6he4MtMOwMlS0TUMTu2PcXUg8+E1lC7eC3UO/RA= -github.com/shurcooL/sanitized_anchor_name v1.0.0 h1:PdmoCO6wvbs+7yrJyMORt4/BmY5IYyJwS/kOiWx8mHo= -github.com/shurcooL/sanitized_anchor_name v1.0.0/go.mod h1:1NzhyTcUVG4SuEtjjoZeVRXNmyL/1OwPU0+IJeTBvfc= -github.com/sirupsen/logrus v1.2.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo= -github.com/sirupsen/logrus v1.4.2/go.mod h1:tLMulIdttU9McNUspp0xgXVQah82FyeX6MwdIuYE2rE= -github.com/sirupsen/logrus v1.6.0/go.mod h1:7uNnSEd1DgxDLC74fIahvMZmmYsHGZGEOFrfsX/uA88= -github.com/skratchdot/open-golang v0.0.0-20200116055534-eef842397966 h1:JIAuq3EEf9cgbU6AtGPK4CTG3Zf6CKMNqf0MHTggAUA= -github.com/skratchdot/open-golang v0.0.0-20200116055534-eef842397966/go.mod h1:sUM3LWHvSMaG192sy56D9F7CNvL7jUJVXoqM1QKLnog= -github.com/spf13/afero v1.1.2/go.mod h1:j4pytiNVoe2o6bmDsKpLACNPDBIoEAkihy7loJ1B0CQ= -github.com/spf13/cast v1.3.0/go.mod h1:Qx5cxh0v+4UWYiBimWS+eyWzqEqokIECu5etghLkUJE= -github.com/spf13/cobra v0.0.5/go.mod h1:3K3wKZymM7VvHMDS9+Akkh4K60UwM26emMESw8tLCHU= -github.com/spf13/cobra v1.5.0 h1:X+jTBEBqF0bHN+9cSMgmfuvv2VHJ9ezmFNf9Y/XstYU= -github.com/spf13/cobra v1.5.0/go.mod h1:dWXEIy2H428czQCjInthrTRUg7yKbok+2Qi/yBIJoUM= -github.com/spf13/jwalterweatherman v1.0.0/go.mod h1:cQK4TGJAtQXfYWX+Ddv3mKDzgVb68N+wFjFa4jdeBTo= -github.com/spf13/pflag v1.0.3/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4= +github.com/sirupsen/logrus v1.9.0 h1:trlNQbNUG3OdDrDil03MCb1H2o9nJ1x4/5LYw7byDE0= +github.com/sirupsen/logrus v1.9.0/go.mod h1:naHLuLoDiP4jHNo9R0sCBMtWGeIprob74mVsIT4qYEQ= +github.com/spf13/cobra v1.6.1 h1:o94oiPyS4KD1mPy2fmcYYHHfCxLqYjJOhGsCHFZtEzA= +github.com/spf13/cobra v1.6.1/go.mod h1:IOw/AERYS7UzyrGinqmz6HLUo219MORXGxhbaJUqzrY= github.com/spf13/pflag v1.0.5 h1:iy+VFUOCP1a+8yFto/drg2CJ5u0yRoB7fZw3DKv/JXA= github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= -github.com/spf13/viper v1.3.2/go.mod h1:ZiWeW+zYFKm7srdB9IoDzzZXaJaI5eL9QjNiN/DMA2s= -github.com/steveyen/gtreap v0.1.0 h1:CjhzTa274PyJLJuMZwIzCO1PfC00oRa8d1Kc78bFXJM= -github.com/steveyen/gtreap v0.1.0/go.mod h1:kl/5J7XbrOmlIbYIXdRHDDE5QxHqpk0cmkT7Z4dM9/Y= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= -github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw= +github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo= github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4= @@ -487,64 +231,33 @@ github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5 github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= -github.com/stretchr/testify v1.8.0 h1:pSgiaMZlXftHpm5L7V1+rVB+AZJydKsMxsQBIJw4PKk= github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU= -github.com/syndtr/goleveldb v1.0.0/go.mod h1:ZVVdQEZoIme9iO1Ch2Jdy24qqXrMMOU6lpPAyBWyWuQ= -github.com/syndtr/goleveldb v1.0.1-0.20210305035536-64b5b1c73954 h1:xQdMZ1WLrgkkvOZ/LDQxjVxMLdby7osSh4ZEVa5sIjs= -github.com/syndtr/goleveldb v1.0.1-0.20210305035536-64b5b1c73954/go.mod h1:u2MKkTVTVJWe5D1rCvame8WqhBd88EuIwODJZ1VHCPM= -github.com/tebeka/snowball v0.4.2/go.mod h1:4IfL14h1lvwZcp1sfXuuc7/7yCsvVffTWxWxCLfFpYg= -github.com/tecbot/gorocksdb v0.0.0-20191217155057-f0fad39f321c h1:g+WoO5jjkqGAzHWCjJB1zZfXPIAaDpzXIEJ0eS6B5Ok= -github.com/tecbot/gorocksdb v0.0.0-20191217155057-f0fad39f321c/go.mod h1:ahpPrc7HpcfEWDQRZEmnXMzHY03mLDYMCxeDzy46i+8= -github.com/tinylib/msgp v1.1.0/go.mod h1:+d+yLhGm8mzTaHzB+wgMYrodPfmZrzkirds8fDWklFE= -github.com/tinylib/msgp v1.1.6 h1:i+SbKraHhnrf9M5MYmvQhFnbLhAXSDWF8WWsuyRdocw= -github.com/tinylib/msgp v1.1.6/go.mod h1:75BAfg2hauQhs3qedfdDZmWAPcFMAvJE5b9rGOMufyw= -github.com/tj/assert v0.0.0-20190920132354-ee03d75cd160 h1:NSWpaDaurcAJY7PkL8Xt0PhZE7qpvbZl5ljd8r6U0bI= -github.com/tj/assert v0.0.0-20190920132354-ee03d75cd160/go.mod h1:mZ9/Rh9oLWpLLDRpvE+3b7gP/C2YyLFYxNmcLnPTMe0= -github.com/tj/go-css v0.0.0-20191108133013-220a796d1705 h1:+UA89aFRjPMqdccHd9A0HLNCRDXIoElaDoW2C1V3TzA= -github.com/tj/go-css v0.0.0-20191108133013-220a796d1705/go.mod h1:e+JPLQ9wyQCgRnPenX2bo7MJoLphBHz5c1WUqaANSeA= -github.com/tj/go-termd v0.0.1 h1:NRrUrpzPj3jVlimGNMdnNOry0vYgvEkMJcJWZkKAeZI= -github.com/tj/go-termd v0.0.1/go.mod h1:qf28T7t3aasdTnAz6ehff7dfebsK+lAKK53duclZ/yQ= +github.com/stretchr/testify v1.8.2 h1:+h33VjcLVPDHtOdpUCuF+7gSuG3yGIftsP1YvFihtJ8= +github.com/stretchr/testify v1.8.2/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4= github.com/tklauser/go-sysconf v0.3.9/go.mod h1:11DU/5sG7UexIrp/O6g35hrWzu0JxlwQ3LSFUzyeuhs= -github.com/tklauser/go-sysconf v0.3.10 h1:IJ1AZGZRWbY8T5Vfk04D9WOA5WSejdflXxP03OUqALw= -github.com/tklauser/go-sysconf v0.3.10/go.mod h1:C8XykCvCb+Gn0oNCWPIlcb0RuglQTYaQ2hGm7jmxEFk= +github.com/tklauser/go-sysconf v0.3.11 h1:89WgdJhk5SNwJfu+GKyYveZ4IaJ7xAkecBo+KdJV0CM= +github.com/tklauser/go-sysconf v0.3.11/go.mod h1:GqXfhXY3kiPa0nAXPDIQIWzJbMCB7AmcWpGR8lSZfqI= github.com/tklauser/numcpus v0.3.0/go.mod h1:yFGUr7TUHQRAhyqBcEg0Ge34zDBAsIvJJcyE6boqnA8= -github.com/tklauser/numcpus v0.4.0/go.mod h1:1+UI3pD8NW14VMwdgJNJ1ESk2UnwhAnz5hMwiKKqXCQ= -github.com/tklauser/numcpus v0.5.0 h1:ooe7gN0fg6myJ0EKoTAf5hebTZrH52px3New/D9iJ+A= -github.com/tklauser/numcpus v0.5.0/go.mod h1:OGzpTxpcIMNGYQdit2BYL1pvk/dSOaJWjKoflh+RQjo= -github.com/ugorji/go/codec v0.0.0-20181204163529-d75b2dcb6bc8/go.mod h1:VFNgLljTbGfSG7qAOspJ7OScBnGdDN/yBr0sguwnwf0= -github.com/valyala/bytebufferpool v1.0.0/go.mod h1:6bBcMArwyJ5K/AmCkWv1jt77kVWyCJ6HpOuEn7z0Csc= -github.com/valyala/fasttemplate v1.0.1/go.mod h1:UQGH1tvbgY+Nz5t2n7tXsz52dQxojPUpymEIMZ47gx8= +github.com/tklauser/numcpus v0.6.0 h1:kebhY2Qt+3U6RNK7UqpYNA+tJ23IBEGKkB7JQBfDYms= +github.com/tklauser/numcpus v0.6.0/go.mod h1:FEZLMke0lhOUG6w2JadTzp0a+Nl8PF/GFkQ5UVIcaL4= github.com/vishvananda/netlink v0.0.0-20210530105856-14e832ae1e8f h1:uKAQrhXS4xtXu+lAQ6Rh8rER4nVuUy9km9ayWDCF2x4= github.com/vishvananda/netlink v0.0.0-20210530105856-14e832ae1e8f/go.mod h1:twkDnbuQxJYemMlGd4JFIcuhgX83tXhKS2B/PRMpOho= -github.com/vishvananda/netns v0.0.0-20200728191858-db3c7e526aae h1:4hwBBUfQCFe3Cym0ZtKyq7L16eZUtYKs+BaHDN6mAns= github.com/vishvananda/netns v0.0.0-20200728191858-db3c7e526aae/go.mod h1:DD4vA1DwXk04H54A1oHXtwZmA0grkVMdPxx/VGLCah0= +github.com/vishvananda/netns v0.0.4 h1:Oeaw1EM2JMxD51g9uhtC0D7erkIjgmj8+JZc26m1YX8= +github.com/vishvananda/netns v0.0.4/go.mod h1:SpkAiCQRtJ6TvvxPnOSyH3BMl6unz3xZlaprSwhNNJM= github.com/vmihailenco/msgpack v4.0.4+incompatible h1:dSLoQfGFAo3F6OoNhwUmLwVgaUXK79GlxNBwueZn0xI= github.com/vmihailenco/msgpack v4.0.4+incompatible/go.mod h1:fy3FlTQTDXWkZ7Bh6AcGMlsjHatGryHQYUTf1ShIgkk= -github.com/willf/bitset v1.1.10 h1:NotGKqX0KwQ72NUzqrjZq5ipPNDQex9lo3WpaS8L2sc= -github.com/willf/bitset v1.1.10/go.mod h1:RjeCKbqT1RxIR/KWY6phxZiaY1IyutSBfGjNPySAYV4= -github.com/xordataexchange/crypt v0.0.3-0.20170626215501-b2862e3d0a77/go.mod h1:aYKd//L2LvnjZzWKhF00oedf4jCCReLcmhLdhm1A27Q= -github.com/yuin/goldmark v1.1.25/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= -github.com/yuin/goldmark v1.1.32/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= -github.com/yuin/goldmark v1.3.5/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1Zlc8k= github.com/yuin/goldmark v1.4.1/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1Zlc8k= github.com/yusufpapurcu/wmi v1.2.2 h1:KBNDSne4vP5mbSWnJbO+51IMOXJB67QiYCSBrubbPRg= github.com/yusufpapurcu/wmi v1.2.2/go.mod h1:SBZ9tNy3G9/m5Oi98Zks0QjeHVDvuK0qfxQmPyzfmi0= -go.etcd.io/bbolt v1.3.5/go.mod h1:G5EMThwa9y8QZGBClrRx5EY+Yw9kAhnjy3bSjsnlVTQ= -go.etcd.io/bbolt v1.3.6 h1:/ecaJf0sk1l4l6V4awd65v2C3ILy7MSj+s/x1ADCIMU= -go.etcd.io/bbolt v1.3.6/go.mod h1:qXsaaIqmgQH0T+OPdb99Bf+PKfBBQVAdyD6TY9G8XM4= -go.etcd.io/etcd/api/v3 v3.5.4 h1:OHVyt3TopwtUQ2GKdd5wu3PmmipR4FTwCqoEjSyRdIc= -go.etcd.io/etcd/api/v3 v3.5.4/go.mod h1:5GB2vv4A4AOn3yk7MftYGHkUfGtDHnEraIjym4dYz5A= -go.etcd.io/etcd/client/pkg/v3 v3.5.4 h1:lrneYvz923dvC14R54XcA7FXoZ3mlGZAgmwhfm7HqOg= -go.etcd.io/etcd/client/pkg/v3 v3.5.4/go.mod h1:IJHfcCEKxYu1Os13ZdwCwIUTUVGYTSAM3YSwc9/Ac1g= -go.etcd.io/etcd/client/v3 v3.5.4 h1:p83BUL3tAYS0OT/r0qglgc3M1JjhM0diV8DSWAhVXv4= -go.etcd.io/etcd/client/v3 v3.5.4/go.mod h1:ZaRkVgBZC+L+dLCjTcF1hRXpgZXQPOvnA/Ak/gq3kiY= -go.opencensus.io v0.21.0/go.mod h1:mSImk1erAIZhrmZN+AvHh14ztQfjbGwt4TtuofqLduU= -go.opencensus.io v0.22.0/go.mod h1:+kGneAE2xo2IficOXnaByMWTGM9T73dGwxeWcUqIpI8= -go.opencensus.io v0.22.2/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= -go.opencensus.io v0.22.3/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= -go.opencensus.io v0.22.4/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= +go.etcd.io/etcd/api/v3 v3.5.7 h1:sbcmosSVesNrWOJ58ZQFitHMdncusIifYcrBfwrlJSY= +go.etcd.io/etcd/api/v3 v3.5.7/go.mod h1:9qew1gCdDDLu+VwmeG+iFpL+QlpHTo7iubavdVDgCAA= +go.etcd.io/etcd/client/pkg/v3 v3.5.7 h1:y3kf5Gbp4e4q7egZdn5T7W9TSHUvkClN6u+Rq9mEOmg= +go.etcd.io/etcd/client/pkg/v3 v3.5.7/go.mod h1:o0Abi1MK86iad3YrWhgUsbGx1pmTS+hrORWc2CamuhY= +go.etcd.io/etcd/client/v3 v3.5.7 h1:u/OhpiuCgYY8awOHlhIhmGIGpxfBU/GZBUP3m/3/Iz4= +go.etcd.io/etcd/client/v3 v3.5.7/go.mod h1:sOWmj9DZUMyAngS7QQwCyAXXAL6WhgTOPLNS/NabQgw= go.uber.org/atomic v1.3.2/go.mod h1:gD2HeocX3+yG+ygLZcrzQJaqmWj9AIm7n08wl/qW/PE= go.uber.org/atomic v1.5.0/go.mod h1:sABNBOSYdrvTF6hTgEIbc7YasKWGhgEQZyfxyTvoXHQ= go.uber.org/atomic v1.6.0/go.mod h1:sABNBOSYdrvTF6hTgEIbc7YasKWGhgEQZyfxyTvoXHQ= @@ -553,402 +266,179 @@ go.uber.org/atomic v1.9.0/go.mod h1:fEN4uk6kAWBTFdckzkM89CLk9XfWZrxpCo0nPH17wJc= go.uber.org/atomic v1.10.0 h1:9qC72Qh0+3MqyJbAn8YU5xVq1frD8bn3JtD2oXtafVQ= go.uber.org/atomic v1.10.0/go.mod h1:LUxbIzbOniOlMKjJjyPfpl4v+PKK2cNJn91OQbhoJI0= go.uber.org/goleak v1.1.10/go.mod h1:8a7PlsEVH3e/a/GLqe5IIrQx6GzcnRmZEufDUTk4A7A= -go.uber.org/goleak v1.1.12 h1:gZAh5/EyT/HQwlpkCy6wTpqfH9H8Lz8zbm3dZh+OyzA= -go.uber.org/goleak v1.1.12/go.mod h1:cwTWslyiVhfpKIDGSZEM2HlOvcqm+tG4zioyIeLoqMQ= +go.uber.org/goleak v1.2.1 h1:NBol2c7O1ZokfZ0LEU9K6Whx/KnwvepVetCUhtKja4A= +go.uber.org/goleak v1.2.1/go.mod h1:qlT2yGI9QafXHhZZLxlSuNsMw3FFLxBr+tBRlmO1xH4= go.uber.org/multierr v1.1.0/go.mod h1:wR5kodmAFQ0UK8QlbwjlSNy0Z68gJhDJUG5sjR94q/0= go.uber.org/multierr v1.3.0/go.mod h1:VgVr7evmIr6uPjLBxg28wmKNXyqE9akIJ5XnfpiKl+4= go.uber.org/multierr v1.4.0/go.mod h1:VgVr7evmIr6uPjLBxg28wmKNXyqE9akIJ5XnfpiKl+4= go.uber.org/multierr v1.5.0/go.mod h1:FeouvMocqHpRaaGuG9EjoKcStLC43Zu/fmqdUMPcKYU= go.uber.org/multierr v1.6.0/go.mod h1:cdWPpRnG4AhwMwsgIHip0KRBQjJy5kYEpYjJxpXp9iU= go.uber.org/multierr v1.7.0/go.mod h1:7EAYxJLBy9rStEaz58O2t4Uvip6FSURkq8/ppBp95ak= -go.uber.org/multierr v1.8.0 h1:dg6GjLku4EH+249NNmoIciG9N/jURbDG+pFlTkhzIC8= -go.uber.org/multierr v1.8.0/go.mod h1:7EAYxJLBy9rStEaz58O2t4Uvip6FSURkq8/ppBp95ak= +go.uber.org/multierr v1.11.0 h1:blXXJkSxSSfBVBlC76pxqeO+LN3aDfLQo+309xJstO0= +go.uber.org/multierr v1.11.0/go.mod h1:20+QtiLqy0Nd6FdQB9TLXag12DsQkrbs3htMFfDN80Y= go.uber.org/tools v0.0.0-20190618225709-2cfd321de3ee/go.mod h1:vJERXedbb3MVM5f9Ejo0C68/HhF8uaILCdgjnY+goOA= go.uber.org/zap v1.9.1/go.mod h1:vwi/ZaCAaUcBkycHslxD9B2zi4UTXhF60s6SWpuDF0Q= go.uber.org/zap v1.12.0/go.mod h1:zwrFLgMcdUuIBviXEYEH1YKNaOBnKXsx2IPda5bBwHM= go.uber.org/zap v1.15.0/go.mod h1:Mb2vm2krFEG5DV0W9qcHBYFtp/Wku1cvYaqPsS/WYfc= -go.uber.org/zap v1.17.0/go.mod h1:MXVU+bhUf/A7Xi2HNOnopQOrmycQ5Ih87HtOu4q5SSo= go.uber.org/zap v1.19.0/go.mod h1:xg/QME4nWcxGxrpdeYfq7UvYrLh66cuVKdrbD1XF/NI= -go.uber.org/zap v1.23.0 h1:OjGQ5KQDEUawVHxNwQgPpiypGHOxo2mNZsOqTak4fFY= -go.uber.org/zap v1.23.0/go.mod h1:D+nX8jyLsMHMYrln8A0rJjFt/T/9/bGgIhAqxv5URuY= -golang.org/x/crypto v0.0.0-20180904163835-0709b304e793/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= -golang.org/x/crypto v0.0.0-20181203042331-505ab145d0a9/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= +go.uber.org/zap v1.24.0 h1:FiJd5l1UOLj0wCgbSE0rwwXHzEdAZS6hiiSnxJN/D60= +go.uber.org/zap v1.24.0/go.mod h1:2kMP+WWQ8aoFoedH3T2sq6iJ2yDWpHbP0f6MQbS9Gkg= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= golang.org/x/crypto v0.0.0-20190510104115-cbcb75029529/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= -golang.org/x/crypto v0.0.0-20190605123033-f99c8df09eb5/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20200323165209-0ec3e9974c59/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/crypto v0.0.0-20210220033148-5ea612d1eb83/go.mod h1:jdWPYTVW3xRLrWPugEBEK3UY2ZEsg3UU495nc5E+M+I= golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= golang.org/x/crypto v0.0.0-20220331220935-ae2d96664a29/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4= -golang.org/x/crypto v0.0.0-20220829220503-c86fa9a7ed90 h1:Y/gsMcFOcR+6S6f3YeMKl5g+dZMEWqcz5Czj/GWYbkM= -golang.org/x/crypto v0.0.0-20220829220503-c86fa9a7ed90/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4= +golang.org/x/crypto v0.7.0 h1:AvwMYaRytfdeVt3u6mLaxYtErKYjxA2OXjJ1HHq6t3A= +golang.org/x/crypto v0.7.0/go.mod h1:pYwdfH91IfpZVANVyUOhSIPZaFoJGxTFbZhFTx+dXZU= golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= -golang.org/x/exp v0.0.0-20190306152737-a1d7652674e8/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= -golang.org/x/exp v0.0.0-20190510132918-efd6b22b2522/go.mod h1:ZjyILWgesfNpC6sMxTJOJm9Kp84zZh5NQWvqDGG3Qr8= -golang.org/x/exp v0.0.0-20190829153037-c13cbed26979/go.mod h1:86+5VVa7VpoJ4kLfm080zCjGlMRFzhUhsZKEZO7MGek= -golang.org/x/exp v0.0.0-20191030013958-a1ab85dbe136/go.mod h1:JXzH8nQsPlswgeRAPE3MuO9GYsAcnJvJ4vnMwN/5qkY= -golang.org/x/exp v0.0.0-20191129062945-2f5052295587/go.mod h1:2RIsYlXP63K8oxa1u096TMicItID8zy7Y6sNkU49FU4= -golang.org/x/exp v0.0.0-20191227195350-da58074b4299/go.mod h1:2RIsYlXP63K8oxa1u096TMicItID8zy7Y6sNkU49FU4= -golang.org/x/exp v0.0.0-20200119233911-0405dc783f0a/go.mod h1:2RIsYlXP63K8oxa1u096TMicItID8zy7Y6sNkU49FU4= -golang.org/x/exp v0.0.0-20200207192155-f17229e696bd/go.mod h1:J/WKrq2StrnmMY6+EHIKF9dgMWnmCNThgcyBT1FY9mM= -golang.org/x/exp v0.0.0-20200224162631-6cc2880d07d6/go.mod h1:3jZMyOhIsHpP37uCMkUooju7aAi5cS1Q23tOzKc+0MU= golang.org/x/exp/typeparams v0.0.0-20220218215828-6cf2b201936e/go.mod h1:AbB0pIl9nAr9wVwH+Z2ZpaocVmF5I4GyWCDIsVjR0bk= -golang.org/x/exp/typeparams v0.0.0-20220613132600-b0d781184e0d h1:+W8Qf4iJtMGKkyAygcKohjxTk4JPsL9DpzApJ22m5Ic= golang.org/x/exp/typeparams v0.0.0-20220613132600-b0d781184e0d/go.mod h1:AbB0pIl9nAr9wVwH+Z2ZpaocVmF5I4GyWCDIsVjR0bk= -golang.org/x/image v0.0.0-20190227222117-0694c2d4d067/go.mod h1:kZ7UVZpmo3dzQBMxlp+ypCbDeSB+sBbTgSJuh5dn5js= -golang.org/x/image v0.0.0-20190802002840-cff245a6509b/go.mod h1:FeLwcggjj3mMvU+oOTbSwawSJRM1uh48EjtB4UJZlP0= +golang.org/x/exp/typeparams v0.0.0-20230321023759-10a507213a29 h1:e7LhZmJ631l59keHP9ssC3sgSn3/oiEHKHKXDkimURY= +golang.org/x/exp/typeparams v0.0.0-20230321023759-10a507213a29/go.mod h1:AbB0pIl9nAr9wVwH+Z2ZpaocVmF5I4GyWCDIsVjR0bk= golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE= golang.org/x/lint v0.0.0-20190227174305-5b3e6a55c961/go.mod h1:wehouNa3lNwaWXcvxsM5YxQ5yQlVC4a0KAMCusXpPoU= -golang.org/x/lint v0.0.0-20190301231843-5614ed5bae6f/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE= golang.org/x/lint v0.0.0-20190313153728-d0100b6bd8b3/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= -golang.org/x/lint v0.0.0-20190409202823-959b441ac422/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= -golang.org/x/lint v0.0.0-20190909230951-414d861bb4ac/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= golang.org/x/lint v0.0.0-20190930215403-16217165b5de/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= -golang.org/x/lint v0.0.0-20191125180803-fdd1cda4f05f/go.mod h1:5qLYkcX4OjUUV8bRuDixDT3tpyyb+LUpUlRWLxfhWrs= -golang.org/x/lint v0.0.0-20200130185559-910be7a94367/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY= -golang.org/x/lint v0.0.0-20200302205851-738671d3881b/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY= golang.org/x/lint v0.0.0-20210508222113-6edffad5e616 h1:VLliZ0d+/avPrXXH+OakdXhpJuEoBZuwh1m2j7U6Iug= golang.org/x/lint v0.0.0-20210508222113-6edffad5e616/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY= -golang.org/x/mobile v0.0.0-20190312151609-d3739f865fa6/go.mod h1:z+o9i4GpDbdi3rU15maQ/Ox0txvL9dWGYEHz965HBQE= -golang.org/x/mobile v0.0.0-20190719004257-d2bd2a29d028/go.mod h1:E/iHnbuqvinMTCcRqshq8CkpyQDoeVncDDYHnLhea+o= golang.org/x/mod v0.0.0-20190513183733-4bf6d317e70e/go.mod h1:mXi4GBBbnImb6dmsKGUJ2LatrhH/nqhxcFungHvyanc= -golang.org/x/mod v0.1.0/go.mod h1:0QHyrYULN0/3qlju5TqG8bIK38QM8yzMo5ekMj3DlcY= golang.org/x/mod v0.1.1-0.20191105210325-c90efee705ee/go.mod h1:QqPTAvyqsEbceGzBzNggFXnrqF1CaUcvgkdR5Ot7KZg= -golang.org/x/mod v0.1.1-0.20191107180719-034126e5016b/go.mod h1:QqPTAvyqsEbceGzBzNggFXnrqF1CaUcvgkdR5Ot7KZg= golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= -golang.org/x/mod v0.4.2/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= -golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4 h1:6zppjxzCulZykYSLyVDYbneBfbaBIQPYMevg0bEwv2s= golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4= +golang.org/x/mod v0.9.0 h1:KENHtAZL2y3NLMYZeHY9DW8HW8V+kQyJsY/V9JlKvCs= +golang.org/x/mod v0.9.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs= golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= -golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= -golang.org/x/net v0.0.0-20181114220301-adae6a3d119a/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20190108225652-1e06a53dbb7e/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20190213061140-3a22650c66bd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= -golang.org/x/net v0.0.0-20190501004415-9ce7a6920f09/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= -golang.org/x/net v0.0.0-20190503192946-f4e77d36d62c/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= golang.org/x/net v0.0.0-20190603091049-60506f45cf65/go.mod h1:HSz+uSET+XFnRR8LxR5pz3Of3rY3CfYBVs4xY44aLks= -golang.org/x/net v0.0.0-20190613194153-d28f0bde5980/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= -golang.org/x/net v0.0.0-20190628185345-da137c7871d7/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= -golang.org/x/net v0.0.0-20190724013045-ca1201d0de80/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= -golang.org/x/net v0.0.0-20191002035440-2ec189313ef0/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= -golang.org/x/net v0.0.0-20191209160850-c0dbc17a3553/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= -golang.org/x/net v0.0.0-20200114155413-6afb5195e5aa/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= -golang.org/x/net v0.0.0-20200202094626-16171245cfb2/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= -golang.org/x/net v0.0.0-20200222125558-5a598a2470a0/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20200226121028-0de0cce0169b/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= -golang.org/x/net v0.0.0-20200301022130-244492dfa37a/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= -golang.org/x/net v0.0.0-20200324143707-d3edc9973b7e/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= -golang.org/x/net v0.0.0-20200501053045-e0ff5e5a1de5/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= -golang.org/x/net v0.0.0-20200506145744-7e3656a0809f/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= -golang.org/x/net v0.0.0-20200513185701-a91f0712d120/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= -golang.org/x/net v0.0.0-20200520004742-59133d7f0dd7/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= -golang.org/x/net v0.0.0-20200520182314-0ba52f642ac2/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= -golang.org/x/net v0.0.0-20200625001655-4c5254603344/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA= -golang.org/x/net v0.0.0-20200707034311-ab3426394381/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA= -golang.org/x/net v0.0.0-20200813134508-3edf25e44fcc/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA= golang.org/x/net v0.0.0-20200822124328-c89045814202/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA= golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= -golang.org/x/net v0.0.0-20210405180319-a5a99cb37ef4/go.mod h1:p54w0d4576C0XHj96bSt6lcn1PtDYWL6XObtHCRCNQM= -golang.org/x/net v0.0.0-20210428140749-89ef3d95e781/go.mod h1:OJAsFXCWl8Ukc7SiCT/9KSuxbyM7479/AVlXFRxuMCk= -golang.org/x/net v0.0.0-20210525063256-abc453219eb5/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/net v0.0.0-20211015210444-4f30a5c0130f/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/net v0.0.0-20211112202133-69e39bad7dc2/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= -golang.org/x/net v0.0.0-20220127200216-cd36cc0744dd/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk= -golang.org/x/net v0.0.0-20220225172249-27dd8689420f/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk= -golang.org/x/net v0.0.0-20220722155237-a158d28d115b h1:PxfKdU9lEEDYjdIzOtC4qFWgkU2rGHdKlKowJSMN9h0= -golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c= +golang.org/x/net v0.8.0 h1:Zrh2ngAOFYneWTAIAPethzeaQLuHwhuBkuV6ZiRnUaQ= +golang.org/x/net v0.8.0/go.mod h1:QVkue5JL9kW//ek3r6jTKnTFis1tRmNAW2P1shuFdJc= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= -golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= -golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= -golang.org/x/oauth2 v0.0.0-20191202225959-858c2ad4c8b6/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= golang.org/x/oauth2 v0.0.0-20200107190931-bf48bf16ab8d/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= -golang.org/x/oauth2 v0.0.0-20210514164344-f6687ab2804c/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= -golang.org/x/oauth2 v0.0.0-20220223155221-ee480838109b/go.mod h1:DAh4E804XQdzx2j+YRIaUnCqCV2RuMz24cGBJ5QYIrc= golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.0.0-20190227155943-e225da77a7e6/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.0.0-20200317015054-43a5402ce75a/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.0.0-20200625203802-6e8e738ad208/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.0.0-20201207232520-09787c993a3a/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20210220032951-036812b2e83c/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.0.0-20220819030929-7fc1605a5dde h1:ejfdSekXMDxDLbRrJMwUk6KnSLZ2McaUCVcIKM+N6jc= -golang.org/x/sync v0.0.0-20220819030929-7fc1605a5dde/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.1.0 h1:wsuoTGHzEhffawBOhz5CYhcrV4IdKZbEyZjBMuTp12o= +golang.org/x/sync v0.1.0/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= -golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= -golang.org/x/sys v0.0.0-20180909124046-d0be0721c37e/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= -golang.org/x/sys v0.0.0-20181116152217-5ac8a444bdc5/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= -golang.org/x/sys v0.0.0-20181128092732-4ed8d59d0b35/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= -golang.org/x/sys v0.0.0-20181205085412-a5c9d58dba9a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= -golang.org/x/sys v0.0.0-20181221143128-b4a75ba826a6/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= -golang.org/x/sys v0.0.0-20190312061237-fead79001313/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20190422165155-953cdadca894/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20190502145724-3ef323f4f1fd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20190507160741-ecd444e8653b/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20190606165138-5da285871e9c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20190624142023-c5567b49c5d0/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20190726091711-fc99dfbffb4e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20190813064441-fde4db37ae7a/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20190904154756-749cb33beabd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190916202348-b4ddaad3f8a3/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20191001151750-bb3f8db39f24/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20191005200804-aed5e4c7ecf9/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191026070338-33540a1f6037/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20191120155948-bd437916bb0e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20191204072324-ce4227a45e2e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20191228213918-04cbcbbfeed8/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200106162015-b016eb3dc98e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200113162924-86b910548bc1/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200116001909-b77594299b42/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200122134326-e047566fdf82/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200202164722-d101bd2416d5/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200212091648-12a6c2dcc1e4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200217220822-9197077df867/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200219091948-cb0a6d8edb6c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200223170610-d5e6a3e2c0ae/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200302150141-5c8b2ff67527/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200323222414-85ca7c5b95cd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200331124033-c3d80250170d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200501052902-10377860bb8e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200511232937-7e40ca221e25/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200515095857-1151b9dac4a9/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200519105757-fe76b779f299/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200523222454-059865788121/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200615200032-f1bc736245b1/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200625212154-ddb9806d33ae/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200728102440-3e129f6d46b1/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200803210538-64077c9b5642/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200814200057-3d37ad5750ed/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200923182605-d9f96fdee20d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20210112080510-489259a85091/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20210124154548-22da62e12c0c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20210330210617-4fbd30eecc44/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20210403161142-5e06dd20ab57/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20210510120138-977fb7262007/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20210603081109-ebe580a85c40/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20210630005230-0f9fa26af87c/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20210816074244-15123e1e1f71/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20210927094055-39ccf1dd6fa6/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20211019181941-9d821ace8654/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20211216021012-1d35b9e2eb4e/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20220114195835-da31bd327af9/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20220128215802-99c3d69c2c27/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20220503163025-988cb79eb6c6/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220627191245-f75cf1eec38b/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20220829200755-d48e67d00261 h1:v6hYoSR9T5oet+pMXwUWkbiVqx/63mlHjefrHmxwfeY= +golang.org/x/sys v0.0.0-20220704084225-05e143d24a9e/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220811171246-fbc7d0a398ab/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220829200755-d48e67d00261/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.2.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.6.0 h1:MVltZSvRTcU2ljQOhs94SXPftV6DCNnZViHeQps87pQ= +golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/term v0.0.0-20201117132131-f5c789dd3221/go.mod h1:Nr5EML6q2oocZ2LXRh80K7BxOlk5/8JxuGnuhpl+muw= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= -golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= -golang.org/x/term v0.0.0-20220722155259-a9ba230a4035 h1:Q5284mrmYTpACcm+eAKjKJH48BBwSyfJqmmGDTtT8Vc= -golang.org/x/term v0.0.0-20220722155259-a9ba230a4035/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= -golang.org/x/text v0.0.0-20170915032832-14c0d48ead0c/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= +golang.org/x/term v0.6.0 h1:clScbb1cHjoCkyRbWwBEUZ5H/tIFu5TAXIqaZD0Gcjw= +golang.org/x/term v0.6.0/go.mod h1:m6U89DPEgQRMq3DNkDClhWw02AUbt2daBVO4cn4Hv9U= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= -golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk= golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= -golang.org/x/text v0.3.5/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= -golang.org/x/text v0.3.7 h1:olpwvP2KacW1ZWvsR7uQhoyTYvKAupfQrRGBFM352Gk= golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= -golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= -golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= -golang.org/x/time v0.0.0-20191024005414-555d28b269f0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= +golang.org/x/text v0.8.0 h1:57P1ETyNKtuIjB4SRd15iJxuhj8Gc416Y78H3qgMh68= +golang.org/x/text v0.8.0/go.mod h1:e1OnstbJyHTd6l/uOt8jFFHp6TRDWZR/bV3emEE/zU8= golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20190114222345-bf090417da8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20190226205152-f727befe758c/go.mod h1:9Yl7xja0Znq3iFh3HoIrodX9oNMXvdceNzlUR8zjMvY= golang.org/x/tools v0.0.0-20190311212946-11955173bddd/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= -golang.org/x/tools v0.0.0-20190312151545-0bb0c0a6e846/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= -golang.org/x/tools v0.0.0-20190312170243-e65039ee4138/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= -golang.org/x/tools v0.0.0-20190425150028-36563e24a262/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= -golang.org/x/tools v0.0.0-20190506145303-2d16b83fe98c/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= golang.org/x/tools v0.0.0-20190524140312-2c0ae7006135/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= -golang.org/x/tools v0.0.0-20190606124116-d0a3d012864b/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= golang.org/x/tools v0.0.0-20190621195816-6e04913cbbac/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= -golang.org/x/tools v0.0.0-20190628153133-6cdbf07be9d0/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= -golang.org/x/tools v0.0.0-20190816200558-6889da9d5479/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= -golang.org/x/tools v0.0.0-20190911174233-4f2ddba30aff/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= -golang.org/x/tools v0.0.0-20191012152004-8de300cfc20a/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20191029041327-9cc4af7d6b2c/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20191029190741-b9c20aec41a5/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20191107010934-f79515f33823/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20191108193012-7d206e10da11/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= -golang.org/x/tools v0.0.0-20191113191852-77e3bb0ad9e7/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= -golang.org/x/tools v0.0.0-20191115202509-3a792d9c32b2/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= -golang.org/x/tools v0.0.0-20191125144606-a911d9008d1f/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= -golang.org/x/tools v0.0.0-20191130070609-6e064ea0cf2d/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= -golang.org/x/tools v0.0.0-20191216173652-a0e659d51361/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= -golang.org/x/tools v0.0.0-20191227053925-7b8e75db28f4/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= -golang.org/x/tools v0.0.0-20200117161641-43d50277825c/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= -golang.org/x/tools v0.0.0-20200122220014-bf1340f18c4a/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= golang.org/x/tools v0.0.0-20200130002326-2f3ba24bd6e7/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= -golang.org/x/tools v0.0.0-20200204074204-1cc6d1ef6c74/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= -golang.org/x/tools v0.0.0-20200207183749-b753a1ba74fa/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= -golang.org/x/tools v0.0.0-20200212150539-ea181f53ac56/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= -golang.org/x/tools v0.0.0-20200224181240-023911ca70b2/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= -golang.org/x/tools v0.0.0-20200227222343-706bc42d1f0d/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= -golang.org/x/tools v0.0.0-20200304193943-95d2e580d8eb/go.mod h1:o4KQGtdN14AW+yjsvvwRTJJuXz8XRtIHtEnmAXLyFUw= -golang.org/x/tools v0.0.0-20200312045724-11d5b4c81c7d/go.mod h1:o4KQGtdN14AW+yjsvvwRTJJuXz8XRtIHtEnmAXLyFUw= -golang.org/x/tools v0.0.0-20200331025713-a30bf2db82d4/go.mod h1:Sl4aGygMT6LrqrWclx+PTx3U+LnKx/seiNR+3G19Ar8= -golang.org/x/tools v0.0.0-20200501065659-ab2804fb9c9d/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= -golang.org/x/tools v0.0.0-20200512131952-2bc93b1c0c88/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= -golang.org/x/tools v0.0.0-20200515010526-7d3b6ebf133d/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= -golang.org/x/tools v0.0.0-20200618134242-20370b0cb4b2/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= golang.org/x/tools v0.0.0-20200619180055-7c47624df98f/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= -golang.org/x/tools v0.0.0-20200729194436-6467de6f59a7/go.mod h1:njjCfa9FT2d7l9Bc6FUM5FLjQPp3cFF28FI3qnDFljA= -golang.org/x/tools v0.0.0-20200804011535-6c149bb5ef0d/go.mod h1:njjCfa9FT2d7l9Bc6FUM5FLjQPp3cFF28FI3qnDFljA= -golang.org/x/tools v0.0.0-20200825202427-b303f430e36d/go.mod h1:njjCfa9FT2d7l9Bc6FUM5FLjQPp3cFF28FI3qnDFljA= -golang.org/x/tools v0.0.0-20201022035929-9cf592e881e9/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= -golang.org/x/tools v0.0.0-20201224043029-2b0845dc783e/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= golang.org/x/tools v0.0.0-20210106214847-113979e3529a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= -golang.org/x/tools v0.1.2/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= -golang.org/x/tools v0.1.5/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= golang.org/x/tools v0.1.11-0.20220513221640-090b14e8501f/go.mod h1:SgwaegtQh8clINPpECJMqnxLv9I09HLqnW3RMqW0CA4= golang.org/x/tools v0.1.11/go.mod h1:SgwaegtQh8clINPpECJMqnxLv9I09HLqnW3RMqW0CA4= -golang.org/x/tools v0.1.12 h1:VveCTK38A2rkS8ZqFY25HIDFscX5X9OoEhJd3quQmXU= -golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc= +golang.org/x/tools v0.7.0 h1:W4OVu8VVOaIO0yzWMNdepAulS7YfoS3Zabrm8DOXXU4= +golang.org/x/tools v0.7.0/go.mod h1:4pg6aUX35JBAogB10C9AtvVL+qowtN4pT3CGSQex14s= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= -google.golang.org/api v0.4.0/go.mod h1:8k5glujaEP+g9n7WNsDg8QP6cUVNI86fCNMcbazEtwE= -google.golang.org/api v0.7.0/go.mod h1:WtwebWUNSVBH/HAw79HIFXZNqEvBhG+Ra+ax0hx3E3M= -google.golang.org/api v0.8.0/go.mod h1:o4eAsZoiT+ibD93RtjEohWalFOjRDx6CVaqeizhEnKg= -google.golang.org/api v0.9.0/go.mod h1:o4eAsZoiT+ibD93RtjEohWalFOjRDx6CVaqeizhEnKg= -google.golang.org/api v0.13.0/go.mod h1:iLdEw5Ide6rF15KTC1Kkl0iskquN2gFfn9o9XIsbkAI= -google.golang.org/api v0.14.0/go.mod h1:iLdEw5Ide6rF15KTC1Kkl0iskquN2gFfn9o9XIsbkAI= -google.golang.org/api v0.15.0/go.mod h1:iLdEw5Ide6rF15KTC1Kkl0iskquN2gFfn9o9XIsbkAI= -google.golang.org/api v0.17.0/go.mod h1:BwFmGc8tA3vsd7r/7kR8DY7iEEGSU04BFxCo5jP/sfE= -google.golang.org/api v0.18.0/go.mod h1:BwFmGc8tA3vsd7r/7kR8DY7iEEGSU04BFxCo5jP/sfE= -google.golang.org/api v0.19.0/go.mod h1:BwFmGc8tA3vsd7r/7kR8DY7iEEGSU04BFxCo5jP/sfE= -google.golang.org/api v0.20.0/go.mod h1:BwFmGc8tA3vsd7r/7kR8DY7iEEGSU04BFxCo5jP/sfE= -google.golang.org/api v0.22.0/go.mod h1:BwFmGc8tA3vsd7r/7kR8DY7iEEGSU04BFxCo5jP/sfE= -google.golang.org/api v0.24.0/go.mod h1:lIXQywCXRcnZPGlsd8NbLnOjtAoL6em04bJ9+z0MncE= -google.golang.org/api v0.28.0/go.mod h1:lIXQywCXRcnZPGlsd8NbLnOjtAoL6em04bJ9+z0MncE= -google.golang.org/api v0.29.0/go.mod h1:Lcubydp8VUV7KeIHD9z2Bys/sm/vGKnG1UHuDBSrHWM= -google.golang.org/api v0.30.0/go.mod h1:QGmEvQ87FHZNiUVJkT14jQNYJ4ZJjdRF23ZXz5138Fc= google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM= google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= -google.golang.org/appengine v1.5.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= -google.golang.org/appengine v1.6.1/go.mod h1:i06prIuMbXzDqacNJfV5OdTW448YApPu5ww/cMBSeb0= -google.golang.org/appengine v1.6.5/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc= google.golang.org/appengine v1.6.6/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc= google.golang.org/appengine v1.6.7 h1:FZR1q0exgwxzPzp/aF+VccGrSfxfPpkBqjIIEq3ru6c= google.golang.org/appengine v1.6.7/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc= google.golang.org/genproto v0.0.0-20180817151627-c66870c02cf8/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc= -google.golang.org/genproto v0.0.0-20190307195333-5fe7a883aa19/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= -google.golang.org/genproto v0.0.0-20190418145605-e7d98fc518a7/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= -google.golang.org/genproto v0.0.0-20190425155659-357c62f0e4bb/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= -google.golang.org/genproto v0.0.0-20190502173448-54afdca5d873/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= -google.golang.org/genproto v0.0.0-20190801165951-fa694d86fc64/go.mod h1:DMBHOl98Agz4BDEuKkezgsaosCRResVns1a3J2ZsMNc= google.golang.org/genproto v0.0.0-20190819201941-24fa4b261c55/go.mod h1:DMBHOl98Agz4BDEuKkezgsaosCRResVns1a3J2ZsMNc= -google.golang.org/genproto v0.0.0-20190911173649-1774047e7e51/go.mod h1:IbNlFCBrqXvoKpeg0TB2l7cyZUmoaFKYIwrEpbDKLA8= -google.golang.org/genproto v0.0.0-20190927181202-20e1ac93f88c/go.mod h1:IbNlFCBrqXvoKpeg0TB2l7cyZUmoaFKYIwrEpbDKLA8= -google.golang.org/genproto v0.0.0-20191108220845-16a3f7862a1a/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= -google.golang.org/genproto v0.0.0-20191115194625-c23dd37a84c9/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= -google.golang.org/genproto v0.0.0-20191216164720-4f79533eabd1/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= -google.golang.org/genproto v0.0.0-20191230161307-f3c370f40bfb/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= -google.golang.org/genproto v0.0.0-20200115191322-ca5a22157cba/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= -google.golang.org/genproto v0.0.0-20200122232147-0452cf42e150/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= -google.golang.org/genproto v0.0.0-20200204135345-fa8e72b47b90/go.mod h1:GmwEX6Z4W5gMy59cAlVYjN9JhxgbQH6Gn+gFDQe2lzA= -google.golang.org/genproto v0.0.0-20200212174721-66ed5ce911ce/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= -google.golang.org/genproto v0.0.0-20200224152610-e50cd9704f63/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= -google.golang.org/genproto v0.0.0-20200228133532-8c2c7df3a383/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= -google.golang.org/genproto v0.0.0-20200305110556-506484158171/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= -google.golang.org/genproto v0.0.0-20200312145019-da6875a35672/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= -google.golang.org/genproto v0.0.0-20200331122359-1ee6d9798940/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= -google.golang.org/genproto v0.0.0-20200430143042-b979b6f78d84/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= -google.golang.org/genproto v0.0.0-20200511104702-f5ebc3bea380/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= google.golang.org/genproto v0.0.0-20200513103714-09dca8ec2884/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= -google.golang.org/genproto v0.0.0-20200515170657-fc4c6c6a6587/go.mod h1:YsZOwe1myG/8QRHRsmBRE1LrgQY60beZKjly0O1fX9U= -google.golang.org/genproto v0.0.0-20200526211855-cb27e3aa2013/go.mod h1:NbSheEEYHJ7i3ixzK3sjbqSGDJWnxyFXZblF3eUsNvo= -google.golang.org/genproto v0.0.0-20200618031413-b414f8b61790/go.mod h1:jDfRM7FcilCzHH/e9qn6dsT145K34l5v+OpcnNgKAAA= -google.golang.org/genproto v0.0.0-20200729003335-053ba62fc06f/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= -google.golang.org/genproto v0.0.0-20200804131852-c06518451d9c/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= -google.golang.org/genproto v0.0.0-20200825200019-8632dd797987/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= -google.golang.org/genproto v0.0.0-20210602131652-f16073e35f0c/go.mod h1:UODoCrxHCcBojKKwX1terBiRUaqAsFqJiF615XL43r0= -google.golang.org/genproto v0.0.0-20220829175752-36a9c930ecbf h1:Q5xNKbTSFwkuaaGaR7CMcXEM5sy19KYdUU8iF8/iRC0= -google.golang.org/genproto v0.0.0-20220829175752-36a9c930ecbf/go.mod h1:dbqgFATTzChvnt+ujMdZwITVAJHFtfyN1qUhDqEiIlk= +google.golang.org/genproto v0.0.0-20230330200707-38013875ee22 h1:n3ThVoQnHbCbnkhZZ1fx3+3fBAisViSwrpbtLV7vydY= +google.golang.org/genproto v0.0.0-20230330200707-38013875ee22/go.mod h1:UUQDJDOlWu4KYeJZffbWgBkS1YFobzKbLVfK69pe0Ak= google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c= -google.golang.org/grpc v1.20.1/go.mod h1:10oTOabMzJvdu6/UiuZezV6QK5dSlG84ov/aaiqXj38= -google.golang.org/grpc v1.21.1/go.mod h1:oYelfM1adQP15Ek0mdvEgi9Df8B9CZIaU1084ijfRaM= google.golang.org/grpc v1.23.0/go.mod h1:Y5yQAOtifL1yxbo5wqy6BxZv8vAUGQwXBOALyacEbxg= -google.golang.org/grpc v1.24.0/go.mod h1:XDChyiUovWa60DnaeDeZmSW86xtLtjtZbwvSiRnRtcA= google.golang.org/grpc v1.25.1/go.mod h1:c3i+UQWmh7LiEpx4sFZnkU36qjEYZ0imhYfXVyQciAY= -google.golang.org/grpc v1.26.0/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk= google.golang.org/grpc v1.27.0/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk= -google.golang.org/grpc v1.27.1/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk= -google.golang.org/grpc v1.28.0/go.mod h1:rpkK4SK4GF4Ach/+MFLZUBavHOvF2JJB5uozKKal+60= -google.golang.org/grpc v1.29.1/go.mod h1:itym6AZVZYACWQqET3MqgPpjcuV5QH3BxFS3IjizoKk= -google.golang.org/grpc v1.30.0/go.mod h1:N36X2cJ7JwdamYAgDz+s+rVMFjt3numwzf/HckM8pak= -google.golang.org/grpc v1.31.0/go.mod h1:N36X2cJ7JwdamYAgDz+s+rVMFjt3numwzf/HckM8pak= google.golang.org/grpc v1.33.1/go.mod h1:fr5YgcSWrqhRRxogOsw7RzIpsmvOZ6IcH4kBYTpR3n0= -google.golang.org/grpc v1.38.0/go.mod h1:NREThFqKR1f3iQ6oBuvc5LadQuXVGo9rkm5ZGrQdJfM= -google.golang.org/grpc v1.49.0 h1:WTLtQzmQori5FUH25Pq4WT22oCsv8USpQ+F6rqtsmxw= -google.golang.org/grpc v1.49.0/go.mod h1:ZgQEeidpAuNRZ8iRrlBKXZQP1ghovWIVhdJRyCDK+GI= -google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8= -google.golang.org/protobuf v0.0.0-20200221191635-4d8936d0db64/go.mod h1:kwYJMbMJ01Woi6D6+Kah6886xMZcty6N08ah7+eCXa0= -google.golang.org/protobuf v0.0.0-20200228230310-ab0ca4ff8a60/go.mod h1:cfTl7dwQJ+fmap5saPgwCLgHXTUD7jkjRqWcaiX5VyM= -google.golang.org/protobuf v1.20.1-0.20200309200217-e05f789c0967/go.mod h1:A+miEFZTKqfCUM6K7xSMQL9OKL/b6hQv+e19PK+JZNE= -google.golang.org/protobuf v1.21.0/go.mod h1:47Nbq4nVaFHyn7ilMalzfO3qCViNmqZ2kzikPIcrTAo= -google.golang.org/protobuf v1.22.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU= -google.golang.org/protobuf v1.23.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU= -google.golang.org/protobuf v1.23.1-0.20200526195155-81db48ad09cc/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU= -google.golang.org/protobuf v1.24.0/go.mod h1:r/3tXBNzIEhYS9I1OUVjXDlt8tc493IdKGjtUeSXeh4= -google.golang.org/protobuf v1.25.0/go.mod h1:9JNX74DMeImyA3h4bdi1ymwjUzf21/xIlbajtzgsN7c= +google.golang.org/grpc v1.54.0 h1:EhTqbhiYeixwWQtAEZAxmV9MGqcjEU2mFx52xCzNyag= +google.golang.org/grpc v1.54.0/go.mod h1:PUSEXI6iWghWaB6lXM4knEgpJNu2qUcKfDtNci3EC2g= google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw= google.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc= -google.golang.org/protobuf v1.28.1 h1:d0NfwRgPtno5B1Wa6L2DAG+KivqkdutMf1UhdNx175w= -google.golang.org/protobuf v1.28.1/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I= -gopkg.in/alecthomas/kingpin.v2 v2.2.6/go.mod h1:FMv+mEhP44yOT+4EoQTLFTRgOQ1FBLkstjWtayDeSgw= +google.golang.org/protobuf v1.30.0 h1:kPPoIgf3TsEvrm0PFe15JQ+570QVxYzEvvHqChK+cng= +google.golang.org/protobuf v1.30.0/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk= gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q= gopkg.in/errgo.v2 v2.1.0/go.mod h1:hNsd1EY+bozCKY1Ytp96fpM3vjJbqLJn88ws8XvfDNI= -gopkg.in/fsnotify.v1 v1.4.7/go.mod h1:Tz8NjZHkW78fSQdbUxIjBTcgA1z1m8ZHf0WmKUhAMys= gopkg.in/ini.v1 v1.67.0 h1:Dgnx+6+nfE+IfzjUEISNeydPJh9AXNNsWbGP9KzCsOA= gopkg.in/ini.v1 v1.67.0/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k= -gopkg.in/natefinch/lumberjack.v2 v2.0.0 h1:1Lc07Kr7qY4U2YPouBjpCLxpiyxIVoxqXgkXLknAOE8= gopkg.in/natefinch/lumberjack.v2 v2.0.0/go.mod h1:l0ndWWf7gzL7RNwBG7wST/UCcT4T24xpD6X8LsfU/+k= -gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7 h1:uRGJdciOHaEIrze2W8Q3AKkepLTh2hOroT7a+7czfdQ= -gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7/go.mod h1:dt/ZhP58zS4L8KSrWDmTeBkI65Dw0HsyUHuEVlX15mw= +gopkg.in/natefinch/lumberjack.v2 v2.2.1 h1:bBRl1b0OH9s/DuPhuXpNl+VtCaJXFZ5/uEFST95x9zc= +gopkg.in/natefinch/lumberjack.v2 v2.2.1/go.mod h1:YD8tP3GAjkrDg1eZH7EGmyESg/lsYskCTPBJVb9jqSc= gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gopkg.in/yaml.v3 v3.0.0-20220512140231-539c8e751b99/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= -honnef.co/go/tools v0.0.0-20190106161140-3f1c8253044a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= -honnef.co/go/tools v0.0.0-20190418001031-e561f6794a2a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= honnef.co/go/tools v0.0.0-20190523083050-ea95bdfd59fc/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= honnef.co/go/tools v0.0.1-2019.2.3/go.mod h1:a3bituU0lyd329TUQxRnasdCoJDkEUEAqEt0JzvZhAg= -honnef.co/go/tools v0.0.1-2020.1.3/go.mod h1:X/FiERA/W4tHapMX5mGpAtMSVEeEUOyHaw9vFzvIQ3k= -honnef.co/go/tools v0.0.1-2020.1.4/go.mod h1:X/FiERA/W4tHapMX5mGpAtMSVEeEUOyHaw9vFzvIQ3k= -honnef.co/go/tools v0.3.2 h1:ytYb4rOqyp1TSa2EPvNVwtPQJctSELKaMyLfqNP4+34= honnef.co/go/tools v0.3.2/go.mod h1:jzwdWgg7Jdq75wlfblQxO4neNaFFSvgc1tD5Wv8U0Yw= -rsc.io/binaryregexp v0.2.0/go.mod h1:qTv7/COck+e2FymRvadv62gMdZztPaShugOCi3I+8D8= -rsc.io/quote/v3 v3.1.0/go.mod h1:yEA65RcK8LyAZtP9Kv3t0HmxON59tX3rD+tICJqUlj0= -rsc.io/sampler v1.3.0/go.mod h1:T1hPZKmBbMNahiBKFy5HrXp6adAjACjK9JXDnKaTXpA= -sigs.k8s.io/yaml v1.2.0/go.mod h1:yfXDCHCao9+ENCvLSE62v9VSji2MKu5jeNfTrofGhJc= +honnef.co/go/tools v0.4.3 h1:o/n5/K5gXqk8Gozvs2cnL0F2S1/g1vcGCAx2vETjITw= +honnef.co/go/tools v0.4.3/go.mod h1:36ZgoUOrqOk1GxwHhyryEkq8FQWkUO2xGuSMhUCcdvA= software.sslmate.com/src/go-pkcs12 v0.2.0 h1:nlFkj7bTysH6VkC4fGphtjXRbezREPgrHuJG20hBGPE= software.sslmate.com/src/go-pkcs12 v0.2.0/go.mod h1:23rNcYsMabIc1otwLpTkCCPwUq6kQsTyowttG/as0kQ= diff --git a/install.sh b/install.sh index 43f2a2e042..bd40111c2e 100755 --- a/install.sh +++ b/install.sh @@ -84,7 +84,7 @@ fi echo "Shell profile: ${bold}$PROFILE${sgr0}" case :$PATH: in - *:$bin_dir:*) : "PATH already contains $bin_dir" ;; + *:$bin_dir:*) echo "PATH already contains $bin_dir" ;; *) printf '\nexport PATH=%s:$PATH\n' "$bin_dir" >> "$PROFILE" echo "$PROFILE has been modified to add tiup to PATH" echo "open a new terminal or ${bold}source ${PROFILE}${sgr0} to use it" diff --git a/pkg/base52/base52.go b/pkg/base52/base52.go index 649e72495f..16357c1bc2 100644 --- a/pkg/base52/base52.go +++ b/pkg/base52/base52.go @@ -45,7 +45,11 @@ func Decode(encoded string) (int64, error) { } var id int64 for i := 0; i < len(encoded); i++ { - id = id*int64(base) + int64(strings.IndexByte(space, encoded[i])) + idx := strings.IndexByte(space, encoded[i]) + if idx < 0 { + return 0, fmt.Errorf("invalid encoded string: '%s' contains invalid character", encoded) + } + id = id*int64(base) + int64(idx) } return id, nil } diff --git a/pkg/base52/base52_test.go b/pkg/base52/base52_test.go index 6db3e09cba..da7d679b3e 100644 --- a/pkg/base52/base52_test.go +++ b/pkg/base52/base52_test.go @@ -33,4 +33,8 @@ func (s *base52Suite) TestDecode(c *C) { decoded, err := Decode("2TPzw7") c.Assert(decoded, Equals, int64(1000000000)) c.Assert(err, IsNil) + + decoded, err = Decode("../../etc/passwd") + c.Assert(decoded, Equals, int64(0)) + c.Assert(err, NotNil) } diff --git a/pkg/cluster/ansible/config.go b/pkg/cluster/ansible/config.go index 570e6303f1..fe9aa7f349 100644 --- a/pkg/cluster/ansible/config.go +++ b/pkg/cluster/ansible/config.go @@ -30,10 +30,10 @@ import ( // ImportConfig copies config files from cluster which deployed through tidb-ansible func ImportConfig(ctx context.Context, name string, clsMeta *spec.ClusterMeta, gOpt operator.Options) error { // there may be already cluster dir, skip create - // if err := os.MkdirAll(meta.ClusterPath(name), 0755); err != nil { + // if err := utils.MkdirAll(meta.ClusterPath(name), 0755); err != nil { // return err // } - // if err := os.WriteFile(meta.ClusterPath(name, "topology.yaml"), yamlFile, 0664); err != nil { + // if err := utils.WriteFile(meta.ClusterPath(name, "topology.yaml"), yamlFile, 0664); err != nil { // return err // } @@ -56,7 +56,7 @@ func ImportConfig(ctx context.Context, name string, clsMeta *spec.ClusterMeta, g spec.ClusterPath(name, "ssh", "id_rsa"), spec.ClusterPath(name, "ssh", "id_rsa.pub")). UserSSH( - inst.GetHost(), + inst.GetManageHost(), inst.GetSSHPort(), clsMeta.User, gOpt.SSHTimeout, @@ -78,7 +78,7 @@ func ImportConfig(ctx context.Context, name string, clsMeta *spec.ClusterMeta, g inst.ComponentName(), inst.GetHost(), inst.GetPort())), - inst.GetHost(), + inst.GetManageHost(), true, 0, false). @@ -90,7 +90,7 @@ func ImportConfig(ctx context.Context, name string, clsMeta *spec.ClusterMeta, g spec.ClusterPath(name, "ssh", "id_rsa"), spec.ClusterPath(name, "ssh", "id_rsa.pub")). UserSSH( - inst.GetHost(), + inst.GetManageHost(), inst.GetSSHPort(), clsMeta.User, gOpt.SSHTimeout, @@ -112,7 +112,7 @@ func ImportConfig(ctx context.Context, name string, clsMeta *spec.ClusterMeta, g inst.ComponentName(), inst.GetHost(), inst.GetPort())), - inst.GetHost(), + inst.GetManageHost(), true, 0, false). @@ -123,7 +123,7 @@ func ImportConfig(ctx context.Context, name string, clsMeta *spec.ClusterMeta, g inst.ComponentName(), inst.GetHost(), inst.GetPort())), - inst.GetHost(), + inst.GetManageHost(), true, 0, false). diff --git a/pkg/cluster/ansible/import.go b/pkg/cluster/ansible/import.go index ae1e90dade..8a8d3a92b5 100644 --- a/pkg/cluster/ansible/import.go +++ b/pkg/cluster/ansible/import.go @@ -29,6 +29,7 @@ import ( "github.com/pingcap/tiup/pkg/cluster/spec" logprinter "github.com/pingcap/tiup/pkg/logger/printer" "github.com/pingcap/tiup/pkg/set" + "github.com/pingcap/tiup/pkg/utils" "github.com/relex/aini" ) @@ -202,7 +203,7 @@ func CommentConfig(clsName string) error { for idx := range lines { lines[idx] = "# " + lines[idx] } - if err := os.WriteFile(path, []byte(strings.Join(lines, "\n")), 0644); err != nil { + if err := utils.WriteFile(path, []byte(strings.Join(lines, "\n")), 0644); err != nil { return errors.Annotatef(err, "write config file %s", path) } diff --git a/pkg/cluster/ansible/import_test.go b/pkg/cluster/ansible/import_test.go index 970bd68c53..8c1f0fe1b6 100644 --- a/pkg/cluster/ansible/import_test.go +++ b/pkg/cluster/ansible/import_test.go @@ -126,6 +126,7 @@ server_configs: pd: {} tidb_dashboard: {} tiflash: {} + tiproxy: {} tiflash-learner: {} pump: {} drainer: {} @@ -135,6 +136,7 @@ server_configs: tidb_servers: [] tikv_servers: [] tiflash_servers: [] +tiproxy_servers: [] pd_servers: [] monitoring_servers: [] `) diff --git a/pkg/cluster/ansible/inventory.go b/pkg/cluster/ansible/inventory.go index ca17d08add..6376dc3c86 100644 --- a/pkg/cluster/ansible/inventory.go +++ b/pkg/cluster/ansible/inventory.go @@ -24,6 +24,7 @@ import ( "github.com/pingcap/tiup/pkg/cluster/executor" "github.com/pingcap/tiup/pkg/cluster/spec" logprinter "github.com/pingcap/tiup/pkg/logger/printer" + "github.com/pingcap/tiup/pkg/tidbver" "github.com/relex/aini" "gopkg.in/ini.v1" "gopkg.in/yaml.v2" @@ -327,8 +328,10 @@ func parseGroupVars(ctx context.Context, dir, ansCfgFile string, clsMeta *spec.C if tcpPort, ok := grpVars["tcp_port"]; ok { tmpIns.TCPPort, _ = strconv.Atoi(tcpPort) } - if httpPort, ok := grpVars["http_port"]; ok { - tmpIns.HTTPPort, _ = strconv.Atoi(httpPort) + if !tidbver.TiFlashNotNeedHTTPPortConfig(clsMeta.Version) { + if httpPort, ok := grpVars["http_port"]; ok { + tmpIns.HTTPPort, _ = strconv.Atoi(httpPort) + } } if flashServicePort, ok := grpVars["flash_service_port"]; ok { tmpIns.FlashServicePort, _ = strconv.Atoi(flashServicePort) @@ -347,8 +350,10 @@ func parseGroupVars(ctx context.Context, dir, ansCfgFile string, clsMeta *spec.C if tcpPort, ok := srv.Vars["tcp_port"]; ok { tmpIns.TCPPort, _ = strconv.Atoi(tcpPort) } - if httpPort, ok := srv.Vars["http_port"]; ok { - tmpIns.HTTPPort, _ = strconv.Atoi(httpPort) + if !tidbver.TiFlashNotNeedHTTPPortConfig(clsMeta.Version) { + if httpPort, ok := srv.Vars["http_port"]; ok { + tmpIns.HTTPPort, _ = strconv.Atoi(httpPort) + } } if flashServicePort, ok := srv.Vars["flash_service_port"]; ok { tmpIns.FlashServicePort, _ = strconv.Atoi(flashServicePort) diff --git a/pkg/cluster/ansible/service_test.go b/pkg/cluster/ansible/service_test.go index ffadc99877..0c5f6044ec 100644 --- a/pkg/cluster/ansible/service_test.go +++ b/pkg/cluster/ansible/service_test.go @@ -21,11 +21,8 @@ import ( var tiflashConfig = ` default_profile = "default" display_name = "TiFlash" -http_port = 11316 listen_host = "0.0.0.0" -mark_cache_size = 5368709120 path = "/data1/test-cluster/leiysky-ansible-test-deploy/tiflash/data/db" -tcp_port = 11315 tmp_path = "/data1/test-cluster/leiysky-ansible-test-deploy/tiflash/data/db/tmp" [flash] diff --git a/pkg/cluster/ansible/test-data/meta.yaml b/pkg/cluster/ansible/test-data/meta.yaml index a6360b90e4..e143d7cd72 100644 --- a/pkg/cluster/ansible/test-data/meta.yaml +++ b/pkg/cluster/ansible/test-data/meta.yaml @@ -21,6 +21,7 @@ topology: pd: {} tidb_dashboard: {} tiflash: {} + tiproxy: {} tiflash-learner: {} pump: {} drainer: {} @@ -77,7 +78,6 @@ topology: ssh_port: 9999 imported: true tcp_port: 9000 - http_port: 8123 flash_service_port: 3930 flash_proxy_port: 20170 flash_proxy_status_port: 20292 @@ -90,7 +90,6 @@ topology: ssh_port: 30000 imported: true tcp_port: 9000 - http_port: 8123 flash_service_port: 3930 flash_proxy_port: 20170 flash_proxy_status_port: 20292 @@ -195,3 +194,4 @@ topology: data_dir: data/alertmanager-9093 arch: amd64 os: linux + tiproxy_servers: [] diff --git a/pkg/cluster/api/dmapi.go b/pkg/cluster/api/dmapi.go index 209d8285b7..243e75e3f4 100644 --- a/pkg/cluster/api/dmapi.go +++ b/pkg/cluster/api/dmapi.go @@ -264,7 +264,7 @@ func (dm *DMMasterClient) OfflineMember(query string, retryOpt *utils.RetryOptio _, err := dm.deleteMember(endpoints) return err }, *retryOpt); err != nil { - return fmt.Errorf("error offline member %s, %v", query, err) + return fmt.Errorf("error offline member %s, %v, %s", query, err, endpoints[0]) } return nil } diff --git a/pkg/cluster/api/pdapi.go b/pkg/cluster/api/pdapi.go index d8bd999728..8be20a52aa 100644 --- a/pkg/cluster/api/pdapi.go +++ b/pkg/cluster/api/pdapi.go @@ -899,6 +899,7 @@ func (pc *PDClient) GetTiKVLabels() (map[string]map[string]string, []map[string] for _, s := range r.Stores { if s.Store.State == metapb.StoreState_Up { lbs := s.Store.GetLabels() + host, port := utils.ParseHostPort(s.Store.GetAddress()) labelsMap := map[string]string{} var labelsArr []string @@ -915,9 +916,9 @@ func (pc *PDClient) GetTiKVLabels() (map[string]map[string]string, []map[string] label := fmt.Sprintf("%s%s%s", "{", strings.Join(labelsArr, ","), "}") storeInfo = append(storeInfo, map[string]LabelInfo{ - strings.Split(s.Store.GetAddress(), ":")[0]: { - Machine: strings.Split(s.Store.GetAddress(), ":")[0], - Port: strings.Split(s.Store.GetAddress(), ":")[1], + host: { + Machine: host, + Port: port, Store: s.Store.GetId(), Status: s.Store.State.String(), Leaders: s.Status.LeaderCount, diff --git a/pkg/cluster/api/tidbapi.go b/pkg/cluster/api/tidbapi.go new file mode 100644 index 0000000000..793ba16656 --- /dev/null +++ b/pkg/cluster/api/tidbapi.go @@ -0,0 +1,77 @@ +// Copyright 2022 PingCAP, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// See the License for the specific language governing permissions and +// limitations under the License. + +package api + +import ( + "context" + "crypto/tls" + "fmt" + "time" + + "github.com/pingcap/tiup/pkg/utils" +) + +// TiDBClient is client for access TiKVCDC Open API +type TiDBClient struct { + urls []string + client *utils.HTTPClient + ctx context.Context +} + +// NewTiDBClient return a `TiDBClient` +func NewTiDBClient(ctx context.Context, addresses []string, timeout time.Duration, tlsConfig *tls.Config) *TiDBClient { + httpPrefix := "http" + if tlsConfig != nil { + httpPrefix = "https" + } + urls := make([]string, 0, len(addresses)) + for _, addr := range addresses { + urls = append(urls, fmt.Sprintf("%s://%s", httpPrefix, addr)) + } + + return &TiDBClient{ + urls: urls, + client: utils.NewHTTPClient(timeout, tlsConfig), + ctx: ctx, + } +} + +func (c *TiDBClient) getEndpoints(api string) (endpoints []string) { + for _, url := range c.urls { + endpoints = append(endpoints, fmt.Sprintf("%s%s", url, api)) + } + return endpoints +} + +// StartUpgrade sends the start upgrade message to the TiDB server +func (c *TiDBClient) StartUpgrade() error { + api := "/upgrade/start" + endpoints := c.getEndpoints(api) + _, err := tryURLs(endpoints, func(endpoint string) ([]byte, error) { + return c.client.Post(c.ctx, endpoint, nil) + }) + + return err +} + +// FinishUpgrade sends the finish upgrade message to the TiDB server +func (c *TiDBClient) FinishUpgrade() error { + api := "/upgrade/finish" + endpoints := c.getEndpoints(api) + _, err := tryURLs(endpoints, func(endpoint string) ([]byte, error) { + return c.client.Post(c.ctx, endpoint, nil) + }) + + return err +} diff --git a/pkg/cluster/executor/executor.go b/pkg/cluster/executor/executor.go index a2c2f20aab..8cc867a991 100644 --- a/pkg/cluster/executor/executor.go +++ b/pkg/cluster/executor/executor.go @@ -118,6 +118,19 @@ func New(etype SSHType, sudo bool, c SSHConfig) (ctxt.Executor, error) { return &CheckPointExecutor{executor, &c}, nil } +// UnwarpCheckPointExecutor unwarp the CheckPointExecutor and return the real executor +// +// Sometimes we just want to get the output of a command, and the CheckPointExecutor will +// always cache the output, it will be a problem when we want to get the real output. +func UnwarpCheckPointExecutor(e ctxt.Executor) ctxt.Executor { + switch e := e.(type) { + case *CheckPointExecutor: + return e.Executor + default: + return e + } +} + func checkLocalIP(ip string) error { ifaces, err := net.Interfaces() if err != nil { diff --git a/pkg/cluster/executor/local.go b/pkg/cluster/executor/local.go index 9a3a13f475..bd1c348329 100644 --- a/pkg/cluster/executor/local.go +++ b/pkg/cluster/executor/local.go @@ -110,7 +110,7 @@ func (l *Local) Execute(ctx context.Context, cmd string, sudo bool, timeout ...t // Transfer implements Executer interface. func (l *Local) Transfer(ctx context.Context, src, dst string, download bool, limit int, _ bool) error { targetPath := filepath.Dir(dst) - if err := utils.CreateDir(targetPath); err != nil { + if err := utils.MkdirAll(targetPath, 0755); err != nil { return err } diff --git a/pkg/cluster/executor/scp.go b/pkg/cluster/executor/scp.go index 2e1cc39d87..3bd30fafe2 100644 --- a/pkg/cluster/executor/scp.go +++ b/pkg/cluster/executor/scp.go @@ -29,6 +29,7 @@ import ( // ScpDownload downloads a file from remote with SCP // The implementation is partially inspired by github.com/dtylman/scp +// Do not support pattern, src and dst must be dir or file path func ScpDownload(session *ssh.Session, client *ssh.Client, src, dst string, limit int, compress bool) error { r, err := session.StdoutPipe() if err != nil { @@ -41,62 +42,6 @@ func ScpDownload(session *ssh.Session, client *ssh.Client, src, dst string, limi return err } - copyF := func() error { - // parse SCP command - line, _, err := bufr.ReadLine() - if err != nil { - return err - } - if line[0] != byte('C') { - return fmt.Errorf("incorrect scp command '%b', should be 'C'", line[0]) - } - - mode, err := strconv.ParseUint(string(line[1:5]), 0, 32) - if err != nil { - return fmt.Errorf("error parsing file mode; %s", err) - } - - // prepare dst file - targetPath := filepath.Dir(dst) - if err := utils.CreateDir(targetPath); err != nil { - return err - } - targetFile, err := os.OpenFile(dst, os.O_RDWR|os.O_CREATE|os.O_TRUNC, fs.FileMode(mode)) - if err != nil { - return err - } - defer targetFile.Close() - - size, err := strconv.Atoi(strings.Fields(string(line))[1]) - if err != nil { - return err - } - - if err := ack(w); err != nil { - return err - } - - // transferring data - n, err := io.CopyN(targetFile, bufr, int64(size)) - if err != nil { - return err - } - if n < int64(size) { - return fmt.Errorf("error downloading via scp, file size mismatch") - } - if err := targetFile.Sync(); err != nil { - return err - } - - return ack(w) - } - - copyErrC := make(chan error, 1) - go func() { - defer w.Close() - copyErrC <- copyF() - }() - remoteArgs := make([]string, 0) if compress { remoteArgs = append(remoteArgs, "-C") @@ -104,7 +49,7 @@ func ScpDownload(session *ssh.Session, client *ssh.Client, src, dst string, limi if limit > 0 { remoteArgs = append(remoteArgs, fmt.Sprintf("-l %d", limit)) } - remoteCmd := fmt.Sprintf("scp %s -f %s", strings.Join(remoteArgs, " "), src) + remoteCmd := fmt.Sprintf("scp -r %s -f %s", strings.Join(remoteArgs, " "), src) err = session.Start(remoteCmd) if err != nil { @@ -114,10 +59,100 @@ func ScpDownload(session *ssh.Session, client *ssh.Client, src, dst string, limi return err } - err = <-copyErrC - if err != nil { - return err + wd := dst + for firstCommand := true; ; firstCommand = false { + // parse scp command + line, err := bufr.ReadString('\n') + if err == io.EOF { + break + } else if err != nil { + return err + } + + switch line[0] { + // ignore ACK from server + case '\x00': + line = line[1:] + case '\x01': + return fmt.Errorf("scp warning: %s", line[1:]) + case '\x02': + return fmt.Errorf("scp error: %s", line[1:]) + } + + switch line[0] { + case 'C': + mode, size, name, err := parseLine(line) + if err != nil { + return err + } + + fp := filepath.Join(wd, name) + // fisrt scp command is 'C' means src is a single file + if firstCommand { + fp = dst + } + + targetFile, err := os.OpenFile(fp, os.O_RDWR|os.O_CREATE|os.O_TRUNC, mode|0700) + if err != nil { + return err + } + defer targetFile.Close() + if err := ack(w); err != nil { + return err + } + + // transferring data + n, err := io.CopyN(targetFile, bufr, size) + if err != nil { + return err + } + if n < size { + return fmt.Errorf("error downloading via scp, file size mismatch") + } + if err := targetFile.Sync(); err != nil { + return err + } + case 'D': + mode, _, name, err := parseLine(line) + if err != nil { + return err + } + + // normally, workdir is like this + wd = filepath.Join(wd, name) + + // fisrt scp command is 'D' means src is a dir + if firstCommand { + fi, err := os.Stat(dst) + if err != nil && !os.IsNotExist(err) { + return err + } else if err == nil && !fi.IsDir() { + return fmt.Errorf("%s cannot be an exist file", wd) + } else if os.IsNotExist(err) { + // dst is not exist, so dst is the target dir + wd = dst + } else { + // dst is exist, dst/name is the target dir + break + } + } + + err = utils.MkdirAll(wd, mode) + if err != nil { + return err + } + case 'E': + wd = filepath.Dir(wd) + default: + return fmt.Errorf("incorrect scp command '%b', should be 'C', 'D' or 'E'", line[0]) + } + + err = ack(w) + if err != nil { + return err + } } + return session.Wait() } @@ -132,3 +167,25 @@ func ack(w io.Writer) error { } return nil } + +func parseLine(line string) (mode fs.FileMode, size int64, name string, err error) { + words := strings.Fields(strings.TrimSuffix(line, "\n")) + if len(words) < 3 { + return 0, 0, "", fmt.Errorf("incorrect scp command param number: %d", len(words)) + } + + modeN, err := strconv.ParseUint(words[0][1:], 0, 32) + if err != nil { + return 0, 0, "", fmt.Errorf("error parsing file mode; %s", err) + } + mode = fs.FileMode(modeN) + + size, err = strconv.ParseInt(words[1], 10, 64) + if err != nil { + return 0, 0, "", err + } + + name = words[2] + + return +} diff --git a/pkg/cluster/executor/ssh.go b/pkg/cluster/executor/ssh.go index 2e6b8f88e3..805013bcf7 100644 --- a/pkg/cluster/executor/ssh.go +++ b/pkg/cluster/executor/ssh.go @@ -217,6 +217,10 @@ func (e *EasySSHExecutor) Transfer(ctx context.Context, src, dst string, downloa defer client.Close() defer session.Close() + err = utils.MkdirAll(filepath.Dir(dst), 0755) + if err != nil { + return nil + } return ScpDownload(session, client, src, dst, limit, compress) } @@ -379,7 +383,7 @@ func (e *NativeSSHExecutor) Transfer(ctx context.Context, src, dst string, downl if download { targetPath := filepath.Dir(dst) - if err := utils.CreateDir(targetPath); err != nil { + if err := utils.MkdirAll(targetPath, 0755); err != nil { return err } args = append(args, fmt.Sprintf("%s@%s:%s", e.Config.User, e.Config.Host, src), dst) diff --git a/pkg/cluster/manager/basic.go b/pkg/cluster/manager/basic.go index 7152f2cef4..ae7be80e91 100644 --- a/pkg/cluster/manager/basic.go +++ b/pkg/cluster/manager/basic.go @@ -277,11 +277,11 @@ func getMonitorHosts(topo spec.Topology) (map[string]hostInfo, set.StringSet) { topo.IterInstance(func(inst spec.Instance) { // add the instance to ignore list if it marks itself as ignore_exporter if inst.IgnoreMonitorAgent() { - noAgentHosts.Insert(inst.GetHost()) + noAgentHosts.Insert(inst.GetManageHost()) } - if _, found := uniqueHosts[inst.GetHost()]; !found { - uniqueHosts[inst.GetHost()] = hostInfo{ + if _, found := uniqueHosts[inst.GetManageHost()]; !found { + uniqueHosts[inst.GetManageHost()] = hostInfo{ ssh: inst.GetSSHPort(), os: inst.OS(), arch: inst.Arch(), diff --git a/pkg/cluster/manager/builder.go b/pkg/cluster/manager/builder.go index 3344adcb82..a945a664f4 100644 --- a/pkg/cluster/manager/builder.go +++ b/pkg/cluster/manager/builder.go @@ -60,10 +60,10 @@ func buildReloadPromAndGrafanaTasks( t := task.NewBuilder(logger) if inst.ComponentName() == spec.ComponentPrometheus { // reload Prometheus - t = t.SystemCtl(inst.GetHost(), inst.ServiceName(), "reload", true, true) + t = t.SystemCtl(inst.GetManageHost(), inst.ServiceName(), "reload", true, true) } else { // restart grafana - t = t.SystemCtl(inst.GetHost(), inst.ServiceName(), "restart", true, false) + t = t.SystemCtl(inst.GetManageHost(), inst.ServiceName(), "restart", true, false) } tasks = append(tasks, t.BuildAsStep(fmt.Sprintf(" - Reload %s -> %s", inst.ComponentName(), inst.ID()))) @@ -102,12 +102,12 @@ func buildScaleOutTask( // Initialize the environments initializedHosts := set.NewStringSet() metadata.GetTopology().IterInstance(func(instance spec.Instance) { - initializedHosts.Insert(instance.GetHost()) + initializedHosts.Insert(instance.GetManageHost()) }) // uninitializedHosts are hosts which haven't been initialized yet uninitializedHosts := make(map[string]hostInfo) // host -> ssh-port, os, arch newPart.IterInstance(func(instance spec.Instance) { - host := instance.GetHost() + host := instance.GetManageHost() if initializedHosts.Exist(host) { return } @@ -134,7 +134,7 @@ func buildScaleOutTask( t := task.NewBuilder(m.logger). RootSSH( - instance.GetHost(), + instance.GetManageHost(), instance.GetSSHPort(), opt.User, s.Password, @@ -152,8 +152,8 @@ func buildScaleOutTask( gOpt.SSHType, globalOptions.SSHType, ). - EnvInit(instance.GetHost(), base.User, base.Group, opt.SkipCreateUser || globalOptions.User == opt.User). - Mkdir(globalOptions.User, instance.GetHost(), dirs...). + EnvInit(instance.GetManageHost(), base.User, base.Group, opt.SkipCreateUser || globalOptions.User == opt.User). + Mkdir(globalOptions.User, instance.GetManageHost(), dirs...). BuildAsStep(fmt.Sprintf(" - Initialized host %s ", host)) envInitTasks = append(envInitTasks, t) }) @@ -164,7 +164,6 @@ func buildScaleOutTask( newPart, m.logger, gOpt, - m.bindVersion, ) sshType := topo.BaseTopo().GlobalOptions.SSHType @@ -172,7 +171,7 @@ func buildScaleOutTask( var iterErr error // Deploy the new topology and refresh the configuration newPart.IterInstance(func(inst spec.Instance) { - version := m.bindVersion(inst.ComponentName(), base.Version) + version := inst.CalculateVersion(base.Version) deployDir := spec.Abs(base.User, inst.DeployDir()) // data dir would be empty for components which don't need it dataDirs := spec.MultiDirAbs(base.User, inst.DataDir()) @@ -186,10 +185,10 @@ func buildScaleOutTask( filepath.Join(deployDir, "scripts"), } // Deploy component - tb := task.NewSimpleUerSSH(m.logger, inst.GetHost(), inst.GetSSHPort(), base.User, gOpt, p, sshType). - Mkdir(base.User, inst.GetHost(), deployDirs...). - Mkdir(base.User, inst.GetHost(), dataDirs...). - Mkdir(base.User, inst.GetHost(), logDir) + tb := task.NewSimpleUerSSH(m.logger, inst.GetManageHost(), inst.GetSSHPort(), base.User, gOpt, p, sshType). + Mkdir(base.User, inst.GetManageHost(), deployDirs...). + Mkdir(base.User, inst.GetManageHost(), dataDirs...). + Mkdir(base.User, inst.GetManageHost(), logDir) srcPath := "" if patchedComponents.Exist(inst.ComponentName()) { @@ -210,12 +209,12 @@ func buildScaleOutTask( tb = tb.DeploySpark(inst, sparkVer.String(), srcPath, deployDir) default: tb.CopyComponent( - inst.ComponentName(), + inst.ComponentSource(), inst.OS(), inst.Arch(), - version, + inst.CalculateVersion(version), srcPath, - inst.GetHost(), + inst.GetManageHost(), deployDir, ) } @@ -235,11 +234,11 @@ func buildScaleOutTask( // data dir would be empty for components which don't need it // Download and copy the latest component to remote if the cluster is imported from Ansible tb := task.NewBuilder(m.logger) + version := inst.CalculateVersion(base.Version) switch compName := inst.ComponentName(); compName { case spec.ComponentGrafana, spec.ComponentPrometheus, spec.ComponentAlertmanager: - version := m.bindVersion(compName, base.Version) tb.Download(compName, inst.OS(), inst.Arch(), version). - CopyComponent(compName, inst.OS(), inst.Arch(), version, "", inst.GetHost(), deployDir) + CopyComponent(compName, inst.OS(), inst.Arch(), version, "", inst.GetManageHost(), deployDir) } deployCompTasks = append(deployCompTasks, tb.BuildAsStep(fmt.Sprintf(" - Deploy instance %s -> %s", inst.ComponentName(), inst.ID()))) } @@ -272,7 +271,6 @@ func buildScaleOutTask( noAgentHosts, topo.BaseTopo().GlobalOptions, topo.BaseTopo().MonitoredOptions, - base.Version, gOpt, p, ) @@ -396,7 +394,7 @@ func buildScaleConfigTasks( // log dir will always be with values, but might not used by the component logDir := spec.Abs(base.User, inst.LogDir()) - t := task.NewSimpleUerSSH(m.logger, inst.GetHost(), inst.GetSSHPort(), base.User, gOpt, p, topo.BaseTopo().GlobalOptions.SSHType). + t := task.NewSimpleUerSSH(m.logger, inst.GetManageHost(), inst.GetSSHPort(), base.User, gOpt, p, topo.BaseTopo().GlobalOptions.SSHType). ScaleConfig( name, base.Version, @@ -429,7 +427,6 @@ func buildMonitoredDeployTask( noAgentHosts set.StringSet, // hosts that do not deploy monitor agents globalOptions *spec.GlobalOptions, monitoredOptions *spec.MonitoredOptions, - version string, gOpt operator.Options, p *tui.SSHConnectionProps, ) (downloadCompTasks []*task.StepDisplay, deployCompTasks []*task.StepDisplay, err error) { @@ -440,7 +437,10 @@ func buildMonitoredDeployTask( uniqueCompOSArch := set.NewStringSet() // monitoring agents for _, comp := range []string{spec.ComponentNodeExporter, spec.ComponentBlackboxExporter} { - version := m.bindVersion(comp, version) + version := monitoredOptions.NodeExporterVersion + if comp == spec.ComponentBlackboxExporter { + version = monitoredOptions.BlackboxExporterVersion + } for host, info := range uniqueHosts { // skip deploying monitoring agents if the instance is marked so if noAgentHosts.Exist(host) { @@ -638,9 +638,9 @@ func buildInitConfigTasks( // Download and copy the latest component to remote if the cluster is imported from Ansible tb := task.NewBuilder(m.logger) if instance.IsImported() { + version := instance.CalculateVersion(base.Version) switch compName { case spec.ComponentGrafana, spec.ComponentPrometheus, spec.ComponentAlertmanager: - version := m.bindVersion(compName, base.Version) tb.Download(compName, instance.OS(), instance.Arch(), version). CopyComponent( compName, @@ -648,7 +648,7 @@ func buildInitConfigTasks( instance.Arch(), version, "", // use default srcPath - instance.GetHost(), + instance.GetManageHost(), deployDir, ) } @@ -683,12 +683,11 @@ func buildDownloadCompTasks( topo spec.Topology, logger *logprinter.Logger, gOpt operator.Options, - bindVersion spec.BindVersion, ) []*task.StepDisplay { var tasks []*task.StepDisplay uniqueTaskList := set.NewStringSet() topo.IterInstance(func(inst spec.Instance) { - key := fmt.Sprintf("%s-%s-%s", inst.ComponentName(), inst.OS(), inst.Arch()) + key := fmt.Sprintf("%s-%s-%s", inst.ComponentSource(), inst.OS(), inst.Arch()) if found := uniqueTaskList.Exist(key); !found { uniqueTaskList.Insert(key) @@ -698,13 +697,13 @@ func buildDownloadCompTasks( // download spark as dependency of tispark tasks = append(tasks, buildDownloadSparkTask(inst, logger, gOpt)) } else { - version = bindVersion(inst.ComponentName(), clusterVersion) + version = inst.CalculateVersion(clusterVersion) } t := task.NewBuilder(logger). - Download(inst.ComponentName(), inst.OS(), inst.Arch(), version). + Download(inst.ComponentSource(), inst.OS(), inst.Arch(), version). BuildAsStep(fmt.Sprintf(" - Download %s:%s (%s/%s)", - inst.ComponentName(), version, inst.OS(), inst.Arch())) + inst.ComponentSource(), version, inst.OS(), inst.Arch())) tasks = append(tasks, t) } }) @@ -842,8 +841,8 @@ func buildCertificateTasks( deployDir := spec.Abs(base.User, inst.DeployDir()) tlsDir := filepath.Join(deployDir, spec.TLSCertKeyDir) - tb := task.NewSimpleUerSSH(m.logger, inst.GetHost(), inst.GetSSHPort(), base.User, gOpt, p, topo.BaseTopo().GlobalOptions.SSHType). - Mkdir(base.User, inst.GetHost(), deployDir, tlsDir) + tb := task.NewSimpleUerSSH(m.logger, inst.GetManageHost(), inst.GetSSHPort(), base.User, gOpt, p, topo.BaseTopo().GlobalOptions.SSHType). + Mkdir(base.User, inst.GetManageHost(), deployDir, tlsDir) ca, err := crypto.ReadCA( name, diff --git a/pkg/cluster/manager/cacert.go b/pkg/cluster/manager/cacert.go index 48cdfe751a..7f3474b311 100644 --- a/pkg/cluster/manager/cacert.go +++ b/pkg/cluster/manager/cacert.go @@ -108,7 +108,7 @@ func (m *Manager) genAndSaveCertificate(clusterName string, globalOptions *spec. if globalOptions.TLSEnabled { // generate CA tlsPath := m.specManager.Path(clusterName, spec.TLSCertKeyDir) - if err := utils.CreateDir(tlsPath); err != nil { + if err := utils.MkdirAll(tlsPath, 0755); err != nil { return nil, err } ca, err := genAndSaveClusterCA(clusterName, tlsPath) diff --git a/pkg/cluster/manager/check.go b/pkg/cluster/manager/check.go index d390d9bc32..ab2d59dfc0 100644 --- a/pkg/cluster/manager/check.go +++ b/pkg/cluster/manager/check.go @@ -182,7 +182,7 @@ func checkSystemInfo( downloadTasks []*task.StepDisplay ) logger := ctx.Value(logprinter.ContextKeyLogger).(*logprinter.Logger) - insightVer := spec.TiDBComponentVersion(spec.ComponentCheckCollector, "") + insightVer := "" uniqueHosts := map[string]int{} // host -> ssh-port uniqueArchList := make(map[string]struct{}) // map["os-arch"]{} @@ -190,7 +190,7 @@ func checkSystemInfo( roleFilter := set.NewStringSet(gOpt.Roles...) nodeFilter := set.NewStringSet(gOpt.Nodes...) - components := topo.ComponentsByUpdateOrder() + components := topo.ComponentsByStartOrder() components = operator.FilterComponent(components, roleFilter) for _, comp := range components { @@ -218,7 +218,7 @@ func checkSystemInfo( // checks that applies to each instance if opt.ExistCluster { t1 = t1.CheckSys( - inst.GetHost(), + inst.GetManageHost(), inst.DeployDir(), task.CheckTypePermission, topo, @@ -227,28 +227,28 @@ func checkSystemInfo( } else { t1 = t1. CheckSys( - inst.GetHost(), + inst.GetManageHost(), inst.DeployDir(), task.ChecktypeIsExist, topo, opt.Opr, ). CheckSys( - inst.GetHost(), + inst.GetManageHost(), inst.DataDir(), task.ChecktypeIsExist, topo, opt.Opr, ). CheckSys( - inst.GetHost(), + inst.GetManageHost(), inst.LogDir(), task.ChecktypeIsExist, topo, opt.Opr, ). CheckSys( - inst.GetHost(), + inst.GetManageHost(), fmt.Sprintf("/etc/systemd/system/%s-%d.service", inst.ComponentName(), inst.GetPort()), task.ChecktypeIsExist, topo, @@ -262,7 +262,7 @@ func checkSystemInfo( // build checking tasks t1 = t1. CheckSys( - inst.GetHost(), + inst.GetManageHost(), dataDir, task.CheckTypeFIO, topo, @@ -271,7 +271,7 @@ func checkSystemInfo( if opt.ExistCluster { t1 = t1.CheckSys( - inst.GetHost(), + inst.GetManageHost(), dataDir, task.CheckTypePermission, topo, @@ -282,11 +282,11 @@ func checkSystemInfo( checkSysTasks = append( checkSysTasks, - t1.BuildAsStep(fmt.Sprintf(" - Checking node %s", inst.GetHost())), + t1.BuildAsStep(fmt.Sprintf(" - Checking node %s", inst.GetManageHost())), ) - if _, found := uniqueHosts[inst.GetHost()]; !found { - uniqueHosts[inst.GetHost()] = inst.GetSSHPort() + if _, found := uniqueHosts[inst.GetManageHost()]; !found { + uniqueHosts[inst.GetManageHost()] = inst.GetSSHPort() insightNodes = append(insightNodes, inst) } } @@ -296,17 +296,15 @@ func checkSystemInfo( if len(existPD) < 1 { return fmt.Errorf("cannot find PD in exist cluster") } - if _, found := uniqueHosts[existPD[0].GetHost()]; !found { + if _, found := uniqueHosts[existPD[0].GetManageHost()]; !found { insightNodes = append(insightNodes, existPD[0]) } - // checks that applies to each host - // if _, found := uniqueHosts[inst.GetHost()]; !found { - // uniqueHosts[inst.GetHost()] = inst.GetSSHPort() + for _, inst := range insightNodes { // build system info collecting tasks t2 := task.NewBuilder(logger). RootSSH( - inst.GetHost(), + inst.GetManageHost(), inst.GetSSHPort(), opt.User, s.Password, @@ -324,28 +322,28 @@ func checkSystemInfo( gOpt.SSHType, topo.GlobalOptions.SSHType, ). - Mkdir(opt.User, inst.GetHost(), filepath.Join(task.CheckToolsPathDir, "bin")). + Mkdir(opt.User, inst.GetManageHost(), filepath.Join(task.CheckToolsPathDir, "bin")). CopyComponent( spec.ComponentCheckCollector, inst.OS(), inst.Arch(), insightVer, "", // use default srcPath - inst.GetHost(), + inst.GetManageHost(), task.CheckToolsPathDir, ). Shell( - inst.GetHost(), + inst.GetManageHost(), filepath.Join(task.CheckToolsPathDir, "bin", "insight"), "", false, ). - BuildAsStep(" - Getting system info of " + utils.JoinHostPort(inst.GetHost(), inst.GetSSHPort())) + BuildAsStep(" - Getting system info of " + utils.JoinHostPort(inst.GetManageHost(), inst.GetSSHPort())) collectTasks = append(collectTasks, t2) t3 := task.NewBuilder(logger). RootSSH( - inst.GetHost(), + inst.GetManageHost(), inst.GetSSHPort(), opt.User, s.Password, @@ -363,8 +361,8 @@ func checkSystemInfo( gOpt.SSHType, topo.GlobalOptions.SSHType, ). - Rmdir(inst.GetHost(), task.CheckToolsPathDir). - BuildAsStep(" - Cleanup check files on " + utils.JoinHostPort(inst.GetHost(), inst.GetSSHPort())) + Rmdir(inst.GetManageHost(), task.CheckToolsPathDir). + BuildAsStep(" - Cleanup check files on " + utils.JoinHostPort(inst.GetManageHost(), inst.GetSSHPort())) cleanTasks = append(cleanTasks, t3) } @@ -697,7 +695,7 @@ func (m *Manager) checkRegionsInfo(clusterName string, topo *spec.Specification, } pdClient := api.NewPDClient( context.WithValue(context.TODO(), logprinter.ContextKeyLogger, m.logger), - topo.GetPDList(), + topo.GetPDListWithManageHost(), time.Second*time.Duration(gOpt.APITimeout), tlsConfig, ) diff --git a/pkg/cluster/manager/cleanup.go b/pkg/cluster/manager/cleanup.go index 895b6c5ad0..0cff6c2390 100644 --- a/pkg/cluster/manager/cleanup.go +++ b/pkg/cluster/manager/cleanup.go @@ -251,10 +251,10 @@ func (c *cleanupFiles) instanceCleanupFiles(topo spec.Topology) { } } - if c.delFileMap[ins.GetHost()] == nil { - c.delFileMap[ins.GetHost()] = set.NewStringSet() + if c.delFileMap[ins.GetManageHost()] == nil { + c.delFileMap[ins.GetManageHost()] = set.NewStringSet() } - c.delFileMap[ins.GetHost()].Join(logPaths).Join(dataPaths).Join(tlsPath) + c.delFileMap[ins.GetManageHost()].Join(logPaths).Join(dataPaths).Join(tlsPath) } } } diff --git a/pkg/cluster/manager/deploy.go b/pkg/cluster/manager/deploy.go index a6779b4fa9..c9b413edd3 100644 --- a/pkg/cluster/manager/deploy.go +++ b/pkg/cluster/manager/deploy.go @@ -17,7 +17,6 @@ import ( "context" "errors" "fmt" - "os" "path/filepath" "strings" @@ -159,7 +158,7 @@ func (m *Manager) Deploy( } } - if err := os.MkdirAll(m.specManager.Path(name), 0755); err != nil { + if err := utils.MkdirAll(m.specManager.Path(name), 0755); err != nil { return errorx.InitializationFailed. Wrap(err, "Failed to create cluster metadata directory '%s'", m.specManager.Path(name)). WithProperty(tui.SuggestionFromString("Please check file system permissions and try again.")) @@ -246,11 +245,11 @@ func (m *Manager) Deploy( } // Download missing component - downloadCompTasks = buildDownloadCompTasks(clusterVersion, topo, m.logger, gOpt, m.bindVersion) + downloadCompTasks = buildDownloadCompTasks(clusterVersion, topo, m.logger, gOpt) // Deploy components to remote topo.IterInstance(func(inst spec.Instance) { - version := m.bindVersion(inst.ComponentName(), clusterVersion) + version := inst.CalculateVersion(clusterVersion) deployDir := spec.Abs(globalOptions.User, inst.DeployDir()) // data dir would be empty for components which don't need it dataDirs := spec.MultiDirAbs(globalOptions.User, inst.DataDir()) @@ -265,9 +264,9 @@ func (m *Manager) Deploy( filepath.Join(deployDir, "scripts"), } - t := task.NewSimpleUerSSH(m.logger, inst.GetHost(), inst.GetSSHPort(), globalOptions.User, gOpt, sshProxyProps, globalOptions.SSHType). - Mkdir(globalOptions.User, inst.GetHost(), deployDirs...). - Mkdir(globalOptions.User, inst.GetHost(), dataDirs...) + t := task.NewSimpleUerSSH(m.logger, inst.GetManageHost(), inst.GetSSHPort(), globalOptions.User, gOpt, sshProxyProps, globalOptions.SSHType). + Mkdir(globalOptions.User, inst.GetManageHost(), deployDirs...). + Mkdir(globalOptions.User, inst.GetManageHost(), dataDirs...) if deployerInstance, ok := inst.(DeployerInstance); ok { deployerInstance.Deploy(t, "", deployDir, version, name, clusterVersion) @@ -286,19 +285,19 @@ func (m *Manager) Deploy( t = t.DeploySpark(inst, sparkVer.String(), "" /* default srcPath */, deployDir) default: t = t.CopyComponent( - inst.ComponentName(), + inst.ComponentSource(), inst.OS(), inst.Arch(), version, "", // use default srcPath - inst.GetHost(), + inst.GetManageHost(), deployDir, ) } } deployCompTasks = append(deployCompTasks, - t.BuildAsStep(fmt.Sprintf(" - Copy %s -> %s", inst.ComponentName(), inst.GetHost())), + t.BuildAsStep(fmt.Sprintf(" - Copy %s -> %s", inst.ComponentName(), inst.GetManageHost())), ) }) @@ -321,7 +320,6 @@ func (m *Manager) Deploy( noAgentHosts, globalOptions, topo.GetMonitoredOptions(), - clusterVersion, gOpt, sshProxyProps, ) diff --git a/pkg/cluster/manager/display.go b/pkg/cluster/manager/display.go index 73b69b4d5c..ce0986f6ba 100644 --- a/pkg/cluster/manager/display.go +++ b/pkg/cluster/manager/display.go @@ -42,14 +42,16 @@ import ( "github.com/pingcap/tiup/pkg/set" "github.com/pingcap/tiup/pkg/tui" "github.com/pingcap/tiup/pkg/utils" - "go.uber.org/zap" ) // DisplayOption represents option of display command type DisplayOption struct { - ClusterName string - ShowUptime bool - ShowProcess bool + ClusterName string + ShowUptime bool + ShowProcess bool + ShowManageHost bool + ShowNuma bool + ShowVersions bool } // InstInfo represents an instance info @@ -57,6 +59,7 @@ type InstInfo struct { ID string `json:"id"` Role string `json:"role"` Host string `json:"host"` + ManageHost string `json:"manage_host"` Ports string `json:"ports"` OsArch string `json:"os_arch"` Status string `json:"status"` @@ -66,6 +69,9 @@ type InstInfo struct { Since string `json:"since"` DataDir string `json:"data_dir"` DeployDir string `json:"deploy_dir"` + NumaNode string `json:"numa_node"` + NumaCores string `json:"numa_cores"` + Version string `json:"version"` ComponentName string Port int @@ -124,6 +130,16 @@ func (m *Manager) Display(dopt DisplayOption, opt operator.Options) error { base := metadata.GetBaseMeta() cyan := color.New(color.FgCyan, color.Bold) + // check if managehost is set + if !dopt.ShowManageHost { + topo.IterInstance(func(inst spec.Instance) { + if inst.GetHost() != inst.GetManageHost() { + dopt.ShowManageHost = true + return + } + }) + } + statusTimeout := time.Duration(opt.APITimeout) * time.Second // display cluster meta var j *JSONOutput @@ -174,13 +190,27 @@ func (m *Manager) Display(dopt DisplayOption, opt operator.Options) error { // display topology var clusterTable [][]string - rowHead := []string{"ID", "Role", "Host", "Ports", "OS/Arch", "Status"} + rowHead := []string{"ID", "Role", "Host"} + + if dopt.ShowManageHost { + rowHead = append(rowHead, "Manage Host") + } + + rowHead = append(rowHead, "Ports", "OS/Arch", "Status") + if dopt.ShowProcess { rowHead = append(rowHead, "Memory", "Memory Limit", "CPU Quota") } if dopt.ShowUptime { rowHead = append(rowHead, "Since") } + if dopt.ShowNuma { + rowHead = append(rowHead, "Numa Node", "Numa Cores") + } + if dopt.ShowVersions { + rowHead = append(rowHead, "Version") + } + rowHead = append(rowHead, "Data Dir", "Deploy Dir") clusterTable = append(clusterTable, rowHead) @@ -190,16 +220,30 @@ func (m *Manager) Display(dopt DisplayOption, opt operator.Options) error { color.CyanString(v.ID), v.Role, v.Host, + } + + if dopt.ShowManageHost { + row = append(row, v.ManageHost) + } + + row = append(row, v.Ports, v.OsArch, - formatInstanceStatus(v.Status), - } + formatInstanceStatus(v.Status)) + if dopt.ShowProcess { row = append(row, v.Memory, v.MemoryLimit, v.CPUquota) } if dopt.ShowUptime { row = append(row, v.Since) } + if dopt.ShowNuma { + row = append(row, v.NumaNode, v.NumaCores) + } + if dopt.ShowVersions { + row = append(row, v.Version) + } + row = append(row, v.DataDir, v.DeployDir) clusterTable = append(clusterTable, row) @@ -207,7 +251,7 @@ func (m *Manager) Display(dopt DisplayOption, opt operator.Options) error { continue } if strings.HasPrefix(v.Status, "Up") || strings.HasPrefix(v.Status, "Healthy") { - instAddr := utils.JoinHostPort(v.Host, v.Port) + instAddr := utils.JoinHostPort(v.ManageHost, v.Port) masterActive = append(masterActive, instAddr) } } @@ -401,7 +445,7 @@ func (m *Manager) DisplayTiKVLabels(dopt DisplayOption, opt operator.Options) er if ins.ComponentName() == spec.ComponentPD { status := ins.Status(ctx, statusTimeout, tlsCfg, masterList...) if strings.HasPrefix(status, "Up") || strings.HasPrefix(status, "Healthy") { - instAddr := utils.JoinHostPort(ins.GetHost(), ins.GetPort()) + instAddr := utils.JoinHostPort(ins.GetManageHost(), ins.GetPort()) mu.Lock() masterActive = append(masterActive, instAddr) mu.Unlock() @@ -529,7 +573,7 @@ func (m *Manager) GetClusterTopology(dopt DisplayOption, opt operator.Options) ( status := ins.Status(ctx, statusTimeout, tlsCfg, masterList...) mu.Lock() if strings.HasPrefix(status, "Up") || strings.HasPrefix(status, "Healthy") { - instAddr := utils.JoinHostPort(ins.GetHost(), ins.GetPort()) + instAddr := utils.JoinHostPort(ins.GetManageHost(), ins.GetPort()) masterActive = append(masterActive, instAddr) } masterStatus[ins.ID()] = status @@ -564,7 +608,7 @@ func (m *Manager) GetClusterTopology(dopt DisplayOption, opt operator.Options) ( switch ins.ComponentName() { case spec.ComponentPD: status = masterStatus[ins.ID()] - instAddr := utils.JoinHostPort(ins.GetHost(), ins.GetPort()) + instAddr := utils.JoinHostPort(ins.GetManageHost(), ins.GetPort()) if dashboardAddr == instAddr { status += "|UI" } @@ -581,11 +625,12 @@ func (m *Manager) GetClusterTopology(dopt DisplayOption, opt operator.Options) ( // Query the service status and uptime if status == "-" || (dopt.ShowUptime && since == "-") || dopt.ShowProcess { - e, found := ctxt.GetInner(ctx).GetExecutor(ins.GetHost()) + e, found := ctxt.GetInner(ctx).GetExecutor(ins.GetManageHost()) if found { var active string + var systemdSince time.Duration nctx := checkpoint.NewContext(ctx) - active, memory, _ = operator.GetServiceStatus(nctx, e, ins.ServiceName()) + active, memory, systemdSince, _ = operator.GetServiceStatus(nctx, e, ins.ServiceName()) if status == "-" { if active == "active" { status = "Up" @@ -594,13 +639,19 @@ func (m *Manager) GetClusterTopology(dopt DisplayOption, opt operator.Options) ( } } if dopt.ShowUptime && since == "-" { - since = formatInstanceSince(parseSystemctlSince(active)) + since = formatInstanceSince(systemdSince) } } } // check if the role is patched roleName := ins.Role() + // get extended name for TiFlash to distinguish disaggregated mode. + if ins.ComponentName() == spec.ComponentTiFlash { + tiflashInstance := ins.(*spec.TiFlashInstance) + tiflashSpec := tiflashInstance.InstanceSpec.(*spec.TiFlashSpec) + roleName += tiflashSpec.GetExtendedRole(ctx, tlsCfg, masterActive...) + } if ins.IsPatched() { roleName += " (patched)" } @@ -610,6 +661,7 @@ func (m *Manager) GetClusterTopology(dopt DisplayOption, opt operator.Options) ( ID: ins.ID(), Role: roleName, Host: ins.GetHost(), + ManageHost: ins.GetManageHost(), Ports: utils.JoinInt(ins.UsedPorts(), "/"), OsArch: tui.OsArch(ins.OS(), ins.Arch()), Status: status, @@ -621,6 +673,9 @@ func (m *Manager) GetClusterTopology(dopt DisplayOption, opt operator.Options) ( ComponentName: ins.ComponentName(), Port: ins.GetPort(), Since: since, + NumaNode: utils.Ternary(ins.GetNumaNode() == "", "-", ins.GetNumaNode()).(string), + NumaCores: utils.Ternary(ins.GetNumaCores() == "", "-", ins.GetNumaCores()).(string), + Version: ins.CalculateVersion(base.Version), }) mu.Unlock() }, opt.Concurrency) @@ -700,37 +755,6 @@ func formatInstanceSince(uptime time.Duration) string { return strings.Join(parts, "") } -// `systemctl status xxx.service` returns as below -// Active: active (running) since Sat 2021-03-27 10:51:11 CST; 41min ago -func parseSystemctlSince(str string) (dur time.Duration) { - // if service is not found or other error, don't need to parse it - if str == "" { - return 0 - } - defer func() { - if dur == 0 { - zap.L().Warn("failed to parse systemctl since", zap.String("value", str)) - } - }() - parts := strings.Split(str, ";") - if len(parts) != 2 { - return - } - parts = strings.Split(parts[0], " ") - if len(parts) < 3 { - return - } - - dateStr := strings.Join(parts[len(parts)-3:], " ") - - tm, err := time.Parse("2006-01-02 15:04:05 MST", dateStr) - if err != nil { - return - } - - return time.Since(tm) -} - // SetSSHKeySet set ssh key set. func SetSSHKeySet(ctx context.Context, privateKeyPath string, publicKeyPath string) error { ctxt.GetInner(ctx).PrivateKeyPath = privateKeyPath @@ -750,7 +774,7 @@ func SetClusterSSH(ctx context.Context, topo spec.Topology, deployUser string, s for _, com := range topo.ComponentsByStartOrder() { for _, in := range com.Instances() { cf := executor.SSHConfig{ - Host: in.GetHost(), + Host: in.GetManageHost(), Port: in.GetSSHPort(), KeyFile: ctxt.GetInner(ctx).PrivateKeyPath, User: deployUser, @@ -761,7 +785,7 @@ func SetClusterSSH(ctx context.Context, topo spec.Topology, deployUser string, s if err != nil { return err } - ctxt.GetInner(ctx).SetExecutor(in.GetHost(), e) + ctxt.GetInner(ctx).SetExecutor(in.GetManageHost(), e) } } @@ -776,13 +800,8 @@ func (m *Manager) DisplayDashboardInfo(clusterName string, timeout time.Duration return err } - pdEndpoints := make([]string, 0) - for _, pd := range metadata.Topology.PDServers { - pdEndpoints = append(pdEndpoints, utils.JoinHostPort(pd.Host, pd.ClientPort)) - } - ctx := context.WithValue(context.Background(), logprinter.ContextKeyLogger, m.logger) - pdAPI := api.NewPDClient(ctx, pdEndpoints, timeout, tlsCfg) + pdAPI := api.NewPDClient(ctx, metadata.Topology.GetPDListWithManageHost(), timeout, tlsCfg) dashboardAddr, err := pdAPI.GetDashboardAddress() if err != nil { return fmt.Errorf("failed to retrieve TiDB Dashboard instance from PD: %s", err) diff --git a/pkg/cluster/manager/exec.go b/pkg/cluster/manager/exec.go index 9252ca2cc1..bdda8c2ba4 100644 --- a/pkg/cluster/manager/exec.go +++ b/pkg/cluster/manager/exec.go @@ -15,7 +15,6 @@ package manager import ( "context" - "strings" "github.com/fatih/color" "github.com/joomcode/errorx" @@ -55,13 +54,13 @@ func (m *Manager) Exec(name string, opt ExecOptions, gOpt operator.Options) erro var shellTasks []task.Task uniqueHosts := map[string]set.StringSet{} // host-sshPort -> {command} topo.IterInstance(func(inst spec.Instance) { - key := utils.JoinHostPort(inst.GetHost(), inst.GetSSHPort()) + key := utils.JoinHostPort(inst.GetManageHost(), inst.GetSSHPort()) if _, found := uniqueHosts[key]; !found { if len(gOpt.Roles) > 0 && !filterRoles.Exist(inst.Role()) { return } - if len(gOpt.Nodes) > 0 && !filterNodes.Exist(inst.GetHost()) { + if len(gOpt.Nodes) > 0 && (!filterNodes.Exist(inst.GetHost()) && !filterNodes.Exist(inst.GetManageHost())) { return } @@ -80,7 +79,7 @@ func (m *Manager) Exec(name string, opt ExecOptions, gOpt operator.Options) erro }) for hostKey, i := range uniqueHosts { - host := strings.Split(hostKey, ":")[0] + host, _ := utils.ParseHostPort(hostKey) for _, cmd := range i.Slice() { shellTasks = append(shellTasks, task.NewBuilder(m.logger). @@ -113,7 +112,7 @@ func (m *Manager) Exec(name string, opt ExecOptions, gOpt operator.Options) erro // print outputs for hostKey, i := range uniqueHosts { - host := strings.Split(hostKey, "-")[0] + host, _ := utils.ParseHostPort(hostKey) for _, cmd := range i.Slice() { stdout, stderr, ok := ctxt.GetInner(execCtx).GetOutputs(hostKey + cmd) if !ok { diff --git a/pkg/cluster/manager/manager.go b/pkg/cluster/manager/manager.go index 7eb9170541..bfe15feddd 100644 --- a/pkg/cluster/manager/manager.go +++ b/pkg/cluster/manager/manager.go @@ -45,7 +45,6 @@ var ( type Manager struct { sysName string specManager *spec.SpecManager - bindVersion spec.BindVersion logger *logprinter.Logger } @@ -53,13 +52,11 @@ type Manager struct { func NewManager( sysName string, specManager *spec.SpecManager, - bindVersion spec.BindVersion, logger *logprinter.Logger, ) *Manager { return &Manager{ sysName: sysName, specManager: specManager, - bindVersion: bindVersion, logger: logger, } } @@ -94,23 +91,39 @@ func (m *Manager) confirmTopology(name, version string, topo spec.Topology, patc fmt.Printf("TLS encryption: %s\n", cyan.Sprint("enabled")) } + // check if managehost is set + manageHost := false + topo.IterInstance(func(inst spec.Instance) { + if inst.GetHost() != inst.GetManageHost() { + manageHost = true + return + } + }) + clusterTable := [][]string{ // Header - {"Role", "Host", "Ports", "OS/Arch", "Directories"}, + {"Role", "Host"}, + } + if manageHost { + clusterTable[0] = append(clusterTable[0], "Manage Host") } + clusterTable[0] = append(clusterTable[0], "Ports", "OS/Arch", "Directories") topo.IterInstance(func(instance spec.Instance) { comp := instance.ComponentName() if patchedRoles.Exist(comp) || instance.IsPatched() { comp += " (patched)" } - clusterTable = append(clusterTable, []string{ - comp, - instance.GetHost(), + instInfo := []string{comp, instance.GetHost()} + if manageHost { + instInfo = append(instInfo, instance.GetManageHost()) + } + instInfo = append(instInfo, utils.JoinInt(instance.UsedPorts(), "/"), tui.OsArch(instance.OS(), instance.Arch()), - strings.Join(instance.UsedDirs(), ","), - }) + strings.Join(instance.UsedDirs(), ",")) + + clusterTable = append(clusterTable, instInfo) }) tui.PrintTable(clusterTable, true) @@ -190,14 +203,14 @@ func (m *Manager) fillHostArchOrOS(s, p *tui.SSHConnectionProps, topo spec.Topol return } - if _, ok := hostArchOrOS[inst.GetHost()]; ok { + if _, ok := hostArchOrOS[inst.GetManageHost()]; ok { return } - hostArchOrOS[inst.GetHost()] = "" + hostArchOrOS[inst.GetManageHost()] = "" tf := task.NewBuilder(m.logger). RootSSH( - inst.GetHost(), + inst.GetManageHost(), inst.GetSSHPort(), user, s.Password, @@ -218,11 +231,11 @@ func (m *Manager) fillHostArchOrOS(s, p *tui.SSHConnectionProps, topo spec.Topol switch fullType { case spec.FullOSType: - tf = tf.Shell(inst.GetHost(), "uname -s", "", false) + tf = tf.Shell(inst.GetManageHost(), "uname -s", "", false) default: - tf = tf.Shell(inst.GetHost(), "uname -m", "", false) + tf = tf.Shell(inst.GetManageHost(), "uname -m", "", false) } - detectTasks = append(detectTasks, tf.BuildAsStep(fmt.Sprintf(" - Detecting node %s %s info", inst.GetHost(), string(fullType)))) + detectTasks = append(detectTasks, tf.BuildAsStep(fmt.Sprintf(" - Detecting node %s %s info", inst.GetManageHost(), string(fullType)))) }) if len(detectTasks) == 0 { return nil diff --git a/pkg/cluster/manager/patch.go b/pkg/cluster/manager/patch.go index ad2dfa7bc7..64936ec952 100644 --- a/pkg/cluster/manager/patch.go +++ b/pkg/cluster/manager/patch.go @@ -19,6 +19,7 @@ import ( "os" "os/exec" "path" + "path/filepath" "strings" "github.com/fatih/color" @@ -77,7 +78,7 @@ func (m *Manager) Patch(name string, packagePath string, opt operator.Options, o if err != nil { return err } - if err := checkPackage(m.bindVersion, m.specManager, name, insts[0].ComponentName(), insts[0].OS(), insts[0].Arch(), packagePath); err != nil { + if err := checkPackage(m.specManager, name, insts[0], insts[0].OS(), insts[0].Arch(), packagePath); err != nil { return err } @@ -85,8 +86,8 @@ func (m *Manager) Patch(name string, packagePath string, opt operator.Options, o for _, inst := range insts { deployDir := spec.Abs(base.User, inst.DeployDir()) tb := task.NewBuilder(m.logger) - tb.BackupComponent(inst.ComponentName(), base.Version, inst.GetHost(), deployDir). - InstallPackage(packagePath, inst.GetHost(), deployDir) + tb.BackupComponent(inst.ComponentName(), base.Version, inst.GetManageHost(), deployDir). + InstallPackage(packagePath, inst.GetManageHost(), deployDir) replacePackageTasks = append(replacePackageTasks, tb.Build()) } @@ -103,7 +104,8 @@ func (m *Manager) Patch(name string, packagePath string, opt operator.Options, o if offline { return nil } - return operator.Upgrade(ctx, topo, opt, tlsCfg, base.Version) + // TBD: should patch be treated as an upgrade? + return operator.Upgrade(ctx, topo, opt, tlsCfg, base.Version, base.Version) }). Build() @@ -138,18 +140,18 @@ func (m *Manager) Patch(name string, packagePath string, opt operator.Options, o return m.specManager.SaveMeta(name, metadata) } -func checkPackage(bindVersion spec.BindVersion, specManager *spec.SpecManager, name, comp, nodeOS, arch, packagePath string) error { +func checkPackage(specManager *spec.SpecManager, name string, inst spec.Instance, nodeOS, arch, packagePath string) error { metadata := specManager.NewMetadata() if err := specManager.Metadata(name, metadata); err != nil { return err } - ver := bindVersion(comp, metadata.GetBaseMeta().Version) + ver := inst.CalculateVersion(metadata.GetBaseMeta().Version) repo, err := clusterutil.NewRepository(nodeOS, arch) if err != nil { return err } - entry, err := repo.ComponentBinEntry(comp, ver) + entry, err := repo.ComponentBinEntry(inst.ComponentSource(), ver) if err != nil { return err } @@ -158,8 +160,8 @@ func checkPackage(bindVersion spec.BindVersion, specManager *spec.SpecManager, n if err != nil { return err } - cacheDir := specManager.Path(name, "cache", comp+"-"+checksum[:7]) - if err := os.MkdirAll(cacheDir, 0755); err != nil { + cacheDir := specManager.Path(name, "cache", inst.ComponentSource()+"-"+checksum[:7]) + if err := utils.MkdirAll(cacheDir, 0755); err != nil { return perrs.Annotatef(err, "create cache directory %s", cacheDir) } if err := exec.Command("tar", "-xvf", packagePath, "-C", cacheDir).Run(); err != nil { @@ -184,7 +186,7 @@ func checkPackage(bindVersion spec.BindVersion, specManager *spec.SpecManager, n } func overwritePatch(specManager *spec.SpecManager, name, comp, packagePath string) error { - if err := os.MkdirAll(specManager.Path(name, spec.PatchDirName), 0755); err != nil { + if err := utils.MkdirAll(specManager.Path(name, spec.PatchDirName), 0755); err != nil { return err } @@ -204,7 +206,13 @@ func overwritePatch(specManager *spec.SpecManager, name, comp, packagePath strin if utils.IsSymExist(symlink) { os.Remove(symlink) } - return os.Symlink(tg, symlink) + + tgRelPath, err := filepath.Rel(filepath.Dir(symlink), tg) + if err != nil { + return err + } + + return os.Symlink(tgRelPath, symlink) } func instancesToPatch(topo spec.Topology, options operator.Options) ([]spec.Instance, error) { diff --git a/pkg/cluster/manager/reload.go b/pkg/cluster/manager/reload.go index 2a78e3fe41..226c66535e 100644 --- a/pkg/cluster/manager/reload.go +++ b/pkg/cluster/manager/reload.go @@ -124,7 +124,7 @@ func (m *Manager) Reload(name string, gOpt operator.Options, skipRestart, skipCo return err } b.Func("Upgrade Cluster", func(ctx context.Context) error { - return operator.Upgrade(ctx, topo, gOpt, tlsCfg, base.Version) + return operator.Upgrade(ctx, topo, gOpt, tlsCfg, base.Version, base.Version) }) } diff --git a/pkg/cluster/manager/rotate_ssh.go b/pkg/cluster/manager/rotate_ssh.go new file mode 100644 index 0000000000..aceb98a2d5 --- /dev/null +++ b/pkg/cluster/manager/rotate_ssh.go @@ -0,0 +1,99 @@ +// Copyright 2023 PingCAP, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// See the License for the specific language governing permissions and +// limitations under the License. + +package manager + +import ( + "context" + "errors" + "fmt" + "os" + + "github.com/joomcode/errorx" + perrs "github.com/pingcap/errors" + "github.com/pingcap/tiup/pkg/cluster/ctxt" + operator "github.com/pingcap/tiup/pkg/cluster/operation" + "github.com/pingcap/tiup/pkg/cluster/spec" + "github.com/pingcap/tiup/pkg/cluster/task" + "github.com/pingcap/tiup/pkg/meta" + "github.com/pingcap/tiup/pkg/tui" +) + +// RotateSSH rotate public keys of target nodes +func (m *Manager) RotateSSH(name string, gOpt operator.Options, skipConfirm bool) error { + metadata, err := m.meta(name) + if err != nil && !errors.Is(perrs.Cause(err), meta.ErrValidate) && + !errors.Is(perrs.Cause(err), spec.ErrNoTiSparkMaster) { + return err + } + + topo := metadata.GetTopology() + base := metadata.GetBaseMeta() + if !skipConfirm { + if err := tui.PromptForConfirmOrAbortError( + "This operation will rotate ssh keys for user '%s' .\nDo you want to continue? [y/N]:", + base.User); err != nil { + return err + } + } + + var rotateSSHTasks []*task.StepDisplay // tasks which are used to initialize environment + uniqueHosts, _ := getMonitorHosts(topo) + for host, hostInfo := range uniqueHosts { + t, err := m.sshTaskBuilder(name, topo, base.User, gOpt) + if err != nil { + return err + } + t = t.RotateSSH(host, base.User, m.specManager.Path(name, "ssh", "new.pub")) + + rotateSSHTasks = append(rotateSSHTasks, t.BuildAsStep(fmt.Sprintf(" - Rotate ssh key on %s:%d", host, hostInfo.ssh))) + } + + builder := task.NewBuilder(m.logger). + Step("+ Generate new SSH keys", + task.NewBuilder(m.logger). + SSHKeyGen(m.specManager.Path(name, "ssh", "new")). + Build(), + m.logger). + ParallelStep("+ rotate ssh keys of target host environments", false, rotateSSHTasks...). + Step("+ overwrite old SSH keys", + task.NewBuilder(m.logger). + Func("rename", func(ctx context.Context) error { + err := os.Rename(m.specManager.Path(name, "ssh", "new.pub"), m.specManager.Path(name, "ssh", "id_rsa.pub")) + if err != nil { + return err + } + err = os.Rename(m.specManager.Path(name, "ssh", "new"), m.specManager.Path(name, "ssh", "id_rsa")) + if err != nil { + return err + } + return nil + }). + Build(), + m.logger) + + ctx := ctxt.New( + context.Background(), + gOpt.Concurrency, + m.logger, + ) + if err := builder.Build().Execute(ctx); err != nil { + if errorx.Cast(err) != nil { + // FIXME: Map possible task errors and give suggestions. + return err + } + return err + } + m.logger.Infof("ssh keys are successfully updated") + return nil +} diff --git a/pkg/cluster/manager/transfer.go b/pkg/cluster/manager/transfer.go index 8ac8615bba..854ba70a64 100644 --- a/pkg/cluster/manager/transfer.go +++ b/pkg/cluster/manager/transfer.go @@ -63,13 +63,13 @@ func (m *Manager) Transfer(name string, opt TransferOptions, gOpt operator.Optio uniqueHosts := map[string]set.StringSet{} // host-sshPort -> {remote-path} topo.IterInstance(func(inst spec.Instance) { - key := fmt.Sprintf("%d-%s", inst.GetSSHPort(), inst.GetHost()) + key := fmt.Sprintf("%d-%s", inst.GetSSHPort(), inst.GetManageHost()) if _, found := uniqueHosts[key]; !found { if len(gOpt.Roles) > 0 && !filterRoles.Exist(inst.Role()) { return } - if len(gOpt.Nodes) > 0 && !filterNodes.Exist(inst.GetHost()) { + if len(gOpt.Nodes) > 0 && (!filterNodes.Exist(inst.GetHost()) || !filterNodes.Exist(inst.GetManageHost())) { return } diff --git a/pkg/cluster/manager/upgrade.go b/pkg/cluster/manager/upgrade.go index db03c5dfe1..aa677a5e53 100644 --- a/pkg/cluster/manager/upgrade.go +++ b/pkg/cluster/manager/upgrade.go @@ -17,12 +17,15 @@ import ( "context" "fmt" "os" + "strings" + "time" "github.com/fatih/color" "github.com/joomcode/errorx" perrs "github.com/pingcap/errors" "github.com/pingcap/tiup/pkg/cluster/clusterutil" "github.com/pingcap/tiup/pkg/cluster/ctxt" + "github.com/pingcap/tiup/pkg/cluster/executor" operator "github.com/pingcap/tiup/pkg/cluster/operation" "github.com/pingcap/tiup/pkg/cluster/spec" "github.com/pingcap/tiup/pkg/cluster/task" @@ -35,7 +38,19 @@ import ( ) // Upgrade the cluster. -func (m *Manager) Upgrade(name string, clusterVersion string, opt operator.Options, skipConfirm, offline bool) error { +func (m *Manager) Upgrade(name string, clusterVersion string, componentVersions map[string]string, opt operator.Options, skipConfirm, offline, ignoreVersionCheck bool) error { + if !skipConfirm && strings.ToLower(opt.DisplayMode) != "json" { + for _, v := range componentVersions { + if v != "" { + m.logger.Warnf(color.YellowString("tiup-cluster does not provide compatibility guarantees or version checks for different component versions. Please be aware of the risks or use it with the assistance of PingCAP support.")) + err := tui.PromptForConfirmOrAbortError("Do you want to continue? [y/N]: ") + if err != nil { + return err + } + break + } + } + } if err := clusterutil.ValidateClusterNameOrError(name); err != nil { return err } @@ -66,43 +81,62 @@ func (m *Manager) Upgrade(name string, clusterVersion string, opt operator.Optio ) if err := versionCompare(base.Version, clusterVersion); err != nil { - return err + if !ignoreVersionCheck { + return err + } + m.logger.Warnf(color.RedString("There is no guarantee that the cluster can be downgraded. Be careful before you continue.")) } + compVersionMsg := "" + for _, comp := range topo.ComponentsByUpdateOrder(base.Version) { + // if component version is not specified, use the cluster version or latest("") + version := componentVersions[comp.Name()] + if version != "" { + comp.SetVersion(version) + } + if len(comp.Instances()) > 0 { + compVersionMsg += fmt.Sprintf("\nwill upgrade component %19s to \"%s\",", "\""+comp.Name()+"\"", comp.CalculateVersion(clusterVersion)) + } + } + monitoredOptions := topo.GetMonitoredOptions() + if monitoredOptions != nil { + if componentVersions[spec.ComponentBlackboxExporter] != "" { + monitoredOptions.BlackboxExporterVersion = componentVersions[spec.ComponentBlackboxExporter] + } + if componentVersions[spec.ComponentNodeExporter] != "" { + monitoredOptions.NodeExporterVersion = componentVersions[spec.ComponentNodeExporter] + } + compVersionMsg += fmt.Sprintf("\nwill upgrade component %19s to \"%s\",", "\"node-exporter\"", monitoredOptions.NodeExporterVersion) + compVersionMsg += fmt.Sprintf("\nwill upgrade component %19s to \"%s\".", "\"blackbox-exporter\"", monitoredOptions.BlackboxExporterVersion) + } + + m.logger.Warnf(`%s +This operation will upgrade %s %s cluster %s to %s:%s`, + color.YellowString("Before the upgrade, it is recommended to read the upgrade guide at https://docs.pingcap.com/tidb/stable/upgrade-tidb-using-tiup and finish the preparation steps."), + m.sysName, + color.HiYellowString(base.Version), + color.HiYellowString(name), + color.HiYellowString(clusterVersion), + compVersionMsg) if !skipConfirm { - if err := tui.PromptForConfirmOrAbortError( - "This operation will upgrade %s %s cluster %s to %s.\nDo you want to continue? [y/N]:", - m.sysName, - color.HiYellowString(base.Version), - color.HiYellowString(name), - color.HiYellowString(clusterVersion)); err != nil { + if err := tui.PromptForConfirmOrAbortError(`Do you want to continue? [y/N]:`); err != nil { return err } m.logger.Infof("Upgrading cluster...") } hasImported := false - for _, comp := range topo.ComponentsByUpdateOrder() { - for _, inst := range comp.Instances() { - compName := inst.ComponentName() - - // ignore monitor agents for instances marked as ignore_exporter - switch compName { - case spec.ComponentNodeExporter, - spec.ComponentBlackboxExporter: - if inst.IgnoreMonitorAgent() { - continue - } - } - - version := m.bindVersion(inst.ComponentName(), clusterVersion) + for _, comp := range topo.ComponentsByUpdateOrder(base.Version) { + compName := comp.Name() + version := comp.CalculateVersion(clusterVersion) + for _, inst := range comp.Instances() { // Download component from repository key := fmt.Sprintf("%s-%s-%s-%s", compName, version, inst.OS(), inst.Arch()) if _, found := uniqueComps[key]; !found { uniqueComps[key] = struct{}{} t := task.NewBuilder(m.logger). - Download(inst.ComponentName(), inst.OS(), inst.Arch(), version). + Download(inst.ComponentSource(), inst.OS(), inst.Arch(), version). Build() downloadCompTasks = append(downloadCompTasks, t) } @@ -118,18 +152,18 @@ func (m *Manager) Upgrade(name string, clusterVersion string, opt operator.Optio // for some component, dataDirs might need to be created due to upgrade // eg: TiCDC support DataDir since v4.0.13 - tb = tb.Mkdir(topo.BaseTopo().GlobalOptions.User, inst.GetHost(), dataDirs...) + tb = tb.Mkdir(topo.BaseTopo().GlobalOptions.User, inst.GetManageHost(), dataDirs...) if inst.IsImported() { switch inst.ComponentName() { case spec.ComponentPrometheus, spec.ComponentGrafana, spec.ComponentAlertmanager: tb.CopyComponent( - inst.ComponentName(), + inst.ComponentSource(), inst.OS(), inst.Arch(), version, "", // use default srcPath - inst.GetHost(), + inst.GetManageHost(), deployDir, ) } @@ -137,8 +171,9 @@ func (m *Manager) Upgrade(name string, clusterVersion string, opt operator.Optio } // backup files of the old version - tb = tb.BackupComponent(inst.ComponentName(), base.Version, inst.GetHost(), deployDir) + tb = tb.BackupComponent(inst.ComponentSource(), base.Version, inst.GetManageHost(), deployDir) + // this interface is not used if deployerInstance, ok := inst.(DeployerInstance); ok { deployerInstance.Deploy(tb, "", deployDir, version, name, clusterVersion) } else { @@ -156,12 +191,12 @@ func (m *Manager) Upgrade(name string, clusterVersion string, opt operator.Optio tb = tb.DeploySpark(inst, sparkVer.String(), "" /* default srcPath */, deployDir) default: tb = tb.CopyComponent( - inst.ComponentName(), + inst.ComponentSource(), inst.OS(), inst.Arch(), version, "", // use default srcPath - inst.GetHost(), + inst.GetManageHost(), deployDir, ) } @@ -185,37 +220,96 @@ func (m *Manager) Upgrade(name string, clusterVersion string, opt operator.Optio } } + var sshProxyProps *tui.SSHConnectionProps = &tui.SSHConnectionProps{} + if opt.SSHType != executor.SSHTypeNone { + var err error + if len(opt.SSHProxyHost) != 0 { + if sshProxyProps, err = tui.ReadIdentityFileOrPassword(opt.SSHProxyIdentity, opt.SSHProxyUsePassword); err != nil { + return err + } + } + } + + uniqueHosts, noAgentHosts := getMonitorHosts(topo) + // Deploy monitor relevant components to remote + dlTasks, dpTasks, err := buildMonitoredDeployTask( + m, + uniqueHosts, + noAgentHosts, + topo.BaseTopo().GlobalOptions, + monitoredOptions, + opt, + sshProxyProps, + ) + if err != nil { + return err + } + + monitorConfigTasks := buildInitMonitoredConfigTasks( + m.specManager, + name, + uniqueHosts, + noAgentHosts, + *topo.BaseTopo().GlobalOptions, + monitoredOptions, + m.logger, + opt.SSHTimeout, + opt.OptTimeout, + opt, + sshProxyProps, + ) + // handle dir scheme changes if hasImported { if err := spec.HandleImportPathMigration(name); err != nil { return err } } - + ctx := ctxt.New( + context.Background(), + opt.Concurrency, + m.logger, + ) tlsCfg, err := topo.TLSConfig(m.specManager.Path(name, spec.TLSCertKeyDir)) if err != nil { return err } + + // make sure the cluster is stopped + if offline && !opt.Force { + running := false + topo.IterInstance(func(ins spec.Instance) { + if !running { + status := ins.Status(ctx, time.Duration(opt.APITimeout), tlsCfg, topo.BaseTopo().MasterList...) + if strings.HasPrefix(status, "Up") || strings.HasPrefix(status, "Healthy") { + running = true + } + } + }, opt.Concurrency) + + if running { + return perrs.Errorf("cluster is running and cannot be upgraded offline") + } + } + b, err := m.sshTaskBuilder(name, topo, base.User, opt) if err != nil { return err } t := b. Parallel(false, downloadCompTasks...). + ParallelStep("download monitored", false, dlTasks...). Parallel(opt.Force, copyCompTasks...). + ParallelStep("deploy monitored", false, dpTasks...). + ParallelStep("refresh monitored config", false, monitorConfigTasks...). Func("UpgradeCluster", func(ctx context.Context) error { if offline { return nil } - return operator.Upgrade(ctx, topo, opt, tlsCfg, base.Version) + return operator.Upgrade(ctx, topo, opt, tlsCfg, base.Version, clusterVersion) }). Build() - ctx := ctxt.New( - context.Background(), - opt.Concurrency, - m.logger, - ) if err := t.Execute(ctx); err != nil { if errorx.Cast(err) != nil { // FIXME: Map possible task errors and give suggestions. @@ -252,10 +346,10 @@ func versionCompare(curVersion, newVersion string) error { } switch semver.Compare(curVersion, newVersion) { - case -1: + case -1, 0: return nil - case 0, 1: - return perrs.Errorf("please specify a higher version than %s", curVersion) + case 1: + return perrs.Errorf("please specify a higher or equle version than %s", curVersion) default: return perrs.Errorf("unreachable") } diff --git a/pkg/cluster/module/wait_for.go b/pkg/cluster/module/wait_for.go index 8a59104e1b..757b0f8fae 100644 --- a/pkg/cluster/module/wait_for.go +++ b/pkg/cluster/module/wait_for.go @@ -21,6 +21,7 @@ import ( "github.com/pingcap/errors" "github.com/pingcap/tiup/pkg/cluster/ctxt" + "github.com/pingcap/tiup/pkg/cluster/executor" "github.com/pingcap/tiup/pkg/utils" "go.uber.org/zap" ) @@ -71,7 +72,7 @@ func (w *WaitFor) Execute(ctx context.Context, e ctxt.Executor) (err error) { } if err := utils.Retry(func() error { // only listing TCP ports - stdout, _, err := e.Execute(ctx, "ss -ltn", false) + stdout, _, err := executor.UnwarpCheckPointExecutor(e).Execute(ctx, "ss -ltn", false) if err == nil { switch w.c.State { case "started": diff --git a/pkg/cluster/operation/action.go b/pkg/cluster/operation/action.go index 4015e0260e..54ad1434a9 100644 --- a/pkg/cluster/operation/action.go +++ b/pkg/cluster/operation/action.go @@ -65,9 +65,9 @@ func Enable( instCount := map[string]int{} cluster.IterInstance(func(inst spec.Instance) { if inst.IgnoreMonitorAgent() { - noAgentHosts.Insert(inst.GetHost()) + noAgentHosts.Insert(inst.GetManageHost()) } else { - instCount[inst.GetHost()]++ + instCount[inst.GetManageHost()]++ } }) @@ -80,7 +80,7 @@ func Enable( for _, inst := range insts { if !inst.IgnoreMonitorAgent() { - instCount[inst.GetHost()]-- + instCount[inst.GetManageHost()]-- } } } @@ -119,7 +119,7 @@ func Start( cluster.IterInstance(func(inst spec.Instance) { if inst.IgnoreMonitorAgent() { - noAgentHosts.Insert(inst.GetHost()) + noAgentHosts.Insert(inst.GetManageHost()) } }) @@ -133,7 +133,7 @@ func Start( errg, _ := errgroup.WithContext(ctx) for _, inst := range insts { if !inst.IgnoreMonitorAgent() { - uniqueHosts.Insert(inst.GetHost()) + uniqueHosts.Insert(inst.GetManageHost()) } if restoreLeader { @@ -185,9 +185,9 @@ func Stop( instCount := map[string]int{} cluster.IterInstance(func(inst spec.Instance) { if inst.IgnoreMonitorAgent() { - noAgentHosts.Insert(inst.GetHost()) + noAgentHosts.Insert(inst.GetManageHost()) } else { - instCount[inst.GetHost()]++ + instCount[inst.GetManageHost()]++ } }) @@ -208,7 +208,7 @@ func Stop( } for _, inst := range insts { if !inst.IgnoreMonitorAgent() { - instCount[inst.GetHost()]-- + instCount[inst.GetManageHost()]-- } } } @@ -354,17 +354,17 @@ func systemctlMonitor(ctx context.Context, hosts []string, noAgentHosts set.Stri } func restartInstance(ctx context.Context, ins spec.Instance, timeout uint64, tlsCfg *tls.Config) error { - e := ctxt.GetInner(ctx).Get(ins.GetHost()) + e := ctxt.GetInner(ctx).Get(ins.GetManageHost()) logger := ctx.Value(logprinter.ContextKeyLogger).(*logprinter.Logger) logger.Infof("\tRestarting instance %s", ins.ID()) if err := systemctl(ctx, e, ins.ServiceName(), "restart", timeout); err != nil { - return toFailedActionError(err, "restart", ins.GetHost(), ins.ServiceName(), ins.LogDir()) + return toFailedActionError(err, "restart", ins.GetManageHost(), ins.ServiceName(), ins.LogDir()) } // Check ready. if err := ins.Ready(ctx, e, timeout, tlsCfg); err != nil { - return toFailedActionError(err, "restart", ins.GetHost(), ins.ServiceName(), ins.LogDir()) + return toFailedActionError(err, "restart", ins.GetManageHost(), ins.ServiceName(), ins.LogDir()) } logger.Infof("\tRestart instance %s success", ins.ID()) @@ -373,7 +373,7 @@ func restartInstance(ctx context.Context, ins spec.Instance, timeout uint64, tls } func enableInstance(ctx context.Context, ins spec.Instance, timeout uint64, isEnable bool) error { - e := ctxt.GetInner(ctx).Get(ins.GetHost()) + e := ctxt.GetInner(ctx).Get(ins.GetManageHost()) logger := ctx.Value(logprinter.ContextKeyLogger).(*logprinter.Logger) action := "disable" @@ -384,7 +384,7 @@ func enableInstance(ctx context.Context, ins spec.Instance, timeout uint64, isEn // Enable/Disable by systemd. if err := systemctl(ctx, e, ins.ServiceName(), action, timeout); err != nil { - return toFailedActionError(err, action, ins.GetHost(), ins.ServiceName(), ins.LogDir()) + return toFailedActionError(err, action, ins.GetManageHost(), ins.ServiceName(), ins.LogDir()) } logger.Infof("\t%s instance %s success", actionPostMsgs[action], ins.ID()) @@ -393,17 +393,17 @@ func enableInstance(ctx context.Context, ins spec.Instance, timeout uint64, isEn } func startInstance(ctx context.Context, ins spec.Instance, timeout uint64, tlsCfg *tls.Config) error { - e := ctxt.GetInner(ctx).Get(ins.GetHost()) + e := ctxt.GetInner(ctx).Get(ins.GetManageHost()) logger := ctx.Value(logprinter.ContextKeyLogger).(*logprinter.Logger) logger.Infof("\tStarting instance %s", ins.ID()) if err := systemctl(ctx, e, ins.ServiceName(), "start", timeout); err != nil { - return toFailedActionError(err, "start", ins.GetHost(), ins.ServiceName(), ins.LogDir()) + return toFailedActionError(err, "start", ins.GetManageHost(), ins.ServiceName(), ins.LogDir()) } // Check ready. if err := ins.Ready(ctx, e, timeout, tlsCfg); err != nil { - return toFailedActionError(err, "start", ins.GetHost(), ins.ServiceName(), ins.LogDir()) + return toFailedActionError(err, "start", ins.GetManageHost(), ins.ServiceName(), ins.LogDir()) } logger.Infof("\tStart instance %s success", ins.ID()) @@ -465,8 +465,8 @@ func EnableComponent(ctx context.Context, instances []spec.Instance, noAgentHost switch name { case spec.ComponentNodeExporter, spec.ComponentBlackboxExporter: - if noAgentHosts.Exist(ins.GetHost()) { - logger.Debugf("Ignored enabling/disabling %s for %s:%d", name, ins.GetHost(), ins.GetPort()) + if noAgentHosts.Exist(ins.GetManageHost()) { + logger.Debugf("Ignored enabling/disabling %s for %s:%d", name, ins.GetManageHost(), ins.GetPort()) continue } } @@ -517,8 +517,8 @@ func StartComponent(ctx context.Context, instances []spec.Instance, noAgentHosts switch name { case spec.ComponentNodeExporter, spec.ComponentBlackboxExporter: - if noAgentHosts.Exist(ins.GetHost()) { - logger.Debugf("Ignored starting %s for %s:%d", name, ins.GetHost(), ins.GetPort()) + if noAgentHosts.Exist(ins.GetManageHost()) { + logger.Debugf("Ignored starting %s for %s:%d", name, ins.GetManageHost(), ins.GetPort()) continue } } @@ -551,12 +551,12 @@ func serialStartInstances(ctx context.Context, instances []spec.Instance, option } func stopInstance(ctx context.Context, ins spec.Instance, timeout uint64) error { - e := ctxt.GetInner(ctx).Get(ins.GetHost()) + e := ctxt.GetInner(ctx).Get(ins.GetManageHost()) logger := ctx.Value(logprinter.ContextKeyLogger).(*logprinter.Logger) - logger.Infof("\tStopping instance %s", ins.GetHost()) + logger.Infof("\tStopping instance %s", ins.GetManageHost()) if err := systemctl(ctx, e, ins.ServiceName(), "stop", timeout); err != nil { - return toFailedActionError(err, "stop", ins.GetHost(), ins.ServiceName(), ins.LogDir()) + return toFailedActionError(err, "stop", ins.GetManageHost(), ins.ServiceName(), ins.LogDir()) } logger.Infof("\tStop %s %s success", ins.ComponentName(), ins.ID()) @@ -589,8 +589,8 @@ func StopComponent(ctx context.Context, switch name { case spec.ComponentNodeExporter, spec.ComponentBlackboxExporter: - if noAgentHosts.Exist(ins.GetHost()) { - logger.Debugf("Ignored stopping %s for %s:%d", name, ins.GetHost(), ins.GetPort()) + if noAgentHosts.Exist(ins.GetManageHost()) { + logger.Debugf("Ignored stopping %s for %s:%d", name, ins.GetManageHost(), ins.GetPort()) continue } case spec.ComponentCDC: @@ -653,3 +653,21 @@ func monitorPortMap(options *spec.MonitoredOptions) map[string]int { spec.ComponentBlackboxExporter: options.BlackboxExporterPort, } } + +func executeSSHCommand(ctx context.Context, action, host, command string) error { + if command == "" { + return nil + } + e, found := ctxt.GetInner(ctx).GetExecutor(host) + if !found { + return fmt.Errorf("no executor") + } + logger := ctx.Value(logprinter.ContextKeyLogger).(*logprinter.Logger) + logger.Infof("\t%s on %s", action, host) + stdout, stderr, err := e.Execute(ctx, command, false) + if err != nil { + return errors.Annotatef(err, "stderr: %s", string(stderr)) + } + logger.Infof("\t%s", stdout) + return nil +} diff --git a/pkg/cluster/operation/check.go b/pkg/cluster/operation/check.go index 44c81ea13f..f04981518e 100644 --- a/pkg/cluster/operation/check.go +++ b/pkg/cluster/operation/check.go @@ -168,13 +168,10 @@ func checkOSInfo(opt *CheckOptions, osInfo *sysinfo.OS) *CheckResult { // check OS vendor switch osInfo.Vendor { case "kylin": - msg := "kylin support is not fully tested, be careful" - result.Err = fmt.Errorf("%s (%s)", result.Msg, msg) - result.Warn = true // VERSION_ID="V10" if ver, _ := strconv.ParseFloat(strings.Trim(osInfo.Version, "V"), 64); ver < 10 { - result.Err = fmt.Errorf("%s %s not supported, use version V10 or higher(%s)", - osInfo.Name, osInfo.Release, msg) + result.Err = fmt.Errorf("%s %s not supported, use version V10 or higher", + osInfo.Name, osInfo.Release) return result } case "amzn": @@ -187,8 +184,6 @@ func checkOSInfo(opt *CheckOptions, osInfo *sysinfo.OS) *CheckResult { } case "centos", "redhat", "rhel", "ol": // check version - // CentOS 8 is known to be not working, and we don't have plan to support it - // as of now, we may add support for RHEL 8 based systems in the future. if ver, _ := strconv.ParseFloat(osInfo.Version, 64); ver < 7 { result.Err = fmt.Errorf("%s %s not supported, use version 8 please", osInfo.Name, osInfo.Release) @@ -216,6 +211,8 @@ func checkOSInfo(opt *CheckOptions, osInfo *sysinfo.OS) *CheckResult { result.Warn = false return result } + case "openEuler": + return result default: result.Err = fmt.Errorf("os vendor %s not supported", osInfo.Vendor) return result @@ -523,7 +520,7 @@ func CheckServices(ctx context.Context, e ctxt.Executor, host, service string, d return result } - active, _, err := GetServiceStatus(ctx, e, service+".service") + active, _, _, err := GetServiceStatus(ctx, e, service+".service") if err != nil { result.Err = err } @@ -581,7 +578,7 @@ func CheckListeningPort(opt *CheckOptions, host string, topo *spec.Specification ports := make(map[int]struct{}) topo.IterInstance(func(inst spec.Instance) { - if inst.GetHost() != host { + if inst.GetManageHost() != host { return } for _, up := range inst.UsedPorts() { @@ -633,7 +630,7 @@ func CheckPartitions(opt *CheckOptions, host string, topo *spec.Specification, r uniqueStores := make(map[string][]storePartitionInfo) // host+partition -> info topo.IterInstance(func(inst spec.Instance) { - if inst.GetHost() != host { + if inst.GetManageHost() != host { return } for _, dataDir := range spec.MultiDirAbs(topo.GlobalOptions.User, inst.DataDir()) { diff --git a/pkg/cluster/operation/destroy.go b/pkg/cluster/operation/destroy.go index 5fae6a1250..1860ab438a 100644 --- a/pkg/cluster/operation/destroy.go +++ b/pkg/cluster/operation/destroy.go @@ -20,10 +20,10 @@ import ( "fmt" "os" "path/filepath" - "strconv" "strings" "time" + "github.com/fatih/color" perrs "github.com/pingcap/errors" "github.com/pingcap/tiup/pkg/cluster/api" "github.com/pingcap/tiup/pkg/cluster/ctxt" @@ -33,6 +33,7 @@ import ( logprinter "github.com/pingcap/tiup/pkg/logger/printer" "github.com/pingcap/tiup/pkg/proxy" "github.com/pingcap/tiup/pkg/set" + "github.com/pingcap/tiup/pkg/utils" ) // Destroy the cluster. @@ -45,7 +46,7 @@ func Destroy( instCount := map[string]int{} cluster.IterInstance(func(inst spec.Instance) { - instCount[inst.GetHost()]++ + instCount[inst.GetManageHost()]++ }) for _, com := range coms { @@ -55,8 +56,8 @@ func Destroy( return perrs.Annotatef(err, "failed to destroy %s", com.Name()) } for _, inst := range insts { - instCount[inst.GetHost()]-- - if instCount[inst.GetHost()] == 0 { + instCount[inst.GetManageHost()]-- + if instCount[inst.GetManageHost()] == 0 { if cluster.GetMonitoredOptions() != nil { if err := DestroyMonitored(ctx, inst, cluster.GetMonitoredOptions(), options.OptTimeout); err != nil && !options.Force { return err @@ -104,7 +105,7 @@ func StopAndDestroyInstance( cluster.IterInstance(func(inst spec.Instance) { if inst.IgnoreMonitorAgent() { - noAgentHosts.Insert(inst.GetHost()) + noAgentHosts.Insert(inst.GetManageHost()) } }) @@ -136,7 +137,7 @@ func StopAndDestroyInstance( monitoredOptions := cluster.GetMonitoredOptions() if monitoredOptions != nil && !instance.IgnoreMonitorAgent() { - if err := StopMonitored(ctx, []string{instance.GetHost()}, noAgentHosts, monitoredOptions, options.OptTimeout); err != nil { + if err := StopMonitored(ctx, []string{instance.GetManageHost()}, noAgentHosts, monitoredOptions, options.OptTimeout); err != nil { if !ignoreErr { return perrs.Annotatef(err, "failed to stop monitor") } @@ -150,7 +151,7 @@ func StopAndDestroyInstance( } } - if err := DeletePublicKey(ctx, instance.GetHost()); err != nil { + if err := DeletePublicKey(ctx, instance.GetManageHost()); err != nil { if !ignoreErr { return perrs.Annotatef(err, "failed to delete public key") } @@ -241,11 +242,11 @@ func DeletePublicKey(ctx context.Context, host string) error { // DestroyMonitored destroy the monitored service. func DestroyMonitored(ctx context.Context, inst spec.Instance, options *spec.MonitoredOptions, timeout uint64) error { - e := ctxt.GetInner(ctx).Get(inst.GetHost()) + e := ctxt.GetInner(ctx).Get(inst.GetManageHost()) logger := ctx.Value(logprinter.ContextKeyLogger).(*logprinter.Logger) - logger.Infof("Destroying monitored %s", inst.GetHost()) - logger.Infof("\tDestroying instance %s", inst.GetHost()) + logger.Infof("Destroying monitored %s", inst.GetManageHost()) + logger.Infof("\tDestroying instance %s", inst.GetManageHost()) // Stop by systemd. delPaths := make([]string, 0) @@ -282,21 +283,21 @@ func DestroyMonitored(ctx context.Context, inst spec.Instance, options *spec.Mon } if err != nil { - return perrs.Annotatef(err, "failed to destroy monitored: %s", inst.GetHost()) + return perrs.Annotatef(err, "failed to destroy monitored: %s", inst.GetManageHost()) } if err := spec.PortStopped(ctx, e, options.NodeExporterPort, timeout); err != nil { - str := fmt.Sprintf("%s failed to destroy node exportoer: %s", inst.GetHost(), err) + str := fmt.Sprintf("%s failed to destroy node exportoer: %s", inst.GetManageHost(), err) logger.Errorf(str) return perrs.Annotatef(err, str) } if err := spec.PortStopped(ctx, e, options.BlackboxExporterPort, timeout); err != nil { - str := fmt.Sprintf("%s failed to destroy blackbox exportoer: %s", inst.GetHost(), err) + str := fmt.Sprintf("%s failed to destroy blackbox exportoer: %s", inst.GetManageHost(), err) logger.Errorf(str) return perrs.Annotatef(err, str) } - logger.Infof("Destroy monitored on %s success", inst.GetHost()) + logger.Infof("Destroy monitored on %s success", inst.GetManageHost()) return nil } @@ -350,10 +351,10 @@ func DestroyComponent(ctx context.Context, instances []spec.Instance, cls spec.T for _, ins := range instances { // Some data of instances will be retained dataRetained := retainDataRoles.Exist(ins.ComponentName()) || - retainDataNodes.Exist(ins.ID()) || retainDataNodes.Exist(ins.GetHost()) + retainDataNodes.Exist(ins.ID()) || retainDataNodes.Exist(ins.GetHost()) || retainDataRoles.Exist(ins.GetManageHost()) - e := ctxt.GetInner(ctx).Get(ins.GetHost()) - logger.Infof("\tDestroying instance %s", ins.GetHost()) + e := ctxt.GetInner(ctx).Get(ins.GetManageHost()) + logger.Infof("\tDestroying instance %s", ins.GetManageHost()) var dataDirs []string if len(ins.DataDir()) > 0 { @@ -370,19 +371,28 @@ func DestroyComponent(ctx context.Context, instances []spec.Instance, cls spec.T for _, dataDir := range dataDirs { // Don't delete the parent directory if any sub-directory retained keepDeployDir = (dataRetained && strings.HasPrefix(dataDir, deployDir)) || keepDeployDir - if !dataRetained && cls.CountDir(ins.GetHost(), dataDir) == 1 { + if !dataRetained && cls.CountDir(ins.GetManageHost(), dataDir) == 1 { // only delete path if it is not used by any other instance in the cluster delPaths.Insert(dataDir) } } + // For TiFlash, we need to delete storage.remote.cache.dir + if ins.ComponentName() == spec.ComponentTiFlash { + tiflashInstance := ins.(*spec.TiFlashInstance) + tiflashSpec := tiflashInstance.InstanceSpec.(*spec.TiFlashSpec) + if remoteCacheDir, ok := tiflashSpec.Config[spec.TiFlashRemoteCacheDir]; ok { + delPaths.Insert(remoteCacheDir.(string)) + } + } + logDir := ins.LogDir() // In TiDB-Ansible, deploy dir are shared by all components on the same // host, so not deleting it. if ins.IsImported() { // not deleting files for imported clusters - if !strings.HasPrefix(logDir, ins.DeployDir()) && cls.CountDir(ins.GetHost(), logDir) == 1 { + if !strings.HasPrefix(logDir, ins.DeployDir()) && cls.CountDir(ins.GetManageHost(), logDir) == 1 { delPaths.Insert(logDir) } logger.Warnf("Deploy dir %s not deleted for TiDB-Ansible imported instance %s.", @@ -396,15 +406,15 @@ func DestroyComponent(ctx context.Context, instances []spec.Instance, cls spec.T delPaths.Insert(filepath.Join(deployDir, spec.TLSCertKeyDir)) } // only delete path if it is not used by any other instance in the cluster - if strings.HasPrefix(logDir, deployDir) && cls.CountDir(ins.GetHost(), logDir) == 1 { + if strings.HasPrefix(logDir, deployDir) && cls.CountDir(ins.GetManageHost(), logDir) == 1 { delPaths.Insert(logDir) } } else { // only delete path if it is not used by any other instance in the cluster - if cls.CountDir(ins.GetHost(), logDir) == 1 { + if cls.CountDir(ins.GetManageHost(), logDir) == 1 { delPaths.Insert(logDir) } - if cls.CountDir(ins.GetHost(), ins.DeployDir()) == 1 { + if cls.CountDir(ins.GetManageHost(), ins.DeployDir()) == 1 { delPaths.Insert(ins.DeployDir()) } } @@ -417,35 +427,31 @@ func DestroyComponent(ctx context.Context, instances []spec.Instance, cls spec.T dpCnt++ } } - if !ins.IsImported() && cls.CountDir(ins.GetHost(), deployDir)-dpCnt == 1 { + if !ins.IsImported() && cls.CountDir(ins.GetManageHost(), deployDir)-dpCnt == 1 { delPaths.Insert(deployDir) } if svc := ins.ServiceName(); svc != "" { delPaths.Insert(fmt.Sprintf("/etc/systemd/system/%s", svc)) } - logger.Debugf("Deleting paths on %s: %s", ins.GetHost(), strings.Join(delPaths.Slice(), " ")) - c := module.ShellModuleConfig{ - Command: fmt.Sprintf("rm -rf %s;", strings.Join(delPaths.Slice(), " ")), - Sudo: true, // the .service files are in a directory owned by root - Chdir: "", - UseShell: false, - } - shell := module.NewShellModule(c) - stdout, stderr, err := shell.Execute(ctx, e) - - if len(stdout) > 0 { - fmt.Println(string(stdout)) - } - if len(stderr) > 0 { - logger.Errorf(string(stderr)) - } + logger.Debugf("Deleting paths on %s: %s", ins.GetManageHost(), strings.Join(delPaths.Slice(), " ")) + for _, delPath := range delPaths.Slice() { + c := module.ShellModuleConfig{ + Command: fmt.Sprintf("rm -rf %s;", delPath), + Sudo: true, // the .service files are in a directory owned by root + Chdir: "", + UseShell: false, + } + shell := module.NewShellModule(c) + _, _, err := shell.Execute(ctx, e) - if err != nil { - return perrs.Annotatef(err, "failed to destroy: %s", ins.GetHost()) + if err != nil { + // Ignore error and continue.For example, deleting a mount point will result in a "Device or resource busy" error. + logger.Warnf(color.YellowString("Warn: failed to delete path \"%s\" on %s.Please check this error message and manually delete if necessary.\nerrmsg: %s", delPath, ins.GetManageHost(), err)) + } } - logger.Infof("Destroy %s success", ins.GetHost()) + logger.Infof("Destroy %s finished", ins.GetManageHost()) logger.Infof("- Destroy %s paths: %v", ins.ComponentName(), delPaths.Slice()) } @@ -477,7 +483,7 @@ func DestroyClusterTombstone( instCount := map[string]int{} for _, component := range cluster.ComponentsByStartOrder() { for _, instance := range component.Instances() { - instCount[instance.GetHost()]++ + instCount[instance.GetManageHost()]++ } } @@ -488,7 +494,7 @@ func DestroyClusterTombstone( pdEndpoints = strings.Split(forcePDEndpoints, ",") logger.Warnf("%s is set, using %s as PD endpoints", EnvNamePDEndpointOverwrite, pdEndpoints) } else { - pdEndpoints = cluster.GetPDList() + pdEndpoints = cluster.GetPDListWithManageHost() } var pdClient = api.NewPDClient(ctx, pdEndpoints, 10*time.Second, tlsCfg) @@ -517,8 +523,8 @@ func DestroyClusterTombstone( instances = filterID(instances, id) for _, instance := range instances { - instCount[instance.GetHost()]-- - err := StopAndDestroyInstance(ctx, cluster, instance, options, true, instCount[instance.GetHost()] == 0, tlsCfg) + instCount[instance.GetManageHost()]-- + err := StopAndDestroyInstance(ctx, cluster, instance, options, true, instCount[instance.GetManageHost()] == 0, tlsCfg) if err != nil { if !options.Force { return err @@ -537,7 +543,7 @@ func DestroyClusterTombstone( continue } - id := s.Host + ":" + strconv.Itoa(s.Port) + id := utils.JoinHostPort(s.Host, s.Port) tombstone, err := pdClient.IsTombStone(id) if err != nil { @@ -567,7 +573,7 @@ func DestroyClusterTombstone( continue } - id := s.Host + ":" + strconv.Itoa(s.FlashServicePort) + id := utils.JoinHostPort(s.Host, s.FlashServicePort) tombstone, err := pdClient.IsTombStone(id) if err != nil { @@ -585,7 +591,7 @@ func DestroyClusterTombstone( } instances := (&spec.TiFlashComponent{Topology: cluster}).Instances() - id = s.Host + ":" + strconv.Itoa(s.GetMainPort()) + id = utils.JoinHostPort(s.Host, s.GetMainPort()) if err := maybeDestroyMonitor(instances, id); err != nil { return nil, err } @@ -598,7 +604,7 @@ func DestroyClusterTombstone( continue } - id := s.Host + ":" + strconv.Itoa(s.Port) + id := utils.JoinHostPort(s.Host, s.Port) tombstone, err := binlogClient.IsPumpTombstone(ctx, id) if err != nil { @@ -628,7 +634,7 @@ func DestroyClusterTombstone( continue } - id := s.Host + ":" + strconv.Itoa(s.Port) + id := utils.JoinHostPort(s.Host, s.Port) tombstone, err := binlogClient.IsDrainerTombstone(ctx, id) if err != nil { diff --git a/pkg/cluster/operation/download.go b/pkg/cluster/operation/download.go index 6e808c6c10..4063fcdeb7 100644 --- a/pkg/cluster/operation/download.go +++ b/pkg/cluster/operation/download.go @@ -37,7 +37,7 @@ func Download(component, nodeOS, arch string, version string) error { fileName := fmt.Sprintf("%s-%s-%s.tar.gz", resName, nodeOS, arch) srcPath := spec.ProfilePath(spec.TiUPPackageCacheDir, fileName) - if err := os.MkdirAll(spec.ProfilePath(spec.TiUPPackageCacheDir), 0755); err != nil { + if err := utils.MkdirAll(spec.ProfilePath(spec.TiUPPackageCacheDir), 0755); err != nil { return err } diff --git a/pkg/cluster/operation/operation.go b/pkg/cluster/operation/operation.go index 6c97324b85..0f50b20377 100644 --- a/pkg/cluster/operation/operation.go +++ b/pkg/cluster/operation/operation.go @@ -15,6 +15,7 @@ package operator import ( "fmt" + "os" "github.com/pingcap/tiup/pkg/cluster/executor" "github.com/pingcap/tiup/pkg/cluster/spec" @@ -45,6 +46,7 @@ type Options struct { SSHProxyIdentity string // the ssh proxy identity file SSHProxyUsePassword bool // use password instead of identity file for ssh proxy connection SSHProxyTimeout uint64 // timeout in seconds when connecting the proxy host + SSHCustomScripts SSHCustomScripts // custom scripts to be executed during the operation // What type of things should we cleanup in clean command CleanupData bool // should we cleanup data @@ -59,6 +61,26 @@ type Options struct { Operation Operation } +// SSHCustomScripts represents the custom ssh script set to be executed during cluster operations +type SSHCustomScripts struct { + BeforeRestartInstance SSHCustomScript + AfterRestartInstance SSHCustomScript +} + +// SSHCustomScript represents a custom ssh script to be executed during cluster operations +type SSHCustomScript struct { + Raw string +} + +// Command returns the ssh command in string format +func (s SSHCustomScript) Command() string { + b, err := os.ReadFile(s.Raw) + if err != nil { + return s.Raw + } + return string(b) +} + // Operation represents the type of cluster operation type Operation byte diff --git a/pkg/cluster/operation/scale_in.go b/pkg/cluster/operation/scale_in.go index 761ac5d0b5..50bc246572 100644 --- a/pkg/cluster/operation/scale_in.go +++ b/pkg/cluster/operation/scale_in.go @@ -19,7 +19,6 @@ import ( "encoding/json" "fmt" "os" - "strconv" "strings" "time" @@ -99,7 +98,7 @@ func ScaleInCluster( for _, component := range cluster.ComponentsByStartOrder() { for _, instance := range component.Instances() { instances[instance.ID()] = instance - instCount[instance.GetHost()]++ + instCount[instance.GetManageHost()]++ } } @@ -200,8 +199,8 @@ func ScaleInCluster( } } - instCount[instance.GetHost()]-- - if err := StopAndDestroyInstance(ctx, cluster, instance, options, true, instCount[instance.GetHost()] == 0, tlsCfg); err != nil { + instCount[instance.GetManageHost()]-- + if err := StopAndDestroyInstance(ctx, cluster, instance, options, true, instCount[instance.GetManageHost()] == 0, tlsCfg); err != nil { logger.Warnf("failed to stop/destroy %s: %v", compName, err) } @@ -295,8 +294,8 @@ func ScaleInCluster( } if !asyncOfflineComps.Exist(instance.ComponentName()) { - instCount[instance.GetHost()]-- - if err := StopAndDestroyInstance(ctx, cluster, instance, options, false, instCount[instance.GetHost()] == 0, tlsCfg); err != nil { + instCount[instance.GetManageHost()]-- + if err := StopAndDestroyInstance(ctx, cluster, instance, options, false, instCount[instance.GetManageHost()] == 0, tlsCfg); err != nil { return err } } else { @@ -321,8 +320,8 @@ func ScaleInCluster( } if !asyncOfflineComps.Exist(instance.ComponentName()) { - instCount[instance.GetHost()]-- - if err := StopAndDestroyInstance(ctx, cluster, instance, options, false, instCount[instance.GetHost()] == 0, tlsCfg); err != nil { + instCount[instance.GetManageHost()]-- + if err := StopAndDestroyInstance(ctx, cluster, instance, options, false, instCount[instance.GetManageHost()] == 0, tlsCfg); err != nil { return err } } else { @@ -341,7 +340,7 @@ func ScaleInCluster( for i := 0; i < len(cluster.TiKVServers); i++ { s := cluster.TiKVServers[i] - id := s.Host + ":" + strconv.Itoa(s.Port) + id := utils.JoinHostPort(s.Host, s.Port) if !deletedNodes.Exist(id) { continue } @@ -351,7 +350,7 @@ func ScaleInCluster( for i := 0; i < len(cluster.TiFlashServers); i++ { s := cluster.TiFlashServers[i] - id := s.Host + ":" + strconv.Itoa(s.TCPPort) + id := utils.JoinHostPort(s.Host, s.TCPPort) if !deletedNodes.Exist(id) { continue } @@ -361,7 +360,7 @@ func ScaleInCluster( for i := 0; i < len(cluster.PumpServers); i++ { s := cluster.PumpServers[i] - id := s.Host + ":" + strconv.Itoa(s.Port) + id := utils.JoinHostPort(s.Host, s.Port) if !deletedNodes.Exist(id) { continue } @@ -371,7 +370,7 @@ func ScaleInCluster( for i := 0; i < len(cluster.Drainers); i++ { s := cluster.Drainers[i] - id := s.Host + ":" + strconv.Itoa(s.Port) + id := utils.JoinHostPort(s.Host, s.Port) if !deletedNodes.Exist(id) { continue } @@ -401,7 +400,7 @@ func deleteMember( return err } case spec.ComponentTiFlash: - addr := instance.GetHost() + ":" + strconv.Itoa(instance.(*spec.TiFlashInstance).GetServicePort()) + addr := utils.JoinHostPort(instance.GetHost(), instance.(*spec.TiFlashInstance).GetServicePort()) if err := pdClient.DelStore(addr, timeoutOpt); err != nil { return err } @@ -410,13 +409,13 @@ func deleteMember( return err } case spec.ComponentDrainer: - addr := instance.GetHost() + ":" + strconv.Itoa(instance.GetPort()) + addr := utils.JoinHostPort(instance.GetHost(), instance.GetPort()) err := binlogClient.OfflineDrainer(ctx, addr) if err != nil { return err } case spec.ComponentPump: - addr := instance.GetHost() + ":" + strconv.Itoa(instance.GetPort()) + addr := utils.JoinHostPort(instance.GetHost(), instance.GetPort()) err := binlogClient.OfflinePump(ctx, addr) if err != nil { return err @@ -441,8 +440,8 @@ func scaleInCDC( g, _ := errgroup.WithContext(ctx) for _, ins := range instances { ins := ins - instCount[ins.GetHost()]++ - destroyNode := instCount[ins.GetHost()] == 0 + instCount[ins.GetManageHost()]++ + destroyNode := instCount[ins.GetManageHost()] == 0 g.Go(func() error { return StopAndDestroyInstance(ctx, cluster, ins, options, true, destroyNode, tlsCfg) }) @@ -453,15 +452,15 @@ func scaleInCDC( deferInstances := make([]spec.Instance, 0, 1) for _, instance := range instances { address := instance.(*spec.CDCInstance).GetAddr() - client := api.NewCDCOpenAPIClient(ctx, []string{address}, 5*time.Second, tlsCfg) + client := api.NewCDCOpenAPIClient(ctx, []string{utils.JoinHostPort(instance.GetManageHost(), instance.GetPort())}, 5*time.Second, tlsCfg) capture, err := client.GetCaptureByAddr(address) if err != nil { // this may be caused by that the instance is not running, or the specified version of cdc does not support open api logger.Debugf("scale-in cdc, get capture by address failed, stop the instance by force, "+ "addr: %s, err: %+v", address, err) - instCount[instance.GetHost()]-- - if err := StopAndDestroyInstance(ctx, cluster, instance, options, true, instCount[instance.GetHost()] == 0, tlsCfg); err != nil { + instCount[instance.GetManageHost()]-- + if err := StopAndDestroyInstance(ctx, cluster, instance, options, true, instCount[instance.GetManageHost()] == 0, tlsCfg); err != nil { return err } continue @@ -473,15 +472,15 @@ func scaleInCDC( continue } - instCount[instance.GetHost()]-- - if err := StopAndDestroyInstance(ctx, cluster, instance, options, false, instCount[instance.GetHost()] == 0, tlsCfg); err != nil { + instCount[instance.GetManageHost()]-- + if err := StopAndDestroyInstance(ctx, cluster, instance, options, false, instCount[instance.GetManageHost()] == 0, tlsCfg); err != nil { return err } } for _, instance := range deferInstances { - instCount[instance.GetHost()]-- - if err := StopAndDestroyInstance(ctx, cluster, instance, options, false, instCount[instance.GetHost()] == 0, tlsCfg); err != nil { + instCount[instance.GetManageHost()]-- + if err := StopAndDestroyInstance(ctx, cluster, instance, options, false, instCount[instance.GetManageHost()] == 0, tlsCfg); err != nil { return err } } diff --git a/pkg/cluster/operation/systemd.go b/pkg/cluster/operation/systemd.go index 497d05db22..8c5ba4e2cc 100644 --- a/pkg/cluster/operation/systemd.go +++ b/pkg/cluster/operation/systemd.go @@ -16,10 +16,12 @@ package operator import ( "context" "strings" + "time" "github.com/pingcap/errors" "github.com/pingcap/tiup/pkg/cluster/ctxt" "github.com/pingcap/tiup/pkg/cluster/module" + "go.uber.org/zap" ) // GetServiceStatus return the Acitive line of status. @@ -34,7 +36,7 @@ import ( Mar 09 13:56:19 ip-172-16-5-70 systemd[1]: Started drainer-8249 service. */ -func GetServiceStatus(ctx context.Context, e ctxt.Executor, name string) (active, memory string, err error) { +func GetServiceStatus(ctx context.Context, e ctxt.Executor, name string) (active, memory string, since time.Duration, err error) { c := module.SystemdModuleConfig{ Unit: name, Action: "status", @@ -50,6 +52,7 @@ func GetServiceStatus(ctx context.Context, e ctxt.Executor, name string) (active switch words[0] { case "Active:": active = words[1] + since = parseSystemctlSince(line) case "Memory:": memory = words[1] } @@ -60,3 +63,34 @@ func GetServiceStatus(ctx context.Context, e ctxt.Executor, name string) (active } return } + +// `systemctl status xxx.service` returns as below +// Active: active (running) since Sat 2021-03-27 10:51:11 CST; 41min ago +func parseSystemctlSince(str string) (dur time.Duration) { + // if service is not found or other error, don't need to parse it + if str == "" { + return 0 + } + defer func() { + if dur == 0 { + zap.L().Warn("failed to parse systemctl since", zap.String("value", str)) + } + }() + parts := strings.Split(str, ";") + if len(parts) != 2 { + return + } + parts = strings.Split(parts[0], " ") + if len(parts) < 3 { + return + } + + dateStr := strings.Join(parts[len(parts)-3:], " ") + + tm, err := time.Parse("2006-01-02 15:04:05 MST", dateStr) + if err != nil { + return + } + + return time.Since(tm) +} diff --git a/pkg/cluster/operation/telemetry.go b/pkg/cluster/operation/telemetry.go index ba2551db58..afa8da6a15 100644 --- a/pkg/cluster/operation/telemetry.go +++ b/pkg/cluster/operation/telemetry.go @@ -61,7 +61,7 @@ func GetNodeInfo( var nodesMu sync.Mutex foundHosts := make(map[string]struct{}) topo.IterInstance(func(inst spec.Instance) { - host := inst.GetHost() + host := inst.GetManageHost() if _, ok := foundHosts[host]; ok { return diff --git a/pkg/cluster/operation/upgrade.go b/pkg/cluster/operation/upgrade.go index e1d77b0e25..5f4ab04b3f 100644 --- a/pkg/cluster/operation/upgrade.go +++ b/pkg/cluster/operation/upgrade.go @@ -19,7 +19,6 @@ import ( "fmt" "os" "reflect" - "strconv" "strings" "time" @@ -30,6 +29,7 @@ import ( logprinter "github.com/pingcap/tiup/pkg/logger/printer" "github.com/pingcap/tiup/pkg/set" "github.com/pingcap/tiup/pkg/tidbver" + "github.com/pingcap/tiup/pkg/utils" "go.uber.org/zap" ) @@ -39,17 +39,18 @@ var ( increaseLimitPoint = checkpoint.Register() ) -// Upgrade the cluster. +// Upgrade the cluster. (actually, it's rolling restart) func Upgrade( ctx context.Context, topo spec.Topology, options Options, tlsCfg *tls.Config, currentVersion string, + targetVersion string, ) error { roleFilter := set.NewStringSet(options.Roles...) nodeFilter := set.NewStringSet(options.Nodes...) - components := topo.ComponentsByUpdateOrder() + components := topo.ComponentsByUpdateOrder(currentVersion) components = FilterComponent(components, roleFilter) logger := ctx.Value(logprinter.ContextKeyLogger).(*logprinter.Logger) @@ -70,6 +71,7 @@ func Upgrade( var origRegionScheduleLimit int var err error + var tidbClient *api.TiDBClient var pdEndpoints []string forcePDEndpoints := os.Getenv(EnvNamePDEndpointOverwrite) // custom set PD endpoint list @@ -79,7 +81,7 @@ func Upgrade( pdEndpoints = strings.Split(forcePDEndpoints, ",") logger.Warnf("%s is set, using %s as PD endpoints", EnvNamePDEndpointOverwrite, pdEndpoints) } else { - pdEndpoints = topo.(*spec.Specification).GetPDList() + pdEndpoints = topo.(*spec.Specification).GetPDListWithManageHost() } pdClient := api.NewPDClient(ctx, pdEndpoints, 10*time.Second, tlsCfg) origLeaderScheduleLimit, origRegionScheduleLimit, err = increaseScheduleLimit(ctx, pdClient) @@ -100,6 +102,21 @@ func Upgrade( } }() } + case spec.ComponentTiDB: + dbs := topo.(*spec.Specification).TiDBServers + endpoints := []string{} + for _, db := range dbs { + endpoints = append(endpoints, utils.JoinHostPort(db.GetManageHost(), db.StatusPort)) + } + + if currentVersion != targetVersion && tidbver.TiDBSupportUpgradeAPI(currentVersion) && tidbver.TiDBSupportUpgradeAPI(targetVersion) { + tidbClient = api.NewTiDBClient(ctx, endpoints, 10*time.Second, tlsCfg) + err = tidbClient.StartUpgrade() + if err != nil { + return err + } + } + default: // do nothing, kept for future usage with other components } @@ -109,9 +126,9 @@ func Upgrade( for _, instance := range instances { // monitors - uniqueHosts.Insert(instance.GetHost()) + uniqueHosts.Insert(instance.GetManageHost()) if instance.IgnoreMonitorAgent() { - noAgentHosts.Insert(instance.GetHost()) + noAgentHosts.Insert(instance.GetManageHost()) } switch component.Name() { case spec.ComponentPD: @@ -143,7 +160,7 @@ func Upgrade( // during the upgrade process, endpoint addresses should not change, so only new the client once. if cdcOpenAPIClient == nil { - cdcOpenAPIClient = api.NewCDCOpenAPIClient(ctx, topo.(*spec.Specification).GetCDCList(), 5*time.Second, tlsCfg) + cdcOpenAPIClient = api.NewCDCOpenAPIClient(ctx, topo.(*spec.Specification).GetCDCListWithManageHost(), 5*time.Second, tlsCfg) } capture, err := cdcOpenAPIClient.GetCaptureByAddr(address) @@ -178,6 +195,19 @@ func Upgrade( return err } } + + switch component.Name() { + case spec.ComponentTiDB: + if currentVersion != targetVersion && tidbver.TiDBSupportUpgradeAPI(currentVersion) && tidbver.TiDBSupportUpgradeAPI(targetVersion) { + err = tidbClient.FinishUpgrade() + if err != nil { + return err + } + } + + default: + // do nothing, kept for future usage with other components + } } if topo.GetMonitoredOptions() == nil { @@ -211,6 +241,11 @@ func upgradeInstance( rollingInstance, isRollingInstance = instance.(spec.RollingUpdateInstance) } + err = executeSSHCommand(ctx, "Executing pre-upgrade command", instance.GetManageHost(), options.SSHCustomScripts.BeforeRestartInstance.Command()) + if err != nil { + return err + } + if isRollingInstance { err := rollingInstance.PreRestart(ctx, topo, int(options.APITimeout), tlsCfg) if err != nil && !options.Force { @@ -229,6 +264,11 @@ func upgradeInstance( } } + err = executeSSHCommand(ctx, "Executing post-upgrade command", instance.GetManageHost(), options.SSHCustomScripts.AfterRestartInstance.Command()) + if err != nil { + return err + } + return nil } @@ -237,7 +277,8 @@ func Addr(ins spec.Instance) string { if ins.GetPort() == 0 || ins.GetPort() == 80 { panic(ins) } - return ins.GetHost() + ":" + strconv.Itoa(ins.GetPort()) + + return utils.JoinHostPort(ins.GetManageHost(), ins.GetPort()) } var ( diff --git a/pkg/cluster/spec/alertmanager.go b/pkg/cluster/spec/alertmanager.go index 3af3b3e7ea..b71392b858 100644 --- a/pkg/cluster/spec/alertmanager.go +++ b/pkg/cluster/spec/alertmanager.go @@ -30,6 +30,7 @@ import ( // AlertmanagerSpec represents the AlertManager topology specification in topology.yaml type AlertmanagerSpec struct { Host string `yaml:"host"` + ManageHost string `yaml:"manage_host,omitempty" validate:"manage_host:editable"` SSHPort int `yaml:"ssh_port,omitempty" validate:"ssh_port:editable"` Imported bool `yaml:"imported,omitempty"` Patched bool `yaml:"patched,omitempty"` @@ -54,7 +55,11 @@ func (s *AlertmanagerSpec) Role() string { // SSH returns the host and SSH port of the instance func (s *AlertmanagerSpec) SSH() (string, int) { - return s.Host, s.SSHPort + host := s.Host + if s.ManageHost != "" { + host = s.ManageHost + } + return host, s.SSHPort } // GetMainPort returns the main port of the instance @@ -62,6 +67,14 @@ func (s *AlertmanagerSpec) GetMainPort() int { return s.WebPort } +// GetManageHost returns the manage host of the instance +func (s *AlertmanagerSpec) GetManageHost() string { + if s.ManageHost != "" { + return s.ManageHost + } + return s.Host +} + // IsImported returns if the node is imported from TiDB-Ansible func (s *AlertmanagerSpec) IsImported() bool { return s.Imported @@ -85,6 +98,21 @@ func (c *AlertManagerComponent) Role() string { return RoleMonitor } +// CalculateVersion implements the Component interface +func (c *AlertManagerComponent) CalculateVersion(_ string) string { + // always not follow cluster version, use ""(latest) by default + version := c.Topology.BaseTopo().AlertManagerVersion + if version != nil { + return *version + } + return "" +} + +// SetVersion implements Component interface. +func (c *AlertManagerComponent) SetVersion(version string) { + *c.Topology.BaseTopo().AlertManagerVersion = version +} + // Instances implements Component interface. func (c *AlertManagerComponent) Instances() []Instance { alertmanagers := c.Topology.BaseTopo().Alertmanagers @@ -98,9 +126,12 @@ func (c *AlertManagerComponent) Instances() []Instance { InstanceSpec: s, Name: c.Name(), Host: s.Host, - ListenHost: s.ListenHost, + ManageHost: s.ManageHost, + ListenHost: utils.Ternary(s.ListenHost != "", s.ListenHost, c.Topology.BaseTopo().GlobalOptions.ListenHost).(string), Port: s.WebPort, SSHP: s.SSHPort, + NumaNode: s.NumaNode, + NumaCores: "", Ports: []int{ s.WebPort, @@ -111,11 +142,12 @@ func (c *AlertManagerComponent) Instances() []Instance { s.DataDir, }, StatusFn: func(_ context.Context, timeout time.Duration, _ *tls.Config, _ ...string) string { - return statusByHost(s.Host, s.WebPort, "/-/ready", timeout, nil) + return statusByHost(s.GetManageHost(), s.WebPort, "/-/ready", timeout, nil) }, UptimeFn: func(_ context.Context, timeout time.Duration, tlsCfg *tls.Config) time.Duration { - return UptimeByHost(s.Host, s.WebPort, timeout, tlsCfg) + return UptimeByHost(s.GetManageHost(), s.WebPort, timeout, tlsCfg) }, + Component: c, }, topo: c.Topology, }) @@ -151,10 +183,11 @@ func (i *AlertManagerInstance) InitConfig( peers = append(peers, utils.JoinHostPort(amspec.Host, amspec.ClusterPort)) } cfg := &scripts.AlertManagerScript{ - WebListenAddr: utils.JoinHostPort(i.GetListenHost(), spec.WebPort), - WebExternalURL: fmt.Sprintf("http://%s", utils.JoinHostPort(spec.Host, spec.WebPort)), - ClusterPeers: peers, - ClusterListenAddr: utils.JoinHostPort(i.GetListenHost(), spec.ClusterPort), + WebListenAddr: utils.JoinHostPort(i.GetListenHost(), spec.WebPort), + WebExternalURL: fmt.Sprintf("http://%s", utils.JoinHostPort(spec.Host, spec.WebPort)), + ClusterPeers: peers, + // ClusterListenAddr cannot use i.GetListenHost due to https://github.com/prometheus/alertmanager/issues/2284 and https://github.com/prometheus/alertmanager/issues/1271 + ClusterListenAddr: utils.JoinHostPort(i.GetHost(), spec.ClusterPort), DeployDir: paths.Deploy, LogDir: paths.Log, @@ -193,7 +226,8 @@ func (i *AlertManagerInstance) InitConfig( if err := i.TransferLocalConfigFile(ctx, e, configPath, dst); err != nil { return err } - return checkConfig(ctx, e, i.ComponentName(), clusterVersion, i.OS(), i.Arch(), i.ComponentName()+".yml", paths, nil) + // version is not used for alertmanager + return checkConfig(ctx, e, i.ComponentName(), i.ComponentSource(), "", i.OS(), i.Arch(), i.ComponentName()+".yml", paths) } // ScaleConfig deploy temporary config on scaling diff --git a/pkg/cluster/spec/bindversion.go b/pkg/cluster/spec/bindversion.go index 6e6ac9fcf8..98a452454e 100644 --- a/pkg/cluster/spec/bindversion.go +++ b/pkg/cluster/spec/bindversion.go @@ -18,24 +18,6 @@ import ( "strings" ) -// TiDBComponentVersion maps the TiDB version to the third components binding version -// Empty version should be treated as the last stable one -func TiDBComponentVersion(comp, version string) string { - switch comp { - case ComponentAlertmanager, - ComponentBlackboxExporter, - ComponentNodeExporter, - ComponentPushwaygate, - ComponentCheckCollector, - ComponentSpark, - ComponentTiSpark, - ComponentTiKVCDC: // TiKV-CDC use individual version. - return "" - default: - return version - } -} - // ComponentSubDir maps a component with version to a subdir if needed func ComponentSubDir(comp, version string) string { if comp == ComponentSpark { diff --git a/pkg/cluster/spec/cdc.go b/pkg/cluster/spec/cdc.go index a626400728..23a03b1c12 100644 --- a/pkg/cluster/spec/cdc.go +++ b/pkg/cluster/spec/cdc.go @@ -34,6 +34,7 @@ import ( // CDCSpec represents the CDC topology specification in topology.yaml type CDCSpec struct { Host string `yaml:"host"` + ManageHost string `yaml:"manage_host,omitempty" validate:"manage_host:editable"` SSHPort int `yaml:"ssh_port,omitempty" validate:"ssh_port:editable"` Imported bool `yaml:"imported,omitempty"` Patched bool `yaml:"patched,omitempty"` @@ -46,6 +47,7 @@ type CDCSpec struct { GCTTL int64 `yaml:"gc-ttl,omitempty" validate:"gc-ttl:editable"` TZ string `yaml:"tz,omitempty" validate:"tz:editable"` TiCDCClusterID string `yaml:"ticdc_cluster_id"` + Source string `yaml:"source,omitempty" validate:"source:editable"` NumaNode string `yaml:"numa_node,omitempty" validate:"numa_node:editable"` Config map[string]any `yaml:"config,omitempty" validate:"config:ignore"` ResourceControl meta.ResourceControl `yaml:"resource_control,omitempty" validate:"resource_control:editable"` @@ -60,7 +62,11 @@ func (s *CDCSpec) Role() string { // SSH returns the host and SSH port of the instance func (s *CDCSpec) SSH() (string, int) { - return s.Host, s.SSHPort + host := s.Host + if s.ManageHost != "" { + host = s.ManageHost + } + return host, s.SSHPort } // GetMainPort returns the main port of the instance @@ -68,6 +74,14 @@ func (s *CDCSpec) GetMainPort() int { return s.Port } +// GetManageHost returns the manage host of the instance +func (s *CDCSpec) GetManageHost() string { + if s.ManageHost != "" { + return s.ManageHost + } + return s.Host +} + // IsImported returns if the node is imported from TiDB-Ansible func (s *CDCSpec) IsImported() bool { return s.Imported @@ -78,6 +92,14 @@ func (s *CDCSpec) IgnoreMonitorAgent() bool { return s.IgnoreExporter } +// GetSource returns source to download the component +func (s *CDCSpec) GetSource() string { + if s.Source == "" { + return ComponentCDC + } + return s.Source +} + // CDCComponent represents CDC component. type CDCComponent struct{ Topology *Specification } @@ -91,6 +113,20 @@ func (c *CDCComponent) Role() string { return ComponentCDC } +// CalculateVersion implements the Component interface +func (c *CDCComponent) CalculateVersion(clusterVersion string) string { + version := c.Topology.ComponentVersions.CDC + if version == "" { + version = clusterVersion + } + return version +} + +// SetVersion implements Component interface. +func (c *CDCComponent) SetVersion(version string) { + c.Topology.ComponentVersions.CDC = version +} + // Instances implements Component interface. func (c *CDCComponent) Instances() []Instance { ins := make([]Instance, 0, len(c.Topology.CDCServers)) @@ -100,8 +136,13 @@ func (c *CDCComponent) Instances() []Instance { InstanceSpec: s, Name: c.Name(), Host: s.Host, + ManageHost: s.ManageHost, + ListenHost: c.Topology.BaseTopo().GlobalOptions.ListenHost, Port: s.Port, SSHP: s.SSHPort, + Source: s.GetSource(), + NumaNode: s.NumaNode, + NumaCores: "", Ports: []int{ s.Port, @@ -110,11 +151,12 @@ func (c *CDCComponent) Instances() []Instance { s.DeployDir, }, StatusFn: func(_ context.Context, timeout time.Duration, tlsCfg *tls.Config, _ ...string) string { - return statusByHost(s.Host, s.Port, "/status", timeout, tlsCfg) + return statusByHost(s.GetManageHost(), s.Port, "/status", timeout, tlsCfg) }, UptimeFn: func(_ context.Context, timeout time.Duration, tlsCfg *tls.Config) time.Duration { - return UptimeByHost(s.Host, s.Port, timeout, tlsCfg) + return UptimeByHost(s.GetManageHost(), s.Port, timeout, tlsCfg) }, + Component: c, }, c.Topology} if s.DataDir != "" { instance.Dirs = append(instance.Dirs, s.DataDir) @@ -167,14 +209,15 @@ func (i *CDCInstance) InitConfig( spec := i.InstanceSpec.(*CDCSpec) globalConfig := topo.ServerConfigs.CDC instanceConfig := spec.Config + version := i.CalculateVersion(clusterVersion) - if !tidbver.TiCDCSupportConfigFile(clusterVersion) { + if !tidbver.TiCDCSupportConfigFile(version) { if len(globalConfig)+len(instanceConfig) > 0 { return errors.New("server_config is only supported with TiCDC version v4.0.13 or later") } } - if !tidbver.TiCDCSupportClusterID(clusterVersion) && spec.TiCDCClusterID != "" { + if !tidbver.TiCDCSupportClusterID(version) && spec.TiCDCClusterID != "" { return errors.New("ticdc_cluster_id is only supported with TiCDC version v6.2.0 or later") } @@ -184,18 +227,18 @@ func (i *CDCInstance) InitConfig( } cfg := &scripts.CDCScript{ Addr: utils.JoinHostPort(i.GetListenHost(), spec.Port), - AdvertiseAddr: i.GetAddr(), + AdvertiseAddr: utils.JoinHostPort(i.GetHost(), i.GetPort()), PD: strings.Join(pds, ","), GCTTL: spec.GCTTL, TZ: spec.TZ, ClusterID: spec.TiCDCClusterID, - DataDirEnabled: tidbver.TiCDCSupportDataDir(clusterVersion), - ConfigFileEnabled: tidbver.TiCDCSupportConfigFile(clusterVersion), + DataDirEnabled: tidbver.TiCDCSupportDataDir(version), + ConfigFileEnabled: tidbver.TiCDCSupportConfigFile(version), TLSEnabled: enableTLS, DeployDir: paths.Deploy, LogDir: paths.Log, - DataDir: utils.Ternary(tidbver.TiCDCSupportSortOrDataDir(clusterVersion), spec.DataDir, "").(string), + DataDir: utils.Ternary(tidbver.TiCDCSupportSortOrDataDir(version), spec.DataDir, "").(string), NumaNode: spec.NumaNode, } @@ -255,7 +298,7 @@ func (i *CDCInstance) PreRestart(ctx context.Context, topo Topology, apiTimeoutS } start := time.Now() - client := api.NewCDCOpenAPIClient(ctx, topo.(*Specification).GetCDCList(), 5*time.Second, tlsCfg) + client := api.NewCDCOpenAPIClient(ctx, topo.(*Specification).GetCDCListWithManageHost(), 5*time.Second, tlsCfg) if err := client.Healthy(); err != nil { logger.Debugf("cdc pre-restart skipped, the cluster unhealthy, trigger hard restart, "+ "addr: %s, err: %+v, elapsed: %+v", address, err, time.Since(start)) @@ -322,7 +365,7 @@ func (i *CDCInstance) PostRestart(ctx context.Context, topo Topology, tlsCfg *tl start := time.Now() address := i.GetAddr() - client := api.NewCDCOpenAPIClient(ctx, []string{address}, 5*time.Second, tlsCfg) + client := api.NewCDCOpenAPIClient(ctx, []string{utils.JoinHostPort(i.GetManageHost(), i.GetPort())}, 5*time.Second, tlsCfg) err := client.IsCaptureAlive() if err != nil { logger.Debugf("cdc post-restart finished, get capture status failed, addr: %s, err: %+v, elapsed: %+v", address, err, time.Since(start)) diff --git a/pkg/cluster/spec/dashboard.go b/pkg/cluster/spec/dashboard.go index cab87f423e..2d236ff5ff 100644 --- a/pkg/cluster/spec/dashboard.go +++ b/pkg/cluster/spec/dashboard.go @@ -29,14 +29,15 @@ import ( // DashboardSpec represents the Dashboard topology specification in topology.yaml type DashboardSpec struct { Host string `yaml:"host"` + ManageHost string `yaml:"manage_host,omitempty" validate:"manage_host:editable"` SSHPort int `yaml:"ssh_port,omitempty" validate:"ssh_port:editable"` - Version string `yaml:"version,omitempty"` Patched bool `yaml:"patched,omitempty"` IgnoreExporter bool `yaml:"ignore_exporter,omitempty"` Port int `yaml:"port" default:"12333"` DeployDir string `yaml:"deploy_dir,omitempty"` DataDir string `yaml:"data_dir,omitempty"` LogDir string `yaml:"log_dir,omitempty"` + Source string `yaml:"source,omitempty" validate:"source:editable"` NumaNode string `yaml:"numa_node,omitempty" validate:"numa_node:editable"` Config map[string]any `yaml:"config,omitempty" validate:"config:ignore"` ResourceControl meta.ResourceControl `yaml:"resource_control,omitempty" validate:"resource_control:editable"` @@ -50,7 +51,7 @@ func (s *DashboardSpec) Status(ctx context.Context, timeout time.Duration, tlsCf timeout = statusQueryTimeout } - state := statusByHost(s.Host, s.Port, "/status", timeout, tlsCfg) + state := statusByHost(s.GetManageHost(), s.Port, "/status", timeout, tlsCfg) return state } @@ -62,7 +63,11 @@ func (s *DashboardSpec) Role() string { // SSH returns the host and SSH port of the instance func (s *DashboardSpec) SSH() (string, int) { - return s.Host, s.SSHPort + host := s.Host + if s.ManageHost != "" { + host = s.ManageHost + } + return host, s.SSHPort } // GetMainPort returns the main port of the instance @@ -70,6 +75,14 @@ func (s *DashboardSpec) GetMainPort() int { return s.Port } +// GetManageHost returns the manage host of the instance +func (s *DashboardSpec) GetManageHost() string { + if s.ManageHost != "" { + return s.ManageHost + } + return s.Host +} + // IsImported returns if the node is imported from TiDB-Ansible func (s *DashboardSpec) IsImported() bool { // TiDB-Ansible do not support dashboard @@ -81,6 +94,14 @@ func (s *DashboardSpec) IgnoreMonitorAgent() bool { return s.IgnoreExporter } +// GetSource returns source to download the component +func (s *DashboardSpec) GetSource() string { + if s.Source == "" { + return ComponentDashboard + } + return s.Source +} + // DashboardComponent represents Drainer component. type DashboardComponent struct{ Topology *Specification } @@ -94,6 +115,20 @@ func (c *DashboardComponent) Role() string { return ComponentDashboard } +// CalculateVersion implements the Component interface +func (c *DashboardComponent) CalculateVersion(clusterVersion string) string { + version := c.Topology.ComponentVersions.Dashboard + if version == "" { + version = clusterVersion + } + return version +} + +// SetVersion implements Component interface. +func (c *DashboardComponent) SetVersion(version string) { + c.Topology.ComponentVersions.Dashboard = version +} + // Instances implements Component interface. func (c *DashboardComponent) Instances() []Instance { ins := make([]Instance, 0, len(c.Topology.Drainers)) @@ -103,8 +138,13 @@ func (c *DashboardComponent) Instances() []Instance { InstanceSpec: s, Name: c.Name(), Host: s.Host, + ManageHost: s.ManageHost, + ListenHost: c.Topology.BaseTopo().GlobalOptions.ListenHost, Port: s.Port, SSHP: s.SSHPort, + Source: s.Source, + NumaNode: s.NumaNode, + NumaCores: "", Ports: []int{ s.Port, @@ -115,8 +155,9 @@ func (c *DashboardComponent) Instances() []Instance { }, StatusFn: s.Status, UptimeFn: func(_ context.Context, timeout time.Duration, tlsCfg *tls.Config) time.Duration { - return UptimeByHost(s.Host, s.Port, timeout, tlsCfg) + return UptimeByHost(s.GetManageHost(), s.Port, timeout, tlsCfg) }, + Component: c, }, c.Topology}) } return ins @@ -168,8 +209,9 @@ func (i *DashboardInstance) InitConfig( pds = append(pds, pdspec.GetAdvertiseClientURL(enableTLS)) } cfg := &scripts.DashboardScript{ + // -h, --host string listen host of the Dashboard Server + Host: i.GetListenHost(), TidbVersion: clusterVersion, - IP: i.GetHost(), DeployDir: paths.Deploy, DataDir: paths.Data[0], LogDir: paths.Log, @@ -200,7 +242,7 @@ func (i *DashboardInstance) InitConfig( return err } - return checkConfig(ctx, e, i.ComponentName(), clusterVersion, i.OS(), i.Arch(), i.ComponentName()+".toml", paths, nil) + return checkConfig(ctx, e, i.ComponentName(), i.ComponentSource(), i.CalculateVersion(clusterVersion), i.OS(), i.Arch(), i.ComponentName()+".toml", paths) } // setTLSConfig set TLS Config to support enable/disable TLS diff --git a/pkg/cluster/spec/drainer.go b/pkg/cluster/spec/drainer.go index 6fe09eee46..402b6bbb49 100644 --- a/pkg/cluster/spec/drainer.go +++ b/pkg/cluster/spec/drainer.go @@ -19,7 +19,6 @@ import ( "fmt" "os" "path/filepath" - "strconv" "strings" "time" @@ -33,6 +32,7 @@ import ( // DrainerSpec represents the Drainer topology specification in topology.yaml type DrainerSpec struct { Host string `yaml:"host"` + ManageHost string `yaml:"manage_host,omitempty" validate:"manage_host:editable"` SSHPort int `yaml:"ssh_port,omitempty" validate:"ssh_port:editable"` Imported bool `yaml:"imported,omitempty"` Patched bool `yaml:"patched,omitempty"` @@ -43,6 +43,7 @@ type DrainerSpec struct { LogDir string `yaml:"log_dir,omitempty"` CommitTS *int64 `yaml:"commit_ts,omitempty" validate:"commit_ts:editable"` // do not use it anymore, exist for compatibility Offline bool `yaml:"offline,omitempty"` + Source string `yaml:"source,omitempty" validate:"source:editable"` NumaNode string `yaml:"numa_node,omitempty" validate:"numa_node:editable"` Config map[string]any `yaml:"config,omitempty" validate:"config:ignore"` ResourceControl meta.ResourceControl `yaml:"resource_control,omitempty" validate:"resource_control:editable"` @@ -56,14 +57,14 @@ func (s *DrainerSpec) Status(ctx context.Context, timeout time.Duration, tlsCfg timeout = statusQueryTimeout } - state := statusByHost(s.Host, s.Port, "/status", timeout, tlsCfg) + state := statusByHost(s.GetManageHost(), s.Port, "/status", timeout, tlsCfg) if s.Offline { binlogClient, err := api.NewBinlogClient(pdList, timeout, tlsCfg) if err != nil { return state } - id := s.Host + ":" + strconv.Itoa(s.Port) + id := utils.JoinHostPort(s.Host, s.Port) tombstone, _ := binlogClient.IsDrainerTombstone(ctx, id) if tombstone { @@ -82,7 +83,11 @@ func (s *DrainerSpec) Role() string { // SSH returns the host and SSH port of the instance func (s *DrainerSpec) SSH() (string, int) { - return s.Host, s.SSHPort + host := s.Host + if s.ManageHost != "" { + host = s.ManageHost + } + return host, s.SSHPort } // GetMainPort returns the main port of the instance @@ -90,6 +95,14 @@ func (s *DrainerSpec) GetMainPort() int { return s.Port } +// GetManageHost returns the manage host of the instance +func (s *DrainerSpec) GetManageHost() string { + if s.ManageHost != "" { + return s.ManageHost + } + return s.Host +} + // IsImported returns if the node is imported from TiDB-Ansible func (s *DrainerSpec) IsImported() bool { return s.Imported @@ -100,6 +113,14 @@ func (s *DrainerSpec) IgnoreMonitorAgent() bool { return s.IgnoreExporter } +// GetSource returns source to download the component +func (s *DrainerSpec) GetSource() string { + if s.Source == "" { + return ComponentDrainer + } + return s.Source +} + // DrainerComponent represents Drainer component. type DrainerComponent struct{ Topology *Specification } @@ -113,6 +134,20 @@ func (c *DrainerComponent) Role() string { return ComponentDrainer } +// CalculateVersion implements the Component interface +func (c *DrainerComponent) CalculateVersion(clusterVersion string) string { + version := c.Topology.ComponentVersions.Drainer + if version == "" { + version = clusterVersion + } + return version +} + +// SetVersion implements Component interface. +func (c *DrainerComponent) SetVersion(version string) { + c.Topology.ComponentVersions.Drainer = version +} + // Instances implements Component interface. func (c *DrainerComponent) Instances() []Instance { ins := make([]Instance, 0, len(c.Topology.Drainers)) @@ -122,8 +157,13 @@ func (c *DrainerComponent) Instances() []Instance { InstanceSpec: s, Name: c.Name(), Host: s.Host, + ManageHost: s.ManageHost, + ListenHost: c.Topology.BaseTopo().GlobalOptions.ListenHost, Port: s.Port, SSHP: s.SSHPort, + Source: s.GetSource(), + NumaNode: s.NumaNode, + NumaCores: "", Ports: []int{ s.Port, @@ -134,8 +174,9 @@ func (c *DrainerComponent) Instances() []Instance { }, StatusFn: s.Status, UptimeFn: func(_ context.Context, timeout time.Duration, tlsCfg *tls.Config) time.Duration { - return UptimeByHost(s.Host, s.Port, timeout, tlsCfg) + return UptimeByHost(s.GetManageHost(), s.Port, timeout, tlsCfg) }, + Component: c, }, c.Topology}) } return ins @@ -181,7 +222,7 @@ func (i *DrainerInstance) InitConfig( } enableTLS := topo.GlobalOptions.TLSEnabled spec := i.InstanceSpec.(*DrainerSpec) - nodeID := i.GetHost() + ":" + strconv.Itoa(i.GetPort()) + nodeID := utils.JoinHostPort(i.GetHost(), i.GetPort()) // keep origin node id if is imported if i.IsImported() { nodeID = "" @@ -251,7 +292,7 @@ func (i *DrainerInstance) InitConfig( return err } - return checkConfig(ctx, e, i.ComponentName(), clusterVersion, i.OS(), i.Arch(), i.ComponentName()+".toml", paths, nil) + return checkConfig(ctx, e, i.ComponentName(), i.ComponentSource(), clusterVersion, i.OS(), i.Arch(), i.ComponentName()+".toml", paths) } // setTLSConfig set TLS Config to support enable/disable TLS diff --git a/pkg/cluster/spec/grafana.go b/pkg/cluster/spec/grafana.go index a9a21454ae..e18a543448 100644 --- a/pkg/cluster/spec/grafana.go +++ b/pkg/cluster/spec/grafana.go @@ -28,12 +28,14 @@ import ( "github.com/pingcap/tiup/pkg/cluster/template/config" "github.com/pingcap/tiup/pkg/cluster/template/scripts" "github.com/pingcap/tiup/pkg/meta" + "github.com/pingcap/tiup/pkg/utils" "gopkg.in/ini.v1" ) // GrafanaSpec represents the Grafana topology specification in topology.yaml type GrafanaSpec struct { Host string `yaml:"host"` + ManageHost string `yaml:"manage_host,omitempty" validate:"manage_host:editable"` SSHPort int `yaml:"ssh_port,omitempty" validate:"ssh_port:editable"` Imported bool `yaml:"imported,omitempty"` Patched bool `yaml:"patched,omitempty"` @@ -62,7 +64,11 @@ func (s *GrafanaSpec) Role() string { // SSH returns the host and SSH port of the instance func (s *GrafanaSpec) SSH() (string, int) { - return s.Host, s.SSHPort + host := s.Host + if s.ManageHost != "" { + host = s.ManageHost + } + return host, s.SSHPort } // GetMainPort returns the main port of the instance @@ -70,6 +76,14 @@ func (s *GrafanaSpec) GetMainPort() int { return s.Port } +// GetManageHost returns the manage host of the instance +func (s *GrafanaSpec) GetManageHost() string { + if s.ManageHost != "" { + return s.ManageHost + } + return s.Host +} + // IsImported returns if the node is imported from TiDB-Ansible func (s *GrafanaSpec) IsImported() bool { return s.Imported @@ -93,6 +107,21 @@ func (c *GrafanaComponent) Role() string { return RoleMonitor } +// CalculateVersion implements the Component interface +func (c *GrafanaComponent) CalculateVersion(clusterVersion string) string { + // always not follow cluster version, use ""(latest) by default + version := c.Topology.BaseTopo().GrafanaVersion + if version != nil && *version != "" { + return *version + } + return clusterVersion +} + +// SetVersion implements Component interface. +func (c *GrafanaComponent) SetVersion(version string) { + *c.Topology.BaseTopo().GrafanaVersion = version +} + // Instances implements Component interface. func (c *GrafanaComponent) Instances() []Instance { servers := c.BaseTopo().Grafanas @@ -105,8 +134,12 @@ func (c *GrafanaComponent) Instances() []Instance { InstanceSpec: s, Name: c.Name(), Host: s.Host, + ManageHost: s.ManageHost, + ListenHost: c.Topology.BaseTopo().GlobalOptions.ListenHost, Port: s.Port, SSHP: s.SSHPort, + NumaNode: "", + NumaCores: "", Ports: []int{ s.Port, @@ -115,11 +148,12 @@ func (c *GrafanaComponent) Instances() []Instance { s.DeployDir, }, StatusFn: func(_ context.Context, timeout time.Duration, _ *tls.Config, _ ...string) string { - return statusByHost(s.Host, s.Port, "", timeout, nil) + return statusByHost(s.GetManageHost(), s.Port, "/login", timeout, nil) }, UptimeFn: func(_ context.Context, timeout time.Duration, tlsCfg *tls.Config) time.Duration { - return UptimeByHost(s.Host, s.Port, timeout, tlsCfg) + return UptimeByHost(s.GetManageHost(), s.Port, timeout, tlsCfg) }, + Component: c, }, topo: c.Topology, }) @@ -166,7 +200,7 @@ func (i *GrafanaInstance) InitConfig( // transfer config spec := i.InstanceSpec.(*GrafanaSpec) - fp = filepath.Join(paths.Cache, fmt.Sprintf("grafana_%s.ini", i.GetHost())) + fp = filepath.Join(paths.Cache, fmt.Sprintf("grafana_%s_%d.ini", i.GetHost(), i.GetPort())) if err := config.NewGrafanaConfig(i.GetHost(), paths.Deploy). WithPort(uint64(i.GetPort())). WithUsername(spec.Username). @@ -235,9 +269,11 @@ func (i *GrafanaInstance) InitConfig( return errors.New("no prometheus found in topology") } fp = filepath.Join(paths.Cache, fmt.Sprintf("datasource_%s.yml", i.GetHost())) - if err := config.NewDatasourceConfig(clusterName, monitors[0].Host). - WithPort(uint64(monitors[0].Port)). - ConfigToFile(fp); err != nil { + datasourceCfg := &config.DatasourceConfig{ + ClusterName: clusterName, + URL: fmt.Sprintf("http://%s", utils.JoinHostPort(monitors[0].Host, monitors[0].Port)), + } + if err := datasourceCfg.ConfigToFile(fp); err != nil { return err } dst = filepath.Join(paths.Deploy, "provisioning", "datasources", "datasource.yml") @@ -302,7 +338,7 @@ func (i *GrafanaInstance) installDashboards(ctx context.Context, e ctxt.Executor return errors.Annotatef(err, "stderr: %s", string(stderr)) } - srcPath := PackagePath(ComponentDMMaster, clusterVersion, i.OS(), i.Arch()) + srcPath := PackagePath(GetDMMasterPackageName(i.topo), clusterVersion, i.OS(), i.Arch()) dstPath := filepath.Join(tmp, filepath.Base(srcPath)) err = e.Transfer(ctx, srcPath, dstPath, false, 0, false) if err != nil { diff --git a/pkg/cluster/spec/instance.go b/pkg/cluster/spec/instance.go index 94b801537d..3281484a80 100644 --- a/pkg/cluster/spec/instance.go +++ b/pkg/cluster/spec/instance.go @@ -41,6 +41,7 @@ const ( ComponentTiKV = "tikv" ComponentPD = "pd" ComponentTiFlash = "tiflash" + ComponentTiProxy = "tiproxy" ComponentGrafana = "grafana" ComponentDrainer = "drainer" ComponentDashboard = "tidb-dashboard" @@ -53,7 +54,6 @@ const ( ComponentDMMaster = "dm-master" ComponentDMWorker = "dm-worker" ComponentPrometheus = "prometheus" - ComponentPushwaygate = "pushgateway" ComponentBlackboxExporter = "blackbox_exporter" ComponentNodeExporter = "node_exporter" ComponentCheckCollector = "insight" @@ -71,6 +71,8 @@ type Component interface { Name() string Role() string Instances() []Instance + CalculateVersion(string) string + SetVersion(string) } // RollingUpdateInstance represent a instance need to transfer state when restart. @@ -89,12 +91,16 @@ type Instance interface { ScaleConfig(ctx context.Context, e ctxt.Executor, topo Topology, clusterName string, clusterVersion string, deployUser string, paths meta.DirPaths) error PrepareStart(ctx context.Context, tlsCfg *tls.Config) error ComponentName() string + ComponentSource() string InstanceName() string ServiceName() string ResourceControl() meta.ResourceControl GetHost() string + GetManageHost() string GetPort() int GetSSHPort() int + GetNumaNode() string + GetNumaCores() string DeployDir() string UsedPorts() []int UsedDirs() []string @@ -106,6 +112,8 @@ type Instance interface { Arch() string IsPatched() bool SetPatched(bool) + CalculateVersion(string) string + // SetVersion(string) setTLSConfig(ctx context.Context, enableTLS bool, configs map[string]any, paths meta.DirPaths) (map[string]any, error) } @@ -137,14 +145,20 @@ type BaseInstance struct { Name string Host string + ManageHost string ListenHost string Port int SSHP int + Source string + NumaNode string + NumaCores string Ports []int Dirs []string StatusFn func(ctx context.Context, timeout time.Duration, tlsCfg *tls.Config, pdHosts ...string) string UptimeFn func(ctx context.Context, timeout time.Duration, tlsCfg *tls.Config) time.Duration + + Component Component } // Ready implements Instance interface @@ -265,7 +279,7 @@ func (i *BaseInstance) MergeServerConfig(ctx context.Context, e ctxt.Executor, g if err != nil { return err } - err = os.WriteFile(fp, conf, os.ModePerm) + err = utils.WriteFile(fp, conf, os.ModePerm) if err != nil { return err } @@ -281,7 +295,7 @@ func (i *BaseInstance) mergeTiFlashLearnerServerConfig(ctx context.Context, e ct if err != nil { return err } - err = os.WriteFile(fp, conf, os.ModePerm) + err = utils.WriteFile(fp, conf, os.ModePerm) if err != nil { return err } @@ -300,6 +314,14 @@ func (i *BaseInstance) ComponentName() string { return i.Name } +// ComponentSource implements Instance interface +func (i *BaseInstance) ComponentSource() string { + if i.Source == "" { + return i.Name + } + return i.Source +} + // InstanceName implements Instance interface func (i *BaseInstance) InstanceName() string { if i.Port > 0 { @@ -328,6 +350,14 @@ func (i *BaseInstance) GetHost() string { return i.Host } +// GetManageHost implements Instance interface +func (i *BaseInstance) GetManageHost() string { + if i.ManageHost != "" { + return i.ManageHost + } + return i.Host +} + // GetListenHost implements Instance interface func (i *BaseInstance) GetListenHost() string { if i.ListenHost == "" { @@ -345,6 +375,16 @@ func (i *BaseInstance) GetSSHPort() int { return i.SSHP } +// GetNumaNode implements Instance interface +func (i *BaseInstance) GetNumaNode() string { + return i.NumaNode +} + +// GetNumaCores implements Instance interface +func (i *BaseInstance) GetNumaCores() string { + return i.NumaCores +} + // DeployDir implements Instance interface func (i *BaseInstance) DeployDir() string { return reflect.Indirect(reflect.ValueOf(i.InstanceSpec)).FieldByName("DeployDir").String() @@ -424,6 +464,11 @@ func (i *BaseInstance) SetPatched(p bool) { v.SetBool(p) } +// CalculateVersion implements the Instance interface +func (i *BaseInstance) CalculateVersion(globalVersion string) string { + return i.Component.CalculateVersion(globalVersion) +} + // PrepareStart checks instance requirements before starting func (i *BaseInstance) PrepareStart(ctx context.Context, tlsCfg *tls.Config) error { return nil diff --git a/pkg/cluster/spec/monitoring.go b/pkg/cluster/spec/monitoring.go index 84304cf4ad..07aafaf5c4 100644 --- a/pkg/cluster/spec/monitoring.go +++ b/pkg/cluster/spec/monitoring.go @@ -38,6 +38,7 @@ import ( // PrometheusSpec represents the Prometheus Server topology specification in topology.yaml type PrometheusSpec struct { Host string `yaml:"host"` + ManageHost string `yaml:"manage_host,omitempty" validate:"manage_host:editable"` SSHPort int `yaml:"ssh_port,omitempty" validate:"ssh_port:editable"` Imported bool `yaml:"imported,omitempty"` Patched bool `yaml:"patched,omitempty"` @@ -50,6 +51,7 @@ type PrometheusSpec struct { NumaNode string `yaml:"numa_node,omitempty" validate:"numa_node:editable"` RemoteConfig Remote `yaml:"remote_config,omitempty" validate:"remote_config:ignore"` ExternalAlertmanagers []ExternalAlertmanager `yaml:"external_alertmanagers" validate:"external_alertmanagers:ignore"` + PushgatewayAddrs []string `yaml:"pushgateway_addrs,omitempty" validate:"pushgateway_addrs:ignore"` Retention string `yaml:"storage_retention,omitempty" validate:"storage_retention:editable"` ResourceControl meta.ResourceControl `yaml:"resource_control,omitempty" validate:"resource_control:editable"` Arch string `yaml:"arch,omitempty"` @@ -79,7 +81,11 @@ func (s *PrometheusSpec) Role() string { // SSH returns the host and SSH port of the instance func (s *PrometheusSpec) SSH() (string, int) { - return s.Host, s.SSHPort + host := s.Host + if s.ManageHost != "" { + host = s.ManageHost + } + return host, s.SSHPort } // GetMainPort returns the main port of the instance @@ -87,6 +93,14 @@ func (s *PrometheusSpec) GetMainPort() int { return s.Port } +// GetManageHost returns the manage host of the instance +func (s *PrometheusSpec) GetManageHost() string { + if s.ManageHost != "" { + return s.ManageHost + } + return s.Host +} + // IsImported returns if the node is imported from TiDB-Ansible func (s *PrometheusSpec) IsImported() bool { return s.Imported @@ -110,6 +124,21 @@ func (c *MonitorComponent) Role() string { return RoleMonitor } +// CalculateVersion implements the Component interface +func (c *MonitorComponent) CalculateVersion(clusterVersion string) string { + // always not follow cluster version, use ""(latest) by default + version := c.Topology.BaseTopo().PrometheusVersion + if version != nil && *version != "" { + return *version + } + return clusterVersion +} + +// SetVersion implements Component interface. +func (c *MonitorComponent) SetVersion(version string) { + *c.Topology.BaseTopo().PrometheusVersion = version +} + // Instances implements Component interface. func (c *MonitorComponent) Instances() []Instance { servers := c.BaseTopo().Monitors @@ -121,8 +150,12 @@ func (c *MonitorComponent) Instances() []Instance { InstanceSpec: s, Name: c.Name(), Host: s.Host, + ManageHost: s.ManageHost, + ListenHost: c.Topology.BaseTopo().GlobalOptions.ListenHost, Port: s.Port, SSHP: s.SSHPort, + NumaNode: s.NumaNode, + NumaCores: "", Ports: []int{ s.Port, @@ -132,11 +165,12 @@ func (c *MonitorComponent) Instances() []Instance { s.DataDir, }, StatusFn: func(_ context.Context, timeout time.Duration, _ *tls.Config, _ ...string) string { - return statusByHost(s.Host, s.Port, "/-/ready", timeout, nil) + return statusByHost(s.GetManageHost(), s.Port, "/-/ready", timeout, nil) }, UptimeFn: func(_ context.Context, timeout time.Duration, tlsCfg *tls.Config) time.Duration { - return UptimeByHost(s.Host, s.Port, timeout, tlsCfg) + return UptimeByHost(s.GetManageHost(), s.Port, timeout, tlsCfg) }, + Component: c, }, c.Topology} if s.NgPort > 0 { mi.BaseInstance.Ports = append(mi.BaseInstance.Ports, s.NgPort) @@ -232,6 +266,13 @@ func (i *MonitorInstance) InitConfig( cfig.AddTiDB(db.Host, uint64(db.StatusPort)) } } + if servers, found := topoHasField("TiProxyServers"); found { + for i := 0; i < servers.Len(); i++ { + db := servers.Index(i).Interface().(*TiProxySpec) + uniqueHosts.Insert(db.Host) + cfig.AddTiProxy(db.Host, uint64(db.StatusPort)) + } + } if servers, found := topoHasField("TiFlashServers"); found { for i := 0; i < servers.Len(); i++ { flash := servers.Index(i).Interface().(*TiFlashSpec) @@ -328,6 +369,7 @@ func (i *MonitorInstance) InitConfig( for _, alertmanager := range spec.ExternalAlertmanagers { cfig.AddAlertmanager(alertmanager.Host, uint64(alertmanager.WebPort)) } + cfig.AddPushgateway(spec.PushgatewayAddrs) if spec.RuleDir != "" { filter := func(name string) bool { return strings.HasSuffix(name, ".rules.yml") } @@ -349,18 +391,24 @@ func (i *MonitorInstance) InitConfig( } if spec.NgPort > 0 { - ngcfg := config.NewNgMonitoringConfig(clusterName, clusterVersion, enableTLS) + pds := []string{} if servers, found := topoHasField("PDServers"); found { for i := 0; i < servers.Len(); i++ { pd := servers.Index(i).Interface().(*PDSpec) - ngcfg.AddPD(pd.Host, uint64(pd.ClientPort)) + pds = append(pds, fmt.Sprintf("\"%s\"", utils.JoinHostPort(pd.Host, pd.ClientPort))) } } - ngcfg.AddIP(i.GetHost()). - AddPort(spec.NgPort). - AddDeployDir(paths.Deploy). - AddDataDir(paths.Data[0]). - AddLog(paths.Log) + ngcfg := &config.NgMonitoringConfig{ + ClusterName: clusterName, + Address: utils.JoinHostPort(i.GetListenHost(), spec.NgPort), + AdvertiseAddress: utils.JoinHostPort(i.GetHost(), spec.NgPort), + PDAddrs: strings.Join(pds, ","), + TLSEnabled: enableTLS, + + DeployDir: paths.Deploy, + DataDir: paths.Data[0], + LogDir: paths.Log, + } if servers, found := topoHasField("Monitors"); found { for i := 0; i < servers.Len(); i++ { @@ -393,7 +441,7 @@ func (i *MonitorInstance) InitConfig( return err } - return checkConfig(ctx, e, i.ComponentName(), clusterVersion, i.OS(), i.Arch(), i.ComponentName()+".yml", paths, nil) + return checkConfig(ctx, e, i.ComponentName(), i.ComponentSource(), clusterVersion, i.OS(), i.Arch(), i.ComponentName()+".yml", paths) } // setTLSConfig set TLS Config to support enable/disable TLS @@ -417,7 +465,7 @@ func (i *MonitorInstance) installRules(ctx context.Context, e ctxt.Executor, dep return errors.Annotatef(err, "stderr: %s", string(stderr)) } - srcPath := PackagePath(ComponentDMMaster, clusterVersion, i.OS(), i.Arch()) + srcPath := PackagePath(GetDMMasterPackageName(i.topo), clusterVersion, i.OS(), i.Arch()) dstPath := filepath.Join(tmp, filepath.Base(srcPath)) err = e.Transfer(ctx, srcPath, dstPath, false, 0, false) @@ -520,7 +568,7 @@ func mergeAdditionalScrapeConf(source string, addition map[string]any) error { if err != nil { return err } - return os.WriteFile(source, bytes, 0644) + return utils.WriteFile(source, bytes, 0644) } func getRetention(retention string) string { diff --git a/pkg/cluster/spec/monitoring_test.go b/pkg/cluster/spec/monitoring_test.go index db91a70546..49730b5889 100644 --- a/pkg/cluster/spec/monitoring_test.go +++ b/pkg/cluster/spec/monitoring_test.go @@ -25,6 +25,7 @@ import ( "github.com/pingcap/tiup/pkg/checkpoint" "github.com/pingcap/tiup/pkg/cluster/executor" "github.com/pingcap/tiup/pkg/meta" + "github.com/pingcap/tiup/pkg/utils" "github.com/stretchr/testify/assert" "gopkg.in/yaml.v3" ) @@ -33,7 +34,7 @@ func TestLocalRuleDirs(t *testing.T) { deployDir, err := os.MkdirTemp("", "tiup-*") assert.Nil(t, err) defer os.RemoveAll(deployDir) - err = os.MkdirAll(path.Join(deployDir, "bin/prometheus"), 0755) + err = utils.MkdirAll(path.Join(deployDir, "bin/prometheus"), 0755) assert.Nil(t, err) localDir, err := filepath.Abs("./testdata/rules") assert.Nil(t, err) @@ -75,7 +76,7 @@ func TestNoLocalRuleDirs(t *testing.T) { deployDir, err := os.MkdirTemp("", "tiup-*") assert.Nil(t, err) defer os.RemoveAll(deployDir) - err = os.MkdirAll(path.Join(deployDir, "bin/prometheus"), 0755) + err = utils.MkdirAll(path.Join(deployDir, "bin/prometheus"), 0755) assert.Nil(t, err) localDir, err := filepath.Abs("./testdata/rules") assert.Nil(t, err) diff --git a/pkg/cluster/spec/parse_topology_test.go b/pkg/cluster/spec/parse_topology_test.go index 3dfb6a60bc..51d0adfb74 100644 --- a/pkg/cluster/spec/parse_topology_test.go +++ b/pkg/cluster/spec/parse_topology_test.go @@ -285,10 +285,6 @@ server_configs: storage.main.dir: [/data1/tiflash] tiflash_servers: - host: 172.16.5.140 - data_dir: /ssd0/tiflash,/ssd1/tiflash,/ssd2/tiflash - config: - storage.main.dir: [/ssd0/tiflash, /ssd1/tiflash, /ssd2/tiflash] - storage.latest.dir: [/ssd0/tiflash, /ssd1/tiflash, /ssd2/tiflash] `, func(file string) { topo := Specification{} err := ParseTopologyYaml(file, &topo) @@ -302,10 +298,6 @@ server_configs: storage.latest.dir: [/data1/tiflash] tiflash_servers: - host: 172.16.5.140 - data_dir: /ssd0/tiflash,/ssd1/tiflash,/ssd2/tiflash - config: - storage.main.dir: [/ssd0/tiflash, /ssd1/tiflash, /ssd2/tiflash] - storage.latest.dir: [/ssd0/tiflash, /ssd1/tiflash, /ssd2/tiflash] `, func(file string) { topo := Specification{} err := ParseTopologyYaml(file, &topo) @@ -354,7 +346,7 @@ tiflash_servers: tiflash_servers: - host: 172.16.5.140 # if storage.main.dir is defined, data_dir will be ignored - data_dir: /hdd0/tiflash + data_dir: /hdd0/tiflash config: storage.main.dir: [/ssd0/tiflash, /ssd1/tiflash, /ssd2/tiflash] `, func(file string) { @@ -408,8 +400,8 @@ tiflash_servers: c.Assert(topo.TiFlashServers[0].LogDir, check.Equals, "/home/tidb/deploy/tiflash-9000/log") }) - // test tiflash storage section defined data dir - // should always define storage.main.dir if any 'storage.*' is defined + // test tiflash storage.latest section defined data dir + // should always define storage.main.dir if 'storage.latest' is defined withTempFile(` tiflash_servers: - host: 172.16.5.140 @@ -423,6 +415,35 @@ tiflash_servers: c.Assert(err, check.NotNil) }) + // test tiflash storage.raft section defined data dir + // should always define storage.main.dir if 'storage.raft' is defined + withTempFile(` +tiflash_servers: + - host: 172.16.5.140 + data_dir: /ssd0/tiflash + config: + #storage.main.dir: [/hdd0/tiflash, /hdd1/tiflash, /hdd2/tiflash] + storage.raft.dir: [/ssd0/tiflash, /ssd1/tiflash, /ssd2/tiflash, /hdd0/tiflash] +`, func(file string) { + topo := Specification{} + err := ParseTopologyYaml(file, &topo) + c.Assert(err, check.NotNil) + }) + + // test tiflash storage.remote section defined data dir + // should be fine even when `storage.main.dir` is not defined. + withTempFile(` +tiflash_servers: + - host: 172.16.5.140 + data_dir: /ssd0/tiflash + config: + storage.remote.dir: /tmp/tiflash/remote +`, func(file string) { + topo := Specification{} + err := ParseTopologyYaml(file, &topo) + c.Assert(err, check.IsNil) + }) + // test tiflash storage section defined data dir // storage.main.dir should always use absolute path withTempFile(` diff --git a/pkg/cluster/spec/pd.go b/pkg/cluster/spec/pd.go index 342f56e23e..f505db7b18 100644 --- a/pkg/cluster/spec/pd.go +++ b/pkg/cluster/spec/pd.go @@ -33,6 +33,7 @@ import ( // PDSpec represents the PD topology specification in topology.yaml type PDSpec struct { Host string `yaml:"host"` + ManageHost string `yaml:"manage_host,omitempty" validate:"manage_host:editable"` ListenHost string `yaml:"listen_host,omitempty"` AdvertiseClientAddr string `yaml:"advertise_client_addr,omitempty"` AdvertisePeerAddr string `yaml:"advertise_peer_addr,omitempty"` @@ -47,6 +48,7 @@ type PDSpec struct { DeployDir string `yaml:"deploy_dir,omitempty"` DataDir string `yaml:"data_dir,omitempty"` LogDir string `yaml:"log_dir,omitempty"` + Source string `yaml:"source,omitempty" validate:"source:editable"` NumaNode string `yaml:"numa_node,omitempty" validate:"numa_node:editable"` Config map[string]any `yaml:"config,omitempty" validate:"config:ignore"` ResourceControl meta.ResourceControl `yaml:"resource_control,omitempty" validate:"resource_control:editable"` @@ -60,7 +62,7 @@ func (s *PDSpec) Status(ctx context.Context, timeout time.Duration, tlsCfg *tls. timeout = statusQueryTimeout } - addr := utils.JoinHostPort(s.Host, s.ClientPort) + addr := utils.JoinHostPort(s.GetManageHost(), s.ClientPort) pc := api.NewPDClient(ctx, []string{addr}, timeout, tlsCfg) // check health @@ -88,7 +90,11 @@ func (s *PDSpec) Role() string { // SSH returns the host and SSH port of the instance func (s *PDSpec) SSH() (string, int) { - return s.Host, s.SSHPort + host := s.Host + if s.ManageHost != "" { + host = s.ManageHost + } + return host, s.SSHPort } // GetMainPort returns the main port of the instance @@ -96,6 +102,14 @@ func (s *PDSpec) GetMainPort() int { return s.ClientPort } +// GetManageHost returns the manage host of the instance +func (s *PDSpec) GetManageHost() string { + if s.ManageHost != "" { + return s.ManageHost + } + return s.Host +} + // IsImported returns if the node is imported from TiDB-Ansible func (s *PDSpec) IsImported() bool { return s.Imported @@ -124,6 +138,14 @@ func (s *PDSpec) GetAdvertisePeerURL(enableTLS bool) string { return fmt.Sprintf("%s://%s", scheme, utils.JoinHostPort(s.Host, s.PeerPort)) } +// GetSource returns source to download the component +func (s *PDSpec) GetSource() string { + if s.Source == "" { + return ComponentPD + } + return s.Source +} + // PDComponent represents PD component. type PDComponent struct{ Topology *Specification } @@ -137,6 +159,20 @@ func (c *PDComponent) Role() string { return ComponentPD } +// CalculateVersion implements the Component interface +func (c *PDComponent) CalculateVersion(clusterVersion string) string { + version := c.Topology.ComponentVersions.PD + if version == "" { + version = clusterVersion + } + return version +} + +// SetVersion implements Component interface. +func (c *PDComponent) SetVersion(version string) { + c.Topology.ComponentVersions.PD = version +} + // Instances implements Component interface. func (c *PDComponent) Instances() []Instance { ins := make([]Instance, 0, len(c.Topology.PDServers)) @@ -148,9 +184,13 @@ func (c *PDComponent) Instances() []Instance { InstanceSpec: s, Name: c.Name(), Host: s.Host, - ListenHost: s.ListenHost, + ManageHost: s.ManageHost, + ListenHost: utils.Ternary(s.ListenHost != "", s.ListenHost, c.Topology.BaseTopo().GlobalOptions.ListenHost).(string), Port: s.ClientPort, SSHP: s.SSHPort, + Source: s.GetSource(), + NumaNode: s.NumaNode, + NumaCores: "", Ports: []int{ s.ClientPort, @@ -162,8 +202,9 @@ func (c *PDComponent) Instances() []Instance { }, StatusFn: s.Status, UptimeFn: func(_ context.Context, timeout time.Duration, tlsCfg *tls.Config) time.Duration { - return UptimeByHost(s.Host, s.ClientPort, timeout, tlsCfg) + return UptimeByHost(s.GetManageHost(), s.ClientPort, timeout, tlsCfg) }, + Component: c, }, topo: c.Topology, }) @@ -195,6 +236,7 @@ func (i *PDInstance) InitConfig( enableTLS := topo.GlobalOptions.TLSEnabled spec := i.InstanceSpec.(*PDSpec) scheme := utils.Ternary(enableTLS, "https", "http").(string) + version := i.CalculateVersion(clusterVersion) initialCluster := []string{} for _, pdspec := range topo.PDServers { @@ -259,7 +301,7 @@ func (i *PDInstance) InitConfig( return err } - return checkConfig(ctx, e, i.ComponentName(), clusterVersion, i.OS(), i.Arch(), i.ComponentName()+".toml", paths, nil) + return checkConfig(ctx, e, i.ComponentName(), i.ComponentSource(), version, i.OS(), i.Arch(), i.ComponentName()+".toml", paths) } // setTLSConfig set TLS Config to support enable/disable TLS @@ -366,7 +408,7 @@ func (i *PDInstance) IsLeader(ctx context.Context, topo Topology, apiTimeoutSeco if !ok { panic("topo should be type of tidb topology") } - pdClient := api.NewPDClient(ctx, tidbTopo.GetPDList(), time.Second*5, tlsCfg) + pdClient := api.NewPDClient(ctx, tidbTopo.GetPDListWithManageHost(), time.Second*5, tlsCfg) return i.checkLeader(pdClient) } @@ -391,7 +433,7 @@ func (i *PDInstance) PreRestart(ctx context.Context, topo Topology, apiTimeoutSe if !ok { panic("topo should be type of tidb topology") } - pdClient := api.NewPDClient(ctx, tidbTopo.GetPDList(), time.Second*5, tlsCfg) + pdClient := api.NewPDClient(ctx, tidbTopo.GetPDListWithManageHost(), time.Second*5, tlsCfg) isLeader, err := i.checkLeader(pdClient) if err != nil { @@ -416,7 +458,7 @@ func (i *PDInstance) PostRestart(ctx context.Context, topo Topology, tlsCfg *tls Delay: time.Second, Timeout: 120 * time.Second, } - currentPDAddrs := []string{utils.JoinHostPort(i.Host, i.Port)} + currentPDAddrs := []string{utils.JoinHostPort(i.GetManageHost(), i.Port)} pdClient := api.NewPDClient(ctx, currentPDAddrs, 5*time.Second, tlsCfg) if err := utils.Retry(pdClient.CheckHealth, timeoutOpt); err != nil { diff --git a/pkg/cluster/spec/profile.go b/pkg/cluster/spec/profile.go index 7ad4ba7c5d..214036d9da 100644 --- a/pkg/cluster/spec/profile.go +++ b/pkg/cluster/spec/profile.go @@ -19,10 +19,9 @@ import ( "path" "path/filepath" - utils2 "github.com/pingcap/tiup/pkg/utils" - "github.com/pingcap/errors" tiuplocaldata "github.com/pingcap/tiup/pkg/localdata" + "github.com/pingcap/tiup/pkg/utils" ) // sub directory names @@ -82,7 +81,7 @@ func Initialize(base string) error { }) initialized = true // make sure the dir exist - return utils2.CreateDir(profileDir) + return utils.MkdirAll(profileDir, 0755) } // ProfileDir returns the full profile directory path of TiUP. diff --git a/pkg/cluster/spec/pump.go b/pkg/cluster/spec/pump.go index 4409e00402..00dc89ddfd 100644 --- a/pkg/cluster/spec/pump.go +++ b/pkg/cluster/spec/pump.go @@ -19,7 +19,6 @@ import ( "fmt" "os" "path/filepath" - "strconv" "strings" "time" @@ -33,6 +32,7 @@ import ( // PumpSpec represents the Pump topology specification in topology.yaml type PumpSpec struct { Host string `yaml:"host"` + ManageHost string `yaml:"manage_host,omitempty" validate:"manage_host:editable"` SSHPort int `yaml:"ssh_port,omitempty" validate:"ssh_port:editable"` Imported bool `yaml:"imported,omitempty"` Patched bool `yaml:"patched,omitempty"` @@ -42,6 +42,7 @@ type PumpSpec struct { DataDir string `yaml:"data_dir,omitempty"` LogDir string `yaml:"log_dir,omitempty"` Offline bool `yaml:"offline,omitempty"` + Source string `yaml:"source,omitempty" validate:"source:editable"` NumaNode string `yaml:"numa_node,omitempty" validate:"numa_node:editable"` Config map[string]any `yaml:"config,omitempty" validate:"config:ignore"` ResourceControl meta.ResourceControl `yaml:"resource_control,omitempty" validate:"resource_control:editable"` @@ -55,14 +56,14 @@ func (s *PumpSpec) Status(ctx context.Context, timeout time.Duration, tlsCfg *tl timeout = statusQueryTimeout } - state := statusByHost(s.Host, s.Port, "/status", timeout, tlsCfg) + state := statusByHost(s.GetManageHost(), s.Port, "/status", timeout, tlsCfg) if s.Offline { binlogClient, err := api.NewBinlogClient(pdList, timeout, tlsCfg) if err != nil { return state } - id := s.Host + ":" + strconv.Itoa(s.Port) + id := utils.JoinHostPort(s.Host, s.Port) tombstone, _ := binlogClient.IsPumpTombstone(ctx, id) if tombstone { @@ -81,7 +82,11 @@ func (s *PumpSpec) Role() string { // SSH returns the host and SSH port of the instance func (s *PumpSpec) SSH() (string, int) { - return s.Host, s.SSHPort + host := s.Host + if s.ManageHost != "" { + host = s.ManageHost + } + return host, s.SSHPort } // GetMainPort returns the main port of the instance @@ -89,6 +94,14 @@ func (s *PumpSpec) GetMainPort() int { return s.Port } +// GetManageHost returns the manage host of the instance +func (s *PumpSpec) GetManageHost() string { + if s.ManageHost != "" { + return s.ManageHost + } + return s.Host +} + // IsImported returns if the node is imported from TiDB-Ansible func (s *PumpSpec) IsImported() bool { return s.Imported @@ -99,6 +112,14 @@ func (s *PumpSpec) IgnoreMonitorAgent() bool { return s.IgnoreExporter } +// GetSource returns source to download the component +func (s *PumpSpec) GetSource() string { + if s.Source == "" { + return ComponentPump + } + return s.Source +} + // PumpComponent represents Pump component. type PumpComponent struct{ Topology *Specification } @@ -112,6 +133,20 @@ func (c *PumpComponent) Role() string { return ComponentPump } +// CalculateVersion implements the Component interface +func (c *PumpComponent) CalculateVersion(clusterVersion string) string { + version := c.Topology.ComponentVersions.Pump + if version == "" { + version = clusterVersion + } + return version +} + +// SetVersion implements Component interface. +func (c *PumpComponent) SetVersion(version string) { + c.Topology.ComponentVersions.Pump = version +} + // Instances implements Component interface. func (c *PumpComponent) Instances() []Instance { ins := make([]Instance, 0, len(c.Topology.PumpServers)) @@ -121,8 +156,13 @@ func (c *PumpComponent) Instances() []Instance { InstanceSpec: s, Name: c.Name(), Host: s.Host, + ManageHost: s.ManageHost, + ListenHost: c.Topology.BaseTopo().GlobalOptions.ListenHost, Port: s.Port, SSHP: s.SSHPort, + Source: s.GetSource(), + NumaNode: s.NumaNode, + NumaCores: "", Ports: []int{ s.Port, @@ -133,8 +173,9 @@ func (c *PumpComponent) Instances() []Instance { }, StatusFn: s.Status, UptimeFn: func(_ context.Context, timeout time.Duration, tlsCfg *tls.Config) time.Duration { - return UptimeByHost(s.Host, s.Port, timeout, tlsCfg) + return UptimeByHost(s.GetManageHost(), s.Port, timeout, tlsCfg) }, + Component: c, }, c.Topology}) } return ins diff --git a/pkg/cluster/spec/server_config.go b/pkg/cluster/spec/server_config.go index d5de0f0bdf..67207a6466 100644 --- a/pkg/cluster/spec/server_config.go +++ b/pkg/cluster/spec/server_config.go @@ -260,10 +260,7 @@ func mergeImported(importConfig []byte, specConfigs ...map[string]any) (map[stri return lhs, nil } -// BindVersion map the cluster version to the third components binding version. -type BindVersion func(comp string, version string) (bindVersion string) - -func checkConfig(ctx context.Context, e ctxt.Executor, componentName, clusterVersion, nodeOS, arch, config string, paths meta.DirPaths, bindVersion BindVersion) error { +func checkConfig(ctx context.Context, e ctxt.Executor, componentName, componentSource, version, nodeOS, arch, config string, paths meta.DirPaths) error { var cmd string configPath := path.Join(paths.Deploy, "conf", config) switch componentName { @@ -277,17 +274,11 @@ func checkConfig(ctx context.Context, e ctxt.Executor, componentName, clusterVer return perrs.Annotate(ErrorCheckConfig, err.Error()) } - clsVer := utils.Version(clusterVersion) - ver := clusterVersion - if clsVer.IsNightly() { - ver = utils.NightlyVersionAlias - } - // FIXME: workaround for nightly versions, need refactor - if bindVersion != nil { - ver = bindVersion(componentName, ver) + if utils.Version(version).IsNightly() { + version = utils.NightlyVersionAlias } - entry, err := repo.ComponentBinEntry(componentName, ver) + entry, err := repo.ComponentBinEntry(componentSource, version) if err != nil { return perrs.Annotate(ErrorCheckConfig, err.Error()) } diff --git a/pkg/cluster/spec/spec.go b/pkg/cluster/spec/spec.go index 1c9fc7b12c..b6c44c1d6d 100644 --- a/pkg/cluster/spec/spec.go +++ b/pkg/cluster/spec/spec.go @@ -28,7 +28,6 @@ import ( "github.com/pingcap/errors" "github.com/pingcap/tiup/pkg/cluster/api" "github.com/pingcap/tiup/pkg/cluster/executor" - "github.com/pingcap/tiup/pkg/cluster/template/scripts" "github.com/pingcap/tiup/pkg/meta" "github.com/pingcap/tiup/pkg/proxy" "github.com/pingcap/tiup/pkg/tidbver" @@ -83,6 +82,7 @@ type ( SSHPort int `yaml:"ssh_port,omitempty" default:"22" validate:"ssh_port:editable"` SSHType executor.SSHType `yaml:"ssh_type,omitempty" default:"builtin"` TLSEnabled bool `yaml:"enable_tls,omitempty"` + ListenHost string `yaml:"listen_host,omitempty" validate:"listen_host:editable"` DeployDir string `yaml:"deploy_dir,omitempty" default:"deploy"` DataDir string `yaml:"data_dir,omitempty" default:"data"` LogDir string `yaml:"log_dir,omitempty"` @@ -94,13 +94,15 @@ type ( // MonitoredOptions represents the monitored node configuration MonitoredOptions struct { - NodeExporterPort int `yaml:"node_exporter_port,omitempty" default:"9100"` - BlackboxExporterPort int `yaml:"blackbox_exporter_port,omitempty" default:"9115"` - DeployDir string `yaml:"deploy_dir,omitempty"` - DataDir string `yaml:"data_dir,omitempty"` - LogDir string `yaml:"log_dir,omitempty"` - NumaNode string `yaml:"numa_node,omitempty" validate:"numa_node:editable"` - ResourceControl meta.ResourceControl `yaml:"resource_control,omitempty" validate:"resource_control:editable"` + NodeExporterPort int `yaml:"node_exporter_port,omitempty" default:"9100"` + BlackboxExporterPort int `yaml:"blackbox_exporter_port,omitempty" default:"9115"` + NodeExporterVersion string `yaml:"node_exporter_version,omitempty"` + BlackboxExporterVersion string `yaml:"blackbox_exporter_version,omitempty"` + DeployDir string `yaml:"deploy_dir,omitempty"` + DataDir string `yaml:"data_dir,omitempty"` + LogDir string `yaml:"log_dir,omitempty"` + NumaNode string `yaml:"numa_node,omitempty" validate:"numa_node:editable"` + ResourceControl meta.ResourceControl `yaml:"resource_control,omitempty" validate:"resource_control:editable"` } // ServerConfigs represents the server runtime configuration @@ -110,6 +112,7 @@ type ( PD map[string]any `yaml:"pd"` Dashboard map[string]any `yaml:"tidb_dashboard"` TiFlash map[string]any `yaml:"tiflash"` + TiProxy map[string]any `yaml:"tiproxy"` TiFlashLearner map[string]any `yaml:"tiflash-learner"` Pump map[string]any `yaml:"pump"` Drainer map[string]any `yaml:"drainer"` @@ -118,25 +121,47 @@ type ( Grafana map[string]string `yaml:"grafana"` } + // ComponentVersions represents the versions of components + ComponentVersions struct { + TiDB string `yaml:"tidb,omitempty"` + TiKV string `yaml:"tikv,omitempty"` + TiFlash string `yaml:"tiflash,omitempty"` + PD string `yaml:"pd,omitempty"` + Dashboard string `yaml:"tidb_dashboard,omitempty"` + Pump string `yaml:"pump,omitempty"` + Drainer string `yaml:"drainer,omitempty"` + CDC string `yaml:"cdc,omitempty"` + TiKVCDC string `yaml:"kvcdc,omitempty"` + TiProxy string `yaml:"tiproxy,omitempty"` + Prometheus string `yaml:"prometheus,omitempty"` + Grafana string `yaml:"grafana,omitempty"` + AlertManager string `yaml:"alertmanager,omitempty"` + // The versions of exporters are placed within the monitored section because they are not explicitly treated as separate components. + // NodeExporter string `yaml:"node_exporter,omitempty"` + // BlackboxExporter string `yaml:"blackbox_exporter,omitempty"` + } + // Specification represents the specification of topology.yaml Specification struct { - GlobalOptions GlobalOptions `yaml:"global,omitempty" validate:"global:editable"` - MonitoredOptions MonitoredOptions `yaml:"monitored,omitempty" validate:"monitored:editable"` - ServerConfigs ServerConfigs `yaml:"server_configs,omitempty" validate:"server_configs:ignore"` - TiDBServers []*TiDBSpec `yaml:"tidb_servers"` - TiKVServers []*TiKVSpec `yaml:"tikv_servers"` - TiFlashServers []*TiFlashSpec `yaml:"tiflash_servers"` - PDServers []*PDSpec `yaml:"pd_servers"` - DashboardServers []*DashboardSpec `yaml:"tidb_dashboard_servers,omitempty"` - PumpServers []*PumpSpec `yaml:"pump_servers,omitempty"` - Drainers []*DrainerSpec `yaml:"drainer_servers,omitempty"` - CDCServers []*CDCSpec `yaml:"cdc_servers,omitempty"` - TiKVCDCServers []*TiKVCDCSpec `yaml:"kvcdc_servers,omitempty"` - TiSparkMasters []*TiSparkMasterSpec `yaml:"tispark_masters,omitempty"` - TiSparkWorkers []*TiSparkWorkerSpec `yaml:"tispark_workers,omitempty"` - Monitors []*PrometheusSpec `yaml:"monitoring_servers"` - Grafanas []*GrafanaSpec `yaml:"grafana_servers,omitempty"` - Alertmanagers []*AlertmanagerSpec `yaml:"alertmanager_servers,omitempty"` + GlobalOptions GlobalOptions `yaml:"global,omitempty" validate:"global:editable"` + MonitoredOptions MonitoredOptions `yaml:"monitored,omitempty" validate:"monitored:editable"` + ComponentVersions ComponentVersions `yaml:"component_versions,omitempty" validate:"component_versions:editable"` + ServerConfigs ServerConfigs `yaml:"server_configs,omitempty" validate:"server_configs:ignore"` + TiDBServers []*TiDBSpec `yaml:"tidb_servers"` + TiKVServers []*TiKVSpec `yaml:"tikv_servers"` + TiFlashServers []*TiFlashSpec `yaml:"tiflash_servers"` + TiProxyServers []*TiProxySpec `yaml:"tiproxy_servers"` + PDServers []*PDSpec `yaml:"pd_servers"` + DashboardServers []*DashboardSpec `yaml:"tidb_dashboard_servers,omitempty"` + PumpServers []*PumpSpec `yaml:"pump_servers,omitempty"` + Drainers []*DrainerSpec `yaml:"drainer_servers,omitempty"` + CDCServers []*CDCSpec `yaml:"cdc_servers,omitempty"` + TiKVCDCServers []*TiKVCDCSpec `yaml:"kvcdc_servers,omitempty"` + TiSparkMasters []*TiSparkMasterSpec `yaml:"tispark_masters,omitempty"` + TiSparkWorkers []*TiSparkWorkerSpec `yaml:"tispark_workers,omitempty"` + Monitors []*PrometheusSpec `yaml:"monitoring_servers"` + Grafanas []*GrafanaSpec `yaml:"grafana_servers,omitempty"` + Alertmanagers []*AlertmanagerSpec `yaml:"alertmanager_servers,omitempty"` } ) @@ -146,9 +171,12 @@ type BaseTopo struct { MonitoredOptions *MonitoredOptions MasterList []string - Monitors []*PrometheusSpec - Grafanas []*GrafanaSpec - Alertmanagers []*AlertmanagerSpec + PrometheusVersion *string + GrafanaVersion *string + AlertManagerVersion *string + Monitors []*PrometheusSpec + Grafanas []*GrafanaSpec + Alertmanagers []*AlertmanagerSpec } // Topology represents specification of the cluster. @@ -162,7 +190,7 @@ type Topology interface { // Instances() []Instance ComponentsByStartOrder() []Component ComponentsByStopOrder() []Component - ComponentsByUpdateOrder() []Component + ComponentsByUpdateOrder(curVer string) []Component IterInstance(fn func(instance Instance), concurrency ...int) GetMonitoredOptions() *MonitoredOptions // count how many time a path is used by instances in cluster @@ -253,12 +281,15 @@ func (s *Specification) Type() string { // BaseTopo implements Topology interface. func (s *Specification) BaseTopo() *BaseTopo { return &BaseTopo{ - GlobalOptions: &s.GlobalOptions, - MonitoredOptions: s.GetMonitoredOptions(), - MasterList: s.GetPDList(), - Monitors: s.Monitors, - Grafanas: s.Grafanas, - Alertmanagers: s.Alertmanagers, + GlobalOptions: &s.GlobalOptions, + MonitoredOptions: s.GetMonitoredOptions(), + MasterList: s.GetPDListWithManageHost(), + PrometheusVersion: &s.ComponentVersions.Prometheus, + GrafanaVersion: &s.ComponentVersions.Grafana, + AlertManagerVersion: &s.ComponentVersions.AlertManager, + Monitors: s.Monitors, + Grafanas: s.Grafanas, + Alertmanagers: s.Alertmanagers, } } @@ -414,11 +445,26 @@ func (s *Specification) GetPDList() []string { return pdList } -// GetCDCList returns a list of CDC API hosts of the current cluster -func (s *Specification) GetCDCList() []string { +// GetPDListWithManageHost returns a list of PD API hosts of the current cluster +func (s *Specification) GetPDListWithManageHost() []string { + var pdList []string + + for _, pd := range s.PDServers { + pdList = append(pdList, utils.JoinHostPort(pd.GetManageHost(), pd.ClientPort)) + } + + return pdList +} + +// GetCDCListWithManageHost returns a list of CDC API hosts of the current cluster +func (s *Specification) GetCDCListWithManageHost() []string { var result []string for _, server := range s.CDCServers { - result = append(result, utils.JoinHostPort(server.Host, server.Port)) + host := server.Host + if server.ManageHost != "" { + host = server.ManageHost + } + result = append(result, utils.JoinHostPort(host, server.Port)) } return result } @@ -461,14 +507,14 @@ func (s *Specification) GetDashboardAddress(ctx context.Context, tlsCfg *tls.Con // GetEtcdClient loads EtcdClient of current cluster func (s *Specification) GetEtcdClient(tlsCfg *tls.Config) (*clientv3.Client, error) { return clientv3.New(clientv3.Config{ - Endpoints: s.GetPDList(), + Endpoints: s.GetPDListWithManageHost(), TLS: tlsCfg, }) } // GetEtcdProxyClient loads EtcdClient of current cluster with TCP proxy func (s *Specification) GetEtcdProxyClient(tlsCfg *tls.Config, tcpProxy *proxy.TCPProxy) (*clientv3.Client, chan struct{}, error) { - closeC := tcpProxy.Run(s.GetPDList()) + closeC := tcpProxy.Run(s.GetPDListWithManageHost()) cli, err := clientv3.New(clientv3.Config{ Endpoints: tcpProxy.GetEndpoints(), TLS: tlsCfg, @@ -480,23 +526,44 @@ func (s *Specification) GetEtcdProxyClient(tlsCfg *tls.Config, tcpProxy *proxy.T func (s *Specification) Merge(that Topology) Topology { spec := that.(*Specification) return &Specification{ - GlobalOptions: s.GlobalOptions, - MonitoredOptions: s.MonitoredOptions, - ServerConfigs: s.ServerConfigs, - TiDBServers: append(s.TiDBServers, spec.TiDBServers...), - TiKVServers: append(s.TiKVServers, spec.TiKVServers...), - PDServers: append(s.PDServers, spec.PDServers...), - DashboardServers: append(s.DashboardServers, spec.DashboardServers...), - TiFlashServers: append(s.TiFlashServers, spec.TiFlashServers...), - PumpServers: append(s.PumpServers, spec.PumpServers...), - Drainers: append(s.Drainers, spec.Drainers...), - CDCServers: append(s.CDCServers, spec.CDCServers...), - TiKVCDCServers: append(s.TiKVCDCServers, spec.TiKVCDCServers...), - TiSparkMasters: append(s.TiSparkMasters, spec.TiSparkMasters...), - TiSparkWorkers: append(s.TiSparkWorkers, spec.TiSparkWorkers...), - Monitors: append(s.Monitors, spec.Monitors...), - Grafanas: append(s.Grafanas, spec.Grafanas...), - Alertmanagers: append(s.Alertmanagers, spec.Alertmanagers...), + GlobalOptions: s.GlobalOptions, + MonitoredOptions: s.MonitoredOptions, + ServerConfigs: s.ServerConfigs, + ComponentVersions: s.ComponentVersions.Merge(spec.ComponentVersions), + TiDBServers: append(s.TiDBServers, spec.TiDBServers...), + TiKVServers: append(s.TiKVServers, spec.TiKVServers...), + PDServers: append(s.PDServers, spec.PDServers...), + DashboardServers: append(s.DashboardServers, spec.DashboardServers...), + TiFlashServers: append(s.TiFlashServers, spec.TiFlashServers...), + TiProxyServers: append(s.TiProxyServers, spec.TiProxyServers...), + PumpServers: append(s.PumpServers, spec.PumpServers...), + Drainers: append(s.Drainers, spec.Drainers...), + CDCServers: append(s.CDCServers, spec.CDCServers...), + TiKVCDCServers: append(s.TiKVCDCServers, spec.TiKVCDCServers...), + TiSparkMasters: append(s.TiSparkMasters, spec.TiSparkMasters...), + TiSparkWorkers: append(s.TiSparkWorkers, spec.TiSparkWorkers...), + Monitors: append(s.Monitors, spec.Monitors...), + Grafanas: append(s.Grafanas, spec.Grafanas...), + Alertmanagers: append(s.Alertmanagers, spec.Alertmanagers...), + } +} + +// Merge returns a new ComponentVersions which sum old ones +func (v *ComponentVersions) Merge(that ComponentVersions) ComponentVersions { + return ComponentVersions{ + TiDB: utils.Ternary(that.TiDB != "", that.TiDB, v.TiDB).(string), + TiKV: utils.Ternary(that.TiKV != "", that.TiKV, v.TiKV).(string), + PD: utils.Ternary(that.PD != "", that.PD, v.PD).(string), + Dashboard: utils.Ternary(that.Dashboard != "", that.Dashboard, v.Dashboard).(string), + TiFlash: utils.Ternary(that.TiFlash != "", that.TiFlash, v.TiFlash).(string), + TiProxy: utils.Ternary(that.TiProxy != "", that.TiProxy, v.TiProxy).(string), + Pump: utils.Ternary(that.Pump != "", that.Pump, v.Pump).(string), + Drainer: utils.Ternary(that.Drainer != "", that.Drainer, v.Drainer).(string), + CDC: utils.Ternary(that.CDC != "", that.CDC, v.CDC).(string), + TiKVCDC: utils.Ternary(that.TiKVCDC != "", that.TiKVCDC, v.TiKVCDC).(string), + Grafana: utils.Ternary(that.Grafana != "", that.Grafana, v.Grafana).(string), + Prometheus: utils.Ternary(that.Prometheus != "", that.Prometheus, v.Prometheus).(string), + AlertManager: utils.Ternary(that.AlertManager != "", that.AlertManager, v.AlertManager).(string), } } @@ -516,15 +583,16 @@ func fillCustomDefaults(globalOptions *GlobalOptions, data any) error { } var ( - globalOptionTypeName = reflect.TypeOf(GlobalOptions{}).Name() - monitorOptionTypeName = reflect.TypeOf(MonitoredOptions{}).Name() - serverConfigsTypeName = reflect.TypeOf(ServerConfigs{}).Name() + globalOptionTypeName = reflect.TypeOf(GlobalOptions{}).Name() + monitorOptionTypeName = reflect.TypeOf(MonitoredOptions{}).Name() + serverConfigsTypeName = reflect.TypeOf(ServerConfigs{}).Name() + componentVersionsTypeName = reflect.TypeOf(ComponentVersions{}).Name() ) // Skip global/monitored options func isSkipField(field reflect.Value) bool { tp := field.Type().Name() - return tp == globalOptionTypeName || tp == monitorOptionTypeName || tp == serverConfigsTypeName + return tp == globalOptionTypeName || tp == monitorOptionTypeName || tp == serverConfigsTypeName || tp == componentVersionsTypeName } func setDefaultDir(parent, role, port string, field reflect.Value) { @@ -695,9 +763,10 @@ func (s *Specification) ComponentsByStopOrder() (comps []Component) { // ComponentsByStartOrder return component in the order need to start. func (s *Specification) ComponentsByStartOrder() (comps []Component) { - // "pd", "dashboard", "tikv", "pump", "tidb", "tiflash", "drainer", "cdc", "tikv-cdc", "prometheus", "grafana", "alertmanager" + // "pd", "dashboard", "tiproxy", "tikv", "pump", "tidb", "tiflash", "drainer", "cdc", "tikv-cdc", "prometheus", "grafana", "alertmanager" comps = append(comps, &PDComponent{s}) comps = append(comps, &DashboardComponent{s}) + comps = append(comps, &TiProxyComponent{s}) comps = append(comps, &TiKVComponent{s}) comps = append(comps, &PumpComponent{s}) comps = append(comps, &TiDBComponent{s}) @@ -714,16 +783,25 @@ func (s *Specification) ComponentsByStartOrder() (comps []Component) { } // ComponentsByUpdateOrder return component in the order need to be updated. -func (s *Specification) ComponentsByUpdateOrder() (comps []Component) { - // "tiflash", "pd", "dashboard", "tikv", "pump", "tidb", "drainer", "cdc", "prometheus", "grafana", "alertmanager" +func (s *Specification) ComponentsByUpdateOrder(curVer string) (comps []Component) { + // Ref: https://github.com/pingcap/tiup/issues/2166 + cdcUpgradeBeforePDTiKVTiDB := tidbver.TiCDCUpgradeBeforePDTiKVTiDB(curVer) + + // "tiflash", <"cdc">, "pd", "dashboard", "tiproxy", "tikv", "pump", "tidb", "drainer", <"cdc>", "prometheus", "grafana", "alertmanager" comps = append(comps, &TiFlashComponent{s}) + if cdcUpgradeBeforePDTiKVTiDB { + comps = append(comps, &CDCComponent{s}) + } comps = append(comps, &PDComponent{s}) comps = append(comps, &DashboardComponent{s}) + comps = append(comps, &TiProxyComponent{s}) comps = append(comps, &TiKVComponent{s}) comps = append(comps, &PumpComponent{s}) comps = append(comps, &TiDBComponent{s}) comps = append(comps, &DrainerComponent{s}) - comps = append(comps, &CDCComponent{s}) + if !cdcUpgradeBeforePDTiKVTiDB { + comps = append(comps, &CDCComponent{s}) + } comps = append(comps, &MonitorComponent{s}) comps = append(comps, &GrafanaComponent{s}) comps = append(comps, &AlertManagerComponent{s}) @@ -789,37 +867,6 @@ func IterHost(topo Topology, fn func(instance Instance)) { } } -// Endpoints returns the PD endpoints configurations -func (s *Specification) Endpoints(user string) []*scripts.PDScript { - var ends []*scripts.PDScript - for _, spec := range s.PDServers { - deployDir := Abs(user, spec.DeployDir) - // data dir would be empty for components which don't need it - dataDir := spec.DataDir - // the default data_dir is relative to deploy_dir - if dataDir != "" && !strings.HasPrefix(dataDir, "/") { - dataDir = filepath.Join(deployDir, dataDir) - } - // log dir will always be with values, but might not used by the component - logDir := Abs(user, spec.LogDir) - - script := &scripts.PDScript{ - Name: spec.Name, - IP: spec.Host, - ListenHost: spec.ListenHost, - DeployDir: deployDir, - DataDir: dataDir, - LogDir: logDir, - NumaNode: spec.NumaNode, - TLSEnabled: s.GlobalOptions.TLSEnabled, - ClientPort: spec.ClientPort, - PeerPort: spec.PeerPort, - } - ends = append(ends, script) - } - return ends -} - // FillHostArchOrOS fills the topology with the given host->arch func (s *Specification) FillHostArchOrOS(hostArch map[string]string, fullType FullHostType) error { if err := FillHostArchOrOS(s, hostArch, fullType); err != nil { @@ -873,6 +920,10 @@ func setHostArchOrOS(field reflect.Value, hostArchOrOS map[string]string, fullTy } host := field.FieldByName("Host") + if field.FieldByName("ManageHost").String() != "" { + host = field.FieldByName("ManageHost") + } + arch := field.FieldByName("Arch") os := field.FieldByName("OS") diff --git a/pkg/cluster/spec/spec_manager.go b/pkg/cluster/spec/spec_manager.go index 79771effd5..241d6fbc2e 100644 --- a/pkg/cluster/spec/spec_manager.go +++ b/pkg/cluster/spec/spec_manager.go @@ -98,7 +98,7 @@ func (s *SpecManager) SaveMeta(clusterName string, meta Metadata) error { return wrapError(err) } - if err := os.MkdirAll(backupDir, 0755); err != nil { + if err := utils.MkdirAll(backupDir, 0755); err != nil { return wrapError(err) } @@ -199,7 +199,7 @@ func (s *SpecManager) GetAllClusters() (map[string]Metadata, error) { // ensureDir ensures that the cluster directory exists. func (s *SpecManager) ensureDir(clusterName string) error { - if err := utils.CreateDir(s.Path(clusterName)); err != nil { + if err := utils.MkdirAll(s.Path(clusterName), 0755); err != nil { return ErrCreateDirFailed. Wrap(err, "Failed to create cluster metadata directory '%s'", s.Path(clusterName)). WithProperty(tui.SuggestionFromString("Please check file system permissions and try again.")) @@ -271,7 +271,7 @@ func (s *SpecManager) NewScaleOutLock(clusterName string, topo Topology) error { return wrapError(err) } - err = os.WriteFile(lockFile, data, 0644) + err = utils.WriteFile(lockFile, data, 0644) if err != nil { return wrapError(err) } diff --git a/pkg/cluster/spec/spec_manager_test.go b/pkg/cluster/spec/spec_manager_test.go index 76926878e2..ec8d38cb29 100644 --- a/pkg/cluster/spec/spec_manager_test.go +++ b/pkg/cluster/spec/spec_manager_test.go @@ -97,7 +97,7 @@ func (t *TestTopology) ComponentsByStopOrder() []Component { return nil } -func (t *TestTopology) ComponentsByUpdateOrder() []Component { +func (t *TestTopology) ComponentsByUpdateOrder(curVer string) []Component { return nil } diff --git a/pkg/cluster/spec/testdata/topology_err.yaml b/pkg/cluster/spec/testdata/topology_err.yaml index 3749140a59..6b64e33afe 100644 --- a/pkg/cluster/spec/testdata/topology_err.yaml +++ b/pkg/cluster/spec/testdata/topology_err.yaml @@ -15,16 +15,6 @@ monitored: # data_dir: "/tidb/data/monitored-9100" # log_dir: "/tidb/deploy/monitored-9100/log" -# # Server configs are used to specify the runtime configuration of TiDB components -# # All configuration items can be found in TiDB docs: -# # - TiDB: https://pingcap.com/docs/stable/reference/configuration/tidb-server/configuration-file/ -# # - TiKV: https://pingcap.com/docs/stable/reference/configuration/tikv-server/configuration-file/ -# # - PD: https://pingcap.com/docs/stable/reference/configuration/pd-server/configuration-file/ -# # All configuration items use points to represent the hierarchy, e.g: -# # readpool.storage.use-unified-pool -# # ^ ^ -# # You can overwrite this configuration via instance-level `config` field - server_configs: tidb: log.slow-threshold: 300 @@ -101,7 +91,6 @@ tikv_servers: # - host: 10.0.1.10 # ssh_port: 22 # tcp_port: 9000 -# http_port: 8123 # flash_service_port: 3930 # flash_proxy_port: 20170 # flash_proxy_status_port: 20292 diff --git a/pkg/cluster/spec/tidb.go b/pkg/cluster/spec/tidb.go index 9581721904..3d793f9a76 100644 --- a/pkg/cluster/spec/tidb.go +++ b/pkg/cluster/spec/tidb.go @@ -32,6 +32,7 @@ import ( // TiDBSpec represents the TiDB topology specification in topology.yaml type TiDBSpec struct { Host string `yaml:"host"` + ManageHost string `yaml:"manage_host,omitempty" validate:"manage_host:editable"` ListenHost string `yaml:"listen_host,omitempty"` AdvertiseAddr string `yaml:"advertise_address,omitempty"` SSHPort int `yaml:"ssh_port,omitempty" validate:"ssh_port:editable"` @@ -42,6 +43,7 @@ type TiDBSpec struct { StatusPort int `yaml:"status_port" default:"10080"` DeployDir string `yaml:"deploy_dir,omitempty"` LogDir string `yaml:"log_dir,omitempty"` + Source string `yaml:"source,omitempty" validate:"source:editable"` NumaNode string `yaml:"numa_node,omitempty" validate:"numa_node:editable"` NumaCores string `yaml:"numa_cores,omitempty" validate:"numa_cores:editable"` Config map[string]any `yaml:"config,omitempty" validate:"config:ignore"` @@ -57,7 +59,11 @@ func (s *TiDBSpec) Role() string { // SSH returns the host and SSH port of the instance func (s *TiDBSpec) SSH() (string, int) { - return s.Host, s.SSHPort + host := s.Host + if s.ManageHost != "" { + host = s.ManageHost + } + return host, s.SSHPort } // GetMainPort returns the main port of the instance @@ -65,6 +71,14 @@ func (s *TiDBSpec) GetMainPort() int { return s.Port } +// GetManageHost returns the manage host of the instance +func (s *TiDBSpec) GetManageHost() string { + if s.ManageHost != "" { + return s.ManageHost + } + return s.Host +} + // IsImported returns if the node is imported from TiDB-Ansible func (s *TiDBSpec) IsImported() bool { return s.Imported @@ -75,6 +89,14 @@ func (s *TiDBSpec) IgnoreMonitorAgent() bool { return s.IgnoreExporter } +// GetSource returns source to download the component +func (s *TiDBSpec) GetSource() string { + if s.Source == "" { + return ComponentTiDB + } + return s.Source +} + // TiDBComponent represents TiDB component. type TiDBComponent struct{ Topology *Specification } @@ -88,6 +110,20 @@ func (c *TiDBComponent) Role() string { return ComponentTiDB } +// CalculateVersion implements the Component interface +func (c *TiDBComponent) CalculateVersion(clusterVersion string) string { + version := c.Topology.ComponentVersions.TiDB + if version == "" { + version = clusterVersion + } + return version +} + +// SetVersion implements Component interface. +func (c *TiDBComponent) SetVersion(version string) { + c.Topology.ComponentVersions.TiDB = version +} + // Instances implements Component interface. func (c *TiDBComponent) Instances() []Instance { ins := make([]Instance, 0, len(c.Topology.TiDBServers)) @@ -97,9 +133,13 @@ func (c *TiDBComponent) Instances() []Instance { InstanceSpec: s, Name: c.Name(), Host: s.Host, - ListenHost: s.ListenHost, + ManageHost: s.ManageHost, + ListenHost: utils.Ternary(s.ListenHost != "", s.ListenHost, c.Topology.BaseTopo().GlobalOptions.ListenHost).(string), Port: s.Port, SSHP: s.SSHPort, + Source: s.Source, + NumaNode: s.NumaNode, + NumaCores: s.NumaCores, Ports: []int{ s.Port, @@ -109,11 +149,12 @@ func (c *TiDBComponent) Instances() []Instance { s.DeployDir, }, StatusFn: func(_ context.Context, timeout time.Duration, tlsCfg *tls.Config, _ ...string) string { - return statusByHost(s.Host, s.StatusPort, "/status", timeout, tlsCfg) + return statusByHost(s.GetManageHost(), s.StatusPort, "/status", timeout, tlsCfg) }, UptimeFn: func(_ context.Context, timeout time.Duration, tlsCfg *tls.Config) time.Duration { - return UptimeByHost(s.Host, s.StatusPort, timeout, tlsCfg) + return UptimeByHost(s.GetManageHost(), s.StatusPort, timeout, tlsCfg) }, + Component: c, }, c.Topology}) } return ins @@ -141,6 +182,7 @@ func (i *TiDBInstance) InitConfig( enableTLS := topo.GlobalOptions.TLSEnabled spec := i.InstanceSpec.(*TiDBSpec) + version := i.CalculateVersion(clusterVersion) pds := []string{} for _, pdspec := range topo.PDServers { @@ -152,7 +194,7 @@ func (i *TiDBInstance) InitConfig( ListenHost: i.GetListenHost(), AdvertiseAddr: utils.Ternary(spec.AdvertiseAddr != "", spec.AdvertiseAddr, spec.Host).(string), PD: strings.Join(pds, ","), - SupportSecboot: tidbver.TiDBSupportSecureBoot(clusterVersion), + SupportSecboot: tidbver.TiDBSupportSecureBoot(version), DeployDir: paths.Deploy, LogDir: paths.Log, @@ -208,7 +250,7 @@ func (i *TiDBInstance) InitConfig( return err } - return checkConfig(ctx, e, i.ComponentName(), clusterVersion, i.OS(), i.Arch(), i.ComponentName()+".toml", paths, nil) + return checkConfig(ctx, e, i.ComponentName(), i.ComponentSource(), version, i.OS(), i.Arch(), i.ComponentName()+".toml", paths) } // setTLSConfig set TLS Config to support enable/disable TLS diff --git a/pkg/cluster/spec/tiflash.go b/pkg/cluster/spec/tiflash.go index d3ccfa1e8e..45ed2e8dd0 100644 --- a/pkg/cluster/spec/tiflash.go +++ b/pkg/cluster/spec/tiflash.go @@ -41,12 +41,13 @@ import ( // TiFlashSpec represents the TiFlash topology specification in topology.yaml type TiFlashSpec struct { Host string `yaml:"host"` + ManageHost string `yaml:"manage_host,omitempty" validate:"manage_host:editable"` SSHPort int `yaml:"ssh_port,omitempty" validate:"ssh_port:editable"` Imported bool `yaml:"imported,omitempty"` Patched bool `yaml:"patched,omitempty"` IgnoreExporter bool `yaml:"ignore_exporter,omitempty"` TCPPort int `yaml:"tcp_port" default:"9000"` - HTTPPort int `yaml:"http_port" default:"8123"` + HTTPPort int `yaml:"http_port" default:"8123"` // Deprecated since v7.1.0 FlashServicePort int `yaml:"flash_service_port" default:"3930"` FlashProxyPort int `yaml:"flash_proxy_port" default:"20170"` FlashProxyStatusPort int `yaml:"flash_proxy_status_port" default:"20292"` @@ -56,6 +57,7 @@ type TiFlashSpec struct { LogDir string `yaml:"log_dir,omitempty"` TmpDir string `yaml:"tmp_path,omitempty"` Offline bool `yaml:"offline,omitempty"` + Source string `yaml:"source,omitempty" validate:"source:editable"` NumaNode string `yaml:"numa_node,omitempty" validate:"numa_node:editable"` NumaCores string `yaml:"numa_cores,omitempty" validate:"numa_cores:editable"` Config map[string]any `yaml:"config,omitempty" validate:"config:ignore"` @@ -75,6 +77,54 @@ func (s *TiFlashSpec) Status(ctx context.Context, timeout time.Duration, tlsCfg return state } +const ( + // EngineLabelKey is the label that indicates the backend of store instance: + // tikv or tiflash. TiFlash instance will contain a label of 'engine: tiflash'. + EngineLabelKey = "engine" + // EngineLabelTiFlash is the label value, which a TiFlash instance will have with + // a label key of EngineLabelKey. + EngineLabelTiFlash = "tiflash" + // EngineLabelTiFlashCompute is for disaggregated tiflash mode, + // it's the lable of tiflash_compute nodes. + EngineLabelTiFlashCompute = "tiflash_compute" + // EngineRoleLabelKey is the label that indicates if the TiFlash instance is a write node. + EngineRoleLabelKey = "engine_role" + // EngineRoleLabelWrite is for disaggregated tiflash write node. + EngineRoleLabelWrite = "write" +) + +// GetExtendedRole get extended name for TiFlash to distinguish disaggregated mode. +func (s *TiFlashSpec) GetExtendedRole(ctx context.Context, tlsCfg *tls.Config, pdList ...string) string { + if len(pdList) < 1 { + return "" + } + storeAddr := utils.JoinHostPort(s.Host, s.FlashServicePort) + pdapi := api.NewPDClient(ctx, pdList, statusQueryTimeout, tlsCfg) + store, err := pdapi.GetCurrentStore(storeAddr) + if err != nil { + return "" + } + isWriteNode := false + isTiFlash := false + for _, label := range store.Store.Labels { + if label.Key == EngineLabelKey { + if label.Value == EngineLabelTiFlashCompute { + return " (compute)" + } + if label.Value == EngineLabelTiFlash { + isTiFlash = true + } + } + if label.Key == EngineRoleLabelKey && label.Value == EngineRoleLabelWrite { + isWriteNode = true + } + if isTiFlash && isWriteNode { + return " (write)" + } + } + return "" +} + // Role returns the component role of the instance func (s *TiFlashSpec) Role() string { return ComponentTiFlash @@ -82,7 +132,11 @@ func (s *TiFlashSpec) Role() string { // SSH returns the host and SSH port of the instance func (s *TiFlashSpec) SSH() (string, int) { - return s.Host, s.SSHPort + host := s.Host + if s.ManageHost != "" { + host = s.ManageHost + } + return host, s.SSHPort } // GetMainPort returns the main port of the instance @@ -90,6 +144,14 @@ func (s *TiFlashSpec) GetMainPort() int { return s.TCPPort } +// GetManageHost returns the manage host of the instance +func (s *TiFlashSpec) GetManageHost() string { + if s.ManageHost != "" { + return s.ManageHost + } + return s.Host +} + // IsImported returns if the node is imported from TiDB-Ansible func (s *TiFlashSpec) IsImported() bool { return s.Imported @@ -100,11 +162,20 @@ func (s *TiFlashSpec) IgnoreMonitorAgent() bool { return s.IgnoreExporter } +// GetSource returns source to download the component +func (s *TiFlashSpec) GetSource() string { + if s.Source == "" { + return ComponentTiFlash + } + return s.Source +} + // key names for storage config const ( TiFlashStorageKeyMainDirs string = "storage.main.dir" TiFlashStorageKeyLatestDirs string = "storage.latest.dir" TiFlashStorageKeyRaftDirs string = "storage.raft.dir" + TiFlashRemoteCacheDir string = "storage.remote.cache.dir" TiFlashRequiredCPUFlags string = "avx2 popcnt movbe" ) @@ -211,6 +282,20 @@ func (c *TiFlashComponent) Role() string { return ComponentTiFlash } +// CalculateVersion implements the Component interface +func (c *TiFlashComponent) CalculateVersion(clusterVersion string) string { + version := c.Topology.ComponentVersions.TiFlash + if version == "" { + version = clusterVersion + } + return version +} + +// SetVersion implements Component interface. +func (c *TiFlashComponent) SetVersion(version string) { + c.Topology.ComponentVersions.TiFlash = version +} + // Instances implements Component interface. func (c *TiFlashComponent) Instances() []Instance { ins := make([]Instance, 0, len(c.Topology.TiFlashServers)) @@ -219,8 +304,13 @@ func (c *TiFlashComponent) Instances() []Instance { InstanceSpec: s, Name: c.Name(), Host: s.Host, + ManageHost: s.ManageHost, + ListenHost: c.Topology.BaseTopo().GlobalOptions.ListenHost, Port: s.GetMainPort(), SSHP: s.SSHPort, + Source: s.GetSource(), + NumaNode: s.NumaNode, + NumaCores: s.NumaCores, Ports: []int{ s.TCPPort, @@ -236,8 +326,9 @@ func (c *TiFlashComponent) Instances() []Instance { }, StatusFn: s.Status, UptimeFn: func(_ context.Context, timeout time.Duration, tlsCfg *tls.Config) time.Duration { - return 0 + return UptimeByHost(s.GetManageHost(), s.StatusPort, timeout, tlsCfg) }, + Component: c, }, c.Topology}) } return ins @@ -308,30 +399,21 @@ func isValidStringArray(key string, config map[string]any, couldEmpty bool) (boo return isKeyDefined, fmt.Errorf("'%s' should be a non-empty string array, please check the tiflash configuration in your yaml file", TiFlashStorageKeyMainDirs) } -// checkTiFlashStorageConfig detect the "storage" section in `config` -// is valid or not. +// checkTiFlashStorageConfig ensures `storage.main` is defined when +// `storage.latest` or `storage.raft` is used. func checkTiFlashStorageConfig(config map[string]any) (bool, error) { - var ( - isStorageDirsDefined bool - err error - ) - if isStorageDirsDefined, err = isValidStringArray(TiFlashStorageKeyMainDirs, config, false); err != nil { - return isStorageDirsDefined, err - } - if !isStorageDirsDefined { - containsStorageSectionKey := func(config map[string]any) (string, bool) { - for k := range config { - if strings.HasPrefix(k, "storage.") { - return k, true - } + isMainStorageDefined, err := isValidStringArray(TiFlashStorageKeyMainDirs, config, false) + if err != nil { + return false, err + } + if !isMainStorageDefined { + for k := range config { + if strings.HasPrefix(k, "storage.latest") || strings.HasPrefix(k, "storage.raft") { + return false, fmt.Errorf("You must set '%s' before setting '%s', please check the tiflash configuration in your yaml file", TiFlashStorageKeyMainDirs, k) } - return "", false - } - if key, contains := containsStorageSectionKey(config); contains { - return isStorageDirsDefined, fmt.Errorf("You must set '%s' before setting '%s', please check the tiflash configuration in your yaml file", TiFlashStorageKeyMainDirs, key) } } - return isStorageDirsDefined, nil + return isMainStorageDefined, nil } // CheckIncorrectConfigs checks incorrect settings @@ -371,15 +453,16 @@ func checkTiFlashStorageConfigWithVersion(clusterVersion string, config map[stri } // InitTiFlashConfig initializes TiFlash config file with the configurations in server_configs -func (i *TiFlashInstance) initTiFlashConfig(ctx context.Context, clusterVersion string, src map[string]any, paths meta.DirPaths) (map[string]any, error) { +func (i *TiFlashInstance) initTiFlashConfig(ctx context.Context, version string, src map[string]any, paths meta.DirPaths) (map[string]any, error) { var ( pathConfig string isStorageDirsDefined bool deprecatedUsersConfig string daemonConfig string + markCacheSize string err error ) - if isStorageDirsDefined, err = checkTiFlashStorageConfigWithVersion(clusterVersion, src); err != nil { + if isStorageDirsDefined, err = checkTiFlashStorageConfigWithVersion(version, src); err != nil { return nil, err } // For backward compatibility, we need to rollback to set 'path' @@ -389,7 +472,7 @@ func (i *TiFlashInstance) initTiFlashConfig(ctx context.Context, clusterVersion pathConfig = fmt.Sprintf(`path: "%s"`, strings.Join(paths.Data, ",")) } - if tidbver.TiFlashDeprecatedUsersConfig(clusterVersion) { + if tidbver.TiFlashDeprecatedUsersConfig(version) { // For v4.0.12 or later, 5.0.0 or later, TiFlash can ignore these `user.*`, `quotas.*` settings deprecatedUsersConfig = "#" } else { @@ -415,18 +498,28 @@ func (i *TiFlashInstance) initTiFlashConfig(ctx context.Context, clusterVersion ` } - spec := i.InstanceSpec.(*TiFlashSpec) - port := "http_port" - enableTLS := i.topo.(*Specification).GlobalOptions.TLSEnabled - tidbStatusAddrs := []string{} for _, tidb := range i.topo.(*Specification).TiDBServers { tidbStatusAddrs = append(tidbStatusAddrs, utils.JoinHostPort(tidb.Host, tidb.StatusPort)) } - if enableTLS { - port = "https_port" + spec := i.InstanceSpec.(*TiFlashSpec) + enableTLS := i.topo.(*Specification).GlobalOptions.TLSEnabled + httpPort := "#" + // For 7.1.0 or later, TiFlash HTTP service is removed, so we don't need to set http_port + if !tidbver.TiFlashNotNeedHTTPPortConfig(version) { + if enableTLS { + httpPort = fmt.Sprintf(`https_port: %d`, spec.HTTPPort) + } else { + httpPort = fmt.Sprintf(`http_port: %d`, spec.HTTPPort) + } } + tcpPort := "#" + // Config tcp_port is only required for TiFlash version < 7.1.0, and is recommended to not specify for TiFlash version >= 7.1.0. + if tidbver.TiFlashRequiresTCPPortConfig(version) { + tcpPort = fmt.Sprintf(`tcp_port: %d`, spec.TCPPort) + } + // set TLS configs spec.Config, err = i.setTLSConfig(ctx, enableTLS, spec.Config, paths) if err != nil { @@ -435,11 +528,13 @@ func (i *TiFlashInstance) initTiFlashConfig(ctx context.Context, clusterVersion topo := Specification{} - if tidbver.TiFlashNotNeedSomeConfig(clusterVersion) { - // For 5.4.0 or later, TiFlash can ignore application.runAsDaemon setting + if tidbver.TiFlashNotNeedSomeConfig(version) { + // For 5.4.0 or later, TiFlash can ignore application.runAsDaemon and mark_cache_size setting daemonConfig = "#" + markCacheSize = "#" } else { daemonConfig = `application.runAsDaemon: true` + markCacheSize = `mark_cache_size: 5368709120` } err = yaml.Unmarshal([]byte(fmt.Sprintf(` server_configs: @@ -447,11 +542,10 @@ server_configs: default_profile: "default" display_name: "TiFlash" listen_host: "%[7]s" - mark_cache_size: 5368709120 tmp_path: "%[11]s" %[1]s - tcp_port: %[3]d - `+port+`: %[4]d + %[3]s + %[4]s flash.tidb_status_addr: "%[5]s" flash.service_addr: "%[6]s" flash.flash_cluster.cluster_manager_path: "%[10]s/bin/tiflash/flash_cluster_manager" @@ -470,20 +564,22 @@ server_configs: raft.pd_addr: "%[9]s" profiles.default.max_memory_usage: 0 %[12]s + %[14]s `, pathConfig, paths.Log, - spec.TCPPort, - spec.HTTPPort, + tcpPort, + httpPort, strings.Join(tidbStatusAddrs, ","), utils.JoinHostPort(spec.Host, spec.FlashServicePort), i.GetListenHost(), spec.StatusPort, - strings.Join(i.getEndpoints(i.topo), ","), + strings.Join(i.topo.(*Specification).GetPDList(), ","), paths.Deploy, fmt.Sprintf("%s/tmp", paths.Data[0]), deprecatedUsersConfig, daemonConfig, + markCacheSize, )), &topo) if err != nil { @@ -664,9 +760,10 @@ func (i *TiFlashInstance) InitConfig( return err } spec := i.InstanceSpec.(*TiFlashSpec) + version := i.CalculateVersion(clusterVersion) cfg := &scripts.TiFlashScript{ - RequiredCPUFlags: getTiFlashRequiredCPUFlagsWithVersion(clusterVersion, spec.Arch), + RequiredCPUFlags: getTiFlashRequiredCPUFlagsWithVersion(version, spec.Arch), DeployDir: paths.Deploy, LogDir: paths.Log, @@ -689,7 +786,7 @@ func (i *TiFlashInstance) InitConfig( return err } - conf, err := i.InitTiFlashLearnerConfig(ctx, clusterVersion, topo.ServerConfigs.TiFlashLearner, paths) + conf, err := i.InitTiFlashLearnerConfig(ctx, version, topo.ServerConfigs.TiFlashLearner, paths) if err != nil { return err } @@ -722,7 +819,7 @@ func (i *TiFlashInstance) InitConfig( } // Init the configuration using cfg and server_configs - if conf, err = i.initTiFlashConfig(ctx, clusterVersion, topo.ServerConfigs.TiFlash, paths); err != nil { + if conf, err = i.initTiFlashConfig(ctx, version, topo.ServerConfigs.TiFlash, paths); err != nil { return err } @@ -753,7 +850,7 @@ func (i *TiFlashInstance) InitConfig( } // Check the configuration of instance level - if conf, err = i.mergeTiFlashInstanceConfig(clusterVersion, conf, spec.Config); err != nil { + if conf, err = i.mergeTiFlashInstanceConfig(version, conf, spec.Config); err != nil { return err } @@ -782,14 +879,6 @@ type replicateConfig struct { EnablePlacementRules string `json:"enable-placement-rules"` } -func (i *TiFlashInstance) getEndpoints(topo Topology) []string { - var endpoints []string - for _, pd := range topo.(*Specification).PDServers { - endpoints = append(endpoints, utils.JoinHostPort(pd.Host, pd.ClientPort)) - } - return endpoints -} - // PrepareStart checks TiFlash requirements before starting func (i *TiFlashInstance) PrepareStart(ctx context.Context, tlsCfg *tls.Config) error { // set enable-placement-rules to true via PDClient @@ -812,7 +901,7 @@ func (i *TiFlashInstance) PrepareStart(ctx context.Context, tlsCfg *tls.Config) topo = i.topo } - endpoints := i.getEndpoints(topo) + endpoints := topo.(*Specification).GetPDListWithManageHost() pdClient := api.NewPDClient(ctx, endpoints, 10*time.Second, tlsCfg) return pdClient.UpdateReplicateConfig(bytes.NewBuffer(enablePlacementRules)) } @@ -821,7 +910,7 @@ func (i *TiFlashInstance) PrepareStart(ctx context.Context, tlsCfg *tls.Config) func (i *TiFlashInstance) Ready(ctx context.Context, e ctxt.Executor, timeout uint64, tlsCfg *tls.Config) error { // FIXME: the timeout is applied twice in the whole `Ready()` process, in the worst // case it might wait double time as other components - if err := PortStarted(ctx, e, i.Port, timeout); err != nil { + if err := PortStarted(ctx, e, i.GetServicePort(), timeout); err != nil { return err } @@ -829,7 +918,7 @@ func (i *TiFlashInstance) Ready(ctx context.Context, e ctxt.Executor, timeout ui if i.topo.BaseTopo().GlobalOptions.TLSEnabled { scheme = "https" } - addr := fmt.Sprintf("%s://%s/tiflash/store-status", scheme, utils.JoinHostPort(i.Host, i.GetStatusPort())) + addr := fmt.Sprintf("%s://%s/tiflash/store-status", scheme, utils.JoinHostPort(i.GetManageHost(), i.GetStatusPort())) req, err := http.NewRequest("GET", addr, nil) if err != nil { return err diff --git a/pkg/cluster/spec/tikv.go b/pkg/cluster/spec/tikv.go index fcc6278076..48f24dcf38 100644 --- a/pkg/cluster/spec/tikv.go +++ b/pkg/cluster/spec/tikv.go @@ -47,6 +47,7 @@ const ( // TiKVSpec represents the TiKV topology specification in topology.yaml type TiKVSpec struct { Host string `yaml:"host"` + ManageHost string `yaml:"manage_host,omitempty" validate:"manage_host:editable"` ListenHost string `yaml:"listen_host,omitempty"` AdvertiseAddr string `yaml:"advertise_addr,omitempty"` SSHPort int `yaml:"ssh_port,omitempty" validate:"ssh_port:editable"` @@ -60,6 +61,7 @@ type TiKVSpec struct { DataDir string `yaml:"data_dir,omitempty"` LogDir string `yaml:"log_dir,omitempty"` Offline bool `yaml:"offline,omitempty"` + Source string `yaml:"source,omitempty" validate:"source:editable"` NumaNode string `yaml:"numa_node,omitempty" validate:"numa_node:editable"` NumaCores string `yaml:"numa_cores,omitempty" validate:"numa_cores:editable"` Config map[string]any `yaml:"config,omitempty" validate:"config:ignore"` @@ -102,7 +104,11 @@ func (s *TiKVSpec) Role() string { // SSH returns the host and SSH port of the instance func (s *TiKVSpec) SSH() (string, int) { - return s.Host, s.SSHPort + host := s.Host + if s.ManageHost != "" { + host = s.ManageHost + } + return host, s.SSHPort } // GetMainPort returns the main port of the instance @@ -110,6 +116,14 @@ func (s *TiKVSpec) GetMainPort() int { return s.Port } +// GetManageHost returns the manage host of the instance +func (s *TiKVSpec) GetManageHost() string { + if s.ManageHost != "" { + return s.ManageHost + } + return s.Host +} + // IsImported returns if the node is imported from TiDB-Ansible func (s *TiKVSpec) IsImported() bool { return s.Imported @@ -150,6 +164,14 @@ func (s *TiKVSpec) Labels() (map[string]string, error) { return lbs, nil } +// GetSource returns source to download the component +func (s *TiKVSpec) GetSource() string { + if s.Source == "" { + return ComponentTiKV + } + return s.Source +} + // TiKVComponent represents TiKV component. type TiKVComponent struct{ Topology *Specification } @@ -163,6 +185,20 @@ func (c *TiKVComponent) Role() string { return ComponentTiKV } +// CalculateVersion implements the Component interface +func (c *TiKVComponent) CalculateVersion(clusterVersion string) string { + version := c.Topology.ComponentVersions.TiKV + if version == "" { + version = clusterVersion + } + return version +} + +// SetVersion implements Component interface. +func (c *TiKVComponent) SetVersion(version string) { + c.Topology.ComponentVersions.TiKV = version +} + // Instances implements Component interface. func (c *TiKVComponent) Instances() []Instance { ins := make([]Instance, 0, len(c.Topology.TiKVServers)) @@ -172,9 +208,13 @@ func (c *TiKVComponent) Instances() []Instance { InstanceSpec: s, Name: c.Name(), Host: s.Host, - ListenHost: s.ListenHost, + ManageHost: s.ManageHost, + ListenHost: utils.Ternary(s.ListenHost != "", s.ListenHost, c.Topology.BaseTopo().GlobalOptions.ListenHost).(string), Port: s.Port, SSHP: s.SSHPort, + Source: s.Source, + NumaNode: s.NumaNode, + NumaCores: s.NumaCores, Ports: []int{ s.Port, @@ -186,8 +226,9 @@ func (c *TiKVComponent) Instances() []Instance { }, StatusFn: s.Status, UptimeFn: func(_ context.Context, timeout time.Duration, tlsCfg *tls.Config) time.Duration { - return UptimeByHost(s.Host, s.StatusPort, timeout, tlsCfg) + return UptimeByHost(s.GetManageHost(), s.StatusPort, timeout, tlsCfg) }, + Component: c, }, c.Topology, 0}) } return ins @@ -285,7 +326,7 @@ func (i *TiKVInstance) InitConfig( return err } - return checkConfig(ctx, e, i.ComponentName(), clusterVersion, i.OS(), i.Arch(), i.ComponentName()+".toml", paths, nil) + return checkConfig(ctx, e, i.ComponentName(), i.ComponentSource(), clusterVersion, i.OS(), i.Arch(), i.ComponentName()+".toml", paths) } // setTLSConfig set TLS Config to support enable/disable TLS @@ -361,7 +402,7 @@ func (i *TiKVInstance) PreRestart(ctx context.Context, topo Topology, apiTimeout return nil } - pdClient := api.NewPDClient(ctx, tidbTopo.GetPDList(), 5*time.Second, tlsCfg) + pdClient := api.NewPDClient(ctx, tidbTopo.GetPDListWithManageHost(), 5*time.Second, tlsCfg) // Make sure there's leader of PD. // Although we evict pd leader when restart pd, @@ -399,7 +440,7 @@ func (i *TiKVInstance) PostRestart(ctx context.Context, topo Topology, tlsCfg *t return nil } - pdClient := api.NewPDClient(ctx, tidbTopo.GetPDList(), 5*time.Second, tlsCfg) + pdClient := api.NewPDClient(ctx, tidbTopo.GetPDListWithManageHost(), 5*time.Second, tlsCfg) // remove store leader evict scheduler after restart if err := pdClient.RemoveStoreEvict(addr(i.InstanceSpec.(*TiKVSpec))); err != nil { @@ -437,7 +478,7 @@ func genLeaderCounter(topo *Specification, tlsCfg *tls.Config) func(string) (int for _, kv := range topo.TiKVServers { kvid := utils.JoinHostPort(kv.Host, kv.Port) if id == kvid { - statusAddress = utils.JoinHostPort(kv.Host, kv.StatusPort) + statusAddress = utils.JoinHostPort(kv.GetManageHost(), kv.StatusPort) break } foundIds = append(foundIds, kvid) diff --git a/pkg/cluster/spec/tikv_cdc.go b/pkg/cluster/spec/tikv_cdc.go index 39da21c692..ee9d6c5c5d 100644 --- a/pkg/cluster/spec/tikv_cdc.go +++ b/pkg/cluster/spec/tikv_cdc.go @@ -34,6 +34,7 @@ import ( // TiKVCDCSpec represents the TiKVCDC topology specification in topology.yaml type TiKVCDCSpec struct { Host string `yaml:"host"` + ManageHost string `yaml:"manage_host,omitempty" validate:"manage_host:editable"` SSHPort int `yaml:"ssh_port,omitempty" validate:"ssh_port:editable"` Imported bool `yaml:"imported,omitempty"` Patched bool `yaml:"patched,omitempty"` @@ -45,6 +46,7 @@ type TiKVCDCSpec struct { Offline bool `yaml:"offline,omitempty"` GCTTL int64 `yaml:"gc-ttl,omitempty" validate:"gc-ttl:editable"` TZ string `yaml:"tz,omitempty" validate:"tz:editable"` + Source string `yaml:"source,omitempty" validate:"source:editable"` NumaNode string `yaml:"numa_node,omitempty" validate:"numa_node:editable"` Config map[string]any `yaml:"config,omitempty" validate:"config:ignore"` ResourceControl meta.ResourceControl `yaml:"resource_control,omitempty" validate:"resource_control:editable"` @@ -59,7 +61,11 @@ func (s *TiKVCDCSpec) Role() string { // SSH returns the host and SSH port of the instance func (s *TiKVCDCSpec) SSH() (string, int) { - return s.Host, s.SSHPort + host := s.Host + if s.ManageHost != "" { + host = s.ManageHost + } + return host, s.SSHPort } // GetMainPort returns the main port of the instance @@ -67,6 +73,14 @@ func (s *TiKVCDCSpec) GetMainPort() int { return s.Port } +// GetManageHost returns the manage host of the instance +func (s *TiKVCDCSpec) GetManageHost() string { + if s.ManageHost != "" { + return s.ManageHost + } + return s.Host +} + // IsImported returns if the node is imported from TiDB-Ansible func (s *TiKVCDCSpec) IsImported() bool { // TiDB-Ansible do not support TiKV-CDC @@ -91,6 +105,26 @@ func (c *TiKVCDCComponent) Role() string { return ComponentTiKVCDC } +// CalculateVersion implements the Component interface +func (c *TiKVCDCComponent) CalculateVersion(clusterVersion string) string { + // always not follow global version, use ""(latest) by default + version := c.Topology.ComponentVersions.TiKVCDC + return version +} + +// SetVersion implements Component interface. +func (c *TiKVCDCComponent) SetVersion(version string) { + c.Topology.ComponentVersions.TiKVCDC = version +} + +// GetSource returns source to download the component +func (s *TiKVCDCSpec) GetSource() string { + if s.Source == "" { + return ComponentTiKVCDC + } + return s.Source +} + // Instances implements Component interface. func (c *TiKVCDCComponent) Instances() []Instance { ins := make([]Instance, 0, len(c.Topology.TiKVCDCServers)) @@ -100,8 +134,13 @@ func (c *TiKVCDCComponent) Instances() []Instance { InstanceSpec: s, Name: c.Name(), Host: s.Host, + ManageHost: s.ManageHost, + ListenHost: c.Topology.BaseTopo().GlobalOptions.ListenHost, Port: s.Port, SSHP: s.SSHPort, + Source: s.GetSource(), + NumaNode: s.NumaNode, + NumaCores: "", Ports: []int{ s.Port, @@ -110,11 +149,12 @@ func (c *TiKVCDCComponent) Instances() []Instance { s.DeployDir, }, StatusFn: func(_ context.Context, timeout time.Duration, tlsCfg *tls.Config, _ ...string) string { - return statusByHost(s.Host, s.Port, "/status", timeout, tlsCfg) + return statusByHost(s.GetManageHost(), s.Port, "/status", timeout, tlsCfg) }, UptimeFn: func(_ context.Context, timeout time.Duration, tlsCfg *tls.Config) time.Duration { - return UptimeByHost(s.Host, s.Port, timeout, tlsCfg) + return UptimeByHost(s.GetManageHost(), s.Port, timeout, tlsCfg) }, + Component: c, }, c.Topology} if s.DataDir != "" { instance.Dirs = append(instance.Dirs, s.DataDir) @@ -220,11 +260,6 @@ func (i *TiKVCDCInstance) setTLSConfig(ctx context.Context, enableTLS bool, conf var _ RollingUpdateInstance = &TiKVCDCInstance{} -// GetAddr return the address of this TiKV-CDC instance -func (i *TiKVCDCInstance) GetAddr() string { - return utils.JoinHostPort(i.GetHost(), i.GetPort()) -} - // PreRestart implements RollingUpdateInstance interface. // All errors are ignored, to trigger hard restart. func (i *TiKVCDCInstance) PreRestart(ctx context.Context, topo Topology, apiTimeoutSeconds int, tlsCfg *tls.Config) error { @@ -238,7 +273,7 @@ func (i *TiKVCDCInstance) PreRestart(ctx context.Context, topo Topology, apiTime panic("logger not found") } - address := i.GetAddr() + address := utils.JoinHostPort(i.GetHost(), i.GetPort()) // cdc rolling upgrade strategy only works if there are more than 2 captures if len(tidbTopo.TiKVCDCServers) <= 1 { logger.Debugf("tikv-cdc pre-restart skipped, only one capture in the topology, addr: %s", address) @@ -246,7 +281,7 @@ func (i *TiKVCDCInstance) PreRestart(ctx context.Context, topo Topology, apiTime } start := time.Now() - client := api.NewTiKVCDCOpenAPIClient(ctx, []string{address}, 5*time.Second, tlsCfg) + client := api.NewTiKVCDCOpenAPIClient(ctx, []string{utils.JoinHostPort(i.GetManageHost(), i.GetPort())}, 5*time.Second, tlsCfg) captures, err := client.GetAllCaptures() if err != nil { logger.Debugf("tikv-cdc pre-restart skipped, cannot get all captures, trigger hard restart, addr: %s, elapsed: %+v", address, time.Since(start)) @@ -297,9 +332,9 @@ func (i *TiKVCDCInstance) PostRestart(ctx context.Context, topo Topology, tlsCfg } start := time.Now() - address := i.GetAddr() + address := utils.JoinHostPort(i.GetHost(), i.GetPort()) - client := api.NewTiKVCDCOpenAPIClient(ctx, []string{address}, 5*time.Second, tlsCfg) + client := api.NewTiKVCDCOpenAPIClient(ctx, []string{utils.JoinHostPort(i.GetManageHost(), i.GetPort())}, 5*time.Second, tlsCfg) err := client.IsCaptureAlive() if err != nil { logger.Debugf("tikv-cdc post-restart finished, get capture status failed, addr: %s, err: %+v, elapsed: %+v", address, err, time.Since(start)) diff --git a/pkg/cluster/spec/tiproxy.go b/pkg/cluster/spec/tiproxy.go new file mode 100644 index 0000000000..15d167ef94 --- /dev/null +++ b/pkg/cluster/spec/tiproxy.go @@ -0,0 +1,328 @@ +// Copyright 2023 PingCAP, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// See the License for the specific language governing permissions and +// limitations under the License. + +package spec + +import ( + "bytes" + "context" + "crypto/tls" + "fmt" + "path/filepath" + "strings" + "time" + + "github.com/pingcap/tiup/pkg/cluster/ctxt" + "github.com/pingcap/tiup/pkg/cluster/template/scripts" + "github.com/pingcap/tiup/pkg/meta" + "github.com/pingcap/tiup/pkg/utils" + "github.com/prometheus/common/expfmt" +) + +func proxyUptimeByHost(host string, port int, timeout time.Duration, tlsCfg *tls.Config) time.Duration { + if timeout < time.Second { + timeout = statusQueryTimeout + } + + scheme := "http" + if tlsCfg != nil { + scheme = "https" + } + url := fmt.Sprintf("%s://%s/api/metrics", scheme, utils.JoinHostPort(host, port)) + + client := utils.NewHTTPClient(timeout, tlsCfg) + + body, err := client.Get(context.TODO(), url) + if err != nil || body == nil { + return 0 + } + + var parser expfmt.TextParser + reader := bytes.NewReader(body) + mf, err := parser.TextToMetricFamilies(reader) + if err != nil { + return 0 + } + + now := time.Now() + for k, v := range mf { + if k == promMetricStartTimeSeconds { + ms := v.GetMetric() + if len(ms) >= 1 { + startTime := ms[0].Gauge.GetValue() + return now.Sub(time.Unix(int64(startTime), 0)) + } + return 0 + } + } + + return 0 +} + +// TiProxySpec represents the TiProxy topology specification in topology.yaml +type TiProxySpec struct { + Host string `yaml:"host"` + ManageHost string `yaml:"manage_host,omitempty" validate:"manage_host:editable"` + SSHPort int `yaml:"ssh_port,omitempty" validate:"ssh_port:editable"` + Port int `yaml:"port" default:"6000"` + StatusPort int `yaml:"status_port" default:"3080"` + DeployDir string `yaml:"deploy_dir,omitempty"` + NumaNode string `yaml:"numa_node,omitempty" validate:"numa_node:editable"` + Config map[string]any `yaml:"config,omitempty" validate:"config:ignore"` + Arch string `yaml:"arch,omitempty"` + OS string `yaml:"os,omitempty"` +} + +// Role returns the component role of the instance +func (s *TiProxySpec) Role() string { + return ComponentTiProxy +} + +// SSH returns the host and SSH port of the instance +func (s *TiProxySpec) SSH() (string, int) { + return s.Host, s.SSHPort +} + +// GetMainPort returns the main port of the instance +func (s *TiProxySpec) GetMainPort() int { + return s.Port +} + +// GetManageHost returns the manage host of the instance +func (s *TiProxySpec) GetManageHost() string { + if s.ManageHost != "" { + return s.ManageHost + } + return s.Host +} + +// IsImported returns if the node is imported from TiDB-Ansible +func (s *TiProxySpec) IsImported() bool { + return false +} + +// IgnoreMonitorAgent returns if the node does not have monitor agents available +func (s *TiProxySpec) IgnoreMonitorAgent() bool { + return false +} + +// TiProxyComponent represents TiProxy component. +type TiProxyComponent struct{ Topology *Specification } + +// Name implements Component interface. +func (c *TiProxyComponent) Name() string { + return ComponentTiProxy +} + +// Role implements Component interface. +func (c *TiProxyComponent) Role() string { + return ComponentTiProxy +} + +// CalculateVersion implements the Component interface +func (c *TiProxyComponent) CalculateVersion(clusterVersion string) string { + // always not follow global version, use ""(latest) by default + version := c.Topology.ComponentVersions.TiProxy + return version +} + +// SetVersion implements Component interface. +func (c *TiProxyComponent) SetVersion(version string) { + c.Topology.ComponentVersions.TiProxy = version +} + +// Instances implements Component interface. +func (c *TiProxyComponent) Instances() []Instance { + ins := make([]Instance, 0, len(c.Topology.TiProxyServers)) + for _, s := range c.Topology.TiProxyServers { + s := s + instance := &TiProxyInstance{BaseInstance{ + InstanceSpec: s, + Name: c.Name(), + Host: s.Host, + ManageHost: s.ManageHost, + ListenHost: c.Topology.BaseTopo().GlobalOptions.ListenHost, + Port: s.Port, + SSHP: s.SSHPort, + Source: ComponentTiProxy, + NumaNode: s.NumaNode, + NumaCores: "", + Ports: []int{ + s.Port, + s.StatusPort, + }, + Dirs: []string{ + s.DeployDir, + }, + StatusFn: func(_ context.Context, timeout time.Duration, tlsCfg *tls.Config, _ ...string) string { + return statusByHost(s.Host, s.StatusPort, "/api/debug/health", timeout, tlsCfg) + }, + UptimeFn: func(_ context.Context, timeout time.Duration, tlsCfg *tls.Config) time.Duration { + return proxyUptimeByHost(s.Host, s.StatusPort, timeout, tlsCfg) + }, + Component: c, + }, c.Topology} + + ins = append(ins, instance) + } + return ins +} + +// TiProxyInstance represent the TiProxy instance. +type TiProxyInstance struct { + BaseInstance + topo Topology +} + +// ScaleConfig deploy temporary config on scaling +func (i *TiProxyInstance) ScaleConfig( + ctx context.Context, + e ctxt.Executor, + topo Topology, + clusterName, + clusterVersion, + user string, + paths meta.DirPaths, +) error { + s := i.topo + defer func() { + i.topo = s + }() + i.topo = mustBeClusterTopo(topo) + + return i.InitConfig(ctx, e, clusterName, clusterVersion, user, paths) +} + +func (i *TiProxyInstance) checkConfig( + cfg map[string]any, + paths meta.DirPaths, +) map[string]any { + topo := i.topo.(*Specification) + spec := i.InstanceSpec.(*TiProxySpec) + enableTLS := topo.GlobalOptions.TLSEnabled + + if cfg == nil { + cfg = make(map[string]any) + } + + pds := []string{} + for _, pdspec := range topo.PDServers { + pds = append(pds, pdspec.GetAdvertiseClientURL(enableTLS)) + } + cfg["proxy.pd-addrs"] = strings.Join(pds, ",") + cfg["proxy.require-backend-tls"] = false + cfg["proxy.addr"] = utils.JoinHostPort(i.GetListenHost(), i.GetPort()) + cfg["api.addr"] = utils.JoinHostPort(i.GetListenHost(), spec.StatusPort) + cfg["log.log-file.filename"] = filepath.Join(paths.Log, "tiproxy.log") + + return cfg +} + +// InitConfig implements Instance interface. +func (i *TiProxyInstance) InitConfig( + ctx context.Context, + e ctxt.Executor, + clusterName, + clusterVersion, + deployUser string, + paths meta.DirPaths, +) error { + topo := i.topo.(*Specification) + if err := i.BaseInstance.InitConfig(ctx, e, topo.GlobalOptions, deployUser, paths); err != nil { + return err + } + spec := i.InstanceSpec.(*TiProxySpec) + globalConfig := topo.ServerConfigs.TiProxy + instanceConfig := i.checkConfig(spec.Config, paths) + + cfg := &scripts.TiProxyScript{ + DeployDir: paths.Deploy, + NumaNode: spec.NumaNode, + } + + fp := filepath.Join(paths.Cache, fmt.Sprintf("run_tiproxy_%s_%d.sh", i.GetHost(), i.GetPort())) + + if err := cfg.ConfigToFile(fp); err != nil { + return err + } + dst := filepath.Join(paths.Deploy, "scripts", "run_tiproxy.sh") + if err := e.Transfer(ctx, fp, dst, false, 0, false); err != nil { + return err + } + + if _, _, err := e.Execute(ctx, "chmod +x "+dst, false); err != nil { + return err + } + + var err error + instanceConfig, err = i.setTLSConfig(ctx, false, instanceConfig, paths) + if err != nil { + return err + } + + return i.MergeServerConfig(ctx, e, globalConfig, instanceConfig, paths) +} + +// setTLSConfig set TLS Config to support enable/disable TLS +func (i *TiProxyInstance) setTLSConfig(ctx context.Context, enableTLS bool, configs map[string]any, paths meta.DirPaths) (map[string]any, error) { + if configs == nil { + configs = make(map[string]any) + } + if enableTLS { + configs["security.cluster-tls.ca"] = fmt.Sprintf("%s/tls/%s", paths.Deploy, TLSCACert) + configs["security.cluster-tls.cert"] = fmt.Sprintf("%s/tls/%s.crt", paths.Deploy, i.Role()) + configs["security.cluster-tls.key"] = fmt.Sprintf("%s/tls/%s.pem", paths.Deploy, i.Role()) + + configs["security.server-tls.ca"] = fmt.Sprintf("%s/tls/%s", paths.Deploy, TLSCACert) + configs["security.server-tls.cert"] = fmt.Sprintf("%s/tls/%s.crt", paths.Deploy, i.Role()) + configs["security.server-tls.key"] = fmt.Sprintf("%s/tls/%s.pem", paths.Deploy, i.Role()) + configs["security.server-tls.skip-ca"] = true + + configs["security.sql-tls.ca"] = fmt.Sprintf("%s/tls/%s", paths.Deploy, TLSCACert) + } else { + // drainer tls config list + tlsConfigs := []string{ + "security.cluster-tls.ca", + "security.cluster-tls.cert", + "security.cluster-tls.key", + "security.server-tls.ca", + "security.server-tls.cert", + "security.server-tls.key", + "security.server-tls.skip-ca", + "security.sql-tls.ca", + } + // delete TLS configs + for _, config := range tlsConfigs { + delete(configs, config) + } + } + + return nil, nil +} + +var _ RollingUpdateInstance = &TiProxyInstance{} + +// GetAddr return the address of this TiProxy instance +func (i *TiProxyInstance) GetAddr() string { + return utils.JoinHostPort(i.GetHost(), i.GetPort()) +} + +// PreRestart implements RollingUpdateInstance interface. +func (i *TiProxyInstance) PreRestart(ctx context.Context, topo Topology, apiTimeoutSeconds int, tlsCfg *tls.Config) error { + return nil +} + +// PostRestart implements RollingUpdateInstance interface. +func (i *TiProxyInstance) PostRestart(ctx context.Context, topo Topology, tlsCfg *tls.Config) error { + return nil +} diff --git a/pkg/cluster/spec/tispark.go b/pkg/cluster/spec/tispark.go index a03c8c4fc3..59aea16985 100644 --- a/pkg/cluster/spec/tispark.go +++ b/pkg/cluster/spec/tispark.go @@ -17,7 +17,6 @@ import ( "context" "crypto/tls" "fmt" - "os" "path/filepath" "reflect" "strings" @@ -38,6 +37,7 @@ import ( // TiSparkMasterSpec is the topology specification for TiSpark master node type TiSparkMasterSpec struct { Host string `yaml:"host"` + ManageHost string `yaml:"manage_host,omitempty" validate:"manage_host:editable"` ListenHost string `yaml:"listen_host,omitempty"` SSHPort int `yaml:"ssh_port,omitempty" validate:"ssh_port:editable"` Imported bool `yaml:"imported,omitempty"` @@ -60,7 +60,11 @@ func (s *TiSparkMasterSpec) Role() string { // SSH returns the host and SSH port of the instance func (s *TiSparkMasterSpec) SSH() (string, int) { - return s.Host, s.SSHPort + host := s.Host + if s.ManageHost != "" { + host = s.ManageHost + } + return host, s.SSHPort } // GetMainPort returns the main port of the instance @@ -81,6 +85,7 @@ func (s *TiSparkMasterSpec) IgnoreMonitorAgent() bool { // TiSparkWorkerSpec is the topology specification for TiSpark slave nodes type TiSparkWorkerSpec struct { Host string `yaml:"host"` + ManageHost string `yaml:"manage_host,omitempty"` ListenHost string `yaml:"listen_host,omitempty"` SSHPort int `yaml:"ssh_port,omitempty" validate:"ssh_port:editable"` Imported bool `yaml:"imported,omitempty"` @@ -101,7 +106,11 @@ func (s *TiSparkWorkerSpec) Role() string { // SSH returns the host and SSH port of the instance func (s *TiSparkWorkerSpec) SSH() (string, int) { - return s.Host, s.SSHPort + host := s.Host + if s.ManageHost != "" { + host = s.ManageHost + } + return host, s.SSHPort } // GetMainPort returns the main port of the instance @@ -132,6 +141,16 @@ func (c *TiSparkMasterComponent) Role() string { return RoleTiSparkMaster } +// CalculateVersion implements the Component interface +func (c *TiSparkMasterComponent) CalculateVersion(clusterVersion string) string { + return "" +} + +// SetVersion implements Component interface. +func (c *TiSparkMasterComponent) SetVersion(version string) { + // should never be calles +} + // Instances implements Component interface. func (c *TiSparkMasterComponent) Instances() []Instance { ins := make([]Instance, 0, len(c.Topology.TiSparkMasters)) @@ -141,9 +160,12 @@ func (c *TiSparkMasterComponent) Instances() []Instance { BaseInstance: BaseInstance{ InstanceSpec: s, Name: c.Name(), + ManageHost: s.ManageHost, Host: s.Host, Port: s.Port, SSHP: s.SSHPort, + NumaNode: "", + NumaCores: "", Ports: []int{ s.Port, @@ -158,6 +180,7 @@ func (c *TiSparkMasterComponent) Instances() []Instance { UptimeFn: func(_ context.Context, timeout time.Duration, tlsCfg *tls.Config) time.Duration { return 0 }, + Component: c, }, topo: c.Topology, }) @@ -274,7 +297,7 @@ func (i *TiSparkMasterInstance) InitConfig( if err != nil { return err } - if err := os.WriteFile(fp, log4jFile, 0644); err != nil { + if err := utils.WriteFile(fp, log4jFile, 0644); err != nil { return err } dst = filepath.Join(paths.Deploy, "conf", "log4j.properties") @@ -311,6 +334,16 @@ func (c *TiSparkWorkerComponent) Role() string { return RoleTiSparkWorker } +// CalculateVersion implements the Component interface +func (c *TiSparkWorkerComponent) CalculateVersion(clusterVersion string) string { + return "" +} + +// SetVersion implements Component interface. +func (c *TiSparkWorkerComponent) SetVersion(version string) { + // should never be called +} + // Instances implements Component interface. func (c *TiSparkWorkerComponent) Instances() []Instance { ins := make([]Instance, 0, len(c.Topology.TiSparkWorkers)) @@ -319,9 +352,12 @@ func (c *TiSparkWorkerComponent) Instances() []Instance { BaseInstance: BaseInstance{ InstanceSpec: s, Name: c.Name(), + ManageHost: s.ManageHost, Host: s.Host, Port: s.Port, SSHP: s.SSHPort, + NumaNode: "", + NumaCores: "", Ports: []int{ s.Port, @@ -336,6 +372,7 @@ func (c *TiSparkWorkerComponent) Instances() []Instance { UptimeFn: func(_ context.Context, timeout time.Duration, tlsCfg *tls.Config) time.Duration { return 0 }, + Component: c, }, topo: c.Topology, }) @@ -441,7 +478,7 @@ func (i *TiSparkWorkerInstance) InitConfig( if err != nil { return err } - if err := os.WriteFile(fp, slaveSh, 0755); err != nil { + if err := utils.WriteFile(fp, slaveSh, 0755); err != nil { return err } dst = filepath.Join(paths.Deploy, "sbin", "start-slave.sh") @@ -455,7 +492,7 @@ func (i *TiSparkWorkerInstance) InitConfig( if err != nil { return err } - if err := os.WriteFile(fp, log4jFile, 0644); err != nil { + if err := utils.WriteFile(fp, log4jFile, 0644); err != nil { return err } dst = filepath.Join(paths.Deploy, "conf", "log4j.properties") diff --git a/pkg/cluster/spec/util.go b/pkg/cluster/spec/util.go index e09a1767f6..26ba27c5ed 100644 --- a/pkg/cluster/spec/util.go +++ b/pkg/cluster/spec/util.go @@ -218,3 +218,16 @@ func PackagePath(comp string, version string, os string, arch string) string { fileName := fmt.Sprintf("%s-%s-%s-%s.tar.gz", comp, version, os, arch) return ProfilePath(TiUPPackageCacheDir, fileName) } + +// GetDMMasterPackageName return package name of the first DMMaster instance +func GetDMMasterPackageName(topo Topology) string { + for _, c := range topo.ComponentsByStartOrder() { + if c.Name() == ComponentDMMaster { + instances := c.Instances() + if len(instances) > 0 { + return instances[0].ComponentSource() + } + } + } + return ComponentDMMaster +} diff --git a/pkg/cluster/spec/validate.go b/pkg/cluster/spec/validate.go index 90a72de04c..486cfbcd8d 100644 --- a/pkg/cluster/spec/validate.go +++ b/pkg/cluster/spec/validate.go @@ -169,7 +169,7 @@ func CheckClusterDirConflict(clusterList map[string]Metadata, clusterName string continue } for _, d2 := range existingEntries { - if d1.instance.GetHost() != d2.instance.GetHost() { + if d1.instance.GetManageHost() != d2.instance.GetManageHost() { continue } @@ -186,12 +186,12 @@ func CheckClusterDirConflict(clusterList map[string]Metadata, clusterName string "ThisDirKind": d1.dirKind, "ThisDir": d1.dir, "ThisComponent": d1.instance.ComponentName(), - "ThisHost": d1.instance.GetHost(), + "ThisHost": d1.instance.GetManageHost(), "ExistCluster": d2.clusterName, "ExistDirKind": d2.dirKind, "ExistDir": d2.dir, "ExistComponent": d2.instance.ComponentName(), - "ExistHost": d2.instance.GetHost(), + "ExistHost": d2.instance.GetManageHost(), } zap.L().Info("Meet deploy directory conflict", zap.Any("info", properties)) return errDeployDirConflict.New("Deploy directory conflicts to an existing cluster").WithProperty(tui.SuggestionFromTemplate(` @@ -219,7 +219,7 @@ Please change to use another directory or another host. // ref https://github.com/pingcap/tiup/issues/1047#issuecomment-761711508 func CheckClusterDirOverlap(entries []DirEntry) error { ignore := func(d1, d2 DirEntry) bool { - return (d1.instance.GetHost() != d2.instance.GetHost()) || + return (d1.instance.GetManageHost() != d2.instance.GetManageHost()) || d1.dir == "" || d2.dir == "" || strings.HasSuffix(d1.dirKind, "deploy directory") || strings.HasSuffix(d2.dirKind, "deploy directory") @@ -258,11 +258,11 @@ func CheckClusterDirOverlap(entries []DirEntry) error { "ThisDirKind": d1.dirKind, "ThisDir": d1.dir, "ThisComponent": d1.instance.ComponentName(), - "ThisHost": d1.instance.GetHost(), + "ThisHost": d1.instance.GetManageHost(), "ThatDirKind": d2.dirKind, "ThatDir": d2.dir, "ThatComponent": d2.instance.ComponentName(), - "ThatHost": d2.instance.GetHost(), + "ThatHost": d2.instance.GetManageHost(), } zap.L().Info("Meet deploy directory overlap", zap.Any("info", properties)) return errDeployDirOverlap.New("Deploy directory overlaps to another instance").WithProperty(tui.SuggestionFromTemplate(` @@ -316,8 +316,8 @@ func CheckClusterPortConflict(clusterList map[string]Metadata, clusterName strin instance: inst, }) } - if !uniqueHosts.Exist(inst.GetHost()) { - uniqueHosts.Insert(inst.GetHost()) + if !uniqueHosts.Exist(inst.GetManageHost()) { + uniqueHosts.Insert(inst.GetManageHost()) existingEntries = append(existingEntries, Entry{ clusterName: name, @@ -349,8 +349,8 @@ func CheckClusterPortConflict(clusterList map[string]Metadata, clusterName strin if mOpt == nil { return } - if !uniqueHosts.Exist(inst.GetHost()) { - uniqueHosts.Insert(inst.GetHost()) + if !uniqueHosts.Exist(inst.GetManageHost()) { + uniqueHosts.Insert(inst.GetManageHost()) currentEntries = append(currentEntries, Entry{ componentName: RoleMonitor, @@ -367,7 +367,7 @@ func CheckClusterPortConflict(clusterList map[string]Metadata, clusterName strin for _, p1 := range currentEntries { for _, p2 := range existingEntries { - if p1.instance.GetHost() != p2.instance.GetHost() { + if p1.instance.GetManageHost() != p2.instance.GetManageHost() { continue } @@ -376,11 +376,11 @@ func CheckClusterPortConflict(clusterList map[string]Metadata, clusterName strin properties := map[string]string{ "ThisPort": strconv.Itoa(p1.port), "ThisComponent": p1.componentName, - "ThisHost": p1.instance.GetHost(), + "ThisHost": p1.instance.GetManageHost(), "ExistCluster": p2.clusterName, "ExistPort": strconv.Itoa(p2.port), "ExistComponent": p2.componentName, - "ExistHost": p2.instance.GetHost(), + "ExistHost": p2.instance.GetManageHost(), } // if one of the instances marks itself as ignore_exporter, do not report // the monitoring agent ports conflict and just skip @@ -449,7 +449,8 @@ type TiKVLabelProvider interface { } func getHostFromAddress(addr string) string { - return strings.Split(addr, ":")[0] + host, _ := utils.ParseHostPort(addr) + return host } // CheckTiKVLabels will check if tikv missing label or have wrong label @@ -839,6 +840,9 @@ func (s *Specification) CountDir(targetHost, dirPrefix string) int { compSpec := reflect.Indirect(compSpecs.Index(index)) deployDir := compSpec.FieldByName("DeployDir").String() host := compSpec.FieldByName("Host").String() + if compSpec.FieldByName("ManageHost").String() != "" { + host = compSpec.FieldByName("ManageHost").String() + } for _, dirType := range dirTypes { j, found := findField(compSpec, dirType) diff --git a/pkg/cluster/task/builder.go b/pkg/cluster/task/builder.go index d416cf7bd9..96e74cbb09 100644 --- a/pkg/cluster/task/builder.go +++ b/pkg/cluster/task/builder.go @@ -142,7 +142,7 @@ func (b *Builder) ClusterSSH( var tasks []Task topo.IterInstance(func(inst spec.Instance) { tasks = append(tasks, &UserSSH{ - host: inst.GetHost(), + host: inst.GetManageHost(), port: inst.GetSSHPort(), deployUser: deployUser, timeout: sshTimeout, @@ -243,25 +243,25 @@ func (b *Builder) BackupComponent(component, fromVer string, host, deployDir str } // InitConfig appends a CopyComponent task to the current task collection -func (b *Builder) InitConfig(clusterName, clusterVersion string, specManager *spec.SpecManager, inst spec.Instance, deployUser string, ignoreCheck bool, paths meta.DirPaths) *Builder { +func (b *Builder) InitConfig(clusterName, version string, specManager *spec.SpecManager, inst spec.Instance, deployUser string, ignoreCheck bool, paths meta.DirPaths) *Builder { // get nightly version var componentVersion utils.Version meta := specManager.NewMetadata() // full version - componentVersion = utils.Version(clusterVersion) + componentVersion = utils.Version(version) if err := specManager.Metadata(clusterName, meta); err == nil { // get nightly version - if clusterVersion == utils.NightlyVersionAlias { - componentVersion, _, err = environment.GlobalEnv().V1Repository().LatestNightlyVersion(inst.ComponentName()) + if version == utils.NightlyVersionAlias { + componentVersion, _, err = environment.GlobalEnv().V1Repository().LatestNightlyVersion(inst.ComponentSource()) if err != nil { - componentVersion = utils.Version(clusterVersion) + componentVersion = utils.Version(version) } } // dm cluster does not require a full nightly version if meta.GetTopology().Type() == spec.TopoTypeDM { - componentVersion = utils.Version(clusterVersion) + componentVersion = utils.Version(version) } } @@ -334,6 +334,16 @@ func (b *Builder) EnvInit(host, deployUser string, userGroup string, skipCreateU return b } +// RotateSSH appends a RotateSSH task to the current task collection +func (b *Builder) RotateSSH(host, deployUser, newPublicKeyPath string) *Builder { + b.tasks = append(b.tasks, &RotateSSH{ + host: host, + deployUser: deployUser, + newPublicKeyPath: newPublicKeyPath, + }) + return b +} + // ClusterOperate appends a cluster operation task. // All the UserSSH needed must be init first. func (b *Builder) ClusterOperate( @@ -437,10 +447,10 @@ func (b *Builder) DeploySpark(inst spec.Instance, sparkVersion, srcPath, deployD inst.Arch(), sparkVersion, srcPath, - inst.GetHost(), + inst.GetManageHost(), deployDir, ).Shell( // spark is under a subdir, move it to deploy dir - inst.GetHost(), + inst.GetManageHost(), fmt.Sprintf( "cp -rf %[1]s %[2]s/ && cp -rf %[3]s/* %[2]s/ && rm -rf %[1]s %[3]s", filepath.Join(deployDir, "bin", sparkSubPath), @@ -455,10 +465,10 @@ func (b *Builder) DeploySpark(inst spec.Instance, sparkVersion, srcPath, deployD inst.Arch(), "", // use the latest stable version srcPath, - inst.GetHost(), + inst.GetManageHost(), deployDir, ).Shell( // move tispark jar to correct path - inst.GetHost(), + inst.GetManageHost(), fmt.Sprintf( "cp -f %[1]s/*.jar %[2]s/jars/ && rm -f %[1]s/*.jar", filepath.Join(deployDir, "bin"), diff --git a/pkg/cluster/task/init_config.go b/pkg/cluster/task/init_config.go index bb4a43bb93..d87b504f42 100644 --- a/pkg/cluster/task/init_config.go +++ b/pkg/cluster/task/init_config.go @@ -16,12 +16,12 @@ package task import ( "context" "fmt" - "os" "github.com/pingcap/errors" "github.com/pingcap/tiup/pkg/cluster/ctxt" "github.com/pingcap/tiup/pkg/cluster/spec" "github.com/pingcap/tiup/pkg/meta" + "github.com/pingcap/tiup/pkg/utils" ) // InitConfig is used to copy all configurations to the target directory of path @@ -38,12 +38,12 @@ type InitConfig struct { // Execute implements the Task interface func (c *InitConfig) Execute(ctx context.Context) error { // Copy to remote server - exec, found := ctxt.GetInner(ctx).GetExecutor(c.instance.GetHost()) + exec, found := ctxt.GetInner(ctx).GetExecutor(c.instance.GetManageHost()) if !found { return ErrNoExecutor } - if err := os.MkdirAll(c.paths.Cache, 0755); err != nil { + if err := utils.MkdirAll(c.paths.Cache, 0755); err != nil { return errors.Annotatef(err, "create cache directory failed: %s", c.paths.Cache) } @@ -52,7 +52,7 @@ func (c *InitConfig) Execute(ctx context.Context) error { if c.ignoreCheck && errors.Cause(err) == spec.ErrorCheckConfig { return nil } - return errors.Annotatef(err, "init config failed: %s:%d", c.instance.GetHost(), c.instance.GetPort()) + return errors.Annotatef(err, "init config failed: %s:%d", c.instance.GetManageHost(), c.instance.GetPort()) } return nil } @@ -65,7 +65,7 @@ func (c *InitConfig) Rollback(ctx context.Context) error { // String implements the fmt.Stringer interface func (c *InitConfig) String() string { return fmt.Sprintf("InitConfig: cluster=%s, user=%s, host=%s, path=%s, %s", - c.clusterName, c.deployUser, c.instance.GetHost(), + c.clusterName, c.deployUser, c.instance.GetManageHost(), c.specManager.Path(c.clusterName, spec.TempConfigPath, c.instance.ServiceName()), c.paths) } diff --git a/pkg/cluster/task/init_config_test.go b/pkg/cluster/task/init_config_test.go index 8f60a2ec31..96f779b3f1 100644 --- a/pkg/cluster/task/init_config_test.go +++ b/pkg/cluster/task/init_config_test.go @@ -69,6 +69,10 @@ func (i *fakeInstance) GetPort() int { return 4000 } +func (i *fakeInstance) GetManageHost() string { + return "1.1.1.1" +} + func Test(t *testing.T) { check.TestingT(t) } var _ = check.Suite(&initConfigSuite{}) @@ -80,7 +84,7 @@ func (s *initConfigSuite) TestCheckConfig(c *check.C) { t := &InitConfig{ clusterName: "test-cluster-name", - clusterVersion: "v4.0.0", + clusterVersion: "v6.0.0", paths: meta.DirPaths{ Cache: "/tmp", }, diff --git a/pkg/cluster/task/monitored_config.go b/pkg/cluster/task/monitored_config.go index c533299f1d..65d91dd0e6 100644 --- a/pkg/cluster/task/monitored_config.go +++ b/pkg/cluster/task/monitored_config.go @@ -16,7 +16,6 @@ package task import ( "context" "fmt" - "os" "path/filepath" "github.com/google/uuid" @@ -29,6 +28,7 @@ import ( system "github.com/pingcap/tiup/pkg/cluster/template/systemd" logprinter "github.com/pingcap/tiup/pkg/logger/printer" "github.com/pingcap/tiup/pkg/meta" + "github.com/pingcap/tiup/pkg/utils" "go.uber.org/zap" ) @@ -56,7 +56,7 @@ func (m *MonitoredConfig) Execute(ctx context.Context) error { return ErrNoExecutor } - if err := os.MkdirAll(m.paths.Cache, 0755); err != nil { + if err := utils.MkdirAll(m.paths.Cache, 0755); err != nil { return err } diff --git a/pkg/cluster/task/rotate_ssh.go b/pkg/cluster/task/rotate_ssh.go new file mode 100644 index 0000000000..0f358fd69f --- /dev/null +++ b/pkg/cluster/task/rotate_ssh.go @@ -0,0 +1,92 @@ +// Copyright 2020 PingCAP, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// See the License for the specific language governing permissions and +// limitations under the License. + +package task + +import ( + "context" + "fmt" + "os" + "strings" + + "github.com/joomcode/errorx" + "github.com/pingcap/tiup/pkg/cluster/ctxt" + "github.com/pingcap/tiup/pkg/cluster/executor" +) + +// RotateSSH is used to rotate ssh key, e.g: +// 1. enable new public key +// 2. revoke old public key +type RotateSSH struct { + host string + deployUser string + newPublicKeyPath string +} + +// Execute implements the Task interface +func (e *RotateSSH) Execute(ctx context.Context) error { + return e.exec(ctx) +} + +func (e *RotateSSH) exec(ctx context.Context) error { + wrapError := func(err error) *errorx.Error { + return ErrEnvInitFailed.Wrap(err, "Failed to Rotate ssh public key on remote host '%s'", e.host) + } + + exec, found := ctxt.GetInner(ctx).GetExecutor(e.host) + if !found { + panic(ErrNoExecutor) + } + + pubKey, err := os.ReadFile(ctxt.GetInner(ctx).PublicKeyPath) + if err != nil { + return wrapError(err) + } + + newPubKey, err := os.ReadFile(e.newPublicKeyPath) + if err != nil { + return wrapError(err) + } + + sshAuthorizedKeys := executor.FindSSHAuthorizedKeysFile(ctx, exec) + + // enable new key + cmd := fmt.Sprintf(`echo %s >> %s`, strings.TrimSpace(string(newPubKey)), sshAuthorizedKeys) + _, _, err = exec.Execute(ctx, cmd, false) + if err != nil { + return wrapError(errEnvInitSubCommandFailed. + Wrap(err, "Failed to write new public key to '%s' for user '%s'", sshAuthorizedKeys, e.deployUser)) + } + + // Revoke old key + cmd = fmt.Sprintf(`sed -i '\|%[1]s|d' %[2]s`, + strings.TrimSpace(string(pubKey)), sshAuthorizedKeys) + + _, _, err = exec.Execute(ctx, cmd, false) + if err != nil { + return wrapError(errEnvInitSubCommandFailed. + Wrap(err, "Failed to revoke old key for user '%s'", e.deployUser)) + } + + return nil +} + +// Rollback implements the Task interface +func (e *RotateSSH) Rollback(ctx context.Context) error { + return ErrUnsupportedRollback +} + +// String implements the fmt.Stringer interface +func (e *RotateSSH) String() string { + return fmt.Sprintf("RotateSSH: user=%s, host=%s", e.deployUser, e.host) +} diff --git a/pkg/cluster/task/scale_config.go b/pkg/cluster/task/scale_config.go index a766e46d62..f336d8071a 100644 --- a/pkg/cluster/task/scale_config.go +++ b/pkg/cluster/task/scale_config.go @@ -16,11 +16,11 @@ package task import ( "context" "fmt" - "os" "github.com/pingcap/tiup/pkg/cluster/ctxt" "github.com/pingcap/tiup/pkg/cluster/spec" "github.com/pingcap/tiup/pkg/meta" + "github.com/pingcap/tiup/pkg/utils" ) // ScaleConfig is used to copy all configurations to the target directory of path @@ -37,13 +37,13 @@ type ScaleConfig struct { // Execute implements the Task interface func (c *ScaleConfig) Execute(ctx context.Context) error { // Copy to remote server - exec, found := ctxt.GetInner(ctx).GetExecutor(c.instance.GetHost()) + exec, found := ctxt.GetInner(ctx).GetExecutor(c.instance.GetManageHost()) if !found { return ErrNoExecutor } c.paths.Cache = c.specManager.Path(c.clusterName, spec.TempConfigPath) - if err := os.MkdirAll(c.paths.Cache, 0755); err != nil { + if err := utils.MkdirAll(c.paths.Cache, 0755); err != nil { return err } @@ -58,5 +58,5 @@ func (c *ScaleConfig) Rollback(ctx context.Context) error { // String implements the fmt.Stringer interface func (c *ScaleConfig) String() string { return fmt.Sprintf("ScaleConfig: cluster=%s, user=%s, host=%s, service=%s, %s", - c.clusterName, c.deployUser, c.instance.GetHost(), c.instance.ServiceName(), c.paths) + c.clusterName, c.deployUser, c.instance.GetManageHost(), c.instance.ServiceName(), c.paths) } diff --git a/pkg/cluster/task/tls.go b/pkg/cluster/task/tls.go index 956148e34c..892b64a6b7 100644 --- a/pkg/cluster/task/tls.go +++ b/pkg/cluster/task/tls.go @@ -64,7 +64,7 @@ func (c *TLSCert) Execute(ctx context.Context) error { } // make sure the cache dir exist - if err := utils.CreateDir(c.paths.Cache); err != nil { + if err := utils.MkdirAll(c.paths.Cache, 0755); err != nil { return err } diff --git a/pkg/cluster/task/update_meta.go b/pkg/cluster/task/update_meta.go index 8e9e3d3e6d..dd6457e1b9 100644 --- a/pkg/cluster/task/update_meta.go +++ b/pkg/cluster/task/update_meta.go @@ -72,6 +72,15 @@ func (u *UpdateMeta) Execute(ctx context.Context) error { } newMeta.Topology.PDServers = pdServers + tiproxyServers := make([]*spec.TiProxySpec, 0) + for i, instance := range (&spec.TiProxyComponent{Topology: topo}).Instances() { + if deleted.Exist(instance.ID()) { + continue + } + tiproxyServers = append(tiproxyServers, topo.TiProxyServers[i]) + } + newMeta.Topology.TiProxyServers = tiproxyServers + dashboardServers := make([]*spec.DashboardSpec, 0) for i, instance := range (&spec.DashboardComponent{Topology: topo}).Instances() { if deleted.Exist(instance.ID()) { @@ -79,7 +88,7 @@ func (u *UpdateMeta) Execute(ctx context.Context) error { } dashboardServers = append(dashboardServers, topo.DashboardServers[i]) } - topo.DashboardServers = dashboardServers + newMeta.Topology.DashboardServers = dashboardServers tiflashServers := make([]*spec.TiFlashSpec, 0) for i, instance := range (&spec.TiFlashComponent{Topology: topo}).Instances() { diff --git a/pkg/cluster/template/config/alertmanager.go b/pkg/cluster/template/config/alertmanager.go index 0b5bebbf0a..e9a8e6f026 100644 --- a/pkg/cluster/template/config/alertmanager.go +++ b/pkg/cluster/template/config/alertmanager.go @@ -14,11 +14,11 @@ package config import ( - "os" "path" "github.com/pingcap/errors" "github.com/pingcap/tiup/embed" + "github.com/pingcap/tiup/pkg/utils" ) // AlertManagerConfig represent the data to generate AlertManager config @@ -50,7 +50,7 @@ func (c *AlertManagerConfig) ConfigToFile(file string) error { if err != nil { return err } - if err := os.WriteFile(file, config, 0755); err != nil { + if err := utils.WriteFile(file, config, 0755); err != nil { return errors.AddStack(err) } return nil diff --git a/pkg/cluster/template/config/blackbox.go b/pkg/cluster/template/config/blackbox.go index d32db5aff5..fb0bc56f17 100644 --- a/pkg/cluster/template/config/blackbox.go +++ b/pkg/cluster/template/config/blackbox.go @@ -15,11 +15,11 @@ package config import ( "bytes" - "os" "path" "text/template" "github.com/pingcap/tiup/embed" + "github.com/pingcap/tiup/pkg/utils" ) // BlackboxConfig represent the data to generate AlertManager config @@ -52,7 +52,7 @@ func (c *BlackboxConfig) ConfigToFile(file string) error { if err != nil { return err } - return os.WriteFile(file, config, 0755) + return utils.WriteFile(file, config, 0755) } // ConfigWithTemplate generate the AlertManager config content by tpl diff --git a/pkg/cluster/template/config/dashboard.go b/pkg/cluster/template/config/dashboard.go index df1aa7020f..7af046e989 100644 --- a/pkg/cluster/template/config/dashboard.go +++ b/pkg/cluster/template/config/dashboard.go @@ -15,11 +15,11 @@ package config import ( "bytes" - "os" "path" "text/template" "github.com/pingcap/tiup/embed" + "github.com/pingcap/tiup/pkg/utils" ) // DashboardConfig represent the data to generate Dashboard config @@ -67,5 +67,5 @@ func (c *DashboardConfig) ConfigToFile(file string) error { if err != nil { return err } - return os.WriteFile(file, config, 0755) + return utils.WriteFile(file, config, 0755) } diff --git a/pkg/cluster/template/config/datasource.go b/pkg/cluster/template/config/datasource.go index d05683726f..5455f90224 100644 --- a/pkg/cluster/template/config/datasource.go +++ b/pkg/cluster/template/config/datasource.go @@ -15,65 +15,36 @@ package config import ( "bytes" - "os" "path" "text/template" "github.com/pingcap/tiup/embed" + "github.com/pingcap/tiup/pkg/utils" ) // DatasourceConfig represent the data to generate Datasource config type DatasourceConfig struct { ClusterName string - IP string - Port uint64 + URL string } -// NewDatasourceConfig returns a DatasourceConfig -func NewDatasourceConfig(cluster, ip string) *DatasourceConfig { - return &DatasourceConfig{ - ClusterName: cluster, - IP: ip, - Port: 9090, - } -} - -// WithPort set Port field of DatasourceConfig -func (c *DatasourceConfig) WithPort(port uint64) *DatasourceConfig { - c.Port = port - return c -} - -// Config generate the config file data. -func (c *DatasourceConfig) Config() ([]byte, error) { +// ConfigToFile write config content to specific path +func (c *DatasourceConfig) ConfigToFile(file string) error { fp := path.Join("templates", "config", "datasource.yml.tpl") tpl, err := embed.ReadTemplate(fp) if err != nil { - return nil, err + return err } - return c.ConfigWithTemplate(string(tpl)) -} -// ConfigWithTemplate generate the Datasource config content by tpl -func (c *DatasourceConfig) ConfigWithTemplate(tpl string) ([]byte, error) { - tmpl, err := template.New("Datasource").Parse(tpl) + tmpl, err := template.New("Datasource").Parse(string(tpl)) if err != nil { - return nil, err + return err } content := bytes.NewBufferString("") if err := tmpl.Execute(content, c); err != nil { - return nil, err - } - - return content.Bytes(), nil -} - -// ConfigToFile write config content to specific path -func (c *DatasourceConfig) ConfigToFile(file string) error { - config, err := c.Config() - if err != nil { return err } - return os.WriteFile(file, config, 0755) + + return utils.WriteFile(file, content.Bytes(), 0755) } diff --git a/pkg/cluster/template/config/grafana.go b/pkg/cluster/template/config/grafana.go index 2f3c732b82..2e097df65c 100644 --- a/pkg/cluster/template/config/grafana.go +++ b/pkg/cluster/template/config/grafana.go @@ -15,11 +15,11 @@ package config import ( "bytes" - "os" "path" "text/template" "github.com/pingcap/tiup/embed" + "github.com/pingcap/tiup/pkg/utils" ) // GrafanaConfig represent the data to generate Grafana config @@ -131,5 +131,5 @@ func (c *GrafanaConfig) ConfigToFile(file string) error { if err != nil { return err } - return os.WriteFile(file, config, 0755) + return utils.WriteFile(file, config, 0755) } diff --git a/pkg/cluster/template/config/ngmonitoring.go b/pkg/cluster/template/config/ngmonitoring.go index a77edf64e0..ae6b97565a 100644 --- a/pkg/cluster/template/config/ngmonitoring.go +++ b/pkg/cluster/template/config/ngmonitoring.go @@ -15,8 +15,6 @@ package config import ( "bytes" - "fmt" - "os" "path" "text/template" @@ -26,95 +24,34 @@ import ( // NgMonitoringConfig represent the data to generate NgMonitoring config type NgMonitoringConfig struct { - ClusterName string - TLSEnabled bool - IP string - Port int - PDAddrs string - DeployDir string - DataDir string - LogDir string -} + ClusterName string + Address string + AdvertiseAddress string + TLSEnabled bool + PDAddrs string -// NewNgMonitoringConfig returns a PrometheusConfig -func NewNgMonitoringConfig(clusterName, clusterVersion string, enableTLS bool) *NgMonitoringConfig { - cfg := &NgMonitoringConfig{ - ClusterName: clusterName, - TLSEnabled: enableTLS, - } - return cfg + DeployDir string + DataDir string + LogDir string } -// AddPD add a PD address -func (c *NgMonitoringConfig) AddPD(ip string, port uint64) *NgMonitoringConfig { - if c.PDAddrs == "" { - c.PDAddrs = fmt.Sprintf("\"%s\"", utils.JoinHostPort(ip, int(port))) - } else { - c.PDAddrs += fmt.Sprintf(",\"%s\"", utils.JoinHostPort(ip, int(port))) +// ConfigToFile write config content to specific path +func (c *NgMonitoringConfig) ConfigToFile(file string) error { + fp := path.Join("templates", "config", "ngmonitoring.toml.tpl") + tpl, err := embed.ReadTemplate(fp) + if err != nil { + return err } - return c -} - -// AddIP add ip to ng-monitoring conf -func (c *NgMonitoringConfig) AddIP(ip string) *NgMonitoringConfig { - c.IP = ip - return c -} - -// AddLog add logdir to ng-monitoring conf -func (c *NgMonitoringConfig) AddLog(dir string) *NgMonitoringConfig { - c.LogDir = dir - return c -} - -// AddDeployDir add logdir to ng-monitoring conf -func (c *NgMonitoringConfig) AddDeployDir(dir string) *NgMonitoringConfig { - c.DeployDir = dir - return c -} - -// AddDataDir add logdir to ng-monitoring conf -func (c *NgMonitoringConfig) AddDataDir(dir string) *NgMonitoringConfig { - c.DataDir = dir - return c -} -// AddPort add port to ng-monitoring conf -func (c *NgMonitoringConfig) AddPort(port int) *NgMonitoringConfig { - c.Port = port - return c -} - -// ConfigWithTemplate generate the Prometheus config content by tpl -func (c *NgMonitoringConfig) ConfigWithTemplate(tpl string) ([]byte, error) { - tmpl, err := template.New("NgMonitoring").Parse(tpl) + tmpl, err := template.New("NgMonitoring").Parse(string(tpl)) if err != nil { - return nil, err + return err } content := bytes.NewBufferString("") if err := tmpl.Execute(content, c); err != nil { - return nil, err - } - - return content.Bytes(), nil -} - -// Config generate the config file data. -func (c *NgMonitoringConfig) Config() ([]byte, error) { - fp := path.Join("templates", "config", "ngmonitoring.toml.tpl") - tpl, err := embed.ReadTemplate(fp) - if err != nil { - return nil, err - } - return c.ConfigWithTemplate(string(tpl)) -} - -// ConfigToFile write config content to specific path -func (c *NgMonitoringConfig) ConfigToFile(file string) error { - config, err := c.Config() - if err != nil { return err } - return os.WriteFile(file, config, 0755) + + return utils.WriteFile(file, content.Bytes(), 0755) } diff --git a/pkg/cluster/template/config/prometheus.go b/pkg/cluster/template/config/prometheus.go index 0329803dec..90d6b00051 100644 --- a/pkg/cluster/template/config/prometheus.go +++ b/pkg/cluster/template/config/prometheus.go @@ -15,7 +15,6 @@ package config import ( "bytes" - "os" "path" "text/template" @@ -32,6 +31,7 @@ type PrometheusConfig struct { TLSEnabled bool NodeExporterAddrs []string TiDBStatusAddrs []string + TiProxyStatusAddrs []string TiKVStatusAddrs []string PDAddrs []string TiFlashStatusAddrs []string @@ -45,7 +45,7 @@ type PrometheusConfig struct { MonitoredServers []string AlertmanagerAddrs []string NGMonitoringAddrs []string - PushgatewayAddr string + PushgatewayAddrs []string BlackboxAddr string GrafanaAddr string HasTiKVAccelerateRules bool @@ -80,6 +80,12 @@ func (c *PrometheusConfig) AddTiDB(ip string, port uint64) *PrometheusConfig { return c } +// AddTiProxy add a TiProxy address +func (c *PrometheusConfig) AddTiProxy(ip string, port uint64) *PrometheusConfig { + c.TiProxyStatusAddrs = append(c.TiProxyStatusAddrs, utils.JoinHostPort(ip, int(port))) + return c +} + // AddTiKV add a TiKV address func (c *PrometheusConfig) AddTiKV(ip string, port uint64) *PrometheusConfig { c.TiKVStatusAddrs = append(c.TiKVStatusAddrs, utils.JoinHostPort(ip, int(port))) @@ -153,8 +159,8 @@ func (c *PrometheusConfig) AddAlertmanager(ip string, port uint64) *PrometheusCo } // AddPushgateway add an pushgateway address -func (c *PrometheusConfig) AddPushgateway(ip string, port uint64) *PrometheusConfig { - c.PushgatewayAddr = utils.JoinHostPort(ip, int(port)) +func (c *PrometheusConfig) AddPushgateway(addresses []string) *PrometheusConfig { + c.PushgatewayAddrs = addresses return c } @@ -231,5 +237,5 @@ func (c *PrometheusConfig) ConfigToFile(file string) error { if err != nil { return err } - return os.WriteFile(file, config, 0755) + return utils.WriteFile(file, config, 0755) } diff --git a/pkg/cluster/template/config/tispark.go b/pkg/cluster/template/config/tispark.go index cb5b460165..598ace4f64 100644 --- a/pkg/cluster/template/config/tispark.go +++ b/pkg/cluster/template/config/tispark.go @@ -15,11 +15,11 @@ package config import ( "bytes" - "os" "path/filepath" "text/template" "github.com/pingcap/tiup/embed" + "github.com/pingcap/tiup/pkg/utils" ) // TiSparkConfig represent the data to generate TiSpark configs @@ -62,7 +62,7 @@ func (c *TiSparkConfig) ConfigToFile(file string) error { if err != nil { return err } - return os.WriteFile(file, config, 0755) + return utils.WriteFile(file, config, 0755) } // ConfigWithTemplate parses the template file diff --git a/pkg/cluster/template/install/local_install.sh.go b/pkg/cluster/template/install/local_install.sh.go index a5a2a45497..b08ee6bb0f 100644 --- a/pkg/cluster/template/install/local_install.sh.go +++ b/pkg/cluster/template/install/local_install.sh.go @@ -13,13 +13,11 @@ package install -import ( - "os" -) +import "github.com/pingcap/tiup/pkg/utils" // WriteLocalInstallScript writes the install script into specified path func WriteLocalInstallScript(path string) error { - return os.WriteFile(path, []byte(script), 0755) + return utils.WriteFile(path, []byte(script), 0755) } var script = `#!/bin/sh diff --git a/pkg/cluster/template/scripts/alertmanager.go b/pkg/cluster/template/scripts/alertmanager.go index 8de5a621ea..8269ce3141 100644 --- a/pkg/cluster/template/scripts/alertmanager.go +++ b/pkg/cluster/template/scripts/alertmanager.go @@ -15,11 +15,11 @@ package scripts import ( "bytes" - "os" "path" "text/template" "github.com/pingcap/tiup/embed" + "github.com/pingcap/tiup/pkg/utils" ) // AlertManagerScript represent the data to generate AlertManager start script @@ -54,5 +54,5 @@ func (c *AlertManagerScript) ConfigToFile(file string) error { return err } - return os.WriteFile(file, content.Bytes(), 0755) + return utils.WriteFile(file, content.Bytes(), 0755) } diff --git a/pkg/cluster/template/scripts/blackbox_exporter.go b/pkg/cluster/template/scripts/blackbox_exporter.go index aa90bed955..b50488a7b7 100644 --- a/pkg/cluster/template/scripts/blackbox_exporter.go +++ b/pkg/cluster/template/scripts/blackbox_exporter.go @@ -15,11 +15,11 @@ package scripts import ( "bytes" - "os" "path" "text/template" "github.com/pingcap/tiup/embed" + "github.com/pingcap/tiup/pkg/utils" ) // BlackboxExporterScript represent the data to generate BlackboxExporter config @@ -67,7 +67,7 @@ func (c *BlackboxExporterScript) ConfigToFile(file string) error { if err != nil { return err } - return os.WriteFile(file, config, 0755) + return utils.WriteFile(file, config, 0755) } // ConfigWithTemplate generate the BlackboxExporter config content by tpl diff --git a/pkg/cluster/template/scripts/cdc.go b/pkg/cluster/template/scripts/cdc.go index 557ad021f6..0c1f91e5e0 100644 --- a/pkg/cluster/template/scripts/cdc.go +++ b/pkg/cluster/template/scripts/cdc.go @@ -15,11 +15,11 @@ package scripts import ( "bytes" - "os" "path" "text/template" "github.com/pingcap/tiup/embed" + "github.com/pingcap/tiup/pkg/utils" ) // CDCScript represent the data to generate cdc config @@ -58,5 +58,5 @@ func (c *CDCScript) ConfigToFile(file string) error { return err } - return os.WriteFile(file, content.Bytes(), 0755) + return utils.WriteFile(file, content.Bytes(), 0755) } diff --git a/pkg/cluster/template/scripts/dashboard.go b/pkg/cluster/template/scripts/dashboard.go index a39cfbe80c..21e361bb1e 100644 --- a/pkg/cluster/template/scripts/dashboard.go +++ b/pkg/cluster/template/scripts/dashboard.go @@ -15,17 +15,17 @@ package scripts import ( "bytes" - "os" "path" "text/template" "github.com/pingcap/tiup/embed" + "github.com/pingcap/tiup/pkg/utils" ) // DashboardScript represent the data to generate cdc config type DashboardScript struct { TidbVersion string - IP string + Host string Port int DeployDir string LogDir string @@ -53,5 +53,5 @@ func (s *DashboardScript) ConfigToFile(file string) error { return err } - return os.WriteFile(file, content.Bytes(), 0755) + return utils.WriteFile(file, content.Bytes(), 0755) } diff --git a/pkg/cluster/template/scripts/dm_master.go b/pkg/cluster/template/scripts/dm_master.go index 3dce4bca17..91bae44686 100644 --- a/pkg/cluster/template/scripts/dm_master.go +++ b/pkg/cluster/template/scripts/dm_master.go @@ -16,7 +16,6 @@ package scripts import ( "bytes" "errors" - "os" "path" "text/template" @@ -26,167 +25,81 @@ import ( // DMMasterScript represent the data to generate TiDB config type DMMasterScript struct { - Name string - Scheme string - IP string - Port int - PeerPort int - DeployDir string - DataDir string - LogDir string - NumaNode string - V1SourcePath string - Endpoints []*DMMasterScript -} - -// NewDMMasterScript returns a DMMasterScript with given arguments -func NewDMMasterScript(name, ip, deployDir, dataDir, logDir string, enableTLS bool) *DMMasterScript { - return &DMMasterScript{ - Name: name, - Scheme: utils.Ternary(enableTLS, "https", "http").(string), - IP: ip, - Port: 8261, - PeerPort: 8291, - DeployDir: deployDir, - DataDir: dataDir, - LogDir: logDir, - } -} - -// WithV1SourcePath set Scheme field of V1SourcePath -func (c *DMMasterScript) WithV1SourcePath(path string) *DMMasterScript { - c.V1SourcePath = path - return c -} - -// WithScheme set Scheme field of NewDMMasterScript -func (c *DMMasterScript) WithScheme(scheme string) *DMMasterScript { - c.Scheme = scheme - return c -} - -// WithPort set Port field of DMMasterScript -func (c *DMMasterScript) WithPort(port int) *DMMasterScript { - c.Port = port - return c -} - -// WithNumaNode set NumaNode field of DMMasterScript -func (c *DMMasterScript) WithNumaNode(numa string) *DMMasterScript { - c.NumaNode = numa - return c -} - -// WithPeerPort set PeerPort field of DMMasterScript -func (c *DMMasterScript) WithPeerPort(port int) *DMMasterScript { - c.PeerPort = port - return c -} + Name string + V1SourcePath string + MasterAddr string + AdvertiseAddr string + PeerURL string + AdvertisePeerURL string + InitialCluster string -// AppendEndpoints add new DMMasterScript to Endpoints field -func (c *DMMasterScript) AppendEndpoints(ends ...*DMMasterScript) *DMMasterScript { - c.Endpoints = append(c.Endpoints, ends...) - return c -} + DeployDir string + DataDir string + LogDir string -// Config generate the config file data. -func (c *DMMasterScript) Config() ([]byte, error) { - fp := path.Join("templates", "scripts", "run_dm-master.sh.tpl") - tpl, err := embed.ReadTemplate(fp) - if err != nil { - return nil, err - } - return c.ConfigWithTemplate(string(tpl)) + NumaNode string } // ConfigToFile write config content to specific path func (c *DMMasterScript) ConfigToFile(file string) error { - config, err := c.Config() + fp := path.Join("templates", "scripts", "run_dm-master.sh.tpl") + tpl, err := embed.ReadTemplate(fp) if err != nil { return err } - return os.WriteFile(file, config, 0755) -} - -// ConfigWithTemplate generate the TiDB config content by tpl -func (c *DMMasterScript) ConfigWithTemplate(tpl string) ([]byte, error) { - tmpl, err := template.New("dm-master").Parse(tpl) + tmpl, err := template.New("dm-master").Parse(string(tpl)) if err != nil { - return nil, err + return err } if c.Name == "" { - return nil, errors.New("empty name") - } - for _, s := range c.Endpoints { - if s.Name == "" { - return nil, errors.New("empty name") - } + return errors.New("empty name") } content := bytes.NewBufferString("") if err := tmpl.Execute(content, c); err != nil { - return nil, err + return err } - return content.Bytes(), nil + return utils.WriteFile(file, content.Bytes(), 0755) } // DMMasterScaleScript represent the data to generate dm-master config on scaling type DMMasterScaleScript struct { - DMMasterScript -} - -// NewDMMasterScaleScript return a new DMMasterScaleScript -func NewDMMasterScaleScript(name, ip, deployDir, dataDir, logDir string, enableTLS bool) *DMMasterScaleScript { - return &DMMasterScaleScript{*NewDMMasterScript(name, ip, deployDir, dataDir, logDir, enableTLS)} -} + Name string + V1SourcePath string + MasterAddr string + AdvertiseAddr string + PeerURL string + AdvertisePeerURL string + Join string -// WithScheme set Scheme field of DMMasterScaleScript -func (c *DMMasterScaleScript) WithScheme(scheme string) *DMMasterScaleScript { - c.Scheme = scheme - return c -} + DeployDir string + DataDir string + LogDir string -// WithPort set Port field of DMMasterScript -func (c *DMMasterScaleScript) WithPort(port int) *DMMasterScaleScript { - c.Port = port - return c + NumaNode string } -// WithNumaNode set NumaNode field of DMMasterScript -func (c *DMMasterScaleScript) WithNumaNode(numa string) *DMMasterScaleScript { - c.NumaNode = numa - return c -} - -// WithPeerPort set PeerPort field of DMMasterScript -func (c *DMMasterScaleScript) WithPeerPort(port int) *DMMasterScaleScript { - c.PeerPort = port - return c -} - -// AppendEndpoints add new DMMasterScript to Endpoints field -func (c *DMMasterScaleScript) AppendEndpoints(ends ...*DMMasterScript) *DMMasterScaleScript { - c.Endpoints = append(c.Endpoints, ends...) - return c -} - -// Config generate the config file data. -func (c *DMMasterScaleScript) Config() ([]byte, error) { +// ConfigToFile write config content to specific path +func (c *DMMasterScaleScript) ConfigToFile(file string) error { fp := path.Join("templates", "scripts", "run_dm-master_scale.sh.tpl") tpl, err := embed.ReadTemplate(fp) if err != nil { - return nil, err + return err } - return c.ConfigWithTemplate(string(tpl)) -} - -// ConfigToFile write config content to specific path -func (c *DMMasterScaleScript) ConfigToFile(file string) error { - config, err := c.Config() + tmpl, err := template.New("dm-master").Parse(string(tpl)) if err != nil { return err } - return os.WriteFile(file, config, 0755) + + if c.Name == "" { + return errors.New("empty name") + } + + content := bytes.NewBufferString("") + if err := tmpl.Execute(content, c); err != nil { + return err + } + return utils.WriteFile(file, content.Bytes(), 0755) } diff --git a/pkg/cluster/template/scripts/dm_worker.go b/pkg/cluster/template/scripts/dm_worker.go index 1bdd1f6536..7f7a3894eb 100644 --- a/pkg/cluster/template/scripts/dm_worker.go +++ b/pkg/cluster/template/scripts/dm_worker.go @@ -15,83 +15,46 @@ package scripts import ( "bytes" - "os" "path" "text/template" "github.com/pingcap/tiup/embed" + "github.com/pingcap/tiup/pkg/utils" ) // DMWorkerScript represent the data to generate TiDB config type DMWorkerScript struct { - Name string - IP string - Port int + Name string + WorkerAddr string + AdvertiseAddr string + Join string + DeployDir string LogDir string - NumaNode string - Endpoints []*DMMasterScript -} -// NewDMWorkerScript returns a DMWorkerScript with given arguments -func NewDMWorkerScript(name, ip, deployDir, logDir string) *DMWorkerScript { - return &DMWorkerScript{ - Name: name, - IP: ip, - Port: 8262, - DeployDir: deployDir, - LogDir: logDir, - } -} - -// WithPort set Port field of DMWorkerScript -func (c *DMWorkerScript) WithPort(port int) *DMWorkerScript { - c.Port = port - return c -} - -// WithNumaNode set NumaNode field of DMWorkerScript -func (c *DMWorkerScript) WithNumaNode(numa string) *DMWorkerScript { - c.NumaNode = numa - return c -} - -// AppendEndpoints add new PDScript to Endpoints field -func (c *DMWorkerScript) AppendEndpoints(ends ...*DMMasterScript) *DMWorkerScript { - c.Endpoints = append(c.Endpoints, ends...) - return c -} + NumaNode string -// Config generate the config file data. -func (c *DMWorkerScript) Config() ([]byte, error) { - fp := path.Join("templates", "scripts", "run_dm-worker.sh.tpl") - tpl, err := embed.ReadTemplate(fp) - if err != nil { - return nil, err - } - return c.ConfigWithTemplate(string(tpl)) + Endpoints []*DMMasterScript + IP string + Port int } // ConfigToFile write config content to specific path func (c *DMWorkerScript) ConfigToFile(file string) error { - config, err := c.Config() + fp := path.Join("templates", "scripts", "run_dm-worker.sh.tpl") + tpl, err := embed.ReadTemplate(fp) if err != nil { return err } - return os.WriteFile(file, config, 0755) -} - -// ConfigWithTemplate generate the DM worker config content by tpl -func (c *DMWorkerScript) ConfigWithTemplate(tpl string) ([]byte, error) { - tmpl, err := template.New("dm-worker").Parse(tpl) + tmpl, err := template.New("dm-worker").Parse(string(tpl)) if err != nil { - return nil, err + return err } content := bytes.NewBufferString("") if err := tmpl.Execute(content, c); err != nil { - return nil, err + return err } - return content.Bytes(), nil + return utils.WriteFile(file, content.Bytes(), 0755) } diff --git a/pkg/cluster/template/scripts/drainer.go b/pkg/cluster/template/scripts/drainer.go index 4ed9b149a3..139a898155 100644 --- a/pkg/cluster/template/scripts/drainer.go +++ b/pkg/cluster/template/scripts/drainer.go @@ -15,11 +15,11 @@ package scripts import ( "bytes" - "os" "path" "text/template" "github.com/pingcap/tiup/embed" + "github.com/pingcap/tiup/pkg/utils" ) // DrainerScript represent the data to generate drainer config @@ -55,5 +55,5 @@ func (c *DrainerScript) ConfigToFile(file string) error { return err } - return os.WriteFile(file, content.Bytes(), 0755) + return utils.WriteFile(file, content.Bytes(), 0755) } diff --git a/pkg/cluster/template/scripts/grafana.go b/pkg/cluster/template/scripts/grafana.go index 4900c2ceda..d7e78d2fec 100644 --- a/pkg/cluster/template/scripts/grafana.go +++ b/pkg/cluster/template/scripts/grafana.go @@ -15,11 +15,11 @@ package scripts import ( "bytes" - "os" "path" "text/template" "github.com/pingcap/tiup/embed" + "github.com/pingcap/tiup/pkg/utils" ) // GrafanaScript represent the data to generate Grafana config @@ -45,5 +45,5 @@ func (c *GrafanaScript) ConfigToFile(file string) error { return err } - return os.WriteFile(file, content.Bytes(), 0755) + return utils.WriteFile(file, content.Bytes(), 0755) } diff --git a/pkg/cluster/template/scripts/monitoring.go b/pkg/cluster/template/scripts/monitoring.go index 5333a2f2b2..70c647d9a8 100644 --- a/pkg/cluster/template/scripts/monitoring.go +++ b/pkg/cluster/template/scripts/monitoring.go @@ -15,11 +15,11 @@ package scripts import ( "bytes" - "os" "path" "text/template" "github.com/pingcap/tiup/embed" + "github.com/pingcap/tiup/pkg/utils" ) // PrometheusScript represent the data to generate Prometheus config @@ -54,5 +54,5 @@ func (c *PrometheusScript) ConfigToFile(file string) error { return err } - return os.WriteFile(file, content.Bytes(), 0755) + return utils.WriteFile(file, content.Bytes(), 0755) } diff --git a/pkg/cluster/template/scripts/node_exporter.go b/pkg/cluster/template/scripts/node_exporter.go index da82cbaeb0..3c437f974f 100644 --- a/pkg/cluster/template/scripts/node_exporter.go +++ b/pkg/cluster/template/scripts/node_exporter.go @@ -15,11 +15,11 @@ package scripts import ( "bytes" - "os" "path" "text/template" "github.com/pingcap/tiup/embed" + "github.com/pingcap/tiup/pkg/utils" ) // NodeExporterScript represent the data to generate NodeExporter config @@ -67,7 +67,7 @@ func (c *NodeExporterScript) ConfigToFile(file string) error { if err != nil { return err } - return os.WriteFile(file, config, 0755) + return utils.WriteFile(file, config, 0755) } // ConfigWithTemplate generate the NodeExporter config content by tpl diff --git a/pkg/cluster/template/scripts/pd.go b/pkg/cluster/template/scripts/pd.go index d4e2c1b8cc..a09eb4be5b 100644 --- a/pkg/cluster/template/scripts/pd.go +++ b/pkg/cluster/template/scripts/pd.go @@ -16,11 +16,11 @@ package scripts import ( "bytes" "errors" - "os" "path" "text/template" "github.com/pingcap/tiup/embed" + "github.com/pingcap/tiup/pkg/utils" ) // PDScript represent the data to generate pd config @@ -30,18 +30,13 @@ type PDScript struct { AdvertiseClientURL string PeerURL string AdvertisePeerURL string - DeployDir string - DataDir string - LogDir string InitialCluster string - IP string - ListenHost string - ClientPort int - PeerPort int - TLSEnabled bool - NumaNode string - Endpoints []*PDScript + DeployDir string + DataDir string + LogDir string + + NumaNode string } // ConfigToFile write config content to specific path @@ -66,7 +61,7 @@ func (c *PDScript) ConfigToFile(file string) error { return err } - return os.WriteFile(file, content.Bytes(), 0755) + return utils.WriteFile(file, content.Bytes(), 0755) } // PDScaleScript represent the data to generate pd config on scaling @@ -77,7 +72,7 @@ type PDScaleScript struct { // NewPDScaleScript return a new PDScaleScript func NewPDScaleScript(pdScript *PDScript, join string) *PDScaleScript { - return &PDScaleScript{PDScript:*pdScript, Join: join} + return &PDScaleScript{PDScript: *pdScript, Join: join} } // ConfigToFile write config content to specific path @@ -102,5 +97,5 @@ func (c *PDScaleScript) ConfigToFile(file string) error { return err } - return os.WriteFile(file, content.Bytes(), 0755) + return utils.WriteFile(file, content.Bytes(), 0755) } diff --git a/pkg/cluster/template/scripts/pump.go b/pkg/cluster/template/scripts/pump.go index 4a050c812b..79ef7cf0c5 100644 --- a/pkg/cluster/template/scripts/pump.go +++ b/pkg/cluster/template/scripts/pump.go @@ -15,11 +15,11 @@ package scripts import ( "bytes" - "os" "path" "text/template" "github.com/pingcap/tiup/embed" + "github.com/pingcap/tiup/pkg/utils" ) // PumpScript represent the data to generate Pump config @@ -53,5 +53,5 @@ func (c *PumpScript) ConfigToFile(file string) error { return err } - return os.WriteFile(file, content.Bytes(), 0755) + return utils.WriteFile(file, content.Bytes(), 0755) } diff --git a/pkg/cluster/template/scripts/tidb.go b/pkg/cluster/template/scripts/tidb.go index c1f6a892db..6d19db2b25 100644 --- a/pkg/cluster/template/scripts/tidb.go +++ b/pkg/cluster/template/scripts/tidb.go @@ -15,11 +15,11 @@ package scripts import ( "bytes" - "os" "path" "text/template" "github.com/pingcap/tiup/embed" + "github.com/pingcap/tiup/pkg/utils" ) // TiDBScript represent the data to generate TiDB config @@ -56,5 +56,5 @@ func (c *TiDBScript) ConfigToFile(file string) error { return err } - return os.WriteFile(file, content.Bytes(), 0755) + return utils.WriteFile(file, content.Bytes(), 0755) } diff --git a/pkg/cluster/template/scripts/tiflash.go b/pkg/cluster/template/scripts/tiflash.go index 4708902edd..c5b1700936 100644 --- a/pkg/cluster/template/scripts/tiflash.go +++ b/pkg/cluster/template/scripts/tiflash.go @@ -15,11 +15,11 @@ package scripts import ( "bytes" - "os" "path" "text/template" "github.com/pingcap/tiup/embed" + "github.com/pingcap/tiup/pkg/utils" ) // TiFlashScript represent the data to generate TiFlash config @@ -51,5 +51,5 @@ func (c *TiFlashScript) ConfigToFile(file string) error { return err } - return os.WriteFile(file, content.Bytes(), 0755) + return utils.WriteFile(file, content.Bytes(), 0755) } diff --git a/pkg/cluster/template/scripts/tikv.go b/pkg/cluster/template/scripts/tikv.go index f70a698b53..d2e1aa0097 100644 --- a/pkg/cluster/template/scripts/tikv.go +++ b/pkg/cluster/template/scripts/tikv.go @@ -15,11 +15,11 @@ package scripts import ( "bytes" - "os" "path" "text/template" "github.com/pingcap/tiup/embed" + "github.com/pingcap/tiup/pkg/utils" ) // TiKVScript represent the data to generate TiKV config @@ -57,5 +57,5 @@ func (c *TiKVScript) ConfigToFile(file string) error { return err } - return os.WriteFile(file, content.Bytes(), 0755) + return utils.WriteFile(file, content.Bytes(), 0755) } diff --git a/pkg/cluster/template/scripts/tikv_cdc.go b/pkg/cluster/template/scripts/tikv_cdc.go index b709d3d79f..a6df29f2cf 100644 --- a/pkg/cluster/template/scripts/tikv_cdc.go +++ b/pkg/cluster/template/scripts/tikv_cdc.go @@ -15,11 +15,11 @@ package scripts import ( "bytes" - "os" "path" "text/template" "github.com/pingcap/tiup/embed" + "github.com/pingcap/tiup/pkg/utils" ) // TiKVCDCScript represent the data to generate cdc config @@ -55,5 +55,5 @@ func (c *TiKVCDCScript) ConfigToFile(file string) error { return err } - return os.WriteFile(file, content.Bytes(), 0755) + return utils.WriteFile(file, content.Bytes(), 0755) } diff --git a/pkg/cluster/template/scripts/tiproxy.go b/pkg/cluster/template/scripts/tiproxy.go new file mode 100644 index 0000000000..d0bd8d5980 --- /dev/null +++ b/pkg/cluster/template/scripts/tiproxy.go @@ -0,0 +1,49 @@ +// Copyright 2023 PingCAP, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// See the License for the specific language governing permissions and +// limitations under the License. + +package scripts + +import ( + "bytes" + "path" + "text/template" + + "github.com/pingcap/tiup/embed" + "github.com/pingcap/tiup/pkg/utils" +) + +// TiProxyScript represent the data to generate tiproxy config +type TiProxyScript struct { + DeployDir string + NumaNode string +} + +// ConfigToFile write config content to specific file. +func (c *TiProxyScript) ConfigToFile(file string) error { + fp := path.Join("templates", "scripts", "run_tiproxy.sh.tpl") + tpl, err := embed.ReadTemplate(fp) + if err != nil { + return err + } + tmpl, err := template.New("TiProxy").Parse(string(tpl)) + if err != nil { + return err + } + + content := bytes.NewBufferString("") + if err := tmpl.Execute(content, c); err != nil { + return err + } + + return utils.WriteFile(file, content.Bytes(), 0755) +} diff --git a/pkg/cluster/template/scripts/tispark.go b/pkg/cluster/template/scripts/tispark.go index 2678f23e63..8a0f517577 100644 --- a/pkg/cluster/template/scripts/tispark.go +++ b/pkg/cluster/template/scripts/tispark.go @@ -15,8 +15,9 @@ package scripts import ( "bytes" - "os" "text/template" + + "github.com/pingcap/tiup/pkg/utils" ) // TiSparkEnv represent the data to generate TiSpark environment config @@ -86,7 +87,7 @@ func (c *TiSparkEnv) ScriptToFile(file string) error { if err != nil { return err } - return os.WriteFile(file, script, 0755) + return utils.WriteFile(file, script, 0755) } // ScriptWithTemplate parses the template file diff --git a/pkg/cluster/template/systemd/system.go b/pkg/cluster/template/systemd/system.go index 5bf4af1fd2..841a15be6a 100644 --- a/pkg/cluster/template/systemd/system.go +++ b/pkg/cluster/template/systemd/system.go @@ -15,11 +15,11 @@ package system import ( "bytes" - "os" "path" "text/template" "github.com/pingcap/tiup/embed" + "github.com/pingcap/tiup/pkg/utils" ) // Config represent the data to generate systemd config @@ -84,7 +84,7 @@ func (c *Config) ConfigToFile(file string) error { if err != nil { return err } - return os.WriteFile(file, config, 0755) + return utils.WriteFile(file, config, 0755) } // Config generate the config file data. diff --git a/pkg/cluster/template/systemd/tispark.go b/pkg/cluster/template/systemd/tispark.go index 4646941306..bb9c59ad3f 100644 --- a/pkg/cluster/template/systemd/tispark.go +++ b/pkg/cluster/template/systemd/tispark.go @@ -15,12 +15,12 @@ package system import ( "bytes" - "os" "path" "strings" "text/template" "github.com/pingcap/tiup/embed" + "github.com/pingcap/tiup/pkg/utils" ) // TiSparkConfig represent the data to generate systemd config @@ -55,7 +55,7 @@ func (c *TiSparkConfig) ConfigToFile(file string) error { if err != nil { return err } - return os.WriteFile(file, config, 0755) + return utils.WriteFile(file, config, 0755) } // Config generate the config file data. diff --git a/pkg/crypto/rand/passwd.go b/pkg/crypto/rand/passwd.go index 132cd3d2d4..924c683091 100644 --- a/pkg/crypto/rand/passwd.go +++ b/pkg/crypto/rand/passwd.go @@ -22,7 +22,7 @@ const ( lowerLetters = "abcdefghijkmnpqrstuvwxyz" upperLetters = "ABCDEFGHJKLMNPQRSTUVWXYZ" digits = "0123456789" - symbols = "!@#$%^&*+-=_" + symbols = "@^*+-_" ) // Password generates a random password diff --git a/pkg/environment/env.go b/pkg/environment/env.go index 4bb55dd368..7c9d86bd29 100644 --- a/pkg/environment/env.go +++ b/pkg/environment/env.go @@ -30,11 +30,6 @@ import ( "golang.org/x/mod/semver" ) -// Name of components -const ( - tiupName = "tiup" -) - var ( // ErrInstallFirst indicates that a component/version is not installed ErrInstallFirst = errors.New("component not installed") @@ -143,9 +138,6 @@ func (env *Environment) UpdateComponents(specs []string, nightly, force bool) er var v1specs []repository.ComponentSpec for _, spec := range specs { component, v := ParseCompVersion(spec) - if component == tiupName { - continue - } if v == "" && nightly { v = utils.NightlyVersionAlias } @@ -212,6 +204,9 @@ func (env *Environment) SelectInstalledVersion(component string, ver utils.Versi }) errInstallFirst := errors.Annotatef(ErrInstallFirst, "use `tiup install %s` to install component `%s` first", component, component) + if !ver.IsEmpty() { + errInstallFirst = errors.Annotatef(ErrInstallFirst, "use `tiup install %s:%s` to install specified version", component, ver.String()) + } if ver.IsEmpty() || string(ver) == utils.NightlyVersionAlias { var selected utils.Version @@ -290,6 +285,45 @@ func (env *Environment) BinaryPath(component string, ver utils.Version) (string, return env.v1Repo.BinaryPath(installPath, component, ver.String()) } +// Link add soft link to $TIUP_HOME/bin/ +func (env *Environment) Link(component string, version utils.Version) error { + version, err := env.SelectInstalledVersion(component, version) + if err != nil { + return err + } + binPath, err := env.BinaryPath(component, version) + if err != nil { + return err + } + + target := env.LocalPath("bin", filepath.Base(binPath)) + backup := target + ".old" + exist := true + _, err = os.Stat(target) + if err != nil { + if !os.IsNotExist(err) { + return err + } + exist = false + } + if exist { + if err := os.Rename(target, backup); err != nil { + fmt.Printf("Backup of `%s` to `%s` failed.\n", target, backup) + return err + } + } + + fmt.Printf("package %s provides these executables: %s\n", component, filepath.Base(binPath)) + + err = os.Symlink(binPath, target) + if err != nil { + defer func() { _ = os.Rename(backup, target) }() + } else { + defer func() { _ = os.Remove(backup) }() + } + return err +} + // ParseCompVersion parses component part from [:version] specification func ParseCompVersion(spec string) (string, utils.Version) { if strings.Contains(spec, ":") { diff --git a/pkg/environment/history.go b/pkg/environment/history.go index a71e303fb7..d7c0ef3a2a 100644 --- a/pkg/environment/history.go +++ b/pkg/environment/history.go @@ -61,7 +61,7 @@ func HistoryRecord(env *Environment, command []string, date time.Time, code int) historyPath := env.LocalPath(HistoryDir) if utils.IsNotExist(historyPath) { - err := os.MkdirAll(historyPath, 0755) + err := utils.MkdirAll(historyPath, 0755) if err != nil { return err } diff --git a/pkg/exec/run.go b/pkg/exec/run.go index caf5e2324d..ed5fb136e8 100644 --- a/pkg/exec/run.go +++ b/pkg/exec/run.go @@ -124,12 +124,12 @@ func PrepareCommand(p *PrepareCommandParams) (*exec.Cmd, error) { binPath := p.BinPath installPath := filepath.Dir(binPath) - if err := os.MkdirAll(p.InstanceDir, 0755); err != nil { + if err := utils.MkdirAll(p.InstanceDir, 0755); err != nil { return nil, err } sd := env.LocalPath(localdata.StorageParentDir, p.Component) - if err := os.MkdirAll(sd, 0755); err != nil { + if err := utils.MkdirAll(sd, 0755); err != nil { return nil, err } @@ -176,7 +176,7 @@ func cmdCheckUpdate(component string, version utils.Version, timeoutSec int) { // timeout for check update go func() { time.Sleep(time.Duration(timeoutSec) * time.Second) - updateC <- color.RedString("timeout!") + updateC <- color.YellowString("timeout(%ds)!", timeoutSec) }() go func() { diff --git a/pkg/localdata/profile.go b/pkg/localdata/profile.go index 264335fc2b..f9a28816ab 100644 --- a/pkg/localdata/profile.go +++ b/pkg/localdata/profile.go @@ -121,10 +121,10 @@ func (p *Profile) ComponentInstalledPath(component string, version utils.Version func (p *Profile) SaveTo(path string, data []byte, perm os.FileMode) error { fullPath := filepath.Join(p.root, path) // create sub directory if needed - if err := os.MkdirAll(filepath.Dir(fullPath), 0755); err != nil { + if err := utils.MkdirAll(filepath.Dir(fullPath), 0755); err != nil { return errors.Trace(err) } - return os.WriteFile(fullPath, data, perm) + return utils.WriteFile(fullPath, data, perm) } // WriteJSON writes struct to a file (in the profile directory) in JSON format diff --git a/pkg/logger/audit.go b/pkg/logger/audit.go index 887c195a9c..39e933601e 100644 --- a/pkg/logger/audit.go +++ b/pkg/logger/audit.go @@ -50,7 +50,7 @@ func OutputAuditLogToFileIfEnabled(dir, fileSuffix string) error { return nil } - if err := utils.CreateDir(dir); err != nil { + if err := utils.MkdirAll(dir, 0755); err != nil { return err } diff --git a/pkg/logger/debug.go b/pkg/logger/debug.go index f683e43879..9f4655b954 100644 --- a/pkg/logger/debug.go +++ b/pkg/logger/debug.go @@ -22,6 +22,7 @@ import ( "github.com/pingcap/tiup/pkg/localdata" "github.com/pingcap/tiup/pkg/tui" + "github.com/pingcap/tiup/pkg/utils" "go.uber.org/zap" "go.uber.org/zap/zapcore" ) @@ -41,7 +42,7 @@ func OutputDebugLog(prefix string) { profile := localdata.InitProfile() logDir = profile.Path("logs") } - if err := os.MkdirAll(logDir, 0755); err != nil { + if err := utils.MkdirAll(logDir, 0755); err != nil { _, _ = fmt.Fprintf(os.Stderr, "\nCreate debug logs(%s) directory failed %v.\n", logDir, err) return } @@ -50,7 +51,7 @@ func OutputDebugLog(prefix string) { fileName := time.Now().Format(fmt.Sprintf("%s-debug-2006-01-02-15-04-05.log", prefix)) filePath := filepath.Join(logDir, fileName) - err := os.WriteFile(filePath, debugBuffer.Bytes(), 0644) + err := utils.WriteFile(filePath, debugBuffer.Bytes(), 0644) if err != nil { _, _ = tui.ColorWarningMsg.Fprint(os.Stderr, "\nWarn: Failed to write error debug log.\n") } else { diff --git a/pkg/repository/clone_mirror.go b/pkg/repository/clone_mirror.go index 2d26e7d09e..2332e888a5 100644 --- a/pkg/repository/clone_mirror.go +++ b/pkg/repository/clone_mirror.go @@ -47,7 +47,6 @@ type CloneOptions struct { // CloneMirror clones a local mirror from the remote repository func CloneMirror(repo *V1Repository, components []string, - tidbClusterVersionMapper func(string) string, targetDir string, selectedVersions []string, options CloneOptions) error { @@ -57,7 +56,7 @@ func CloneMirror(repo *V1Repository, fmt.Printf("Start to clone mirror, targetDir is %s, source mirror is %s, selectedVersions are [%s]\n", targetDir, repo.Mirror().Source(), strings.Join(selectedVersions, ",")) fmt.Println("If this does not meet expectations, please abort this process, read `tiup mirror clone --help` and run again") - if err := os.MkdirAll(targetDir, 0755); err != nil { + if err := utils.MkdirAll(targetDir, 0755); err != nil { return err } @@ -65,10 +64,10 @@ func CloneMirror(repo *V1Repository, tmpDir := filepath.Join(targetDir, fmt.Sprintf("_tmp_%d", time.Now().UnixNano())) keyDir := filepath.Join(targetDir, "keys") - if err := os.MkdirAll(tmpDir, 0755); err != nil { + if err := utils.MkdirAll(tmpDir, 0755); err != nil { return err } - if err := os.MkdirAll(keyDir, 0755); err != nil { + if err := utils.MkdirAll(keyDir, 0755); err != nil { return err } defer os.RemoveAll(tmpDir) @@ -149,7 +148,7 @@ func CloneMirror(repo *V1Repository, snapshot := v1manifest.NewSnapshot(initTime) snapshot.SetExpiresAt(expiresAt) - componentManifests, err := cloneComponents(repo, components, selectedVersions, tidbClusterVersionMapper, targetDir, tmpDir, options) + componentManifests, err := cloneComponents(repo, components, selectedVersions, targetDir, tmpDir, options) if err != nil { return err } @@ -246,7 +245,6 @@ func CloneMirror(repo *V1Repository, func cloneComponents(repo *V1Repository, components, selectedVersions []string, - tidbClusterVersionMapper func(string) string, targetDir, tmpDir string, options CloneOptions) (map[string]*v1manifest.Component, error) { compManifests := map[string]*v1manifest.Component{} @@ -265,7 +263,10 @@ func cloneComponents(repo *V1Repository, return nil, errors.Annotatef(err, "fetch component '%s' manifest failed", name) } - vs := combineVersions(options.Components[name], tidbClusterVersionMapper, manifest, options.OSs, options.Archs, selectedVersions) + vs, err := combineVersions(options.Components[name], manifest, options.OSs, options.Archs, selectedVersions) + if err != nil { + return nil, err + } var newManifest *v1manifest.Component if options.Full { @@ -317,8 +318,8 @@ func cloneComponents(repo *V1Repository, continue } name, versionItem := name, versionItem + tickets <- struct{}{} errG.Go(func() error { - tickets <- struct{}{} defer func() { <-tickets }() err := download(targetDir, tmpDir, repo, &versionItem) @@ -419,33 +420,18 @@ func checkVersion(options CloneOptions, versions set.StringSet, version string) return false } -func combineVersions(versions *[]string, - tidbClusterVersionMapper func(string) string, +func combineVersions(componentVersions *[]string, manifest *v1manifest.Component, oss, archs, - selectedVersions []string) set.StringSet { - if (versions == nil || len(*versions) < 1) && len(selectedVersions) < 1 { - return nil - } - - if bindver := tidbClusterVersionMapper(manifest.ID); bindver != "" { - return set.NewStringSet(bindver) + globalVersions []string) (set.StringSet, error) { + if (componentVersions == nil || len(*componentVersions) < 1) && len(globalVersions) < 1 { + return nil, errors.New("no version specified") } result := set.NewStringSet() - if versions != nil && len(*versions) > 0 { - result = set.NewStringSet(*versions...) - } - - // Some components version binding to TiDB - coreSuites := set.NewStringSet("tidb", "tikv", "pd", "tiflash", "prometheus", "grafana", "ctl", "cdc") - for _, os := range oss { for _, arch := range archs { platform := PlatformString(os, arch) - versions := manifest.VersionList(platform) - if versions == nil { - continue - } + versionList := manifest.VersionList(platform) // set specified version with latest tag if result.Exist(utils.LatestVersionAlias) { @@ -455,41 +441,63 @@ func combineVersions(versions *[]string, } } - for _, selectedVersion := range selectedVersions { - if selectedVersion == utils.NightlyVersionAlias { - selectedVersion = manifest.Nightly - } + if componentVersions != nil && len(*componentVersions) > 0 { + for _, selectedVersion := range *componentVersions { + fmt.Printf("%s %s/%s selected version is %s\n", manifest.ID, os, arch, selectedVersion) + if selectedVersion == utils.NightlyVersionAlias { + selectedVersion = manifest.Nightly + } - if selectedVersion == utils.LatestVersionAlias { - latest := manifest.LatestVersion(platform) - if latest == "" { - continue + if selectedVersion == utils.LatestVersionAlias { + latest := manifest.LatestVersion(platform) + if latest != "" { + fmt.Printf("%s %s/%s found the lastest version %s\n", manifest.ID, os, arch, latest) + // set latest version + selectedVersion = latest + } } - fmt.Printf("%s %s/%s found the lastest version %s\n", manifest.ID, os, arch, latest) - // set latest version - selectedVersion = latest + _, found := versionList[selectedVersion] + if !found { + return nil, errors.Errorf("version %s not found in %s %s/%s", selectedVersion, manifest.ID, os, arch) + } + result.Insert(selectedVersion) } + } else { + for _, selectedVersion := range globalVersions { + if selectedVersion == utils.NightlyVersionAlias { + selectedVersion = manifest.Nightly + } - _, found := versions[selectedVersion] - // Some TiUP components won't be bound version with TiDB, if cannot find - // selected version we download the latest version to as a alternative - if !found && !coreSuites.Exist(manifest.ID) { - // Use the latest stable versionS if the selected version doesn't exist in specific platform - latest := manifest.LatestVersion(platform) - if latest == "" { - continue + if selectedVersion == utils.LatestVersionAlias { + latest := manifest.LatestVersion(platform) + if latest == "" { + continue + } + + fmt.Printf("%s %s/%s found the lastest version %s\n", manifest.ID, os, arch, latest) + // set latest version + selectedVersion = latest } - if selectedVersion != utils.LatestVersionAlias { - fmt.Printf("%s %s/%s %s not found, using %s instead.\n", manifest.ID, os, arch, selectedVersion, latest) + + _, found := versionList[selectedVersion] + // Some TiUP components won't be bound version with TiDB, if cannot find + // selected version we download the latest version to as a alternative + if !found { + // Use the latest stable versionS if the selected version doesn't exist in specific platform + latest := manifest.LatestVersion(platform) + if latest == "" { + continue + } + if selectedVersion != utils.LatestVersionAlias { + fmt.Printf("%s %s/%s %s not found, using %s instead.\n", manifest.ID, os, arch, selectedVersion, latest) + } + selectedVersion = latest } - selectedVersion = latest - } - if !result.Exist(selectedVersion) { result.Insert(selectedVersion) } } } } - return result + return result, nil } diff --git a/pkg/repository/mirror.go b/pkg/repository/mirror.go index 283cbafa67..7fb446f704 100644 --- a/pkg/repository/mirror.go +++ b/pkg/repository/mirror.go @@ -15,6 +15,7 @@ package repository import ( "bytes" + "crypto/tls" "encoding/json" stderrors "errors" "fmt" @@ -220,7 +221,7 @@ func (l *localFilesystem) Download(resource, targetDir string) error { } defer reader.Close() - if err := os.MkdirAll(targetDir, 0755); err != nil { + if err := utils.MkdirAll(targetDir, 0755); err != nil { return errors.Trace(err) } outPath := filepath.Join(targetDir, resource) @@ -293,6 +294,14 @@ func (l *httpMirror) downloadFile(url string, to string, maxSize int64) (io.Read }(time.Now()) client := grab.NewClient() + + // workaround to resolve cdn error "tls: protocol version not supported" + client.HTTPClient.(*http.Client).Transport = &http.Transport{ + Proxy: http.ProxyFromEnvironment, + // avoid using http/2 by setting non-nil TLSClientConfig + TLSClientConfig: &tls.Config{}, + } + client.UserAgent = fmt.Sprintf("tiup/%s", version.NewTiUPVersion().SemVer()) req, err := grab.NewRequest(to, url) if err != nil { @@ -326,6 +335,7 @@ L: } progress.SetCurrent(resp.BytesComplete()) case <-resp.Done: + progress.SetCurrent(resp.BytesComplete()) progress.Finish() break L } @@ -435,7 +445,7 @@ func (l *httpMirror) Publish(manifest *v1manifest.Manifest, info model.Component } manifestAddr := fmt.Sprintf("%s/api/v1/component/%s/%s%s", l.Source(), sid, manifest.Signed.(*v1manifest.Component).ID, qstr) - client := http.Client{Timeout: time.Minute} + client := http.Client{Timeout: 5 * time.Minute} resp, err := client.Post(manifestAddr, "text/json", bodyBuf) if err != nil { return err @@ -504,7 +514,7 @@ func (l *httpMirror) Download(resource, targetDir string) error { return err } - if err := os.MkdirAll(targetDir, 0755); err != nil { + if err := utils.MkdirAll(targetDir, 0755); err != nil { return errors.Trace(err) } return utils.Move(tmpFilePath, dstFilePath) @@ -546,7 +556,7 @@ func (l *MockMirror) Download(resource, targetDir string) error { return errors.Annotatef(ErrNotFound, "resource %s", resource) } - if err := os.MkdirAll(targetDir, 0755); err != nil { + if err := utils.MkdirAll(targetDir, 0755); err != nil { return err } target := filepath.Join(targetDir, resource) diff --git a/pkg/repository/progress.go b/pkg/repository/progress.go index 951dc99a88..f8df7e2cdf 100644 --- a/pkg/repository/progress.go +++ b/pkg/repository/progress.go @@ -52,6 +52,5 @@ func (p *ProgressBar) SetCurrent(size int64) { // Finish implement the DownloadProgress interface func (p *ProgressBar) Finish() { - p.bar.SetCurrent(p.size) p.bar.Finish() } diff --git a/pkg/repository/store/sync.go b/pkg/repository/store/sync.go index ae3cb2bc81..afcc2cc74d 100644 --- a/pkg/repository/store/sync.go +++ b/pkg/repository/store/sync.go @@ -57,7 +57,7 @@ func (s *fsSyncer) Sync(srcDir string) error { unix := time.Now().UnixNano() dstDir := path.Join(s.root, fmt.Sprintf("commit-%d", unix)) - if err := os.MkdirAll(dstDir, 0755); err != nil { + if err := utils.MkdirAll(dstDir, 0755); err != nil { return err } diff --git a/pkg/repository/testdata/manifests/root.json b/pkg/repository/testdata/manifests/root.json index eb7de68e7b..ca16fcc0cb 100644 --- a/pkg/repository/testdata/manifests/root.json +++ b/pkg/repository/testdata/manifests/root.json @@ -1 +1 @@ -{"signatures":[{"keyid":"a61b695e2b86097d993e94e99fd15ec6d8fc8e9522948c9ff21c2f2c881093ae","sig":"aptcBUxk5mIY3a2lqNChLU6dXUP/tcaIiwv7e/nQFZjsL3YOm9VG8OQzc4Sy+sv1DqFDvmwNZVp4FyfbPluXRyRgMRYJ194Z9Z1GXHQBmaWaVhMX9qpwzcSe8XNsReaDugHl+iJYfFif34vVm6fbhQgwxggXgJ37TbScVe3OB4LZ7RuSo3RExj3ojMkZr5wWtb4nFKd0VwfGH7KROmz06Pdok/k5PmI7spBLCFUWSAheVThA4jFXtlFPXGmbgBnPMRaRR5uWf59QFD0YQ4xTsJvWLHVmLtIgvDQnouVKYURXooCbYj/OfnvWIQIy3HzuEJC09ZQJokzpluhXaI8J+g=="},{"keyid":"b128ee6a42e2665bd45aa2fa4a7b6e098cfedb3911154f300634b2c056214b9e","sig":"wENeklsuDdoRNLCvvzsD3sJRymbAjMNT/ZTWsPX7tdPtfvq6+jyvvAgXiPMM92Kvb/WUVjBp5lwD5txGfD/2QLWRER9eDbz0XoZ1hGbAERQCWW7BZIc0tgc7EDLoPUxNQBMVYvHi1XTn1kEZqXHu8lrDLSpL2jIRMJOuLqJv6mmBraxGOc+loIwH2hQkXCvp8C8fW9M/bK5YKzuV3FafKyLuCkqMV7AIkKH2PztPXEVDQ3Mhrxzz9P1q7d6E07Hee9FP/xSm2B4MR9eqHiQTJW+aH/9BI60OaWLNxOg2GEr3t72j7mgDZq4d5i99zWwDG3YyWNw7AXtcuPPvKl6Vzg=="},{"keyid":"818c3ab5ddd59a79d4cff76ebb594be56e5a630c05cfcae382f32e07899e7ab7","sig":"PVoqno+m+HiHzkE+wMskafuHU/WUGw/eT/QRkXKgmAdDce6C5yvEVM1H37OpMPvfAjf70Q9opdaxBpkiTo3637EJbHGo7Eqs61hyIYecbYP9/LMinBSQZNOaTdtMUMYB28Cm9rERZCekxs2izLnkwX0py+V3IvfATOKy8jes13fiUF44A27J0X0n8601WicrOrDK37c0pSg66sd4xg5TucMcDsdxlGRW6+puvXLjAt2xk+qHSzbFEoMramfagWBF+hveKcv+buoZ81+f/H+3VBiNAxtTQLgDDqHI5IqAf1HlT20ThNimmN9q8bonirxYx6xhn0y4HvXLOn6nwMrqLQ=="},{"keyid":"9b3cea98f6f23cc11813b12d0526a1b6cfb3761008f0882c9caa8db742d63002","sig":"VOIfdL0qGiHvHmCAu4vLG2s5Ny1+zBZ7ZovPFo8eJT8ppquUDR7POzNBlk+xgteCfUFx56imsvBxKu77zCIe2yT2FcwYdpzcDzaJqOxsEhdusZa7gXdvDVs+JYoZMtRWrjo9InfXVpT8s8SOKglQAnuwcS5/Y+5T5czdXqeWOW9Lnk0Xpt09PUrW3tqrVKhWq1WqRSIayCNV1nnlM3Tj2240DiuYiTEIZn95p8ThB0ZzNLPVNzrvs5/7ZjTdmYCeDSNRwseHO3XFPUOtApEvSWNwjhH4x/YJOo64hqwnaO99utN4jGOKcXkHwQpKPDWKRecgo5+n8VzEZeA+XKLvow=="},{"keyid":"5607181203a2fb60b9d725109388ccb19ccdc236a4b1d1441fbea7ad07616c4a","sig":"QkVHaGa0K8UBc146leEndu2RFQY6tBzoZXGQ9osUyzLVpn3706LLxwD70zOqDhKHaWnnyf5sd3wJrzlhHXjpz3p380xgparaSyEky5d7OB7s5mBToy2bnA7f85IX3sRq7eyb4gcZfFTiZaTT+ygWzPClJaZsiF8p+XzQSkBXXycn3xl8XKtP81+CQOwhdhF9l1oXSEVFKjwMZC9WG4X9ipLS1MPr7J08YVxiMpMKiXtTQtlPdbqP1r65dWDtdJ1y7zMcatWjYGY4JakIrawIcfYJlYrMpAGUzyxx/Tm21d85b+NzyNN7sMpoUKLNHS3vUHOD2a9vSCiwFTjIrsxJDQ=="}],"signed":{"_type":"root","expires":"2023-07-26T11:18:30+08:00","roles":{"index":{"keys":{"7fce7ec4f9c36d51dec7ec96065bb64958b743e46ea8141da668cd2ce58a9e61":{"keytype":"rsa","keyval":{"public":"-----BEGIN PUBLIC KEY-----\nMIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEAn5kVA7MlBfSe7EBaExjl\nKbwoDkn1aYi74s29mFgtRo8nejbrVvZQMCIUhvKc0pFa/l9JD/QY6/nAOCE1lpzi\nwwNkSntfOo3p3HQIR+Ut7hZ4Sxfe/5JagGo3LQ+Hd3EJWUxyEfQ/Bff07F3XAbqM\n5+cKNrdsKWZJcPiJDW621qGwCx52f+gzl9bnFe4/hx34OUgirwqh5DS+LhIO+/yt\nbOiN1AyjQKlnb8lUnblElS4Njd+F4io5VzSrZYi2+4AbTkO6wLwbsWHMzXfv9qwn\nvllufOHpB6EwiQ/xBOMuvJJymHnZvs8AH4SuydQIXLaJuv1ysFaBs0KB/ktbakSK\nLwIDAQAB\n-----END PUBLIC KEY-----\n"},"scheme":"rsassa-pss-sha256"}},"threshold":1,"url":"/index.json"},"root":{"keys":{"5607181203a2fb60b9d725109388ccb19ccdc236a4b1d1441fbea7ad07616c4a":{"keytype":"rsa","keyval":{"public":"-----BEGIN PUBLIC KEY-----\nMIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEAyDwCfCl30vhyJW7fB1bs\npRYKtBKzl7o0qnJTm+IksjQ8RXxj8osUpMLmSvOzCaJ5Wxe+Pm1LpSTDbbubbgvd\nnmEFL6228sifviNIu2HlIl+agfzmXuJ9OBlzGUaI4gAd1Z6pF6+mjlcjz2PbWF84\nAbXZdK49uluqulp7HrGB/qNjGcIRUCHgDU4nnq0OkI1BZZSKm9ovonqDkIK76x/S\niAD9OjKsjQ/s57tE+5WTVObKpfrfK0JeHdpAUsA/2n4L1Z6FmZD4LZWqb0i+C7xj\nMElC99KtjlwRntcjeVWG9YjU8AcEN0n1gON9S2oRdyyAzDTgGb7WueDnn6qstt5w\nSQIDAQAB\n-----END PUBLIC KEY-----\n"},"scheme":"rsassa-pss-sha256"},"818c3ab5ddd59a79d4cff76ebb594be56e5a630c05cfcae382f32e07899e7ab7":{"keytype":"rsa","keyval":{"public":"-----BEGIN PUBLIC KEY-----\nMIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEA37V4zn8sYuhC5HbIos/6\nf3zhnr7WSnV0GtrOj2q6tCvo/1rGXDKbL8KJgLBHcQT6P732sm5BmZ+y8hRXZDqy\njTBqT70Vc0fGfCkH5Z0FU8WWlhQ2Ck3W20TxXPJY7p+wnuUCmcLzD7cKwUGOshjs\nyozqAMQ7C/HSmJFCUY8PaASvQCCg/YyMr3Ft8xpyc5IGmIi4NkVjf/JWIUFFv7kr\nIkRbT5K1nKJ2earot1TIcWUY2O4OvbhtueJ+IN0Lkdg+v98W/pvlF+ji24DBlcGw\nLdOeGRt5z6GnHKlAuZ/tbjRHeXnq1KxMETlXCgu2xCwsSK/10SE8oTaXbFoMHNBN\nAQIDAQAB\n-----END PUBLIC KEY-----\n"},"scheme":"rsassa-pss-sha256"},"9b3cea98f6f23cc11813b12d0526a1b6cfb3761008f0882c9caa8db742d63002":{"keytype":"rsa","keyval":{"public":"-----BEGIN PUBLIC KEY-----\nMIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEAsOgQkwLOh31QV9OpbO9v\n6o83durJFGPOnVXZiab83pKaSk7HEK9WzXBq0BaPvtFwSfROVdpgtopri5lZi+uH\naMKLUn5F8XRnSMl/7m5vM4XpZZYa4aQId4TWdbFtTu31eHGZ3eEC5nDRJ5NhZOJd\nKLFBu/xmxrh/eNZt4QbdWLZayjHnzyoy5AnfNTR6nJgPAv+rBOqyqT/r14q4Pngh\n3z0I3pNFr5qmxsp013XV+kgOW1F7zT7IMU8xRIgo85UWUNhax0/bjY/2NI1Z+WjR\nyhZmUBMVYWvfw97xDUrvBvrJxZPgg0lGvxJC6LF2dM7wgLaNx9khT6HMBVxjxLMs\nDQIDAQAB\n-----END PUBLIC KEY-----\n"},"scheme":"rsassa-pss-sha256"},"a61b695e2b86097d993e94e99fd15ec6d8fc8e9522948c9ff21c2f2c881093ae":{"keytype":"rsa","keyval":{"public":"-----BEGIN PUBLIC KEY-----\nMIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEAnayxhw6KeoKK+Ax9RW6v\n66YjrpRpGLewLmSSAzJGX8nL5/a2nEbXbeF9po265KcBSFWol8jLBsmG56ruwwxp\noWWhJPncqGqy8wMeRMmTf7ATGa+tk+To7UAQD0MYzt7rRlIdpqi9Us3J6076Z83k\n2sxFnX9sVflhOsotGWL7hmrn/CJWxKsO6OVCoqbIlnJV8xFazE2eCfaDTIEEEgnh\nLIGDsmv1AN8ImUIn/hyKcm1PfhDZrF5qhEVhfz5D8aX3cUcEJw8BvCaNloXyHf+y\nDKjqO/dJ7YFWVt7nPqOvaEkBQGMd54ETJ/BbO9r3WTsjXKleoPovBSQ/oOxApypb\nNQIDAQAB\n-----END PUBLIC KEY-----\n"},"scheme":"rsassa-pss-sha256"},"b128ee6a42e2665bd45aa2fa4a7b6e098cfedb3911154f300634b2c056214b9e":{"keytype":"rsa","keyval":{"public":"-----BEGIN PUBLIC KEY-----\nMIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEA0kgU3v3SxYzm5J6P+GPP\nmy6toBnKYtTViAVmpUJiIEjdZ9NLpoJU0na9q0CD8sIgo2Js/W/owJUvSj6rm8us\nsu/Ve5KsoJN6zca2am1uZ5IKnc48i0mCv76WXawCxM+NFGqSCMJcltlhj3fC/GDS\ngu+BiIbrgR1PgJf6Jk6l7uMJdN3TL6JJQcEC4lz+2hj5zoVNYkq06ZC79j2tPDCI\nkTAYGF/TAAVLH08/kGH5ZeRPlVKJ7cwW3OniLM5NeFnS8+shRNb6AYr7xju3Ikbw\nDo14ipIghBI0iAxn6Lvr/iilc7TM7RWJ4OiTrmK3SQSJ+U6H2N2/I5OGEHBEKzbA\nOQIDAQAB\n-----END PUBLIC KEY-----\n"},"scheme":"rsassa-pss-sha256"}},"threshold":3,"url":"/root.json"},"snapshot":{"keys":{"8660a9f40687fb33e6f8ad563f21ee81b9ce7b91c90827cc7ae2416c5e0e94e9":{"keytype":"rsa","keyval":{"public":"-----BEGIN PUBLIC KEY-----\nMIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEAqTZx29eJR5EumjqM4YTb\nFlKbim1GNYmtbCLH51BbU2lt46ddmfGvtGsxTD3mIZ/GEHVFv6Aei3xx5nIfhGP0\nrG78JRz394uU8Pd62DiIFWYizr5o+ZBZu29D2YK5ZtxoLFpgt0ibnINK2NcesDC8\nSqfIUbMiQFT6yB/MYD275SjfRGHOeYTPmKdjMJrhLL2cfIPYnQ0QFYIyMvXBG1Fj\nU0rc9UclYQHh9YheIDVYI9YCo/DWP3KFfRJpoTjQRGoPSK9TXcpCAEzQpEG3jOek\n9PdV9Ol6/O8JbrFwXWF3LhkUThg+zCjV4qHtP4oqp5QCqzTQTXGQ9qxWUSlHi4Eu\nIwIDAQAB\n-----END PUBLIC KEY-----\n"},"scheme":"rsassa-pss-sha256"}},"threshold":1,"url":"/snapshot.json"},"timestamp":{"keys":{"66d4ea1da00076c822a6e1b4df5eb1e529eb38f6edcedff323e62f2bfe3eaddd":{"keytype":"rsa","keyval":{"public":"-----BEGIN PUBLIC KEY-----\nMIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEAuzTgV5iKhMnunUDxt4PB\npYqTMPaJN/ZdOOsP6cS3DeCE/EcYGfgCjvP7KD3gjG98VDBTVcuwZClSy+/zvHhV\nIq7VWu+yxQL5c6oa1xpCyHoA96JiLIDPhmqEdscdRybcRQ2CYywzKA8jSwEQCnEK\nc8a74ceY352l/MEcOem0+AtKrOjqcjbXCayDwC9yTg/c78bkp+4T8AhSWgt6Tlrt\nY8jLE7zwojFtIYtMwobWRIW2O3nJDXiSBbTPG3M9kF1G43INshSdBcuq5Tmy8lpE\n/XiG/E7+hP63Hm+KAcdvl553Zs7pLhAZxV0kqlApqRRwhscw+JQci8sVONun5t9t\nNwIDAQAB\n-----END PUBLIC KEY-----\n"},"scheme":"rsassa-pss-sha256"}},"threshold":1,"url":"/timestamp.json"}},"spec_version":"0.1.0","version":3}} \ No newline at end of file +{"signatures":[{"keyid":"a61b695e2b86097d993e94e99fd15ec6d8fc8e9522948c9ff21c2f2c881093ae","sig":"kJdviRCj2xr6Mqb8maOJZUnBgHJqLI5V2MqhQL9mxqsHFE/6z3bmT1OIpaG31b0COhIg0gtR60Y/rUu1YRTs8UpUJa+9bHVcLxrg6PuewzT34ZjtsGpQ8PDhe2j99EULN85rq7jV+HRwdZLHVZXI1P25I0wy1AGtprVo13ZB4h5CBy4ZEUlmae0QKbH+Ej9ouVaeBlyw1JHLew8I1eWEj7x9WYMkO6spqDtPhrIYjqxnh9zUBAvMNQzv3ysu0KM+D4Pr6kALRE9G3AMmbCEzhy+C+r/FT7egNv28kJvxoQjgfEMABYLgLU/Gp7qRVbEbthH4XI8IohmiCH6NavQKBA=="},{"keyid":"9b3cea98f6f23cc11813b12d0526a1b6cfb3761008f0882c9caa8db742d63002","sig":"cQAJkEEpUdauI7TN2nrEF/wZbPKa/Rhu/xEDZgm5tBAc0LKhD48aLH0VdxapytSf1IWk+MatEK/vVMA2sppqrJTGyJ5Ue+JEUe+JhcTAFW0sUzTHUgR5ARTFqDR0RnKysPO/a9BMZsJfyEfS2j2Nlx7iE/2PeMVc2GhhSEY+cHE000ZFXxaxuWYjYzd35LeLQotMWmtsEx0JYrqHgXzMU0yJ0QikE1JUd8uQ4LMvd53iPIEuHxKyq87oIMfvx+gKhbGvgQx3XMYI54qMP+fumPVYQ3alYX5FB+vFX2MFqzl7GxAQJCd57xsanRhKSAkG2LKvEUZuCfTtqiwqb13Lyg=="},{"keyid":"5607181203a2fb60b9d725109388ccb19ccdc236a4b1d1441fbea7ad07616c4a","sig":"cPqcr9zhryFMREpIm81u+vAy1anzJFEFwWk6gCSRNW8lZuGQPgF66qc09YGHe6wGFHmlAm7CTFKh561W8zJfi6IbGS7wUArkaNV5fgPkxg2FDGCC5tHs8we3fgyv+kgr6yzGwQVHl6v4J8S5bkK1MM7GbMGAP6dmf774f6ysLOkjGurAg7UwrZey8/ZQKCjzysnAxyRC+/4QhUfAC5OPs2hI+BUjPDTET3f1Icgq0f9CZfUZ4nW1utMd4/sbQKowNVfZ8j62r8awUuOGmmJbuqLs6ifkfKCXw5ZfrT9G8ySxbI8CshWd54Ck6c7WsyRjY4Y9TB+Oh3pJ+ieE91HJKg=="},{"keyid":"d499c8ca3c2018885037da89bdfa327e21a0e6a15e2ca8dabe5e78a3edf9c91c","sig":"lnfUXTabAURE8BKzbGBJ4oXtDYbpXl1hsEtzOfjJwBae+c9qMusYMR9aU3ZOm2hlAasUXf+bfIy5NoDZMuDC8roYD3UHokC2qrym1h3VgfVl1aMlWDAgtIhNQuHVbKGyPBDMW+O0gDIV2RPwTwcedZHxLbakMqgHaSI1SPFu4ryoLbOsBr2wxQSVKUXRhcvQFvgRJS884TfIYmY3v4EIHvSfHrnPmDmMFrpMUCVzseDPMG/FEWamzjk0GnRAZiwv5NVjbtIAtIUxBASZRWQr1h176k03jwPodT/iynNTNMy/2WHMwUSrVxeFB+aXI9+Pbl/OHw3UAz2qp3kS2ny0aQ=="},{"keyid":"70033289cef8e5105914b22ecc7f15e271d0650d5216d1ab0bd67f8440411bb6","sig":"QilsuGt2x4NjB8uTQeFaUb8sfRwhwcEKGqBBAXLPtgCLm7GXxRb4RXO8nPQ0o5Kg1nZp7t899Q5nMzOO6Qc5ng3vTOwDW2cDSVhyllPKJAzgJC5uwHQdQi9y3vSiNb4j2mpkQbyfu9va5yUiAZzSoJHuRv+aperJHvv+Ev/kxqlvy/4TtXtMPebG2qc08K/WdXBA0S++oWhsC8J5mFgBJBFXFJ6ewLlRJK7DmZKEG+0vtaOeCLOsRnKFSQ3rfrMYdBnGU32+NtlmTU4rjpZ/HdSwgi4K8tDVoo1CE9EwvLvZ1oKpHyRiZWmviRCy81WVB/kpYgfSR4u1CQ02CgZVkQ=="}],"signed":{"_type":"root","expires":"2024-07-26T11:18:30+08:00","roles":{"index":{"keys":{"7fce7ec4f9c36d51dec7ec96065bb64958b743e46ea8141da668cd2ce58a9e61":{"keytype":"rsa","keyval":{"public":"-----BEGIN PUBLIC KEY-----\nMIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEAn5kVA7MlBfSe7EBaExjl\nKbwoDkn1aYi74s29mFgtRo8nejbrVvZQMCIUhvKc0pFa/l9JD/QY6/nAOCE1lpzi\nwwNkSntfOo3p3HQIR+Ut7hZ4Sxfe/5JagGo3LQ+Hd3EJWUxyEfQ/Bff07F3XAbqM\n5+cKNrdsKWZJcPiJDW621qGwCx52f+gzl9bnFe4/hx34OUgirwqh5DS+LhIO+/yt\nbOiN1AyjQKlnb8lUnblElS4Njd+F4io5VzSrZYi2+4AbTkO6wLwbsWHMzXfv9qwn\nvllufOHpB6EwiQ/xBOMuvJJymHnZvs8AH4SuydQIXLaJuv1ysFaBs0KB/ktbakSK\nLwIDAQAB\n-----END PUBLIC KEY-----\n"},"scheme":"rsassa-pss-sha256"}},"threshold":1,"url":"/index.json"},"root":{"keys":{"5607181203a2fb60b9d725109388ccb19ccdc236a4b1d1441fbea7ad07616c4a":{"keytype":"rsa","keyval":{"public":"-----BEGIN PUBLIC KEY-----\nMIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEAyDwCfCl30vhyJW7fB1bs\npRYKtBKzl7o0qnJTm+IksjQ8RXxj8osUpMLmSvOzCaJ5Wxe+Pm1LpSTDbbubbgvd\nnmEFL6228sifviNIu2HlIl+agfzmXuJ9OBlzGUaI4gAd1Z6pF6+mjlcjz2PbWF84\nAbXZdK49uluqulp7HrGB/qNjGcIRUCHgDU4nnq0OkI1BZZSKm9ovonqDkIK76x/S\niAD9OjKsjQ/s57tE+5WTVObKpfrfK0JeHdpAUsA/2n4L1Z6FmZD4LZWqb0i+C7xj\nMElC99KtjlwRntcjeVWG9YjU8AcEN0n1gON9S2oRdyyAzDTgGb7WueDnn6qstt5w\nSQIDAQAB\n-----END PUBLIC KEY-----\n"},"scheme":"rsassa-pss-sha256"},"70033289cef8e5105914b22ecc7f15e271d0650d5216d1ab0bd67f8440411bb6":{"keytype":"rsa","keyval":{"public":"-----BEGIN PUBLIC KEY-----\nMIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEApU5RHP0K+Fgkzm9L0yHR\n4CEqLLwHr7hQxjWOaq5K0UfaOKcsPQZ1SkJ/AMppz7ovzwOU4hcy0wJOV7ms6ACk\nS3hte2GlH/xp+OzWiRnI4qJ6GRrAe+ototj1ZMGvpLK4ifxkKaY6vuWFFAeS0fSe\nPHUGAl5v+PaJWgDNQTRmuAu5oCaYP6oT6VKHj6ulLAgAOqWsBSJiK3oIRcWPR+uI\nIW/9BV158wfmxAw1+7ch1RD44+1vV3+Eo94alvVZIAfcJqDS3XGr2Hfd/YWGj1d2\nD26eblBJoQt0L2E2EL8igu1sudVkMZ3NAIfmBrOWUxHEbIjYeKvXPbaSGdC+FoXD\nrwIDAQAB\n-----END PUBLIC KEY-----\n"},"scheme":"rsassa-pss-sha256"},"9b3cea98f6f23cc11813b12d0526a1b6cfb3761008f0882c9caa8db742d63002":{"keytype":"rsa","keyval":{"public":"-----BEGIN PUBLIC KEY-----\nMIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEAsOgQkwLOh31QV9OpbO9v\n6o83durJFGPOnVXZiab83pKaSk7HEK9WzXBq0BaPvtFwSfROVdpgtopri5lZi+uH\naMKLUn5F8XRnSMl/7m5vM4XpZZYa4aQId4TWdbFtTu31eHGZ3eEC5nDRJ5NhZOJd\nKLFBu/xmxrh/eNZt4QbdWLZayjHnzyoy5AnfNTR6nJgPAv+rBOqyqT/r14q4Pngh\n3z0I3pNFr5qmxsp013XV+kgOW1F7zT7IMU8xRIgo85UWUNhax0/bjY/2NI1Z+WjR\nyhZmUBMVYWvfw97xDUrvBvrJxZPgg0lGvxJC6LF2dM7wgLaNx9khT6HMBVxjxLMs\nDQIDAQAB\n-----END PUBLIC KEY-----\n"},"scheme":"rsassa-pss-sha256"},"a61b695e2b86097d993e94e99fd15ec6d8fc8e9522948c9ff21c2f2c881093ae":{"keytype":"rsa","keyval":{"public":"-----BEGIN PUBLIC KEY-----\nMIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEAnayxhw6KeoKK+Ax9RW6v\n66YjrpRpGLewLmSSAzJGX8nL5/a2nEbXbeF9po265KcBSFWol8jLBsmG56ruwwxp\noWWhJPncqGqy8wMeRMmTf7ATGa+tk+To7UAQD0MYzt7rRlIdpqi9Us3J6076Z83k\n2sxFnX9sVflhOsotGWL7hmrn/CJWxKsO6OVCoqbIlnJV8xFazE2eCfaDTIEEEgnh\nLIGDsmv1AN8ImUIn/hyKcm1PfhDZrF5qhEVhfz5D8aX3cUcEJw8BvCaNloXyHf+y\nDKjqO/dJ7YFWVt7nPqOvaEkBQGMd54ETJ/BbO9r3WTsjXKleoPovBSQ/oOxApypb\nNQIDAQAB\n-----END PUBLIC KEY-----\n"},"scheme":"rsassa-pss-sha256"},"d499c8ca3c2018885037da89bdfa327e21a0e6a15e2ca8dabe5e78a3edf9c91c":{"keytype":"rsa","keyval":{"public":"-----BEGIN PUBLIC KEY-----\nMIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEA5oDytiywLDOSpIBovZxx\nNlZJg5Gk3O9kpiOQ0XnD+L2LV+a2dJU1KmBOoGCUr2TNaGTPihAStjpFIsW4c7Ye\nB2RjUFUrXRf3mvc3n4fACayenxtnCleSR4gKkAdHqqPCiWHT5TAtybKSHuHAluUL\nkMvavUZjIPMj0YYB0R8Re7BjU+zxnipJosTbbPQ7fa3+x2VAHc066Y9qp1YucdpB\nMZ3UwtSVNK7aCbFZvKPwAm22fnDYmMbYFeTz/rrl8k+rKTM37d4D3mURC9xDJxIP\nXVaU2dBImYjoFcY0/5oBU5vr1sj2sdUH+3G5AUr6iCL+XJLiwA1x24jKA6mUjQ93\ndwIDAQAB\n-----END PUBLIC KEY-----\n"},"scheme":"rsassa-pss-sha256"}},"threshold":3,"url":"/root.json"},"snapshot":{"keys":{"8660a9f40687fb33e6f8ad563f21ee81b9ce7b91c90827cc7ae2416c5e0e94e9":{"keytype":"rsa","keyval":{"public":"-----BEGIN PUBLIC KEY-----\nMIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEAqTZx29eJR5EumjqM4YTb\nFlKbim1GNYmtbCLH51BbU2lt46ddmfGvtGsxTD3mIZ/GEHVFv6Aei3xx5nIfhGP0\nrG78JRz394uU8Pd62DiIFWYizr5o+ZBZu29D2YK5ZtxoLFpgt0ibnINK2NcesDC8\nSqfIUbMiQFT6yB/MYD275SjfRGHOeYTPmKdjMJrhLL2cfIPYnQ0QFYIyMvXBG1Fj\nU0rc9UclYQHh9YheIDVYI9YCo/DWP3KFfRJpoTjQRGoPSK9TXcpCAEzQpEG3jOek\n9PdV9Ol6/O8JbrFwXWF3LhkUThg+zCjV4qHtP4oqp5QCqzTQTXGQ9qxWUSlHi4Eu\nIwIDAQAB\n-----END PUBLIC KEY-----\n"},"scheme":"rsassa-pss-sha256"}},"threshold":1,"url":"/snapshot.json"},"timestamp":{"keys":{"66d4ea1da00076c822a6e1b4df5eb1e529eb38f6edcedff323e62f2bfe3eaddd":{"keytype":"rsa","keyval":{"public":"-----BEGIN PUBLIC KEY-----\nMIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEAuzTgV5iKhMnunUDxt4PB\npYqTMPaJN/ZdOOsP6cS3DeCE/EcYGfgCjvP7KD3gjG98VDBTVcuwZClSy+/zvHhV\nIq7VWu+yxQL5c6oa1xpCyHoA96JiLIDPhmqEdscdRybcRQ2CYywzKA8jSwEQCnEK\nc8a74ceY352l/MEcOem0+AtKrOjqcjbXCayDwC9yTg/c78bkp+4T8AhSWgt6Tlrt\nY8jLE7zwojFtIYtMwobWRIW2O3nJDXiSBbTPG3M9kF1G43INshSdBcuq5Tmy8lpE\n/XiG/E7+hP63Hm+KAcdvl553Zs7pLhAZxV0kqlApqRRwhscw+JQci8sVONun5t9t\nNwIDAQAB\n-----END PUBLIC KEY-----\n"},"scheme":"rsassa-pss-sha256"}},"threshold":1,"url":"/timestamp.json"}},"spec_version":"0.1.0","version":4}} \ No newline at end of file diff --git a/pkg/repository/testdata/polluted/bin/root.json b/pkg/repository/testdata/polluted/bin/root.json index eb7de68e7b..ca16fcc0cb 100644 --- a/pkg/repository/testdata/polluted/bin/root.json +++ b/pkg/repository/testdata/polluted/bin/root.json @@ -1 +1 @@ -{"signatures":[{"keyid":"a61b695e2b86097d993e94e99fd15ec6d8fc8e9522948c9ff21c2f2c881093ae","sig":"aptcBUxk5mIY3a2lqNChLU6dXUP/tcaIiwv7e/nQFZjsL3YOm9VG8OQzc4Sy+sv1DqFDvmwNZVp4FyfbPluXRyRgMRYJ194Z9Z1GXHQBmaWaVhMX9qpwzcSe8XNsReaDugHl+iJYfFif34vVm6fbhQgwxggXgJ37TbScVe3OB4LZ7RuSo3RExj3ojMkZr5wWtb4nFKd0VwfGH7KROmz06Pdok/k5PmI7spBLCFUWSAheVThA4jFXtlFPXGmbgBnPMRaRR5uWf59QFD0YQ4xTsJvWLHVmLtIgvDQnouVKYURXooCbYj/OfnvWIQIy3HzuEJC09ZQJokzpluhXaI8J+g=="},{"keyid":"b128ee6a42e2665bd45aa2fa4a7b6e098cfedb3911154f300634b2c056214b9e","sig":"wENeklsuDdoRNLCvvzsD3sJRymbAjMNT/ZTWsPX7tdPtfvq6+jyvvAgXiPMM92Kvb/WUVjBp5lwD5txGfD/2QLWRER9eDbz0XoZ1hGbAERQCWW7BZIc0tgc7EDLoPUxNQBMVYvHi1XTn1kEZqXHu8lrDLSpL2jIRMJOuLqJv6mmBraxGOc+loIwH2hQkXCvp8C8fW9M/bK5YKzuV3FafKyLuCkqMV7AIkKH2PztPXEVDQ3Mhrxzz9P1q7d6E07Hee9FP/xSm2B4MR9eqHiQTJW+aH/9BI60OaWLNxOg2GEr3t72j7mgDZq4d5i99zWwDG3YyWNw7AXtcuPPvKl6Vzg=="},{"keyid":"818c3ab5ddd59a79d4cff76ebb594be56e5a630c05cfcae382f32e07899e7ab7","sig":"PVoqno+m+HiHzkE+wMskafuHU/WUGw/eT/QRkXKgmAdDce6C5yvEVM1H37OpMPvfAjf70Q9opdaxBpkiTo3637EJbHGo7Eqs61hyIYecbYP9/LMinBSQZNOaTdtMUMYB28Cm9rERZCekxs2izLnkwX0py+V3IvfATOKy8jes13fiUF44A27J0X0n8601WicrOrDK37c0pSg66sd4xg5TucMcDsdxlGRW6+puvXLjAt2xk+qHSzbFEoMramfagWBF+hveKcv+buoZ81+f/H+3VBiNAxtTQLgDDqHI5IqAf1HlT20ThNimmN9q8bonirxYx6xhn0y4HvXLOn6nwMrqLQ=="},{"keyid":"9b3cea98f6f23cc11813b12d0526a1b6cfb3761008f0882c9caa8db742d63002","sig":"VOIfdL0qGiHvHmCAu4vLG2s5Ny1+zBZ7ZovPFo8eJT8ppquUDR7POzNBlk+xgteCfUFx56imsvBxKu77zCIe2yT2FcwYdpzcDzaJqOxsEhdusZa7gXdvDVs+JYoZMtRWrjo9InfXVpT8s8SOKglQAnuwcS5/Y+5T5czdXqeWOW9Lnk0Xpt09PUrW3tqrVKhWq1WqRSIayCNV1nnlM3Tj2240DiuYiTEIZn95p8ThB0ZzNLPVNzrvs5/7ZjTdmYCeDSNRwseHO3XFPUOtApEvSWNwjhH4x/YJOo64hqwnaO99utN4jGOKcXkHwQpKPDWKRecgo5+n8VzEZeA+XKLvow=="},{"keyid":"5607181203a2fb60b9d725109388ccb19ccdc236a4b1d1441fbea7ad07616c4a","sig":"QkVHaGa0K8UBc146leEndu2RFQY6tBzoZXGQ9osUyzLVpn3706LLxwD70zOqDhKHaWnnyf5sd3wJrzlhHXjpz3p380xgparaSyEky5d7OB7s5mBToy2bnA7f85IX3sRq7eyb4gcZfFTiZaTT+ygWzPClJaZsiF8p+XzQSkBXXycn3xl8XKtP81+CQOwhdhF9l1oXSEVFKjwMZC9WG4X9ipLS1MPr7J08YVxiMpMKiXtTQtlPdbqP1r65dWDtdJ1y7zMcatWjYGY4JakIrawIcfYJlYrMpAGUzyxx/Tm21d85b+NzyNN7sMpoUKLNHS3vUHOD2a9vSCiwFTjIrsxJDQ=="}],"signed":{"_type":"root","expires":"2023-07-26T11:18:30+08:00","roles":{"index":{"keys":{"7fce7ec4f9c36d51dec7ec96065bb64958b743e46ea8141da668cd2ce58a9e61":{"keytype":"rsa","keyval":{"public":"-----BEGIN PUBLIC KEY-----\nMIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEAn5kVA7MlBfSe7EBaExjl\nKbwoDkn1aYi74s29mFgtRo8nejbrVvZQMCIUhvKc0pFa/l9JD/QY6/nAOCE1lpzi\nwwNkSntfOo3p3HQIR+Ut7hZ4Sxfe/5JagGo3LQ+Hd3EJWUxyEfQ/Bff07F3XAbqM\n5+cKNrdsKWZJcPiJDW621qGwCx52f+gzl9bnFe4/hx34OUgirwqh5DS+LhIO+/yt\nbOiN1AyjQKlnb8lUnblElS4Njd+F4io5VzSrZYi2+4AbTkO6wLwbsWHMzXfv9qwn\nvllufOHpB6EwiQ/xBOMuvJJymHnZvs8AH4SuydQIXLaJuv1ysFaBs0KB/ktbakSK\nLwIDAQAB\n-----END PUBLIC KEY-----\n"},"scheme":"rsassa-pss-sha256"}},"threshold":1,"url":"/index.json"},"root":{"keys":{"5607181203a2fb60b9d725109388ccb19ccdc236a4b1d1441fbea7ad07616c4a":{"keytype":"rsa","keyval":{"public":"-----BEGIN PUBLIC KEY-----\nMIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEAyDwCfCl30vhyJW7fB1bs\npRYKtBKzl7o0qnJTm+IksjQ8RXxj8osUpMLmSvOzCaJ5Wxe+Pm1LpSTDbbubbgvd\nnmEFL6228sifviNIu2HlIl+agfzmXuJ9OBlzGUaI4gAd1Z6pF6+mjlcjz2PbWF84\nAbXZdK49uluqulp7HrGB/qNjGcIRUCHgDU4nnq0OkI1BZZSKm9ovonqDkIK76x/S\niAD9OjKsjQ/s57tE+5WTVObKpfrfK0JeHdpAUsA/2n4L1Z6FmZD4LZWqb0i+C7xj\nMElC99KtjlwRntcjeVWG9YjU8AcEN0n1gON9S2oRdyyAzDTgGb7WueDnn6qstt5w\nSQIDAQAB\n-----END PUBLIC KEY-----\n"},"scheme":"rsassa-pss-sha256"},"818c3ab5ddd59a79d4cff76ebb594be56e5a630c05cfcae382f32e07899e7ab7":{"keytype":"rsa","keyval":{"public":"-----BEGIN PUBLIC KEY-----\nMIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEA37V4zn8sYuhC5HbIos/6\nf3zhnr7WSnV0GtrOj2q6tCvo/1rGXDKbL8KJgLBHcQT6P732sm5BmZ+y8hRXZDqy\njTBqT70Vc0fGfCkH5Z0FU8WWlhQ2Ck3W20TxXPJY7p+wnuUCmcLzD7cKwUGOshjs\nyozqAMQ7C/HSmJFCUY8PaASvQCCg/YyMr3Ft8xpyc5IGmIi4NkVjf/JWIUFFv7kr\nIkRbT5K1nKJ2earot1TIcWUY2O4OvbhtueJ+IN0Lkdg+v98W/pvlF+ji24DBlcGw\nLdOeGRt5z6GnHKlAuZ/tbjRHeXnq1KxMETlXCgu2xCwsSK/10SE8oTaXbFoMHNBN\nAQIDAQAB\n-----END PUBLIC KEY-----\n"},"scheme":"rsassa-pss-sha256"},"9b3cea98f6f23cc11813b12d0526a1b6cfb3761008f0882c9caa8db742d63002":{"keytype":"rsa","keyval":{"public":"-----BEGIN PUBLIC KEY-----\nMIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEAsOgQkwLOh31QV9OpbO9v\n6o83durJFGPOnVXZiab83pKaSk7HEK9WzXBq0BaPvtFwSfROVdpgtopri5lZi+uH\naMKLUn5F8XRnSMl/7m5vM4XpZZYa4aQId4TWdbFtTu31eHGZ3eEC5nDRJ5NhZOJd\nKLFBu/xmxrh/eNZt4QbdWLZayjHnzyoy5AnfNTR6nJgPAv+rBOqyqT/r14q4Pngh\n3z0I3pNFr5qmxsp013XV+kgOW1F7zT7IMU8xRIgo85UWUNhax0/bjY/2NI1Z+WjR\nyhZmUBMVYWvfw97xDUrvBvrJxZPgg0lGvxJC6LF2dM7wgLaNx9khT6HMBVxjxLMs\nDQIDAQAB\n-----END PUBLIC KEY-----\n"},"scheme":"rsassa-pss-sha256"},"a61b695e2b86097d993e94e99fd15ec6d8fc8e9522948c9ff21c2f2c881093ae":{"keytype":"rsa","keyval":{"public":"-----BEGIN PUBLIC KEY-----\nMIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEAnayxhw6KeoKK+Ax9RW6v\n66YjrpRpGLewLmSSAzJGX8nL5/a2nEbXbeF9po265KcBSFWol8jLBsmG56ruwwxp\noWWhJPncqGqy8wMeRMmTf7ATGa+tk+To7UAQD0MYzt7rRlIdpqi9Us3J6076Z83k\n2sxFnX9sVflhOsotGWL7hmrn/CJWxKsO6OVCoqbIlnJV8xFazE2eCfaDTIEEEgnh\nLIGDsmv1AN8ImUIn/hyKcm1PfhDZrF5qhEVhfz5D8aX3cUcEJw8BvCaNloXyHf+y\nDKjqO/dJ7YFWVt7nPqOvaEkBQGMd54ETJ/BbO9r3WTsjXKleoPovBSQ/oOxApypb\nNQIDAQAB\n-----END PUBLIC KEY-----\n"},"scheme":"rsassa-pss-sha256"},"b128ee6a42e2665bd45aa2fa4a7b6e098cfedb3911154f300634b2c056214b9e":{"keytype":"rsa","keyval":{"public":"-----BEGIN PUBLIC KEY-----\nMIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEA0kgU3v3SxYzm5J6P+GPP\nmy6toBnKYtTViAVmpUJiIEjdZ9NLpoJU0na9q0CD8sIgo2Js/W/owJUvSj6rm8us\nsu/Ve5KsoJN6zca2am1uZ5IKnc48i0mCv76WXawCxM+NFGqSCMJcltlhj3fC/GDS\ngu+BiIbrgR1PgJf6Jk6l7uMJdN3TL6JJQcEC4lz+2hj5zoVNYkq06ZC79j2tPDCI\nkTAYGF/TAAVLH08/kGH5ZeRPlVKJ7cwW3OniLM5NeFnS8+shRNb6AYr7xju3Ikbw\nDo14ipIghBI0iAxn6Lvr/iilc7TM7RWJ4OiTrmK3SQSJ+U6H2N2/I5OGEHBEKzbA\nOQIDAQAB\n-----END PUBLIC KEY-----\n"},"scheme":"rsassa-pss-sha256"}},"threshold":3,"url":"/root.json"},"snapshot":{"keys":{"8660a9f40687fb33e6f8ad563f21ee81b9ce7b91c90827cc7ae2416c5e0e94e9":{"keytype":"rsa","keyval":{"public":"-----BEGIN PUBLIC KEY-----\nMIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEAqTZx29eJR5EumjqM4YTb\nFlKbim1GNYmtbCLH51BbU2lt46ddmfGvtGsxTD3mIZ/GEHVFv6Aei3xx5nIfhGP0\nrG78JRz394uU8Pd62DiIFWYizr5o+ZBZu29D2YK5ZtxoLFpgt0ibnINK2NcesDC8\nSqfIUbMiQFT6yB/MYD275SjfRGHOeYTPmKdjMJrhLL2cfIPYnQ0QFYIyMvXBG1Fj\nU0rc9UclYQHh9YheIDVYI9YCo/DWP3KFfRJpoTjQRGoPSK9TXcpCAEzQpEG3jOek\n9PdV9Ol6/O8JbrFwXWF3LhkUThg+zCjV4qHtP4oqp5QCqzTQTXGQ9qxWUSlHi4Eu\nIwIDAQAB\n-----END PUBLIC KEY-----\n"},"scheme":"rsassa-pss-sha256"}},"threshold":1,"url":"/snapshot.json"},"timestamp":{"keys":{"66d4ea1da00076c822a6e1b4df5eb1e529eb38f6edcedff323e62f2bfe3eaddd":{"keytype":"rsa","keyval":{"public":"-----BEGIN PUBLIC KEY-----\nMIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEAuzTgV5iKhMnunUDxt4PB\npYqTMPaJN/ZdOOsP6cS3DeCE/EcYGfgCjvP7KD3gjG98VDBTVcuwZClSy+/zvHhV\nIq7VWu+yxQL5c6oa1xpCyHoA96JiLIDPhmqEdscdRybcRQ2CYywzKA8jSwEQCnEK\nc8a74ceY352l/MEcOem0+AtKrOjqcjbXCayDwC9yTg/c78bkp+4T8AhSWgt6Tlrt\nY8jLE7zwojFtIYtMwobWRIW2O3nJDXiSBbTPG3M9kF1G43INshSdBcuq5Tmy8lpE\n/XiG/E7+hP63Hm+KAcdvl553Zs7pLhAZxV0kqlApqRRwhscw+JQci8sVONun5t9t\nNwIDAQAB\n-----END PUBLIC KEY-----\n"},"scheme":"rsassa-pss-sha256"}},"threshold":1,"url":"/timestamp.json"}},"spec_version":"0.1.0","version":3}} \ No newline at end of file +{"signatures":[{"keyid":"a61b695e2b86097d993e94e99fd15ec6d8fc8e9522948c9ff21c2f2c881093ae","sig":"kJdviRCj2xr6Mqb8maOJZUnBgHJqLI5V2MqhQL9mxqsHFE/6z3bmT1OIpaG31b0COhIg0gtR60Y/rUu1YRTs8UpUJa+9bHVcLxrg6PuewzT34ZjtsGpQ8PDhe2j99EULN85rq7jV+HRwdZLHVZXI1P25I0wy1AGtprVo13ZB4h5CBy4ZEUlmae0QKbH+Ej9ouVaeBlyw1JHLew8I1eWEj7x9WYMkO6spqDtPhrIYjqxnh9zUBAvMNQzv3ysu0KM+D4Pr6kALRE9G3AMmbCEzhy+C+r/FT7egNv28kJvxoQjgfEMABYLgLU/Gp7qRVbEbthH4XI8IohmiCH6NavQKBA=="},{"keyid":"9b3cea98f6f23cc11813b12d0526a1b6cfb3761008f0882c9caa8db742d63002","sig":"cQAJkEEpUdauI7TN2nrEF/wZbPKa/Rhu/xEDZgm5tBAc0LKhD48aLH0VdxapytSf1IWk+MatEK/vVMA2sppqrJTGyJ5Ue+JEUe+JhcTAFW0sUzTHUgR5ARTFqDR0RnKysPO/a9BMZsJfyEfS2j2Nlx7iE/2PeMVc2GhhSEY+cHE000ZFXxaxuWYjYzd35LeLQotMWmtsEx0JYrqHgXzMU0yJ0QikE1JUd8uQ4LMvd53iPIEuHxKyq87oIMfvx+gKhbGvgQx3XMYI54qMP+fumPVYQ3alYX5FB+vFX2MFqzl7GxAQJCd57xsanRhKSAkG2LKvEUZuCfTtqiwqb13Lyg=="},{"keyid":"5607181203a2fb60b9d725109388ccb19ccdc236a4b1d1441fbea7ad07616c4a","sig":"cPqcr9zhryFMREpIm81u+vAy1anzJFEFwWk6gCSRNW8lZuGQPgF66qc09YGHe6wGFHmlAm7CTFKh561W8zJfi6IbGS7wUArkaNV5fgPkxg2FDGCC5tHs8we3fgyv+kgr6yzGwQVHl6v4J8S5bkK1MM7GbMGAP6dmf774f6ysLOkjGurAg7UwrZey8/ZQKCjzysnAxyRC+/4QhUfAC5OPs2hI+BUjPDTET3f1Icgq0f9CZfUZ4nW1utMd4/sbQKowNVfZ8j62r8awUuOGmmJbuqLs6ifkfKCXw5ZfrT9G8ySxbI8CshWd54Ck6c7WsyRjY4Y9TB+Oh3pJ+ieE91HJKg=="},{"keyid":"d499c8ca3c2018885037da89bdfa327e21a0e6a15e2ca8dabe5e78a3edf9c91c","sig":"lnfUXTabAURE8BKzbGBJ4oXtDYbpXl1hsEtzOfjJwBae+c9qMusYMR9aU3ZOm2hlAasUXf+bfIy5NoDZMuDC8roYD3UHokC2qrym1h3VgfVl1aMlWDAgtIhNQuHVbKGyPBDMW+O0gDIV2RPwTwcedZHxLbakMqgHaSI1SPFu4ryoLbOsBr2wxQSVKUXRhcvQFvgRJS884TfIYmY3v4EIHvSfHrnPmDmMFrpMUCVzseDPMG/FEWamzjk0GnRAZiwv5NVjbtIAtIUxBASZRWQr1h176k03jwPodT/iynNTNMy/2WHMwUSrVxeFB+aXI9+Pbl/OHw3UAz2qp3kS2ny0aQ=="},{"keyid":"70033289cef8e5105914b22ecc7f15e271d0650d5216d1ab0bd67f8440411bb6","sig":"QilsuGt2x4NjB8uTQeFaUb8sfRwhwcEKGqBBAXLPtgCLm7GXxRb4RXO8nPQ0o5Kg1nZp7t899Q5nMzOO6Qc5ng3vTOwDW2cDSVhyllPKJAzgJC5uwHQdQi9y3vSiNb4j2mpkQbyfu9va5yUiAZzSoJHuRv+aperJHvv+Ev/kxqlvy/4TtXtMPebG2qc08K/WdXBA0S++oWhsC8J5mFgBJBFXFJ6ewLlRJK7DmZKEG+0vtaOeCLOsRnKFSQ3rfrMYdBnGU32+NtlmTU4rjpZ/HdSwgi4K8tDVoo1CE9EwvLvZ1oKpHyRiZWmviRCy81WVB/kpYgfSR4u1CQ02CgZVkQ=="}],"signed":{"_type":"root","expires":"2024-07-26T11:18:30+08:00","roles":{"index":{"keys":{"7fce7ec4f9c36d51dec7ec96065bb64958b743e46ea8141da668cd2ce58a9e61":{"keytype":"rsa","keyval":{"public":"-----BEGIN PUBLIC KEY-----\nMIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEAn5kVA7MlBfSe7EBaExjl\nKbwoDkn1aYi74s29mFgtRo8nejbrVvZQMCIUhvKc0pFa/l9JD/QY6/nAOCE1lpzi\nwwNkSntfOo3p3HQIR+Ut7hZ4Sxfe/5JagGo3LQ+Hd3EJWUxyEfQ/Bff07F3XAbqM\n5+cKNrdsKWZJcPiJDW621qGwCx52f+gzl9bnFe4/hx34OUgirwqh5DS+LhIO+/yt\nbOiN1AyjQKlnb8lUnblElS4Njd+F4io5VzSrZYi2+4AbTkO6wLwbsWHMzXfv9qwn\nvllufOHpB6EwiQ/xBOMuvJJymHnZvs8AH4SuydQIXLaJuv1ysFaBs0KB/ktbakSK\nLwIDAQAB\n-----END PUBLIC KEY-----\n"},"scheme":"rsassa-pss-sha256"}},"threshold":1,"url":"/index.json"},"root":{"keys":{"5607181203a2fb60b9d725109388ccb19ccdc236a4b1d1441fbea7ad07616c4a":{"keytype":"rsa","keyval":{"public":"-----BEGIN PUBLIC KEY-----\nMIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEAyDwCfCl30vhyJW7fB1bs\npRYKtBKzl7o0qnJTm+IksjQ8RXxj8osUpMLmSvOzCaJ5Wxe+Pm1LpSTDbbubbgvd\nnmEFL6228sifviNIu2HlIl+agfzmXuJ9OBlzGUaI4gAd1Z6pF6+mjlcjz2PbWF84\nAbXZdK49uluqulp7HrGB/qNjGcIRUCHgDU4nnq0OkI1BZZSKm9ovonqDkIK76x/S\niAD9OjKsjQ/s57tE+5WTVObKpfrfK0JeHdpAUsA/2n4L1Z6FmZD4LZWqb0i+C7xj\nMElC99KtjlwRntcjeVWG9YjU8AcEN0n1gON9S2oRdyyAzDTgGb7WueDnn6qstt5w\nSQIDAQAB\n-----END PUBLIC KEY-----\n"},"scheme":"rsassa-pss-sha256"},"70033289cef8e5105914b22ecc7f15e271d0650d5216d1ab0bd67f8440411bb6":{"keytype":"rsa","keyval":{"public":"-----BEGIN PUBLIC KEY-----\nMIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEApU5RHP0K+Fgkzm9L0yHR\n4CEqLLwHr7hQxjWOaq5K0UfaOKcsPQZ1SkJ/AMppz7ovzwOU4hcy0wJOV7ms6ACk\nS3hte2GlH/xp+OzWiRnI4qJ6GRrAe+ototj1ZMGvpLK4ifxkKaY6vuWFFAeS0fSe\nPHUGAl5v+PaJWgDNQTRmuAu5oCaYP6oT6VKHj6ulLAgAOqWsBSJiK3oIRcWPR+uI\nIW/9BV158wfmxAw1+7ch1RD44+1vV3+Eo94alvVZIAfcJqDS3XGr2Hfd/YWGj1d2\nD26eblBJoQt0L2E2EL8igu1sudVkMZ3NAIfmBrOWUxHEbIjYeKvXPbaSGdC+FoXD\nrwIDAQAB\n-----END PUBLIC KEY-----\n"},"scheme":"rsassa-pss-sha256"},"9b3cea98f6f23cc11813b12d0526a1b6cfb3761008f0882c9caa8db742d63002":{"keytype":"rsa","keyval":{"public":"-----BEGIN PUBLIC KEY-----\nMIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEAsOgQkwLOh31QV9OpbO9v\n6o83durJFGPOnVXZiab83pKaSk7HEK9WzXBq0BaPvtFwSfROVdpgtopri5lZi+uH\naMKLUn5F8XRnSMl/7m5vM4XpZZYa4aQId4TWdbFtTu31eHGZ3eEC5nDRJ5NhZOJd\nKLFBu/xmxrh/eNZt4QbdWLZayjHnzyoy5AnfNTR6nJgPAv+rBOqyqT/r14q4Pngh\n3z0I3pNFr5qmxsp013XV+kgOW1F7zT7IMU8xRIgo85UWUNhax0/bjY/2NI1Z+WjR\nyhZmUBMVYWvfw97xDUrvBvrJxZPgg0lGvxJC6LF2dM7wgLaNx9khT6HMBVxjxLMs\nDQIDAQAB\n-----END PUBLIC KEY-----\n"},"scheme":"rsassa-pss-sha256"},"a61b695e2b86097d993e94e99fd15ec6d8fc8e9522948c9ff21c2f2c881093ae":{"keytype":"rsa","keyval":{"public":"-----BEGIN PUBLIC KEY-----\nMIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEAnayxhw6KeoKK+Ax9RW6v\n66YjrpRpGLewLmSSAzJGX8nL5/a2nEbXbeF9po265KcBSFWol8jLBsmG56ruwwxp\noWWhJPncqGqy8wMeRMmTf7ATGa+tk+To7UAQD0MYzt7rRlIdpqi9Us3J6076Z83k\n2sxFnX9sVflhOsotGWL7hmrn/CJWxKsO6OVCoqbIlnJV8xFazE2eCfaDTIEEEgnh\nLIGDsmv1AN8ImUIn/hyKcm1PfhDZrF5qhEVhfz5D8aX3cUcEJw8BvCaNloXyHf+y\nDKjqO/dJ7YFWVt7nPqOvaEkBQGMd54ETJ/BbO9r3WTsjXKleoPovBSQ/oOxApypb\nNQIDAQAB\n-----END PUBLIC KEY-----\n"},"scheme":"rsassa-pss-sha256"},"d499c8ca3c2018885037da89bdfa327e21a0e6a15e2ca8dabe5e78a3edf9c91c":{"keytype":"rsa","keyval":{"public":"-----BEGIN PUBLIC KEY-----\nMIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEA5oDytiywLDOSpIBovZxx\nNlZJg5Gk3O9kpiOQ0XnD+L2LV+a2dJU1KmBOoGCUr2TNaGTPihAStjpFIsW4c7Ye\nB2RjUFUrXRf3mvc3n4fACayenxtnCleSR4gKkAdHqqPCiWHT5TAtybKSHuHAluUL\nkMvavUZjIPMj0YYB0R8Re7BjU+zxnipJosTbbPQ7fa3+x2VAHc066Y9qp1YucdpB\nMZ3UwtSVNK7aCbFZvKPwAm22fnDYmMbYFeTz/rrl8k+rKTM37d4D3mURC9xDJxIP\nXVaU2dBImYjoFcY0/5oBU5vr1sj2sdUH+3G5AUr6iCL+XJLiwA1x24jKA6mUjQ93\ndwIDAQAB\n-----END PUBLIC KEY-----\n"},"scheme":"rsassa-pss-sha256"}},"threshold":3,"url":"/root.json"},"snapshot":{"keys":{"8660a9f40687fb33e6f8ad563f21ee81b9ce7b91c90827cc7ae2416c5e0e94e9":{"keytype":"rsa","keyval":{"public":"-----BEGIN PUBLIC KEY-----\nMIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEAqTZx29eJR5EumjqM4YTb\nFlKbim1GNYmtbCLH51BbU2lt46ddmfGvtGsxTD3mIZ/GEHVFv6Aei3xx5nIfhGP0\nrG78JRz394uU8Pd62DiIFWYizr5o+ZBZu29D2YK5ZtxoLFpgt0ibnINK2NcesDC8\nSqfIUbMiQFT6yB/MYD275SjfRGHOeYTPmKdjMJrhLL2cfIPYnQ0QFYIyMvXBG1Fj\nU0rc9UclYQHh9YheIDVYI9YCo/DWP3KFfRJpoTjQRGoPSK9TXcpCAEzQpEG3jOek\n9PdV9Ol6/O8JbrFwXWF3LhkUThg+zCjV4qHtP4oqp5QCqzTQTXGQ9qxWUSlHi4Eu\nIwIDAQAB\n-----END PUBLIC KEY-----\n"},"scheme":"rsassa-pss-sha256"}},"threshold":1,"url":"/snapshot.json"},"timestamp":{"keys":{"66d4ea1da00076c822a6e1b4df5eb1e529eb38f6edcedff323e62f2bfe3eaddd":{"keytype":"rsa","keyval":{"public":"-----BEGIN PUBLIC KEY-----\nMIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEAuzTgV5iKhMnunUDxt4PB\npYqTMPaJN/ZdOOsP6cS3DeCE/EcYGfgCjvP7KD3gjG98VDBTVcuwZClSy+/zvHhV\nIq7VWu+yxQL5c6oa1xpCyHoA96JiLIDPhmqEdscdRybcRQ2CYywzKA8jSwEQCnEK\nc8a74ceY352l/MEcOem0+AtKrOjqcjbXCayDwC9yTg/c78bkp+4T8AhSWgt6Tlrt\nY8jLE7zwojFtIYtMwobWRIW2O3nJDXiSBbTPG3M9kF1G43INshSdBcuq5Tmy8lpE\n/XiG/E7+hP63Hm+KAcdvl553Zs7pLhAZxV0kqlApqRRwhscw+JQci8sVONun5t9t\nNwIDAQAB\n-----END PUBLIC KEY-----\n"},"scheme":"rsassa-pss-sha256"}},"threshold":1,"url":"/timestamp.json"}},"spec_version":"0.1.0","version":4}} \ No newline at end of file diff --git a/pkg/repository/v1_repository.go b/pkg/repository/v1_repository.go index 6c82500504..d7ed787de9 100644 --- a/pkg/repository/v1_repository.go +++ b/pkg/repository/v1_repository.go @@ -807,9 +807,6 @@ func findVersionFromManifest(id, constraint, platform string, manifest *v1manife versions := manifest.VersionList(platform) verList := make([]string, 0, len(versions)) for v := range versions { - if v == manifest.Nightly { - continue - } verList = append(verList, v) } sort.Slice(verList, func(p, q int) bool { diff --git a/pkg/repository/v1manifest/local_manifests.go b/pkg/repository/v1manifest/local_manifests.go index c2310cc573..10bbd97e0b 100644 --- a/pkg/repository/v1manifest/local_manifests.go +++ b/pkg/repository/v1manifest/local_manifests.go @@ -123,11 +123,11 @@ func (ms *FsManifests) save(manifest *Manifest, filename string) error { path := filepath.Join(ms.profile.Root(), localdata.ManifestParentDir, filename) // create sub directory if needed - if err := os.MkdirAll(filepath.Dir(path), 0755); err != nil { + if err := utils.MkdirAll(filepath.Dir(path), 0755); err != nil { return errors.Trace(err) } - err = os.WriteFile(path, bytes, 0644) + err = utils.WriteFile(path, bytes, os.ModePerm) if err != nil { return err } @@ -239,7 +239,7 @@ func (ms *FsManifests) InstallComponent(reader io.Reader, targetDir, component, return utils.Untar(reader, targetDir) } - if err := os.MkdirAll(targetDir, 0755); err != nil { + if err := utils.MkdirAll(targetDir, 0755); err != nil { return errors.Trace(err) } writer, err := os.OpenFile(filepath.Join(targetDir, filename), os.O_WRONLY|os.O_CREATE|os.O_TRUNC, os.ModePerm) diff --git a/pkg/repository/v1manifest/repo.go b/pkg/repository/v1manifest/repo.go index 67d8f6c3ea..d457a0fa6f 100644 --- a/pkg/repository/v1manifest/repo.go +++ b/pkg/repository/v1manifest/repo.go @@ -120,7 +120,7 @@ func SaveKeyInfo(key *KeyInfo, ty, dir string) (string, error) { } } if utils.IsNotExist(dir) { - if err := os.MkdirAll(dir, 0755); err != nil { + if err := utils.MkdirAll(dir, 0755); err != nil { return "", errors.Annotate(err, "create key directory") } } diff --git a/pkg/repository/v1manifest/testdata/polluted/bin/root.json b/pkg/repository/v1manifest/testdata/polluted/bin/root.json index eb7de68e7b..ca16fcc0cb 100644 --- a/pkg/repository/v1manifest/testdata/polluted/bin/root.json +++ b/pkg/repository/v1manifest/testdata/polluted/bin/root.json @@ -1 +1 @@ -{"signatures":[{"keyid":"a61b695e2b86097d993e94e99fd15ec6d8fc8e9522948c9ff21c2f2c881093ae","sig":"aptcBUxk5mIY3a2lqNChLU6dXUP/tcaIiwv7e/nQFZjsL3YOm9VG8OQzc4Sy+sv1DqFDvmwNZVp4FyfbPluXRyRgMRYJ194Z9Z1GXHQBmaWaVhMX9qpwzcSe8XNsReaDugHl+iJYfFif34vVm6fbhQgwxggXgJ37TbScVe3OB4LZ7RuSo3RExj3ojMkZr5wWtb4nFKd0VwfGH7KROmz06Pdok/k5PmI7spBLCFUWSAheVThA4jFXtlFPXGmbgBnPMRaRR5uWf59QFD0YQ4xTsJvWLHVmLtIgvDQnouVKYURXooCbYj/OfnvWIQIy3HzuEJC09ZQJokzpluhXaI8J+g=="},{"keyid":"b128ee6a42e2665bd45aa2fa4a7b6e098cfedb3911154f300634b2c056214b9e","sig":"wENeklsuDdoRNLCvvzsD3sJRymbAjMNT/ZTWsPX7tdPtfvq6+jyvvAgXiPMM92Kvb/WUVjBp5lwD5txGfD/2QLWRER9eDbz0XoZ1hGbAERQCWW7BZIc0tgc7EDLoPUxNQBMVYvHi1XTn1kEZqXHu8lrDLSpL2jIRMJOuLqJv6mmBraxGOc+loIwH2hQkXCvp8C8fW9M/bK5YKzuV3FafKyLuCkqMV7AIkKH2PztPXEVDQ3Mhrxzz9P1q7d6E07Hee9FP/xSm2B4MR9eqHiQTJW+aH/9BI60OaWLNxOg2GEr3t72j7mgDZq4d5i99zWwDG3YyWNw7AXtcuPPvKl6Vzg=="},{"keyid":"818c3ab5ddd59a79d4cff76ebb594be56e5a630c05cfcae382f32e07899e7ab7","sig":"PVoqno+m+HiHzkE+wMskafuHU/WUGw/eT/QRkXKgmAdDce6C5yvEVM1H37OpMPvfAjf70Q9opdaxBpkiTo3637EJbHGo7Eqs61hyIYecbYP9/LMinBSQZNOaTdtMUMYB28Cm9rERZCekxs2izLnkwX0py+V3IvfATOKy8jes13fiUF44A27J0X0n8601WicrOrDK37c0pSg66sd4xg5TucMcDsdxlGRW6+puvXLjAt2xk+qHSzbFEoMramfagWBF+hveKcv+buoZ81+f/H+3VBiNAxtTQLgDDqHI5IqAf1HlT20ThNimmN9q8bonirxYx6xhn0y4HvXLOn6nwMrqLQ=="},{"keyid":"9b3cea98f6f23cc11813b12d0526a1b6cfb3761008f0882c9caa8db742d63002","sig":"VOIfdL0qGiHvHmCAu4vLG2s5Ny1+zBZ7ZovPFo8eJT8ppquUDR7POzNBlk+xgteCfUFx56imsvBxKu77zCIe2yT2FcwYdpzcDzaJqOxsEhdusZa7gXdvDVs+JYoZMtRWrjo9InfXVpT8s8SOKglQAnuwcS5/Y+5T5czdXqeWOW9Lnk0Xpt09PUrW3tqrVKhWq1WqRSIayCNV1nnlM3Tj2240DiuYiTEIZn95p8ThB0ZzNLPVNzrvs5/7ZjTdmYCeDSNRwseHO3XFPUOtApEvSWNwjhH4x/YJOo64hqwnaO99utN4jGOKcXkHwQpKPDWKRecgo5+n8VzEZeA+XKLvow=="},{"keyid":"5607181203a2fb60b9d725109388ccb19ccdc236a4b1d1441fbea7ad07616c4a","sig":"QkVHaGa0K8UBc146leEndu2RFQY6tBzoZXGQ9osUyzLVpn3706LLxwD70zOqDhKHaWnnyf5sd3wJrzlhHXjpz3p380xgparaSyEky5d7OB7s5mBToy2bnA7f85IX3sRq7eyb4gcZfFTiZaTT+ygWzPClJaZsiF8p+XzQSkBXXycn3xl8XKtP81+CQOwhdhF9l1oXSEVFKjwMZC9WG4X9ipLS1MPr7J08YVxiMpMKiXtTQtlPdbqP1r65dWDtdJ1y7zMcatWjYGY4JakIrawIcfYJlYrMpAGUzyxx/Tm21d85b+NzyNN7sMpoUKLNHS3vUHOD2a9vSCiwFTjIrsxJDQ=="}],"signed":{"_type":"root","expires":"2023-07-26T11:18:30+08:00","roles":{"index":{"keys":{"7fce7ec4f9c36d51dec7ec96065bb64958b743e46ea8141da668cd2ce58a9e61":{"keytype":"rsa","keyval":{"public":"-----BEGIN PUBLIC KEY-----\nMIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEAn5kVA7MlBfSe7EBaExjl\nKbwoDkn1aYi74s29mFgtRo8nejbrVvZQMCIUhvKc0pFa/l9JD/QY6/nAOCE1lpzi\nwwNkSntfOo3p3HQIR+Ut7hZ4Sxfe/5JagGo3LQ+Hd3EJWUxyEfQ/Bff07F3XAbqM\n5+cKNrdsKWZJcPiJDW621qGwCx52f+gzl9bnFe4/hx34OUgirwqh5DS+LhIO+/yt\nbOiN1AyjQKlnb8lUnblElS4Njd+F4io5VzSrZYi2+4AbTkO6wLwbsWHMzXfv9qwn\nvllufOHpB6EwiQ/xBOMuvJJymHnZvs8AH4SuydQIXLaJuv1ysFaBs0KB/ktbakSK\nLwIDAQAB\n-----END PUBLIC KEY-----\n"},"scheme":"rsassa-pss-sha256"}},"threshold":1,"url":"/index.json"},"root":{"keys":{"5607181203a2fb60b9d725109388ccb19ccdc236a4b1d1441fbea7ad07616c4a":{"keytype":"rsa","keyval":{"public":"-----BEGIN PUBLIC KEY-----\nMIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEAyDwCfCl30vhyJW7fB1bs\npRYKtBKzl7o0qnJTm+IksjQ8RXxj8osUpMLmSvOzCaJ5Wxe+Pm1LpSTDbbubbgvd\nnmEFL6228sifviNIu2HlIl+agfzmXuJ9OBlzGUaI4gAd1Z6pF6+mjlcjz2PbWF84\nAbXZdK49uluqulp7HrGB/qNjGcIRUCHgDU4nnq0OkI1BZZSKm9ovonqDkIK76x/S\niAD9OjKsjQ/s57tE+5WTVObKpfrfK0JeHdpAUsA/2n4L1Z6FmZD4LZWqb0i+C7xj\nMElC99KtjlwRntcjeVWG9YjU8AcEN0n1gON9S2oRdyyAzDTgGb7WueDnn6qstt5w\nSQIDAQAB\n-----END PUBLIC KEY-----\n"},"scheme":"rsassa-pss-sha256"},"818c3ab5ddd59a79d4cff76ebb594be56e5a630c05cfcae382f32e07899e7ab7":{"keytype":"rsa","keyval":{"public":"-----BEGIN PUBLIC KEY-----\nMIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEA37V4zn8sYuhC5HbIos/6\nf3zhnr7WSnV0GtrOj2q6tCvo/1rGXDKbL8KJgLBHcQT6P732sm5BmZ+y8hRXZDqy\njTBqT70Vc0fGfCkH5Z0FU8WWlhQ2Ck3W20TxXPJY7p+wnuUCmcLzD7cKwUGOshjs\nyozqAMQ7C/HSmJFCUY8PaASvQCCg/YyMr3Ft8xpyc5IGmIi4NkVjf/JWIUFFv7kr\nIkRbT5K1nKJ2earot1TIcWUY2O4OvbhtueJ+IN0Lkdg+v98W/pvlF+ji24DBlcGw\nLdOeGRt5z6GnHKlAuZ/tbjRHeXnq1KxMETlXCgu2xCwsSK/10SE8oTaXbFoMHNBN\nAQIDAQAB\n-----END PUBLIC KEY-----\n"},"scheme":"rsassa-pss-sha256"},"9b3cea98f6f23cc11813b12d0526a1b6cfb3761008f0882c9caa8db742d63002":{"keytype":"rsa","keyval":{"public":"-----BEGIN PUBLIC KEY-----\nMIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEAsOgQkwLOh31QV9OpbO9v\n6o83durJFGPOnVXZiab83pKaSk7HEK9WzXBq0BaPvtFwSfROVdpgtopri5lZi+uH\naMKLUn5F8XRnSMl/7m5vM4XpZZYa4aQId4TWdbFtTu31eHGZ3eEC5nDRJ5NhZOJd\nKLFBu/xmxrh/eNZt4QbdWLZayjHnzyoy5AnfNTR6nJgPAv+rBOqyqT/r14q4Pngh\n3z0I3pNFr5qmxsp013XV+kgOW1F7zT7IMU8xRIgo85UWUNhax0/bjY/2NI1Z+WjR\nyhZmUBMVYWvfw97xDUrvBvrJxZPgg0lGvxJC6LF2dM7wgLaNx9khT6HMBVxjxLMs\nDQIDAQAB\n-----END PUBLIC KEY-----\n"},"scheme":"rsassa-pss-sha256"},"a61b695e2b86097d993e94e99fd15ec6d8fc8e9522948c9ff21c2f2c881093ae":{"keytype":"rsa","keyval":{"public":"-----BEGIN PUBLIC KEY-----\nMIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEAnayxhw6KeoKK+Ax9RW6v\n66YjrpRpGLewLmSSAzJGX8nL5/a2nEbXbeF9po265KcBSFWol8jLBsmG56ruwwxp\noWWhJPncqGqy8wMeRMmTf7ATGa+tk+To7UAQD0MYzt7rRlIdpqi9Us3J6076Z83k\n2sxFnX9sVflhOsotGWL7hmrn/CJWxKsO6OVCoqbIlnJV8xFazE2eCfaDTIEEEgnh\nLIGDsmv1AN8ImUIn/hyKcm1PfhDZrF5qhEVhfz5D8aX3cUcEJw8BvCaNloXyHf+y\nDKjqO/dJ7YFWVt7nPqOvaEkBQGMd54ETJ/BbO9r3WTsjXKleoPovBSQ/oOxApypb\nNQIDAQAB\n-----END PUBLIC KEY-----\n"},"scheme":"rsassa-pss-sha256"},"b128ee6a42e2665bd45aa2fa4a7b6e098cfedb3911154f300634b2c056214b9e":{"keytype":"rsa","keyval":{"public":"-----BEGIN PUBLIC KEY-----\nMIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEA0kgU3v3SxYzm5J6P+GPP\nmy6toBnKYtTViAVmpUJiIEjdZ9NLpoJU0na9q0CD8sIgo2Js/W/owJUvSj6rm8us\nsu/Ve5KsoJN6zca2am1uZ5IKnc48i0mCv76WXawCxM+NFGqSCMJcltlhj3fC/GDS\ngu+BiIbrgR1PgJf6Jk6l7uMJdN3TL6JJQcEC4lz+2hj5zoVNYkq06ZC79j2tPDCI\nkTAYGF/TAAVLH08/kGH5ZeRPlVKJ7cwW3OniLM5NeFnS8+shRNb6AYr7xju3Ikbw\nDo14ipIghBI0iAxn6Lvr/iilc7TM7RWJ4OiTrmK3SQSJ+U6H2N2/I5OGEHBEKzbA\nOQIDAQAB\n-----END PUBLIC KEY-----\n"},"scheme":"rsassa-pss-sha256"}},"threshold":3,"url":"/root.json"},"snapshot":{"keys":{"8660a9f40687fb33e6f8ad563f21ee81b9ce7b91c90827cc7ae2416c5e0e94e9":{"keytype":"rsa","keyval":{"public":"-----BEGIN PUBLIC KEY-----\nMIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEAqTZx29eJR5EumjqM4YTb\nFlKbim1GNYmtbCLH51BbU2lt46ddmfGvtGsxTD3mIZ/GEHVFv6Aei3xx5nIfhGP0\nrG78JRz394uU8Pd62DiIFWYizr5o+ZBZu29D2YK5ZtxoLFpgt0ibnINK2NcesDC8\nSqfIUbMiQFT6yB/MYD275SjfRGHOeYTPmKdjMJrhLL2cfIPYnQ0QFYIyMvXBG1Fj\nU0rc9UclYQHh9YheIDVYI9YCo/DWP3KFfRJpoTjQRGoPSK9TXcpCAEzQpEG3jOek\n9PdV9Ol6/O8JbrFwXWF3LhkUThg+zCjV4qHtP4oqp5QCqzTQTXGQ9qxWUSlHi4Eu\nIwIDAQAB\n-----END PUBLIC KEY-----\n"},"scheme":"rsassa-pss-sha256"}},"threshold":1,"url":"/snapshot.json"},"timestamp":{"keys":{"66d4ea1da00076c822a6e1b4df5eb1e529eb38f6edcedff323e62f2bfe3eaddd":{"keytype":"rsa","keyval":{"public":"-----BEGIN PUBLIC KEY-----\nMIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEAuzTgV5iKhMnunUDxt4PB\npYqTMPaJN/ZdOOsP6cS3DeCE/EcYGfgCjvP7KD3gjG98VDBTVcuwZClSy+/zvHhV\nIq7VWu+yxQL5c6oa1xpCyHoA96JiLIDPhmqEdscdRybcRQ2CYywzKA8jSwEQCnEK\nc8a74ceY352l/MEcOem0+AtKrOjqcjbXCayDwC9yTg/c78bkp+4T8AhSWgt6Tlrt\nY8jLE7zwojFtIYtMwobWRIW2O3nJDXiSBbTPG3M9kF1G43INshSdBcuq5Tmy8lpE\n/XiG/E7+hP63Hm+KAcdvl553Zs7pLhAZxV0kqlApqRRwhscw+JQci8sVONun5t9t\nNwIDAQAB\n-----END PUBLIC KEY-----\n"},"scheme":"rsassa-pss-sha256"}},"threshold":1,"url":"/timestamp.json"}},"spec_version":"0.1.0","version":3}} \ No newline at end of file +{"signatures":[{"keyid":"a61b695e2b86097d993e94e99fd15ec6d8fc8e9522948c9ff21c2f2c881093ae","sig":"kJdviRCj2xr6Mqb8maOJZUnBgHJqLI5V2MqhQL9mxqsHFE/6z3bmT1OIpaG31b0COhIg0gtR60Y/rUu1YRTs8UpUJa+9bHVcLxrg6PuewzT34ZjtsGpQ8PDhe2j99EULN85rq7jV+HRwdZLHVZXI1P25I0wy1AGtprVo13ZB4h5CBy4ZEUlmae0QKbH+Ej9ouVaeBlyw1JHLew8I1eWEj7x9WYMkO6spqDtPhrIYjqxnh9zUBAvMNQzv3ysu0KM+D4Pr6kALRE9G3AMmbCEzhy+C+r/FT7egNv28kJvxoQjgfEMABYLgLU/Gp7qRVbEbthH4XI8IohmiCH6NavQKBA=="},{"keyid":"9b3cea98f6f23cc11813b12d0526a1b6cfb3761008f0882c9caa8db742d63002","sig":"cQAJkEEpUdauI7TN2nrEF/wZbPKa/Rhu/xEDZgm5tBAc0LKhD48aLH0VdxapytSf1IWk+MatEK/vVMA2sppqrJTGyJ5Ue+JEUe+JhcTAFW0sUzTHUgR5ARTFqDR0RnKysPO/a9BMZsJfyEfS2j2Nlx7iE/2PeMVc2GhhSEY+cHE000ZFXxaxuWYjYzd35LeLQotMWmtsEx0JYrqHgXzMU0yJ0QikE1JUd8uQ4LMvd53iPIEuHxKyq87oIMfvx+gKhbGvgQx3XMYI54qMP+fumPVYQ3alYX5FB+vFX2MFqzl7GxAQJCd57xsanRhKSAkG2LKvEUZuCfTtqiwqb13Lyg=="},{"keyid":"5607181203a2fb60b9d725109388ccb19ccdc236a4b1d1441fbea7ad07616c4a","sig":"cPqcr9zhryFMREpIm81u+vAy1anzJFEFwWk6gCSRNW8lZuGQPgF66qc09YGHe6wGFHmlAm7CTFKh561W8zJfi6IbGS7wUArkaNV5fgPkxg2FDGCC5tHs8we3fgyv+kgr6yzGwQVHl6v4J8S5bkK1MM7GbMGAP6dmf774f6ysLOkjGurAg7UwrZey8/ZQKCjzysnAxyRC+/4QhUfAC5OPs2hI+BUjPDTET3f1Icgq0f9CZfUZ4nW1utMd4/sbQKowNVfZ8j62r8awUuOGmmJbuqLs6ifkfKCXw5ZfrT9G8ySxbI8CshWd54Ck6c7WsyRjY4Y9TB+Oh3pJ+ieE91HJKg=="},{"keyid":"d499c8ca3c2018885037da89bdfa327e21a0e6a15e2ca8dabe5e78a3edf9c91c","sig":"lnfUXTabAURE8BKzbGBJ4oXtDYbpXl1hsEtzOfjJwBae+c9qMusYMR9aU3ZOm2hlAasUXf+bfIy5NoDZMuDC8roYD3UHokC2qrym1h3VgfVl1aMlWDAgtIhNQuHVbKGyPBDMW+O0gDIV2RPwTwcedZHxLbakMqgHaSI1SPFu4ryoLbOsBr2wxQSVKUXRhcvQFvgRJS884TfIYmY3v4EIHvSfHrnPmDmMFrpMUCVzseDPMG/FEWamzjk0GnRAZiwv5NVjbtIAtIUxBASZRWQr1h176k03jwPodT/iynNTNMy/2WHMwUSrVxeFB+aXI9+Pbl/OHw3UAz2qp3kS2ny0aQ=="},{"keyid":"70033289cef8e5105914b22ecc7f15e271d0650d5216d1ab0bd67f8440411bb6","sig":"QilsuGt2x4NjB8uTQeFaUb8sfRwhwcEKGqBBAXLPtgCLm7GXxRb4RXO8nPQ0o5Kg1nZp7t899Q5nMzOO6Qc5ng3vTOwDW2cDSVhyllPKJAzgJC5uwHQdQi9y3vSiNb4j2mpkQbyfu9va5yUiAZzSoJHuRv+aperJHvv+Ev/kxqlvy/4TtXtMPebG2qc08K/WdXBA0S++oWhsC8J5mFgBJBFXFJ6ewLlRJK7DmZKEG+0vtaOeCLOsRnKFSQ3rfrMYdBnGU32+NtlmTU4rjpZ/HdSwgi4K8tDVoo1CE9EwvLvZ1oKpHyRiZWmviRCy81WVB/kpYgfSR4u1CQ02CgZVkQ=="}],"signed":{"_type":"root","expires":"2024-07-26T11:18:30+08:00","roles":{"index":{"keys":{"7fce7ec4f9c36d51dec7ec96065bb64958b743e46ea8141da668cd2ce58a9e61":{"keytype":"rsa","keyval":{"public":"-----BEGIN PUBLIC KEY-----\nMIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEAn5kVA7MlBfSe7EBaExjl\nKbwoDkn1aYi74s29mFgtRo8nejbrVvZQMCIUhvKc0pFa/l9JD/QY6/nAOCE1lpzi\nwwNkSntfOo3p3HQIR+Ut7hZ4Sxfe/5JagGo3LQ+Hd3EJWUxyEfQ/Bff07F3XAbqM\n5+cKNrdsKWZJcPiJDW621qGwCx52f+gzl9bnFe4/hx34OUgirwqh5DS+LhIO+/yt\nbOiN1AyjQKlnb8lUnblElS4Njd+F4io5VzSrZYi2+4AbTkO6wLwbsWHMzXfv9qwn\nvllufOHpB6EwiQ/xBOMuvJJymHnZvs8AH4SuydQIXLaJuv1ysFaBs0KB/ktbakSK\nLwIDAQAB\n-----END PUBLIC KEY-----\n"},"scheme":"rsassa-pss-sha256"}},"threshold":1,"url":"/index.json"},"root":{"keys":{"5607181203a2fb60b9d725109388ccb19ccdc236a4b1d1441fbea7ad07616c4a":{"keytype":"rsa","keyval":{"public":"-----BEGIN PUBLIC KEY-----\nMIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEAyDwCfCl30vhyJW7fB1bs\npRYKtBKzl7o0qnJTm+IksjQ8RXxj8osUpMLmSvOzCaJ5Wxe+Pm1LpSTDbbubbgvd\nnmEFL6228sifviNIu2HlIl+agfzmXuJ9OBlzGUaI4gAd1Z6pF6+mjlcjz2PbWF84\nAbXZdK49uluqulp7HrGB/qNjGcIRUCHgDU4nnq0OkI1BZZSKm9ovonqDkIK76x/S\niAD9OjKsjQ/s57tE+5WTVObKpfrfK0JeHdpAUsA/2n4L1Z6FmZD4LZWqb0i+C7xj\nMElC99KtjlwRntcjeVWG9YjU8AcEN0n1gON9S2oRdyyAzDTgGb7WueDnn6qstt5w\nSQIDAQAB\n-----END PUBLIC KEY-----\n"},"scheme":"rsassa-pss-sha256"},"70033289cef8e5105914b22ecc7f15e271d0650d5216d1ab0bd67f8440411bb6":{"keytype":"rsa","keyval":{"public":"-----BEGIN PUBLIC KEY-----\nMIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEApU5RHP0K+Fgkzm9L0yHR\n4CEqLLwHr7hQxjWOaq5K0UfaOKcsPQZ1SkJ/AMppz7ovzwOU4hcy0wJOV7ms6ACk\nS3hte2GlH/xp+OzWiRnI4qJ6GRrAe+ototj1ZMGvpLK4ifxkKaY6vuWFFAeS0fSe\nPHUGAl5v+PaJWgDNQTRmuAu5oCaYP6oT6VKHj6ulLAgAOqWsBSJiK3oIRcWPR+uI\nIW/9BV158wfmxAw1+7ch1RD44+1vV3+Eo94alvVZIAfcJqDS3XGr2Hfd/YWGj1d2\nD26eblBJoQt0L2E2EL8igu1sudVkMZ3NAIfmBrOWUxHEbIjYeKvXPbaSGdC+FoXD\nrwIDAQAB\n-----END PUBLIC KEY-----\n"},"scheme":"rsassa-pss-sha256"},"9b3cea98f6f23cc11813b12d0526a1b6cfb3761008f0882c9caa8db742d63002":{"keytype":"rsa","keyval":{"public":"-----BEGIN PUBLIC KEY-----\nMIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEAsOgQkwLOh31QV9OpbO9v\n6o83durJFGPOnVXZiab83pKaSk7HEK9WzXBq0BaPvtFwSfROVdpgtopri5lZi+uH\naMKLUn5F8XRnSMl/7m5vM4XpZZYa4aQId4TWdbFtTu31eHGZ3eEC5nDRJ5NhZOJd\nKLFBu/xmxrh/eNZt4QbdWLZayjHnzyoy5AnfNTR6nJgPAv+rBOqyqT/r14q4Pngh\n3z0I3pNFr5qmxsp013XV+kgOW1F7zT7IMU8xRIgo85UWUNhax0/bjY/2NI1Z+WjR\nyhZmUBMVYWvfw97xDUrvBvrJxZPgg0lGvxJC6LF2dM7wgLaNx9khT6HMBVxjxLMs\nDQIDAQAB\n-----END PUBLIC KEY-----\n"},"scheme":"rsassa-pss-sha256"},"a61b695e2b86097d993e94e99fd15ec6d8fc8e9522948c9ff21c2f2c881093ae":{"keytype":"rsa","keyval":{"public":"-----BEGIN PUBLIC KEY-----\nMIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEAnayxhw6KeoKK+Ax9RW6v\n66YjrpRpGLewLmSSAzJGX8nL5/a2nEbXbeF9po265KcBSFWol8jLBsmG56ruwwxp\noWWhJPncqGqy8wMeRMmTf7ATGa+tk+To7UAQD0MYzt7rRlIdpqi9Us3J6076Z83k\n2sxFnX9sVflhOsotGWL7hmrn/CJWxKsO6OVCoqbIlnJV8xFazE2eCfaDTIEEEgnh\nLIGDsmv1AN8ImUIn/hyKcm1PfhDZrF5qhEVhfz5D8aX3cUcEJw8BvCaNloXyHf+y\nDKjqO/dJ7YFWVt7nPqOvaEkBQGMd54ETJ/BbO9r3WTsjXKleoPovBSQ/oOxApypb\nNQIDAQAB\n-----END PUBLIC KEY-----\n"},"scheme":"rsassa-pss-sha256"},"d499c8ca3c2018885037da89bdfa327e21a0e6a15e2ca8dabe5e78a3edf9c91c":{"keytype":"rsa","keyval":{"public":"-----BEGIN PUBLIC KEY-----\nMIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEA5oDytiywLDOSpIBovZxx\nNlZJg5Gk3O9kpiOQ0XnD+L2LV+a2dJU1KmBOoGCUr2TNaGTPihAStjpFIsW4c7Ye\nB2RjUFUrXRf3mvc3n4fACayenxtnCleSR4gKkAdHqqPCiWHT5TAtybKSHuHAluUL\nkMvavUZjIPMj0YYB0R8Re7BjU+zxnipJosTbbPQ7fa3+x2VAHc066Y9qp1YucdpB\nMZ3UwtSVNK7aCbFZvKPwAm22fnDYmMbYFeTz/rrl8k+rKTM37d4D3mURC9xDJxIP\nXVaU2dBImYjoFcY0/5oBU5vr1sj2sdUH+3G5AUr6iCL+XJLiwA1x24jKA6mUjQ93\ndwIDAQAB\n-----END PUBLIC KEY-----\n"},"scheme":"rsassa-pss-sha256"}},"threshold":3,"url":"/root.json"},"snapshot":{"keys":{"8660a9f40687fb33e6f8ad563f21ee81b9ce7b91c90827cc7ae2416c5e0e94e9":{"keytype":"rsa","keyval":{"public":"-----BEGIN PUBLIC KEY-----\nMIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEAqTZx29eJR5EumjqM4YTb\nFlKbim1GNYmtbCLH51BbU2lt46ddmfGvtGsxTD3mIZ/GEHVFv6Aei3xx5nIfhGP0\nrG78JRz394uU8Pd62DiIFWYizr5o+ZBZu29D2YK5ZtxoLFpgt0ibnINK2NcesDC8\nSqfIUbMiQFT6yB/MYD275SjfRGHOeYTPmKdjMJrhLL2cfIPYnQ0QFYIyMvXBG1Fj\nU0rc9UclYQHh9YheIDVYI9YCo/DWP3KFfRJpoTjQRGoPSK9TXcpCAEzQpEG3jOek\n9PdV9Ol6/O8JbrFwXWF3LhkUThg+zCjV4qHtP4oqp5QCqzTQTXGQ9qxWUSlHi4Eu\nIwIDAQAB\n-----END PUBLIC KEY-----\n"},"scheme":"rsassa-pss-sha256"}},"threshold":1,"url":"/snapshot.json"},"timestamp":{"keys":{"66d4ea1da00076c822a6e1b4df5eb1e529eb38f6edcedff323e62f2bfe3eaddd":{"keytype":"rsa","keyval":{"public":"-----BEGIN PUBLIC KEY-----\nMIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEAuzTgV5iKhMnunUDxt4PB\npYqTMPaJN/ZdOOsP6cS3DeCE/EcYGfgCjvP7KD3gjG98VDBTVcuwZClSy+/zvHhV\nIq7VWu+yxQL5c6oa1xpCyHoA96JiLIDPhmqEdscdRybcRQ2CYywzKA8jSwEQCnEK\nc8a74ceY352l/MEcOem0+AtKrOjqcjbXCayDwC9yTg/c78bkp+4T8AhSWgt6Tlrt\nY8jLE7zwojFtIYtMwobWRIW2O3nJDXiSBbTPG3M9kF1G43INshSdBcuq5Tmy8lpE\n/XiG/E7+hP63Hm+KAcdvl553Zs7pLhAZxV0kqlApqRRwhscw+JQci8sVONun5t9t\nNwIDAQAB\n-----END PUBLIC KEY-----\n"},"scheme":"rsassa-pss-sha256"}},"threshold":1,"url":"/timestamp.json"}},"spec_version":"0.1.0","version":4}} \ No newline at end of file diff --git a/pkg/telemetry/meta.go b/pkg/telemetry/meta.go index 56ff123986..04228c3b7f 100644 --- a/pkg/telemetry/meta.go +++ b/pkg/telemetry/meta.go @@ -23,6 +23,7 @@ import ( "github.com/pingcap/tiup/pkg/crypto/rand" "github.com/pingcap/tiup/pkg/environment" "github.com/pingcap/tiup/pkg/localdata" + "github.com/pingcap/tiup/pkg/utils" "gopkg.in/yaml.v2" ) @@ -37,8 +38,6 @@ const ( DisableStatus Status = "disable" ) -const defaultStatus = EnableStatus - // Meta data of telemetry. type Meta struct { UUID string `yaml:"uuid,omitempty"` @@ -78,8 +77,7 @@ func LoadFrom(fname string) (meta *Meta, err error) { if err != nil { if os.IsNotExist(err) { - meta = NewMeta() - return meta, meta.SaveTo(fname) + return &Meta{}, nil } return } @@ -89,11 +87,11 @@ func LoadFrom(fname string) (meta *Meta, err error) { // populate UUID and secret if not set var updated bool - if meta.UUID == "" { + if meta.UUID == "" && meta.Status == EnableStatus { meta.UUID = NewUUID() updated = true } - if meta.Secret == "" { + if meta.Secret == "" && meta.Status == EnableStatus { meta.Secret = NewSecret() updated = true } @@ -111,13 +109,13 @@ func (m *Meta) SaveTo(fname string) error { return errors.AddStack(err) } - return os.WriteFile(fname, data, 0644) + return utils.WriteFile(fname, data, 0644) } // GetMeta read the telemeta from disk func GetMeta(env *environment.Environment) (meta *Meta, fname string, err error) { dir := env.Profile().Path(localdata.TelemetryDir) - err = os.MkdirAll(dir, 0755) + err = utils.MkdirAll(dir, 0755) if err != nil { return } diff --git a/pkg/telemetry/meta_test.go b/pkg/telemetry/meta_test.go index d4376cb9db..6d7a736bfd 100644 --- a/pkg/telemetry/meta_test.go +++ b/pkg/telemetry/meta_test.go @@ -23,8 +23,8 @@ func (s *teleSuite) TestTelemetry(c *check.C) { // Should no error and get a default meta. meta, err := LoadFrom(fname) c.Assert(err, check.IsNil) - c.Assert(meta.Status, check.Equals, defaultStatus) - c.Assert(len(meta.UUID), check.Greater, 0) + c.Assert(meta.Status, check.Equals, Status("")) + c.Assert(len(meta.UUID), check.Equals, 0) // Save and load back err = meta.SaveTo(fname) diff --git a/pkg/telemetry/testdata/single/nilvalue.yaml b/pkg/telemetry/testdata/single/nilvalue.yaml index c447ab13f2..147bafdcde 100644 --- a/pkg/telemetry/testdata/single/nilvalue.yaml +++ b/pkg/telemetry/testdata/single/nilvalue.yaml @@ -39,7 +39,6 @@ tikv_servers: tiflash_servers: - host: 172.19.0.101 tcp_port: 19000 - http_port: 18123 flash_service_port: 13930 flash_proxy_port: 10170 flash_proxy_status_port: 10292 diff --git a/pkg/tidbver/tidbver.go b/pkg/tidbver/tidbver.go index b7d49abe13..eae857f78e 100644 --- a/pkg/tidbver/tidbver.go +++ b/pkg/tidbver/tidbver.go @@ -26,6 +26,13 @@ func TiDBSupportSecureBoot(version string) bool { return semver.Compare(version, "v5.3.0") >= 0 || strings.Contains(version, "nightly") } +// TiDBSupportUpgradeAPI return if given version of TiDB support upgrade API +func TiDBSupportUpgradeAPI(version string) bool { + return semver.Compare(version, "v7.4.0") >= 0 || + (semver.MajorMinor(version) == "v7.1" && semver.Compare(version, "v7.1.2") >= 0) || + strings.Contains(version, "nightly") +} + // TiKVSupportAdvertiseStatusAddr return if given version of TiKV support --advertise-status-addr func TiKVSupportAdvertiseStatusAddr(version string) bool { // TiKV support --advertise-status-addr since v4.0.1 @@ -61,12 +68,43 @@ func TiFlashDeprecatedUsersConfig(version string) bool { return semver.Compare(version, "v4.0.12") >= 0 && version != "v5.0.0-rc" || strings.Contains(version, "nightly") } +// TiFlashNotNeedHTTPPortConfig return if given version of TiFlash do not need http_port config +func TiFlashNotNeedHTTPPortConfig(version string) bool { + return semver.Compare(version, "v7.1.0") >= 0 || strings.Contains(version, "nightly") +} + +// TiFlashRequiresTCPPortConfig return if given version of TiFlash requires tcp_port config. +// TiFlash 7.1.0 and later versions won't listen to tpc_port if the config is not given, which is recommended. +// However this config is required for pre-7.1.0 versions because TiFlash will listen to it anyway, +// and we must make sure the port is being configured as specified in the topology file, +// otherwise multiple TiFlash instances will conflict. +func TiFlashRequiresTCPPortConfig(version string) bool { + return semver.Compare(version, "v7.1.0") < 0 && !strings.Contains(version, "nightly") +} + // TiFlashNotNeedSomeConfig return if given version of TiFlash do not need some config like runAsDaemon func TiFlashNotNeedSomeConfig(version string) bool { // https://github.com/pingcap/tiup/pull/1673 return semver.Compare(version, "v5.4.0") >= 0 || strings.Contains(version, "nightly") } +// TiFlashPlaygroundNewStartMode return true if the given version of TiFlash could be started +// with the new implementation in TiUP playground. +func TiFlashPlaygroundNewStartMode(version string) bool { + return semver.Compare(version, "v7.1.0") >= 0 || strings.Contains(version, "nightly") +} + +// TiDBSupportDisagg returns true if the given version of TiDB and TiFlash supports +// disaggregated mode. +func TiDBSupportDisagg(version string) bool { + return semver.Compare(version, "v7.0.0") >= 0 || strings.Contains(version, "nightly") +} + +// PDSupportMicroServices returns true if the given version of PD supports micro services. +func PDSupportMicroServices(version string) bool { + return semver.Compare(version, "v7.3.0") >= 0 || strings.Contains(version, "nightly") +} + // TiCDCSupportConfigFile return if given version of TiCDC support config file func TiCDCSupportConfigFile(version string) bool { // config support since v4.0.13, ignore v5.0.0-rc @@ -99,6 +137,11 @@ func TiCDCSupportRollingUpgrade(version string) bool { return semver.Compare(version, "v6.3.0") >= 0 || strings.Contains(version, "nightly") } +// TiCDCUpgradeBeforePDTiKVTiDB return if the given version of TiCDC should upgrade TiCDC before PD and TiKV +func TiCDCUpgradeBeforePDTiKVTiDB(version string) bool { + return semver.Compare(version, "v5.1.0") >= 0 || strings.Contains(version, "nightly") +} + // NgMonitorDeployByDefault return if given version of TiDB cluster should contain ng-monitoring func NgMonitorDeployByDefault(version string) bool { return semver.Compare(version, "v5.4.0") >= 0 || strings.Contains(version, "nightly") diff --git a/pkg/tui/progress/display_props.go b/pkg/tui/progress/display_props.go index 48dd5d2462..5e01a62734 100644 --- a/pkg/tui/progress/display_props.go +++ b/pkg/tui/progress/display_props.go @@ -96,6 +96,7 @@ type DisplayProps struct { Prefix string `json:"prefix,omitempty"` Suffix string `json:"suffix,omitempty"` // If `Mode == Done / Error`, Suffix is not printed Mode Mode `json:"mode,omitempty"` + Detail string `json:"detail,omitempty"` } // String implements string diff --git a/pkg/tui/progress/example_single_bar_test.go b/pkg/tui/progress/example_single_bar_test.go index 0c1f66724a..514a91796f 100644 --- a/pkg/tui/progress/example_single_bar_test.go +++ b/pkg/tui/progress/example_single_bar_test.go @@ -1,6 +1,7 @@ package progress_test import ( + "errors" "strconv" "testing" "time" @@ -42,9 +43,44 @@ func ExampleSingleBar() { b.StopRenderLoop() } +func ExampleSingleBar_err() { + b := progress.NewSingleBar("Prefix") + + b.UpdateDisplay(&progress.DisplayProps{ + Prefix: "Prefix", + Suffix: "Suffix", + }) + + n := 3 + + go func() { + time.Sleep(time.Second) + for i := 0; i < n; i++ { + b.UpdateDisplay(&progress.DisplayProps{ + Prefix: "Prefix" + strconv.Itoa(i), + Suffix: "Suffix" + strconv.Itoa(i), + }) + time.Sleep(time.Second) + } + }() + + b.StartRenderLoop() + + time.Sleep(time.Second * time.Duration(n+1)) + + b.UpdateDisplay(&progress.DisplayProps{ + Mode: progress.ModeError, + Prefix: "Prefix", + Detail: errors.New("expected failure").Error(), + }) + + b.StopRenderLoop() +} + func TestExampleOutput(t *testing.T) { if !testing.Verbose() { return } ExampleSingleBar() + ExampleSingleBar_err() } diff --git a/pkg/tui/progress/single_bar.go b/pkg/tui/progress/single_bar.go index 854f27c160..1f5a454c4e 100644 --- a/pkg/tui/progress/single_bar.go +++ b/pkg/tui/progress/single_bar.go @@ -31,7 +31,7 @@ type singleBarCore struct { func (b *singleBarCore) renderDoneOrError(w io.Writer, dp *DisplayProps) { width := int(termSizeWidth.Load()) - var tail string + var tail, detail string var tailColor *color.Color switch dp.Mode { case ModeDone: @@ -51,7 +51,10 @@ func (b *singleBarCore) renderDoneOrError(w io.Writer, dp *DisplayProps) { } else { displayPrefix = runewidth.Truncate(dp.Prefix, width-prefixWidth, "") } - _, _ = fmt.Fprintf(w, "%s ... %s", displayPrefix, tailColor.Sprint(tail)) + if len(dp.Detail) > 0 { + detail = ": " + dp.Detail + } + _, _ = fmt.Fprintf(w, "%s ... %s%s", displayPrefix, tailColor.Sprint(tail), detail) } func (b *singleBarCore) renderSpinner(w io.Writer, dp *DisplayProps) { diff --git a/pkg/utils/http_client.go b/pkg/utils/http_client.go index ed48f60ea1..8e9b14b61d 100644 --- a/pkg/utils/http_client.go +++ b/pkg/utils/http_client.go @@ -103,7 +103,7 @@ func (c *HTTPClient) Download(ctx context.Context, url, filePath string) error { return fmt.Errorf("target file %s already exists", filePath) } - if err := CreateDir(filepath.Dir(filePath)); err != nil { + if err := MkdirAll(filepath.Dir(filePath), 0755); err != nil { return err } diff --git a/pkg/utils/ioutil.go b/pkg/utils/ioutil.go index 7e6431d346..5d56fbedcc 100644 --- a/pkg/utils/ioutil.go +++ b/pkg/utils/ioutil.go @@ -26,6 +26,7 @@ import ( "path/filepath" "strings" "sync" + "syscall" "time" "github.com/otiai10/copy" @@ -100,11 +101,21 @@ func Tar(writer io.Writer, from string) error { tarW := tar.NewWriter(compressW) defer tarW.Close() + // NOTE: filepath.Walk does not follow the symbolic link. return filepath.Walk(from, func(path string, info fs.FileInfo, err error) error { if err != nil { return err } - header, _ := tar.FileInfoHeader(info, "") + + link := "" + if info.Mode()&fs.ModeSymlink != 0 { + link, err = os.Readlink(path) + if err != nil { + return err + } + } + + header, _ := tar.FileInfoHeader(info, link) header.Name, _ = filepath.Rel(from, path) // skip "." if header.Name == "." { @@ -115,7 +126,7 @@ func Tar(writer io.Writer, from string) error { if err != nil { return err } - if !info.IsDir() { + if info.Mode().IsRegular() { fd, err := os.Open(path) if err != nil { return err @@ -140,7 +151,7 @@ func Untar(reader io.Reader, to string) error { decFile := func(hdr *tar.Header) error { file := path.Join(to, hdr.Name) - err := os.MkdirAll(filepath.Dir(file), 0755) + err := MkdirAll(filepath.Dir(file), 0755) if err != nil { return err } @@ -164,7 +175,7 @@ func Untar(reader io.Reader, to string) error { } switch hdr.Typeflag { case tar.TypeDir: - if err := os.MkdirAll(path.Join(to, hdr.Name), hdr.FileInfo().Mode()); err != nil { + if err := MkdirAll(path.Join(to, hdr.Name), hdr.FileInfo().Mode()); err != nil { return errors.Trace(err) } case tar.TypeSymlink: @@ -242,17 +253,6 @@ func Move(src, dst string) error { return errors.Trace(os.RemoveAll(src)) } -// CreateDir creates the directory if it not exists. -func CreateDir(path string) error { - if _, err := os.Stat(path); err != nil { - if os.IsNotExist(err) { - return os.MkdirAll(path, 0755) - } - return err - } - return nil -} - // Checksum returns the sha1 sum of target file func Checksum(file string) (string, error) { tarball, err := os.OpenFile(file, os.O_RDONLY, 0) @@ -372,3 +372,62 @@ func SaveFileWithBackup(path string, data []byte, backupDir string) error { return nil } + +// MkdirAll basically copied from os.MkdirAll, but use max(parent permission,minPerm) +func MkdirAll(path string, minPerm os.FileMode) error { + // Fast path: if we can tell whether path is a directory or file, stop with success or error. + dir, err := os.Stat(path) + if err == nil { + if dir.IsDir() { + return nil + } + return &os.PathError{Op: "mkdir", Path: path, Err: syscall.ENOTDIR} + } + + // Slow path: make sure parent exists and then call Mkdir for path. + i := len(path) + for i > 0 && os.IsPathSeparator(path[i-1]) { // Skip trailing path separator. + i-- + } + + j := i + for j > 0 && !os.IsPathSeparator(path[j-1]) { // Scan backward over element. + j-- + } + + if j > 1 { + // Create parent. + err = MkdirAll(path[:j-1], minPerm) + if err != nil { + return err + } + } + + perm := minPerm + fi, err := os.Stat(filepath.Dir(path)) + if err == nil { + perm |= fi.Mode().Perm() + } + + // Parent now exists; invoke Mkdir and use its result; inheritance parent perm. + err = os.Mkdir(path, perm) + if err != nil { + // Handle arguments like "foo/." by + // double-checking that directory doesn't exist. + dir, err1 := os.Lstat(path) + if err1 == nil && dir.IsDir() { + return nil + } + return err + } + return nil +} + +// WriteFile call os.WriteFile, but use max(parent permission,minPerm) +func WriteFile(name string, data []byte, perm os.FileMode) error { + fi, err := os.Stat(filepath.Dir(name)) + if err == nil { + perm |= (fi.Mode().Perm() & 0666) + } + return os.WriteFile(name, data, perm) +} diff --git a/pkg/utils/retry.go b/pkg/utils/retry.go index 0a6c65a409..a958b8c0b2 100644 --- a/pkg/utils/retry.go +++ b/pkg/utils/retry.go @@ -77,8 +77,9 @@ func Retry(doFunc func() error, opts ...RetryOption) error { // call the function var attemptCount int64 + var err error for attemptCount = 0; attemptCount < cfg.Attempts; attemptCount++ { - if err := doFunc(); err == nil { + if err = doFunc(); err == nil { return nil } @@ -91,7 +92,7 @@ func Retry(doFunc func() error, opts ...RetryOption) error { } } - return fmt.Errorf("operation exceeds the max retry attempts of %d", cfg.Attempts) + return fmt.Errorf("operation exceeds the max retry attempts of %d. error of last attempt: %s", cfg.Attempts, err) } // IsTimeoutOrMaxRetry return true if it's timeout or reach max retry. diff --git a/pkg/utils/utils.go b/pkg/utils/utils.go index 6ce59c45db..4853ff7517 100644 --- a/pkg/utils/utils.go +++ b/pkg/utils/utils.go @@ -76,3 +76,12 @@ func Ternary(condition bool, a, b any) any { func JoinHostPort(host string, port int) string { return net.JoinHostPort(host, strconv.Itoa(port)) } + +// ParseHostPort Prase host and port +func ParseHostPort(hostport string) (host, port string) { + colon := strings.LastIndex(hostport, ":") + + host = strings.TrimSuffix(strings.TrimPrefix(hostport[:colon], "["), "]") + port = hostport[colon+1:] + return +} diff --git a/pkg/version/version.go b/pkg/version/version.go index b680d10139..8645e63060 100644 --- a/pkg/version/version.go +++ b/pkg/version/version.go @@ -1,4 +1,4 @@ -// Copyright 2021 PingCAP, Inc. +// Copyright 2023 PingCAP, Inc. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. @@ -21,9 +21,9 @@ var ( // TiUPVerMajor is the major version of TiUP TiUPVerMajor = 1 // TiUPVerMinor is the minor version of TiUP - TiUPVerMinor = 11 + TiUPVerMinor = 13 // TiUPVerPatch is the patch version of TiUP - TiUPVerPatch = 0 + TiUPVerPatch = 1 // TiUPVerName is an alternative name of the version TiUPVerName = "tiup" // GitHash is the current git commit hash diff --git a/server/package/package.go b/server/package/package.go index 29978aae28..84d9b38b4f 100644 --- a/server/package/package.go +++ b/server/package/package.go @@ -19,6 +19,7 @@ import ( "os/exec" "runtime" + "github.com/pingcap/tiup/pkg/utils" "github.com/spf13/cobra" ) @@ -76,7 +77,7 @@ func execute() error { } func pack(targets []string, options packageOptions) error { - if err := os.MkdirAll("package", 0755); err != nil { + if err := utils.MkdirAll("package", 0755); err != nil { return err } diff --git a/server/session/session.go b/server/session/session.go index 087845b6e5..9eeed3b03c 100644 --- a/server/session/session.go +++ b/server/session/session.go @@ -23,6 +23,7 @@ import ( "github.com/pingcap/errors" "github.com/pingcap/tiup/pkg/localdata" logprinter "github.com/pingcap/tiup/pkg/logger/printer" + "github.com/pingcap/tiup/pkg/utils" ) // Max alive time of a session @@ -66,7 +67,7 @@ func (s *sessionManager) Write(id, name string, reader io.Reader) error { } pkgDir := path.Join(dataDir, "packages") - if err := os.MkdirAll(pkgDir, 0755); err != nil { + if err := utils.MkdirAll(pkgDir, 0755); err != nil { return errors.Annotate(err, "create package dir") } @@ -98,7 +99,7 @@ func (s *sessionManager) Read(id string) (string, io.ReadCloser, error) { } pkgDir := path.Join(dataDir, "packages") - if err := os.MkdirAll(pkgDir, 0755); err != nil { + if err := utils.MkdirAll(pkgDir, 0755); err != nil { return "", nil, errors.Annotate(err, "create package dir") } diff --git a/tests/tiup-cluster/root.json b/tests/tiup-cluster/root.json index 45278e8382..6a57d89c25 100644 --- a/tests/tiup-cluster/root.json +++ b/tests/tiup-cluster/root.json @@ -1 +1 @@ -{"signatures":[{"keyid":"9b3cea98f6f23cc11813b12d0526a1b6cfb3761008f0882c9caa8db742d63002","sig":"cHrebh9dB5z0ypRdAGKso4xBhCp6mY8pZ3ul6Ggzc5+WGU1rOGHEJPS3SP1F7FaBUu4A4aytvQ1fn0fzL20HyiwtSXu7nFzrO5MiGMNa3afyGYxGjHNWZZmYm/6+eZ9fqt7erTrtJgvuiV4VgmzCbQ3uIFEt89tRlsmkFIQ2bry4Z9ml06b/zT243pO7uInU2On+W3/ggluIoATubggzNxhB0OSqREwNfEUACq4N5UzqRwinCURISk/xUHU4/n/P9VraSocXjcMCryrsvaYHEq0AZMNNUPhSn5ow4kPqeELgb0NZxvG5wo276LngCQUqk7FvcxWSH9gMR3VcBc3OnQ=="},{"keyid":"b128ee6a42e2665bd45aa2fa4a7b6e098cfedb3911154f300634b2c056214b9e","sig":"aFlunIdjZAnm2/tY9QqKGGM2XxVM/cvNpmHrs3Z9BoGzLhOoubhPi70cEfoE2/NQX43SDKr97KT6cPY3XpkNa7mQtVxXEX1/hxc0U6qRFwdAfnV0EzuJeBfn33pfhA/TKVd6FGOEOtrLYrzigLFOuPYrQrvvfWEJGL9merz6kpjXEtTDX+378TQ5wzvphNYDVI3Hp8S+cvatwn+85agzlc4zOrRkp9TMz+vnJZSwQ6eoYqrHFQkuACto+N500lizueeEOSt2Ke17TjxeZIka192XlojHp3iiIm0TsI04DJCAsQ7yliT22BkI/NO+DhWecruJIfb7KfaTmxSG+nddiA=="},{"keyid":"2001b18089c9a865cebee793dbc23f9b1f5ce457f96b8d0b2c0909c26b00c643","sig":"H2qSlXtLWHiECouClS86Enw7pCdbM1duEyG3IcblU8pDmiu72T8SNsz4nq3TZw0hFSqAVw5udVenQ1d3sxQDix1/DCYi6PhkM/ghhyWr6+Ko2mJMT+ejqcCYFvbrb2UoeDEeS85obzi2UPELNC5iBt8iJlRtLzRvpKN+0pTwKhusnLUgan9ZLnRJsZdw2U67W2u/dABxpYsbXDYGa/4M/MkA/NEZhJ53VCvxr8RweBdEeVfKmzF/Ipkxn49aB4hlPgH/2E015U9gu7UQffiWRf+ww3eHgzjp1ItJhclUTbI0Q7BBJ4A5oYWP9DLRab+07y8+iNZmj7wTPvCS+V0+Fw=="},{"keyid":"545c31fd615bbaa4c5424509a9305eb280e019996b043a576dc12b758aa0890a","sig":"PAK7AKXTx61/MKT7ml1QrDaPlbIYZ+UmG+6xgDN/P3ForyqvLgTiqif7ZqZMlUOEXEo/aOZIcJS5yt6xp+XrQT5Dj0OAemmn7kHaYuhl5A3HyMksm2TBGxQXRByyrqsD5OtzWKHXgOIByMBBFnJbPqp3cDU+SoRYizVW114iDo0PXPXQPeX/ffVWkw7kP3l4JBgcUhCrLCND6temGJT7hAMkfPWeDeuuNlyy3ajXH+ENm6DR/RthFi1ys1rAu2j6bDwOpOu3FIzFFdGTs0fJz8tJeLIUejoDJCOwSh7qnpeIBM7cDJlT9sZrVUL5Oluelj4tgoJCO1BN8o7myVCGDw=="},{"keyid":"3f32d76cac38c9d87232ed7570132deb343a68bce4b12d365b8ab61085e1a633","sig":"2UTi6cgCF/EELQtGWU2ENk458JrFymHn1S8aVv0q3B1YoHR92Mm5Ck0FbFkfg9ySOWiEqaesIvtC8FafrpTnbVUrivYnjKOXGzvVBBeQR/Ri44rql9x+6WzjJ4p6/M+FqZEpHisx0I5AFYf2fQkL1Hztmv6yhQTrW3GOh9SVRYlZwEqjeDqm4wIFGZ6amaMr7xu2zxlTEMtz4Gd9zlZVN4Wr6qeYtkW7YLZcFECTzpwHgHJswzljhG5XTxevI/Tz4eKYOgyQCtPC/JS0HOMiTVc6d2XRs5NNvP2a4/LH1tHZRupMAibklwqiwLKfMwVl+tncb9hedxUSMPmo/CMrxw=="}],"signed":{"_type":"root","expires":"2021-05-26T11:18:30+08:00","roles":{"index":{"keys":{"7fce7ec4f9c36d51dec7ec96065bb64958b743e46ea8141da668cd2ce58a9e61":{"keytype":"rsa","keyval":{"public":"-----BEGIN PUBLIC KEY-----\nMIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEAn5kVA7MlBfSe7EBaExjl\nKbwoDkn1aYi74s29mFgtRo8nejbrVvZQMCIUhvKc0pFa/l9JD/QY6/nAOCE1lpzi\nwwNkSntfOo3p3HQIR+Ut7hZ4Sxfe/5JagGo3LQ+Hd3EJWUxyEfQ/Bff07F3XAbqM\n5+cKNrdsKWZJcPiJDW621qGwCx52f+gzl9bnFe4/hx34OUgirwqh5DS+LhIO+/yt\nbOiN1AyjQKlnb8lUnblElS4Njd+F4io5VzSrZYi2+4AbTkO6wLwbsWHMzXfv9qwn\nvllufOHpB6EwiQ/xBOMuvJJymHnZvs8AH4SuydQIXLaJuv1ysFaBs0KB/ktbakSK\nLwIDAQAB\n-----END PUBLIC KEY-----\n"},"scheme":"rsassa-pss-sha256"}},"threshold":1,"url":"/index.json"},"root":{"keys":{"2001b18089c9a865cebee793dbc23f9b1f5ce457f96b8d0b2c0909c26b00c643":{"keytype":"rsa","keyval":{"public":"-----BEGIN PUBLIC KEY-----\nMIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEA1LB2sQGelCEaKTKPzdim\n5V0RrdDOyZTSsZlWjzqZQn7lIRKG9Yjah//ReBdy3gmTwbZWUYzGFeclS+1+H05f\nvvxJUN1ttNgy6xsKql6s1ZhdwoBLbkjTqHjbRRQ2+fMJQdhusb1TXEP5Vut2jlyo\nSoGSa9mDC0VbGW9Xs/4HqfyH6m4dV6GeFYwDUX0ok6l6DHk28UIFyieKITFNkrKv\n5xoUPS3P49tX7wprXiFBKiP1Tr72O+GSTBFXuUhPASBVCXoxj7g5fB024P464ku/\nTHECfX1F5q7htz2zkgn7V9A9kedASwoqbrC5glHXfrfiQOctHkyKaGLswWe+8OAp\n5wIDAQAB\n-----END PUBLIC KEY-----\n"},"scheme":"rsassa-pss-sha256"},"3f32d76cac38c9d87232ed7570132deb343a68bce4b12d365b8ab61085e1a633":{"keytype":"rsa","keyval":{"public":"-----BEGIN PUBLIC KEY-----\nMIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEA2usmq9K1PdMUEr2AUmC/\n1l8RxbAIhyKzoA0O7zEpIunjU58BP5Ht3APPGAXqg3/lyeTcn8/h+ADEZElrgXCl\nnbsR37R3zmj+/z/M1icp/1O6gaIpHEUz5U2ryPkj9vy5TVE+7uB12x7SBDg3w5mk\nHc0NVPhHBqD9tvXlzzZSHqNjRdM+g5OdCGVzKpt/2LrCIv6MSxoxsqNP3N8I8Tb/\nunwVfhjRjXG23mjLukGPjxJUoXO38WujVWER5ZafhyZDt9VFNdPoGl2kN23aQM4p\n/gyeUgwTfeVqlXYOeErfI9AznnJA12WHTmIMNWpz3NK+c3P4GGTOZcRjnfr0J9wa\ndQIDAQAB\n-----END PUBLIC KEY-----\n"},"scheme":"rsassa-pss-sha256"},"545c31fd615bbaa4c5424509a9305eb280e019996b043a576dc12b758aa0890a":{"keytype":"rsa","keyval":{"public":"-----BEGIN PUBLIC KEY-----\nMIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEAxzlxJStPfdjMERTUs2GH\nMKMPAflgMTId3rhKsyLGAoRraAE3+crZEkXz+sEgCSW7590qDofcZYFeS9QOebD2\nI1/PYbDqMOwWkRSta6BRJyhgGKmG8QuxiYQQEQSgBhTQap3jnxiduXiZ+6uTiNkS\n44/Z12GN+vXLDLCVBlxFZx2Am9QFVCyP7f9Dxj0EkaVKRGu6+utjaWGyQLq5splk\nNbFvMLYJLkzrk8dzLwr1E85NRCAVLnRJR4fYllglJmJi6laHdOgXf9GOL1vQ/qUh\nRXqYkGiZ/15vurMMyUaIdzLY95XHw6vsjOwV9kBs8z/cxBVLxpNWUiOBsfDpmBc3\nCwIDAQAB\n-----END PUBLIC KEY-----\n"},"scheme":"rsassa-pss-sha256"},"9b3cea98f6f23cc11813b12d0526a1b6cfb3761008f0882c9caa8db742d63002":{"keytype":"rsa","keyval":{"public":"-----BEGIN PUBLIC KEY-----\nMIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEAsOgQkwLOh31QV9OpbO9v\n6o83durJFGPOnVXZiab83pKaSk7HEK9WzXBq0BaPvtFwSfROVdpgtopri5lZi+uH\naMKLUn5F8XRnSMl/7m5vM4XpZZYa4aQId4TWdbFtTu31eHGZ3eEC5nDRJ5NhZOJd\nKLFBu/xmxrh/eNZt4QbdWLZayjHnzyoy5AnfNTR6nJgPAv+rBOqyqT/r14q4Pngh\n3z0I3pNFr5qmxsp013XV+kgOW1F7zT7IMU8xRIgo85UWUNhax0/bjY/2NI1Z+WjR\nyhZmUBMVYWvfw97xDUrvBvrJxZPgg0lGvxJC6LF2dM7wgLaNx9khT6HMBVxjxLMs\nDQIDAQAB\n-----END PUBLIC KEY-----\n"},"scheme":"rsassa-pss-sha256"},"b128ee6a42e2665bd45aa2fa4a7b6e098cfedb3911154f300634b2c056214b9e":{"keytype":"rsa","keyval":{"public":"-----BEGIN PUBLIC KEY-----\nMIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEA0kgU3v3SxYzm5J6P+GPP\nmy6toBnKYtTViAVmpUJiIEjdZ9NLpoJU0na9q0CD8sIgo2Js/W/owJUvSj6rm8us\nsu/Ve5KsoJN6zca2am1uZ5IKnc48i0mCv76WXawCxM+NFGqSCMJcltlhj3fC/GDS\ngu+BiIbrgR1PgJf6Jk6l7uMJdN3TL6JJQcEC4lz+2hj5zoVNYkq06ZC79j2tPDCI\nkTAYGF/TAAVLH08/kGH5ZeRPlVKJ7cwW3OniLM5NeFnS8+shRNb6AYr7xju3Ikbw\nDo14ipIghBI0iAxn6Lvr/iilc7TM7RWJ4OiTrmK3SQSJ+U6H2N2/I5OGEHBEKzbA\nOQIDAQAB\n-----END PUBLIC KEY-----\n"},"scheme":"rsassa-pss-sha256"}},"threshold":3,"url":"/root.json"},"snapshot":{"keys":{"8660a9f40687fb33e6f8ad563f21ee81b9ce7b91c90827cc7ae2416c5e0e94e9":{"keytype":"rsa","keyval":{"public":"-----BEGIN PUBLIC KEY-----\nMIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEAqTZx29eJR5EumjqM4YTb\nFlKbim1GNYmtbCLH51BbU2lt46ddmfGvtGsxTD3mIZ/GEHVFv6Aei3xx5nIfhGP0\nrG78JRz394uU8Pd62DiIFWYizr5o+ZBZu29D2YK5ZtxoLFpgt0ibnINK2NcesDC8\nSqfIUbMiQFT6yB/MYD275SjfRGHOeYTPmKdjMJrhLL2cfIPYnQ0QFYIyMvXBG1Fj\nU0rc9UclYQHh9YheIDVYI9YCo/DWP3KFfRJpoTjQRGoPSK9TXcpCAEzQpEG3jOek\n9PdV9Ol6/O8JbrFwXWF3LhkUThg+zCjV4qHtP4oqp5QCqzTQTXGQ9qxWUSlHi4Eu\nIwIDAQAB\n-----END PUBLIC KEY-----\n"},"scheme":"rsassa-pss-sha256"}},"threshold":1,"url":"/snapshot.json"},"timestamp":{"keys":{"66d4ea1da00076c822a6e1b4df5eb1e529eb38f6edcedff323e62f2bfe3eaddd":{"keytype":"rsa","keyval":{"public":"-----BEGIN PUBLIC KEY-----\nMIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEAuzTgV5iKhMnunUDxt4PB\npYqTMPaJN/ZdOOsP6cS3DeCE/EcYGfgCjvP7KD3gjG98VDBTVcuwZClSy+/zvHhV\nIq7VWu+yxQL5c6oa1xpCyHoA96JiLIDPhmqEdscdRybcRQ2CYywzKA8jSwEQCnEK\nc8a74ceY352l/MEcOem0+AtKrOjqcjbXCayDwC9yTg/c78bkp+4T8AhSWgt6Tlrt\nY8jLE7zwojFtIYtMwobWRIW2O3nJDXiSBbTPG3M9kF1G43INshSdBcuq5Tmy8lpE\n/XiG/E7+hP63Hm+KAcdvl553Zs7pLhAZxV0kqlApqRRwhscw+JQci8sVONun5t9t\nNwIDAQAB\n-----END PUBLIC KEY-----\n"},"scheme":"rsassa-pss-sha256"}},"threshold":1,"url":"/timestamp.json"}},"spec_version":"0.1.0","version":1}} \ No newline at end of file +{"signatures":[{"keyid":"9b3cea98f6f23cc11813b12d0526a1b6cfb3761008f0882c9caa8db742d63002","sig":"kmRKAh18iBSxTU20AlqCFYV5w4OgK72RGqN5sGkDy5I+xxKlMgq2xvlWH329bpRdtmajd6wozb6ibEhiijWei7DnQCkJPVw1Y5DdKyRAHjM8ZYeawVNaUCh2VH13is6cbzSPp30CKZ5whtuERAgkLsX/M0E6bterzhQR//PTNfO41/NOgXnyqGgOslzM46N86rCBPcQwHcyHV0voXX1zGpCSDLkODEbi6ILweJl0iNnaBF+lBzvW6N+oqmEUUzbH7tLSCsJVyj3tLSlPvbTO8BoWnkN1k67WNkBBfA9vD0s46j2hWGuTMApiUa5Iou6/N0JI9Sb2kZAle15TJQJvEA=="},{"keyid":"a61b695e2b86097d993e94e99fd15ec6d8fc8e9522948c9ff21c2f2c881093ae","sig":"YNGnO9+vcnfwI0ELBdJZh1KEu62NIPZcV2TKfxEBaj7g/Fm1HO3ReAH36dew9MytDPRuIdCXYHVSOQfiBQCuJCF7k/hRQwEFlrJWHMuN2rz7YLqTUyBzkE/PbAEOTtl4C/Q8bxHZ6tHdOJAUIBsXTy4yRZVempUtIULjPNh2d6BPt7x61+3RwhhrPHKFxy2I1hVaZIMfO94Ofb4iwd0UJ1YuQdDqIP75+YtoYUsBFKee4AqvRgKrp5rqQvn0CGOOreOIXvTsszvuzxR26kWvOPPkS3zJRwTu7TW0pE0AsUgEKHKGDoShNdcmx+hGz8mn8BpAivxvJhqqJsz9nazkZA=="},{"keyid":"5607181203a2fb60b9d725109388ccb19ccdc236a4b1d1441fbea7ad07616c4a","sig":"X2FykukrMhf2qeK4bgOTy272VMFJeHAQYNGayckbjfdfToTfnRt+mwV1x5jUFTq39XsEPC6wgTs3ZWQ/3DJ1Wv7oyytN6QTPvblcFnAHNdCNGU2QceyG1N9efRbnpL+VzaWVmpwqrV2DjSHNRYMJsCfwbuC9ZCZ4HFL+bv/InZnM7Zxg7Yrl3Vzs6gOYnlZr19vgQOW/n7CCYgE2X9iS+Y3/8ALFtgU+CJEjAXK37N769H+kYG+IuuhSobdhBk/ie+oXTxm/Y5BjNBvBqetUVpCdqw/PpJ+vpN7KBGGVg3ij10wA2a1B+CA1dxjF6Tzg/HAnNRidZrbmWrkc8I69IA=="},{"keyid":"d499c8ca3c2018885037da89bdfa327e21a0e6a15e2ca8dabe5e78a3edf9c91c","sig":"wo6iQdse3ADk4SirmdjCdjOnm9sg6ztKEAFfi2BRZE1Z8FlnzUp8S+OztmgRqsVVfGEDAwgWJeimRjohznKBz0tS3bZjzLdHllw0ZHv1n+i8vaBVA5uvpZuzjpCdQbNEh4o9yrVv7hqyFVva5LEMJelOkWV6TvdgdPztHC7B7neyNm7I/0QI3mG8HHFen/CX2CHJu6OQ29XHjbe79VnTbo2ujK9b3ZIHivRMpND0/DBKxSWuXvQ2BSC1nYNDBoJ2AneOQ+aUM6WMUQst8ct9pGlZszyZX/8A1Pbj/e1ToPgkEDSqg2w+Bxty0allc4nyaijzBDo6SoKoWfXKi9wq0Q=="},{"keyid":"70033289cef8e5105914b22ecc7f15e271d0650d5216d1ab0bd67f8440411bb6","sig":"OoK5e9uZ6xu4b6WQjbJge2hsJR10DJ70vEsFRY26C52m5WOrhOJeDxthy5wF01P3odB5hWAAKYDRzB+wUy/h05O33ZXxYrbtqDkwkBYOPRr0X24MDz2nD+etOXyed69V77xi62JYSbl9aP5ItKDGrrYDkOzkQcP2q87eaVJxRJEm0qrA4Jtz8oZ/nOZwX9vZRmd/qNorye/On8yxbtueqCkTVKULI0c/CwBw/rAr+THa1alPqn24djS3MqBst+DDRWR3996M9Cz5agWBASo6TSXNoUnIOGyza7zqoNsdrePLHCbInUWuJ+lN2ilGTgl6ERyYoc0+QFmRaVljG5pGcw=="}],"signed":{"_type":"root","expires":"2024-07-26T11:18:30+08:00","roles":{"index":{"keys":{"7fce7ec4f9c36d51dec7ec96065bb64958b743e46ea8141da668cd2ce58a9e61":{"keytype":"rsa","keyval":{"public":"-----BEGIN PUBLIC KEY-----\nMIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEAn5kVA7MlBfSe7EBaExjl\nKbwoDkn1aYi74s29mFgtRo8nejbrVvZQMCIUhvKc0pFa/l9JD/QY6/nAOCE1lpzi\nwwNkSntfOo3p3HQIR+Ut7hZ4Sxfe/5JagGo3LQ+Hd3EJWUxyEfQ/Bff07F3XAbqM\n5+cKNrdsKWZJcPiJDW621qGwCx52f+gzl9bnFe4/hx34OUgirwqh5DS+LhIO+/yt\nbOiN1AyjQKlnb8lUnblElS4Njd+F4io5VzSrZYi2+4AbTkO6wLwbsWHMzXfv9qwn\nvllufOHpB6EwiQ/xBOMuvJJymHnZvs8AH4SuydQIXLaJuv1ysFaBs0KB/ktbakSK\nLwIDAQAB\n-----END PUBLIC KEY-----\n"},"scheme":"rsassa-pss-sha256"}},"threshold":1,"url":"/index.json"},"root":{"keys":{"5607181203a2fb60b9d725109388ccb19ccdc236a4b1d1441fbea7ad07616c4a":{"keytype":"rsa","keyval":{"public":"-----BEGIN PUBLIC KEY-----\nMIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEAyDwCfCl30vhyJW7fB1bs\npRYKtBKzl7o0qnJTm+IksjQ8RXxj8osUpMLmSvOzCaJ5Wxe+Pm1LpSTDbbubbgvd\nnmEFL6228sifviNIu2HlIl+agfzmXuJ9OBlzGUaI4gAd1Z6pF6+mjlcjz2PbWF84\nAbXZdK49uluqulp7HrGB/qNjGcIRUCHgDU4nnq0OkI1BZZSKm9ovonqDkIK76x/S\niAD9OjKsjQ/s57tE+5WTVObKpfrfK0JeHdpAUsA/2n4L1Z6FmZD4LZWqb0i+C7xj\nMElC99KtjlwRntcjeVWG9YjU8AcEN0n1gON9S2oRdyyAzDTgGb7WueDnn6qstt5w\nSQIDAQAB\n-----END PUBLIC KEY-----\n"},"scheme":"rsassa-pss-sha256"},"70033289cef8e5105914b22ecc7f15e271d0650d5216d1ab0bd67f8440411bb6":{"keytype":"rsa","keyval":{"public":"-----BEGIN PUBLIC KEY-----\nMIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEApU5RHP0K+Fgkzm9L0yHR\n4CEqLLwHr7hQxjWOaq5K0UfaOKcsPQZ1SkJ/AMppz7ovzwOU4hcy0wJOV7ms6ACk\nS3hte2GlH/xp+OzWiRnI4qJ6GRrAe+ototj1ZMGvpLK4ifxkKaY6vuWFFAeS0fSe\nPHUGAl5v+PaJWgDNQTRmuAu5oCaYP6oT6VKHj6ulLAgAOqWsBSJiK3oIRcWPR+uI\nIW/9BV158wfmxAw1+7ch1RD44+1vV3+Eo94alvVZIAfcJqDS3XGr2Hfd/YWGj1d2\nD26eblBJoQt0L2E2EL8igu1sudVkMZ3NAIfmBrOWUxHEbIjYeKvXPbaSGdC+FoXD\nrwIDAQAB\n-----END PUBLIC KEY-----\n"},"scheme":"rsassa-pss-sha256"},"9b3cea98f6f23cc11813b12d0526a1b6cfb3761008f0882c9caa8db742d63002":{"keytype":"rsa","keyval":{"public":"-----BEGIN PUBLIC KEY-----\nMIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEAsOgQkwLOh31QV9OpbO9v\n6o83durJFGPOnVXZiab83pKaSk7HEK9WzXBq0BaPvtFwSfROVdpgtopri5lZi+uH\naMKLUn5F8XRnSMl/7m5vM4XpZZYa4aQId4TWdbFtTu31eHGZ3eEC5nDRJ5NhZOJd\nKLFBu/xmxrh/eNZt4QbdWLZayjHnzyoy5AnfNTR6nJgPAv+rBOqyqT/r14q4Pngh\n3z0I3pNFr5qmxsp013XV+kgOW1F7zT7IMU8xRIgo85UWUNhax0/bjY/2NI1Z+WjR\nyhZmUBMVYWvfw97xDUrvBvrJxZPgg0lGvxJC6LF2dM7wgLaNx9khT6HMBVxjxLMs\nDQIDAQAB\n-----END PUBLIC KEY-----\n"},"scheme":"rsassa-pss-sha256"},"a61b695e2b86097d993e94e99fd15ec6d8fc8e9522948c9ff21c2f2c881093ae":{"keytype":"rsa","keyval":{"public":"-----BEGIN PUBLIC KEY-----\nMIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEAnayxhw6KeoKK+Ax9RW6v\n66YjrpRpGLewLmSSAzJGX8nL5/a2nEbXbeF9po265KcBSFWol8jLBsmG56ruwwxp\noWWhJPncqGqy8wMeRMmTf7ATGa+tk+To7UAQD0MYzt7rRlIdpqi9Us3J6076Z83k\n2sxFnX9sVflhOsotGWL7hmrn/CJWxKsO6OVCoqbIlnJV8xFazE2eCfaDTIEEEgnh\nLIGDsmv1AN8ImUIn/hyKcm1PfhDZrF5qhEVhfz5D8aX3cUcEJw8BvCaNloXyHf+y\nDKjqO/dJ7YFWVt7nPqOvaEkBQGMd54ETJ/BbO9r3WTsjXKleoPovBSQ/oOxApypb\nNQIDAQAB\n-----END PUBLIC KEY-----\n"},"scheme":"rsassa-pss-sha256"},"d499c8ca3c2018885037da89bdfa327e21a0e6a15e2ca8dabe5e78a3edf9c91c":{"keytype":"rsa","keyval":{"public":"-----BEGIN PUBLIC KEY-----\nMIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEA5oDytiywLDOSpIBovZxx\nNlZJg5Gk3O9kpiOQ0XnD+L2LV+a2dJU1KmBOoGCUr2TNaGTPihAStjpFIsW4c7Ye\nB2RjUFUrXRf3mvc3n4fACayenxtnCleSR4gKkAdHqqPCiWHT5TAtybKSHuHAluUL\nkMvavUZjIPMj0YYB0R8Re7BjU+zxnipJosTbbPQ7fa3+x2VAHc066Y9qp1YucdpB\nMZ3UwtSVNK7aCbFZvKPwAm22fnDYmMbYFeTz/rrl8k+rKTM37d4D3mURC9xDJxIP\nXVaU2dBImYjoFcY0/5oBU5vr1sj2sdUH+3G5AUr6iCL+XJLiwA1x24jKA6mUjQ93\ndwIDAQAB\n-----END PUBLIC KEY-----\n"},"scheme":"rsassa-pss-sha256"}},"threshold":3,"url":"/root.json"},"snapshot":{"keys":{"8660a9f40687fb33e6f8ad563f21ee81b9ce7b91c90827cc7ae2416c5e0e94e9":{"keytype":"rsa","keyval":{"public":"-----BEGIN PUBLIC KEY-----\nMIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEAqTZx29eJR5EumjqM4YTb\nFlKbim1GNYmtbCLH51BbU2lt46ddmfGvtGsxTD3mIZ/GEHVFv6Aei3xx5nIfhGP0\nrG78JRz394uU8Pd62DiIFWYizr5o+ZBZu29D2YK5ZtxoLFpgt0ibnINK2NcesDC8\nSqfIUbMiQFT6yB/MYD275SjfRGHOeYTPmKdjMJrhLL2cfIPYnQ0QFYIyMvXBG1Fj\nU0rc9UclYQHh9YheIDVYI9YCo/DWP3KFfRJpoTjQRGoPSK9TXcpCAEzQpEG3jOek\n9PdV9Ol6/O8JbrFwXWF3LhkUThg+zCjV4qHtP4oqp5QCqzTQTXGQ9qxWUSlHi4Eu\nIwIDAQAB\n-----END PUBLIC KEY-----\n"},"scheme":"rsassa-pss-sha256"}},"threshold":1,"url":"/snapshot.json"},"timestamp":{"keys":{"66d4ea1da00076c822a6e1b4df5eb1e529eb38f6edcedff323e62f2bfe3eaddd":{"keytype":"rsa","keyval":{"public":"-----BEGIN PUBLIC KEY-----\nMIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEAuzTgV5iKhMnunUDxt4PB\npYqTMPaJN/ZdOOsP6cS3DeCE/EcYGfgCjvP7KD3gjG98VDBTVcuwZClSy+/zvHhV\nIq7VWu+yxQL5c6oa1xpCyHoA96JiLIDPhmqEdscdRybcRQ2CYywzKA8jSwEQCnEK\nc8a74ceY352l/MEcOem0+AtKrOjqcjbXCayDwC9yTg/c78bkp+4T8AhSWgt6Tlrt\nY8jLE7zwojFtIYtMwobWRIW2O3nJDXiSBbTPG3M9kF1G43INshSdBcuq5Tmy8lpE\n/XiG/E7+hP63Hm+KAcdvl553Zs7pLhAZxV0kqlApqRRwhscw+JQci8sVONun5t9t\nNwIDAQAB\n-----END PUBLIC KEY-----\n"},"scheme":"rsassa-pss-sha256"}},"threshold":1,"url":"/timestamp.json"}},"spec_version":"0.1.0","version":4}} \ No newline at end of file diff --git a/tests/tiup-cluster/script/tikv_cdc.sh b/tests/tiup-cluster/script/tikv_cdc.sh index c9f73357a2..0e1877cb98 100755 --- a/tests/tiup-cluster/script/tikv_cdc.sh +++ b/tests/tiup-cluster/script/tikv_cdc.sh @@ -68,8 +68,9 @@ function tikv_cdc_test() { # Patch if [[ ! -z "$tikv_cdc_patch" ]]; then + wget https://tiup-mirrors.pingcap.com/tikv-cdc-v${tikv_cdc_patch}-linux-amd64.tar.gz tiup install tikv-cdc:v${tikv_cdc_patch} - tiup-cluster --yes patch $name ~/.tiup/storage/cluster/packages/tikv-cdc-v${tikv_cdc_patch}-linux-amd64.tar.gz -R tikv-cdc --offline + tiup-cluster --yes patch $name ./tikv-cdc-v${tikv_cdc_patch}-linux-amd64.tar.gz -R tikv-cdc --offline tiup-cluster display $name | grep "tikv-cdc (patched)" fi @@ -81,9 +82,9 @@ function tikv_cdc_test() { # test patch overwrite if [[ ! -z "$tikv_cdc_patch" ]]; then - tiup-cluster --yes patch $name ~/.tiup/storage/cluster/packages/tikv-cdc-v${tikv_cdc_patch}-linux-amd64.tar.gz -R tikv-cdc --overwrite + tiup-cluster --yes patch $name ./tikv-cdc-v${tikv_cdc_patch}-linux-amd64.tar.gz -R tikv-cdc --overwrite # overwrite with the same tarball twice - tiup-cluster --yes patch $name ~/.tiup/storage/cluster/packages/tikv-cdc-v${tikv_cdc_patch}-linux-amd64.tar.gz -R tikv-cdc --overwrite + tiup-cluster --yes patch $name ./tikv-cdc-v${tikv_cdc_patch}-linux-amd64.tar.gz -R tikv-cdc --overwrite fi tiup-cluster --yes stop $name diff --git a/tests/tiup-dm/root.json b/tests/tiup-dm/root.json index 45278e8382..6a57d89c25 100644 --- a/tests/tiup-dm/root.json +++ b/tests/tiup-dm/root.json @@ -1 +1 @@ -{"signatures":[{"keyid":"9b3cea98f6f23cc11813b12d0526a1b6cfb3761008f0882c9caa8db742d63002","sig":"cHrebh9dB5z0ypRdAGKso4xBhCp6mY8pZ3ul6Ggzc5+WGU1rOGHEJPS3SP1F7FaBUu4A4aytvQ1fn0fzL20HyiwtSXu7nFzrO5MiGMNa3afyGYxGjHNWZZmYm/6+eZ9fqt7erTrtJgvuiV4VgmzCbQ3uIFEt89tRlsmkFIQ2bry4Z9ml06b/zT243pO7uInU2On+W3/ggluIoATubggzNxhB0OSqREwNfEUACq4N5UzqRwinCURISk/xUHU4/n/P9VraSocXjcMCryrsvaYHEq0AZMNNUPhSn5ow4kPqeELgb0NZxvG5wo276LngCQUqk7FvcxWSH9gMR3VcBc3OnQ=="},{"keyid":"b128ee6a42e2665bd45aa2fa4a7b6e098cfedb3911154f300634b2c056214b9e","sig":"aFlunIdjZAnm2/tY9QqKGGM2XxVM/cvNpmHrs3Z9BoGzLhOoubhPi70cEfoE2/NQX43SDKr97KT6cPY3XpkNa7mQtVxXEX1/hxc0U6qRFwdAfnV0EzuJeBfn33pfhA/TKVd6FGOEOtrLYrzigLFOuPYrQrvvfWEJGL9merz6kpjXEtTDX+378TQ5wzvphNYDVI3Hp8S+cvatwn+85agzlc4zOrRkp9TMz+vnJZSwQ6eoYqrHFQkuACto+N500lizueeEOSt2Ke17TjxeZIka192XlojHp3iiIm0TsI04DJCAsQ7yliT22BkI/NO+DhWecruJIfb7KfaTmxSG+nddiA=="},{"keyid":"2001b18089c9a865cebee793dbc23f9b1f5ce457f96b8d0b2c0909c26b00c643","sig":"H2qSlXtLWHiECouClS86Enw7pCdbM1duEyG3IcblU8pDmiu72T8SNsz4nq3TZw0hFSqAVw5udVenQ1d3sxQDix1/DCYi6PhkM/ghhyWr6+Ko2mJMT+ejqcCYFvbrb2UoeDEeS85obzi2UPELNC5iBt8iJlRtLzRvpKN+0pTwKhusnLUgan9ZLnRJsZdw2U67W2u/dABxpYsbXDYGa/4M/MkA/NEZhJ53VCvxr8RweBdEeVfKmzF/Ipkxn49aB4hlPgH/2E015U9gu7UQffiWRf+ww3eHgzjp1ItJhclUTbI0Q7BBJ4A5oYWP9DLRab+07y8+iNZmj7wTPvCS+V0+Fw=="},{"keyid":"545c31fd615bbaa4c5424509a9305eb280e019996b043a576dc12b758aa0890a","sig":"PAK7AKXTx61/MKT7ml1QrDaPlbIYZ+UmG+6xgDN/P3ForyqvLgTiqif7ZqZMlUOEXEo/aOZIcJS5yt6xp+XrQT5Dj0OAemmn7kHaYuhl5A3HyMksm2TBGxQXRByyrqsD5OtzWKHXgOIByMBBFnJbPqp3cDU+SoRYizVW114iDo0PXPXQPeX/ffVWkw7kP3l4JBgcUhCrLCND6temGJT7hAMkfPWeDeuuNlyy3ajXH+ENm6DR/RthFi1ys1rAu2j6bDwOpOu3FIzFFdGTs0fJz8tJeLIUejoDJCOwSh7qnpeIBM7cDJlT9sZrVUL5Oluelj4tgoJCO1BN8o7myVCGDw=="},{"keyid":"3f32d76cac38c9d87232ed7570132deb343a68bce4b12d365b8ab61085e1a633","sig":"2UTi6cgCF/EELQtGWU2ENk458JrFymHn1S8aVv0q3B1YoHR92Mm5Ck0FbFkfg9ySOWiEqaesIvtC8FafrpTnbVUrivYnjKOXGzvVBBeQR/Ri44rql9x+6WzjJ4p6/M+FqZEpHisx0I5AFYf2fQkL1Hztmv6yhQTrW3GOh9SVRYlZwEqjeDqm4wIFGZ6amaMr7xu2zxlTEMtz4Gd9zlZVN4Wr6qeYtkW7YLZcFECTzpwHgHJswzljhG5XTxevI/Tz4eKYOgyQCtPC/JS0HOMiTVc6d2XRs5NNvP2a4/LH1tHZRupMAibklwqiwLKfMwVl+tncb9hedxUSMPmo/CMrxw=="}],"signed":{"_type":"root","expires":"2021-05-26T11:18:30+08:00","roles":{"index":{"keys":{"7fce7ec4f9c36d51dec7ec96065bb64958b743e46ea8141da668cd2ce58a9e61":{"keytype":"rsa","keyval":{"public":"-----BEGIN PUBLIC KEY-----\nMIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEAn5kVA7MlBfSe7EBaExjl\nKbwoDkn1aYi74s29mFgtRo8nejbrVvZQMCIUhvKc0pFa/l9JD/QY6/nAOCE1lpzi\nwwNkSntfOo3p3HQIR+Ut7hZ4Sxfe/5JagGo3LQ+Hd3EJWUxyEfQ/Bff07F3XAbqM\n5+cKNrdsKWZJcPiJDW621qGwCx52f+gzl9bnFe4/hx34OUgirwqh5DS+LhIO+/yt\nbOiN1AyjQKlnb8lUnblElS4Njd+F4io5VzSrZYi2+4AbTkO6wLwbsWHMzXfv9qwn\nvllufOHpB6EwiQ/xBOMuvJJymHnZvs8AH4SuydQIXLaJuv1ysFaBs0KB/ktbakSK\nLwIDAQAB\n-----END PUBLIC KEY-----\n"},"scheme":"rsassa-pss-sha256"}},"threshold":1,"url":"/index.json"},"root":{"keys":{"2001b18089c9a865cebee793dbc23f9b1f5ce457f96b8d0b2c0909c26b00c643":{"keytype":"rsa","keyval":{"public":"-----BEGIN PUBLIC KEY-----\nMIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEA1LB2sQGelCEaKTKPzdim\n5V0RrdDOyZTSsZlWjzqZQn7lIRKG9Yjah//ReBdy3gmTwbZWUYzGFeclS+1+H05f\nvvxJUN1ttNgy6xsKql6s1ZhdwoBLbkjTqHjbRRQ2+fMJQdhusb1TXEP5Vut2jlyo\nSoGSa9mDC0VbGW9Xs/4HqfyH6m4dV6GeFYwDUX0ok6l6DHk28UIFyieKITFNkrKv\n5xoUPS3P49tX7wprXiFBKiP1Tr72O+GSTBFXuUhPASBVCXoxj7g5fB024P464ku/\nTHECfX1F5q7htz2zkgn7V9A9kedASwoqbrC5glHXfrfiQOctHkyKaGLswWe+8OAp\n5wIDAQAB\n-----END PUBLIC KEY-----\n"},"scheme":"rsassa-pss-sha256"},"3f32d76cac38c9d87232ed7570132deb343a68bce4b12d365b8ab61085e1a633":{"keytype":"rsa","keyval":{"public":"-----BEGIN PUBLIC KEY-----\nMIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEA2usmq9K1PdMUEr2AUmC/\n1l8RxbAIhyKzoA0O7zEpIunjU58BP5Ht3APPGAXqg3/lyeTcn8/h+ADEZElrgXCl\nnbsR37R3zmj+/z/M1icp/1O6gaIpHEUz5U2ryPkj9vy5TVE+7uB12x7SBDg3w5mk\nHc0NVPhHBqD9tvXlzzZSHqNjRdM+g5OdCGVzKpt/2LrCIv6MSxoxsqNP3N8I8Tb/\nunwVfhjRjXG23mjLukGPjxJUoXO38WujVWER5ZafhyZDt9VFNdPoGl2kN23aQM4p\n/gyeUgwTfeVqlXYOeErfI9AznnJA12WHTmIMNWpz3NK+c3P4GGTOZcRjnfr0J9wa\ndQIDAQAB\n-----END PUBLIC KEY-----\n"},"scheme":"rsassa-pss-sha256"},"545c31fd615bbaa4c5424509a9305eb280e019996b043a576dc12b758aa0890a":{"keytype":"rsa","keyval":{"public":"-----BEGIN PUBLIC KEY-----\nMIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEAxzlxJStPfdjMERTUs2GH\nMKMPAflgMTId3rhKsyLGAoRraAE3+crZEkXz+sEgCSW7590qDofcZYFeS9QOebD2\nI1/PYbDqMOwWkRSta6BRJyhgGKmG8QuxiYQQEQSgBhTQap3jnxiduXiZ+6uTiNkS\n44/Z12GN+vXLDLCVBlxFZx2Am9QFVCyP7f9Dxj0EkaVKRGu6+utjaWGyQLq5splk\nNbFvMLYJLkzrk8dzLwr1E85NRCAVLnRJR4fYllglJmJi6laHdOgXf9GOL1vQ/qUh\nRXqYkGiZ/15vurMMyUaIdzLY95XHw6vsjOwV9kBs8z/cxBVLxpNWUiOBsfDpmBc3\nCwIDAQAB\n-----END PUBLIC KEY-----\n"},"scheme":"rsassa-pss-sha256"},"9b3cea98f6f23cc11813b12d0526a1b6cfb3761008f0882c9caa8db742d63002":{"keytype":"rsa","keyval":{"public":"-----BEGIN PUBLIC KEY-----\nMIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEAsOgQkwLOh31QV9OpbO9v\n6o83durJFGPOnVXZiab83pKaSk7HEK9WzXBq0BaPvtFwSfROVdpgtopri5lZi+uH\naMKLUn5F8XRnSMl/7m5vM4XpZZYa4aQId4TWdbFtTu31eHGZ3eEC5nDRJ5NhZOJd\nKLFBu/xmxrh/eNZt4QbdWLZayjHnzyoy5AnfNTR6nJgPAv+rBOqyqT/r14q4Pngh\n3z0I3pNFr5qmxsp013XV+kgOW1F7zT7IMU8xRIgo85UWUNhax0/bjY/2NI1Z+WjR\nyhZmUBMVYWvfw97xDUrvBvrJxZPgg0lGvxJC6LF2dM7wgLaNx9khT6HMBVxjxLMs\nDQIDAQAB\n-----END PUBLIC KEY-----\n"},"scheme":"rsassa-pss-sha256"},"b128ee6a42e2665bd45aa2fa4a7b6e098cfedb3911154f300634b2c056214b9e":{"keytype":"rsa","keyval":{"public":"-----BEGIN PUBLIC KEY-----\nMIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEA0kgU3v3SxYzm5J6P+GPP\nmy6toBnKYtTViAVmpUJiIEjdZ9NLpoJU0na9q0CD8sIgo2Js/W/owJUvSj6rm8us\nsu/Ve5KsoJN6zca2am1uZ5IKnc48i0mCv76WXawCxM+NFGqSCMJcltlhj3fC/GDS\ngu+BiIbrgR1PgJf6Jk6l7uMJdN3TL6JJQcEC4lz+2hj5zoVNYkq06ZC79j2tPDCI\nkTAYGF/TAAVLH08/kGH5ZeRPlVKJ7cwW3OniLM5NeFnS8+shRNb6AYr7xju3Ikbw\nDo14ipIghBI0iAxn6Lvr/iilc7TM7RWJ4OiTrmK3SQSJ+U6H2N2/I5OGEHBEKzbA\nOQIDAQAB\n-----END PUBLIC KEY-----\n"},"scheme":"rsassa-pss-sha256"}},"threshold":3,"url":"/root.json"},"snapshot":{"keys":{"8660a9f40687fb33e6f8ad563f21ee81b9ce7b91c90827cc7ae2416c5e0e94e9":{"keytype":"rsa","keyval":{"public":"-----BEGIN PUBLIC KEY-----\nMIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEAqTZx29eJR5EumjqM4YTb\nFlKbim1GNYmtbCLH51BbU2lt46ddmfGvtGsxTD3mIZ/GEHVFv6Aei3xx5nIfhGP0\nrG78JRz394uU8Pd62DiIFWYizr5o+ZBZu29D2YK5ZtxoLFpgt0ibnINK2NcesDC8\nSqfIUbMiQFT6yB/MYD275SjfRGHOeYTPmKdjMJrhLL2cfIPYnQ0QFYIyMvXBG1Fj\nU0rc9UclYQHh9YheIDVYI9YCo/DWP3KFfRJpoTjQRGoPSK9TXcpCAEzQpEG3jOek\n9PdV9Ol6/O8JbrFwXWF3LhkUThg+zCjV4qHtP4oqp5QCqzTQTXGQ9qxWUSlHi4Eu\nIwIDAQAB\n-----END PUBLIC KEY-----\n"},"scheme":"rsassa-pss-sha256"}},"threshold":1,"url":"/snapshot.json"},"timestamp":{"keys":{"66d4ea1da00076c822a6e1b4df5eb1e529eb38f6edcedff323e62f2bfe3eaddd":{"keytype":"rsa","keyval":{"public":"-----BEGIN PUBLIC KEY-----\nMIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEAuzTgV5iKhMnunUDxt4PB\npYqTMPaJN/ZdOOsP6cS3DeCE/EcYGfgCjvP7KD3gjG98VDBTVcuwZClSy+/zvHhV\nIq7VWu+yxQL5c6oa1xpCyHoA96JiLIDPhmqEdscdRybcRQ2CYywzKA8jSwEQCnEK\nc8a74ceY352l/MEcOem0+AtKrOjqcjbXCayDwC9yTg/c78bkp+4T8AhSWgt6Tlrt\nY8jLE7zwojFtIYtMwobWRIW2O3nJDXiSBbTPG3M9kF1G43INshSdBcuq5Tmy8lpE\n/XiG/E7+hP63Hm+KAcdvl553Zs7pLhAZxV0kqlApqRRwhscw+JQci8sVONun5t9t\nNwIDAQAB\n-----END PUBLIC KEY-----\n"},"scheme":"rsassa-pss-sha256"}},"threshold":1,"url":"/timestamp.json"}},"spec_version":"0.1.0","version":1}} \ No newline at end of file +{"signatures":[{"keyid":"9b3cea98f6f23cc11813b12d0526a1b6cfb3761008f0882c9caa8db742d63002","sig":"kmRKAh18iBSxTU20AlqCFYV5w4OgK72RGqN5sGkDy5I+xxKlMgq2xvlWH329bpRdtmajd6wozb6ibEhiijWei7DnQCkJPVw1Y5DdKyRAHjM8ZYeawVNaUCh2VH13is6cbzSPp30CKZ5whtuERAgkLsX/M0E6bterzhQR//PTNfO41/NOgXnyqGgOslzM46N86rCBPcQwHcyHV0voXX1zGpCSDLkODEbi6ILweJl0iNnaBF+lBzvW6N+oqmEUUzbH7tLSCsJVyj3tLSlPvbTO8BoWnkN1k67WNkBBfA9vD0s46j2hWGuTMApiUa5Iou6/N0JI9Sb2kZAle15TJQJvEA=="},{"keyid":"a61b695e2b86097d993e94e99fd15ec6d8fc8e9522948c9ff21c2f2c881093ae","sig":"YNGnO9+vcnfwI0ELBdJZh1KEu62NIPZcV2TKfxEBaj7g/Fm1HO3ReAH36dew9MytDPRuIdCXYHVSOQfiBQCuJCF7k/hRQwEFlrJWHMuN2rz7YLqTUyBzkE/PbAEOTtl4C/Q8bxHZ6tHdOJAUIBsXTy4yRZVempUtIULjPNh2d6BPt7x61+3RwhhrPHKFxy2I1hVaZIMfO94Ofb4iwd0UJ1YuQdDqIP75+YtoYUsBFKee4AqvRgKrp5rqQvn0CGOOreOIXvTsszvuzxR26kWvOPPkS3zJRwTu7TW0pE0AsUgEKHKGDoShNdcmx+hGz8mn8BpAivxvJhqqJsz9nazkZA=="},{"keyid":"5607181203a2fb60b9d725109388ccb19ccdc236a4b1d1441fbea7ad07616c4a","sig":"X2FykukrMhf2qeK4bgOTy272VMFJeHAQYNGayckbjfdfToTfnRt+mwV1x5jUFTq39XsEPC6wgTs3ZWQ/3DJ1Wv7oyytN6QTPvblcFnAHNdCNGU2QceyG1N9efRbnpL+VzaWVmpwqrV2DjSHNRYMJsCfwbuC9ZCZ4HFL+bv/InZnM7Zxg7Yrl3Vzs6gOYnlZr19vgQOW/n7CCYgE2X9iS+Y3/8ALFtgU+CJEjAXK37N769H+kYG+IuuhSobdhBk/ie+oXTxm/Y5BjNBvBqetUVpCdqw/PpJ+vpN7KBGGVg3ij10wA2a1B+CA1dxjF6Tzg/HAnNRidZrbmWrkc8I69IA=="},{"keyid":"d499c8ca3c2018885037da89bdfa327e21a0e6a15e2ca8dabe5e78a3edf9c91c","sig":"wo6iQdse3ADk4SirmdjCdjOnm9sg6ztKEAFfi2BRZE1Z8FlnzUp8S+OztmgRqsVVfGEDAwgWJeimRjohznKBz0tS3bZjzLdHllw0ZHv1n+i8vaBVA5uvpZuzjpCdQbNEh4o9yrVv7hqyFVva5LEMJelOkWV6TvdgdPztHC7B7neyNm7I/0QI3mG8HHFen/CX2CHJu6OQ29XHjbe79VnTbo2ujK9b3ZIHivRMpND0/DBKxSWuXvQ2BSC1nYNDBoJ2AneOQ+aUM6WMUQst8ct9pGlZszyZX/8A1Pbj/e1ToPgkEDSqg2w+Bxty0allc4nyaijzBDo6SoKoWfXKi9wq0Q=="},{"keyid":"70033289cef8e5105914b22ecc7f15e271d0650d5216d1ab0bd67f8440411bb6","sig":"OoK5e9uZ6xu4b6WQjbJge2hsJR10DJ70vEsFRY26C52m5WOrhOJeDxthy5wF01P3odB5hWAAKYDRzB+wUy/h05O33ZXxYrbtqDkwkBYOPRr0X24MDz2nD+etOXyed69V77xi62JYSbl9aP5ItKDGrrYDkOzkQcP2q87eaVJxRJEm0qrA4Jtz8oZ/nOZwX9vZRmd/qNorye/On8yxbtueqCkTVKULI0c/CwBw/rAr+THa1alPqn24djS3MqBst+DDRWR3996M9Cz5agWBASo6TSXNoUnIOGyza7zqoNsdrePLHCbInUWuJ+lN2ilGTgl6ERyYoc0+QFmRaVljG5pGcw=="}],"signed":{"_type":"root","expires":"2024-07-26T11:18:30+08:00","roles":{"index":{"keys":{"7fce7ec4f9c36d51dec7ec96065bb64958b743e46ea8141da668cd2ce58a9e61":{"keytype":"rsa","keyval":{"public":"-----BEGIN PUBLIC KEY-----\nMIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEAn5kVA7MlBfSe7EBaExjl\nKbwoDkn1aYi74s29mFgtRo8nejbrVvZQMCIUhvKc0pFa/l9JD/QY6/nAOCE1lpzi\nwwNkSntfOo3p3HQIR+Ut7hZ4Sxfe/5JagGo3LQ+Hd3EJWUxyEfQ/Bff07F3XAbqM\n5+cKNrdsKWZJcPiJDW621qGwCx52f+gzl9bnFe4/hx34OUgirwqh5DS+LhIO+/yt\nbOiN1AyjQKlnb8lUnblElS4Njd+F4io5VzSrZYi2+4AbTkO6wLwbsWHMzXfv9qwn\nvllufOHpB6EwiQ/xBOMuvJJymHnZvs8AH4SuydQIXLaJuv1ysFaBs0KB/ktbakSK\nLwIDAQAB\n-----END PUBLIC KEY-----\n"},"scheme":"rsassa-pss-sha256"}},"threshold":1,"url":"/index.json"},"root":{"keys":{"5607181203a2fb60b9d725109388ccb19ccdc236a4b1d1441fbea7ad07616c4a":{"keytype":"rsa","keyval":{"public":"-----BEGIN PUBLIC KEY-----\nMIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEAyDwCfCl30vhyJW7fB1bs\npRYKtBKzl7o0qnJTm+IksjQ8RXxj8osUpMLmSvOzCaJ5Wxe+Pm1LpSTDbbubbgvd\nnmEFL6228sifviNIu2HlIl+agfzmXuJ9OBlzGUaI4gAd1Z6pF6+mjlcjz2PbWF84\nAbXZdK49uluqulp7HrGB/qNjGcIRUCHgDU4nnq0OkI1BZZSKm9ovonqDkIK76x/S\niAD9OjKsjQ/s57tE+5WTVObKpfrfK0JeHdpAUsA/2n4L1Z6FmZD4LZWqb0i+C7xj\nMElC99KtjlwRntcjeVWG9YjU8AcEN0n1gON9S2oRdyyAzDTgGb7WueDnn6qstt5w\nSQIDAQAB\n-----END PUBLIC KEY-----\n"},"scheme":"rsassa-pss-sha256"},"70033289cef8e5105914b22ecc7f15e271d0650d5216d1ab0bd67f8440411bb6":{"keytype":"rsa","keyval":{"public":"-----BEGIN PUBLIC KEY-----\nMIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEApU5RHP0K+Fgkzm9L0yHR\n4CEqLLwHr7hQxjWOaq5K0UfaOKcsPQZ1SkJ/AMppz7ovzwOU4hcy0wJOV7ms6ACk\nS3hte2GlH/xp+OzWiRnI4qJ6GRrAe+ototj1ZMGvpLK4ifxkKaY6vuWFFAeS0fSe\nPHUGAl5v+PaJWgDNQTRmuAu5oCaYP6oT6VKHj6ulLAgAOqWsBSJiK3oIRcWPR+uI\nIW/9BV158wfmxAw1+7ch1RD44+1vV3+Eo94alvVZIAfcJqDS3XGr2Hfd/YWGj1d2\nD26eblBJoQt0L2E2EL8igu1sudVkMZ3NAIfmBrOWUxHEbIjYeKvXPbaSGdC+FoXD\nrwIDAQAB\n-----END PUBLIC KEY-----\n"},"scheme":"rsassa-pss-sha256"},"9b3cea98f6f23cc11813b12d0526a1b6cfb3761008f0882c9caa8db742d63002":{"keytype":"rsa","keyval":{"public":"-----BEGIN PUBLIC KEY-----\nMIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEAsOgQkwLOh31QV9OpbO9v\n6o83durJFGPOnVXZiab83pKaSk7HEK9WzXBq0BaPvtFwSfROVdpgtopri5lZi+uH\naMKLUn5F8XRnSMl/7m5vM4XpZZYa4aQId4TWdbFtTu31eHGZ3eEC5nDRJ5NhZOJd\nKLFBu/xmxrh/eNZt4QbdWLZayjHnzyoy5AnfNTR6nJgPAv+rBOqyqT/r14q4Pngh\n3z0I3pNFr5qmxsp013XV+kgOW1F7zT7IMU8xRIgo85UWUNhax0/bjY/2NI1Z+WjR\nyhZmUBMVYWvfw97xDUrvBvrJxZPgg0lGvxJC6LF2dM7wgLaNx9khT6HMBVxjxLMs\nDQIDAQAB\n-----END PUBLIC KEY-----\n"},"scheme":"rsassa-pss-sha256"},"a61b695e2b86097d993e94e99fd15ec6d8fc8e9522948c9ff21c2f2c881093ae":{"keytype":"rsa","keyval":{"public":"-----BEGIN PUBLIC KEY-----\nMIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEAnayxhw6KeoKK+Ax9RW6v\n66YjrpRpGLewLmSSAzJGX8nL5/a2nEbXbeF9po265KcBSFWol8jLBsmG56ruwwxp\noWWhJPncqGqy8wMeRMmTf7ATGa+tk+To7UAQD0MYzt7rRlIdpqi9Us3J6076Z83k\n2sxFnX9sVflhOsotGWL7hmrn/CJWxKsO6OVCoqbIlnJV8xFazE2eCfaDTIEEEgnh\nLIGDsmv1AN8ImUIn/hyKcm1PfhDZrF5qhEVhfz5D8aX3cUcEJw8BvCaNloXyHf+y\nDKjqO/dJ7YFWVt7nPqOvaEkBQGMd54ETJ/BbO9r3WTsjXKleoPovBSQ/oOxApypb\nNQIDAQAB\n-----END PUBLIC KEY-----\n"},"scheme":"rsassa-pss-sha256"},"d499c8ca3c2018885037da89bdfa327e21a0e6a15e2ca8dabe5e78a3edf9c91c":{"keytype":"rsa","keyval":{"public":"-----BEGIN PUBLIC KEY-----\nMIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEA5oDytiywLDOSpIBovZxx\nNlZJg5Gk3O9kpiOQ0XnD+L2LV+a2dJU1KmBOoGCUr2TNaGTPihAStjpFIsW4c7Ye\nB2RjUFUrXRf3mvc3n4fACayenxtnCleSR4gKkAdHqqPCiWHT5TAtybKSHuHAluUL\nkMvavUZjIPMj0YYB0R8Re7BjU+zxnipJosTbbPQ7fa3+x2VAHc066Y9qp1YucdpB\nMZ3UwtSVNK7aCbFZvKPwAm22fnDYmMbYFeTz/rrl8k+rKTM37d4D3mURC9xDJxIP\nXVaU2dBImYjoFcY0/5oBU5vr1sj2sdUH+3G5AUr6iCL+XJLiwA1x24jKA6mUjQ93\ndwIDAQAB\n-----END PUBLIC KEY-----\n"},"scheme":"rsassa-pss-sha256"}},"threshold":3,"url":"/root.json"},"snapshot":{"keys":{"8660a9f40687fb33e6f8ad563f21ee81b9ce7b91c90827cc7ae2416c5e0e94e9":{"keytype":"rsa","keyval":{"public":"-----BEGIN PUBLIC KEY-----\nMIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEAqTZx29eJR5EumjqM4YTb\nFlKbim1GNYmtbCLH51BbU2lt46ddmfGvtGsxTD3mIZ/GEHVFv6Aei3xx5nIfhGP0\nrG78JRz394uU8Pd62DiIFWYizr5o+ZBZu29D2YK5ZtxoLFpgt0ibnINK2NcesDC8\nSqfIUbMiQFT6yB/MYD275SjfRGHOeYTPmKdjMJrhLL2cfIPYnQ0QFYIyMvXBG1Fj\nU0rc9UclYQHh9YheIDVYI9YCo/DWP3KFfRJpoTjQRGoPSK9TXcpCAEzQpEG3jOek\n9PdV9Ol6/O8JbrFwXWF3LhkUThg+zCjV4qHtP4oqp5QCqzTQTXGQ9qxWUSlHi4Eu\nIwIDAQAB\n-----END PUBLIC KEY-----\n"},"scheme":"rsassa-pss-sha256"}},"threshold":1,"url":"/snapshot.json"},"timestamp":{"keys":{"66d4ea1da00076c822a6e1b4df5eb1e529eb38f6edcedff323e62f2bfe3eaddd":{"keytype":"rsa","keyval":{"public":"-----BEGIN PUBLIC KEY-----\nMIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEAuzTgV5iKhMnunUDxt4PB\npYqTMPaJN/ZdOOsP6cS3DeCE/EcYGfgCjvP7KD3gjG98VDBTVcuwZClSy+/zvHhV\nIq7VWu+yxQL5c6oa1xpCyHoA96JiLIDPhmqEdscdRybcRQ2CYywzKA8jSwEQCnEK\nc8a74ceY352l/MEcOem0+AtKrOjqcjbXCayDwC9yTg/c78bkp+4T8AhSWgt6Tlrt\nY8jLE7zwojFtIYtMwobWRIW2O3nJDXiSBbTPG3M9kF1G43INshSdBcuq5Tmy8lpE\n/XiG/E7+hP63Hm+KAcdvl553Zs7pLhAZxV0kqlApqRRwhscw+JQci8sVONun5t9t\nNwIDAQAB\n-----END PUBLIC KEY-----\n"},"scheme":"rsassa-pss-sha256"}},"threshold":1,"url":"/timestamp.json"}},"spec_version":"0.1.0","version":4}} \ No newline at end of file diff --git a/tests/tiup-playground/test_playground.sh b/tests/tiup-playground/test_playground.sh index 73200c127e..335ba945de 100755 --- a/tests/tiup-playground/test_playground.sh +++ b/tests/tiup-playground/test_playground.sh @@ -62,7 +62,7 @@ sleep 3 trap "kill_all" EXIT # wait start cluster successfully -timeout 300 grep -q "CLUSTER START SUCCESSFULLY" <(tail -f $outfile) +timeout 300 grep -q "TiDB Playground Cluster is started" <(tail -f $outfile) tiup-playground display | grep -qv "exit" tiup-playground scale-out --db 2 @@ -110,7 +110,7 @@ tiup-playground $TIDB_VERSION > $outfile 2>&1 & sleep 3 # wait start cluster successfully -timeout 300 grep -q "CLUSTER START SUCCESSFULLY" <(tail -f $outfile) +timeout 300 grep -q "TiDB Playground Cluster is started" <(tail -f $outfile) cat $outfile | grep ":3930" | grep -q "Done" @@ -120,7 +120,7 @@ outfile_1=/tmp/tiup-playground-test_1.out # no TiFlash to speed up tiup-playground $TIDB_VERSION --tag $TAG --db 2 --tiflash 0 > $outfile_1 2>&1 & sleep 3 -timeout 300 grep -q "CLUSTER START SUCCESSFULLY" <(tail -f $outfile_1) +timeout 300 grep -q "TiDB Playground Cluster is started" <(tail -f $outfile_1) tiup-playground --tag $TAG display | grep -qv "exit" # TiDB scale-out to 4 @@ -144,7 +144,7 @@ sleep 100 echo -e "\033[0;36m<<< Run TiKV-CDC test >>>\033[0m" tiup-playground $TIDB_VERSION --db 1 --pd 1 --kv 1 --tiflash 0 --kvcdc 1 --kvcdc.version v1.0.0 > $outfile 2>&1 & sleep 3 -timeout 300 grep -q "CLUSTER START SUCCESSFULLY" <(tail -f $outfile) +timeout 300 grep -q "TiDB Playground Cluster is started" <(tail -f $outfile) tiup-playground display | grep -qv "exit" # scale out tiup-playground scale-out --kvcdc 2 diff --git a/tests/tiup/test_tiup.sh b/tests/tiup/test_tiup.sh index f04fc7ffdf..9fb3b51270 100755 --- a/tests/tiup/test_tiup.sh +++ b/tests/tiup/test_tiup.sh @@ -27,6 +27,7 @@ tiup list tiup tiup help tiup install tidb:v5.2.2 +tiup install tidb:v3.0.13 tiup update tidb tiup update tidb --nightly tiup --binary tidb:nightly diff --git a/tools/check/go.mod b/tools/check/go.mod index 6786b96213..02c753a2d8 100644 --- a/tools/check/go.mod +++ b/tools/check/go.mod @@ -1,26 +1,20 @@ module github.com/pingcap/tidb/_tools +go 1.21 + require ( - github.com/alecthomas/template v0.0.0-20160405071501-a0175ee3bccc // indirect - github.com/alecthomas/units v0.0.0-20151022065526-2efee857e7cf // indirect - github.com/chzchzchz/goword v0.0.0-20170907005317-a9744cb52b03 - github.com/client9/misspell v0.3.4 - github.com/dnephin/govet v0.0.0-20171012192244-4a96d43e39d3 - github.com/google/shlex v0.0.0-20181106134648-c34317bd91bf // indirect - github.com/gordonklaus/ineffassign v0.0.0-20180909121442-1003c8bd00dc - github.com/kisielk/errcheck v1.2.0 - github.com/mgechev/revive v1.2.4 - github.com/nicksnyder/go-i18n v1.10.0 // indirect - github.com/pelletier/go-toml v1.2.0 // indirect - github.com/securego/gosec v0.0.0-20181211171558-12400f9a1ca7 - golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898 // indirect - gopkg.in/alecthomas/gometalinter.v2 v2.0.12 // indirect - gopkg.in/alecthomas/gometalinter.v3 v3.0.0 // indirect - gopkg.in/alecthomas/kingpin.v2 v2.2.6 // indirect - gopkg.in/alecthomas/kingpin.v3-unstable v3.0.0-20170321130658-9670b87a702e // indirect - gopkg.in/yaml.v2 v2.2.2 // indirect - honnef.co/go/tools v0.0.0-20180920025451-e3ad64cb4ed3 - k8s.io/klog v1.0.0 // indirect + github.com/BurntSushi/toml v1.2.1 // indirect + github.com/chavacava/garif v0.0.0-20230227094218-b8c73b2037b8 // indirect + github.com/fatih/color v1.15.0 // indirect + github.com/fatih/structtag v1.2.0 // indirect + github.com/mattn/go-colorable v0.1.13 // indirect + github.com/mattn/go-isatty v0.0.17 // indirect + github.com/mattn/go-runewidth v0.0.9 // indirect + github.com/mgechev/dots v0.0.0-20210922191527-e955255bf517 // indirect + github.com/mgechev/revive v1.3.2 // indirect + github.com/mitchellh/go-homedir v1.1.0 // indirect + github.com/olekukonko/tablewriter v0.0.5 // indirect + github.com/pkg/errors v0.9.1 // indirect + golang.org/x/sys v0.8.0 // indirect + golang.org/x/tools v0.9.1 // indirect ) - -go 1.18 diff --git a/tools/check/go.sum b/tools/check/go.sum index 259615e4aa..6936e004e9 100644 --- a/tools/check/go.sum +++ b/tools/check/go.sum @@ -1,152 +1,42 @@ -github.com/BurntSushi/toml v0.3.0 h1:e1/Ivsx3Z0FVTV0NSOv/aVgbUWyQuzj7DDnFblkRvsY= -github.com/BurntSushi/toml v0.3.0/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= -github.com/BurntSushi/toml v0.3.1 h1:WXkYYl6Yr3qBf1K79EBnL4mak0OimBfB0XUf9Vl28OQ= -github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= -github.com/BurntSushi/toml v1.2.0 h1:Rt8g24XnyGTyglgET/PRUNlrUeu9F5L+7FilkXfZgs0= -github.com/BurntSushi/toml v1.2.0/go.mod h1:CxXYINrC8qIiEnFrOxCa7Jy5BFHlXnUU2pbicEuybxQ= -github.com/alecthomas/template v0.0.0-20160405071501-a0175ee3bccc/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc= -github.com/alecthomas/units v0.0.0-20151022065526-2efee857e7cf/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0= -github.com/chavacava/garif v0.0.0-20220630083739-93517212f375 h1:E7LT642ysztPWE0dfz43cWOvMiF42DyTRC+eZIaO4yI= -github.com/chavacava/garif v0.0.0-20220630083739-93517212f375/go.mod h1:4m1Rv7xfuwWPNKXlThldNuJvutYM6J95wNuuVmn55To= -github.com/chzchzchz/goword v0.0.0-20170907005317-a9744cb52b03/go.mod h1:uFE9hX+zXEwvyUThZ4gDb9vkAwc5DoHUnRSEpH0VrOs= -github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw= +github.com/BurntSushi/toml v1.2.1 h1:9F2/+DoOYIOksmaJFPw1tGFy1eDnIJXg+UHjuD8lTak= +github.com/BurntSushi/toml v1.2.1/go.mod h1:CxXYINrC8qIiEnFrOxCa7Jy5BFHlXnUU2pbicEuybxQ= +github.com/chavacava/garif v0.0.0-20230227094218-b8c73b2037b8 h1:W9o46d2kbNL06lq7UNDPV0zYLzkrde/bjIqO02eoll0= +github.com/chavacava/garif v0.0.0-20230227094218-b8c73b2037b8/go.mod h1:gakxgyXaaPkxvLw1XQxNGK4I37ys9iBRzNUx/B7pUCo= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= -github.com/dnephin/govet v0.0.0-20171012192244-4a96d43e39d3/go.mod h1:pPTX0MEEoAnfbrAGFj4nSVNhl6YbugRj6eardUZdtGo= -github.com/fatih/color v1.7.0 h1:DkWD4oS2D8LGGgTQ6IvwJJXSL5Vp2ffcQg58nFV38Ys= -github.com/fatih/color v1.7.0/go.mod h1:Zm6kSWBoL9eyXnKyktHP6abPY2pDugNf5KwzbycvMj4= -github.com/fatih/color v1.9.0 h1:8xPHl4/q1VyqGIPif1F+1V3Y3lSmrq01EabUW3CoW5s= -github.com/fatih/color v1.9.0/go.mod h1:eQcE1qtQxscV5RaZvpXrrb8Drkc3/DdQ+uUYCNjL+zU= -github.com/fatih/color v1.13.0 h1:8LOYc1KYPPmyKMuN8QV2DNRWNbLo6LZ0iLs8+mlH53w= -github.com/fatih/color v1.13.0/go.mod h1:kLAiJbzzSOZDVNGyDpeOxJ47H46qBXwg5ILebYFFOfk= -github.com/fatih/structtag v1.0.0 h1:pTHj65+u3RKWYPSGaU290FpI/dXxTaHdVwVwbcPKmEc= -github.com/fatih/structtag v1.0.0/go.mod h1:IKitwq45uXL/yqi5mYghiD3w9H6eTOvI9vnk8tXMphA= +github.com/fatih/color v1.15.0 h1:kOqh6YHBtK8aywxGerMG2Eq3H6Qgoqeo13Bk2Mv/nBs= +github.com/fatih/color v1.15.0/go.mod h1:0h5ZqXfHYED7Bhv2ZJamyIOUej9KtShiJESRwBDUSsw= github.com/fatih/structtag v1.2.0 h1:/OdNE99OxoI/PqaW/SuSK9uxxT3f/tcSZgon/ssNSx4= github.com/fatih/structtag v1.2.0/go.mod h1:mBJUNpUnHmRKrKlQQlmCrh5PuhftFbNv8Ys4/aAZl94= -github.com/go-logr/logr v0.1.0/go.mod h1:ixOQHD9gLJUVQQ2ZOR7zLEifBX6tGkNJF4QyIY7sIas= -github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= -github.com/google/shlex v0.0.0-20181106134648-c34317bd91bf/go.mod h1:RpwtwJQFrIEPstU94h88MWPXP2ektJZ8cZ0YntAmXiE= -github.com/gordonklaus/ineffassign v0.0.0-20180909121442-1003c8bd00dc/go.mod h1:cuNKsD1zp2v6XfE/orVX2QE1LC+i254ceGcVeDT3pTU= -github.com/kisielk/errcheck v1.2.0 h1:reN85Pxc5larApoH1keMBiu2GWtPqXQ1nc9gx+jOU+E= -github.com/kisielk/errcheck v1.2.0/go.mod h1:/BMXB+zMLi60iA8Vv6Ksmxu/1UDYcXs4uQLJ+jE2L00= -github.com/kisielk/gotool v0.0.0-20161130080628-0de1eaf82fa3/go.mod h1:jxZFDH7ILpTPQTk+E2s+z4CUas9lVNjIuKR4c5/zKgM= -github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= -github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= -github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= -github.com/mattn/go-colorable v0.0.9 h1:UVL0vNpWh04HeJXV0KLcaT7r06gOH2l4OW6ddYRUIY4= -github.com/mattn/go-colorable v0.0.9/go.mod h1:9vuHe8Xs5qXnSaW/c/ABM9alt+Vo+STaOChaDxuIBZU= -github.com/mattn/go-colorable v0.1.4 h1:snbPLB8fVfU9iwbbo30TPtbLRzwWu6aJS6Xh4eaaviA= -github.com/mattn/go-colorable v0.1.4/go.mod h1:U0ppj6V5qS13XJ6of8GYAs25YV2eR4EVcfRqFIhoBtE= -github.com/mattn/go-colorable v0.1.9 h1:sqDoxXbdeALODt0DAeJCVp38ps9ZogZEAXjus69YV3U= -github.com/mattn/go-colorable v0.1.9/go.mod h1:u6P/XSegPjTcexA+o6vUJrdnUu04hMope9wVRipJSqc= -github.com/mattn/go-isatty v0.0.4 h1:bnP0vzxcAdeI1zdubAl5PjU6zsERjGZb7raWodagDYs= -github.com/mattn/go-isatty v0.0.4/go.mod h1:M+lRXTBqGeGNdLjl/ufCoiOlB5xdOkqRJdNxMWT7Zi4= -github.com/mattn/go-isatty v0.0.8/go.mod h1:Iq45c/XA43vh69/j3iqttzPXn0bhXyGjM0Hdxcsrc5s= -github.com/mattn/go-isatty v0.0.11 h1:FxPOTFNqGkuDUGi3H/qkUbQO4ZiBa2brKq5r0l8TGeM= -github.com/mattn/go-isatty v0.0.11/go.mod h1:PhnuNfih5lzO57/f3n+odYbM4JtupLOxQOAqxQCu2WE= -github.com/mattn/go-isatty v0.0.12/go.mod h1:cbi8OIDigv2wuxKPP5vlRcQ1OAZbq2CE4Kysco4FUpU= -github.com/mattn/go-isatty v0.0.14 h1:yVuAays6BHfxijgZPzw+3Zlu5yQgKGP2/hcQbHb7S9Y= -github.com/mattn/go-isatty v0.0.14/go.mod h1:7GGIvUiUoEMVVmxf/4nioHXj79iQHKdU27kJ6hsGG94= -github.com/mattn/go-runewidth v0.0.3 h1:a+kO+98RDGEfo6asOGMmpodZq4FNtnGP54yps8BzLR4= -github.com/mattn/go-runewidth v0.0.3/go.mod h1:LwmH8dsx7+W8Uxz3IHJYH5QSwggIsqBzpuz5H//U1FU= -github.com/mattn/go-runewidth v0.0.7 h1:Ei8KR0497xHyKJPAv59M1dkC+rOZCMBJ+t3fZ+twI54= -github.com/mattn/go-runewidth v0.0.7/go.mod h1:H031xJmbD/WCDINGzjvQ9THkh0rPKHF+m2gUSrubnMI= +github.com/mattn/go-colorable v0.1.13 h1:fFA4WZxdEF4tXPZVKMLwD8oUnCTTo08duU7wxecdEvA= +github.com/mattn/go-colorable v0.1.13/go.mod h1:7S9/ev0klgBDR4GtXTXX8a3vIGJpMovkB8vQcUbaXHg= +github.com/mattn/go-isatty v0.0.16/go.mod h1:kYGgaQfpe5nmfYZH+SKPsOc2e4SrIfOl2e/yFXSvRLM= +github.com/mattn/go-isatty v0.0.17 h1:BTarxUcIeDqL27Mc+vyvdWYSL28zpIhv3RoTdsLMPng= +github.com/mattn/go-isatty v0.0.17/go.mod h1:kYGgaQfpe5nmfYZH+SKPsOc2e4SrIfOl2e/yFXSvRLM= github.com/mattn/go-runewidth v0.0.9 h1:Lm995f3rfxdpd6TSmuVCHVb/QhupuXlYr8sCI/QdE+0= github.com/mattn/go-runewidth v0.0.9/go.mod h1:H031xJmbD/WCDINGzjvQ9THkh0rPKHF+m2gUSrubnMI= -github.com/mgechev/dots v0.0.0-20180605013149-8e09d8ea2757 h1:KTwJ7Lo3KDKMknRYN5JEFRGIM4IkG59QjFFM2mxsMEU= -github.com/mgechev/dots v0.0.0-20180605013149-8e09d8ea2757/go.mod h1:KQ7+USdGKfpPjXk4Ga+5XxQM4Lm4e3gAogrreFAYpOg= -github.com/mgechev/dots v0.0.0-20190921121421-c36f7dcfbb81 h1:QASJXOGm2RZ5Ardbc86qNFvby9AqkLDibfChMtAg5QM= -github.com/mgechev/dots v0.0.0-20190921121421-c36f7dcfbb81/go.mod h1:KQ7+USdGKfpPjXk4Ga+5XxQM4Lm4e3gAogrreFAYpOg= github.com/mgechev/dots v0.0.0-20210922191527-e955255bf517 h1:zpIH83+oKzcpryru8ceC6BxnoG8TBrhgAvRg8obzup0= github.com/mgechev/dots v0.0.0-20210922191527-e955255bf517/go.mod h1:KQ7+USdGKfpPjXk4Ga+5XxQM4Lm4e3gAogrreFAYpOg= -github.com/mgechev/revive v0.0.0-20181210140514-b4cc152955fb h1:bLiKpCHe+ngBsF1o7DjZTmoffHEy2gdQ/+9NunuJ4ZY= -github.com/mgechev/revive v0.0.0-20181210140514-b4cc152955fb/go.mod h1:pVHj2KvxEhotJ6Lmr7zb3YgNMX1QKt8cyp6fdPHOrzU= -github.com/mgechev/revive v1.0.2 h1:v0NxxQ7fSFz/u1NQydPo6EGdq7va0J1BtsZmae6kzUg= -github.com/mgechev/revive v1.0.2/go.mod h1:rb0dQy1LVAxW9SWy5R3LPUjevzUbUS316U5MFySA2lo= -github.com/mgechev/revive v1.2.4 h1:+2Hd/S8oO2H0Ikq2+egtNwQsVhAeELHjxjIUFX5ajLI= -github.com/mgechev/revive v1.2.4/go.mod h1:iAWlQishqCuj4yhV24FTnKSXGpbAA+0SckXB8GQMX/Q= +github.com/mgechev/revive v1.3.2 h1:Wb8NQKBaALBJ3xrrj4zpwJwqwNA6nDpyJSEQWcCka6U= +github.com/mgechev/revive v1.3.2/go.mod h1:UCLtc7o5vg5aXCwdUTU1kEBQ1v+YXPAkYDIDXbrs5I0= github.com/mitchellh/go-homedir v1.1.0 h1:lukF9ziXFxDFPkA1vsr5zpc1XuPDn/wFntq5mG+4E0Y= github.com/mitchellh/go-homedir v1.1.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0= -github.com/mozilla/tls-observatory v0.0.0-20180409132520-8791a200eb40/go.mod h1:SrKMQvPiws7F7iqYp8/TX+IhxCYhzr6N/1yb8cwHsGk= -github.com/nbutton23/zxcvbn-go v0.0.0-20160627004424-a22cb81b2ecd/go.mod h1:o96djdrsSGy3AWPyBgZMAGfxZNfgntdJG+11KU4QvbU= -github.com/nicksnyder/go-i18n v1.10.0/go.mod h1:HrK7VCrbOvQoUAQ7Vpy7i87N7JZZZ7R2xBGjv0j365Q= -github.com/olekukonko/tablewriter v0.0.0-20180912035003-be2c049b30cc h1:rQ1O4ZLYR2xXHXgBCCfIIGnuZ0lidMQw2S5n1oOv+Wg= -github.com/olekukonko/tablewriter v0.0.0-20180912035003-be2c049b30cc/go.mod h1:vsDQFd/mU46D+Z4whnwzcISnGGzXWMclvtLoiIKAKIo= -github.com/olekukonko/tablewriter v0.0.4 h1:vHD/YYe1Wolo78koG299f7V/VAS08c6IpCLn+Ejf/w8= -github.com/olekukonko/tablewriter v0.0.4/go.mod h1:zq6QwlOf5SlnkVbMSr5EoBv3636FWnp+qbPhuoO21uA= github.com/olekukonko/tablewriter v0.0.5 h1:P2Ga83D34wi1o9J6Wh1mRuqd4mF/x/lgBS7N7AbDhec= github.com/olekukonko/tablewriter v0.0.5/go.mod h1:hPp6KlRPjbx+hW8ykQs1w3UBbZlj6HuIJcUGPhkA7kY= -github.com/onsi/ginkgo v0.0.0-20170829012221-11459a886d9c/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= -github.com/onsi/gomega v0.0.0-20170829124025-dcabb60a477c/go.mod h1:C1qb7wdrVGGVU+Z6iS04AVkA3Q65CEZX59MT0QO5uiA= -github.com/pelletier/go-toml v1.2.0/go.mod h1:5z9KED0ma1S8pY6P1sdut58dfprrGBbd/94hg7ilaic= -github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= -github.com/ryanuber/go-glob v0.0.0-20170128012129-256dc444b735/go.mod h1:807d1WSdnB0XRJzKNil9Om6lcp/3a0v4qIHxIXzX/Yc= -github.com/securego/gosec v0.0.0-20181211171558-12400f9a1ca7/go.mod h1:m3KbCTwh9vLhm6AKBjE+ALesKilKcQHezI1uVOti0Ks= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw= -github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= +github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo= github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU= -github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY= -golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= -golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= -golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= -golang.org/x/mod v0.1.1-0.20191105210325-c90efee705ee/go.mod h1:QqPTAvyqsEbceGzBzNggFXnrqF1CaUcvgkdR5Ot7KZg= -golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4= -golang.org/x/net v0.0.0-20170915142106-8351a756f30f/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= -golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= -golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= -golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= -golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c= -golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sys v0.0.0-20171026204733-164713f0dfce/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= -golang.org/x/sys v0.0.0-20180909124046-d0be0721c37e/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= -golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= -golang.org/x/sys v0.0.0-20190222072716-a9d3bda3a223/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= -golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20191026070338-33540a1f6037 h1:YyJpGZS1sBuBCzLAR1VEpK193GlqGZbnPFnPV/5Rsb4= -golang.org/x/sys v0.0.0-20191026070338-33540a1f6037/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200116001909-b77594299b42/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200223170610-d5e6a3e2c0ae/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20210630005230-0f9fa26af87c/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f h1:v4INt8xihDGvnrfjMDVXGxw9wrfxYyCjk0KbXjhR55s= -golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= -golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= -golang.org/x/text v0.0.0-20170915090833-1cbadb444a80/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= -golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= -golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= -golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= -golang.org/x/tools v0.0.0-20170915040203-e531a2a1c15f/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= -golang.org/x/tools v0.0.0-20180911133044-677d2ff680c1/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= -golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= -golang.org/x/tools v0.0.0-20181030221726-6c7e314b6563/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= -golang.org/x/tools v0.0.0-20190925020647-22afafe3322a h1:3GxqzBPBt1O2dIiPnzldQ5d25CAMWJFBZTpqxLPfjs8= -golang.org/x/tools v0.0.0-20190925020647-22afafe3322a/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= -golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= -golang.org/x/tools v0.0.0-20200225230052-807dcd883420 h1:4RJNOV+2rLxMEfr6QIpC7GEv9MjD6ApGXTCLrNF9+eA= -golang.org/x/tools v0.0.0-20200225230052-807dcd883420/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= -golang.org/x/tools v0.1.12 h1:VveCTK38A2rkS8ZqFY25HIDFscX5X9OoEhJd3quQmXU= -golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc= -golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= -golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= -gopkg.in/alecthomas/gometalinter.v2 v2.0.12/go.mod h1:NDRytsqEZyolNuAgTzJkZMkSQM7FIKyzVzGhjB/qfYo= -gopkg.in/alecthomas/gometalinter.v3 v3.0.0/go.mod h1:sE0aqUDPY4ibZWdfOxx4ZVG9CD+Y5I1H+Snwv8a3r/s= -gopkg.in/alecthomas/kingpin.v2 v2.2.6/go.mod h1:FMv+mEhP44yOT+4EoQTLFTRgOQ1FBLkstjWtayDeSgw= -gopkg.in/alecthomas/kingpin.v3-unstable v3.0.0-20170321130658-9670b87a702e/go.mod h1:3HH7i1SgMqlzxCcBmUHW657sD4Kvv9sC3HpL3YukzwA= +github.com/stretchr/testify v1.8.2/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4= +golang.org/x/sys v0.0.0-20220811171246-fbc7d0a398ab/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.8.0 h1:EBmGv8NaZBZTWvrbjNoL6HVt+IVy3QDQpJs7VRIw3tU= +golang.org/x/sys v0.8.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/tools v0.9.1 h1:8WMNJAz3zrtPmnYC7ISf5dEn3MT0gY7jBJfw27yrrLo= +golang.org/x/tools v0.9.1/go.mod h1:owI94Op576fPu3cIGQeHs3joujW/2Oc6MtlxbF5dfNc= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= -gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= -gopkg.in/yaml.v2 v2.0.0-20170812160011-eb3733d160e7/go.mod h1:JAlM8MvJe8wmxCU4Bli9HhUf9+ttbYbLASfIpnQbh74= -gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= -honnef.co/go/tools v0.0.0-20180920025451-e3ad64cb4ed3/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= -k8s.io/klog v1.0.0/go.mod h1:4Bi6QPql/J/LkTDqv7R/cd3hPo4k2DG6Ptcz060Ez5I= diff --git a/tools/check/golangci.yaml b/tools/check/golangci.yaml index e7400d95f7..db7cc69929 100644 --- a/tools/check/golangci.yaml +++ b/tools/check/golangci.yaml @@ -5,6 +5,9 @@ linters-settings: errcheck: exclude-functions: - (*mime/multipart.Form).RemoveAll + gocritic: + disabled-checks: + - ifElseChain linters: disable-all: true @@ -16,7 +19,7 @@ linters: - gosimple - govet - ineffassign - - interfacer + # - interfacer - misspell - rowserrcheck - staticcheck diff --git a/tools/check/revive.toml b/tools/check/revive.toml index 1e0cea1e1c..ac23908265 100644 --- a/tools/check/revive.toml +++ b/tools/check/revive.toml @@ -32,7 +32,6 @@ warningCode = 0 [rule.empty-block] [rule.empty-lines] [rule.get-return] -[rule.early-return] [rule.unnecessary-stmt] [rule.struct-tag] [rule.string-of-int]