Data race when getting cluster info

[rleungx]

Flaky Test

Which jobs are failing

==================
WARNING: DATA RACE
Write at 0x00c002517228 by goroutine 1477:
  github.com/tikv/pd/server.(*Server).startServer()
      /home/runner/work/pd/pd/server/server.go:424 +0x114
  github.com/tikv/pd/server.(*Server).Run()
      /home/runner/work/pd/pd/server/server.go:582 +0x1c7
  github.com/tikv/pd/tests.(*TestServer).Run()
      /home/runner/work/pd/pd/tests/cluster.go:125 +0x1e4
  github.com/tikv/pd/tests.(*TestCluster).RunServer.func1()
      /home/runner/work/pd/pd/tests/cluster.go:568 +0x44

Previous read at 0x00c002517228 by goroutine 1615:
  github.com/tikv/pd/server.(*GrpcServer).errorHeader()
      /home/runner/work/pd/pd/server/grpc_service.go:2025 +0x311
  github.com/tikv/pd/server.(*GrpcServer).wrapErrorToHeader()
      /home/runner/work/pd/pd/server/grpc_service.go:2017 +0x199
  github.com/tikv/pd/server.(*GrpcServer).GetClusterInfo()
      /home/runner/work/pd/pd/server/grpc_service.go:135 +0x11f
  github.com/pingcap/kvproto/pkg/pdpb._PD_GetClusterInfo_Handler.func1()
      /home/runner/go/pkg/mod/github.com/pingcap/[email protected]/pkg/pdpb/pdpb.pb.go:9552 +0x82
  go.etcd.io/etcd/etcdserver/api/v3rpc.Server.ChainUnaryServer.func5.1()
      /home/runner/go/pkg/mod/github.com/grpc-ecosystem/[email protected]/chain.go:31 +0x223
  github.com/grpc-ecosystem/go-grpc-prometheus.init.(*ServerMetrics).UnaryServerInterceptor.func3()
      /home/runner/go/pkg/mod/github.com/grpc-ecosystem/[email protected]/server_metrics.go:107 +0xbb
  go.etcd.io/etcd/etcdserver/api/v3rpc.Server.ChainUnaryServer.func5.1()
      /home/runner/go/pkg/mod/github.com/grpc-ecosystem/[email protected]/chain.go:34 +0x182
  go.etcd.io/etcd/etcdserver/api/v3rpc.Server.newUnaryInterceptor.func4()
      /home/runner/go/pkg/mod/go.etcd.io/[email protected]/etcdserver/api/v3rpc/interceptor.go:70 +0x2f1
  go.etcd.io/etcd/etcdserver/api/v3rpc.Server.ChainUnaryServer.func5.1()
      /home/runner/go/pkg/mod/github.com/grpc-ecosystem/[email protected]/chain.go:34 +0x182
  go.etcd.io/etcd/etcdserver/api/v3rpc.Server.newLogUnaryInterceptor.func3()
      /home/runner/go/pkg/mod/go.etcd.io/[email protected]/etcdserver/api/v3rpc/interceptor.go:77 +0xba
  go.etcd.io/etcd/etcdserver/api/v3rpc.Server.ChainUnaryServer.func5()
      /home/runner/go/pkg/mod/github.com/grpc-ecosystem/[email protected]/chain.go:39 +0x32e
  github.com/pingcap/kvproto/pkg/pdpb._PD_GetClusterInfo_Handler()
      /home/runner/go/pkg/mod/github.com/pingcap/[email protected]/pkg/pdpb/pdpb.pb.go:9554 +0x1c3
  google.golang.org/grpc.(*Server).processUnaryRPC()
      /home/runner/go/pkg/mod/google.golang.org/[email protected]/server.go:1024 +0x1494
  google.golang.org/grpc.(*Server).handleStream()
      /home/runner/go/pkg/mod/google.golang.org/[email protected]/server.go:1313 +0xf72
  google.golang.org/grpc.(*Server).serveStreams.func1.1()
      /home/runner/go/pkg/mod/google.golang.org/[email protected]/server.go:722 +0xe6

Goroutine 1477 (running) created at:
  github.com/tikv/pd/tests.(*TestCluster).RunServer()
      /home/runner/work/pd/pd/tests/cluster.go:568 +0x124
  github.com/tikv/pd/tests.(*TestCluster).RunServers()
      /home/runner/work/pd/pd/tests/cluster.go:576 +0x105
  github.com/tikv/pd/tests.(*TestCluster).RunInitialServers()
      /home/runner/work/pd/pd/tests/cluster.go:592 +0x339
  github.com/tikv/pd/tests/integrations/mcs/tso.TestTSOServerStartFirst()
      /home/runner/work/pd/pd/tests/integrations/mcs/tso/api_test.go:164 +0x5f1
  github.com/pingcap/failpoint.parseTerm()
      /home/runner/go/pkg/mod/github.com/pingcap/[email protected]/terms.go:149 +0x364
  github.com/pingcap/failpoint.parse()
      /home/runner/go/pkg/mod/github.com/pingcap/[email protected]/terms.go:126 +0xa5
  github.com/pingcap/failpoint.newTerms()
      /home/runner/go/pkg/mod/github.com/pingcap/[email protected]/terms.go:98 +0x3e
  github.com/pingcap/failpoint.(*Failpoint).Enable()
      /home/runner/go/pkg/mod/github.com/pingcap/[email protected]/failpoint.go:54 +0x3e
  github.com/pingcap/failpoint.(*Failpoints).Enable()
      /home/runner/go/pkg/mod/github.com/pingcap/[email protected]/failpoints.go:105 +0x276
  github.com/pingcap/failpoint.Enable()
      /home/runner/go/pkg/mod/github.com/pingcap/[email protected]/failpoints.go:225 +0xb0
  github.com/tikv/pd/tests/integrations/mcs/tso.TestTSOServerStartFirst()
      /home/runner/work/pd/pd/tests/integrations/mcs/tso/api_test.go:142 +0xb1
  testing.tRunner()
      /opt/hostedtoolcache/go/1.21.1/x64/src/testing/testing.go:1595 +0x238
  testing.(*T).Run.func1()
      /opt/hostedtoolcache/go/1.21.1/x64/src/testing/testing.go:1648 +0x44

Goroutine 1615 (running) created at:
  google.golang.org/grpc.(*Server).serveStreams.func1()
      /home/runner/go/pkg/mod/google.golang.org/[email protected]/server.go:720 +0x185
  google.golang.org/grpc/internal/transport.(*http2Server).operateHeaders()
      /home/runner/go/pkg/mod/google.golang.org/[email protected]/internal/transport/http2_server.go:447 +0x25a1
  google.golang.org/grpc/internal/transport.(*http2Server).HandleStreams()
      /home/runner/go/pkg/mod/google.golang.org/[email protected]/internal/transport/http2_server.go:488 +0x259
  google.golang.org/grpc.(*Server).serveStreams()
      /home/runner/go/pkg/mod/google.golang.org/[email protected]/server.go:718 +0x1d6
  google.golang.org/grpc.(*Server).handleRawConn.func1()
      /home/runner/go/pkg/mod/google.golang.org/[email protected]/server.go:679 +0x46
==================

CI link

https://github.com/tikv/pd/actions/runs/6446858210/job/17502509920

Reason for failure (if possible)

Anything else

[rleungx]

The issue will be tracked through #7772. Close it.