Minimize throttling for happy path of probes.

1. Load the current replicas via the local cache. 2. Add jitter to the probe intervals to spread out host apiserver calls. 3. Make client-go QPS and Burst configurable via CLI flags. 4. Metrics for probe sub-command.
gardener · Jun 23, 2020 · 55d0dff · 55d0dff
1 parent f518a81
commit 55d0dff
Show file tree

Hide file tree

Showing 107 changed files with 21,429 additions and 43 deletions.
diff --git a/cmd/probe.go b/cmd/probe.go
@@ -57,6 +57,9 @@ func runProbe(cmd *cobra.Command, args []string) {
 	klog.V(2).Infoln("master: ", deployedNamespace)
 	klog.V(2).Infoln("deployed-namespace: ", masterURL)
 	klog.V(2).Infoln("concurrent-syncs: ", concurrentSyncs)
+	klog.V(2).Infoln("qps: ", qps)
+	klog.V(2).Infoln("burst: ", burst)
+	klog.V(2).Infoln("port: ", port)
 
 	// set up signals so we handle the first shutdown signal gracefully
 	stopCh := setupSignalHandler()
@@ -73,6 +76,9 @@ func runProbe(cmd *cobra.Command, args []string) {
 		klog.Fatalf("Error parsing kubeconfig file: %s", err.Error())
 	}
 
+	config.QPS = qps
+	config.Burst = burst
+
 	clientset, err := kubernetes.NewForConfig(config)
 	if err != nil {
 		klog.Fatalf("Error creating k8s clientset: %s", err.Error())
@@ -94,7 +100,7 @@ func runProbe(cmd *cobra.Command, args []string) {
 	leaderElectionClient := kubernetes.NewForConfigOrDie(rest.AddUserAgent(config, "dependency-watchdog-election"))
 	recorder := createRecorder(leaderElectionClient)
 	run := func(ctx context.Context) {
-
+		go serveMetrics()
 		klog.Info("Starting endpoint controller.")
 		if err = controller.Run(concurrentSyncs); err != nil {
 			klog.Fatalf("Error running controller: %s", err.Error())

diff --git a/cmd/root.go b/cmd/root.go
@@ -20,12 +20,14 @@ import (
 	"context"
 	"flag"
 	"fmt"
+	"net/http"
 	"os"
 	"os/signal"
 	"syscall"
 	"time"
 
 	"github.com/gardener/dependency-watchdog/pkg/restarter"
+	"github.com/prometheus/client_golang/prometheus/promhttp"
 	"github.com/spf13/cobra"
 	"github.com/spf13/pflag"
 	v1 "k8s.io/api/core/v1"
@@ -44,6 +46,7 @@ import (
 const (
 	defaultWatchDuration   = "2m"
 	defaultConcurrentSyncs = 1
+	defaultPort            = 9643
 )
 
 var (
@@ -55,6 +58,9 @@ var (
 	dependencyWatchdogAgentName = "dependency-watchdog"
 	defaultSyncDuration         = 30 * time.Second
 	concurrentSyncs             = defaultConcurrentSyncs
+	qps                         float32
+	burst                       int
+	port                        int
 
 	onlyOneSignalHandler = make(chan struct{})
 	shutdownSignals      = []os.Signal{os.Interrupt, syscall.SIGTERM}
@@ -97,6 +103,9 @@ func init() {
 	rootCmd.PersistentFlags().StringVar(&deployedNamespace, "deployed-namespace", "default", "namespace into which the dependency-watchdog is deployed")
 	rootCmd.PersistentFlags().StringVar(&masterURL, "master", "", "The address of the Kubernetes API server. Overrides any value in kubeconfig. Only required if out-of-cluster.")
 	rootCmd.PersistentFlags().IntVar(&concurrentSyncs, "concurrent-syncs", defaultConcurrentSyncs, "The number of workers performing reconcilation concurrently.")
+	rootCmd.PersistentFlags().Float32Var(&qps, "qps", rest.DefaultQPS, "Throttling QPS configuration for the client to host apiserver.")
+	rootCmd.PersistentFlags().IntVar(&burst, "burst", rest.DefaultBurst, "Throttling burst configuration for the client to host apiserver.")
+	rootCmd.PersistentFlags().IntVar(&port, "port", defaultPort, "The port on which health and prometheus metrics are exposed.")
 	rootCmd.Flags().StringVar(&strWatchDuration, "watch-duration", defaultWatchDuration, "The duration to watch dependencies after the service is ready.")
 
 	klog.InitFlags(nil)
@@ -113,6 +122,9 @@ func runRoot(cmd *cobra.Command, args []string) {
 	klog.V(2).Infoln("deployed-namespace: ", masterURL)
 	klog.V(2).Infoln("concurrent-syncs: ", concurrentSyncs)
 	klog.V(2).Infoln("watch-duration: ", strWatchDuration)
+	klog.V(2).Infoln("qps: ", qps)
+	klog.V(2).Infoln("burst: ", burst)
+	klog.V(2).Infoln("port: ", port)
 
 	watchDuration, err := time.ParseDuration(strWatchDuration)
 	if err != nil {
@@ -134,6 +146,9 @@ func runRoot(cmd *cobra.Command, args []string) {
 		klog.Fatalf("Error parsing kubeconfig file: %s", err.Error())
 	}
 
+	config.QPS = qps
+	config.Burst = burst
+
 	clientset, err := kubernetes.NewForConfig(config)
 	if err != nil {
 		klog.Fatalf("Error creating k8s clientset: %s", err.Error())
@@ -151,7 +166,7 @@ func runRoot(cmd *cobra.Command, args []string) {
 	leaderElectionClient := kubernetes.NewForConfigOrDie(rest.AddUserAgent(config, "dependency-watchdog-election"))
 	recorder := createRecorder(leaderElectionClient)
 	run := func(ctx context.Context) {
-
+		go serveMetrics()
 		klog.Info("Starting endpoint controller.")
 		if err = controller.Run(concurrentSyncs); err != nil {
 			klog.Fatalf("Error running controller: %s", err.Error())
@@ -221,3 +236,8 @@ func setupSignalHandler() (stopCh <-chan struct{}) {
 
 	return stop
 }
+
+func serveMetrics() error {
+	http.Handle("/metrics", promhttp.Handler())
+	return http.ListenAndServe(fmt.Sprintf("%s%d", ":", port), nil)
+}
diff --git a/go.mod b/go.mod
@@ -9,6 +9,7 @@ require (
 	github.com/imdario/mergo v0.3.8 // indirect
 	github.com/onsi/ginkgo v1.12.2
 	github.com/onsi/gomega v1.10.1
+	github.com/prometheus/client_golang v1.0.0
 	github.com/spf13/cobra v0.0.5
 	github.com/spf13/pflag v1.0.5
 	golang.org/x/sys v0.0.0-20200523222454-059865788121 // indirect

diff --git a/go.sum b/go.sum
@@ -26,6 +26,7 @@ github.com/armon/consul-api v0.0.0-20180202201655-eb2c6b5be1b6/go.mod h1:grANhF5
 github.com/asaskevich/govalidator v0.0.0-20180720115003-f9ffefc3facf/go.mod h1:lB+ZfQJz7igIIfQNfa7Ml4HSf2uFQQRzpGGRXenZAgY=
 github.com/asaskevich/govalidator v0.0.0-20190424111038-f61b66f89f4a/go.mod h1:lB+ZfQJz7igIIfQNfa7Ml4HSf2uFQQRzpGGRXenZAgY=
 github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973/go.mod h1:Dwedo/Wpr24TaqPxmxbtue+5NUziq4I4S80YR8gNf3Q=
+github.com/beorn7/perks v1.0.0 h1:HWo1m869IqiPhD389kmkxeTalrjNbbJTC8LXupb+sl0=
 github.com/beorn7/perks v1.0.0/go.mod h1:KWe93zE9D1o94FZ5RNwFwVgaQK1VOXiVxmqh+CedLV8=
 github.com/bgentry/speakeasy v0.1.0/go.mod h1:+zsyZBPWlz7T6j88CTgSN5bM796AkVf0kBD4zp0CCIs=
 github.com/blang/semver v3.5.0+incompatible/go.mod h1:kRBLl5iJ+tD4TcOOxsy/0fnwebNt5EWlYSAyrTnjyyk=
@@ -218,6 +219,7 @@ github.com/mailru/easyjson v0.7.0/go.mod h1:KAzv3t3aY1NaHWoQz1+4F1ccyAH66Jk7yos7
 github.com/mattn/go-colorable v0.0.9/go.mod h1:9vuHe8Xs5qXnSaW/c/ABM9alt+Vo+STaOChaDxuIBZU=
 github.com/mattn/go-isatty v0.0.4/go.mod h1:M+lRXTBqGeGNdLjl/ufCoiOlB5xdOkqRJdNxMWT7Zi4=
 github.com/mattn/go-runewidth v0.0.2/go.mod h1:LwmH8dsx7+W8Uxz3IHJYH5QSwggIsqBzpuz5H//U1FU=
+github.com/matttproud/golang_protobuf_extensions v1.0.1 h1:4hp9jkHxhMHkqkrB3Ix0jegS5sx/RkqARlsWZ6pIwiU=
 github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5Ld7szi9bcBfOoFv/3dc6xSMkL2PC0=
 github.com/mitchellh/go-homedir v1.1.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0=
 github.com/mitchellh/mapstructure v1.1.2/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh9fWfEaFds41c1Y=
@@ -260,13 +262,17 @@ github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZb
 github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
 github.com/pquerna/cachecontrol v0.0.0-20171018203845-0dec1b30a021/go.mod h1:prYjPmNq4d1NPVmpShWobRqXY3q7Vp+80DqgxxUrUIA=
 github.com/prometheus/client_golang v0.9.1/go.mod h1:7SWBe2y4D6OKWSNQJUaRYU/AaXPKyh/dDVn+NZz0KFw=
+github.com/prometheus/client_golang v1.0.0 h1:vrDKnkGzuGvhNAL56c7DBz29ZL+KxnoR0x7enabFceM=
 github.com/prometheus/client_golang v1.0.0/go.mod h1:db9x61etRT2tGnBNRi70OPL5FsnadC4Ky3P0J6CfImo=
 github.com/prometheus/client_model v0.0.0-20180712105110-5c3871d89910/go.mod h1:MbSGuTsp3dbXC40dX6PRTWyKYBIrTGTE9sqQNg2J8bo=
 github.com/prometheus/client_model v0.0.0-20190129233127-fd36f4220a90/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA=
 github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA=
+github.com/prometheus/client_model v0.2.0 h1:uq5h0d+GuxiXLJLNABMgp2qUWDPiLvgCzz2dUR+/W/M=
 github.com/prometheus/client_model v0.2.0/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA=
+github.com/prometheus/common v0.4.1 h1:K0MGApIoQvMw27RTdJkPbr3JZ7DNbtxQNyi5STVM6Kw=
 github.com/prometheus/common v0.4.1/go.mod h1:TNfzLD0ON7rHzMJeJkieUDPYmFC7Snx/y86RQel1bk4=
 github.com/prometheus/procfs v0.0.0-20181005140218-185b4288413d/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk=
+github.com/prometheus/procfs v0.0.2 h1:6LJUbpNm42llc4HRCuvApCSWB/WfhuNo9K98Q9sNGfs=
 github.com/prometheus/procfs v0.0.2/go.mod h1:TjEm7ze935MbeOT/UhFTIMYKhuLP4wbCsTZCD3I8kEA=
 github.com/rogpeppe/fastuuid v0.0.0-20150106093220-6724a57986af/go.mod h1:XWv6SoW27p1b0cqNHllgS5HIMJraePCO15w5zCzIWYg=
 github.com/russross/blackfriday v1.5.2/go.mod h1:JO/DiYxRf+HjHt06OyowR9PTA263kcR/rfWxYHBV53g=