From 4c708d146d49ced277da05cb8ee2f5ff7c306b7c Mon Sep 17 00:00:00 2001 From: Tong Cai Date: Mon, 10 Dec 2018 00:01:28 +0800 Subject: [PATCH 1/3] Add kubelet-certificate-authority flag This allow user to specify separate CA for kubelet --- cmd/metrics-server/app/start.go | 4 +++- pkg/sources/summary/configs.go | 15 ++++++++++++++- 2 files changed, 17 insertions(+), 2 deletions(-) diff --git a/cmd/metrics-server/app/start.go b/cmd/metrics-server/app/start.go index 192f2318c..a6ee530ed 100644 --- a/cmd/metrics-server/app/start.go +++ b/cmd/metrics-server/app/start.go @@ -61,6 +61,7 @@ func NewCommandStartMetricsServer(out, errOut io.Writer, stopCh <-chan struct{}) flags.IntVar(&o.KubeletPort, "kubelet-port", o.KubeletPort, "The port to use to connect to Kubelets.") flags.StringVar(&o.Kubeconfig, "kubeconfig", o.Kubeconfig, "The path to the kubeconfig used to connect to the Kubernetes API server and the Kubelets (defaults to in-cluster config)") flags.StringSliceVar(&o.KubeletPreferredAddressTypes, "kubelet-preferred-address-types", o.KubeletPreferredAddressTypes, "The priority of node address types to use when determining which address to use to connect to a particular node") + flags.StringVar(&o.KubeltClientConfigOverrides.CAFile, "kubelet-certificate-authority", "", "Path to a client cert file for TLS.") flags.MarkDeprecated("deprecated-kubelet-completely-insecure", "This is rarely the right option, since it leaves kubelet communication completely insecure. If you encounter auth errors, make sure you've enabled token webhook auth on the Kubelet, and if you're in a test cluster with self-signed Kubelet certificates, consider using kubelet-insecure-tls instead.") @@ -89,6 +90,7 @@ type MetricsServerOptions struct { KubeletPort int InsecureKubeletTLS bool KubeletPreferredAddressTypes []string + KubeltClientConfigOverrides summary.KubeletClientConfigOverrides DeprecatedCompletelyInsecureKubelet bool } @@ -171,7 +173,7 @@ func (o MetricsServerOptions) Run(stopCh <-chan struct{}) error { informerFactory := informers.NewSharedInformerFactory(kubeClient, 0) // set up the source manager - kubeletConfig := summary.GetKubeletConfig(clientConfig, o.KubeletPort, o.InsecureKubeletTLS, o.DeprecatedCompletelyInsecureKubelet) + kubeletConfig := summary.GetKubeletConfig(clientConfig, o.KubeltClientConfigOverrides, o.KubeletPort, o.InsecureKubeletTLS, o.DeprecatedCompletelyInsecureKubelet) kubeletClient, err := summary.KubeletClientFor(kubeletConfig) if err != nil { return fmt.Errorf("unable to construct a client to connect to the kubelets: %v", err) diff --git a/pkg/sources/summary/configs.go b/pkg/sources/summary/configs.go index e5da658f3..5190643f0 100644 --- a/pkg/sources/summary/configs.go +++ b/pkg/sources/summary/configs.go @@ -21,7 +21,8 @@ import ( ) // GetKubeletConfig fetches connection config for connecting to the Kubelet. -func GetKubeletConfig(baseKubeConfig *rest.Config, port int, insecureTLS bool, completelyInsecure bool) *KubeletClientConfig { +func GetKubeletConfig(baseKubeConfig *rest.Config, overrides KubeletClientConfigOverrides, + port int, insecureTLS bool, completelyInsecure bool) *KubeletClientConfig { cfg := rest.CopyConfig(baseKubeConfig) if completelyInsecure { cfg = rest.AnonymousClientConfig(cfg) // don't use auth to avoid leaking auth details to insecure endpoints @@ -30,6 +31,11 @@ func GetKubeletConfig(baseKubeConfig *rest.Config, port int, insecureTLS bool, c cfg.TLSClientConfig.Insecure = true cfg.TLSClientConfig.CAData = nil cfg.TLSClientConfig.CAFile = "" + } else { + if len(overrides.CAFile) > 0 { + cfg.TLSClientConfig.CAFile = overrides.CAFile + cfg.TLSClientConfig.CAData = nil + } } kubeletConfig := &KubeletClientConfig{ Port: port, @@ -40,6 +46,13 @@ func GetKubeletConfig(baseKubeConfig *rest.Config, port int, insecureTLS bool, c return kubeletConfig } +// KubeletClientConfigOverrides overrides some tls and auth configuration for connecting to Kubelets. +type KubeletClientConfigOverrides struct { + // Trusted root certificates for server + CAFile string + // TODO: also add KeyFile, CertFile, BearerToken +} + // KubeletClientConfig represents configuration for connecting to Kubelets. type KubeletClientConfig struct { Port int From 98c39626e2d06e9fc323cf91b0061302b11f400d Mon Sep 17 00:00:00 2001 From: Tong Cai Date: Wed, 26 Dec 2018 00:09:00 +0800 Subject: [PATCH 2/3] Fix comments 1. Better flag description 2. Copy clientConfig and override it before invoke GetKubeletConfig function --- cmd/metrics-server/app/start.go | 9 +++++++-- pkg/sources/summary/configs.go | 9 +-------- 2 files changed, 8 insertions(+), 10 deletions(-) diff --git a/cmd/metrics-server/app/start.go b/cmd/metrics-server/app/start.go index a6ee530ed..3795900c4 100644 --- a/cmd/metrics-server/app/start.go +++ b/cmd/metrics-server/app/start.go @@ -61,7 +61,7 @@ func NewCommandStartMetricsServer(out, errOut io.Writer, stopCh <-chan struct{}) flags.IntVar(&o.KubeletPort, "kubelet-port", o.KubeletPort, "The port to use to connect to Kubelets.") flags.StringVar(&o.Kubeconfig, "kubeconfig", o.Kubeconfig, "The path to the kubeconfig used to connect to the Kubernetes API server and the Kubelets (defaults to in-cluster config)") flags.StringSliceVar(&o.KubeletPreferredAddressTypes, "kubelet-preferred-address-types", o.KubeletPreferredAddressTypes, "The priority of node address types to use when determining which address to use to connect to a particular node") - flags.StringVar(&o.KubeltClientConfigOverrides.CAFile, "kubelet-certificate-authority", "", "Path to a client cert file for TLS.") + flags.StringVar(&o.KubeltClientConfigOverrides.CAFile, "kubelet-certificate-authority", "", "Path to the CA to use to validate the Kubelet's serving certificates.") flags.MarkDeprecated("deprecated-kubelet-completely-insecure", "This is rarely the right option, since it leaves kubelet communication completely insecure. If you encounter auth errors, make sure you've enabled token webhook auth on the Kubelet, and if you're in a test cluster with self-signed Kubelet certificates, consider using kubelet-insecure-tls instead.") @@ -173,7 +173,12 @@ func (o MetricsServerOptions) Run(stopCh <-chan struct{}) error { informerFactory := informers.NewSharedInformerFactory(kubeClient, 0) // set up the source manager - kubeletConfig := summary.GetKubeletConfig(clientConfig, o.KubeltClientConfigOverrides, o.KubeletPort, o.InsecureKubeletTLS, o.DeprecatedCompletelyInsecureKubelet) + kubeletRestCfg := rest.CopyConfig(clientConfig) + if len(o.KubeltClientConfigOverrides.CAFile) > 0 { + kubeletRestCfg.TLSClientConfig.CAFile = o.KubeltClientConfigOverrides.CAFile + kubeletRestCfg.TLSClientConfig.CAData = nil + } + kubeletConfig := summary.GetKubeletConfig(kubeletRestCfg, o.KubeletPort, o.InsecureKubeletTLS, o.DeprecatedCompletelyInsecureKubelet) kubeletClient, err := summary.KubeletClientFor(kubeletConfig) if err != nil { return fmt.Errorf("unable to construct a client to connect to the kubelets: %v", err) diff --git a/pkg/sources/summary/configs.go b/pkg/sources/summary/configs.go index 5190643f0..d1275351f 100644 --- a/pkg/sources/summary/configs.go +++ b/pkg/sources/summary/configs.go @@ -21,9 +21,7 @@ import ( ) // GetKubeletConfig fetches connection config for connecting to the Kubelet. -func GetKubeletConfig(baseKubeConfig *rest.Config, overrides KubeletClientConfigOverrides, - port int, insecureTLS bool, completelyInsecure bool) *KubeletClientConfig { - cfg := rest.CopyConfig(baseKubeConfig) +func GetKubeletConfig(cfg *rest.Config, port int, insecureTLS bool, completelyInsecure bool) *KubeletClientConfig { if completelyInsecure { cfg = rest.AnonymousClientConfig(cfg) // don't use auth to avoid leaking auth details to insecure endpoints cfg.TLSClientConfig = rest.TLSClientConfig{} // empty TLS config --> no TLS @@ -31,11 +29,6 @@ func GetKubeletConfig(baseKubeConfig *rest.Config, overrides KubeletClientConfig cfg.TLSClientConfig.Insecure = true cfg.TLSClientConfig.CAData = nil cfg.TLSClientConfig.CAFile = "" - } else { - if len(overrides.CAFile) > 0 { - cfg.TLSClientConfig.CAFile = overrides.CAFile - cfg.TLSClientConfig.CAData = nil - } } kubeletConfig := &KubeletClientConfig{ Port: port, From f724d6a025f853ecda606a040c9f060a7b9388f5 Mon Sep 17 00:00:00 2001 From: Tong Cai Date: Sat, 2 Feb 2019 19:28:50 +0800 Subject: [PATCH 3/3] Fix nits --- cmd/metrics-server/app/start.go | 8 ++++---- pkg/sources/summary/configs.go | 7 ------- 2 files changed, 4 insertions(+), 11 deletions(-) diff --git a/cmd/metrics-server/app/start.go b/cmd/metrics-server/app/start.go index 3795900c4..94f03a21c 100644 --- a/cmd/metrics-server/app/start.go +++ b/cmd/metrics-server/app/start.go @@ -61,7 +61,7 @@ func NewCommandStartMetricsServer(out, errOut io.Writer, stopCh <-chan struct{}) flags.IntVar(&o.KubeletPort, "kubelet-port", o.KubeletPort, "The port to use to connect to Kubelets.") flags.StringVar(&o.Kubeconfig, "kubeconfig", o.Kubeconfig, "The path to the kubeconfig used to connect to the Kubernetes API server and the Kubelets (defaults to in-cluster config)") flags.StringSliceVar(&o.KubeletPreferredAddressTypes, "kubelet-preferred-address-types", o.KubeletPreferredAddressTypes, "The priority of node address types to use when determining which address to use to connect to a particular node") - flags.StringVar(&o.KubeltClientConfigOverrides.CAFile, "kubelet-certificate-authority", "", "Path to the CA to use to validate the Kubelet's serving certificates.") + flags.StringVar(&o.KubeletCAFile, "kubelet-certificate-authority", "", "Path to the CA to use to validate the Kubelet's serving certificates.") flags.MarkDeprecated("deprecated-kubelet-completely-insecure", "This is rarely the right option, since it leaves kubelet communication completely insecure. If you encounter auth errors, make sure you've enabled token webhook auth on the Kubelet, and if you're in a test cluster with self-signed Kubelet certificates, consider using kubelet-insecure-tls instead.") @@ -90,7 +90,7 @@ type MetricsServerOptions struct { KubeletPort int InsecureKubeletTLS bool KubeletPreferredAddressTypes []string - KubeltClientConfigOverrides summary.KubeletClientConfigOverrides + KubeletCAFile string DeprecatedCompletelyInsecureKubelet bool } @@ -174,8 +174,8 @@ func (o MetricsServerOptions) Run(stopCh <-chan struct{}) error { // set up the source manager kubeletRestCfg := rest.CopyConfig(clientConfig) - if len(o.KubeltClientConfigOverrides.CAFile) > 0 { - kubeletRestCfg.TLSClientConfig.CAFile = o.KubeltClientConfigOverrides.CAFile + if len(o.KubeletCAFile) > 0 { + kubeletRestCfg.TLSClientConfig.CAFile = o.KubeletCAFile kubeletRestCfg.TLSClientConfig.CAData = nil } kubeletConfig := summary.GetKubeletConfig(kubeletRestCfg, o.KubeletPort, o.InsecureKubeletTLS, o.DeprecatedCompletelyInsecureKubelet) diff --git a/pkg/sources/summary/configs.go b/pkg/sources/summary/configs.go index d1275351f..7628042cc 100644 --- a/pkg/sources/summary/configs.go +++ b/pkg/sources/summary/configs.go @@ -39,13 +39,6 @@ func GetKubeletConfig(cfg *rest.Config, port int, insecureTLS bool, completelyIn return kubeletConfig } -// KubeletClientConfigOverrides overrides some tls and auth configuration for connecting to Kubelets. -type KubeletClientConfigOverrides struct { - // Trusted root certificates for server - CAFile string - // TODO: also add KeyFile, CertFile, BearerToken -} - // KubeletClientConfig represents configuration for connecting to Kubelets. type KubeletClientConfig struct { Port int