From 6b2a19f1190d7820caa0ffc301dc865f7d4d59e2 Mon Sep 17 00:00:00 2001 From: Paul Laffitte Date: Fri, 7 Jun 2024 23:45:49 +0200 Subject: [PATCH 1/7] feat(metrics): add status in cached_images metric --- internal/controller/collector.go | 11 ++++------- 1 file changed, 4 insertions(+), 7 deletions(-) diff --git a/internal/controller/collector.go b/internal/controller/collector.go index f3b971a8..6d080eea 100644 --- a/internal/controller/collector.go +++ b/internal/controller/collector.go @@ -53,7 +53,7 @@ var ( cachedImagesMetric = prometheus.BuildFQName(kuikMetrics.Namespace, subsystem, "cached_images") cachedImagesHelp = "Number of images expected to be cached" - cachedImagesDesc = prometheus.NewDesc(cachedImagesMetric, cachedImagesHelp, []string{"cached", "expiring"}, nil) + cachedImagesDesc = prometheus.NewDesc(cachedImagesMetric, cachedImagesHelp, []string{"status", "cached", "expiring"}, nil) ) func RegisterMetrics(client client.Client) { @@ -70,10 +70,6 @@ func RegisterMetrics(client client.Client) { ) } -func cachedImagesWithLabelValues(gaugeVec *prometheus.GaugeVec, cachedImage *kuikv1alpha1.CachedImage) prometheus.Gauge { - return gaugeVec.WithLabelValues(strconv.FormatBool(cachedImage.Status.IsCached), strconv.FormatBool(cachedImage.Spec.ExpiresAt != nil)) -} - type ControllerCollector struct { client.Client } @@ -90,10 +86,11 @@ func (c *ControllerCollector) Collect(ch chan<- prometheus.Metric) { Name: cachedImagesMetric, Help: cachedImagesHelp, }, - []string{"cached", "expiring"}, + []string{"status", "cached", "expiring"}, ) for _, cachedImage := range cachedImageList.Items { - cachedImagesWithLabelValues(cachedImageGaugeVec, &cachedImage).Inc() + cachedImageGauge := cachedImageGaugeVec.WithLabelValues(cachedImage.Status.Phase, strconv.FormatBool(cachedImage.Status.IsCached), strconv.FormatBool(cachedImage.Spec.ExpiresAt != nil)) + cachedImageGauge.Inc() } cachedImageGaugeVec.Collect(ch) } else { From a1e6a0e2eae5a347e7f8d2dc6eacb0470d75c533 Mon Sep 17 00:00:00 2001 From: Paul Laffitte Date: Fri, 7 Jun 2024 23:47:19 +0200 Subject: [PATCH 2/7] feat(metrics): add repositories metric --- internal/controller/collector.go | 71 +++++++++++++++++++++++++------- 1 file changed, 56 insertions(+), 15 deletions(-) diff --git a/internal/controller/collector.go b/internal/controller/collector.go index 6d080eea..f837cf59 100644 --- a/internal/controller/collector.go +++ b/internal/controller/collector.go @@ -54,6 +54,10 @@ var ( cachedImagesMetric = prometheus.BuildFQName(kuikMetrics.Namespace, subsystem, "cached_images") cachedImagesHelp = "Number of images expected to be cached" cachedImagesDesc = prometheus.NewDesc(cachedImagesMetric, cachedImagesHelp, []string{"status", "cached", "expiring"}, nil) + + repositoriesMetric = prometheus.BuildFQName(kuikMetrics.Namespace, subsystem, "repositories") + repositoriesHelp = "Number of repositories" + repositoriesDesc = prometheus.NewDesc(repositoriesMetric, repositoriesHelp, []string{"status"}, nil) ) func RegisterMetrics(client client.Client) { @@ -76,28 +80,65 @@ type ControllerCollector struct { func (c *ControllerCollector) Describe(ch chan<- *prometheus.Desc) { ch <- cachedImagesDesc + ch <- repositoriesDesc } func (c *ControllerCollector) Collect(ch chan<- prometheus.Metric) { - cachedImageList := &kuikv1alpha1.CachedImageList{} - if err := c.List(context.Background(), cachedImageList); err == nil { - cachedImageGaugeVec := prometheus.NewGaugeVec( - prometheus.GaugeOpts{ - Name: cachedImagesMetric, - Help: cachedImagesHelp, - }, - []string{"status", "cached", "expiring"}, - ) - for _, cachedImage := range cachedImageList.Items { - cachedImageGauge := cachedImageGaugeVec.WithLabelValues(cachedImage.Status.Phase, strconv.FormatBool(cachedImage.Status.IsCached), strconv.FormatBool(cachedImage.Spec.ExpiresAt != nil)) - cachedImageGauge.Inc() - } - cachedImageGaugeVec.Collect(ch) + if cachedImagesGaugeVec, err := c.getCachedImagesMetric(); err == nil { + cachedImagesGaugeVec.Collect(ch) + } else { + log.FromContext(context.Background()).Error(err, "could not collect "+cachedImagesMetric+" metric") + } + + if repositoriesGaugeVec, err := c.getRepositoriesMetric(); err == nil { + repositoriesGaugeVec.Collect(ch) } else { - log.FromContext(context.TODO()).Error(err, "could not collect "+cachedImagesMetric+" metric") + log.FromContext(context.Background()).Error(err, "could not collect "+repositoriesMetric+" metric") } } +func (c *ControllerCollector) getCachedImagesMetric() (*prometheus.GaugeVec, error) { + cachedImageList := &kuikv1alpha1.CachedImageList{} + if err := c.List(context.Background(), cachedImageList); err != nil { + return nil, err + } + + cachedImagesGaugeVec := prometheus.NewGaugeVec( + prometheus.GaugeOpts{ + Name: cachedImagesMetric, + Help: cachedImagesHelp, + }, + []string{"status", "cached", "expiring"}, + ) + for _, cachedImage := range cachedImageList.Items { + cachedImagesGauge := cachedImagesGaugeVec.WithLabelValues(cachedImage.Status.Phase, strconv.FormatBool(cachedImage.Status.IsCached), strconv.FormatBool(cachedImage.Spec.ExpiresAt != nil)) + cachedImagesGauge.Inc() + } + + return cachedImagesGaugeVec, nil +} + +func (c *ControllerCollector) getRepositoriesMetric() (*prometheus.GaugeVec, error) { + repositoriesList := &kuikv1alpha1.RepositoryList{} + if err := c.List(context.Background(), repositoriesList); err != nil { + return nil, err + } + + repositoriesGaugeVec := prometheus.NewGaugeVec( + prometheus.GaugeOpts{ + Name: repositoriesMetric, + Help: repositoriesHelp, + }, + []string{"status"}, + ) + for _, repository := range repositoriesList.Items { + repositoriesGauge := repositoriesGaugeVec.WithLabelValues(repository.Status.Phase) + repositoriesGauge.Inc() + } + + return repositoriesGaugeVec, nil +} + func SetLeader(leader bool) { if leader { isLeader.Set(1) From 3314d498e0ef0afa675c82b412db2ebf28fd4b17 Mon Sep 17 00:00:00 2001 From: Paul Laffitte Date: Thu, 13 Jun 2024 11:40:14 +0200 Subject: [PATCH 3/7] refactor: remove unused variable --- cmd/cache/main.go | 1 - internal/controller/collector.go | 2 -- 2 files changed, 3 deletions(-) diff --git a/cmd/cache/main.go b/cmd/cache/main.go index 2798398b..d4f44129 100644 --- a/cmd/cache/main.go +++ b/cmd/cache/main.go @@ -154,7 +154,6 @@ func main() { kuikController.SetLeader(true) }() - kuikController.ProbeAddr = probeAddr kuikController.RegisterMetrics(mgr.GetClient()) setupLog.Info("starting manager") diff --git a/internal/controller/collector.go b/internal/controller/collector.go index f837cf59..1caac222 100644 --- a/internal/controller/collector.go +++ b/internal/controller/collector.go @@ -14,8 +14,6 @@ import ( const subsystem = "controller" -var ProbeAddr = "" - var ( ImagePutInCache = prometheus.NewCounter( prometheus.CounterOpts{ From e71f30c6a99f26d3d68b42935b72dd74142ca3b8 Mon Sep 17 00:00:00 2001 From: Paul Laffitte Date: Thu, 13 Jun 2024 16:32:56 +0200 Subject: [PATCH 4/7] feat(metrics): add containers_with_cached_images metric --- api/kuik/v1alpha1/cachedimage_utils.go | 15 +++++ internal/controller/collector.go | 76 ++++++++++++++++++++-- internal/controller/core/pod_controller.go | 10 +-- 3 files changed, 89 insertions(+), 12 deletions(-) diff --git a/api/kuik/v1alpha1/cachedimage_utils.go b/api/kuik/v1alpha1/cachedimage_utils.go index cd37905b..1945841d 100644 --- a/api/kuik/v1alpha1/cachedimage_utils.go +++ b/api/kuik/v1alpha1/cachedimage_utils.go @@ -2,6 +2,7 @@ package v1alpha1 import ( "context" + "strings" "github.com/distribution/reference" "github.com/enix/kube-image-keeper/internal/registry" @@ -39,3 +40,17 @@ func (r *CachedImage) GetPullSecrets(apiReader client.Reader) ([]corev1.Secret, return pullSecrets, nil } + +func CachedImageNameFromSourceImage(sourceImage string) (string, error) { + ref, err := reference.ParseAnyReference(sourceImage) + if err != nil { + return "", err + } + + sanitizedName := registry.SanitizeName(ref.String()) + if !strings.Contains(sourceImage, ":") { + sanitizedName += "-latest" + } + + return sanitizedName, nil +} diff --git a/internal/controller/collector.go b/internal/controller/collector.go index 1caac222..1aa08a5c 100644 --- a/internal/controller/collector.go +++ b/internal/controller/collector.go @@ -4,9 +4,14 @@ import ( "context" "strconv" + "github.com/enix/kube-image-keeper/api/kuik/v1alpha1" kuikv1alpha1 "github.com/enix/kube-image-keeper/api/kuik/v1alpha1" + "github.com/enix/kube-image-keeper/internal/controller/core" kuikMetrics "github.com/enix/kube-image-keeper/internal/metrics" + "github.com/enix/kube-image-keeper/internal/registry" "github.com/prometheus/client_golang/prometheus" + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "sigs.k8s.io/controller-runtime/pkg/client" "sigs.k8s.io/controller-runtime/pkg/log" "sigs.k8s.io/controller-runtime/pkg/metrics" @@ -56,6 +61,10 @@ var ( repositoriesMetric = prometheus.BuildFQName(kuikMetrics.Namespace, subsystem, "repositories") repositoriesHelp = "Number of repositories" repositoriesDesc = prometheus.NewDesc(repositoriesMetric, repositoriesHelp, []string{"status"}, nil) + + containersWithCachedImageMetric = prometheus.BuildFQName(kuikMetrics.Namespace, subsystem, "containers_with_cached_image") + containersWithCachedImageHelp = "Number of containers that have been rewritten to use a cached image" + containersWithCachedImageDesc = prometheus.NewDesc(containersWithCachedImageMetric, containersWithCachedImageHelp, []string{"status", "cached"}, nil) ) func RegisterMetrics(client client.Client) { @@ -79,6 +88,7 @@ type ControllerCollector struct { func (c *ControllerCollector) Describe(ch chan<- *prometheus.Desc) { ch <- cachedImagesDesc ch <- repositoriesDesc + ch <- containersWithCachedImageDesc } func (c *ControllerCollector) Collect(ch chan<- prometheus.Metric) { @@ -93,6 +103,12 @@ func (c *ControllerCollector) Collect(ch chan<- prometheus.Metric) { } else { log.FromContext(context.Background()).Error(err, "could not collect "+repositoriesMetric+" metric") } + + if containersWithCachedImageGaugeVec, err := c.getContainersWithCachedImageMetric(); err == nil { + containersWithCachedImageGaugeVec.Collect(ch) + } else { + log.FromContext(context.Background()).Error(err, "could not collect "+containersWithCachedImageMetric+" metric") + } } func (c *ControllerCollector) getCachedImagesMetric() (*prometheus.GaugeVec, error) { @@ -109,13 +125,66 @@ func (c *ControllerCollector) getCachedImagesMetric() (*prometheus.GaugeVec, err []string{"status", "cached", "expiring"}, ) for _, cachedImage := range cachedImageList.Items { - cachedImagesGauge := cachedImagesGaugeVec.WithLabelValues(cachedImage.Status.Phase, strconv.FormatBool(cachedImage.Status.IsCached), strconv.FormatBool(cachedImage.Spec.ExpiresAt != nil)) - cachedImagesGauge.Inc() + cachedImagesGaugeVec. + WithLabelValues(cachedImage.Status.Phase, strconv.FormatBool(cachedImage.Status.IsCached), strconv.FormatBool(cachedImage.Spec.ExpiresAt != nil)). + Inc() } return cachedImagesGaugeVec, nil } +func (c *ControllerCollector) getContainersWithCachedImageMetric() (*prometheus.GaugeVec, error) { + cachedImageList := &kuikv1alpha1.CachedImageList{} + if err := c.List(context.Background(), cachedImageList); err != nil { + return nil, err + } + + cachedImages := map[string]v1alpha1.CachedImage{} + for _, cachedImage := range cachedImageList.Items { + cachedImages[cachedImage.Name] = cachedImage + } + + podList := &corev1.PodList{} + labelSelector := metav1.LabelSelector{ + MatchLabels: map[string]string{ + core.LabelManagedName: "true", + }, + } + selector, err := metav1.LabelSelectorAsSelector(&labelSelector) + if err != nil { + return nil, err + } + if err := c.List(context.Background(), podList, &client.ListOptions{LabelSelector: selector}); err != nil { + return nil, err + } + + containersWithCachedImageGaugeVec := prometheus.NewGaugeVec( + prometheus.GaugeOpts{ + Name: containersWithCachedImageMetric, + Help: containersWithCachedImageHelp, + }, + []string{"status", "cached"}, + ) + for _, pod := range podList.Items { + for _, container := range pod.Spec.Containers { + annotationKey := registry.ContainerAnnotationKey(container.Name, false) + if sourceImage, ok := pod.ObjectMeta.Annotations[annotationKey]; ok { + cachedImageName, err := v1alpha1.CachedImageNameFromSourceImage(sourceImage) + if err != nil { + return nil, err + } + if cachedImage, ok := cachedImages[cachedImageName]; ok { + containersWithCachedImageGaugeVec. + WithLabelValues(cachedImage.Status.Phase, strconv.FormatBool(cachedImage.Status.IsCached)). + Inc() + } + } + } + } + + return containersWithCachedImageGaugeVec, nil +} + func (c *ControllerCollector) getRepositoriesMetric() (*prometheus.GaugeVec, error) { repositoriesList := &kuikv1alpha1.RepositoryList{} if err := c.List(context.Background(), repositoriesList); err != nil { @@ -130,8 +199,7 @@ func (c *ControllerCollector) getRepositoriesMetric() (*prometheus.GaugeVec, err []string{"status"}, ) for _, repository := range repositoriesList.Items { - repositoriesGauge := repositoriesGaugeVec.WithLabelValues(repository.Status.Phase) - repositoriesGauge.Inc() + repositoriesGaugeVec.WithLabelValues(repository.Status.Phase).Inc() } return repositoriesGaugeVec, nil diff --git a/internal/controller/core/pod_controller.go b/internal/controller/core/pod_controller.go index 15839a6c..2ed8fa5f 100644 --- a/internal/controller/core/pod_controller.go +++ b/internal/controller/core/pod_controller.go @@ -3,7 +3,6 @@ package core import ( "context" _ "crypto/sha256" - "strings" "golang.org/x/exp/maps" apierrors "k8s.io/apimachinery/pkg/api/errors" @@ -11,7 +10,7 @@ import ( "k8s.io/apimachinery/pkg/selection" "k8s.io/apimachinery/pkg/types" - "github.com/distribution/reference" + "github.com/enix/kube-image-keeper/api/kuik/v1alpha1" kuikv1alpha1 "github.com/enix/kube-image-keeper/api/kuik/v1alpha1" "github.com/enix/kube-image-keeper/internal/registry" corev1 "k8s.io/api/core/v1" @@ -250,16 +249,11 @@ func desiredCachedImagesForContainers(ctx context.Context, containers []corev1.C } func cachedImageFromSourceImage(sourceImage string) (*kuikv1alpha1.CachedImage, error) { - ref, err := reference.ParseAnyReference(sourceImage) + sanitizedName, err := v1alpha1.CachedImageNameFromSourceImage(sourceImage) if err != nil { return nil, err } - sanitizedName := registry.SanitizeName(ref.String()) - if !strings.Contains(sourceImage, ":") { - sanitizedName += "-latest" - } - cachedImage := kuikv1alpha1.CachedImage{ TypeMeta: metav1.TypeMeta{APIVersion: kuikv1alpha1.GroupVersion.String(), Kind: "CachedImage"}, ObjectMeta: metav1.ObjectMeta{ From 517c6e71e41d7bdfe3dc9a134ac76a7135a5f872 Mon Sep 17 00:00:00 2001 From: Paul Laffitte Date: Fri, 14 Jun 2024 17:25:05 +0200 Subject: [PATCH 5/7] refactor(metrics): move collector to controller package --- internal/controller/collector.go | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/internal/controller/collector.go b/internal/controller/collector.go index 1aa08a5c..dc9a3b60 100644 --- a/internal/controller/collector.go +++ b/internal/controller/collector.go @@ -5,7 +5,6 @@ import ( "strconv" "github.com/enix/kube-image-keeper/api/kuik/v1alpha1" - kuikv1alpha1 "github.com/enix/kube-image-keeper/api/kuik/v1alpha1" "github.com/enix/kube-image-keeper/internal/controller/core" kuikMetrics "github.com/enix/kube-image-keeper/internal/metrics" "github.com/enix/kube-image-keeper/internal/registry" @@ -42,7 +41,7 @@ var ( Name: "is_leader", Help: "Whether or not this replica is a leader. 1 if it is, 0 otherwise.", }) - up = prometheus.NewGaugeFunc(prometheus.GaugeOpts{ + Up = prometheus.NewGaugeFunc(prometheus.GaugeOpts{ Namespace: kuikMetrics.Namespace, Subsystem: subsystem, Name: "up", @@ -74,7 +73,7 @@ func RegisterMetrics(client client.Client) { ImageRemovedFromCache, kuikMetrics.NewInfo(subsystem), isLeader, - up, + Up, &ControllerCollector{ Client: client, }, @@ -112,7 +111,7 @@ func (c *ControllerCollector) Collect(ch chan<- prometheus.Metric) { } func (c *ControllerCollector) getCachedImagesMetric() (*prometheus.GaugeVec, error) { - cachedImageList := &kuikv1alpha1.CachedImageList{} + cachedImageList := &v1alpha1.CachedImageList{} if err := c.List(context.Background(), cachedImageList); err != nil { return nil, err } @@ -134,7 +133,7 @@ func (c *ControllerCollector) getCachedImagesMetric() (*prometheus.GaugeVec, err } func (c *ControllerCollector) getContainersWithCachedImageMetric() (*prometheus.GaugeVec, error) { - cachedImageList := &kuikv1alpha1.CachedImageList{} + cachedImageList := &v1alpha1.CachedImageList{} if err := c.List(context.Background(), cachedImageList); err != nil { return nil, err } @@ -186,7 +185,7 @@ func (c *ControllerCollector) getContainersWithCachedImageMetric() (*prometheus. } func (c *ControllerCollector) getRepositoriesMetric() (*prometheus.GaugeVec, error) { - repositoriesList := &kuikv1alpha1.RepositoryList{} + repositoriesList := &v1alpha1.RepositoryList{} if err := c.List(context.Background(), repositoriesList); err != nil { return nil, err } From 327254fe64eec6e3502fc5738765e904f9b0582d Mon Sep 17 00:00:00 2001 From: Paul Laffitte Date: Thu, 11 Jul 2024 16:40:53 +0200 Subject: [PATCH 6/7] feat(metrics): add image_caching_request metric --- api/kuik/v1alpha1/cachedimage_utils.go | 9 +++++++++ internal/controller/collector.go | 10 ++++++++++ internal/controller/kuik/cachedimage_controller.go | 10 +++++++--- 3 files changed, 26 insertions(+), 3 deletions(-) diff --git a/api/kuik/v1alpha1/cachedimage_utils.go b/api/kuik/v1alpha1/cachedimage_utils.go index 1945841d..6475ead6 100644 --- a/api/kuik/v1alpha1/cachedimage_utils.go +++ b/api/kuik/v1alpha1/cachedimage_utils.go @@ -21,6 +21,15 @@ func (r *CachedImage) Repository() (reference.Named, error) { return named, nil } +func (r *CachedImage) Upstream() (string, error) { + named, err := r.Repository() + if err != nil { + return "", err + } + + return reference.Domain(named), nil +} + func (r *CachedImage) GetPullSecrets(apiReader client.Reader) ([]corev1.Secret, error) { named, err := r.Repository() if err != nil { diff --git a/internal/controller/collector.go b/internal/controller/collector.go index dc9a3b60..d61408e1 100644 --- a/internal/controller/collector.go +++ b/internal/controller/collector.go @@ -19,6 +19,15 @@ import ( const subsystem = "controller" var ( + ImageCachingRequest = prometheus.NewCounterVec( + prometheus.CounterOpts{ + Namespace: kuikMetrics.Namespace, + Subsystem: subsystem, + Name: "image_caching_request", + Help: "Number of request to cache an image", + }, + []string{"successful", "upstream_registry"}, + ) ImagePutInCache = prometheus.NewCounter( prometheus.CounterOpts{ Namespace: kuikMetrics.Namespace, @@ -69,6 +78,7 @@ var ( func RegisterMetrics(client client.Client) { // Register custom metrics with the global prometheus registry metrics.Registry.MustRegister( + ImageCachingRequest, ImagePutInCache, ImageRemovedFromCache, kuikMetrics.NewInfo(subsystem), diff --git a/internal/controller/kuik/cachedimage_controller.go b/internal/controller/kuik/cachedimage_controller.go index 8aa44153..6e8b3898 100644 --- a/internal/controller/kuik/cachedimage_controller.go +++ b/internal/controller/kuik/cachedimage_controller.go @@ -4,6 +4,7 @@ import ( "context" "crypto/x509" "net/http" + "strconv" "strings" "time" @@ -239,8 +240,14 @@ func (r *CachedImageReconciler) Reconcile(ctx context.Context, req ctrl.Request) putImageInCache = false } if putImageInCache { + upstream, err := cachedImage.Upstream() + if err != nil { + return ctrl.Result{}, err + } + r.Recorder.Eventf(&cachedImage, "Normal", "Caching", "Start caching image %s", cachedImage.Spec.SourceImage) err = r.cacheImage(&cachedImage) + kuikController.ImageCachingRequest.WithLabelValues(strconv.FormatBool(err == nil), upstream).Inc() if err != nil { log.Error(err, "failed to cache image") r.Recorder.Eventf(&cachedImage, "Warning", "CacheFailed", "Failed to cache image %s, reason: %s", cachedImage.Spec.SourceImage, err) @@ -401,9 +408,6 @@ func (r *CachedImageReconciler) SetupWithManager(mgr ctrl.Manager, maxConcurrent &corev1.Pod{}, handler.EnqueueRequestsFromMapFunc(r.cachedImagesRequestFromPod), builder.WithPredicates(predicate.Funcs{ - // GenericFunc: func(e event.GenericEvent) bool { - // return true - // }, DeleteFunc: func(e event.DeleteEvent) bool { pod := e.Object.(*corev1.Pod) var currentPod corev1.Pod From cb93bb11a7bd53a5e43e4241d97383dd699eeee7 Mon Sep 17 00:00:00 2001 From: Paul Laffitte Date: Thu, 11 Jul 2024 18:38:46 +0200 Subject: [PATCH 7/7] fix: handle errors in podsWithDeletingCachedImages --- internal/controller/core/pod_controller.go | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/internal/controller/core/pod_controller.go b/internal/controller/core/pod_controller.go index 2ed8fa5f..44de8fdf 100644 --- a/internal/controller/core/pod_controller.go +++ b/internal/controller/core/pod_controller.go @@ -161,7 +161,13 @@ func (r *PodReconciler) podsWithDeletingCachedImages(ctx context.Context, obj cl } var podList corev1.PodList - podRequirements, _ := labels.NewRequirement(LabelManagedName, selection.Equals, []string{"true"}) + podRequirements, err := labels.NewRequirement(LabelManagedName, selection.Equals, []string{"true"}) + if err != nil { + // errors cannot be handled in a better way for now (see https://github.com/kubernetes-sigs/controller-runtime/issues/1996) + // maybe we don't need to enqueue all Pods related to this CachedImage but only those in the status UsedBy + log.Error(err, "could not list pods") + return nil + } selector := labels.NewSelector() selector = selector.Add(*podRequirements) if err := r.List(ctx, &podList, &client.ListOptions{