Prune stale per-container version metrics to stop unbounded Prometheus series growth

pkg/metrics/metrics.go

@@ -118,6 +118,14 @@ func (m *Metrics) AddImage(namespace, pod, container, containerType, imageURL st
 		isLatestF = 1.0
 	}
 
+	labels := buildContainerPartialLabels(namespace, pod, container, containerType)
+
+	// Remove any existing "current state" gauge series for this container before
+	// registering the newest values. Otherwise each version change leaves behind
+	// a distinct Prometheus series due to the current/latest version labels.
+	m.containerImageVersion.DeletePartialMatch(labels)
+	m.containerImageChecked.DeletePartialMatch(labels)
+
 	m.containerImageVersion.With(
 		buildFullLabels(namespace, pod, container, containerType, imageURL, currentVersion, latestVersion),
 	).Set(isLatestF)

pkg/metrics/metrics_test.go

@@ -15,6 +15,7 @@ import (
 
 	"github.com/prometheus/client_golang/prometheus"
 	"github.com/prometheus/client_golang/prometheus/testutil"
+	dto "github.com/prometheus/client_model/go"
 
 	corev1 "k8s.io/api/core/v1"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
@@ -72,6 +73,69 @@ func TestCache(t *testing.T) {
 	}
 }
 
+func TestAddImageReplacesExistingVersionMetrics(t *testing.T) {
+	reg := prometheus.NewRegistry()
+	m := New(logrus.NewEntry(logrus.New()), reg, fakek8s)
+
+	m.AddImage("namespace", "pod", "container", "container", "url", false, "1.0.0", "1.1.0")
+	m.AddImage("namespace", "pod", "container", "container", "url", true, "1.1.0", "1.1.0")
+
+	assert.Equal(t, 1,
+		testutil.CollectAndCount(m.containerImageVersion.MetricVec, MetricNamespace+"_is_latest_version"),
+	)
+	assert.Equal(t, 1,
+		testutil.CollectAndCount(m.containerImageChecked.MetricVec, MetricNamespace+"_last_checked"),
+	)
+
+	metricFamilies, err := reg.Gather()
+	require.NoError(t, err)
+	assert.False(t, hasMetricWithLabels(metricFamilies, MetricNamespace+"_is_latest_version", map[string]string{
+		"namespace":       "namespace",
+		"pod":             "pod",
+		"container":       "container",
+		"container_type":  "container",
+		"image":           "url",
+		"current_version": "1.0.0",
+		"latest_version":  "1.1.0",
+	}))
+
+	currentMetric, err := m.containerImageVersion.GetMetricWith(
+		buildFullLabels("namespace", "pod", "container", "container", "url", "1.1.0", "1.1.0"),
+	)
+	require.NoError(t, err)
+	assert.Equal(t, float64(1), testutil.ToFloat64(currentMetric))
+}
+
+func hasMetricWithLabels(metricFamilies []*dto.MetricFamily, name string, labels map[string]string) bool {
+	for _, metricFamily := range metricFamilies {
+		if metricFamily.GetName() != name {
+			continue
+		}
+
+		for _, metric := range metricFamily.GetMetric() {
+			if metricHasLabels(metric, labels) {
+				return true
+			}
+		}
+	}
+
+	return false
+}
+
+func metricHasLabels(metric *dto.Metric, labels map[string]string) bool {
+	if len(metric.GetLabel()) != len(labels) {
+		return false
+	}
+
+	for _, label := range metric.GetLabel() {
+		if labels[label.GetName()] != label.GetValue() {
+			return false
+		}
+	}
+
+	return true
+}
+
 // TestErrorsReporting verifies that the error metric increments correctly
 func TestErrorsReporting(t *testing.T) {
 	m := New(logrus.NewEntry(logrus.New()), prometheus.NewRegistry(), fakek8s)