diff --git a/prometheus-metrics-config/src/main/java/io/prometheus/metrics/config/MetricsProperties.java b/prometheus-metrics-config/src/main/java/io/prometheus/metrics/config/MetricsProperties.java index 6c8942713..ae930804d 100644 --- a/prometheus-metrics-config/src/main/java/io/prometheus/metrics/config/MetricsProperties.java +++ b/prometheus-metrics-config/src/main/java/io/prometheus/metrics/config/MetricsProperties.java @@ -28,6 +28,7 @@ public class MetricsProperties { private static final String SUMMARY_QUANTILE_ERRORS = "summaryQuantileErrors"; private static final String SUMMARY_MAX_AGE_SECONDS = "summaryMaxAgeSeconds"; private static final String SUMMARY_NUMBER_OF_AGE_BUCKETS = "summaryNumberOfAgeBuckets"; + private static final String USE_OTEL_METRICS = "useOtelMetrics"; @Nullable private final Boolean exemplarsEnabled; @Nullable private final Boolean histogramNativeOnly; @@ -42,6 +43,7 @@ public class MetricsProperties { @Nullable private final List summaryQuantileErrors; @Nullable private final Long summaryMaxAgeSeconds; @Nullable private final Integer summaryNumberOfAgeBuckets; + @Nullable private final Boolean useOtelMetrics; public MetricsProperties( @Nullable Boolean exemplarsEnabled, @@ -56,7 +58,8 @@ public MetricsProperties( @Nullable List summaryQuantiles, @Nullable List summaryQuantileErrors, @Nullable Long summaryMaxAgeSeconds, - @Nullable Integer summaryNumberOfAgeBuckets) { + @Nullable Integer summaryNumberOfAgeBuckets, + @Nullable Boolean useOtelMetrics) { this( exemplarsEnabled, histogramNativeOnly, @@ -71,6 +74,7 @@ public MetricsProperties( summaryQuantileErrors, summaryMaxAgeSeconds, summaryNumberOfAgeBuckets, + useOtelMetrics, ""); } @@ -88,6 +92,7 @@ private MetricsProperties( @Nullable List summaryQuantileErrors, @Nullable Long summaryMaxAgeSeconds, @Nullable Integer summaryNumberOfAgeBuckets, + @Nullable Boolean useOtelMetrics, String configPropertyPrefix) { this.exemplarsEnabled = exemplarsEnabled; this.histogramNativeOnly = isHistogramNativeOnly(histogramClassicOnly, histogramNativeOnly); @@ -109,6 +114,7 @@ private MetricsProperties( : unmodifiableList(new ArrayList<>(summaryQuantileErrors)); this.summaryMaxAgeSeconds = summaryMaxAgeSeconds; this.summaryNumberOfAgeBuckets = summaryNumberOfAgeBuckets; + this.useOtelMetrics = useOtelMetrics; validate(configPropertyPrefix); } @@ -334,6 +340,12 @@ public Integer getSummaryNumberOfAgeBuckets() { return summaryNumberOfAgeBuckets; } + /** See {@code Summary.Builder.useOtelMetrics()} */ + @Nullable + public Boolean useOtelMetrics() { + return useOtelMetrics; + } + /** * Note that this will remove entries from {@code properties}. This is because we want to know if * there are unused properties remaining after all properties have been loaded. @@ -354,6 +366,7 @@ static MetricsProperties load(String prefix, Map properties) Util.loadDoubleList(prefix + "." + SUMMARY_QUANTILE_ERRORS, properties), Util.loadLong(prefix + "." + SUMMARY_MAX_AGE_SECONDS, properties), Util.loadInteger(prefix + "." + SUMMARY_NUMBER_OF_AGE_BUCKETS, properties), + Util.loadBoolean(prefix + "." + USE_OTEL_METRICS, properties), prefix); } @@ -375,6 +388,7 @@ public static class Builder { @Nullable private List summaryQuantileErrors; @Nullable private Long summaryMaxAgeSeconds; @Nullable private Integer summaryNumberOfAgeBuckets; + @Nullable private Boolean useOtelMetrics; private Builder() {} @@ -392,7 +406,8 @@ public MetricsProperties build() { summaryQuantiles, summaryQuantileErrors, summaryMaxAgeSeconds, - summaryNumberOfAgeBuckets); + summaryNumberOfAgeBuckets, + useOtelMetrics); } /** See {@link MetricsProperties#getExemplarsEnabled()} */ @@ -476,5 +491,11 @@ public Builder summaryNumberOfAgeBuckets(@Nullable Integer summaryNumberOfAgeBuc this.summaryNumberOfAgeBuckets = summaryNumberOfAgeBuckets; return this; } + + /** See {@link MetricsProperties#useOtelMetrics()} */ + public Builder useOtelMetrics(@Nullable Boolean useOtelMetrics) { + this.useOtelMetrics = useOtelMetrics; + return this; + } } } diff --git a/prometheus-metrics-instrumentation-jvm/src/main/java/io/prometheus/metrics/instrumentation/jvm/JvmGarbageCollectorMetrics.java b/prometheus-metrics-instrumentation-jvm/src/main/java/io/prometheus/metrics/instrumentation/jvm/JvmGarbageCollectorMetrics.java index 262e2df5f..a38f8f350 100644 --- a/prometheus-metrics-instrumentation-jvm/src/main/java/io/prometheus/metrics/instrumentation/jvm/JvmGarbageCollectorMetrics.java +++ b/prometheus-metrics-instrumentation-jvm/src/main/java/io/prometheus/metrics/instrumentation/jvm/JvmGarbageCollectorMetrics.java @@ -1,6 +1,8 @@ package io.prometheus.metrics.instrumentation.jvm; +import com.sun.management.GarbageCollectionNotificationInfo; import io.prometheus.metrics.config.PrometheusProperties; +import io.prometheus.metrics.core.metrics.Histogram; import io.prometheus.metrics.core.metrics.SummaryWithCallback; import io.prometheus.metrics.model.registry.PrometheusRegistry; import io.prometheus.metrics.model.snapshots.Quantiles; @@ -9,6 +11,8 @@ import java.lang.management.ManagementFactory; import java.util.List; import javax.annotation.Nullable; +import javax.management.NotificationEmitter; +import javax.management.openmbean.CompositeData; /** * JVM Garbage Collector metrics. The {@link JvmGarbageCollectorMetrics} are registered as part of @@ -39,6 +43,7 @@ public class JvmGarbageCollectorMetrics { private static final String JVM_GC_COLLECTION_SECONDS = "jvm_gc_collection_seconds"; + private static final String JVM_GC_DURATION = "jvm.gc.duration"; private final PrometheusProperties config; private final List garbageCollectorBeans; @@ -50,7 +55,14 @@ private JvmGarbageCollectorMetrics( } private void register(PrometheusRegistry registry) { + if (Boolean.TRUE.equals(config.getDefaultMetricProperties().useOtelMetrics())) { + registerGCDurationHistogram(registry); + } else { + registerGCDurationSummary(registry); + } + } + private void registerGCDurationSummary(PrometheusRegistry registry) { SummaryWithCallback.builder(config) .name(JVM_GC_COLLECTION_SECONDS) .help("Time spent in a given JVM garbage collector in seconds.") @@ -69,6 +81,44 @@ private void register(PrometheusRegistry registry) { .register(registry); } + private void registerGCDurationHistogram(PrometheusRegistry registry) { + double[] buckets = {0.01, 0.1, 1, 10}; + + Histogram gcDurationHistogram = + Histogram.builder(config) + .name(JVM_GC_DURATION) + .help("Duration of JVM garbage collection actions.") + .labelNames("jvm.gc.action", "jvm.gc.name", "jvm.gc.cause") + .classicUpperBounds(buckets) + .register(registry); + + for (GarbageCollectorMXBean gcBean : garbageCollectorBeans) { + + if (!(gcBean instanceof NotificationEmitter)) { + continue; + } + + ((NotificationEmitter) gcBean) + .addNotificationListener( + (notification, handback) -> { + if (!GarbageCollectionNotificationInfo.GARBAGE_COLLECTION_NOTIFICATION.equals( + notification.getType())) { + return; + } + + GarbageCollectionNotificationInfo info = + GarbageCollectionNotificationInfo.from( + (CompositeData) notification.getUserData()); + + gcDurationHistogram + .labelValues(info.getGcAction(), info.getGcName(), info.getGcCause()) + .observe(Unit.millisToSeconds(info.getGcInfo().getDuration())); + }, + null, + null); + } + } + public static Builder builder() { return new Builder(PrometheusProperties.get()); } diff --git a/prometheus-metrics-instrumentation-jvm/src/test/java/io/prometheus/metrics/instrumentation/jvm/JvmGarbageCollectorMetricsTest.java b/prometheus-metrics-instrumentation-jvm/src/test/java/io/prometheus/metrics/instrumentation/jvm/JvmGarbageCollectorMetricsTest.java index 177f29d2e..598d7f1a6 100644 --- a/prometheus-metrics-instrumentation-jvm/src/test/java/io/prometheus/metrics/instrumentation/jvm/JvmGarbageCollectorMetricsTest.java +++ b/prometheus-metrics-instrumentation-jvm/src/test/java/io/prometheus/metrics/instrumentation/jvm/JvmGarbageCollectorMetricsTest.java @@ -1,26 +1,38 @@ package io.prometheus.metrics.instrumentation.jvm; +import static com.sun.management.GarbageCollectionNotificationInfo.GARBAGE_COLLECTION_NOTIFICATION; import static io.prometheus.metrics.instrumentation.jvm.TestUtil.convertToOpenMetricsFormat; import static org.assertj.core.api.Assertions.assertThat; +import static org.mockito.ArgumentCaptor.forClass; +import static org.mockito.Mockito.*; import static org.mockito.Mockito.times; import static org.mockito.Mockito.verify; import static org.mockito.Mockito.when; +import io.prometheus.metrics.config.MetricsProperties; +import io.prometheus.metrics.config.PrometheusProperties; import io.prometheus.metrics.model.registry.MetricNameFilter; import io.prometheus.metrics.model.registry.PrometheusRegistry; import io.prometheus.metrics.model.snapshots.MetricSnapshots; import java.io.IOException; import java.lang.management.GarbageCollectorMXBean; import java.util.Arrays; +import java.util.Collections; +import java.util.HashMap; +import java.util.Map; import java.util.concurrent.TimeUnit; +import javax.management.Notification; +import javax.management.NotificationEmitter; +import javax.management.NotificationListener; +import javax.management.openmbean.*; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; -import org.mockito.Mockito; +import org.mockito.ArgumentCaptor; class JvmGarbageCollectorMetricsTest { - private final GarbageCollectorMXBean mockGcBean1 = Mockito.mock(GarbageCollectorMXBean.class); - private final GarbageCollectorMXBean mockGcBean2 = Mockito.mock(GarbageCollectorMXBean.class); + private final GarbageCollectorMXBean mockGcBean1 = mock(GarbageCollectorMXBean.class); + private final GarbageCollectorMXBean mockGcBean2 = mock(GarbageCollectorMXBean.class); @BeforeEach public void setUp() { @@ -58,7 +70,9 @@ public void testGoodCase() throws IOException { @Test public void testIgnoredMetricNotScraped() { MetricNameFilter filter = - MetricNameFilter.builder().nameMustNotBeEqualTo("jvm_gc_collection_seconds").build(); + MetricNameFilter.builder() + .nameMustNotBeEqualTo("jvm_gc_collection_seconds", "jvm_gc_duration") + .build(); PrometheusRegistry registry = new PrometheusRegistry(); JvmGarbageCollectorMetrics.builder() @@ -70,4 +84,148 @@ public void testIgnoredMetricNotScraped() { verify(mockGcBean1, times(0)).getCollectionCount(); assertThat(snapshots.size()).isZero(); } + + @Test + public void testNonOtelMetricsAbsentWhenUseOtelEnabled() { + + PrometheusRegistry registry = new PrometheusRegistry(); + PrometheusProperties properties = + PrometheusProperties.builder() + .defaultMetricsProperties(MetricsProperties.builder().useOtelMetrics(true).build()) + .build(); + JvmGarbageCollectorMetrics.builder(properties) + .garbageCollectorBeans(Arrays.asList(mockGcBean1, mockGcBean2)) + .register(registry); + registry.scrape(); + + verify(mockGcBean1, times(0)).getCollectionTime(); + verify(mockGcBean1, times(0)).getCollectionCount(); + } + + @Test + @SuppressWarnings("rawtypes") + public void testGCDurationHistogramLabels() throws Exception { + GarbageCollectorMXBean mockGcBean = + mock( + GarbageCollectorMXBean.class, + withSettings().extraInterfaces(NotificationEmitter.class)); + when(mockGcBean.getName()).thenReturn("MyGC"); + + PrometheusProperties properties = + PrometheusProperties.builder() + .defaultMetricsProperties(MetricsProperties.builder().useOtelMetrics(true).build()) + .build(); + + PrometheusRegistry registry = new PrometheusRegistry(); + JvmGarbageCollectorMetrics.builder(properties) + .garbageCollectorBeans(Collections.singletonList(mockGcBean)) + .register(registry); + + NotificationListener listener; + ArgumentCaptor captor = forClass(NotificationListener.class); + verify((NotificationEmitter) mockGcBean) + .addNotificationListener(captor.capture(), isNull(), isNull()); + listener = captor.getValue(); + + TabularType memoryTabularType = getMemoryTabularType(); + TabularData memoryBefore = new TabularDataSupport(memoryTabularType); + TabularData memoryAfter = new TabularDataSupport(memoryTabularType); + + CompositeType gcInfoType = + new CompositeType( + "sun.management.BaseGcInfoCompositeType", + "gcInfo", + new String[] { + "id", "startTime", "endTime", "duration", "memoryUsageBeforeGc", "memoryUsageAfterGc" + }, + new String[] { + "id", "startTime", "endTime", "duration", "memoryUsageBeforeGc", "memoryUsageAfterGc" + }, + new OpenType[] { + SimpleType.LONG, + SimpleType.LONG, + SimpleType.LONG, + SimpleType.LONG, + memoryTabularType, + memoryTabularType + }); + + java.util.Map gcInfoMap = new HashMap<>(); + gcInfoMap.put("id", 0L); + gcInfoMap.put("startTime", 100L); + gcInfoMap.put("endTime", 200L); + gcInfoMap.put("duration", 100L); + gcInfoMap.put("memoryUsageBeforeGc", memoryBefore); + gcInfoMap.put("memoryUsageAfterGc", memoryAfter); + + CompositeData notificationData = getGcNotificationData(gcInfoType, gcInfoMap); + + Notification notification = + new Notification( + GARBAGE_COLLECTION_NOTIFICATION, mockGcBean, 1, System.currentTimeMillis(), "gc"); + notification.setUserData(notificationData); + + listener.handleNotification(notification, null); + + MetricSnapshots snapshots = registry.scrape(); + + String expected = + """ + {"jvm.gc.duration_bucket","jvm.gc.action"="end of minor GC","jvm.gc.cause"="testCause","jvm.gc.name"="MyGC",le="0.01"} 0 + {"jvm.gc.duration_bucket","jvm.gc.action"="end of minor GC","jvm.gc.cause"="testCause","jvm.gc.name"="MyGC",le="0.1"} 1 + {"jvm.gc.duration_bucket","jvm.gc.action"="end of minor GC","jvm.gc.cause"="testCause","jvm.gc.name"="MyGC",le="1.0"} 1 + {"jvm.gc.duration_bucket","jvm.gc.action"="end of minor GC","jvm.gc.cause"="testCause","jvm.gc.name"="MyGC",le="10.0"} 1 + {"jvm.gc.duration_bucket","jvm.gc.action"="end of minor GC","jvm.gc.cause"="testCause","jvm.gc.name"="MyGC",le="+Inf"} 1 + {"jvm.gc.duration_count","jvm.gc.action"="end of minor GC","jvm.gc.cause"="testCause","jvm.gc.name"="MyGC"} 1 + {"jvm.gc.duration_sum","jvm.gc.action"="end of minor GC","jvm.gc.cause"="testCause","jvm.gc.name"="MyGC"} 0.1 + """; + + String metrics = convertToOpenMetricsFormat(snapshots); + + assertThat(metrics).contains(expected); + } + + private TabularType getMemoryTabularType() throws OpenDataException { + CompositeType memoryUsageType = + new CompositeType( + "java.lang.management.MemoryUsage", + "MemoryUsage", + new String[] {"init", "used", "committed", "max"}, + new String[] {"init", "used", "committed", "max"}, + new OpenType[] {SimpleType.LONG, SimpleType.LONG, SimpleType.LONG, SimpleType.LONG}); + + CompositeType memoryUsageEntryType = + new CompositeType( + "memoryUsageEntry", + "memoryUsageEntry", + new String[] {"key", "value"}, + new String[] {"key", "value"}, + new OpenType[] {SimpleType.STRING, memoryUsageType}); + + return new TabularType( + "memoryUsageTabular", "memoryUsageTabular", memoryUsageEntryType, new String[] {"key"}); + } + + private static CompositeData getGcNotificationData( + CompositeType gcInfoType, Map gcInfoMap) throws OpenDataException { + CompositeData gcInfoData = new CompositeDataSupport(gcInfoType, gcInfoMap); + + CompositeType notificationType = + new CompositeType( + "sun.management.BaseGarbageCollectionNotifInfoCompositeType", + "GarbageCollectionNotificationInfo", + new String[] {"gcAction", "gcName", "gcCause", "gcInfo"}, + new String[] {"gcAction", "gcName", "gcCause", "gcInfo"}, + new OpenType[] { + SimpleType.STRING, SimpleType.STRING, SimpleType.STRING, gcInfoType + }); + + Map notifMap = new HashMap<>(); + notifMap.put("gcAction", "end of minor GC"); + notifMap.put("gcName", "MyGC"); + notifMap.put("gcCause", "testCause"); + notifMap.put("gcInfo", gcInfoData); + + return new CompositeDataSupport(notificationType, notifMap); + } }