diff --git a/confidence-proto/src/main/proto/confidence/telemetry.proto b/confidence-proto/src/main/proto/confidence/telemetry.proto index 61099b22..d0cc4ebb 100644 --- a/confidence-proto/src/main/proto/confidence/telemetry.proto +++ b/confidence-proto/src/main/proto/confidence/telemetry.proto @@ -5,6 +5,17 @@ option java_package = "com.spotify.telemetry.v1"; option java_multiple_files = true; option java_outer_classname = "TelemetryProto"; +import "google/protobuf/descriptor.proto"; + +message MetricAnnotation { + string name = 1; + string unit = 2; +} + +extend google.protobuf.EnumValueOptions { + MetricAnnotation metric = 50641; +} + enum Platform { PLATFORM_UNSPECIFIED = 0; PLATFORM_JAVA = 1; @@ -32,12 +43,61 @@ message LibraryTraces { message Trace { TraceId id = 1; - // only used for timed events. - optional uint64 millisecond_duration = 2; + + oneof traceData { + uint64 millisecond_duration = 2 [deprecated = true]; + RequestTrace request_trace = 3; + CountTrace count_trace = 4; + EvaluationTrace evaluation_trace = 5; + } + + message CountTrace {} + + message RequestTrace { + uint64 millisecond_duration = 1; + Status status = 2; + + enum Status { + STATUS_UNSPECIFIED = 0; + STATUS_SUCCESS = 1; + STATUS_ERROR = 2; + STATUS_TIMEOUT = 3; + STATUS_CACHED = 4; + } + } + + message EvaluationTrace { + EvaluationReason reason = 1; + EvaluationErrorCode error_code = 2; + + enum EvaluationReason { + EVALUATION_REASON_UNSPECIFIED = 0; + EVALUATION_REASON_TARGETING_MATCH = 1; + EVALUATION_REASON_DEFAULT = 2; + EVALUATION_REASON_STALE = 3; + EVALUATION_REASON_DISABLED = 4; + EVALUATION_REASON_CACHED = 5; + EVALUATION_REASON_STATIC = 6; + EVALUATION_REASON_SPLIT = 7; + EVALUATION_REASON_ERROR = 8; + } + + enum EvaluationErrorCode { + EVALUATION_ERROR_CODE_UNSPECIFIED = 0; + EVALUATION_ERROR_CODE_PROVIDER_NOT_READY = 1; + EVALUATION_ERROR_CODE_FLAG_NOT_FOUND = 2; + EVALUATION_ERROR_CODE_PARSE_ERROR = 3; + EVALUATION_ERROR_CODE_TYPE_MISMATCH = 4; + EVALUATION_ERROR_CODE_TARGETING_KEY_MISSING = 5; + EVALUATION_ERROR_CODE_INVALID_CONTEXT = 6; + EVALUATION_ERROR_CODE_PROVIDER_FATAL = 7; + EVALUATION_ERROR_CODE_GENERAL = 8; + } + } } enum Library { - LIBRARY_UNSPECIFIED = 0; + LIBRARY_UNKNOWN = 0; LIBRARY_CONFIDENCE = 1; LIBRARY_OPEN_FEATURE = 2; LIBRARY_REACT = 3; @@ -45,9 +105,8 @@ message LibraryTraces { enum TraceId { TRACE_ID_UNSPECIFIED = 0; - TRACE_ID_RESOLVE_LATENCY = 1; - TRACE_ID_STALE_FLAG = 2; - TRACE_ID_FLAG_TYPE_MISMATCH = 3; - TRACE_ID_WITH_CONTEXT = 4; + TRACE_ID_RESOLVE_LATENCY = 1 [(metric) = {name: "resolve_latency", unit: "ms"}]; + TRACE_ID_STALE_FLAG = 2 [deprecated = true, (metric) = {name: "stale_flag"}]; + TRACE_ID_FLAG_EVALUATION = 3 [(metric) = {name: "flag_evaluation"}]; } } diff --git a/openfeature-provider/src/main/java/com/spotify/confidence/ConfidenceFeatureProvider.java b/openfeature-provider/src/main/java/com/spotify/confidence/ConfidenceFeatureProvider.java index 8f7bc644..c2905a1b 100644 --- a/openfeature-provider/src/main/java/com/spotify/confidence/ConfidenceFeatureProvider.java +++ b/openfeature-provider/src/main/java/com/spotify/confidence/ConfidenceFeatureProvider.java @@ -192,6 +192,7 @@ public ProviderEvaluation getObjectEvaluation( } final ResolvedFlag resolvedFlag = resolveFlagResponse.getResolvedFlags(0); + final String reason = resolvedFlag.getReason().toString(); if (resolvedFlag.getVariant().isEmpty()) { log.debug( @@ -199,6 +200,7 @@ public ProviderEvaluation getObjectEvaluation( "The server returned no assignment for the flag '%s'. Typically, this happens " + "if no configured rules matches the given evaluation context.", flagPath.getFlag())); + confidence.client().trackEvaluation(reason, null); return ProviderEvaluation.builder() .value(defaultValue) .reason( @@ -216,10 +218,10 @@ public ProviderEvaluation getObjectEvaluation( value = defaultValue; } - // regular resolve was successful + confidence.client().trackEvaluation(reason, null); return ProviderEvaluation.builder() .value(value) - .reason(resolvedFlag.getReason().toString()) + .reason(reason) .variant(resolvedFlag.getVariant()) .build(); } diff --git a/openfeature-provider/src/test/java/com/spotify/confidence/FeatureProviderTest.java b/openfeature-provider/src/test/java/com/spotify/confidence/FeatureProviderTest.java index 65fc5f88..ac246b2a 100644 --- a/openfeature-provider/src/test/java/com/spotify/confidence/FeatureProviderTest.java +++ b/openfeature-provider/src/test/java/com/spotify/confidence/FeatureProviderTest.java @@ -77,7 +77,8 @@ void beforeEach() { final FlagResolverClientImpl flagResolver = new FlagResolverClientImpl( new GrpcFlagResolver("fake-secret", channel, telemetryInterceptor, 1_000), telemetry); - final Confidence confidence = Confidence.create(fakeEventSender, flagResolver, "clientKey"); + final Confidence confidence = + Confidence.create(fakeEventSender, flagResolver, "clientKey", telemetry); final FeatureProvider featureProvider = new ConfidenceFeatureProvider(confidence); openFeatureAPI = OpenFeatureAPI.getInstance(); @@ -525,10 +526,21 @@ public void resolvesContainHeaderWithTelemetryData() { assertThat(libraryTracesList).hasSize(1); final LibraryTraces traces = libraryTracesList.get(0); assertThat(traces.getLibrary()).isEqualTo(LibraryTraces.Library.LIBRARY_OPEN_FEATURE); - assertThat(traces.getTracesList()).hasSize(1); - final LibraryTraces.Trace trace = traces.getTraces(0); - assertThat(trace.getId()).isEqualTo(LibraryTraces.TraceId.TRACE_ID_RESOLVE_LATENCY); - assertThat(trace.getMillisecondDuration()).isNotNegative(); + assertThat(traces.getTracesList()).hasSize(2); + final LibraryTraces.Trace latencyTrace = traces.getTraces(0); + assertThat(latencyTrace.getId()).isEqualTo(LibraryTraces.TraceId.TRACE_ID_RESOLVE_LATENCY); + assertThat(latencyTrace.getRequestTrace().getMillisecondDuration()).isNotNegative(); + assertThat(latencyTrace.getRequestTrace().getStatus()) + .isEqualTo(LibraryTraces.Trace.RequestTrace.Status.STATUS_SUCCESS); + final LibraryTraces.Trace evaluationTrace = traces.getTraces(1); + assertThat(evaluationTrace.getId()).isEqualTo(LibraryTraces.TraceId.TRACE_ID_FLAG_EVALUATION); + assertThat(evaluationTrace.getEvaluationTrace().getReason()) + .isEqualTo( + LibraryTraces.Trace.EvaluationTrace.EvaluationReason.EVALUATION_REASON_TARGETING_MATCH); + assertThat(evaluationTrace.getEvaluationTrace().getErrorCode()) + .isEqualTo( + LibraryTraces.Trace.EvaluationTrace.EvaluationErrorCode + .EVALUATION_ERROR_CODE_UNSPECIFIED); client.getIntegerDetails("flag.prop-Y", 1000, SAMPLE_CONTEXT); } diff --git a/sdk-java/src/main/java/com/spotify/confidence/Confidence.java b/sdk-java/src/main/java/com/spotify/confidence/Confidence.java index 53458194..8f688e6e 100644 --- a/sdk-java/src/main/java/com/spotify/confidence/Confidence.java +++ b/sdk-java/src/main/java/com/spotify/confidence/Confidence.java @@ -128,8 +128,10 @@ public FlagEvaluation getEvaluation(String key, T defaultValue) { try { return getEvaluationFuture(key, defaultValue).get(); } catch (Exception e) { - return new FlagEvaluation<>( - defaultValue, "", "ERROR", ErrorType.INTERNAL_ERROR, e.getMessage()); + final FlagEvaluation evaluation = + new FlagEvaluation<>(defaultValue, "", "ERROR", ErrorType.INTERNAL_ERROR, e.getMessage()); + client().trackEvaluation(evaluation.getReason(), evaluation.getErrorType().orElse(null)); + return evaluation; } } @@ -166,8 +168,9 @@ public CompletableFuture> getEvaluationFuture(String key, if (resolvedFlag.getVariant().isEmpty()) { final String errorMessage = String.format( - "The server returned no assignment for the flag '%s'. Typically, this happens " - + "if no configured rules matches the given evaluation context.", + "The server returned no assignment for the flag '%s'. Typically, this" + + " happens if no configured rules matches the given evaluation" + + " context.", flagPath.getFlag()); log.debug(errorMessage); return new FlagEvaluation<>( @@ -196,10 +199,27 @@ public CompletableFuture> getEvaluationFuture(String key, } } }) - .exceptionally(handleFlagEvaluationError(defaultValue)); + .thenApply( + evaluation -> { + client() + .trackEvaluation( + evaluation.getReason(), evaluation.getErrorType().orElse(null)); + return evaluation; + }) + .exceptionally( + e -> { + final FlagEvaluation evaluation = + handleFlagEvaluationError(defaultValue).apply(e); + client() + .trackEvaluation( + evaluation.getReason(), evaluation.getErrorType().orElse(null)); + return evaluation; + }); } catch (Exception e) { - return CompletableFuture.completedFuture(handleFlagEvaluationError(defaultValue).apply(e)); + final FlagEvaluation evaluation = handleFlagEvaluationError(defaultValue).apply(e); + client().trackEvaluation(evaluation.getReason(), evaluation.getErrorType().orElse(null)); + return CompletableFuture.completedFuture(evaluation); } } @@ -218,7 +238,8 @@ public void logResolveTesterHint(ResolvedFlag resolvedFlag) { Base64.getEncoder().encodeToString(jsonPrinter.print(resolveTesterLogging).getBytes()); final String logMessage = String.format( - "Check your flag evaluation for '%s' by copy pasting the payload to the Resolve tester '%s'", + "Check your flag evaluation for '%s' by copy pasting the payload to the Resolve" + + " tester '%s'", flag, base64); log.debug(logMessage); } catch (InvalidProtocolBufferException e) { @@ -236,11 +257,20 @@ static Confidence create( EventSenderEngine eventSenderEngine, FlagResolverClient flagResolverClient, String clientSecret) { + return create(eventSenderEngine, flagResolverClient, clientSecret, null); + } + + @VisibleForTesting + static Confidence create( + EventSenderEngine eventSenderEngine, + FlagResolverClient flagResolverClient, + String clientSecret, + @Nullable Telemetry telemetry) { final Closer closer = Closer.create(); closer.register(eventSenderEngine); closer.register(flagResolverClient); return new RootInstance( - new ClientDelegate(closer, flagResolverClient, eventSenderEngine, clientSecret)); + new ClientDelegate(closer, flagResolverClient, eventSenderEngine, clientSecret, telemetry)); } public static Confidence.Builder builder(String clientSecret) { @@ -252,16 +282,19 @@ static class ClientDelegate implements FlagResolverClient, EventSenderEngine { private final FlagResolverClient flagResolverClient; private final EventSenderEngine eventSenderEngine; private String clientSecret; + @Nullable private final Telemetry telemetry; ClientDelegate( Closeable closeable, FlagResolverClient flagResolverClient, EventSenderEngine eventSenderEngine, - String clientSecret) { + String clientSecret, + @Nullable Telemetry telemetry) { this.closeable = closeable; this.flagResolverClient = flagResolverClient; this.eventSenderEngine = eventSenderEngine; this.clientSecret = clientSecret; + this.telemetry = telemetry; } @Override @@ -281,6 +314,14 @@ public CompletableFuture resolveFlags( return flagResolverClient.resolveFlags(flag, context); } + void trackEvaluation(String resolveReason, @Nullable ErrorType errorType) { + if (telemetry != null) { + telemetry.appendEvaluation( + Telemetry.mapReason(resolveReason, errorType), + Telemetry.mapErrorCode(resolveReason, errorType)); + } + } + @Override public void close() throws IOException { closeable.close(); @@ -417,7 +458,8 @@ public Confidence build() { closer.register(flagResolverClient); closer.register(eventSenderEngine); return new RootInstance( - new ClientDelegate(closer, flagResolverClient, eventSenderEngine, clientSecret)); + new ClientDelegate( + closer, flagResolverClient, eventSenderEngine, clientSecret, telemetry)); } private void registerChannelForShutdown(ManagedChannel channel) { diff --git a/sdk-java/src/main/java/com/spotify/confidence/ConfidenceStub.java b/sdk-java/src/main/java/com/spotify/confidence/ConfidenceStub.java index b6969d39..322e7bb4 100644 --- a/sdk-java/src/main/java/com/spotify/confidence/ConfidenceStub.java +++ b/sdk-java/src/main/java/com/spotify/confidence/ConfidenceStub.java @@ -161,7 +161,7 @@ public List getCallHistory() { // Mock implementation of ClientDelegate private static class MockClientDelegate extends ClientDelegate { private MockClientDelegate() { - super(null, null, null, ""); + super(null, null, null, "", null); } @Override diff --git a/sdk-java/src/main/java/com/spotify/confidence/ErrorType.java b/sdk-java/src/main/java/com/spotify/confidence/ErrorType.java index e0e15da7..154db7d1 100644 --- a/sdk-java/src/main/java/com/spotify/confidence/ErrorType.java +++ b/sdk-java/src/main/java/com/spotify/confidence/ErrorType.java @@ -6,5 +6,7 @@ public enum ErrorType { INVALID_VALUE_PATH, INVALID_CONTEXT, INTERNAL_ERROR, - NETWORK_ERROR + NETWORK_ERROR, + PARSE_ERROR, + PROVIDER_NOT_READY } diff --git a/sdk-java/src/main/java/com/spotify/confidence/Telemetry.java b/sdk-java/src/main/java/com/spotify/confidence/Telemetry.java index 4c835693..c47425d4 100644 --- a/sdk-java/src/main/java/com/spotify/confidence/Telemetry.java +++ b/sdk-java/src/main/java/com/spotify/confidence/Telemetry.java @@ -5,10 +5,10 @@ import com.spotify.telemetry.v1.Monitoring; import com.spotify.telemetry.v1.Platform; import java.util.concurrent.ConcurrentLinkedQueue; +import javax.annotation.Nullable; public class Telemetry { - private final ConcurrentLinkedQueue latencyTraces = - new ConcurrentLinkedQueue<>(); + private final ConcurrentLinkedQueue traces = new ConcurrentLinkedQueue<>(); private final boolean isProvider; public Telemetry() { @@ -20,13 +20,94 @@ public Telemetry(boolean isProvider) { } public void appendLatency(long latency) { - latencyTraces.add( + traces.add( LibraryTraces.Trace.newBuilder() .setId(LibraryTraces.TraceId.TRACE_ID_RESOLVE_LATENCY) - .setMillisecondDuration(latency) + .setRequestTrace( + LibraryTraces.Trace.RequestTrace.newBuilder() + .setMillisecondDuration(latency) + .setStatus(LibraryTraces.Trace.RequestTrace.Status.STATUS_SUCCESS) + .build()) .build()); } + public void appendEvaluation( + LibraryTraces.Trace.EvaluationTrace.EvaluationReason reason, + LibraryTraces.Trace.EvaluationTrace.EvaluationErrorCode errorCode) { + traces.add( + LibraryTraces.Trace.newBuilder() + .setId(LibraryTraces.TraceId.TRACE_ID_FLAG_EVALUATION) + .setEvaluationTrace( + LibraryTraces.Trace.EvaluationTrace.newBuilder() + .setReason(reason) + .setErrorCode(errorCode) + .build()) + .build()); + } + + public static LibraryTraces.Trace.EvaluationTrace.EvaluationReason mapReason( + String resolveReason, @Nullable ErrorType errorType) { + if (errorType != null) { + return LibraryTraces.Trace.EvaluationTrace.EvaluationReason.EVALUATION_REASON_ERROR; + } + switch (resolveReason) { + case "RESOLVE_REASON_MATCH": + return LibraryTraces.Trace.EvaluationTrace.EvaluationReason + .EVALUATION_REASON_TARGETING_MATCH; + case "RESOLVE_REASON_NO_SEGMENT_MATCH": + case "RESOLVE_REASON_NO_TREATMENT_MATCH": + return LibraryTraces.Trace.EvaluationTrace.EvaluationReason.EVALUATION_REASON_DEFAULT; + case "RESOLVE_REASON_FLAG_ARCHIVED": + return LibraryTraces.Trace.EvaluationTrace.EvaluationReason.EVALUATION_REASON_DISABLED; + case "RESOLVE_REASON_TARGETING_KEY_ERROR": + case "RESOLVE_REASON_ERROR": + case "RESOLVE_REASON_UNRECOGNIZED_TARGETING_RULE": + case "RESOLVE_REASON_MATERIALIZATION_NOT_SUPPORTED": + return LibraryTraces.Trace.EvaluationTrace.EvaluationReason.EVALUATION_REASON_ERROR; + default: + return LibraryTraces.Trace.EvaluationTrace.EvaluationReason.EVALUATION_REASON_UNSPECIFIED; + } + } + + public static LibraryTraces.Trace.EvaluationTrace.EvaluationErrorCode mapErrorCode( + String resolveReason, @Nullable ErrorType errorType) { + if (errorType != null) { + switch (errorType) { + case FLAG_NOT_FOUND: + return LibraryTraces.Trace.EvaluationTrace.EvaluationErrorCode + .EVALUATION_ERROR_CODE_FLAG_NOT_FOUND; + case INVALID_VALUE_TYPE: + return LibraryTraces.Trace.EvaluationTrace.EvaluationErrorCode + .EVALUATION_ERROR_CODE_TYPE_MISMATCH; + case INVALID_CONTEXT: + return LibraryTraces.Trace.EvaluationTrace.EvaluationErrorCode + .EVALUATION_ERROR_CODE_INVALID_CONTEXT; + case PARSE_ERROR: + return LibraryTraces.Trace.EvaluationTrace.EvaluationErrorCode + .EVALUATION_ERROR_CODE_PARSE_ERROR; + case PROVIDER_NOT_READY: + return LibraryTraces.Trace.EvaluationTrace.EvaluationErrorCode + .EVALUATION_ERROR_CODE_PROVIDER_NOT_READY; + default: + return LibraryTraces.Trace.EvaluationTrace.EvaluationErrorCode + .EVALUATION_ERROR_CODE_GENERAL; + } + } + switch (resolveReason) { + case "RESOLVE_REASON_TARGETING_KEY_ERROR": + return LibraryTraces.Trace.EvaluationTrace.EvaluationErrorCode + .EVALUATION_ERROR_CODE_TARGETING_KEY_MISSING; + case "RESOLVE_REASON_ERROR": + case "RESOLVE_REASON_UNRECOGNIZED_TARGETING_RULE": + case "RESOLVE_REASON_MATERIALIZATION_NOT_SUPPORTED": + return LibraryTraces.Trace.EvaluationTrace.EvaluationErrorCode + .EVALUATION_ERROR_CODE_GENERAL; + default: + return LibraryTraces.Trace.EvaluationTrace.EvaluationErrorCode + .EVALUATION_ERROR_CODE_UNSPECIFIED; + } + } + public Monitoring getSnapshot() { final Monitoring snapshot = getSnapshotInternal(); clear(); @@ -42,7 +123,7 @@ public Monitoring getSnapshotInternal() { ? LibraryTraces.Library.LIBRARY_OPEN_FEATURE : LibraryTraces.Library.LIBRARY_CONFIDENCE) .setLibraryVersion(ConfidenceUtils.getSdkVersion()) - .addAllTraces(latencyTraces) + .addAllTraces(traces) .build(); return Monitoring.newBuilder() @@ -52,7 +133,7 @@ public Monitoring getSnapshotInternal() { } private void clear() { - latencyTraces.clear(); + traces.clear(); } public boolean isProvider() { diff --git a/sdk-java/src/test/java/com/spotify/confidence/ConfidenceIntegrationTest.java b/sdk-java/src/test/java/com/spotify/confidence/ConfidenceIntegrationTest.java index 32010273..701320bb 100644 --- a/sdk-java/src/test/java/com/spotify/confidence/ConfidenceIntegrationTest.java +++ b/sdk-java/src/test/java/com/spotify/confidence/ConfidenceIntegrationTest.java @@ -79,7 +79,7 @@ void beforeEach() { final FlagResolverClientImpl flagResolver = new FlagResolverClientImpl( new GrpcFlagResolver("fake-secret", channel, telemetryInterceptor, 1_000), telemetry); - confidence = Confidence.create(fakeEventSender, flagResolver, ""); + confidence = Confidence.create(fakeEventSender, flagResolver, "", telemetry); } @AfterAll @@ -454,10 +454,21 @@ public void resolvesContainHeaderWithTelemetryData() { assertThat(libraryTracesList).hasSize(1); final LibraryTraces traces = libraryTracesList.get(0); assertThat(traces.getLibrary()).isEqualTo(LibraryTraces.Library.LIBRARY_CONFIDENCE); - assertThat(traces.getTracesList()).hasSize(1); - final LibraryTraces.Trace trace = traces.getTraces(0); - assertThat(trace.getId()).isEqualTo(LibraryTraces.TraceId.TRACE_ID_RESOLVE_LATENCY); - assertThat(trace.getMillisecondDuration()).isNotNegative(); + assertThat(traces.getTracesList()).hasSize(2); + final LibraryTraces.Trace latencyTrace = traces.getTraces(0); + assertThat(latencyTrace.getId()).isEqualTo(LibraryTraces.TraceId.TRACE_ID_RESOLVE_LATENCY); + assertThat(latencyTrace.getRequestTrace().getMillisecondDuration()).isNotNegative(); + assertThat(latencyTrace.getRequestTrace().getStatus()) + .isEqualTo(LibraryTraces.Trace.RequestTrace.Status.STATUS_SUCCESS); + final LibraryTraces.Trace evaluationTrace = traces.getTraces(1); + assertThat(evaluationTrace.getId()).isEqualTo(LibraryTraces.TraceId.TRACE_ID_FLAG_EVALUATION); + assertThat(evaluationTrace.getEvaluationTrace().getReason()) + .isEqualTo( + LibraryTraces.Trace.EvaluationTrace.EvaluationReason.EVALUATION_REASON_TARGETING_MATCH); + assertThat(evaluationTrace.getEvaluationTrace().getErrorCode()) + .isEqualTo( + LibraryTraces.Trace.EvaluationTrace.EvaluationErrorCode + .EVALUATION_ERROR_CODE_UNSPECIFIED); confidence.withContext(SAMPLE_CONTEXT).getEvaluation("flag.prop-Y", 1000);