From fc920ed51592ccc7da477591212d58582dfa554b Mon Sep 17 00:00:00 2001 From: cleverchuk Date: Thu, 19 Feb 2026 16:25:13 -0500 Subject: [PATCH 1/2] migrated joboe --- agent/build.gradle.kts | 5 +++++ custom/build.gradle.kts | 8 +++++++- .../extensions/config/HttpSettingsReaderDelegate.java | 4 ++-- .../config/parser/json/ProfilerSettingParser.java | 6 +++--- dependencyManagement/build.gradle.kts | 2 +- testing/agent-for-testing/build.gradle.kts | 5 +++++ 6 files changed, 23 insertions(+), 7 deletions(-) diff --git a/agent/build.gradle.kts b/agent/build.gradle.kts index 6da42bd4..66ce2863 100644 --- a/agent/build.gradle.kts +++ b/agent/build.gradle.kts @@ -50,6 +50,11 @@ dependencies { bootstrapLibs("com.solarwinds.joboe:core") bootstrapLibs("com.solarwinds.joboe:metrics") + bootstrapLibs("com.solarwinds.joboe:config") + bootstrapLibs("com.solarwinds.joboe:sampling") + bootstrapLibs("com.solarwinds.joboe:logging") + + bootstrapLibs("org.json:json") upstreamAgent("io.opentelemetry.javaagent:opentelemetry-javaagent") } diff --git a/custom/build.gradle.kts b/custom/build.gradle.kts index 3866a49e..7c3bbe64 100644 --- a/custom/build.gradle.kts +++ b/custom/build.gradle.kts @@ -23,6 +23,10 @@ dependencies { compileOnly(project(":libs:shared")) compileOnly("com.solarwinds.joboe:core") + compileOnly("com.solarwinds.joboe:config") + compileOnly("com.solarwinds.joboe:sampling") + compileOnly("com.solarwinds.joboe:logging") + compileOnly("org.projectlombok:lombok") compileOnly("com.solarwinds.joboe:metrics") annotationProcessor("org.projectlombok:lombok") @@ -43,8 +47,10 @@ dependencies { compileOnly("io.opentelemetry:opentelemetry-sdk-extension-incubator") compileOnly("io.opentelemetry:opentelemetry-exporter-otlp") + compileOnly("com.google.code.gson:gson") + implementation("org.json:json") + testImplementation(project(":libs:shared")) - testImplementation("org.json:json") testImplementation("com.solarwinds.joboe:core") testImplementation("io.opentelemetry:opentelemetry-api-incubator") diff --git a/custom/src/main/java/com/solarwinds/opentelemetry/extensions/config/HttpSettingsReaderDelegate.java b/custom/src/main/java/com/solarwinds/opentelemetry/extensions/config/HttpSettingsReaderDelegate.java index e00724ee..c189d9e1 100644 --- a/custom/src/main/java/com/solarwinds/opentelemetry/extensions/config/HttpSettingsReaderDelegate.java +++ b/custom/src/main/java/com/solarwinds/opentelemetry/extensions/config/HttpSettingsReaderDelegate.java @@ -16,14 +16,14 @@ package com.solarwinds.opentelemetry.extensions.config; +import com.google.gson.Gson; +import com.google.gson.GsonBuilder; import com.solarwinds.joboe.config.ConfigManager; import com.solarwinds.joboe.config.ConfigProperty; import com.solarwinds.joboe.config.ProxyConfig; import com.solarwinds.joboe.logging.Logger; import com.solarwinds.joboe.logging.LoggerFactory; import com.solarwinds.joboe.sampling.Settings; -import com.solarwinds.joboe.shaded.google.gson.Gson; -import com.solarwinds.joboe.shaded.google.gson.GsonBuilder; import io.opentelemetry.api.internal.InstrumentationUtil; import java.io.BufferedReader; import java.io.IOException; diff --git a/custom/src/main/java/com/solarwinds/opentelemetry/extensions/config/parser/json/ProfilerSettingParser.java b/custom/src/main/java/com/solarwinds/opentelemetry/extensions/config/parser/json/ProfilerSettingParser.java index 036c47fa..20fcd544 100644 --- a/custom/src/main/java/com/solarwinds/opentelemetry/extensions/config/parser/json/ProfilerSettingParser.java +++ b/custom/src/main/java/com/solarwinds/opentelemetry/extensions/config/parser/json/ProfilerSettingParser.java @@ -21,11 +21,11 @@ import com.solarwinds.joboe.core.profiler.ProfilerSetting; import com.solarwinds.joboe.logging.Logger; import com.solarwinds.joboe.logging.LoggerFactory; -import com.solarwinds.joboe.shaded.org.json.JSONArray; -import com.solarwinds.joboe.shaded.org.json.JSONException; -import com.solarwinds.joboe.shaded.org.json.JSONObject; import java.util.HashSet; import java.util.Set; +import org.json.JSONArray; +import org.json.JSONException; +import org.json.JSONObject; public class ProfilerSettingParser implements ConfigParser { private static final Logger logger = LoggerFactory.getLogger(); diff --git a/dependencyManagement/build.gradle.kts b/dependencyManagement/build.gradle.kts index c9592f3f..0880c06b 100644 --- a/dependencyManagement/build.gradle.kts +++ b/dependencyManagement/build.gradle.kts @@ -7,7 +7,7 @@ val otelSdkVersion = "1.59.0" val mockitoVersion = "5.2.0" val byteBuddyVersion = "1.18.4" -val joboeVersion = "10.0.27" +val joboeVersion = "11.0.0" val opentelemetryJavaagentAlpha = "$otelAgentVersion-alpha" val opentelemetryAlpha = "$otelSdkVersion-alpha" diff --git a/testing/agent-for-testing/build.gradle.kts b/testing/agent-for-testing/build.gradle.kts index 2c88ba43..adae5eab 100644 --- a/testing/agent-for-testing/build.gradle.kts +++ b/testing/agent-for-testing/build.gradle.kts @@ -48,6 +48,11 @@ dependencies { bootstrapLibs("com.solarwinds.joboe:core") bootstrapLibs("com.solarwinds.joboe:metrics") + bootstrapLibs("com.solarwinds.joboe:config") + bootstrapLibs("com.solarwinds.joboe:sampling") + bootstrapLibs("com.solarwinds.joboe:logging") + + bootstrapLibs("org.json:json") upstreamAgent("io.opentelemetry.javaagent:opentelemetry-agent-for-testing") } From b2c82556cfc855c5f86d3656cab2d8943a030499 Mon Sep 17 00:00:00 2001 From: cleverchuk Date: Thu, 5 Mar 2026 11:40:55 -0500 Subject: [PATCH 2/2] integrate joboe --- .github/scripts/shading-check.sh | 24 + .github/workflows/push.yml | 23 +- agent-lambda/build.gradle.kts | 6 +- agent/build.gradle.kts | 9 +- bootstrap/build.gradle.kts | 6 +- build.gradle.kts | 2 +- buildSrc/build.gradle.kts | 2 + ...nds.instrumentation-conventions.gradle.kts | 1 - .../solarwinds.java-conventions.gradle.kts | 15 +- .../solarwinds.shadow-conventions.gradle.kts | 18 + custom/build.gradle.kts | 21 +- .../extensions/SolarwindsAgentListener.java | 3 - .../SolarwindsProfilingSpanProcessor.java | 2 +- .../config/HttpSettingsReaderDelegate.java | 9 +- .../livereload/ConfigurationFileWatcher.java | 15 +- ...toConfigurationCustomizerProviderImpl.java | 2 +- custom/src/main/resources/ao-collector.crt | 24 - .../SolarwindsProfilingSpanProcessorTest.java | 4 +- .../HttpSettingsReaderDelegateTest.java | 7 +- .../ConfigurationFileWatcherTest.java | 2 +- dependencyManagement/build.gradle.kts | 14 +- .../hibernate-6.0/javaagent/build.gradle.kts | 2 +- .../instrumentation-shared/build.gradle.kts | 4 +- .../jdbc/javaagent/build.gradle.kts | 5 +- libs/config/.gitignore | 1 + libs/config/build.gradle.kts | 11 + .../joboe/config/ConfigContainer.java | 276 ++++ .../solarwinds/joboe/config/ConfigGroup.java | 28 + .../joboe/config/ConfigManager.java | 93 ++ .../solarwinds/joboe/config/ConfigParser.java | 33 + .../joboe/config/ConfigProperty.java | 276 ++++ .../solarwinds/joboe/config/ConfigReader.java | 38 + .../joboe/config/ConfigSourceType.java | 22 + .../joboe/config/EnvConfigReader.java | 77 + .../joboe/config/InvalidConfigException.java | 59 + .../InvalidConfigReadSourceException.java | 91 ++ .../InvalidConfigServiceKeyException.java | 35 + .../config/JavaRuntimeVersionChecker.java | 29 + .../joboe/config/JavaVersionComparator.java | 57 + .../joboe/config/JsonConfigReader.java | 95 ++ .../joboe/config/LogTraceIdScope.java | 23 + .../joboe/config/LogTraceIdSetting.java | 37 + .../solarwinds/joboe/config/ProxyConfig.java | 34 + .../joboe/config/ServiceKeyUtils.java | 76 + .../joboe/config/ConfigPropertyTest.java | 49 + .../joboe/config/EnvConfigReaderTest.java | 55 + .../config/JavaRuntimeVersionCheckerTest.java | 47 + .../joboe/config/JsonConfigReaderTest.java | 53 + libs/config/src/test/resources/invalid.json | 8 + libs/config/src/test/resources/valid.json | 5 + libs/core/.gitignore | 1 + libs/core/build.gradle.kts | 58 + .../joboe/core/AtomicEventReporterStats.java | 81 + .../joboe/core/BsonBufferException.java | 23 + .../com/solarwinds/joboe/core/Constants.java | 49 + .../com/solarwinds/joboe/core/Context.java | 226 +++ .../java/com/solarwinds/joboe/core/Event.java | 97 ++ .../com/solarwinds/joboe/core/EventImpl.java | 687 ++++++++ .../solarwinds/joboe/core/EventReporter.java | 26 + .../joboe/core/EventReporterException.java | 28 + .../core/EventReporterQueueFullException.java | 23 + .../joboe/core/EventReporterStats.java | 30 + .../joboe/core/EventValueConverter.java | 325 ++++ .../com/solarwinds/joboe/core/HostId.java | 157 ++ .../core/NonThreadLocalTestReporter.java | 62 + .../com/solarwinds/joboe/core/NoopEvent.java | 67 + .../solarwinds/joboe/core/OboeException.java | 32 + .../joboe/core/QueuingEventReporter.java | 273 ++++ .../joboe/core/ReporterFactory.java | 81 + .../solarwinds/joboe/core/TestReporter.java | 177 +++ .../com/solarwinds/joboe/core/TestingEnv.java | 37 + .../solarwinds/joboe/core/UDPReporter.java | 112 ++ .../solarwinds/joboe/core/XTraceHeader.java | 31 + .../joboe/core/ebson/BasicObjectId.java | 106 ++ .../joboe/core/ebson/BasicTimestamp.java | 85 + .../joboe/core/ebson/BsonBinary.java | 172 ++ .../joboe/core/ebson/BsonBytes.java | 123 ++ .../joboe/core/ebson/BsonDocument.java | 250 +++ .../joboe/core/ebson/BsonDocuments.java | 344 ++++ .../joboe/core/ebson/BsonObject.java | 276 ++++ .../joboe/core/ebson/BsonObjectId.java | 33 + .../joboe/core/ebson/BsonReader.java | 40 + .../joboe/core/ebson/BsonTimestamp.java | 29 + .../joboe/core/ebson/BsonToken.java | 84 + .../joboe/core/ebson/BsonWriter.java | 39 + .../joboe/core/ebson/DefaultDocument.java | 68 + .../core/ebson/DefaultDocumentBuilder.java | 75 + .../joboe/core/ebson/DefaultPredicate.java | 142 ++ .../joboe/core/ebson/DefaultReader.java | 230 +++ .../joboe/core/ebson/DefaultWriter.java | 223 +++ .../joboe/core/ebson/MultiValList.java | 34 + .../joboe/core/ebson/package-info.java | 5 + .../joboe/core/profiler/Profiler.java | 765 +++++++++ .../joboe/core/profiler/ProfilerSetting.java | 81 + .../com/solarwinds/joboe/core/rpc/Client.java | 60 + .../joboe/core/rpc/ClientException.java | 31 + .../joboe/core/rpc/ClientFatalException.java | 37 + .../joboe/core/rpc/ClientLoggingCallback.java | 177 +++ .../joboe/core/rpc/ClientManagerProvider.java | 47 + .../core/rpc/ClientRecoverableException.java | 38 + .../rpc/ClientRejectedExecutionException.java | 31 + .../joboe/core/rpc/EncodingType.java | 21 + .../joboe/core/rpc/HeartbeatScheduler.java | 22 + .../solarwinds/joboe/core/rpc/HostType.java | 22 + .../joboe/core/rpc/KeepAliveMonitor.java | 60 + .../joboe/core/rpc/ProtocolClient.java | 51 + .../joboe/core/rpc/ProtocolClientFactory.java | 26 + .../com/solarwinds/joboe/core/rpc/Result.java | 33 + .../solarwinds/joboe/core/rpc/ResultCode.java | 29 + .../solarwinds/joboe/core/rpc/RpcClient.java | 917 +++++++++++ .../joboe/core/rpc/RpcClientManager.java | 134 ++ .../RpcClientRejectedExecutionException.java | 35 + .../joboe/core/rpc/RpcSettings.java | 151 ++ .../joboe/core/rpc/SettingsResult.java | 32 + .../joboe/core/rpc/grpc/GrpcClient.java | 444 ++++++ .../core/rpc/grpc/GrpcClientManager.java | 93 ++ .../core/settings/OboeSettingsException.java | 34 + .../core/settings/PollingSettingsFetcher.java | 156 ++ .../core/settings/RpcSettingsReader.java | 73 + .../joboe/core/settings/SettingsReader.java | 30 + .../joboe/core/settings/SettingsUtil.java | 80 + .../core/settings/SimpleSettingsFetcher.java | 78 + .../core/settings/TestSettingsReader.java | 218 +++ .../core/util/AzureInstanceIdReader.java | 21 + .../joboe/core/util/BackTraceCache.java | 57 + .../joboe/core/util/BackTraceUtil.java | 101 ++ .../solarwinds/joboe/core/util/BsonUtils.java | 71 + .../joboe/core/util/DaemonThreadFactory.java | 58 + .../joboe/core/util/DummyHostInfoReader.java | 32 + .../solarwinds/joboe/core/util/ExecUtils.java | 117 ++ .../core/util/HeartbeatSchedulerProvider.java | 29 + .../joboe/core/util/HostInfoReader.java | 23 + .../core/util/HostInfoReaderProvider.java | 25 + .../joboe/core/util/HostInfoUtils.java | 130 ++ .../joboe/core/util/HostMetadataReader.java | 23 + .../joboe/core/util/HostNameReader.java | 21 + .../solarwinds/joboe/core/util/HttpUtils.java | 46 + .../joboe/core/util/JavaProcessUtils.java | 55 + .../core/util/NetworkAddressInfoReader.java | 21 + .../util/RuntimeHostInfoReaderProvider.java | 27 + .../joboe/core/util/ServerHostInfoReader.java | 1290 +++++++++++++++ .../solarwinds/joboe/core/util/SslUtils.java | 194 +++ .../solarwinds/joboe/core/util/TestUtils.java | 116 ++ .../solarwinds/joboe/core/util/TimeUtils.java | 273 ++++ .../joboe/core/util/UamsClientIdReader.java | 149 ++ .../core/util/diagnostic/DiagnosticTools.java | 445 ++++++ .../diagnostic/InvalidArgumentsException.java | 23 + .../core/AtomicEventReporterStatsTest.java | 44 + .../solarwinds/joboe/core/ContextTest.java | 366 +++++ .../solarwinds/joboe/core/EventImplTest.java | 562 +++++++ .../joboe/core/EventValueConverterTest.java | 178 +++ .../joboe/core/QueuingEventReporterTest.java | 52 + .../joboe/core/ReporterFactoryTest.java | 62 + .../core/TestExecutionExceptionRpcClient.java | 69 + .../joboe/core/TestReporterTest.java | 85 + .../solarwinds/joboe/core/TestRpcClient.java | 106 ++ .../core/TestSubmitRejectionRpcClient.java | 65 + .../core/profiler/CircuitBreakerTest.java | 130 ++ .../joboe/core/profiler/ProfileTest.java | 242 +++ .../com/solarwinds/joboe/core/rpc/README.md | 26 + .../joboe/core/rpc/RpcClientManagerTest.java | 90 ++ .../joboe/core/rpc/RpcClientTest.java | 1085 +++++++++++++ .../joboe/core/rpc/grpc/GrpcClientTest.java | 583 +++++++ .../core/rpc/grpc/test-collector-private.pem | 28 + .../joboe/core/rpc/invalid-collector.crt | 32 + .../joboe/core/rpc/test-collector-public.pem | 21 + .../settings/PollingSettingsFetcherTest.java | 413 +++++ .../joboe/core/settings/SettingsUtilTest.java | 94 ++ .../joboe/core/util/DockerInfoReaderTest.java | 83 + .../util/HeartbeatSchedulerProviderTest.java | 40 + .../core/util/JavaVersionComparatorTest.java | 37 + .../joboe/core/util/K8sReaderTest.java | 51 + .../RuntimeHostInfoReaderProviderTest.java | 30 + .../core/util/ServerHostInfoReaderTest.java | 107 ++ .../solarwinds/joboe/core/util/SuSE-release | 3 + .../joboe/core/util/TimeUtilsTest.java | 94 ++ .../solarwinds/joboe/core/util/debian_version | 1 + .../joboe/core/util/docker-cgroup-ce | 13 + .../core/util/docker-cgroup-cri-containerd | 11 + .../joboe/core/util/docker-cgroup-ecs | 13 + .../joboe/core/util/docker-cgroup-empty | 0 .../joboe/core/util/docker-cgroup-invalid | 1 + .../joboe/core/util/docker-cgroup-kubepods | 11 + .../joboe/core/util/docker-cgroup-non-docker | 10 + .../joboe/core/util/docker-cgroup-standard | 11 + .../joboe/core/util/docker-cgroup-standard-2 | 11 + .../solarwinds/joboe/core/util/gentoo-release | 1 + .../solarwinds/joboe/core/util/lsb-release | 4 + .../com/solarwinds/joboe/core/util/namespace | 1 + .../com/solarwinds/joboe/core/util/poduid | 40 + .../solarwinds/joboe/core/util/redhat-release | 1 + .../joboe/core/util/slackware-version | 1 + .../joboe/core/util/system-release-cpe | 1 + .../resources/solarwinds-apm-settings-raw | Bin 0 -> 377 bytes libs/lambda/build.gradle.kts | 15 +- .../extensions/FileSettingsReader.java | 4 +- libs/logging/.gitignore | 1 + libs/logging/build.gradle.kts | 5 + .../joboe/logging/CompositeStream.java | 43 + .../joboe/logging/FileLoggerStream.java | 371 +++++ .../solarwinds/joboe/logging/LogSetting.java | 88 ++ .../com/solarwinds/joboe/logging/Logger.java | 289 ++++ .../joboe/logging/LoggerConfiguration.java | 32 + .../joboe/logging/LoggerFactory.java | 42 + .../joboe/logging/LoggerStream.java | 23 + .../joboe/logging/LoggerThreadFactory.java | 58 + .../joboe/logging/SystemErrStream.java | 33 + .../joboe/logging/SystemOutStream.java | 33 + .../joboe/logging/FileLoggerStreamTest.java | 281 ++++ .../solarwinds/joboe/logging/LoggerTest.java | 313 ++++ .../joboe/logging/TestLoggerProcess.java | 47 + libs/sampling/.gitignore | 1 + libs/sampling/build.gradle.kts | 13 + .../joboe/sampling/BinaryUtils.java | 71 + .../solarwinds/joboe/sampling/Constants.java | 49 + .../sampling/DevURandomSeedGenerator.java | 68 + .../solarwinds/joboe/sampling/HexUtils.java | 53 + .../solarwinds/joboe/sampling/Metadata.java | 636 ++++++++ .../joboe/sampling/ResourceMatcher.java | 21 + .../joboe/sampling/SampleRateSource.java | 42 + .../joboe/sampling/SamplingConfiguration.java | 38 + .../joboe/sampling/SamplingException.java | 25 + .../sampling/SecureRandomSeedGenerator.java | 47 + .../joboe/sampling/SeedException.java | 42 + .../joboe/sampling/SeedGenerator.java | 33 + .../solarwinds/joboe/sampling/Settings.java | 95 ++ .../joboe/sampling/SettingsArg.java | 341 ++++ .../sampling/SettingsArgChangeListener.java | 57 + .../joboe/sampling/SettingsFetcher.java | 48 + .../joboe/sampling/SettingsListener.java | 26 + .../joboe/sampling/SettingsManager.java | 155 ++ .../joboe/sampling/TokenBucket.java | 116 ++ .../joboe/sampling/TokenBucketType.java | 32 + .../joboe/sampling/TraceConfig.java | 130 ++ .../joboe/sampling/TraceConfigs.java | 71 + .../joboe/sampling/TraceDecision.java | 75 + .../joboe/sampling/TraceDecisionUtil.java | 530 +++++++ .../joboe/sampling/TracingMode.java | 65 + .../joboe/sampling/XTraceOptionsResponse.java | 98 ++ .../joboe/sampling/XorShiftRNG.java | 98 ++ .../joboe/sampling/XtraceOption.java | 159 ++ .../joboe/sampling/XtraceOptions.java | 462 ++++++ .../joboe/sampling/MetadataTest.java | 204 +++ .../joboe/sampling/SettingsArgTest.java | 109 ++ .../joboe/sampling/SettingsManagerTest.java | 169 ++ .../joboe/sampling/SettingsStub.java | 165 ++ .../joboe/sampling/TokenBucketTest.java | 135 ++ .../joboe/sampling/TraceDecisionUtilTest.java | 1403 +++++++++++++++++ .../joboe/sampling/XtraceOptionTest.java | 32 + .../sampling/XtraceOptionsResponseTest.java | 225 +++ .../joboe/sampling/XtraceOptionsTest.java | 385 +++++ .../src/test/resources/hmac-signature.txt | 1 + libs/shared/build.gradle.kts | 17 +- .../extensions/SamplingUtil.java | 8 +- .../opentelemetry/extensions/SharedNames.java | 6 +- .../SolarwindsContextPropagator.java | 6 +- .../extensions/SolarwindsSampler.java | 6 +- .../extensions/TransactionNameManager.java | 6 +- .../extensions/TriggerTraceContextKey.java | 4 +- .../config/parser/json/LogSettingParser.java | 7 - .../extensions/SamplingUtilTest.java | 12 +- .../SolarwindsContextPropagatorTest.java | 9 +- .../extensions/SolarwindsSamplerTest.java | 10 +- long-running-test-arch/xk6/go.sum | 45 + settings.gradle.kts | 4 + spotbugs-exclude.xml | 65 + testing/agent-for-testing/build.gradle.kts | 9 +- 267 files changed, 27381 insertions(+), 167 deletions(-) create mode 100755 .github/scripts/shading-check.sh delete mode 100644 custom/src/main/resources/ao-collector.crt create mode 100644 libs/config/.gitignore create mode 100644 libs/config/build.gradle.kts create mode 100644 libs/config/src/main/java/com/solarwinds/joboe/config/ConfigContainer.java create mode 100644 libs/config/src/main/java/com/solarwinds/joboe/config/ConfigGroup.java create mode 100644 libs/config/src/main/java/com/solarwinds/joboe/config/ConfigManager.java create mode 100644 libs/config/src/main/java/com/solarwinds/joboe/config/ConfigParser.java create mode 100644 libs/config/src/main/java/com/solarwinds/joboe/config/ConfigProperty.java create mode 100644 libs/config/src/main/java/com/solarwinds/joboe/config/ConfigReader.java create mode 100644 libs/config/src/main/java/com/solarwinds/joboe/config/ConfigSourceType.java create mode 100644 libs/config/src/main/java/com/solarwinds/joboe/config/EnvConfigReader.java create mode 100644 libs/config/src/main/java/com/solarwinds/joboe/config/InvalidConfigException.java create mode 100644 libs/config/src/main/java/com/solarwinds/joboe/config/InvalidConfigReadSourceException.java create mode 100644 libs/config/src/main/java/com/solarwinds/joboe/config/InvalidConfigServiceKeyException.java create mode 100644 libs/config/src/main/java/com/solarwinds/joboe/config/JavaRuntimeVersionChecker.java create mode 100644 libs/config/src/main/java/com/solarwinds/joboe/config/JavaVersionComparator.java create mode 100644 libs/config/src/main/java/com/solarwinds/joboe/config/JsonConfigReader.java create mode 100644 libs/config/src/main/java/com/solarwinds/joboe/config/LogTraceIdScope.java create mode 100644 libs/config/src/main/java/com/solarwinds/joboe/config/LogTraceIdSetting.java create mode 100644 libs/config/src/main/java/com/solarwinds/joboe/config/ProxyConfig.java create mode 100644 libs/config/src/main/java/com/solarwinds/joboe/config/ServiceKeyUtils.java create mode 100644 libs/config/src/test/java/com/solarwinds/joboe/config/ConfigPropertyTest.java create mode 100644 libs/config/src/test/java/com/solarwinds/joboe/config/EnvConfigReaderTest.java create mode 100644 libs/config/src/test/java/com/solarwinds/joboe/config/JavaRuntimeVersionCheckerTest.java create mode 100644 libs/config/src/test/java/com/solarwinds/joboe/config/JsonConfigReaderTest.java create mode 100644 libs/config/src/test/resources/invalid.json create mode 100644 libs/config/src/test/resources/valid.json create mode 100644 libs/core/.gitignore create mode 100644 libs/core/build.gradle.kts create mode 100644 libs/core/src/main/java/com/solarwinds/joboe/core/AtomicEventReporterStats.java create mode 100644 libs/core/src/main/java/com/solarwinds/joboe/core/BsonBufferException.java create mode 100644 libs/core/src/main/java/com/solarwinds/joboe/core/Constants.java create mode 100644 libs/core/src/main/java/com/solarwinds/joboe/core/Context.java create mode 100644 libs/core/src/main/java/com/solarwinds/joboe/core/Event.java create mode 100644 libs/core/src/main/java/com/solarwinds/joboe/core/EventImpl.java create mode 100644 libs/core/src/main/java/com/solarwinds/joboe/core/EventReporter.java create mode 100644 libs/core/src/main/java/com/solarwinds/joboe/core/EventReporterException.java create mode 100644 libs/core/src/main/java/com/solarwinds/joboe/core/EventReporterQueueFullException.java create mode 100644 libs/core/src/main/java/com/solarwinds/joboe/core/EventReporterStats.java create mode 100644 libs/core/src/main/java/com/solarwinds/joboe/core/EventValueConverter.java create mode 100644 libs/core/src/main/java/com/solarwinds/joboe/core/HostId.java create mode 100644 libs/core/src/main/java/com/solarwinds/joboe/core/NonThreadLocalTestReporter.java create mode 100644 libs/core/src/main/java/com/solarwinds/joboe/core/NoopEvent.java create mode 100644 libs/core/src/main/java/com/solarwinds/joboe/core/OboeException.java create mode 100644 libs/core/src/main/java/com/solarwinds/joboe/core/QueuingEventReporter.java create mode 100644 libs/core/src/main/java/com/solarwinds/joboe/core/ReporterFactory.java create mode 100644 libs/core/src/main/java/com/solarwinds/joboe/core/TestReporter.java create mode 100644 libs/core/src/main/java/com/solarwinds/joboe/core/TestingEnv.java create mode 100644 libs/core/src/main/java/com/solarwinds/joboe/core/UDPReporter.java create mode 100644 libs/core/src/main/java/com/solarwinds/joboe/core/XTraceHeader.java create mode 100644 libs/core/src/main/java/com/solarwinds/joboe/core/ebson/BasicObjectId.java create mode 100644 libs/core/src/main/java/com/solarwinds/joboe/core/ebson/BasicTimestamp.java create mode 100644 libs/core/src/main/java/com/solarwinds/joboe/core/ebson/BsonBinary.java create mode 100644 libs/core/src/main/java/com/solarwinds/joboe/core/ebson/BsonBytes.java create mode 100644 libs/core/src/main/java/com/solarwinds/joboe/core/ebson/BsonDocument.java create mode 100644 libs/core/src/main/java/com/solarwinds/joboe/core/ebson/BsonDocuments.java create mode 100644 libs/core/src/main/java/com/solarwinds/joboe/core/ebson/BsonObject.java create mode 100644 libs/core/src/main/java/com/solarwinds/joboe/core/ebson/BsonObjectId.java create mode 100644 libs/core/src/main/java/com/solarwinds/joboe/core/ebson/BsonReader.java create mode 100644 libs/core/src/main/java/com/solarwinds/joboe/core/ebson/BsonTimestamp.java create mode 100644 libs/core/src/main/java/com/solarwinds/joboe/core/ebson/BsonToken.java create mode 100644 libs/core/src/main/java/com/solarwinds/joboe/core/ebson/BsonWriter.java create mode 100644 libs/core/src/main/java/com/solarwinds/joboe/core/ebson/DefaultDocument.java create mode 100644 libs/core/src/main/java/com/solarwinds/joboe/core/ebson/DefaultDocumentBuilder.java create mode 100644 libs/core/src/main/java/com/solarwinds/joboe/core/ebson/DefaultPredicate.java create mode 100644 libs/core/src/main/java/com/solarwinds/joboe/core/ebson/DefaultReader.java create mode 100644 libs/core/src/main/java/com/solarwinds/joboe/core/ebson/DefaultWriter.java create mode 100644 libs/core/src/main/java/com/solarwinds/joboe/core/ebson/MultiValList.java create mode 100644 libs/core/src/main/java/com/solarwinds/joboe/core/ebson/package-info.java create mode 100644 libs/core/src/main/java/com/solarwinds/joboe/core/profiler/Profiler.java create mode 100644 libs/core/src/main/java/com/solarwinds/joboe/core/profiler/ProfilerSetting.java create mode 100644 libs/core/src/main/java/com/solarwinds/joboe/core/rpc/Client.java create mode 100644 libs/core/src/main/java/com/solarwinds/joboe/core/rpc/ClientException.java create mode 100644 libs/core/src/main/java/com/solarwinds/joboe/core/rpc/ClientFatalException.java create mode 100644 libs/core/src/main/java/com/solarwinds/joboe/core/rpc/ClientLoggingCallback.java create mode 100644 libs/core/src/main/java/com/solarwinds/joboe/core/rpc/ClientManagerProvider.java create mode 100644 libs/core/src/main/java/com/solarwinds/joboe/core/rpc/ClientRecoverableException.java create mode 100644 libs/core/src/main/java/com/solarwinds/joboe/core/rpc/ClientRejectedExecutionException.java create mode 100644 libs/core/src/main/java/com/solarwinds/joboe/core/rpc/EncodingType.java create mode 100644 libs/core/src/main/java/com/solarwinds/joboe/core/rpc/HeartbeatScheduler.java create mode 100644 libs/core/src/main/java/com/solarwinds/joboe/core/rpc/HostType.java create mode 100644 libs/core/src/main/java/com/solarwinds/joboe/core/rpc/KeepAliveMonitor.java create mode 100644 libs/core/src/main/java/com/solarwinds/joboe/core/rpc/ProtocolClient.java create mode 100644 libs/core/src/main/java/com/solarwinds/joboe/core/rpc/ProtocolClientFactory.java create mode 100644 libs/core/src/main/java/com/solarwinds/joboe/core/rpc/Result.java create mode 100644 libs/core/src/main/java/com/solarwinds/joboe/core/rpc/ResultCode.java create mode 100644 libs/core/src/main/java/com/solarwinds/joboe/core/rpc/RpcClient.java create mode 100644 libs/core/src/main/java/com/solarwinds/joboe/core/rpc/RpcClientManager.java create mode 100644 libs/core/src/main/java/com/solarwinds/joboe/core/rpc/RpcClientRejectedExecutionException.java create mode 100644 libs/core/src/main/java/com/solarwinds/joboe/core/rpc/RpcSettings.java create mode 100644 libs/core/src/main/java/com/solarwinds/joboe/core/rpc/SettingsResult.java create mode 100644 libs/core/src/main/java/com/solarwinds/joboe/core/rpc/grpc/GrpcClient.java create mode 100644 libs/core/src/main/java/com/solarwinds/joboe/core/rpc/grpc/GrpcClientManager.java create mode 100644 libs/core/src/main/java/com/solarwinds/joboe/core/settings/OboeSettingsException.java create mode 100644 libs/core/src/main/java/com/solarwinds/joboe/core/settings/PollingSettingsFetcher.java create mode 100644 libs/core/src/main/java/com/solarwinds/joboe/core/settings/RpcSettingsReader.java create mode 100644 libs/core/src/main/java/com/solarwinds/joboe/core/settings/SettingsReader.java create mode 100644 libs/core/src/main/java/com/solarwinds/joboe/core/settings/SettingsUtil.java create mode 100644 libs/core/src/main/java/com/solarwinds/joboe/core/settings/SimpleSettingsFetcher.java create mode 100644 libs/core/src/main/java/com/solarwinds/joboe/core/settings/TestSettingsReader.java create mode 100644 libs/core/src/main/java/com/solarwinds/joboe/core/util/AzureInstanceIdReader.java create mode 100644 libs/core/src/main/java/com/solarwinds/joboe/core/util/BackTraceCache.java create mode 100644 libs/core/src/main/java/com/solarwinds/joboe/core/util/BackTraceUtil.java create mode 100644 libs/core/src/main/java/com/solarwinds/joboe/core/util/BsonUtils.java create mode 100644 libs/core/src/main/java/com/solarwinds/joboe/core/util/DaemonThreadFactory.java create mode 100644 libs/core/src/main/java/com/solarwinds/joboe/core/util/DummyHostInfoReader.java create mode 100644 libs/core/src/main/java/com/solarwinds/joboe/core/util/ExecUtils.java create mode 100644 libs/core/src/main/java/com/solarwinds/joboe/core/util/HeartbeatSchedulerProvider.java create mode 100644 libs/core/src/main/java/com/solarwinds/joboe/core/util/HostInfoReader.java create mode 100644 libs/core/src/main/java/com/solarwinds/joboe/core/util/HostInfoReaderProvider.java create mode 100644 libs/core/src/main/java/com/solarwinds/joboe/core/util/HostInfoUtils.java create mode 100644 libs/core/src/main/java/com/solarwinds/joboe/core/util/HostMetadataReader.java create mode 100644 libs/core/src/main/java/com/solarwinds/joboe/core/util/HostNameReader.java create mode 100644 libs/core/src/main/java/com/solarwinds/joboe/core/util/HttpUtils.java create mode 100644 libs/core/src/main/java/com/solarwinds/joboe/core/util/JavaProcessUtils.java create mode 100644 libs/core/src/main/java/com/solarwinds/joboe/core/util/NetworkAddressInfoReader.java create mode 100644 libs/core/src/main/java/com/solarwinds/joboe/core/util/RuntimeHostInfoReaderProvider.java create mode 100644 libs/core/src/main/java/com/solarwinds/joboe/core/util/ServerHostInfoReader.java create mode 100644 libs/core/src/main/java/com/solarwinds/joboe/core/util/SslUtils.java create mode 100644 libs/core/src/main/java/com/solarwinds/joboe/core/util/TestUtils.java create mode 100644 libs/core/src/main/java/com/solarwinds/joboe/core/util/TimeUtils.java create mode 100644 libs/core/src/main/java/com/solarwinds/joboe/core/util/UamsClientIdReader.java create mode 100644 libs/core/src/main/java/com/solarwinds/joboe/core/util/diagnostic/DiagnosticTools.java create mode 100644 libs/core/src/main/java/com/solarwinds/joboe/core/util/diagnostic/InvalidArgumentsException.java create mode 100644 libs/core/src/test/java/com/solarwinds/joboe/core/AtomicEventReporterStatsTest.java create mode 100644 libs/core/src/test/java/com/solarwinds/joboe/core/ContextTest.java create mode 100644 libs/core/src/test/java/com/solarwinds/joboe/core/EventImplTest.java create mode 100644 libs/core/src/test/java/com/solarwinds/joboe/core/EventValueConverterTest.java create mode 100644 libs/core/src/test/java/com/solarwinds/joboe/core/QueuingEventReporterTest.java create mode 100644 libs/core/src/test/java/com/solarwinds/joboe/core/ReporterFactoryTest.java create mode 100644 libs/core/src/test/java/com/solarwinds/joboe/core/TestExecutionExceptionRpcClient.java create mode 100644 libs/core/src/test/java/com/solarwinds/joboe/core/TestReporterTest.java create mode 100644 libs/core/src/test/java/com/solarwinds/joboe/core/TestRpcClient.java create mode 100644 libs/core/src/test/java/com/solarwinds/joboe/core/TestSubmitRejectionRpcClient.java create mode 100644 libs/core/src/test/java/com/solarwinds/joboe/core/profiler/CircuitBreakerTest.java create mode 100644 libs/core/src/test/java/com/solarwinds/joboe/core/profiler/ProfileTest.java create mode 100644 libs/core/src/test/java/com/solarwinds/joboe/core/rpc/README.md create mode 100644 libs/core/src/test/java/com/solarwinds/joboe/core/rpc/RpcClientManagerTest.java create mode 100644 libs/core/src/test/java/com/solarwinds/joboe/core/rpc/RpcClientTest.java create mode 100644 libs/core/src/test/java/com/solarwinds/joboe/core/rpc/grpc/GrpcClientTest.java create mode 100644 libs/core/src/test/java/com/solarwinds/joboe/core/rpc/grpc/test-collector-private.pem create mode 100644 libs/core/src/test/java/com/solarwinds/joboe/core/rpc/invalid-collector.crt create mode 100644 libs/core/src/test/java/com/solarwinds/joboe/core/rpc/test-collector-public.pem create mode 100644 libs/core/src/test/java/com/solarwinds/joboe/core/settings/PollingSettingsFetcherTest.java create mode 100644 libs/core/src/test/java/com/solarwinds/joboe/core/settings/SettingsUtilTest.java create mode 100644 libs/core/src/test/java/com/solarwinds/joboe/core/util/DockerInfoReaderTest.java create mode 100644 libs/core/src/test/java/com/solarwinds/joboe/core/util/HeartbeatSchedulerProviderTest.java create mode 100644 libs/core/src/test/java/com/solarwinds/joboe/core/util/JavaVersionComparatorTest.java create mode 100644 libs/core/src/test/java/com/solarwinds/joboe/core/util/K8sReaderTest.java create mode 100644 libs/core/src/test/java/com/solarwinds/joboe/core/util/RuntimeHostInfoReaderProviderTest.java create mode 100644 libs/core/src/test/java/com/solarwinds/joboe/core/util/ServerHostInfoReaderTest.java create mode 100644 libs/core/src/test/java/com/solarwinds/joboe/core/util/SuSE-release create mode 100644 libs/core/src/test/java/com/solarwinds/joboe/core/util/TimeUtilsTest.java create mode 100644 libs/core/src/test/java/com/solarwinds/joboe/core/util/debian_version create mode 100644 libs/core/src/test/java/com/solarwinds/joboe/core/util/docker-cgroup-ce create mode 100644 libs/core/src/test/java/com/solarwinds/joboe/core/util/docker-cgroup-cri-containerd create mode 100644 libs/core/src/test/java/com/solarwinds/joboe/core/util/docker-cgroup-ecs create mode 100644 libs/core/src/test/java/com/solarwinds/joboe/core/util/docker-cgroup-empty create mode 100644 libs/core/src/test/java/com/solarwinds/joboe/core/util/docker-cgroup-invalid create mode 100644 libs/core/src/test/java/com/solarwinds/joboe/core/util/docker-cgroup-kubepods create mode 100644 libs/core/src/test/java/com/solarwinds/joboe/core/util/docker-cgroup-non-docker create mode 100644 libs/core/src/test/java/com/solarwinds/joboe/core/util/docker-cgroup-standard create mode 100644 libs/core/src/test/java/com/solarwinds/joboe/core/util/docker-cgroup-standard-2 create mode 100644 libs/core/src/test/java/com/solarwinds/joboe/core/util/gentoo-release create mode 100644 libs/core/src/test/java/com/solarwinds/joboe/core/util/lsb-release create mode 100644 libs/core/src/test/java/com/solarwinds/joboe/core/util/namespace create mode 100644 libs/core/src/test/java/com/solarwinds/joboe/core/util/poduid create mode 100644 libs/core/src/test/java/com/solarwinds/joboe/core/util/redhat-release create mode 100644 libs/core/src/test/java/com/solarwinds/joboe/core/util/slackware-version create mode 100644 libs/core/src/test/java/com/solarwinds/joboe/core/util/system-release-cpe create mode 100644 libs/core/src/test/resources/solarwinds-apm-settings-raw create mode 100644 libs/logging/.gitignore create mode 100644 libs/logging/build.gradle.kts create mode 100644 libs/logging/src/main/java/com/solarwinds/joboe/logging/CompositeStream.java create mode 100644 libs/logging/src/main/java/com/solarwinds/joboe/logging/FileLoggerStream.java create mode 100644 libs/logging/src/main/java/com/solarwinds/joboe/logging/LogSetting.java create mode 100644 libs/logging/src/main/java/com/solarwinds/joboe/logging/Logger.java create mode 100644 libs/logging/src/main/java/com/solarwinds/joboe/logging/LoggerConfiguration.java create mode 100644 libs/logging/src/main/java/com/solarwinds/joboe/logging/LoggerFactory.java create mode 100644 libs/logging/src/main/java/com/solarwinds/joboe/logging/LoggerStream.java create mode 100644 libs/logging/src/main/java/com/solarwinds/joboe/logging/LoggerThreadFactory.java create mode 100644 libs/logging/src/main/java/com/solarwinds/joboe/logging/SystemErrStream.java create mode 100644 libs/logging/src/main/java/com/solarwinds/joboe/logging/SystemOutStream.java create mode 100644 libs/logging/src/test/java/com/solarwinds/joboe/logging/FileLoggerStreamTest.java create mode 100644 libs/logging/src/test/java/com/solarwinds/joboe/logging/LoggerTest.java create mode 100644 libs/logging/src/test/java/com/solarwinds/joboe/logging/TestLoggerProcess.java create mode 100644 libs/sampling/.gitignore create mode 100644 libs/sampling/build.gradle.kts create mode 100644 libs/sampling/src/main/java/com/solarwinds/joboe/sampling/BinaryUtils.java create mode 100644 libs/sampling/src/main/java/com/solarwinds/joboe/sampling/Constants.java create mode 100644 libs/sampling/src/main/java/com/solarwinds/joboe/sampling/DevURandomSeedGenerator.java create mode 100644 libs/sampling/src/main/java/com/solarwinds/joboe/sampling/HexUtils.java create mode 100644 libs/sampling/src/main/java/com/solarwinds/joboe/sampling/Metadata.java create mode 100644 libs/sampling/src/main/java/com/solarwinds/joboe/sampling/ResourceMatcher.java create mode 100644 libs/sampling/src/main/java/com/solarwinds/joboe/sampling/SampleRateSource.java create mode 100644 libs/sampling/src/main/java/com/solarwinds/joboe/sampling/SamplingConfiguration.java create mode 100644 libs/sampling/src/main/java/com/solarwinds/joboe/sampling/SamplingException.java create mode 100644 libs/sampling/src/main/java/com/solarwinds/joboe/sampling/SecureRandomSeedGenerator.java create mode 100644 libs/sampling/src/main/java/com/solarwinds/joboe/sampling/SeedException.java create mode 100644 libs/sampling/src/main/java/com/solarwinds/joboe/sampling/SeedGenerator.java create mode 100644 libs/sampling/src/main/java/com/solarwinds/joboe/sampling/Settings.java create mode 100644 libs/sampling/src/main/java/com/solarwinds/joboe/sampling/SettingsArg.java create mode 100644 libs/sampling/src/main/java/com/solarwinds/joboe/sampling/SettingsArgChangeListener.java create mode 100644 libs/sampling/src/main/java/com/solarwinds/joboe/sampling/SettingsFetcher.java create mode 100644 libs/sampling/src/main/java/com/solarwinds/joboe/sampling/SettingsListener.java create mode 100644 libs/sampling/src/main/java/com/solarwinds/joboe/sampling/SettingsManager.java create mode 100644 libs/sampling/src/main/java/com/solarwinds/joboe/sampling/TokenBucket.java create mode 100644 libs/sampling/src/main/java/com/solarwinds/joboe/sampling/TokenBucketType.java create mode 100644 libs/sampling/src/main/java/com/solarwinds/joboe/sampling/TraceConfig.java create mode 100644 libs/sampling/src/main/java/com/solarwinds/joboe/sampling/TraceConfigs.java create mode 100644 libs/sampling/src/main/java/com/solarwinds/joboe/sampling/TraceDecision.java create mode 100644 libs/sampling/src/main/java/com/solarwinds/joboe/sampling/TraceDecisionUtil.java create mode 100644 libs/sampling/src/main/java/com/solarwinds/joboe/sampling/TracingMode.java create mode 100644 libs/sampling/src/main/java/com/solarwinds/joboe/sampling/XTraceOptionsResponse.java create mode 100644 libs/sampling/src/main/java/com/solarwinds/joboe/sampling/XorShiftRNG.java create mode 100644 libs/sampling/src/main/java/com/solarwinds/joboe/sampling/XtraceOption.java create mode 100644 libs/sampling/src/main/java/com/solarwinds/joboe/sampling/XtraceOptions.java create mode 100644 libs/sampling/src/test/java/com/solarwinds/joboe/sampling/MetadataTest.java create mode 100644 libs/sampling/src/test/java/com/solarwinds/joboe/sampling/SettingsArgTest.java create mode 100644 libs/sampling/src/test/java/com/solarwinds/joboe/sampling/SettingsManagerTest.java create mode 100644 libs/sampling/src/test/java/com/solarwinds/joboe/sampling/SettingsStub.java create mode 100644 libs/sampling/src/test/java/com/solarwinds/joboe/sampling/TokenBucketTest.java create mode 100644 libs/sampling/src/test/java/com/solarwinds/joboe/sampling/TraceDecisionUtilTest.java create mode 100644 libs/sampling/src/test/java/com/solarwinds/joboe/sampling/XtraceOptionTest.java create mode 100644 libs/sampling/src/test/java/com/solarwinds/joboe/sampling/XtraceOptionsResponseTest.java create mode 100644 libs/sampling/src/test/java/com/solarwinds/joboe/sampling/XtraceOptionsTest.java create mode 100644 libs/sampling/src/test/resources/hmac-signature.txt create mode 100644 long-running-test-arch/xk6/go.sum create mode 100644 spotbugs-exclude.xml diff --git a/.github/scripts/shading-check.sh b/.github/scripts/shading-check.sh new file mode 100755 index 00000000..8a5f6e34 --- /dev/null +++ b/.github/scripts/shading-check.sh @@ -0,0 +1,24 @@ +code=0 +for path in $(jar -tf agent/build/libs/solarwinds-apm-agent.jar | grep -E -v '^((com/solarwinds|inst|io/open|META))') +do + PACKAGE=$(echo "$path" | awk -F/ '{print $2}') + if [ -n "$PACKAGE" ] && [ "$PACKAGE" != "annotation" ]; then + echo "Package ($path) is not shaded" + code=1 + fi +done + +if [[ code -ne 0 ]]; then + exit $code +fi + +lambda=0 +for path in $(jar -tf agent-lambda/build/libs/solarwinds-apm-agent-lambda.jar | grep -E -v '^((com/solarwinds|inst|io/open|META))') +do + PACKAGE=$(echo "$path" | awk -F/ '{print $2}') + if [ -n "$PACKAGE" ] && [ "$PACKAGE" != "annotation" ]; then + echo "Package ($path) is not shaded" + lambda=1 + fi +done +exit $lambda \ No newline at end of file diff --git a/.github/workflows/push.yml b/.github/workflows/push.yml index 073d8bfe..fe82bb24 100644 --- a/.github/workflows/push.yml +++ b/.github/workflows/push.yml @@ -163,28 +163,7 @@ jobs: run: ./gradlew test - name: Check shading - run: | - code=0 - for path in $(jar -tf agent/build/libs/solarwinds-apm-agent.jar | grep -E -v '^((com/solarwinds|inst|io/open|META))') - do - PACKAGE=$(echo "$path" | awk -F/ '{print $2}') - if [ -n "$PACKAGE" ] && [ "$PACKAGE" != "annotation" ]; then - echo "Package ($path) is not shaded" - code=1 - fi - done - exit $code - - lambda=0 - for path in $(jar -tf agent-lambda/build/libs/solarwinds-apm-agent-lambda.jar | grep -E -v '^((com/solarwinds|inst|io/open|META))') - do - PACKAGE=$(echo "$path" | awk -F/ '{print $2}') - if [ -n "$PACKAGE" ] && [ "$PACKAGE" != "annotation" ]; then - echo "Package ($path) is not shaded" - lambda=1 - fi - done - exit $lambda + run: ./.github/scripts/shading-check.sh smoke-test-linux: runs-on: ubuntu-latest diff --git a/agent-lambda/build.gradle.kts b/agent-lambda/build.gradle.kts index 866a7cad..be202804 100644 --- a/agent-lambda/build.gradle.kts +++ b/agent-lambda/build.gradle.kts @@ -47,10 +47,10 @@ dependencies { javaagentLibs(project(":instrumentation")) bootstrapLibs(project(":bootstrap")) - bootstrapLibs("com.solarwinds.joboe:config") + bootstrapLibs(project(":libs:config")) bootstrapLibs("org.json:json") - bootstrapLibs("com.solarwinds.joboe:sampling") - bootstrapLibs("com.solarwinds.joboe:logging") + bootstrapLibs(project(":libs:sampling")) + bootstrapLibs(project(":libs:logging")) upstreamAgent("io.opentelemetry.javaagent:opentelemetry-javaagent") } diff --git a/agent/build.gradle.kts b/agent/build.gradle.kts index 66ce2863..418d2b1f 100644 --- a/agent/build.gradle.kts +++ b/agent/build.gradle.kts @@ -47,12 +47,11 @@ dependencies { javaagentLibs(project(":instrumentation")) bootstrapLibs(project(":bootstrap")) - bootstrapLibs("com.solarwinds.joboe:core") - bootstrapLibs("com.solarwinds.joboe:metrics") + bootstrapLibs(project(":libs:core")) - bootstrapLibs("com.solarwinds.joboe:config") - bootstrapLibs("com.solarwinds.joboe:sampling") - bootstrapLibs("com.solarwinds.joboe:logging") + bootstrapLibs(project(":libs:config")) + bootstrapLibs(project(":libs:sampling")) + bootstrapLibs(project(":libs:logging")) bootstrapLibs("org.json:json") upstreamAgent("io.opentelemetry.javaagent:opentelemetry-javaagent") diff --git a/bootstrap/build.gradle.kts b/bootstrap/build.gradle.kts index 62d720c7..f9964aa6 100644 --- a/bootstrap/build.gradle.kts +++ b/bootstrap/build.gradle.kts @@ -3,9 +3,9 @@ plugins { } dependencies { - compileOnly("com.solarwinds.joboe:config") - compileOnly("com.solarwinds.joboe:logging") - compileOnly("com.solarwinds.joboe:sampling") + compileOnly(project(":libs:config")) + compileOnly(project(":libs:logging")) + compileOnly(project(":libs:sampling")) compileOnly("io.opentelemetry:opentelemetry-api") compileOnly("io.opentelemetry.semconv:opentelemetry-semconv") diff --git a/build.gradle.kts b/build.gradle.kts index c013d6a5..1d4507e3 100644 --- a/build.gradle.kts +++ b/build.gradle.kts @@ -18,7 +18,7 @@ plugins{ id("io.github.gradle-nexus.publish-plugin") version "2.0.0" } -val swoAgentVersion = "3.1.3" +val swoAgentVersion = "3.1.4" extra["swoAgentVersion"] = swoAgentVersion group = "com.solarwinds" version = if (System.getenv("SNAPSHOT_BUILD").toBoolean()) "$swoAgentVersion-SNAPSHOT" else swoAgentVersion diff --git a/buildSrc/build.gradle.kts b/buildSrc/build.gradle.kts index 51a472f4..31d9a2e6 100644 --- a/buildSrc/build.gradle.kts +++ b/buildSrc/build.gradle.kts @@ -40,8 +40,10 @@ repositories { dependencies { implementation(gradleApi()) implementation("com.diffplug.spotless:spotless-plugin-gradle:8.2.1") + implementation("io.freefair.gradle:lombok-plugin:8.13") implementation("io.opentelemetry.instrumentation:gradle-plugins:2.25.0-alpha") implementation("com.gradleup.shadow:shadow-gradle-plugin:9.3.1") implementation("com.github.gmazzo.buildconfig:com.github.gmazzo.buildconfig.gradle.plugin:6.0.7") + implementation("com.github.spotbugs:com.github.spotbugs.gradle.plugin:6.4.8") } diff --git a/buildSrc/src/main/kotlin/solarwinds.instrumentation-conventions.gradle.kts b/buildSrc/src/main/kotlin/solarwinds.instrumentation-conventions.gradle.kts index 926fffe7..2b6922a7 100644 --- a/buildSrc/src/main/kotlin/solarwinds.instrumentation-conventions.gradle.kts +++ b/buildSrc/src/main/kotlin/solarwinds.instrumentation-conventions.gradle.kts @@ -36,7 +36,6 @@ dependencies { compileOnly("net.bytebuddy:byte-buddy") // Used by byte-buddy but not brought in as a transitive dependency. - compileOnly("com.google.code.findbugs:annotations") compileOnly("com.google.auto.service:auto-service") annotationProcessor("com.google.auto.service:auto-service") diff --git a/buildSrc/src/main/kotlin/solarwinds.java-conventions.gradle.kts b/buildSrc/src/main/kotlin/solarwinds.java-conventions.gradle.kts index abe6f04e..3b3f6f9d 100644 --- a/buildSrc/src/main/kotlin/solarwinds.java-conventions.gradle.kts +++ b/buildSrc/src/main/kotlin/solarwinds.java-conventions.gradle.kts @@ -18,7 +18,9 @@ import com.solarwinds.instrumentation.gradle.SolarwindsJavaExtension plugins { java checkstyle + id("io.freefair.lombok") id("solarwinds.spotless-conventions") + id("com.github.spotbugs") } repositories { @@ -93,8 +95,7 @@ dependencies { testImplementation("io.opentelemetry.semconv:opentelemetry-semconv") testImplementation("io.opentelemetry.instrumentation:opentelemetry-instrumentation-api") - testImplementation("com.solarwinds.joboe:core") - testImplementation("com.solarwinds.joboe:metrics") + testImplementation(project(":libs:core")) testImplementation("org.junit-pioneer:junit-pioneer") testImplementation("org.junit.jupiter:junit-jupiter-params") @@ -117,11 +118,7 @@ tasks { "-Xlint:all", // disable annotation ownership warnings "-Xlint:-processing", - "-Werror", - // FIXME: Refactor generic service provider interfaces (e.g., ConfigParser) to avoid rawtypes warnings - // Disable AutoService verify check to prevent rawtypes warnings for generic service provider interfaces - // The @SuppressWarnings("rawtypes") annotation is not recognized in certain Gradle 9 compilation contexts - "-Averify=false" + "-Werror" ) ) @@ -146,4 +143,8 @@ tasks { checkstyle { configFile = file("$rootDir/checkstyle.xml") +} + +spotbugs { + excludeFilter.set(file("$rootDir/spotbugs-exclude.xml")) } \ No newline at end of file diff --git a/buildSrc/src/main/kotlin/solarwinds.shadow-conventions.gradle.kts b/buildSrc/src/main/kotlin/solarwinds.shadow-conventions.gradle.kts index 118646f7..595e63e2 100644 --- a/buildSrc/src/main/kotlin/solarwinds.shadow-conventions.gradle.kts +++ b/buildSrc/src/main/kotlin/solarwinds.shadow-conventions.gradle.kts @@ -53,4 +53,22 @@ tasks.withType().configureEach { relocate("com.github.benmanes", "com.solarwinds.joboe.shaded.caffeine") relocate("org.checkerframework", "com.solarwinds.joboe.shaded.checkerframework") relocate("org.json", "com.solarwinds.joboe.shaded.org.json") + + relocate("com.solarwinds.trace", "com.solarwinds.joboe.shaded.trace") + relocate("android.annotation", "com.solarwinds.joboe.shaded.android.annotation") + relocate("javax.annotation", "com.solarwinds.joboe.shaded.javax.annotation") + + relocate("cloud", "com.solarwinds.joboe.shaded.cloud") + relocate("google", "com.solarwinds.joboe.shaded.google2") + relocate("com.google", "com.solarwinds.joboe.shaded.google") + + relocate("javax.xml", "com.solarwinds.joboe.shaded.javax.xml") + relocate("io.grpc", "com.solarwinds.joboe.shaded.io.grpc") + relocate("io.netty", "com.solarwinds.joboe.shaded.io.netty") + + relocate("io.perfmark", "com.solarwinds.joboe.shaded.io.perfmark") + relocate("javax.activation", "com.solarwinds.joboe.shaded.javax.activation") + relocate("org.jspecify", "com.solarwinds.joboe.shaded.org.jspecify") + + relocate("org.codehaus.mojo", "com.solarwinds.joboe.shaded.org.codehaus.mojo") } diff --git a/custom/build.gradle.kts b/custom/build.gradle.kts index 7c3bbe64..24f205f2 100644 --- a/custom/build.gradle.kts +++ b/custom/build.gradle.kts @@ -21,15 +21,11 @@ plugins { dependencies { compileOnly(project(":bootstrap")) compileOnly(project(":libs:shared")) - compileOnly("com.solarwinds.joboe:core") + compileOnly(project(":libs:core")) - compileOnly("com.solarwinds.joboe:config") - compileOnly("com.solarwinds.joboe:sampling") - compileOnly("com.solarwinds.joboe:logging") - - compileOnly("org.projectlombok:lombok") - compileOnly("com.solarwinds.joboe:metrics") - annotationProcessor("org.projectlombok:lombok") + compileOnly(project(":libs:config")) + compileOnly(project(":libs:sampling")) + compileOnly(project(":libs:logging")) compileOnly("com.google.auto.service:auto-service") annotationProcessor("com.google.auto.service:auto-service") @@ -50,14 +46,21 @@ dependencies { compileOnly("com.google.code.gson:gson") implementation("org.json:json") + testImplementation(project(":libs:config")) + testImplementation(project(":libs:sampling")) testImplementation(project(":libs:shared")) - testImplementation("com.solarwinds.joboe:core") + testImplementation(project(":libs:core")) testImplementation("io.opentelemetry:opentelemetry-api-incubator") testImplementation("io.opentelemetry:opentelemetry-sdk-extension-incubator") testImplementation("io.opentelemetry:opentelemetry-exporter-otlp") } +tasks.named("compileJava") { + // Disable AutoService verify check to prevent rawtypes warnings for generic service provider interfaces + options.compilerArgs.add("-Averify=false") +} + tasks.withType(Checkstyle::class).configureEach { exclude("**/BuildConfig.java") exclude("**/transaction/**") diff --git a/custom/src/main/java/com/solarwinds/opentelemetry/extensions/SolarwindsAgentListener.java b/custom/src/main/java/com/solarwinds/opentelemetry/extensions/SolarwindsAgentListener.java index 3b83921a..0314bbc8 100644 --- a/custom/src/main/java/com/solarwinds/opentelemetry/extensions/SolarwindsAgentListener.java +++ b/custom/src/main/java/com/solarwinds/opentelemetry/extensions/SolarwindsAgentListener.java @@ -32,7 +32,6 @@ import com.solarwinds.joboe.core.util.HostInfoUtils; import com.solarwinds.joboe.logging.Logger; import com.solarwinds.joboe.logging.LoggerFactory; -import com.solarwinds.joboe.metrics.SystemMonitorController; import com.solarwinds.joboe.sampling.SettingsManager; import com.solarwinds.opentelemetry.core.AgentState; import com.solarwinds.opentelemetry.extensions.config.HttpSettingsFetcher; @@ -150,8 +149,6 @@ private void registerShutdownTasks() { new Thread("SolarwindsAPM-shutdown-hook") { @Override public void run() { - SystemMonitorController - .stop(); // stop system monitors, this might flush extra messages to reporters if (ReporterProvider.getEventReporter() != null) { ReporterProvider.getEventReporter() .close(); // close event reporter properly to give it chance to send out pending diff --git a/custom/src/main/java/com/solarwinds/opentelemetry/extensions/SolarwindsProfilingSpanProcessor.java b/custom/src/main/java/com/solarwinds/opentelemetry/extensions/SolarwindsProfilingSpanProcessor.java index 8c4c410c..b7ba9b40 100644 --- a/custom/src/main/java/com/solarwinds/opentelemetry/extensions/SolarwindsProfilingSpanProcessor.java +++ b/custom/src/main/java/com/solarwinds/opentelemetry/extensions/SolarwindsProfilingSpanProcessor.java @@ -25,7 +25,6 @@ import com.solarwinds.joboe.logging.Logger; import com.solarwinds.joboe.logging.LoggerFactory; import com.solarwinds.joboe.sampling.Metadata; -import com.solarwinds.joboe.shaded.javax.annotation.Nonnull; import com.solarwinds.opentelemetry.core.Util; import io.opentelemetry.api.trace.Span; import io.opentelemetry.api.trace.SpanContext; @@ -33,6 +32,7 @@ import io.opentelemetry.sdk.trace.ReadWriteSpan; import io.opentelemetry.sdk.trace.ReadableSpan; import io.opentelemetry.sdk.trace.internal.ExtendedSpanProcessor; +import javax.annotation.Nonnull; /** Span process to perform code profiling */ public class SolarwindsProfilingSpanProcessor implements ExtendedSpanProcessor { diff --git a/custom/src/main/java/com/solarwinds/opentelemetry/extensions/config/HttpSettingsReaderDelegate.java b/custom/src/main/java/com/solarwinds/opentelemetry/extensions/config/HttpSettingsReaderDelegate.java index c189d9e1..b5640096 100644 --- a/custom/src/main/java/com/solarwinds/opentelemetry/extensions/config/HttpSettingsReaderDelegate.java +++ b/custom/src/main/java/com/solarwinds/opentelemetry/extensions/config/HttpSettingsReaderDelegate.java @@ -32,6 +32,7 @@ import java.net.InetSocketAddress; import java.net.Proxy; import java.net.URL; +import java.nio.charset.StandardCharsets; import java.util.concurrent.atomic.AtomicReference; public class HttpSettingsReaderDelegate { @@ -56,7 +57,8 @@ public Settings fetchSettings(String url, String authorizationHeader) { responseCode, errorResponse)); } else { try (BufferedReader reader = - new BufferedReader(new InputStreamReader(connection.getInputStream()))) { + new BufferedReader( + new InputStreamReader(connection.getInputStream(), StandardCharsets.UTF_8))) { settings.set(JsonSettingWrapper.wrap(gson.fromJson(reader, JsonSetting.class))); } } @@ -79,14 +81,15 @@ public Settings fetchSettings(String url, String authorizationHeader) { private String getErrorMessage(HttpURLConnection connection) { String errorResponse; try (BufferedReader errorReader = - new BufferedReader(new InputStreamReader(connection.getErrorStream()))) { + new BufferedReader( + new InputStreamReader(connection.getErrorStream(), StandardCharsets.UTF_8))) { StringBuilder errorBuilder = new StringBuilder(); String line; while ((line = errorReader.readLine()) != null) { errorBuilder.append(line); } errorResponse = errorBuilder.toString(); - } catch (Exception e) { + } catch (IOException e) { errorResponse = "Unable to read error response"; } return errorResponse; diff --git a/custom/src/main/java/com/solarwinds/opentelemetry/extensions/config/livereload/ConfigurationFileWatcher.java b/custom/src/main/java/com/solarwinds/opentelemetry/extensions/config/livereload/ConfigurationFileWatcher.java index 3b447f34..3e9d1051 100644 --- a/custom/src/main/java/com/solarwinds/opentelemetry/extensions/config/livereload/ConfigurationFileWatcher.java +++ b/custom/src/main/java/com/solarwinds/opentelemetry/extensions/config/livereload/ConfigurationFileWatcher.java @@ -27,6 +27,7 @@ import java.util.concurrent.ScheduledExecutorService; import java.util.concurrent.ScheduledFuture; import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicReference; public final class ConfigurationFileWatcher { private final Path directory; @@ -39,7 +40,7 @@ public final class ConfigurationFileWatcher { private final long watchPeriod; - private static ConfigurationFileWatcher INSTANCE; + private static final AtomicReference INSTANCE = new AtomicReference<>(); private ScheduledFuture scheduledWatch; @@ -62,13 +63,15 @@ public static void restartWatch( WatchService watchService, ScheduledExecutorService scheduledExecutorService, Runnable fileChangeListener) { - if (INSTANCE != null) { - INSTANCE.cancelWatch(); - } - INSTANCE = + ConfigurationFileWatcher next = new ConfigurationFileWatcher( directory, watchPeriod, watchService, scheduledExecutorService, fileChangeListener); - INSTANCE.startWatch(); + + ConfigurationFileWatcher previous = INSTANCE.getAndSet(next); + if (previous != null) { + previous.cancelWatch(); + } + next.startWatch(); } private void watch() { diff --git a/custom/src/main/java/com/solarwinds/opentelemetry/extensions/config/provider/AutoConfigurationCustomizerProviderImpl.java b/custom/src/main/java/com/solarwinds/opentelemetry/extensions/config/provider/AutoConfigurationCustomizerProviderImpl.java index d1448d0b..e598304c 100644 --- a/custom/src/main/java/com/solarwinds/opentelemetry/extensions/config/provider/AutoConfigurationCustomizerProviderImpl.java +++ b/custom/src/main/java/com/solarwinds/opentelemetry/extensions/config/provider/AutoConfigurationCustomizerProviderImpl.java @@ -21,7 +21,6 @@ import com.solarwinds.joboe.config.JavaRuntimeVersionChecker; import com.solarwinds.joboe.logging.Logger; import com.solarwinds.joboe.logging.LoggerFactory; -import com.solarwinds.joboe.shaded.javax.annotation.Nonnull; import com.solarwinds.opentelemetry.extensions.MetricExporterCustomizer; import com.solarwinds.opentelemetry.extensions.ResourceCustomizer; import com.solarwinds.opentelemetry.extensions.SolarwindsPropertiesSupplier; @@ -31,6 +30,7 @@ import com.solarwinds.opentelemetry.extensions.config.SpanExporterCustomizer; import io.opentelemetry.sdk.autoconfigure.spi.AutoConfigurationCustomizer; import io.opentelemetry.sdk.autoconfigure.spi.AutoConfigurationCustomizerProvider; +import javax.annotation.Nonnull; /** * An implementation of {@link AutoConfigurationCustomizer} which serves as the bootstrap for our diff --git a/custom/src/main/resources/ao-collector.crt b/custom/src/main/resources/ao-collector.crt deleted file mode 100644 index 1872b7dc..00000000 --- a/custom/src/main/resources/ao-collector.crt +++ /dev/null @@ -1,24 +0,0 @@ ------BEGIN CERTIFICATE----- -MIID8TCCAtmgAwIBAgIJAMoDz7Npas2/MA0GCSqGSIb3DQEBCwUAMIGOMQswCQYD -VQQGEwJVUzETMBEGA1UECAwKQ2FsaWZvcm5pYTEWMBQGA1UEBwwNU2FuIEZyYW5j -aXNjbzEVMBMGA1UECgwMTGlicmF0byBJbmMuMRUwEwYDVQQDDAxBcHBPcHRpY3Mg -Q0ExJDAiBgkqhkiG9w0BCQEWFXN1cHBvcnRAYXBwb3B0aWNzLmNvbTAeFw0xNzA5 -MTUyMjAxMzlaFw0yNzA5MTMyMjAxMzlaMIGOMQswCQYDVQQGEwJVUzETMBEGA1UE -CAwKQ2FsaWZvcm5pYTEWMBQGA1UEBwwNU2FuIEZyYW5jaXNjbzEVMBMGA1UECgwM -TGlicmF0byBJbmMuMRUwEwYDVQQDDAxBcHBPcHRpY3MgQ0ExJDAiBgkqhkiG9w0B -CQEWFXN1cHBvcnRAYXBwb3B0aWNzLmNvbTCCASIwDQYJKoZIhvcNAQEBBQADggEP -ADCCAQoCggEBAOxO0wsGba3iI4r3L5BMST0rAO/gGaUhpQre6nRwVTmPCnLw1bmn -GdiFgYv/oRRwU+VieumHSQqoOmyFrg+ajGmvUDp2WqQ0It+XhcbaHFiAp2H7+mLf -cUH6S43/em0WUxZHeRzRupRDyO1bX6Hh2jgxykivlFrn5HCIQD5Hx1/SaZoW9v2n -oATCbgFOiPW6kU/AVs4R0VBujon13HCehVelNKkazrAEBT1i6RvdOB6aQQ32seW+ -gLV5yVWSPEJvA9ZJqad/nQ8EQUMSSlVN191WOjp4bGpkJE1svs7NmM+Oja50W56l -qOH5eWermr/8qWjdPlDJ+I0VkgN0UyHVuRECAwEAAaNQME4wHQYDVR0OBBYEFOuL -KDTFhRQXwlBRxhPqhukrNYeRMB8GA1UdIwQYMBaAFOuLKDTFhRQXwlBRxhPqhukr -NYeRMAwGA1UdEwQFMAMBAf8wDQYJKoZIhvcNAQELBQADggEBAJQtH446NZhjusy6 -iCyvmnD95ybfNPDpjHmNx5n9Y6w9n+9y1o3732HUJE+WjvbLS3h1o7wujGKMcRJn -7I7eTDd26ZhLvnh5/AitYjdxrtUkQDgyxwLFJKhZu0ik2vXqj0fL961/quJL8Gyp -hNj3Nf7WMohQMSohEmCCX2sHyZGVGYmQHs5omAtkH/NNySqmsWNcpgd3M0aPDRBZ -5VFreOSGKBTJnoLNqods/S9RV0by84hm3j6aQ/tMDIVE9VCJtrE6evzC0MWyVFwR -ftgwcxyEq5SkiR+6BCwdzAMqADV37TzXDHLjwSrMIrgLV5xZM20Kk6chxI5QAr/f -7tsqAxw= ------END CERTIFICATE----- diff --git a/custom/src/test/java/com/solarwinds/opentelemetry/extensions/SolarwindsProfilingSpanProcessorTest.java b/custom/src/test/java/com/solarwinds/opentelemetry/extensions/SolarwindsProfilingSpanProcessorTest.java index ac4157a7..963e2d33 100644 --- a/custom/src/test/java/com/solarwinds/opentelemetry/extensions/SolarwindsProfilingSpanProcessorTest.java +++ b/custom/src/test/java/com/solarwinds/opentelemetry/extensions/SolarwindsProfilingSpanProcessorTest.java @@ -74,9 +74,9 @@ class SolarwindsProfilingSpanProcessorTest { private MockedStatic profilerMock; - private final String traceId = "0123456789abcdef0123456789abcdef"; + private static final String traceId = "0123456789abcdef0123456789abcdef"; - private final String spanId = "0123456789abcdef"; + private static final String spanId = "0123456789abcdef"; @BeforeEach void setup() { diff --git a/custom/src/test/java/com/solarwinds/opentelemetry/extensions/config/HttpSettingsReaderDelegateTest.java b/custom/src/test/java/com/solarwinds/opentelemetry/extensions/config/HttpSettingsReaderDelegateTest.java index 32436c39..0c0a3abe 100644 --- a/custom/src/test/java/com/solarwinds/opentelemetry/extensions/config/HttpSettingsReaderDelegateTest.java +++ b/custom/src/test/java/com/solarwinds/opentelemetry/extensions/config/HttpSettingsReaderDelegateTest.java @@ -38,6 +38,7 @@ import java.io.InputStream; import java.net.HttpURLConnection; import java.net.URL; +import java.nio.charset.StandardCharsets; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; import org.junit.jupiter.api.extension.ExtendWith; @@ -80,7 +81,8 @@ void setup() { @Test void testFetchSettings_Success() throws IOException { - InputStream inputStream = new ByteArrayInputStream(TEST_JSON_RESPONSE.getBytes()); + InputStream inputStream = + new ByteArrayInputStream(TEST_JSON_RESPONSE.getBytes(StandardCharsets.UTF_8)); doReturn(mockConnection) .when(tested) @@ -112,7 +114,8 @@ void testFetchSettings_Success() throws IOException { @Test void testFetchSettings_HttpError() throws IOException { String errorMessage = "Bad Request - Invalid parameters"; - InputStream errorStream = new ByteArrayInputStream(errorMessage.getBytes()); + InputStream errorStream = + new ByteArrayInputStream(errorMessage.getBytes(StandardCharsets.UTF_8)); doReturn(mockConnection) .when(tested) diff --git a/custom/src/test/java/com/solarwinds/opentelemetry/extensions/config/livereload/ConfigurationFileWatcherTest.java b/custom/src/test/java/com/solarwinds/opentelemetry/extensions/config/livereload/ConfigurationFileWatcherTest.java index f674a98b..3e53a776 100644 --- a/custom/src/test/java/com/solarwinds/opentelemetry/extensions/config/livereload/ConfigurationFileWatcherTest.java +++ b/custom/src/test/java/com/solarwinds/opentelemetry/extensions/config/livereload/ConfigurationFileWatcherTest.java @@ -49,7 +49,7 @@ class ConfigurationFileWatcherTest { @Captor private ArgumentCaptor runnableArgumentCaptor; - private final long watchPeriod = 1; + private static final long watchPeriod = 1; @Test void verifyThatOverflowEventsAreIgnored() throws IOException { diff --git a/dependencyManagement/build.gradle.kts b/dependencyManagement/build.gradle.kts index 0880c06b..f520a7e3 100644 --- a/dependencyManagement/build.gradle.kts +++ b/dependencyManagement/build.gradle.kts @@ -7,7 +7,6 @@ val otelSdkVersion = "1.59.0" val mockitoVersion = "5.2.0" val byteBuddyVersion = "1.18.4" -val joboeVersion = "11.0.0" val opentelemetryJavaagentAlpha = "$otelAgentVersion-alpha" val opentelemetryAlpha = "$otelSdkVersion-alpha" @@ -58,19 +57,10 @@ dependencies { api("net.bytebuddy:byte-buddy:${byteBuddyVersion}") api("com.google.auto.service:auto-service:$autoservice") - api("org.projectlombok:lombok:1.18.42") - api("com.solarwinds.joboe:core:$joboeVersion") - api("com.solarwinds.joboe:metrics:$joboeVersion") - - api("com.solarwinds.joboe:config:$joboeVersion") - api("com.solarwinds.joboe:logging:$joboeVersion") - api("com.solarwinds.joboe:sampling:$joboeVersion") - - api("org.json:json:20251224") - api("com.google.code.gson:gson:2.13.2") + api("org.json:json:20250517") + api("com.google.code.gson:gson:2.10.1") api("com.github.ben-manes.caffeine:caffeine:2.9.3") - api("com.google.code.findbugs:annotations:3.0.1u2") api("io.opentelemetry.contrib:opentelemetry-span-stacktrace:$otelJavaContribVersion") api("io.opentelemetry.semconv:opentelemetry-semconv-incubating:$opentelemetrySemconvAlpha") diff --git a/instrumentation/hibernate/hibernate-6.0/javaagent/build.gradle.kts b/instrumentation/hibernate/hibernate-6.0/javaagent/build.gradle.kts index 2f435155..34feed52 100644 --- a/instrumentation/hibernate/hibernate-6.0/javaagent/build.gradle.kts +++ b/instrumentation/hibernate/hibernate-6.0/javaagent/build.gradle.kts @@ -7,7 +7,7 @@ dependencies { implementation(project(":instrumentation:instrumentation-shared")) compileOnly("org.hibernate:hibernate-core:6.0.0.Final") - compileOnly("com.solarwinds.joboe:logging") + compileOnly(project(":libs:logging")) compileOnly("io.opentelemetry:opentelemetry-sdk-trace") compileOnly("io.opentelemetry.semconv:opentelemetry-semconv") diff --git a/instrumentation/instrumentation-shared/build.gradle.kts b/instrumentation/instrumentation-shared/build.gradle.kts index a83d6d58..5108d7df 100644 --- a/instrumentation/instrumentation-shared/build.gradle.kts +++ b/instrumentation/instrumentation-shared/build.gradle.kts @@ -20,9 +20,11 @@ plugins { dependencies { compileOnly(project(":bootstrap")) - compileOnly("com.solarwinds.joboe:config") + compileOnly(project(":libs:config")) + compileOnly(project(":libs:logging")) compileOnly("io.opentelemetry.semconv:opentelemetry-semconv") + testImplementation(project(":libs:config")) testImplementation(project(":instrumentation:instrumentation-shared")) } diff --git a/instrumentation/jdbc/javaagent/build.gradle.kts b/instrumentation/jdbc/javaagent/build.gradle.kts index 7f9bfa70..4508a99d 100644 --- a/instrumentation/jdbc/javaagent/build.gradle.kts +++ b/instrumentation/jdbc/javaagent/build.gradle.kts @@ -20,17 +20,18 @@ plugins { dependencies { compileOnly(project(":bootstrap")) - compileOnly("com.solarwinds.joboe:config") + compileOnly(project(":libs:config")) implementation(project(":instrumentation:instrumentation-shared")) compileOnly("org.json:json") - compileOnly("com.solarwinds.joboe:logging") + compileOnly(project(":libs:logging")) compileOnly("io.opentelemetry:opentelemetry-sdk-trace") compileOnly("io.opentelemetry.semconv:opentelemetry-semconv") compileOnly("com.github.ben-manes.caffeine:caffeine") testImplementation(project(":instrumentation:jdbc:javaagent")) + testImplementation(project(":libs:config")) testImplementation(project(":instrumentation:instrumentation-shared")) testImplementation(platform("org.testcontainers:testcontainers-bom:2.0.3")) diff --git a/libs/config/.gitignore b/libs/config/.gitignore new file mode 100644 index 00000000..567609b1 --- /dev/null +++ b/libs/config/.gitignore @@ -0,0 +1 @@ +build/ diff --git a/libs/config/build.gradle.kts b/libs/config/build.gradle.kts new file mode 100644 index 00000000..cc09f1e8 --- /dev/null +++ b/libs/config/build.gradle.kts @@ -0,0 +1,11 @@ +plugins { + id("solarwinds.java-conventions") +} + +description = "config" + +dependencies { + implementation(project(":libs:logging")) + compileOnly("org.json:json") + testImplementation("org.json:json") +} diff --git a/libs/config/src/main/java/com/solarwinds/joboe/config/ConfigContainer.java b/libs/config/src/main/java/com/solarwinds/joboe/config/ConfigContainer.java new file mode 100644 index 00000000..64964a11 --- /dev/null +++ b/libs/config/src/main/java/com/solarwinds/joboe/config/ConfigContainer.java @@ -0,0 +1,276 @@ +/* + * © SolarWinds Worldwide, LLC. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.solarwinds.joboe.config; + +import com.solarwinds.joboe.logging.Logger; +import com.solarwinds.joboe.logging.LoggerFactory; +import java.io.Serializable; +import java.math.BigDecimal; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import org.json.JSONArray; +import org.json.JSONException; + +/** + * This container serves several purposes + * + *

+ * + *

    + *
  1. Contains configuration values, the subset method prevents irrelevant access to config + * parameters + *
  2. Type conversion/validation base on string values as parameter in method putByStringValue + *
+ * + *

Take note that this container does not allow overwriting existing values. If values are + * inserted multiple times with the same key, only the first insert will be handled, all subsequent + * inserts are ignored + * + * @author Patson Luk + */ +public class ConfigContainer { + private static final Logger logger = LoggerFactory.getLogger(); + + // The map that contains all the config info. First grouped by ConfigGroup (MONITOR, AGENT etc), + // then by each ConfigProperty. Not directly accessible from the outside + private final Map> configMaps = + new HashMap>(); + + /** + * Subset method should be used when parameters are passed down to other code. It should only pass + * the set of parameters that are relevant to that particular module, not everything + * + *

For example, the JMX monitoring code should not have access to the agent parameter such as + * the sampling rate + * + * @param groups groups of ConfigGroup to retain + * @return + */ + public ConfigContainer subset(ConfigGroup... groups) { + ConfigContainer subset = new ConfigContainer(); + for (ConfigGroup group : groups) { + subset.configMaps.put(group, this.configMaps.get(group)); + } + + return subset; + } + + /** + * Gets a configuration property value from a Property key + * + * @param propertyKey + * @return the property value based on the propertyKey + */ + public Object get(ConfigProperty propertyKey) { + Map configMap = configMaps.get(propertyKey.getGroup()); + + if (configMap != null) { + return configMap.get(propertyKey); + } else { // no config for this group configured + return null; + } + } + + /** + * Checks whether a property key is set in the container + * + * @param propertyKey + * @return whether the property key is set in the container + */ + public boolean containsProperty(ConfigProperty propertyKey) { + Map configMap = configMaps.get(propertyKey.getGroup()); + + if (configMap != null) { + return configMap.containsKey(propertyKey); + } else { + return false; + } + } + + public Object remove(ConfigProperty propertyKey) { + Map configs = configMaps.get(propertyKey.getGroup()); + if (configs != null) { + return configs.remove(propertyKey); + } + return null; + } + + public void put(ConfigProperty propertyKey, Object value) throws InvalidConfigException { + put(propertyKey, value, false); + } + + public void put(ConfigProperty propertyKey, Object value, boolean override) + throws InvalidConfigException { + Map configMap = configMaps.get(propertyKey.getGroup()); + + if (configMap == null) { // The Group is not initialized, put a new map into the configMap + configMap = new HashMap(); + configMaps.put(propertyKey.getGroup(), configMap); + } + + if (!override + && configMap.containsKey( + propertyKey)) { // the key was already inserted before, do NOT overwrite + if (!configMap.get(propertyKey).equals(value)) { + logger.debug( + "key [" + + propertyKey + + "] is already defined with value [" + + configMap.get(propertyKey) + + "]. Ignoring new value [" + + value + + "]"); + } + } else { + if (value == null) { // Do not allow null via put by string value + throw new InvalidConfigException( + propertyKey, " Does not support null property value", null); + } else { + configMap.put(propertyKey, value); + } + } + } + + /** + * Insert a property value by its String representation. Take note that the String argument will + * be converted to the type defined in the ConfigProperty.typeClass and converted by + * the {@link ConfigParser} if one is defined within {@link ConfigProperty} + * + *

It will ignore the operation if the key already exists + * + * @param propertyKey + * @param stringValue + * @throws InvalidConfigException if the stringValue cannot be converted to the expected type + * defined in the propertyKey + */ + public void putByStringValue(ConfigProperty propertyKey, String stringValue) + throws InvalidConfigException { + Object value = getValue(stringValue, propertyKey); + if (value == null) { // Do not allow null via put by string value + throw new InvalidConfigException( + propertyKey, + "Does not support null property value, the value [" + + stringValue + + "] got converted to null value", + null); + } else { + put(propertyKey, value); + } + } + + /** + * Converts and returns the Object of typeClass base on the valueString. Take note that this + * method should only handle the basic Wrapper types like Integer, BigDecimal or String. Do not + * attempt to handle anything too specific here as the Container is supposed to be generic + * + * @param valueString + * @param configProperty + * @return the Object of typeClass base on the valueString. null if the conversion failed + * @throws InvalidConfigException + */ + @SuppressWarnings("unchecked") + private static Object getValue( + String valueString, ConfigProperty configProperty) throws InvalidConfigException { + Serializable javaValue; + Class typeClass = (Class) configProperty.getTypeClass(); + ConfigParser processor = (ConfigParser) configProperty.getConfigParser(); + + try { + if (typeClass != String.class) { // do not attempt to trim String type + valueString = valueString.trim(); + } + + if (typeClass == String.class) { + javaValue = valueString; + } else if (typeClass == Long.class) { + javaValue = Long.valueOf(valueString); + } else if (typeClass == BigDecimal.class) { + javaValue = new BigDecimal(valueString); + } else if (typeClass == Integer.class) { + javaValue = Integer.valueOf(valueString); + } else if (typeClass == Boolean.class) { + if ("true".equalsIgnoreCase(valueString) + || "false" + .equalsIgnoreCase( + valueString)) { // use strict handling, do not allow unexpected/invalid values + javaValue = Boolean.valueOf(valueString); + } else { + throw new InvalidConfigException( + configProperty, "[" + valueString + "] is not valid boolean value"); + } + } else if (typeClass == String[].class) { + javaValue = parseJsonStringArray(valueString); + } else { + logger.warn("Unknown type for configuration: " + typeClass.getName()); + javaValue = valueString; + } + } catch (IllegalArgumentException e) { + throw new InvalidConfigException( + configProperty, + "Failed to parse value [" + + valueString + + "] of class [" + + typeClass.getName() + + "], message: " + + e.getMessage(), + e); + } + + if (processor != null) { + try { + return processor.convert((T) javaValue); + } catch (InvalidConfigException e) { + throw new InvalidConfigException( + configProperty, e.getMessage(), e.getCause()); // set the violating config property + } catch (ClassCastException e) { + throw new InvalidConfigException( + configProperty, + "Failed to read config value " + javaValue + ", message : " + e.getMessage(), + e); + } + } else { + return javaValue; + } + } + + private static String[] parseJsonStringArray(String stringArrayValue) { + List list = new ArrayList(); + try { + JSONArray stringArray = new JSONArray(stringArrayValue); + for (int i = 0; i < stringArray.length(); i++) { + list.add((String) stringArray.get(i)); + } + } catch (JSONException e) { + logger.warn(e.getMessage()); + throw new IllegalArgumentException( + "Cannot parse the string value " + + stringArrayValue + + " as json array : " + + e.getMessage(), + e); + } + + return list.toArray(new String[list.size()]); + } + + @Override + public String toString() { + return configMaps.toString(); + } +} diff --git a/libs/config/src/main/java/com/solarwinds/joboe/config/ConfigGroup.java b/libs/config/src/main/java/com/solarwinds/joboe/config/ConfigGroup.java new file mode 100644 index 00000000..3904bd6d --- /dev/null +++ b/libs/config/src/main/java/com/solarwinds/joboe/config/ConfigGroup.java @@ -0,0 +1,28 @@ +/* + * © SolarWinds Worldwide, LLC. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.solarwinds.joboe.config; + +/** + * Grouping for the configuration properties defined in ConfigProperty + * + * @author Patson Luk + */ +public enum ConfigGroup { + AGENT, + MONITOR, + PROFILER +} diff --git a/libs/config/src/main/java/com/solarwinds/joboe/config/ConfigManager.java b/libs/config/src/main/java/com/solarwinds/joboe/config/ConfigManager.java new file mode 100644 index 00000000..9d7fb796 --- /dev/null +++ b/libs/config/src/main/java/com/solarwinds/joboe/config/ConfigManager.java @@ -0,0 +1,93 @@ +/* + * © SolarWinds Worldwide, LLC. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.solarwinds.joboe.config; + +import com.solarwinds.joboe.logging.Logger; +import com.solarwinds.joboe.logging.LoggerFactory; + +public class ConfigManager { + private static final Logger logger = LoggerFactory.getLogger(); + private ConfigContainer configs; + + private static final ConfigManager SINGLETON = new ConfigManager(); // only a singleton for now + + private ConfigManager() {} + + public static void initialize(ConfigContainer configs) { + SINGLETON.configs = configs; + } + + /** For internal testing - reset the states of the ConfigManager */ + public static void reset() { + SINGLETON.configs = null; + } + + public static void setConfig(ConfigProperty configKey, Object value) + throws InvalidConfigException { + if (SINGLETON.configs == null) { + SINGLETON.configs = new ConfigContainer(); + } + SINGLETON.configs.put(configKey, value, true); + } + + public static void removeConfig(ConfigProperty configKey) { + if (SINGLETON.configs != null) { + SINGLETON.configs.remove(configKey); + } + } + + /** + * Convenience method for other code to read the configuration value of the Agent + * + * @param configKey + * @return the configuration value of the provided key. Take note that this might be null if the + * configuration property is not required + */ + public static Object getConfig(ConfigProperty configKey) { + if (SINGLETON.configs == null) { + logger.debug( + "Failed to read config property [" + + configKey + + "] as agent is not initialized properly, config is null!"); + return null; + } + + return SINGLETON.configs.get(configKey); + } + + @SuppressWarnings("unchecked") + public static T getConfigOptional(ConfigProperty configKey, T defaultValue) { + if (SINGLETON.configs == null) { + logger.debug( + "Failed to read config property [" + + configKey + + "] as agent is not initialized properly, config is null!"); + return defaultValue; + } + Object value = SINGLETON.configs.get(configKey); + return value != null ? (T) value : defaultValue; + } + + public static ConfigContainer getConfigs(ConfigGroup... groups) { + if (SINGLETON.configs == null) { + logger.debug("Agent is not initialized properly, config is null!"); + return new ConfigContainer(); + } + + return SINGLETON.configs.subset(groups); + } +} diff --git a/libs/config/src/main/java/com/solarwinds/joboe/config/ConfigParser.java b/libs/config/src/main/java/com/solarwinds/joboe/config/ConfigParser.java new file mode 100644 index 00000000..b85d41f4 --- /dev/null +++ b/libs/config/src/main/java/com/solarwinds/joboe/config/ConfigParser.java @@ -0,0 +1,33 @@ +/* + * © SolarWinds Worldwide, LLC. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.solarwinds.joboe.config; + +public interface ConfigParser { + /** + * Convert the input of type T into result of R. If there are any error during the conversion, it + * should throw InvalidConfigException + * + * @param input + * @return + * @throws InvalidConfigException + */ + R convert(T input) throws InvalidConfigException; + + default String configKey() { + return ""; + } +} diff --git a/libs/config/src/main/java/com/solarwinds/joboe/config/ConfigProperty.java b/libs/config/src/main/java/com/solarwinds/joboe/config/ConfigProperty.java new file mode 100644 index 00000000..a687d304 --- /dev/null +++ b/libs/config/src/main/java/com/solarwinds/joboe/config/ConfigProperty.java @@ -0,0 +1,276 @@ +/* + * © SolarWinds Worldwide, LLC. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.solarwinds.joboe.config; + +import java.io.Serializable; +import java.util.HashMap; +import java.util.Map; +import lombok.Getter; + +/** + * Lists and describes the properties used in configurations with its value type (Java Class Type). + * + *

It also defines the corresponding keys of the properties when used as Agent Arguments (via + * -javaagent) or in configuration file. + * + *

Lookups are provided in order to retrieve the ConfigProperty from Agent Argument + * and from Key (property file key) + * + * @author Patson Luk + */ +public enum ConfigProperty { + AGENT_CONFIG( + new ConfigKey(null, EnvPrefix.PRODUCT + "CONFIG_FILE"), ConfigGroup.AGENT, String.class), + AGENT_EXPORT_LOGS_ENABLED( + new ConfigKey("agent.exportLogsEnabled", EnvPrefix.PRODUCT + "EXPORT_LOGS_ENABLED"), + ConfigGroup.AGENT, + Boolean.class), + AGENT_EXPORT_METRICS_ENABLED( + new ConfigKey("agent.exportMetricsEnabled", EnvPrefix.PRODUCT + "EXPORT_METRICS_ENABLED"), + ConfigGroup.AGENT, + Boolean.class), + AGENT_SPAN_STACKTRACE_FILTERS( + new ConfigKey("agent.spanStacktraceFilters", EnvPrefix.PRODUCT + "SPAN_STACKTRACE_FILTERS"), + ConfigGroup.AGENT, + String.class), + AGENT_TRANSACTION_NAME( + new ConfigKey(null, EnvPrefix.PRODUCT + "TRANSACTION_NAME"), ConfigGroup.AGENT, String.class), + AGENT_CONFIG_FILE_WATCH_PERIOD( + new ConfigKey("agent.configFileWatchPeriod"), ConfigGroup.AGENT, Long.class), + AGENT_EVENTS_SEND_CAPACITY( + new ConfigKey(null, EnvPrefix.PRODUCT + "EVENTS_SEND_CAPACITY"), + ConfigGroup.AGENT, + Integer.class), + AGENT_DEBUG(new ConfigKey(null, null), ConfigGroup.AGENT, Boolean.class), + AGENT_LOGGING( + new ConfigKey("agent.logging", EnvPrefix.PRODUCT + "DEBUG_LEVEL"), + ConfigGroup.AGENT, + String.class), + AGENT_TRACING_MODE(new ConfigKey("agent.tracingMode", null), ConfigGroup.AGENT, String.class), + AGENT_SERVICE_KEY( + new ConfigKey("agent.serviceKey", EnvPrefix.PRODUCT + "SERVICE_KEY"), + ConfigGroup.AGENT, + String.class), + AGENT_SQL_QUERY_MAX_LENGTH( + new ConfigKey("agent.sqlQueryMaxLength", EnvPrefix.PRODUCT + "MAX_SQL_QUERY_LENGTH"), + ConfigGroup.AGENT, + Integer.class), + AGENT_URL_SAMPLE_RATE(new ConfigKey("agent.urlSampleRates"), ConfigGroup.AGENT, String.class), + AGENT_TIME_ADJUST_INTERVAL( + new ConfigKey("agent.timeAdjustInterval"), ConfigGroup.AGENT, Integer.class), + AGENT_CONTEXT_TTL(new ConfigKey("agent.contextTtl"), ConfigGroup.AGENT, Integer.class), + AGENT_CONTEXT_MAX_EVENTS( + new ConfigKey("agent.contextMaxEvents"), ConfigGroup.AGENT, Integer.class), + AGENT_CONTEXT_MAX_BACKTRACES( + new ConfigKey("agent.contextMaxBacktraces"), ConfigGroup.AGENT, Integer.class), + AGENT_HOSTNAME_ALIAS( + new ConfigKey("agent.hostnameAlias", EnvPrefix.PRODUCT + "HOSTNAME_ALIAS"), + ConfigGroup.AGENT, + String.class), + AGENT_LOG_FILE( + new ConfigKey(null, EnvPrefix.PRODUCT + "JAVA_LOG_FILE"), ConfigGroup.AGENT, String.class), + AGENT_COLLECTOR( + new ConfigKey("agent.collector", EnvPrefix.PRODUCT + "COLLECTOR"), + ConfigGroup.AGENT, + String.class), + AGENT_COLLECTOR_SERVER_CERT_LOCATION( + new ConfigKey(null, EnvPrefix.PRODUCT + "TRUSTEDPATH"), ConfigGroup.AGENT, String.class), + AGENT_EVENTS_FLUSH_INTERVAL( + new ConfigKey(null, EnvPrefix.PRODUCT + "EVENTS_FLUSH_INTERVAL"), + ConfigGroup.AGENT, + Integer.class), + AGENT_TRANSACTION_NAME_PATTERN( + new ConfigKey("transaction.namePattern"), ConfigGroup.AGENT, String.class), + AGENT_DOMAIN_PREFIXED_TRANSACTION_NAME( + new ConfigKey("transaction.prependDomain"), ConfigGroup.AGENT, Boolean.class), + AGENT_TRANSACTION_SETTINGS( + new ConfigKey("agent.transactionSettings"), ConfigGroup.AGENT, String.class), + AGENT_TRANSACTION_NAMING_SCHEMES( + new ConfigKey("agent.transactionNameSchemes"), ConfigGroup.AGENT, String.class), + // AGENT_INTERNAL_TRANSACTION_SETTINGS should NOT be specified directly in the json file. This is + // used to store the result after comparing AGENT_TRANSACTION_SETTINGS and AGENT_URL_SAMPLE_RATE - + // not an ideal solution as this is confusing + AGENT_INTERNAL_TRANSACTION_SETTINGS( + new ConfigKey("agent.internal.transactionSettings"), ConfigGroup.AGENT, String.class), + AGENT_EC2_METADATA_TIMEOUT( + new ConfigKey("agent.ec2MetadataTimeout", EnvPrefix.PRODUCT + "EC2_METADATA_TIMEOUT"), + ConfigGroup.AGENT, + Integer.class), + AGENT_AZURE_VM_METADATA_TIMEOUT( + new ConfigKey( + "agent.azureVmMetadataTimeout", EnvPrefix.PRODUCT + "AZURE_VM_METADATA_TIMEOUT"), + ConfigGroup.AGENT, + Integer.class), + AGENT_AZURE_VM_METADATA_VERSION( + new ConfigKey( + "agent.azureVmMetadataVersion", EnvPrefix.PRODUCT + "AZURE_VM_METADATA_VERSION"), + ConfigGroup.AGENT, + String.class), + AGENT_COLLECTOR_TIMEOUT( + new ConfigKey("agent.collectorTimeout", EnvPrefix.PRODUCT + "COLLECTOR_TIMEOUT"), + ConfigGroup.AGENT, + Integer.class), + AGENT_TRIGGER_TRACE_ENABLED( + new ConfigKey("agent.triggerTrace", EnvPrefix.PRODUCT + "TRIGGER_TRACE"), + ConfigGroup.AGENT, + String.class), + AGENT_PROXY( + new ConfigKey("agent.proxy", EnvPrefix.PRODUCT + "PROXY"), ConfigGroup.AGENT, String.class), + AGENT_GRPC_COMPRESSION( + new ConfigKey(null, EnvPrefix.PRODUCT + "GRPC_COMPRESSION"), + ConfigGroup.AGENT, + String.class), // not advertised + AGENT_SQL_TAG( + new ConfigKey("agent.sqlTag", EnvPrefix.PRODUCT + "SQL_TAG"), + ConfigGroup.AGENT, + Boolean.class), + AGENT_SQL_TAG_PREPARED( + new ConfigKey("agent.sqlTagPrepared", EnvPrefix.PRODUCT + "SQL_TAG_PREPARED"), + ConfigGroup.AGENT, + Boolean.class), + AGENT_SQL_TAG_DATABASES( + new ConfigKey("agent.sqlTagDatabases", EnvPrefix.PRODUCT + "SQL_TAG_DATABASES"), + ConfigGroup.AGENT, + String.class), + MONITOR_JMX_SCOPES(new ConfigKey("monitor.jmx.scopes"), ConfigGroup.MONITOR, String.class), + MONITOR_JMX_ENABLE(new ConfigKey("monitor.jmx.enable"), ConfigGroup.MONITOR, Boolean.class), + MONITOR_JMX_MAX_ENTRY(new ConfigKey("monitor.jmx.maxEntry"), ConfigGroup.MONITOR, Integer.class), + MONITOR_METRICS_FLUSH_INTERVAL( + new ConfigKey(null, EnvPrefix.PRODUCT + "METRICS_FLUSH_INTERVAL"), + ConfigGroup.MONITOR, + Integer.class), + + MONITOR_SPAN_METRICS_ENABLE( + new ConfigKey("monitor.spanMetrics.enable", EnvPrefix.PRODUCT + "SPAN_METRICS_ENABLE"), + ConfigGroup.MONITOR, + Boolean.class), + + PROFILER(new ConfigKey("profiler"), ConfigGroup.PROFILER, String.class), + PROFILER_ENABLED_ENV_VAR( + new ConfigKey(null, EnvPrefix.PRODUCT + "PROFILER_ENABLED"), + ConfigGroup.PROFILER, + Boolean.class), + PROFILER_INTERVAL_ENV_VAR( + new ConfigKey(null, EnvPrefix.PRODUCT + "PROFILER_INTERVAL"), + ConfigGroup.PROFILER, + Integer.class); + private final ConfigKey configKey; + + /** + * -- GETTER -- + * + * @return the Java Class of the property value + */ + @Getter private final Class typeClass; + + private final ConfigGroup group; + @Getter private ConfigParser configParser; + + public static class EnvPrefix { + public static final String PRODUCT = "SW_APM_"; + } + + private static class ConfigKey { + private final String configFileKey; + private final String environmentVariableKey; + + public ConfigKey(String configFileKey) { + this(configFileKey, null); + } + + public ConfigKey(String configFileKey, String environmentVariableKey) { + super(); + this.configFileKey = configFileKey; + this.environmentVariableKey = environmentVariableKey; + } + } + + /** + * @param configKey keys used to map to this property + * @param group + * @param typeClass the Java Class of the property value + */ + ConfigProperty(ConfigKey configKey, ConfigGroup group, Class typeClass) { + this.configKey = configKey; + this.typeClass = typeClass; + this.group = group; + registerLookup(this); + } + + public void setParser(ConfigParser configParser) { + this.configParser = configParser; + } + + private static void registerLookup(ConfigProperty property) { + String configFileKey = property.configKey.configFileKey; + if (configFileKey != null) { + // put the key into the lookup map + ConfigPropertyRegistry.CONFIG_FILE_KEY_TO_PARAMETER.put(configFileKey, property); + } + + String environmentVariableKey = property.configKey.environmentVariableKey; + if (environmentVariableKey != null) { + // put the key into the lookup map + ConfigPropertyRegistry.ENVIRONMENT_VARIABLE_KEY_TO_PARAMETER.put( + environmentVariableKey, property); + } + } + + /** + * @param key the key used in configuration property file + * @return the corresponding ConfigProperty by the key. Null if the property is not defined under + * that key + */ + public static ConfigProperty fromConfigFileKey(String key) { + return ConfigPropertyRegistry.CONFIG_FILE_KEY_TO_PARAMETER.get(key); + } + + public String getConfigFileKey() { + return configKey.configFileKey; + } + + public String getEnvironmentVariableKey() { + return configKey.environmentVariableKey; + } + + /** + * @param environmentVariableKey the environment variable key used + * @return the corresponding ConfigProperty by the environment variable key used. Null if the + * property is not defined under that environment variable key + */ + public static ConfigProperty fromEnvironmentVariableKey(String environmentVariableKey) { + return ConfigPropertyRegistry.ENVIRONMENT_VARIABLE_KEY_TO_PARAMETER.get(environmentVariableKey); + } + + public static Map getEnviromentVariableMap() { + return ConfigPropertyRegistry.ENVIRONMENT_VARIABLE_KEY_TO_PARAMETER; + } + + /** + * @return the Grouping of this ConfigProperty + */ + ConfigGroup getGroup() { + return group; + } + + static class ConfigPropertyRegistry { + private static final Map CONFIG_FILE_KEY_TO_PARAMETER = + new HashMap(); + private static final Map ENVIRONMENT_VARIABLE_KEY_TO_PARAMETER = + new HashMap(); + } +} diff --git a/libs/config/src/main/java/com/solarwinds/joboe/config/ConfigReader.java b/libs/config/src/main/java/com/solarwinds/joboe/config/ConfigReader.java new file mode 100644 index 00000000..8b65bb42 --- /dev/null +++ b/libs/config/src/main/java/com/solarwinds/joboe/config/ConfigReader.java @@ -0,0 +1,38 @@ +/* + * © SolarWinds Worldwide, LLC. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.solarwinds.joboe.config; + +import com.solarwinds.joboe.logging.Logger; +import com.solarwinds.joboe.logging.LoggerFactory; +import lombok.Getter; + +public abstract class ConfigReader { + protected final Logger logger = LoggerFactory.getLogger(); + @Getter private final ConfigSourceType configSourceType; + + protected ConfigReader(ConfigSourceType configSourceType) { + this.configSourceType = configSourceType; + } + + /** + * Reads the configuration and puts the result in {@link ConfigContainer} + * + * @param container the container which this config reader should write result into + * @throws Exception + */ + public abstract void read(ConfigContainer container) throws InvalidConfigException; +} diff --git a/libs/config/src/main/java/com/solarwinds/joboe/config/ConfigSourceType.java b/libs/config/src/main/java/com/solarwinds/joboe/config/ConfigSourceType.java new file mode 100644 index 00000000..8e449b1c --- /dev/null +++ b/libs/config/src/main/java/com/solarwinds/joboe/config/ConfigSourceType.java @@ -0,0 +1,22 @@ +/* + * © SolarWinds Worldwide, LLC. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.solarwinds.joboe.config; + +public enum ConfigSourceType { + ENV_VAR, + JSON_FILE +} diff --git a/libs/config/src/main/java/com/solarwinds/joboe/config/EnvConfigReader.java b/libs/config/src/main/java/com/solarwinds/joboe/config/EnvConfigReader.java new file mode 100644 index 00000000..058beaa5 --- /dev/null +++ b/libs/config/src/main/java/com/solarwinds/joboe/config/EnvConfigReader.java @@ -0,0 +1,77 @@ +/* + * © SolarWinds Worldwide, LLC. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.solarwinds.joboe.config; + +import java.util.ArrayList; +import java.util.List; +import java.util.Map; +import java.util.Map.Entry; + +/** + * Reads from system environment variables for {@link ConfigProperty} + * + * @author Patson Luk + */ +public class EnvConfigReader extends ConfigReader { + private final Map env; + + public EnvConfigReader(Map env) { + super(ConfigSourceType.ENV_VAR); + this.env = env; + } + + @Override + public void read(ConfigContainer container) throws InvalidConfigException { + List exceptions = new ArrayList(); + for (Entry envNameEntry : + ConfigProperty.getEnviromentVariableMap().entrySet()) { + String envName = envNameEntry.getKey(); + if (env.containsKey(envName)) { + String value = env.get(envName); + try { + container.putByStringValue(envNameEntry.getValue(), value); + + String maskedValue; + if (envNameEntry.getValue() == ConfigProperty.AGENT_SERVICE_KEY) { + maskedValue = ServiceKeyUtils.maskServiceKey(value); + } else { + maskedValue = value; + } + logger.info( + "System environment variable [" + + envName + + "] value [" + + maskedValue + + "] maps to agent property " + + envNameEntry.getValue()); + } catch (InvalidConfigException e) { + logger.warn( + "Invalid System environment variable [" + envName + "] value [" + value + "]"); + exceptions.add(e); + } + } + } + + if (!exceptions.isEmpty()) { + logger.warn( + "Found " + + exceptions.size() + + " exception(s) while reading config from environment variables"); + throw exceptions.get(0); // report the first exception encountered + } + } +} diff --git a/libs/config/src/main/java/com/solarwinds/joboe/config/InvalidConfigException.java b/libs/config/src/main/java/com/solarwinds/joboe/config/InvalidConfigException.java new file mode 100644 index 00000000..76c480e0 --- /dev/null +++ b/libs/config/src/main/java/com/solarwinds/joboe/config/InvalidConfigException.java @@ -0,0 +1,59 @@ +/* + * © SolarWinds Worldwide, LLC. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.solarwinds.joboe.config; + +import lombok.Getter; + +public class InvalidConfigException extends Exception { + private static final long serialVersionUID = 1L; + protected String originalMessage; + @Getter protected ConfigProperty configProperty; + + public InvalidConfigException(String message) { + this(message, null); + } + + public InvalidConfigException(Throwable cause) { + this(null, cause); + } + + public InvalidConfigException(String message, Throwable cause) { + this(null, message, cause); + } + + public InvalidConfigException(ConfigProperty configProperty, String message) { + this(configProperty, message, null); + } + + public InvalidConfigException(ConfigProperty configProperty, String message, Throwable cause) { + super(message, cause); + this.originalMessage = message; + this.configProperty = configProperty; + } + + @Override + public String getMessage() { + if (configProperty == null) { + return super.getMessage(); + } else { + return "Found error in config. Config key: " + + configProperty.name() + + " " + + super.getMessage(); + } + } +} diff --git a/libs/config/src/main/java/com/solarwinds/joboe/config/InvalidConfigReadSourceException.java b/libs/config/src/main/java/com/solarwinds/joboe/config/InvalidConfigReadSourceException.java new file mode 100644 index 00000000..87a3b9d7 --- /dev/null +++ b/libs/config/src/main/java/com/solarwinds/joboe/config/InvalidConfigReadSourceException.java @@ -0,0 +1,91 @@ +/* + * © SolarWinds Worldwide, LLC. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.solarwinds.joboe.config; + +import lombok.Getter; + +/** + * Invalid config while reading from a specific {@Link ConfigSourceType} + * + *

This contains extra info on the source type read and a ConfigContainer with the config read so + * far + */ +public class InvalidConfigReadSourceException extends InvalidConfigException { + private static final long serialVersionUID = 1L; + private final ConfigSourceType configSourceType; + @Getter private ConfigContainer configContainerBeforeException = null; + private String physicalLocation = null; + + public InvalidConfigReadSourceException( + ConfigProperty configProperty, + ConfigSourceType sourceType, + String physicalLocation, + ConfigContainer configContainerBeforeException, + InvalidConfigException exception) { + super(configProperty, exception.originalMessage, exception); + this.physicalLocation = physicalLocation; + this.configSourceType = sourceType; + this.configContainerBeforeException = configContainerBeforeException; + this.configProperty = exception.getConfigProperty(); + } + + @Override + public String getMessage() { + if (configProperty == null && configSourceType == null) { + return super.getMessage(); + } else { + StringBuilder message = new StringBuilder("Found error in config. "); + if (configSourceType != null) { + message.append("Location: " + getLocation(configSourceType, physicalLocation) + "."); + } + if (configProperty != null) { + message.append( + "Config key: " + + (configSourceType != null + ? getConfigPropertyLabel(configSourceType, configProperty) + : configProperty.name()) + + "."); + } + + message.append(" " + originalMessage); + return message.toString(); + } + } + + private static String getConfigPropertyLabel( + ConfigSourceType configSourceType, ConfigProperty configProperty) { + switch (configSourceType) { + case ENV_VAR: + return configProperty.getEnvironmentVariableKey(); + case JSON_FILE: + return configProperty.getConfigFileKey(); + default: + return configProperty.name(); + } + } + + private static String getLocation(ConfigSourceType configSourceType, String physicalLocation) { + switch (configSourceType) { + case ENV_VAR: + return "Environment variable"; + case JSON_FILE: + return "JSON config file at " + physicalLocation; + default: + return "Unknown location"; + } + } +} diff --git a/libs/config/src/main/java/com/solarwinds/joboe/config/InvalidConfigServiceKeyException.java b/libs/config/src/main/java/com/solarwinds/joboe/config/InvalidConfigServiceKeyException.java new file mode 100644 index 00000000..b273500d --- /dev/null +++ b/libs/config/src/main/java/com/solarwinds/joboe/config/InvalidConfigServiceKeyException.java @@ -0,0 +1,35 @@ +/* + * © SolarWinds Worldwide, LLC. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.solarwinds.joboe.config; + +public class InvalidConfigServiceKeyException extends InvalidConfigException { + + /** */ + private static final long serialVersionUID = 1L; + + public InvalidConfigServiceKeyException(String message) { + super(message); + } + + public InvalidConfigServiceKeyException(String message, Throwable cause) { + super(message, cause); + } + + public InvalidConfigServiceKeyException(Throwable cause) { + super(cause); + } +} diff --git a/libs/config/src/main/java/com/solarwinds/joboe/config/JavaRuntimeVersionChecker.java b/libs/config/src/main/java/com/solarwinds/joboe/config/JavaRuntimeVersionChecker.java new file mode 100644 index 00000000..deafe44c --- /dev/null +++ b/libs/config/src/main/java/com/solarwinds/joboe/config/JavaRuntimeVersionChecker.java @@ -0,0 +1,29 @@ +/* + * © SolarWinds Worldwide, LLC. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.solarwinds.joboe.config; + +public class JavaRuntimeVersionChecker { + public static final String minVersionSupported = "1.8.0_252"; + + public static boolean isJdkVersionSupported() { + return isJdkVersionGreaterOrEqualToRef(minVersionSupported, System.getProperty("java.version")); + } + + public static boolean isJdkVersionGreaterOrEqualToRef(String reference, String version) { + return JavaVersionComparator.compare(reference, version) <= 0; + } +} diff --git a/libs/config/src/main/java/com/solarwinds/joboe/config/JavaVersionComparator.java b/libs/config/src/main/java/com/solarwinds/joboe/config/JavaVersionComparator.java new file mode 100644 index 00000000..99003f40 --- /dev/null +++ b/libs/config/src/main/java/com/solarwinds/joboe/config/JavaVersionComparator.java @@ -0,0 +1,57 @@ +/* + * © SolarWinds Worldwide, LLC. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.solarwinds.joboe.config; + +import java.util.ArrayList; +import java.util.List; + +public class JavaVersionComparator { + public static int compare(String v1, String v2) { + return new Version(v1).compare(new Version(v2)); + } + + private static class Version { + List versions = new ArrayList<>(); + + Version(String v) { + String[] arr = v.split("[_.]"); + for (String ver : arr) { + try { + versions.add(Integer.valueOf(ver)); + } catch (NumberFormatException e) { + versions.add(0); + } + } + } + + int compare(Version other) { + int selfIdx = 0, otherIdx = 0; + while (selfIdx < versions.size() && otherIdx < other.versions.size()) { + int selfVersion = versions.get(selfIdx); + int otherVersion = other.versions.get(otherIdx); + + if (selfVersion != otherVersion) { + return selfVersion - otherVersion; + } + selfIdx++; + otherIdx++; + } + if (selfIdx == versions.size() && otherIdx == other.versions.size()) return 0; + return selfIdx == versions.size() ? -1 : 1; + } + } +} diff --git a/libs/config/src/main/java/com/solarwinds/joboe/config/JsonConfigReader.java b/libs/config/src/main/java/com/solarwinds/joboe/config/JsonConfigReader.java new file mode 100644 index 00000000..0cad2702 --- /dev/null +++ b/libs/config/src/main/java/com/solarwinds/joboe/config/JsonConfigReader.java @@ -0,0 +1,95 @@ +/* + * © SolarWinds Worldwide, LLC. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.solarwinds.joboe.config; + +import java.io.InputStream; +import java.util.ArrayList; +import java.util.List; +import org.json.JSONException; +import org.json.JSONObject; +import org.json.JSONTokener; + +/** + * A Reader that reads the input config file in JSON + * + * @author Patson Luk + */ +public class JsonConfigReader extends ConfigReader { + private final InputStream configStream; + + /** + * @param configStream The input stream of the configuration file + */ + public JsonConfigReader(InputStream configStream) { + super(ConfigSourceType.JSON_FILE); + this.configStream = configStream; + } + + @Override + public void read(ConfigContainer container) throws InvalidConfigException { + if (configStream == null) { + throw new InvalidConfigException("Cannot find any valid configuration for agent"); + } + + List exceptions = new ArrayList(); + JSONObject jsonObject; + try { + jsonObject = new JSONObject(new JSONTokener(configStream)); + + for (Object keyAsObject : jsonObject.keySet()) { + ConfigProperty key = + ConfigProperty.fromConfigFileKey( + (String) + keyAsObject); // attempt to retrieve the corresponding ConfigProperty as key + + if (key == null) { + exceptions.add( + new InvalidConfigException( + "Invalid line in configuration file : key [" + keyAsObject + "] is invalid")); + } else { + try { + Object value = jsonObject.get((String) keyAsObject); + if (value != null) { + container.putByStringValue(key, value.toString()); + } else { + // should not be null since it is read from JSON file + exceptions.add(new InvalidConfigException(key, "Unexpected null value", null)); + } + } catch (JSONException e) { + exceptions.add( + new InvalidConfigException( + "Json exception while processing config for key [" + + keyAsObject + + "] : " + + e.getMessage(), + e)); + } + } + } + } catch (JSONException e) { + exceptions.add( + new InvalidConfigException( + "Json exception while processing config : " + e.getMessage(), e)); + } + + if (!exceptions.isEmpty()) { + logger.warn( + "Found " + exceptions.size() + " exception(s) while reading config from config file"); + throw exceptions.get(0); // report the first exception encountered + } + } +} diff --git a/libs/config/src/main/java/com/solarwinds/joboe/config/LogTraceIdScope.java b/libs/config/src/main/java/com/solarwinds/joboe/config/LogTraceIdScope.java new file mode 100644 index 00000000..e03564b2 --- /dev/null +++ b/libs/config/src/main/java/com/solarwinds/joboe/config/LogTraceIdScope.java @@ -0,0 +1,23 @@ +/* + * © SolarWinds Worldwide, LLC. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.solarwinds.joboe.config; + +public enum LogTraceIdScope { + ENABLED, + DISABLED, + SAMPLED_ONLY +} diff --git a/libs/config/src/main/java/com/solarwinds/joboe/config/LogTraceIdSetting.java b/libs/config/src/main/java/com/solarwinds/joboe/config/LogTraceIdSetting.java new file mode 100644 index 00000000..1520119f --- /dev/null +++ b/libs/config/src/main/java/com/solarwinds/joboe/config/LogTraceIdSetting.java @@ -0,0 +1,37 @@ +/* + * © SolarWinds Worldwide, LLC. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.solarwinds.joboe.config; + +import lombok.Getter; + +/** + * Setting that contains scope information for each log injection category. + * + *

There are 2 categories right now - "autoInsert" and "mdc" + * + * @author pluk + */ +@Getter +public class LogTraceIdSetting { + private final LogTraceIdScope autoInsertScope; + private final LogTraceIdScope mdcScope; + + public LogTraceIdSetting(LogTraceIdScope autoInsertScope, LogTraceIdScope mdcScope) { + this.autoInsertScope = autoInsertScope; + this.mdcScope = mdcScope; + } +} diff --git a/libs/config/src/main/java/com/solarwinds/joboe/config/ProxyConfig.java b/libs/config/src/main/java/com/solarwinds/joboe/config/ProxyConfig.java new file mode 100644 index 00000000..4ae29d58 --- /dev/null +++ b/libs/config/src/main/java/com/solarwinds/joboe/config/ProxyConfig.java @@ -0,0 +1,34 @@ +/* + * © SolarWinds Worldwide, LLC. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.solarwinds.joboe.config; + +import lombok.Getter; + +@Getter +public class ProxyConfig { + private final String host; + private final int port; + private final String username; + private final String password; + + public ProxyConfig(String host, int port, String username, String password) { + this.host = host; + this.port = port; + this.username = username; + this.password = password; + } +} diff --git a/libs/config/src/main/java/com/solarwinds/joboe/config/ServiceKeyUtils.java b/libs/config/src/main/java/com/solarwinds/joboe/config/ServiceKeyUtils.java new file mode 100644 index 00000000..0f888bb3 --- /dev/null +++ b/libs/config/src/main/java/com/solarwinds/joboe/config/ServiceKeyUtils.java @@ -0,0 +1,76 @@ +/* + * © SolarWinds Worldwide, LLC. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.solarwinds.joboe.config; + +public class ServiceKeyUtils { + public static final char SEPARATOR_CHARACTER = + ':'; // char that separates customer key and service name within the service key + public static final int SERVICE_NAME_MAX_LENGTH = 255; + + public static String maskServiceKey(String serviceKey) { + if (serviceKey == null) { + return serviceKey; + } + + final int headCharacterCount = 4; + final int tailCharacterCount = 4; + final char maskCharacter = '*'; + + int separatorIndex = serviceKey.indexOf(SEPARATOR_CHARACTER); + String serviceName = getServiceName(serviceKey); + String customerKey = + separatorIndex != -1 ? serviceKey.substring(0, separatorIndex) : serviceKey; + + if (customerKey.length() > headCharacterCount + tailCharacterCount) { + StringBuilder mask = new StringBuilder(); + for (int i = 0; i < customerKey.length() - (headCharacterCount + tailCharacterCount); i++) { + mask.append(maskCharacter); + } + + customerKey = + customerKey.substring(0, headCharacterCount) + + mask + + customerKey.substring(customerKey.length() - tailCharacterCount); + } + + return serviceName != null ? customerKey + SEPARATOR_CHARACTER + serviceName : customerKey; + } + + public static String getServiceName(String serviceKey) { + int separatorIndex = serviceKey.indexOf(SEPARATOR_CHARACTER); + return separatorIndex != -1 ? serviceKey.substring(separatorIndex + 1) : null; + } + + public static String getApiKey(String serviceKey) { + int separatorIndex = serviceKey.indexOf(SEPARATOR_CHARACTER); + return separatorIndex != -1 ? serviceKey.substring(0, separatorIndex) : serviceKey; + } + + public static String transformServiceKey(String serviceKey) { + String serviceName = getServiceName(serviceKey); + + if (serviceName != null) { + serviceName = serviceName.toLowerCase().replaceAll("\\s", "-").replaceAll("[^\\w.:_-]", ""); + if (serviceName.length() > SERVICE_NAME_MAX_LENGTH) { + serviceName = serviceName.substring(0, SERVICE_NAME_MAX_LENGTH); + } + return getApiKey(serviceKey) + SEPARATOR_CHARACTER + serviceName; + } else { + return serviceKey; + } + } +} diff --git a/libs/config/src/test/java/com/solarwinds/joboe/config/ConfigPropertyTest.java b/libs/config/src/test/java/com/solarwinds/joboe/config/ConfigPropertyTest.java new file mode 100644 index 00000000..4cdedbc7 --- /dev/null +++ b/libs/config/src/test/java/com/solarwinds/joboe/config/ConfigPropertyTest.java @@ -0,0 +1,49 @@ +/* + * © SolarWinds Worldwide, LLC. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.solarwinds.joboe.config; + +import static org.junit.jupiter.api.Assertions.assertEquals; + +import java.util.ArrayList; +import java.util.Arrays; +import java.util.List; +import org.junit.jupiter.api.Test; + +public class ConfigPropertyTest { + + @Test + public void testThatKeysAreNotDuplicated() { + List fileKey = new ArrayList<>(); + List envKey = new ArrayList<>(); + + Arrays.stream(ConfigProperty.values()) + .forEach( + configProperty -> { + if (configProperty.getConfigFileKey() != null) + fileKey.add(configProperty.getConfigFileKey()); + + if (configProperty.getEnvironmentVariableKey() != null) + envKey.add(configProperty.getEnvironmentVariableKey()); + }); + + long fileKeyCount = fileKey.stream().distinct().count(); + long envKeyCount = envKey.stream().distinct().count(); + + assertEquals(fileKey.size(), fileKeyCount); + assertEquals(envKey.size(), envKeyCount); + } +} diff --git a/libs/config/src/test/java/com/solarwinds/joboe/config/EnvConfigReaderTest.java b/libs/config/src/test/java/com/solarwinds/joboe/config/EnvConfigReaderTest.java new file mode 100644 index 00000000..d8fef1ff --- /dev/null +++ b/libs/config/src/test/java/com/solarwinds/joboe/config/EnvConfigReaderTest.java @@ -0,0 +1,55 @@ +/* + * © SolarWinds Worldwide, LLC. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.solarwinds.joboe.config; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.fail; + +import java.util.HashMap; +import java.util.Map; +import org.junit.jupiter.api.Test; + +public class EnvConfigReaderTest { + @Test + public void testValidRead() throws InvalidConfigException { + Map vars = new HashMap(); + vars.put(ConfigProperty.EnvPrefix.PRODUCT + "SERVICE_KEY", "some key"); + EnvConfigReader reader = new EnvConfigReader(vars); + ConfigContainer container = new ConfigContainer(); + reader.read(container); + + assertEquals("some key", container.get(ConfigProperty.AGENT_SERVICE_KEY)); + } + + /** Even if some values are invalid, it should still read the rest */ + @Test + public void testPartialRead() { + Map vars = new HashMap(); + vars.put(ConfigProperty.EnvPrefix.PRODUCT + "SERVICE_KEY", "some key"); + vars.put(ConfigProperty.EnvPrefix.PRODUCT + "MAX_SQL_QUERY_LENGTH", "2.1"); + EnvConfigReader reader = new EnvConfigReader(vars); + ConfigContainer container = new ConfigContainer(); + try { + reader.read(container); + fail("Expected " + InvalidConfigException.class.getName() + " but it's not thrown"); + } catch (InvalidConfigException e) { + // expected + } + + assertEquals("some key", container.get(ConfigProperty.AGENT_SERVICE_KEY)); + } +} diff --git a/libs/config/src/test/java/com/solarwinds/joboe/config/JavaRuntimeVersionCheckerTest.java b/libs/config/src/test/java/com/solarwinds/joboe/config/JavaRuntimeVersionCheckerTest.java new file mode 100644 index 00000000..d44b91c0 --- /dev/null +++ b/libs/config/src/test/java/com/solarwinds/joboe/config/JavaRuntimeVersionCheckerTest.java @@ -0,0 +1,47 @@ +/* + * © SolarWinds Worldwide, LLC. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.solarwinds.joboe.config; + +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertTrue; + +import org.junit.jupiter.api.Test; +import org.junitpioneer.jupiter.SetSystemProperty; + +class JavaRuntimeVersionCheckerTest { + + @Test + @SetSystemProperty(key = "java.version", value = "1.8.0_250") + void returnFalse() { + boolean jdkVersionSupported = JavaRuntimeVersionChecker.isJdkVersionSupported(); + assertFalse(jdkVersionSupported); + } + + @Test + void returnTrueWhenReferenceAndVersionAreTheSame() { + boolean jdkVersionSupported = + JavaRuntimeVersionChecker.isJdkVersionGreaterOrEqualToRef("11", "11"); + assertTrue(jdkVersionSupported); + } + + @Test + void returnTrueWhenVersionIsGreater() { + boolean jdkVersionSupported = + JavaRuntimeVersionChecker.isJdkVersionGreaterOrEqualToRef("11", "17"); + assertTrue(jdkVersionSupported); + } +} diff --git a/libs/config/src/test/java/com/solarwinds/joboe/config/JsonConfigReaderTest.java b/libs/config/src/test/java/com/solarwinds/joboe/config/JsonConfigReaderTest.java new file mode 100644 index 00000000..5567d7b3 --- /dev/null +++ b/libs/config/src/test/java/com/solarwinds/joboe/config/JsonConfigReaderTest.java @@ -0,0 +1,53 @@ +/* + * © SolarWinds Worldwide, LLC. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.solarwinds.joboe.config; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.fail; + +import org.junit.jupiter.api.Test; + +public class JsonConfigReaderTest { + @Test + public void testValidRead() throws InvalidConfigException { + JsonConfigReader reader = + new JsonConfigReader(JsonConfigReaderTest.class.getResourceAsStream("/valid.json")); + ConfigContainer container = new ConfigContainer(); + reader.read(container); + + assertEquals("info", container.get(ConfigProperty.AGENT_LOGGING)); + assertEquals("some key", container.get(ConfigProperty.AGENT_SERVICE_KEY)); + } + + /** Even if some values are invalid, it should still read the rest */ + @Test + public void testPartialRead() { + JsonConfigReader reader = + new JsonConfigReader(JsonConfigReaderTest.class.getResourceAsStream("/invalid.json")); + ConfigContainer container = new ConfigContainer(); + try { + reader.read(container); + fail("Expected " + InvalidConfigException.class.getName() + " but it's not thrown"); + } catch (InvalidConfigException e) { + // expected + } + + // the rest of the values should be read + assertEquals("info", container.get(ConfigProperty.AGENT_LOGGING)); + assertEquals("some key", container.get(ConfigProperty.AGENT_SERVICE_KEY)); + } +} diff --git a/libs/config/src/test/resources/invalid.json b/libs/config/src/test/resources/invalid.json new file mode 100644 index 00000000..fc53021e --- /dev/null +++ b/libs/config/src/test/resources/invalid.json @@ -0,0 +1,8 @@ +{ + "unknownKey": "abc", + "agent.aaa": "abc", + "agent.serviceKey":"some key", + "agent.logging":"info", + "agent.jdbcInstAll":false +} + diff --git a/libs/config/src/test/resources/valid.json b/libs/config/src/test/resources/valid.json new file mode 100644 index 00000000..c4b60778 --- /dev/null +++ b/libs/config/src/test/resources/valid.json @@ -0,0 +1,5 @@ +{ + "agent.serviceKey":"some key", + "agent.logging":"info" +} + diff --git a/libs/core/.gitignore b/libs/core/.gitignore new file mode 100644 index 00000000..567609b1 --- /dev/null +++ b/libs/core/.gitignore @@ -0,0 +1 @@ +build/ diff --git a/libs/core/build.gradle.kts b/libs/core/build.gradle.kts new file mode 100644 index 00000000..99c4abfe --- /dev/null +++ b/libs/core/build.gradle.kts @@ -0,0 +1,58 @@ +plugins { + id("solarwinds.java-conventions") +} + +description = "core" + +dependencies { + implementation(project(":libs:logging")) + implementation(project(":libs:config")) + implementation(project(":libs:sampling")) + + implementation("io.grpc:grpc-netty:1.79.0") + implementation("io.grpc:grpc-stub:1.79.0") + implementation("io.grpc:grpc-protobuf:1.79.0") + + compileOnly("org.json:json") + compileOnly("io.opentelemetry:opentelemetry-api") + compileOnly("io.opentelemetry:opentelemetry-context") + + implementation("javax.xml.bind:jaxb-api:2.3.1") + + implementation("com.solarwinds:apm-proto:1.0.8") { + exclude(group = "com.google.guava", module = "guava") + exclude(group = "io.grpc") + } + + compileOnly("com.google.auto.service:auto-service") + annotationProcessor("com.google.auto.service:auto-service") + + testImplementation("org.json:json") + testImplementation("io.opentelemetry:opentelemetry-api") + testImplementation("io.opentelemetry:opentelemetry-context") +} + +sourceSets { + main { java { exclude("**/*.template") } } +} + +tasks.withType().configureEach { + exclude("**/*.template") +} + +tasks.named("compileJava") { + // Disable AutoService verify check to prevent rawtypes warnings for generic service provider interfaces + options.compilerArgs.add("-Averify=false") +} + +tasks.withType().configureEach { + // Suppress warnings from migrated/vendored legacy code (hdrHistogram, ebson, etc.) + options.compilerArgs.addAll( + listOf( + "-Xlint:-serial", + "-Xlint:-rawtypes", + "-Xlint:-dep-ann", + "-Xlint:-deprecation" + ) + ) +} diff --git a/libs/core/src/main/java/com/solarwinds/joboe/core/AtomicEventReporterStats.java b/libs/core/src/main/java/com/solarwinds/joboe/core/AtomicEventReporterStats.java new file mode 100644 index 00000000..211c5b33 --- /dev/null +++ b/libs/core/src/main/java/com/solarwinds/joboe/core/AtomicEventReporterStats.java @@ -0,0 +1,81 @@ +/* + * © SolarWinds Worldwide, LLC. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.solarwinds.joboe.core; + +import java.util.concurrent.atomic.AtomicLong; +import java.util.function.Supplier; + +/** + * Stats on reporter + * + * @author pluk + */ +public class AtomicEventReporterStats { + private final AtomicLong sentCount = new AtomicLong(); + private final AtomicLong overflowedCount = new AtomicLong(); + private final AtomicLong failedCount = new AtomicLong(); + private final AtomicLong queueLargestCount = new AtomicLong(); + private final AtomicLong processedCount = new AtomicLong(); + + private final Supplier queueSizeSupplier; + + public AtomicEventReporterStats(Supplier queueSizeSupplier) { + this.queueSizeSupplier = queueSizeSupplier; + } + + public void incrementSentCount(long increment) { + sentCount.addAndGet(increment); + } + + public void incrementOverflowedCount(long increment) { + overflowedCount.addAndGet(increment); + } + + public void incrementFailedCount(long increment) { + failedCount.addAndGet(increment); + } + + public void incrementProcessedCount(long increment) { + processedCount.addAndGet(increment); + } + + public void setQueueCount(long currentCount) { + synchronized (queueLargestCount) { + if (currentCount > queueLargestCount.get()) { + queueLargestCount.set(currentCount); + } + } + } + + public EventReporterStats consumeStats() { + long sentCount = this.sentCount.getAndSet(0); + long overflowedCount = this.overflowedCount.getAndSet(0); + long failedCount = this.failedCount.getAndSet(0); + + long queueLargestCount = + this.queueLargestCount.getAndSet( + queueSizeSupplier.get()); // reset to current queue size as the largest + long processedCount = this.processedCount.getAndSet(0); + return EventReporterStats.builder() + .sentCount(sentCount) + .failedCount(failedCount) + .overflowedCount(overflowedCount) + .queueLargestCount(queueLargestCount) + .processedCount(processedCount) + .build(); + } +} diff --git a/libs/core/src/main/java/com/solarwinds/joboe/core/BsonBufferException.java b/libs/core/src/main/java/com/solarwinds/joboe/core/BsonBufferException.java new file mode 100644 index 00000000..d4c2ac2a --- /dev/null +++ b/libs/core/src/main/java/com/solarwinds/joboe/core/BsonBufferException.java @@ -0,0 +1,23 @@ +/* + * © SolarWinds Worldwide, LLC. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.solarwinds.joboe.core; + +public class BsonBufferException extends Exception { + public BsonBufferException(Throwable cause) { + super(cause); + } +} diff --git a/libs/core/src/main/java/com/solarwinds/joboe/core/Constants.java b/libs/core/src/main/java/com/solarwinds/joboe/core/Constants.java new file mode 100644 index 00000000..6fe04568 --- /dev/null +++ b/libs/core/src/main/java/com/solarwinds/joboe/core/Constants.java @@ -0,0 +1,49 @@ +/* + * © SolarWinds Worldwide, LLC. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.solarwinds.joboe.core; + +/** Constants used throughout jboe code (from oboe.h) */ +public class Constants { + + public static final int TASK_ID_LEN = 16, + OP_ID_LEN = 8, + MAX_METADATA_PACK_LEN = 512, + MASK_TASK_ID_LEN = 0x03, + MASK_OP_ID_LEN = 0x08, + MASK_HAS_OPTIONS = 0x04, // unused? + + // MAX_UDP_PKT_SZ = 65507, // (65535 max IP packet size - 20 IPv4 header - 8 UDP + // header) + MAX_EVENT_BUFFER_SIZE = + 512 * 1024, // 512kB. This should not be bound by UDP size anymore with SSL reporting, + // though we still want to have some limit + MAX_BACK_TRACE_TOP_LINE_COUNT = 100, + MAX_BACK_TRACE_BOTTOM_LINE_COUNT = 20, + MAX_BACK_TRACE_LINE_COUNT = MAX_BACK_TRACE_TOP_LINE_COUNT + MAX_BACK_TRACE_BOTTOM_LINE_COUNT, + XTR_UDP_PORT = 7831; + public static final String SW_W3C_KEY_PREFIX = "sw.", + XTR_ASYNC_KEY = "Async", + XTR_EDGE_KEY = SW_W3C_KEY_PREFIX + "parent_span_id", + XTR_AO_EDGE_KEY = "Edge", + XTR_THREAD_ID_KEY = "TID", + XTR_HOSTNAME_KEY = "Hostname", + XTR_METADATA_KEY = SW_W3C_KEY_PREFIX + "trace_context", + XTR_XTRACE = "X-Trace", + XTR_PROCESS_ID_KEY = "PID", + XTR_TIMESTAMP_U_KEY = "Timestamp_u", + XTR_UDP_HOST = "127.0.0.1"; +} diff --git a/libs/core/src/main/java/com/solarwinds/joboe/core/Context.java b/libs/core/src/main/java/com/solarwinds/joboe/core/Context.java new file mode 100644 index 00000000..a8363eb9 --- /dev/null +++ b/libs/core/src/main/java/com/solarwinds/joboe/core/Context.java @@ -0,0 +1,226 @@ +/* + * © SolarWinds Worldwide, LLC. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.solarwinds.joboe.core; + +import com.solarwinds.joboe.logging.Logger; +import com.solarwinds.joboe.logging.LoggerFactory; +import com.solarwinds.joboe.sampling.Metadata; +import com.solarwinds.joboe.sampling.SamplingException; +import com.solarwinds.joboe.sampling.SettingsArg; +import com.solarwinds.joboe.sampling.SettingsArgChangeListener; +import com.solarwinds.joboe.sampling.SettingsManager; +import io.opentelemetry.api.trace.Span; +import io.opentelemetry.api.trace.SpanContext; +import java.util.Arrays; + +public class Context { + private static final ThreadLocal skipInheritingContextThreadLocal = + new ThreadLocal(); + private static final Logger logger = LoggerFactory.getLogger(); + private static boolean inheritContext = + true; // whether child thread should inherits metadata (clone) from parent + + // Thread local storage for metadata. This is inheritable so that child threads pick up the + // parent's context. + private static final InheritableThreadLocal mdThreadLocal = + new InheritableThreadLocal() { + @Override + protected Metadata initialValue() { + Metadata md = new Metadata(); + return md; + } + + @Override + protected Metadata childValue(Metadata parentMetadata) { + if (!inheritContext + || (skipInheritingContextThreadLocal.get() != null + && skipInheritingContextThreadLocal.get())) { + return new Metadata(); // do not propagate context here, return an empty context + } else { + Metadata clonedMetadata = new Metadata(parentMetadata); + // if parent span is sampled, that means a parent span exists then this is a child span + // spawn off from a thread from parent span, mark this as asynchronous + if (parentMetadata.isSampled()) { + clonedMetadata.setIsAsync(true); + } + return clonedMetadata; + } + } + }; + + static { + SettingsManager.registerListener( + new SettingsArgChangeListener( + SettingsArg + .DISABLE_INHERIT_CONTEXT) { // listen to Settings change on inheriting context + @Override + public void onChange(Boolean newValue) { + if (newValue != null) { + inheritContext = !newValue; + } else { + inheritContext = true; // by default we inherit context + } + } + }); + } + + public static Event createEvent() { + return createEventWithContext(getMetadata(), true); + } + + public static Event createEventWithID(String metadataID) throws SamplingException { + return createEventWithIDAndContext(metadataID, getMetadata()); + } + + public static Event createEventWithContext(Metadata context) { + return createEventWithContext(context, true); // by default add edge (not trace start) + } + + public static Event createEventWithContext(Metadata context, boolean addEdge) { + if (shouldCreateEvent(context)) { + return new EventImpl(context, addEdge); + } else { + return new NoopEvent(context); + } + } + + public static Event createEventWithIDAndContext(String metadataID, Metadata currentContext) + throws SamplingException { + if (shouldCreateEvent(currentContext)) { + return new EventImpl(currentContext, metadataID, true); + } else { + return new NoopEvent(currentContext); + } + } + + public static Event createEventWithGeneratedMetadata(Metadata generatedMetadata) { + return createEventWithGeneratedMetadata(null, generatedMetadata); + } + + public static Event createEventWithGeneratedMetadata( + Metadata parentMetadata, Metadata generatedMetadata) { + if (shouldCreateEvent(generatedMetadata)) { + return new EventImpl(parentMetadata, generatedMetadata); + } else { + return new NoopEvent(generatedMetadata); + } + } + + private static boolean shouldCreateEvent(Metadata context) { + if (!context.isSampled()) { + return false; + } + + if (!context.incrNumEvents()) { + // Should not invalidate as metrics should still be captured, setting it to not sampled might + // also impact logic that assume a "sampled" entry point should match a "sampled" exit point + return false; + } + + if (context.isExpired(System.currentTimeMillis())) { + // Consider an error condition (context leaking) Hence we should expire the context metadata + // to stop further processing/leaking + context.invalidate(); + return false; + } + + return true; + } + + /** Returns metadata for current thread */ + public static Metadata getMetadata() { + Metadata md = mdThreadLocal.get(); + if (Arrays.equals(md.getTaskID(), Metadata.unsetTaskID)) { + SpanContext sc = Span.current().getSpanContext(); + if (sc.isValid()) { + return new Metadata(sc); + } + } + return md; + } + + /** + * Sets metadata for current thread. + * + *

Note: This sets the metadata in the local thread storage only. It does NOT update the + * OpenTelemetry context. To propagate context via OpenTelemetry, please use {@code + * io.opentelemetry.context.Context.makeCurrent()}. + * + * @param md + * @deprecated Use OpenTelemetry Context propagation instead. + */ + @Deprecated + public static void setMetadata(Metadata md) { + setMap(md); + } + + /** + * Sets metadata for this thread from hex string + * + *

Note: This sets the metadata in the local thread storage only. It does NOT update the + * OpenTelemetry context. To propagate context via OpenTelemetry, please use {@code + * io.opentelemetry.context.Context.makeCurrent()}. + * + * @param hexStr + * @throws SamplingException + * @deprecated Use OpenTelemetry Context propagation instead. + */ + @Deprecated + public static void setMetadata(String hexStr) throws SamplingException { + Metadata md = new Metadata(); + md.fromHexString(hexStr); + setMap(md); + } + + /** + * Clears metadata for current thread. + * + *

Note: This only clears the local thread storage. It does not affect any active + * OpenTelemetry Scope. + * + * @deprecated Use OpenTelemetry Context propagation instead. + */ + @Deprecated + public static void clearMetadata() { + setMap(new Metadata()); + } + + private static void setMap(Metadata md) { + mdThreadLocal.set(md); + } + + public static boolean isValid() { + return getMetadata().isValid(); + } + + /** + * Set whether the any child threads spawned by the current thread should skip inheriting a clone + * of current context + * + *

It is known that in thread pool handling, using inheritable thread local has problem of + * leaking context. (unable to clear the context in the spawned thread afterwards) This can be set + * to true if the context propagation is handled by other means in order to avoid the leaking + * problem. + * + *

By default this is false + * + * @param skipInheritingContext + */ + public static void setSkipInheritingContext(boolean skipInheritingContext) { + skipInheritingContextThreadLocal.set(skipInheritingContext); + } +} diff --git a/libs/core/src/main/java/com/solarwinds/joboe/core/Event.java b/libs/core/src/main/java/com/solarwinds/joboe/core/Event.java new file mode 100644 index 00000000..77005e77 --- /dev/null +++ b/libs/core/src/main/java/com/solarwinds/joboe/core/Event.java @@ -0,0 +1,97 @@ +/* + * © SolarWinds Worldwide, LLC. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.solarwinds.joboe.core; + +import com.solarwinds.joboe.sampling.Metadata; +import java.nio.ByteBuffer; +import java.util.Map; + +public abstract class Event { + protected final Metadata metadata; + + protected Event(Metadata metadata) { + this.metadata = metadata; + } + + /** + * Add key/value pair to event + * + * @param key + * @param value + */ + public abstract void addInfo(String key, Object value); + + /** + * Add all key /value pairs to event + * + * @param infoMap + */ + public abstract void addInfo(Map infoMap); + + /** + * Add all key/value pairs to event. This assumes that info contains alternating Name/Value pairs + * (String, Object). + * + * @param info + */ + public abstract void addInfo(Object... info); + + public abstract void addEdge(Metadata md); + + public abstract void addEdge(String hexstr); + + /** Marks event as Asynchronous. (One could also add this k/v manually.) */ + public abstract void setAsync(); + + public final Metadata getMetadata() { + return metadata; + } + + /** + * Report event to agent + * + * @param reporter + */ + public abstract void report(EventReporter reporter); + + /** + * Report event to agent + * + * @param md metadata from context - if null, then no context metadata check and update will be + * done + * @param reporter Event Reporter + */ + public abstract void report(Metadata md, EventReporter reporter); + + public abstract byte[] toBytes() throws BsonBufferException; + + public abstract ByteBuffer toByteBuffer() throws BsonBufferException; + + /** + * Sets timestamp in microsecond since epoch time + * + * @param timestamp + */ + public abstract void setTimestamp(long timestamp); + + /** + * Sets an explicit thread ID to be reported in the event + * + * @param threadId + */ + public abstract void setThreadId(Long threadId); +} diff --git a/libs/core/src/main/java/com/solarwinds/joboe/core/EventImpl.java b/libs/core/src/main/java/com/solarwinds/joboe/core/EventImpl.java new file mode 100644 index 00000000..46a04dd1 --- /dev/null +++ b/libs/core/src/main/java/com/solarwinds/joboe/core/EventImpl.java @@ -0,0 +1,687 @@ +/* + * © SolarWinds Worldwide, LLC. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.solarwinds.joboe.core; + +import static com.solarwinds.joboe.core.Constants.*; + +import com.solarwinds.joboe.core.ebson.BsonDocument; +import com.solarwinds.joboe.core.ebson.BsonDocuments; +import com.solarwinds.joboe.core.ebson.BsonToken; +import com.solarwinds.joboe.core.ebson.BsonWriter; +import com.solarwinds.joboe.core.ebson.MultiValList; +import com.solarwinds.joboe.core.util.HostInfoUtils; +import com.solarwinds.joboe.core.util.JavaProcessUtils; +import com.solarwinds.joboe.core.util.TimeUtils; +import com.solarwinds.joboe.logging.Logger; +import com.solarwinds.joboe.logging.LoggerFactory; +import com.solarwinds.joboe.sampling.Metadata; +import com.solarwinds.joboe.sampling.SamplingException; +import java.lang.reflect.Array; +import java.nio.BufferOverflowException; +import java.nio.ByteBuffer; +import java.nio.ByteOrder; +import java.util.*; +import java.util.Map.Entry; + +public class EventImpl extends Event { + private static final Logger logger = LoggerFactory.getLogger(); + + private BsonDocument.Builder bsonBuilder; + private final MultiValList edges = new MultiValList(1); + private boolean isAsync = false; + + private boolean isEntry = false; + private boolean isExit = false; + + private Long threadId = null; + + private Long timestamp = null; + + static final int MAX_KEY_COUNT = 1024; + private static final Collection BASIC_KEYS = + Arrays.asList( + "Layer", + "Label", + Constants.XTR_ASYNC_KEY, + Constants.XTR_EDGE_KEY, + Constants.XTR_AO_EDGE_KEY, + Constants.XTR_THREAD_ID_KEY, + Constants.XTR_HOSTNAME_KEY, + Constants.XTR_METADATA_KEY, + Constants.XTR_XTRACE, + Constants.XTR_PROCESS_ID_KEY, + Constants.XTR_TIMESTAMP_U_KEY); + + // Buffer: used for building BSON byte stream, one-per-thread so we don't keep reallocating + // buffers + private static final ThreadLocal BUFFER = + new ThreadLocal() { + @Override + protected ByteBuffer initialValue() { + return ByteBuffer.allocate(MAX_EVENT_BUFFER_SIZE).order(ByteOrder.LITTLE_ENDIAN); + } + }; + + public EventImpl(Metadata ctxMetadata, boolean addEdge) { + super(new Metadata(ctxMetadata)); + if (ctxMetadata.isSampled()) { // only further init the metadata if it's being traced + init(); + if (addEdge) { + addEdge(ctxMetadata); + } + } + } + + /** + * Creates an event with a previously determined metadataID . See ServletInstrumentation for an + * example where this was needed. + */ + public EventImpl(Metadata ctxMetadata, String metadataID, boolean addEdge) + throws SamplingException { + super(new Metadata(metadataID)); + initOverride(); + if (addEdge) { + addEdge(ctxMetadata); + } + } + + /** + * Creates an event with a previously determined metadata + * + * @param parentMetadata + * @param eventMetadata + */ + EventImpl(Metadata parentMetadata, Metadata eventMetadata) { + super(eventMetadata); + initOverride(); + if (parentMetadata != null) { + addEdge(parentMetadata); + } + } + + private void init() { + metadata.randomizeOpID(); + bsonBuilder = BsonDocuments.builder(); + String traceContext = metadata.toHexString(); + bsonBuilder.put(XTR_METADATA_KEY, traceContext); + bsonBuilder.put(XTR_XTRACE, w3cContextToXTrace(traceContext)); + } + + /** + * Sets XTrace ID - used in special cases where we need to override (See ServlerInstrumentation + * for an example) + */ + private void initOverride() { + // We do NOT randomize here because we assume we are constructed with a metadata that was + // already randomized + bsonBuilder = BsonDocuments.builder(); + String traceContext = metadata.toHexString(); + bsonBuilder.put(XTR_METADATA_KEY, traceContext); + bsonBuilder.put(XTR_XTRACE, w3cContextToXTrace(traceContext)); + } + + protected static String w3cContextToXTrace(String w3cContext) { + String[] arr = w3cContext.split("-"); + if (arr.length != 4) { + return ""; + } + + String padding = "00000000"; // eight zeros + return "2B" + arr[1].toUpperCase() + padding + arr[2].toUpperCase() + arr[3]; + } + + /* (non-Javadoc) + * @see com.solarwinds.joboe.core.Event#addInfo(java.lang.String, java.lang.Object) + */ + @Override + public void addInfo(String key, Object value) { + if (metadata.isSampled()) { + insertToBsonBuilder(key, value); + } + } + + /* (non-Javadoc) + * @see com.solarwinds.joboe.core.Event#addInfo(java.util.Map) + */ + @Override + public void addInfo(Map infoMap) { + if (metadata.isSampled()) { + for (Map.Entry entry : infoMap.entrySet()) { + insertToBsonBuilder(entry.getKey(), entry.getValue()); + } + } + } + + /* (non-Javadoc) + * @see com.solarwinds.joboe.core.Event#addInfo(java.lang.Object) + */ + @Override + public void addInfo(Object... info) { + if (metadata.isSampled()) { + if (info.length % 2 == 1) { + throw new RuntimeException("Even number of arguments expected"); + } + + for (int i = 0; i < info.length / 2; i++) { + if (!(info[i * 2] instanceof String)) { + throw new RuntimeException("String expected."); + } + insertToBsonBuilder((String) info[i * 2], info[i * 2 + 1]); + } + } + } + + /** + * Inserts custom keys and values into the Bson map, certain checks might be performed (for + * example checking whether it's entry/exit event), But the key and value will always be inserted + * into the Bson map (hence not validations) + * + * @param key + * @param value + */ + private void insertToBsonBuilder(String key, Object value) { + if ("Label".equals(key) && value instanceof String) { + if ("entry".equals(value)) { + isEntry = true; + } else if ("exit".equals(value)) { + isExit = true; + } + } else if ("Backtrace".equals(key) + && !metadata.incrNumBacktraces()) { // do not add backtrace if limit is exceeded + return; + } + + bsonBuilder.put(key, value); + } + + /* (non-Javadoc) + * @see com.solarwinds.joboe.core.Event#addEdge(com.solarwinds.joboe.sampling.Metadata) + */ + @Override + public void addEdge(Metadata md) { + if (metadata.isSampled() + && md.isSampled() + && metadata.isTaskEqual(md) + && !edges.contains(md.opHexString())) { + edges.add(md.opHexString()); + } + } + + /* (non-Javadoc) + * @see com.solarwinds.joboe.core.Event#addEdge(java.lang.String) + */ + @Override + public void addEdge(String hexstr) { + try { + addEdge(new Metadata(hexstr)); + } catch (SamplingException ex) { + logger.debug("Invalid XTrace ID: " + hexstr, ex); + } + } + + /* (non-Javadoc) + * @see com.solarwinds.joboe.core.Event#setAsync() + */ + @Override + public void setAsync() { + this.isAsync = true; + } + + /* (non-Javadoc) + * @see com.solarwinds.joboe.core.Event#report(com.solarwinds.joboe.core.EventReporter) + */ + @Override + public void report(EventReporter reporter) { + report(Context.getMetadata(), reporter); + } + + /* (non-Javadoc) + * @see com.solarwinds.joboe.core.Event#report(com.solarwinds.joboe.sampling.Metadata, com.solarwinds.joboe.core.EventReporter) + */ + @Override + public void report(Metadata contextMetadata, EventReporter reporter) { + if (reporter == null) { + return; + } + + if (contextMetadata != null) { + if (!metadata.isSampled()) { + return; + } + + // Event metadata must have the same taskID as the event + if (!metadata.isTaskEqual(contextMetadata)) { + return; + } + + // Event metadata must has a different opID + if (metadata.isOpEqual(contextMetadata)) { + return; + } + } + + // Add common key/value pairs + addEdges(); + addTimestamps(); + addProcessInfo(); + addHostname(); + + if (isAsync) { // if the event is marked explicitly as async + addAsync(); + } else if (contextMetadata != null + && contextMetadata.isAsync()) { // or if the associated context is marked as async + if (isEntry) { + if (contextMetadata.incrementAndGetAsyncLayerLevel() == 1) { + addAsync(); // only report async on entry event of the top extent within an async stack + } + } else if (isExit) { + contextMetadata.decrementAndGetAsyncLayerLevel(); + } + } + + try { + reporter.send(this); + // Update the context's opID to that of the event + if (contextMetadata != null) { + contextMetadata.setOpID(metadata); + } + } catch (EventReporterException e) { + logger.trace( + "Failed to send out event, exception message [" + + e.getMessage() + + "]. Please take note that existing code flow should not be affected, this might only impact the instrumentation of current trace"); + } catch (Throwable ex) { + logger.error( + "Failed to send out event, exception message [" + + ex.getMessage() + + "]. Please take note that existing code flow should not be affected, this might only impact the instrumentation of current trace", + ex); + } + } + + /* (non-Javadoc) + * @see com.solarwinds.joboe.core.Event#toBytes() + */ + @Override + public byte[] toBytes() throws BsonBufferException { + BsonDocument doc = bsonBuilder.build(); + BsonWriter writer = BsonToken.DOCUMENT.writer(); + ByteBuffer buffer = BUFFER.get(); + buffer.clear(); // cast for JDK 8- runtime compatibility + + boolean retry = false; + try { + writer.writeTo(buffer, doc); + } catch (BufferOverflowException e) { // cannot use multi-catch for 1.6 source compatibility + retry = true; + } catch (IllegalArgumentException e) { + retry = true; + } + + if (retry) { + logger.warn("The KVs are too big to be converted. Trimming down the KVs..."); + doc = trimDoc(doc); + + buffer.clear(); // cast for JDK 8- runtime compatibility + + if (doc != null) { + RuntimeException bsonException = null; + try { + writer.writeTo(buffer, doc); // try again + } catch (BufferOverflowException e) { // cannot use multi-catch for 1.6 source compatibility + bsonException = e; + } catch (IllegalArgumentException e) { + bsonException = e; + } + + // still failing after retry, throw exception + if (bsonException != null) { + throw new BsonBufferException(bsonException); + } + } + } + + buffer.flip(); // cast for JDK 8- runtime compatibility + + byte[] bytes = new byte[buffer.remaining()]; // allocate an array with the actual size required + buffer.get(bytes); + return bytes; + } + + /* (non-Javadoc) + * @see com.solarwinds.joboe.core.Event#toByteBuffer() + */ + @Override + public ByteBuffer toByteBuffer() throws BsonBufferException { + return ByteBuffer.wrap(toBytes()); + } + + /** + * Trims the BsonDocument using a conservative strategy. This should only be invoked as a fall + * back to when a BufferOverflowException is encountered during event conversion. Each event + * creator (instrumentation, metrics collector) should try their best to avoid overflowing the + * event + * + * @param doc + * @return trimmed BsonDocument + */ + private BsonDocument trimDoc(BsonDocument doc) { + BsonDocument.Builder newBuilder = BsonDocuments.builder(); + + int bytesAllowed = MAX_EVENT_BUFFER_SIZE; // bytes allowed to build the new doc(event) + + // First extract basic key values, those should not be subject to trimming + Map basicKeyValues = extractBasicKeyValues(doc); + + // calculate bytes taken by the basic key values + for (Entry basicKeyValue : basicKeyValues.entrySet()) { + bytesAllowed -= getBsonByteSize(basicKeyValue.getKey()); + try { + bytesAllowed -= getBsonByteSize(basicKeyValue.getValue()); + } catch (IllegalArgumentException e) { + logger.warn( + "Unknown value type for basic key [" + + basicKeyValue.getKey() + + "]. Type [" + + basicKeyValue.getValue().getClass().getName() + + "]"); + } + } + + if (bytesAllowed < 0) { // should not happen...unless we add in some crazy basic keys or the + // MAX_EVENT_BUFFER_SIZE is unreasonably small + logger.warn("Cannot send an event as the basic key values fail to fit in the event!"); + return null; + } + + newBuilder.putAll(basicKeyValues); // add all basic keys + + Map otherKeyValues = new LinkedHashMap(doc); + otherKeyValues + .keySet() + .removeAll(BASIC_KEYS); // now the otherKeyValues contain all the non-basic keys + + // First try to trim down the number of entries based on MAX_KEY_COUNT + trimKeyValues(otherKeyValues); + + // Now iterate through each KV entry and check whether trimming to its value is necessary + int maxBytePerKeyValue = + bytesAllowed + / otherKeyValues + .size(); // just an approximation. Take note this is a conservative estimation and + // will not fully utilize all the space available. + + logger.debug( + "Trimming KVs on other key count " + + otherKeyValues.size() + + " maxBytePerKeyValue " + + maxBytePerKeyValue); + + // test buffer to check accumulative size of the bson document to be built + ByteBuffer testBuffer = + ByteBuffer.allocate(MAX_EVENT_BUFFER_SIZE).order(ByteOrder.LITTLE_ENDIAN); + BsonWriter testWriter = BsonToken.DOCUMENT.writer(); + + testWriter.writeTo(testBuffer, newBuilder.build()); + + for (Entry entry : otherKeyValues.entrySet()) { + // check if key alone would fit + if (getBsonByteSize(entry.getKey()) >= maxBytePerKeyValue) { + logger.warn( + "Dropping event entry with key [" + + entry.getKey().substring(0, 10) + + "...] as the key is too long"); + continue; + } + + int maxByteForValue = + maxBytePerKeyValue - getBsonByteSize(entry.getKey()); // the byte remaining for the value + + // check if the value would fit in the remaining byte + Object value = entry.getValue(); + if (value instanceof String) { // trim if string is too long + if (getBsonByteSize(value) > maxByteForValue) { + value = ((String) value).substring(0, maxByteForValue / 2); // 2 byte each character + } + } else if (value instanceof Object[]) { // trim if array is too long + Object[] valueArray = (Object[]) value; + + int arrayValueSize = 0; + + // iterate the array calculate how much byte has been taken, if it exceeds maxBytePerValue, + // drop the current array element and the rest in the array + int arrayWalker; + for (arrayWalker = 0; arrayWalker < valueArray.length; arrayWalker++) { + Object arrayElement = valueArray[arrayWalker]; + try { + arrayValueSize += getBsonByteSize(arrayElement); + arrayValueSize += + 8; // also it keeps track of the index of the array. adding 8 as a conservative + // estimate + } catch (IllegalArgumentException e) { + logger.warn( + "Found unknown type [" + + arrayElement.getClass().getName() + + "] in KV " + + entry.getKey() + + " while trimming the doc. Trimming the rest of the elements in the array..."); + break; + } + + if (arrayValueSize > maxByteForValue) { + logger.warn( + "Found oversized array in KV " + + entry.getKey() + + " while trimming the doc. Trimming the elements with index from " + + arrayWalker); + break; + } + } + + if (arrayWalker < valueArray.length) { // trim the array in this KV + Object[] trimmedArray = new Object[arrayWalker]; + System.arraycopy(valueArray, 0, trimmedArray, 0, arrayWalker); + value = trimmedArray; + } + } else { // not string or object[], not going to trim + try { + logger.debug( + "Skip " + + entry.getKey() + + " for trimming as it is type " + + (entry.getValue() != null ? entry.getValue().getClass().getName() : "null ") + + ". Estimated size is " + + getBsonByteSize(entry.getValue())); + } catch (IllegalArgumentException e) { + logger.warn( + "Skip " + + entry.getKey() + + " for trimming as it is type " + + (entry.getValue() != null ? entry.getValue().getClass().getName() : "null ") + + ". Size cannot be estimated: " + + e.getMessage(), + e); + } + } + + // try to serialize it to ensure it would not throw exception, take note that the test buffer + // accumulates each KV as a separate test document + // and each test document is just a single KV bson + // This comparison should work as the size of the actual bson document which contains all KVs + // should be slightly smaller than many single KV bson documents combined + BsonDocument.Builder testBuilder = BsonDocuments.builder(); + testBuilder.put(entry.getKey(), value); + int lastPosition = testBuffer.position(); + try { + testWriter.writeTo(testBuffer, testBuilder.build()); + + // safe to add this to builder + newBuilder.put(entry.getKey(), value); + } catch (BufferOverflowException e) { + logger.warn( + "Failed to write KV [" + + entry.getKey() + + "] to event, as adding it does not fit the bytebuffer, skipping this KV!"); + testBuffer.position( + lastPosition); // roll back to last position. Cast for JDK 8- runtime compatibility + } + } + + BsonDocument newDocument = newBuilder.build(); + + return newDocument; + } + + /** + * Returns the estimate byte size of the object in bson. Take note that this only account for Bson + * object type + * + * @param object to use for byte size estimate + */ + private static int getBsonByteSize(Object object) { + if (object instanceof String) { // if it's a String element check if it's too long + return ((String) object).length() * 2; // 2 byte each character + } else if (object instanceof Boolean) { + return 1; + } else if (object instanceof Byte) { + return 1; + } else if (object instanceof Double) { + return Double.SIZE / 8; + } else if (object instanceof Integer) { + return Integer.SIZE / 8; + } else if (object instanceof Long) { + return Long.SIZE / 8; + } else if (object != null && object.getClass().isArray()) { + int size = 0; + int length = Array.getLength(object); + for (int i = 0; i < length; i++) { + Object obj = Array.get(object, i); + size += getBsonByteSize(obj); + } + return size; + } else if (object instanceof MultiValList) { + MultiValList list = (MultiValList) object; + int size = 0; + for (Object element : list) { + size += getBsonByteSize(element); + } + return size; + } else if (object instanceof Collection) { + Collection collection = (Collection) object; + int size = 0; + Iterator iterator = collection.iterator(); + while (iterator.hasNext()) { + size += getBsonByteSize(iterator.next()); + } + return size; + } else if (object instanceof Map) { + Map map = (Map) object; + int size = 0; + for (Entry entry : map.entrySet()) { + size += getBsonByteSize(entry.getKey()) + getBsonByteSize(entry.getValue()); + } + return size; + } else if (object == null) { + return 1; + } else { // other unknown types. To be safe, just drop from here + throw new IllegalArgumentException( + "unknown size for type [" + object.getClass().getName() + "]"); + } + } + + /** + * Trim the input KV map such that it contains up to MAX_KEY_COUNT entries. + * + * @param keyValues + * @return a new Map instance of the trimmed entries + */ + private static void trimKeyValues(Map keyValues) { + if (keyValues.size() > MAX_KEY_COUNT) { + logger.warn( + "Found " + keyValues.size() + " KVs in the event, trimming it down to " + MAX_KEY_COUNT); + Set trimmedKeys = new HashSet(); + + for (Entry entry : keyValues.entrySet()) { + if (trimmedKeys.size() >= MAX_KEY_COUNT) { + break; + } + trimmedKeys.add(entry.getKey()); + } + + keyValues.keySet().retainAll(trimmedKeys); + } + } + + /** + * @param keyValues + * @return a new instance of Map of key values that with keys defined in BASIC_KEYS + */ + private static Map extractBasicKeyValues(Map keyValues) { + Map basicKeyValues = new LinkedHashMap(keyValues); + basicKeyValues.keySet().retainAll(BASIC_KEYS); + return basicKeyValues; + } + + private void addTimestamps() { + if (timestamp == null) { + timestamp = TimeUtils.getTimestampMicroSeconds(); + } + + addInfo(XTR_TIMESTAMP_U_KEY, timestamp); + } + + private void addHostname() { + addInfo(XTR_HOSTNAME_KEY, HostInfoUtils.getHostName()); + } + + private void addProcessInfo() { + addInfo(XTR_THREAD_ID_KEY, threadId != null ? threadId : Thread.currentThread().getId()); + addInfo(XTR_PROCESS_ID_KEY, JavaProcessUtils.getPid()); + } + + private void addEdges() { + if (!edges.isEmpty()) { + addInfo(XTR_EDGE_KEY, edges); + addInfo(XTR_AO_EDGE_KEY, toAOEdges(edges)); + } + } + + private MultiValList toAOEdges(MultiValList edges) { + MultiValList aoEdges = new MultiValList<>(); + for (String edge : edges) { + aoEdges.add(edge.toUpperCase()); + } + return aoEdges; + } + + private void addAsync() { + addInfo(XTR_ASYNC_KEY, true); + } + + /* (non-Javadoc) + * @see com.solarwinds.joboe.core.Event#setTimestamp(long) + */ + @Override + public void setTimestamp(long timestamp) { + this.timestamp = timestamp; + } + + @Override + public void setThreadId(Long threadId) { + this.threadId = threadId; + } +} diff --git a/libs/core/src/main/java/com/solarwinds/joboe/core/EventReporter.java b/libs/core/src/main/java/com/solarwinds/joboe/core/EventReporter.java new file mode 100644 index 00000000..251e656d --- /dev/null +++ b/libs/core/src/main/java/com/solarwinds/joboe/core/EventReporter.java @@ -0,0 +1,26 @@ +/* + * © SolarWinds Worldwide, LLC. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.solarwinds.joboe.core; + +/** All event reporters must implement this interface. */ +public interface EventReporter { + void send(Event event) throws EventReporterException; + + EventReporterStats consumeStats(); + + void close(); +} diff --git a/libs/core/src/main/java/com/solarwinds/joboe/core/EventReporterException.java b/libs/core/src/main/java/com/solarwinds/joboe/core/EventReporterException.java new file mode 100644 index 00000000..5a26a942 --- /dev/null +++ b/libs/core/src/main/java/com/solarwinds/joboe/core/EventReporterException.java @@ -0,0 +1,28 @@ +/* + * © SolarWinds Worldwide, LLC. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.solarwinds.joboe.core; + +public class EventReporterException extends Exception { + + public EventReporterException(String message, Throwable cause) { + super(message, cause); + } + + public EventReporterException(String message) { + super(message); + } +} diff --git a/libs/core/src/main/java/com/solarwinds/joboe/core/EventReporterQueueFullException.java b/libs/core/src/main/java/com/solarwinds/joboe/core/EventReporterQueueFullException.java new file mode 100644 index 00000000..16170123 --- /dev/null +++ b/libs/core/src/main/java/com/solarwinds/joboe/core/EventReporterQueueFullException.java @@ -0,0 +1,23 @@ +/* + * © SolarWinds Worldwide, LLC. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.solarwinds.joboe.core; + +public class EventReporterQueueFullException extends EventReporterException { + public EventReporterQueueFullException(String message) { + super(message); + } +} diff --git a/libs/core/src/main/java/com/solarwinds/joboe/core/EventReporterStats.java b/libs/core/src/main/java/com/solarwinds/joboe/core/EventReporterStats.java new file mode 100644 index 00000000..e2dd9bd6 --- /dev/null +++ b/libs/core/src/main/java/com/solarwinds/joboe/core/EventReporterStats.java @@ -0,0 +1,30 @@ +/* + * © SolarWinds Worldwide, LLC. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.solarwinds.joboe.core; + +import lombok.Builder; +import lombok.Value; + +@Value +@Builder +public class EventReporterStats { + long sentCount; + long overflowedCount; + long failedCount; + long queueLargestCount; + long processedCount; +} diff --git a/libs/core/src/main/java/com/solarwinds/joboe/core/EventValueConverter.java b/libs/core/src/main/java/com/solarwinds/joboe/core/EventValueConverter.java new file mode 100644 index 00000000..55950e17 --- /dev/null +++ b/libs/core/src/main/java/com/solarwinds/joboe/core/EventValueConverter.java @@ -0,0 +1,325 @@ +/* + * © SolarWinds Worldwide, LLC. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.solarwinds.joboe.core; + +import com.solarwinds.joboe.logging.Logger; +import com.solarwinds.joboe.logging.LoggerFactory; +import java.math.BigDecimal; +import java.math.BigInteger; +import java.net.InetAddress; +import java.net.URL; +import java.util.Arrays; +import java.util.Collection; +import java.util.Date; +import java.util.HashSet; +import java.util.LinkedHashMap; +import java.util.Map; +import java.util.Set; +import java.util.UUID; + +/** + * Converter that converts any object into object that are safe to used as the value of Event Info + * + * @author Patson Luk + */ +public class EventValueConverter { + protected static final Logger logger = LoggerFactory.getLogger(); + + // types that can put directly to the Event info AND the type has to be final (no isInstanceOf + // checks are going to be performed) + private static final Set> EXPECTED_SIMPLE_TYPES = + new HashSet>( + Arrays.asList( + new Class[] { + Boolean.class, Double.class, Integer.class, Long.class, + })); + + // types that requires conversion/validations before putting into the Event info, more common + // class first for better performance (since we might do instanceOf check) + protected final Map, Converter> EXPECTED_SPECIAL_TYPES = + new LinkedHashMap, Converter>(); + + // By default, always output the class name only if its an unknown parameter type + private static final Converter DEFAULT_HANDLER = new ClassNameParameterHandler(); + + public static final int DEFAULT_MAX_LENGTH = 1024; + + // Max character counts allowed by any parameter that has string representation as tracked value + public final int maxValueLength; + private final Logger.Level logLevel; + + public EventValueConverter() { + this(DEFAULT_MAX_LENGTH); + } + + public EventValueConverter(int maxValueLength) { + this(maxValueLength, Logger.Level.INFO); + } + + public EventValueConverter(int maxValueLength, Logger.Level logLevel) { + EXPECTED_SPECIAL_TYPES.put(String.class, new SimpleParameterHandler()); + EXPECTED_SPECIAL_TYPES.put(Date.class, new SimpleParameterHandler()); + EXPECTED_SPECIAL_TYPES.put(byte[].class, new ByteArrayParameterHandler()); + EXPECTED_SPECIAL_TYPES.put(URL.class, new ToStringParameterHandler()); + EXPECTED_SPECIAL_TYPES.put( + BigDecimal.class, new BigDecimalParameterHandler()); // Event does not support BigDecimal + EXPECTED_SPECIAL_TYPES.put( + BigInteger.class, new BigIntegerParameterHandler()); // Event does not support BigInteger + EXPECTED_SPECIAL_TYPES.put( + Float.class, new FloatParameterHandler()); // Event does not support Float + EXPECTED_SPECIAL_TYPES.put( + Short.class, new ShortParameterHandler()); // Event does not support Short + EXPECTED_SPECIAL_TYPES.put( + Byte.class, new ByteParameterHandler()); // Event does not support Byte + EXPECTED_SPECIAL_TYPES.put(Character.class, new ToStringParameterHandler()); + EXPECTED_SPECIAL_TYPES.put(InetAddress.class, new ToStringParameterHandler()); + EXPECTED_SPECIAL_TYPES.put(Collection.class, new CollectionParameterHandler()); + EXPECTED_SPECIAL_TYPES.put(Map.class, new MapParameterHandler()); + EXPECTED_SPECIAL_TYPES.put(UUID.class, new ToStringParameterHandler()); + + this.maxValueLength = maxValueLength; + this.logLevel = logLevel; + } + + /** + * Converts the rawValue into an object that is acceptable as info value of Event + * + * @param rawValue + * @return + */ + public Object convertToEventValue(Object rawValue) { + if (rawValue == null) { + return null; + } else if (EXPECTED_SIMPLE_TYPES.contains( + rawValue + .getClass())) { // quick check to avoid overhead of checking for each of the entry, most + // objects should fall into this condition + return rawValue; // safe to put directly to the map + } else { + return getValueFromSpecialType(rawValue); + } + } + + /** + * Gets the result value that we will use for tracking, some parameter values might need + * conversions + * + * @param parameter the input parameter, the one used by the setXXX and setObject methods + * @return the result value that we will stored and used when reporting the parameters in Trace + */ + @SuppressWarnings("unchecked") + private Object getValueFromSpecialType(Object parameter) { + Converter converter = null; + + // quick check to avoid overhead of checking for each of the entries + if (EXPECTED_SPECIAL_TYPES.containsKey(parameter.getClass())) { + converter = EXPECTED_SPECIAL_TYPES.get(parameter.getClass()); + } else { // have to iterate thru the special types, should not be common + for (Class specialType : EXPECTED_SPECIAL_TYPES.keySet()) { + if (specialType.isInstance(parameter)) { + converter = EXPECTED_SPECIAL_TYPES.get(specialType); + break; + } + } + } + + if (converter == null) { + logger.debug( + "Class [" + + parameter.getClass().getName() + + "] is not in the expected list of classes for PreparedStatement parameter. Using the default handler"); + converter = DEFAULT_HANDLER; + } + + // use the handler to retrieve the result value + Object value = converter.getValue(parameter); + + // trim the result value if it is a String + if (value instanceof String) { + String stringValue = (String) value; + if (stringValue.length() > maxValueLength) { + logger.log( + logLevel, + "Parameter truncated as it is too long [" + stringValue.length() + "] characters"); + int truncateCount = stringValue.length() - maxValueLength; + stringValue = + stringValue.substring(0, maxValueLength) + + "...(" + + truncateCount + + " characters truncated)"; + } + return stringValue; + } else { + return value; + } + } + + /** + * Handler that takes in an input parameter value and convert it in something we can use for + * tracking (in BSON map) + * + * @author Patson Luk + * @param Type of the Parameter + * @param Expected Type of the result value + */ + protected abstract static class Converter { + protected abstract R getValue(T parameter); + } + + /** + * Gets the String value of the object by using the toString() method of the input parameter + * + * @author Patson Luk + */ + public static class ToStringParameterHandler extends Converter { + @Override + protected String getValue(Object parameter) { + return parameter.toString(); + } + } + + /** + * Gets the fully qualified class name of the input parameter + * + * @author Patson Luk + */ + public static class ClassNameParameterHandler extends Converter { + @Override + protected String getValue(Object parameter) { + return "(" + + parameter.getClass().getName() + + ") id [" + + System.identityHashCode(parameter) + + "]"; + } + } + + /** + * Byte array handler. Display the length of the Byte array + * + * @author Patson Luk + */ + private static class ByteArrayParameterHandler extends Converter { + @Override + protected String getValue(byte[] parameter) { + return "(Byte array " + + parameter.length + + " Bytes) id [" + + System.identityHashCode(parameter) + + "]"; + } + } + + private static class CollectionParameterHandler extends Converter, String> { + @Override + protected String getValue(Collection parameter) { + return "(Collection of class [" + + parameter.getClass().getName() + + "] with " + + parameter.size() + + " Elements) id [" + + System.identityHashCode(parameter) + + "]"; + } + } + + private static class MapParameterHandler extends Converter, String> { + @Override + protected String getValue(Map parameter) { + return "(Map of class [" + + parameter.getClass().getName() + + "] with " + + parameter.size() + + " Elements) id [" + + System.identityHashCode(parameter) + + "]"; + } + } + + /** + * BigDecimal handler. Convert to double instead + * + * @author Patson Luk + */ + private static class BigDecimalParameterHandler extends Converter { + @Override + protected Double getValue(BigDecimal parameter) { + return parameter.doubleValue(); + } + } + + /** + * BigInteger handler. Convert to long instead + * + * @author Patson Luk + */ + private static class BigIntegerParameterHandler extends Converter { + @Override + protected Long getValue(BigInteger parameter) { + return parameter.longValue(); + } + } + + /** + * Float handler. Convert to double instead + * + * @author Patson Luk + */ + private static class FloatParameterHandler extends Converter { + @Override + protected Double getValue(Float parameter) { + return parameter.doubleValue(); + } + } + + /** + * Short handler. Convert to integer instead + * + * @author Patson Luk + */ + private static class ShortParameterHandler extends Converter { + @Override + protected Integer getValue(Short parameter) { + return parameter.intValue(); + } + } + + /** + * Byte handler. Convert to integer instead + * + * @author Patson Luk + */ + private static class ByteParameterHandler extends Converter { + @Override + protected Integer getValue(Byte parameter) { + return (int) parameter; + } + } + + /** + * Simple Parameter handler. No special handling is done, the parameter will simply be returned + * "as is" + * + * @author Patson Luk + */ + private static class SimpleParameterHandler extends Converter { + @Override + protected Object getValue(Object parameter) { + return parameter; + } + } +} diff --git a/libs/core/src/main/java/com/solarwinds/joboe/core/HostId.java b/libs/core/src/main/java/com/solarwinds/joboe/core/HostId.java new file mode 100644 index 00000000..afedd727 --- /dev/null +++ b/libs/core/src/main/java/com/solarwinds/joboe/core/HostId.java @@ -0,0 +1,157 @@ +/* + * © SolarWinds Worldwide, LLC. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.solarwinds.joboe.core; + +import static com.solarwinds.joboe.core.util.ServerHostInfoReader.setIfNotNull; + +import com.solarwinds.joboe.core.rpc.HostType; +import com.solarwinds.trace.ingestion.proto.Collector; +import java.util.Collections; +import java.util.List; +import lombok.Builder; +import lombok.Data; +import lombok.Value; +import org.json.JSONException; +import org.json.JSONObject; + +/** + * A combination of values to identify a host that makes the RPC call + * + * @author pluk + */ +@Data +@Builder +public class HostId { + private final String hostname; + private final int pid; + @Builder.Default private List macAddresses = Collections.emptyList(); + private final String ec2InstanceId; + private final String ec2AvailabilityZone; + private final String dockerContainerId; + private final String herokuDynoId; + private final String azureAppServiceInstanceId; + @Builder.Default private final HostType hostType = HostType.PERSISTENT; + private final String uamsClientId; + private final String uuid; + private final AwsMetadata awsMetadata; + private final AzureVmMetadata azureVmMetadata; + private final K8sMetadata k8sMetadata; + + @Value + @Builder + public static class AzureVmMetadata { + @Builder.Default String cloudProvider = "azure"; + @Builder.Default String cloudPlatform = "azure_vm"; + String cloudRegion; + String cloudAccountId; + String hostId; + String hostName; + String azureVmName; + String azureVmSize; + String azureVmScaleSetName; + String azureResourceGroupName; + + public static AzureVmMetadata fromJson(String payload) throws JSONException { + JSONObject jsonObject = new JSONObject(payload); + JSONObject compute = jsonObject.getJSONObject("compute"); + return AzureVmMetadata.builder() + .hostId(compute.getString("vmId")) + .cloudRegion(compute.getString("location")) + .hostName(compute.getString("name")) + .azureVmName(compute.getString("name")) + .cloudAccountId(compute.getString("subscriptionId")) + .azureVmSize(compute.getString("vmSize")) + .azureVmScaleSetName(compute.getString("vmScaleSetName")) + .azureResourceGroupName(compute.getString("resourceGroupName")) + .build(); + } + + public Collector.Azure toGrpc() { + return Collector.Azure.newBuilder() + .setCloudProvider(cloudProvider) + .setCloudPlatform(cloudPlatform) + .setCloudRegion(cloudRegion) + .setCloudAccountId(cloudAccountId) + .setHostId(hostId) + .setHostName(hostName) + .setAzureVmName(azureVmName) + .setAzureVmSize(azureVmSize) + .setAzureVmScaleSetName(azureVmScaleSetName) + .setAzureResourceGroupName(azureResourceGroupName) + .build(); + } + } + + @Value + @Builder + public static class AwsMetadata { + @Builder.Default String cloudProvider = "aws"; + @Builder.Default String cloudPlatform = "aws_ec2"; + String cloudAccountId; + String cloudRegion; + String cloudAvailabilityZone; + String hostId; + String hostImageId; + String hostName; + String hostType; + + public static AwsMetadata fromJson(String json, String hostName) throws JSONException { + if (json == null) return null; + JSONObject jsonObject = new JSONObject(json); + return AwsMetadata.builder() + .cloudRegion(jsonObject.getString("region")) + .cloudAccountId(jsonObject.getString("accountId")) + .cloudAvailabilityZone(jsonObject.getString("availabilityZone")) + .hostId(jsonObject.getString("instanceId")) + .hostImageId(jsonObject.getString("imageId")) + .hostType(jsonObject.getString("instanceType")) + .hostName(hostName) + .build(); + } + + public Collector.Aws toGrpc() { + return Collector.Aws.newBuilder() + .setCloudProvider(cloudProvider) + .setCloudPlatform(cloudPlatform) + .setCloudAccountId(cloudAccountId) + .setCloudRegion(cloudRegion) + .setCloudAvailabilityZone(cloudAvailabilityZone) + .setHostId(hostId) + .setHostImageId(hostImageId) + .setHostName(hostName) + .setHostType(hostType) + .build(); + } + } + + @Value + @Builder + public static class K8sMetadata { + String namespace; + String podName; + String podUid; + + public Collector.K8s toGrpc() { + Collector.K8s.Builder builder = Collector.K8s.newBuilder(); + setIfNotNull(builder::setNamespace, namespace); + + setIfNotNull(builder::setPodName, podName); + setIfNotNull(builder::setPodUid, podUid); + return builder.build(); + } + } +} diff --git a/libs/core/src/main/java/com/solarwinds/joboe/core/NonThreadLocalTestReporter.java b/libs/core/src/main/java/com/solarwinds/joboe/core/NonThreadLocalTestReporter.java new file mode 100644 index 00000000..a991ab52 --- /dev/null +++ b/libs/core/src/main/java/com/solarwinds/joboe/core/NonThreadLocalTestReporter.java @@ -0,0 +1,62 @@ +/* + * © SolarWinds Worldwide, LLC. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.solarwinds.joboe.core; + +import com.solarwinds.joboe.logging.Logger; +import com.solarwinds.joboe.logging.LoggerFactory; +import java.util.Deque; +import java.util.LinkedList; + +/** + * Mock reporter similar to {@link TestReporter}, only that this one does not work in thread local + * manner + * + * @author pluk + */ +class NonThreadLocalTestReporter extends TestReporter { + private final Logger logger = LoggerFactory.getLogger(); + + private final Deque byteBufferList = new LinkedList(); + + NonThreadLocalTestReporter() {} + + @Override + public synchronized void reset() { + byteBufferList.clear(); + } + + @Override + public synchronized void send(Event event) { + try { + byte[] buf = event.toBytes(); + logger.debug("Sent " + buf.length + " bytes"); + byteBufferList.add(buf); + } catch (BsonBufferException e) { + logger.error("Failed to send events : " + e.getMessage(), e); + } + } + + @Override + public synchronized byte[] getLastSent() { + return byteBufferList.getLast(); + } + + @Override + public synchronized Deque getBufList() { + return new LinkedList<>(byteBufferList); // return a clone + } +} diff --git a/libs/core/src/main/java/com/solarwinds/joboe/core/NoopEvent.java b/libs/core/src/main/java/com/solarwinds/joboe/core/NoopEvent.java new file mode 100644 index 00000000..55233fb2 --- /dev/null +++ b/libs/core/src/main/java/com/solarwinds/joboe/core/NoopEvent.java @@ -0,0 +1,67 @@ +/* + * © SolarWinds Worldwide, LLC. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.solarwinds.joboe.core; + +import com.solarwinds.joboe.sampling.Metadata; +import java.nio.ByteBuffer; +import java.util.Map; + +public class NoopEvent extends Event { + NoopEvent(Metadata metadata) { + super(metadata); + } + + @Override + public void addInfo(String key, Object value) {} + + @Override + public void addInfo(Map infoMap) {} + + @Override + public void addInfo(Object... info) {} + + @Override + public void addEdge(Metadata md) {} + + @Override + public void addEdge(String hexstr) {} + + @Override + public void setAsync() {} + + @Override + public void report(EventReporter reporter) {} + + @Override + public void report(Metadata md, EventReporter reporter) {} + + @Override + public byte[] toBytes() { + return null; + } + + @Override + public ByteBuffer toByteBuffer() { + return null; + } + + @Override + public void setTimestamp(long timestamp) {} + + @Override + public void setThreadId(Long threadId) {} +} diff --git a/libs/core/src/main/java/com/solarwinds/joboe/core/OboeException.java b/libs/core/src/main/java/com/solarwinds/joboe/core/OboeException.java new file mode 100644 index 00000000..d12d224d --- /dev/null +++ b/libs/core/src/main/java/com/solarwinds/joboe/core/OboeException.java @@ -0,0 +1,32 @@ +/* + * © SolarWinds Worldwide, LLC. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.solarwinds.joboe.core; + +/** Generic Oboe Exception */ +public class OboeException extends Exception { + public OboeException(String message, Throwable cause) { + super(message, cause); + } + + public OboeException(Throwable cause) { + super(cause); + } + + public OboeException(String msg) { + super(msg); + } +} diff --git a/libs/core/src/main/java/com/solarwinds/joboe/core/QueuingEventReporter.java b/libs/core/src/main/java/com/solarwinds/joboe/core/QueuingEventReporter.java new file mode 100644 index 00000000..35cef8f8 --- /dev/null +++ b/libs/core/src/main/java/com/solarwinds/joboe/core/QueuingEventReporter.java @@ -0,0 +1,273 @@ +/* + * © SolarWinds Worldwide, LLC. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.solarwinds.joboe.core; + +import com.solarwinds.joboe.config.ConfigManager; +import com.solarwinds.joboe.config.ConfigProperty; +import com.solarwinds.joboe.core.rpc.Client; +import com.solarwinds.joboe.core.rpc.ClientLoggingCallback; +import com.solarwinds.joboe.core.rpc.Result; +import com.solarwinds.joboe.core.rpc.ResultCode; +import com.solarwinds.joboe.core.util.DaemonThreadFactory; +import com.solarwinds.joboe.logging.Logger; +import com.solarwinds.joboe.logging.LoggerFactory; +import com.solarwinds.joboe.sampling.SettingsArg; +import com.solarwinds.joboe.sampling.SettingsArgChangeListener; +import com.solarwinds.joboe.sampling.SettingsManager; +import java.util.ArrayList; +import java.util.List; +import java.util.concurrent.BlockingQueue; +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; +import java.util.concurrent.LinkedBlockingQueue; +import java.util.concurrent.ScheduledExecutorService; +import java.util.concurrent.TimeUnit; + +/** + * A reporter that accepts events into a queue w/o blocking but sends out events synchronously. + * + *

Accepts events are inserted into a queue and being consumed and sent out synchronously. If + * sending rate is slower than the queuing rate, then this report will attempt to send out events in + * batches + * + * @author pluk + */ +public class QueuingEventReporter implements EventReporter { + static final int QUEUE_CAPACITY = 10000; + static final int SEND_CAPACITY; + private final BlockingQueue eventQueue = new LinkedBlockingQueue(QUEUE_CAPACITY); + protected ExecutorService executorService = + Executors.newSingleThreadExecutor(DaemonThreadFactory.newInstance("queuing-event-reporter")); + + private final ClientLoggingCallback loggingCallback = + new ClientLoggingCallback("send events"); + + protected static Logger logger = LoggerFactory.getLogger(); + private final AtomicEventReporterStats stats = new AtomicEventReporterStats(eventQueue::size); + + private static final long REPORT_QUEUE_FULL_INTERVAL = 60 * 1000; // 1 minute + static final int DEFAULT_FLUSH_INTERVAL = 2; // 2 second + + static int flushInterval = DEFAULT_FLUSH_INTERVAL; // in unit of second + + static { + if (ConfigManager.getConfig(ConfigProperty.AGENT_EVENTS_FLUSH_INTERVAL) instanceof Integer) { + setFlushInterval( + (Integer) ConfigManager.getConfig(ConfigProperty.AGENT_EVENTS_FLUSH_INTERVAL)); + } + SettingsManager.registerListener( + new SettingsArgChangeListener(SettingsArg.EVENTS_FLUSH_INTERVAL) { + @Override + public void onChange(Integer newValue) { + if (newValue != null) { + setFlushInterval(newValue); + } else { // reset to default + setFlushInterval(DEFAULT_FLUSH_INTERVAL); + } + } + }); + + SEND_CAPACITY = + ConfigManager.getConfigOptional(ConfigProperty.AGENT_EVENTS_SEND_CAPACITY, 1000); + } + + private boolean reportedQueueFull = false; + private long reportedQueueFullTime = 0; + + private final SendRunnable sendRunnable; + + protected final Client client; + + static void setFlushInterval(int eventsFlushInterval) { + if (eventsFlushInterval >= 0) { + flushInterval = eventsFlushInterval; + logger.debug("Event flush interval set to " + eventsFlushInterval + "s"); + } else { + logger.warn("Event flush interval value " + eventsFlushInterval + " is not valid"); + } + } + + public QueuingEventReporter(Client client) { + this.client = client; + sendRunnable = new SendRunnable(); + executorService.execute(sendRunnable); + } + + private class SendRunnable implements Runnable { + private boolean exitSignalled = false; + private CountDownLatch countDownLatch = new CountDownLatch(1); + private final ScheduledExecutorService scheduledExecutorService = + Executors.newScheduledThreadPool(1, DaemonThreadFactory.newInstance("send-event-delay")); + + @Override + public void run() { + while (!exitSignalled || !eventQueue.isEmpty()) { + List sendingEvents = new ArrayList(); + try { + sendingEvents.add( + eventQueue.take()); // this blocks until at least one event is available; + + eventQueue.drainTo( + sendingEvents, SEND_CAPACITY - sendingEvents.size()); // drain the rest of the queue + + // sleep a while to batch up events, but only sleep if there's no build up, if the + // sendingEvents reaches capacity, we should send events right the way + if (sendingEvents.size() < SEND_CAPACITY) { + try { + waitForNextSend(); + } catch (InterruptedException e) { + logger.debug("Queuing event wait interrupted"); + } + eventQueue.drainTo( + sendingEvents, SEND_CAPACITY - sendingEvents.size()); // try draining again + } + + Result result = synchronousSend(sendingEvents); + ResultCode resultCode = result.getResultCode(); + + if (resultCode.isError()) { + logger.debug("Failed to send out " + sendingEvents.size() + " events"); + stats.incrementFailedCount(sendingEvents.size()); + } else { + stats.incrementSentCount(sendingEvents.size()); + } + + } catch (Exception e) { + // do not retry the message, just log the problem + logger.debug( + "Failed to send " + + sendingEvents.size() + + " events, exception found: " + + e.getMessage()); // Should not be too noisy + stats.incrementFailedCount(sendingEvents.size()); + } finally { + stats.incrementProcessedCount(sendingEvents.size()); + } + } + scheduledExecutorService.shutdownNow(); + } + + /** Signals sending out events immediately */ + protected void sendNow() { + if (countDownLatch != null && countDownLatch.getCount() > 0) { + logger.debug("SendNow signaled for event reporter"); + countDownLatch.countDown(); + } + } + + /** Blocks until the flushInterval elapsed or sendNow is signaled */ + private void waitForNextSend() throws InterruptedException { + countDownLatch = new CountDownLatch(1); + scheduledExecutorService.schedule( + () -> countDownLatch.countDown(), flushInterval, TimeUnit.SECONDS); + countDownLatch.await(); + } + } + + /** + * Signals to sends an event via this reporter. Take note that the actual outbound request might + * be sent later on + * + * @throws EventReporterException if the reporter cannot accepts this event, for example the queue + * is full + */ + @Override + public void send(Event event) throws EventReporterException { + if (!eventQueue.offer(event)) { + stats.incrementOverflowedCount(1); + stats.incrementProcessedCount(1); + + if (!reportedQueueFull) { + long currentTime = System.currentTimeMillis(); + if (currentTime - reportedQueueFullTime + >= REPORT_QUEUE_FULL_INTERVAL) { // at most once every minute + logger.warn( + "Fail to report tracing event as the event queue is full in the reporter"); // only + // print + // warning + // on + // first + // queue + // overflow + reportedQueueFull = true; + reportedQueueFullTime = currentTime; + } + } + + throw new EventReporterQueueFullException("Cannot send event as the reporter queue is full"); + } else { + stats.setQueueCount(eventQueue.size()); + + if (eventQueue.size() + >= SEND_CAPACITY) { // should start sending early (if sleeping) as it's filling up + sendRunnable.sendNow(); + } + + reportedQueueFull = false; + } + } + + public void flush() { + sendRunnable.sendNow(); + } + + /** + * Gets and clears the current reporter stats + * + * @return + */ + @Override + public EventReporterStats consumeStats() { + return stats.consumeStats(); + } + + /** + * Closes this reporter orderly. Allow all submitted events to be processed with a default timeout + */ + @Override + public void close() { + logger.debug("Closing queueing event reporter, signaling shut down after sending all events"); + if (client.getStatus() != Client.Status.OK) { + logger.debug("RPC client is not OK. Shutting down the service now"); + executorService.shutdownNow(); + } else { + executorService.shutdown(); + } + + sendRunnable.exitSignalled = true; + try { + client.close(); + boolean termination = executorService.awaitTermination(5, TimeUnit.SECONDS); + logger.debug(String.format("Event reporter service shut down: [%s]", termination)); + } catch (InterruptedException ignore) { + } + } + + /** + * This should only return upon completion of sending all the provided events (success or failure) + * + * @param events + * @return + * @throws Exception + */ + public Result synchronousSend(List events) throws Exception { + return client + .postEvents(events, loggingCallback) + .get(); // block until the client has finished the request + } +} diff --git a/libs/core/src/main/java/com/solarwinds/joboe/core/ReporterFactory.java b/libs/core/src/main/java/com/solarwinds/joboe/core/ReporterFactory.java new file mode 100644 index 00000000..e4a5f8b6 --- /dev/null +++ b/libs/core/src/main/java/com/solarwinds/joboe/core/ReporterFactory.java @@ -0,0 +1,81 @@ +/* + * © SolarWinds Worldwide, LLC. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.solarwinds.joboe.core; + +import com.solarwinds.joboe.core.rpc.Client; +import com.solarwinds.joboe.logging.Logger; +import com.solarwinds.joboe.logging.LoggerFactory; +import java.io.IOException; +import lombok.Getter; + +/** + * Provide methods to create {@link EventReporter} + * + * @author pluk + */ +public class ReporterFactory { + private static final Logger logger = LoggerFactory.getLogger(); + + @Getter(lazy = true) + private static final ReporterFactory instance = new ReporterFactory(); + + private ReporterFactory() {} + + /** + * Builds a {@link UDPReporter}. Take note that this might create a singleton if the system has + * restrictions on UDP bind address/port, in such an environment, the singleton will have the + * host/port set to the first call to this method, any other calls following with different host + * and port would NOT reset the host/port + * + * @param host Destination host + * @param port Destination port + * @return + * @throws IOException + */ + UDPReporter createUdpReporter(String host, Integer port) throws IOException { + if (host == null || port == null) { + logger.error("Cannot build UDPReporter. Host and/or port params are null!"); + return null; + } + return new UDPReporter(host, port); + } + + /** + * Builds a {@link TestReporter}, take note that this reporter collects events from all threads + * + * @return + */ + public TestReporter createTestReporter() { + return createTestReporter(false); + } + + /** + * Builds a {@link TestReporter}, which works in thread local manner according to the parameter + * isThreadLocal + * + * @param isThreadLocal whether the TestReporter built should work in thread local + * manner + * @return + */ + public TestReporter createTestReporter(boolean isThreadLocal) { + return isThreadLocal ? new TestReporter() : new NonThreadLocalTestReporter(); + } + + public QueuingEventReporter createQueuingEventReporter(Client client) { + return new QueuingEventReporter(client); + } +} diff --git a/libs/core/src/main/java/com/solarwinds/joboe/core/TestReporter.java b/libs/core/src/main/java/com/solarwinds/joboe/core/TestReporter.java new file mode 100644 index 00000000..55d31619 --- /dev/null +++ b/libs/core/src/main/java/com/solarwinds/joboe/core/TestReporter.java @@ -0,0 +1,177 @@ +/* + * © SolarWinds Worldwide, LLC. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.solarwinds.joboe.core; + +import com.solarwinds.joboe.core.ebson.BsonDocument; +import com.solarwinds.joboe.core.ebson.BsonDocuments; +import com.solarwinds.joboe.logging.Logger; +import com.solarwinds.joboe.logging.LoggerFactory; +import java.nio.ByteBuffer; +import java.nio.ByteOrder; +import java.util.ArrayList; +import java.util.Deque; +import java.util.HashMap; +import java.util.HashSet; +import java.util.LinkedList; +import java.util.List; +import java.util.Map; +import java.util.Map.Entry; +import java.util.Set; + +/** + * Mock reporter: stores all 'sent' bytes in memory, for testing. Take note that this reporter works + * in a ThreadLocal manner such that events collected/retrieved are only visible to the thread + * itself + * + * @author pluk + */ +public class TestReporter implements EventReporter { + private static final Logger logger = LoggerFactory.getLogger(); + + private final ThreadLocal> byteBufList = ThreadLocal.withInitial(LinkedList::new); + + public TestReporter() {} + + public void reset() { + byteBufList.set(new LinkedList<>()); + } + + @Override + public void send(Event event) { + try { + byte[] buf = event.toBytes(); + logger.debug("Sent " + buf.length + " bytes"); + byteBufList.get().add(buf); + } catch (BsonBufferException e) { + logger.error("Failed to send events : " + e.getMessage(), e); + } + } + + public byte[] getLastSent() { + return byteBufList.get().getLast(); + } + + public List getSentEventsAsBsonDocument() { + List documents = new ArrayList(); + for (byte[] eventBytes : getBufList()) { + documents.add(getBsonDocumentFromBytes(eventBytes)); + } + + return documents; + } + + public List getSentEvents(boolean includeInitEvents) { + List events = new ArrayList(); + Set initLayers = new HashSet(); + for (byte[] eventBytes : getBufList()) { + boolean isInitEvent = false; + Map kvs = new HashMap(); + + for (Map.Entry kv : getEntriesFromBytes(eventBytes)) { + // make sure it is not the init event, we do not count init events here + if (!includeInitEvents) { + if ("__Init".equals(kv.getKey())) { // then this is the init entry event + isInitEvent = true; + } else if ("Layer".equals(kv.getKey())) { + if (initLayers.contains(kv.getValue().toString())) { // then this is the init exit event + isInitEvent = true; + } + } + } + kvs.put(kv.getKey(), kv.getValue()); + } + + if (!isInitEvent) { + events.add(new DeserializedEvent(kvs)); + } else { + // track the layer name, we want to get rid of the corresponding exit init event ooo + initLayers.add((String) kvs.get("Layer")); + } + } + return events; + } + + public List getSentEvents() { + return getSentEvents(false); + } + + public Deque getBufList() { + return byteBufList.get(); + } + + /** + * Event does not provide getInfo method, so we need this helper method to get the info in order + * to verify the result + * + * @param event + * @param key + * @return + */ + public static Object getValueFromEvent(Event event, String key) { + ByteBuffer buffer = + ByteBuffer.allocate(Constants.MAX_EVENT_BUFFER_SIZE).order(ByteOrder.LITTLE_ENDIAN); + try { + buffer.put(event.toBytes()); + buffer.flip(); + + BsonDocument doc = BsonDocuments.readFrom(buffer); + return doc.get(key); + } catch (BsonBufferException e) { + logger.error("Failed to get value from event : " + e.getMessage(), e); + return null; + } + } + + private static Set> getEntriesFromBytes(byte[] bytes) { + return getBsonDocumentFromBytes(bytes).entrySet(); + } + + private static BsonDocument getBsonDocumentFromBytes(byte[] bytes) { + ByteBuffer buffer = + ByteBuffer.allocate(Constants.MAX_EVENT_BUFFER_SIZE).order(ByteOrder.LITTLE_ENDIAN); + buffer.put(bytes); + buffer.flip(); + + return BsonDocuments.readFrom(buffer); + } + + public static class DeserializedEvent { + private Map kvs = new HashMap(); + + private DeserializedEvent(Map kvs) { + this.kvs = kvs; + } + + public Map getSentEntries() { + return new HashMap(kvs); + } + + @Override + public String toString() { + return getSentEntries().toString(); + } + } + + @Override + public EventReporterStats consumeStats() { + int sentEvent = getBufList().size(); + return new EventReporterStats(sentEvent, 0, 0, sentEvent, 0); + } + + @Override + public void close() {} +} diff --git a/libs/core/src/main/java/com/solarwinds/joboe/core/TestingEnv.java b/libs/core/src/main/java/com/solarwinds/joboe/core/TestingEnv.java new file mode 100644 index 00000000..a028aa12 --- /dev/null +++ b/libs/core/src/main/java/com/solarwinds/joboe/core/TestingEnv.java @@ -0,0 +1,37 @@ +/* + * © SolarWinds Worldwide, LLC. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.solarwinds.joboe.core; + +import com.solarwinds.joboe.core.settings.TestSettingsReader; +import lombok.Getter; + +@Getter +public class TestingEnv { + private final TestReporter tracingReporter; + private final TestSettingsReader settingsReader; + private final TestReporter profilingReporter; + + public TestingEnv( + TestReporter tracingReporter, + TestReporter profilingReporter, + TestSettingsReader settingsReader) { + super(); + this.tracingReporter = tracingReporter; + this.profilingReporter = profilingReporter; + this.settingsReader = settingsReader; + } +} diff --git a/libs/core/src/main/java/com/solarwinds/joboe/core/UDPReporter.java b/libs/core/src/main/java/com/solarwinds/joboe/core/UDPReporter.java new file mode 100644 index 00000000..4f85b7c3 --- /dev/null +++ b/libs/core/src/main/java/com/solarwinds/joboe/core/UDPReporter.java @@ -0,0 +1,112 @@ +/* + * © SolarWinds Worldwide, LLC. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.solarwinds.joboe.core; + +import static com.solarwinds.joboe.core.Constants.XTR_UDP_HOST; +import static com.solarwinds.joboe.core.Constants.XTR_UDP_PORT; + +import com.solarwinds.joboe.logging.Logger; +import com.solarwinds.joboe.logging.LoggerFactory; +import java.io.IOException; +import java.net.DatagramPacket; +import java.net.DatagramSocket; +import java.net.InetAddress; + +public class UDPReporter implements EventReporter { + private static final Logger logger = LoggerFactory.getLogger(); + private DatagramSocket socket; + + private InetAddress addr; + private int port; + + /** + * Creates UDP reporter with default destination Host and Port. + * + *

WARNING: this is ONLY for internal usage and will be removed from public access later on In + * order to build UDP reporter, please use {@link ReporterFactory} instead + * + * @throws IOException + */ + public UDPReporter() throws IOException { + this(XTR_UDP_HOST, XTR_UDP_PORT); + } + + /** + * Creates UDP reporter with provided destination Host and Port + * + *

WARNING: this is ONLY for internal usage and will be removed from public access later on In + * order to build UDP reporter, please use {@link ReporterFactory} instead + * + * @param host + * @param port + * @throws IOException + */ + public UDPReporter(String host, int port) throws IOException { + this(host, port, null, null); + } + + /** + * Create UDP report with provided destination Host and Port. Also bind the socket to the local + * address and port provided + * + * @param host + * @param port + * @param datagramLocalAddress + * @param datagramLocalPort + * @throws IOException + */ + UDPReporter(String host, int port, String datagramLocalAddress, Integer datagramLocalPort) + throws IOException { + init(host, port, datagramLocalAddress, datagramLocalPort); + } + + protected void init(String host, int port, String datagramLocalAddress, Integer datagramLocalPort) + throws IOException { + if (datagramLocalAddress != null && datagramLocalPort != null) { + socket = new DatagramSocket(datagramLocalPort, InetAddress.getByName(datagramLocalAddress)); + } else { + socket = new DatagramSocket(); + } + addr = InetAddress.getByName(host); + this.port = port; + } + + @Override + public void send(Event event) throws EventReporterException { + byte[] buf; + try { + buf = event.toBytes(); + } catch (BsonBufferException e) { + logger.error("Failed to get value from event : " + e.getMessage(), e); + throw new EventReporterException(e.getMessage(), e); + } + DatagramPacket pkt = new DatagramPacket(buf, buf.length, addr, port); + try { + socket.send(pkt); + } catch (IOException e) { + throw new EventReporterException(e.getMessage(), e); + } + } + + @Override + public EventReporterStats consumeStats() { + return new EventReporterStats(0, 0, 0, 0, 0); // not implemented + } + + @Override + public void close() {} +} diff --git a/libs/core/src/main/java/com/solarwinds/joboe/core/XTraceHeader.java b/libs/core/src/main/java/com/solarwinds/joboe/core/XTraceHeader.java new file mode 100644 index 00000000..d7c930cf --- /dev/null +++ b/libs/core/src/main/java/com/solarwinds/joboe/core/XTraceHeader.java @@ -0,0 +1,31 @@ +/* + * © SolarWinds Worldwide, LLC. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.solarwinds.joboe.core; + +/** + * Known X-Trace headers used by our Agent. Take note that these headers are not tied to any + * protocol (Http for instance). It is the caller who should map the input headers to the + * corresponding X-Trace headers of this enum + * + * @author Patson Luk + */ +public enum XTraceHeader { + TRACE_ID, + SPAN_ID, + TRACE_OPTIONS, + TRACE_OPTIONS_SIGNATURE +} diff --git a/libs/core/src/main/java/com/solarwinds/joboe/core/ebson/BasicObjectId.java b/libs/core/src/main/java/com/solarwinds/joboe/core/ebson/BasicObjectId.java new file mode 100644 index 00000000..92aa6c70 --- /dev/null +++ b/libs/core/src/main/java/com/solarwinds/joboe/core/ebson/BasicObjectId.java @@ -0,0 +1,106 @@ +/* + * © SolarWinds Worldwide, LLC. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.solarwinds.joboe.core.ebson; + +import com.google.common.base.Objects; +import java.nio.ByteBuffer; +import java.nio.ByteOrder; +import javax.xml.bind.DatatypeConverter; + +final class BasicObjectId implements BsonObjectId { + + private static final int TIME_LENGTH = 4; + private static final int MACHINE_ID_LENGTH = 3; + private static final int PROCESS_ID_LENGTH = 2; + private static final int INCREMENT_LENGTH = 3; + private static final int OBJECT_ID_LENGTH = + TIME_LENGTH + MACHINE_ID_LENGTH + PROCESS_ID_LENGTH + INCREMENT_LENGTH; + + private final ByteBuffer time; + private final ByteBuffer machineId; + private final ByteBuffer processId; + private final ByteBuffer increment; + + private final ByteBuffer objectId = + ByteBuffer.allocate(OBJECT_ID_LENGTH).order(ByteOrder.BIG_ENDIAN); + + BasicObjectId(ByteBuffer buffer) { + int oldPosition = buffer.position(); + + int oldLimit = buffer.limit(); + buffer.limit(buffer.position() + OBJECT_ID_LENGTH); + objectId.put(buffer).flip(); + buffer.limit(oldLimit); + + assert buffer.position() == oldPosition + OBJECT_ID_LENGTH; + + time = ByteBuffer.wrap(objectId.array(), 0, TIME_LENGTH).order(ByteOrder.BIG_ENDIAN); + machineId = + ByteBuffer.wrap(objectId.array(), TIME_LENGTH, MACHINE_ID_LENGTH) + .order(ByteOrder.LITTLE_ENDIAN); + processId = + ByteBuffer.wrap(objectId.array(), MACHINE_ID_LENGTH, PROCESS_ID_LENGTH) + .order(ByteOrder.LITTLE_ENDIAN); + increment = + ByteBuffer.wrap(objectId.array(), PROCESS_ID_LENGTH, INCREMENT_LENGTH) + .order(ByteOrder.BIG_ENDIAN); + } + + @Override + public ByteBuffer objectId() { + return objectId.asReadOnlyBuffer(); + } + + @Override + public ByteBuffer time() { + return time.asReadOnlyBuffer(); + } + + @Override + public ByteBuffer machineId() { + return machineId.asReadOnlyBuffer(); + } + + @Override + public ByteBuffer processId() { + return processId.asReadOnlyBuffer(); + } + + @Override + public ByteBuffer increment() { + return increment.asReadOnlyBuffer(); + } + + @Override + public int hashCode() { + return Objects.hashCode(objectId); + } + + @Override + public boolean equals(Object object) { + if (object instanceof BsonObjectId) { + BsonObjectId other = (BsonObjectId) object; + return objectId.equals(other.objectId()); + } + return false; + } + + @Override + public String toString() { + return DatatypeConverter.printHexBinary(objectId.array()).toLowerCase(); + } +} diff --git a/libs/core/src/main/java/com/solarwinds/joboe/core/ebson/BasicTimestamp.java b/libs/core/src/main/java/com/solarwinds/joboe/core/ebson/BasicTimestamp.java new file mode 100644 index 00000000..239d75c0 --- /dev/null +++ b/libs/core/src/main/java/com/solarwinds/joboe/core/ebson/BasicTimestamp.java @@ -0,0 +1,85 @@ +/* + * © SolarWinds Worldwide, LLC. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.solarwinds.joboe.core.ebson; + +import com.google.common.base.Objects; +import java.nio.ByteBuffer; +import java.nio.ByteOrder; +import javax.xml.bind.DatatypeConverter; + +final class BasicTimestamp implements BsonTimestamp { + + private static final int TIME_LENGTH = 4; + private static final int INCREMENT_LENGTH = 4; + private static final int TIMESTAMP_LENGTH = TIME_LENGTH + INCREMENT_LENGTH; + + private final ByteBuffer time; + private final ByteBuffer increment; + + private final ByteBuffer timestamp = + ByteBuffer.allocate(TIMESTAMP_LENGTH).order(ByteOrder.LITTLE_ENDIAN); + + BasicTimestamp(ByteBuffer buffer) { + int oldPosition = buffer.position(); + + int oldLimit = buffer.limit(); + buffer.limit(buffer.position() + TIMESTAMP_LENGTH); + timestamp.put(buffer).flip(); + buffer.limit(oldLimit); + + assert buffer.position() == oldPosition + TIMESTAMP_LENGTH; + + time = ByteBuffer.wrap(timestamp.array(), 0, TIME_LENGTH).order(ByteOrder.LITTLE_ENDIAN); + increment = + ByteBuffer.wrap(timestamp.array(), INCREMENT_LENGTH, TIME_LENGTH) + .order(ByteOrder.LITTLE_ENDIAN); + } + + @Override + public ByteBuffer timestamp() { + return timestamp.asReadOnlyBuffer(); + } + + @Override + public ByteBuffer time() { + return time.asReadOnlyBuffer(); + } + + @Override + public ByteBuffer increment() { + return increment.asReadOnlyBuffer(); + } + + @Override + public int hashCode() { + return Objects.hashCode(timestamp); + } + + @Override + public boolean equals(Object object) { + if (object instanceof BsonTimestamp) { + BsonTimestamp other = (BsonTimestamp) object; + return timestamp.equals(other.timestamp()); + } + return false; + } + + @Override + public String toString() { + return DatatypeConverter.printHexBinary(timestamp.array()).toLowerCase(); + } +} diff --git a/libs/core/src/main/java/com/solarwinds/joboe/core/ebson/BsonBinary.java b/libs/core/src/main/java/com/solarwinds/joboe/core/ebson/BsonBinary.java new file mode 100644 index 00000000..775e1ed0 --- /dev/null +++ b/libs/core/src/main/java/com/solarwinds/joboe/core/ebson/BsonBinary.java @@ -0,0 +1,172 @@ +/* + * © SolarWinds Worldwide, LLC. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.solarwinds.joboe.core.ebson; + +import com.google.common.base.Preconditions; +import com.google.common.base.Predicate; +import com.google.common.base.Predicates; +import javax.annotation.Nullable; + +/** Representation of a BSON binary (sub-)type. */ +public enum BsonBinary { + + /** + * Generic binary. + * + *

Note: the most commonly used binary sub-type and should be the 'default' for drivers + * and tools. + */ + GENERIC( + BsonBytes.GENERIC, DefaultPredicate.GENERIC, DefaultReader.GENERIC, DefaultWriter.GENERIC), + + /** Function binary. */ + FUNCTION(BsonBytes.FUNCTION), + + /** + * Old binary. + * + *

Note: this used to be the default subtype, but was deprecated in favor of the generic + * binary type. + */ + OLD(BsonBytes.OLD), + + /** UUID binary. */ + UUID(BsonBytes.UUID), + + /** MD5 binary. */ + MD5(BsonBytes.MD5), + + /** User-defined binary. */ + USER(BsonBytes.USER); + + private final byte terminal; + + private Predicate> predicate; + private BsonReader reader; + private BsonWriter writer; + + BsonBinary(byte terminal) { + this(terminal, Predicates.alwaysFalse(), null, null); + } + + BsonBinary(byte terminal, Predicate> predicate, BsonReader reader, BsonWriter writer) { + this.terminal = terminal; + this.predicate = predicate; + this.reader = reader; + this.writer = writer; + } + + /** + * Returns this binary's associated terminal. + * + * @return this binary's associated terminal + */ + public byte terminal() { + return terminal; + } + + /** + * Returns this binary's associated {@linkplain Predicate predicate}. + * + * @return this binary's associated predicate + * @throws IllegalStateException if this binary does not have an associated predicate + */ + public Predicate> predicate() { + Preconditions.checkState(predicate != null, "'%s' does not have an associated predicate", this); + return predicate; + } + + /** + * Associates {@link Predicate predicate} with this binary. + * + * @param predicate the predicate to be associated with this binary + */ + public void predicate(Predicate> predicate) { + Preconditions.checkNotNull(predicate, "cannot associate a null predicate with '%s'", this); + this.predicate = predicate; + } + + /** + * Returns this binary's associated {@linkplain BsonReader reader}. + * + * @return this binary's associated reader + * @throws IllegalStateException if this binary does not have an associated reader + */ + public BsonReader reader() { + Preconditions.checkState(reader != null, "'%s' does not have an associated reader", this); + return reader; + } + + /** + * Associates {@link BsonReader reader} with this binary. + * + * @param reader the reader to be associated with this binary + */ + public void reader(BsonReader reader) { + Preconditions.checkNotNull(reader, "cannot associate a null reader with '%s'", this); + this.reader = reader; + } + + /** + * Returns this binary's associated {@linkplain BsonWriter writer}. + * + * @return this binary's associated writer + * @throws IllegalStateException if this binary does not have an associated writer + */ + public BsonWriter writer() { + Preconditions.checkState(writer != null, "'%s' does not have an associated writer", this); + return writer; + } + + /** + * Associates {@link BsonWriter writer} with this binary. + * + * @param writer the writer to be associated with this binary + */ + public void writer(BsonWriter writer) { + Preconditions.checkNotNull(writer, "cannot associate a null writer with '%s'", this); + this.writer = writer; + } + + /** + * Returns the binary representing {@code clazz}. + * + * @param clazz the class to return a binary representation for + * @return the binary representing {@code clazz} + * @throws IllegalArgumentException if no binary representing {@code clazz} was found + */ + public static BsonBinary find(@Nullable Class clazz) { + for (BsonBinary binary : values()) if (binary.predicate().apply(clazz)) return binary; + throw new IllegalArgumentException( + String.format("no binary " + "representing the '%s' type value was found", clazz)); + } + + /** + * Returns the binary representing {@code terminal}. + * + * @param terminal the terminal to return a binary representation for + * @return the binary representing {@code terminal} + * @throws IllegalArgumentException if no binary representing {@code terminal} was found + */ + public static BsonBinary find(byte terminal) { + for (BsonBinary binary : values()) if (binary.terminal() - terminal == 0) return binary; + throw new IllegalArgumentException( + String.format( + "no binary representing " + "the '%s' terminal value was found", + Byte.valueOf(terminal))); + } +} diff --git a/libs/core/src/main/java/com/solarwinds/joboe/core/ebson/BsonBytes.java b/libs/core/src/main/java/com/solarwinds/joboe/core/ebson/BsonBytes.java new file mode 100644 index 00000000..c7589f75 --- /dev/null +++ b/libs/core/src/main/java/com/solarwinds/joboe/core/ebson/BsonBytes.java @@ -0,0 +1,123 @@ +/* + * © SolarWinds Worldwide, LLC. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.solarwinds.joboe.core.ebson; + +/** Frequently used byte values. */ +public final class BsonBytes { + + /** EOF (-1). */ + public static final byte EOF = -1; + + /** EOO (0). */ + public static final byte EOO = 0; + + /** DOUBLE (1). */ + public static final byte DOUBLE = 1; + + /** STRING (2). */ + public static final byte STRING = 2; + + /** EMBEDDED (3). */ + public static final byte EMBEDDED = 3; + + /** ARRAY (4). */ + public static final byte ARRAY = 4; + + /** BINARY (5). */ + public static final byte BINARY = 5; + + /** GENERIC (0). */ + public static final byte GENERIC = 0; + + /** FUNCTION (1). */ + public static final byte FUNCTION = 1; + + /** OLD (2). */ + public static final byte OLD = 2; + + /** UUID (3). */ + public static final byte UUID = 3; + + /** MD5 (5). */ + public static final byte MD5 = 5; + + /** USER (128). */ + public static final byte USER = (byte) 128; + + /** + * UNDEFINED (6). + * + * @deprecated See the BSON specification for details. + */ + @Deprecated public static final byte UNDEFINED = 6; + + /** OBJECT_ID (7). */ + public static final byte OBJECT_ID = 7; + + /** BOOLEAN (8). */ + public static final byte BOOLEAN = 8; + + /** FALSE (0). */ + public static final byte FALSE = 0; + + /** TRUE (1). */ + public static final byte TRUE = 1; + + /** UTC_DATE_TIME (9). */ + public static final byte UTC_DATE_TIME = 9; + + /** NULL (10). */ + public static final byte NULL = 10; + + /** REGULAR_EXPRESSION (11). */ + public static final byte REGULAR_EXPRESSION = 11; + + /** + * DB_POINTER (12). + * + * @deprecated See the following link for + * more details. + */ + @Deprecated public static final byte DB_POINTER = 12; + + /** JAVASCRIPT_CODE (13). */ + public static final byte JAVASCRIPT_CODE = 13; + + /** SYMBOL (14). */ + public static final byte SYMBOL = 14; + + /** JAVASCRIPT_CODE_WITH_SCOPE (15). */ + public static final byte JAVASCRIPT_CODE_WITH_SCOPE = 15; + + /** INT32 (16). */ + public static final byte INT32 = 16; + + /** TIMESTAMP (17). */ + public static final byte TIMESTAMP = 17; + + /** INT64 (18). */ + public static final byte INT64 = 18; + + /** MAX_KEY (127). */ + public static final byte MAX_KEY = 127; + + /** MIN_KEY (255). */ + public static final byte MIN_KEY = (byte) 255; + + private BsonBytes() {} +} diff --git a/libs/core/src/main/java/com/solarwinds/joboe/core/ebson/BsonDocument.java b/libs/core/src/main/java/com/solarwinds/joboe/core/ebson/BsonDocument.java new file mode 100644 index 00000000..dd0760ba --- /dev/null +++ b/libs/core/src/main/java/com/solarwinds/joboe/core/ebson/BsonDocument.java @@ -0,0 +1,250 @@ +/* + * © SolarWinds Worldwide, LLC. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.solarwinds.joboe.core.ebson; + +import java.util.Collection; +import java.util.Map; +import java.util.Set; +import javax.annotation.CheckForNull; +import javax.annotation.Nullable; + +/** + * Immutable representation of a BSON document. + * + *

Notes: + * + *

    + *
  • None of the {@linkplain Map map interface's} optional operations are supported. + *
  • Use the {@linkplain BsonDocuments documents utility class} to create new documents. + *
+ */ +public interface BsonDocument extends Map { + + /** + * Returns this document's size (the number of its keys). + * + * @return this document's size + */ + @Override + int size(); + + /** + * Returns true if this document contains no key-value pairs; false otherwise. + * + * @return true if this document contains no key-value pairs; false otherwise + */ + @Override + boolean isEmpty(); + + /** + * Returns an immutable view of this document's keys. + * + * @return an immutable view of this document's keys. + */ + @Override + Set keySet(); + + /** + * Returns an immutable view of this document's values. + * + * @return an immutable view of this document's values + */ + @Override + Collection values(); + + /** + * Returns an immutable view of this document's key-value pairs. + * + * @return an immutable view of this document's key-value pairs + */ + @Override + Set> entrySet(); + + /** + * Returns the value associated with {@code key} in a type-safe manner. + * + *

Note: {@code type} can and should be null only iff the value associated with + * {@code key} is expected to be null. + * + * @param key the key whose associated value is to be returned + * @param type the expected type of the value associated with {@code key} + * @param the expected type of the value associated with {@code key} + * @return the value associated with {@code key} in a type-safe manner + * @throws NullPointerException if {@code key} is null + * @throws IllegalArgumentException if this document does not contain {@code key} or if the value + * associated with it is not assignment-compatible with {@code type} + * @throws ClassCastException if {@code key} is not a string + */ + @Nullable T get(Object key, @Nullable Class type); + + /** + * Returns the value associated with {@code key}. + * + * @param key the key whose associated value is to be returned + * @return the value associated with {@code key} + * @throws NullPointerException if {@code key} is null + * @throws IllegalArgumentException if this document does not contain {@code key} + * @throws ClassCastException if {@code key} is not a string + */ + @Override + @CheckForNull + Object get(Object key); + + /** + * Returns true if {@code key} is contained by this document; false otherwise. + * + * @param key the key to be tested if it is contained by this document + * @return true if {@code key} is contained by this document; false otherwise + * @throws NullPointerException if {@code key} is null + * @throws ClassCastException if {@code key} is not a string + */ + @Override + boolean containsKey(Object key); + + /** + * Returns true if {@code value} is contained by this document; false otherwise. + * + * @param value the value to be tested if it is contained by this document + * @return true if {@code value} is contained by this document; false otherwise + */ + @Override + boolean containsValue(@Nullable Object value); + + /** + * Returns this document's hash code (calculated using its {@linkplain #entrySet() key-value + * pairs}). + * + * @return this document's hash code + */ + @Override + int hashCode(); + + /** + * Returns true if {@code object} is the same as this document; false otherwise. + * + *

{@code object} is the same as this document iff: + * + *

    + *
  • {@code this == object} or + *
  • {@code object instanceof Map} and + *
  • {@code this.entrySet().equals(object.entrySet())}. + *
+ * + * @param object the reference object with which to compare + * @return true if {@code object} is the same as this document; false otherwise + */ + @Override + boolean equals(@CheckForNull Object object); + + /** + * Returns this document's textual representation. + * + *

The general format is the following: + * + *

    + *
  • Empty: {} + *
  • Single key-value pair: {key: value} + *
  • Multiple key-value pair: {key1: value1, key2: value2, ...} + *
  • Embedded: {key: {key: value}} + *
  • ... + *
+ * + * @return this document's textual representation + */ + @Override + String toString(); + + /** + * Not supported. + * + * @param key not supported + * @return not supported + * @throws UnsupportedOperationException on every invocation of this method + */ + @Override + @Nullable Object remove(Object key); + + /** + * Not supported. + * + * @throws UnsupportedOperationException on every invocation of this method + */ + @Override + void clear(); + + /** + * Not supported. + * + * @param map not supported + * @throws UnsupportedOperationException on every invocation of this method + */ + @Override + void putAll(Map map); + + /** + * {@linkplain BsonDocument Document} builder. + * + *

Notes: + * + *

    + *
  • Calling a builder's {@linkplain #build} method does not clear its state, however + * subsequent invocations of it returns new immutable documents. + *
  • Document builders can be acquired via the {@linkplain BsonDocuments documents utility + * class}. + *
+ */ + interface Builder { + + /** + * Adds {@code key} and its associated {@code value} to the document being built. + * + * @param key the key to be added to the document being built + * @param value the value associated with {@code key} + * @return this builder + * @throws NullPointerException if {@code key} is null + * @throws IllegalArgumentException if {@code key} is already present + */ + Builder put(String key, @Nullable Object value); + + /** + * Adds {@code map}'s key-value pairs to the document being built. + * + * @param map the map whose key-value pairs are to be added to the document being built + * @return this builder + * @throws NullPointerException if {@code map} or any of its keys are null + * @throws IllegalArgumentException if {@code map} contains keys that are already present + */ + Builder putAll(Map map); + + /** + * A patched method to allow same key with multiple values (MULTIVAL), this should only be used + * for reading + * + * @param key + * @param value + * @return + */ + Builder putAllowMultiVal(String key, @Nullable Object value); + + /** + * Returns a new document using the contents of this builder. + * + * @return a new document using the contents of this builder + */ + BsonDocument build(); + } +} diff --git a/libs/core/src/main/java/com/solarwinds/joboe/core/ebson/BsonDocuments.java b/libs/core/src/main/java/com/solarwinds/joboe/core/ebson/BsonDocuments.java new file mode 100644 index 00000000..65fcb96c --- /dev/null +++ b/libs/core/src/main/java/com/solarwinds/joboe/core/ebson/BsonDocuments.java @@ -0,0 +1,344 @@ +/* + * © SolarWinds Worldwide, LLC. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.solarwinds.joboe.core.ebson; + +import java.nio.ByteBuffer; +import java.util.Map; +import javax.annotation.Nullable; + +/** Utility class for working with {@linkplain BsonDocument documents}. */ +public final class BsonDocuments { + + private BsonDocuments() {} + + /** + * Reads a new document from {@code buffer}. + * + * @param buffer the buffer that contains a document's serialized data + * @return a new document from {@code buffer} + * @throws NullPointerException if {@code buffer} is null + * @throws IllegalArgumentException if {@code buffer} is not using little-endian byte ordering + */ + public static BsonDocument readFrom(ByteBuffer buffer) { + return (BsonDocument) BsonToken.DOCUMENT.reader().readFrom(buffer); + } + + /** + * Writes {@code document} to {@code buffer}. + * + * @param buffer the buffer to write to + * @param document the document to be written into {@code buffer} + * @throws NullPointerException if {@code buffer} or {@code document} is null + * @throws IllegalArgumentException if {@code buffer} is not using little-endian byte ordering + */ + public static void writeTo(ByteBuffer buffer, BsonDocument document) { + BsonToken.DOCUMENT.writer().writeTo(buffer, document); + } + + /** + * Returns a new document containing {@code map}'s key-value pairs. + * + * @param map the map whose key-value pairs will be used to initialize the new document + * @return a new document containing {@code map}'s key-value pairs + * @throws NullPointerException if {@code map} or any of its keys are null + */ + public static BsonDocument copyOf(Map map) { + return builder().putAll(map).build(); + } + + // @checkstyle:off ParameterNumber|JavadocMethod + + /** + * Returns a new document containing the key-value pairs: k1: v1, k2: v2, etc. + * + * @return a new document containing the key-value pairs: k1: v1, k2: v2 etc. + * @throws NullPointerException if any key-value pair's key is null + * @throws IllegalArgumentException if there are duplicate keys + */ + public static BsonDocument of( + String k1, + @Nullable Object v1, + String k2, + @Nullable Object v2, + String k3, + @Nullable Object v3, + String k4, + @Nullable Object v4, + String k5, + @Nullable Object v5, + String k6, + @Nullable Object v6, + String k7, + @Nullable Object v7, + String k8, + @Nullable Object v8, + String k9, + @Nullable Object v9, + String k10, + @Nullable Object v10) { + return builder() + .put(k1, v1) + .put(k2, v2) + .put(k3, v3) + .put(k4, v4) + .put(k5, v5) + .put(k6, v6) + .put(k7, v7) + .put(k8, v8) + .put(k9, v9) + .put(k10, v10) + .build(); + } + + /** + * Returns a new document containing the key-value pairs: k1: v1, k2: v2, etc. + * + * @return a new document containing the key-value pairs: k1: v1, k2: v2 etc. + * @throws NullPointerException if any key-value pair's key is null + * @throws IllegalArgumentException if there are duplicate keys + */ + public static BsonDocument of( + String k1, + @Nullable Object v1, + String k2, + @Nullable Object v2, + String k3, + @Nullable Object v3, + String k4, + @Nullable Object v4, + String k5, + @Nullable Object v5, + String k6, + @Nullable Object v6, + String k7, + @Nullable Object v7, + String k8, + @Nullable Object v8, + String k9, + @Nullable Object v9) { + return builder() + .put(k1, v1) + .put(k2, v2) + .put(k3, v3) + .put(k4, v4) + .put(k5, v5) + .put(k6, v6) + .put(k7, v7) + .put(k8, v8) + .put(k9, v9) + .build(); + } + + /** + * Returns a new document containing the key-value pairs: k1: v1, k2: v2, etc. + * + * @return a new document containing the key-value pairs: k1: v1, k2: v2 etc. + * @throws NullPointerException if any key-value pair's key is null + * @throws IllegalArgumentException if there are duplicate keys + */ + public static BsonDocument of( + String k1, + @Nullable Object v1, + String k2, + @Nullable Object v2, + String k3, + @Nullable Object v3, + String k4, + @Nullable Object v4, + String k5, + @Nullable Object v5, + String k6, + @Nullable Object v6, + String k7, + @Nullable Object v7, + String k8, + @Nullable Object v8) { + return builder() + .put(k1, v1) + .put(k2, v2) + .put(k3, v3) + .put(k4, v4) + .put(k5, v5) + .put(k6, v6) + .put(k7, v7) + .put(k8, v8) + .build(); + } + + /** + * Returns a new document containing the key-value pairs: k1: v1, k2: v2, etc. + * + * @return a new document containing the key-value pairs: k1: v1, k2: v2 etc. + * @throws NullPointerException if any key-value pair's key is null + * @throws IllegalArgumentException if there are duplicate keys + */ + public static BsonDocument of( + String k1, + @Nullable Object v1, + String k2, + @Nullable Object v2, + String k3, + @Nullable Object v3, + String k4, + @Nullable Object v4, + String k5, + @Nullable Object v5, + String k6, + @Nullable Object v6, + String k7, + @Nullable Object v7) { + return builder() + .put(k1, v1) + .put(k2, v2) + .put(k3, v3) + .put(k4, v4) + .put(k5, v5) + .put(k6, v6) + .put(k7, v7) + .build(); + } + + /** + * Returns a new document containing the key-value pairs: k1: v1, k2: v2, etc. + * + * @return a new document containing the key-value pairs: k1: v1, k2: v2 etc. + * @throws NullPointerException if any key-value pair's key is null + * @throws IllegalArgumentException if there are duplicate keys + */ + public static BsonDocument of( + String k1, + @Nullable Object v1, + String k2, + @Nullable Object v2, + String k3, + @Nullable Object v3, + String k4, + @Nullable Object v4, + String k5, + @Nullable Object v5, + String k6, + @Nullable Object v6) { + return builder() + .put(k1, v1) + .put(k2, v2) + .put(k3, v3) + .put(k4, v4) + .put(k5, v5) + .put(k6, v6) + .build(); + } + + /** + * Returns a new document containing the key-value pairs: k1: v1, k2: v2, etc. + * + * @return a new document containing the key-value pairs: k1: v1, k2: v2 etc. + * @throws NullPointerException if any key-value pair's key is null + * @throws IllegalArgumentException if there are duplicate keys + */ + public static BsonDocument of( + String k1, + @Nullable Object v1, + String k2, + @Nullable Object v2, + String k3, + @Nullable Object v3, + String k4, + @Nullable Object v4, + String k5, + @Nullable Object v5) { + return builder().put(k1, v1).put(k2, v2).put(k3, v3).put(k4, v4).put(k5, v5).build(); + } + + /** + * Returns a new document containing the key-value pairs: k1: v1, k2: v2, etc. + * + * @return a new document containing the key-value pairs: k1: v1, k2: v2 etc. + * @throws NullPointerException if any key-value pair's key is null + * @throws IllegalArgumentException if there are duplicate keys + */ + public static BsonDocument of( + String k1, + @Nullable Object v1, + String k2, + @Nullable Object v2, + String k3, + @Nullable Object v3, + String k4, + @Nullable Object v4) { + return builder().put(k1, v1).put(k2, v2).put(k3, v3).put(k4, v4).build(); + } + + /** + * Returns a new document containing the key-value pairs: k1: v1, k2: v2, etc. + * + * @return a new document containing the key-value pairs: k1: v1, k2: v2 etc. + * @throws NullPointerException if any key-value pair's key is null + * @throws IllegalArgumentException if there are duplicate keys + */ + public static BsonDocument of( + String k1, + @Nullable Object v1, + String k2, + @Nullable Object v2, + String k3, + @Nullable Object v3) { + return builder().put(k1, v1).put(k2, v2).put(k3, v3).build(); + } + + /** + * Returns a new document containing the key-value pairs: k1: v1, k2: v2. + * + * @return a new document containing the key-value pairs: k1: v1, k2: v2 + * @throws NullPointerException if any key-value pair's key is null + * @throws IllegalArgumentException if there are duplicate keys + */ + public static BsonDocument of(String k1, @Nullable Object v1, String k2, @Nullable Object v2) { + return builder().put(k1, v1).put(k2, v2).build(); + } + + // @checkstyle:on ParameterNumber|JavadocMethod + + /** + * Returns a new document containing {@code key} and its associated {@code value}. + * + * @param key the key that will be used to initialize the new document + * @param value the value associated with {@code key} + * @return a new document containing {@code key} and its associated {@code value} + * @throws NullPointerException if {@code key} is null + */ + public static BsonDocument of(String key, @Nullable Object value) { + return builder().put(key, value).build(); + } + + /** + * Returns the empty document. + * + * @return the empty document + */ + public static BsonDocument of() { + return builder().build(); + } + + /** + * Returns a new {@linkplain BsonDocument.Builder document builder}. + * + * @return a new document builder + */ + public static BsonDocument.Builder builder() { + return new DefaultDocumentBuilder(); + } +} diff --git a/libs/core/src/main/java/com/solarwinds/joboe/core/ebson/BsonObject.java b/libs/core/src/main/java/com/solarwinds/joboe/core/ebson/BsonObject.java new file mode 100644 index 00000000..3637e557 --- /dev/null +++ b/libs/core/src/main/java/com/solarwinds/joboe/core/ebson/BsonObject.java @@ -0,0 +1,276 @@ +/* + * © SolarWinds Worldwide, LLC. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.solarwinds.joboe.core.ebson; + +import com.google.common.base.Preconditions; +import com.google.common.base.Predicate; +import com.google.common.base.Predicates; +import javax.annotation.Nullable; + +/** + * Representation of a BSON object type. + * + * @see BsonToken + * @see BsonBinary + */ +public enum BsonObject { + + /** 64-bit IEEE 754 floating point. */ + DOUBLE(BsonBytes.DOUBLE, DefaultPredicate.DOUBLE, DefaultReader.DOUBLE, DefaultWriter.DOUBLE), + + /** UTF-8 string. */ + STRING(BsonBytes.STRING, DefaultPredicate.STRING, DefaultReader.STRING, DefaultWriter.STRING), + + /** Embedded {@linkplain BsonToken#DOCUMENT document}. */ + EMBEDDED( + BsonBytes.EMBEDDED, + DefaultPredicate.EMBEDDED, + BsonToken.DOCUMENT.reader(), + BsonToken.DOCUMENT.writer()), + + /** + * Special embedded {@linkplain #EMBEDDED document}. + * + *

Note: an array is a document whose keys are integer values starting with 0 and + * continuing sequentially. + */ + ARRAY(BsonBytes.ARRAY, DefaultPredicate.ARRAY, DefaultReader.ARRAY, DefaultWriter.ARRAY), + + /** Binary data. */ + BINARY(BsonBytes.BINARY, DefaultPredicate.BINARY, DefaultReader.BINARY, DefaultWriter.BINARY), + + /** + * Undefined. + * + * @deprecated See the BSON specification for details. + */ + @Deprecated + UNDEFINED(BsonBytes.UNDEFINED), + + /** + * Object ID. + * + *

Note: special MongoDB related type. + */ + OBJECT_ID( + BsonBytes.OBJECT_ID, + DefaultPredicate.OBJECT_ID, + DefaultReader.OBJECT_ID, + DefaultWriter.OBJECT_ID), + + /** + * Boolean. + * + *

Note: 0 is false; 1 is true. + */ + BOOLEAN( + BsonBytes.BOOLEAN, DefaultPredicate.BOOLEAN, DefaultReader.BOOLEAN, DefaultWriter.BOOLEAN), + + /** + * UTC date-time. + * + *

Note: milliseconds since the Unix epoch. + */ + UTC_DATE_TIME( + BsonBytes.UTC_DATE_TIME, + DefaultPredicate.UTC_DATE_TIME, + DefaultReader.UTC_DATE_TIME, + DefaultWriter.UTC_DATE_TIME), + + /** null. */ + NULL(BsonBytes.NULL, DefaultPredicate.NULL, DefaultReader.NULL, DefaultWriter.NULL), + + /** Regular expression. */ + REGULAR_EXPRESSION( + BsonBytes.REGULAR_EXPRESSION, + DefaultPredicate.REGULAR_EXPRESSION, + DefaultReader.REGULAR_EXPRESSION, + DefaultWriter.REGULAR_EXPRESSION), + + /** + * DB pointer. + * + * @deprecated See the following link for + * more details. + */ + @Deprecated + DB_POINTER(BsonBytes.DB_POINTER), + + /** JavaScript code. */ + JAVASCRIPT_CODE(BsonBytes.JAVASCRIPT_CODE), + + /** + * Symbol. + * + *

Note: similar to a string but for languages with a distinct symbol type. + */ + SYMBOL(BsonBytes.SYMBOL), + + /** JavaScript code with scope. */ + JAVASCRIPT_CODE_WITH_SCOPE(BsonBytes.JAVASCRIPT_CODE_WITH_SCOPE), + + /** 32-bit signed integer. */ + INT32(BsonBytes.INT32, DefaultPredicate.INT32, DefaultReader.INT32, DefaultWriter.INT32), + + /** + * Timestamp. + * + *

Note: special internal type used by MongoDB replication and sharding. + */ + TIMESTAMP( + BsonBytes.TIMESTAMP, + DefaultPredicate.TIMESTAMP, + DefaultReader.TIMESTAMP, + DefaultWriter.TIMESTAMP), + + /** 64-bit signed integer. */ + INT64(BsonBytes.INT64, DefaultPredicate.INT64, DefaultReader.INT64, DefaultWriter.INT64), + + /** + * Max key. + * + *

Note: special type which compares higher than all other possible {@code BSON} element + * values. + */ + MAX_KEY(BsonBytes.MAX_KEY), + + /** + * Min key. + * + *

Note: special type which compares lower than all other possible {@code BSON} element + * values. + */ + MIN_KEY(BsonBytes.MIN_KEY); + + private final byte terminal; + + private Predicate> predicate; + private BsonReader reader; + private BsonWriter writer; + + BsonObject(byte terminal) { + this(terminal, Predicates.alwaysFalse(), null, null); + } + + BsonObject(byte terminal, Predicate> predicate, BsonReader reader, BsonWriter writer) { + this.terminal = terminal; + this.predicate = predicate; + this.reader = reader; + this.writer = writer; + } + + /** + * Returns this object's associated terminal. + * + * @return this object's associated terminal + */ + public byte terminal() { + return terminal; + } + + /** + * Returns this object's associated {@linkplain Predicate predicate}. + * + * @return this object's associated predicate + * @throws IllegalStateException if this object does not have an associated predicate + */ + public Predicate> predicate() { + Preconditions.checkState(predicate != null, "'%s' does not have an associated predicate", this); + return predicate; + } + + /** + * Associates {@link Predicate predicate} with this object. + * + * @param predicate the predicate to be associated with this object + */ + public void predicate(Predicate> predicate) { + Preconditions.checkNotNull(predicate, "cannot associate a null predicate with '%s'", this); + this.predicate = predicate; + } + + /** + * Returns this object's associated {@linkplain BsonReader reader}. + * + * @return this object's associated reader + * @throws IllegalStateException if this object does not have an associated reader + */ + public BsonReader reader() { + Preconditions.checkState(reader != null, "'%s' does not have an associated reader", this); + return reader; + } + + /** + * Associates {@link BsonReader reader} with this object. + * + * @param reader the reader to be associated with this object + */ + public void reader(BsonReader reader) { + Preconditions.checkNotNull(predicate, "cannot associate a null reader with '%s'", this); + this.reader = reader; + } + + /** + * Returns this object's associated {@linkplain BsonWriter writer}. + * + * @return this object's associated writer + * @throws IllegalStateException if this object does not have an associated writer + */ + public BsonWriter writer() { + Preconditions.checkState(writer != null, "'%s' does not have an associated writer", this); + return writer; + } + + /** + * Associates {@link BsonWriter writer} with this object. + * + * @param writer the writer to be associated with this object + */ + public void writer(BsonWriter writer) { + Preconditions.checkNotNull(writer, "cannot associate a null writer with '%s'", this); + this.writer = writer; + } + + /** + * Returns the object representing {@code clazz}. + * + * @param clazz the class to return a object representation for + * @return the object representing {@code clazz} + * @throws IllegalArgumentException if no object representing {@code clazz} was found + */ + public static BsonObject find(@Nullable Class clazz) { + for (BsonObject object : values()) if (object.predicate().apply(clazz)) return object; + throw new IllegalArgumentException( + String.format("no object " + "representing the '%s' type value was found", clazz)); + } + + /** + * Returns the object representing {@code terminal}. + * + * @param terminal the terminal to return a object representation for + * @return the object representing {@code terminal} + * @throws IllegalArgumentException if no object representing {@code terminal} was found + */ + public static BsonObject find(byte terminal) { + for (BsonObject object : values()) if (object.terminal() - terminal == 0) return object; + throw new IllegalArgumentException( + String.format( + "no object representing " + "the '%s' terminal value was found", + Byte.valueOf(terminal))); + } +} diff --git a/libs/core/src/main/java/com/solarwinds/joboe/core/ebson/BsonObjectId.java b/libs/core/src/main/java/com/solarwinds/joboe/core/ebson/BsonObjectId.java new file mode 100644 index 00000000..1e180a66 --- /dev/null +++ b/libs/core/src/main/java/com/solarwinds/joboe/core/ebson/BsonObjectId.java @@ -0,0 +1,33 @@ +/* + * © SolarWinds Worldwide, LLC. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.solarwinds.joboe.core.ebson; + +import java.nio.ByteBuffer; + +// @checkstyle:off . +public interface BsonObjectId { + + ByteBuffer objectId(); + + ByteBuffer time(); + + ByteBuffer machineId(); + + ByteBuffer processId(); + + ByteBuffer increment(); +} diff --git a/libs/core/src/main/java/com/solarwinds/joboe/core/ebson/BsonReader.java b/libs/core/src/main/java/com/solarwinds/joboe/core/ebson/BsonReader.java new file mode 100644 index 00000000..68760907 --- /dev/null +++ b/libs/core/src/main/java/com/solarwinds/joboe/core/ebson/BsonReader.java @@ -0,0 +1,40 @@ +/* + * © SolarWinds Worldwide, LLC. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.solarwinds.joboe.core.ebson; + +import java.nio.ByteBuffer; +import javax.annotation.Nullable; + +/** + * Reads Java object(s) from {@linkplain ByteBuffer buffers}, deserialized from bytes as specified + * by the BSON specification. + * + *

Note: buffers supplied to {@linkplain #readFrom} must use little-endian byte ordering. + */ +public interface BsonReader { + + /** + * Reads an arbitrary amount of bytes from {@code buffer} and constructs a new object from what + * was read. + * + * @param buffer the buffer to read from + * @return a new object from the bytes read from {@code buffer} + * @throws NullPointerException if {@code buffer} is null + * @throws IllegalArgumentException if {@code buffer} is not using little-endian byte ordering + */ + @Nullable Object readFrom(ByteBuffer buffer); +} diff --git a/libs/core/src/main/java/com/solarwinds/joboe/core/ebson/BsonTimestamp.java b/libs/core/src/main/java/com/solarwinds/joboe/core/ebson/BsonTimestamp.java new file mode 100644 index 00000000..7490505c --- /dev/null +++ b/libs/core/src/main/java/com/solarwinds/joboe/core/ebson/BsonTimestamp.java @@ -0,0 +1,29 @@ +/* + * © SolarWinds Worldwide, LLC. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.solarwinds.joboe.core.ebson; + +import java.nio.ByteBuffer; + +// @checkstyle:off . +public interface BsonTimestamp { + + ByteBuffer timestamp(); + + ByteBuffer time(); + + ByteBuffer increment(); +} diff --git a/libs/core/src/main/java/com/solarwinds/joboe/core/ebson/BsonToken.java b/libs/core/src/main/java/com/solarwinds/joboe/core/ebson/BsonToken.java new file mode 100644 index 00000000..6384e093 --- /dev/null +++ b/libs/core/src/main/java/com/solarwinds/joboe/core/ebson/BsonToken.java @@ -0,0 +1,84 @@ +/* + * © SolarWinds Worldwide, LLC. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.solarwinds.joboe.core.ebson; + +import com.google.common.base.Preconditions; + +/** Representation of a BSON token type. */ +public enum BsonToken { + + /** BSON document. */ + DOCUMENT(DefaultReader.DOCUMENT, DefaultWriter.DOCUMENT), + + /** Key-value pair in a {@linkplain #DOCUMENT document}. */ + FIELD(DefaultReader.FIELD, DefaultWriter.FIELD), + + /** Key in a {@linkplain #FIELD field} ({@code \0} delimited UTF-8 string). */ + KEY(DefaultReader.KEY, DefaultWriter.KEY); + + private BsonReader reader; + private BsonWriter writer; + + BsonToken(BsonReader reader, BsonWriter writer) { + this.reader = reader; + this.writer = writer; + } + + BsonToken() {} + + /** + * Returns this token's associated {@linkplain BsonReader reader}. + * + * @return this token's associated reader + * @throws IllegalStateException if this token does not have an associated reader + */ + public BsonReader reader() { + Preconditions.checkState(reader != null, "'%s' does not have an associated reader", this); + return reader; + } + + /** + * Associates {@link BsonReader reader} with this token. + * + * @param reader the reader to be associated with this token + */ + public void reader(BsonReader reader) { + Preconditions.checkNotNull(reader, "cannot associate a null reader with '%s'", this); + this.reader = reader; + } + + /** + * Returns this token's associated {@linkplain BsonWriter writer}. + * + * @return this token's associated writer + * @throws IllegalStateException if this token does not have an associated writer + */ + public BsonWriter writer() { + Preconditions.checkState(writer != null, "'%s' does not have an associated writer", this); + return writer; + } + + /** + * Associates {@link BsonWriter writer} with this token. + * + * @param writer the writer to be associated with this token + */ + public void writer(BsonWriter writer) { + Preconditions.checkNotNull(reader, "cannot associate a null writer with '%s'", this); + this.writer = writer; + } +} diff --git a/libs/core/src/main/java/com/solarwinds/joboe/core/ebson/BsonWriter.java b/libs/core/src/main/java/com/solarwinds/joboe/core/ebson/BsonWriter.java new file mode 100644 index 00000000..8ea8b2f7 --- /dev/null +++ b/libs/core/src/main/java/com/solarwinds/joboe/core/ebson/BsonWriter.java @@ -0,0 +1,39 @@ +/* + * © SolarWinds Worldwide, LLC. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.solarwinds.joboe.core.ebson; + +import java.nio.ByteBuffer; +import javax.annotation.Nullable; + +/** + * Writes Java object(s) to {@linkplain ByteBuffer buffers}, serialized to bytes as specified by the + * BSON specification. + * + *

Note: buffers supplied to {@linkplain #writeTo} must use little-endian byte ordering. + */ +public interface BsonWriter { + + /** + * Writes {@code reference} to {@code buffer}. + * + * @param buffer the buffer {@code reference} will be written to + * @param reference the reference to be written into {@code buffer} + * @throws NullPointerException if {@code buffer} is null + * @throws IllegalArgumentException if {@code buffer} is not using little-endian byte ordering + */ + void writeTo(ByteBuffer buffer, @Nullable Object reference); +} diff --git a/libs/core/src/main/java/com/solarwinds/joboe/core/ebson/DefaultDocument.java b/libs/core/src/main/java/com/solarwinds/joboe/core/ebson/DefaultDocument.java new file mode 100644 index 00000000..a1498cf9 --- /dev/null +++ b/libs/core/src/main/java/com/solarwinds/joboe/core/ebson/DefaultDocument.java @@ -0,0 +1,68 @@ +/* + * © SolarWinds Worldwide, LLC. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.solarwinds.joboe.core.ebson; + +import com.google.common.base.Joiner; +import com.google.common.base.Preconditions; +import com.google.common.collect.ForwardingMap; +import com.google.common.collect.Maps; +import java.util.Collections; +import java.util.Map; + +final class DefaultDocument extends ForwardingMap implements BsonDocument { + + private final Map delegate; + + DefaultDocument(Map map) { + delegate = Collections.unmodifiableMap(Maps.newLinkedHashMap(map)); + } + + @Override + public T get(Object key, Class type) { + Object value = get(key); + Preconditions.checkArgument( + type == null ? value == null : type.isInstance(value), + "expected '%s' instead of '%s'", + value == null ? null : value.getClass(), + type); + return type == null ? null : type.cast(value); + } + + @Override + public Object get(Object key) { + Preconditions.checkArgument(containsKey(key), "key: '%s' is missing", key); + return super.get(key); + } + + @Override + public boolean containsKey(Object key) { + Preconditions.checkNotNull(key, "null key"); + if (!(key instanceof String)) + throw new ClassCastException(String.format("key: '%s' is not a string", key)); + return super.containsKey(key); + } + + @Override + public String toString() { + return "{" + Joiner.on(", ").withKeyValueSeparator(": ").useForNull("null").join(this) + "}"; + } + + @Override + protected Map delegate() { + return delegate; + } +} diff --git a/libs/core/src/main/java/com/solarwinds/joboe/core/ebson/DefaultDocumentBuilder.java b/libs/core/src/main/java/com/solarwinds/joboe/core/ebson/DefaultDocumentBuilder.java new file mode 100644 index 00000000..f8bf1b35 --- /dev/null +++ b/libs/core/src/main/java/com/solarwinds/joboe/core/ebson/DefaultDocumentBuilder.java @@ -0,0 +1,75 @@ +/* + * © SolarWinds Worldwide, LLC. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.solarwinds.joboe.core.ebson; + +import com.google.common.base.Preconditions; +import com.google.common.collect.Maps; +import java.util.Collections; +import java.util.Map; +import java.util.Map.Entry; +import javax.annotation.Nullable; + +final class DefaultDocumentBuilder implements BsonDocument.Builder { + + private final Map builder; + + DefaultDocumentBuilder() { + builder = Maps.newLinkedHashMap(); + } + + @Override + public BsonDocument.Builder putAll(Map map) { + Preconditions.checkNotNull(map, "null map"); + for (Entry entry : map.entrySet()) put(entry.getKey(), entry.getValue()); + return this; + } + + @Override + public BsonDocument.Builder put(String key, @Nullable Object value) { + Preconditions.checkNotNull(key, "null key"); + Preconditions.checkArgument(!builder.containsKey(key), "key: '%s' is already present", key); + builder.put(key, value); + return this; + } + + @Override + @SuppressWarnings("unchecked") + public BsonDocument.Builder putAllowMultiVal(String key, @Nullable Object value) { + Preconditions.checkNotNull(key, "null key"); + if (builder.containsKey(key)) { + Object existingValue = builder.get(key); + MultiValList list; + if (existingValue instanceof MultiValList) { + list = (MultiValList) existingValue; + } else { + // convert the existing value into a MultiValList + list = new MultiValList(); + list.add(existingValue); + builder.put(key, list); + } + list.add(value); + } else { + builder.put(key, value); + } + return this; + } + + @Override + public BsonDocument build() { + return new DefaultDocument(builder.isEmpty() ? Collections.emptyMap() : builder); + } +} diff --git a/libs/core/src/main/java/com/solarwinds/joboe/core/ebson/DefaultPredicate.java b/libs/core/src/main/java/com/solarwinds/joboe/core/ebson/DefaultPredicate.java new file mode 100644 index 00000000..47ed8768 --- /dev/null +++ b/libs/core/src/main/java/com/solarwinds/joboe/core/ebson/DefaultPredicate.java @@ -0,0 +1,142 @@ +/* + * © SolarWinds Worldwide, LLC. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.solarwinds.joboe.core.ebson; + +import com.google.common.base.Predicate; +import java.util.Collection; +import java.util.Date; +import java.util.Map; +import java.util.regex.Pattern; + +enum DefaultPredicate implements Predicate> { + DOUBLE { + + @Override + public boolean apply(Class input) { + return input != null && Double.class.isAssignableFrom(input); + } + }, + + STRING { + + @Override + public boolean apply(Class input) { + return input != null && String.class.isAssignableFrom(input); + } + }, + + EMBEDDED { + + @Override + public boolean apply(Class input) { + return input != null && Map.class.isAssignableFrom(input); + } + }, + + ARRAY { + + @Override + public boolean apply(Class input) { + return input != null + && !byte[].class.isAssignableFrom(input) + && (Collection.class.isAssignableFrom(input) || input.isArray()); + } + }, + + BINARY { + + @Override + public boolean apply(Class input) { + return GENERIC.apply(input); + } + }, + + GENERIC { + + @Override + public boolean apply(Class input) { + return input != null && byte[].class.isAssignableFrom(input); + } + }, + + OBJECT_ID { + + @Override + public boolean apply(Class input) { + return input != null && BsonObjectId.class.isAssignableFrom(input); + } + }, + + BOOLEAN { + + @Override + public boolean apply(Class input) { + return input != null && Boolean.class.isAssignableFrom(input); + } + }, + + UTC_DATE_TIME { + + @Override + public boolean apply(Class input) { + return input != null && Date.class.isAssignableFrom(input); + } + }, + + NULL { + + @Override + public boolean apply(Class input) { + return input == null; + } + }, + + REGULAR_EXPRESSION { + + @Override + public boolean apply(Class input) { + return input != null && Pattern.class.isAssignableFrom(input); + } + }, + + INT32 { + + @Override + public boolean apply(Class input) { + return input != null && Integer.class.isAssignableFrom(input); + } + }, + + TIMESTAMP { + + @Override + public boolean apply(Class input) { + return input != null && BsonTimestamp.class.isAssignableFrom(input); + } + }, + + INT64 { + + @Override + public boolean apply(Class input) { + return input != null && Long.class.isAssignableFrom(input); + } + }; + + @Override + public abstract boolean apply(Class input); +} diff --git a/libs/core/src/main/java/com/solarwinds/joboe/core/ebson/DefaultReader.java b/libs/core/src/main/java/com/solarwinds/joboe/core/ebson/DefaultReader.java new file mode 100644 index 00000000..3fc9778a --- /dev/null +++ b/libs/core/src/main/java/com/solarwinds/joboe/core/ebson/DefaultReader.java @@ -0,0 +1,230 @@ +/* + * © SolarWinds Worldwide, LLC. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.solarwinds.joboe.core.ebson; + +import com.google.common.base.Charsets; +import com.google.common.base.Preconditions; +import com.google.common.collect.Maps; +import com.google.common.primitives.Bytes; +import com.google.common.primitives.Ints; +import java.nio.ByteBuffer; +import java.nio.ByteOrder; +import java.util.Date; +import java.util.Map.Entry; +import java.util.regex.Pattern; + +enum DefaultReader implements BsonReader { + DOCUMENT { + + @Override + public Object checkedReadFrom(ByteBuffer buffer) { + int documentLength = buffer.getInt(); + BsonReader fieldReader = BsonToken.FIELD.reader(); + BsonDocument.Builder document = BsonDocuments.builder(); + if (documentLength > Ints.BYTES + 1) + do { + @SuppressWarnings("unchecked") + Entry entry = (Entry) fieldReader.readFrom(buffer); + document.putAllowMultiVal(entry.getKey(), entry.getValue()); // allow MULTIVAL + } while (buffer.get(buffer.position()) != BsonBytes.EOO); + buffer.get(); + return document.build(); + } + }, + + FIELD { + + @Override + public Object checkedReadFrom(ByteBuffer buffer) { + BsonObject bsonObject = BsonObject.find(buffer.get()); + BsonReader keyReader = BsonToken.KEY.reader(); + BsonReader valueReader = bsonObject.reader(); + return Maps.immutableEntry(keyReader.readFrom(buffer), valueReader.readFrom(buffer)); + } + }, + + KEY { + + @Override + public Object checkedReadFrom(ByteBuffer buffer) { + byte[] bytes = new byte[] {}; + byte[] read = new byte[] {BsonBytes.EOF}; + while ((read[0] = buffer.get()) != BsonBytes.EOO) { + bytes = Bytes.concat(bytes, read); + } + return new String(bytes, Charsets.UTF_8); + } + }, + + DOUBLE { + + @Override + public Double checkedReadFrom(ByteBuffer buffer) { + return Double.valueOf(buffer.getDouble()); + } + }, + + STRING { + + @Override + public Object checkedReadFrom(ByteBuffer buffer) { + int stringLength = buffer.getInt(); + byte[] bytes = new byte[stringLength - 1]; + buffer.get(bytes).get(); + return new String(bytes, Charsets.UTF_8); + } + }, + + ARRAY { + + @Override + public Object checkedReadFrom(ByteBuffer buffer) { + return BsonToken.DOCUMENT.reader().readFrom(buffer); + } + }, + + BINARY { + + @Override + public Object checkedReadFrom(ByteBuffer buffer) { + int binaryLength = buffer.getInt(); + byte terminal = buffer.get(); + BsonBinary bsonBinary = BsonBinary.find(terminal); + int oldLimit = buffer.limit(); + buffer.limit(buffer.position() + binaryLength); + byte[] binary = (byte[]) bsonBinary.reader().readFrom(buffer); + buffer.limit(oldLimit); + return binary; + } + }, + + GENERIC { + + @Override + public Object checkedReadFrom(ByteBuffer buffer) { + byte[] binary = new byte[buffer.remaining()]; + buffer.get(binary); + return binary; + } + }, + + OBJECT_ID { + + @Override + Object checkedReadFrom(ByteBuffer buffer) { + return new BasicObjectId(buffer); + } + }, + + BOOLEAN { + + @Override + public Object checkedReadFrom(ByteBuffer buffer) { + return Boolean.valueOf(buffer.get() == BsonBytes.TRUE); + } + }, + + UTC_DATE_TIME { + + @Override + public Object checkedReadFrom(ByteBuffer buffer) { + return new Date(buffer.getLong()); + } + }, + + NULL { + + @Override + public Object checkedReadFrom(ByteBuffer buffer) { + return null; + } + }, + + REGULAR_EXPRESSION { + + @Override + public Object checkedReadFrom(ByteBuffer buffer) { + BsonReader keyReader = BsonToken.KEY.reader(); + String pattern = (String) keyReader.readFrom(buffer); + String options = (String) keyReader.readFrom(buffer); + return Pattern.compile(pattern, optionsToFlags(options)); + } + + private int optionsToFlags(String options) { + int flags = 0; + for (char option : options.toCharArray()) flags |= flags + optionToFlag(option); + return flags; + } + + // @do-not-check CyclomaticComplexity + private int optionToFlag(char option) { + int flag = 0; + switch (option) { + case 'i': + flag = Pattern.CASE_INSENSITIVE; + break; + case 'm': + flag = Pattern.MULTILINE; + break; + case 's': + flag = Pattern.DOTALL; + break; + case 'x': + flag = Pattern.COMMENTS; + break; + default: + break; + } + return flag; + } + }, + + INT32 { + + @Override + public Object checkedReadFrom(ByteBuffer buffer) { + return Integer.valueOf(buffer.getInt()); + } + }, + + TIMESTAMP { + + @Override + Object checkedReadFrom(ByteBuffer buffer) { + return new BasicTimestamp(buffer); + } + }, + + INT64 { + + @Override + public Object checkedReadFrom(ByteBuffer buffer) { + return Long.valueOf(buffer.getLong()); + } + }; + + @Override + public final Object readFrom(ByteBuffer buffer) { + Preconditions.checkNotNull(buffer, "null buffer"); + Preconditions.checkArgument( + buffer.order() == ByteOrder.LITTLE_ENDIAN, + "buffer has big-endian byte order; expected little-endian"); + return checkedReadFrom(buffer); + } + + abstract Object checkedReadFrom(ByteBuffer buffer); +} diff --git a/libs/core/src/main/java/com/solarwinds/joboe/core/ebson/DefaultWriter.java b/libs/core/src/main/java/com/solarwinds/joboe/core/ebson/DefaultWriter.java new file mode 100644 index 00000000..e28c0e27 --- /dev/null +++ b/libs/core/src/main/java/com/solarwinds/joboe/core/ebson/DefaultWriter.java @@ -0,0 +1,223 @@ +/* + * © SolarWinds Worldwide, LLC. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.solarwinds.joboe.core.ebson; + +import com.google.common.base.Charsets; +import com.google.common.base.Joiner; +import com.google.common.collect.Maps; +import com.google.common.collect.Sets; +import com.google.common.primitives.Ints; +import java.lang.reflect.Array; +import java.nio.ByteBuffer; +import java.util.Collection; +import java.util.Date; +import java.util.List; +import java.util.Map; +import java.util.Map.Entry; +import java.util.SortedSet; +import java.util.regex.Pattern; + +enum DefaultWriter implements BsonWriter { + DOCUMENT { + + @Override + public void writeTo(ByteBuffer buffer, Object reference) { + int markedPosition = buffer.position(); + buffer.position(markedPosition + Ints.BYTES); + BsonWriter fieldWriter = BsonToken.FIELD.writer(); + for (Entry entry : ((Map) reference).entrySet()) + fieldWriter.writeTo(buffer, entry); + buffer.put(BsonBytes.EOO); + buffer.putInt(markedPosition, buffer.position() - markedPosition); + } + }, + + FIELD { + + @Override + public void writeTo(ByteBuffer buffer, Object reference) { + Entry entry = (Entry) reference; + if (entry.getValue() instanceof MultiValList) { + MULTIVAL.writeTo(buffer, reference); + } else { + BsonObject bsonObject = + BsonObject.find(entry.getValue() == null ? null : entry.getValue().getClass()); + buffer.put(bsonObject.terminal()); + BsonToken.KEY.writer().writeTo(buffer, entry.getKey()); + bsonObject.writer().writeTo(buffer, entry.getValue()); + } + } + }, + + MULTIVAL { + + @Override + public void writeTo(ByteBuffer buffer, Object reference) { + Entry entry = (Entry) reference; + List list = (List) entry.getValue(); + // Write multiple key/values, each with the same key + for (Object item : list) { + BsonObject bsonObject = BsonObject.find(item == null ? null : item.getClass()); + buffer.put(bsonObject.terminal()); + BsonToken.KEY.writer().writeTo(buffer, entry.getKey()); + bsonObject.writer().writeTo(buffer, item); + } + } + }, + + KEY { + + @Override + public void writeTo(ByteBuffer buffer, Object reference) { + buffer.put(((String) reference).getBytes(Charsets.UTF_8)).put(BsonBytes.EOO); + } + }, + + DOUBLE { + + @Override + public void writeTo(ByteBuffer buffer, Object reference) { + buffer.putDouble(((Double) reference).doubleValue()); + } + }, + + STRING { + + @Override + public void writeTo(ByteBuffer buffer, Object reference) { + byte[] bytes; + bytes = ((String) reference).getBytes(Charsets.UTF_8); + buffer.putInt(bytes.length + 1).put(bytes).put(BsonBytes.EOO); + } + }, + + ARRAY { + + @Override + public void writeTo(ByteBuffer buffer, Object reference) { + Object array = + reference instanceof Collection ? ((Collection) reference).toArray() : reference; + Map document = Maps.newLinkedHashMap(); + for (int i = 0; i < Array.getLength(array); i++) + document.put(String.valueOf(i), Array.get(array, i)); + BsonToken.DOCUMENT.writer().writeTo(buffer, document); + } + }, + + BINARY { + + @Override + public void writeTo(ByteBuffer buffer, Object reference) { + byte[] binary = (byte[]) reference; + buffer.putInt(binary.length); + BsonBinary bsonBinary = BsonBinary.find(binary.getClass()); + buffer.put(bsonBinary.terminal()); + bsonBinary.writer().writeTo(buffer, binary); + } + }, + + GENERIC { + + @Override + public void writeTo(ByteBuffer buffer, Object reference) { + buffer.put((byte[]) reference); + } + }, + + OBJECT_ID { + + @Override + public void writeTo(ByteBuffer buffer, Object reference) { + buffer.put(((BsonObjectId) reference).objectId()); + } + }, + + BOOLEAN { + + @Override + public void writeTo(ByteBuffer buffer, Object reference) { + buffer.put(((Boolean) reference).booleanValue() ? BsonBytes.TRUE : BsonBytes.FALSE); + } + }, + + UTC_DATE_TIME { + + @Override + public void writeTo(ByteBuffer buffer, Object reference) { + buffer.putLong(((Date) reference).getTime()); + } + }, + + NULL { + + @Override + public void writeTo(ByteBuffer buffer, Object reference) {} + }, + + REGULAR_EXPRESSION { + + @Override + public void writeTo(ByteBuffer buffer, Object reference) { + Pattern regularExpression = (Pattern) reference; + BsonWriter keyWriter = BsonToken.KEY.writer(); + keyWriter.writeTo(buffer, regularExpression.pattern()); + keyWriter.writeTo(buffer, flagsToOptions(regularExpression.flags())); + } + + // @do-not-check CyclomaticComplexity + private String flagsToOptions(int flags) { + SortedSet options = Sets.newTreeSet(); + if (hasFlag(flags, Pattern.CASE_INSENSITIVE)) options.add(Character.valueOf('i')); + + if (hasFlag(flags, Pattern.COMMENTS)) options.add(Character.valueOf('x')); + + if (hasFlag(flags, Pattern.DOTALL)) options.add(Character.valueOf('s')); + + if (hasFlag(flags, Pattern.MULTILINE)) options.add(Character.valueOf('m')); + + return Joiner.on("").join(options); + } + + private boolean hasFlag(int flags, int flag) { + return (flags & flag) != 0; + } + }, + + INT32 { + + @Override + public void writeTo(ByteBuffer buffer, Object reference) { + buffer.putInt(((Integer) reference).intValue()); + } + }, + + TIMESTAMP { + + @Override + public void writeTo(ByteBuffer buffer, Object reference) { + buffer.put(((BsonTimestamp) reference).timestamp()); + } + }, + + INT64 { + + @Override + public void writeTo(ByteBuffer buffer, Object reference) { + buffer.putLong(((Long) reference).longValue()); + } + } +} diff --git a/libs/core/src/main/java/com/solarwinds/joboe/core/ebson/MultiValList.java b/libs/core/src/main/java/com/solarwinds/joboe/core/ebson/MultiValList.java new file mode 100644 index 00000000..f74a8075 --- /dev/null +++ b/libs/core/src/main/java/com/solarwinds/joboe/core/ebson/MultiValList.java @@ -0,0 +1,34 @@ +/* + * © SolarWinds Worldwide, LLC. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.solarwinds.joboe.core.ebson; + +import java.util.ArrayList; + +/** + * Hack that allows multiple key/value pairs with the same key to be added to a BSON document. This + * list just contains the values. We look for it when generating the BSON in DefaultWriter. + */ +public class MultiValList extends ArrayList { + + public MultiValList() { + super(); + } + + public MultiValList(int initialCapacity) { + super(initialCapacity); + } +} diff --git a/libs/core/src/main/java/com/solarwinds/joboe/core/ebson/package-info.java b/libs/core/src/main/java/com/solarwinds/joboe/core/ebson/package-info.java new file mode 100644 index 00000000..6f9569b4 --- /dev/null +++ b/libs/core/src/main/java/com/solarwinds/joboe/core/ebson/package-info.java @@ -0,0 +1,5 @@ +/** BSON encoder/decoder. */ +@ParametersAreNonnullByDefault +package com.solarwinds.joboe.core.ebson; + +import javax.annotation.ParametersAreNonnullByDefault; diff --git a/libs/core/src/main/java/com/solarwinds/joboe/core/profiler/Profiler.java b/libs/core/src/main/java/com/solarwinds/joboe/core/profiler/Profiler.java new file mode 100644 index 00000000..e6fce24a --- /dev/null +++ b/libs/core/src/main/java/com/solarwinds/joboe/core/profiler/Profiler.java @@ -0,0 +1,765 @@ +/* + * © SolarWinds Worldwide, LLC. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.solarwinds.joboe.core.profiler; + +import com.solarwinds.joboe.core.Context; +import com.solarwinds.joboe.core.Event; +import com.solarwinds.joboe.core.EventReporter; +import com.solarwinds.joboe.core.util.DaemonThreadFactory; +import com.solarwinds.joboe.core.util.TimeUtils; +import com.solarwinds.joboe.logging.Logger; +import com.solarwinds.joboe.logging.LoggerFactory; +import com.solarwinds.joboe.sampling.Metadata; +import com.solarwinds.joboe.sampling.SettingsArg; +import com.solarwinds.joboe.sampling.SettingsArgChangeListener; +import com.solarwinds.joboe.sampling.SettingsManager; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collections; +import java.util.HashMap; +import java.util.HashSet; +import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.ConcurrentMap; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; +import java.util.concurrent.Future; +import java.util.concurrent.TimeUnit; +import lombok.Getter; + +/** + * A Sampling Profiler that runs a single background thread to take stack trace snapshots of a list + * of "tracked threads" on a given interval + * + *

The list of "tracked threads" are set by other external logic and is not controlled by this + * profiler + * + * @author pluk + */ +public class Profiler { + private static final ConcurrentMap profileByTraceId = + new ConcurrentHashMap(); // for quicker lookup + private static final String SOLARWINDS_THREAD_PREFIX = "Solarwinds-"; + + private static final Logger logger = LoggerFactory.getLogger(); + + public static EventReporter reporter; + + @Getter private static Status status = Status.UNINITIALIZED; + + public enum Status { + UNINITIALIZED, + PAUSED_CIRCUIT_BREAKER, + RUNNING, + STOPPING, + STOPPED + } + + private static int + interval; // current interval to take snapshots on, could be changed dynamically + + private static ProfilerSetting localSetting; // profiler settings from local config + private static Future samplerFuture; + + static final int MAX_REPORTED_FRAME_DEPTH = 400; + + /** Listens to interval changes beamed down by collector */ + private static void addIntervalChangeListener() { + SettingsManager.registerListener( + new SettingsArgChangeListener(SettingsArg.PROFILING_INTERVAL) { + @Override + public void onChange(Integer newValue) { + logger.info( + "Collector sends new profiling interval : " + + (newValue != null ? newValue.toString() : "(empty)")); + if (newValue + != null) { // value from collector also has higher precedence than local settings + interval = newValue; + } else { + interval = localSetting.getInterval(); + } + + logger.info("Updated profiling interval to : " + interval); + + if (interval <= 0) { // stop profiler if new value is negative + if (status != Status.STOPPED) { // only if profiler is not stopped yet + logger.info( + "Profiler stopping after interval update from remote collector, previous status: " + + status); + stop(); + } + } else { // otherwise start profiler if it was stopped + if (status == Status.STOPPED) { + logger.info("Profiler starting after interval update from remote collector"); + start(); + } + } + } + }); + } + + /** + * Initializes profiler by providing the ProfilerSetting. The Profiler will start listening for + * remote config changes + * + *

The agent might be put into either standby mode (if `interval` is 0) or running mode if + * `interval` is valid + * + *

Ignores calls if profiler is NOT in `UNINITIALIZED` state + * + * @param setting + * @param reporter reporter used to export the captured data (in {@link Event} output format) + */ + public static void initialize(ProfilerSetting setting, EventReporter reporter) { + if (status == Status.UNINITIALIZED) { + localSetting = setting; + interval = localSetting.getInterval(); + + status = + Status + .STOPPED; // switch to stopped as the profiler is going to be initialized but has not + // started profiling yet + + Profiler.reporter = reporter; + + if (interval != 0) { + logger.debug("Starting profiler worker, previous status: " + status); + start(); + } else { + logger.debug("No profiling started. Profiler is on standby, previous status: " + status); + } + + addIntervalChangeListener(); // add listener here to avoid race condition on starting the + // profiler + } else { + logger.info("Profiler is already initialized, ignoring initialize operation"); + } + } + + private static void start() { + if (status == Status.STOPPED) { + run(); + } else { + logger.warn("Cannot start a profiler when it's in status " + status); + } + } + + /** Starts the background thread that takes snapshots on `interval` */ + static void run() { + status = Status.RUNNING; + + final CircuitBreaker circuitBreaker = + new CircuitBreaker( + localSetting.getCircuitBreakerDurationThreshold(), + localSetting.getCircuitBreakerCountThreshold()); + + ExecutorService service = + Executors.newFixedThreadPool(1, DaemonThreadFactory.newInstance("profiling-sampler")); + samplerFuture = + service.submit( + () -> { + while (status != Status.STOPPING) { + status = Status.RUNNING; + try { + ProfilingDurationInfo durationInfo = + checkThreads(); // take and report snapshots on the list on tracked threads + long duration = durationInfo.duration; + + long circuitBreakerPause = + circuitBreaker.getPause( + duration); // consult with the circuit break on whether the last operation + // would trigger a pause + if (circuitBreakerPause > 0) { // circuit breaker is triggered, pausing + status = Status.PAUSED_CIRCUIT_BREAKER; + logger.info( + "Pause profiling for " + + circuitBreakerPause + + " secs. Previous profiling operation took " + + duration + + "ms. That's total of " + + circuitBreaker.getBreakCountThreshold() + + " consecutive profiling operation(s) that exceeded the circuit breaker duration threshold " + + circuitBreaker.getBreakDurationThreshold() + + " ms"); + TimeUnit.SECONDS.sleep(circuitBreakerPause); + } else { + // the previous sleep computes modulo hence can have a sleep time in range 1 - + // 20ms for default + // which is inconsistent with documentation. + TimeUnit.MILLISECONDS.sleep(interval); + } + } catch (InterruptedException e) { + logger.debug( + "Profiler interrupted: " + + e.getMessage()); // hard to tell whether this is triggered by JVM + // shutdown + status = Status.STOPPING; // flag it to stop + } catch (Throwable e) { + logger.warn("Profiler interrupted unexpectedly: " + e.getMessage(), e); + status = Status.STOPPING; // flag it to stop + } + } + status = Status.STOPPED; + }); + service.shutdown(); + } + + /** + * Stops the background thread that takes snapshots on `interval`, blocks until the thread is dead + */ + public static void stop() { + logger.info("Stopping Profiler"); + status = Status.STOPPING; // flag that the profiler is signaled for stopping + if (samplerFuture != null) { + samplerFuture.cancel(true); + } + logger.info("Profiler is stopped"); + } + + /** + * Takes and reports snapshots on the tracked threads + * + * @return the duration of the checkThreads operation + */ + private static ProfilingDurationInfo checkThreads() { + if (profileByTraceId.isEmpty()) { + return new ProfilingDurationInfo(-1, Collections.emptyList()); + } + + long start = System.currentTimeMillis(); + + List taskIds = new ArrayList(); + for (Profile profile : new HashSet(profileByTraceId.values())) { + for (Thread thread : profile.getActiveThreads()) { + long snapshotTimestamp = TimeUtils.getTimestampMicroSeconds(); + StackTraceElement[] stackTrace = thread.getStackTrace(); + profile.record(thread, stackTrace, snapshotTimestamp); + } + for (SnapshotTracker tracker : profile.snapshotTrackersByThread.values()) { + taskIds.add(tracker.metadata.taskHexString()); + } + } + long end = System.currentTimeMillis(); + long duration = end - start; + + return new ProfilingDurationInfo(duration, taskIds); + } + + private static class ProfilingDurationInfo { + private final long duration; + private final List taskIds; + + private ProfilingDurationInfo(long duration, List taskIds) { + this.duration = duration; + this.taskIds = taskIds; + } + } + + /** + * Adds a thread to be tracked for profiling + * + * @param thread + * @param metadata + * @param traceId + * @return + */ + public static boolean addProfiledThread(Thread thread, Metadata metadata, String traceId) { + if (thread.getName() != null + && thread + .getName() + .startsWith(SOLARWINDS_THREAD_PREFIX)) { // do not instrument our own threads + return false; + } + + if (status != Status.RUNNING) { + logger.debug( + "Add profile thread operation skipped as profiler is not running, status : " + status); + return false; + } + + Profile profile; + profile = profileByTraceId.get(traceId); + if (profile == null) { // then this task is instrumented the first time, add profile + profile = new Profile(); + profileByTraceId.put(traceId, profile); + } + + if (profile.startProfilingOnThread(thread, metadata)) { + logger.debug( + "Started profiling on Thread id: " + + thread.getId() + + " name: " + + thread.getName() + + " for trace: " + + traceId); + return true; + } else { + return false; + } + } + + /** + * Stops profiling on all threads triggered by this parent (tracing) span + * + * @param traceId + * @return + */ + public static Profile stopProfile(String traceId) { + Profile profile = profileByTraceId.remove(traceId); + + if (profile != null) { + profile.stop(); + } + + return profile; + } + + /** + * Stops profiling on a particular thread + * + * @param profiledThread + * @param traceId + */ + public static void removeProfiledThread(Thread profiledThread, String traceId) { + Profile profile = profileByTraceId.get(traceId); + if (profile != null) { + if (profile.stopProfilingOnThread(profiledThread)) { + logger.debug( + "Stopped profiling on Thread id: " + + profiledThread.getId() + + " name: " + + profiledThread.getName() + + " for trace " + + traceId); + } + } + } + + /** + * A Profile is created per trace that has profiling triggered. It keeps a map of threads being + * profiled. + * + *

For example if a servlet triggers profiling, a `Profile` instance would be created for that. + * + *

If later on more threads related to the same servlet call are tracked, they will be added to + * this same Profile instance. + * + * @author pluk + */ + public static class Profile { + private final Map snapshotTrackersByThread = + new ConcurrentHashMap(); + + private final ProfilerSetting setting; + + @Getter private boolean sampled = false; + + private Event snapshotEntry; + + private Metadata entryMetadata; + + private Profile() { + this(Profiler.localSetting); + } + + Profile(ProfilerSetting setting) { + this.setting = setting; + } + + /** + * Creates an "entry" event for the profiling span with a "SpanRef" pointing back to the parent + * "tracing" span that triggers/create this profile + * + * @param parentMetadata + * @return + */ + private Metadata createProfileSpanEntry(Metadata parentMetadata) { + entryMetadata = new Metadata(parentMetadata); + + snapshotEntry = Context.createEventWithContext(entryMetadata, false); + snapshotEntry.setTimestamp(TimeUtils.getTimestampMicroSeconds()); + snapshotEntry.addInfo( + "Label", "entry", + "Spec", "profiling", + "Language", "java", + "Interval", interval, + "SpanRef", parentMetadata.opHexString()); + + return snapshotEntry.getMetadata(); + } + + public void stop() { + for (SnapshotTracker tracker : snapshotTrackersByThread.values()) { + tracker.stop(); + createProfileSpanExit(tracker); + } + snapshotTrackersByThread.clear(); + } + + /** + * Records and reports (if not omitted) the stack trace provided by in the parameters + * + * @param thread + * @param stack + * @param collectionTime time in microseconde when a snapshot was collected + */ + public void record(Thread thread, StackTraceElement[] stack, long collectionTime) { + long threadId = thread.getId(); + SnapshotTracker tracker = snapshotTrackersByThread.get(thread); + int framesExited; + + StackTraceElement[] newFrames; + + int originalFramesCount = stack.length; // get the framesCount before trimming + stack = trimStack(stack); + + if (tracker != null && !tracker.stopped) { + if (tracker.stack == null) { + framesExited = 0; + newFrames = stack; + } else { + // start matching previous stack with current stack, from the back of the list (bottom of + // calling stack frame) to front (top stack frame) + int currentCallFrameWalker = stack.length - 1; + int previousCallFrameWalker = tracker.stack.length - 1; + while (previousCallFrameWalker >= 0 && currentCallFrameWalker >= 0) { + StackTraceElement previousCallFrame = tracker.stack[previousCallFrameWalker]; + StackTraceElement currentCallFrame = stack[currentCallFrameWalker]; + if (!previousCallFrame.equals( + currentCallFrame)) { // diverges, exit here and count frames pop + break; + } + + currentCallFrameWalker--; + previousCallFrameWalker--; + } + + framesExited = previousCallFrameWalker + 1; + if (currentCallFrameWalker >= 0) { + newFrames = Arrays.copyOfRange(stack, 0, currentCallFrameWalker + 1); + } else { + newFrames = null; + } + } + + if (newFrames != null + || framesExited > 0) { // only update and report if things have changed + synchronized (tracker) { + if (!tracker.stopped) { + reportSnapshot( + tracker.metadata, + framesExited, + tracker.snapshotsOmitted.isEmpty() + ? Collections.emptyList() + : new ArrayList(tracker.snapshotsOmitted), + newFrames, + originalFramesCount, + threadId, + collectionTime); + } + } + tracker.stack = stack; + tracker.snapshotsOmitted.clear(); // reset snapshots omitted + } else { + if (tracker.metadata.isExpired( + collectionTime / 1000 /*collectionTime is in µs from the caller scope*/)) { + logger.warn( + String.format( + "Metadata has expired and we're stopping profiling on thread - %s. Trace took to long!", + thread.getName())); + stopProfilingOnThread(thread); + } else { + tracker.snapshotsOmitted.add(collectionTime); + } + } + } + } + + /** + * Trim the stack by removing top frames that matches ProfilerSetting.getExcludePackages or if + * it's deeper than MAX_REPORTED_FRAME_DEPTH + * + * @param stack + * @return + */ + private StackTraceElement[] trimStack(StackTraceElement[] stack) { + StackTraceElement[] trimmedStack; + if (stack.length > MAX_REPORTED_FRAME_DEPTH) { + trimmedStack = + Arrays.copyOfRange(stack, stack.length - MAX_REPORTED_FRAME_DEPTH, stack.length); + } else { + trimmedStack = stack; + } + + if (setting.getExcludePackages().isEmpty()) { // no trimming required + return trimmedStack; + } + // traverse from top to bottom + for (int i = 0; i < trimmedStack.length; i++) { + StackTraceElement frame = trimmedStack[i]; + boolean isExcludedFrame = false; + for (String excludePackage : setting.getExcludePackages()) { + String frameClassName = frame.getClassName(); + if (frameClassName.startsWith(excludePackage + ".")) { + isExcludedFrame = true; + break; + } + } + + if (!isExcludedFrame) { // this current frame does not match any of the exclude prefix, that + // means the rest of this stack should be reported + if (i == 0) { + return trimmedStack; // no change + } else { + return Arrays.copyOfRange(trimmedStack, i, trimmedStack.length); + } + } + } + + return new StackTraceElement[0]; // everything is excluded... + } + + /** + * Starts profiling on a thread with the parent (tracing) span + * + * @param thread + * @param parentMetadata + * @return + */ + boolean startProfilingOnThread(Thread thread, Metadata parentMetadata) { + if (!snapshotTrackersByThread.containsKey(thread)) { + Metadata snapshotMetadata = createProfileSpanEntry(parentMetadata); + SnapshotTracker tracker = new SnapshotTracker(snapshotMetadata); + snapshotTrackersByThread.put(thread, tracker); + return true; + } else { // this thread is already tracked + return false; + } + } + + /** + * Stops profiling on this particular thread + * + * @param thread + * @return + */ + boolean stopProfilingOnThread(Thread thread) { + SnapshotTracker tracker = snapshotTrackersByThread.remove(thread); + if (tracker != null) { + tracker.stop(); + createProfileSpanExit(tracker); + } + return tracker != null; + } + + private void createProfileSpanExit(SnapshotTracker tracker) { + if (!sampled) return; + Event snapshotExit = Context.createEventWithContext(tracker.metadata); + snapshotExit.addInfo( + "Label", "exit", + "Spec", "profiling", + "SnapshotsOmitted", tracker.snapshotsOmitted); + + synchronized (tracker) { + snapshotExit.addEdge(tracker.metadata); + snapshotExit.report(tracker.metadata, Profiler.reporter); + } + } + + /** + * Get a list of threads currently tracked by this Profile + * + * @return + */ + public Set getActiveThreads() { + return new HashSet(snapshotTrackersByThread.keySet()); + } + + SnapshotTracker getSnapshotTracker(Thread thread) { + return snapshotTrackersByThread.get(thread); + } + + private void reportSnapshot( + Metadata metadata, + int framesExited, + List snapshotsOmitted, + StackTraceElement[] newFrames, + int framesCount, + long threadId, + long timestamp) { + Event event; + + event = Context.createEventWithContext(metadata); + event.addInfo( + "Label", "info", + "Spec", "profiling", + "FramesExited", framesExited, + "SnapshotsOmitted", snapshotsOmitted, + "FramesCount", framesCount); + event.setTimestamp(timestamp); + event.setThreadId(threadId); + + if (newFrames != null) { + List> newFramesValue = new ArrayList>(); + for (StackTraceElement newFrame : newFrames) { + Map frameKeyValues = new HashMap(); + String className = newFrame.getClassName(); + if (className != null) { + frameKeyValues.put("C", className); + } + String fileName = newFrame.getFileName(); + if (fileName != null) { + frameKeyValues.put("F", fileName); + } + int lineNumber = newFrame.getLineNumber(); + if (lineNumber > 0) { + frameKeyValues.put("L", lineNumber); + } + String methodName = newFrame.getMethodName(); + if (methodName != null) { + frameKeyValues.put("M", methodName); + } + newFramesValue.add(frameKeyValues); + } + + event.addInfo("NewFrames", newFramesValue); + } + + if (!sampled) { + snapshotEntry.report(entryMetadata, Profiler.reporter); + sampled = true; + + snapshotEntry = null; + entryMetadata = null; + } + + event.report(metadata, reporter); + } + } + + /** + * Keeps the state of tracked thread to enable snapshot reporting on a thread. + * + *

State is important as: 1. Enables the check if the current snapshot can be omitted if it's + * identical to the previously reported one 2. Enables synchronization to avoid reporting + * snapshots if the thread (or it's parent span) is flagged to stop profiling + * + * @author pluk + */ + public static class SnapshotTracker { + private StackTraceElement[] stack; + private final Metadata metadata; + private boolean stopped = false; + @Getter private final ArrayList snapshotsOmitted = new ArrayList(); + + public SnapshotTracker(Metadata metadata) { + this.metadata = metadata; + } + + private void stop() { + stopped = true; + } + } + + /** + * A stateful circuit breaker that acts on consecutive calls to `getPause` when the duration + * parameter is a above or below the `durationThreshold` + * + *

The goal of this stateful circuit breaker is to break when the operation is consistently + * slow and increase the pause time exponentially (up to a max) if the system remains slow + * + *

However, if the system resumes back to normal, it should reset the pause time. + * + * @author pluk + */ + static class CircuitBreaker { + private int consecutiveBadCount = 0; + private int consecutiveGoodCount = 0; + private final int durationThreshold; + private final int countThreshold; + static final int INITIAL_CIRCUIT_BREAKER_PAUSE = 60; + static final int MAX_CIRCUIT_BREAKER_PAUSE = 60 * 60; + static final double PAUSE_MULTIPLIER = 1.5; + private long nextPause = INITIAL_CIRCUIT_BREAKER_PAUSE; + + CircuitBreaker(int durationThreshold, int countThreshold) { + this.durationThreshold = durationThreshold; + this.countThreshold = countThreshold; + } + + /** + * This might mutate the current circuit breaker states, depending on the current state and the + * duration parameters: + * + *

At first the circuit breaker starts with a "Normal" state When there are n (defined by + * `countThreshold`) consecutive `getPause` calls with param `duration` above the + * `durationThreshold`, the circuit breaker will go into the "Break" state "Break" state will be + * transitioned into a "Restored but broken recently" state when there's a new `getPause` call + * "Restored but broken recently" state will be transitioned to "Normal" state if there are n + * consecutive `getPause` calls with param `duration` below or equal to the `durationThreshold` + * + *

And below are the behaviors of this method in various states/transitions: + * + *

When transition to "Normal" state, `nextPause` is set to INITIAL_CIRCUIT_BREAKER_PAUSE + * When in "Normal" state, `getPause` returns 0 When transition to "Break" state, `getPause` + * returns `nextPause` then `nextPause` is multiplied by `PAUSE_MULTIPLIER` When transition to + * or in "Restored but broken recently" state, `getPause` returns 0 + * + * @param duration + * @return + */ + public long getPause(long duration) { + long pause = 0; + if (duration <= durationThreshold) { + if (consecutiveGoodCount < countThreshold) { + consecutiveGoodCount++; + if (consecutiveGoodCount == countThreshold) { + nextPause = + INITIAL_CIRCUIT_BREAKER_PAUSE; // reset pause as duration is good for the last + // countThreshold consecutive operations + } + consecutiveBadCount = 0; // reset consecutive bad count + } + } else { + if (consecutiveBadCount < countThreshold) { + consecutiveBadCount++; + if (consecutiveBadCount == countThreshold) { // trigger circuit breaker + pause = nextPause; + + nextPause = (long) (nextPause * PAUSE_MULTIPLIER); + nextPause = Math.min(nextPause, MAX_CIRCUIT_BREAKER_PAUSE); + consecutiveBadCount = + 0; // also reset the bad count, if we get more consecutive bad duration, we want to + // increase the threshold further + } + consecutiveGoodCount = 0; // reset consecutive good count + } + } + return pause; + } + + public int getBreakDurationThreshold() { + return durationThreshold; + } + + public int getBreakCountThreshold() { + return countThreshold; + } + } +} diff --git a/libs/core/src/main/java/com/solarwinds/joboe/core/profiler/ProfilerSetting.java b/libs/core/src/main/java/com/solarwinds/joboe/core/profiler/ProfilerSetting.java new file mode 100644 index 00000000..8fa40c37 --- /dev/null +++ b/libs/core/src/main/java/com/solarwinds/joboe/core/profiler/ProfilerSetting.java @@ -0,0 +1,81 @@ +/* + * © SolarWinds Worldwide, LLC. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.solarwinds.joboe.core.profiler; + +import java.io.Serializable; +import java.util.Arrays; +import java.util.HashSet; +import java.util.Set; +import lombok.EqualsAndHashCode; +import lombok.Getter; + +@EqualsAndHashCode +public class ProfilerSetting implements Serializable { + public static final int DEFAULT_INTERVAL = 20; + public static final int MIN_INTERVAL = 10; + public static final int DEFAULT_CIRCUIT_BREAKER_DURATION_THRESHOLD = 100; + public static final int DEFAULT_CIRCUIT_BREAKER_COUNT_THRESHOLD = 2; + public static final Set DEFAULT_EXCLUDE_PACKAGES = + new HashSet(Arrays.asList("java", "javax", "com.sun", "sun", "sunw")); + private final boolean isEnabled; + @Getter private final Set excludePackages; + @Getter private final int interval; + @Getter private final int circuitBreakerDurationThreshold; + @Getter private final int circuitBreakerCountThreshold; + + public ProfilerSetting( + boolean isEnabled, + Set excludePackages, + int interval, + int circuitBreakerDurationThreshold, + int circuitBreakerCountThreshold) { + super(); + this.isEnabled = isEnabled; + this.excludePackages = excludePackages; + this.interval = interval; + this.circuitBreakerDurationThreshold = circuitBreakerDurationThreshold; + this.circuitBreakerCountThreshold = circuitBreakerCountThreshold; + } + + public ProfilerSetting(boolean isEnabled, int interval) { + this( + isEnabled, + DEFAULT_EXCLUDE_PACKAGES, + interval, + DEFAULT_CIRCUIT_BREAKER_DURATION_THRESHOLD, + DEFAULT_CIRCUIT_BREAKER_COUNT_THRESHOLD); + } + + public boolean isEnabled() { + return isEnabled; + } + + @Override + public String toString() { + return "ProfilerSetting [isEnabled=" + + isEnabled + + ", excludePackages=" + + excludePackages + + ", interval=" + + interval + + ", circuitBreakerDurationThreshold=" + + circuitBreakerDurationThreshold + + ", circuitBreakerCountThreshold=" + + circuitBreakerCountThreshold + + "]"; + } +} diff --git a/libs/core/src/main/java/com/solarwinds/joboe/core/rpc/Client.java b/libs/core/src/main/java/com/solarwinds/joboe/core/rpc/Client.java new file mode 100644 index 00000000..86afa2cc --- /dev/null +++ b/libs/core/src/main/java/com/solarwinds/joboe/core/rpc/Client.java @@ -0,0 +1,60 @@ +/* + * © SolarWinds Worldwide, LLC. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.solarwinds.joboe.core.rpc; + +import com.solarwinds.joboe.core.Event; +import java.util.List; +import java.util.Map; +import java.util.concurrent.Future; + +/** + * RPC client that provides various api methods to our collector + * + * @author pluk + */ +public interface Client { + Future postEvents(List events, Callback callback) throws ClientException; + + Future postMetrics(List> messages, Callback callback) + throws ClientException; + + Future postStatus(List> messages, Callback callback) + throws ClientException; + + Future getSettings(String version, Callback callback) + throws ClientException; + + void close(); + + Status getStatus(); + + enum ClientType { + GRPC + } + + enum Status { + NOT_CONNECTED, + OK, + FAILURE + } + + interface Callback { + void complete(T result); + + void fail(Exception e); + } +} diff --git a/libs/core/src/main/java/com/solarwinds/joboe/core/rpc/ClientException.java b/libs/core/src/main/java/com/solarwinds/joboe/core/rpc/ClientException.java new file mode 100644 index 00000000..aea23fba --- /dev/null +++ b/libs/core/src/main/java/com/solarwinds/joboe/core/rpc/ClientException.java @@ -0,0 +1,31 @@ +/* + * © SolarWinds Worldwide, LLC. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.solarwinds.joboe.core.rpc; + +public class ClientException extends Exception { + public ClientException(Throwable cause) { + super(cause); + } + + public ClientException(String message) { + super(message); + } + + public ClientException(String message, Throwable cause) { + super(message, cause); + } +} diff --git a/libs/core/src/main/java/com/solarwinds/joboe/core/rpc/ClientFatalException.java b/libs/core/src/main/java/com/solarwinds/joboe/core/rpc/ClientFatalException.java new file mode 100644 index 00000000..daef1f1e --- /dev/null +++ b/libs/core/src/main/java/com/solarwinds/joboe/core/rpc/ClientFatalException.java @@ -0,0 +1,37 @@ +/* + * © SolarWinds Worldwide, LLC. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.solarwinds.joboe.core.rpc; + +/** + * Indicates exception that cannot be recovered (hence should not retry on the same operation) + * + * @author pluk + */ +public class ClientFatalException extends ClientException { + + public ClientFatalException(String message) { + super(message); + } + + public ClientFatalException(Throwable cause) { + super(cause); + } + + public ClientFatalException(String message, Throwable cause) { + super(message, cause); + } +} diff --git a/libs/core/src/main/java/com/solarwinds/joboe/core/rpc/ClientLoggingCallback.java b/libs/core/src/main/java/com/solarwinds/joboe/core/rpc/ClientLoggingCallback.java new file mode 100644 index 00000000..d8f1bcf2 --- /dev/null +++ b/libs/core/src/main/java/com/solarwinds/joboe/core/rpc/ClientLoggingCallback.java @@ -0,0 +1,177 @@ +/* + * © SolarWinds Worldwide, LLC. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.solarwinds.joboe.core.rpc; + +import com.solarwinds.joboe.core.rpc.Client.Callback; +import com.solarwinds.joboe.logging.Logger; +import com.solarwinds.joboe.logging.Logger.Level; +import com.solarwinds.joboe.logging.LoggerFactory; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.ConcurrentMap; + +/** + * Provides a Thrift Callback that performs general logging on various Thrift {@link Result}. + * + *

Takes into consideration of the current Logger's logging verbosity level + * + * @author pluk + * @param + */ +public class ClientLoggingCallback implements Callback { + private final Logger logger = LoggerFactory.getLogger(); + private final String operation; + + private final Callback delegation; + + /** + * @param operation The operation string to be included in the logging message for identification + * purpose + */ + public ClientLoggingCallback(String operation) { + this.operation = operation; + + if (logger.shouldLog(Level.DEBUG)) { + delegation = new FinerLoggingCallback(); + } else { + delegation = new LoggingCallback(); + } + } + + @Override + public void complete(T result) { + delegation.complete(result); + } + + @Override + public void fail(Exception e) { + delegation.fail(e); + } + + /** + * Callback that logs every result and exception, with full stacktrace for exceptions + * + * @author pluk + */ + private class FinerLoggingCallback implements Callback { + + @Override + public void complete(T result) { + ResultCode resultCode = result.getResultCode(); + String arg = result.getArg(); + if (resultCode.isError()) { + logger.warn( + "Failed operation [" + + operation + + "] due to Result code [" + + resultCode + + "] arg [" + + arg + + "]"); + } else { + logger.debug( + "Completed operation [" + + operation + + "] with Result code [" + + resultCode + + "] arg [" + + arg + + "]"); + } + } + + @Override + public void fail(Exception e) { + logger.warn( + "Client operation [" + operation + "] failed with exception message : " + e.getMessage(), + e); + } + } + + /** + * Callback that logs only the first occurrence of repeating error result code/repeating + * exception. No stack trace reported + * + * @author pluk + */ + private class LoggingCallback implements Callback { + private final ConcurrentMap logInfoByResultCode = + new ConcurrentHashMap(); + private static final int REPORT_INTERVAL = 60 * 1000; // 1 min + + { + for (ResultCode resultCode : ResultCode.values()) { + logInfoByResultCode.put(resultCode, new LogInfo()); + } + } + + @Override + public void complete(Result result) { + ResultCode resultCode = result.getResultCode(); + + if (resultCode.isError()) { + long time = System.currentTimeMillis(); + + synchronized (resultCode) { + LogInfo logInfo = logInfoByResultCode.get(resultCode); + logInfo.occurrence++; + + if (time - logInfo.lastLogTime >= REPORT_INTERVAL) { + if (logInfo.occurrence > 1) { + logger.warn( + "Failed operation [" + + operation + + "] due to Result code [" + + resultCode + + "] arg [" + + result.getArg() + + "] " + + logInfo.occurrence + + " occurrences since last error"); + } else { + logger.warn( + "Failed operation [" + + operation + + "] due to Result code [" + + resultCode + + "] arg [" + + result.getArg() + + "]"); + } + + logInfo.reset(time); + } + } + } + } + + @Override + public void fail(Exception e) { + // do not log exception here as we only want to warn something for persisting exception, which + // is something this logger cannot determine + } + + private class LogInfo { + private int occurrence; + private long lastLogTime = 0; + + private void reset(long time) { + occurrence = 0; + lastLogTime = time; + } + } + } +} diff --git a/libs/core/src/main/java/com/solarwinds/joboe/core/rpc/ClientManagerProvider.java b/libs/core/src/main/java/com/solarwinds/joboe/core/rpc/ClientManagerProvider.java new file mode 100644 index 00000000..5371d508 --- /dev/null +++ b/libs/core/src/main/java/com/solarwinds/joboe/core/rpc/ClientManagerProvider.java @@ -0,0 +1,47 @@ +/* + * © SolarWinds Worldwide, LLC. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.solarwinds.joboe.core.rpc; + +import com.solarwinds.joboe.core.rpc.grpc.GrpcClientManager; +import com.solarwinds.joboe.logging.Logger; +import com.solarwinds.joboe.logging.LoggerFactory; +import java.util.HashMap; +import java.util.Map; +import java.util.Optional; + +public class ClientManagerProvider { + private ClientManagerProvider() {} + + private static final Logger logger = LoggerFactory.getLogger(); + private static final Map registeredManagers = + new HashMap<>(); + + static { + registeredManagers.put(Client.ClientType.GRPC, new GrpcClientManager()); + } + + public static Optional getClientManager(Client.ClientType clientType) { + logger.debug("Using " + clientType + " for rpc calls"); + return Optional.ofNullable(registeredManagers.get(clientType)); + } + + public static void closeAllManagers() { + for (RpcClientManager clientManager : registeredManagers.values()) { + clientManager.close(); + } + } +} diff --git a/libs/core/src/main/java/com/solarwinds/joboe/core/rpc/ClientRecoverableException.java b/libs/core/src/main/java/com/solarwinds/joboe/core/rpc/ClientRecoverableException.java new file mode 100644 index 00000000..74c5810c --- /dev/null +++ b/libs/core/src/main/java/com/solarwinds/joboe/core/rpc/ClientRecoverableException.java @@ -0,0 +1,38 @@ +/* + * © SolarWinds Worldwide, LLC. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.solarwinds.joboe.core.rpc; + +/** + * Indicates exception that can be recovered (hence could retry on the same operation, and it might + * become successful) + * + * @author pluk + */ +public class ClientRecoverableException extends ClientException { + + public ClientRecoverableException(String message) { + super(message); + } + + public ClientRecoverableException(Throwable cause) { + super(cause); + } + + public ClientRecoverableException(String message, Throwable cause) { + super(message, cause); + } +} diff --git a/libs/core/src/main/java/com/solarwinds/joboe/core/rpc/ClientRejectedExecutionException.java b/libs/core/src/main/java/com/solarwinds/joboe/core/rpc/ClientRejectedExecutionException.java new file mode 100644 index 00000000..3283b313 --- /dev/null +++ b/libs/core/src/main/java/com/solarwinds/joboe/core/rpc/ClientRejectedExecutionException.java @@ -0,0 +1,31 @@ +/* + * © SolarWinds Worldwide, LLC. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.solarwinds.joboe.core.rpc; + +import java.util.concurrent.RejectedExecutionException; + +@SuppressWarnings("serial") +public class ClientRejectedExecutionException extends ClientException { + + public ClientRejectedExecutionException(RejectedExecutionException cause) { + super(cause); + } + + public ClientRejectedExecutionException(String message) { + super(message); + } +} diff --git a/libs/core/src/main/java/com/solarwinds/joboe/core/rpc/EncodingType.java b/libs/core/src/main/java/com/solarwinds/joboe/core/rpc/EncodingType.java new file mode 100644 index 00000000..4da4ab4f --- /dev/null +++ b/libs/core/src/main/java/com/solarwinds/joboe/core/rpc/EncodingType.java @@ -0,0 +1,21 @@ +/* + * © SolarWinds Worldwide, LLC. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.solarwinds.joboe.core.rpc; + +public enum EncodingType { + BSON +} diff --git a/libs/core/src/main/java/com/solarwinds/joboe/core/rpc/HeartbeatScheduler.java b/libs/core/src/main/java/com/solarwinds/joboe/core/rpc/HeartbeatScheduler.java new file mode 100644 index 00000000..a9db9148 --- /dev/null +++ b/libs/core/src/main/java/com/solarwinds/joboe/core/rpc/HeartbeatScheduler.java @@ -0,0 +1,22 @@ +/* + * © SolarWinds Worldwide, LLC. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.solarwinds.joboe.core.rpc; + +@FunctionalInterface +public interface HeartbeatScheduler { + void schedule(); +} diff --git a/libs/core/src/main/java/com/solarwinds/joboe/core/rpc/HostType.java b/libs/core/src/main/java/com/solarwinds/joboe/core/rpc/HostType.java new file mode 100644 index 00000000..aa6e4d86 --- /dev/null +++ b/libs/core/src/main/java/com/solarwinds/joboe/core/rpc/HostType.java @@ -0,0 +1,22 @@ +/* + * © SolarWinds Worldwide, LLC. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.solarwinds.joboe.core.rpc; + +public enum HostType { + PERSISTENT, + AWS_LAMBDA +} diff --git a/libs/core/src/main/java/com/solarwinds/joboe/core/rpc/KeepAliveMonitor.java b/libs/core/src/main/java/com/solarwinds/joboe/core/rpc/KeepAliveMonitor.java new file mode 100644 index 00000000..1eee51eb --- /dev/null +++ b/libs/core/src/main/java/com/solarwinds/joboe/core/rpc/KeepAliveMonitor.java @@ -0,0 +1,60 @@ +/* + * © SolarWinds Worldwide, LLC. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.solarwinds.joboe.core.rpc; + +import com.solarwinds.joboe.core.util.DaemonThreadFactory; +import com.solarwinds.joboe.logging.LoggerFactory; +import java.util.concurrent.Executors; +import java.util.concurrent.ScheduledExecutorService; +import java.util.concurrent.ScheduledFuture; +import java.util.concurrent.TimeUnit; +import java.util.function.Supplier; + +public class KeepAliveMonitor implements HeartbeatScheduler { + private final ScheduledExecutorService keepAliveService; + private ScheduledFuture keepAliveFuture; + private final Runnable keepAliveRunnable; + private static final long KEEP_ALIVE_INTERVAL = 20; // in seconds + + public KeepAliveMonitor(Supplier protocolClient, String serviceKey, Object lock) { + keepAliveService = + Executors.newScheduledThreadPool(1, DaemonThreadFactory.newInstance("keep-alive")); + keepAliveRunnable = + () -> { + synchronized (lock) { + try { + protocolClient.get().doPing(serviceKey); + schedule(); // reschedule another keep alive ping + } catch (Exception e) { + LoggerFactory.getLogger().debug("Keep alive ping failed [" + e.getMessage() + "]", e); + // do not re-schedule another keep alive ping if it was having issues + } + } + }; + schedule(); + } + + @Override + public synchronized void schedule() { + if (keepAliveFuture != null) { + keepAliveFuture.cancel(false); + } + + keepAliveFuture = + keepAliveService.schedule(keepAliveRunnable, KEEP_ALIVE_INTERVAL, TimeUnit.SECONDS); + } +} diff --git a/libs/core/src/main/java/com/solarwinds/joboe/core/rpc/ProtocolClient.java b/libs/core/src/main/java/com/solarwinds/joboe/core/rpc/ProtocolClient.java new file mode 100644 index 00000000..ea9212f2 --- /dev/null +++ b/libs/core/src/main/java/com/solarwinds/joboe/core/rpc/ProtocolClient.java @@ -0,0 +1,51 @@ +/* + * © SolarWinds Worldwide, LLC. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.solarwinds.joboe.core.rpc; + +import com.solarwinds.joboe.core.Event; +import java.util.List; +import java.util.Map; + +/** + * Protocol (Thrift/gRPC etc) specific client that usually wraps and forward call to the underlying + * code generated client + * + * @see com.solarwinds.joboe.core.rpc.Client.ClientType + */ +public interface ProtocolClient { + int MAX_CALL_SIZE = 4 * 1024 * 1024; // in bytes, an approximation + int INITIAL_MESSAGE_SIZE = 64 * 1024; // starts with 64 kB per message + int MAX_MESSAGE_SIZE = + MAX_CALL_SIZE; // the max buffer used for the message can at most be the same as max call size + + /** + * Shuts down this protocol client, should do cleanup to shutdown any underlying generated + * client/connection if applicable + */ + void shutdown(); + + Result doPostEvents(String serviceKey, List events) throws ClientException; + + Result doPostMetrics(String serviceKey, List> messages) + throws ClientException; + + Result doPostStatus(String serviceKey, List> messages) throws ClientException; + + SettingsResult doGetSettings(String serviceKey, String version) throws ClientException; + + void doPing(String serviceKey) throws Exception; +} diff --git a/libs/core/src/main/java/com/solarwinds/joboe/core/rpc/ProtocolClientFactory.java b/libs/core/src/main/java/com/solarwinds/joboe/core/rpc/ProtocolClientFactory.java new file mode 100644 index 00000000..31a05220 --- /dev/null +++ b/libs/core/src/main/java/com/solarwinds/joboe/core/rpc/ProtocolClientFactory.java @@ -0,0 +1,26 @@ +/* + * © SolarWinds Worldwide, LLC. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.solarwinds.joboe.core.rpc; + +/** + * Factory to build {@link ProtocolClient} instances + * + * @param Actual type of the ProtocolClient + */ +public interface ProtocolClientFactory { + C buildClient(String host, int port) throws ClientException; +} diff --git a/libs/core/src/main/java/com/solarwinds/joboe/core/rpc/Result.java b/libs/core/src/main/java/com/solarwinds/joboe/core/rpc/Result.java new file mode 100644 index 00000000..11e14bc8 --- /dev/null +++ b/libs/core/src/main/java/com/solarwinds/joboe/core/rpc/Result.java @@ -0,0 +1,33 @@ +/* + * © SolarWinds Worldwide, LLC. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.solarwinds.joboe.core.rpc; + +import lombok.Getter; + +@Getter +public class Result { + private final String warning; + private final ResultCode resultCode; + private final String arg; + + public Result(ResultCode resultCode, String arg, String warning) { + super(); + this.resultCode = resultCode; + this.arg = arg; + this.warning = warning; + } +} diff --git a/libs/core/src/main/java/com/solarwinds/joboe/core/rpc/ResultCode.java b/libs/core/src/main/java/com/solarwinds/joboe/core/rpc/ResultCode.java new file mode 100644 index 00000000..98d602d5 --- /dev/null +++ b/libs/core/src/main/java/com/solarwinds/joboe/core/rpc/ResultCode.java @@ -0,0 +1,29 @@ +/* + * © SolarWinds Worldwide, LLC. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.solarwinds.joboe.core.rpc; + +public enum ResultCode { + OK, + TRY_LATER, + INVALID_API_KEY, + LIMIT_EXCEEDED, + REDIRECT; + + public boolean isError() { + return !(this == OK || this == REDIRECT); + } +} diff --git a/libs/core/src/main/java/com/solarwinds/joboe/core/rpc/RpcClient.java b/libs/core/src/main/java/com/solarwinds/joboe/core/rpc/RpcClient.java new file mode 100644 index 00000000..d47bc359 --- /dev/null +++ b/libs/core/src/main/java/com/solarwinds/joboe/core/rpc/RpcClient.java @@ -0,0 +1,917 @@ +/* + * © SolarWinds Worldwide, LLC. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.solarwinds.joboe.core.rpc; + +import com.solarwinds.joboe.core.Event; +import com.solarwinds.joboe.core.util.DaemonThreadFactory; +import com.solarwinds.joboe.core.util.HeartbeatSchedulerProvider; +import com.solarwinds.joboe.logging.Logger; +import com.solarwinds.joboe.logging.Logger.Level; +import com.solarwinds.joboe.logging.LoggerFactory; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.Map.Entry; +import java.util.concurrent.*; +import lombok.Getter; + +/** + * High level client that other code logic should use to make RPC calls to collector for various + * operations such as posting events, getting trace settings. + * + *

This client takes care of connection management/retries automatically. The {@link ResultCode} + * returned in the {@link Result} indicates the result of the "last attempt" before either a + * successful response or all retries are exhausted. + * + *

For example if the final {@link ResultCode} is TRY_LATER, then that means this client has + * already attempted to retry according to backup strategy as specified by {@link RetryParams} and + * yet failed to get an OK response. Hence it returns the result of last attempt - TRY_LATER. + * + *

Take note that this client is protocol agnostic, the actual underlying protocol used in + * determined by the {@link ProtocolClientFactory} provided in the constructor. + */ +public class RpcClient implements com.solarwinds.joboe.core.rpc.Client { + private static final Logger logger = LoggerFactory.getLogger(); + + protected String host; + protected int port; + private final String serviceKey; + private boolean reportedConnectError = false; + private boolean reportedRejectedExecutionError = false; + private boolean isClosing = + false; // indicates whether this current Collector client is closing permanently + private Status connectionStatus = Status.NOT_CONNECTED; + + protected ProtocolClient protocolClient; + private final ProtocolClientFactory protocolClientFactory; + + private static final int QUEUE_CAPACITY = 100; + + private static final int TIMEOUT = 10 * 1000; // 10 secs + + public enum TaskType { + POST_EVENTS(true), + POST_METRICS(true), + POST_STATUS(true), + GET_SETTINGS(true), + CONNECTION_INIT(false); + private final boolean threadpoolRequired; + + TaskType(boolean threadpoolRequired) { + this.threadpoolRequired = threadpoolRequired; + } + } + + private final Map services = + new HashMap< + RpcClient.TaskType, + ExecutorService>(); // separate thread pool (single thread) for each message type, see + // https://github.com/librato/joboe/issues/565 + + private final RetryParamConstants + defaultRetryParamConstants; // defines various retry param constants such as init delay, max + // delay and max retry on various Result code + + private final HeartbeatScheduler heartbeatScheduler; + + /** + * @param host host of the collector + * @param port port of the collector + * @param serviceKey serviceKey for this instrumented application (api token + service name) + * @param protocolClientFactory factory used to instantiated the ProtocolClient + * @param taskTypes taskTypes this client is expected to handle, if none is defined then this + * client will support all operations + */ + public RpcClient( + String host, + int port, + String serviceKey, + ProtocolClientFactory protocolClientFactory, + TaskType... taskTypes) { + this( + host, port, serviceKey, RetryParamConstants.getDefault(), protocolClientFactory, taskTypes); + } + + /** + * @param host host of the collector + * @param port port of the collector + * @param serviceKey serviceKey for this instrumented application (api token + service name) + * @param retryParamConstants defines various retry param constants such as init delay, max delay + * and max retry on various ResultCode + * @param protocolClientFactory factory used to instantiated the ProtocolClient + * @param taskTypes taskTypes this client is expected to handle, if none is defined then this + * client will support all operations + */ + public RpcClient( + String host, + int port, + String serviceKey, + RetryParamConstants retryParamConstants, + ProtocolClientFactory protocolClientFactory, + TaskType... taskTypes) { + this.host = host; + this.port = port; + this.serviceKey = serviceKey; + this.defaultRetryParamConstants = retryParamConstants; + this.protocolClientFactory = protocolClientFactory; + + asyncInitClient(); + + if (taskTypes == null || taskTypes.length == 0) { + taskTypes = TaskType.values(); + } + + for (TaskType taskType : taskTypes) { + if (taskType.threadpoolRequired) { + services.put( + taskType, + new ThreadPoolExecutor( + 1, + 1, + 0L, + TimeUnit.MILLISECONDS, + new LinkedBlockingQueue(QUEUE_CAPACITY), + DaemonThreadFactory.newInstance(taskType.name().toLowerCase() + "-executor"))); + } + } + + heartbeatScheduler = + HeartbeatSchedulerProvider.createHeartbeatScheduler(() -> protocolClient, serviceKey, this); + } + + private Future submit(final Callable clientCall, final TaskType taskType) + throws RpcClientRejectedExecutionException { + ExecutorService executor = services.get(taskType); + if (executor == null) { + throw new RpcClientRejectedExecutionException( + "Cannot submit job of taskType [" + + taskType + + "] as this collector client only handles " + + services.keySet()); + } + + FutureTask task = + new FutureTask( + new Callable() { + @Override + public T call() throws ClientException { + T result = null; + + RetryParams retryParams = new RetryParams(taskType); + do { + result = handleClientCall(clientCall, retryParams); + } while (!RpcClient.this.isClosing + && retryParams + .retry()); // retry on the retryParams (for example ResultCode == TRY_AGAIN + // or connection recovered from failure) + + if (result == null) { // cannot even get a result object from the server + throw new ClientException( + "Failed to get response of taskType [" + + taskType + + "] from collector after " + + retryParams.currentRetryCounts + + " tries"); + } + + return result; + } + + /** + * Handles the actual collector call. This call blocks and modifies the underlying + * protocol client connection if either connection exception arises or if a "redirect" + * result is received. It is implemented this way as those conditions applies to the + * any tasks submitted to this RpcClient instance hence blocking is required + * + * @param clientCall + * @param retryParams + * @return the result of the rpc call, take note that this could be null if the + * protocol client is reconnected during a failed call + * @throws ClientException if fatal(unrecoverable) exception is found or if retry on + * failed connection has exceeded its limit + */ + private T handleClientCall(Callable clientCall, RetryParams retryParams) + throws ClientException { + synchronized ( + RpcClient.this) { // synchronize on the RpcClient instance as we do not want + // concurrent operation on the underlying protocol generated client + try { + if (!checkClient()) { // check/initialize the connection + return null; + } + + T result = clientCall.call(); // actual call to the underlying protocol client + + if (reportedConnectError) { // successfully made a call, if this connection had + // error previously, we should report that + // connection is recovered + logger.info( + "Protocol client [" + + taskType + + "] successfully recovered : " + + host + + ":" + + port); + reportedConnectError = false; // reset the flag + } + reportedRejectedExecutionError = + false; // successfully made a call, if any subsequent call is rejected due + // to full queue, it should print a new warning + + if (result.getResultCode() == ResultCode.TRY_LATER) { + retryParams.flagRetry(RetryType.TRY_LATER); + } else if (result.getResultCode() == ResultCode.LIMIT_EXCEEDED) { + retryParams.flagRetry(RetryType.LIMIT_EXCEED); + } else if (result.getResultCode() == ResultCode.REDIRECT) { + if (retryParams.flagRetry( + RetryType.REDIRECT, + true)) { // flag retry on redirect, also clear other params + resetClient( + result.getArg()); // reset the client based on the redirect params + } + } + + // update connection status + connectionStatus = + result.getResultCode().isError() ? Status.FAILURE : Status.OK; + + heartbeatScheduler.schedule(); // connection is healthy, update keep alive + + return result; + } catch (Exception e) { + if (RpcClient.this.isClosing) { + logger.debug( + "Found exception during collector Client shutdown. This is probably not critical as the client is shutting down : " + + e.getMessage(), + e); + return null; + } else if (e instanceof ClientRecoverableException) { + logConnectException(e, taskType); + reconnectClient(); // retry the connection, this blocks until connection is + // re-established + retryParams.flagRetry(RetryType.SERVER_ERROR); // retry after a server error + return null; // return null as the result as call result is unresolved + } else if (e instanceof ClientException) { // cannot recover + logger.warn( + "Error sending message to collector (fatal exception) [" + + taskType + + "] : " + + e.getClass().getName() + + " message: " + + e.getMessage()); // fatal exception so it's okay to be verbose + throw (ClientException) e; // re-throw it, does not make sense to retry + } else { + logger.warn( + "Error sending message to collector (fatal exception) [" + + taskType + + "] : " + + e.getClass().getName() + + " message: " + + e.getMessage()); // unexpected exception so it's okay to be verbose + throw new ClientException(e); // re-throw it, does not make sense to retry + } + } + } + } + + /** + * Blocks until a connection is successfully re-established or max retry attempt is + * reached + */ + private void reconnectClient() { + shutdownProtocolClient(); + initClient(); + } + }); + + try { + executor.execute(task); + } catch (RejectedExecutionException e) { + if (!executor.isShutdown()) { // do not report exception if it's getting shutdown + handleRejectedExecutionException(e); + throw new RpcClientRejectedExecutionException(e); + } + } + + return task; + } + + /** + * Checks if underlying protocol client is available. Initialize the underlying client if it's not + * yet available. + * + *

This method blocks according to the defaultRetryParamConstants. By default, this should + * block indefinitely until connection is successfully initialized + */ + private synchronized boolean checkClient() { + if (protocolClient == null) { + initClient(); + } + return protocolClient != null; + } + + /** Resets the underlying protocol outbound client based on the String arg (for host/port) */ + private void resetClient(String arg) throws ClientFatalException { + shutdownProtocolClient(); // shut down the current generated client + if (arg != null && !arg.isEmpty()) { + String[] tokens = arg.split(":"); + + String newHost; + Integer newPort = null; + if (tokens.length == 1) { + logger.warn( + "Redirect from Collector but couldn't locate port number from the response arg: [" + + arg + + "], using previous port: " + + this.port); + newHost = tokens[0]; + } else { + newHost = tokens[0]; + try { + newPort = Integer.parseInt(tokens[1]); + } catch (NumberFormatException e) { + throw new ClientFatalException( + "Failed to perform collector Redirect. Invalid port number [" + + tokens[1] + + "] found in arg [" + + arg + + "]"); + } + } + + this.host = newHost; + if (newPort != null) { + this.port = newPort; + } + logger.info("Collector Redirect to " + this.host + ":" + this.port); + + initClient(); + } else { + throw new ClientFatalException( + "Failed to perform collector Redirect. Redirect args is empty"); + } + } + + @Override + protected void finalize() throws Throwable { + close(); + super.finalize(); + } + + private final void asyncInitClient() { + ExecutorService executorService = + Executors.newSingleThreadExecutor(DaemonThreadFactory.newInstance("init-rpc-client")); + executorService.submit(() -> initClient()); + executorService.shutdown(); + } + + /** + * Initialize the underlying protocol channel (create connection and perform a ping check) + * + *

It blocks and retries on failure unless max retry for CONNECTION_FAILURE is reached + * according to defaultRetryParamConstants. + */ + private synchronized void initClient() { + RetryParams retryParams = new RetryParams(TaskType.CONNECTION_INIT); + + boolean initClient = !RpcClient.this.isClosing; + while (initClient) { + logger.debug("Creating collector client : " + host + ":" + port); + try { + protocolClient = protocolClientFactory.buildClient(host, port); + logger.debug("Created collector client : " + host + ":" + port); + + connectionStatus = Status.OK; + return; + } catch (Exception e) { + connectionStatus = Status.NOT_CONNECTED; + if (retryParams.getCurrentRetryCount(RetryType.CONNECTION_FAILURE) + == 20) { // prolonged outage + logCriticalConnectException(e, TaskType.CONNECTION_INIT); + } else { + logConnectException(e, TaskType.CONNECTION_INIT); + } + + retryParams.flagRetry(RetryType.CONNECTION_FAILURE); + shutdownProtocolClient(); // always shuts down underlying and retry in this case + } + + initClient = !RpcClient.this.isClosing && retryParams.retry(); + } + } + + /** + * Logs on every connection exception if applicable + * + * @param e + * @param taskType + */ + private void logConnectException(Exception e, TaskType taskType) { + if (logger.shouldLog( + Level.DEBUG)) { // if logging level is debug, then log this exception always + logger.warn( + "SSL client connection to collector [" + + host + + ":" + + port + + "] failed [" + + taskType + + "], message : " + + e.getMessage(), + e); + reportedConnectError = true; + } + } + + /** + * Logs on critical connection exception - for example prolonged connection init failure + * + * @param e + * @param taskType + */ + private void logCriticalConnectException(Exception e, TaskType taskType) { + if (logger.shouldLog(Level.DEBUG) + || !reportedConnectError) { // Warn it on the first time only for INFO+ logging settings + if (logger.shouldLog(Level.DEBUG)) { // only report full trace if it's DEBUG logging settings + logger.warn( + "SSL client connection to collector [" + + host + + ":" + + port + + "] failed after retries [" + + taskType + + "], message : " + + e.getMessage(), + e); + } else { + logger.warn( + "SSL client connection to collector [" + + host + + ":" + + port + + "] failed after retries [" + + taskType + + "], message : " + + e.getMessage()); + } + reportedConnectError = true; + } + } + + private void handleRejectedExecutionException(RejectedExecutionException e) { + if (logger.shouldLog(Level.DEBUG) + || !reportedRejectedExecutionError) { // Warn it on the first time only for INFO+ logging + // settings + logger.warn( + "Rejected operation on Collector client side, probably due to full queue : " + + e.getMessage()); + reportedRejectedExecutionError = true; + } + } + + /** + * Posts tracing events to collector + * + * @param events tracing events to be posted + * @param callback callback to invoke if the operation is completed, null means no callback + * @return Future of the Result + * @throws ClientException + */ + @Override + public Future postEvents(final List events, final Callback callback) + throws ClientException { + return submit( + new CallableWithCallback(callback) { + @Override + public Result doCall() throws Exception { + return protocolClient.doPostEvents(serviceKey, events); + } + }, + TaskType.POST_EVENTS); + } + + /** + * Posts metrics messages to collector + * + * @param messages metrics messages to be posted + * @param callback callback to invoke if the operation is completed, null means no callback + * @return Future of the Result + * @throws ClientException + */ + @Override + public Future postMetrics( + final List> messages, final Callback callback) + throws ClientException { + return submit( + new CallableWithCallback(callback) { + @Override + public Result doCall() throws Exception { + return protocolClient.doPostMetrics(serviceKey, messages); + } + }, + TaskType.POST_METRICS); + } + + /** + * Posts status messages (for example init, framework usage) to collector + * + * @param messages status messages to be posted + * @param callback callback to invoke if the operation is completed, null means no callback + * @return Future of the Result + * @throws ClientException + */ + @Override + public Future postStatus( + final List> messages, final Callback callback) + throws ClientException { + return submit( + new CallableWithCallback(callback) { + @Override + public Result doCall() throws Exception { + return protocolClient.doPostStatus(serviceKey, messages); + } + }, + TaskType.POST_STATUS); + } + + /** + * Gets tracing settings from the collector + * + * @param version version of the settings structure being requested + * @param callback callback to invoke if the operation is completed, null means no callback + * @return Future of the SettingsResult + * @throws ClientException + */ + @Override + public Future getSettings( + final String version, final Callback callback) throws ClientException { + return submit( + new CallableWithCallback(callback) { + @Override + public SettingsResult doCall() throws Exception { + return protocolClient.doGetSettings(serviceKey, version); + } + }, + TaskType.GET_SETTINGS); + } + + /** + * Closes this high level rpc client and the underlying protocol client. + * + *

Stops accepting new rpc calls (new calls will be rejected with + * RpcClientRejectedExecutionException) and finishes the remaining calls. + * + *

This client cannot be reused anymore after closing. + */ + @Override + public void close() { + isClosing = true; + + // stop all the job queue executor services, but it should still process jobs that are queued + for (ExecutorService service : services.values()) { + if (connectionStatus != Status.OK) { + logger.debug( + "Force shutting down the collector client executor to avoid hanging due to connection retry"); + service.shutdownNow(); + } else { + logger.debug("Shutting down the collector client executor"); + service.shutdown(); + } + } + + shutdownProtocolClient(); + } + + private synchronized void shutdownProtocolClient() { + if (protocolClient != null) { + protocolClient.shutdown(); + protocolClient = null; + } + } + + /** + * Gets the current connection {@link com.solarwinds.joboe.core.rpc.Client.Status} + * + * @return + */ + @Override + public Status getStatus() { + return connectionStatus; + } + + @Override + public String toString() { + return "RpcClient [host=" + host + ", port=" + port + "]"; + } + + private abstract static class CallableWithCallback implements Callable { + private final Callback callback; + + private CallableWithCallback(Callback callback) { + this.callback = callback; + } + + private static String previousReportedWarning = + null; // could be modified by multiple thread, but it's rather harmless even if that happens + + @Override + public final T call() throws Exception { + try { + T result = + doCall(); // might be more performant to separate the load generation with the actual + // collector outbound call + if (callback != null) { + callback.complete(result); + } + + String warning = result.getWarning(); + if (warning != null && !warning.isEmpty()) { + if (logger.shouldLog(Level.DEBUG) || !warning.equals(previousReportedWarning)) { + logger.warn("RPC call warning : [" + warning + "]"); + previousReportedWarning = warning; + } + } else if (result.getResultCode() + == ResultCode.OK) { // then reset the previous warning, since it's OK now + previousReportedWarning = null; + } + + return result; + } catch (Exception e) { + if (callback != null) { + callback.fail(e); + } + throw e; + } + } + + public abstract T doCall() throws Exception; + } + + /** + * Identifies reasoning for a retry operation + * + * @author pluk + */ + @Getter + enum RetryType { + TRY_LATER(true), // retry as result code is TRY_LATER + LIMIT_EXCEED(true), // retry as result code is LIMIT_EXCEED + CONNECTION_FAILURE(true), // retry due to failed connection + SERVER_ERROR(true), // retry after a error triggered by server + CONNECTION_RECOVERED(false), // retry due to connection successfully recovered from failure + REDIRECT(false); // retry as result code is REDIRECT + + /** + * -- GETTER -- + * + * @return whether this result code is considered a "failure" result code + */ + private final boolean failure; + + RetryType(boolean failure) { + this.failure = failure; + } + } + + /** Default values for retry params */ + public static class RetryParamConstants { + private final Map> initRetryDelaysByTaskType = + new HashMap>(); + private final Map> maxRetryDelaysByTaskType = + new HashMap>(); + private final Map> maxRetryCountsByTaskType = + new HashMap>(); + + private static final int DEFAULT_INIT_DELAY = 500; + private static final int DEFAULT_MAX_DELAY = 60000; + private static final int DEAFULT_MAX_RETRY_COUNT = 20; + + /** + * Retrieves the default param constants, which has init delay 500 ms, max delay 60000 ms, and + * max retry count 20. + * + *

Take note that for task type "GET_SETTING", the max retry count is set to 0 on failure + * RetryType (no retry on failure RetryType) + */ + static RetryParamConstants getDefault() { + + return new RetryParamConstants( + DEFAULT_INIT_DELAY, DEFAULT_MAX_DELAY, DEAFULT_MAX_RETRY_COUNT); + } + + private final void setMaxRetryCountsOnFailure(TaskType taskType, Integer maxRetryCount) { + for (Entry maxRetryCountEntry : + maxRetryCountsByTaskType.get(taskType).entrySet()) { + if (maxRetryCountEntry.getKey().isFailure()) { + maxRetryCountEntry.setValue(maxRetryCount); + } + } + } + + /** for internal testing purposes */ + public RetryParamConstants(int initRetryDelay, int maxRetryDelay, int maxRetryCount) { + for (TaskType taskType : TaskType.values()) { + Map initRetryDelays = new HashMap<>(); + initRetryDelays.put(RetryType.TRY_LATER, initRetryDelay); + initRetryDelays.put(RetryType.LIMIT_EXCEED, initRetryDelay); + initRetryDelays.put(RetryType.CONNECTION_FAILURE, initRetryDelay); + initRetryDelays.put(RetryType.SERVER_ERROR, initRetryDelay); + initRetryDelaysByTaskType.put(taskType, initRetryDelays); + + Map maxRetryDelays = new HashMap<>(); + maxRetryDelays.put(RetryType.TRY_LATER, maxRetryDelay); + maxRetryDelays.put(RetryType.LIMIT_EXCEED, maxRetryDelay); + maxRetryDelays.put(RetryType.CONNECTION_FAILURE, maxRetryDelay); + maxRetryDelays.put(RetryType.SERVER_ERROR, maxRetryDelay); + maxRetryDelaysByTaskType.put(taskType, maxRetryDelays); + + Map maxRetryCounts = new HashMap<>(); + maxRetryCounts.put(RetryType.TRY_LATER, maxRetryCount); + maxRetryCounts.put(RetryType.LIMIT_EXCEED, maxRetryCount); + maxRetryCounts.put( + RetryType.CONNECTION_FAILURE, + null); // CONNECITON_FAILURE - it should always retry indefinitely, unless specified + // otherwise + maxRetryCounts.put(RetryType.SERVER_ERROR, maxRetryCount); + maxRetryCounts.put(RetryType.REDIRECT, maxRetryCount); + maxRetryCountsByTaskType.put(taskType, maxRetryCounts); + } + + setMaxRetryCountsOnFailure( + TaskType.GET_SETTINGS, + 0); // special case...no retry for GET_SETTINGS on failure RetryType + setMaxRetryCountsOnFailure( + TaskType.CONNECTION_INIT, + null); // special case. for CONNECTION_INIT, we keep retrying until it's successful + } + } + + /** + * Represents the state of retrying. Provides ways to flag retry and performs sleep based {@link + * RetryType} + */ + class RetryParams { + private static final double RETRY_MULTIPLIER = 1.5; + private final Map currentRetryDelays = new HashMap(); + private final Map currentRetryCounts = new HashMap(); + + private boolean shouldRetry; + private Integer activeDelay; + private final TaskType taskType; + + RetryParams(TaskType taskType) { + this.taskType = taskType; + } + + /** + * Flags for a retry with a given reason in the {@link RetryType}, this will affect the outcome + * of the next {@link RetryParams#retry()} call. + * + *

Take note that flagging a retry with this method alone does not determine the final + * outcome of the next {@link RetryParams#retry()} call, the current states of the + * RetryParams would also be assessed (for example if the current retry count has + * exceeded the limit) + * + * @param retryType + * @return whether flagging is successful based on the current state + */ + public boolean flagRetry(RetryType retryType) { + return flagRetry(retryType, false); + } + + /** + * Flags for a retry with a given reason in the {@link RetryType}, this will affect the outcome + * of the next {@link RetryParams#retry()} call. + * + *

Take note that flagging a retry with this method alone does not determine the final + * outcome of the next {@link RetryParams#retry()} call, the current states of the + * RetryParams would also be assessed (for example if the current retry count has + * exceeded the limit) + * + * @param retryType + * @param resetOtherParams whether to reset the states of all params with other RetryType + * + * @return whether flagging is successful based on the current state + */ + public boolean flagRetry(RetryType retryType, boolean resetOtherParams) { + // first increment delay + Integer delay = currentRetryDelays.get(retryType); + if (delay == null) { + delay = getInitRetryDelay(taskType, retryType); + if (delay == null) { + delay = 0; + } + } else { + Integer maxRetryDelay = getMaxRetryDelay(taskType, retryType); + delay = + maxRetryDelay != null + ? Math.min((int) (delay * RETRY_MULTIPLIER), maxRetryDelay) + : (int) (delay * RETRY_MULTIPLIER); + } + + if (resetOtherParams) { + currentRetryDelays.clear(); + } + + currentRetryDelays.put(retryType, delay); + + // then increment count + Integer retryCount = currentRetryCounts.get(retryType); + if (retryCount == null) { + retryCount = 1; + } else { + retryCount++; + } + + if (resetOtherParams) { + currentRetryCounts.clear(); + } + + currentRetryCounts.put(retryType, retryCount); + + // update the should retry flag + Integer maxRetryCount = + getMaxRetryCount(taskType, retryType); // could be null if there's no limit to it + shouldRetry = maxRetryCount == null || retryCount <= maxRetryCount; + if (!shouldRetry) { + logger.debug( + "Not going to retry message as max retry count [" + + maxRetryCount + + "] is exceeded, cause: " + + retryType); + activeDelay = 0; + } else { + logger.debug( + "Flagging to retry message with delay [" + delay + "] ms, cause: " + retryType); + activeDelay = delay; + } + + return shouldRetry; + } + + private Integer getMaxRetryCount(TaskType taskType, RetryType retryType) { + return defaultRetryParamConstants.maxRetryCountsByTaskType.get(taskType).get(retryType); + } + + private Integer getMaxRetryDelay(TaskType taskType, RetryType retryType) { + return defaultRetryParamConstants.maxRetryDelaysByTaskType.get(taskType).get(retryType); + } + + private Integer getInitRetryDelay(TaskType taskType, RetryType retryType) { + return defaultRetryParamConstants.initRetryDelaysByTaskType.get(taskType).get(retryType); + } + + private int getCurrentRetryCount(RetryType retryType) { + return currentRetryCounts.containsKey(retryType) ? currentRetryCounts.get(retryType) : 0; + } + + /** + * Determines whether a retry should be performed based on previous call of {@link + * RetryParams#flagRetry(RetryType) } with consideration of current states for retry + * restrictions on the "TaskType" of this RetryParam refers to. + * + *

This might enforce time wait (thread sleep) if a retry should be performed with delay + * + * @return whether retry should be performed + */ + public boolean retry() { + if (shouldRetry) { + try { + if (activeDelay > 0) { + logger.debug("Collector client retry sleeping for " + activeDelay + " millisecs"); + TimeUnit.MILLISECONDS.sleep(activeDelay); + } + } catch (InterruptedException e) { + logger.debug("Collector client retry sleep is interrupted, message: " + e.getMessage()); + return false; // should not retry if sleep is interrupted + } finally { + activeDelay = 0; + shouldRetry = false; // reset + } + return true; + } else { + return false; + } + } + } + + /** + * A keep alive monitor that sends a ping message if it has been idle for 20 seconds + * + * @author pluk + */ +} diff --git a/libs/core/src/main/java/com/solarwinds/joboe/core/rpc/RpcClientManager.java b/libs/core/src/main/java/com/solarwinds/joboe/core/rpc/RpcClientManager.java new file mode 100644 index 00000000..1ac5226e --- /dev/null +++ b/libs/core/src/main/java/com/solarwinds/joboe/core/rpc/RpcClientManager.java @@ -0,0 +1,134 @@ +/* + * © SolarWinds Worldwide, LLC. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.solarwinds.joboe.core.rpc; + +import com.solarwinds.joboe.config.ConfigManager; +import com.solarwinds.joboe.config.ConfigProperty; +import com.solarwinds.joboe.core.rpc.Client.ClientType; +import com.solarwinds.joboe.logging.Logger; +import com.solarwinds.joboe.logging.LoggerFactory; +import java.io.File; +import java.net.MalformedURLException; +import java.net.URL; + +/** + * Manages creation of PRC {@link Client} + * + * @author pluk + */ +public abstract class RpcClientManager { + private static final Logger logger = LoggerFactory.getLogger(); + + static final String DEFAULT_HOST = + "apm.collector.na-01.cloud.solarwinds.com"; // default collector host: NH production + + static final int DEFAULT_PORT = 443; // default collector port + + protected static String collectorHost; + + protected static int collectorPort; + + protected static URL collectorCertLocation; + + static { + init( + (String) ConfigManager.getConfig(ConfigProperty.AGENT_COLLECTOR), + (String) ConfigManager.getConfig(ConfigProperty.AGENT_COLLECTOR_SERVER_CERT_LOCATION)); + } + + static void init(String collector, String collectorCertValue) { + if (collector != null) { // use system env if defined + setHostAndPortByVariable(collector); + } else { + collectorHost = DEFAULT_HOST; + collectorPort = DEFAULT_PORT; + } + + if (collectorCertValue != null) { + logger.info("Setting RPC Client to use server cert at [" + collectorCertValue + "]"); + try { + File collectorCert = new File(collectorCertValue); + if (collectorCert.exists()) { + collectorCertLocation = collectorCert.toURI().toURL(); + } else { + logger.warn( + "Failed to load RPC collector server certificate from location [" + + collectorCertValue + + "], file does not exist!"); + } + + } catch (MalformedURLException e) { + logger.warn( + "Failed to load RPC collector server certificate from location [" + + collectorCertValue + + "], using default location instead!"); + } + } + } + + private static void setHostAndPortByVariable(String variable) { + String[] tokens = variable.split(":"); + if (tokens.length == 1) { + collectorHost = tokens[0]; + collectorPort = DEFAULT_PORT; + } else { + collectorHost = tokens[0]; + try { + collectorPort = Integer.parseInt(tokens[1]); + } catch (NumberFormatException e) { + logger.warn( + "Failed to parse the port number from " + + ConfigProperty.AGENT_COLLECTOR.getEnvironmentVariableKey() + + " : [" + + variable + + "]"); + collectorPort = DEFAULT_PORT; + } + } + + logger.info( + "Using RPC collector host [" + + collectorHost + + "] port [" + + collectorPort + + "] for RPC client creation"); + } + + public static Client getClient(OperationType operationType) throws ClientException { + return getClient( + operationType, (String) ConfigManager.getConfig(ConfigProperty.AGENT_SERVICE_KEY)); + } + + public static Client getClient(OperationType operationType, String serviceKey) + throws ClientException { + return ClientManagerProvider.getClientManager(ClientType.GRPC) + .orElseThrow(() -> new ClientException("Unknown Client type requested")) + .getClientImpl(operationType, serviceKey); + } + + protected abstract void close(); + + protected abstract Client getClientImpl(OperationType operationType, String serviceKey); + + public enum OperationType { + TRACING, + PROFILING, + SETTINGS, + METRICS, + STATUS + } +} diff --git a/libs/core/src/main/java/com/solarwinds/joboe/core/rpc/RpcClientRejectedExecutionException.java b/libs/core/src/main/java/com/solarwinds/joboe/core/rpc/RpcClientRejectedExecutionException.java new file mode 100644 index 00000000..3f34cd98 --- /dev/null +++ b/libs/core/src/main/java/com/solarwinds/joboe/core/rpc/RpcClientRejectedExecutionException.java @@ -0,0 +1,35 @@ +/* + * © SolarWinds Worldwide, LLC. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.solarwinds.joboe.core.rpc; + +import java.util.concurrent.RejectedExecutionException; + +/** + * Indicates collector RPC client rejects the operation before making any actual outbound requests + * + * @author pluk + */ +public class RpcClientRejectedExecutionException extends ClientRejectedExecutionException { + + public RpcClientRejectedExecutionException(RejectedExecutionException cause) { + super(cause); + } + + public RpcClientRejectedExecutionException(String message) { + super(message); + } +} diff --git a/libs/core/src/main/java/com/solarwinds/joboe/core/rpc/RpcSettings.java b/libs/core/src/main/java/com/solarwinds/joboe/core/rpc/RpcSettings.java new file mode 100644 index 00000000..d8ce7d9d --- /dev/null +++ b/libs/core/src/main/java/com/solarwinds/joboe/core/rpc/RpcSettings.java @@ -0,0 +1,151 @@ +/* + * © SolarWinds Worldwide, LLC. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.solarwinds.joboe.core.rpc; + +import com.solarwinds.joboe.logging.Logger; +import com.solarwinds.joboe.logging.LoggerFactory; +import com.solarwinds.joboe.sampling.Settings; +import com.solarwinds.joboe.sampling.SettingsArg; +import com.solarwinds.joboe.sampling.TraceDecisionUtil; +import java.nio.ByteBuffer; +import java.util.HashMap; +import java.util.Map; +import java.util.Map.Entry; + +/** Settings extracted from RPC calls */ +public class RpcSettings extends com.solarwinds.joboe.sampling.Settings { + private static final Logger logger = LoggerFactory.getLogger(); + + private final short type; // required + private final short flags; // required + private final long timestamp; // required, in millsec + private final long value; // required + private final long ttl; // time to live this settings record + private final Map, Object> args = + new HashMap, Object>(); // other arguments + + public RpcSettings( + String stringFlags, long timestamp, long value, long ttl, Map args) { + this.type = Settings.OBOE_SETTINGS_TYPE_DEFAULT_SAMPLE_RATE; + this.flags = convertFlagsFromStringToShort(stringFlags); + this.timestamp = timestamp; + if (value < 0) { + logger.warn( + "Received invalid sample rate from RPC client : " + + value + + ", must be between 0 and " + + TraceDecisionUtil.SAMPLE_RESOLUTION); + this.value = 0; + } else if (value > TraceDecisionUtil.SAMPLE_RESOLUTION) { + logger.warn( + "Received invalid sample rate from RPC client : " + + value + + ", must be between 0 and " + + TraceDecisionUtil.SAMPLE_RESOLUTION); + this.value = TraceDecisionUtil.SAMPLE_RESOLUTION; + } else { + this.value = value; + } + this.ttl = ttl; + readArgs(args); + } + + /** + * For internal testing purpose only + * + * @param source + * @param timestamp a new timestamp + */ + public RpcSettings(RpcSettings source, long timestamp) { + this.type = source.type; + this.flags = source.flags; + this.timestamp = timestamp; // take the new timestamp + this.value = source.value; + this.ttl = source.ttl; + this.args.putAll(source.args); + } + + /** + * Read arguments from the input map + * + * @param inputArgs + */ + private void readArgs(Map inputArgs) { + for (Entry inputArg : inputArgs.entrySet()) { + SettingsArg arg = SettingsArg.fromKey(inputArg.getKey()); + if (arg == null) { + logger.debug("Cannot recognize argument [" + inputArg.getKey() + "], ignoring..."); + } else { + args.put(arg, arg.readValue(inputArg.getValue())); + } + } + } + + public static short convertFlagsFromStringToShort(String stringFlags) { + short flags = 0; + String[] flagTokens = stringFlags.split(","); + for (String flagToken : flagTokens) { + if ("OVERRIDE".equals(flagToken)) { + flags |= OBOE_SETTINGS_FLAG_OVERRIDE; + } else if ("SAMPLE_START".equals(flagToken)) { + flags |= OBOE_SETTINGS_FLAG_SAMPLE_START; + } else if ("SAMPLE_THROUGH".equals(flagToken)) { + flags |= OBOE_SETTINGS_FLAG_SAMPLE_THROUGH; + } else if ("SAMPLE_THROUGH_ALWAYS".equals(flagToken)) { + flags |= OBOE_SETTINGS_FLAG_SAMPLE_THROUGH_ALWAYS; + } else if ("TRIGGER_TRACE".equals(flagToken)) { + flags |= OBOE_SETTINGS_FLAG_TRIGGER_TRACE_ENABLED; + } else if ("SAMPLE_BUCKET_ENABLED".equals(flagToken)) { // not used anymore + flags |= OBOE_SETTINGS_FLAG_SAMPLE_BUCKET_ENABLED; + } else { + logger.debug("Unknown flag found from settings: " + flagToken); + } + } + return flags; + } + + @Override + public long getValue() { + return value; + } + + @Override + public long getTimestamp() { + return timestamp; + } + + @Override + public short getType() { + return type; + } + + @Override + public short getFlags() { + return flags; + } + + @Override + public long getTtl() { + return ttl; + } + + @Override + @SuppressWarnings("unchecked") + public T getArgValue(SettingsArg arg) { + return (T) args.get(arg); + } +} diff --git a/libs/core/src/main/java/com/solarwinds/joboe/core/rpc/SettingsResult.java b/libs/core/src/main/java/com/solarwinds/joboe/core/rpc/SettingsResult.java new file mode 100644 index 00000000..1d5fd47f --- /dev/null +++ b/libs/core/src/main/java/com/solarwinds/joboe/core/rpc/SettingsResult.java @@ -0,0 +1,32 @@ +/* + * © SolarWinds Worldwide, LLC. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.solarwinds.joboe.core.rpc; + +import com.solarwinds.joboe.sampling.Settings; +import java.util.List; +import lombok.Getter; + +@Getter +public class SettingsResult extends Result { + private final List settings; + + public SettingsResult( + ResultCode resultCode, String arg, String warning, List settings) { + super(resultCode, arg, warning); + this.settings = settings; + } +} diff --git a/libs/core/src/main/java/com/solarwinds/joboe/core/rpc/grpc/GrpcClient.java b/libs/core/src/main/java/com/solarwinds/joboe/core/rpc/grpc/GrpcClient.java new file mode 100644 index 00000000..82e11610 --- /dev/null +++ b/libs/core/src/main/java/com/solarwinds/joboe/core/rpc/grpc/GrpcClient.java @@ -0,0 +1,444 @@ +/* + * © SolarWinds Worldwide, LLC. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.solarwinds.joboe.core.rpc.grpc; + +import static com.solarwinds.joboe.core.settings.SettingsUtil.transformToLocalSettings; +import static com.solarwinds.joboe.core.util.HostInfoUtils.getHostId; +import static com.solarwinds.joboe.core.util.ServerHostInfoReader.setIfNotNull; + +import com.google.protobuf.ByteString; +import com.solarwinds.joboe.config.ConfigManager; +import com.solarwinds.joboe.config.ConfigProperty; +import com.solarwinds.joboe.config.ProxyConfig; +import com.solarwinds.joboe.core.BsonBufferException; +import com.solarwinds.joboe.core.Event; +import com.solarwinds.joboe.core.HostId; +import com.solarwinds.joboe.core.rpc.ClientException; +import com.solarwinds.joboe.core.rpc.ClientFatalException; +import com.solarwinds.joboe.core.rpc.ClientRecoverableException; +import com.solarwinds.joboe.core.rpc.ProtocolClient; +import com.solarwinds.joboe.core.rpc.ProtocolClientFactory; +import com.solarwinds.joboe.core.rpc.Result; +import com.solarwinds.joboe.core.rpc.ResultCode; +import com.solarwinds.joboe.core.rpc.SettingsResult; +import com.solarwinds.joboe.core.util.BsonUtils; +import com.solarwinds.joboe.core.util.HostInfoUtils; +import com.solarwinds.joboe.core.util.SslUtils; +import com.solarwinds.joboe.logging.Logger; +import com.solarwinds.joboe.logging.LoggerFactory; +import com.solarwinds.trace.ingestion.proto.Collector; +import com.solarwinds.trace.ingestion.proto.TraceCollectorGrpc; +import io.grpc.HttpConnectProxiedSocketAddress; +import io.grpc.ManagedChannel; +import io.grpc.Status; +import io.grpc.StatusRuntimeException; +import io.grpc.netty.GrpcSslContexts; +import io.grpc.netty.NettyChannelBuilder; +import java.io.IOException; +import java.net.InetSocketAddress; +import java.net.URL; +import java.security.GeneralSecurityException; +import java.util.ArrayList; +import java.util.List; +import java.util.Map; +import java.util.concurrent.TimeUnit; +import javax.net.ssl.SSLException; +import javax.net.ssl.TrustManagerFactory; + +/** + * Collector protocol client that uses the gRPC protocol. + * + *

Wraps the gRPC client generated from the collector.proto definition + */ +public class GrpcClient implements ProtocolClient { + private static final Logger logger = LoggerFactory.getLogger(); + private final TraceCollectorGrpc.TraceCollectorBlockingStub client; + private final GrpcHostIdManager hostIdManager = new GrpcHostIdManager(); + + private static final int INITIAL_MESSAGE_SIZE = 64 * 1024; // 64 kB + + private final int deadlineSeconds; + + GrpcClient(TraceCollectorGrpc.TraceCollectorBlockingStub blockingStub) { + this.client = blockingStub; + Integer configuredDeadLine = + (Integer) ConfigManager.getConfig(ConfigProperty.AGENT_COLLECTOR_TIMEOUT); + this.deadlineSeconds = configuredDeadLine == null ? 10 : configuredDeadLine; + } + + @Override + public void shutdown() { + if (client != null && client.getChannel() instanceof ManagedChannel) { + ((ManagedChannel) client.getChannel()).shutdown(); + } + } + + @Override + public Result doPostEvents(String serviceKey, List events) throws ClientException { + return postInBatch(serviceKey, events, EVENT_SERIALIZER, POST_EVENTS_ACTION); + } + + @Override + public Result doPostMetrics(String serviceKey, List> messages) + throws ClientException { + return postInBatch(serviceKey, messages, KEY_VALUE_MAP_SERIALIZER, POST_METRICS_ACTION); + } + + @Override + public Result doPostStatus(String serviceKey, List> messages) + throws ClientException { + return postInBatch(serviceKey, messages, KEY_VALUE_MAP_SERIALIZER, POST_STATUS_ACTION); + } + + @Override + public SettingsResult doGetSettings(String serviceKey, String version) throws ClientException { + // For getSettings call, we decided to fill in `hostname` only for `HostID` for consistency with + // other agent implementation + // Unlike other language agent, this change does not give any performance boost to java agent + Collector.SettingsRequest request = + Collector.SettingsRequest.newBuilder() + .setApiKey(serviceKey) + .setClientVersion(version) + .setIdentity(hostIdManager.getHostnameOnlyHostID()) + .build(); + try { + Collector.SettingsResult result = + client.withDeadlineAfter(deadlineSeconds, TimeUnit.SECONDS).getSettings(request); + return transformToLocalSettings(result); + + } catch (StatusRuntimeException e) { + throw new ClientRecoverableException( + "gRPC Operation failed : [get settings] status [" + e.getStatus() + "]", e); + } + } + + @Override + public void doPing(String serviceKey) throws ClientException { + try { + client.ping(Collector.PingRequest.newBuilder().setApiKey(serviceKey).build()); + } catch (StatusRuntimeException e) { + throw new ClientRecoverableException( + "gRPC Operation failed : [ping] status [" + e.getStatus() + "]", e); + } + } + + private Result postInBatch( + String serviceKey, List items, Serializer serializer, PostAction postAction) + throws ClientException { + List> itemsByCalls = new ArrayList>(); + List byteStrings = new ArrayList(); + itemsByCalls.add(byteStrings); + + long callSize = 0; + for (T item : items) { + try { + ByteString itemByteString = serializer.serialize(item); + int size = itemByteString.size(); + if (callSize + size > MAX_CALL_SIZE) { + byteStrings = new ArrayList(); + itemsByCalls.add(byteStrings); + callSize = size; + } else { + callSize += size; + } + byteStrings.add(itemByteString); + + } catch (BsonBufferException e) { + logger.warn( + "Failed to perform action [" + + postAction.getDescription() + + "] due to buffer exception : " + + e.getMessage(), + e); + throw new ClientFatalException(e); + } + } + + Collector.MessageResult resultMessage = null; + for (List itemsAsByteString : itemsByCalls) { + Collector.HostID hostId = hostIdManager.getHostID(); + Collector.MessageRequest.Builder builder = + Collector.MessageRequest.newBuilder() + .setApiKey(serviceKey) + .setIdentity(hostId) + .setEncoding(Collector.EncodingType.BSON); + logger.debug( + postAction.getDescription() + + " " + + itemsAsByteString.size() + + " item(s) using gRPC client hostId=" + + hostId); + try { + builder.addAllMessages(itemsAsByteString); + resultMessage = + postAction.post( + client.withDeadlineAfter(deadlineSeconds, TimeUnit.SECONDS), builder.build()); + } catch (StatusRuntimeException e) { + if (e.getStatus().getCode() == Status.RESOURCE_EXHAUSTED.getCode()) { + throw new ClientFatalException( + "gRPC Operation failed : [post events] status [" + + e.getStatus() + + "]. This is not recoverable due to exhausted resource.", + e); + } else { + throw new ClientRecoverableException( + "gRPC Operation failed : [post events] status [" + e.getStatus() + "]", e); + } + } + } + + if (resultMessage == null) { + return new Result(null, null, null); + } + + return new Result( + ResultCode.valueOf(resultMessage.getResult().name()), + resultMessage.getArg(), + resultMessage.getWarning()); + } + + interface Serializer { + ByteString serialize(T item) throws BsonBufferException; + } + + interface PostAction { + Collector.MessageResult post( + TraceCollectorGrpc.TraceCollectorBlockingStub client, + Collector.MessageRequest messageRequest) + throws StatusRuntimeException; + + String getDescription(); + } + + private static final Serializer> KEY_VALUE_MAP_SERIALIZER = + item -> + ByteString.copyFrom( + BsonUtils.convertMapToBson(item, INITIAL_MESSAGE_SIZE, MAX_MESSAGE_SIZE)); + + private static final Serializer EVENT_SERIALIZER = + event -> ByteString.copyFrom(event.toBytes()); + + private static final PostAction POST_EVENTS_ACTION = + new PostAction() { + @Override + public Collector.MessageResult post( + TraceCollectorGrpc.TraceCollectorBlockingStub client, + Collector.MessageRequest request) { + return client.postEvents(request); + } + + @Override + public String getDescription() { + return "Post Events"; + } + }; + + private static final PostAction POST_STATUS_ACTION = + new PostAction() { + @Override + public Collector.MessageResult post( + TraceCollectorGrpc.TraceCollectorBlockingStub client, + Collector.MessageRequest request) { + return client.postStatus(request); + } + + @Override + public String getDescription() { + return "Post Status Message"; + } + }; + + private static final PostAction POST_METRICS_ACTION = + new PostAction() { + @Override + public Collector.MessageResult post( + TraceCollectorGrpc.TraceCollectorBlockingStub client, + Collector.MessageRequest request) { + return client.postMetrics(request); + } + + @Override + public String getDescription() { + return "Post Metrics"; + } + }; + + static class GrpcProtocolClientFactory implements ProtocolClientFactory { + private final TrustManagerFactory trustManagerFactory; + private final ProxyConfig proxyConfig = + (ProxyConfig) ConfigManager.getConfig(ConfigProperty.AGENT_PROXY); + private static final String compression; + private static final String DEFAULT_COMPRESSION = "gzip"; + + static { + String compressionString = + ((String) ConfigManager.getConfig(ConfigProperty.AGENT_GRPC_COMPRESSION)); + compression = + compressionString != null ? compressionString.toLowerCase() : DEFAULT_COMPRESSION; + logger.debug("Using compression " + compression + " for gRPC client"); + } + + GrpcProtocolClientFactory(URL serverCertLocation) throws IOException, GeneralSecurityException { + TrustManagerFactory factory = null; + if (serverCertLocation != null) { + // factory = InsecureTrustManagerFactory.INSTANCE; //for testing + try { + factory = SslUtils.getTrustManagerFactory(serverCertLocation); + } catch (GeneralSecurityException e) { + logger.warn( + "Failed to initialize trust manager factory for GRPC client: " + e.getMessage(), e); + } catch (IOException e) { + logger.warn( + "Failed to initialize trust manager factory for GRPC client: " + e.getMessage(), e); + } + } + + this.trustManagerFactory = factory; + } + + @Override + public ProtocolClient buildClient(String host, int port) throws ClientFatalException { + // netty + NettyChannelBuilder channelBuilder = NettyChannelBuilder.forAddress(host, port); + + if (proxyConfig != null) { + channelBuilder = + channelBuilder.proxyDetector( + targetServerAddress -> { + HttpConnectProxiedSocketAddress.Builder builder = + HttpConnectProxiedSocketAddress.newBuilder() + .setProxyAddress( + new InetSocketAddress(proxyConfig.getHost(), proxyConfig.getPort())) + .setTargetAddress((InetSocketAddress) targetServerAddress); + if (proxyConfig.getUsername() != null) { + builder = builder.setUsername(proxyConfig.getUsername()); + } + if (proxyConfig.getPassword() != null) { + builder = builder.setPassword(proxyConfig.getPassword()); + } + return builder.build(); + }); + } + + if (trustManagerFactory != null) { + try { + channelBuilder = + channelBuilder.sslContext( + GrpcSslContexts.forClient().trustManager(trustManagerFactory).build()); + } catch (SSLException e) { + throw new ClientFatalException("Failed to init client SSL"); + } + } + ManagedChannel channel = channelBuilder.build(); + + TraceCollectorGrpc.TraceCollectorBlockingStub stub = + TraceCollectorGrpc.newBlockingStub(channel); + if (!"none".equals(compression)) { + stub = stub.withCompression(compression); + } + + return new GrpcClient(stub); + } + } + + private static class GrpcHostIdManager { + private HostId localHostId; + private Collector.HostID grpcHostID; + private Collector.HostID grpcHostnameOnlyHostID; + private String localHostname; + + private GrpcHostIdManager() {} + + private Collector.HostID getHostID() { + HostId hostId = getHostId(); + boolean loadGrpcHostId; + if (hostId == localHostId || hostId.equals(localHostId)) { + loadGrpcHostId = grpcHostID == null; + } else { + localHostId = hostId; + loadGrpcHostId = true; + } + + if (loadGrpcHostId) { + grpcHostID = toGrpcHostID(localHostId); + } + + return grpcHostID; + } + + private Collector.HostID getHostnameOnlyHostID() { + String hostname = HostInfoUtils.getHostName(); + boolean loadGrpcHostId; + if (hostname.equals(localHostname)) { + loadGrpcHostId = grpcHostnameOnlyHostID == null; + } else { + localHostname = hostname; + loadGrpcHostId = true; + } + + if (loadGrpcHostId) { + grpcHostnameOnlyHostID = toGrpcHostnameOnlyHostID(localHostname); + } + return grpcHostnameOnlyHostID; + } + + private static Collector.HostID toGrpcHostID(HostId hostId) { + Collector.HostID.Builder builder = Collector.HostID.newBuilder(); + setIfNotNull(builder::setHostname, hostId.getHostname()); + setIfNotNull(builder::setEc2InstanceID, hostId.getEc2InstanceId()); + + setIfNotNull(builder::setEc2AvailabilityZone, hostId.getEc2AvailabilityZone()); + setIfNotNull(builder::setDockerContainerID, hostId.getDockerContainerId()); + setIfNotNull(builder::setHerokuDynoID, hostId.getHerokuDynoId()); + + setIfNotNull(builder::setAzAppServiceInstanceID, hostId.getAzureAppServiceInstanceId()); + setIfNotNull(builder::setUamsClientID, hostId.getUamsClientId()); + setIfNotNull(builder::setUuid, hostId.getUuid()); + + if (hostId.getAzureVmMetadata() != null) { + builder.setAzureMetadata(hostId.getAzureVmMetadata().toGrpc()); + } + + if (hostId.getAwsMetadata() != null) { + builder.setAwsMetadata(hostId.getAwsMetadata().toGrpc()); + } + + if (hostId.getK8sMetadata() != null) { + builder.setK8SMetadata(hostId.getK8sMetadata().toGrpc()); + } + + return builder + .setHostType(Collector.HostType.valueOf(hostId.getHostType().name())) + .addAllMacAddresses(hostId.getMacAddresses()) + .setPid(hostId.getPid()) + .build(); + } + + /** + * For getSettings call, we decided to fill in `hostname` only for `HostID` for consistency with + * other agent implementation. + * + *

Unlike other language agent, this change does not give any performance boost to java + * agent. + * + * @return + */ + private static Collector.HostID toGrpcHostnameOnlyHostID(String hostname) { + return Collector.HostID.newBuilder().setHostname(hostname).build(); + } + } +} diff --git a/libs/core/src/main/java/com/solarwinds/joboe/core/rpc/grpc/GrpcClientManager.java b/libs/core/src/main/java/com/solarwinds/joboe/core/rpc/grpc/GrpcClientManager.java new file mode 100644 index 00000000..13b9bf70 --- /dev/null +++ b/libs/core/src/main/java/com/solarwinds/joboe/core/rpc/grpc/GrpcClientManager.java @@ -0,0 +1,93 @@ +/* + * © SolarWinds Worldwide, LLC. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.solarwinds.joboe.core.rpc.grpc; + +import com.solarwinds.joboe.core.rpc.Client; +import com.solarwinds.joboe.core.rpc.RpcClient; +import com.solarwinds.joboe.core.rpc.RpcClientManager; +import com.solarwinds.joboe.logging.Logger; +import com.solarwinds.joboe.logging.LoggerFactory; + +/** + * Manages creation of {@link GrpcClient}. Currently, only 2 gRPC client instances are allowed per + * JVM process - one for Tracing operation and another one for all others + * + * @author pluk + */ +public class GrpcClientManager extends RpcClientManager { + private static final Logger logger = LoggerFactory.getLogger(); + private static Client tracingRpcClient; // one instance for tracing purpose + private static boolean clientsInitialized = false; + private static Client nonTracingRpcClient; // one instance for everything else + + public GrpcClientManager() {} + + @Override + protected Client getClientImpl(OperationType operationType, String serviceKey) { + synchronized (this) { + if (!clientsInitialized) { + initializeClients(serviceKey); + clientsInitialized = true; + } + } + + if (operationType == OperationType.TRACING || operationType == OperationType.PROFILING) { + return tracingRpcClient; + } else { + return nonTracingRpcClient; + } + } + + private void initializeClients(String serviceKey) { + try { + tracingRpcClient = + new RpcClient( + collectorHost, + collectorPort, + serviceKey, + new GrpcClient.GrpcProtocolClientFactory(collectorCertLocation), + RpcClient.TaskType.POST_EVENTS); + } catch (Exception e) { + logger.warn("Failed to initialize rpc tracing client: " + e.getMessage(), e); + } + + try { + nonTracingRpcClient = + new RpcClient( + collectorHost, + collectorPort, + serviceKey, + new GrpcClient.GrpcProtocolClientFactory(collectorCertLocation), + RpcClient.TaskType.GET_SETTINGS, + RpcClient.TaskType.POST_METRICS, + RpcClient.TaskType.POST_STATUS); + } catch (Exception e) { + logger.warn("Failed to initialize rpc non-tracing client: " + e.getMessage(), e); + } + } + + @Override + public void close() { + if (tracingRpcClient != null) { + tracingRpcClient.close(); + } + + if (nonTracingRpcClient != null) { + nonTracingRpcClient.close(); + } + } +} diff --git a/libs/core/src/main/java/com/solarwinds/joboe/core/settings/OboeSettingsException.java b/libs/core/src/main/java/com/solarwinds/joboe/core/settings/OboeSettingsException.java new file mode 100644 index 00000000..52858e4d --- /dev/null +++ b/libs/core/src/main/java/com/solarwinds/joboe/core/settings/OboeSettingsException.java @@ -0,0 +1,34 @@ +/* + * © SolarWinds Worldwide, LLC. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.solarwinds.joboe.core.settings; + +import com.solarwinds.joboe.core.OboeException; + +/** Reports exception arise during settings (rates/bucket) read operations */ +public class OboeSettingsException extends OboeException { + public OboeSettingsException(String msg) { + super(msg); + } + + public OboeSettingsException(String message, Throwable cause) { + super(message, cause); + } + + public OboeSettingsException(Throwable cause) { + super(cause); + } +} diff --git a/libs/core/src/main/java/com/solarwinds/joboe/core/settings/PollingSettingsFetcher.java b/libs/core/src/main/java/com/solarwinds/joboe/core/settings/PollingSettingsFetcher.java new file mode 100644 index 00000000..15535f4b --- /dev/null +++ b/libs/core/src/main/java/com/solarwinds/joboe/core/settings/PollingSettingsFetcher.java @@ -0,0 +1,156 @@ +/* + * © SolarWinds Worldwide, LLC. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.solarwinds.joboe.core.settings; + +import com.solarwinds.joboe.core.util.DaemonThreadFactory; +import com.solarwinds.joboe.logging.Logger; +import com.solarwinds.joboe.logging.LoggerFactory; +import com.solarwinds.joboe.sampling.Settings; +import com.solarwinds.joboe.sampling.SettingsFetcher; +import com.solarwinds.joboe.sampling.SettingsListener; +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; +import java.util.concurrent.Future; +import java.util.concurrent.TimeUnit; + +/** + * A {@link SettingsFetcher} that polls settings from the provided {@link SettingsReader} at a given + * refresh interval + * + *

Every time a {@link Settings} is fetched, the registered listener would be notified + * + * @author pluk + */ +public class PollingSettingsFetcher implements SettingsFetcher { + private static final Logger logger = LoggerFactory.getLogger(); + private static final int DEAFULT_REFRESH_INTERVAL = 30; // 30 secs + + private final int refreshInterval; // pauses between each reader call in seconds + private final SettingsReader reader; + + private final ExecutorService executorService = + Executors.newSingleThreadExecutor(DaemonThreadFactory.newInstance("poll-settings")); + private Settings currentSettings; + + private SettingsListener listener; + private final CountDownLatch firstTryLatch = new CountDownLatch(1); + private boolean active = true; + + public PollingSettingsFetcher(SettingsReader reader) { + this(reader, DEAFULT_REFRESH_INTERVAL); + } + + public PollingSettingsFetcher(SettingsReader reader, int refreshInterval) { + this.reader = reader; + this.refreshInterval = refreshInterval; + startWorker(); + } + + @Override + public Settings getSettings() { + synchronized (this) { + if (isExpired(currentSettings)) { + currentSettings = null; + } + } + return currentSettings; + } + + private boolean isExpired(Settings settings) { + boolean isExpired = false; + if (settings != null) { + long ttlInMillisec = settings.getTtl() * 1000; + isExpired = settings.getTimestamp() + ttlInMillisec < System.currentTimeMillis(); + if (isExpired) { + logger.warn("Settings for [" + settings + "] has expired"); + } + } + return isExpired; + } + + @Override + public CountDownLatch isSettingsAvailableLatch() { + return firstTryLatch; + } + + @Override + public void registerListener(SettingsListener listener) { + this.listener = listener; + } + + @Override + public void close() { + active = false; + executorService.shutdown(); + if (reader != null) { + reader.close(); + } + } + + /** + * Starts background worker to update settings record with refreshInterval sec pauses + * in between + * + * @return + */ + private Future startWorker() { + return executorService.submit( + new Runnable() { + @Override + public void run() { + logger.debug("Starting background worker to update settings"); + boolean running = true; + while (PollingSettingsFetcher.this.active + && running) { // periodically poll for settings + Settings newSettings = null; + try { + newSettings = reader.getSettings(); + } catch (OboeSettingsException e) { + logger.debug( + "Failed to get settings : " + e.getMessage(), + e); // Should not be too noisy as this might happen for intermittent connection + // problem + } + + if (newSettings != null) { + synchronized (this) { + currentSettings = newSettings; + } + + firstTryLatch.countDown(); // count down on the latch to flag that settings is ready + if (listener != null) { + listener.onSettingsRetrieved(newSettings); + } + } else { // purge expired ones + logger.debug("Failed to retrieve settings from rpc call"); + } + + try { + TimeUnit.SECONDS.sleep(refreshInterval); // sleep for the defined interval + } catch (InterruptedException e) { + logger.debug( + PollingSettingsFetcher.class.getName() + + " worker is interrupted : " + + e.getMessage()); + running = false; + } + } + } + }); + } +} diff --git a/libs/core/src/main/java/com/solarwinds/joboe/core/settings/RpcSettingsReader.java b/libs/core/src/main/java/com/solarwinds/joboe/core/settings/RpcSettingsReader.java new file mode 100644 index 00000000..fd46d6c9 --- /dev/null +++ b/libs/core/src/main/java/com/solarwinds/joboe/core/settings/RpcSettingsReader.java @@ -0,0 +1,73 @@ +/* + * © SolarWinds Worldwide, LLC. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.solarwinds.joboe.core.settings; + +import com.solarwinds.joboe.core.rpc.Client; +import com.solarwinds.joboe.core.rpc.ClientLoggingCallback; +import com.solarwinds.joboe.core.rpc.ResultCode; +import com.solarwinds.joboe.core.rpc.SettingsResult; +import com.solarwinds.joboe.logging.Logger; +import com.solarwinds.joboe.logging.LoggerFactory; +import com.solarwinds.joboe.sampling.Settings; + +/** + * Reads {@link Settings} from the source provided by the rpc {@link Client} specified during + * instantiation + */ +public class RpcSettingsReader implements SettingsReader { + private static final Logger logger = LoggerFactory.getLogger(); + + private final Client rpcClient; + private final ClientLoggingCallback loggingCallback = + new ClientLoggingCallback("get service settings"); + + private static final String SSL_CLIENT_VERSION = "2"; + + /** + * @param rpcClient client to use for retrieving settings + */ + public RpcSettingsReader(Client rpcClient) { + this.rpcClient = rpcClient; + } + + /* (non-Javadoc) + * @see com.solarwinds.joboe.core.SettingsReader#getLayerSampleRate(java.lang.String) + */ + @Override + public Settings getSettings() throws OboeSettingsException { + SettingsResult result; + try { + result = rpcClient.getSettings(SSL_CLIENT_VERSION, loggingCallback).get(); + if (result.getResultCode() == ResultCode.OK) { + logger.debug("Got settings from collector: " + result.getSettings().get(0)); + return result.getSettings().get(0); + } + } catch (Exception e) { + throw new OboeSettingsException("Exception from RPC call: " + e.getMessage(), e); + } + + throw new OboeSettingsException( + "Rpc call returns non OK status code: " + result.getResultCode()); + } + + @Override + public void close() { + if (rpcClient != null) { + rpcClient.close(); + } + } +} diff --git a/libs/core/src/main/java/com/solarwinds/joboe/core/settings/SettingsReader.java b/libs/core/src/main/java/com/solarwinds/joboe/core/settings/SettingsReader.java new file mode 100644 index 00000000..0e4153f0 --- /dev/null +++ b/libs/core/src/main/java/com/solarwinds/joboe/core/settings/SettingsReader.java @@ -0,0 +1,30 @@ +/* + * © SolarWinds Worldwide, LLC. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.solarwinds.joboe.core.settings; + +import com.solarwinds.joboe.sampling.Settings; + +/** + * Reads {@link Settings} + * + * @author pluk + */ +public interface SettingsReader { + Settings getSettings() throws OboeSettingsException; + + void close(); +} diff --git a/libs/core/src/main/java/com/solarwinds/joboe/core/settings/SettingsUtil.java b/libs/core/src/main/java/com/solarwinds/joboe/core/settings/SettingsUtil.java new file mode 100644 index 00000000..2740eb2b --- /dev/null +++ b/libs/core/src/main/java/com/solarwinds/joboe/core/settings/SettingsUtil.java @@ -0,0 +1,80 @@ +/* + * © SolarWinds Worldwide, LLC. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.solarwinds.joboe.core.settings; + +import com.google.protobuf.ByteString; +import com.solarwinds.joboe.core.rpc.ResultCode; +import com.solarwinds.joboe.core.rpc.RpcSettings; +import com.solarwinds.joboe.core.rpc.SettingsResult; +import com.solarwinds.joboe.sampling.Settings; +import com.solarwinds.trace.ingestion.proto.Collector; +import java.nio.ByteBuffer; +import java.util.*; +import lombok.AccessLevel; +import lombok.NoArgsConstructor; + +@NoArgsConstructor(access = AccessLevel.PRIVATE) +public final class SettingsUtil { + + public static SettingsResult transformToLocalSettings(Collector.SettingsResult result) { + List settings = new ArrayList<>(); + if (result.getResult() == Collector.ResultCode.OK) { + for (Collector.OboeSetting oboeSetting : result.getSettingsList()) { + settings.add(convertSetting(oboeSetting)); + } + } + + return new SettingsResult( + ResultCode.valueOf(result.getResult().name()), + result.getArg(), + result.getWarning(), + settings); + } + + public static Settings convertSetting(Collector.OboeSetting grpcOboeSetting) { + Map convertedArguments = new HashMap(); + + for (Map.Entry argumentEntry : + grpcOboeSetting.getArgumentsMap().entrySet()) { + convertedArguments.put( + argumentEntry.getKey(), argumentEntry.getValue().asReadOnlyByteBuffer()); + } + + return new RpcSettings( + grpcOboeSetting.getFlags().toStringUtf8(), + System.currentTimeMillis(), // use local timestamp for now, as it is easier to compare ttl + // with it + grpcOboeSetting.getValue(), + grpcOboeSetting.getTtl(), + convertedArguments); + } + + public static short convertType(Collector.OboeSettingType grpcType) { + switch (grpcType) { + case DEFAULT_SAMPLE_RATE: + return Settings.OBOE_SETTINGS_TYPE_DEFAULT_SAMPLE_RATE; + case LAYER_SAMPLE_RATE: + return Settings.OBOE_SETTINGS_TYPE_LAYER_SAMPLE_RATE; + case LAYER_APP_SAMPLE_RATE: + return Settings.OBOE_SETTINGS_TYPE_LAYER_APP_SAMPLE_RATE; + case LAYER_HTTPHOST_SAMPLE_RATE: + return Settings.OBOE_SETTINGS_TYPE_LAYER_HTTPHOST_SAMPLE_RATE; + default: + return -1; + } + } +} diff --git a/libs/core/src/main/java/com/solarwinds/joboe/core/settings/SimpleSettingsFetcher.java b/libs/core/src/main/java/com/solarwinds/joboe/core/settings/SimpleSettingsFetcher.java new file mode 100644 index 00000000..0b5e49d9 --- /dev/null +++ b/libs/core/src/main/java/com/solarwinds/joboe/core/settings/SimpleSettingsFetcher.java @@ -0,0 +1,78 @@ +/* + * © SolarWinds Worldwide, LLC. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.solarwinds.joboe.core.settings; + +import com.solarwinds.joboe.sampling.Settings; +import com.solarwinds.joboe.sampling.SettingsFetcher; +import com.solarwinds.joboe.sampling.SettingsListener; +import java.util.concurrent.CountDownLatch; + +/** + * A testing fetcher that returns {@link Settings} by directly reads from the {@link + * TestSettingsReader} provided in constructor + * + *

Notifies {@link SettingsListener} whenever the reader has settings changes + * + * @author pluk + */ +public class SimpleSettingsFetcher implements SettingsFetcher { + private final TestSettingsReader reader; + private SettingsListener listener; + + public SimpleSettingsFetcher(TestSettingsReader reader) { + this.reader = reader; + + reader.onSettingsChanged(() -> fetch()); + } + + @Override + public Settings getSettings() { + try { + return reader.getSettings(); + } catch (OboeSettingsException e) { + return null; + } + } + + @Override + public void registerListener(SettingsListener listener) { + this.listener = listener; + } + + private void fetch() { + if (listener != null) { + Settings newSettings = getSettings(); + listener.onSettingsRetrieved(newSettings); + } + } + + @Override + public CountDownLatch isSettingsAvailableLatch() { + return new CountDownLatch(0); + } + + @Override + public void close() { + if (reader != null) { + reader.close(); + } + } + + public SettingsReader getReader() { + return reader; + } +} diff --git a/libs/core/src/main/java/com/solarwinds/joboe/core/settings/TestSettingsReader.java b/libs/core/src/main/java/com/solarwinds/joboe/core/settings/TestSettingsReader.java new file mode 100644 index 00000000..f641a270 --- /dev/null +++ b/libs/core/src/main/java/com/solarwinds/joboe/core/settings/TestSettingsReader.java @@ -0,0 +1,218 @@ +/* + * © SolarWinds Worldwide, LLC. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.solarwinds.joboe.core.settings; + +import com.solarwinds.joboe.sampling.Settings; +import com.solarwinds.joboe.sampling.SettingsArg; +import com.solarwinds.joboe.sampling.TracingMode; +import java.util.HashMap; +import java.util.Map; + +public class TestSettingsReader implements SettingsReader { + private volatile Settings layerSettings; + private SettingsChangeCallback settingsChangeCallback; + + @Override + public Settings getSettings() throws OboeSettingsException { + return layerSettings; + } + + public void put(Settings settings) { + layerSettings = settings; + settingsChanged(); + } + + public void setAll(Settings allSettings) { + layerSettings = allSettings; + settingsChanged(); + } + + public void reset() { + layerSettings = null; + settingsChanged(); + } + + @Override + public void close() {} + + /** + * Registers Callback on settings changed + * + * @param settingsChangeCallback + */ + public void onSettingsChanged(SettingsChangeCallback settingsChangeCallback) { + this.settingsChangeCallback = settingsChangeCallback; + } + + private void settingsChanged() { + if (settingsChangeCallback != null) { + settingsChangeCallback.settingsChanged(); + } + } + + public interface SettingsChangeCallback { + void settingsChanged(); + } + + public static class SettingsMockupBuilder { + private short flags = 0; + private Integer sampleRate = null; + private final Map, Object> args = new HashMap, Object>(); + + public static final double DEFAULT_TOKEN_BUCKET_RATE = 8.0; + public static final double DEFAULT_TOKEN_BUCKET_CAPACITY = 16.0; + private short settingsType = Settings.OBOE_SETTINGS_TYPE_DEFAULT_SAMPLE_RATE; + + public SettingsMockupBuilder() { + addDefaultArgs(); + } + + private void addDefaultArgs() { + args.put(SettingsArg.BUCKET_RATE, DEFAULT_TOKEN_BUCKET_RATE); + args.put(SettingsArg.BUCKET_CAPACITY, DEFAULT_TOKEN_BUCKET_CAPACITY); + args.put(SettingsArg.RELAXED_BUCKET_RATE, DEFAULT_TOKEN_BUCKET_RATE); + args.put(SettingsArg.RELAXED_BUCKET_CAPACITY, DEFAULT_TOKEN_BUCKET_CAPACITY); + args.put(SettingsArg.STRICT_BUCKET_RATE, DEFAULT_TOKEN_BUCKET_RATE); + args.put(SettingsArg.STRICT_BUCKET_CAPACITY, DEFAULT_TOKEN_BUCKET_CAPACITY); + } + + public SettingsMockupBuilder withFlags( + boolean isStart, + boolean isThrough, + boolean isThroughAlways, + boolean isTriggerTraceEnabled, + boolean isOverride) { + flags = getFlags(isStart, isThrough, isThroughAlways, isTriggerTraceEnabled, isOverride); + return this; + } + + public SettingsMockupBuilder withFlags(TracingMode tracingMode) { + flags |= tracingMode.toFlags(); + return this; + } + + public SettingsMockupBuilder withSampleRate(int sampleRate) { + this.sampleRate = sampleRate; + return this; + } + + public SettingsMockupBuilder withSettingsArg(SettingsArg arg, T value) { + args.put(arg, value); + return this; + } + + public SettingsMockupBuilder withSettingsArgs(Map, ?> args) { + this.args.clear(); + this.args.putAll(args); + return this; + } + + public SettingsMockupBuilder withSettingsType(short settingsType) { + this.settingsType = settingsType; + return this; + } + + public SettingsMockup build() { + return new SettingsMockup(settingsType, flags, sampleRate, args); + } + + private static short getFlags( + boolean isStart, + boolean isThrough, + boolean isThroughAlways, + boolean isTriggerTraceEnabled, + boolean isOverride) { + byte flags = 0; + if (isOverride) { + flags |= Settings.OBOE_SETTINGS_FLAG_OVERRIDE; + } + + if (isStart) { + flags |= Settings.OBOE_SETTINGS_FLAG_SAMPLE_START; + } + + if (isThrough) { + flags |= Settings.OBOE_SETTINGS_FLAG_SAMPLE_THROUGH; + } + + if (isThroughAlways) { + flags |= Settings.OBOE_SETTINGS_FLAG_SAMPLE_THROUGH_ALWAYS; + } + + if (isTriggerTraceEnabled) { + flags |= Settings.OBOE_SETTINGS_FLAG_TRIGGER_TRACE_ENABLED; + } + + return flags; + } + } + + /** + * A mockup of Settings. This is necessary to alter the behavior of LayerUtil on the fly without + * modifying the internal logic flow + * + * @author Patson Luk + * @see Settings + */ + public static class SettingsMockup extends Settings { + private final short settingsType; + private short flags = 0; + private int sampleRate = 1000000; + private final Map, Object> args; + + private SettingsMockup( + short settingsType, short flags, Integer sampleRate, Map, Object> args) { + this.settingsType = settingsType; + this.flags = flags; + if (sampleRate != null) { + this.sampleRate = sampleRate; + } + this.args = args; + } + + @Override + public long getValue() { + return sampleRate; + } + + @Override + public long getTimestamp() { + return 0; + } + + @Override + public short getFlags() { + return flags; + } + + @Override + @SuppressWarnings("unchecked") + public T getArgValue(SettingsArg arg) { + return (T) args.get(arg); + } + + @Override + public short getType() { + return settingsType; + } + + @Override + public long getTtl() { + return 0; + } + } +} diff --git a/libs/core/src/main/java/com/solarwinds/joboe/core/util/AzureInstanceIdReader.java b/libs/core/src/main/java/com/solarwinds/joboe/core/util/AzureInstanceIdReader.java new file mode 100644 index 00000000..8bb327d9 --- /dev/null +++ b/libs/core/src/main/java/com/solarwinds/joboe/core/util/AzureInstanceIdReader.java @@ -0,0 +1,21 @@ +/* + * © SolarWinds Worldwide, LLC. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.solarwinds.joboe.core.util; + +public interface AzureInstanceIdReader { + String getAzureInstanceId(); +} diff --git a/libs/core/src/main/java/com/solarwinds/joboe/core/util/BackTraceCache.java b/libs/core/src/main/java/com/solarwinds/joboe/core/util/BackTraceCache.java new file mode 100644 index 00000000..96f270c4 --- /dev/null +++ b/libs/core/src/main/java/com/solarwinds/joboe/core/util/BackTraceCache.java @@ -0,0 +1,57 @@ +/* + * © SolarWinds Worldwide, LLC. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.solarwinds.joboe.core.util; + +import com.google.common.cache.Cache; +import com.google.common.cache.CacheBuilder; +import java.util.List; +import java.util.concurrent.TimeUnit; + +public class BackTraceCache { + private static Cache, String> backTraceCache = + null; // delay building cache as it causes problem in jboss see + // https://github.com/librato/joboe/issues/594 + private static volatile boolean enabled = false; + + static String getBackTraceString(List stackTrace) { + Cache, String> cache = getCache(); + return cache != null ? cache.getIfPresent(stackTrace) : null; + } + + static void putBackTraceString(List stackTrace, String stackTraceString) { + Cache, String> cache = getCache(); + if (cache != null) { + cache.put(stackTrace, stackTraceString); + } + } + + public static final void enable() { + enabled = true; + } + + private static Cache, String> getCache() { + if (backTraceCache == null && enabled) { + backTraceCache = + CacheBuilder.newBuilder() + .maximumSize(20) + .expireAfterAccess(3600, TimeUnit.SECONDS) + .build(); // 1 hour cache + } + + return backTraceCache; + } +} diff --git a/libs/core/src/main/java/com/solarwinds/joboe/core/util/BackTraceUtil.java b/libs/core/src/main/java/com/solarwinds/joboe/core/util/BackTraceUtil.java new file mode 100644 index 00000000..efa68208 --- /dev/null +++ b/libs/core/src/main/java/com/solarwinds/joboe/core/util/BackTraceUtil.java @@ -0,0 +1,101 @@ +/* + * © SolarWinds Worldwide, LLC. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.solarwinds.joboe.core.util; + +import com.solarwinds.joboe.core.Constants; +import com.solarwinds.joboe.logging.Logger; +import com.solarwinds.joboe.logging.LoggerFactory; +import java.util.Arrays; +import java.util.List; + +public class BackTraceUtil { + private static final Logger logger = LoggerFactory.getLogger(); + + public static StackTraceElement[] getBackTrace(int skipElements) { + StackTraceElement[] stackTrace = Thread.currentThread().getStackTrace(); + + int startPosition = + 2 + skipElements; // Starts with 2: To exclude the getStackTrace() and addBackTrace calls + // themselves. Also adds the number of skipElements provided in the + // argument to skip elements + + if (startPosition >= stackTrace.length) { + logger.debug( + "Attempt to skip [" + + skipElements + + "] elements in addBackTrace is invalid, no stack trace element is left!"); + return new StackTraceElement[0]; + } + + int targetStackTraceLength = stackTrace.length - startPosition; + StackTraceElement[] targetStackTrace = new StackTraceElement[targetStackTraceLength]; + System.arraycopy(stackTrace, startPosition, targetStackTrace, 0, targetStackTraceLength); + + return targetStackTrace; + } + + public static String backTraceToString(StackTraceElement[] stackTrace) { + List wrappedStackTrace = + Arrays.asList(stackTrace); // wrap it so hashCode and equals work + + String cachedValue = BackTraceCache.getBackTraceString(wrappedStackTrace); + if (cachedValue != null) { + return cachedValue; + } + + StringBuffer st = new StringBuffer(); + + if (stackTrace.length + > Constants.MAX_BACK_TRACE_LINE_COUNT) { // then we will have to skip some lines + appendStackTrace( + stackTrace, 0, Constants.MAX_BACK_TRACE_TOP_LINE_COUNT, st); // add the top lines + + st.append( + "...Skipped " + (stackTrace.length - Constants.MAX_BACK_TRACE_LINE_COUNT) + " line(s)\n"); + + appendStackTrace( + stackTrace, + stackTrace.length - Constants.MAX_BACK_TRACE_BOTTOM_LINE_COUNT, + Constants.MAX_BACK_TRACE_BOTTOM_LINE_COUNT, + st); // add the bottom lines + } else { + appendStackTrace(stackTrace, 0, stackTrace.length, st); // add everything + } + + String value = st.toString(); + + BackTraceCache.putBackTraceString(wrappedStackTrace, value); + + return value; + } + + /** + * Build the stackTrace output and append the result to the buffer provided + * + * @param stackTrace The source of the stack trace array + * @param startPosition + * @param lineCount + * @param buffer The buffer that stores the result + */ + private static void appendStackTrace( + StackTraceElement[] stackTrace, int startPosition, int lineCount, StringBuffer buffer) { + for (int i = startPosition; i < startPosition + lineCount && i < stackTrace.length; i++) { + buffer.append(stackTrace[i].toString()); + buffer.append("\n"); + } + } +} diff --git a/libs/core/src/main/java/com/solarwinds/joboe/core/util/BsonUtils.java b/libs/core/src/main/java/com/solarwinds/joboe/core/util/BsonUtils.java new file mode 100644 index 00000000..edc68544 --- /dev/null +++ b/libs/core/src/main/java/com/solarwinds/joboe/core/util/BsonUtils.java @@ -0,0 +1,71 @@ +/* + * © SolarWinds Worldwide, LLC. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.solarwinds.joboe.core.util; + +import com.solarwinds.joboe.core.BsonBufferException; +import com.solarwinds.joboe.core.ebson.BsonDocument; +import com.solarwinds.joboe.core.ebson.BsonDocuments; +import com.solarwinds.joboe.core.ebson.BsonToken; +import com.solarwinds.joboe.core.ebson.BsonWriter; +import java.nio.BufferOverflowException; +import java.nio.ByteBuffer; +import java.nio.ByteOrder; +import java.util.Map; + +public class BsonUtils { + private BsonUtils() {} + + public static ByteBuffer convertMapToBson( + Map map, int bufferSize, int maxBufferSize) throws BsonBufferException { + BsonDocument doc = BsonDocuments.copyOf(map); + BsonWriter writer = BsonToken.DOCUMENT.writer(); + ByteBuffer buffer = ByteBuffer.allocate(bufferSize).order(ByteOrder.LITTLE_ENDIAN); + + RuntimeException bufferException; + try { + writer.writeTo(buffer, doc); + buffer.flip(); // cast for JDK 8- runtime compatibility + return buffer; + } catch (BufferOverflowException e) { // cannot use multi-catch due to 1.6 compatibility + bufferException = e; + } catch ( + IllegalArgumentException + e) { // IllegalArumgnentException could be thrown from BsonWriter.DOCUMENT.writeto if + // buffer position is greater than limit - 4 + bufferException = e; + } + + if (bufferException != null) { + buffer.clear(); // cast for JDK 8- runtime compatibility + if (bufferSize * 2 <= maxBufferSize) { + return convertMapToBson(map, bufferSize * 2, maxBufferSize); + } else { + throw new BsonBufferException(bufferException); + } + } else { + // shouldn't really run to here base on current logic flow, but just to play safe + return buffer; + } + } + + @SuppressWarnings("unchecked") + public static Map convertBsonToMap(ByteBuffer byteBuffer) { + Object o = BsonToken.DOCUMENT.reader().readFrom(byteBuffer); + + return (Map) o; + } +} diff --git a/libs/core/src/main/java/com/solarwinds/joboe/core/util/DaemonThreadFactory.java b/libs/core/src/main/java/com/solarwinds/joboe/core/util/DaemonThreadFactory.java new file mode 100644 index 00000000..0dd769a5 --- /dev/null +++ b/libs/core/src/main/java/com/solarwinds/joboe/core/util/DaemonThreadFactory.java @@ -0,0 +1,58 @@ +/* + * © SolarWinds Worldwide, LLC. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.solarwinds.joboe.core.util; + +import com.solarwinds.joboe.logging.Logger; +import com.solarwinds.joboe.logging.LoggerFactory; +import java.util.concurrent.Executors; +import java.util.concurrent.ThreadFactory; +import java.util.concurrent.atomic.AtomicInteger; + +public class DaemonThreadFactory implements ThreadFactory { + private static final Logger logger = LoggerFactory.getLogger(); + private final String threadName; + private static final String THREAD_NAME_PREFIX = "SolarwindsAPM"; + private final AtomicInteger count = new AtomicInteger(0); + + private DaemonThreadFactory(String threadName) { + this.threadName = + threadName != null ? THREAD_NAME_PREFIX + "-" + threadName : THREAD_NAME_PREFIX; + } + + public static DaemonThreadFactory newInstance(String threadName) { + return new DaemonThreadFactory(threadName); + } + + @Override + public Thread newThread(Runnable r) { + Thread t = Executors.defaultThreadFactory().newThread(r); + t.setDaemon(true); + t.setName(threadName + "-" + count.incrementAndGet()); + try { + // Set contextClassLoader to null to avoid memory leak error message during tomcat shutdown + // see http://wiki.apache.org/tomcat/MemoryLeakProtection#cclThreadSpawnedByWebApp + // It is ok to set it to null as we do not need servlet container class loader for spawned + // thread as they should only reference core sdk code or classes included in the agent jar + t.setContextClassLoader(null); + } catch (SecurityException e) { + logger.warn( + "Cannot set the context class loader of System Monitor threads to null. Tomcat might display warning message of memory leak during shutdown"); + } + + return t; + } +} diff --git a/libs/core/src/main/java/com/solarwinds/joboe/core/util/DummyHostInfoReader.java b/libs/core/src/main/java/com/solarwinds/joboe/core/util/DummyHostInfoReader.java new file mode 100644 index 00000000..e4c407fb --- /dev/null +++ b/libs/core/src/main/java/com/solarwinds/joboe/core/util/DummyHostInfoReader.java @@ -0,0 +1,32 @@ +/* + * © SolarWinds Worldwide, LLC. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.solarwinds.joboe.core.util; + +import com.solarwinds.joboe.core.HostId; + +public class DummyHostInfoReader implements HostInfoReader, HostNameReader { + + @Override + public String getHostName() { + return ""; + } + + @Override + public HostId getHostId() { + return HostId.builder().hostname(getHostName()).build(); + } +} diff --git a/libs/core/src/main/java/com/solarwinds/joboe/core/util/ExecUtils.java b/libs/core/src/main/java/com/solarwinds/joboe/core/util/ExecUtils.java new file mode 100644 index 00000000..ef7d74ee --- /dev/null +++ b/libs/core/src/main/java/com/solarwinds/joboe/core/util/ExecUtils.java @@ -0,0 +1,117 @@ +/* + * © SolarWinds Worldwide, LLC. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.solarwinds.joboe.core.util; + +import com.solarwinds.joboe.logging.Logger; +import com.solarwinds.joboe.logging.LoggerFactory; +import java.io.BufferedReader; +import java.io.IOException; +import java.io.InputStream; +import java.io.InputStreamReader; +import java.util.concurrent.*; + +public class ExecUtils { + private static final Logger logger = LoggerFactory.getLogger(); + private static final int EXEC_TIMEOUT = 5000; // 5 seconds for exec + + /** + * Executes the command and returns the result as string + * + * @param command + * @throws Exception if the exec command failed to execute + * @return + */ + public static String exec(String command, String newLine) throws Exception { + ExecutorService executorService = + Executors.newCachedThreadPool(DaemonThreadFactory.newInstance("exec")); + + try { + Process process = Runtime.getRuntime().exec(command); + process + .getOutputStream() + .close(); // to indicate that we do not want to write anything to the process input, for + // powershell this is necessary otherwise it will hang + + Future errorStreamFuture = + executorService.submit(new ReadStreamCallable(process.getErrorStream(), newLine)); + Future inputStreamFuture = + executorService.submit(new ReadStreamCallable(process.getInputStream(), newLine)); + + String errorResult = errorStreamFuture.get(EXEC_TIMEOUT, TimeUnit.SECONDS); + String standardResult = inputStreamFuture.get(EXEC_TIMEOUT, TimeUnit.SECONDS); + + if (!errorResult.isEmpty()) { + logger.debug("exec " + command + " output to error stream : " + errorResult); + } + + return standardResult; + } finally { + executorService.shutdown(); + } + } + + public static String exec(String command) throws Exception { + return exec(command, null); + } + + private static class ReadStreamCallable implements Callable { + private final InputStream inputStream; + private String newLine; + + private ReadStreamCallable(InputStream inputStream) { + this.inputStream = inputStream; + } + + private ReadStreamCallable(InputStream inputStream, String newLine) { + this.inputStream = inputStream; + this.newLine = newLine; + } + + @Override + public String call() { + BufferedReader bufferedReader = null; + try { + bufferedReader = new BufferedReader(new InputStreamReader(inputStream)); + String line; + StringBuilder stringBuilder = new StringBuilder(); + + while ((line = bufferedReader.readLine()) != null) { + if (line.isEmpty()) { + continue; + } + stringBuilder.append(line); + if (newLine != null) { + stringBuilder.append(newLine); + } + } + + return stringBuilder.toString(); + } catch (IOException ex) { + logger.warn("exec failed with: " + ex.getMessage()); + return null; + } finally { + if (bufferedReader != null) { + try { + bufferedReader.close(); + } catch (IOException e) { + logger.warn("Failed to close reader for exec"); + } + } + } + } + } +} diff --git a/libs/core/src/main/java/com/solarwinds/joboe/core/util/HeartbeatSchedulerProvider.java b/libs/core/src/main/java/com/solarwinds/joboe/core/util/HeartbeatSchedulerProvider.java new file mode 100644 index 00000000..8e2e1ff3 --- /dev/null +++ b/libs/core/src/main/java/com/solarwinds/joboe/core/util/HeartbeatSchedulerProvider.java @@ -0,0 +1,29 @@ +/* + * © SolarWinds Worldwide, LLC. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.solarwinds.joboe.core.util; + +import com.solarwinds.joboe.core.rpc.HeartbeatScheduler; +import com.solarwinds.joboe.core.rpc.KeepAliveMonitor; +import com.solarwinds.joboe.core.rpc.ProtocolClient; +import java.util.function.Supplier; + +public class HeartbeatSchedulerProvider { + public static HeartbeatScheduler createHeartbeatScheduler( + Supplier protocolClient, String serviceKey, Object lock) { + return new KeepAliveMonitor(protocolClient, serviceKey, lock); + } +} diff --git a/libs/core/src/main/java/com/solarwinds/joboe/core/util/HostInfoReader.java b/libs/core/src/main/java/com/solarwinds/joboe/core/util/HostInfoReader.java new file mode 100644 index 00000000..5c40af90 --- /dev/null +++ b/libs/core/src/main/java/com/solarwinds/joboe/core/util/HostInfoReader.java @@ -0,0 +1,23 @@ +/* + * © SolarWinds Worldwide, LLC. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.solarwinds.joboe.core.util; + +import com.solarwinds.joboe.core.HostId; + +public interface HostInfoReader { + HostId getHostId(); +} diff --git a/libs/core/src/main/java/com/solarwinds/joboe/core/util/HostInfoReaderProvider.java b/libs/core/src/main/java/com/solarwinds/joboe/core/util/HostInfoReaderProvider.java new file mode 100644 index 00000000..e01a10a7 --- /dev/null +++ b/libs/core/src/main/java/com/solarwinds/joboe/core/util/HostInfoReaderProvider.java @@ -0,0 +1,25 @@ +/* + * © SolarWinds Worldwide, LLC. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.solarwinds.joboe.core.util; + +/** + * The provider which offers concrete instance of the HostInfoReader interface. This keeps service + * loader away from instantiating the concrete HostInfoReader classes. + */ +public interface HostInfoReaderProvider { + HostInfoReader getHostInfoReader(); +} diff --git a/libs/core/src/main/java/com/solarwinds/joboe/core/util/HostInfoUtils.java b/libs/core/src/main/java/com/solarwinds/joboe/core/util/HostInfoUtils.java new file mode 100644 index 00000000..07178d9a --- /dev/null +++ b/libs/core/src/main/java/com/solarwinds/joboe/core/util/HostInfoUtils.java @@ -0,0 +1,130 @@ +/* + * © SolarWinds Worldwide, LLC. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.solarwinds.joboe.core.util; + +import com.solarwinds.joboe.core.HostId; +import com.solarwinds.joboe.logging.Logger; +import com.solarwinds.joboe.logging.LoggerFactory; +import java.util.*; +import lombok.Getter; + +/** + * Helper to extract information on the host this JVM runs on + * + * @author pluk + */ +public class HostInfoUtils { + private static final Logger logger = LoggerFactory.getLogger(); + private static HostInfoReader reader; + + static { + for (HostInfoReaderProvider provider : ServiceLoader.load(HostInfoReaderProvider.class)) { + logger.debug("Use HostInfoReaderProvider implementation " + provider.getClass().getName()); + HostInfoUtils.reader = provider.getHostInfoReader(); + break; // use the first implementation found in the path + } + if (HostInfoUtils.reader == null) { + reader = new DummyHostInfoReader(); + } + } + + public static String getAzureInstanceId() { + if (reader instanceof AzureInstanceIdReader) + return ((AzureInstanceIdReader) reader).getAzureInstanceId(); + return "unknown-azure-instance"; + } + + @Getter + public enum OsType { + LINUX("Linux"), + WINDOWS("Windows"), + MAC("Mac"), + UNKNOWN("Unknown"); + private final String label; + + OsType(String label) { + this.label = label; + } + } + + private HostInfoUtils() { + // prevent instantiations + } + + public static OsType getOsType() { + String osName = System.getProperty("os.name"); + if (osName == null) { + logger.warn("Failed to identify OS, system property `os.name` is null"); + return OsType.UNKNOWN; + } else if (osName.toLowerCase().startsWith("windows")) { + return OsType.WINDOWS; + } else if (osName.toLowerCase().startsWith("linux")) { + return OsType.LINUX; + } else if (osName.toLowerCase().startsWith("mac")) { + return OsType.MAC; + } else { + logger.info("Unsupported OS type detected: " + osName); + return OsType.UNKNOWN; + } + } + + public static void init(HostInfoReader reader) { + HostInfoUtils.reader = reader; + } + + public static String getHostName() { + if (reader instanceof HostNameReader) { + return ((HostNameReader) reader).getHostName(); + } + return "unknown-host"; + } + + public static HostId getHostId() { + return reader.getHostId(); + } + + /** + * Get a map of host information + * + * @return + */ + public static Map getHostMetadata() { + if (reader instanceof HostMetadataReader) { + return ((HostMetadataReader) reader).getHostMetadata(); + } + return new HashMap<>(); + } + + public static NetworkAddressInfo getNetworkAddressInfo() { + if (reader instanceof NetworkAddressInfoReader) { + return ((NetworkAddressInfoReader) reader).getNetworkAddressInfo(); + } + return new NetworkAddressInfo(Collections.emptyList(), Collections.emptyList()); + } + + @Getter + public static class NetworkAddressInfo { + public NetworkAddressInfo(List ipAddresses, List macAddresses) { + super(); + this.ipAddresses = ipAddresses; + this.macAddresses = macAddresses; + } + + private final List ipAddresses; + private final List macAddresses; + } +} diff --git a/libs/core/src/main/java/com/solarwinds/joboe/core/util/HostMetadataReader.java b/libs/core/src/main/java/com/solarwinds/joboe/core/util/HostMetadataReader.java new file mode 100644 index 00000000..53694540 --- /dev/null +++ b/libs/core/src/main/java/com/solarwinds/joboe/core/util/HostMetadataReader.java @@ -0,0 +1,23 @@ +/* + * © SolarWinds Worldwide, LLC. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.solarwinds.joboe.core.util; + +import java.util.Map; + +public interface HostMetadataReader { + Map getHostMetadata(); +} diff --git a/libs/core/src/main/java/com/solarwinds/joboe/core/util/HostNameReader.java b/libs/core/src/main/java/com/solarwinds/joboe/core/util/HostNameReader.java new file mode 100644 index 00000000..6f5de4cc --- /dev/null +++ b/libs/core/src/main/java/com/solarwinds/joboe/core/util/HostNameReader.java @@ -0,0 +1,21 @@ +/* + * © SolarWinds Worldwide, LLC. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.solarwinds.joboe.core.util; + +public interface HostNameReader { + String getHostName(); +} diff --git a/libs/core/src/main/java/com/solarwinds/joboe/core/util/HttpUtils.java b/libs/core/src/main/java/com/solarwinds/joboe/core/util/HttpUtils.java new file mode 100644 index 00000000..15599f78 --- /dev/null +++ b/libs/core/src/main/java/com/solarwinds/joboe/core/util/HttpUtils.java @@ -0,0 +1,46 @@ +/* + * © SolarWinds Worldwide, LLC. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.solarwinds.joboe.core.util; + +public final class HttpUtils { + private HttpUtils() {} // no instantiation on this class is allowed + + public static String trimQueryParameters(String uri) { + if (uri == null) { + return null; + } + + int querySeparatorPosition = uri.indexOf('?'); + if (querySeparatorPosition == -1) { + return uri; + } else { + return uri.substring(0, querySeparatorPosition); + } + } + + public static boolean isServerErrorStatusCode(int statusCode) { + return statusCode / 100 == 5; // all 5xx + } + + public static boolean isClientErrorStatusCode(int statusCode) { + return statusCode / 100 == 4; // all 4xx + } + + public static boolean isErrorStatusCode(int statusCode) { + return isServerErrorStatusCode(statusCode) || isClientErrorStatusCode(statusCode); + } +} diff --git a/libs/core/src/main/java/com/solarwinds/joboe/core/util/JavaProcessUtils.java b/libs/core/src/main/java/com/solarwinds/joboe/core/util/JavaProcessUtils.java new file mode 100644 index 00000000..aa13bbfd --- /dev/null +++ b/libs/core/src/main/java/com/solarwinds/joboe/core/util/JavaProcessUtils.java @@ -0,0 +1,55 @@ +/* + * © SolarWinds Worldwide, LLC. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.solarwinds.joboe.core.util; + +import java.lang.management.ManagementFactory; + +/** + * Helper to extract information on the running JVM process + * + * @author pluk + */ +public final class JavaProcessUtils { + private static Integer pid = null; + + private JavaProcessUtils() { + // prevent instantiations + } + + /** + * Copied from Event.getPID() Retrieves PID from current Java process + * + * @return + */ + public static int getPid() { + // You'd think getting the PID would be simple: + // http://arhipov.blogspot.com/2011/01/java-snippet-get-pid-at-runtime.html + if (pid == null) { + String nameOfRunningVM = ManagementFactory.getRuntimeMXBean().getName(); + int p = nameOfRunningVM.indexOf('@'); + String pidStr = nameOfRunningVM.substring(0, p); + try { + pid = Integer.parseInt(pidStr); + } catch (NumberFormatException ex) { + // Shouldn't happen + pid = -1; + } + } + + return pid; + } +} diff --git a/libs/core/src/main/java/com/solarwinds/joboe/core/util/NetworkAddressInfoReader.java b/libs/core/src/main/java/com/solarwinds/joboe/core/util/NetworkAddressInfoReader.java new file mode 100644 index 00000000..e2d1cf17 --- /dev/null +++ b/libs/core/src/main/java/com/solarwinds/joboe/core/util/NetworkAddressInfoReader.java @@ -0,0 +1,21 @@ +/* + * © SolarWinds Worldwide, LLC. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.solarwinds.joboe.core.util; + +public interface NetworkAddressInfoReader { + HostInfoUtils.NetworkAddressInfo getNetworkAddressInfo(); +} diff --git a/libs/core/src/main/java/com/solarwinds/joboe/core/util/RuntimeHostInfoReaderProvider.java b/libs/core/src/main/java/com/solarwinds/joboe/core/util/RuntimeHostInfoReaderProvider.java new file mode 100644 index 00000000..47abec6b --- /dev/null +++ b/libs/core/src/main/java/com/solarwinds/joboe/core/util/RuntimeHostInfoReaderProvider.java @@ -0,0 +1,27 @@ +/* + * © SolarWinds Worldwide, LLC. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.solarwinds.joboe.core.util; + +import com.google.auto.service.AutoService; + +@AutoService(HostInfoReaderProvider.class) +public class RuntimeHostInfoReaderProvider implements HostInfoReaderProvider { + @Override + public HostInfoReader getHostInfoReader() { + return ServerHostInfoReader.INSTANCE; + } +} diff --git a/libs/core/src/main/java/com/solarwinds/joboe/core/util/ServerHostInfoReader.java b/libs/core/src/main/java/com/solarwinds/joboe/core/util/ServerHostInfoReader.java new file mode 100644 index 00000000..319acd3e --- /dev/null +++ b/libs/core/src/main/java/com/solarwinds/joboe/core/util/ServerHostInfoReader.java @@ -0,0 +1,1290 @@ +/* + * © SolarWinds Worldwide, LLC. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.solarwinds.joboe.core.util; + +import com.solarwinds.joboe.config.ConfigManager; +import com.solarwinds.joboe.config.ConfigProperty; +import com.solarwinds.joboe.core.Context; +import com.solarwinds.joboe.core.HostId; +import com.solarwinds.joboe.logging.Logger; +import com.solarwinds.joboe.logging.LoggerFactory; +import com.solarwinds.joboe.sampling.Metadata; +import io.opentelemetry.api.internal.InstrumentationUtil; +import java.io.BufferedReader; +import java.io.File; +import java.io.FileNotFoundException; +import java.io.FileReader; +import java.io.IOException; +import java.io.InputStreamReader; +import java.io.OutputStream; +import java.lang.management.ManagementFactory; +import java.net.HttpURLConnection; +import java.net.InetAddress; +import java.net.NetworkInterface; +import java.net.Proxy; +import java.net.SocketException; +import java.net.URL; +import java.net.UnknownHostException; +import java.nio.charset.StandardCharsets; +import java.nio.file.Files; +import java.nio.file.Paths; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collections; +import java.util.Enumeration; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.Optional; +import java.util.UUID; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.Executors; +import java.util.concurrent.ScheduledExecutorService; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicReference; +import java.util.function.Consumer; +import java.util.function.Function; +import java.util.function.Supplier; +import java.util.regex.Matcher; +import java.util.regex.Pattern; +import java.util.stream.Stream; +import lombok.Getter; +import org.json.JSONException; + +/** + * Helper to extract information on the host this JVM runs on + * + * @author pluk + */ +public class ServerHostInfoReader + implements HostInfoReader, + AzureInstanceIdReader, + HostMetadataReader, + NetworkAddressInfoReader, + HostNameReader { + public static final ServerHostInfoReader INSTANCE = new ServerHostInfoReader(); + private static final Logger logger = LoggerFactory.getLogger(); + + private static final int HOST_ID_CHECK_INTERVAL = 60; + + private static final int TIMEOUT_DEFAULT = 1000; + + private static String distro; + private static HostId hostId; // lazily initialized to avoid cyclic init + private static final String hostname = loadHostName(); + private static String uuid; + private static boolean checkedDistro = false; + static HostInfoUtils.OsType osType = HostInfoUtils.getOsType(); + + static final String HOSTNAME_SEPARATOR = ";"; + + enum DistroType { + AMAZON, + UBUNTU, + DEBIAN, + REDHAT_BASED, + SUSE, + SLACKWARE, + GENTOO + } + + static final Map DISTRO_FILE_NAMES = new HashMap(); + + public static final String HOSTNAME_ALIAS_KEY = "ConfiguredHostname"; + + private static final String METADATA_SERVICE_URL = "http://169.254.169.254"; + + static { + DISTRO_FILE_NAMES.put(DistroType.AMAZON, "/etc/system-release-cpe"); + DISTRO_FILE_NAMES.put(DistroType.REDHAT_BASED, "/etc/redhat-release"); + DISTRO_FILE_NAMES.put(DistroType.UBUNTU, "/etc/lsb-release"); + DISTRO_FILE_NAMES.put(DistroType.DEBIAN, "/etc/debian_version"); + DISTRO_FILE_NAMES.put(DistroType.SUSE, "/etc/SuSE-release"); + DISTRO_FILE_NAMES.put(DistroType.SLACKWARE, "/etc/slackware-version"); + DISTRO_FILE_NAMES.put(DistroType.GENTOO, "/etc/gentoo-release"); + } + + private ServerHostInfoReader() { + // prevent instantiations + } + + public String getAwsInstanceId() { + return Ec2InstanceReader.getInstanceId(); + } + + public String getAwsAvailabilityZone() { + return Ec2InstanceReader.getAvailabilityZone(); + } + + public String getDockerContainerId() { + return DockerInfoReader.getDockerId(); + } + + public String getHerokuDynoId() { + return HerokuDynoReader.getDynoId(); + } + + public String getUuid() { + if (uuid == null) { + uuid = UUID.randomUUID().toString(); + } + + return uuid; + } + + @Override + public String getAzureInstanceId() { + return AzureReader.getAppInstanceId(); + } + + // The network adapter status. It's used for Windows only. + @Getter + enum NicStatus { + NOT_PRESENT("Not Present"), + UP("Up"), + DISCONNECTED("Disconnected"); + + private final String desc; + private static final Map ENUM_MAP; + + static { + Map map = new ConcurrentHashMap(); + for (NicStatus instance : NicStatus.values()) { + map.put(instance.getDesc(), instance); + } + ENUM_MAP = Collections.unmodifiableMap(map); + } + + NicStatus(String desc) { + this.desc = desc; + } + + public static NicStatus fromDesc(String desc) { + return ENUM_MAP.get(desc); + } + } + + /** + * Get the network adapters status by executing a command on Windows. + * + * @param nicStatusMap + */ + private static void buildNicStatusMap(Map nicStatusMap) { + String output; + try { + output = + ExecUtils.exec( + "powershell.exe Get-NetAdapter -IncludeHidden | Select-Object InterfaceDescription,Status | Format-Table -AutoSize", + System.getProperty("line.separator")); + } catch (Exception e) { + logger.info("Failed to obtain nic status from exec `Get-NetAdapter` : " + e.getMessage()); + return; + } + + String[] lines = output.split(System.getProperty("line.separator")); + if (lines.length < 3) { + logger.info( + "No enough data received from exec `Get-NetAdapter`(" + + lines.length + + "): " + + Arrays.toString(lines)); + return; + } + + String header = lines[0]; + int statusStartPoint = header.indexOf("Status"); + if (statusStartPoint == -1) { + logger.info( + "Failed to obtain nic status as the header `Status` is not found. " + + Arrays.toString(lines)); + return; + } + + lines = Arrays.copyOfRange(lines, 2, lines.length); + + for (String line : lines) { + if (statusStartPoint > line.length()) { + continue; + } + String name = line.substring(0, statusStartPoint).trim(); + NicStatus status = NicStatus.fromDesc(line.substring(statusStartPoint).trim()); + logger.debug("Get device display name=" + name + ", status=" + status); + nicStatusMap.put(name, status); + } + } + + /** + * Extracts network interface info from the system. Take note that loop back, point-to-point and + * non-physical addresses are excluded + * + * @return + */ + @Override + public HostInfoUtils.NetworkAddressInfo getNetworkAddressInfo() { + try { + List ips = new ArrayList(); + List macAddresses = new ArrayList(); + + // map of device id -> status + Map nicStatusMap = new HashMap(); + boolean isWindowsIPv4 = false; + if (osType.equals(HostInfoUtils.OsType.WINDOWS) + && Boolean.getBoolean("java.net.preferIPv4Stack")) { + buildNicStatusMap(nicStatusMap); + isWindowsIPv4 = true; + } + + for (NetworkInterface networkInterface : + Collections.list(NetworkInterface.getNetworkInterfaces())) { + try { + logger.debug( + "Found network interface " + + networkInterface.getName() + + " " + + networkInterface.getDisplayName()); + if (!networkInterface.isLoopback() + && !networkInterface.isPointToPoint() + && isPhysicalInterface(networkInterface) + && !isGhostHyperV(networkInterface)) { + logger.debug( + "Processing physical network interface " + + networkInterface.getName() + + " " + + networkInterface.getDisplayName()); + boolean hasIp = false; + for (InetAddress address : Collections.list(networkInterface.getInetAddresses())) { + String ipAddress = address.getHostAddress(); + logger.debug("Found ip address " + ipAddress); + if (!ips.contains(ipAddress)) { + ips.add(ipAddress); + } + hasIp = true; + } + + if (isWindowsIPv4) { // extra check for windows IPv4 preferred environment, see + // https://github.com/librato/joboe/pull/1090 for details + NicStatus status = nicStatusMap.get(networkInterface.getDisplayName()); + logger.debug( + "Checking " + + networkInterface.getName() + + ", " + + networkInterface.getDisplayName() + + ", status=" + + status); + if (!(NicStatus.UP.equals(status) || NicStatus.DISCONNECTED.equals(status))) { + // ignore disabled/null NIC on Windows when "-Djava.net.preferIPv4Stack=true" is set + logger.debug( + "Ignore disabled/null network adapter " + + networkInterface.getDisplayName() + + ", status=" + + status); + continue; + } + // We cannot simply filter out NICs without an IP for all the scenarios. This is + // because if an NIC is DISCONNECTED, it will have + // some kind of `link local` IP address (169.254. 0.0/16). The link local IP can be + // fetched by the Go API (used by the + // host agent) but not by the Java Windows API when `java.net.preferIPv4Stack`=true. + // Therefore, if the isWindowsIPv4 is true (check that when it's true), we'll accept + // the DISCONNECTED NIC without + // considering if it has an IP. For all other cases, that NIC will be filtered out if + // it doesn't have an IP address. + if (NicStatus.UP.equals(status) && !hasIp) { + logger.debug( + "Ignore network adapter which is up but no IP assigned: " + + networkInterface.getDisplayName()); + continue; + } + } else if (!hasIp) { + logger.debug("Ignore network adapter without an IP: " + networkInterface.getName()); + continue; + } + // add mac addresses too + byte[] hwAddr = networkInterface.getHardwareAddress(); + if ((hwAddr != null) && (hwAddr.length != 0)) { + String macAddress = getMacAddressFromBytes(hwAddr); + logger.debug("Found MAC address " + macAddress); + if (!macAddresses.contains(macAddress)) { + macAddresses.add(macAddress); + } + } + } + } catch (NoSuchMethodError e) { + logger.debug( + "Failed to get network info for " + + networkInterface.getName() + + ", probably running JDK 1.5 or earlier"); + } catch (SocketException e) { + logger.debug( + "Failed to get network info for " + + networkInterface.getName() + + ":" + + e.getMessage()); + } + } + logger.debug("All MAC addresses accepted: " + Arrays.toString(macAddresses.toArray())); + return new HostInfoUtils.NetworkAddressInfo(ips, macAddresses); + } catch (SocketException e) { + logger.warn("Failed to get network info : " + e.getMessage()); + } + return null; + } + + /** + * Determines whether a network interface is physical (for Linux only). + * + *

By our definition, a network interface is considered physical if its link in /sys/class/net + * does NOT contain "/virtual/" ... + * + *

Take note that this definition is different from NetworkInterface.isVirtual() + * + * @param networkInterface + * @return + */ + private static boolean isPhysicalInterface(NetworkInterface networkInterface) { + if (osType + != HostInfoUtils.OsType + .LINUX) { // not applicable to non-linux systems, consider all interfaces physical + return true; + } + + // cannot use java.nio.file.Files.readSymbolicLink as it's not available in jdk 1.6 + String interfaceFilePath = "/sys/class/net/" + networkInterface.getName(); + File interfaceFile = new File(interfaceFilePath); + try { + return !interfaceFile.getCanonicalPath().contains("/virtual/"); + } catch (IOException e) { + logger.warn( + "Error identifying network interface in " + + interfaceFilePath + + " message : " + + e.getMessage(), + e); + return true; // having problem identifying the interface, treat it as physical + } + } + + /** + * To detect whether it is a ghost Microsoft Hyper-V Network Adapter. + * + *

Since in IPv4 preferred environment, `NetworkInterface.getAll` might return ghost Microsoft + * Hyper-V Network Adapter + * + *

... for details + * + * @param networkInterface + * @return false if it's NOT a Microsoft Hyper-V Network Adapter or it's UP + */ + private static boolean isGhostHyperV(NetworkInterface networkInterface) { + final String hyperVPrefix = "Microsoft Hyper-V Network Adapter"; + try { + String displayName = networkInterface.getDisplayName(); + return displayName != null + && displayName.startsWith(hyperVPrefix) + && !networkInterface.isUp(); + } catch (SocketException e) { + logger.debug("Cannot call isUp on " + networkInterface.getDisplayName(), e); + return false; + } + } + + private static String getMacAddressFromBytes(byte[] bytes) { + StringBuilder sb = new StringBuilder(18); + for (byte b : bytes) { + if (sb.length() > 0) { + sb.append(':'); + } + sb.append(String.format("%02x", b)); + } + return sb.toString().toUpperCase(); + } + + @Override + public String getHostName() { + return hostname; + } + + @Override + public HostId getHostId() { + if (hostId == null) { + Metadata existingMetdataContext = null; + try { + existingMetdataContext = Context.getMetadata(); + Context.clearMetadata(); // make sure our init route does not accidentally trigger tracing + // instrumentations + + // synchronously get it once first to ensure it's available at the return statement + hostId = buildHostId(); + // also start the background checker here + startHostIdChecker(); + } finally { + Context.setMetadata(existingMetdataContext); // set the existing context back + } + } + return hostId; + } + + public static String loadHostName() { + String hostName = loadHostNameFromExec(); + + if (hostName != null) { + return hostName; + } + + hostName = loadHostNameFromInetAddress(); + + if (hostName != null) { + return hostName; + } + + // We've failed to get an IP address, so as a last resort... + hostName = "unknown_hostname"; + return hostName; + } + + /** + * This was copied from JAgentInfo.java + * + *

Returns system host name using external 'hostname' command. + * + *

This feels lame, but there is no other way to get a hostname that definitively matches the + * gethostname() C library function we use elsewhere (liboboe, tracelyzer, etc.) without resorting + * to JNI. We only do this at startup so I think we can live with it. + * + *

'InetAddress.getHostName()' and 'getCanonicalHostName()' may vary or fail based on DNS + * settings, /etc/hosts settings, etc. and just aren't guaranteed to be the same as gethostname(), + * even though on many systems they are. + * + *

Also see ... + */ + private static String loadHostNameFromExec() { + try { + return ExecUtils.exec("hostname"); + } catch (Exception e) { + // in some "slim" os, it might not have `hostname`. For example Orcale linux slim + logger.info("Failed to obtain host name from exec `hostname` : " + e.getMessage()); + return null; + } + } + + /** + * This was copied from JAgentInfo.java + * + * @return + */ + private static String loadHostNameFromInetAddress() { + try { + InetAddress addr = InetAddress.getLocalHost(); + return addr.getCanonicalHostName(); + } catch (UnknownHostException hostEx) { + // Our hostname doesn't resolve, likely a mis-configured host, so fallback to using the first + // non-loopback IP address + try { + Enumeration netInts = NetworkInterface.getNetworkInterfaces(); + NetworkInterface netInt; + Enumeration addrs; + InetAddress addr; + + while (netInts.hasMoreElements()) { + netInt = netInts.nextElement(); + addrs = netInt.getInetAddresses(); + + while (addrs.hasMoreElements()) { + addr = addrs.nextElement(); + + if (!addr.isLoopbackAddress() && !addr.isLinkLocalAddress()) { + return addr.getHostAddress(); + } + } + } + + } catch (SocketException socketEx) { + logger.warn("Unable to retrieve network interfaces", socketEx); + } + + return null; + } + } + + public static String getDistro() { + if (!checkedDistro) { + if (osType == HostInfoUtils.OsType.LINUX) { + distro = getLinuxDistro(); + } else if (osType == HostInfoUtils.OsType.WINDOWS) { + distro = getWindowsDistro(); + } + checkedDistro = true; + } + return distro; // could be null for unsupported system + } + + @Override + /** + * Get a map of host information + * + * @return + */ + public Map getHostMetadata() { + HashMap infoMap = new HashMap(); + + String hostnameAlias = (String) ConfigManager.getConfig(ConfigProperty.AGENT_HOSTNAME_ALIAS); + if (hostnameAlias != null) { + infoMap.put(HOSTNAME_ALIAS_KEY, hostnameAlias); + } + + addOsInfo(infoMap); + addNetworkAddresses(infoMap); + + return infoMap; + } + + private static void addOsInfo(Map infoMap) { + infoMap.put("UnameSysName", osType.getLabel()); + infoMap.put("UnameVersion", ManagementFactory.getOperatingSystemMXBean().getVersion()); + + String distro = getDistro(); + if (distro != null) { + infoMap.put("Distro", distro); + } + } + + private void addNetworkAddresses(Map infoMap) { + HostInfoUtils.NetworkAddressInfo networkInfo = getNetworkAddressInfo(); + if (networkInfo != null) { + infoMap.put("IPAddresses", networkInfo.getIpAddresses()); + } + } + + /** + * Identifies the distro value on a Linux system + * + *

Code logic copied from ... + * + * @return + */ + private static String getLinuxDistro() { + // Note: Order of checking is important because some distros share same file names but with + // different function. + // Keep this order: redhat based -> ubuntu -> debian + + BufferedReader fileReader = null; + + try { + if ((fileReader = getFileReader(DistroType.REDHAT_BASED)) != null) { + // Redhat, CentOS, Fedora + return getRedHatBasedDistro(fileReader); + } else if ((fileReader = getFileReader(DistroType.AMAZON)) != null) { + // Amazon Linux + return getAmazonLinuxDistro(fileReader); + } else if ((fileReader = getFileReader(DistroType.UBUNTU)) != null) { + // Ubuntu + return getUbuntuDistro(fileReader); + } else if ((fileReader = getFileReader(DistroType.DEBIAN)) != null) { + // Debian + return getDebianDistro(fileReader); + } else if ((fileReader = getFileReader(DistroType.SUSE)) != null) { + // Novell SuSE + return getNovellSuseDistro(fileReader); + } else if ((fileReader = getFileReader(DistroType.SLACKWARE)) != null) { + // Slackware + return getSlackwareDistro(fileReader); + } else if ((fileReader = getFileReader(DistroType.SLACKWARE)) != null) { + return getGentooDistro(fileReader); + } else { + return "Unknown"; + } + } catch (IOException e) { + logger.warn("Problem reading distro file : " + e.getMessage(), e); + return "Unknown"; + } finally { + if (fileReader != null) { + try { + fileReader.close(); + } catch (IOException e) { + logger.warn(e.getMessage(), e); + } + } + } + } + + private static String getWindowsDistro() { + return ManagementFactory.getOperatingSystemMXBean().getName(); + } + + static String getGentooDistro(BufferedReader fileReader) throws IOException { + String line; + return (line = fileReader.readLine()) != null ? line : "Gentoo unknown"; + } + + static String getSlackwareDistro(BufferedReader fileReader) throws IOException { + String line; + return (line = fileReader.readLine()) != null ? line : "Slackware unknown"; + } + + static String getNovellSuseDistro(BufferedReader fileReader) throws IOException { + String line; + return (line = fileReader.readLine()) != null ? line : "Novell SuSE unknown"; + } + + static String getDebianDistro(BufferedReader fileReader) throws IOException { + String line; + return (line = fileReader.readLine()) != null ? ("Debian " + line) : "Debian unknown"; + } + + static String getRedHatBasedDistro(BufferedReader fileReader) throws IOException { + String line; + return (line = fileReader.readLine()) != null ? line : "Red Hat based unknown"; + } + + static String getAmazonLinuxDistro(BufferedReader fileReader) throws IOException { + String line; + if ((line = fileReader.readLine()) != null) { + String[] tokens = line.split(":"); + if (tokens.length >= 5) { + String patch = tokens[4]; + return "Amzn Linux " + patch; + } + } + return "Amzn Linux unknown"; + } + + static String getUbuntuDistro(BufferedReader fileReader) throws IOException { + String line; + while ((line = fileReader.readLine()) != null) { + String[] tokens = line.split("="); + if (tokens.length >= 2 && "DISTRIB_DESCRIPTION".equals(tokens[0])) { + // trim trailing/leading quotes + String patch = tokens[1]; + if (patch.startsWith("\"")) { + patch = patch.substring(1); + } + if (patch.endsWith("\"")) { + patch = patch.substring(0, patch.length() - 1); + } + + return patch; + } + } + return "Ubuntu unknown"; + } + + public static void setIfNotNull(Consumer setter, V value) { + if (value != null) { + setter.accept(value); + } + } + + public static void setAlternateIfNull(Consumer setter, V value, Supplier alternate) { + if (value != null) { + setter.accept(value); + } else { + setIfNotNull(setter, alternate.get()); + } + } + + private static BufferedReader getFileReader(DistroType distroType) { + String path = DISTRO_FILE_NAMES.get(distroType); + if (path == null) { + logger.warn("Unexpected distroType lookup: " + distroType); + return null; + } + File file = new File(path); + try { + return file.exists() ? new BufferedReader(new FileReader(file)) : null; + } catch (FileNotFoundException e) { + logger.warn(e.getMessage(), e); + return null; + } + } + + private void startHostIdChecker() { + ScheduledExecutorService executorService = + Executors.newScheduledThreadPool(1, DaemonThreadFactory.newInstance("host-id-checker")); + // align to minute, figure out the delay in ms + long delay = 60 * 1000 - System.currentTimeMillis() % (60 * 1000); + executorService.scheduleAtFixedRate( + () -> hostId = buildHostId(), delay, HOST_ID_CHECK_INTERVAL * 1000, TimeUnit.MILLISECONDS); + } + + private HostId buildHostId() { + HostInfoUtils.NetworkAddressInfo networkAddressInfo = getNetworkAddressInfo(); + List macAddresses = + networkAddressInfo != null ? networkAddressInfo.getMacAddresses() : Collections.emptyList(); + // assume all that uses HostInfoUtils are persistent server, can be improved to recognize + // different types later + return HostId.builder() + .hostname(getHostName()) + .pid(JavaProcessUtils.getPid()) + .macAddresses(macAddresses) + .ec2InstanceId(Ec2InstanceReader.getInstanceId()) + .ec2AvailabilityZone(Ec2InstanceReader.getAvailabilityZone()) + .dockerContainerId(DockerInfoReader.getDockerId()) + .herokuDynoId(HerokuDynoReader.getDynoId()) + .azureAppServiceInstanceId(AzureReader.getAppInstanceId()) + .uamsClientId(UamsClientIdReader.getUamsClientId()) + .uuid(getUuid()) + .awsMetadata(Ec2InstanceReader.getInstance().awsMetadata) + .azureVmMetadata(AzureReader.getInstance().azureVmMetadata) + .k8sMetadata(K8sReader.getInstance().getK8sMetadata()) + .build(); + } + + public static class Ec2InstanceReader { + private static final int TIMEOUT_MIN = 0; // do not wait at all, disable retrieval + private static final int TIMEOUT_MAX = 3000; + + private static final int TIMEOUT = getTimeout(); + + private static int getTimeout() { + Integer configValue = + (Integer) ConfigManager.getConfig(ConfigProperty.AGENT_EC2_METADATA_TIMEOUT); + if (configValue == null) { + return TIMEOUT_DEFAULT; + } else if (configValue > TIMEOUT_MAX) { + logger.warn( + "EC2 metadata read timeout cannot be greater than " + + TIMEOUT_MAX + + " millisec but found [" + + configValue + + "]. Using " + + TIMEOUT_MAX + + " instead."); + return TIMEOUT_MAX; + } else if (configValue < TIMEOUT_MIN) { + logger.warn( + "EC2 metadata read timeout cannot be smaller than " + + TIMEOUT_MIN + + " millisec but found [" + + configValue + + "]. Using " + + TIMEOUT_MIN + + " instead, which essentially disable reading EC2 metadata"); + return TIMEOUT_MIN; + } else { + return configValue; + } + } + + /** System property for overriding the Amazon EC2 Instance Metadata Service endpoint. */ + public static final String EC2_METADATA_SERVICE_OVERRIDE_SYSTEM_PROPERTY = + "com.amazonaws.sdk.ec2MetadataServiceEndpointOverride"; + + private static final String EC2_METADATA_SERVICE_URL = "http://169.254.169.254"; + private static final String INSTANCE_ID_PATH = "latest/meta-data/instance-id"; + private static final String AVAILABILITY_ZONE_PATH = + "latest/meta-data/placement/availability-zone"; + + private String instanceId; + private String availabilityZone; + + private HostId.AwsMetadata awsMetadata; + + @Getter(lazy = true) + private static final Ec2InstanceReader instance = new Ec2InstanceReader(); + + public static String getInstanceId() { + return getInstance().instanceId; + } + + public static String getAvailabilityZone() { + return getInstance().availabilityZone; + } + + private Ec2InstanceReader() { + initialize(); + } + + private void initialize() { + if (TIMEOUT == TIMEOUT_MIN) { // disable reader + return; + } + String token = getApiToken(); + instanceId = getResourceOnEndpoint(INSTANCE_ID_PATH, token); + if (instanceId != null) { // only proceed if instance id can be found + availabilityZone = getResourceOnEndpoint(AVAILABILITY_ZONE_PATH, token); + logger.debug( + "Found EC2 instance id " + instanceId + " availability zone: " + availabilityZone); + } + + awsMetadata = getMetadata(token); + } + + private static String useIMDSv1(String relativePath) { + AtomicReference result = new AtomicReference<>(); + InstrumentationUtil.suppressInstrumentation( + () -> { + HttpURLConnection connection = null; + try { + URL url = new URL(getMetadataHost() + "/" + relativePath); + connection = (HttpURLConnection) url.openConnection(Proxy.NO_PROXY); + + connection.setRequestMethod("GET"); + connection.setReadTimeout(TIMEOUT); + connection.setConnectTimeout(TIMEOUT); + + int statusCode = connection.getResponseCode(); + if (statusCode >= 200 && statusCode < 300) { + try (BufferedReader reader = + new BufferedReader(new InputStreamReader(connection.getInputStream()))) { + StringBuilder sb = new StringBuilder(); + String line; + while ((line = reader.readLine()) != null) { + sb.append(line); + } + + String payload = sb.toString(); + logger.debug(String.format("Retrieved metadata using IMDSv1: %s", payload)); + result.set(payload); + } + } else { + logger.debug( + String.format( + "Failed to retrieved metadata using IMDSv1: status code=%d", statusCode)); + } + + } catch (IOException exception) { + logger.debug(String.format("Error retrieving metadata using IMDSv1: %s", exception)); + } finally { + if (connection != null) { + connection.disconnect(); + } + } + }); + return result.get(); + } + + private static String getApiToken() { + AtomicReference result = new AtomicReference<>(); + InstrumentationUtil.suppressInstrumentation( + () -> { + HttpURLConnection connection = null; + try { + URL url = new URL(getMetadataHost() + "/latest/api/token"); + connection = (HttpURLConnection) url.openConnection(Proxy.NO_PROXY); + connection.setRequestMethod("PUT"); + + connection.setReadTimeout(TIMEOUT); + connection.setConnectTimeout(TIMEOUT); + connection.setRequestProperty("Content-Type", "application/json"); + + connection.setRequestProperty("X-aws-ec2-metadata-token-ttl-seconds", "3600"); + connection.setDoOutput(true); + try (OutputStream os = connection.getOutputStream()) { + os.write("{}".getBytes(StandardCharsets.UTF_8)); + } + + int statusCode = connection.getResponseCode(); + if (statusCode >= 200 && statusCode < 300) { + try (BufferedReader reader = + new BufferedReader(new InputStreamReader(connection.getInputStream()))) { + StringBuilder sb = new StringBuilder(); + String line; + while ((line = reader.readLine()) != null) { + sb.append(line); + } + result.set(sb.toString()); + } + } else { + logger.debug( + String.format( + "Failed to retrieved metadata request token: status code=%d", statusCode)); + } + + } catch (IOException e) { + logger.debug(String.format("Error getting token for IMDSv2: %s", e)); + } finally { + if (connection != null) { + connection.disconnect(); + } + } + }); + return result.get(); + } + + private static String useIMDSv2(String relativePath, String apiToken) { + AtomicReference result = new AtomicReference<>(); + InstrumentationUtil.suppressInstrumentation( + () -> { + HttpURLConnection connection = null; + try { + URL url = new URL(getMetadataHost() + "/" + relativePath); + connection = (HttpURLConnection) url.openConnection(Proxy.NO_PROXY); + connection.setRequestMethod("GET"); + + connection.setReadTimeout(TIMEOUT); + connection.setConnectTimeout(TIMEOUT); + connection.setRequestProperty("X-aws-ec2-metadata-token", apiToken); + + int statusCode = connection.getResponseCode(); + if (statusCode >= 200 && statusCode < 300) { + try (BufferedReader reader = + new BufferedReader(new InputStreamReader(connection.getInputStream()))) { + StringBuilder sb = new StringBuilder(); + String line; + while ((line = reader.readLine()) != null) { + sb.append(line); + } + String payload = sb.toString(); + logger.debug(String.format("Retrieved metadata using IMDSv2: %s", payload)); + result.set(payload); + } + } else { + logger.debug( + String.format( + "Failed to retrieved metadata using IMDSv2: status code=%d", statusCode)); + } + + } catch (IOException e) { + logger.debug(String.format("Error retrieving metadata using IMDSv2: %s", e)); + } finally { + if (connection != null) { + connection.disconnect(); + } + } + }); + return result.get(); + } + + private static String getResourceOnEndpoint(String relativePath, String token) { + String result = null; + if (token != null) { + result = useIMDSv2(relativePath, token); + } + + if (result == null) { + result = useIMDSv1(relativePath); + } + return result; + } + + private static HostId.AwsMetadata getMetadata(String token) { + String payload = getResourceOnEndpoint("latest/dynamic/instance-identity/document", token); + if (payload != null) { + try { + HostId.AwsMetadata metadata = + HostId.AwsMetadata.fromJson( + payload, getResourceOnEndpoint("latest/meta-data/hostname", token)); + logger.debug(String.format("Aws Metadata: %s", metadata)); + return metadata; + + } catch (JSONException e) { + logger.debug( + String.format("Error converting json to AwsMetadata model: %s\n %s", payload, e)); + } + } + return null; + } + + private static String getMetadataHost() { + String host = System.getProperty(EC2_METADATA_SERVICE_OVERRIDE_SYSTEM_PROPERTY); + return host != null ? host : METADATA_SERVICE_URL; + } + } + + public static class DockerInfoReader { + public static final String DEFAULT_LINUX_DOCKER_FILE_LOCATION = "/proc/self/cgroup"; + + private String dockerId; + + static final Pattern CONTAINER_ID_REGEX = Pattern.compile("[a-f0-9]{64}"); + + @Getter(lazy = true) + private static final DockerInfoReader instance = new DockerInfoReader(); + + public static String getDockerId() { + return getInstance().dockerId; + } + + private DockerInfoReader() { + if (osType == HostInfoUtils.OsType.LINUX) { + initializeLinux(DEFAULT_LINUX_DOCKER_FILE_LOCATION); + } else if (osType == HostInfoUtils.OsType.WINDOWS) { + initializeWindows(); + } + + if (dockerId != null) { + logger.debug("Found Docker instance ID :" + this.dockerId); + } else { + logger.debug("Cannot locate Docker id, not a Docker container"); + } + } + + private static Optional getIdFromLine(String line) { + // This cgroup output line should have the container id in it + int lastSlashIdx = line.lastIndexOf('/'); + if (lastSlashIdx < 0) { + return Optional.empty(); + } + + String containerId; + + String lastSection = line.substring(lastSlashIdx + 1); + int colonIdx = lastSection.lastIndexOf(':'); + + if (colonIdx != -1) { + // since containerd v1.5.0+, containerId is divided by the last colon when the cgroupDriver + // is + // systemd: + // https://github.com/containerd/containerd/blob/release/1.5/pkg/cri/server/helpers_linux.go#L64 + containerId = lastSection.substring(colonIdx + 1); + } else { + int startIdx = lastSection.lastIndexOf('-'); + int endIdx = lastSection.lastIndexOf('.'); + + startIdx = startIdx == -1 ? 0 : startIdx + 1; + if (endIdx == -1) { + endIdx = lastSection.length(); + } + if (startIdx > endIdx) { + return Optional.empty(); + } + + containerId = lastSection.substring(startIdx, endIdx); + } + + if (CONTAINER_ID_REGEX.matcher(containerId).matches()) { + logger.info(String.format("Found container id: {%s} in line: {%s}", containerId, line)); + return Optional.of(containerId); + + } else { + logger.debug(String.format("No container id in line: {%s}", line)); + return Optional.empty(); + } + } + + void initializeLinux(String dockerFileLocation) { + dockerId = fileReader(dockerFileLocation, DockerInfoReader::getIdFromLine); + } + + /** + * Initializes Windows docker ID if the java process is running within a Windows docker + * container + * + *

Determines if it is a Windows container by checking if `cexecsvc` exists in `powershell + * get-process` + * + *

If so, set the host name as Docker ID + */ + private void initializeWindows() { + try { + String getContainerTypeResult = + ExecUtils.exec( + "powershell Get-ItemProperty -Path HKLM:\\SYSTEM\\CurrentControlSet\\Control\\ -Name \"ContainerType\""); + if (getContainerTypeResult != null && !getContainerTypeResult.isEmpty()) { + dockerId = ServerHostInfoReader.INSTANCE.getHostName(); + } + } catch (Exception e) { + logger.info( + "Failed to identify whether this windows system is a docker container: " + + e.getMessage()); + } + } + } + + public static class HerokuDynoReader { + private static final String DYNO_ENV_VARIABLE = "DYNO"; + + @Getter(lazy = true) + private static final HerokuDynoReader instance = new HerokuDynoReader(); + + private final String dynoId; + + public static String getDynoId() { + return getInstance().dynoId; + } + + private HerokuDynoReader() { + this.dynoId = System.getenv(DYNO_ENV_VARIABLE); + if (this.dynoId != null) { + logger.debug("Found Heroku Dyno ID: " + this.dynoId); + } + } + } + + public static class AzureReader { + private static final String INSTANCE_ID_ENV_VARIABLE = "WEBSITE_INSTANCE_ID"; + + @Getter(lazy = true) + private static final AzureReader instance = new AzureReader(); + + private static final String DEFAULT_METADATA_VERSION = "2021-12-13"; + private final String appInstanceId; + + private final HostId.AzureVmMetadata azureVmMetadata; + + public static String getAppInstanceId() { + return getInstance().appInstanceId; + } + + private HostId.AzureVmMetadata getVmMetadata() { + Integer timeout = + ConfigManager.getConfigOptional( + ConfigProperty.AGENT_AZURE_VM_METADATA_TIMEOUT, TIMEOUT_DEFAULT); + String metadataVersionCfg = + (String) ConfigManager.getConfig(ConfigProperty.AGENT_AZURE_VM_METADATA_VERSION); + + final String metadataVersion = + metadataVersionCfg != null ? metadataVersionCfg : DEFAULT_METADATA_VERSION; + AtomicReference result = new AtomicReference<>(); + + InstrumentationUtil.suppressInstrumentation( + () -> { + HttpURLConnection connection = null; + try { + URL url = + new URL( + String.format( + "%s%s%s", + METADATA_SERVICE_URL, + "/metadata/instance?api-version=", + metadataVersion)); + + connection = (HttpURLConnection) url.openConnection(Proxy.NO_PROXY); + connection.setRequestMethod("GET"); + connection.setReadTimeout(timeout); + + connection.setConnectTimeout(timeout); + connection.setRequestProperty("Metadata", "true"); + int statusCode = connection.getResponseCode(); + + logger.debug(String.format("Azure IMDS status code: %s", statusCode)); + if (statusCode >= 200 && statusCode < 300) { + try (BufferedReader reader = + new BufferedReader(new InputStreamReader(connection.getInputStream()))) { + StringBuilder sb = new StringBuilder(); + + String line; + while ((line = reader.readLine()) != null) { + sb.append(line); + } + + String payload = sb.toString(); + logger.debug(String.format("Azure IMDS payload: %s", payload)); + result.set(HostId.AzureVmMetadata.fromJson(payload)); + } + } + + } catch (IOException | JSONException exception) { + logger.debug("Error retrieving vmId from IMDS", exception); + } finally { + if (connection != null) { + connection.disconnect(); + } + } + }); + return result.get(); + } + + private AzureReader() { + this.appInstanceId = System.getenv(INSTANCE_ID_ENV_VARIABLE); + if (this.appInstanceId != null) { + logger.debug("Found Azure instance ID: " + this.appInstanceId); + } + azureVmMetadata = getVmMetadata(); + logger.debug(String.format("Azure vm metadata: %s", azureVmMetadata)); + } + } + + public static class K8sReader { + public static String SW_K8S_POD_NAMESPACE = "SW_K8S_POD_NAMESPACE"; + + public static String SW_K8S_POD_NAME = "SW_K8S_POD_NAME"; + + public static String SW_K8S_POD_UID = "SW_K8S_POD_UID"; + + public static String NAMESPACE_FILE_LOC_LINUX = + "/var/run/secrets/kubernetes.io/serviceaccount/namespace"; + + public static String NAMESPACE_FILE_LOC_WINDOWS = + "C:\\var\\run\\secrets\\kubernetes.io\\serviceaccount\\namespace"; + + public static String POD_UUID_FILE_LOC = "/proc/self/mountinfo"; + + static final Pattern POD_UID_REGEX = + Pattern.compile("[0-9a-fA-F]{8}-([0-9a-fA-F]{4}-){3}[0-9a-fA-F]{12}"); + + private final HostId.K8sMetadata k8sMetadata; + + @Getter(lazy = true) + private static final K8sReader instance = new K8sReader(); + + public K8sReader() { + HostId.K8sMetadata.K8sMetadataBuilder builder = HostId.K8sMetadata.builder(); + Map env = System.getenv(); + setAlternateIfNull( + builder::namespace, env.get(SW_K8S_POD_NAMESPACE), K8sReader::getNamespace); + setAlternateIfNull( + builder::podName, env.get(SW_K8S_POD_NAME), ServerHostInfoReader.INSTANCE::getHostName); + + setAlternateIfNull(builder::podUid, env.get(SW_K8S_POD_UID), K8sReader::getPodId); + k8sMetadata = builder.build(); + } + + private static String getNamespace() { + if (osType == HostInfoUtils.OsType.LINUX) { + return fileReader(NAMESPACE_FILE_LOC_LINUX, line -> Optional.of(line.trim())); + + } else if (osType == HostInfoUtils.OsType.WINDOWS) { + return fileReader(NAMESPACE_FILE_LOC_WINDOWS, line -> Optional.of(line.trim())); + } + + return null; + } + + private static String getPodId() { + return fileReader( + POD_UUID_FILE_LOC, + line -> { + Matcher matcher = POD_UID_REGEX.matcher(line); + String podId = null; + if (matcher.find()) { + podId = line.substring(matcher.start(), matcher.end()); + logger.info(String.format("Found pod uid: %s", podId)); + } else { + logger.debug(String.format("Pod uid not found on line: %s", line)); + } + return Optional.ofNullable(podId); + }); + } + + public HostId.K8sMetadata getK8sMetadata() { + if (k8sMetadata.getNamespace() == null) { + return null; + } + return k8sMetadata; + } + } + + public static V fileReader(String filepath, Function> lineParser) { + Optional value = Optional.empty(); + try (Stream lineStream = Files.lines(Paths.get(filepath))) { + value = + lineStream + .filter(line -> !line.isEmpty()) + .map(lineParser) + .filter(Optional::isPresent) + .findFirst() + .orElse(Optional.empty()); + + } catch (IOException e) { + logger.debug(String.format("Error reading file(%s). %s", filepath, e.getMessage())); + } + return value.orElse(null); + } +} diff --git a/libs/core/src/main/java/com/solarwinds/joboe/core/util/SslUtils.java b/libs/core/src/main/java/com/solarwinds/joboe/core/util/SslUtils.java new file mode 100644 index 00000000..68f39552 --- /dev/null +++ b/libs/core/src/main/java/com/solarwinds/joboe/core/util/SslUtils.java @@ -0,0 +1,194 @@ +/* + * © SolarWinds Worldwide, LLC. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.solarwinds.joboe.core.util; + +import com.solarwinds.joboe.logging.Logger; +import com.solarwinds.joboe.logging.LoggerFactory; +import java.io.IOException; +import java.io.InputStream; +import java.net.URL; +import java.security.GeneralSecurityException; +import java.security.KeyStore; +import java.security.cert.Certificate; +import java.security.cert.CertificateException; +import java.security.cert.CertificateFactory; +import java.security.cert.X509Certificate; +import java.util.*; +import javax.net.ssl.SSLContext; +import javax.net.ssl.TrustManager; +import javax.net.ssl.TrustManagerFactory; +import javax.net.ssl.X509TrustManager; + +public class SslUtils { + private static final Logger logger = LoggerFactory.getLogger(); + + private SslUtils() {} + + public static TrustManagerFactory getTrustManagerFactory(URL serverCertLocation) + throws IOException, GeneralSecurityException { + // Trusted keystore manager => so this client can trust the collector! + CertificateFactory certificateFactory = CertificateFactory.getInstance("X.509"); + InputStream caInput = serverCertLocation.openStream(); + + Certificate certificate; + try { + certificate = certificateFactory.generateCertificate(caInput); + } finally { + caInput.close(); + } + + // trusted keystore + KeyStore trustedKeyStore = KeyStore.getInstance(KeyStore.getDefaultType()); + + trustedKeyStore.load(null, null); + trustedKeyStore.setCertificateEntry("ca", certificate); + + // Create a TrustManager that trusts the CAs in our KeyStore + String algorithm = TrustManagerFactory.getDefaultAlgorithm(); + TrustManagerFactory trustManagerFactory = TrustManagerFactory.getInstance(algorithm); + trustManagerFactory.init(trustedKeyStore); + + // end Trusted keystore manager + return trustManagerFactory; + } + + /** + * Wrap the input trustManagers into a single manager, which simply iterate through the input + * managers and check whether the cert presented has host name matches the `host` input parameter. + * + *

This is necessary for either java 6- that does not support + * `SslParameters#setEndpointIdentificationAlgorithm("HTTPS")` or when there's cert host name + * override + * + * @param trustManagers + * @param host + * @return an array of one manager, which wraps all the input trust managers + */ + private static TrustManager[] explicitHostCheckManager( + final TrustManager[] trustManagers, final String host) { + X509TrustManager manager = + new X509TrustManager() { + @Override + public void checkClientTrusted(X509Certificate[] x509Certificates, String s) + throws CertificateException { + for (TrustManager trustManager : trustManagers) { + ((X509TrustManager) trustManager).checkClientTrusted(x509Certificates, s); + } + } + + @Override + public void checkServerTrusted(X509Certificate[] x509Certificates, String s) + throws CertificateException { + for (TrustManager trustManager : trustManagers) { + ((X509TrustManager) trustManager).checkServerTrusted(x509Certificates, s); + } + + boolean hostnameMatch = false; + for (X509Certificate x509Certificate : x509Certificates) { + Collection> alternativeNameCollection = + x509Certificate.getSubjectAlternativeNames(); + if (alternativeNameCollection != null) { + for (List alternativeNames : alternativeNameCollection) { + if (alternativeNames.get(1).equals(host)) { + hostnameMatch = true; + break; + } + } + } + } + if (!hostnameMatch) { + throw new CertificateException( + "Certificate hostname and requested hostname don't match"); + } + } + + @Override + public X509Certificate[] getAcceptedIssuers() { + return new X509Certificate[0]; + } + }; + return new TrustManager[] {manager}; + } + + public static SSLContext getSslContext(URL serverCertLocation) + throws IOException, GeneralSecurityException { + return getSslContext(serverCertLocation, null); + } + + public static SSLContext getSslContext(URL serverCertLocation, String explicitHostCheck) + throws IOException, GeneralSecurityException { + + // Create an SSLContext that uses our TrustManager + TrustManagerFactory factory = getTrustManagerFactory(serverCertLocation); + TrustManager[] managers = factory.getTrustManagers(); + if (explicitHostCheck != null) { + managers = explicitHostCheckManager(managers, explicitHostCheck); + } + + return getSslContext(managers); + } + + /** + * Obtain a SSLContext, if the default of the JVM is TLSv1, it will try to see whether there's + * support for TLSv1.2 or TLSv1.1 and return SSLContext with the newer version For JVM default + * that is NOT TLSv1, it will just return SSLContext with the default SSL version + * + * @return SSLContext that prefers protocol version higher than TLSv1 + * @throws GeneralSecurityException + */ + private static SSLContext getSslContext(TrustManager[] trustManagers) + throws GeneralSecurityException { + SSLContext context = SSLContext.getInstance("TLS"); + context.init(null, trustManagers, null); + + try { + if (isDefaultTLSv1(context)) { // we should try higher version + List supportedProtocols = + Arrays.asList(context.getSupportedSSLParameters().getProtocols()); + if (supportedProtocols.contains("TLSv1.2")) { + context = SSLContext.getInstance("TLSv1.2"); + context.init(null, trustManagers, null); + return context; + } else if (supportedProtocols.contains("TLSv1.1")) { + context = SSLContext.getInstance("TLSv1.1"); + context.init(null, trustManagers, null); + return context; + } else { + logger.warn("SSL default protocol is TLSv1 and no TLSv1.1 nor TLSv1.2 support found"); + return context; + } + } else { // the default is NOT TLSv1. Just use whatever the default is + return context; + } + } catch (NoSuchMethodError e) { + logger.warn("Cannot check SSL protocol version as it's running JDK 1.6-"); + return context; + } + } + + /** Returns whether the only enabled SSL protocol is TLSv1 */ + private static boolean isDefaultTLSv1(SSLContext context) { + List enabledProtocols = + new ArrayList( + Arrays.asList( + context.getDefaultSSLParameters().getProtocols())); // the list has to be mutable + + // need to filter out protocols such as SSLv2Hello, SSLv3 etc + enabledProtocols.removeIf(protocol -> !protocol.startsWith("TLSv")); + return enabledProtocols.size() == 1 && "TLSv1".equals(enabledProtocols.get(0)); + } +} diff --git a/libs/core/src/main/java/com/solarwinds/joboe/core/util/TestUtils.java b/libs/core/src/main/java/com/solarwinds/joboe/core/util/TestUtils.java new file mode 100644 index 00000000..7d329407 --- /dev/null +++ b/libs/core/src/main/java/com/solarwinds/joboe/core/util/TestUtils.java @@ -0,0 +1,116 @@ +/* + * © SolarWinds Worldwide, LLC. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.solarwinds.joboe.core.util; + +import com.solarwinds.joboe.core.ReporterFactory; +import com.solarwinds.joboe.core.TestReporter; +import com.solarwinds.joboe.core.profiler.Profiler; +import com.solarwinds.joboe.core.profiler.ProfilerSetting; +import com.solarwinds.joboe.core.settings.*; +import com.solarwinds.joboe.sampling.SamplingConfiguration; +import com.solarwinds.joboe.sampling.Settings; +import com.solarwinds.joboe.sampling.SettingsArg; +import com.solarwinds.joboe.sampling.SettingsManager; +import com.solarwinds.joboe.sampling.TraceDecisionUtil; +import com.solarwinds.joboe.sampling.TracingMode; + +public abstract class TestUtils { + private static final TestDefaultSettings DEFAULT_SETTINGS = new TestDefaultSettings(); + + private TestUtils() {} + + public static TestSettingsReader initSettingsReader() { + TestSettingsReader reader = new TestSettingsReader(); + SettingsManager.initialize( + new SimpleSettingsFetcher(reader), SamplingConfiguration.builder().build()); + return reader; + } + + public static TestReporter initTraceReporter() { + try { + return ReporterFactory.getInstance().createTestReporter(); + } catch (Exception e) { + return null; + } + } + + public static TestReporter initProfilingReporter(ProfilerSetting profilerSetting) { + try { + TestReporter testProfilingReporter = ReporterFactory.getInstance().createTestReporter(); + Profiler.initialize(profilerSetting, testProfilingReporter); + return testProfilingReporter; + } catch (Exception e) { + return null; + } + } + + public static Settings getDefaultSettings() { + return DEFAULT_SETTINGS; + } + + private static class TestDefaultSettings extends Settings { + public static final Double DEFAULT_BUCKET_CAPACITY = 16.0; + public static final Double DEFAULT_BUCKET_RATE = 8.0; + public static final long DEFAULT_SAMPLE_RATE = TraceDecisionUtil.SAMPLE_RESOLUTION; // 100% + public static final short DEFAULT_TYPE = Settings.OBOE_SETTINGS_TYPE_DEFAULT_SAMPLE_RATE; + + @Override + public long getValue() { + return DEFAULT_SAMPLE_RATE; + } + + @Override + public long getTimestamp() { + return TimeUtils.getTimestampMicroSeconds(); + } + + @Override + public short getType() { + return DEFAULT_TYPE; + } + + @Override + public short getFlags() { + return TracingMode.ALWAYS.toFlags(); + } + + @Override + public long getTtl() { + return Integer.MAX_VALUE; // don't use long, otherwise it might overflow... + } + + @Override + @SuppressWarnings("unchecked") + public T getArgValue(SettingsArg arg) { + if (arg.equals(SettingsArg.BUCKET_CAPACITY)) { + return (T) DEFAULT_BUCKET_CAPACITY; + } else if (arg.equals(SettingsArg.BUCKET_RATE)) { + return (T) DEFAULT_BUCKET_RATE; + } else if (arg.equals(SettingsArg.RELAXED_BUCKET_CAPACITY)) { + return (T) DEFAULT_BUCKET_CAPACITY; + } else if (arg.equals(SettingsArg.RELAXED_BUCKET_RATE)) { + return (T) DEFAULT_BUCKET_RATE; + } else if (arg.equals(SettingsArg.STRICT_BUCKET_CAPACITY)) { + return (T) DEFAULT_BUCKET_CAPACITY; + } else if (arg.equals(SettingsArg.STRICT_BUCKET_RATE)) { + return (T) DEFAULT_BUCKET_RATE; + } + + return null; + } + } +} diff --git a/libs/core/src/main/java/com/solarwinds/joboe/core/util/TimeUtils.java b/libs/core/src/main/java/com/solarwinds/joboe/core/util/TimeUtils.java new file mode 100644 index 00000000..e738a2ed --- /dev/null +++ b/libs/core/src/main/java/com/solarwinds/joboe/core/util/TimeUtils.java @@ -0,0 +1,273 @@ +/* + * © SolarWinds Worldwide, LLC. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.solarwinds.joboe.core.util; + +import com.solarwinds.joboe.config.ConfigManager; +import com.solarwinds.joboe.config.ConfigProperty; +import com.solarwinds.joboe.logging.Logger; +import com.solarwinds.joboe.logging.LoggerFactory; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collections; +import java.util.List; +import java.util.concurrent.TimeUnit; + +/** + * Utils that provide timestamp in microsecond precision since the epoch time + * + *

In this static initializer, we take the current millisec as the "base" time (times 1000 so we + * get the "base" time in micro second unit) and take the current nanotime as the "reference" point + * (as zero nanosecs from the "base" time). Take note that this is what the winoboe has essentially + * been doing to get microsec precision timestamps on windows + * + *

When a timestamp is requested, we take the current nanotime minus the "reference" point, which + * gives us the "offset" from the "base" time. The offset is then added to the "base" time which + * finally returns the timestamp in microsecond. + * + *

This Utils uses a background worker to adjust the base time periodically to address Time shift + * and drifting problem + * + *

More information in ... + * + *

Auto-adjust the + * + * @author pluk + */ +public class TimeUtils { + private static final Logger logger = LoggerFactory.getLogger(); + private static final long reference; // reference in nanosecond + private static long base; // base in microsecond + private static final int DEFAULT_TIME_ADJUST_INTERVAL = 600; // in terms of sec + private static final int MIN_TIME_ADJUST_INTERVAL = 10; // in terms of sec + private static final int FLIP_SAMPLE_COUNT = 50; + private static final int BAD_TIMESTAMP_THRESHOLD = + 10; // how many bad timestamps before aborting a time adjust operation + + static { + base = System.currentTimeMillis() * 1000; + reference = + System.nanoTime(); // reference in nano second, do NOT convert this to microsecond here to + // account for the possibility of numerical overflow + + startAdjustBaseWorker( + (Integer) ConfigManager.getConfig(ConfigProperty.AGENT_TIME_ADJUST_INTERVAL)); + } + + public static long getTimestampMicroSeconds() { + long currentNano = System.nanoTime(); + long offsetMicro = + (currentNano - reference) / 1000; // should work properly even if there's numerical overflow + + long result = base + offsetMicro; + return result; + } + + /** + * Starts background working to periodically adjust the base time to address Time drifting and + * shift problem as documented in ... + * + * @param paramValue + * @return + */ + private static boolean startAdjustBaseWorker(Integer paramValue) { + final int timeAdjustInterval; + if (paramValue != null) { + if (!(paramValue >= MIN_TIME_ADJUST_INTERVAL || paramValue == 0)) { + logger.warn( + ConfigProperty.AGENT_TIME_ADJUST_INTERVAL.getConfigFileKey() + + " should be 0 (disabled) or positive integer value (seconds) >= " + + MIN_TIME_ADJUST_INTERVAL + + ", but found [" + + paramValue + + "]. Disabling time adjustment"); + timeAdjustInterval = 0; + } else { + timeAdjustInterval = paramValue; + } + } else { + timeAdjustInterval = DEFAULT_TIME_ADJUST_INTERVAL; + } + + if (timeAdjustInterval > 0) { + // using simple thread instead, see deadlock problem as documented in + // https://github.com/librato/joboe/issues/608 + DaemonThreadFactory.newInstance("time-adjustment") + .newThread( + () -> { + try { + while (true) { + adjustBase(); + TimeUnit.SECONDS.sleep(timeAdjustInterval); + } + } catch (InterruptedException e) { + logger.debug( + "Adjust time worker thread interrupted, probably due to shutdown : " + + e.getMessage()); + } + }) + .start(); + + return true; + } else { + return false; + } + } + + /** Adjusts the base time */ + private static void adjustBase() { + List dataSystem = new ArrayList(); + List badDiffs = new ArrayList(); + + Statistics statsSystem; + long newAdjustment; + + // take note of the currentTimeMillis "flip" time and set the micro second to align that as 0th + // micro second + long lastMillisec = System.currentTimeMillis(); + + while (dataSystem.size() < FLIP_SAMPLE_COUNT) { + long systemTime = System.currentTimeMillis(); + long diff = systemTime - lastMillisec; + + if (diff == 1) { // if time just flip + long microTime = getTimestampMicroSeconds(); + long diffSystem = microTime - systemTime * 1000; + dataSystem.add(diffSystem); + lastMillisec = systemTime; + } else if (diff > 1) { // busy system? not accurate + lastMillisec = systemTime; + badDiffs.add(diff); + if (badDiffs.size() + >= BAD_TIMESTAMP_THRESHOLD) { // avoid endless looping if a system never manage to get 2 + // timestamps within a millisecond + logger.debug( + "Aborting current time adjustment as system appears to be busy. Will try again in next cycle. Bad diffs: " + + badDiffs); + return; + } + } else if (diff + < 0) { // something goes terribly wrong, perhaps clock reset, do not keep trying + return; + } + } + + statsSystem = new Statistics(dataSystem); + newAdjustment = statsSystem.getMedian().longValue(); + + base -= newAdjustment; + } + + public static class Statistics> { + private final List data; + private final int size; + private boolean sorted = false; + + @SuppressWarnings({"varargs", "unchecked"}) + public Statistics(T... data) { + this.data = Arrays.asList(data); + size = data.length; + } + + public Statistics(List data) { + this.data = data; + size = data.size(); + } + + @SuppressWarnings("unchecked") + public T getSum() { + Double sum = 0.0; + for (T a : data) { + sum += a.doubleValue(); + } + return (T) sum; + } + + public double getMean() { + return getSum().doubleValue() / size; + } + + public double getVariance() { + double mean = getMean(); + double temp = 0; + for (T a : data) { + double diff = a.doubleValue() - mean; + temp += diff * diff; + } + return temp / size; + } + + public double getStdDev() { + return Math.sqrt(getVariance()); + } + + public Number getMedian() { + sortIfNotSorted(); + if (size % 2 == 0) { + return (data.get((size / 2) - 1).doubleValue() + data.get(size / 2).doubleValue()) / 2.0; + } else { + return data.get(size / 2); + } + } + + public T getPercentile(double fraction) { + sortIfNotSorted(); + + int percentileMark = (int) (size * fraction); + if (percentileMark == size) { + percentileMark -= 1; + } + return data.get(percentileMark); + } + + public T getMax() { + sortIfNotSorted(); + return data.get(data.size() - 1); + } + + public T getMin() { + sortIfNotSorted(); + return data.get(0); + } + + private synchronized void sortIfNotSorted() { + if (!sorted) { + Collections.sort(data); + sorted = true; + } + } + + @Override + public String toString() { + return "Statistics{" + + "min=" + + getMin() + + ", max=" + + getMax() + + ", mean=" + + getMean() + + ", median=" + + getPercentile(0.5) + + ", p90=" + + getPercentile(0.9) + + ", p99=" + + getPercentile(0.99) + + ", sample size=" + + data.size() + + '}'; + } + } +} diff --git a/libs/core/src/main/java/com/solarwinds/joboe/core/util/UamsClientIdReader.java b/libs/core/src/main/java/com/solarwinds/joboe/core/util/UamsClientIdReader.java new file mode 100644 index 00000000..6d725221 --- /dev/null +++ b/libs/core/src/main/java/com/solarwinds/joboe/core/util/UamsClientIdReader.java @@ -0,0 +1,149 @@ +/* + * © SolarWinds Worldwide, LLC. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.solarwinds.joboe.core.util; + +import com.solarwinds.joboe.logging.Logger; +import com.solarwinds.joboe.logging.LoggerFactory; +import io.opentelemetry.api.internal.InstrumentationUtil; +import java.io.BufferedReader; +import java.io.FileReader; +import java.io.IOException; +import java.io.InputStreamReader; +import java.net.HttpURLConnection; +import java.net.Proxy; +import java.net.URL; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.Paths; +import java.nio.file.attribute.FileTime; +import java.time.Instant; +import java.util.Optional; +import java.util.concurrent.atomic.AtomicReference; +import org.json.JSONException; +import org.json.JSONObject; + +public class UamsClientIdReader { + private static final Logger logger = LoggerFactory.getLogger(); + private static final HostInfoUtils.OsType osType = HostInfoUtils.getOsType(); + private static final Path uamsClientIdPath; + private static final AtomicReference uamsClientId = new AtomicReference<>(); + private static final AtomicReference lastModified = + new AtomicReference<>(FileTime.from(Instant.EPOCH)); + + static { + if (osType == HostInfoUtils.OsType.WINDOWS) { + String programData = System.getenv("PROGRAMDATA"); + if (programData == null) { + programData = "C:\\ProgramData\\"; + } + uamsClientIdPath = Paths.get(programData, "SolarWinds", "UAMSClient", "uamsclientid"); + } else { + uamsClientIdPath = Paths.get("/", "opt", "solarwinds", "uamsclient", "var", "uamsclientid"); + } + logger.debug("Set uamsclientid path to " + uamsClientIdPath); + } + + public static String getUamsClientId() { + try { + FileTime modifiedTime = Files.getLastModifiedTime(uamsClientIdPath); + if (!lastModified.get().equals(modifiedTime)) { + lastModified.set(modifiedTime); + uamsClientId.set(sanitize(readFirstLine(uamsClientIdPath))); + logger.debug( + "Updated uamsclientid to " + uamsClientId.get() + ", lastModifiedTime=" + modifiedTime); + } + } catch (IOException e) { + logger.debug(String.format("Cannot read the file[%s] due error: %s", uamsClientIdPath, e)); + getUamsClientIdViaRestApi().ifPresent(uamsClientId::set); + } + return uamsClientId.get(); + } + + static Optional getUamsClientIdViaRestApi() { + AtomicReference> result = new AtomicReference<>(Optional.empty()); + InstrumentationUtil.suppressInstrumentation( + () -> { + HttpURLConnection connection = null; + try { + connection = + (HttpURLConnection) + new URL("http://127.0.0.1:2113/info/uamsclient").openConnection(Proxy.NO_PROXY); + connection.setRequestMethod("GET"); + + int statusCode = connection.getResponseCode(); + if (statusCode >= 200 && statusCode < 300) { + String payload; + try (BufferedReader reader = + new BufferedReader(new InputStreamReader(connection.getInputStream()))) { + StringBuilder sb = new StringBuilder(); + String line; + while ((line = reader.readLine()) != null) { + sb.append(line); + } + payload = sb.toString(); + } + JSONObject jsonPayload = new JSONObject(payload); + String clientId = jsonPayload.getString("uamsclient_id"); + + logger.debug( + String.format( + "Got UAMS client ID(%s) from API, using hardcoded endpoint", clientId)); + result.set(Optional.ofNullable(clientId)); + } else { + logger.debug( + String.format( + "Request to UAMS REST endpoint failed. Status=%d, payload=%s", + statusCode, null)); + } + + } catch (IOException | JSONException exception) { + logger.debug(String.format("Error reading from UAMS REST endpoint\n%s", exception)); + } finally { + if (connection != null) { + connection.disconnect(); + } + } + }); + return result.get(); + } + + private static String readFirstLine(Path filePath) throws IOException { + String line = null; + try (BufferedReader br = new BufferedReader(new FileReader(filePath.toFile()))) { + line = br.readLine(); + } + return line; + } + + private static String sanitize(String id) { + String res = null; + try { + if (id.length() != 36) { // UUID in 8-4-4-4-12 format + throw new IllegalArgumentException("incorrect length"); + } + + String[] parts = id.split("-"); + if (parts.length != 5) { + throw new IllegalArgumentException("incorrect format"); + } + res = id; + } catch (IllegalArgumentException e) { + logger.debug("Discarded invalid UAMS client id: " + id, e); + } + return res; + } +} diff --git a/libs/core/src/main/java/com/solarwinds/joboe/core/util/diagnostic/DiagnosticTools.java b/libs/core/src/main/java/com/solarwinds/joboe/core/util/diagnostic/DiagnosticTools.java new file mode 100644 index 00000000..f766ab5b --- /dev/null +++ b/libs/core/src/main/java/com/solarwinds/joboe/core/util/diagnostic/DiagnosticTools.java @@ -0,0 +1,445 @@ +/* + * © SolarWinds Worldwide, LLC. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.solarwinds.joboe.core.util.diagnostic; + +import com.solarwinds.joboe.config.ConfigContainer; +import com.solarwinds.joboe.config.ConfigGroup; +import com.solarwinds.joboe.config.ConfigManager; +import com.solarwinds.joboe.config.ConfigProperty; +import com.solarwinds.joboe.config.InvalidConfigException; +import com.solarwinds.joboe.config.JavaRuntimeVersionChecker; +import com.solarwinds.joboe.config.ServiceKeyUtils; +import com.solarwinds.joboe.core.rpc.Client; +import com.solarwinds.joboe.core.rpc.ClientException; +import com.solarwinds.joboe.core.rpc.ResultCode; +import com.solarwinds.joboe.core.rpc.RpcClientManager; +import com.solarwinds.joboe.core.rpc.RpcClientManager.OperationType; +import com.solarwinds.joboe.logging.LogSetting; +import com.solarwinds.joboe.logging.Logger; +import com.solarwinds.joboe.logging.LoggerConfiguration; +import com.solarwinds.joboe.logging.LoggerFactory; +import java.io.FileNotFoundException; +import java.io.PrintStream; +import java.nio.file.Path; +import java.util.Collections; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.concurrent.ExecutionException; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.TimeoutException; + +/** + * Diagnostic tools that verify service key and connectivity by connecting to the collector server + * + *

This is packaged into the java agent jar and can be invoked as: + * + *

java -Djava.security.debug=certpath,provider -Djavax.net.debug=ssl:session -cp + * solarwinds-apm-agent.jar com.solarwinds.joboe.core.util.diagnostic.DiagnosticTools + * service_key=YourSolarWindsApiToken:YourServiceName [optional parameters] + * + *

For example: java -Djava.security.debug=certpath,provider -Djavax.net.debug=ssl:session -cp + * solarwinds-apm-agent.jar com.solarwinds.joboe.core.util.diagnostic.DiagnosticTools + * service_key=0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef:my-service + * timeout=10000 log_file=solarwinds-apm-diagnostics.log + * + * @author Patson + */ +public class DiagnosticTools { + private static final int DEFAULT_TIMEOUT = 8000; // 8 seconds + private static final Logger logger = LoggerFactory.getLogger(); + private static final int APPOPTICS_API_TOKEN_LENGTH = 64; + private static final int SWOKEN_API_TOKEN_LENGTH = 71; + + private static int timeout = DEFAULT_TIMEOUT; + private static final LogSetting DEFAULT_LOG_SETTING = + new LogSetting(Logger.Level.DEBUG, true, true, null, null, null); + + public static void main(String[] args) { + if (!JavaRuntimeVersionChecker.isJdkVersionSupported()) { + logger.error( + "The current Java runtime version is not supported. Minimum version=" + + JavaRuntimeVersionChecker.minVersionSupported); + return; + } + + Result result = null; + try { + ConfigContainer container = new ConfigContainer(); + String serviceKey = null; + if (args.length == 0) { + logger.info("No parameters string is provided, using defaults"); + printUsage(); + } else { + Map parameters = parseParameters(args); + String logFileString = parameters.get(ParameterKey.LOG_FILE); + + if (logFileString != null) { + PrintStream logFileStream = getLogFile(logFileString); + if (logFileStream != null) { + System.setOut(logFileStream); + System.setErr(logFileStream); + } + } + + serviceKey = parameters.get(ParameterKey.SERVICE_KEY); + String timeoutFromParameter = parameters.get(ParameterKey.TIMEOUT); + if (timeoutFromParameter != null) { + try { + timeout = Integer.parseInt(timeoutFromParameter); + } catch (NumberFormatException e) { + logger.warn( + "Cannot parse timeout from argument [" + + timeoutFromParameter + + "]. Using default " + + DEFAULT_TIMEOUT + + " ms instead."); + } + } + + String collector = parameters.get(ParameterKey.COLLECTOR); + if (collector != null) { + container.putByStringValue(ConfigProperty.AGENT_COLLECTOR, collector); + logger.info("Using collector endpoint " + container.get(ConfigProperty.AGENT_COLLECTOR)); + } + } + + logger.info("Using service key " + ServiceKeyUtils.maskServiceKey(serviceKey)); + container.put(ConfigProperty.AGENT_LOGGING, DEFAULT_LOG_SETTING); + ConfigContainer subset = container.subset(ConfigGroup.AGENT); + LoggerFactory.init( + LoggerConfiguration.builder() + .logFile((Path) subset.get(ConfigProperty.AGENT_LOG_FILE)) + .logSetting((LogSetting) subset.get(ConfigProperty.AGENT_LOGGING)) + .debug( + subset.get(ConfigProperty.AGENT_DEBUG) != null + && (boolean) subset.get(ConfigProperty.AGENT_DEBUG)) + .build()); + + ConfigManager.initialize(container); + result = testServiceKey(serviceKey); + } catch (InvalidArgumentsException | InvalidConfigException e) { + logger.warn(e.getMessage()); + printUsage(); + result = Result.invalidArguments(args); + } finally { + if (result != null) { + logger.info(result.toString()); + System.exit(result.resultType.exitCode); + } + } + } + + private static void printUsage() { + logger.info( + "Usage example : java -Djava.security.debug=certpath,provider -Djavax.net.debug=ssl:session -cp solarwinds-apm-agent.jar com.solarwinds.joboe.core.util.diagnostic.DiagnosticTools service_key=YourSolarWindsApiToken:YourServiceName"); + logger.info( + "All other program parameters except for the service_key are optional and in format of [key]=[value], available parameters are:"); + logger.info("service_key : Service key to be used for the diagnostics"); + logger.info("timeout : Max time to wait for the diagnostics to finish"); + logger.info( + "log_file : File location to print the logs to, could either be relative or absolute path"); + logger.info("collector : The collector endpoint for the diagnostics tool to connect to"); + } + + private enum ParameterKey { + SERVICE_KEY("service_key"), + COLLECTOR("collector"), + TIMEOUT("timeout"), + LOG_FILE("log_file"), + AGENT_CONFIG("config"); + + private final String key; + private static final Map map = + new HashMap(); + + static { + for (ParameterKey parameter : ParameterKey.values()) { + map.put(parameter.key, parameter); + } + } + + ParameterKey(String key) { + this.key = key; + } + + private static ParameterKey fromKey(String key) { + return map.get(key); + } + } + + private static PrintStream getLogFile(String logFileString) { + try { + return new PrintStream(logFileString); + } catch (FileNotFoundException e) { + logger.warn("Cannot write to log file [" + logFileString + "]"); + return null; + } + } + + private static Map parseParameters(String[] argments) + throws InvalidArgumentsException { + Map parameters = new HashMap(); + for (String keyValue : argments) { + int separator = keyValue.indexOf('='); + if (separator <= 0) { + throw new InvalidArgumentsException( + "parameter entry [" + + keyValue + + "] is unexpected, should be in the form of [key]=[value]"); + } else { + String keyString = keyValue.substring(0, separator); + ParameterKey key = ParameterKey.fromKey(keyString); + if (key == null) { + throw new InvalidArgumentsException( + "parameter key [" + keyString + "] is not a valid parameter key"); + } + parameters.put(key, keyValue.substring(separator + 1)); + } + } + + return parameters; + } + + private static class Result { + private final ResultType resultType; + private final String message; + + private static final Result OK = new Result(ResultType.OK, "Diagnostics successful"); + private static final Result TIMEOUT_FAILED = + new Result( + ResultType.CONNECTION_FAILURE, + "Failed to get response from SolarWinds server after waiting for " + + timeout + + " milliseconds"); + private static final Result TRY_LATER = + new Result(ResultType.TRY_LATER, "SolarWinds server returned non-ok status code TRY_LATER"); + private static final Result LIMIT_EXCEEDED = + new Result( + ResultType.LIMIT_EXCEEDED, + "SolarWinds server returned non-ok status code LIMIT_EXCEEDED"); + + private Result(ResultType resultType) { + this(resultType, null); + } + + private Result(ResultType resultType, String message) { + this.resultType = resultType; + this.message = message; + } + + private static Result ok() { + return OK; + } + + private static Result invalidServiceKey(String serviceKey, String warning) { + return new Result( + ResultType.INVALID_SERVICE_KEY, + "Service key [" + + ServiceKeyUtils.maskServiceKey(serviceKey) + + "] is invalid : " + + warning); + } + + private static Result connectionFailure(ClientException e) { + StringBuilder message = + new StringBuilder("Failed to connect to AppOptics collector due to connection problem."); + + if (e.getMessage() != null) { + message.append(" Reason: " + e.getMessage()); + } + + return new Result(ResultType.CONNECTION_FAILURE_FATAL, message.toString()); + } + + private static Result timeout() { + return TIMEOUT_FAILED; + } + + private static Result unexpectedException(Exception e) { + StringBuilder message = + new StringBuilder( + "Failed to connect to SolarWinds collector due to unexpected exception."); + + if (e != null && e.getMessage() != null) { + message.append(" Reason: " + e.getMessage()); + } + + return new Result(ResultType.UNKNOWN_ERROR, message.toString()); + } + + private static Result invalidFormatArgument(String argumentKey, String argumentValue) { + return new Result( + ResultType.INVALID_FORMAT_ARGUMENT, + "Invalid format for argument [" + argumentKey + "] with value [" + argumentValue + "]"); + } + + private static Result invalidArguments(String[] arguments) { + String argumentString = ""; + for (String argument : arguments) { + if (argument.contains("=") && argument.split("=")[0].equals(ParameterKey.SERVICE_KEY.key)) { + String[] arg = argument.split("="); + argumentString = + argumentString + " " + arg[0] + " " + ServiceKeyUtils.maskServiceKey(arg[1]); + } else { + argumentString += argument + " "; + } + } + argumentString = argumentString.trim(); + return new Result( + ResultType.INVALID_ARGUMENTS, "[" + argumentString + "] is not a valid argument"); + } + + private static Result tryLater() { + return TRY_LATER; + } + + private static Result limitExceeded() { + return LIMIT_EXCEEDED; + } + + @Override + public String toString() { + return "[" + resultType + "]" + (message != null ? " message: " + message : ""); + } + } + + private enum ResultType { + OK(0), + UNKNOWN_ERROR(101), + INVALID_FORMAT_ARGUMENT(102), + INVALID_ARGUMENTS(103), + INVALID_SERVICE_KEY(104), + CONNECTION_FAILURE(105), + TRY_LATER(106), + LIMIT_EXCEEDED(107), + CONNECTION_FAILURE_FATAL(108); + + private final int exitCode; + + ResultType(int exitCode) { + this.exitCode = exitCode; + } + } + + private static Result testServiceKey(String serviceKey) { + if (!isValidServiceKeyFormat(serviceKey)) { + return Result.invalidFormatArgument( + ParameterKey.SERVICE_KEY.key, ServiceKeyUtils.maskServiceKey(serviceKey)); + } + + Client client = null; + try { + client = RpcClientManager.getClient(OperationType.STATUS, serviceKey); + com.solarwinds.joboe.core.rpc.Result rpcCallResult = + client.postStatus(generateDiagnosticMessage(), null).get(timeout, TimeUnit.MILLISECONDS); + + if (!rpcCallResult + .getWarning() + .isEmpty()) { // warning for postStatus call is likely for service key errors, see + // https://swicloud.atlassian.net/browse/AO-16547?focusedCommentId=197795 + return Result.invalidServiceKey(serviceKey, rpcCallResult.getWarning()); + } else if (rpcCallResult.getResultCode() == ResultCode.TRY_LATER) { + return Result.tryLater(); + } else if (rpcCallResult.getResultCode() == ResultCode.LIMIT_EXCEEDED) { + return Result.limitExceeded(); + } else { // all other result code are considered as successful + return Result.ok(); + } + + } catch (ClientException e) { + return Result.connectionFailure(e); + } catch (ExecutionException e) { + if (e.getCause() instanceof com.solarwinds.joboe.core.rpc.ClientException) { + return Result.connectionFailure((ClientException) e.getCause()); + } else { + return Result.unexpectedException(e); + } + } catch (TimeoutException e) { + return Result.timeout(); + } catch (Exception e) { + return Result.unexpectedException(e); + } finally { + if (client != null) { + client.close(); + } + } + } + + private static boolean isValidServiceKeyFormat(String serviceKey) { + if (serviceKey == null) { + logger.warn("Service key is not defined!"); + return false; + } + + String[] tokens = serviceKey.split(":", 2); + if (tokens.length != 2) { + logger.warn( + "Service Key [" + + ServiceKeyUtils.maskServiceKey(serviceKey) + + "] format is incorrect - not in :"); + return false; + } + + String apiToken = tokens[0]; + String serviceName = tokens[1]; + + if (apiToken.length() != APPOPTICS_API_TOKEN_LENGTH + && apiToken.length() != SWOKEN_API_TOKEN_LENGTH) { + logger.warn( + "Service Key [" + + ServiceKeyUtils.maskServiceKey(serviceKey) + + "] format is incorrect - API token is not in " + + APPOPTICS_API_TOKEN_LENGTH + + " or " + + SWOKEN_API_TOKEN_LENGTH + + " characters. Found " + + apiToken.length() + + " character(s)"); + return false; + } + + if (serviceName.trim().isEmpty()) { + logger.warn( + "Service Name should not be empty in Service Key [" + + ServiceKeyUtils.maskServiceKey(serviceKey) + + "]"); + return false; + } + + return true; + } + + private static List> generateDiagnosticMessage() { + String version = DiagnosticTools.class.getPackage().getImplementationVersion(); + logger.info("Fetched version " + version + " in the diagnostic tool."); + Map initMessage = new HashMap(); + + initMessage.put("__Diagnostic", true); + + if (version != null) { + initMessage.put("Java.SolarWindsAPM.Version", version); + } + + initMessage.put( + "DiagnosticTimestamp", System.currentTimeMillis() / 1000); // seconds since epoch + + logger.info("Connectivity check message: " + initMessage); + + return Collections.singletonList(initMessage); + } +} diff --git a/libs/core/src/main/java/com/solarwinds/joboe/core/util/diagnostic/InvalidArgumentsException.java b/libs/core/src/main/java/com/solarwinds/joboe/core/util/diagnostic/InvalidArgumentsException.java new file mode 100644 index 00000000..d2bf3836 --- /dev/null +++ b/libs/core/src/main/java/com/solarwinds/joboe/core/util/diagnostic/InvalidArgumentsException.java @@ -0,0 +1,23 @@ +/* + * © SolarWinds Worldwide, LLC. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.solarwinds.joboe.core.util.diagnostic; + +public class InvalidArgumentsException extends Exception { + public InvalidArgumentsException(String message) { + super(message); + } +} diff --git a/libs/core/src/test/java/com/solarwinds/joboe/core/AtomicEventReporterStatsTest.java b/libs/core/src/test/java/com/solarwinds/joboe/core/AtomicEventReporterStatsTest.java new file mode 100644 index 00000000..06b8fbdd --- /dev/null +++ b/libs/core/src/test/java/com/solarwinds/joboe/core/AtomicEventReporterStatsTest.java @@ -0,0 +1,44 @@ +/* + * © SolarWinds Worldwide, LLC. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.solarwinds.joboe.core; + +import static org.junit.jupiter.api.Assertions.*; + +import org.junit.jupiter.api.Test; + +class AtomicEventReporterStatsTest { + + private final AtomicEventReporterStats tested = new AtomicEventReporterStats(() -> 10); + + @Test + void testConsumeStats() { + tested.incrementFailedCount(1); + tested.incrementProcessedCount(1); + tested.incrementOverflowedCount(1); + + tested.incrementSentCount(1); + tested.setQueueCount(1); + + EventReporterStats expected = new EventReporterStats(1, 1, 1, 1, 1); + EventReporterStats actual = tested.consumeStats(); + assertEquals(expected, actual); + + expected = new EventReporterStats(0, 0, 0, 10, 0); + actual = tested.consumeStats(); + assertEquals(expected, actual); + } +} diff --git a/libs/core/src/test/java/com/solarwinds/joboe/core/ContextTest.java b/libs/core/src/test/java/com/solarwinds/joboe/core/ContextTest.java new file mode 100644 index 00000000..05516b3b --- /dev/null +++ b/libs/core/src/test/java/com/solarwinds/joboe/core/ContextTest.java @@ -0,0 +1,366 @@ +/* + * © SolarWinds Worldwide, LLC. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.solarwinds.joboe.core; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertNotSame; +import static org.junit.jupiter.api.Assertions.assertNull; +import static org.junit.jupiter.api.Assertions.assertSame; +import static org.junit.jupiter.api.Assertions.assertTrue; + +import com.solarwinds.joboe.core.TestReporter.DeserializedEvent; +import com.solarwinds.joboe.core.settings.TestSettingsReader; +import com.solarwinds.joboe.core.settings.TestSettingsReader.SettingsMockupBuilder; +import com.solarwinds.joboe.core.util.TestUtils; +import com.solarwinds.joboe.sampling.Metadata; +import com.solarwinds.joboe.sampling.SettingsArg; +import com.solarwinds.joboe.sampling.TraceDecisionUtil; +import com.solarwinds.joboe.sampling.TracingMode; +import java.util.ArrayList; +import java.util.List; +import java.util.Map; +import java.util.concurrent.ExecutionException; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; +import java.util.concurrent.Future; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicReference; +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.Test; + +public class ContextTest { + private static final TestSettingsReader testSettingsReader = TestUtils.initSettingsReader(); + private static final TestReporter tracingReporter = TestUtils.initTraceReporter(); + + @AfterEach + protected void tearDown() throws Exception { + + testSettingsReader.reset(); + tracingReporter.reset(); + } + + @Test + public void testContext() throws Exception { + + // Clear metadata: it may have been set by a previous test + Context.clearMetadata(); + + // Make sure we can get metadata from our context: + final Metadata md = Context.getMetadata(); + assertFalse(md.isValid()); + + md.randomize(); + assertTrue(md.isValid()); + + Metadata md2 = Context.getMetadata(); + assertSame(md, md2); + + // Make sure we can set IDs: + Metadata rndMd = new Metadata(); + rndMd.randomize(); + + Context.setMetadata(rndMd.toHexString()); + assertEquals(Context.getMetadata().toHexString(), rndMd.toHexString()); + + // Verify that the context is inherited in child thread + final Metadata parentMD = Context.getMetadata(); + final AtomicReference assertionError = new AtomicReference(); + + Thread thr = + new Thread( + () -> { + try { + Metadata childMD = Context.getMetadata(); + assertNotSame(parentMD, childMD); // different object + assertEquals(parentMD.toHexString(), childMD.toHexString()); // but same metadata + assertTrue(childMD.isValid()); + } catch (Error e) { + assertionError.set(e); + } + }); + + thr.start(); + thr.join(); + + if (assertionError.get() != null) { + throw assertionError.get(); + } + } + + @Test + public void testInheritContext() throws InterruptedException { + Metadata context = Context.getMetadata(); + context.randomize(); // make a valid context + + TestThread thread; + + thread = new TestThread(); + thread.start(); + thread.join(); + + assertEquals( + context.toHexString(), thread.threadContext.toHexString()); // should have same xtrace id + assertNotSame( + context, + thread + .threadContext); // but they should not be the same object, the inherited context should + // be a clone + + // test config flag + Context.setSkipInheritingContext(true); + thread = new TestThread(); + thread.start(); + thread.join(); + Context.setSkipInheritingContext(false); + + assertFalse(thread.threadContext.isValid()); // no context inherited + // disable inherit context + testSettingsReader.put( + new SettingsMockupBuilder() + .withFlags(TracingMode.ALWAYS) + .withSampleRate(TraceDecisionUtil.SAMPLE_RESOLUTION) + .withSettingsArg(SettingsArg.DISABLE_INHERIT_CONTEXT, true) + .build()); + thread = new TestThread(); + thread.start(); + thread.join(); + + assertFalse(thread.threadContext.isValid()); // no context inherited + + // re-enable inherit context + testSettingsReader.put( + new SettingsMockupBuilder() + .withFlags(TracingMode.ALWAYS) + .withSampleRate(TraceDecisionUtil.SAMPLE_RESOLUTION) + .build()); + thread = new TestThread(); + thread.start(); + thread.join(); + + assertEquals( + context.toHexString(), thread.threadContext.toHexString()); // should have same xtrace id + assertNotSame( + context, + thread + .threadContext); // but they should not be the same object, the inherited context should + // be a clone + } + + @Test + public void testCreateEventWithContextTtl() throws InterruptedException { + Context.clearMetadata(); // this triggers creation of new metadata + Context.getMetadata().randomize(true); // make it a valid context + + // warm-up run...as first call can take a few secs (for getting host info) + Event event = Context.createEvent(); + event.report(tracingReporter); + tracingReporter.reset(); + + Context.clearMetadata(); // this triggers creation of new metadata + Context.getMetadata().randomize(); // make it a valid context + int newTtl = 2; + // set to 2 secs + testSettingsReader.put( + new SettingsMockupBuilder() + .withFlags(TracingMode.ALWAYS) + .withSampleRate(TraceDecisionUtil.SAMPLE_RESOLUTION) + .withSettingsArg(SettingsArg.MAX_CONTEXT_AGE, newTtl) + .build()); + + event = Context.createEvent(); + event.report(tracingReporter); // should be okay + + List deserializedEvents = tracingReporter.getSentEvents(); + assertEquals(1, deserializedEvents.size()); // ok, since it's within 2 secs + tracingReporter.reset(); + + TimeUnit.SECONDS.sleep(newTtl + 1); + + event = Context.createEvent(); + event.report(tracingReporter); // not reported as this is more than 2 secs since creation + deserializedEvents = tracingReporter.getSentEvents(); + assertTrue(deserializedEvents.isEmpty()); // no events + + Context.clearMetadata(); // ensure reset metadata also reset ttl + Context.getMetadata().randomize(); // make it a valid context + + event = Context.createEvent(); + event.report(tracingReporter); // this event should now be reported as this is a new metadata + + deserializedEvents = tracingReporter.getSentEvents(); + assertEquals(1, deserializedEvents.size()); + + tracingReporter.reset(); + } + + @Test + public void testCreateEventWithMaxEvents() throws InterruptedException { + Context.clearMetadata(); // this triggers creation of new metadata + Context.getMetadata().randomize(true); // make it a valid context + + List deserializedEvents; + + // now simulate a max event change + Context.clearMetadata(); // this triggers creation of new metadata + Context.getMetadata().randomize(true); // make it a valid context + int newMaxEvent = 10; + testSettingsReader.put( + new SettingsMockupBuilder() + .withFlags(TracingMode.ALWAYS) + .withSampleRate(TraceDecisionUtil.SAMPLE_RESOLUTION) + .withSettingsArg(SettingsArg.MAX_CONTEXT_EVENTS, newMaxEvent) + .build()); + + for (int i = 0; i < newMaxEvent; i++) { + Event event = Context.createEvent(); + event.report(tracingReporter); + } + + deserializedEvents = tracingReporter.getSentEvents(); + assertEquals(newMaxEvent, deserializedEvents.size()); // ok within limit + tracingReporter.reset(); + + Event noopEvent = Context.createEvent(); // exceeding limit + noopEvent.report(tracingReporter); + deserializedEvents = tracingReporter.getSentEvents(); + tracingReporter.reset(); + assertTrue(deserializedEvents.isEmpty()); // no events + + assertTrue(Context.isValid()); // context should still be valid + assertTrue(Context.getMetadata().isSampled()); // and it should still be flagged as sampled + + Context.clearMetadata(); // this triggers creation of new metadata + Context.getMetadata().randomize(true); // make it a valid context + + ExecutorService threadPool = Executors.newCachedThreadPool(); + + for (int i = 0; i < newMaxEvent + 50; i++) { + threadPool.submit( + () -> { + Event event = Context.createEvent(); + event.report(tracingReporter); + }); + } + + threadPool.shutdown(); + threadPool.awaitTermination(10, TimeUnit.SECONDS); + + assertEquals( + newMaxEvent, + tracingReporter + .getSentEvents() + .size()); // collected event count should be newMaxEvent (not +1) + tracingReporter.reset(); + + noopEvent = Context.createEvent(); // exceeding limit, this should return a noop + noopEvent.report(tracingReporter); + deserializedEvents = tracingReporter.getSentEvents(); + assertTrue(deserializedEvents.isEmpty()); // no events + } + + @Test + public void testMaxBacktraces() throws InterruptedException, ExecutionException { + Context.clearMetadata(); // this triggers creation of new metadata + Context.getMetadata().randomize(true); // make it a valid context + + List deserializedEvents; + + // now simulate a max event change + Context.clearMetadata(); // this triggers creation of new metadata + Context.getMetadata().randomize(true); // make it a valid context + int newBacktraces = 10; + testSettingsReader.put( + new SettingsMockupBuilder() + .withFlags(TracingMode.ALWAYS) + .withSampleRate(TraceDecisionUtil.SAMPLE_RESOLUTION) + .withSettingsArg(SettingsArg.MAX_CONTEXT_BACKTRACES, newBacktraces) + .build()); + + for (int i = 0; i < newBacktraces; i++) { + Event event = Context.createEvent(); + event.addInfo("Backtrace", "some backtrace"); + event.addInfo("OtherKv", "some value"); + event.report(tracingReporter); + } + + deserializedEvents = tracingReporter.getSentEvents(); + assertEquals(10, deserializedEvents.size()); + for (DeserializedEvent deseralizedEvent : deserializedEvents) { + assertEquals("some backtrace", deseralizedEvent.getSentEntries().get("Backtrace")); + assertEquals("some value", deseralizedEvent.getSentEntries().get("OtherKv")); + } + tracingReporter.reset(); + + Event event = Context.createEvent(); + event.addInfo("Backtrace", "some backtrace"); // exceeding limit + event.addInfo("OtherKv", "some value"); // this should still get through + event.report(tracingReporter); + + deserializedEvents = tracingReporter.getSentEvents(); + assertEquals(1, deserializedEvents.size()); + DeserializedEvent deseralizedEvent = deserializedEvents.get(0); + assertNull(deseralizedEvent.getSentEntries().get("Backtrace")); + assertEquals("some value", deseralizedEvent.getSentEntries().get("OtherKv")); + tracingReporter.reset(); + + Context.clearMetadata(); // this triggers creation of new metadata + Context.getMetadata().randomize(true); // make it a valid context + + ExecutorService threadPool = Executors.newCachedThreadPool(); + + List> futures = new ArrayList>(); + for (int i = 0; i < newBacktraces + 10; i++) { + futures.add( + threadPool.submit( + () -> { + Event event1 = Context.createEvent(); + event1.addInfo("Backtrace", "some backtrace"); // 10 backtraces will not be reported + event1.addInfo("OtherKv", "some value"); + event1.report(tracingReporter); + })); + } + + threadPool.shutdown(); + threadPool.awaitTermination(10, TimeUnit.SECONDS); + + for (Future future : futures) { + future.get(); // check no assertion exception thrown in the runnable + } + + int backTraceCollected = 0; + for (DeserializedEvent deserializedEvent : tracingReporter.getSentEvents()) { + Map sentEntries = deserializedEvent.getSentEntries(); + if (sentEntries.containsKey("Backtrace")) { + backTraceCollected++; + } + assertEquals("some value", sentEntries.get("OtherKv")); + } + + tracingReporter.reset(); + assertEquals(newBacktraces, backTraceCollected); + } + + private static class TestThread extends Thread { + private Metadata threadContext; + + @Override + public void run() { + threadContext = Context.getMetadata(); + } + } +} diff --git a/libs/core/src/test/java/com/solarwinds/joboe/core/EventImplTest.java b/libs/core/src/test/java/com/solarwinds/joboe/core/EventImplTest.java new file mode 100644 index 00000000..9120cf3f --- /dev/null +++ b/libs/core/src/test/java/com/solarwinds/joboe/core/EventImplTest.java @@ -0,0 +1,562 @@ +/* + * © SolarWinds Worldwide, LLC. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.solarwinds.joboe.core; + +import static com.solarwinds.joboe.core.Constants.XTR_ASYNC_KEY; +import static com.solarwinds.joboe.core.Constants.XTR_EDGE_KEY; +import static com.solarwinds.joboe.core.Constants.XTR_HOSTNAME_KEY; +import static com.solarwinds.joboe.core.Constants.XTR_METADATA_KEY; +import static com.solarwinds.joboe.core.Constants.XTR_PROCESS_ID_KEY; +import static com.solarwinds.joboe.core.Constants.XTR_THREAD_ID_KEY; +import static com.solarwinds.joboe.core.Constants.XTR_TIMESTAMP_U_KEY; +import static com.solarwinds.joboe.core.Constants.XTR_XTRACE; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertNull; +import static org.junit.jupiter.api.Assertions.assertTrue; + +import com.solarwinds.joboe.config.InvalidConfigException; +import com.solarwinds.joboe.core.TestReporter.DeserializedEvent; +import com.solarwinds.joboe.core.ebson.BsonDocument; +import com.solarwinds.joboe.core.ebson.BsonReader; +import com.solarwinds.joboe.core.ebson.BsonToken; +import com.solarwinds.joboe.core.util.TestUtils; +import com.solarwinds.joboe.sampling.Metadata; +import com.solarwinds.joboe.sampling.SamplingConfiguration; +import java.net.DatagramPacket; +import java.net.DatagramSocket; +import java.nio.ByteBuffer; +import java.nio.ByteOrder; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.logging.Logger; +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; + +public class EventImplTest { + private final Logger log = Logger.getLogger(getClass().getName()); + private static final TestReporter reporter = TestUtils.initTraceReporter(); + + @BeforeEach + void setup() { + Metadata.setup(SamplingConfiguration.builder().build()); + } + + @AfterEach + protected void tearDown() throws Exception { + reporter.reset(); + Context.clearMetadata(); + } + + /* Test that events can be sent and decoded, using a mock sender. */ + @Test + public void testLocalSendEvent() throws Exception { + + // Metadata for the context: + Metadata md = new Metadata(); + md.randomize(); + log.info("Context Metadata: " + md.toHexString()); + String origMdOp = md.opHexString(); + + // Dummy up an event: + Event evt = new EventImpl(md, true); + + String testKey = "test"; + String testVal = "testFromJava"; + + evt.addInfo(testKey, testVal); + log.info("Event Metadata: " + evt.getMetadata().toHexString()); + + // Send it: + TestReporter reporter = ReporterFactory.getInstance().createTestReporter(); + evt.report(md, reporter); + + // Decode what was "sent": + ByteBuffer buf = ByteBuffer.wrap(reporter.getLastSent()).order(ByteOrder.LITTLE_ENDIAN); + BsonReader reader = BsonToken.DOCUMENT.reader(); + BsonDocument doc = (BsonDocument) reader.readFrom(buf); + + // Check that we received expected values: + assertEquals(doc.get(XTR_METADATA_KEY), evt.getMetadata().toHexString()); + assertEquals(doc.get(XTR_EDGE_KEY), origMdOp); + assertEquals(doc.get(testKey), testVal); + + // If these aren't there the test will fail due to an exception: + Integer pid = (Integer) doc.get(XTR_PROCESS_ID_KEY); + Long tid = (Long) doc.get(XTR_THREAD_ID_KEY); + String host = (String) doc.get(XTR_HOSTNAME_KEY); + Long time_u = (Long) doc.get(XTR_TIMESTAMP_U_KEY); + + log.info( + "Received: host: " + host + " pid: " + pid + " tid: " + tid + " timestamp_u:" + time_u); + } + + /** Tests network send/receive by sending UDP to ourselves */ + @Test + public void testNetworkSendEvent() throws Exception { + + // Metadata for the context: + Metadata md = new Metadata(); + md.randomize(); + log.info("Context Metadata: " + md.toHexString()); + String origMdOp = md.opHexString(); + + // Dummy up an event: + Event evt = new EventImpl(md, true); + + // Set up a UDP receiver: + int udpPort = 45352; + final DatagramSocket listener = new DatagramSocket(udpPort); + listener.setSoTimeout(5000); + byte[] rcvBuf = new byte[16384]; + final DatagramPacket pkt = new DatagramPacket(rcvBuf, rcvBuf.length); + + // Send something to ourselves + EventReporter reporter = ReporterFactory.getInstance().createUdpReporter("127.0.0.1", udpPort); + evt.report(md, reporter); + + // Did we get it? + listener.receive(pkt); + log.info("Received: " + pkt.getLength()); + + // Decode what was sent + ByteBuffer buf = ByteBuffer.wrap(pkt.getData()).order(ByteOrder.LITTLE_ENDIAN); + BsonReader reader = BsonToken.DOCUMENT.reader(); + BsonDocument doc = (BsonDocument) reader.readFrom(buf); + + // Check that we received expected values: + assertEquals(doc.get(XTR_METADATA_KEY), evt.getMetadata().toHexString()); + assertEquals(doc.get(XTR_EDGE_KEY), origMdOp); + + // If these aren't there the test will fail due to an exception: + Integer pid = (Integer) doc.get(XTR_PROCESS_ID_KEY); + Long tid = (Long) doc.get(XTR_THREAD_ID_KEY); + String host = (String) doc.get(XTR_HOSTNAME_KEY); + Long time_u = (Long) doc.get(XTR_TIMESTAMP_U_KEY); + + log.info( + "Received: host: " + host + " pid: " + pid + " tid: " + tid + " timestamp_u:" + time_u); + } + + @Test + public void testEventReport() throws Exception { + Metadata contextMetadata = new Metadata(); + contextMetadata.randomize(true); + + Event event; + event = new EventImpl(new Metadata(), false); // invalid context + event.report(contextMetadata, reporter); + assert (reporter.getSentEvents().isEmpty()); + + Metadata metadata = new Metadata(); + metadata.randomize(false); // not sampled + event = new EventImpl(metadata, false); // not sampled context + event.report(contextMetadata, reporter); + assert (reporter.getSentEvents().isEmpty()); + + event = + new EventImpl(contextMetadata, contextMetadata.toHexString(), false); // same task and op id + event.report(contextMetadata, reporter); + assert (reporter.getSentEvents().isEmpty()); + + Metadata differentTaskIdMetadata = new Metadata(); + differentTaskIdMetadata.randomize(true); + event = new EventImpl(differentTaskIdMetadata, false); // different task Id + event.report(contextMetadata, reporter); + assert (reporter.getSentEvents().isEmpty()); + + event = + new EventImpl(contextMetadata, false); // valid metadata - same task ID but different op ID + event.report(contextMetadata, null); // but reporter is null + assert (reporter.getSentEvents().isEmpty()); + + event = + new EventImpl(contextMetadata, false); // valid metadata - same task ID but different op ID + event.report(contextMetadata, reporter); // ok, should report event + assertEquals(1, reporter.getSentEvents().size()); + } + + /** + * Over-sized event with a large KV + * + * @throws InvalidConfigException + */ + @Test + public void testOversizedEvent1() throws InvalidConfigException { + String bigValue = new String(new char[1000000]); + + Event testEvent = startTrace(); + + testEvent.addInfo("key1", bigValue); + testEvent.addInfo("key2", new String[] {bigValue, bigValue, bigValue}); + testEvent.addInfo("key3", 0); + testEvent.addInfo("key4", false); + testEvent.addInfo("key5", new String[0]); + testEvent.addInfo("key6", new String[] {"hi"}); + testEvent.addInfo("key7", new Object[] {1, bigValue}); + + Metadata testEdge = new Metadata(Context.getMetadata()); + testEdge.randomizeOpID(); + testEvent.addEdge(testEdge.toHexString()); + + TestReporter reporter = ReporterFactory.getInstance().createTestReporter(); + testEvent.report(reporter); + + // Decode what was "sent": + ByteBuffer buf = ByteBuffer.wrap(reporter.getLastSent()).order(ByteOrder.LITTLE_ENDIAN); + BsonReader reader = BsonToken.DOCUMENT.reader(); + BsonDocument doc = (BsonDocument) reader.readFrom(buf); + + // Check that we received expected values: + assertTrue(bigValue.startsWith((String) doc.get("key1"))); + assertEquals( + 0, + ((BsonDocument) doc.get("key2")) + .size()); // bson converts it to BsonDocument, should not include any as the value is + // too big to even fit in one + assertEquals(0, doc.get("key3")); + assertEquals(false, doc.get("key4")); + assertEquals(0, ((BsonDocument) doc.get("key5")).size()); // bson converts it to BsonDocument + assertEquals(1, ((BsonDocument) doc.get("key6")).size()); // bson converts it to BsonDocument + assertEquals( + 1, + ((BsonDocument) doc.get("key7")) + .size()); // bson converts it to BsonDocument, should not include the 2nd argument as + // it's too big + assertEquals(testEdge.opHexString(), doc.get(Constants.XTR_EDGE_KEY)); + } + + /** + * Over-sized event with too many KV pairs + * + * @throws InvalidConfigException + */ + @Test + public void testOversizedEvent2() throws InvalidConfigException { + Event testEvent = startTrace(); + + for (int i = 0; i < 10000; i++) { + testEvent.addInfo(String.valueOf(i), i); + } + + TestReporter reporter = ReporterFactory.getInstance().createTestReporter(); + testEvent.report(reporter); + + // Decode what was "sent": + ByteBuffer buf = ByteBuffer.wrap(reporter.getLastSent()).order(ByteOrder.LITTLE_ENDIAN); + BsonReader reader = BsonToken.DOCUMENT.reader(); + BsonDocument doc = (BsonDocument) reader.readFrom(buf); + + assertTrue(doc.containsKey(Constants.XTR_THREAD_ID_KEY)); + assertTrue(doc.containsKey(Constants.XTR_HOSTNAME_KEY)); + assertTrue(doc.containsKey(Constants.XTR_METADATA_KEY)); + assertTrue(doc.containsKey(XTR_XTRACE)); + assertTrue(doc.containsKey(Constants.XTR_PROCESS_ID_KEY)); + assertTrue(doc.containsKey(Constants.XTR_TIMESTAMP_U_KEY)); + + // Check that we received expected max entries defined in Event.MAX_KEY_COUNT + for (int i = 0; i < EventImpl.MAX_KEY_COUNT - 7; i++) { // minus the 7 important keys above + assertEquals(doc.get(String.valueOf(i)), i); + } + } + + /** + * Over-sized event with single KV as a huge array + * + * @throws InvalidConfigException + */ + @Test + public void testOversizedEvent3() throws InvalidConfigException { + final int ARRAY_SIZE = 100000; + Object[] hugeArray = new Object[ARRAY_SIZE]; + for (int i = 0; i < ARRAY_SIZE; i++) { + hugeArray[i] = 0; + } + + Event testEvent = startTrace(); + testEvent.addInfo("HugeArray", hugeArray); + + TestReporter reporter = ReporterFactory.getInstance().createTestReporter(); + testEvent.report(reporter); + + // Decode what was "sent": + ByteBuffer buf = ByteBuffer.wrap(reporter.getLastSent()).order(ByteOrder.LITTLE_ENDIAN); + BsonReader reader = BsonToken.DOCUMENT.reader(); + BsonDocument doc = (BsonDocument) reader.readFrom(buf); + + assertTrue(doc.containsKey(Constants.XTR_THREAD_ID_KEY)); + assertTrue(doc.containsKey(Constants.XTR_HOSTNAME_KEY)); + assertTrue(doc.containsKey(Constants.XTR_METADATA_KEY)); + assertTrue(doc.containsKey(XTR_XTRACE)); + assertTrue(doc.containsKey(Constants.XTR_PROCESS_ID_KEY)); + assertTrue(doc.containsKey(Constants.XTR_TIMESTAMP_U_KEY)); + + // Check that at least part of the array is included + assertTrue(doc.containsKey("HugeArray")); + } + + /** + * Over-sized event with single KV as a very long string + * + * @throws InvalidConfigException + */ + @Test + public void testOversizedEvent4() throws InvalidConfigException { + final int STRING_APPEND_COUNT = 100000; + StringBuffer longString = new StringBuffer(); + for (int i = 0; i < STRING_APPEND_COUNT; i++) { + longString.append(i); + } + + Event testEvent = startTrace(); + testEvent.addInfo("LongString", longString.toString()); + + TestReporter reporter = ReporterFactory.getInstance().createTestReporter(); + testEvent.report(reporter); + + // Decode what was "sent": + ByteBuffer buf = ByteBuffer.wrap(reporter.getLastSent()).order(ByteOrder.LITTLE_ENDIAN); + BsonReader reader = BsonToken.DOCUMENT.reader(); + BsonDocument doc = (BsonDocument) reader.readFrom(buf); + + assertTrue(doc.containsKey(Constants.XTR_THREAD_ID_KEY)); + assertTrue(doc.containsKey(Constants.XTR_HOSTNAME_KEY)); + assertTrue(doc.containsKey(Constants.XTR_METADATA_KEY)); + assertTrue(doc.containsKey(XTR_XTRACE)); + assertTrue(doc.containsKey(Constants.XTR_PROCESS_ID_KEY)); + assertTrue(doc.containsKey(Constants.XTR_TIMESTAMP_U_KEY)); + + // Check that at least part of the longString is included + assertTrue(doc.containsKey("LongString")); + } + + /** + * Over-sized event with alot of KVs and long keys and values + * + * @throws InvalidConfigException + */ + @Test + public void testOversizedEvent5() throws InvalidConfigException { + final int STRING_APPEND_COUNT = 1000; + StringBuffer longString = new StringBuffer(); + for (int i = 0; i < STRING_APPEND_COUNT; i++) { + longString.append(i); + } + final String longPrefix = longString.toString(); + + Event testEvent = startTrace(); + for (int i = 0; i < 100; i++) { + testEvent.addInfo(longPrefix + i, longPrefix + 1); + } + + TestReporter reporter = ReporterFactory.getInstance().createTestReporter(); + testEvent.report(reporter); + + // Decode what was "sent": + ByteBuffer buf = ByteBuffer.wrap(reporter.getLastSent()).order(ByteOrder.LITTLE_ENDIAN); + BsonReader reader = BsonToken.DOCUMENT.reader(); + BsonDocument doc = (BsonDocument) reader.readFrom(buf); + + assertTrue(doc.containsKey(Constants.XTR_THREAD_ID_KEY)); + assertTrue(doc.containsKey(Constants.XTR_HOSTNAME_KEY)); + assertTrue(doc.containsKey(Constants.XTR_METADATA_KEY)); + assertTrue(doc.containsKey(XTR_XTRACE)); + assertTrue(doc.containsKey(Constants.XTR_PROCESS_ID_KEY)); + assertTrue(doc.containsKey(Constants.XTR_TIMESTAMP_U_KEY)); + + // non of the long KVs made it as the key itself probably does not fit for the byte allocate for + // each entry. This is ok as this is one extreme case + } + + /** + * Over-sized event with large map value + * + * @throws InvalidConfigException + */ + @Test + public void testOversizedEvent6() throws InvalidConfigException { + int TEST_KEY_COUNT = 32 * 1024; + String PREFIX = "漢字"; + Map map = new HashMap(); + for (int i = 0; i < TEST_KEY_COUNT; i++) { + map.put(PREFIX + i, PREFIX); + } + + Event testEvent = startTrace(); + testEvent.addInfo("large-map", map); + + testEvent.addInfo("k1", 1); // these 2 should still make it + testEvent.addInfo("k2", "2"); + + TestReporter reporter = ReporterFactory.getInstance().createTestReporter(); + testEvent.report(reporter); + + // Decode what was "sent": + ByteBuffer buf = ByteBuffer.wrap(reporter.getLastSent()).order(ByteOrder.LITTLE_ENDIAN); + BsonReader reader = BsonToken.DOCUMENT.reader(); + BsonDocument doc = (BsonDocument) reader.readFrom(buf); + + assertTrue(doc.containsKey(Constants.XTR_THREAD_ID_KEY)); + assertTrue(doc.containsKey(Constants.XTR_HOSTNAME_KEY)); + assertTrue(doc.containsKey(Constants.XTR_METADATA_KEY)); + assertTrue(doc.containsKey(XTR_XTRACE)); + assertTrue(doc.containsKey(Constants.XTR_PROCESS_ID_KEY)); + assertTrue(doc.containsKey(Constants.XTR_TIMESTAMP_U_KEY)); + + // large-map should be dropped + assertFalse(doc.containsKey("large-map")); + // the 2 smaller KVs should be there + assertEquals(1, doc.get("k1")); + assertEquals("2", doc.get("k2")); + } + + /** + * Over-sized event with large collection value + * + * @throws InvalidConfigException + */ + @Test + public void testOversizedEvent7() throws InvalidConfigException { + int TEST_KEY_COUNT = 32 * 1024; + String PREFIX = "漢字"; + List list = new ArrayList(); + for (int i = 0; i < TEST_KEY_COUNT; i++) { + list.add(PREFIX + i); + } + + Event testEvent = startTrace(); + testEvent.addInfo("long-list", list); + + testEvent.addInfo("k1", 1); // these 2 should still make it + testEvent.addInfo("k2", "2"); + + TestReporter reporter = ReporterFactory.getInstance().createTestReporter(); + testEvent.report(reporter); + + // Decode what was "sent": + ByteBuffer buf = ByteBuffer.wrap(reporter.getLastSent()).order(ByteOrder.LITTLE_ENDIAN); + BsonReader reader = BsonToken.DOCUMENT.reader(); + BsonDocument doc = (BsonDocument) reader.readFrom(buf); + + assertTrue(doc.containsKey(Constants.XTR_THREAD_ID_KEY)); + assertTrue(doc.containsKey(Constants.XTR_HOSTNAME_KEY)); + assertTrue(doc.containsKey(Constants.XTR_METADATA_KEY)); + assertTrue(doc.containsKey(XTR_XTRACE)); + assertTrue(doc.containsKey(Constants.XTR_PROCESS_ID_KEY)); + assertTrue(doc.containsKey(Constants.XTR_TIMESTAMP_U_KEY)); + + // long-list should be dropped + assertFalse(doc.containsKey("long-list")); + // the 2 smaller KVs should be there + assertEquals(1, doc.get("k1")); + assertEquals("2", doc.get("k2")); + } + + @Test + public void testAsyncByMarkedEvent() { + Context.getMetadata().randomize(true); + + Event event = Context.createEvent(); + event.addInfo( + "Layer", "test", + "Label", "entry"); + event.setAsync(); + event.report(reporter); + + event = Context.createEvent(); + event.addInfo( + "Layer", "test", + "Label", "exit"); + event.setAsync(); + event.report(reporter); + + for (DeserializedEvent deserializedEvent : reporter.getSentEvents()) { + assertEquals(true, deserializedEvent.getSentEntries().get(Constants.XTR_ASYNC_KEY)); + } + } + + @Test + public void testAsyncByMarkedMetadata() { + Context.getMetadata().randomize(true); + Context.getMetadata().setIsAsync(true); + + Event event = Context.createEvent(); + event.addInfo("Layer", "test", "Label", "entry"); + event.report(reporter); + + event = Context.createEvent(); + event.addInfo("Layer", "test-nested", "Label", "entry"); + event.report(reporter); + + event = Context.createEvent(); + event.addInfo("Layer", "test-nested", "Label", "exit"); + event.report(reporter); + + event = Context.createEvent(); + event.addInfo("Layer", "test", "Label", "exit"); + event.report(reporter); + + event = Context.createEvent(); + event.addInfo("Layer", "test", "Label", "entry"); + event.report(reporter); + + event = Context.createEvent(); + event.addInfo("Layer", "test-nested", "Label", "entry"); + event.report(reporter); + + event = Context.createEvent(); + event.addInfo("Layer", "test-nested", "Label", "exit"); + event.report(reporter); + + event = Context.createEvent(); + event.addInfo("Layer", "test", "Label", "exit"); + event.report(reporter); + + List deserializedEvents = reporter.getSentEvents(); + + assertEquals(8, deserializedEvents.size()); + + // only the first (0) and the fifth (4) + for (int i = 0; i < deserializedEvents.size(); i++) { + if (i == 0 || i == 4) { + assertEquals(true, deserializedEvents.get(i).getSentEntries().get(Constants.XTR_ASYNC_KEY)); + } else { + assertNull(deserializedEvents.get(i).getSentEntries().get(XTR_ASYNC_KEY)); + } + } + } + + @Test + public void testW3cContextToXTrace() { + assertEquals( + "2BA6A6D97A748BFC9F91A4DC46A0D15BBB00000000B6968E14AC09A25A01", + EventImpl.w3cContextToXTrace("00-a6a6d97a748bfc9f91a4dc46a0d15bbb-b6968e14ac09a25a-01")); + assertEquals( + "2BA6A6D97A748BFC9F91A4DC46A0D15BBB00000000B6968E14AC09A25A00", + EventImpl.w3cContextToXTrace("00-a6a6d97a748bfc9f91a4dc46a0d15bbb-b6968e14ac09a25a-00")); + } + + private Event startTrace() { + Metadata md = new Metadata(); + md.randomize(true); + Context.setMetadata(md); + return Context.createEventWithContext(md, false); + } +} diff --git a/libs/core/src/test/java/com/solarwinds/joboe/core/EventValueConverterTest.java b/libs/core/src/test/java/com/solarwinds/joboe/core/EventValueConverterTest.java new file mode 100644 index 00000000..a9d596a6 --- /dev/null +++ b/libs/core/src/test/java/com/solarwinds/joboe/core/EventValueConverterTest.java @@ -0,0 +1,178 @@ +/* + * © SolarWinds Worldwide, LLC. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.solarwinds.joboe.core; + +import static org.junit.jupiter.api.Assertions.assertEquals; + +import java.math.BigDecimal; +import java.math.BigInteger; +import java.net.InetAddress; +import java.net.URL; +import java.sql.Time; +import java.util.ArrayList; +import java.util.Date; +import java.util.HashMap; +import java.util.UUID; +import org.junit.jupiter.api.Test; + +/** + * Converts object into values compatible to BSON + * + * @author pluk + */ +public class EventValueConverterTest { + private static final int MAX_VALUE_LENGTH = 100; + private final EventValueConverter converter = new EventValueConverter(MAX_VALUE_LENGTH); + Object o; + int objectId; + + @Test + public void testSimpleTypeBooleans() { + assertEquals(true, converter.convertToEventValue(true)); + } + + @Test + public void testSimpleTypeDoubles() { + assertEquals((double) 0, converter.convertToEventValue((double) 0)); + } + + @Test + public void testSimpleTypeIntegers() { + assertEquals(0, converter.convertToEventValue(0)); + } + + @Test + public void testSimpleTypeLongs() { + assertEquals((long) 0, converter.convertToEventValue((long) 0)); + } + + @Test + public void testSimpleTypeEmptyString() { + assertEquals("", converter.convertToEventValue("")); + } + + @Test + public void testSpecialTypeDate() throws Exception { + Date date = new Date(); + assertEquals(date, converter.convertToEventValue(date)); + } + + @Test + public void testSpecialTypeTime() throws Exception { + Time time = new Time(System.currentTimeMillis()); + assertEquals(time, converter.convertToEventValue(time)); + } + + @Test + public void testSpecialTypeBigDecimal() throws Exception { + assertEquals((double) 0, converter.convertToEventValue(BigDecimal.ZERO)); + } + + @Test + public void testSpecialTypeBigInteger() throws Exception { + assertEquals((long) 0, converter.convertToEventValue(BigInteger.ZERO)); + } + + @Test + public void testSpecialTypeFloat() throws Exception { + assertEquals((double) 0, converter.convertToEventValue((float) 0)); + } + + @Test + public void testSpecialTypeShort() throws Exception { + assertEquals(0, converter.convertToEventValue((short) 0)); + } + + @Test + public void testSpecialTypeByte() throws Exception { + assertEquals(0, converter.convertToEventValue((byte) 0)); + } + + @Test + public void testSpecialTypeChar() throws Exception { + assertEquals("a", converter.convertToEventValue('a')); + } + + @Test + public void testSpecialTypeLongString() throws Exception { + String rawString = new String(new byte[10000]); + int truncateCount = rawString.length() - MAX_VALUE_LENGTH; + String finalString = + rawString.substring(0, MAX_VALUE_LENGTH) + + "...(" + + truncateCount + + " characters truncated)"; + assertEquals(finalString, converter.convertToEventValue(rawString)); + } + + @Test + public void testSpecialTypeByteArray() throws Exception { + o = new byte[0]; + objectId = System.identityHashCode(o); + assertEquals("(Byte array 0 Bytes) id [" + objectId + "]", converter.convertToEventValue(o)); + } + + @Test + public void testSpecialTypeURL() throws Exception { + o = new URL("http://www.google.com"); + assertEquals(o.toString(), converter.convertToEventValue(o)); + } + + @Test + public void testSpecialTypeIPAddress() throws Exception { + o = InetAddress.getByAddress(new byte[4]); + assertEquals(o.toString(), converter.convertToEventValue(o)); + o = InetAddress.getByAddress(new byte[16]); + assertEquals(o.toString(), converter.convertToEventValue(o)); + } + + @Test + public void testSpecialTypeArrayList() throws Exception { + o = new ArrayList(); + objectId = System.identityHashCode(o); + assertEquals( + "(Collection of class [" + + ArrayList.class.getName() + + "] with 0 Elements) id [" + + objectId + + "]", + converter.convertToEventValue(o)); + } + + @Test + public void testSpecialTypeHashMap() throws Exception { + o = new HashMap(); + objectId = System.identityHashCode(o); + assertEquals( + "(Map of class [" + HashMap.class.getName() + "] with 0 Elements) id [" + objectId + "]", + converter.convertToEventValue(o)); + } + + @Test + public void testSpecialTypeUUID() throws Exception { + o = UUID.randomUUID(); + assertEquals(o.toString(), converter.convertToEventValue(o)); + } + + @Test + public void testSpecialTypeObject() throws Exception { + o = new Object(); + objectId = System.identityHashCode(o); + assertEquals( + "(" + Object.class.getName() + ") id [" + objectId + "]", converter.convertToEventValue(o)); + } +} diff --git a/libs/core/src/test/java/com/solarwinds/joboe/core/QueuingEventReporterTest.java b/libs/core/src/test/java/com/solarwinds/joboe/core/QueuingEventReporterTest.java new file mode 100644 index 00000000..446e2c17 --- /dev/null +++ b/libs/core/src/test/java/com/solarwinds/joboe/core/QueuingEventReporterTest.java @@ -0,0 +1,52 @@ +/* + * © SolarWinds Worldwide, LLC. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.solarwinds.joboe.core; + +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.ArgumentMatchers.anyList; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.when; + +import com.solarwinds.joboe.core.rpc.Client; +import com.solarwinds.joboe.core.rpc.Result; +import com.solarwinds.joboe.core.rpc.ResultCode; +import java.util.Collections; +import java.util.concurrent.Future; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.ExtendWith; +import org.mockito.InjectMocks; +import org.mockito.Mock; +import org.mockito.junit.jupiter.MockitoExtension; + +@ExtendWith(MockitoExtension.class) +class QueuingEventReporterTest { + + @InjectMocks private QueuingEventReporter tested; + + @Mock private Client clientMock; + + @Mock private Future futureMock; + + @Test + void testSynchronousSend() throws Exception { + when(clientMock.postEvents(anyList(), any())).thenReturn(futureMock); + when(futureMock.get()).thenReturn(new Result(ResultCode.OK, "", "")); + + tested.synchronousSend(Collections.emptyList()); + verify(clientMock).postEvents(anyList(), any()); + } +} diff --git a/libs/core/src/test/java/com/solarwinds/joboe/core/ReporterFactoryTest.java b/libs/core/src/test/java/com/solarwinds/joboe/core/ReporterFactoryTest.java new file mode 100644 index 00000000..243da636 --- /dev/null +++ b/libs/core/src/test/java/com/solarwinds/joboe/core/ReporterFactoryTest.java @@ -0,0 +1,62 @@ +/* + * © SolarWinds Worldwide, LLC. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.solarwinds.joboe.core; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertNotNull; + +import com.solarwinds.joboe.core.rpc.Client; +import java.lang.reflect.Field; +import java.net.InetAddress; +import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.ExtendWith; +import org.mockito.Mock; +import org.mockito.junit.jupiter.MockitoExtension; + +@ExtendWith(MockitoExtension.class) +public class ReporterFactoryTest { + + private static ReporterFactory tested; + + @Mock private Client clientMock; + + @BeforeAll + static void setup() { + tested = ReporterFactory.getInstance(); + } + + @Test + public void testbuildNonDefaultUdpReporter() throws Exception { + UDPReporter reporter = tested.createUdpReporter("localhost", 9999); + + Field addressField = reporter.getClass().getDeclaredField("addr"); + addressField.setAccessible(true); + + Field portField = reporter.getClass().getDeclaredField("port"); + portField.setAccessible(true); + + InetAddress address = (InetAddress) addressField.get(reporter); + assertEquals(InetAddress.getByName("localhost"), address); + assertEquals(9999, portField.get(reporter)); + } + + @Test + void testCreateQueuingEventReporter() { + assertNotNull(tested.createQueuingEventReporter(clientMock)); + } +} diff --git a/libs/core/src/test/java/com/solarwinds/joboe/core/TestExecutionExceptionRpcClient.java b/libs/core/src/test/java/com/solarwinds/joboe/core/TestExecutionExceptionRpcClient.java new file mode 100644 index 00000000..d19b4c6c --- /dev/null +++ b/libs/core/src/test/java/com/solarwinds/joboe/core/TestExecutionExceptionRpcClient.java @@ -0,0 +1,69 @@ +/* + * © SolarWinds Worldwide, LLC. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.solarwinds.joboe.core; + +import com.solarwinds.joboe.core.rpc.Client; +import com.solarwinds.joboe.core.rpc.ClientException; +import com.solarwinds.joboe.core.rpc.Result; +import com.solarwinds.joboe.core.rpc.SettingsResult; +import java.util.List; +import java.util.Map; +import java.util.concurrent.Callable; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; +import java.util.concurrent.Future; + +public class TestExecutionExceptionRpcClient implements Client { + private final ExecutorService service = Executors.newSingleThreadExecutor(); + + @Override + public Future postEvents(List events, Callback callback) { + return service.submit(new ExceptionCallable()); + } + + @Override + public Future postMetrics(List> messages, Callback callback) { + return service.submit(new ExceptionCallable()); + } + + @Override + public Future postStatus(List> messages, Callback callback) { + return service.submit(new ExceptionCallable()); + } + + @Override + public Future getSettings(String version, Callback callback) { + return service.submit(new ExceptionCallable()); + } + + private static class ExceptionCallable implements Callable { + @Override + public T call() throws Exception { + throw new ClientException("testing exception from exception rpc client"); + } + } + + @Override + public Status getStatus() { + return Status.OK; + } + + @Override + public void close() { + service.shutdown(); + } +} diff --git a/libs/core/src/test/java/com/solarwinds/joboe/core/TestReporterTest.java b/libs/core/src/test/java/com/solarwinds/joboe/core/TestReporterTest.java new file mode 100644 index 00000000..85b58e88 --- /dev/null +++ b/libs/core/src/test/java/com/solarwinds/joboe/core/TestReporterTest.java @@ -0,0 +1,85 @@ +/* + * © SolarWinds Worldwide, LLC. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.solarwinds.joboe.core; + +import static org.junit.jupiter.api.Assertions.assertEquals; + +import com.solarwinds.joboe.config.InvalidConfigException; +import com.solarwinds.joboe.sampling.Metadata; +import com.solarwinds.joboe.sampling.SamplingConfiguration; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; + +public class TestReporterTest { + + @BeforeEach + void setup() { + Metadata.setup(SamplingConfiguration.builder().build()); + } + + @Test + public void testReporterSameThread() throws InvalidConfigException { + final TestReporter threadLocalReporter = ReporterFactory.getInstance().createTestReporter(true); + final TestReporter nonThreadLocalReporter = + ReporterFactory.getInstance().createTestReporter(false); + + Event event; + + event = startTrace(); + event.report(threadLocalReporter); + + event = startTrace(); + event.report(nonThreadLocalReporter); + + assertEquals(1, threadLocalReporter.getSentEvents().size()); + assertEquals(1, nonThreadLocalReporter.getSentEvents().size()); + } + + @Test + public void testReporterDifferentThread() throws InvalidConfigException, InterruptedException { + final TestReporter threadLocalReporter = ReporterFactory.getInstance().createTestReporter(true); + final TestReporter nonThreadLocalReporter = + ReporterFactory.getInstance().createTestReporter(false); + + Thread thread = + new Thread( + () -> { + Event event; + + event = startTrace(); + event.report(threadLocalReporter); + + event = startTrace(); + event.report(nonThreadLocalReporter); + }); + + thread.start(); + thread.join(); + + assertEquals( + 0, + threadLocalReporter.getSentEvents().size()); // different thread, should not get the event + assertEquals(1, nonThreadLocalReporter.getSentEvents().size()); + } + + private Event startTrace() { + Metadata md = new Metadata(); + md.randomize(true); + Context.setMetadata(md); + return Context.createEventWithContext(md, false); + } +} diff --git a/libs/core/src/test/java/com/solarwinds/joboe/core/TestRpcClient.java b/libs/core/src/test/java/com/solarwinds/joboe/core/TestRpcClient.java new file mode 100644 index 00000000..f9fffff9 --- /dev/null +++ b/libs/core/src/test/java/com/solarwinds/joboe/core/TestRpcClient.java @@ -0,0 +1,106 @@ +/* + * © SolarWinds Worldwide, LLC. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.solarwinds.joboe.core; + +import com.solarwinds.joboe.core.rpc.Client; +import com.solarwinds.joboe.core.rpc.Result; +import com.solarwinds.joboe.core.rpc.ResultCode; +import com.solarwinds.joboe.core.rpc.SettingsResult; +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; +import java.util.Map; +import java.util.concurrent.CompletableFuture; +import java.util.concurrent.Future; +import lombok.Getter; + +public class TestRpcClient implements Client { + private final long delay; + + @Getter private final List postedEvents = new ArrayList(); + + @Getter + private final List> postedMetrics = new ArrayList>(); + + @Getter + private final List> postedStatus = new ArrayList>(); + + private final Result stringResult; + private final SettingsResult settingsResult; + + public TestRpcClient(long delay) { + this(delay, ResultCode.OK); + } + + public TestRpcClient(long delay, ResultCode resultCode) { + super(); + this.delay = delay; + stringResult = new Result(resultCode, "", ""); + settingsResult = new SettingsResult(resultCode, "", "", Collections.emptyList()); + } + + @Override + public Future postEvents(List events, Callback callback) { + sleep(); + postedEvents.addAll(events); + return CompletableFuture.completedFuture(stringResult); + } + + @Override + public Future postMetrics(List> messages, Callback callback) { + sleep(); + postedMetrics.addAll(messages); + return CompletableFuture.completedFuture(stringResult); + } + + @Override + public Future postStatus(List> messages, Callback callback) { + sleep(); + postedStatus.addAll(messages); + return CompletableFuture.completedFuture(stringResult); + } + + @Override + public Future getSettings(String version, Callback callback) { + sleep(); + return CompletableFuture.completedFuture(settingsResult); + } + + @Override + public void close() {} + + @Override + public Status getStatus() { + return Status.OK; + } + + private void sleep() { + if (delay > 0) { + try { + Thread.sleep(delay); + } catch (InterruptedException e) { + e.printStackTrace(); + } + } + } + + public void reset() { + postedEvents.clear(); + postedMetrics.clear(); + postedStatus.clear(); + } +} diff --git a/libs/core/src/test/java/com/solarwinds/joboe/core/TestSubmitRejectionRpcClient.java b/libs/core/src/test/java/com/solarwinds/joboe/core/TestSubmitRejectionRpcClient.java new file mode 100644 index 00000000..95751a41 --- /dev/null +++ b/libs/core/src/test/java/com/solarwinds/joboe/core/TestSubmitRejectionRpcClient.java @@ -0,0 +1,65 @@ +/* + * © SolarWinds Worldwide, LLC. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.solarwinds.joboe.core; + +import com.solarwinds.joboe.core.rpc.Client; +import com.solarwinds.joboe.core.rpc.ClientException; +import com.solarwinds.joboe.core.rpc.ClientRejectedExecutionException; +import com.solarwinds.joboe.core.rpc.Result; +import com.solarwinds.joboe.core.rpc.SettingsResult; +import java.util.List; +import java.util.Map; +import java.util.concurrent.Future; +import java.util.concurrent.RejectedExecutionException; + +public class TestSubmitRejectionRpcClient implements Client { + @Override + public Future postEvents(List events, Callback callback) + throws ClientException { + throw new ClientRejectedExecutionException( + new RejectedExecutionException("Testing submit client exception")); + } + + @Override + public Future postMetrics(List> messages, Callback callback) + throws ClientException { + throw new ClientRejectedExecutionException( + new RejectedExecutionException("Testing submit client exception")); + } + + @Override + public Future postStatus(List> messages, Callback callback) + throws ClientException { + throw new ClientRejectedExecutionException( + new RejectedExecutionException("Testing submit client exception")); + } + + @Override + public Future getSettings(String version, Callback callback) + throws ClientException { + throw new ClientRejectedExecutionException( + new RejectedExecutionException("Testing submit client exception")); + } + + @Override + public void close() {} + + @Override + public Status getStatus() { + return Status.OK; + } +} diff --git a/libs/core/src/test/java/com/solarwinds/joboe/core/profiler/CircuitBreakerTest.java b/libs/core/src/test/java/com/solarwinds/joboe/core/profiler/CircuitBreakerTest.java new file mode 100644 index 00000000..94189bd0 --- /dev/null +++ b/libs/core/src/test/java/com/solarwinds/joboe/core/profiler/CircuitBreakerTest.java @@ -0,0 +1,130 @@ +/* + * © SolarWinds Worldwide, LLC. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.solarwinds.joboe.core.profiler; + +import static org.junit.jupiter.api.Assertions.assertEquals; + +import com.solarwinds.joboe.core.profiler.Profiler.CircuitBreaker; +import org.junit.jupiter.api.Test; + +public class CircuitBreakerTest { + @Test + public void testPause() { + CircuitBreaker circuitBreaker = new CircuitBreaker(100, 3); + + /** + * Copied from description on CircuitBreaker.pauseMethod + * + *

This might mutate the current circuit breaker states, depending on the current state and + * the duration parameters: + * + *

At first the circuit breaker starts with a "Normal" state When there are n (defined by + * `countThreshold`) consecutive `getPause` calls with param `duration` above the + * `durationThreshold`, the circuit breaker will go into the "Break" state "Break" state will be + * transitioned into a "Restored but broken recently" state when there's a new `getPause` call + * "Restored but broken recently" state will be transitioned to "Normal" state if there are n + * consecutive `getPause` calls with param `duration` below or equal to the `durationThreshold` + * + *

And below are the behaviors of this method in various states/transitions: + * + *

When transition to "Normal" state, `nextPause` is set to INITIAL_CIRCUIT_BREAKER_PAUSE + * When in "Normal" state, `getPause` returns 0 When transition to "Break" state, `getPause` + * returns `nextPause` then `nextPause` is multiplied by `PAUSE_MULTIPLIER` When transition to + * or in "Restored but broken recently" state, `getPause` returns 0 + */ + assertEquals(0, circuitBreaker.getPause(0)); // no break + assertEquals(0, circuitBreaker.getPause(100)); // no break, still within threshold + assertEquals( + 0, + circuitBreaker.getPause( + 101)); // no break, above threshold but only 1 consecutive occurrence + assertEquals( + 0, + circuitBreaker.getPause( + 101)); // no break, above threshold but only 2 consecutive occurrences + assertEquals( + CircuitBreaker.INITIAL_CIRCUIT_BREAKER_PAUSE, + circuitBreaker.getPause(101)); // break, above threshold and 3 consecutive occurrences + assertEquals( + 0, + circuitBreaker.getPause( + 101)); // no break, above threshold and should transition into "Restored but broken + // recently" state + assertEquals( + 0, + circuitBreaker.getPause( + 101)); // no break, above threshold but only 2 consecutive occurrences + assertEquals( + (int) (CircuitBreaker.INITIAL_CIRCUIT_BREAKER_PAUSE * CircuitBreaker.PAUSE_MULTIPLIER), + circuitBreaker.getPause( + 101)); // break again with increased pause, above threshold and 3 consecutive + // occurrences + assertEquals( + 0, + circuitBreaker.getPause( + 101)); // no break, above threshold and should transition into "Restored but broken + // recently" state + assertEquals( + 0, + circuitBreaker.getPause( + 0)); // no break - not resetting nextPause yet, below threshold but only 1 consecutive + // occurrence + assertEquals( + 0, + circuitBreaker.getPause( + 101)); // no break, above threshold but only 1 consecutive occurrence + assertEquals( + 0, + circuitBreaker.getPause( + 101)); // no break, above threshold but only 2 consecutive occurrence + assertEquals( + (int) + (CircuitBreaker.INITIAL_CIRCUIT_BREAKER_PAUSE + * CircuitBreaker.PAUSE_MULTIPLIER + * CircuitBreaker.PAUSE_MULTIPLIER), + circuitBreaker.getPause( + 101)); // break again with increased pause, above threshold and 3 consecutive + // occurrences + assertEquals( + 0, + circuitBreaker.getPause( + 0)); // no break - not resetting nextPause yet, below threshold but only 1 consecutive + // occurrence + assertEquals( + 0, + circuitBreaker.getPause( + 0)); // no break - not resetting nextPause yet, below threshold but only 2 consecutive + // occurrences + assertEquals( + 0, + circuitBreaker.getPause( + 0)); // no break - resetting nextPause, below threshold and 3 consecutive occurrences + assertEquals( + 0, + circuitBreaker.getPause( + 101)); // no break, above threshold but only 1 consecutive occurrence + assertEquals( + 0, + circuitBreaker.getPause( + 101)); // no break, above threshold but only 2 consecutive occurrence + assertEquals( + CircuitBreaker.INITIAL_CIRCUIT_BREAKER_PAUSE, + circuitBreaker.getPause( + 101)); // break but only pause for INITIAL_CIRCUIT_BREAKER_PAUSE, above threshold and 3 + // consecutive occurrences + } +} diff --git a/libs/core/src/test/java/com/solarwinds/joboe/core/profiler/ProfileTest.java b/libs/core/src/test/java/com/solarwinds/joboe/core/profiler/ProfileTest.java new file mode 100644 index 00000000..4a4307c3 --- /dev/null +++ b/libs/core/src/test/java/com/solarwinds/joboe/core/profiler/ProfileTest.java @@ -0,0 +1,242 @@ +/* + * © SolarWinds Worldwide, LLC. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.solarwinds.joboe.core.profiler; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertNull; +import static org.junit.jupiter.api.Assertions.assertTrue; + +import com.solarwinds.joboe.core.TestReporter; +import com.solarwinds.joboe.core.TestReporter.DeserializedEvent; +import com.solarwinds.joboe.core.profiler.Profiler.Profile; +import com.solarwinds.joboe.core.util.TestUtils; +import com.solarwinds.joboe.sampling.Metadata; +import com.solarwinds.joboe.sampling.SamplingConfiguration; +import com.solarwinds.joboe.sampling.SamplingException; +import java.util.Arrays; +import java.util.Collections; +import java.util.List; +import java.util.Map; +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.api.Test; + +public class ProfileTest { + + private static final ProfilerSetting profilerSetting = + new ProfilerSetting( + true, + Collections.emptySet(), + ProfilerSetting.DEFAULT_INTERVAL, + ProfilerSetting.DEFAULT_CIRCUIT_BREAKER_DURATION_THRESHOLD, + ProfilerSetting.DEFAULT_CIRCUIT_BREAKER_COUNT_THRESHOLD); + private static TestReporter profilingReporter; + + @BeforeAll + static /*static because reporter is static in Profiler*/ void setup() { + profilingReporter = TestUtils.initProfilingReporter(profilerSetting); + } + + @AfterEach + void tearDown() { + profilingReporter.reset(); + } + + @Test + void partialValidateEntryEventShapeOnStartProfilingOnThread() throws SamplingException { + Thread thread = Thread.currentThread(); + Profile profile = new Profile(profilerSetting); + Metadata.setup(SamplingConfiguration.builder().build()); + + profile.startProfilingOnThread( + thread, new Metadata("00-970026c88092a447d3b2bba3be3be2fc-0c8fc43138df813a-01")); + simulateStackChange(profile); + List events = profilingReporter.getSentEvents(); + Map entryEvent = events.get(0).getSentEntries(); + + assertEquals("entry", entryEvent.get("Label")); + assertEquals("profiling", entryEvent.get("Spec")); + + assertEquals("java", entryEvent.get("Language")); + assertEquals("0c8fc43138df813a", entryEvent.get("SpanRef")); + assertEquals(thread.getId(), entryEvent.get("TID")); + } + + @Test + void doNotCreateTerminalEventsWhenSampleIsNotCollected() throws SamplingException { + Thread thread = Thread.currentThread(); + Profile profile = new Profile(profilerSetting); + Metadata.setup(SamplingConfiguration.builder().build()); + + profile.startProfilingOnThread( + thread, new Metadata("00-970026c88092a447d3b2bba3be3be2fc-0c8fc43138df813a-01")); + profile.stopProfilingOnThread(thread); + List events = profilingReporter.getSentEvents(); + + assertTrue(events.isEmpty()); + } + + @Test + void createTerminalEventsWhenSampleIsCollected() throws SamplingException { + Thread thread = Thread.currentThread(); + Profile profile = new Profile(profilerSetting); + Metadata.setup(SamplingConfiguration.builder().build()); + + profile.startProfilingOnThread( + thread, new Metadata("00-970026c88092a447d3b2bba3be3be2fc-0c8fc43138df813a-01")); + simulateStackChange(profile); + profile.stopProfilingOnThread(thread); + + List events = profilingReporter.getSentEvents(); + assertEquals(3, events.size()); + } + + @Test + @SuppressWarnings("unchecked") + void partialValidateInfoEventOnRecord() throws SamplingException { + Thread thread = Thread.currentThread(); + StackTraceElement[] stackTrace = thread.getStackTrace(); + Profile profile = new Profile(profilerSetting); + Metadata.setup(SamplingConfiguration.builder().build()); + + profile.startProfilingOnThread( + thread, new Metadata("00-970026c88092a447d3b2bba3be3be2fc-0c8fc43138df813a-01")); + profile.record(thread, stackTrace, 2); + List infoEvents = profilingReporter.getSentEvents(); + + Map snapshotEvent = infoEvents.get(1).getSentEntries(); + assertEquals("info", snapshotEvent.get("Label")); + assertEquals("profiling", snapshotEvent.get("Spec")); + + assertEquals(stackTrace.length, snapshotEvent.get("FramesCount")); + assertEquals(thread.getId(), snapshotEvent.get("TID")); + assertNewFrames( + Arrays.copyOfRange(stackTrace, 0, 3), + (Map>) snapshotEvent.get("NewFrames")); + } + + @Test + @SuppressWarnings("unchecked") + void partialValidateInfoEventOnRecordWithSkippedFrame() throws SamplingException { + Thread thread = Thread.currentThread(); + StackTraceElement[] stackTrace = thread.getStackTrace(); + Profile profile = new Profile(profilerSetting); + Metadata.setup(SamplingConfiguration.builder().build()); + + profile.startProfilingOnThread( + thread, new Metadata("00-970026c88092a447d3b2bba3be3be2fc-0c8fc43138df813a-01")); + profile.record(thread, stackTrace, 2); + profile.record(thread, stackTrace, 2); + + StackTraceElement[] newFrames = simulateStackChange(profile); + List infoEvents = profilingReporter.getSentEvents(); + + Map snapshotEvent = infoEvents.get(2).getSentEntries(); + assertEquals("info", snapshotEvent.get("Label")); + assertEquals("profiling", snapshotEvent.get("Spec")); + + assertFalse(((Map) snapshotEvent.get("SnapshotsOmitted")).isEmpty()); + assertEquals("profiling", snapshotEvent.get("Spec")); + assertEquals(2, snapshotEvent.get("FramesExited")); + + assertEquals(newFrames.length, snapshotEvent.get("FramesCount")); + assertEquals(thread.getId(), snapshotEvent.get("TID")); + assertNewFrames( + Arrays.copyOfRange(newFrames, 0, 3), + (Map>) snapshotEvent.get("NewFrames")); + } + + @Test + @SuppressWarnings("unchecked") + void partialValidateInfoEventOnRecordWithFrameChange() throws SamplingException { + Thread thread = Thread.currentThread(); + StackTraceElement[] stackTrace = thread.getStackTrace(); + Profile profile = new Profile(profilerSetting); + Metadata.setup(SamplingConfiguration.builder().build()); + + profile.startProfilingOnThread( + thread, new Metadata("00-970026c88092a447d3b2bba3be3be2fc-0c8fc43138df813a-01")); + profile.record(thread, stackTrace, 2); + StackTraceElement[] newFrames = simulateStackChange(profile); + + profile.record(thread, stackTrace, 2); + List infoEvents = profilingReporter.getSentEvents(); + + Map snapshotEvent = infoEvents.get(2).getSentEntries(); + assertEquals("info", snapshotEvent.get("Label")); + assertEquals("profiling", snapshotEvent.get("Spec")); + + assertTrue(((Map) snapshotEvent.get("SnapshotsOmitted")).isEmpty()); + assertEquals("profiling", snapshotEvent.get("Spec")); + assertEquals(2, snapshotEvent.get("FramesExited")); + + assertEquals(newFrames.length, snapshotEvent.get("FramesCount")); + assertEquals(thread.getId(), snapshotEvent.get("TID")); + assertNewFrames( + Arrays.copyOfRange(newFrames, 0, 3), + (Map>) snapshotEvent.get("NewFrames")); + } + + @Test + void verifyThatProfilingIsStoppedWhenMetadataExpires() throws SamplingException { + Thread thread = Thread.currentThread(); + StackTraceElement[] stackTrace = thread.getStackTrace(); + Profile profile = new Profile(profilerSetting); + + profile.startProfilingOnThread( + thread, new Metadata("00-970026c88092a447d3b2bba3be3be2fc-0c8fc43138df813a-01")); + Profiler.SnapshotTracker snapshotTracker = profile.getSnapshotTracker(thread); + profile.record(thread, stackTrace, 2); + + profile.record(thread, stackTrace, 2); + profile.record(thread, stackTrace, 2); + Metadata.setup(SamplingConfiguration.builder().ttl(0).build()); // set ttl 0 to force expiration + + profile.record(thread, stackTrace, (System.currentTimeMillis() * 1000) << 1); + /* + we expect 2 because the two calls to profile.record(..) above should be skipped, and the stack didn't change; + the last call should trigger a stop due to expiration. + */ + assertEquals(2, snapshotTracker.getSnapshotsOmitted().size()); + // we expect null because profiling should be stopped by now + assertNull(profile.getSnapshotTracker(thread)); + } + + private void assertNewFrames( + StackTraceElement[] expectedNewFrames, Map> actualNewFrames) { + for (int i = 0; i < expectedNewFrames.length; i++) { + StackTraceElement expectedFrame = expectedNewFrames[i]; + Map actualFrame = actualNewFrames.get(String.valueOf(i)); + assertEquals(expectedFrame.getClassName(), actualFrame.get("C")); + + assertEquals(expectedFrame.getMethodName(), actualFrame.get("M")); + assertEquals(expectedFrame.getFileName(), actualFrame.get("F")); + if (expectedFrame.getLineNumber() >= 0) { + assertEquals(expectedFrame.getLineNumber(), actualFrame.get("L")); + } + } + } + + private StackTraceElement[] simulateStackChange(Profile profile) { + Thread thread = Thread.currentThread(); + StackTraceElement[] stackTrace = thread.getStackTrace(); + profile.record(thread, stackTrace, 2); + + return stackTrace; + } +} diff --git a/libs/core/src/test/java/com/solarwinds/joboe/core/rpc/README.md b/libs/core/src/test/java/com/solarwinds/joboe/core/rpc/README.md new file mode 100644 index 00000000..a8204cc8 --- /dev/null +++ b/libs/core/src/test/java/com/solarwinds/joboe/core/rpc/README.md @@ -0,0 +1,26 @@ +This is to document the cert generation for test servers + +## gRPC +gRPC take the server public key (cert) and private key pair, so it was simply generated by: +``` +openssl req -x509 -days 365000 -nodes -newkey rsa:2048 -keyout test-collector-private.pem -out test-collector-public.pem +``` +(with CN localhost) + +Server uses both the cert (test-collector-public.pem) and the private key (test-collector-private.pem) +Client uses only the cert (test-collector-public.pem) + +## Thrift + +Thrift takes JKS "keystore" (which contains both cert (public key) and private key), first generate a pkcs12 keystore +``` +openssl pkcs12 -export -in test-collector-public.pem -inkey test-collector-private.pem -out test-collector-keystore.p12 -name localhost -passin pass:labrat1214 -passout pass:labrat1214 +``` +Now import the pkcs12 keystore into JKS keystore +``` +keytool -importkeystore -srckeystore test-collector-keystore.p12 -srcstoretype PKCS12 -srcstorepass labrat1214 -alias localhost -deststorepass labrat1214 -destkeypass labrat1214 -destkeystore test-collector-keystore.jks +``` + +Server uses the JKS keystore (take note the p12 is just an intermediate file, not used at the end) +Client uses the cert (test-collector-public.pem) + diff --git a/libs/core/src/test/java/com/solarwinds/joboe/core/rpc/RpcClientManagerTest.java b/libs/core/src/test/java/com/solarwinds/joboe/core/rpc/RpcClientManagerTest.java new file mode 100644 index 00000000..d74f731f --- /dev/null +++ b/libs/core/src/test/java/com/solarwinds/joboe/core/rpc/RpcClientManagerTest.java @@ -0,0 +1,90 @@ +/* + * © SolarWinds Worldwide, LLC. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.solarwinds.joboe.core.rpc; + +import static org.junit.jupiter.api.Assertions.assertEquals; + +import java.io.File; +import java.lang.reflect.Field; +import org.junit.jupiter.api.Test; + +public class RpcClientManagerTest { + private static final String TEST_SERVER_CERT_LOCATION = + "src/test/java/com/solarwinds/joboe/core/rpc/test-collector-public.pem"; + + @Test + public void testValidEnvironmentVariables() throws Exception { + RpcClientManager.init("unit-test-collector1:1234", TEST_SERVER_CERT_LOCATION); + + // verify the fields + Field field; + field = RpcClientManager.class.getDeclaredField("collectorHost"); + field.setAccessible(true); + assertEquals("unit-test-collector1", field.get(null)); + + field = RpcClientManager.class.getDeclaredField("collectorPort"); + field.setAccessible(true); + assertEquals(1234, field.get(null)); + + field = RpcClientManager.class.getDeclaredField("collectorCertLocation"); + field.setAccessible(true); + assertEquals(new File(TEST_SERVER_CERT_LOCATION).toURI().toURL(), field.get(null)); + + RpcClientManager.init("unit-test-collector2", null); + field = RpcClientManager.class.getDeclaredField("collectorHost"); + field.setAccessible(true); + assertEquals("unit-test-collector2", field.get(null)); + + field = RpcClientManager.class.getDeclaredField("collectorPort"); + field.setAccessible(true); + assertEquals(RpcClientManager.DEFAULT_PORT, field.get(null)); + + RpcClientManager.init(null, null); // revert + } + + @Test + public void testInvalidEnvironmentVariables() throws Exception { + RpcClientManager.init("unit-test-collector:not-a-number", null); + + // verify the fields, port number should fallback to default + Field field; + field = RpcClientManager.class.getDeclaredField("collectorHost"); + field.setAccessible(true); + assertEquals("unit-test-collector", field.get(null)); + + field = RpcClientManager.class.getDeclaredField("collectorPort"); + field.setAccessible(true); + assertEquals(RpcClientManager.DEFAULT_PORT, field.get(null)); + + RpcClientManager.init(null, null); // revert + } + + @Test + public void testDefaultEnviromentVariables() throws Exception { + RpcClientManager.init(null, null); + + // verify the fields + Field field; + field = RpcClientManager.class.getDeclaredField("collectorHost"); + field.setAccessible(true); + assertEquals(RpcClientManager.DEFAULT_HOST, field.get(null)); + + field = RpcClientManager.class.getDeclaredField("collectorPort"); + field.setAccessible(true); + assertEquals(RpcClientManager.DEFAULT_PORT, field.get(null)); + } +} diff --git a/libs/core/src/test/java/com/solarwinds/joboe/core/rpc/RpcClientTest.java b/libs/core/src/test/java/com/solarwinds/joboe/core/rpc/RpcClientTest.java new file mode 100644 index 00000000..a61f3d04 --- /dev/null +++ b/libs/core/src/test/java/com/solarwinds/joboe/core/rpc/RpcClientTest.java @@ -0,0 +1,1085 @@ +/* + * © SolarWinds Worldwide, LLC. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.solarwinds.joboe.core.rpc; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.junit.jupiter.api.Assertions.fail; + +import com.solarwinds.joboe.core.BsonBufferException; +import com.solarwinds.joboe.core.Constants; +import com.solarwinds.joboe.core.Context; +import com.solarwinds.joboe.core.Event; +import com.solarwinds.joboe.core.ebson.BsonDocument; +import com.solarwinds.joboe.core.ebson.BsonDocuments; +import com.solarwinds.joboe.core.rpc.RpcClient.TaskType; +import com.solarwinds.joboe.core.settings.PollingSettingsFetcherTest; +import com.solarwinds.joboe.core.util.TimeUtils; +import com.solarwinds.joboe.sampling.Metadata; +import com.solarwinds.joboe.sampling.SamplingConfiguration; +import com.solarwinds.joboe.sampling.Settings; +import com.solarwinds.joboe.sampling.SettingsArg; +import java.io.File; +import java.io.IOException; +import java.net.InetSocketAddress; +import java.net.Socket; +import java.net.URL; +import java.nio.ByteBuffer; +import java.nio.ByteOrder; +import java.security.GeneralSecurityException; +import java.util.*; +import java.util.Map.Entry; +import java.util.concurrent.ExecutionException; +import java.util.concurrent.Future; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.TimeoutException; +import java.util.concurrent.atomic.AtomicBoolean; +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; + +public abstract class RpcClientTest { + private static final int TEST_SERVER_PORT_BASE = 10148; + protected static final String TEST_SERVER_HOST = "localhost"; + protected static final String TEST_CLIENT_ID = "123"; + protected List testEvents = generateTestEvents(); + protected Event bigEvent = generateBigEvent(); + + private static final String TEST_SERVER_CERT_LOCATION = + "src/test/java/com/solarwinds/joboe/core/rpc/test-collector-public.pem"; + private static final String INVALID_CERT_LOCATION = + "src/test/java/com/solarwinds/joboe/core/rpc/invalid-collector.crt"; + private static int portWalker = TEST_SERVER_PORT_BASE; + private static int testServerPort = TEST_SERVER_PORT_BASE; + protected static final List TEST_SETTINGS = generateTestSettings(); + + private static final RpcClient.RetryParamConstants QUICK_RETRY = + new RpcClient.RetryParamConstants(100, 200, 3); + + protected TestCollector testCollector; + + @BeforeEach + public void setUp() throws Exception { + testServerPort = locateAvailablePort(); + testCollector = startCollector(testServerPort); + + Metadata.setup(SamplingConfiguration.builder().build()); + testEvents = generateTestEvents(); + bigEvent = generateBigEvent(); + } + + @AfterEach + public void tearDown() throws Exception { + testCollector.stop(); + } + + protected interface TestCollector { + List stop(); + + List flush(); + + Map getCallCountStats(); + } + + protected abstract TestCollector startCollector(int port) throws IOException; + + protected abstract TestCollector startRedirectCollector(int port, String redirectArg) + throws IOException; + + protected abstract TestCollector startRatedCollector(int port, ResultCode limitExceededCode) + throws IOException; + + protected abstract TestCollector startBiasedTestCollector(int port) throws IOException; + + // Test server that throws Runtime exception on every other message + protected abstract TestCollector startErroneousTestCollector(int port, double errorPercentage) + throws IOException; + + protected abstract void startSoftDisabledTestCollector(int port, String warning) + throws IOException; + + protected static String getServerPublicKeyLocation() { + return TEST_SERVER_CERT_LOCATION; + } + + private static List generateTestSettings() { + List settings = new ArrayList(); + + Map arguments = new HashMap(); + + arguments.put( + SettingsArg.BUCKET_CAPACITY.getKey(), SettingsArg.BUCKET_CAPACITY.toByteBuffer(32.0)); + arguments.put(SettingsArg.BUCKET_RATE.getKey(), SettingsArg.BUCKET_RATE.toByteBuffer(2.0)); + + settings.add( + new RpcSettings( + PollingSettingsFetcherTest.DEFAULT_FLAGS_STRING, + TimeUtils.getTimestampMicroSeconds(), + 1000000, + 600, + arguments)); + return settings; + } + + private static List generateTestEvents() { + List testEvents = new ArrayList(); + Context.getMetadata().randomize(); + Event entryEvent = Context.createEvent(); + entryEvent.addInfo("Layer", "test-thrift"); + entryEvent.addInfo("Label", "entry"); + testEvents.add(entryEvent); + + Event exitEvent = Context.createEvent(); + exitEvent.addInfo("Layer", "test-thrift"); + exitEvent.addInfo("Label", "exit"); + testEvents.add(exitEvent); + + return testEvents; + } + + private static Event generateBigEvent() { + Context.getMetadata().randomize(); + Event testEvent = Context.createEvent(); + for (int i = 0; i < 100; i++) { // each event is around 100 kb + testEvent.addInfo(String.valueOf(i), new byte[1024]); + } + + return testEvent; + } + + protected abstract ProtocolClientFactory getProtocolClientFactory(URL certUrl) + throws IOException, GeneralSecurityException; + + @Test + public void testConnectValidServer() throws Exception { + System.out.println("running testConnectValidServer"); + Client client = null; + try { + client = + new RpcClient( + TEST_SERVER_HOST, + testServerPort, + TEST_CLIENT_ID, + getProtocolClientFactory(new File(getServerPublicKeyLocation()).toURI().toURL())); + assertEquals( + com.solarwinds.joboe.core.rpc.ResultCode.OK, + client.postEvents(testEvents, null).get().getResultCode()); + assertEventEquals(testEvents, testCollector.flush()); + } finally { + if (client != null) { + client.close(); + } + } + } + + @Test + public void testConnectInvalidServer() throws Exception { + System.out.println("running testConnectInvalidServer"); + Client client = null; + try { + client = + new RpcClient( + TEST_SERVER_HOST, + testServerPort, + TEST_CLIENT_ID, + QUICK_RETRY, + getProtocolClientFactory(new File(INVALID_CERT_LOCATION).toURI().toURL())); + } catch (Exception e) { + // expected; + System.out.println("Fret not! Expected ^^"); + } finally { + if (client != null) { + client.close(); + } + } + } + + @Test + public void testConnectDeadServer() throws Exception { + System.out.println("running testConnectDeadServer"); + Client client = null; + try { + client = + new RpcClient( + TEST_SERVER_HOST, + 19876, + TEST_CLIENT_ID, + QUICK_RETRY, + getProtocolClientFactory(new File(getServerPublicKeyLocation()).toURI().toURL())); + client.postEvents(testEvents, null).get(5, TimeUnit.SECONDS); + fail("Expect exception thrown, but no exception found!"); + } catch (TimeoutException | ExecutionException e) { + // expected; + System.out.println("Fret not! Expected ^^"); + } finally { + if (client != null) { + client.close(); + } + } + } + + @Test + public void testConnectLazyServer() throws Exception { + int lazyServerPort = locateAvailablePort(); + + TestCollector lazyServer = null; + Client client = null; + + try { + client = + new RpcClient( + TEST_SERVER_HOST, + lazyServerPort, + TEST_CLIENT_ID, + getProtocolClientFactory(new File(getServerPublicKeyLocation()).toURI().toURL())); + Future futureResult = client.postEvents(testEvents, null); + + TimeUnit.SECONDS.sleep(5); // lazy!! + lazyServer = startCollector(lazyServerPort); // now start the lazy server + + assertEquals(com.solarwinds.joboe.core.rpc.ResultCode.OK, futureResult.get().getResultCode()); + assertEventEquals(testEvents, lazyServer.flush()); + + } finally { + if (client != null) { + client.close(); + } + lazyServer.stop(); + } + } + + @Test + public void testPostManyEvents() throws Exception { + System.out.println("running testPostManyEvents"); + Client client = null; + + try { + client = + new RpcClient( + TEST_SERVER_HOST, + testServerPort, + TEST_CLIENT_ID, + getProtocolClientFactory(new File(getServerPublicKeyLocation()).toURI().toURL())); + + List events = new ArrayList(); + + for (int i = 0; i < 1000; i++) { + events.addAll(testEvents); + } + + assertEquals( + com.solarwinds.joboe.core.rpc.ResultCode.OK, + client.postEvents(events, null).get().getResultCode()); + assertEventEquals(events, testCollector.flush()); + } finally { + if (client != null) { + client.close(); + } + } + } + + @Test + public void testPostManyBigEvents() throws Exception { + System.out.println("running testPostManyBigEvents"); + Client client = null; + + try { + client = + new RpcClient( + TEST_SERVER_HOST, + testServerPort, + TEST_CLIENT_ID, + getProtocolClientFactory(new File(getServerPublicKeyLocation()).toURI().toURL())); + + List events = new ArrayList(); + + int eventCount = 1000; + long totalSize = (long) eventCount * bigEvent.toBytes().length; // just an approximation + + for (int i = 0; i < eventCount; i++) { // 1000 events, each event is around 100kb + events.add(bigEvent); + } + + assertEquals( + com.solarwinds.joboe.core.rpc.ResultCode.OK, + client.postEvents(events, null).get().getResultCode()); + assertEventEquals(events, testCollector.flush()); + + Map callCountStats = testCollector.getCallCountStats(); + long minimumCallCount = totalSize / ProtocolClient.MAX_CALL_SIZE; // just an approximation + assert (callCountStats.get(TaskType.POST_EVENTS) + >= minimumCallCount); // should be more than 10 calls for post Events due to the split up + } finally { + if (client != null) { + client.close(); + } + } + } + + @Test + public void testGetSettings() throws Exception { + System.out.println("running testGetSettings"); + Client client = null; + + try { + client = + new RpcClient( + TEST_SERVER_HOST, + testServerPort, + TEST_CLIENT_ID, + getProtocolClientFactory(new File(getServerPublicKeyLocation()).toURI().toURL())); + + com.solarwinds.joboe.core.rpc.SettingsResult result = client.getSettings("", null).get(); + assertEquals(com.solarwinds.joboe.core.rpc.ResultCode.OK, result.getResultCode()); + assertEquals(TEST_SETTINGS.size(), result.getSettings().size()); + + for (int i = 0; i < TEST_SETTINGS.size(); i++) { + RpcSettings expectedSetting = TEST_SETTINGS.get(i); + Settings receivedSetting = result.getSettings().get(i); + assertEquals(expectedSetting.getType(), receivedSetting.getType()); + + short expectedFlags = expectedSetting.getFlags(); + assertEquals(expectedFlags, receivedSetting.getFlags()); + assertEquals(expectedSetting.getValue(), receivedSetting.getValue()); + assertEquals( + expectedSetting.getArgValue(SettingsArg.BUCKET_CAPACITY), + receivedSetting.getArgValue(SettingsArg.BUCKET_CAPACITY), + 0); + assertEquals( + expectedSetting.getArgValue(SettingsArg.BUCKET_RATE), + receivedSetting.getArgValue(SettingsArg.BUCKET_RATE), + 0); + } + } finally { + if (client != null) { + client.close(); + } + } + } + + @Test + public void testGetSettingsDeadServer() throws Exception { + System.out.println("running testGetSettingsDeadServer"); + Client client = null; + try { + client = + new RpcClient( + TEST_SERVER_HOST, + 19876, + TEST_CLIENT_ID, + getProtocolClientFactory(new File(getServerPublicKeyLocation()).toURI().toURL())); + client.getSettings("", null).get(5, TimeUnit.SECONDS); + fail("Expect exception thrown, but no exception found!"); + } catch (TimeoutException | ExecutionException e) { + // expected; + System.out.println("Fret not! Expected ^^"); + } finally { + if (client != null) { + client.close(); + } + } + } + + /** + * With non empty warning + * + * @throws Exception + */ + @Test + public void testGetSettingSoftDisabled() throws Exception { + System.out.println("running testGetSettings with warning (soft-disabled)"); + Client client = null; + int softDisabledServerPort = locateAvailablePort(); + + String warning = "Test warning"; + startSoftDisabledTestCollector(softDisabledServerPort, warning); + try { + client = + new RpcClient( + TEST_SERVER_HOST, + softDisabledServerPort, + TEST_CLIENT_ID, + getProtocolClientFactory(new File(getServerPublicKeyLocation()).toURI().toURL())); + + com.solarwinds.joboe.core.rpc.SettingsResult result = client.getSettings("", null).get(); + assertEquals(com.solarwinds.joboe.core.rpc.ResultCode.OK, result.getResultCode()); + assertEquals(warning, result.getWarning()); + } finally { + if (client != null) { + client.close(); + } + } + } + + @Test + public void testPostStatus() throws Exception { + System.out.println("running testPostStatus"); + Client client = null; + + try { + client = + new RpcClient( + TEST_SERVER_HOST, + testServerPort, + TEST_CLIENT_ID, + getProtocolClientFactory(new File(getServerPublicKeyLocation()).toURI().toURL())); + + Map testMessage = new HashMap(); + testMessage.put("SomeString", "123"); + testMessage.put("SomeInteger", 456); + testMessage.put("SomeDouble", 0.789); + testMessage.put("SomeBoolean", true); + + Map subMap = new HashMap(testMessage); + testMessage.put("SomeMap", subMap); + + Result result = client.postStatus(Collections.singletonList(testMessage), null).get(); + assertEquals(com.solarwinds.joboe.core.rpc.ResultCode.OK, result.getResultCode()); + + List receivedMessages = testCollector.flush(); + assertEquals(1, receivedMessages.size()); + + Set> recievedEntries = + getEntriesFromBytes(receivedMessages.get(0).array()); + assertEquals(testMessage.entrySet(), recievedEntries); + } finally { + if (client != null) { + client.close(); + } + } + } + + @Test + public void testPostStatusBigMessage() throws Exception { + System.out.println("running testPostStatusBigMessage"); + Client client = null; + final int ENTRY_COUNT = 500; + final int ENTRY_SIZE = 1024; + try { + client = + new RpcClient( + TEST_SERVER_HOST, + testServerPort, + TEST_CLIENT_ID, + getProtocolClientFactory(new File(getServerPublicKeyLocation()).toURI().toURL())); + + List> testMessages = new ArrayList>(); + Map testMessage = new HashMap(); + for (int j = 0; j < ENTRY_COUNT; j++) { + testMessage.put(String.valueOf(j), new Byte[ENTRY_SIZE]); + } + testMessages.add(testMessage); + + client + .postStatus(testMessages, null) + .get(); // big but it's within the max size defined in ThriftClient MAX_MESSAGE_SIZE + } finally { + if (client != null) { + client.close(); + } + } + } + + @Test + public void testPostStatusHugeMessage() throws Exception { + System.out.println("running testPostStatusHugeMessage"); + Client client = null; + final int ENTRY_COUNT = 2000; + final int ENTRY_SIZE = 1024; + try { + client = + new RpcClient( + TEST_SERVER_HOST, + testServerPort, + TEST_CLIENT_ID, + getProtocolClientFactory(new File(getServerPublicKeyLocation()).toURI().toURL())); + + List> testMessages = new ArrayList>(); + Map testMessage = new HashMap(); + for (int j = 0; j < ENTRY_COUNT; j++) { + testMessage.put(String.valueOf(j), new Byte[ENTRY_SIZE]); + } + testMessages.add(testMessage); + + client.postStatus(testMessages, null).get(); // should throw exception + fail("Expected " + ExecutionException.class.getName() + " but it was not thrown"); + } catch (ExecutionException e) { + if (!(e.getCause() instanceof ClientFatalException)) { + fail( + "Expected instance of " + + ClientFatalException.class.getName() + + " but found " + + e.getCause().getClass().getName()); + } + // expected; + System.out.println("Fret not! Expected ^^"); + } finally { + if (client != null) { + client.close(); + } + } + } + + // TODO isn't postMetrics essentially the same as postStatus??? + + @Test + public void testPostMetrics() throws Exception { + System.out.println("running testPostMetrics"); + Client client = null; + + try { + client = + new RpcClient( + TEST_SERVER_HOST, + testServerPort, + TEST_CLIENT_ID, + getProtocolClientFactory(new File(getServerPublicKeyLocation()).toURI().toURL())); + + Map testMessage = new HashMap(); + testMessage.put("SomeString", "123"); + testMessage.put("SomeInteger", 456); + testMessage.put("SomeDouble", 0.789); + testMessage.put("SomeBoolean", true); + + Map subMap = new HashMap(testMessage); + testMessage.put("SomeMap", subMap); + + Result result = client.postMetrics(Collections.singletonList(testMessage), null).get(); + assertEquals(com.solarwinds.joboe.core.rpc.ResultCode.OK, result.getResultCode()); + + List receivedMessages = testCollector.flush(); + assertEquals(1, receivedMessages.size()); + + Set> recievedEntries = + getEntriesFromBytes(receivedMessages.get(0).array()); + assertEquals(testMessage.entrySet(), recievedEntries); + } finally { + if (client != null) { + client.close(); + } + } + } + + @Test + public void testConnectionInit() throws Exception { + System.out.println("running testConnectionInit"); + Client client = null; + + try { + client = + new RpcClient( + TEST_SERVER_HOST, + testServerPort, + TEST_CLIENT_ID, + getProtocolClientFactory(new File(getServerPublicKeyLocation()).toURI().toURL())); + + Map testMessage = new HashMap(); + Result result = + client + .postMetrics(Collections.singletonList(testMessage), null) + .get(); // post something to ensure connection init is triggered and sent + assertEquals(com.solarwinds.joboe.core.rpc.ResultCode.OK, result.getResultCode()); + + List receivedMessages = testCollector.flush(); + assertEquals(1, receivedMessages.size()); + + } finally { + if (client != null) { + client.close(); + } + } + } + + @Test + public void testRedirectLoop() throws Exception { + System.out.println("running testRedirectLoop"); + int redirectPort = locateAvailablePort(); + TestCollector redirectServer = + startRedirectCollector(redirectPort, "localhost:" + redirectPort); // redirect loop + + Client client = null; + + try { + client = + new RpcClient( + TEST_SERVER_HOST, + redirectPort, + TEST_CLIENT_ID, + getProtocolClientFactory(new File(getServerPublicKeyLocation()).toURI().toURL())); + + assertEquals( + com.solarwinds.joboe.core.rpc.ResultCode.REDIRECT, + client.postEvents(testEvents, null).get().getResultCode()); + } finally { + if (client != null) { + client.close(); + } + redirectServer.stop(); + } + } + + @Test + public void testValidRedirect() throws Exception { + System.out.println("running testValidRedirect"); + int redirectPort1 = locateAvailablePort(); + int redirectPort2 = redirectPort1 + 1; + TestCollector redirectServer1 = + startRedirectCollector(redirectPort1, "localhost:" + redirectPort2); + TestCollector redirectServer2 = + startRedirectCollector( + redirectPort2, "localhost:" + testServerPort); // redirect back to correct server + + Client client = null; + + try { + client = + new RpcClient( + TEST_SERVER_HOST, + redirectPort1, + TEST_CLIENT_ID, + getProtocolClientFactory(new File(getServerPublicKeyLocation()).toURI().toURL())); + + assertEquals( + com.solarwinds.joboe.core.rpc.ResultCode.OK, + client.postEvents(testEvents, null).get().getResultCode()); + assertEventEquals(testEvents, testCollector.flush()); + } finally { + if (client != null) { + client.close(); + } + redirectServer1.stop(); + redirectServer2.stop(); + } + } + + @Test + public void testInvalidRedirectArg() throws Exception { + System.out.println("running testInvalidRedirectArg"); + int redirectPort = locateAvailablePort(); + TestCollector redirectServer = + startRedirectCollector( + redirectPort, "http://localhost:" + redirectPort); // invalid redirect arg format + Client client = null; + + try { + client = + new RpcClient( + TEST_SERVER_HOST, + redirectPort, + TEST_CLIENT_ID, + getProtocolClientFactory(new File(getServerPublicKeyLocation()).toURI().toURL())); + client.postEvents(testEvents, null).get(); + fail("Expect exception thrown, but no exception found!"); + } catch (ExecutionException e) { + // expected + System.out.println("Fret not! Expected ^^"); + } finally { + if (client != null) { + client.close(); + } + redirectServer.stop(); + } + } + + @Test + public void testInvalidRedirectTarget() throws Exception { + System.out.println("running testInvalidRedirectTarget"); + int redirectPort = locateAvailablePort(); + TestCollector redirectServer = + startRedirectCollector( + redirectPort, "unknown-host-aieeeeeeee:" + redirectPort); // invalid redirect arg format + Client client = null; + + try { + client = + new RpcClient( + TEST_SERVER_HOST, + redirectPort, + TEST_CLIENT_ID, + QUICK_RETRY, + getProtocolClientFactory(new File(getServerPublicKeyLocation()).toURI().toURL())); + + client.postEvents(testEvents, null).get(5, TimeUnit.SECONDS); + + fail("Expect exception thrown, but no exception found!"); + } catch (TimeoutException | ExecutionException e) { + // expected; + System.out.println("Fret not! Expected ^^"); + } finally { + if (client != null) { + client.close(); + } + redirectServer.stop(); + } + } + + /** + * Tests against thrift server that processes on certain speed, once exceeded it will return try + * later. + * + *

This verifies the retry mechanism of this client + * + * @throws Exception + */ + @Test + public void testTryLater() throws Exception { + System.out.println("running testTryLater"); + int tryLaterPort = locateAvailablePort(); + + TestCollector tryLaterCollector = startRatedCollector(tryLaterPort, ResultCode.TRY_LATER); + Client client = null; + + try { + client = + new RpcClient( + TEST_SERVER_HOST, + tryLaterPort, + TEST_CLIENT_ID, + getProtocolClientFactory(new File(getServerPublicKeyLocation()).toURI().toURL())); + + List> futures = new ArrayList>(); + List sentEvents = new ArrayList(); + for (int i = 0; i < 10; i++) { + futures.add( + client.postEvents( + testEvents, + null)); // post events, the handler will return TRY_LATER most of the time, but the + // client should be able to retry the request until it's OK eventually + sentEvents.addAll(testEvents); + } + + for (Future future : futures) { + assertEquals(com.solarwinds.joboe.core.rpc.ResultCode.OK, future.get().getResultCode()); + } + + assertEventEquals(sentEvents, tryLaterCollector.flush()); + } finally { + if (client != null) { + client.close(); + } + tryLaterCollector.stop(); + } + } + + /** + * Tests against thrift server that processes on certain speed, once exceeded it will return limit + * exceed + * + *

This verifies the retry mechanism of this client + * + * @throws Exception + */ + @Test + public void testLimitExceed() throws Exception { + System.out.println("running testLimitExceed"); + int tryLaterPort = locateAvailablePort(); + + TestCollector tryLaterServer = startRatedCollector(tryLaterPort, ResultCode.LIMIT_EXCEEDED); + Client client = null; + + try { + client = + new RpcClient( + TEST_SERVER_HOST, + tryLaterPort, + TEST_CLIENT_ID, + getProtocolClientFactory(new File(getServerPublicKeyLocation()).toURI().toURL())); + + List> futures = new ArrayList>(); + List sentEvents = new ArrayList(); + for (int i = 0; i < 10; i++) { + futures.add( + client.postEvents( + testEvents, + null)); // post events, the handler will return LIMIT_EXCEEDED most of the time, but + // the client should be able to retry the request until it's OK eventually + sentEvents.addAll(testEvents); + } + + for (Future future : futures) { + assertEquals(com.solarwinds.joboe.core.rpc.ResultCode.OK, future.get().getResultCode()); + } + + assertEventEquals(sentEvents, tryLaterServer.flush()); + } finally { + if (client != null) { + client.close(); + } + tryLaterServer.stop(); + } + } + + @Test + public void testUnstableServer() throws Exception { + System.out.println("running testUnstableServer"); + final int unstableServerPort = locateAvailablePort(); + + final AtomicBoolean keepRunning = new AtomicBoolean(true); + + final List collectorEvents = new ArrayList(); + + Thread serverThread = + new Thread() { + @Override + public void run() { + while (keepRunning.get()) { + try { + while (isPortNotAvailable()) { + System.out.println( + "unstable server port [" + + unstableServerPort + + "] is not yet available... sleeping"); + TimeUnit.SECONDS.sleep(1); + } + + TestCollector unstableServer = startCollector(unstableServerPort); + System.out.println("unstable server serving for 1 sec..."); + Thread.sleep(1000); + System.out.println("unstable server coming down..."); + List receivedEvents = unstableServer.stop(); + collectorEvents.addAll(receivedEvents); + } catch (Exception e) { + e.printStackTrace(); + } + } + } + }; + + serverThread.start(); + + Client client = null; + + try { + client = + new RpcClient( + TEST_SERVER_HOST, + unstableServerPort, + TEST_CLIENT_ID, + getProtocolClientFactory(new File(getServerPublicKeyLocation()).toURI().toURL())); + + List> futures = new ArrayList>(); + List sentEvents = new ArrayList(); + for (int i = 0; i < 100; i++) { + Context.getMetadata().randomize(); + Event entryEvent = Context.createEvent(); + entryEvent.addInfo("Layer", "test-" + i); + entryEvent.addInfo("Label", "info"); + + futures.add( + client.postEvents( + Collections.singletonList(entryEvent), + null)); // post alot of events, and it is supposed to retry if connection is gone + // intermittedly + Thread.sleep(100); // some sleep in between so the the post might hit the server downtime + sentEvents.add(entryEvent); + } + + for (Future future : futures) { + assertEquals(com.solarwinds.joboe.core.rpc.ResultCode.OK, future.get().getResultCode()); + } + + keepRunning.set(false); + serverThread.join(); // wait for the server thread to complete (so server is stopped properly) + + assertEventEquals(sentEvents, collectorEvents); + } finally { + if (client != null) { + client.close(); + } + } + } + + @Test + public void testOccasionalErrorServer() throws Exception { + System.out.println("running testOccasionalErrorServer"); + int errorServerPort = locateAvailablePort(); + + TestCollector errorServer = + startErroneousTestCollector(errorServerPort, 0.5); // half of them run into exception + Client client = null; + + try { + client = + new RpcClient( + TEST_SERVER_HOST, + errorServerPort, + TEST_CLIENT_ID, + getProtocolClientFactory(new File(getServerPublicKeyLocation()).toURI().toURL())); + + List> futures = new ArrayList>(); + List sentEvents = new ArrayList(); + for (int i = 0; i < 10; i++) { + futures.add( + client.postEvents( + testEvents, + null)); // post events, the handler will return LIMIT_EXCEEDED most of the time, but + // the client should be able to retry the request until it's OK eventually + sentEvents.addAll(testEvents); + } + + // TODO + for (Future future : futures) { // should all be OK eventually after retry + assertEquals(com.solarwinds.joboe.core.rpc.ResultCode.OK, future.get().getResultCode()); + } + + assertEventEquals(sentEvents, errorServer.flush()); + } finally { + if (client != null) { + client.close(); + } + errorServer.stop(); + } + } + + @Test + public void testErrorServer() throws Exception { + System.out.println("running testErrorServer"); + int errorServerPort = locateAvailablePort(); + + TestCollector errorServer = + startErroneousTestCollector(errorServerPort, 1); // always return error + Client client = null; + + try { + client = + new RpcClient( + TEST_SERVER_HOST, + errorServerPort, + TEST_CLIENT_ID, + QUICK_RETRY, + getProtocolClientFactory(new File(getServerPublicKeyLocation()).toURI().toURL())); + + client.postEvents(testEvents, null).get(); // will fail and give up base on QUICK_RETRY + + fail("Expect exception because of retry failures, but it's not thrown!"); + } catch (ExecutionException e) { + if (!(e.getCause() instanceof ClientException)) { + fail("Expect ClientException because of retry failures, but found " + e.getCause()); + } + // expected + } finally { + if (client != null) { + client.close(); + } + errorServer.stop(); + } + } + + /** + * Test different calls processing should block each other based on different status code. For + * example a server might return TRY_LATER for postMetrics but get settings calls might be OK, + * therefore the getSettings calls should not get held up by postMetrics's failure + * + * @throws Exception + */ + @Test + public void testBiasedServer() throws Exception { + System.out.println("running testBiasedServer"); + int biasedServerPort = locateAvailablePort(); + + TestCollector basiedServer = startBiasedTestCollector(biasedServerPort); + Client client = + new RpcClient( + TEST_SERVER_HOST, + biasedServerPort, + TEST_CLIENT_ID, + getProtocolClientFactory(new File(getServerPublicKeyLocation()).toURI().toURL())); + assertThrows( + TimeoutException.class, + () -> + client.postMetrics(new ArrayList>(), null).get(5, TimeUnit.SECONDS), + "Not expecting to return any result for this call!"); // this is supposed to get held up + // because of TRY_LAYER) + + assertEquals( + com.solarwinds.joboe.core.rpc.ResultCode.OK, + client.getSettings("", null).get().getResultCode()); // this should be successful + client.close(); + basiedServer.stop(); + } + + protected void assertEventEquals(List testEvents, List receivedMessages) + throws BsonBufferException { + assertEquals(testEvents.size(), receivedMessages.size()); + for (int i = 0; i < testEvents.size(); i++) { + assertEquals(testEvents.get(i).toByteBuffer(), receivedMessages.get(i)); + } + } + + protected static Set> getEntriesFromBytes(byte[] bytes) { + return getBsonDocumentFromBytes(bytes).entrySet(); + } + + protected static BsonDocument getBsonDocumentFromBytes(byte[] bytes) { + ByteBuffer buffer = + ByteBuffer.allocate(Constants.MAX_EVENT_BUFFER_SIZE).order(ByteOrder.LITTLE_ENDIAN); + buffer.put(bytes); + buffer.flip(); + + return BsonDocuments.readFrom(buffer); + } + + /** Checks to see if a specific port is available. */ + public static synchronized int locateAvailablePort() { + int MAX_PORT = portWalker + 2000; // huh shouldn't hit it + while (portWalker <= MAX_PORT) { + if (isPortNotAvailable()) { + portWalker++; + } else { + return portWalker; + } + } + + return 0; + } + + private static boolean isPortNotAvailable() { + Socket s = new Socket(); + try { + s.connect(new InetSocketAddress("localhost", portWalker), 100); + // that means the port is occupied + return true; + } catch (IOException e) { + // that means port is available + return false; + } finally { + try { + s.close(); + } catch (IOException e) { + e.printStackTrace(); + } + } + } + + protected static class ErrorState { + private final double errorPercentage; + private double sum = 0.0; + + public ErrorState(double errorPercentage) { + this.errorPercentage = errorPercentage; + } + + public boolean isNextAsError() { + sum += errorPercentage; + boolean isError = sum >= 1.0; + if (isError) { + sum = sum - (int) sum; + } + return isError; + } + } +} diff --git a/libs/core/src/test/java/com/solarwinds/joboe/core/rpc/grpc/GrpcClientTest.java b/libs/core/src/test/java/com/solarwinds/joboe/core/rpc/grpc/GrpcClientTest.java new file mode 100644 index 00000000..2b1ced66 --- /dev/null +++ b/libs/core/src/test/java/com/solarwinds/joboe/core/rpc/grpc/GrpcClientTest.java @@ -0,0 +1,583 @@ +/* + * © SolarWinds Worldwide, LLC. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.solarwinds.joboe.core.rpc.grpc; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.fail; + +import com.google.protobuf.ByteString; +import com.solarwinds.joboe.core.rpc.Client; +import com.solarwinds.joboe.core.rpc.ClientFatalException; +import com.solarwinds.joboe.core.rpc.ProtocolClient; +import com.solarwinds.joboe.core.rpc.ProtocolClientFactory; +import com.solarwinds.joboe.core.rpc.ResultCode; +import com.solarwinds.joboe.core.rpc.RpcClient; +import com.solarwinds.joboe.core.rpc.RpcClient.TaskType; +import com.solarwinds.joboe.core.rpc.RpcClientTest; +import com.solarwinds.joboe.core.rpc.RpcSettings; +import com.solarwinds.joboe.core.settings.PollingSettingsFetcherTest; +import com.solarwinds.joboe.core.util.TimeUtils; +import com.solarwinds.joboe.sampling.SettingsArg; +import com.solarwinds.trace.ingestion.proto.Collector; +import com.solarwinds.trace.ingestion.proto.TraceCollectorGrpc; +import io.grpc.Server; +import io.grpc.ServerBuilder; +import io.grpc.Status; +import io.grpc.stub.StreamObserver; +import java.io.File; +import java.io.IOException; +import java.net.URL; +import java.nio.ByteBuffer; +import java.security.GeneralSecurityException; +import java.util.ArrayList; +import java.util.Collections; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.concurrent.ExecutionException; +import java.util.concurrent.atomic.AtomicBoolean; +import lombok.Getter; +import org.junit.jupiter.api.Test; + +public class GrpcClientTest extends RpcClientTest { + private static final String TEST_SERVER_PRIVATE_KEY_LOCATION = + "src/test/java/com/solarwinds/joboe/core/rpc/grpc/test-collector-private.pem"; + + private static final List TEST_OBOE_SETTINGS = + convertToOboeSettings(TEST_SETTINGS); + + @SuppressWarnings("unchecked") + private static List convertToOboeSettings(List settings) { + List oboeSettings = new ArrayList(); + + for (RpcSettings fromEntry : settings) { + Map arguments = new HashMap(); + for (SettingsArg arg : SettingsArg.values()) { + Object argValue = fromEntry.getArgValue(arg); + if (argValue != null) { + arguments.put(arg.getKey(), ByteString.copyFrom(arg.toByteBuffer(argValue))); + } + } + Collector.OboeSetting setting = + Collector.OboeSetting.newBuilder() + .setFlags(ByteString.copyFromUtf8(PollingSettingsFetcherTest.DEFAULT_FLAGS_STRING)) + .setTimestamp(TimeUtils.getTimestampMicroSeconds()) + .setValue(1000000) + .setTtl(600) + .putAllArguments(arguments) + .build(); + oboeSettings.add(setting); + } + return oboeSettings; + } + + @Test + public void testExhaustedServer() throws Exception { + System.out.println("running testExhaustedServer"); + int exhaustedServerPort = locateAvailablePort(); + startExhaustedServer(exhaustedServerPort); + Client client = null; + + try { + client = + new RpcClient( + TEST_SERVER_HOST, + exhaustedServerPort, + TEST_CLIENT_ID, + getProtocolClientFactory(new File(getServerPublicKeyLocation()).toURI().toURL())); + + assertEquals( + com.solarwinds.joboe.core.rpc.ResultCode.OK, + client.postEvents(testEvents, null).get().getResultCode()); + fail("Expected " + ClientFatalException.class.getName() + " but found none"); + } catch (ExecutionException e) { + if (!(e.getCause() instanceof ClientFatalException)) { + fail("Expected " + ClientFatalException.class.getName() + " but found " + e.getCause()); + } + } finally { + if (client != null) { + client.close(); + } + } + } + + @Override + protected TestCollector startCollector(int port) throws IOException { + return new GrpcCollector(port, new GrpcCollectorService()); + } + + @Override + protected TestCollector startRedirectCollector(int port, String redirectArg) throws IOException { + return new GrpcCollector(port, new GrpcRedirectCollectorService(redirectArg)); + } + + @Override + protected TestCollector startRatedCollector(int port, ResultCode limitExceededCode) + throws IOException { + return new GrpcCollector( + port, + new GrpcRatedCollectorService(10, Collector.ResultCode.valueOf(limitExceededCode.name()))); + } + + @Override + protected TestCollector startBiasedTestCollector(int port) throws IOException { + return new GrpcCollector( + port, + new GrpcBiasedCollectorService( + Collections.singletonMap(TaskType.POST_METRICS, Collector.ResultCode.TRY_LATER))); + } + + @Override + protected TestCollector startErroneousTestCollector(int port, double errorPercentage) + throws IOException { + return new GrpcCollector(port, new GrpcErroneousCollectorService(errorPercentage)); + } + + @Override + protected void startSoftDisabledTestCollector(int port, String warning) throws IOException { + new GrpcCollector(port, new GrpcCollectorService(Collector.ResultCode.OK, "", warning)); + } + + private void startExhaustedServer(int port) throws IOException { + new GrpcCollector( + port, + new GrpcCollectorService() { + @Override + public void postEvents( + Collector.MessageRequest request, + StreamObserver responseObserver) { + responseObserver.onError( + Status.RESOURCE_EXHAUSTED + .withDescription("Testing resource exhaustion on server side") + .asRuntimeException()); + } + }); + } + + @Override + protected ProtocolClientFactory getProtocolClientFactory(URL certUrl) + throws IOException, GeneralSecurityException { + return new GrpcClient.GrpcProtocolClientFactory(certUrl); + } + + private static class GrpcCollector implements TestCollector { + private final Server server; + private final GrpcCollectorService service; + + GrpcCollector(int port, GrpcCollectorService service) throws IOException { + ServerBuilder builder = + ServerBuilder.forPort(port) + .useTransportSecurity( + new File(getServerPublicKeyLocation()), + new File(TEST_SERVER_PRIVATE_KEY_LOCATION)) + .maxInboundMessageSize( + ProtocolClient.MAX_CALL_SIZE + 1024 * 1024) // accept a slightly bigger message + .addService(service); + this.server = builder.build(); + this.service = service; + server.start(); + System.out.println( + "Grpc collector started at " + port + " with service GrpcCollectorService"); + } + + @Override + public List stop() { + if (server != null) { + server.shutdown(); + try { + server.awaitTermination(); + } catch (InterruptedException ignore) { + } + } + return flush(); + } + + @Override + public List flush() { + return service.flush(); + } + + @Override + public Map getCallCountStats() { + return service.getCallCountStats(); + } + } + + private static class GrpcCollectorService extends TraceCollectorGrpc.TraceCollectorImplBase { + private final Collector.ResultCode resultCode; + private final String arg; + private final String warning; + protected List buffer = + new ArrayList(); // what has been received so far + protected final Collector.MessageResult result; + private static final Collector.MessageResult PING_RESULT = + Collector.MessageResult.newBuilder().setResult(Collector.ResultCode.OK).setArg("").build(); + @Getter private final Map callCountStats = new HashMap(); + + public GrpcCollectorService() { + this(Collector.ResultCode.OK, "", ""); + } + + public GrpcCollectorService(Collector.ResultCode resultCode, String arg, String warning) { + this.resultCode = resultCode; + this.arg = arg; + this.warning = warning; + this.result = + Collector.MessageResult.newBuilder() + .setResult(resultCode) + .setArg(arg) + .setWarning("") + .build(); + } + + @Override + public void postEvents( + Collector.MessageRequest request, + StreamObserver responseObserver) { + buffer.addAll(request.getMessagesList()); + responseObserver.onNext(result); + responseObserver.onCompleted(); + + incrementCallCountStats(TaskType.POST_EVENTS); + } + + @Override + public void postMetrics( + Collector.MessageRequest request, + StreamObserver responseObserver) { + buffer.addAll(request.getMessagesList()); + responseObserver.onNext(result); + responseObserver.onCompleted(); + + incrementCallCountStats(TaskType.POST_METRICS); + } + + @Override + public void postStatus( + Collector.MessageRequest request, + StreamObserver responseObserver) { + buffer.addAll(request.getMessagesList()); + responseObserver.onNext(result); + responseObserver.onCompleted(); + + incrementCallCountStats(TaskType.POST_STATUS); + } + + @Override + public void getSettings( + Collector.SettingsRequest request, + StreamObserver responseObserver) { + Collector.SettingsResult settingsResult = + Collector.SettingsResult.newBuilder() + .setResult(resultCode) + .setArg(arg) + .setWarning(warning) + .addAllSettings(TEST_OBOE_SETTINGS) + .build(); + responseObserver.onNext(settingsResult); + responseObserver.onCompleted(); + + incrementCallCountStats(TaskType.GET_SETTINGS); + } + + private void incrementCallCountStats(TaskType taskType) { + Long existingCount = callCountStats.get(taskType); + if (existingCount == null) { + existingCount = 0L; + } + callCountStats.put(taskType, ++existingCount); + } + + @Override + public void ping( + Collector.PingRequest request, StreamObserver responseObserver) { + responseObserver.onNext(PING_RESULT); + responseObserver.onCompleted(); + } + + public List flush() { + List messages = new ArrayList<>(); + + for (ByteString entry : buffer) { + messages.add(ByteBuffer.wrap(entry.toByteArray())); + } + + buffer.clear(); + + return messages; + } + } + + private static class GrpcBiasedCollectorService extends GrpcCollectorService { + private final Map resultCodeByTaskType; + + public GrpcBiasedCollectorService(Map resultCodeByTaskType) { + super(); + this.resultCodeByTaskType = resultCodeByTaskType; + } + + @Override + public void postEvents( + Collector.MessageRequest request, + StreamObserver responseObserver) { + if (resultCodeByTaskType.containsKey(TaskType.POST_EVENTS)) { + responseObserver.onNext( + Collector.MessageResult.newBuilder() + .setResult(resultCodeByTaskType.get(TaskType.POST_EVENTS)) + .setArg("") + .build()); + responseObserver.onCompleted(); + } else { + super.postEvents(request, responseObserver); + } + } + + @Override + public void postMetrics( + Collector.MessageRequest request, + StreamObserver responseObserver) { + if (resultCodeByTaskType.containsKey(TaskType.POST_METRICS)) { + responseObserver.onNext( + Collector.MessageResult.newBuilder() + .setResult(resultCodeByTaskType.get(TaskType.POST_METRICS)) + .setArg("") + .build()); + responseObserver.onCompleted(); + } else { + super.postMetrics(request, responseObserver); + } + } + + @Override + public void postStatus( + Collector.MessageRequest request, + StreamObserver responseObserver) { + if (resultCodeByTaskType.containsKey(TaskType.POST_STATUS)) { + responseObserver.onNext( + Collector.MessageResult.newBuilder() + .setResult(resultCodeByTaskType.get(TaskType.POST_STATUS)) + .setArg("") + .build()); + responseObserver.onCompleted(); + } else { + super.postStatus(request, responseObserver); + } + } + + @Override + public void getSettings( + Collector.SettingsRequest request, + StreamObserver responseObserver) { + if (resultCodeByTaskType.containsKey(TaskType.GET_SETTINGS)) { + responseObserver.onNext( + Collector.SettingsResult.newBuilder() + .setResult(resultCodeByTaskType.get(TaskType.GET_SETTINGS)) + .setArg("") + .addAllSettings(TEST_OBOE_SETTINGS) + .build()); + responseObserver.onCompleted(); + } else { + super.getSettings(request, responseObserver); + } + } + } + + private static class GrpcRedirectCollectorService extends GrpcCollectorService { + private final Collector.MessageResult REDIRECT_RESULT; + private final String redirectArg; + + public GrpcRedirectCollectorService(String redirectArg) { + super(); + this.redirectArg = redirectArg; + REDIRECT_RESULT = + Collector.MessageResult.newBuilder() + .setResult(Collector.ResultCode.REDIRECT) + .setArg(redirectArg) + .build(); + } + + @Override + public void postEvents( + Collector.MessageRequest request, + StreamObserver responseObserver) { + responseObserver.onNext(REDIRECT_RESULT); + responseObserver.onCompleted(); + } + + @Override + public void postMetrics( + Collector.MessageRequest request, + StreamObserver responseObserver) { + responseObserver.onNext(REDIRECT_RESULT); + responseObserver.onCompleted(); + } + + @Override + public void postStatus( + Collector.MessageRequest request, + StreamObserver responseObserver) { + responseObserver.onNext(REDIRECT_RESULT); + responseObserver.onCompleted(); + } + + @Override + public void getSettings( + Collector.SettingsRequest request, + StreamObserver responseObserver) { + Collector.SettingsResult settingsResult = + Collector.SettingsResult.newBuilder() + .setResult(Collector.ResultCode.REDIRECT) + .setArg(redirectArg) + .addAllSettings(TEST_OBOE_SETTINGS) + .build(); + responseObserver.onNext(settingsResult); + responseObserver.onCompleted(); + } + + @Override + public void ping( + Collector.PingRequest request, StreamObserver responseObserver) { + responseObserver.onNext(REDIRECT_RESULT); + responseObserver.onCompleted(); + } + } + + private static class GrpcRatedCollectorService extends GrpcCollectorService { + private final int processingSpeedPerMessage; + private final Collector.ResultCode fullResultCode; + private final AtomicBoolean isProcessingAtomic = new AtomicBoolean(false); + + public GrpcRatedCollectorService( + int processingTimePerMessage, Collector.ResultCode fullResultCode) { + super(); + this.processingSpeedPerMessage = processingTimePerMessage; + this.fullResultCode = fullResultCode; + } + + @Override + public void postEvents( + Collector.MessageRequest request, + StreamObserver responseObserver) { + processMessage(request.getMessagesList(), responseObserver); + } + + @Override + public void postMetrics( + Collector.MessageRequest request, + StreamObserver responseObserver) { + processMessage(request.getMessagesList(), responseObserver); + } + + @Override + public void postStatus( + Collector.MessageRequest request, + StreamObserver responseObserver) { + processMessage(request.getMessagesList(), responseObserver); + } + + @Override + public void getSettings( + Collector.SettingsRequest request, + StreamObserver responseObserver) { + // do nothing + } + + @Override + public void ping( + Collector.PingRequest request, StreamObserver responseObserver) { + processMessage(Collections.emptyList(), responseObserver); + } + + @SuppressWarnings("unchecked") + private void processMessage(final List messages, StreamObserver responseObserver) { + boolean alreadyProcessing = isProcessingAtomic.getAndSet(true); + if (alreadyProcessing) { + responseObserver.onNext( + Collector.MessageResult.newBuilder().setResult(fullResultCode).setArg("").build()); + responseObserver.onCompleted(); + } else { + buffer.addAll(messages); + + // since the server is blocking, we have to return the result but delay setting back the + // isProcessingAtomic flag to indicate that it's busy + new Thread( + () -> { + try { + Thread.sleep((long) messages.size() * processingSpeedPerMessage); + } catch (InterruptedException ignore) { + } + isProcessingAtomic.set(false); + }) + .start(); + + responseObserver.onNext( + Collector.MessageResult.newBuilder() + .setResult(Collector.ResultCode.OK) + .setArg("") + .build()); + responseObserver.onCompleted(); + } + } + } + + private static class GrpcErroneousCollectorService extends GrpcCollectorService { + private final ErrorState errorState; + + public GrpcErroneousCollectorService(double errorPercentage) { + super(); + this.errorState = new ErrorState(errorPercentage); + } + + @Override + public void postEvents( + Collector.MessageRequest request, + StreamObserver responseObserver) { + checkError(); + super.postEvents(request, responseObserver); + } + + @Override + public void postMetrics( + Collector.MessageRequest request, + StreamObserver responseObserver) { + checkError(); + super.postMetrics(request, responseObserver); + } + + @Override + public void postStatus( + Collector.MessageRequest request, + StreamObserver responseObserver) { + checkError(); + super.postStatus(request, responseObserver); + } + + @Override + public void getSettings( + Collector.SettingsRequest request, + StreamObserver responseObserver) { + checkError(); + super.getSettings(request, responseObserver); + } + + private void checkError() throws RuntimeException { + if (errorState.isNextAsError()) { + throw new RuntimeException("Test exception from erroneous server"); + } + } + } +} diff --git a/libs/core/src/test/java/com/solarwinds/joboe/core/rpc/grpc/test-collector-private.pem b/libs/core/src/test/java/com/solarwinds/joboe/core/rpc/grpc/test-collector-private.pem new file mode 100644 index 00000000..e471be7e --- /dev/null +++ b/libs/core/src/test/java/com/solarwinds/joboe/core/rpc/grpc/test-collector-private.pem @@ -0,0 +1,28 @@ +-----BEGIN PRIVATE KEY----- +MIIEvwIBADANBgkqhkiG9w0BAQEFAASCBKkwggSlAgEAAoIBAQDGcjQDADGPuYFF ++d+EO420lyEoZuRGx+PSU4D1+Ir0TvTyxvv3WlPea3gfIG5quqlDrbST12HIwTSr ++e4OdE2lx6SuO58zl6i95OsAEMmnJQj1G6Z9xxT1wNYMFTGml+t0vWwmW7id061A +9Yuxe5N73xyPqeHkz+4fP/pxkhPJ7LdfVPDImUjCNUtzjvySQhBRWnYq3p6BCO1S +Utd/EBEUiqUKmM0oMlWlEv/DGfYTZfUoRUsViZKSb7GnFv4owrtE0V3C5NTuPqn1 +YJpiT2gRFC1lqaEVDgm/6xy9jr3QO73fzuj83vadi/TYt2pkpD1S6pWrS8btNdQ5 +GEo0JTiLAgMBAAECggEBAI4Uchy75MAsZtv8/QUlxl1H3xuYH1R2BS0vUCPLoWEt +rr6rrPb6GxYiB8zxYVzU7B9inOlEyeP8QIPo24JJztYkzElasq8zpELhRUe0vUwI +fhNPirJ++QjC2f2opvXJy3C3tlj4ToPhbCgYJb4a5gtIQKCzVuKF5M1G9z6dAcIN +kIQrP2a9Ltr13JlQRfU2X04hrbhXt21un7FfCWWkLclNtjzY3TqZMzMBomGaZ2iA +PqozRh9grYy0FGP0/mAAHhr8R1zsipZoGHTFAVrkNwZ/E2nMzWSfCM56jrmC7iYb +ot1jgIZ+ilz2laTLju3LzealT5F5ZmpZ+ZgpirhdpRECgYEA6TfG0JPSu7vk1/0t +9MY6a3UN93eVUI0jMVoolfM4FFHKtesvRvUvOdWbkK4ThfqfcpZ9gBskCoCuNfpY +F2e4yGCCvWqe+iRReYCwR5WYfQf+444Yv6KS0aDT6aufk2ZRtNdG1q1Hg4QOyDRZ +OZa2tZ7nMf1cq2WaeZWErFj6qWcCgYEA2dTeDnmEKojegJUzLCWR1pJVcX89kJLN ++SmRwq7Oo51hO9Q/CJYe+nD0ndfRJoDPSaEwKWphGKcxo4YyrEiafCT3yfdrwPSe +0Qn5mxy7iMK2f6/H1iVSelu4OmnMHntGo0agzxKnmu4ZSP5+Ar/hZd0Lrl2sSOQg +EBFG5DcmbT0CgYBqiVJHIeAYZoLpr/x4Xr19LSHONFB/VZoIB3mW1l592cdSRzd3 +oLWMI+pGs16zy4NfIyP9i2hxa8spWU04k+czkfLneHdbKZAWgxUD/nCEXUywws7H +bArJvEBR9FaXTRxyEg2IL+wFRiRCjLdduV2JpidTDLxyh52DgSv0V0labQKBgQDX +2OKgnTClpTI1X6JxYFj+scoQyPCMTavj9ZkFvInt/ojW7B35uCfCKiN6NNx+tqyw +XRSINW05LJM3YkbcCKVr1oXij1UqwjqNEMFRPktl3OtR4zC1tg3gSPpoh2VH3wfD +yryV8/o4vy428laTCueiNELa6N9K6yIKSdRhV3SwUQKBgQDU9U5O9SYQkBkd7P2F +ql0FJW5TIAZf4bFwDH0ARyEDvnOCk6LnIlNselRWvPYkkZQPnteYqgXw5hgqrvAa +YrbXBtsZo2I6JJZLezzRlqqKC8xrSAFdvRoAtktDKflO6F91VP8Y8PFQfQ6MgMde +CXy8gUopxn3VvE2b9WfmdzXgpQ== +-----END PRIVATE KEY----- diff --git a/libs/core/src/test/java/com/solarwinds/joboe/core/rpc/invalid-collector.crt b/libs/core/src/test/java/com/solarwinds/joboe/core/rpc/invalid-collector.crt new file mode 100644 index 00000000..4f98aec0 --- /dev/null +++ b/libs/core/src/test/java/com/solarwinds/joboe/core/rpc/invalid-collector.crt @@ -0,0 +1,32 @@ +-----BEGIN CERTIFICATE----- +MIIFgTCCA2mgAwIBAgIJAPq0Hlbgw/RhMA0GCSqGSIb3DQEBCwUAMFcxCzAJBgNV +BAYTAkFVMRMwEQYDVQQIDApTb21lLVN0YXRlMSEwHwYDVQQKDBhJbnRlcm5ldCBX +aWRnaXRzIFB0eSBMdGQxEDAOBgNVBAMMB3RoaXMuY2EwHhcNMTYxMTAxMjExNzA5 +WhcNMTcxMTAxMjExNzA5WjBXMQswCQYDVQQGEwJBVTETMBEGA1UECAwKU29tZS1T +dGF0ZTEhMB8GA1UECgwYSW50ZXJuZXQgV2lkZ2l0cyBQdHkgTHRkMRAwDgYDVQQD +DAd0aGlzLmNhMIICIjANBgkqhkiG9w0BAQEFAAOCAg8AMIICCgKCAgEAxf93He+L +mWRBFjwDdw621+LS4N/esEozlznJouO9rszO1O+KQDOmnPDCzJuEUNQM7yvs5KC/ +hKh5X/7csUOdAmyh7Z3FK6QEFds4O5ASZXw+MEadCkBp0iBijWUCnWmkQeTIFkzm +gybWPhZzIfOVqqJFUBr8PP1s3peQ9/5Krl2NV/xTyVKI+TLtenwELZezA9YP5g5c +2vkEPFqETvpFlxB59eJJV+lKVyntbXgaHqIvNo/2Yj3DvkUhLFBtgwIXFhUREY5T +zGn0Ra/blhnpd8/XTWZuRwqQvDYydYT0gI69c0JH7HUnIGFKlUBN71Ohu0Ia59Ju +XvLY4Ur3Fl1EyLvxBtOxjUShBSBmry9Nu0O7uZq9C30kne01ISnOk1Iz0LyJUeG6 +kPjQH8NHMhQplUVTriNAGOqEaLzeWVZ2ziB8Rklca4/btUjob3e4ElbpdRjwKOGY +1cwcVpvMVxCjeT9HY+tLCPWhDG47ynBpfcidO1pUYTphxA6VDyqx72AOW9WM8QVH +wqiB4eMCjIrf4qes6t4BETQzYp6D+j2mJzoqYoLB+vSGwWiNpE9I6+TP8fegtXy9 +TDzAOkh1ERMwO6b9q3lM4Z2LR5JXR0GYLPvUiqsZJM/DF2oATS/rO2dSvp9fj93s +j1lfZJzpKw863TEEH1TBLKhJy6c23A46ERsCAwEAAaNQME4wHQYDVR0OBBYEFJNw +TfJnmI23XNdjq6c96eJjhZwuMB8GA1UdIwQYMBaAFJNwTfJnmI23XNdjq6c96eJj +hZwuMAwGA1UdEwQFMAMBAf8wDQYJKoZIhvcNAQELBQADggIBAKnioRMslZ8OyOS7 +VQ88iSx8ZKIziS2qjkaqgyhaUUMPsg9HC+w+1xoAjtUMtZsg+/vuZsgj9RYrOokU +L49L31gAaSfGtUyRXZLg8yXX+xRxLthqiJHsynwIJT0sWBo7cIs1KpfGay/BtGS2 +PeeiqANRiGzF+PQ9kXrZBj1EXYV6uhEZNI5rSVxWrQ4DDEuWib2l5lTklS9HSTR4 +jXIok9KMAzsZH+i119Srw2Z+6xKnXiVTMRWDEyZXRaTjL1fNzJLEpbcbjTHISHZD +X8cgAXOjm7hcey8oLmn6gRP7msOCRvzU9F/yRdzfQ1JSI0ok7gvzFL5SaD9Hw4DM +gSY+fd1yGXU56XBznIn15BgSIk70WDTJBMXTwnZTeNSG51jCMlilvGGqyo1VYP35 +zr1KrGzivN+XE//O6uZsUf29r7jnwBZrQsgCkCHKscI3DAhF3iOSus8BheV+WBiy +YquUKOgHaDfW8JX/mxL18Fpv97flBjB5IP+I7k+ZfWtOavMTz+4aVzJqg5++fPH4 +cikbap0ZQzdIaUiKPEYps+eOh5DY37uBIkUB1306ijbC5O4CWyyMaIwDwDrd71VY +Zikm1Vig5V5O1C4z9wV0P6Pbh+lynPBFf0BFhJBfwkbsLfIjpWMIibGP/sjAqJPl +/an+7YEvNI18bJzPVv0cu672XEIa +-----END CERTIFICATE----- diff --git a/libs/core/src/test/java/com/solarwinds/joboe/core/rpc/test-collector-public.pem b/libs/core/src/test/java/com/solarwinds/joboe/core/rpc/test-collector-public.pem new file mode 100644 index 00000000..9277d53e --- /dev/null +++ b/libs/core/src/test/java/com/solarwinds/joboe/core/rpc/test-collector-public.pem @@ -0,0 +1,21 @@ +-----BEGIN CERTIFICATE----- +MIIDhzCCAm+gAwIBAgIJAKjfIgpNsQzSMA0GCSqGSIb3DQEBCwUAMFkxCzAJBgNV +BAYTAkFVMRMwEQYDVQQIDApTb21lLVN0YXRlMSEwHwYDVQQKDBhJbnRlcm5ldCBX +aWRnaXRzIFB0eSBMdGQxEjAQBgNVBAMMCWxvY2FsaG9zdDAgFw0xOTA5MTExOTMy +MDBaGA8zMDE5MDExMjE5MzIwMFowWTELMAkGA1UEBhMCQVUxEzARBgNVBAgMClNv +bWUtU3RhdGUxITAfBgNVBAoMGEludGVybmV0IFdpZGdpdHMgUHR5IEx0ZDESMBAG +A1UEAwwJbG9jYWxob3N0MIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEA +xnI0AwAxj7mBRfnfhDuNtJchKGbkRsfj0lOA9fiK9E708sb791pT3mt4HyBuarqp +Q620k9dhyME0q/nuDnRNpcekrjufM5eoveTrABDJpyUI9RumfccU9cDWDBUxppfr +dL1sJlu4ndOtQPWLsXuTe98cj6nh5M/uHz/6cZITyey3X1TwyJlIwjVLc478kkIQ +UVp2Kt6egQjtUlLXfxARFIqlCpjNKDJVpRL/wxn2E2X1KEVLFYmSkm+xpxb+KMK7 +RNFdwuTU7j6p9WCaYk9oERQtZamhFQ4Jv+scvY690Du9387o/N72nYv02LdqZKQ9 +UuqVq0vG7TXUORhKNCU4iwIDAQABo1AwTjAdBgNVHQ4EFgQUhYliC2RzD3CtOyWh +VaaDTystD+owHwYDVR0jBBgwFoAUhYliC2RzD3CtOyWhVaaDTystD+owDAYDVR0T +BAUwAwEB/zANBgkqhkiG9w0BAQsFAAOCAQEAF0jaO4CmUO6qfPafbIjf5J7q6F7+ +Y07Y+Yuqfor6Xtq1tKgy5aE/JHIcHYQg1WOd/dIlwSMAayakFIopt90KNTzgQApN +Yth/TEwEhm00Szkh09P564wPTwcKAOcTJNu+jFsDLd2iUcAlsykTRYkWe+hy2219 +sOAPntCua2MrtI6+NJj1uuUw4GlfmRy2nnLBYtS4atrvzn5OtLqlYRLgLDLY/nJa +KpyfOsn3Bdd0xjsJncvdn8CnSJ7sMJbayYVauhJb1ZqPzKZ0LUajYAJY3XL1Idpm +s/WazEeTSV9vQ/HAECq+Q6DUgt6NI1IINxs7CpS57a0pAp4xNS3SPLQTsg== +-----END CERTIFICATE----- diff --git a/libs/core/src/test/java/com/solarwinds/joboe/core/settings/PollingSettingsFetcherTest.java b/libs/core/src/test/java/com/solarwinds/joboe/core/settings/PollingSettingsFetcherTest.java new file mode 100644 index 00000000..4893e4ba --- /dev/null +++ b/libs/core/src/test/java/com/solarwinds/joboe/core/settings/PollingSettingsFetcherTest.java @@ -0,0 +1,413 @@ +/* + * © SolarWinds Worldwide, LLC. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.solarwinds.joboe.core.settings; + +import static org.junit.jupiter.api.Assertions.*; + +import com.solarwinds.joboe.core.Event; +import com.solarwinds.joboe.core.rpc.*; +import com.solarwinds.joboe.sampling.Settings; +import com.solarwinds.joboe.sampling.SettingsArg; +import com.solarwinds.joboe.sampling.SettingsFetcher; +import com.solarwinds.joboe.sampling.SettingsListener; +import com.solarwinds.joboe.sampling.TraceDecisionUtil; +import java.nio.ByteBuffer; +import java.nio.ByteOrder; +import java.util.*; +import java.util.concurrent.*; +import org.junit.jupiter.api.Test; + +public class PollingSettingsFetcherTest { + private static final int SAMPLE_RATE_FOR_DEFAULT_LAYER = 400000; + private static final List MOCK_SETTINGS = new ArrayList(); + public static final short DEFAULT_FLAGS = + RpcSettings.OBOE_SETTINGS_FLAG_TRIGGER_TRACE_ENABLED + | RpcSettings.OBOE_SETTINGS_FLAG_SAMPLE_START + | RpcSettings.OBOE_SETTINGS_FLAG_SAMPLE_THROUGH + | RpcSettings.OBOE_SETTINGS_FLAG_SAMPLE_THROUGH_ALWAYS; + + public static final String DEFAULT_FLAGS_STRING = + "TRIGGER_TRACE,SAMPLE_THROUGH_ALWAYS,SAMPLE_THROUGH,SAMPLE_START"; + + private static final Map ARGS = new HashMap(); + private static final double BUCKET_RATE = 2; + private static final double BUCKET_CAPACITY = 32; + private static final int REFRESH_INTERVAL = 1; + private static final int TTL = 5; // set TTL > READER_REFRESH_INTERVAL for expired settings test + + static { + ByteBuffer buffer; + buffer = ByteBuffer.allocate(8).order(ByteOrder.LITTLE_ENDIAN); + buffer.putDouble(BUCKET_CAPACITY); + buffer.rewind(); + ARGS.put(SettingsArg.BUCKET_CAPACITY.getKey(), buffer); + + buffer = ByteBuffer.allocate(8).order(ByteOrder.LITTLE_ENDIAN); + buffer.putDouble(BUCKET_RATE); + buffer.rewind(); + ARGS.put(SettingsArg.BUCKET_RATE.getKey(), buffer); + + MOCK_SETTINGS.add( + new RpcSettings( + DEFAULT_FLAGS_STRING, + System.currentTimeMillis(), + SAMPLE_RATE_FOR_DEFAULT_LAYER, + TTL, + ARGS)); + } + + @Test + public void testSettings() throws Exception { + Client client = new MockRpcClient(MOCK_SETTINGS); + SettingsFetcher fetcher = getFetcher(client); + + Settings settings = null; + + // Test some values what we know are stored in the above file: + assert fetcher != null; + settings = fetcher.getSettings(); + assertEquals(SAMPLE_RATE_FOR_DEFAULT_LAYER, (int) settings.getValue()); + assertEquals(DEFAULT_FLAGS, settings.getFlags()); + + // old settings should not return bucket capacity nor rate + assertEquals(BUCKET_CAPACITY, settings.getArgValue(SettingsArg.BUCKET_CAPACITY)); + assertEquals(BUCKET_RATE, settings.getArgValue(SettingsArg.BUCKET_RATE)); + + fetcher.close(); + } + + @Test + public void testSettingsCache() throws Exception { + Client client = new OneHitWonderClient(MOCK_SETTINGS); + SettingsFetcher fetcher = getFetcher(client); + + assert fetcher != null; + Settings settings = fetcher.getSettings(); // first hit should be okay + assertEquals(SAMPLE_RATE_FOR_DEFAULT_LAYER, (int) settings.getValue()); + + settings = fetcher.getSettings(); // second hit + assertEquals(SAMPLE_RATE_FOR_DEFAULT_LAYER, (int) settings.getValue()); + + fetcher.close(); + } + + @Test + public void testSettingsCacheExpired() throws Exception { + Client client = new OneHitWonderClient(MOCK_SETTINGS); + SettingsFetcher fetcher = getFetcher(client); + + assert fetcher != null; + Settings settings = fetcher.getSettings(); // cached Settings not yet expired + assertEquals(SAMPLE_RATE_FOR_DEFAULT_LAYER, (int) settings.getValue()); + + int wait = TTL - 1; // wait shorter than TTL so settings should not yet expire on dead client + + TimeUnit.SECONDS.sleep(wait); + + settings = fetcher.getSettings(); // cached Settings not yet expired + assertEquals(SAMPLE_RATE_FOR_DEFAULT_LAYER, (int) settings.getValue()); + + TimeUnit.SECONDS.sleep( + 3); // sleep for 3 more seconds, now it's TTL + 2 (and 2 is bigger than REFRESH_INTERVAL), + // hence the Settings should be expired + System.out.println("waking up"); + settings = fetcher.getSettings(); // second hit after record expired, should return null + assertNull(settings); + + fetcher.close(); + } + + @Test + public void testInvalidArgs() throws Exception { + SettingsFetcher fetcher; + Settings settings; + Settings sourceSettings; + + // test remote settings that give empty map for args + sourceSettings = + new RpcSettings( + DEFAULT_FLAGS_STRING, System.currentTimeMillis(), 100000, TTL, Collections.emptyMap()); + Client client = new MockRpcClient(Collections.singletonList(sourceSettings)); + fetcher = getFetcher(client); + settings = fetcher.getSettings(); + assertEquals(100000, (int) settings.getValue()); + assertEquals(DEFAULT_FLAGS, settings.getFlags()); + assertNull(settings.getArgValue(SettingsArg.BUCKET_CAPACITY)); + assertNull(settings.getArgValue(SettingsArg.BUCKET_RATE)); + + // test remote settings that give empty values for args that triggers BufferUnderflow + Map args = new HashMap(); + args.put(SettingsArg.BUCKET_CAPACITY.getKey(), ByteBuffer.allocate(0)); + args.put(SettingsArg.BUCKET_RATE.getKey(), ByteBuffer.allocate(0)); + args.put(SettingsArg.METRIC_FLUSH_INTERVAL.getKey(), ByteBuffer.allocate(0)); + sourceSettings = + new RpcSettings(DEFAULT_FLAGS_STRING, System.currentTimeMillis(), 100000, TTL, args); + fetcher = getFetcher(new MockRpcClient(Collections.singletonList(sourceSettings))); + settings = fetcher.getSettings(); + assertEquals(100000, (int) settings.getValue()); + assertEquals(DEFAULT_FLAGS, settings.getFlags()); + assertNull(settings.getArgValue(SettingsArg.BUCKET_CAPACITY)); + assertNull(settings.getArgValue(SettingsArg.BUCKET_RATE)); + assertNull(settings.getArgValue(SettingsArg.METRIC_FLUSH_INTERVAL)); + + // test remote settings that give valid values for args + args = new HashMap(); + ByteBuffer buffer; + buffer = ByteBuffer.allocate(8).order(ByteOrder.LITTLE_ENDIAN); + buffer.putDouble(1); + buffer.rewind(); + args.put(SettingsArg.BUCKET_CAPACITY.getKey(), buffer); + buffer = ByteBuffer.allocate(8).order(ByteOrder.LITTLE_ENDIAN); + buffer.putDouble(2); + buffer.rewind(); + args.put(SettingsArg.BUCKET_RATE.getKey(), buffer); + buffer = ByteBuffer.allocate(4).order(ByteOrder.LITTLE_ENDIAN); + buffer.putInt(3); + buffer.rewind(); + args.put(SettingsArg.METRIC_FLUSH_INTERVAL.getKey(), buffer); + sourceSettings = + new RpcSettings(DEFAULT_FLAGS_STRING, System.currentTimeMillis(), 100000, TTL, args); + fetcher = getFetcher(new MockRpcClient(Collections.singletonList(sourceSettings))); + settings = fetcher.getSettings(); + assertEquals(100000, (int) settings.getValue()); + assertEquals(DEFAULT_FLAGS, settings.getFlags()); + assertEquals(1.0, settings.getArgValue(SettingsArg.BUCKET_CAPACITY)); + assertEquals(2.0, settings.getArgValue(SettingsArg.BUCKET_RATE)); + assertEquals((Integer) 3, settings.getArgValue(SettingsArg.METRIC_FLUSH_INTERVAL)); + + fetcher.close(); + } + + @Test + public void testInvalidSampleRate() throws Exception { + SettingsFetcher fetcher; + Settings settings; + Settings sourceSettings; + + // test remote settings that gives sample rate that is greater than 1000000 + sourceSettings = + new RpcSettings(DEFAULT_FLAGS_STRING, System.currentTimeMillis(), 1111111, TTL, ARGS); + Client client = new MockRpcClient(Collections.singletonList(sourceSettings)); + fetcher = getFetcher(client); + assert fetcher != null; + settings = fetcher.getSettings(); + assertEquals( + TraceDecisionUtil.SAMPLE_RESOLUTION, + (int) settings.getValue()); // should be adjusted to 1000000 + assertEquals(DEFAULT_FLAGS, settings.getFlags()); + assertEquals(BUCKET_CAPACITY, settings.getArgValue(SettingsArg.BUCKET_CAPACITY)); + assertEquals(BUCKET_RATE, settings.getArgValue(SettingsArg.BUCKET_RATE)); + + // test remote settings that gives sample rate that is negative + sourceSettings = + new RpcSettings(DEFAULT_FLAGS_STRING, System.currentTimeMillis(), -1, TTL, ARGS); + fetcher = getFetcher(new MockRpcClient(Collections.singletonList(sourceSettings))); + settings = fetcher.getSettings(); + assertEquals(0, (int) settings.getValue()); // should be adjusted to 0 + assertEquals(DEFAULT_FLAGS, settings.getFlags()); + assertEquals(BUCKET_CAPACITY, settings.getArgValue(SettingsArg.BUCKET_CAPACITY)); + assertEquals(BUCKET_RATE, settings.getArgValue(SettingsArg.BUCKET_RATE)); + + fetcher.close(); + } + + @Test + public void testExecutionException() throws Exception { + Client client = new ExecutionExceptionClient(); + SettingsFetcher fetcher = getFetcher(client); + + assert fetcher != null; + Settings settings = fetcher.getSettings(); + assertNull(settings); + + fetcher.close(); + } + + @Test + public void testSettingsListener() throws InterruptedException { + RpcSettings settings; + settings = new RpcSettings(DEFAULT_FLAGS_STRING, System.currentTimeMillis(), 1, TTL, ARGS); + Client client = new MockRpcClient(Collections.singletonList(settings)); + + SettingsFetcher fetcher = + new PollingSettingsFetcher(new RpcSettingsReader(client), 1); // refresh every second + + TestSettingsListener testSettingsListener = new TestSettingsListener(); + fetcher.registerListener(testSettingsListener); + + TimeUnit.SECONDS.sleep(2); // should be enough time for an update + + assertNotNull(testSettingsListener.settingsRetrieved); + + fetcher.close(); + } + + private static class TestSettingsListener implements SettingsListener { + private Settings settingsRetrieved; + + @Override + public void onSettingsRetrieved(Settings newSettings) { + settingsRetrieved = newSettings; + } + } + + private SettingsFetcher getFetcher(Client rpcClient) { + RpcSettingsReader reader = new RpcSettingsReader(rpcClient); + try { + SettingsFetcher fetcher = new PollingSettingsFetcher(reader, REFRESH_INTERVAL); + fetcher.isSettingsAvailableLatch().await(10, TimeUnit.SECONDS); + return fetcher; + } catch (InterruptedException e) { + e.printStackTrace(); + return null; + } + } + + private abstract static class TestClient implements Client { + private final List settings; + + protected TestClient(List settings) { + this.settings = settings; + } + + @Override + public final Future postEvents(List events, Callback callback) + throws ClientException { + return null; // not used + } + + @Override + public final Future postMetrics( + List> messages, Callback callback) throws ClientException { + return null; // not used + } + + @Override + public final Future postStatus( + List> messages, Callback callback) throws ClientException { + return null; // not used + } + + /** + * Clone the settings as the buffer within the original settings cannot be shared + * + * @return + */ + protected List getClonedSettings() { + List clonedSettings = new ArrayList(); + for (Settings settingsEntry : settings) { + clonedSettings.add( + new RpcSettings( + (RpcSettings) settingsEntry, + System.currentTimeMillis())); // create clones with the current timestamp + } + + return clonedSettings; + } + } + + private static class MockRpcClient extends TestClient { + private final ExecutorService service = Executors.newSingleThreadExecutor(); + + private MockRpcClient(List settings) { + super(settings); + } + + @Override + public Future getSettings(String version, Callback callback) + throws ClientException { + return service.submit( + () -> { + // need to create deep clone for each buffer args + return new SettingsResult(ResultCode.OK, "", "", getClonedSettings()); + }); + } + + @Override + public void close() { + service.shutdown(); + } + + @Override + public Status getStatus() { + return Status.OK; + } + } + + private static class OneHitWonderClient extends TestClient { + private boolean hasHit = false; + private final ExecutorService service = Executors.newSingleThreadExecutor(); + + protected OneHitWonderClient(List settings) { + super(settings); + } + + @Override + public Future getSettings(String version, Callback callback) { + if (!hasHit) { + hasHit = true; // hit once! next one should return exception + return service.submit(() -> new SettingsResult(ResultCode.OK, "", "", getClonedSettings())); + } else { + return service.submit( + () -> { + throw new RuntimeException("Testing exception"); + }); + } + } + + @Override + public void close() { + service.shutdown(); + } + + @Override + public Status getStatus() { + return Status.OK; + } + } + + private static class ExecutionExceptionClient extends TestClient { + private final ExecutorService service = Executors.newSingleThreadExecutor(); + + protected ExecutionExceptionClient() { + super(null); + } + + @Override + public Future getSettings(String version, Callback callback) { + return getExecutionException(); + } + + private Future + getExecutionException() { // could have used CompletableFuture but it's jdk 8... + return service.submit( + () -> { + throw new Exception("test"); + }); + } + + @Override + public void close() { + service.shutdown(); + } + + @Override + public Status getStatus() { + return Status.OK; + } + } +} diff --git a/libs/core/src/test/java/com/solarwinds/joboe/core/settings/SettingsUtilTest.java b/libs/core/src/test/java/com/solarwinds/joboe/core/settings/SettingsUtilTest.java new file mode 100644 index 00000000..b83c50b4 --- /dev/null +++ b/libs/core/src/test/java/com/solarwinds/joboe/core/settings/SettingsUtilTest.java @@ -0,0 +1,94 @@ +/* + * © SolarWinds Worldwide, LLC. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.solarwinds.joboe.core.settings; + +import static org.junit.jupiter.api.Assertions.*; +import static org.mockito.Mockito.when; + +import com.google.protobuf.InvalidProtocolBufferException; +import com.solarwinds.joboe.core.rpc.ResultCode; +import com.solarwinds.joboe.core.rpc.RpcSettings; +import com.solarwinds.joboe.core.rpc.SettingsResult; +import com.solarwinds.joboe.sampling.Settings; +import com.solarwinds.trace.ingestion.proto.Collector; +import java.io.File; +import java.io.IOException; +import java.nio.file.Files; +import java.nio.file.Paths; +import java.util.Collections; +import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.ExtendWith; +import org.mockito.Mock; +import org.mockito.junit.jupiter.MockitoExtension; + +@ExtendWith(MockitoExtension.class) +class SettingsUtilTest { + + private static byte[] settingsBlob; + + @Mock private RpcSettings settingsMock; + + @BeforeAll + static void setup() throws IOException { + settingsBlob = + Files.readAllBytes( + Paths.get(new File("src/test/resources/solarwinds-apm-settings-raw").getPath())); + } + + @Test + void testTransformToKVSetting() { + when(settingsMock.getTtl()).thenReturn(60L); + SettingsResult settingsResult = + new SettingsResult(ResultCode.OK, "arg", "we up", Collections.singletonList(settingsMock)); + + boolean anyMatch = + settingsResult.getSettings().stream().anyMatch(settings -> settings.getTtl() == 60); + assertTrue(anyMatch); + } + + @Test + void testTransformToLocalSettings() throws InvalidProtocolBufferException { + SettingsResult settingsResult = + SettingsUtil.transformToLocalSettings(Collector.SettingsResult.parseFrom(settingsBlob)); + assertEquals(ResultCode.OK, settingsResult.getResultCode()); + } + + @Test + void testConvertSetting() throws InvalidProtocolBufferException { + Collector.SettingsResult settingsResult = Collector.SettingsResult.parseFrom(settingsBlob); + Settings settings = SettingsUtil.convertSetting(settingsResult.getSettings(0)); + assertEquals(120, settings.getTtl()); + } + + @Test + void testConvertType() { + assertEquals(-1, SettingsUtil.convertType(Collector.OboeSettingType.UNRECOGNIZED)); + assertEquals( + Settings.OBOE_SETTINGS_TYPE_DEFAULT_SAMPLE_RATE, + SettingsUtil.convertType(Collector.OboeSettingType.DEFAULT_SAMPLE_RATE)); + assertEquals( + Settings.OBOE_SETTINGS_TYPE_LAYER_SAMPLE_RATE, + SettingsUtil.convertType(Collector.OboeSettingType.LAYER_SAMPLE_RATE)); + assertEquals( + Settings.OBOE_SETTINGS_TYPE_LAYER_APP_SAMPLE_RATE, + SettingsUtil.convertType(Collector.OboeSettingType.LAYER_APP_SAMPLE_RATE)); + assertEquals( + Settings.OBOE_SETTINGS_TYPE_LAYER_HTTPHOST_SAMPLE_RATE, + SettingsUtil.convertType(Collector.OboeSettingType.LAYER_HTTPHOST_SAMPLE_RATE)); + } +} diff --git a/libs/core/src/test/java/com/solarwinds/joboe/core/util/DockerInfoReaderTest.java b/libs/core/src/test/java/com/solarwinds/joboe/core/util/DockerInfoReaderTest.java new file mode 100644 index 00000000..77dc5698 --- /dev/null +++ b/libs/core/src/test/java/com/solarwinds/joboe/core/util/DockerInfoReaderTest.java @@ -0,0 +1,83 @@ +/* + * © SolarWinds Worldwide, LLC. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.solarwinds.joboe.core.util; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertNull; + +import com.solarwinds.joboe.core.util.ServerHostInfoReader.DockerInfoReader; +import java.io.IOException; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; + +public class DockerInfoReaderTest { + private static final String TEST_FILE_PREFIX = + "src/test/java/com/solarwinds/joboe/core/util/docker-cgroup-"; // using a rather static path. + + // Using + // Class.getResourceAsStream + // does not work in test (vs + // main) + + @BeforeEach + protected void tearDown() throws Exception { + DockerInfoReader.getInstance() + .initializeLinux(DockerInfoReader.DEFAULT_LINUX_DOCKER_FILE_LOCATION); // reset to default + } + + @Test + public void testReadDockerContainerId() throws IOException { + DockerInfoReader.getInstance().initializeLinux(TEST_FILE_PREFIX + "standard"); + assertEquals( + "0531ff3c6395131175507ac7e94fdf387f2a2dea81961e6c96f6ac5ccd7ede3f", + DockerInfoReader.getDockerId()); + + DockerInfoReader.getInstance().initializeLinux(TEST_FILE_PREFIX + "standard-2"); + assertEquals( + "0531ff3c6395131175507ac7e94fdf387f2a2dea81961e6c96f6ac5ccd7ede3f", + DockerInfoReader.getDockerId()); + + DockerInfoReader.getInstance().initializeLinux(TEST_FILE_PREFIX + "ce"); + assertEquals( + "93d377d55070d2463493706ba7194d119c3efb1c2e7929f36da183ffe71d72a8", + DockerInfoReader.getDockerId()); + + DockerInfoReader.getInstance().initializeLinux(TEST_FILE_PREFIX + "ecs"); + assertEquals( + "93d377d55070d2463493706ba7194d119c3efb1c2e7929f36da183ffe71d72a8", + DockerInfoReader.getDockerId()); + + DockerInfoReader.getInstance().initializeLinux(TEST_FILE_PREFIX + "cri-containerd"); + assertEquals( + "0531ff3c6395131175507ac7e94fdf387f2a2dea81961e6c96f6ac5ccd7ede3f", + DockerInfoReader.getDockerId()); + + DockerInfoReader.getInstance().initializeLinux(TEST_FILE_PREFIX + "kubepods"); + assertEquals( + "0531ff3c6395131175507ac7e94fdf387f2a2dea81961e6c96f6ac5ccd7ede3f", + DockerInfoReader.getDockerId()); + + DockerInfoReader.getInstance().initializeLinux(TEST_FILE_PREFIX + "empty"); + assertNull(DockerInfoReader.getDockerId()); + + DockerInfoReader.getInstance().initializeLinux(TEST_FILE_PREFIX + "non-docker"); + assertNull(DockerInfoReader.getDockerId()); + + DockerInfoReader.getInstance().initializeLinux(TEST_FILE_PREFIX + "invalid"); + assertNull(DockerInfoReader.getDockerId()); + } +} diff --git a/libs/core/src/test/java/com/solarwinds/joboe/core/util/HeartbeatSchedulerProviderTest.java b/libs/core/src/test/java/com/solarwinds/joboe/core/util/HeartbeatSchedulerProviderTest.java new file mode 100644 index 00000000..c7fec0cc --- /dev/null +++ b/libs/core/src/test/java/com/solarwinds/joboe/core/util/HeartbeatSchedulerProviderTest.java @@ -0,0 +1,40 @@ +/* + * © SolarWinds Worldwide, LLC. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.solarwinds.joboe.core.util; + +import static org.junit.jupiter.api.Assertions.assertTrue; + +import com.solarwinds.joboe.core.rpc.KeepAliveMonitor; +import com.solarwinds.joboe.core.rpc.ProtocolClient; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.ExtendWith; +import org.mockito.Mock; +import org.mockito.junit.jupiter.MockitoExtension; + +@ExtendWith(MockitoExtension.class) +class HeartbeatSchedulerProviderTest { + + @Mock private ProtocolClient protocolClientStub; + + @Test + void testThatKeepAliveMonitorIsCreatedWhenNotLambda() { + assertTrue( + HeartbeatSchedulerProvider.createHeartbeatScheduler( + () -> protocolClientStub, "some key", "locker") + instanceof KeepAliveMonitor); + } +} diff --git a/libs/core/src/test/java/com/solarwinds/joboe/core/util/JavaVersionComparatorTest.java b/libs/core/src/test/java/com/solarwinds/joboe/core/util/JavaVersionComparatorTest.java new file mode 100644 index 00000000..0933d2e5 --- /dev/null +++ b/libs/core/src/test/java/com/solarwinds/joboe/core/util/JavaVersionComparatorTest.java @@ -0,0 +1,37 @@ +/* + * © SolarWinds Worldwide, LLC. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.solarwinds.joboe.core.util; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertTrue; + +import com.solarwinds.joboe.config.JavaVersionComparator; +import org.junit.jupiter.api.Test; + +public class JavaVersionComparatorTest { + @Test + public void testVersionCompare() { + assertTrue(JavaVersionComparator.compare("1.8.0", "17.0.1") < 0); + assertTrue(JavaVersionComparator.compare("1.8.0", "1.8.0_252") < 0); + assertTrue(JavaVersionComparator.compare("1.8.0_252", "1.8.0_332") < 0); + assertTrue(JavaVersionComparator.compare("1.8.0", "1.8.1") < 0); + assertTrue(JavaVersionComparator.compare("1.8.0", "1.9.1") < 0); + assertTrue(JavaVersionComparator.compare("1.8.2", "1.10.1") < 0); + assertEquals(0, JavaVersionComparator.compare("1.8.0", "1.8.0")); + assertTrue(JavaVersionComparator.compare("16.8.2", "17.0.1") < 0); + } +} diff --git a/libs/core/src/test/java/com/solarwinds/joboe/core/util/K8sReaderTest.java b/libs/core/src/test/java/com/solarwinds/joboe/core/util/K8sReaderTest.java new file mode 100644 index 00000000..594c0975 --- /dev/null +++ b/libs/core/src/test/java/com/solarwinds/joboe/core/util/K8sReaderTest.java @@ -0,0 +1,51 @@ +/* + * © SolarWinds Worldwide, LLC. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.solarwinds.joboe.core.util; + +import static org.junit.jupiter.api.Assertions.assertEquals; + +import com.solarwinds.joboe.core.HostId; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; + +public class K8sReaderTest { + private ServerHostInfoReader.K8sReader tested; + + @BeforeEach + protected void setUp() throws Exception { + ServerHostInfoReader.K8sReader.NAMESPACE_FILE_LOC_LINUX = + "src/test/java/com/solarwinds/joboe/core/util/namespace"; + + ServerHostInfoReader.K8sReader.POD_UUID_FILE_LOC = + "src/test/java/com/solarwinds/joboe/core/util/poduid"; + ServerHostInfoReader.osType = HostInfoUtils.OsType.LINUX; + tested = new ServerHostInfoReader.K8sReader(); + } + + @Test + public void testReadK8sMetadata() { + HostId.K8sMetadata actual = tested.getK8sMetadata(); + HostId.K8sMetadata expected = + HostId.K8sMetadata.builder() + .namespace("o11y-platform") + .podUid("9dcdb600-4156-4b7b-afcc-f8c06cb0e474") + .podName(ServerHostInfoReader.INSTANCE.getHostName()) + .build(); + + assertEquals(expected, actual); + } +} diff --git a/libs/core/src/test/java/com/solarwinds/joboe/core/util/RuntimeHostInfoReaderProviderTest.java b/libs/core/src/test/java/com/solarwinds/joboe/core/util/RuntimeHostInfoReaderProviderTest.java new file mode 100644 index 00000000..470d7e48 --- /dev/null +++ b/libs/core/src/test/java/com/solarwinds/joboe/core/util/RuntimeHostInfoReaderProviderTest.java @@ -0,0 +1,30 @@ +/* + * © SolarWinds Worldwide, LLC. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.solarwinds.joboe.core.util; + +import static org.junit.jupiter.api.Assertions.assertTrue; + +import org.junit.jupiter.api.Test; + +class RuntimeHostInfoReaderProviderTest { + private final RuntimeHostInfoReaderProvider tested = new RuntimeHostInfoReaderProvider(); + + @Test + void returnServerHostInfoReader() { + assertTrue(tested.getHostInfoReader() instanceof ServerHostInfoReader); + } +} diff --git a/libs/core/src/test/java/com/solarwinds/joboe/core/util/ServerHostInfoReaderTest.java b/libs/core/src/test/java/com/solarwinds/joboe/core/util/ServerHostInfoReaderTest.java new file mode 100644 index 00000000..0efc540d --- /dev/null +++ b/libs/core/src/test/java/com/solarwinds/joboe/core/util/ServerHostInfoReaderTest.java @@ -0,0 +1,107 @@ +/* + * © SolarWinds Worldwide, LLC. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.solarwinds.joboe.core.util; + +import static org.junit.jupiter.api.Assertions.assertEquals; + +import com.solarwinds.joboe.core.HostId; +import com.solarwinds.joboe.core.util.HostInfoUtils.NetworkAddressInfo; +import java.io.*; +import java.util.Map; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; + +public class ServerHostInfoReaderTest { + private static final String TEST_FILE_FOLDER = + "src/test/java/com/solarwinds/joboe/core/util/"; // using a rather static path. Using + // Class.getResourceAsStream does not work in + // test (vs main) + private final ServerHostInfoReader reader = ServerHostInfoReader.INSTANCE; + + @BeforeEach + void setup() { + ServerHostInfoReader.DockerInfoReader.getInstance() + .initializeLinux( + ServerHostInfoReader.DockerInfoReader + .DEFAULT_LINUX_DOCKER_FILE_LOCATION); // reset to default + } + + @Test + public void testGetDistroParsing() throws IOException { + assertEquals( + "Red Hat Enterprise Linux Server release 6.5 (Santiago)", + ServerHostInfoReader.getRedHatBasedDistro( + getFileReader(ServerHostInfoReader.DistroType.REDHAT_BASED))); + assertEquals( + "Amzn Linux 2015.09", + ServerHostInfoReader.getAmazonLinuxDistro( + getFileReader(ServerHostInfoReader.DistroType.AMAZON))); + assertEquals( + "Ubuntu 10.04.2 LTS", + ServerHostInfoReader.getUbuntuDistro( + getFileReader(ServerHostInfoReader.DistroType.UBUNTU))); + assertEquals( + "Debian 7.7", + ServerHostInfoReader.getDebianDistro( + getFileReader(ServerHostInfoReader.DistroType.DEBIAN))); + assertEquals( + "SUSE Linux Enterprise Server 10 (x86_64)", + ServerHostInfoReader.getNovellSuseDistro( + getFileReader(ServerHostInfoReader.DistroType.SUSE))); + assertEquals( + "Slackware-x86_64 13.0", + ServerHostInfoReader.getSlackwareDistro( + getFileReader(ServerHostInfoReader.DistroType.SLACKWARE))); + assertEquals( + "Gentoo Base System release 2.1", + ServerHostInfoReader.getGentooDistro( + getFileReader(ServerHostInfoReader.DistroType.GENTOO))); + } + + private BufferedReader getFileReader(ServerHostInfoReader.DistroType distroType) + throws FileNotFoundException { + String path = ServerHostInfoReader.DISTRO_FILE_NAMES.get(distroType); + String fileName = new File(path).getName(); + + return new BufferedReader(new FileReader(TEST_FILE_FOLDER + fileName)); + } + + @Test + public void testGetHostMetadata() { + Map hostMetadata = reader.getHostMetadata(); + + assert (hostMetadata.containsKey("UnameSysName")); + assert (hostMetadata.containsKey("UnameVersion")); + HostInfoUtils.OsType osType = HostInfoUtils.getOsType(); + assert osType != HostInfoUtils.OsType.LINUX && osType != HostInfoUtils.OsType.WINDOWS + || (hostMetadata.containsKey("Distro")); + NetworkAddressInfo networkInfo = reader.getNetworkAddressInfo(); + if (networkInfo != null) { + assert networkInfo.getIpAddresses().isEmpty() || (hostMetadata.containsKey("IPAddresses")); + } + } + + @Test + public void testGetHostId() { + HostId hostId = reader.getHostId(); + assertEquals(reader.getHostName(), hostId.getHostname()); + assertEquals(reader.getAwsInstanceId(), hostId.getEc2InstanceId()); + assertEquals(reader.getAwsAvailabilityZone(), hostId.getEc2AvailabilityZone()); + assertEquals(reader.getHerokuDynoId(), hostId.getHerokuDynoId()); + assertEquals(reader.getNetworkAddressInfo().getMacAddresses(), hostId.getMacAddresses()); + } +} diff --git a/libs/core/src/test/java/com/solarwinds/joboe/core/util/SuSE-release b/libs/core/src/test/java/com/solarwinds/joboe/core/util/SuSE-release new file mode 100644 index 00000000..c7841938 --- /dev/null +++ b/libs/core/src/test/java/com/solarwinds/joboe/core/util/SuSE-release @@ -0,0 +1,3 @@ +SUSE Linux Enterprise Server 10 (x86_64) +VERSION = 10 +PATCHLEVEL = 4 \ No newline at end of file diff --git a/libs/core/src/test/java/com/solarwinds/joboe/core/util/TimeUtilsTest.java b/libs/core/src/test/java/com/solarwinds/joboe/core/util/TimeUtilsTest.java new file mode 100644 index 00000000..790fb21b --- /dev/null +++ b/libs/core/src/test/java/com/solarwinds/joboe/core/util/TimeUtilsTest.java @@ -0,0 +1,94 @@ +/* + * © SolarWinds Worldwide, LLC. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.solarwinds.joboe.core.util; + +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertTrue; + +import java.lang.reflect.Method; +import java.util.ArrayList; +import java.util.List; +import org.junit.jupiter.api.Test; + +public class TimeUtilsTest { + private static final int RUN_COUNT = 10000; + private static final int WARM_UP_COUNT = 1000; + + @Test + public void testTimeBase() { + List diffsInMicroSec = new ArrayList(); + + for (int i = 0; i < RUN_COUNT; i++) { + long timeUtilsMirco = TimeUtils.getTimestampMicroSeconds(); + long systemMilli = System.currentTimeMillis(); + + if (i > WARM_UP_COUNT) { + diffsInMicroSec.add(Math.abs(systemMilli * 1000 - timeUtilsMirco)); + } + } + + TimeUtils.Statistics statistics = new TimeUtils.Statistics(diffsInMicroSec); + + long median = statistics.getMedian().longValue(); + assertTrue( + median < 100000, + "Found median " + + median); // a rather generous median, this test is just to make sure value is not + // totally incorrect as unstable system (especially windows) can sometimes + // give bad shift + } + + @Test + public void testTimeDifference() throws InterruptedException { + List durationDiscrepanciesInMicroSec; + TimeUtils.Statistics statistics; + + durationDiscrepanciesInMicroSec = new ArrayList(); + for (int i = 0; i < RUN_COUNT; i++) { + long startMicro = TimeUtils.getTimestampMicroSeconds(); + Thread.sleep(1); + long endMicro = TimeUtils.getTimestampMicroSeconds(); + + assertTrue( + endMicro > startMicro, + "end micro is smaller than startMicro: " + + endMicro + + " vs " + + startMicro); // this is the most important! Time should not go backwards + + if (i > WARM_UP_COUNT) { + durationDiscrepanciesInMicroSec.add(Math.abs((endMicro - startMicro) - 1000)); + } + } + + statistics = new TimeUtils.Statistics(durationDiscrepanciesInMicroSec); + long differenceMedian = statistics.getMedian().longValue(); + assertTrue( + differenceMedian < 1000, + "Found median " + differenceMedian); // discrepancy of p95 should be less than 1 millisec + } + + @Test + public void testTimeAdjustWorkerInit() throws Exception { + Method startAdjustBaseWorkerMethod = + TimeUtils.class.getDeclaredMethod("startAdjustBaseWorker", Integer.class); + startAdjustBaseWorkerMethod.setAccessible(true); + + assertFalse((Boolean) startAdjustBaseWorkerMethod.invoke(null, 0)); + assertTrue((Boolean) startAdjustBaseWorkerMethod.invoke(null, 100)); + } +} diff --git a/libs/core/src/test/java/com/solarwinds/joboe/core/util/debian_version b/libs/core/src/test/java/com/solarwinds/joboe/core/util/debian_version new file mode 100644 index 00000000..120096f1 --- /dev/null +++ b/libs/core/src/test/java/com/solarwinds/joboe/core/util/debian_version @@ -0,0 +1 @@ +7.7 \ No newline at end of file diff --git a/libs/core/src/test/java/com/solarwinds/joboe/core/util/docker-cgroup-ce b/libs/core/src/test/java/com/solarwinds/joboe/core/util/docker-cgroup-ce new file mode 100644 index 00000000..0a5cdaf5 --- /dev/null +++ b/libs/core/src/test/java/com/solarwinds/joboe/core/util/docker-cgroup-ce @@ -0,0 +1,13 @@ +13:name=systemd:/docker-ce/docker/93d377d55070d2463493706ba7194d119c3efb1c2e7929f36da183ffe71d72a8 +12:pids:/docker-ce/docker/93d377d55070d2463493706ba7194d119c3efb1c2e7929f36da183ffe71d72a8 +11:hugetlb:/docker-ce/docker/93d377d55070d2463493706ba7194d119c3efb1c2e7929f36da183ffe71d72a8 +10:net_prio:/docker-ce/docker/93d377d55070d2463493706ba7194d119c3efb1c2e7929f36da183ffe71d72a8 +9:perf_event:/docker-ce/docker/93d377d55070d2463493706ba7194d119c3efb1c2e7929f36da183ffe71d72a8 +8:net_cls:/docker-ce/docker/93d377d55070d2463493706ba7194d119c3efb1c2e7929f36da183ffe71d72a8 +7:freezer:/docker-ce/docker/93d377d55070d2463493706ba7194d119c3efb1c2e7929f36da183ffe71d72a8 +6:devices:/docker-ce/docker/93d377d55070d2463493706ba7194d119c3efb1c2e7929f36da183ffe71d72a8 +5:memory:/docker-ce/docker/93d377d55070d2463493706ba7194d119c3efb1c2e7929f36da183ffe71d72a8 +4:blkio:/docker-ce/docker/93d377d55070d2463493706ba7194d119c3efb1c2e7929f36da183ffe71d72a8 +3:cpuacct:/docker-ce/docker/93d377d55070d2463493706ba7194d119c3efb1c2e7929f36da183ffe71d72a8 +2:cpu:/docker-ce/docker/93d377d55070d2463493706ba7194d119c3efb1c2e7929f36da183ffe71d72a8 +1:cpuset:/docker-ce/docker/93d377d55070d2463493706ba7194d119c3efb1c2e7929f36da183ffe71d72a8 \ No newline at end of file diff --git a/libs/core/src/test/java/com/solarwinds/joboe/core/util/docker-cgroup-cri-containerd b/libs/core/src/test/java/com/solarwinds/joboe/core/util/docker-cgroup-cri-containerd new file mode 100644 index 00000000..f20f3830 --- /dev/null +++ b/libs/core/src/test/java/com/solarwinds/joboe/core/util/docker-cgroup-cri-containerd @@ -0,0 +1,11 @@ +11:devices:/kubepods.slice/kubepods-besteffort.slice/kubepods-besteffort-pod349404940_49404049404.slice/cri-containerd-0531ff3c6395131175507ac7e94fdf387f2a2dea81961e6c96f6ac5ccd7ede3f.scope +10:hugetlb:/kubepods.slice/kubepods-besteffort.slice/kubepods-besteffort-pod349404940_49404049404.slice/cri-containerd-0531ff3c6395131175507ac7e94fdf387f2a2dea81961e6c96f6ac5ccd7ede3f.scope +9:cpu,cpu_acct:/kubepods.slice/kubepods-besteffort.slice/kubepods-besteffort-pod349404940_49404049404.slice/cri-containerd-0531ff3c6395131175507ac7e94fdf387f2a2dea81961e6c96f6ac5ccd7ede3f.scope +8:memory:/kubepods.slice/kubepods-besteffort.slice/kubepods-besteffort-pod349404940_49404049404.slice/cri-containerd-0531ff3c6395131175507ac7e94fdf387f2a2dea81961e6c96f6ac5ccd7ede3f.scope +7:pids:/kubepods.slice/kubepods-besteffort.slice/kubepods-besteffort-pod349404940_49404049404.slice/cri-containerd-0531ff3c6395131175507ac7e94fdf387f2a2dea81961e6c96f6ac5ccd7ede3f.scope +6:perf_event:/kubepods.slice/kubepods-besteffort.slice/kubepods-besteffort-pod349404940_49404049404.slice/cri-containerd-0531ff3c6395131175507ac7e94fdf387f2a2dea81961e6c96f6ac5ccd7ede3f.scope +5:freezer:/kubepods.slice/kubepods-besteffort.slice/kubepods-besteffort-pod349404940_49404049404.slice/cri-containerd-0531ff3c6395131175507ac7e94fdf387f2a2dea81961e6c96f6ac5ccd7ede3f.scope +4:net_cls,net_prio:/kubepods.slice/kubepods-besteffort.slice/kubepods-besteffort-pod349404940_49404049404.slice/cri-containerd-0531ff3c6395131175507ac7e94fdf387f2a2dea81961e6c96f6ac5ccd7ede3f.scope +3:blkio:/kubepods.slice/kubepods-besteffort.slice/kubepods-besteffort-pod349404940_49404049404.slice/cri-containerd-0531ff3c6395131175507ac7e94fdf387f2a2dea81961e6c96f6ac5ccd7ede3f.scope +2:cpuset:/kubepods.slice/kubepods-besteffort.slice/kubepods-besteffort-pod349404940_49404049404.slice/cri-containerd-0531ff3c6395131175507ac7e94fdf387f2a2dea81961e6c96f6ac5ccd7ede3f.scope +1:name=systemd:/kubepods.slice/kubepods-besteffort.slice/kubepods-besteffort-pod349404940_49404049404.slice/cri-containerd-0531ff3c6395131175507ac7e94fdf387f2a2dea81961e6c96f6ac5ccd7ede3f.scope \ No newline at end of file diff --git a/libs/core/src/test/java/com/solarwinds/joboe/core/util/docker-cgroup-ecs b/libs/core/src/test/java/com/solarwinds/joboe/core/util/docker-cgroup-ecs new file mode 100644 index 00000000..ed78ea3e --- /dev/null +++ b/libs/core/src/test/java/com/solarwinds/joboe/core/util/docker-cgroup-ecs @@ -0,0 +1,13 @@ +13:name=systemd:/ecs/task-arn/93d377d55070d2463493706ba7194d119c3efb1c2e7929f36da183ffe71d72a8 +12:pids:/ecs/task-arn/93d377d55070d2463493706ba7194d119c3efb1c2e7929f36da183ffe71d72a8 +11:hugetlb:/ecs/task-arn/93d377d55070d2463493706ba7194d119c3efb1c2e7929f36da183ffe71d72a8 +10:net_prio:/ecs/task-arn/93d377d55070d2463493706ba7194d119c3efb1c2e7929f36da183ffe71d72a8 +9:perf_event:/ecs/task-arn/93d377d55070d2463493706ba7194d119c3efb1c2e7929f36da183ffe71d72a8 +8:net_cls:/ecs/task-arn/93d377d55070d2463493706ba7194d119c3efb1c2e7929f36da183ffe71d72a8 +7:freezer:/ecs/task-arn/93d377d55070d2463493706ba7194d119c3efb1c2e7929f36da183ffe71d72a8 +6:devices:/ecs/task-arn/93d377d55070d2463493706ba7194d119c3efb1c2e7929f36da183ffe71d72a8 +5:memory:/ecs/task-arn/93d377d55070d2463493706ba7194d119c3efb1c2e7929f36da183ffe71d72a8 +4:blkio:/ecs/task-arn/93d377d55070d2463493706ba7194d119c3efb1c2e7929f36da183ffe71d72a8 +3:cpuacct:/ecs/task-arn/93d377d55070d2463493706ba7194d119c3efb1c2e7929f36da183ffe71d72a8 +2:cpu:/ecs/task-arn/93d377d55070d2463493706ba7194d119c3efb1c2e7929f36da183ffe71d72a8 +1:cpuset:/ecs/task-arn/93d377d55070d2463493706ba7194d119c3efb1c2e7929f36da183ffe71d72a8 \ No newline at end of file diff --git a/libs/core/src/test/java/com/solarwinds/joboe/core/util/docker-cgroup-empty b/libs/core/src/test/java/com/solarwinds/joboe/core/util/docker-cgroup-empty new file mode 100644 index 00000000..e69de29b diff --git a/libs/core/src/test/java/com/solarwinds/joboe/core/util/docker-cgroup-invalid b/libs/core/src/test/java/com/solarwinds/joboe/core/util/docker-cgroup-invalid new file mode 100644 index 00000000..ec00544e --- /dev/null +++ b/libs/core/src/test/java/com/solarwinds/joboe/core/util/docker-cgroup-invalid @@ -0,0 +1 @@ +10:memory:/docker/non-container-id diff --git a/libs/core/src/test/java/com/solarwinds/joboe/core/util/docker-cgroup-kubepods b/libs/core/src/test/java/com/solarwinds/joboe/core/util/docker-cgroup-kubepods new file mode 100644 index 00000000..12bd176c --- /dev/null +++ b/libs/core/src/test/java/com/solarwinds/joboe/core/util/docker-cgroup-kubepods @@ -0,0 +1,11 @@ +11:devices:/kubepods/besteffort/pod349404940_49404049404/0531ff3c6395131175507ac7e94fdf387f2a2dea81961e6c96f6ac5ccd7ede3f +10:hugetlb:/kubepods/besteffort/pod349404940_49404049404/0531ff3c6395131175507ac7e94fdf387f2a2dea81961e6c96f6ac5ccd7ede3f +9:cpu,cpu_acct:/kubepods/besteffort/pod349404940_49404049404/0531ff3c6395131175507ac7e94fdf387f2a2dea81961e6c96f6ac5ccd7ede3f +8:memory:/kubepods/besteffort/pod349404940_49404049404/0531ff3c6395131175507ac7e94fdf387f2a2dea81961e6c96f6ac5ccd7ede3f +7:pids:/kubepods/besteffort/pod349404940_49404049404/0531ff3c6395131175507ac7e94fdf387f2a2dea81961e6c96f6ac5ccd7ede3f +6:perf_event:/kubepods/besteffort/pod349404940_49404049404/0531ff3c6395131175507ac7e94fdf387f2a2dea81961e6c96f6ac5ccd7ede3f +5:freezer:/kubepods/besteffort/pod349404940_49404049404/0531ff3c6395131175507ac7e94fdf387f2a2dea81961e6c96f6ac5ccd7ede3f +4:net_cls,net_prio:/kubepods/besteffort/pod349404940_49404049404/0531ff3c6395131175507ac7e94fdf387f2a2dea81961e6c96f6ac5ccd7ede3f +3:blkio:/kubepods/besteffort/pod349404940_49404049404/0531ff3c6395131175507ac7e94fdf387f2a2dea81961e6c96f6ac5ccd7ede3f +2:cpuset:/kubepods/besteffort/pod349404940_49404049404/0531ff3c6395131175507ac7e94fdf387f2a2dea81961e6c96f6ac5ccd7ede3f +1:name=systemd:/kubepods/besteffort/pod349404940_49404049404/0531ff3c6395131175507ac7e94fdf387f2a2dea81961e6c96f6ac5ccd7ede3f \ No newline at end of file diff --git a/libs/core/src/test/java/com/solarwinds/joboe/core/util/docker-cgroup-non-docker b/libs/core/src/test/java/com/solarwinds/joboe/core/util/docker-cgroup-non-docker new file mode 100644 index 00000000..c93c71ea --- /dev/null +++ b/libs/core/src/test/java/com/solarwinds/joboe/core/util/docker-cgroup-non-docker @@ -0,0 +1,10 @@ +10:memory:/user.slice/user-1000.slice +9:blkio:/user.slice/user-1000.slice +8:net_cls,net_prio:/ +7:cpu,cpuacct:/user.slice/user-1000.slice +6:perf_event:/ +5:freezer:/ +4:cpuset:/ +3:pids:/user.slice/user-1000.slice +2:devices:/user.slice/user-1000.slice +1:name=systemd:/user.slice/user-1000.slice/session-3.scope \ No newline at end of file diff --git a/libs/core/src/test/java/com/solarwinds/joboe/core/util/docker-cgroup-standard b/libs/core/src/test/java/com/solarwinds/joboe/core/util/docker-cgroup-standard new file mode 100644 index 00000000..6434495c --- /dev/null +++ b/libs/core/src/test/java/com/solarwinds/joboe/core/util/docker-cgroup-standard @@ -0,0 +1,11 @@ +11:cpuset:/docker/0531ff3c6395131175507ac7e94fdf387f2a2dea81961e6c96f6ac5ccd7ede3f +10:memory:/docker/0531ff3c6395131175507ac7e94fdf387f2a2dea81961e6c96f6ac5ccd7ede3f +9:net_cls,net_prio:/docker/0531ff3c6395131175507ac7e94fdf387f2a2dea81961e6c96f6ac5ccd7ede3f +8:pids:/docker/0531ff3c6395131175507ac7e94fdf387f2a2dea81961e6c96f6ac5ccd7ede3f +7:blkio:/docker/0531ff3c6395131175507ac7e94fdf387f2a2dea81961e6c96f6ac5ccd7ede3f +6:cpu,cpuacct:/docker/0531ff3c6395131175507ac7e94fdf387f2a2dea81961e6c96f6ac5ccd7ede3f +5:perf_event:/docker/0531ff3c6395131175507ac7e94fdf387f2a2dea81961e6c96f6ac5ccd7ede3f +4:freezer:/docker/0531ff3c6395131175507ac7e94fdf387f2a2dea81961e6c96f6ac5ccd7ede3f +3:hugetlb:/docker/0531ff3c6395131175507ac7e94fdf387f2a2dea81961e6c96f6ac5ccd7ede3f +2:devices:/docker/0531ff3c6395131175507ac7e94fdf387f2a2dea81961e6c96f6ac5ccd7ede3f +1:name=systemd:/docker/0531ff3c6395131175507ac7e94fdf387f2a2dea81961e6c96f6ac5ccd7ede3f \ No newline at end of file diff --git a/libs/core/src/test/java/com/solarwinds/joboe/core/util/docker-cgroup-standard-2 b/libs/core/src/test/java/com/solarwinds/joboe/core/util/docker-cgroup-standard-2 new file mode 100644 index 00000000..6938c567 --- /dev/null +++ b/libs/core/src/test/java/com/solarwinds/joboe/core/util/docker-cgroup-standard-2 @@ -0,0 +1,11 @@ +11:cpuset:abc +10:memory:abc +9:net_cls,net_prio:abc +8:pids:abc +7:blkio:abc +6:cpu,cpuacct:abc +5:perf_event:abc +4:freezer:abc +3:hugetlb:abc +2:devices:abc +1:name=systemd:/docker/0531ff3c6395131175507ac7e94fdf387f2a2dea81961e6c96f6ac5ccd7ede3f \ No newline at end of file diff --git a/libs/core/src/test/java/com/solarwinds/joboe/core/util/gentoo-release b/libs/core/src/test/java/com/solarwinds/joboe/core/util/gentoo-release new file mode 100644 index 00000000..e3636f5b --- /dev/null +++ b/libs/core/src/test/java/com/solarwinds/joboe/core/util/gentoo-release @@ -0,0 +1 @@ +Gentoo Base System release 2.1 \ No newline at end of file diff --git a/libs/core/src/test/java/com/solarwinds/joboe/core/util/lsb-release b/libs/core/src/test/java/com/solarwinds/joboe/core/util/lsb-release new file mode 100644 index 00000000..3a9d6913 --- /dev/null +++ b/libs/core/src/test/java/com/solarwinds/joboe/core/util/lsb-release @@ -0,0 +1,4 @@ +DISTRIB_ID=Ubuntu +DISTRIB_RELEASE=10.04 +DISTRIB_CODENAME=lucid +DISTRIB_DESCRIPTION="Ubuntu 10.04.2 LTS" \ No newline at end of file diff --git a/libs/core/src/test/java/com/solarwinds/joboe/core/util/namespace b/libs/core/src/test/java/com/solarwinds/joboe/core/util/namespace new file mode 100644 index 00000000..e9f96ad6 --- /dev/null +++ b/libs/core/src/test/java/com/solarwinds/joboe/core/util/namespace @@ -0,0 +1 @@ +o11y-platform \ No newline at end of file diff --git a/libs/core/src/test/java/com/solarwinds/joboe/core/util/poduid b/libs/core/src/test/java/com/solarwinds/joboe/core/util/poduid new file mode 100644 index 00000000..9cfb7315 --- /dev/null +++ b/libs/core/src/test/java/com/solarwinds/joboe/core/util/poduid @@ -0,0 +1,40 @@ +5095 3607 0:432 / / ro,relatime master:975 - overlay overlay rw,lowerdir=/var/lib/containerd/io.containerd.snapshotter.v1.overlayfs/snapshots/2416/fs:/var/lib/containerd/io.containerd.snapshotter.v1.overlayfs/snapshots/2415/fs:/var/lib/containerd/io.containerd.snapshotter.v1.overlayfs/snapshots/2414/fs:/var/lib/containerd/io.containerd.snapshotter.v1.overlayfs/snapshots/2413/fs:/var/lib/containerd/io.containerd.snapshotter.v1.overlayfs/snapshots/2412/fs:/var/lib/containerd/io.containerd.snapshotter.v1.overlayfs/snapshots/2411/fs:/var/lib/containerd/io.containerd.snapshotter.v1.overlayfs/snapshots/2410/fs:/var/lib/containerd/io.containerd.snapshotter.v1.overlayfs/snapshots/2409/fs:/var/lib/containerd/io.containerd.snapshotter.v1.overlayfs/snapshots/2408/fs,upperdir=/var/lib/containerd/io.containerd.snapshotter.v1.overlayfs/snapshots/2417/fs,workdir=/var/lib/containerd/io.containerd.snapshotter.v1.overlayfs/snapshots/2417/work,xino=off +5096 5095 0:433 / /proc rw,nosuid,nodev,noexec,relatime - proc proc rw +5097 5095 0:434 / /dev rw,nosuid - tmpfs tmpfs rw,size=65536k,mode=755 +5098 5097 0:435 / /dev/pts rw,nosuid,noexec,relatime - devpts devpts rw,gid=5,mode=620,ptmxmode=666 +5099 5097 0:402 / /dev/mqueue rw,nosuid,nodev,noexec,relatime - mqueue mqueue rw +5100 5095 0:407 / /sys ro,nosuid,nodev,noexec,relatime - sysfs sysfs ro +5101 5100 0:436 / /sys/fs/cgroup rw,nosuid,nodev,noexec,relatime - tmpfs tmpfs rw,mode=755 +5102 5101 0:30 /kubepods/burstable/pod9dcdb600-4156-4b7b-afcc-f8c06cb0e474/b92a703b826d4494978d810adf100d06c5ade2539b714d984ae3c8fef3449964 /sys/fs/cgroup/systemd ro,nosuid,nodev,noexec,relatime master:11 - cgroup cgroup rw,xattr,name=systemd +5103 5101 0:33 /kubepods/burstable/pod9dcdb600-4156-4b7b-afcc-f8c06cb0e474/b92a703b826d4494978d810adf100d06c5ade2539b714d984ae3c8fef3449964 /sys/fs/cgroup/blkio ro,nosuid,nodev,noexec,relatime master:15 - cgroup cgroup rw,blkio +5130 5101 0:34 /kubepods/burstable/pod9dcdb600-4156-4b7b-afcc-f8c06cb0e474/b92a703b826d4494978d810adf100d06c5ade2539b714d984ae3c8fef3449964 /sys/fs/cgroup/cpu,cpuacct ro,nosuid,nodev,noexec,relatime master:16 - cgroup cgroup rw,cpu,cpuacct +5131 5101 0:35 /kubepods/burstable/pod9dcdb600-4156-4b7b-afcc-f8c06cb0e474/b92a703b826d4494978d810adf100d06c5ade2539b714d984ae3c8fef3449964 /sys/fs/cgroup/devices ro,nosuid,nodev,noexec,relatime master:17 - cgroup cgroup rw,devices +5132 5101 0:36 /kubepods/burstable/pod9dcdb600-4156-4b7b-afcc-f8c06cb0e474/b92a703b826d4494978d810adf100d06c5ade2539b714d984ae3c8fef3449964 /sys/fs/cgroup/freezer ro,nosuid,nodev,noexec,relatime master:18 - cgroup cgroup rw,freezer +5133 5101 0:37 /kubepods/burstable/pod9dcdb600-4156-4b7b-afcc-f8c06cb0e474/b92a703b826d4494978d810adf100d06c5ade2539b714d984ae3c8fef3449964 /sys/fs/cgroup/memory ro,nosuid,nodev,noexec,relatime master:19 - cgroup cgroup rw,memory +5134 5101 0:38 /kubepods/burstable/pod9dcdb600-4156-4b7b-afcc-f8c06cb0e474/b92a703b826d4494978d810adf100d06c5ade2539b714d984ae3c8fef3449964 /sys/fs/cgroup/cpuset ro,nosuid,nodev,noexec,relatime master:20 - cgroup cgroup rw,cpuset +5162 5101 0:39 /kubepods/burstable/pod9dcdb600-4156-4b7b-afcc-f8c06cb0e474/b92a703b826d4494978d810adf100d06c5ade2539b714d984ae3c8fef3449964 /sys/fs/cgroup/hugetlb ro,nosuid,nodev,noexec,relatime master:21 - cgroup cgroup rw,hugetlb +5163 5101 0:40 /kubepods/burstable/pod9dcdb600-4156-4b7b-afcc-f8c06cb0e474/b92a703b826d4494978d810adf100d06c5ade2539b714d984ae3c8fef3449964 /sys/fs/cgroup/net_cls,net_prio ro,nosuid,nodev,noexec,relatime master:22 - cgroup cgroup rw,net_cls,net_prio +5164 5101 0:41 /kubepods/burstable/pod9dcdb600-4156-4b7b-afcc-f8c06cb0e474/b92a703b826d4494978d810adf100d06c5ade2539b714d984ae3c8fef3449964 /sys/fs/cgroup/pids ro,nosuid,nodev,noexec,relatime master:23 - cgroup cgroup rw,pids +5165 5101 0:42 / /sys/fs/cgroup/rdma ro,nosuid,nodev,noexec,relatime master:24 - cgroup cgroup rw,rdma +5166 5101 0:43 /kubepods/burstable/pod9dcdb600-4156-4b7b-afcc-f8c06cb0e474/b92a703b826d4494978d810adf100d06c5ade2539b714d984ae3c8fef3449964 /sys/fs/cgroup/perf_event ro,nosuid,nodev,noexec,relatime master:25 - cgroup cgroup rw,perf_event +5167 5095 8:1 /var/lib/kubelet/pods/9dcdb600-4156-4b7b-afcc-f8c06cb0e474/volumes/kubernetes.io~empty-dir/temp-dir /tmp rw,relatime - ext4 /dev/sda1 rw,discard +5175 5095 8:1 /var/lib/kubelet/pods/9dcdb600-4156-4b7b-afcc-f8c06cb0e474/etc-hosts /etc/hosts rw,relatime - ext4 /dev/sda1 rw,discard +5176 5097 8:1 /var/lib/kubelet/pods/9dcdb600-4156-4b7b-afcc-f8c06cb0e474/containers/trace-hopper/12b93bce /dev/termination-log rw,relatime - ext4 /dev/sda1 rw,discard +5177 5095 8:1 /var/lib/containerd/io.containerd.grpc.v1.cri/sandboxes/a079f12d3215697a69dbb86b58dd6d6957c54c8588b9dfa6b210e22ec70f279e/hostname /etc/hostname ro,relatime - ext4 /dev/sda1 rw,discard +5181 5095 8:1 /var/lib/containerd/io.containerd.grpc.v1.cri/sandboxes/a079f12d3215697a69dbb86b58dd6d6957c54c8588b9dfa6b210e22ec70f279e/resolv.conf /etc/resolv.conf ro,relatime - ext4 /dev/sda1 rw,discard +5189 5097 0:400 / /dev/shm rw,nosuid,nodev,noexec,relatime - tmpfs shm rw,size=65536k +5203 5095 8:1 /var/lib/kubelet/pods/9dcdb600-4156-4b7b-afcc-f8c06cb0e474/volumes/kubernetes.io~configmap/trace-hopper-vol/..2023_05_11_12_36_52.1754749256/trace-hopper.yaml /app/resources/trace-hopper.yaml ro,relatime - ext4 /dev/sda1 rw,discard +5204 5095 8:1 /var/lib/kubelet/pods/9dcdb600-4156-4b7b-afcc-f8c06cb0e474/volumes/kubernetes.io~configmap/trace-hopper-vol/..2023_05_11_12_36_52.1754749256/javaagent.json /app/resources/javaagent.json ro,relatime - ext4 /dev/sda1 rw,discard +5205 5095 0:399 / /run/secrets/kubernetes.io/serviceaccount ro,relatime - tmpfs tmpfs rw,size=4954828k +3608 5096 0:433 /bus /proc/bus ro,nosuid,nodev,noexec,relatime - proc proc rw +3609 5096 0:433 /fs /proc/fs ro,nosuid,nodev,noexec,relatime - proc proc rw +3610 5096 0:433 /irq /proc/irq ro,nosuid,nodev,noexec,relatime - proc proc rw +3611 5096 0:433 /sys /proc/sys ro,nosuid,nodev,noexec,relatime - proc proc rw +3676 5096 0:433 /sysrq-trigger /proc/sysrq-trigger ro,nosuid,nodev,noexec,relatime - proc proc rw +3677 5096 0:437 / /proc/acpi ro,relatime - tmpfs tmpfs ro +3679 5096 0:434 /null /proc/kcore rw,nosuid - tmpfs tmpfs rw,size=65536k,mode=755 +3680 5096 0:434 /null /proc/keys rw,nosuid - tmpfs tmpfs rw,size=65536k,mode=755 +3681 5096 0:434 /null /proc/timer_list rw,nosuid - tmpfs tmpfs rw,size=65536k,mode=755 +3682 5096 0:434 /null /proc/sched_debug rw,nosuid - tmpfs tmpfs rw,size=65536k,mode=755 +3683 5096 0:438 / /proc/scsi ro,relatime - tmpfs tmpfs ro +3684 5100 0:452 / /sys/firmware ro,relatime - tmpfs tmpfs ro \ No newline at end of file diff --git a/libs/core/src/test/java/com/solarwinds/joboe/core/util/redhat-release b/libs/core/src/test/java/com/solarwinds/joboe/core/util/redhat-release new file mode 100644 index 00000000..0fa975a2 --- /dev/null +++ b/libs/core/src/test/java/com/solarwinds/joboe/core/util/redhat-release @@ -0,0 +1 @@ +Red Hat Enterprise Linux Server release 6.5 (Santiago) \ No newline at end of file diff --git a/libs/core/src/test/java/com/solarwinds/joboe/core/util/slackware-version b/libs/core/src/test/java/com/solarwinds/joboe/core/util/slackware-version new file mode 100644 index 00000000..fb17e310 --- /dev/null +++ b/libs/core/src/test/java/com/solarwinds/joboe/core/util/slackware-version @@ -0,0 +1 @@ +Slackware-x86_64 13.0 \ No newline at end of file diff --git a/libs/core/src/test/java/com/solarwinds/joboe/core/util/system-release-cpe b/libs/core/src/test/java/com/solarwinds/joboe/core/util/system-release-cpe new file mode 100644 index 00000000..a4522960 --- /dev/null +++ b/libs/core/src/test/java/com/solarwinds/joboe/core/util/system-release-cpe @@ -0,0 +1 @@ +cpe:/o:amazon:linux:2015.09:ga \ No newline at end of file diff --git a/libs/core/src/test/resources/solarwinds-apm-settings-raw b/libs/core/src/test/resources/solarwinds-apm-settings-raw new file mode 100644 index 0000000000000000000000000000000000000000..55b263c619302a2d84f8dfbf9aac33c1be541565 GIT binary patch literal 377 zcmb38#w6qx?C2Zd;~F0v;usX71LcHx1o?-$d&E2XggZtC!$h4zoxNQ{;$8h5oqSwf zbV7nW-Q8V-;zNQQon0l)+*!1OP2oU`t(6oPpHpdac4~=pVnJeZW=W+G2Ll8+Sc!3Q zLF9rGOHyItAMCAUxI}zYONug+i`{Zci!(g)N>Yo;5_5!DY=Ang47hlLGt=`DOG=AU zy;CcN6by>fO0o=HBO}bcyb25qU6OLV%CnP`Bb_~?EDh6=Laj8oWI~EE)6-Lnf>Lu5 yD^gRiyU4`BN`*@Tn@TkIsdGs~RRx1wS%Td-2?r}>E^#ahA*Rim2?9Iq9V!6zigE}5 literal 0 HcmV?d00001 diff --git a/libs/lambda/build.gradle.kts b/libs/lambda/build.gradle.kts index e600a102..ac2fdbb2 100644 --- a/libs/lambda/build.gradle.kts +++ b/libs/lambda/build.gradle.kts @@ -21,13 +21,11 @@ plugins { dependencies { compileOnly(project(":libs:shared")) compileOnly("com.google.code.gson:gson") - compileOnly("org.projectlombok:lombok") - annotationProcessor("org.projectlombok:lombok") - compileOnly("com.solarwinds.joboe:config") - compileOnly("com.solarwinds.joboe:sampling") + compileOnly(project(":libs:config")) + compileOnly(project(":libs:sampling")) - compileOnly("com.solarwinds.joboe:logging") + compileOnly(project(":libs:logging")) compileOnly("com.google.auto.service:auto-service") annotationProcessor("com.google.auto.service:auto-service") @@ -40,9 +38,16 @@ dependencies { compileOnly("io.opentelemetry.javaagent:opentelemetry-javaagent-extension-api") compileOnly(project(":bootstrap")) + testImplementation(project(":libs:config")) + testImplementation(project(":libs:sampling")) testImplementation(project(":libs:shared")) } +tasks.named("compileJava") { + // Disable AutoService verify check to prevent rawtypes warnings for generic service provider interfaces + options.compilerArgs.add("-Averify=false") +} + swoJava { minJavaVersionSupported.set(JavaVersion.VERSION_1_8) } diff --git a/libs/lambda/src/main/java/com/solarwinds/opentelemetry/extensions/FileSettingsReader.java b/libs/lambda/src/main/java/com/solarwinds/opentelemetry/extensions/FileSettingsReader.java index 06380b11..46a952bd 100644 --- a/libs/lambda/src/main/java/com/solarwinds/opentelemetry/extensions/FileSettingsReader.java +++ b/libs/lambda/src/main/java/com/solarwinds/opentelemetry/extensions/FileSettingsReader.java @@ -27,6 +27,7 @@ import com.solarwinds.opentelemetry.extensions.config.JsonSettingWrapper; import java.io.IOException; import java.lang.reflect.Type; +import java.nio.charset.StandardCharsets; import java.nio.file.Files; import java.nio.file.Paths; import java.util.List; @@ -48,7 +49,8 @@ public Settings getSettings() throws SamplingException { try { byte[] bytes = Files.readAllBytes(Paths.get(settingsFilePath)); List kvSetting = - JsonSettingWrapper.fromJsonSettings(gson.fromJson(new String(bytes), type)); + JsonSettingWrapper.fromJsonSettings( + gson.fromJson(new String(bytes, StandardCharsets.UTF_8), type)); logger.debug(String.format("Got settings from file: %s", kvSetting)); if (!kvSetting.isEmpty()) { diff --git a/libs/logging/.gitignore b/libs/logging/.gitignore new file mode 100644 index 00000000..567609b1 --- /dev/null +++ b/libs/logging/.gitignore @@ -0,0 +1 @@ +build/ diff --git a/libs/logging/build.gradle.kts b/libs/logging/build.gradle.kts new file mode 100644 index 00000000..78105556 --- /dev/null +++ b/libs/logging/build.gradle.kts @@ -0,0 +1,5 @@ +plugins { + id("solarwinds.java-conventions") +} + +description = "logging" diff --git a/libs/logging/src/main/java/com/solarwinds/joboe/logging/CompositeStream.java b/libs/logging/src/main/java/com/solarwinds/joboe/logging/CompositeStream.java new file mode 100644 index 00000000..29fe3677 --- /dev/null +++ b/libs/logging/src/main/java/com/solarwinds/joboe/logging/CompositeStream.java @@ -0,0 +1,43 @@ +/* + * © SolarWinds Worldwide, LLC. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.solarwinds.joboe.logging; + +class CompositeStream implements LoggerStream { + private final LoggerStream[] streams; + + CompositeStream(LoggerStream... streams) { + this.streams = streams; + } + + @Override + public void println(String value) { + for (LoggerStream stream : streams) { + stream.println(value); + } + } + + @Override + public void printStackTrace(Throwable throwable) { + for (LoggerStream stream : streams) { + stream.printStackTrace(throwable); + } + } + + LoggerStream[] getStreams() { + return streams; + } +} diff --git a/libs/logging/src/main/java/com/solarwinds/joboe/logging/FileLoggerStream.java b/libs/logging/src/main/java/com/solarwinds/joboe/logging/FileLoggerStream.java new file mode 100644 index 00000000..448c0363 --- /dev/null +++ b/libs/logging/src/main/java/com/solarwinds/joboe/logging/FileLoggerStream.java @@ -0,0 +1,371 @@ +/* + * © SolarWinds Worldwide, LLC. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.solarwinds.joboe.logging; + +import static com.solarwinds.joboe.logging.Logger.STD_STREAM_LOGGER; + +import java.io.BufferedWriter; +import java.io.Closeable; +import java.io.IOException; +import java.io.PrintWriter; +import java.nio.channels.Channels; +import java.nio.channels.FileChannel; +import java.nio.channels.FileLock; +import java.nio.charset.Charset; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.Paths; +import java.nio.file.StandardCopyOption; +import java.nio.file.StandardOpenOption; +import java.util.concurrent.Callable; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.LinkedBlockingQueue; +import java.util.concurrent.RejectedExecutionException; +import java.util.concurrent.ThreadPoolExecutor; +import java.util.concurrent.TimeUnit; + +/** + * Logger stream that writes to a File via a FileChannel + * + *

This implements a rolling behavior which rolls the log file and create backup files up to + * `maxBackup` if the current log file size exceeds `maxSize`. + * + *

Take note that multiple java processes might run this logger and the rolling operation should + * be blocking across processes so that the log file should only be rolled once. + * + *

If another process has rolled the file, the current logger might still write to the renamed + * (rolled) file for a short while but should be switching to the new log file eventually. However, + * it is guaranteed that no log should be lost if the rolling is done by another process. + */ +class FileLoggerStream implements LoggerStream, Closeable { + private static final int MAX_LOCK_TRY = 5; + private static final int LOCK_RETRY_SLEEP = 100; // in millseconds + private static final int CLOSE_TIMEOUT = 5; + private static final int MESSAGE_QUEUE_SIZE = 100000; + private static final int RETRY_INTERVAL = 60000; // only retry every 60 seconds + private final Path logFilePath; + private final Path + logFileLockPath; // need a separate lock file, cannot use the existing log file as it could be + // moved by other processes + FileChannel fileChannel; + private PrintWriter filePrintWriter; + private Long lastRollFailure; // last time of file rolling failure + private Long lastChannelReset; // last time the file channel was reset + private final long maxSize; // in bytes + private final int maxBackup; + private final ExecutorService service = + new ThreadPoolExecutor( + 1, + 1, + 0L, + TimeUnit.MILLISECONDS, + new LinkedBlockingQueue(MESSAGE_QUEUE_SIZE), + LoggerThreadFactory.newInstance("file-logger")); + + FileLoggerStream(Path logFilePath, int maxSizeInBytes, int maxBackup) throws IOException { + resetChannel(logFilePath); + this.logFilePath = logFilePath; + String tempDir = System.getProperty("java.io.tmpdir"); + if (tempDir != null && Files.isWritable(Paths.get(tempDir))) { + this.logFileLockPath = Paths.get(tempDir, logFilePath.getFileName() + ".lock"); + } else { + this.logFileLockPath = Paths.get(logFilePath.toString() + ".lock"); + } + Logger.INSTANCE.debug("Using " + logFileLockPath + " for log file lock for file rolling"); + + this.maxSize = maxSizeInBytes; + this.maxBackup = maxBackup; + } + + Path getLogFilePath() { + return logFilePath; + } + + private void resetChannel(Path logFilePath) throws IOException { + resetChannel(logFilePath, true); + } + + /** + * Resets the file channel and print writer + * + *

If forced, then it always resets it, otherwise reset would only happen if it has been less + * than RETRY_INTERVAL since last attempt + * + * @param logFilePath + * @param forced + * @throws IOException + */ + private void resetChannel(Path logFilePath, boolean forced) throws IOException { + boolean resetChannel; + long time = System.currentTimeMillis(); + if (forced) { + resetChannel = true; + } else { + resetChannel = lastChannelReset == null || time - lastChannelReset >= RETRY_INTERVAL; + } + + if (resetChannel) { + closeIo(); + + this.fileChannel = + FileChannel.open( + logFilePath, + StandardOpenOption.APPEND, + StandardOpenOption.CREATE, + StandardOpenOption.WRITE); + this.filePrintWriter = + new PrintWriter( + new BufferedWriter( + Channels.newWriter(fileChannel, Charset.defaultCharset().newEncoder(), -1))); + + lastChannelReset = time; + } + } + + @Override + public void println(final String value) { + try { + service.submit( + new LogTask() { + @Override + protected void log() { + filePrintWriter.println(value); + filePrintWriter.flush(); + } + }); + } catch (RejectedExecutionException e) { + // use STD_STREAM_LOGGER as we do not want to submit more messages to this file logger + STD_STREAM_LOGGER.debug( + "Failed to log message to file as queue is full " + e.getMessage(), e); + } + } + + @Override + public void printStackTrace(final Throwable throwable) { + try { + service.submit( + new LogTask() { + @Override + protected void log() { + throwable.printStackTrace(filePrintWriter); + filePrintWriter.flush(); + } + }); + } catch (RejectedExecutionException e) { + // use STD_STREAM_LOGGER as we do not want to submit more messages to this file logger + STD_STREAM_LOGGER.debug( + "Failed to log message to file as queue is full " + e.getMessage(), e); + } + } + + @Override + public void close() { + close(CLOSE_TIMEOUT); + } + + public void close(long closeTimeoutInSec) { + service.shutdown(); + try { + service.awaitTermination(closeTimeoutInSec, TimeUnit.SECONDS); + closeIo(); + } catch (Exception e) { + // ok, it's closing anyway + } + } + + private void closeIo() throws IOException { + if (filePrintWriter != null) { + filePrintWriter.flush(); + filePrintWriter.close(); + } + if (fileChannel != null) { + fileChannel.close(); + } + } + + private abstract class LogTask implements Callable { + @Override + public Boolean call() { + try { + checkFile(); + } catch (Throwable e) { + // use STD_STREAM_LOGGER as we do not want to submit more messages to this file logger which + // might trigger more error + STD_STREAM_LOGGER.warn( + "Failed to check the log file for size limit : " + e.getMessage(), e); + } + log(); + return true; + } + + protected abstract void log(); + } + + /** + * Checks and resets if the current log file channel exceeds the max size and perform file rolling + * if necessary + * + *

Take note that this needs to take into consideration of: 1. 2 threads within the same JVM + * can possibly call this concurrently, even though current implementation runs on a single + * threaded pool, it could change in the future. 2. 2 JVM processes can possibly call this + * concurrently (for example the shutdown process of tomcat is a separate process from the running + * tomcat), and it should only roll once. + */ + private boolean checkFile() throws IOException { + if (isChannelExceedingSizeLimit()) { + boolean forcedReset = false; + try { + if (shouldRollFile()) { // then see if files should be rolled + Logger.INSTANCE.debug( + "Log file [" + + logFilePath + + "] exceeds " + + (maxSize / 1024 / 1024) + + " MB, attempt to roll the log files."); + boolean rolled = lockAndRollFile(); + forcedReset = rolled; // always reset the channel if the rolling was successful + } + } finally { + // if the channel exceeds size limit and the file rolling was successful, then it's required + // to + // reset the channel. Otherwise, we will try to reset the channel but w/o a forced reset, + // this is + // to avoid resetting the channel too rapidly if file rolling fails repeatedly due to + // persistent + // problem such as external file locking. Take note that it still makes sense to try + // resetting even + // if file rolling failed, for example another process might have rolled the file so this + // process + // should attempt to reset file channel to point to the new log file + resetChannel(logFilePath, forcedReset); + } + } + return true; + } + + private boolean isChannelExceedingSizeLimit() throws IOException { + return fileChannel.size() > maxSize; + } + + private boolean shouldRollFile() throws IOException { + return Files.size(logFilePath) > maxSize + && (lastRollFailure == null + || System.currentTimeMillis() - lastRollFailure >= RETRY_INTERVAL); + } + + /** + * Attempts to acquire the lock for rolling the file. + * + *

If the lock is acquired, then perform file rolling + * + * @return whether the file was rolled successfully + * @throws IOException + */ + boolean lockAndRollFile() throws IOException { + FileLockAndChannel fileLockAndChannel = null; + + try { + fileLockAndChannel = tryLock(); + boolean rolled = false; + if (fileLockAndChannel == null) { + // ok to log to the file logger as we know this would not trigger rapid file locking error + // as it's guarded by RETRY_INTERVAL + Logger.INSTANCE.warn( + "Failed to acquire lock on log file after " + + MAX_LOCK_TRY + + " tries during the attempt to roll."); + } else { // locked acquired, check again to ensure the files are not rolled while waiting for + // the lock + if (shouldRollFile()) { + try { + doRollFile(); + rolled = true; + Logger.INSTANCE.debug("Successfully rolled the log files"); + lastRollFailure = null; // reset failure timestamp + } catch (IOException e) { + // ok to log to the file logger as we know this would not trigger rapid file locking + // error as it's guarded by RETRY_INTERVAL + Logger.INSTANCE.warn( + "Failed to roll the log files, exception message: " + e.getMessage(), e); + lastRollFailure = System.currentTimeMillis(); + throw e; + } + } + } + return rolled; + } finally { + if (fileLockAndChannel != null) { + fileLockAndChannel.fileLock + .release(); // release the lock first before closing the file channel + fileLockAndChannel.lockChannel.close(); + } + } + } + + private void doRollFile() throws IOException { + // delete the one with highest index if exist + Path lastBackupFilePath = Paths.get(logFilePath.toString() + "." + maxBackup); + Files.deleteIfExists(lastBackupFilePath); + + // traverse the backup file and bump the index up by one, from the highest index - 1 first + for (int i = maxBackup - 1; i > 0; i--) { + Path backupFilePath = Paths.get(logFilePath + "." + i); + Path newBackupFilePath = Paths.get(logFilePath + "." + (i + 1)); + if (Files.exists(backupFilePath)) { + Files.move(backupFilePath, newBackupFilePath, StandardCopyOption.ATOMIC_MOVE); + } + } + // move the current log file + Files.move(logFilePath, Paths.get(logFilePath + ".1"), StandardCopyOption.ATOMIC_MOVE); + } + + FileLockAndChannel tryLock() { + int tryCount = 0; + FileLock lock; + while (++tryCount <= MAX_LOCK_TRY) { + try { + FileChannel lockChannel = + FileChannel.open( + logFileLockPath, + StandardOpenOption.CREATE, + StandardOpenOption.WRITE); // do not use DELETE_ON_EXIT here. Not working sometimes + lock = lockChannel.tryLock(); + if (lock != null) { + return new FileLockAndChannel(lock, lockChannel); + } + } catch (IOException e) { + Logger.INSTANCE.debug("Failed to obtain lockChannel : " + e.getMessage(), e); + } + try { + TimeUnit.MILLISECONDS.sleep(LOCK_RETRY_SLEEP); + } catch (InterruptedException e) { + Logger.INSTANCE.debug("Retry lock sleep interrupted", e); + } + } + return null; + } + + private static class FileLockAndChannel { + private final FileLock fileLock; + private final FileChannel lockChannel; + + private FileLockAndChannel(FileLock fileLock, FileChannel lockChannel) { + this.fileLock = fileLock; + this.lockChannel = lockChannel; + } + } +} diff --git a/libs/logging/src/main/java/com/solarwinds/joboe/logging/LogSetting.java b/libs/logging/src/main/java/com/solarwinds/joboe/logging/LogSetting.java new file mode 100644 index 00000000..117017d9 --- /dev/null +++ b/libs/logging/src/main/java/com/solarwinds/joboe/logging/LogSetting.java @@ -0,0 +1,88 @@ +/* + * © SolarWinds Worldwide, LLC. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.solarwinds.joboe.logging; + +import java.io.Serializable; +import java.nio.file.Path; +import java.util.Objects; +import lombok.Getter; + +@Getter +public class LogSetting implements Serializable { + private static final long serialVersionUID = 1L; + private final Logger.Level level; + private final boolean stdoutEnabled; + private final boolean stderrEnabled; + private final Path logFilePath; + private final int logFileMaxSize; + private final int logFileMaxBackup; + + public static final int DEFAULT_FILE_MAX_SIZE = 10; // in MB + public static final int DEFAULT_FILE_MAX_BACKUP = 5; // files to be kept + + public LogSetting( + Logger.Level level, + boolean stdoutEnabled, + boolean stderrEnabled, + Path logFilePath, + Integer logFileMaxSize, + Integer logFileMaxBackup) { + this.level = level; + this.stdoutEnabled = stdoutEnabled; + this.stderrEnabled = stderrEnabled; + this.logFilePath = logFilePath; + this.logFileMaxSize = logFileMaxSize != null ? logFileMaxSize : DEFAULT_FILE_MAX_SIZE; + this.logFileMaxBackup = logFileMaxBackup != null ? logFileMaxBackup : DEFAULT_FILE_MAX_BACKUP; + } + + @Override + public boolean equals(Object o) { + if (this == o) return true; + if (o == null || getClass() != o.getClass()) return false; + LogSetting that = (LogSetting) o; + return stdoutEnabled == that.stdoutEnabled + && stderrEnabled == that.stderrEnabled + && logFileMaxSize == that.logFileMaxSize + && logFileMaxBackup == that.logFileMaxBackup + && level == that.level + && Objects.equals(logFilePath, that.logFilePath); + } + + @Override + public int hashCode() { + return Objects.hash( + level, stdoutEnabled, stderrEnabled, logFilePath, logFileMaxSize, logFileMaxBackup); + } + + @Override + public String toString() { + return "LogSetting{" + + "level=" + + level + + ", stdoutEnabled=" + + stdoutEnabled + + ", stderrEnabled=" + + stderrEnabled + + ", logFilePath=" + + logFilePath + + ", logFileMaxSize=" + + logFileMaxSize + + ", logFileMaxBackup=" + + logFileMaxBackup + + '}'; + } +} diff --git a/libs/logging/src/main/java/com/solarwinds/joboe/logging/Logger.java b/libs/logging/src/main/java/com/solarwinds/joboe/logging/Logger.java new file mode 100644 index 00000000..96f0914f --- /dev/null +++ b/libs/logging/src/main/java/com/solarwinds/joboe/logging/Logger.java @@ -0,0 +1,289 @@ +/* + * © SolarWinds Worldwide, LLC. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.solarwinds.joboe.logging; + +import java.io.IOException; +import java.nio.file.Path; +import java.nio.file.Paths; +import java.text.DateFormat; +import java.text.SimpleDateFormat; +import java.util.ArrayList; +import java.util.Calendar; +import java.util.Collections; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.Set; +import lombok.Getter; + +/** + * A wrapper class around the java.util.logger. Take note that this is very similar to other logging + * frameworks (commons.logging, log4j etc)'s Logger and should be replaced by those implementations + * if we later on switch to those frameworks + * + * @author Patson Luk + */ +@Getter +public class Logger { + public static final String SOLARWINDSS_TAG = "[SolarWinds APM]"; + + private static final Level DEFAULT_LOGGING = Level.INFO; + private static final ThreadLocal DATE_FORMAT = + new ThreadLocal() { + @Override + protected SimpleDateFormat initialValue() { + return new SimpleDateFormat("MMM dd, yyyy hh:mm:ss.SSS aa"); + } + }; + private static final LogSetting DEFAULT_LOG_SETTING = + new LogSetting(DEFAULT_LOGGING, true, true, null, null, null); + + static final Logger INSTANCE = new Logger(); + static final Logger STD_STREAM_LOGGER = + new Logger(); // an internal logger that uses stderr and stdout streams only + + private Level loggerLevel = DEFAULT_LOGGING; + private LoggerStream errorStream = new SystemErrStream(); + private LoggerStream infoStream = new SystemOutStream(); + + private static final String INSTANCE_ID_ENV_VARIABLE = "WEBSITE_INSTANCE_ID"; + + Logger() {} + + LoggerStream getLogFileStream(Path logFilePath, int logFileMaxSize, int logFileMaxBackup) { + // special case for azure, prefix the instance id to the log file name + String azureInstanceId = System.getenv(INSTANCE_ID_ENV_VARIABLE); + ; + if (azureInstanceId != null) { + Path fileName = logFilePath.getFileName(); + String prefixedFileName = azureInstanceId + "-" + fileName.toString(); + logFilePath = + logFilePath.getParent() != null + ? Paths.get(logFilePath.getParent().toString(), prefixedFileName) + : Paths.get(prefixedFileName); + } + + try { + info("Java agent log location: " + logFilePath.toAbsolutePath()); + return new FileLoggerStream(logFilePath, logFileMaxSize * 1024 * 1024, logFileMaxBackup); + } catch (IOException e) { + warn( + "Failed to redirect logs to [" + logFilePath.toAbsolutePath() + "] : " + e.getMessage(), + e); + return null; + } + } + + LoggerStream getErrorStream() { + return errorStream; + } + + LoggerStream getInfoStream() { + return infoStream; + } + + public void fatal(String message) { + this.log(Level.FATAL, message); + } + + public void fatal(String message, Throwable throwable) { + this.log(Level.FATAL, message, throwable); + } + + public void error(String message) { + this.log(Level.ERROR, message); + } + + public void error(String message, Throwable throwable) { + this.log(Level.ERROR, message, throwable); + } + + public void warn(String message) { + this.log(Level.WARNING, message); + } + + public void warn(String message, Throwable throwable) { + this.log(Level.WARNING, message, throwable); + } + + public void info(String message) { + this.log(Level.INFO, message); + } + + public void info(String message, Throwable throwable) { + this.log(Level.INFO, message, throwable); + } + + public void debug(String message) { + this.log(Level.DEBUG, message); + } + + public void debug(String message, Throwable throwable) { + this.log(Level.DEBUG, message, throwable); + } + + public void trace(String message) { + this.log(Level.TRACE, message); + } + + public void trace(String message, Throwable throwable) { + this.log(Level.TRACE, message, throwable); + } + + public void log(Level level, String message) { + log(level, message, null); + } + + public void log(Level level, String message, Throwable t) { + if (level == null) { + if (shouldLog(Level.ERROR)) { // missing level in the input has severity level of Level.ERROR + print(Level.ERROR, "Missing log Level for this log message!"); + } + if (shouldLog( + DEFAULT_LOGGING)) { // the logging message itself takes the default logging level + print(DEFAULT_LOGGING, message, t); + } + } else if (shouldLog(level)) { + print(level, message, t); + } + } + + private void print(Level level, String message) { + print(level, message, null); + } + + private void print(Level level, String message, Throwable t) { + LoggerStream output = + level.compareTo(Level.INFO) < 0 + ? errorStream + : infoStream; // most severe level declared first + output.println(getFormattedMessage(level, message)); + + if (t != null) { + output.printStackTrace(t); + } + } + + private static String getFormattedMessage(Level level, String message) { + String timestamp = DATE_FORMAT.get().format(Calendar.getInstance().getTime()); + String label = timestamp + " " + level.toString() + " " + SOLARWINDSS_TAG + " "; + + return message != null ? label + message : label; + } + + public boolean shouldLog(Level messageLevel) { + return messageLevel.compareTo(loggerLevel) <= 0; // most severe level is declared first + } + + void configure(LoggerConfiguration config) { + LogSetting logSetting = config.getLogSetting(); + if (logSetting == null) { + logSetting = DEFAULT_LOG_SETTING; + } + + // level + if (config + .isDebug()) { // backward compatibility, use loggingLevel=DEBUG if the argument is set to + // false + loggerLevel = Level.DEBUG; + } else { + loggerLevel = logSetting.getLevel() != null ? logSetting.getLevel() : DEFAULT_LOGGING; + } + + // log file location + Path path = config.getLogFile(); + if (path == null) { + path = logSetting.getLogFilePath(); + } + + // other log file parameters + LoggerStream logFileStream = null; + if (path != null) { + String javaVersion = System.getProperty("java.version"); + + if (javaVersion.startsWith("1.6")) { + + info("Cannot set up log file at [" + path + "] as it is only supported for JDK 7 or later"); + } else { + logFileStream = + getLogFileStream( + path, logSetting.getLogFileMaxSize(), logSetting.getLogFileMaxBackup()); + } + } + + List errorStreams = new ArrayList(); + List infoStreams = new ArrayList(); + + if (logSetting.isStderrEnabled()) { + errorStreams.add(SystemErrStream.INSTANCE); + } + if (logSetting.isStdoutEnabled()) { + infoStreams.add(SystemOutStream.INSTANCE); + } + if (logFileStream != null) { + errorStreams.add(logFileStream); + infoStreams.add(logFileStream); + } + + if (errorStreams.size() == 1) { + this.errorStream = errorStreams.get(0); + } else { + this.errorStream = + new CompositeStream(errorStreams.toArray(new LoggerStream[errorStreams.size()])); + } + + if (infoStreams.size() == 1) { + this.infoStream = infoStreams.get(0); + } else { + this.infoStream = + new CompositeStream(infoStreams.toArray(new LoggerStream[infoStreams.size()])); + } + } + + @Getter + public enum Level { + OFF("off"), + FATAL("fatal"), + ERROR("error"), + WARNING("warn"), + INFO("info"), + DEBUG("debug"), + TRACE("trace"); + + private static final Map map = new HashMap(); + + static { + for (Level level : Level.values()) { + map.put(level.label, level); + } + } + + private final String label; + + Level(String label) { + this.label = label; + } + + public static Level fromLabel(String label) { + return map.get(label); + } + + public static Set getAllLabels() { + return Collections.unmodifiableSet(map.keySet()); + } + } +} diff --git a/libs/logging/src/main/java/com/solarwinds/joboe/logging/LoggerConfiguration.java b/libs/logging/src/main/java/com/solarwinds/joboe/logging/LoggerConfiguration.java new file mode 100644 index 00000000..ab04aad4 --- /dev/null +++ b/libs/logging/src/main/java/com/solarwinds/joboe/logging/LoggerConfiguration.java @@ -0,0 +1,32 @@ +/* + * © SolarWinds Worldwide, LLC. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.solarwinds.joboe.logging; + +import java.nio.file.Path; +import lombok.Builder; +import lombok.Value; + +@Value +@Builder +public class LoggerConfiguration { + @Builder.Default + LogSetting logSetting = new LogSetting(Logger.Level.INFO, true, true, null, null, null); + + boolean debug; + + Path logFile; +} diff --git a/libs/logging/src/main/java/com/solarwinds/joboe/logging/LoggerFactory.java b/libs/logging/src/main/java/com/solarwinds/joboe/logging/LoggerFactory.java new file mode 100644 index 00000000..fa58a445 --- /dev/null +++ b/libs/logging/src/main/java/com/solarwinds/joboe/logging/LoggerFactory.java @@ -0,0 +1,42 @@ +/* + * © SolarWinds Worldwide, LLC. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.solarwinds.joboe.logging; + +import lombok.Getter; + +/** + * Factory that returns logger instance. Take note that this is very similar to other logging + * frameworks (commons.logging, log4j etc) factory and should be replaced by those factory if we + * later on switch to those frameworks + * + * @author Patson Luk + */ +public class LoggerFactory { + // Only allow a single logger. We do not need multiple loggers as we handle everything using the + // same handlers at this moment + // Take note that we do not instantiate this in the init routine as we wish to make the logger + // always available (even before init is run) + // Just that code that uses the logger before calling init will not follow the configuration + // defined in the configration file ConfigProperty.AGENT_DEBUG + + @Getter private static final Logger logger = Logger.INSTANCE; + + /** Initialize the logger factory. This is used to set the logging level of the logger */ + public static void init(LoggerConfiguration config) { + logger.configure(config); + } +} diff --git a/libs/logging/src/main/java/com/solarwinds/joboe/logging/LoggerStream.java b/libs/logging/src/main/java/com/solarwinds/joboe/logging/LoggerStream.java new file mode 100644 index 00000000..0ff07204 --- /dev/null +++ b/libs/logging/src/main/java/com/solarwinds/joboe/logging/LoggerStream.java @@ -0,0 +1,23 @@ +/* + * © SolarWinds Worldwide, LLC. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.solarwinds.joboe.logging; + +interface LoggerStream { + void println(String value); + + void printStackTrace(Throwable throwable); +} diff --git a/libs/logging/src/main/java/com/solarwinds/joboe/logging/LoggerThreadFactory.java b/libs/logging/src/main/java/com/solarwinds/joboe/logging/LoggerThreadFactory.java new file mode 100644 index 00000000..440dfde6 --- /dev/null +++ b/libs/logging/src/main/java/com/solarwinds/joboe/logging/LoggerThreadFactory.java @@ -0,0 +1,58 @@ +/* + * © SolarWinds Worldwide, LLC. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.solarwinds.joboe.logging; + +import java.util.concurrent.Executors; +import java.util.concurrent.ThreadFactory; +import java.util.concurrent.atomic.AtomicInteger; + +public class LoggerThreadFactory implements ThreadFactory { + private final String threadName; + private static final Logger logger = LoggerFactory.getLogger(); + private final AtomicInteger count = new AtomicInteger(0); + private final ThreadFactory threadFactory = Executors.defaultThreadFactory(); + + private LoggerThreadFactory(String threadName) { + String THREAD_NAME_PREFIX = "SolarwindsAPM"; + this.threadName = + threadName != null ? THREAD_NAME_PREFIX + "-" + threadName : THREAD_NAME_PREFIX; + } + + public static LoggerThreadFactory newInstance(String threadName) { + return new LoggerThreadFactory(threadName); + } + + @Override + public Thread newThread(Runnable runnable) { + Thread thread = threadFactory.newThread(runnable); + thread.setDaemon(true); + thread.setName(threadName + "-" + count.incrementAndGet()); + + try { + // Set contextClassLoader to null to avoid memory leak error message during tomcat shutdown + // see http://wiki.apache.org/tomcat/MemoryLeakProtection#cclThreadSpawnedByWebApp + // It is ok to set it to null as we do not need servlet container class loader for spawned + // thread as they should only reference core sdk code or classes included in the agent jar + thread.setContextClassLoader(null); + } catch (SecurityException e) { + logger.warn( + "Cannot set the context class loader of System Monitor threads to null. Tomcat might display warning message of memory leak during shutdown"); + } + + return thread; + } +} diff --git a/libs/logging/src/main/java/com/solarwinds/joboe/logging/SystemErrStream.java b/libs/logging/src/main/java/com/solarwinds/joboe/logging/SystemErrStream.java new file mode 100644 index 00000000..5fa15f43 --- /dev/null +++ b/libs/logging/src/main/java/com/solarwinds/joboe/logging/SystemErrStream.java @@ -0,0 +1,33 @@ +/* + * © SolarWinds Worldwide, LLC. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.solarwinds.joboe.logging; + +class SystemErrStream implements LoggerStream { + static final SystemErrStream INSTANCE = new SystemErrStream(); + + SystemErrStream() {} + + @Override + public void println(String value) { + System.err.println(value); + } + + @Override + public void printStackTrace(Throwable throwable) { + throwable.printStackTrace(System.err); + } +} diff --git a/libs/logging/src/main/java/com/solarwinds/joboe/logging/SystemOutStream.java b/libs/logging/src/main/java/com/solarwinds/joboe/logging/SystemOutStream.java new file mode 100644 index 00000000..4b0d228b --- /dev/null +++ b/libs/logging/src/main/java/com/solarwinds/joboe/logging/SystemOutStream.java @@ -0,0 +1,33 @@ +/* + * © SolarWinds Worldwide, LLC. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.solarwinds.joboe.logging; + +class SystemOutStream implements LoggerStream { + static final SystemOutStream INSTANCE = new SystemOutStream(); + + SystemOutStream() {} + + @Override + public void println(String value) { + System.out.println(value); + } + + @Override + public void printStackTrace(Throwable throwable) { + throwable.printStackTrace(System.out); + } +} diff --git a/libs/logging/src/test/java/com/solarwinds/joboe/logging/FileLoggerStreamTest.java b/libs/logging/src/test/java/com/solarwinds/joboe/logging/FileLoggerStreamTest.java new file mode 100644 index 00000000..793bc617 --- /dev/null +++ b/libs/logging/src/test/java/com/solarwinds/joboe/logging/FileLoggerStreamTest.java @@ -0,0 +1,281 @@ +/* + * © SolarWinds Worldwide, LLC. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.solarwinds.joboe.logging; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; + +import java.io.BufferedReader; +import java.io.IOException; +import java.io.InputStreamReader; +import java.net.URI; +import java.net.URISyntaxException; +import java.nio.file.Files; +import java.nio.file.NoSuchFileException; +import java.nio.file.Path; +import java.nio.file.Paths; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.concurrent.TimeUnit; +import org.junit.jupiter.api.Test; + +public class FileLoggerStreamTest { + @Test + public void testBasicOperation() throws IOException { + Path logFilePath = getTestLogFilePath("basic-test.log"); + Files.deleteIfExists(logFilePath); + + // test creating new log file + try (FileLoggerStream loggerStream = new FileLoggerStream(logFilePath, 1024 * 1024, 1)) { + loggerStream.println("test1"); + } + + List lines = Files.readAllLines(logFilePath); + assertEquals(1, lines.size()); + assertEquals("test1", lines.get(0)); + + // test appending an existing log file + try (FileLoggerStream loggerStream = new FileLoggerStream(logFilePath, 1024 * 1024, 1)) { + loggerStream.println("test2"); + } + + lines = Files.readAllLines(logFilePath); + assertEquals(2, lines.size()); + assertEquals("test1", lines.get(0)); + assertEquals("test2", lines.get(1)); + + try (FileLoggerStream loggerStream = new FileLoggerStream(logFilePath, 1024 * 1024, 1)) { + loggerStream.printStackTrace(new RuntimeException("Test Exception")); + } + lines = Files.readAllLines(logFilePath); + assert (lines.get(2).contains(RuntimeException.class.getName())); + assert (lines.get(2).contains("Test Exception")); + assert (lines.get(3).contains(FileLoggerStreamTest.class.getName())); + } + + @Test + public void testFileRolling() throws IOException, InterruptedException { + Path logFilePath = getTestLogFilePath("file-rolling.log"); + Path backupPath = Paths.get(logFilePath.toString() + ".1"); + Files.deleteIfExists(logFilePath); + Files.deleteIfExists(backupPath); + + int targetSizePerCall = 10; + int stringArgumentSize = targetSizePerCall - System.lineSeparator().length(); + + // test creating new log file + try (FileLoggerStream loggerStream = new FileLoggerStream(logFilePath, 10, 1)) { + loggerStream.println( + getTestString(1, stringArgumentSize)); // 10 bytes, current log file size 0 + loggerStream.println( + getTestString(2, stringArgumentSize)); // 10 bytes, current log file size 10, no rolling + loggerStream.println( + getTestString(3, stringArgumentSize)); // 10 bytes, current log file size 20, roll + } + + List lines = Files.readAllLines(backupPath); + assertEquals(2, lines.size()); + assertEquals(getTestString(1, stringArgumentSize), lines.get(0)); + assertEquals(getTestString(2, stringArgumentSize), lines.get(1)); + + lines = Files.readAllLines(logFilePath); + assertEquals(1, lines.size()); + assertEquals(getTestString(3, stringArgumentSize), lines.get(0)); + + try (FileLoggerStream loggerStream = new FileLoggerStream(logFilePath, 10, 1)) { + // continue with the same log file and backups + loggerStream.println( + getTestString(4, stringArgumentSize)); // 10 bytes, current log file size 10, no rolling + loggerStream.println( + getTestString( + 5, stringArgumentSize)); // 10 bytes, current log file size 20, roll, the existing .1 + // backup will be replaced with the new one + } + + // there should not be a .2 backup + assertFalse(Files.exists(Paths.get(logFilePath + ".2"))); + + System.out.println(backupPath.toAbsolutePath()); + lines = Files.readAllLines(backupPath); + assertEquals(2, lines.size()); + assertEquals(getTestString(3, stringArgumentSize), lines.get(0)); + assertEquals(getTestString(4, stringArgumentSize), lines.get(1)); + + lines = Files.readAllLines(logFilePath); + assertEquals(1, lines.size()); + assertEquals(getTestString(5, stringArgumentSize), lines.get(0)); + } + + /** Test rolling by concurrent java processes */ + @Test + public void testConcurrentFileRolling() throws Exception { + String testClassDirectory = + getClass().getProtectionDomain().getCodeSource().getLocation().getPath(); + String classDirectory = + Logger.class.getProtectionDomain().getCodeSource().getLocation().getPath(); + + String classPathString; + String osName = System.getProperty("os.name"); + if ("windows".equals(osName)) { + classPathString = '"' + testClassDirectory + ';' + classDirectory + '"'; + } else { + classPathString = testClassDirectory + ':' + classDirectory; + } + + Path logFilePath = getTestLogFilePath("concurrent-test.log"); + + int maxSize = 1024 * 1024; // 1MB + int maxBackup = LogSetting.DEFAULT_FILE_MAX_BACKUP; + int iterationCount = 1024; + int processCount = 4; + int lineSize = 1024; + int printStringSize = lineSize - System.lineSeparator().length(); + + List processes = new ArrayList(); + cleanup(logFilePath); + for (int i = 0; i < processCount; i++) { + String printString = getTestString(i, printStringSize); + String[] command = + new String[] { + "java", + "-cp", + classPathString, + TestLoggerProcess.class.getName(), + String.valueOf(logFilePath.toAbsolutePath()), + String.valueOf(maxSize), + String.valueOf(maxBackup), + String.valueOf(iterationCount), + printString + }; + Process process = Runtime.getRuntime().exec(command); + System.out.println("Executing command " + Arrays.toString(command) + " process " + process); + processes.add(process); + } + + Map readProcessThreads = new HashMap<>(); + for (Process process : processes) { + final BufferedReader inputReader = + new BufferedReader(new InputStreamReader(process.getInputStream())); + final BufferedReader errorReader = + new BufferedReader(new InputStreamReader(process.getErrorStream())); + ReadProcessThread readProcessThread = + new ReadProcessThread(process.toString(), inputReader, errorReader); + readProcessThreads.put(process, readProcessThread); + readProcessThread.start(); + } + + for (Process process : processes) { + process.waitFor(); + readProcessThreads.get(process).stopReading(); + } + + int expectedLogFileCount = + (processCount * iterationCount * lineSize) / maxSize + + 1; // + 1 for the overflow as "divide by maxSize" truncates + List allLogLines = new ArrayList(); + allLogLines.addAll(Files.readAllLines(logFilePath)); + + for (int i = 1; i <= expectedLogFileCount - 1; i++) { // -1 will be the backups + try { + List lines = Files.readAllLines(Paths.get(logFilePath + "." + i)); + allLogLines.addAll(lines); + System.out.println(i + " => " + lines.size()); + } catch (NoSuchFileException e) { + // it's okay, sometimes it might fit in just expectedLogFileCount - 1 files + } + } + + for (String logLine : allLogLines) { + assertEquals(printStringSize, logLine.length()); + } + + assertEquals(processCount * iterationCount, allLogLines.size()); + for (int i = 0; i < processCount; i++) { + int matchingStringCount = 0; + String expectedLogLine = getTestString(i, printStringSize); + for (String logLine : allLogLines) { + if (expectedLogLine.equals(logLine)) { + matchingStringCount++; + } + } + assertEquals(iterationCount, matchingStringCount, "Failed at index [" + i + "]"); + } + } + + private void cleanup(Path logFilePath) throws IOException { + Files.deleteIfExists(logFilePath); + + int i = 1; + while (Files.deleteIfExists(Paths.get(logFilePath + "." + (i++)))) {} + } + + private String getTestString(int index, int targetByteSize) { + StringBuilder result = new StringBuilder(String.valueOf(index)); + while (result.length() < targetByteSize) { + result.append('x'); + } + return result.toString(); + } + + private Path getTestLogFilePath(String fileName) { + try { + return Paths.get( + URI.create( + getClass().getProtectionDomain().getCodeSource().getLocation().toURI() + fileName)); + } catch (URISyntaxException e) { + e.printStackTrace(); + return null; + } + } + + static class ReadProcessThread extends Thread { + private final BufferedReader inputReader, errorReader; + private final String prefix; + private boolean shouldRun = true; + + ReadProcessThread(String prefix, BufferedReader inputReader, BufferedReader errorReader) { + this.inputReader = inputReader; + this.errorReader = errorReader; + this.prefix = prefix; + } + + public void stopReading() { + shouldRun = false; + } + + @Override + public void run() { + while (shouldRun) { + try { + String line; + while ((line = errorReader.readLine()) != null) { + System.out.println(prefix + " : " + line); + } + while ((line = inputReader.readLine()) != null) { + System.out.println(prefix + " : " + line); + } + TimeUnit.SECONDS.sleep(1); + } catch (Exception e) { + e.printStackTrace(); + } + } + } + } +} diff --git a/libs/logging/src/test/java/com/solarwinds/joboe/logging/LoggerTest.java b/libs/logging/src/test/java/com/solarwinds/joboe/logging/LoggerTest.java new file mode 100644 index 00000000..09e0e695 --- /dev/null +++ b/libs/logging/src/test/java/com/solarwinds/joboe/logging/LoggerTest.java @@ -0,0 +1,313 @@ +/* + * © SolarWinds Worldwide, LLC. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.solarwinds.joboe.logging; + +import static org.junit.jupiter.api.Assertions.assertEquals; + +import java.io.ByteArrayOutputStream; +import java.io.PrintStream; +import java.lang.reflect.Field; +import lombok.Getter; +import org.junit.jupiter.api.Test; + +public class LoggerTest { + + @Getter + private static class ProxyLoggerStream implements LoggerStream { + private final ByteArrayOutputStream proxyOutput = new ByteArrayOutputStream(); + private final PrintStream proxyStream = new PrintStream(proxyOutput); + + @Override + public void println(String value) { + proxyStream.println(value); + } + + @Override + public void printStackTrace(Throwable throwable) { + throwable.printStackTrace(proxyStream); + } + } + + private void setProxyStreams(Logger logger, ProxyLoggerStream out, ProxyLoggerStream err) + throws Exception { + Field outStreamField = Logger.class.getDeclaredField("infoStream"); + outStreamField.setAccessible(true); + outStreamField.set(logger, out); + + Field errorStreamField = Logger.class.getDeclaredField("errorStream"); + errorStreamField.setAccessible(true); + errorStreamField.set(logger, err); + } + + @Test + public void testTrace() throws Exception { + Logger logger = new Logger(); + + logger.configure( + LoggerConfiguration.builder().logSetting(getLogSetting(Logger.Level.TRACE)).build()); + + ProxyLoggerStream proxyOutStream = new ProxyLoggerStream(); + ProxyLoggerStream proxyErrStream = new ProxyLoggerStream(); + ByteArrayOutputStream proxyOut = proxyOutStream.getProxyOutput(); + ByteArrayOutputStream proxyErr = proxyErrStream.getProxyOutput(); + setProxyStreams(logger, proxyOutStream, proxyErrStream); + + sendTestMessages(logger); + + String outMessage = proxyOut.toString(); + String errMessage = proxyErr.toString(); + + assertEquals(2, countOccurrence(outMessage, "trace-message")); + assertEquals(1, countOccurrence(outMessage, "trace-exception")); + assertEquals(2, countOccurrence(outMessage, "debug-message")); + assertEquals(1, countOccurrence(outMessage, "debug-exception")); + assertEquals(2, countOccurrence(outMessage, "info-message")); + assertEquals(1, countOccurrence(outMessage, "info-exception")); + assertEquals(2, countOccurrence(errMessage, "warn-message")); + assertEquals(1, countOccurrence(errMessage, "warn-exception")); + assertEquals(2, countOccurrence(errMessage, "error-message")); + assertEquals(1, countOccurrence(errMessage, "error-exception")); + assertEquals(2, countOccurrence(errMessage, "fatal-message")); + assertEquals(1, countOccurrence(errMessage, "fatal-exception")); + } + + @Test + public void testDebug() throws Exception { + Logger logger = new Logger(); + + logger.configure( + LoggerConfiguration.builder().logSetting(getLogSetting(Logger.Level.DEBUG)).build()); + + ProxyLoggerStream proxyOutStream = new ProxyLoggerStream(); + ProxyLoggerStream proxyErrStream = new ProxyLoggerStream(); + ByteArrayOutputStream proxyOut = proxyOutStream.getProxyOutput(); + ByteArrayOutputStream proxyErr = proxyErrStream.getProxyOutput(); + setProxyStreams(logger, proxyOutStream, proxyErrStream); + + sendTestMessages(logger); + + String outMessage = proxyOut.toString(); + String errMessage = proxyErr.toString(); + + assertEquals(0, countOccurrence(outMessage, "trace-message")); + assertEquals(0, countOccurrence(outMessage, "trace-exception")); + assertEquals(2, countOccurrence(outMessage, "debug-message")); + assertEquals(1, countOccurrence(outMessage, "debug-exception")); + assertEquals(2, countOccurrence(outMessage, "info-message")); + assertEquals(1, countOccurrence(outMessage, "info-exception")); + assertEquals(2, countOccurrence(errMessage, "warn-message")); + assertEquals(1, countOccurrence(errMessage, "warn-exception")); + assertEquals(2, countOccurrence(errMessage, "error-message")); + assertEquals(1, countOccurrence(errMessage, "error-exception")); + assertEquals(2, countOccurrence(errMessage, "fatal-message")); + assertEquals(1, countOccurrence(errMessage, "fatal-exception")); + } + + @Test + public void testInfo() throws Exception { + Logger logger = new Logger(); + + logger.configure( + LoggerConfiguration.builder().logSetting(getLogSetting(Logger.Level.INFO)).build()); + + ProxyLoggerStream proxyOutStream = new ProxyLoggerStream(); + ProxyLoggerStream proxyErrStream = new ProxyLoggerStream(); + ByteArrayOutputStream proxyOut = proxyOutStream.getProxyOutput(); + ByteArrayOutputStream proxyErr = proxyErrStream.getProxyOutput(); + setProxyStreams(logger, proxyOutStream, proxyErrStream); + + sendTestMessages(logger); + + String outMessage = proxyOut.toString(); + String errMessage = proxyErr.toString(); + + assertEquals(0, countOccurrence(outMessage, "trace-message")); + assertEquals(0, countOccurrence(outMessage, "trace-exception")); + assertEquals(0, countOccurrence(outMessage, "debug-message")); + assertEquals(0, countOccurrence(outMessage, "debug-exception")); + assertEquals(2, countOccurrence(outMessage, "info-message")); + assertEquals(1, countOccurrence(outMessage, "info-exception")); + assertEquals(2, countOccurrence(errMessage, "warn-message")); + assertEquals(1, countOccurrence(errMessage, "warn-exception")); + assertEquals(2, countOccurrence(errMessage, "error-message")); + assertEquals(1, countOccurrence(errMessage, "error-exception")); + assertEquals(2, countOccurrence(errMessage, "fatal-message")); + assertEquals(1, countOccurrence(errMessage, "fatal-exception")); + } + + @Test + public void testWarn() throws Exception { + Logger logger = new Logger(); + + logger.configure( + LoggerConfiguration.builder().logSetting(getLogSetting(Logger.Level.WARNING)).build()); + + ProxyLoggerStream proxyOutStream = new ProxyLoggerStream(); + ProxyLoggerStream proxyErrStream = new ProxyLoggerStream(); + ByteArrayOutputStream proxyOut = proxyOutStream.getProxyOutput(); + ByteArrayOutputStream proxyErr = proxyErrStream.getProxyOutput(); + setProxyStreams(logger, proxyOutStream, proxyErrStream); + + sendTestMessages(logger); + + String outMessage = proxyOut.toString(); + String errMessage = proxyErr.toString(); + + assertEquals(0, countOccurrence(outMessage, "trace-message")); + assertEquals(0, countOccurrence(outMessage, "trace-exception")); + assertEquals(0, countOccurrence(outMessage, "debug-message")); + assertEquals(0, countOccurrence(outMessage, "debug-exception")); + assertEquals(0, countOccurrence(outMessage, "info-message")); + assertEquals(0, countOccurrence(outMessage, "info-exception")); + assertEquals(2, countOccurrence(errMessage, "warn-message")); + assertEquals(1, countOccurrence(errMessage, "warn-exception")); + assertEquals(2, countOccurrence(errMessage, "error-message")); + assertEquals(1, countOccurrence(errMessage, "error-exception")); + assertEquals(2, countOccurrence(errMessage, "fatal-message")); + assertEquals(1, countOccurrence(errMessage, "fatal-exception")); + } + + @Test + public void testError() throws Exception { + Logger logger = new Logger(); + logger.configure( + LoggerConfiguration.builder().logSetting(getLogSetting(Logger.Level.ERROR)).build()); + + ProxyLoggerStream proxyOutStream = new ProxyLoggerStream(); + ProxyLoggerStream proxyErrStream = new ProxyLoggerStream(); + ByteArrayOutputStream proxyOut = proxyOutStream.getProxyOutput(); + ByteArrayOutputStream proxyErr = proxyErrStream.getProxyOutput(); + setProxyStreams(logger, proxyOutStream, proxyErrStream); + + sendTestMessages(logger); + + String outMessage = proxyOut.toString(); + String errMessage = proxyErr.toString(); + + assertEquals(0, countOccurrence(outMessage, "trace-message")); + assertEquals(0, countOccurrence(outMessage, "trace-exception")); + assertEquals(0, countOccurrence(outMessage, "debug-message")); + assertEquals(0, countOccurrence(outMessage, "debug-exception")); + assertEquals(0, countOccurrence(outMessage, "info-message")); + assertEquals(0, countOccurrence(outMessage, "info-exception")); + assertEquals(0, countOccurrence(errMessage, "warn-message")); + assertEquals(0, countOccurrence(errMessage, "warn-exception")); + assertEquals(2, countOccurrence(errMessage, "error-message")); + assertEquals(1, countOccurrence(errMessage, "error-exception")); + assertEquals(2, countOccurrence(errMessage, "fatal-message")); + assertEquals(1, countOccurrence(errMessage, "fatal-exception")); + } + + @Test + public void testFatal() throws Exception { + Logger logger = new Logger(); + logger.configure( + LoggerConfiguration.builder().logSetting(getLogSetting(Logger.Level.FATAL)).build()); + + ProxyLoggerStream proxyOutStream = new ProxyLoggerStream(); + ProxyLoggerStream proxyErrStream = new ProxyLoggerStream(); + ByteArrayOutputStream proxyOut = proxyOutStream.getProxyOutput(); + ByteArrayOutputStream proxyErr = proxyErrStream.getProxyOutput(); + setProxyStreams(logger, proxyOutStream, proxyErrStream); + + sendTestMessages(logger); + + String outMessage = proxyOut.toString(); + String errMessage = proxyErr.toString(); + + assertEquals(0, countOccurrence(outMessage, "trace-message")); + assertEquals(0, countOccurrence(outMessage, "trace-exception")); + assertEquals(0, countOccurrence(outMessage, "debug-message")); + assertEquals(0, countOccurrence(outMessage, "debug-exception")); + assertEquals(0, countOccurrence(outMessage, "info-message")); + assertEquals(0, countOccurrence(outMessage, "info-exception")); + assertEquals(0, countOccurrence(errMessage, "warn-message")); + assertEquals(0, countOccurrence(errMessage, "warn-exception")); + assertEquals(0, countOccurrence(errMessage, "error-message")); + assertEquals(0, countOccurrence(errMessage, "error-exception")); + assertEquals(2, countOccurrence(errMessage, "fatal-message")); + assertEquals(1, countOccurrence(errMessage, "fatal-exception")); + } + + @Test + public void testOff() throws Exception { + Logger logger = new Logger(); + logger.configure( + LoggerConfiguration.builder().logSetting(getLogSetting(Logger.Level.OFF)).build()); + + ProxyLoggerStream proxyOutStream = new ProxyLoggerStream(); + ProxyLoggerStream proxyErrStream = new ProxyLoggerStream(); + ByteArrayOutputStream proxyOut = proxyOutStream.getProxyOutput(); + ByteArrayOutputStream proxyErr = proxyErrStream.getProxyOutput(); + setProxyStreams(logger, proxyOutStream, proxyErrStream); + + sendTestMessages(logger); + + String outMessage = proxyOut.toString(); + String errMessage = proxyErr.toString(); + + assertEquals(0, countOccurrence(outMessage, "trace-message")); + assertEquals(0, countOccurrence(outMessage, "trace-exception")); + assertEquals(0, countOccurrence(outMessage, "debug-message")); + assertEquals(0, countOccurrence(outMessage, "debug-exception")); + assertEquals(0, countOccurrence(outMessage, "info-message")); + assertEquals(0, countOccurrence(outMessage, "info-exception")); + assertEquals(0, countOccurrence(errMessage, "warn-message")); + assertEquals(0, countOccurrence(errMessage, "warn-exception")); + assertEquals(0, countOccurrence(errMessage, "error-message")); + assertEquals(0, countOccurrence(errMessage, "error-exception")); + assertEquals(0, countOccurrence(errMessage, "fatal-message")); + assertEquals(0, countOccurrence(errMessage, "fatal-exception")); + } + + private void sendTestMessages(Logger logger) { + logger.trace("trace-message"); + logger.trace("trace-message", new Exception("trace-exception")); + + logger.debug("debug-message"); + logger.debug("debug-message", new Exception("debug-exception")); + + logger.info("info-message"); + logger.info("info-message", new Exception("info-exception")); + + logger.warn("warn-message"); + logger.warn("warn-message", new Exception("warn-exception")); + + logger.error("error-message"); + logger.error("error-message", new Exception("error-exception")); + + logger.fatal("fatal-message"); + logger.fatal("fatal-message", new Exception("fatal-exception")); + } + + private static int countOccurrence(String input, String phase) { + int occurence = 0; + + int index; + + while ((index = input.indexOf(phase)) != -1) { + occurence++; + input = input.substring(index + phase.length()); + } + + return occurence; + } + + private static LogSetting getLogSetting(Logger.Level logLevel) { + return new LogSetting(logLevel, true, true, null, null, null); + } +} diff --git a/libs/logging/src/test/java/com/solarwinds/joboe/logging/TestLoggerProcess.java b/libs/logging/src/test/java/com/solarwinds/joboe/logging/TestLoggerProcess.java new file mode 100644 index 00000000..188ad501 --- /dev/null +++ b/libs/logging/src/test/java/com/solarwinds/joboe/logging/TestLoggerProcess.java @@ -0,0 +1,47 @@ +/* + * © SolarWinds Worldwide, LLC. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.solarwinds.joboe.logging; + +import java.io.IOException; +import java.nio.file.Paths; + +public class TestLoggerProcess { + // args: logFileName, maxSize, maxBackup, iterationCount, printString + public static void main(String[] args) throws IOException, InterruptedException { + + String logFileName = args[0]; + int maxSize = Integer.parseInt(args[1]); // in bytes + int maxBackup = Integer.parseInt(args[2]); + int iterationCount = Integer.parseInt(args[3]); + String printString = args[4]; + + long start = System.currentTimeMillis(); + FileLoggerStream loggerStream = + new FileLoggerStream(Paths.get(logFileName), maxSize, maxBackup); + try { + for (int i = 0; i < iterationCount; i++) { + loggerStream.println(printString); + } + } catch (Throwable e) { + e.printStackTrace(); + } finally { + loggerStream.close(10); + } + long end = System.currentTimeMillis(); + System.out.println("Per operation : " + (end - start) * 1.0 / iterationCount); + } +} diff --git a/libs/sampling/.gitignore b/libs/sampling/.gitignore new file mode 100644 index 00000000..567609b1 --- /dev/null +++ b/libs/sampling/.gitignore @@ -0,0 +1 @@ +build/ diff --git a/libs/sampling/build.gradle.kts b/libs/sampling/build.gradle.kts new file mode 100644 index 00000000..1ac1a9a2 --- /dev/null +++ b/libs/sampling/build.gradle.kts @@ -0,0 +1,13 @@ +plugins { + id("solarwinds.java-conventions") +} + +description = "sampling" + +dependencies { + implementation(project(":libs:logging")) + compileOnly("io.opentelemetry:opentelemetry-api") + implementation("com.github.ben-manes.caffeine:caffeine") + + testImplementation("io.opentelemetry:opentelemetry-api") +} diff --git a/libs/sampling/src/main/java/com/solarwinds/joboe/sampling/BinaryUtils.java b/libs/sampling/src/main/java/com/solarwinds/joboe/sampling/BinaryUtils.java new file mode 100644 index 00000000..6b990b22 --- /dev/null +++ b/libs/sampling/src/main/java/com/solarwinds/joboe/sampling/BinaryUtils.java @@ -0,0 +1,71 @@ +/* + * © SolarWinds Worldwide, LLC. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.solarwinds.joboe.sampling; + +/** + * Utility methods for working with binary and hex data. + * + * @author Daniel Dyer + */ +public final class BinaryUtils { + // Mask for casting a byte to an int, bit-by-bit (with + // bitwise AND) with no special consideration for the sign bit. + private static final int BITWISE_BYTE_TO_INT = 0x000000FF; + + private static final char[] HEX_CHARS = { + '0', '1', '2', '3', '4', '5', '6', '7', + '8', '9', 'A', 'B', 'C', 'D', 'E', 'F' + }; + + private BinaryUtils() { + // Prevents instantiation of utility class. + } + + /** + * Take four bytes from the specified position in the specified block and convert them into a + * 32-bit int, using the big-endian convention. + * + * @param bytes The data to read from. + * @param offset The position to start reading the 4-byte int from. + * @return The 32-bit integer represented by the four bytes. + */ + public static int convertBytesToInt(byte[] bytes, int offset) { + return (BITWISE_BYTE_TO_INT & bytes[offset + 3]) + | ((BITWISE_BYTE_TO_INT & bytes[offset + 2]) << 8) + | ((BITWISE_BYTE_TO_INT & bytes[offset + 1]) << 16) + | ((BITWISE_BYTE_TO_INT & bytes[offset]) << 24); + } + + /** + * Convert an array of bytes into an array of ints. 4 bytes from the input data map to a single + * int in the output data. + * + * @param bytes The data to read from. + * @return An array of 32-bit integers constructed from the data. + * @since 1.1 + */ + public static int[] convertBytesToInts(byte[] bytes) { + if (bytes.length % 4 != 0) { + throw new IllegalArgumentException("Number of input bytes must be a multiple of 4."); + } + int[] ints = new int[bytes.length / 4]; + for (int i = 0; i < ints.length; i++) { + ints[i] = convertBytesToInt(bytes, i * 4); + } + return ints; + } +} diff --git a/libs/sampling/src/main/java/com/solarwinds/joboe/sampling/Constants.java b/libs/sampling/src/main/java/com/solarwinds/joboe/sampling/Constants.java new file mode 100644 index 00000000..263ade6d --- /dev/null +++ b/libs/sampling/src/main/java/com/solarwinds/joboe/sampling/Constants.java @@ -0,0 +1,49 @@ +/* + * © SolarWinds Worldwide, LLC. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.solarwinds.joboe.sampling; + +/** Constants used throughout jboe code (from oboe.h) */ +public class Constants { + + public static final int TASK_ID_LEN = 16, + OP_ID_LEN = 8, + MAX_METADATA_PACK_LEN = 512, + MASK_TASK_ID_LEN = 0x03, + MASK_OP_ID_LEN = 0x08, + MASK_HAS_OPTIONS = 0x04, // unused? + + // MAX_UDP_PKT_SZ = 65507, // (65535 max IP packet size - 20 IPv4 header - 8 UDP + // header) + MAX_EVENT_BUFFER_SIZE = + 512 * 1024, // 512kB. This should not be bound by UDP size anymore with SSL reporting, + // though we still want to have some limit + MAX_BACK_TRACE_TOP_LINE_COUNT = 100, + MAX_BACK_TRACE_BOTTOM_LINE_COUNT = 20, + MAX_BACK_TRACE_LINE_COUNT = MAX_BACK_TRACE_TOP_LINE_COUNT + MAX_BACK_TRACE_BOTTOM_LINE_COUNT, + XTR_UDP_PORT = 7831; + public static final String SW_W3C_KEY_PREFIX = "sw.", + XTR_ASYNC_KEY = "Async", + XTR_EDGE_KEY = SW_W3C_KEY_PREFIX + "parent_span_id", + XTR_AO_EDGE_KEY = "Edge", + XTR_THREAD_ID_KEY = "TID", + XTR_HOSTNAME_KEY = "Hostname", + XTR_METADATA_KEY = SW_W3C_KEY_PREFIX + "trace_context", + XTR_XTRACE = "X-Trace", + XTR_PROCESS_ID_KEY = "PID", + XTR_TIMESTAMP_U_KEY = "Timestamp_u", + XTR_UDP_HOST = "127.0.0.1"; +} diff --git a/libs/sampling/src/main/java/com/solarwinds/joboe/sampling/DevURandomSeedGenerator.java b/libs/sampling/src/main/java/com/solarwinds/joboe/sampling/DevURandomSeedGenerator.java new file mode 100644 index 00000000..e539c9f5 --- /dev/null +++ b/libs/sampling/src/main/java/com/solarwinds/joboe/sampling/DevURandomSeedGenerator.java @@ -0,0 +1,68 @@ +/* + * © SolarWinds Worldwide, LLC. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.solarwinds.joboe.sampling; + +import java.io.File; +import java.io.FileInputStream; +import java.io.IOException; + +/** + * Copied from com.tracelytics.ext.uncommons.maths.random.DevRandomSeedGenerator, but + * use /dev/urandom instead of /dev/random + * + * @author pluk + */ +public class DevURandomSeedGenerator implements SeedGenerator { + private static final File DEV_U_RANDOM = new File("/dev/urandom"); + + @Override + public byte[] generateSeed(int length) throws SeedException { + FileInputStream file = null; + try { + file = new FileInputStream(DEV_U_RANDOM); + byte[] randomSeed = new byte[length]; + int count = 0; + while (count < length) { + int bytesRead = file.read(randomSeed, count, length - count); + if (bytesRead == -1) { + throw new SeedException("EOF encountered reading random data."); + } + count += bytesRead; + } + return randomSeed; + } catch (IOException ex) { + throw new SeedException("Failed reading from " + DEV_U_RANDOM.getName(), ex); + } catch (SecurityException ex) { + // Might be thrown if resource access is restricted (such as in + // an applet sandbox). + throw new SeedException("SecurityManager prevented access to " + DEV_U_RANDOM.getName(), ex); + } finally { + if (file != null) { + try { + file.close(); + } catch (IOException ex) { + // Ignore. + } + } + } + } + + @Override + public String toString() { + return "/dev/urandom"; + } +} diff --git a/libs/sampling/src/main/java/com/solarwinds/joboe/sampling/HexUtils.java b/libs/sampling/src/main/java/com/solarwinds/joboe/sampling/HexUtils.java new file mode 100644 index 00000000..9f808c13 --- /dev/null +++ b/libs/sampling/src/main/java/com/solarwinds/joboe/sampling/HexUtils.java @@ -0,0 +1,53 @@ +/* + * © SolarWinds Worldwide, LLC. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.solarwinds.joboe.sampling; + +public class HexUtils { + private static final char[] hexTable = { + '0', '1', '2', '3', '4', '5', '6', '7', '8', '9', 'A', 'B', 'C', 'D', 'E', 'F' + }; + + public static String bytesToHex(byte[] bytes) { + return bytesToHex(bytes, bytes.length); + } + + public static String bytesToHex(byte[] bytes, int len) { + char[] hexChars = new char[len * 2]; + int v; + for (int j = 0; j < len; j++) { + v = bytes[j] & 0xFF; + hexChars[j * 2] = hexTable[v / 16]; + hexChars[j * 2 + 1] = hexTable[v % 16]; + } + return new String(hexChars); + } + + public static byte[] hexToBytes(String s) throws SamplingException { + int len = s.length(); + + if ((len % 2) != 0 || len > Constants.MAX_METADATA_PACK_LEN) { + throw new SamplingException("Invalid string length"); + } + + byte[] buf = new byte[len / 2]; + for (int i = 0; i < len; i += 2) { + buf[i / 2] = + (byte) ((Character.digit(s.charAt(i), 16) << 4) + Character.digit(s.charAt(i + 1), 16)); + } + return buf; + } +} diff --git a/libs/sampling/src/main/java/com/solarwinds/joboe/sampling/Metadata.java b/libs/sampling/src/main/java/com/solarwinds/joboe/sampling/Metadata.java new file mode 100644 index 00000000..078e6fa0 --- /dev/null +++ b/libs/sampling/src/main/java/com/solarwinds/joboe/sampling/Metadata.java @@ -0,0 +1,636 @@ +/* + * © SolarWinds Worldwide, LLC. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.solarwinds.joboe.sampling; + +import com.solarwinds.joboe.logging.Logger; +import com.solarwinds.joboe.logging.LoggerFactory; +import io.opentelemetry.api.trace.SpanContext; +import java.util.Arrays; +import java.util.Random; +import java.util.concurrent.atomic.AtomicInteger; +import lombok.Getter; + +/** + * Oboe Metadata: Task and Op IDs Note that this is migrated from AO's X-Trace ID and it complies + * with the W3C trace context spec: ... + */ +public class Metadata { + private static final Logger logger = LoggerFactory.getLogger(); + + @Getter private byte[] taskID; + @Getter private byte[] opID; + + private int taskLen = Constants.TASK_ID_LEN; + private int opLen = Constants.OP_ID_LEN; + public static final int METADATA_BUF_SIZE = + 1 + Constants.TASK_ID_LEN + Constants.OP_ID_LEN + 1 + 3; + public static final int METADATA_HEX_STRING_SIZE = + (1 + Constants.TASK_ID_LEN + Constants.OP_ID_LEN + 1) * 2 + 3; + + @Getter private byte flags; + + private boolean isAsync; + + public static final char[] hexTable = { + '0', '1', '2', '3', '4', '5', '6', '7', '8', '9', 'a', 'b', 'c', 'd', 'e', 'f' + }; + public static final byte[] unsetTaskID = new byte[Constants.TASK_ID_LEN]; // initialized to zero + public static final byte[] unsetOpID = new byte[Constants.OP_ID_LEN]; // initialized to zero + + private static int ttl; // in millisec + public static final int DEFAULT_TTL = 20 * 60 * 1000; // 20 mins by default, in unit of millisec + private static int maxEvents; + private static int maxBacktraces; + public static final int DEFAULT_MAX_EVENTS = 100000; // max 100k events per trace by default + public static final int DEFAULT_MAX_BACKTRACES = 1000; // max 1000 backtraces per trace by default + + public static final int CURRENT_VERSION = 0; // current W3C trace context version as of 2021 + public static final String CURRENT_VERSION_HEXSTRING = + "" + hexTable[CURRENT_VERSION >>> 4] + hexTable[CURRENT_VERSION & 0x0F]; + public static final String HEXSTRING_DELIMETER = "-"; + + private long creationTimestamp; // creation timestamp in millisec + @Getter private Long traceId; + private AtomicInteger numEvents = new AtomicInteger(); + private AtomicInteger numBacktraces = new AtomicInteger(); + @Getter private boolean reportMetrics = false; + + @Getter private static boolean initializedStatics = false; + + public Metadata() { + initialize(); + } + + public Metadata(String hexStr) throws SamplingException { + if (initializedStatics) { + initialize(); + fromHexString(hexStr); + } else { + throw new SamplingException("Must call Metadata#setup first"); + } + } + + public Metadata(SpanContext spanContext) { + initialize(); + if (spanContext.isValid()) { + System.arraycopy(spanContext.getTraceIdBytes(), 0, this.taskID, 0, Constants.TASK_ID_LEN); + System.arraycopy(spanContext.getSpanIdBytes(), 0, this.opID, 0, Constants.OP_ID_LEN); + this.flags = spanContext.getTraceFlags().asByte(); + } + } + + public Metadata(Metadata toClone) { + if (toClone.isExpired( + System.currentTimeMillis())) { // stop this source from spreading expired metadata + toClone.invalidate(); + } + + initialize(); + this.taskLen = toClone.taskLen; + this.opLen = toClone.opLen; + this.isAsync = toClone.isAsync; + this.creationTimestamp = toClone.creationTimestamp; + this.numEvents = + toClone.numEvents; // use the same instance of numEvent as we want to keep a centralized + // counter for all clones + this.numBacktraces = + toClone.numBacktraces; // use the same instance of numBacktraces as we want to keep a + // centralized counter for all clones + System.arraycopy(toClone.taskID, 0, this.taskID, 0, Constants.TASK_ID_LEN); + System.arraycopy(toClone.opID, 0, this.opID, 0, Constants.OP_ID_LEN); + this.flags = toClone.flags; + this.traceId = toClone.traceId; + this.reportMetrics = toClone.reportMetrics; + } + + public static void setup(SamplingConfiguration samplingConfiguration) { + initializedStatics = true; + ttl = samplingConfiguration.getTtl(); + maxBacktraces = samplingConfiguration.getMaxBacktraces(); + + maxEvents = samplingConfiguration.getMaxEvents(); + addTtlChangeListener(); + addMaxEventsChangeListener(); + + addMaxBacktracesChangeListener(); + } + + /** Listens to dynamic ttl change from Settings */ + private static void addTtlChangeListener() { + SettingsManager.registerListener( + new SettingsArgChangeListener(SettingsArg.MAX_CONTEXT_AGE) { + @Override + public void onChange(Integer newValue) { + if (newValue != null) { + ttl = newValue * 1000; // convert from seconds to milliseconds + } else { // reset back to default + ttl = DEFAULT_TTL; + } + } + }); + } + + private static void addMaxEventsChangeListener() { + SettingsManager.registerListener( + new SettingsArgChangeListener(SettingsArg.MAX_CONTEXT_EVENTS) { + @Override + public void onChange(Integer newValue) { + maxEvents = (newValue != null) ? newValue : DEFAULT_MAX_EVENTS; + } + }); + } + + private static void addMaxBacktracesChangeListener() { + SettingsManager.registerListener( + new SettingsArgChangeListener(SettingsArg.MAX_CONTEXT_BACKTRACES) { + @Override + public void onChange(Integer newValue) { + maxBacktraces = (newValue != null) ? newValue : DEFAULT_MAX_BACKTRACES; + } + }); + } + + /** Clears Task and Op IDs */ + public void initialize() { + taskID = new byte[Constants.TASK_ID_LEN]; + taskLen = Constants.TASK_ID_LEN; + opID = new byte[Constants.OP_ID_LEN]; + opLen = Constants.OP_ID_LEN; + flags = 0x0; + isAsync = false; + traceId = null; + numEvents.set(0); + numBacktraces.set(0); + creationTimestamp = System.currentTimeMillis(); + reportMetrics = false; + } + + /** Invalidates this metadata. For now, it will just set all the bits back to zeros */ + public void invalidate() { + initialize(); + } + + public void randomize() { + randomize(true); + } + + /** Randomizes and resets the state of this Metadata */ + public void randomize(boolean isSampled) { + initialize(); + randomizeTaskID(); + if (isSampled) { + randomizeOpID(); + } + setSampled(isSampled); + } + + public void randomizeTaskID() { + random.nextBytes(taskID); + // just in case if it really generates all zeros, then flip the last byte to a non zero value + if (!isValid()) { + taskID[taskLen - 1] = 0x1; + } + } + + public void randomizeOpID() { + random.nextBytes(opID); + } + + public void setOpID(Metadata md) { + this.opLen = md.opLen; + setOpID(md.opID); + } + + public void setOpID(String opId) throws SamplingException { + setOpID(hexToBytes(opId)); + } + + public void setOpID(byte[] opId) { + System.arraycopy(opId, 0, this.opID, 0, Constants.OP_ID_LEN); + } + + /** + * Whether the metadata has a valid task id - the operation has gone through a valid entry point + * (might or might not be sampled) + * + * @return + */ + public boolean isValid() { + return !Arrays.equals(taskID, unsetTaskID); + } + + /** + * Whether the metadata has SAMPLED flag turned on - the operation is sampled to generate tracing + * events + * + * @return + */ + public boolean isSampled() { + return getFlag(Flag.SAMPLED); + } + + public boolean getFlag(Flag flag) { + return (flags & flag.mask) == flag.mask; + } + + public void setSampled(boolean sampled) { + setFlag(Flag.SAMPLED, sampled); + } + + public void setFlag(Flag flag, boolean value) { + if (value) { + flags |= flag.mask; + } else { + flags &= (~flag.mask); + } + } + + /** + * Increase the event counter tracking the number of events for this trace by 1 and return whether + * the counter is within valid limits after the increment + * + *

The event counter persists across copies of this Metadata (used in the same trace). + * + * @return whether the counter is within valid limits after the increment + */ + public boolean incrNumEvents() { + int currentCount = numEvents.incrementAndGet(); + if (currentCount == maxEvents + 1) { // only report it once on first limit exceeded + logger.info( + "Exceeded maximum number of events allowed per trace [" + + maxEvents + + "] for task ID [" + + taskHexString() + + "]"); + } + return currentCount <= maxEvents; + } + + /** + * Increase the backtrace counter tracking the number of backtraces for this trace by 1 and return + * whether the counter is within valid limits after the increment + * + *

The event counter persists across copies of this Metadata (used in the same trace). + * + * @return whether the counter is within valid limits after the increment + */ + public boolean incrNumBacktraces() { + int currentCount = numBacktraces.incrementAndGet(); + if (currentCount == maxBacktraces + 1) { // only report it once on first limit exceeded + logger.info( + "Exceeded maximum number of backtraces allowed per trace [" + + maxBacktraces + + "] for task ID [" + + taskHexString() + + "]"); + } + return currentCount <= maxBacktraces; + } + + /** + * @param reportingTimestamp the timestamp to be reported in millisec + * @return + */ + public boolean isExpired(long reportingTimestamp) { + if (isValid()) { + boolean expired = reportingTimestamp - creationTimestamp > ttl; + if (expired) { + logger.info("Context of " + toHexString() + " has been expired"); + } + return expired; + } else { // default invalid context never expires + return false; + } + } + + public boolean isTaskEqual(Metadata md) { + return Arrays.equals(taskID, md.taskID); + } + + public boolean isOpEqual(Metadata md) { + return Arrays.equals(opID, md.opID); + } + + /** + * Packs metadata into byte buffer. Note that the parameter `version` is currently used for + * testing only. + * + * @param version + * @return + */ + public byte[] getPackedMetadata(int version) { + byte[] buf = new byte[METADATA_BUF_SIZE]; + int writeMarker = 0; + + // Header with version and lengths: + buf[writeMarker++] = (byte) version; + + // Task and Op ID data: + buf[writeMarker++] = '-'; + + System.arraycopy(taskID, 0, buf, writeMarker, taskLen); + writeMarker += taskLen; + + buf[writeMarker++] = '-'; + + System.arraycopy(opID, 0, buf, writeMarker, opLen); + writeMarker += opLen; + + buf[writeMarker++] = '-'; + + buf[writeMarker] = flags; + + return buf; + } + + /** Populates this object from packed metadata contained in the byte buffer */ + public void unpackMetadata(byte[] buf) throws SamplingException { + + if (buf == null || buf.length < 1) { + throw new SamplingException("Byte buffer is not valid"); + } + + int version = buf[0]; + if (version != CURRENT_VERSION) { + throw new SamplingException( + "Unexpected version. Found " + version + " but expected " + CURRENT_VERSION); + } + + int expectedLen = + 1 + + Constants.TASK_ID_LEN + + Constants.OP_ID_LEN + + 1 + + 3; // header + task id + op id + flags + delimiters + if (buf.length < expectedLen) { + throw new SamplingException("Invalid buffer length: expected " + expectedLen); + } + + int readMarker = 2; // the version and the delimiter + System.arraycopy(buf, readMarker, taskID, 0, taskLen); + readMarker += taskLen; + readMarker++; // the '-' delimiter + System.arraycopy(buf, readMarker, opID, 0, opLen); + readMarker += opLen; + readMarker++; // the '-' delimiter + this.flags = buf[readMarker]; + + this.creationTimestamp = + System.currentTimeMillis(); // a new task id, consider this a new metadata + } + + /** Returns hex representation of this metadata instance */ + public String toHexString() { + return bytesToHex(getPackedMetadata(CURRENT_VERSION)); + } + + @Override + public String toString() { + return toHexString(); + } + + /** + * Gets a hex string representation by setting an explicit version number. For internal use only + * + * @param versionOverride + * @return + */ + public String toHexString(int versionOverride) { + return bytesToHex(getPackedMetadata(versionOverride)); + } + + public String opHexString() { + return bytesToHex(opID, opLen); + } + + public String taskHexString() { + return bytesToHex(taskID, taskLen); + } + + /** Populates this metadata instance from a hex string */ + public void fromHexString(String s) throws SamplingException { + unpackMetadata(hexToBytes(s)); + } + + @Override + public int hashCode() { + final int prime = 31; + int result = 1; + result = prime * result + flags; + result = prime * result + (isAsync ? 1231 : 1237); + result = prime * result + Arrays.hashCode(opID); + result = prime * result + opLen; + result = prime * result + Arrays.hashCode(taskID); + result = prime * result + taskLen; + return result; + } + + @Override + public boolean equals(Object obj) { + if (this == obj) return true; + if (obj == null) return false; + if (getClass() != obj.getClass()) return false; + Metadata other = (Metadata) obj; + if (flags != other.flags) return false; + if (isAsync != other.isAsync) return false; + if (!Arrays.equals(opID, other.opID)) return false; + if (opLen != other.opLen) return false; + if (!Arrays.equals(taskID, other.taskID)) return false; + return taskLen == other.taskLen; + } + + public static String bytesToHex(byte[] bytes) { + return bytesToHex(bytes, bytes.length); + } + + /** + * This method checks if the position is one of the delimiter positions in a W3C trace context. + * + * @param len + * @param index + * @return + */ + private static boolean isW3CDelimiterPos(int len, int index) { + if (len != METADATA_BUF_SIZE) { + return false; + } + return index == 1 || index == 18 || index == 27; + } + + private static String bytesToHex(byte[] bytes, int len) { + StringBuilder sb = new StringBuilder(); + int v; + for (int i = 0; i < len; i++) { + v = bytes[i] & 0xFF; + + if (isW3CDelimiterPos(len, i) && v == '-') { + sb.append('-'); + continue; + } + sb.append(hexTable[v >>> 4]); + sb.append(hexTable[v & 0x0F]); + } + return sb.toString(); + } + + // TODO + private byte[] hexToBytes(String s) throws SamplingException { + int len = s.length(); + + if (len > METADATA_HEX_STRING_SIZE) { + throw new SamplingException("Invalid string length"); + } + + byte[] buf = new byte[METADATA_BUF_SIZE]; + for (int i = 0, j = 0; i < len; j++) { + if (s.charAt(i) == '-') { + buf[j] = '-'; + i++; + continue; + } + buf[j] = + (byte) + ((Character.digit(s.charAt(i), 16) << 4) + + (i + 1 < len ? Character.digit(s.charAt(i + 1), 16) : 0)); + i += 2; + } + return buf; + } + + public boolean isAsync() { + return isAsync; + } + + public void setIsAsync(boolean isAsync) { + this.isAsync = isAsync; + } + + public void setTraceId(Long traceId) { + this.traceId = traceId; + } + + public void setReportMetrics(boolean reportMetrics) { + this.reportMetrics = reportMetrics; + } + + /** + * Checks if the xTraceId is compatible with this current agent + * + * @param xTraceId + * @return + * @exception NullPointerException if xTraceId is null + */ + public static boolean isCompatible(String xTraceId) { + try { + new Metadata(xTraceId); + return true; + } catch (SamplingException e) { + logger.debug("X-Trace ID [" + xTraceId + "] not compatible : " + e.getMessage()); + return false; + } + } + + /** + * asyncLayerLevel is used to track layerLevel for Metadata that is marked as Async. + * + *

This is used to determine whether the current level of an extent within an async stack + * (indicated by the isAsync flag of the Metadata instance) is a top level extent, such that it's + * eligible to be flagged as async. + * + *

More details at + * + *

Take note that this approach would not work if the entry and exit events do not use the same + * Metadata instance (for example exit event restores the context using plain string that some + * instrumentation on asynchronous constructs that have entry and exit events are on different + * threads). + * + *

This is considered rare cases as even though we are trying to address Async flag here, this + * is not to be confused with the async constructed itself. + * + *

The async flag here addresses operations that runs on a separate thread (hence asynchronous + * relative to the thread that spawns it), but the operations themselves start and end on the same + * thread usually. (as opposed to those async construct whose entry and exit events are on + * different threads) + * + *

Take note that even if this flag is off balance (due to concerns above), it will have rather + * mild impact as: + * + *

    + *
  1. This only affect whether we flag a top level extent in the async call stack as + * asynchronous or not + *
  2. In multi-threaded trace, the metadata usually gets cloned in forked extents, so incorrect + * asyncLayerLevel should not pollute other threads nor traces + *
+ */ + private int asyncLayerLevel = 0; + + public int incrementAndGetAsyncLayerLevel() { + return ++asyncLayerLevel; + } + + public int decrementAndGetAsyncLayerLevel() { + return --asyncLayerLevel; + } + + static int getMaxEvents() { + return maxEvents; + } + + static int getTtl() { + return ttl; + } + + static int getMaxBacktraces() { + return maxBacktraces; + } + + // Shared random number generator to avoid overhead: + private static Random random = null; + + static { + // This RNG implementation is MUCH (over 10x) faster than Java's built-in SecureRandom + // See http://maths.uncommons.org/ + try { + try { + // try /dev/urandom first, as using SecureRandomSeedGenerator could trigger slow startup due + // to /dev/random blocking (entropy exhaustion) + random = new XorShiftRNG(new DevURandomSeedGenerator()); + } catch (SeedException e) { + logger.debug( + "Failed to use /dev/urandom as seed generator. Error message : " + e.getMessage()); + // try using the SecureRandomSeedGenerator instead + random = new XorShiftRNG(new SecureRandomSeedGenerator()); + } + } catch (Exception ex) { + logger.error(ex.getMessage(), ex); // should never happen + } + } + + public String getCompactTraceId() { + return taskHexString() + "-" + (isSampled() ? "1" : "0"); + } + + private enum Flag { + SAMPLED((byte) 0x1); + + private final byte mask; + + Flag(byte mask) { + this.mask = mask; + } + } +} diff --git a/libs/sampling/src/main/java/com/solarwinds/joboe/sampling/ResourceMatcher.java b/libs/sampling/src/main/java/com/solarwinds/joboe/sampling/ResourceMatcher.java new file mode 100644 index 00000000..0dee49a5 --- /dev/null +++ b/libs/sampling/src/main/java/com/solarwinds/joboe/sampling/ResourceMatcher.java @@ -0,0 +1,21 @@ +/* + * © SolarWinds Worldwide, LLC. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.solarwinds.joboe.sampling; + +public interface ResourceMatcher { + boolean matches(String signal); +} diff --git a/libs/sampling/src/main/java/com/solarwinds/joboe/sampling/SampleRateSource.java b/libs/sampling/src/main/java/com/solarwinds/joboe/sampling/SampleRateSource.java new file mode 100644 index 00000000..07973245 --- /dev/null +++ b/libs/sampling/src/main/java/com/solarwinds/joboe/sampling/SampleRateSource.java @@ -0,0 +1,42 @@ +/* + * © SolarWinds Worldwide, LLC. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.solarwinds.joboe.sampling; + +/** + * Matches oboe C library OBOE_SAMPLE_RATE_SOURCE values + * + *

See oboe_inst_macros.h + */ +public enum SampleRateSource { + FILE(1), // locally configured rate, could be from file (agent.sampleRate or url patterns) or JVM + // args + DEFAULT(2), + OBOE(3), + LAST_OBOE(4), + DEFAULT_MISCONFIGURED(5), + OBOE_DEFAULT(6); + + private final int value; + + SampleRateSource(int value) { + this.value = value; + } + + public int value() { + return value; + } +} diff --git a/libs/sampling/src/main/java/com/solarwinds/joboe/sampling/SamplingConfiguration.java b/libs/sampling/src/main/java/com/solarwinds/joboe/sampling/SamplingConfiguration.java new file mode 100644 index 00000000..eb778049 --- /dev/null +++ b/libs/sampling/src/main/java/com/solarwinds/joboe/sampling/SamplingConfiguration.java @@ -0,0 +1,38 @@ +/* + * © SolarWinds Worldwide, LLC. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.solarwinds.joboe.sampling; + +import lombok.Builder; +import lombok.Value; + +@Value +@Builder +public class SamplingConfiguration { + @Builder.Default int ttl = 1_200_000; // 20 minutes by default, in unit of millisecond; + + @Builder.Default int maxEvents = 100_000; // max 100k events per trace by default + + @Builder.Default int maxBacktraces = 1000; // max 1000 backtraces per trace by default + + @Builder.Default boolean triggerTraceEnabled = true; + + Integer sampleRate; + + TracingMode tracingMode; + + TraceConfigs internalTransactionSettings; +} diff --git a/libs/sampling/src/main/java/com/solarwinds/joboe/sampling/SamplingException.java b/libs/sampling/src/main/java/com/solarwinds/joboe/sampling/SamplingException.java new file mode 100644 index 00000000..020515a3 --- /dev/null +++ b/libs/sampling/src/main/java/com/solarwinds/joboe/sampling/SamplingException.java @@ -0,0 +1,25 @@ +/* + * © SolarWinds Worldwide, LLC. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.solarwinds.joboe.sampling; + +public class SamplingException extends Exception { + private static final long serialVersionUID = 1L; + + public SamplingException(String message) { + super(message); + } +} diff --git a/libs/sampling/src/main/java/com/solarwinds/joboe/sampling/SecureRandomSeedGenerator.java b/libs/sampling/src/main/java/com/solarwinds/joboe/sampling/SecureRandomSeedGenerator.java new file mode 100644 index 00000000..90be8539 --- /dev/null +++ b/libs/sampling/src/main/java/com/solarwinds/joboe/sampling/SecureRandomSeedGenerator.java @@ -0,0 +1,47 @@ +/* + * © SolarWinds Worldwide, LLC. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.solarwinds.joboe.sampling; + +import java.security.SecureRandom; + +/** + * {@link SeedGenerator} implementation that uses Java's bundled {@link SecureRandom} RNG to + * generate random seed data. + * + *

The advantage of using SecureRandom for seeding but not as the primary RNG is that we can use + * it to seed RNGs that are much faster than SecureRandom. + * + *

This is the only seeding strategy that is guaranteed to work on all platforms and therefore is + * provided as a fall-back option should none of the other provided {@link SeedGenerator} + * implementations be useable. + * + * @author Daniel Dyer + */ +public class SecureRandomSeedGenerator implements SeedGenerator { + private static final SecureRandom SOURCE = new SecureRandom(); + + /** {@inheritDoc} */ + @Override + public byte[] generateSeed(int length) throws SeedException { + return SOURCE.generateSeed(length); + } + + @Override + public String toString() { + return "java.security.SecureRandom"; + } +} diff --git a/libs/sampling/src/main/java/com/solarwinds/joboe/sampling/SeedException.java b/libs/sampling/src/main/java/com/solarwinds/joboe/sampling/SeedException.java new file mode 100644 index 00000000..5e3f7218 --- /dev/null +++ b/libs/sampling/src/main/java/com/solarwinds/joboe/sampling/SeedException.java @@ -0,0 +1,42 @@ +/* + * © SolarWinds Worldwide, LLC. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.solarwinds.joboe.sampling; + +/** + * Exception thrown by {@link SeedGenerator} implementations when they are unable to generate a new + * seed for an RNG. + * + * @author Daniel Dyer + */ +public class SeedException extends Exception { + private static final long serialVersionUID = 1L; + + /** + * @param message Details of the problem. + */ + public SeedException(String message) { + super(message); + } + + /** + * @param message Details of the problem. + * @param cause The root cause of the problem. + */ + public SeedException(String message, Throwable cause) { + super(message, cause); + } +} diff --git a/libs/sampling/src/main/java/com/solarwinds/joboe/sampling/SeedGenerator.java b/libs/sampling/src/main/java/com/solarwinds/joboe/sampling/SeedGenerator.java new file mode 100644 index 00000000..1dee9e41 --- /dev/null +++ b/libs/sampling/src/main/java/com/solarwinds/joboe/sampling/SeedGenerator.java @@ -0,0 +1,33 @@ +/* + * © SolarWinds Worldwide, LLC. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.solarwinds.joboe.sampling; + +/** + * Strategy interface for seeding random number generators. + * + * @author Daniel Dyer + */ +public interface SeedGenerator { + /** + * Generate a seed value for a random number generator. + * + * @param length The length of the seed to generate (in bytes). + * @return A byte array containing the seed data. + * @throws SeedException If a seed cannot be generated for any reason. + */ + byte[] generateSeed(int length) throws SeedException; +} diff --git a/libs/sampling/src/main/java/com/solarwinds/joboe/sampling/Settings.java b/libs/sampling/src/main/java/com/solarwinds/joboe/sampling/Settings.java new file mode 100644 index 00000000..32c3ca00 --- /dev/null +++ b/libs/sampling/src/main/java/com/solarwinds/joboe/sampling/Settings.java @@ -0,0 +1,95 @@ +/* + * © SolarWinds Worldwide, LLC. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.solarwinds.joboe.sampling; + +/** + * Settings that contains information such as sample rate, tracing mode and other arguments + * + * @author pluk + */ +public abstract class Settings { + public static final short OBOE_SETTINGS_FLAG_INVALID = 0x1, + OBOE_SETTINGS_FLAG_OVERRIDE = 0x2, + OBOE_SETTINGS_FLAG_SAMPLE_START = 0x4, + OBOE_SETTINGS_FLAG_SAMPLE_THROUGH = 0x8, + OBOE_SETTINGS_FLAG_SAMPLE_THROUGH_ALWAYS = 0x10, + OBOE_SETTINGS_FLAG_TRIGGER_TRACE_ENABLED = 0x20, + OBOE_SETTINGS_FLAG_SAMPLE_BUCKET_ENABLED = + 0x40; // NOT USED This flag is to indicates whether the args position in settings contains + // valid bucket rate and bucket capacity in order to avoid errors reading old + // settings. It does not directly control whether token bucket check should be + // enforced + public static final short OBOE_SETTINGS_TYPE_SKIP = 0, + OBOE_SETTINGS_TYPE_STOP = 1, + OBOE_SETTINGS_TYPE_DEFAULT_SAMPLE_RATE = 2, + OBOE_SETTINGS_TYPE_LAYER_SAMPLE_RATE = 3, + OBOE_SETTINGS_TYPE_LAYER_APP_SAMPLE_RATE = 4, // not used + OBOE_SETTINGS_TYPE_LAYER_HTTPHOST_SAMPLE_RATE = 5; + + /** + * Returns value, or null if value is invalid (indicating refresh is required.) + * + * @return + */ + public abstract long getValue(); + + public abstract long getTimestamp(); + + public abstract short getType(); + + public abstract short getFlags(); + + public abstract long getTtl(); + + public abstract T getArgValue(SettingsArg arg); + + public final boolean isDefault() { + return (getType() == OBOE_SETTINGS_TYPE_DEFAULT_SAMPLE_RATE); + } + + @Override + public String toString() { + return "[Settings: timestamp=" + + getTimestamp() + + " type=" + + getType() + + " flags=" + + getFlags() + + " value=" + + getValue() + + " ttl=" + + getTtl() + + " args=" + + getArgsString() + + " ]"; + } + + private String getArgsString() { + StringBuilder builder = new StringBuilder(); + for (SettingsArg arg : SettingsArg.values()) { + Object value = getArgValue(arg); + if (value != null) { + if (arg == SettingsArg.TRACE_OPTIONS_SECRET) { + builder.append(arg.getKey()).append("=, "); + } else { + builder.append(arg.getKey()).append("=").append(value).append(", "); + } + } + } + return builder.toString(); + } +} diff --git a/libs/sampling/src/main/java/com/solarwinds/joboe/sampling/SettingsArg.java b/libs/sampling/src/main/java/com/solarwinds/joboe/sampling/SettingsArg.java new file mode 100644 index 00000000..abfaa860 --- /dev/null +++ b/libs/sampling/src/main/java/com/solarwinds/joboe/sampling/SettingsArg.java @@ -0,0 +1,341 @@ +/* + * © SolarWinds Worldwide, LLC. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.solarwinds.joboe.sampling; + +import com.solarwinds.joboe.logging.Logger; +import com.solarwinds.joboe.logging.LoggerFactory; +import java.nio.BufferUnderflowException; +import java.nio.ByteBuffer; +import java.nio.ByteOrder; +import java.util.Arrays; +import java.util.Collection; +import java.util.HashMap; +import java.util.Map; +import lombok.Getter; + +/** + * Setting arguments used in {@link Settings} + * + * @author pluk + * @param the corresponding type of the argument value + */ +public abstract class SettingsArg { + private static final Map> keyToArgs = + new HashMap>(); + protected Logger logger = LoggerFactory.getLogger(); + + public static final SettingsArg BUCKET_CAPACITY = new DoubleSettingsArg("BucketCapacity"); + public static final SettingsArg BUCKET_RATE = new DoubleSettingsArg("BucketRate"); + public static final SettingsArg METRIC_FLUSH_INTERVAL = + new IntegerSettingsArg("MetricsFlushInterval"); + public static final SettingsArg MAX_TRANSACTIONS = + new IntegerSettingsArg("MaxTransactions"); + public static final SettingsArg MAX_CONTEXT_AGE = + new IntegerSettingsArg("MaxContextAge"); + public static final SettingsArg MAX_CONTEXT_EVENTS = + new IntegerSettingsArg("MaxContextEvents"); + public static final SettingsArg MAX_CONTEXT_BACKTRACES = + new IntegerSettingsArg("MaxContextBacktraces"); + public static final SettingsArg DISABLE_INHERIT_CONTEXT = + new BooleanSettingsArg("DisableInheritContext"); + public static final SettingsArg MAX_CUSTOM_METRICS = + new IntegerSettingsArg("MaxCustomMetrics"); + public static final SettingsArg EVENTS_FLUSH_INTERVAL = + new IntegerSettingsArg("EventsFlushInterval"); + public static final SettingsArg PROFILING_INTERVAL = + new IntegerSettingsArg("ProfilingInterval"); + public static final SettingsArg TRACE_OPTIONS_SECRET = + new ByteArraySettingsArg("SignatureKey"); + public static final SettingsArg RELAXED_BUCKET_CAPACITY = + new DoubleSettingsArg("TriggerRelaxedBucketCapacity"); + public static final SettingsArg RELAXED_BUCKET_RATE = + new DoubleSettingsArg("TriggerRelaxedBucketRate"); + public static final SettingsArg STRICT_BUCKET_CAPACITY = + new DoubleSettingsArg("TriggerStrictBucketCapacity"); + public static final SettingsArg STRICT_BUCKET_RATE = + new DoubleSettingsArg("TriggerStrictBucketRate"); + + /** + * -- GETTER -- Gets the string key of this SettingsArg + * + * @return + */ + @Getter protected final String key; + + private SettingsArg(String key) { + this.key = key; + keyToArgs.put(key, this); + } + + /** + * Reads byteBuffer and returns the converted argument value + * + * @param byteBuffer + * @return + */ + public abstract T readValue(ByteBuffer byteBuffer); + + /** Converts the given {@link Object} to T */ + public abstract T readValue(Object object); + + /** + * Reads the typed value of this SettingsArg and convert it to ByteBuffer + * + * @param fromValue + * @return + */ + public abstract ByteBuffer toByteBuffer(T fromValue); + + /** + * Checks if the values of this settings arg type are considered equal. By default uses the equals + * method. + * + * @param value1 + * @param value2 + * @return + */ + public boolean areValuesEqual(T value1, T value2) { + if (value1 == value2) { + return true; + } else if (value1 == null) { + return false; + } else { + return value1.equals(value2); + } + } + + /** + * Gets the SettingsArg corresponds to this key + * + * @param key + * @return + */ + public static SettingsArg fromKey(String key) { + return keyToArgs.get(key); + } + + /** + * Gets all the available (instantiated so far) SettingsArg instances + * + * @return + */ + public static Collection> values() { + return keyToArgs.values(); + } + + @Override + public String toString() { + return "SettingsArg [key=" + key + "]"; + } + + public static class DoubleSettingsArg extends SettingsArg { + DoubleSettingsArg(String key) { + super(key); + } + + @Override + public Double readValue(ByteBuffer byteBuffer) { + try { + return byteBuffer.order(ByteOrder.LITTLE_ENDIAN).getDouble(); + } catch (BufferUnderflowException e) { + logger.warn( + "Cannot find valid value for arg [" + + key + + "] from settings : " + + e.getClass().getName()); + return null; + } finally { + byteBuffer.rewind(); // cast for JDK 8- runtime compatibility + } + } + + @Override + public Double readValue(Object object) { + if (object instanceof Integer) { + return ((Integer) object).doubleValue(); + + } else if (object instanceof Double) { + return (Double) object; + } + + return null; + } + + @Override + public ByteBuffer toByteBuffer(Double value) { + if (value != null) { + ByteBuffer buffer = ByteBuffer.allocate(8).order(ByteOrder.LITTLE_ENDIAN); + buffer.putDouble(value); + + buffer.rewind(); + + return buffer; + } else { + return null; + } + } + } + + public static class IntegerSettingsArg extends SettingsArg { + IntegerSettingsArg(String key) { + super(key); + } + + @Override + public Integer readValue(ByteBuffer byteBuffer) { + try { + return byteBuffer.order(ByteOrder.LITTLE_ENDIAN).getInt(); + } catch (BufferUnderflowException e) { + logger.warn( + "Cannot find valid value for arg [" + + key + + "] from settings : " + + e.getClass().getName()); + return null; + } finally { + byteBuffer.rewind(); // cast for JDK 8- runtime compatibility + } + } + + @Override + public Integer readValue(Object object) { + if (object instanceof Double) { + return ((Double) object).intValue(); + + } else if (object instanceof Integer) { + return (Integer) object; + } + + return null; + } + + @Override + public ByteBuffer toByteBuffer(Integer value) { + if (value != null) { + ByteBuffer buffer = ByteBuffer.allocate(4).order(ByteOrder.LITTLE_ENDIAN); + buffer.putInt(value); + + buffer.rewind(); + + return buffer; + } else { + return null; + } + } + } + + public static class BooleanSettingsArg extends SettingsArg { + BooleanSettingsArg(String key) { + super(key); + } + + @Override + public Boolean readValue(ByteBuffer byteBuffer) { + try { + int value = byteBuffer.order(ByteOrder.LITTLE_ENDIAN).getInt(); + return value != 0; // any non-zero is considered as True + } catch (BufferUnderflowException e) { + logger.warn( + "Cannot find valid value for arg [" + + key + + "] from settings : " + + e.getClass().getName()); + return null; + } finally { + byteBuffer.rewind(); // cast for JDK 8- runtime compatibility + } + } + + @Override + public Boolean readValue(Object object) { + if (object instanceof Boolean) { + return (Boolean) object; + } + + return null; + } + + @Override + public ByteBuffer toByteBuffer(Boolean value) { + if (value != null) { + ByteBuffer buffer = ByteBuffer.allocate(4).order(ByteOrder.LITTLE_ENDIAN); + buffer.putInt(value ? 1 : 0); + + buffer.rewind(); + + return buffer; + } else { + return null; + } + } + } + + public static class ByteArraySettingsArg extends SettingsArg { + ByteArraySettingsArg(String key) { + super(key); + } + + @Override + public byte[] readValue(ByteBuffer byteBuffer) { + try { + byte[] value = new byte[byteBuffer.remaining()]; + byteBuffer.order(ByteOrder.LITTLE_ENDIAN).get(value); + return value; + } catch (BufferUnderflowException e) { + logger.warn( + "Cannot find valid value for arg [" + + key + + "] from settings : " + + e.getClass().getName()); + return null; + } finally { + byteBuffer.rewind(); + } + } + + @Override + public byte[] readValue(Object object) { + if (object instanceof byte[]) { + return (byte[]) object; + + } else if (object instanceof String) { + return ((String) object).getBytes(); + } + + return null; + } + + @Override + public ByteBuffer toByteBuffer(byte[] value) { + if (value != null) { + ByteBuffer buffer = ByteBuffer.allocate(value.length).order(ByteOrder.LITTLE_ENDIAN); + buffer.put(value); + + buffer.rewind(); + + return buffer; + } else { + return null; + } + } + + @Override + public boolean areValuesEqual(byte[] value1, byte[] value2) { + return Arrays.equals(value1, value2); + } + } +} diff --git a/libs/sampling/src/main/java/com/solarwinds/joboe/sampling/SettingsArgChangeListener.java b/libs/sampling/src/main/java/com/solarwinds/joboe/sampling/SettingsArgChangeListener.java new file mode 100644 index 00000000..e857ad05 --- /dev/null +++ b/libs/sampling/src/main/java/com/solarwinds/joboe/sampling/SettingsArgChangeListener.java @@ -0,0 +1,57 @@ +/* + * © SolarWinds Worldwide, LLC. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.solarwinds.joboe.sampling; + +import lombok.Getter; + +/** + * Listens to change of {@link SettingsArg} from {@link Settings} + * + *

This only gets notified when the value has been changed. + * + *

Take note that changing from null value to non-null and vice versa are considered as change + * too + * + * @author pluk + * @param + */ +@Getter +public abstract class SettingsArgChangeListener { + private final SettingsArg type; + private T lastValue; + + public SettingsArgChangeListener(SettingsArg type) { + this.type = type; + } + + public final void onValue(T value) { + boolean changed; + + if (lastValue != null) { + changed = !type.areValuesEqual(lastValue, value); + } else { + changed = (value != null); + } + + if (changed) { + lastValue = value; + onChange(value); + } + } + + protected abstract void onChange(T newValue); +} diff --git a/libs/sampling/src/main/java/com/solarwinds/joboe/sampling/SettingsFetcher.java b/libs/sampling/src/main/java/com/solarwinds/joboe/sampling/SettingsFetcher.java new file mode 100644 index 00000000..f78a51cf --- /dev/null +++ b/libs/sampling/src/main/java/com/solarwinds/joboe/sampling/SettingsFetcher.java @@ -0,0 +1,48 @@ +/* + * © SolarWinds Worldwide, LLC. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.solarwinds.joboe.sampling; + +import java.util.concurrent.CountDownLatch; + +/** + * Fetches {@link Settings}, fetching strategy based on concrete implementation + * + * Provides Settings via 2 mechanisms: + *

    + *
  1. Direct retrieval from getSettings
    2. + *
  2. Notify subscribing {@link SettingsListener} from registerListener, + * the fetcher implementation determines whenever notification should be sent to subscribing SettingsListener + *
    3. + *
+ * + * Take note that other logics should inquire about Settings via {@link SettingsManager} instead of this fetcher directly + * + * @author pluk + * + */ +public interface SettingsFetcher { + String DEFAULT_LAYER = ""; + + // Settings getSettings(String serviceName); + Settings getSettings(); + + void registerListener(SettingsListener listener); + + CountDownLatch isSettingsAvailableLatch(); + + void close(); +} diff --git a/libs/sampling/src/main/java/com/solarwinds/joboe/sampling/SettingsListener.java b/libs/sampling/src/main/java/com/solarwinds/joboe/sampling/SettingsListener.java new file mode 100644 index 00000000..4f0e660d --- /dev/null +++ b/libs/sampling/src/main/java/com/solarwinds/joboe/sampling/SettingsListener.java @@ -0,0 +1,26 @@ +/* + * © SolarWinds Worldwide, LLC. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.solarwinds.joboe.sampling; + +/** + * Listens to notification of {@link Settings} from {@link SettingsFetcher} + * + * @author pluk + */ +public interface SettingsListener { + void onSettingsRetrieved(Settings newSettings); +} diff --git a/libs/sampling/src/main/java/com/solarwinds/joboe/sampling/SettingsManager.java b/libs/sampling/src/main/java/com/solarwinds/joboe/sampling/SettingsManager.java new file mode 100644 index 00000000..30729f7d --- /dev/null +++ b/libs/sampling/src/main/java/com/solarwinds/joboe/sampling/SettingsManager.java @@ -0,0 +1,155 @@ +/* + * © SolarWinds Worldwide, LLC. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.solarwinds.joboe.sampling; + +import com.solarwinds.joboe.logging.Logger; +import com.solarwinds.joboe.logging.LoggerFactory; +import java.util.HashSet; +import java.util.Map; +import java.util.Map.Entry; +import java.util.Set; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.TimeUnit; +import lombok.Getter; + +/** + * Manages {@link Settings} of per jvm process. All Settings should be retrieved via + * this manager. initialize or initializeFetcher must be invoked before + * this manager returns any valid Settings Take note that there are 2 ways to inquire + * about {@link SettingsArg} of Settings from this manager + * + *
    + *
  1. Direct inquiry from getSettings and extract SettingsArg from the + * result + *
  2. Subscribe a {@link SettingsArgChangeListener} to this manager + *
+ * + * @author pluk + */ +@Getter +public final class SettingsManager { + private static SettingsFetcher fetcher; + private static final Map, Set>> listeners = + new ConcurrentHashMap, Set>>(); + private static final Logger logger = LoggerFactory.getLogger(); + + @Getter + private static SamplingConfiguration samplingConfiguration = + SamplingConfiguration.builder().build(); + + public static CountDownLatch initialize( + SettingsFetcher fetcher, SamplingConfiguration samplingConfiguration) { + initializeFetcher(fetcher); + Metadata.setup(samplingConfiguration); + SettingsManager.samplingConfiguration = samplingConfiguration; + + return fetcher.isSettingsAvailableLatch(); + } + + /** + * Initializes this manager with a provided {@link SettingsFetcher}. Direct call to this is only + * for internal tests + */ + public static void initializeFetcher(SettingsFetcher fetcher) { + SettingsManager.fetcher = fetcher; + fetcher.registerListener( + newSettings -> { + // figure out the difference and notify listeners + for (Entry, Set>> entry : + listeners.entrySet()) { + SettingsArg listenedToArg = entry.getKey(); + Object newValue = newSettings != null ? newSettings.getArgValue(listenedToArg) : null; + + for (SettingsArgChangeListener listener : entry.getValue()) { + notifyValue(listener, newValue); + } + } + }); + } + + @SuppressWarnings("unchecked") + private static void notifyValue(SettingsArgChangeListener listener, Object value) { + ((SettingsArgChangeListener) listener).onValue((T) value); + } + + /** + * Registers a {@link SettingsArgChangeListener} to this manager to listen to changes on {@link + * SettingsArg}. + * + *

The caller will get notified immediately once on the initial value upon calling this method + * + * @param listener + */ + public static void registerListener(SettingsArgChangeListener listener) { + Set> listenersOfThisType = + listeners.computeIfAbsent( + listener.getType(), k -> new HashSet>()); + listenersOfThisType.add(listener); + + Settings currentSettings = getSettings(); + if (currentSettings != null) { + notifyValue(listener, currentSettings.getArgValue(listener.getType())); + } + } + + public static void removeListener(SettingsArgChangeListener listener) { + Set> listenersOfThisType = listeners.get(listener.getType()); + if (listenersOfThisType != null) { + listenersOfThisType.remove(listener); + } + } + + /** + * Gets the Settings of this current process. Might return null if no Settings + * is available yet + * + * @return + */ + public static Settings getSettings() { + return getSettings(0, null); + } + + /** + * Gets the Settings of this current process. If a Settings is not yet + * available, this method will block either until Settings is available or the + * timeout elapses + * + * @param timeout + * @param unit + * @return + */ + public static Settings getSettings(long timeout, TimeUnit unit) { + if (fetcher != null) { + if (timeout > 0) { + try { + if (!fetcher.isSettingsAvailableLatch().await(timeout, unit)) { + logger.warn("Settings are not avaialable after waiting for " + timeout + " " + unit); + return null; + } + } catch (InterruptedException e) { + logger.warn("Settings are not avaialable as latch await is interrupted"); + return null; + } + } + return fetcher.getSettings(); + } else { + logger.debug("Settings are not yet available as initialization has not been completed yet"); + return null; + } + } +} diff --git a/libs/sampling/src/main/java/com/solarwinds/joboe/sampling/TokenBucket.java b/libs/sampling/src/main/java/com/solarwinds/joboe/sampling/TokenBucket.java new file mode 100644 index 00000000..bd0e9374 --- /dev/null +++ b/libs/sampling/src/main/java/com/solarwinds/joboe/sampling/TokenBucket.java @@ -0,0 +1,116 @@ +/* + * © SolarWinds Worldwide, LLC. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.solarwinds.joboe.sampling; + +/** + * A bucket that contains tokens to be consumed. The bucket is refilled based on the replenish rate. + * + *

This class is converted from ... + * + * @author Patson Luk + */ +public class TokenBucket { + private double ratePerSecond; // replenish rate in second + private double + capacity; // capacity of the bucket, available token should never exceed this number + private double availableTokens; + private long lastCheck; + + public TokenBucket(double bucketCapacity, double ratePerSecond) { + this.ratePerSecond = ratePerSecond; + this.capacity = bucketCapacity; + this.availableTokens = bucketCapacity; + + this.lastCheck = + System.currentTimeMillis(); // do not use System.nanoTime as we need JRE 1.5 support + } + + /** + * Consumes one available token + * + * @return true if available token is consumed, false if there is no available token to be + * consumed + */ + public boolean consume() { + return consume(1); + } + + /** + * Consumes # of available token as the size provided + * + * @param size # of tokens to be consumed + * @return + */ + public synchronized boolean consume(int size) { + updateAvailable(); + + if (availableTokens >= size) { + availableTokens -= size; + + return true; + } else { + return false; + } + } + + /** Updates the available tokens based on last checked time and replenish rate */ + private synchronized void updateAvailable() { + if (availableTokens < capacity) { + // calculate time since last check + long now = System.currentTimeMillis(); + long delta = now - lastCheck; + + // return if "time went backwards", possible on some VMs or with NTP + if (delta <= 0) { + return; + } + + // compute number of new tokens generated since last check + double newTokens = ratePerSecond * (double) delta / 1000; // delta in millisec + if (availableTokens + newTokens < capacity) { + availableTokens = availableTokens + newTokens; + } else { + availableTokens = capacity; + } + + // update time of last check + lastCheck = now; + } + } + + /** + * Sets the replenish rate + * + * @param ratePerSecond Replenish rate per second + */ + synchronized void setRatePerSecond(double ratePerSecond) { + this.ratePerSecond = ratePerSecond; + } + + /** + * Sets the capacity. This would adjust the available token if available token > capacity + * + * @param capacity + */ + synchronized void setCapacity(double capacity) { + this.capacity = capacity; + if (availableTokens > capacity) { + availableTokens = capacity; + } + } +} diff --git a/libs/sampling/src/main/java/com/solarwinds/joboe/sampling/TokenBucketType.java b/libs/sampling/src/main/java/com/solarwinds/joboe/sampling/TokenBucketType.java new file mode 100644 index 00000000..a7e2a846 --- /dev/null +++ b/libs/sampling/src/main/java/com/solarwinds/joboe/sampling/TokenBucketType.java @@ -0,0 +1,32 @@ +/* + * © SolarWinds Worldwide, LLC. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.solarwinds.joboe.sampling; + +import lombok.Getter; + +@Getter +public enum TokenBucketType { + REGULAR("regular"), + STRICT("strict"), + RELAXED("relaxed"); + + private final String label; + + TokenBucketType(String label) { + this.label = label; + } +} diff --git a/libs/sampling/src/main/java/com/solarwinds/joboe/sampling/TraceConfig.java b/libs/sampling/src/main/java/com/solarwinds/joboe/sampling/TraceConfig.java new file mode 100644 index 00000000..ddbd74db --- /dev/null +++ b/libs/sampling/src/main/java/com/solarwinds/joboe/sampling/TraceConfig.java @@ -0,0 +1,130 @@ +/* + * © SolarWinds Worldwide, LLC. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.solarwinds.joboe.sampling; + +import java.util.Collections; +import java.util.Map; +import lombok.Getter; + +/** + * Sample Rate Configuration + * + * @see TraceDecisionUtil + */ +public class TraceConfig { + private final Integer sampleRate; + @Getter private final SampleRateSource sampleRateSource; + private final Short flags; + @Getter private final Map bucketCapacities; + @Getter private final Map bucketRates; + + public TraceConfig(Integer sampleRate, SampleRateSource sampleRateSource, Short flags) { + this(sampleRate, sampleRateSource, flags, Collections.emptyMap(), Collections.emptyMap()); + } + + public TraceConfig( + Integer sampleRate, + SampleRateSource sampleRateSource, + Short flags, + Map bucketCapacities, + Map bucketRates) { + this.sampleRate = sampleRate; + this.sampleRateSource = sampleRateSource; + this.flags = flags; + this.bucketCapacities = bucketCapacities; + this.bucketRates = bucketRates; + } + + public int getSampleRate() { + return sampleRate; + } + + public int getSampleRateSourceValue() { + return sampleRateSource.value(); + } + + public double getBucketCapacity(TokenBucketType bucketType) { + return bucketCapacities.containsKey(bucketType) ? bucketCapacities.get(bucketType) : 0; + } + + public double getBucketRate(TokenBucketType bucketType) { + return bucketRates.containsKey(bucketType) ? bucketRates.get(bucketType) : 0; + } + + public boolean hasOverrideFlag() { + return flags != null + && (flags & Settings.OBOE_SETTINGS_FLAG_OVERRIDE) == Settings.OBOE_SETTINGS_FLAG_OVERRIDE; + } + + public boolean hasSampleStartFlag() { + return flags != null + && (flags & Settings.OBOE_SETTINGS_FLAG_SAMPLE_START) + == Settings.OBOE_SETTINGS_FLAG_SAMPLE_START; + } + + public boolean hasSampleThroughFlag() { + return flags != null + && (flags & Settings.OBOE_SETTINGS_FLAG_SAMPLE_THROUGH) + == Settings.OBOE_SETTINGS_FLAG_SAMPLE_THROUGH; + } + + public boolean hasSampleThroughAlwaysFlag() { + return flags != null + && (flags & Settings.OBOE_SETTINGS_FLAG_SAMPLE_THROUGH_ALWAYS) + == Settings.OBOE_SETTINGS_FLAG_SAMPLE_THROUGH_ALWAYS; + } + + public boolean isMetricsEnabled() { + return flags != null + && (flags + & (Settings.OBOE_SETTINGS_FLAG_SAMPLE_START + | Settings.OBOE_SETTINGS_FLAG_SAMPLE_THROUGH_ALWAYS)) + != 0; // for now if those 2 flags are not on, we assume it's metrics disabled + } + + public boolean hasSampleTriggerTraceFlag() { + return (flags & Settings.OBOE_SETTINGS_FLAG_TRIGGER_TRACE_ENABLED) + == Settings.OBOE_SETTINGS_FLAG_TRIGGER_TRACE_ENABLED; + } + + short getFlags() { + return flags; + } + + public boolean isFlagsConfigured() { + return flags != null; + } + + public boolean isSampleRateConfigured() { + return sampleRate != null; + } + + @Override + public String toString() { + return "SampleRateConfig [sampleRate=" + + sampleRate + + ", sampleRateSource=" + + sampleRateSource + + ", flags=" + + flags + + ", bucketCapacities=" + + bucketCapacities + + ", bucketRates=" + + bucketRates + + "]"; + } +} diff --git a/libs/sampling/src/main/java/com/solarwinds/joboe/sampling/TraceConfigs.java b/libs/sampling/src/main/java/com/solarwinds/joboe/sampling/TraceConfigs.java new file mode 100644 index 00000000..4d8b3e72 --- /dev/null +++ b/libs/sampling/src/main/java/com/solarwinds/joboe/sampling/TraceConfigs.java @@ -0,0 +1,71 @@ +/* + * © SolarWinds Worldwide, LLC. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.solarwinds.joboe.sampling; + +import com.github.benmanes.caffeine.cache.Cache; +import com.github.benmanes.caffeine.cache.Caffeine; +import java.io.Serializable; +import java.util.List; +import java.util.Map; +import java.util.Map.Entry; + +/** + * Container that stores {@link TraceConfig} mapped by URL + * + * @author pluk + */ +public class TraceConfigs implements Serializable { + private static final long serialVersionUID = 1L; + + private final Map traceConfigsByMatcher; + + private final Cache lruCache = + Caffeine.newBuilder().maximumSize(1048).build(); + + private final Cache lruCacheKey = Caffeine.newBuilder().maximumSize(1048).build(); + + public TraceConfigs(Map traceConfigsByMatcher) { + this.traceConfigsByMatcher = traceConfigsByMatcher; + } + + public TraceConfig getTraceConfig(List signals) { + StringBuilder key = new StringBuilder(); + signals.forEach(key::append); + TraceConfig result = null; + + if (lruCacheKey.getIfPresent(key.toString()) != null) { + return lruCache.getIfPresent(key.toString()); + } + + outer: + for (Entry entry : traceConfigsByMatcher.entrySet()) { + for (String signal : signals) { + if (entry.getKey().matches(signal)) { + result = entry.getValue(); + break outer; + } + } + } + + if (result != null) { + lruCache.put(key.toString(), result); + } + + lruCacheKey.put(key.toString(), key.toString()); + return result; + } +} diff --git a/libs/sampling/src/main/java/com/solarwinds/joboe/sampling/TraceDecision.java b/libs/sampling/src/main/java/com/solarwinds/joboe/sampling/TraceDecision.java new file mode 100644 index 00000000..562f9388 --- /dev/null +++ b/libs/sampling/src/main/java/com/solarwinds/joboe/sampling/TraceDecision.java @@ -0,0 +1,75 @@ +/* + * © SolarWinds Worldwide, LLC. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.solarwinds.joboe.sampling; + +import lombok.Getter; + +/** + * Contains the result of trace decision (whether a request is sampled and whether metrics should be + * reported) + * + *

Also contains the {@link TraceConfig} used for the trace decision-making logic, this could be + * null if no config was found + * + * @author pluk + */ +@Getter +public class TraceDecision { + private final boolean sampled; + private final boolean reportMetrics; + private final TraceConfig traceConfig; + private final boolean bucketExhausted; + private final TraceDecisionUtil.RequestType requestType; + private final Metadata incomingMetadata; + + public TraceDecision(boolean sampled, boolean reportMetrics, TraceConfig traceConfig) { + this(sampled, reportMetrics, false, traceConfig, TraceDecisionUtil.RequestType.REGULAR); + } + + public TraceDecision( + boolean sampled, + boolean reportMetrics, + TraceConfig traceConfig, + TraceDecisionUtil.RequestType requestType) { + this(sampled, reportMetrics, false, traceConfig, requestType); + } + + public TraceDecision( + boolean sampled, + boolean reportMetrics, + boolean bucketExhausted, + TraceConfig traceConfig, + TraceDecisionUtil.RequestType requestType) { + this(sampled, reportMetrics, bucketExhausted, traceConfig, requestType, null); + } + + public TraceDecision( + boolean sampled, + boolean reportMetrics, + boolean bucketExhausted, + TraceConfig traceConfig, + TraceDecisionUtil.RequestType requestType, + Metadata incomingMetadata) { + super(); + this.sampled = sampled; + this.reportMetrics = reportMetrics; + this.bucketExhausted = bucketExhausted; + this.traceConfig = traceConfig; + this.requestType = requestType; + this.incomingMetadata = incomingMetadata; + } +} diff --git a/libs/sampling/src/main/java/com/solarwinds/joboe/sampling/TraceDecisionUtil.java b/libs/sampling/src/main/java/com/solarwinds/joboe/sampling/TraceDecisionUtil.java new file mode 100644 index 00000000..b56766d4 --- /dev/null +++ b/libs/sampling/src/main/java/com/solarwinds/joboe/sampling/TraceDecisionUtil.java @@ -0,0 +1,530 @@ +/* + * © SolarWinds Worldwide, LLC. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.solarwinds.joboe.sampling; + +import com.solarwinds.joboe.logging.Logger; +import com.solarwinds.joboe.logging.LoggerFactory; +import java.util.AbstractMap; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.Random; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.atomic.AtomicInteger; +import lombok.Getter; + +public class TraceDecisionUtil { + public static final int SAMPLE_RESOLUTION = 1000000; + private static final Logger logger = LoggerFactory.getLogger(); + + // Used to control the rate and max traces can be generated + private static final ConcurrentHashMap tokenBuckets = + new ConcurrentHashMap(); + + private static final Random rand = new Random(); + + private static final Map requestCounters = + new HashMap(); + + private static final Map lastTraceConfigs = + new ConcurrentHashMap(); + + // map of flags to keep of whether a settings error was reported, only report it once + private static final ConcurrentHashMap reportedSettingsError = + new ConcurrentHashMap(); + + static { + for (MetricType type : MetricType.values()) { + requestCounters.put(type, new AtomicInteger(0)); + } + } + + /** + * Determines if we should trace this request. + * + * @param layer + * @param inXTraceID incoming validated X-Trace-Id, if it is not valid, it should be null + * @param xTraceOptions incoming X-Trace-Options, null if not defined + * @param signals resource if the request has one, for example URL for web request, job name for + * jobs + * @return trace decision, should always be nonnull + */ + public static TraceDecision shouldTraceRequest( + String layer, String inXTraceID, XtraceOptions xTraceOptions, List signals) { + boolean isTriggerTrace; + try { + RequestType requestType = getRequestType(inXTraceID, xTraceOptions); + isTriggerTrace = requestType.isTriggerTrace(); + + // First get the config from remote source (SRv1) + TraceConfig remoteConfig = getRemoteTraceConfig(); + + if (remoteConfig == null) { // settings not readable, do not trace + logger.debug( + "Failed to fetch settings records, not tracing"); // debug message, otherwise it could + // be very noisy if remote config is + // not found + return new TraceDecision(false, false, null, requestType); + } + + // Get the local sample rate (either from file, defaults, JVM, url matching etc) + TraceConfig localConfig = getLocalTraceConfig(signals); + + TraceConfig config = + computeTraceConfig( + remoteConfig, + localConfig, + SettingsManager.getSamplingConfiguration().isTriggerTraceEnabled()); + + if (!config.isSampleRateConfigured()) { + logger.debug( + "Cannot trace request as sample rate is undefined"); // debug message, otherwise it + // could be very noisy + return new TraceDecision(false, false, config, requestType); + } + + if (!config.isFlagsConfigured()) { + logger.debug( + "Cannot trace request as flags are undefined"); // debug message, otherwise it could be + // very noisy + return new TraceDecision(false, false, config, requestType); + } + + boolean isReportMetrics = isReportMetricsByConfig(config); + if (xTraceOptions != null && xTraceOptions.getAuthenticationStatus().isFailure()) { + logger.debug("Bad x-tv-options-signature, not tracing"); + return new TraceDecision(false, isReportMetrics, null, requestType); + } + + if (config + .hasSampleStartFlag()) { // only makes sense to record the sample rate if start flag is + // ON. as it is the only case that the sample rate is actually + // used + recordLastTraceConfig(config, layer); + } + + Metadata inMetadata = null; + if (inXTraceID != null) { + inMetadata = validateMetadata(inXTraceID); + } + boolean isSampled = isSampledByConfig(inMetadata, config, isTriggerTrace); + boolean bucketExhausted = false; + + // perform token bucket check if it is a new trace + if (isSampled) { + if (inMetadata == null) { + TokenBucket tokenBucket = + getTokenBucket( + requestType.bucketType, + remoteConfig.getBucketCapacity(requestType.bucketType), + remoteConfig.getBucketRate( + requestType + .bucketType)); // passes in remoteConfig as this is the only source of + // token bucket parameters + if (tokenBucket.consume()) { // check whether there are tokens left to be consumed + incrementMetrics(MetricType.TRACE_COUNT); // count all new traffic that will be traced + if (requestType.isTriggerTrace()) { + incrementMetrics(MetricType.TRIGGERED_TRACE_COUNT); + } + } else { + logger.trace("No Tokens available in the Token Bucket. Not tracing this request"); + + incrementMetrics(MetricType.TOKEN_BUCKET_EXHAUSTION); + isSampled = false; // flip it to false due to exhausted bucket + bucketExhausted = true; + } + } else { + incrementMetrics(MetricType.TRACE_COUNT); // count all through traffic that will be traced + } + } + + return new TraceDecision( + isSampled, isReportMetrics, bucketExhausted, config, requestType, inMetadata); + } finally { + incrementMetrics(MetricType.THROUGHPUT); + } + } + + private static Metadata validateMetadata(String inXTraceID) { + if (!Metadata.isCompatible(inXTraceID)) { + logger.debug("Not accepting X-Trace ID [" + inXTraceID + "] for trace continuation"); + return null; + } + + try { + Metadata inMetadata = new Metadata(inXTraceID); + if (!inMetadata.isValid()) { + logger.debug("Invalid incoming x-trace ID [" + inXTraceID + "]"); + return null; + } + return inMetadata; + } catch (SamplingException e) { + logger.warn("Failed to parse x-trace ID [" + inXTraceID + "] " + e.getMessage()); + return null; + } + } + + private static Map.Entry getTagEntry(String key, Object value) { + return new AbstractMap.SimpleEntry<>(key, value); + } + + private static RequestType getRequestType(String inXTraceId, XtraceOptions xTraceOptions) { + if (xTraceOptions == null + || inXTraceId + != null) { // if there's an incoming valid x-trace ID then trigger trace option is + // ignored + return RequestType.REGULAR; + } + RequestType requestType; + XtraceOptions.AuthenticationStatus authenticationStatus = + xTraceOptions.getAuthenticationStatus(); + if (authenticationStatus.isFailure()) { + requestType = RequestType.BAD_SIGNATURE; + } else if (!xTraceOptions.getOptionValue(XtraceOption.TRIGGER_TRACE)) { + requestType = RequestType.REGULAR; + } else if (authenticationStatus.isAuthenticated()) { + requestType = RequestType.AUTHENTICATED_TRIGGER_TRACE; + } else { + requestType = RequestType.UNAUTHENTICATED_TRIGGER_TRACE; + } + return requestType; + } + + /** + * Retrieves the token bucket based on requestType. Update the bucket with newBucketCapacity and + * newBucketRate + * + * @param bucketType - token bucket type + * @param newBucketCapacity new Bucket capacity + * @param newBucketRate new Bucket replenish rate + * @return token bucket of the given layer updated with the newBucketCapacity and newBucketRate + * provided + */ + static TokenBucket getTokenBucket( + TokenBucketType bucketType, double newBucketCapacity, double newBucketRate) { + TokenBucket bucket = tokenBuckets.get(bucketType); + if (bucket == null) { + bucket = new TokenBucket(newBucketCapacity, newBucketRate); + tokenBuckets.put(bucketType, bucket); + } else { + bucket.setCapacity(newBucketCapacity); + bucket.setRatePerSecond(newBucketRate); + } + + return bucket; + } + + /** + * Retrieves Trace Config from a remote source + * + * @return TraceConfig if a settings is found, null if the settings cannot be found/ not yet + * available + */ + public static TraceConfig getRemoteTraceConfig() { + TraceConfig config = null; + Settings settings = SettingsManager.getSettings(); + + if (settings != null) { + Map bucketCapacities = + getBucketSettings( + settings, + SettingsArg.BUCKET_CAPACITY, + SettingsArg.RELAXED_BUCKET_CAPACITY, + SettingsArg.STRICT_BUCKET_CAPACITY); + Map bucketRates = + getBucketSettings( + settings, + SettingsArg.BUCKET_RATE, + SettingsArg.RELAXED_BUCKET_RATE, + SettingsArg.STRICT_BUCKET_RATE); + + config = + new TraceConfig( + (int) settings.getValue(), + settings.isDefault() ? SampleRateSource.OBOE_DEFAULT : SampleRateSource.OBOE, + settings.getFlags(), + bucketCapacities, + bucketRates); + } + + return config; + } + + private static Map getBucketSettings( + Settings settings, + SettingsArg regularBucketArg, + SettingsArg relaxedBucketArg, + SettingsArg strictBucketArg) { + Map result = new HashMap(); + result.put(TokenBucketType.REGULAR, getBucketSettingsArg(settings, regularBucketArg)); + result.put(TokenBucketType.RELAXED, getBucketSettingsArg(settings, relaxedBucketArg)); + result.put(TokenBucketType.STRICT, getBucketSettingsArg(settings, strictBucketArg)); + + return result; + } + + /** + * Validates the bucket settings, if it's invalid, 0 will be returned + * + * @param settings + * @param arg + * @return + */ + private static double getBucketSettingsArg(Settings settings, SettingsArg arg) { + Double value = settings.getArgValue(arg); + if (value == null) { + logger.debug("Cannot read settings arg " + arg); + return 0; + } else if (value < 0) { + logger.warn("Invalid negative value in settings arg " + arg); + return 0; + } + + return value; + } + + /** + * Gets the Trace Config from local system based on the URL (if available) of the request + * + * @param signals URL of a web based request, null otherwise + * @return A URL specific Trace Config will be returned if a match is found, otherwise the + * universal Trace Config from local settings. + */ + public static TraceConfig getLocalTraceConfig(List signals) { + TraceConfig localTraceConfig = getLocalSpecificTraceConfig(signals); + if (localTraceConfig == null) { + localTraceConfig = getLocalUniversalTraceConfig(); + } + + return localTraceConfig; + } + + /** + * Retrieves universal (general settings that apply to the java process) Trace Config from local + * Settings such as jvm arguments and java agent config file + * + * @return + */ + private static TraceConfig getLocalUniversalTraceConfig() { + Integer sampleRate = SettingsManager.getSamplingConfiguration().getSampleRate(); + TracingMode tracingMode = SettingsManager.getSamplingConfiguration().getTracingMode(); + return new TraceConfig( + sampleRate, + sampleRate != null ? SampleRateSource.FILE : SampleRateSource.DEFAULT, + tracingMode != null ? tracingMode.toFlags() : null); + } + + /** + * Retrieves Resource specific Trace Config from local settings (by URL, job names etc) + * + * @param signals URL of the web request, null otherwise + * @return Resource specific Trace Config if a match is found from local settings, otherwise null + * is returned + */ + private static TraceConfig getLocalSpecificTraceConfig(List signals) { + if (signals != null + && SettingsManager.getSamplingConfiguration().getInternalTransactionSettings() != null) { + return SettingsManager.getSamplingConfiguration() + .getInternalTransactionSettings() + .getTraceConfig(signals); + } else { + return null; + } + } + + /** + * Computes the trace config based on the precedence list. + * + * @param remoteConfig should not be null + * @param localConfig + * @return + */ + static TraceConfig computeTraceConfig( + TraceConfig remoteConfig, TraceConfig localConfig, boolean localTriggerTraceEnabled) { + boolean hasRemoteConfigOverride = remoteConfig.hasOverrideFlag(); + + // consider sample rate: + TraceConfig sampleRateConfig; + + if (hasRemoteConfigOverride && localConfig.isSampleRateConfigured()) { + sampleRateConfig = + remoteConfig.getSampleRate() <= localConfig.getSampleRate() ? remoteConfig : localConfig; + } else if (localConfig.isSampleRateConfigured()) { + sampleRateConfig = localConfig; + } else { + sampleRateConfig = remoteConfig; + } + + Integer sampleRate = sampleRateConfig.getSampleRate(); + SampleRateSource sampleRateSource = sampleRateConfig.getSampleRateSource(); + + // consider tracing flags: + short flags; + if (hasRemoteConfigOverride && localConfig.isFlagsConfigured()) { // get the Lower flags + flags = (short) (remoteConfig.getFlags() & localConfig.getFlags()); + } else if (localConfig.isFlagsConfigured()) { + flags = localConfig.getFlags(); + } else { + flags = remoteConfig.getFlags(); + } + + // special case : consider trigger trace local config if explicitly disabled + if (!localTriggerTraceEnabled) { + flags = (short) (flags & ~Settings.OBOE_SETTINGS_FLAG_TRIGGER_TRACE_ENABLED); // disable the + // OBOE_SETTINGS_FLAG_TRIGGER_TRACE_ENABLED bit + } + + return new TraceConfig( + sampleRate, + sampleRateSource, + flags, + remoteConfig.getBucketCapacities(), + remoteConfig.getBucketRates()); // token bucket parameters are always from remote config + } + + private static boolean sampled(int sampleRate) { + return (sampleRate == SAMPLE_RESOLUTION + || (sampleRate < SAMPLE_RESOLUTION && rand.nextInt(SAMPLE_RESOLUTION) <= sampleRate)); + } + + /** + * Checks whether we should sample based on the TraceConfig and various criteria + * + * @param inMetadata + * @param config + * @return whether we should trace + */ + private static boolean isSampledByConfig( + Metadata inMetadata, TraceConfig config, boolean isTriggerTrace) { + if (inMetadata == null) { // new trace + if (isTriggerTrace) { // trigger trace only matters for new traces + return config.hasSampleTriggerTraceFlag(); + } else { + if (config.hasSampleStartFlag()) { + incrementMetrics(MetricType.SAMPLE_COUNT); + return sampled(config.getSampleRate()); + } else { + return false; + } + } + } else { // continue trace + Metadata continuingMetadata = inMetadata; + + if (!continuingMetadata + .isSampled()) { // sampling decision from previous service decides not to trace + return false; + } else // this flag is not being used currently + if (config.hasSampleThroughAlwaysFlag()) { + incrementMetrics(MetricType.THROUGH_TRACE_COUNT); + return true; + } else return config.hasSampleThroughFlag() && sampled(config.getSampleRate()); + } + } + + private static boolean isReportMetricsByConfig(TraceConfig config) { + return config + .isMetricsEnabled(); // report metrics as far as the metrics is enabled in config. No + // propagation for now + } + + /** + * Increments the count for the given metricType. + * + * @param metricType + */ + private static void incrementMetrics(MetricType metricType) { + requestCounters.get(metricType).incrementAndGet(); + } + + /** + * Records the last accessed sample rate for metric reporting functionality + * + * @param config last accessed sample rate + * @param layer + */ + private static void recordLastTraceConfig(TraceConfig config, String layer) { + synchronized (lastTraceConfigs) { + lastTraceConfigs.put(layer, config); + } + } + + /** + * Consumes and return a clone of metrics data map of the Metric type as argument. Take note of + * the side effect that, after the call, the existing metric data would be consumed and cleared + * + * @param type Metric type + * @return a clone of the metric data map of the Metric type, null if the Metric is not available + * for that type + */ + public static int consumeMetricsData(MetricType type) { + AtomicInteger metricData = requestCounters.get(type); + return metricData.getAndSet(0); + } + + /** + * Consumes and return a clone of map of last accessed sample rate settings. Take note of the side + * effect that, after the call, the existing last accessed sampler ate settings would be consumed + * and cleared + * + * @return a lone of map of last accessed sample rate settings + */ + public static Map consumeLastTraceConfigs() { + Map configs; + synchronized (lastTraceConfigs) { + configs = new HashMap(lastTraceConfigs); + lastTraceConfigs.clear(); + } + + return configs; + } + + /** For internal testing usage only */ + public static void reset() { + tokenBuckets.clear(); + + for (AtomicInteger counter : requestCounters.values()) { + counter.set(0); + } + lastTraceConfigs.clear(); + reportedSettingsError.clear(); + } + + public enum MetricType { + THROUGHPUT, + TOKEN_BUCKET_EXHAUSTION, + TRACE_COUNT, + SAMPLE_COUNT, + THROUGH_TRACE_COUNT, + TRIGGERED_TRACE_COUNT // ,SIGNED_REQUEST_COUNT + } + + @Getter + public enum RequestType { + REGULAR(false, TokenBucketType.REGULAR), + AUTHENTICATED_TRIGGER_TRACE(true, TokenBucketType.RELAXED), + UNAUTHENTICATED_TRIGGER_TRACE(true, TokenBucketType.STRICT), + BAD_SIGNATURE(false, TokenBucketType.REGULAR); + + private final boolean triggerTrace; + private final TokenBucketType bucketType; + + RequestType(boolean triggerTrace, TokenBucketType bucketType) { + this.triggerTrace = triggerTrace; + this.bucketType = bucketType; + } + } +} diff --git a/libs/sampling/src/main/java/com/solarwinds/joboe/sampling/TracingMode.java b/libs/sampling/src/main/java/com/solarwinds/joboe/sampling/TracingMode.java new file mode 100644 index 00000000..f1b9bb33 --- /dev/null +++ b/libs/sampling/src/main/java/com/solarwinds/joboe/sampling/TracingMode.java @@ -0,0 +1,65 @@ +/* + * © SolarWinds Worldwide, LLC. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.solarwinds.joboe.sampling; + +import static com.solarwinds.joboe.sampling.Settings.OBOE_SETTINGS_FLAG_SAMPLE_START; +import static com.solarwinds.joboe.sampling.Settings.OBOE_SETTINGS_FLAG_SAMPLE_THROUGH_ALWAYS; +import static com.solarwinds.joboe.sampling.Settings.OBOE_SETTINGS_FLAG_TRIGGER_TRACE_ENABLED; + +import lombok.Getter; + +@Getter +public enum TracingMode { + ALWAYS("always"), // deprecated + ENABLED("enabled"), + NEVER("never"), // deprecated + DISABLED("disabled"); + + private final String stringValue; + + TracingMode(String stringValue) { + this.stringValue = stringValue; + } + + public static TracingMode fromString(String stringValue) { + for (TracingMode mode : values()) { + if (mode.stringValue.equals(stringValue)) { + return mode; + } + } + + return null; + } + + // convert agent tracing mode to settings flags + // XXX: Using THROUGH_ALWAYS to maintain previous behaviour when setting sample rate from command + // line/config + public short toFlags() { + switch (this) { + case ALWAYS: + case ENABLED: + return OBOE_SETTINGS_FLAG_SAMPLE_START + | OBOE_SETTINGS_FLAG_SAMPLE_THROUGH_ALWAYS + | OBOE_SETTINGS_FLAG_TRIGGER_TRACE_ENABLED; + + case NEVER: + case DISABLED: + default: + return 0x00; + } + } +} diff --git a/libs/sampling/src/main/java/com/solarwinds/joboe/sampling/XTraceOptionsResponse.java b/libs/sampling/src/main/java/com/solarwinds/joboe/sampling/XTraceOptionsResponse.java new file mode 100644 index 00000000..825a5e83 --- /dev/null +++ b/libs/sampling/src/main/java/com/solarwinds/joboe/sampling/XTraceOptionsResponse.java @@ -0,0 +1,98 @@ +/* + * © SolarWinds Worldwide, LLC. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.solarwinds.joboe.sampling; + +import java.util.*; +import java.util.Map.Entry; + +/** + * Computes the response from {@link XtraceOptions} processing. + * + *

Contains the key/value produced from the computation. + */ +public class XTraceOptionsResponse { + + public static XTraceOptionsResponse computeResponse( + XtraceOptions options, TraceDecision traceDecision, boolean isServiceRoot) { + if (options == null) { + return null; + } + + XTraceOptionsResponse response = new XTraceOptionsResponse(); + + if (options + .getAuthenticationStatus() + .isFailure()) { // if auth failure, we will only reply with the auth option + response.setValue("auth", options.getAuthenticationStatus().getReason()); + } else { + if (options.getAuthenticationStatus().isAuthenticated()) { + response.setValue("auth", "ok"); + } + boolean isTriggerTrace = options.getOptionValue(XtraceOption.TRIGGER_TRACE); + if (isTriggerTrace) { + if (!isServiceRoot) { // a continued trace, trigger trace flag has no effect + response.setValue("trigger-trace", "ignored"); + } else if (traceDecision.isSampled()) { + response.setValue("trigger-trace", "ok"); + } else if (traceDecision.getTraceConfig() == null) { + response.setValue("trigger-trace", "settings-not-available"); + } else if (traceDecision.getTraceConfig().getFlags() == TracingMode.DISABLED.toFlags()) { + response.setValue("trigger-trace", "tracing-disabled"); + } else if (!traceDecision.getTraceConfig().hasSampleTriggerTraceFlag()) { + response.setValue("trigger-trace", "trigger-tracing-disabled"); + } else if (traceDecision.isBucketExhausted()) { + response.setValue("trigger-trace", "rate-exceeded"); + } else { + response.setValue("trigger-trace", "unknown-failure"); + } + } else { + response.setValue("trigger-trace", "not-requested"); + } + + for (XtraceOptions.XTraceOptionException exception : options.getExceptions()) { + exception.appendToResponse(response); + } + } + + return response; + } + + private final Map keyValues = new LinkedHashMap(); + + private XTraceOptionsResponse() {} + + public String getValue(String key) { + return keyValues.get(key); + } + + public void setValue(String key, String value) { + keyValues.put(key, value); + } + + @Override + public String toString() { + StringBuilder builder = new StringBuilder(); + for (Entry entry : keyValues.entrySet()) { + builder.append(entry.getKey() + "=" + entry.getValue() + ";"); + } + + if (builder.length() > 0) { + builder.deleteCharAt(builder.length() - 1); // remove the last dangling ; + } + return builder.toString(); + } +} diff --git a/libs/sampling/src/main/java/com/solarwinds/joboe/sampling/XorShiftRNG.java b/libs/sampling/src/main/java/com/solarwinds/joboe/sampling/XorShiftRNG.java new file mode 100644 index 00000000..d6cac4cd --- /dev/null +++ b/libs/sampling/src/main/java/com/solarwinds/joboe/sampling/XorShiftRNG.java @@ -0,0 +1,98 @@ +/* + * © SolarWinds Worldwide, LLC. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.solarwinds.joboe.sampling; + +import java.util.Random; +import java.util.concurrent.locks.ReentrantLock; + +/** + * Very fast pseudo random number generator. See this page + * for a description. + * + * @author Daniel Dyer + * @since 1.2 + */ +public class XorShiftRNG extends Random { + private static final long serialVersionUID = 1L; + private static final int SEED_SIZE_BYTES = 20; // Needs 5 32-bit integers. + + // Previously used an array for state but using separate fields proved to be + // faster. + private int state1; + private int state2; + private int state3; + private int state4; + private int state5; + + private final byte[] seed; + + // Lock to prevent concurrent modification of the RNG's internal state. + private final ReentrantLock lock = new ReentrantLock(); + + /** + * Seed the RNG using the provided seed generation strategy. + * + * @param seedGenerator The seed generation strategy that will provide the seed value for this + * RNG. + * @throws SeedException If there is a problem generating a seed. + */ + public XorShiftRNG(SeedGenerator seedGenerator) throws SeedException { + this(seedGenerator.generateSeed(SEED_SIZE_BYTES)); + } + + /** + * Creates an RNG and seeds it with the specified seed data. + * + * @param seed The seed data used to initialise the RNG. + */ + public XorShiftRNG(byte[] seed) { + if (seed == null || seed.length != SEED_SIZE_BYTES) { + throw new IllegalArgumentException("XOR shift RNG requires 160 bits of seed data."); + } + this.seed = seed.clone(); + int[] state = BinaryUtils.convertBytesToInts(seed); + this.state1 = state[0]; + this.state2 = state[1]; + this.state3 = state[2]; + this.state4 = state[3]; + this.state5 = state[4]; + } + + /** Returns the seed bytes used to initialise this RNG. */ + public byte[] getSeed() { + return seed.clone(); + } + + /** {@inheritDoc} */ + @Override + protected int next(int bits) { + try { + lock.lock(); + int t = (state1 ^ (state1 >> 7)); + state1 = state2; + state2 = state3; + state3 = state4; + state4 = state5; + state5 = (state5 ^ (state5 << 6)) ^ (t ^ (t << 13)); + int value = (state2 + state2 + 1) * state5; + return value >>> (32 - bits); + } finally { + lock.unlock(); + } + } +} diff --git a/libs/sampling/src/main/java/com/solarwinds/joboe/sampling/XtraceOption.java b/libs/sampling/src/main/java/com/solarwinds/joboe/sampling/XtraceOption.java new file mode 100644 index 00000000..55897a81 --- /dev/null +++ b/libs/sampling/src/main/java/com/solarwinds/joboe/sampling/XtraceOption.java @@ -0,0 +1,159 @@ +/* + * © SolarWinds Worldwide, LLC. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.solarwinds.joboe.sampling; + +import com.solarwinds.joboe.logging.Logger; +import com.solarwinds.joboe.logging.LoggerFactory; +import java.util.HashMap; +import java.util.Map; +import lombok.Getter; + +/** + * An option key within X-Trace-Options header. This does NOT store the option value + * + * @param The value type of the option + * @see XtraceOptions + */ +@Getter +public class XtraceOption { + + private static final Logger LOGGER = LoggerFactory.getLogger(); + private static final Map> keyLookup = + new HashMap>(); + public static final XtraceOption TRIGGER_TRACE = + new XtraceOption("trigger-trace", null, false); + public static final XtraceOption SW_KEYS = + new XtraceOption("sw-keys", ValueParser.STRING_VALUE_PARSER); + public static final XtraceOption TS = + new XtraceOption("ts", ValueParser.LONG_VALUE_PARSER); + public static final String CUSTOM_KV_PREFIX = "custom-"; + + private final V defaultValue; + @Getter private final String key; + private final ValueParser parser; + private boolean isCustomKv = false; + + /** + * @param key + * @param parser null parser indicates that this is a key only option + */ + private XtraceOption(String key, ValueParser parser) { + this(key, parser, null); + } + + private XtraceOption(String key, ValueParser parser, V defaultValue) { + this.key = key; + this.defaultValue = defaultValue; + this.parser = parser; + + keyLookup.put(key, this); + } + + public static XtraceOption fromKey(String key) { + if (key.contains( + " ")) { // invalid key if it contains any space. Not using regex here as it could be pretty + // slow + return null; + } + + XtraceOption option = keyLookup.get(key); + if (option != null) { + return option; + } else if (isCustomKv(key)) { + option = new XtraceOption<>(key, ValueParser.STRING_VALUE_PARSER); + option.isCustomKv = true; + return option; + } else { + return null; + } + } + + @Override + public boolean equals(Object o) { + if (this == o) return true; + if (o == null || getClass() != o.getClass()) return false; + + XtraceOption that = (XtraceOption) o; + + return key.equals(that.key); + } + + @Override + public int hashCode() { + return key.hashCode(); + } + + /** + * Whether this option is a custom one that starts with {@link XtraceOption#CUSTOM_KV_PREFIX} + * + * @return + */ + public boolean isCustomKv() { + return this.isCustomKv; + } + + /** + * Whether this option should appear in key-value pair or not. + * + * @return + */ + public boolean isKeyOnlyOption() { + return parser == null; + } + + private static boolean isCustomKv(String key) { + return key.startsWith(CUSTOM_KV_PREFIX); + } + + private interface ValueParser { + V parse(String stringValue) throws IllegalArgumentException; + + BooleanValueParser BOOLEAN_VALUE_PARSER = new BooleanValueParser(); + StringValueParser STRING_VALUE_PARSER = new StringValueParser(); + LongValueParser LONG_VALUE_PARSER = new LongValueParser(); + } + + public V parseValueFromString(String value) + throws XtraceOptions.InvalidValueXTraceOptionException { + try { + return parser != null ? parser.parse(value) : null; + } catch (IllegalArgumentException e) { + throw new XtraceOptions.InvalidValueXTraceOptionException(this, value); + } + } + + private static class BooleanValueParser implements ValueParser { + @Override + public Boolean parse(String stringValue) { + return "1".equals(stringValue) || Boolean.valueOf(stringValue); + } + } + + private static class StringValueParser implements ValueParser { + @Override + public String parse(String stringValue) { + return stringValue; + } + } + + private static class LongValueParser implements ValueParser { + @Override + public Long parse(String stringValue) throws NumberFormatException { + return Long.parseLong(stringValue); + } + } +} diff --git a/libs/sampling/src/main/java/com/solarwinds/joboe/sampling/XtraceOptions.java b/libs/sampling/src/main/java/com/solarwinds/joboe/sampling/XtraceOptions.java new file mode 100644 index 00000000..e511211b --- /dev/null +++ b/libs/sampling/src/main/java/com/solarwinds/joboe/sampling/XtraceOptions.java @@ -0,0 +1,462 @@ +/* + * © SolarWinds Worldwide, LLC. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.solarwinds.joboe.sampling; + +import com.solarwinds.joboe.logging.Logger; +import com.solarwinds.joboe.logging.LoggerFactory; +import java.security.GeneralSecurityException; +import java.util.ArrayList; +import java.util.Collections; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; +import java.util.Objects; +import javax.crypto.Mac; +import javax.crypto.spec.SecretKeySpec; +import lombok.Getter; + +/** + * Represents the data model from the `X-Trace-Options` request header. + * + *

Provides a static method to construct the {@link XtraceOptions} instance by parsing and + * authenticating the `X-Trace-Options` with signature check {@link SignatureAuthenticator}(Current + * authentication uses HMAC-SHA1). Authentication status and exceptions associated with the + * operation are also exposed on the result instance. + * + *

Provides a method to look up {@link XtraceOption} with the corresponding typed value. + */ +public class XtraceOptions { + private static final Logger logger = LoggerFactory.getLogger(); + static final long TIMESTAMP_MAX_DELTA = 5 * 60; // 5 minutes in seconds + private static SignatureAuthenticator authenticator; + + private final Map, ?> options; + static final String ENTRY_SEPARATOR = ";"; + static final String KEY_VALUE_SEPARATOR = "="; + + /** + * -- GETTER -- Gets the exceptions occurred during the construction of XTraceOptions by + * + * @return + */ + @Getter private final List exceptions; + + /** + * -- GETTER -- Gets the authentication status after invocation of + * + * @return + */ + @Getter private final AuthenticationStatus authenticationStatus; + + static { + SettingsManager.registerListener( + new SettingsArgChangeListener(SettingsArg.TRACE_OPTIONS_SECRET) { + @Override + public void onChange(byte[] newValue) { + if (newValue != null) { + authenticator = new HmacSignatureAuthenticator(newValue); + } else { + authenticator = null; // remove the existing authenticator + } + } + }); + } + + XtraceOptions( + Map, ?> options, + List exceptions, + AuthenticationStatus authenticationStatus) { + this.options = options; + this.exceptions = exceptions; + this.authenticationStatus = authenticationStatus; + } + + /** + * Extracts XTraceOptions by parsing the `traceOptionString`. + * + *

If `traceOptionsSignature` is provided, then authenticates the options with the + * authenticator + * + * @param traceOptionsString + * @param traceOptionsSignature + * @return An XTraceOptions instance after the parsing and authentication; null if + * `traceOptionsString` is null. Take note that any parsing or authentication failure will be + * recorded in the returning instance and can be extracted by {@link + * XtraceOptions#getAuthenticationStatus()} and {@link XtraceOptions#getExceptions()} methods. + */ + public static XtraceOptions getXTraceOptions( + String traceOptionsString, String traceOptionsSignature) { + return getXTraceOptions(traceOptionsString, traceOptionsSignature, authenticator); + } + + static XtraceOptions getXTraceOptions( + String traceOptionsString, + String traceOptionsSignature, + SignatureAuthenticator authenticator) { + if (traceOptionsString == null) { + return null; + } + + List exceptions = new ArrayList(); + + Map, Object> options = new LinkedHashMap, Object>(); + for (String optionEntry : traceOptionsString.split(ENTRY_SEPARATOR)) { + optionEntry = optionEntry.trim(); + int separatorIndex = optionEntry.indexOf(KEY_VALUE_SEPARATOR); + String optionKey; + if (separatorIndex >= 0) { + optionKey = optionEntry.substring(0, separatorIndex); + } else { // check whether it is key only option + optionKey = optionEntry; + } + + optionKey = optionKey.trim(); + + if (optionKey.isEmpty()) { // skip empty key + if (!optionEntry.isEmpty()) { + logger.debug( + "Skipped entry [" + optionEntry + "] in X-Trace-Options as the key is empty"); + } + continue; + } + + XtraceOption option = XtraceOption.fromKey(optionKey); + if (option != null) { + if (option.isKeyOnlyOption()) { + if (separatorIndex > 0) { // do not allow key only option with a value + exceptions.add(new InvalidFormatXTraceOptionException(option, optionEntry)); + } else { + if (!options.containsKey(option)) { + options.put(option, true); + } else { + logger.debug( + "Duplicated option [" + + option.getKey() + + "] found in X-Trace-Options, ignoring..."); + } + } + } else { + if (separatorIndex < 0) { + exceptions.add(new InvalidFormatXTraceOptionException(option, optionEntry)); + } + String optionValueString = optionEntry.substring(separatorIndex + 1).trim(); + try { + if (!options.containsKey(option)) { + options.put(option, option.parseValueFromString(optionValueString)); + } else { + logger.debug( + "Duplicated option [" + + option.getKey() + + "] with value [" + + optionValueString + + "] found in X-Trace-Options, ignoring..."); + } + } catch (InvalidValueXTraceOptionException e) { + exceptions.add(e); + } + } + } else { + exceptions.add(new UnknownXTraceOptionException(optionKey)); + } + } + + // authenticate + AuthenticationStatus authenticationStatus = + authenticate( + traceOptionsString, + (Long) options.get(XtraceOption.TS), + traceOptionsSignature, + authenticator); + + if (authenticationStatus.isFailure()) { // if authentication failed, ignore all xtrace options + return new XtraceOptions( + Collections.emptyMap(), Collections.emptyList(), authenticationStatus); + } else { + for (XTraceOptionException exception : exceptions) { + logger.debug(exception.getMessage()); + } + return new XtraceOptions(options, exceptions, authenticationStatus); + } + } + + /** + * Authenticates the `optionString` with: + * + *

    + *
  1. Check if the `timestamp` provided is within the accepted time range (iff + * `traceOptionsSignature` is non null + *
  2. Authenticate the `optionString` and `traceOptionsSignature` with the provided signature + * `authenticator` + *
+ * + * Returns the authentication status based on the provided parameters + * + * @param optionsString + * @param timestamp + * @param traceOptionsSignature + * @param authenticator + * @return {@link AuthenticationStatus#NOT_AUTHENTICATED} if `traceOptionsSignature` is null; + * otherwise the result of the authentication + */ + static AuthenticationStatus authenticate( + String optionsString, + Long timestamp, + String traceOptionsSignature, + SignatureAuthenticator authenticator) { + if (traceOptionsSignature == null) { + return AuthenticationStatus.NOT_AUTHENTICATED; + } + + if (timestamp == null) { + return AuthenticationStatus.failure("bad-timestamp"); + } + if (!isTimestampWithRange(timestamp)) { + return AuthenticationStatus.failure("bad-timestamp"); + } + + if (authenticator == null) { + return AuthenticationStatus.failure("authenticator-unavailable"); + } else { + if (authenticator.authenticate(optionsString, traceOptionsSignature)) { + return AuthenticationStatus.OK; + } else { + return AuthenticationStatus.failure("bad-signature"); + } + } + } + + private static boolean isTimestampWithRange(long timestamp) { + long currentTimeInSeconds = System.currentTimeMillis() / 1000; + return timestamp >= currentTimeInSeconds - TIMESTAMP_MAX_DELTA + && timestamp <= currentTimeInSeconds + TIMESTAMP_MAX_DELTA; + } + + /** + * Gets the value of the option. If it's not defined then the default value of the option will be + * returned. + * + *

Take note that default value could be null too. + * + * @param option + * @param + * @return + */ + @SuppressWarnings("unchecked") + public T getOptionValue(XtraceOption option) { + return options.containsKey(option) ? (T) options.get(option) : option.getDefaultValue(); + } + + /** + * Gets the custom KVs X-Trace-Options with key that starts with {@link + * XtraceOption#CUSTOM_KV_PREFIX} + * + * @return + */ + @SuppressWarnings("unchecked") + public Map, String> getCustomKvs() { + Map, String> customKvs = new LinkedHashMap, String>(); + for (Map.Entry, ?> entry : options.entrySet()) { + XtraceOption option = entry.getKey(); + if (option.isCustomKv()) { + customKvs.put((XtraceOption) option, (String) entry.getValue()); + } + } + return customKvs; + } + + public abstract static class XTraceOptionException extends Exception { + private static final long serialVersionUID = 1L; + + XTraceOptionException(String message, Exception cause) { + super(message, cause); + } + + XTraceOptionException(String message) { + super(message); + } + + abstract void appendToResponse(XTraceOptionsResponse response); + } + + @Getter + public abstract static class InvalidXTraceOptionException extends XTraceOptionException { + private static final long serialVersionUID = 1L; + protected String invalidOptionKey; + private static final String RESPONSE_KEY = "ignored"; + + protected InvalidXTraceOptionException(String invalidOptionKey, String message) { + super(message); + this.invalidOptionKey = invalidOptionKey; + } + + @Override + void appendToResponse(XTraceOptionsResponse response) { + String existingIgnoredOptions = response.getValue(RESPONSE_KEY); + String newIgnoredOptions; + if (existingIgnoredOptions == null) { + newIgnoredOptions = invalidOptionKey; + } else { + newIgnoredOptions = existingIgnoredOptions + "," + invalidOptionKey; + } + + response.setValue(RESPONSE_KEY, newIgnoredOptions); + } + } + + /** The X-Trace-Options key is unknown */ + static class UnknownXTraceOptionException extends InvalidXTraceOptionException { + private static final long serialVersionUID = 1L; + + UnknownXTraceOptionException(String unknownOptionkey) { + super(unknownOptionkey, "Unknown key " + unknownOptionkey + " in X-Trace-Options header"); + } + } + + /** The x-trace-option value is not the expected type/value */ + public static class InvalidValueXTraceOptionException extends InvalidXTraceOptionException { + private static final long serialVersionUID = 1L; + + InvalidValueXTraceOptionException(XtraceOption optionKey, String invalidValue) { + super( + optionKey.getKey(), + "Invalid value [" + + invalidValue + + "] for option [" + + optionKey.getKey() + + "] in X-Trace-Options header"); + } + } + + /** + * The x-trace-option entry is in invalid format, for example it expects a key/value but the + * separator cannot be found + */ + static class InvalidFormatXTraceOptionException extends InvalidXTraceOptionException { + private static final long serialVersionUID = 1L; + + InvalidFormatXTraceOptionException(XtraceOption optionKey, String entry) { + super( + optionKey.getKey(), + "Invalid format for option entry [" + entry + "] in X-Trace-Options header"); + } + } + + @Getter + public static class AuthenticationStatus { + public static final AuthenticationStatus OK = new AuthenticationStatus(false, true, null); + public static final AuthenticationStatus NOT_AUTHENTICATED = + new AuthenticationStatus(false, false, null); + + /** + * -- GETTER -- Gets the reason of the authentication failure. If the authentication was + * successfully or no authentication is done, then this will be null + * + * @return + */ + private final String reason; + + /** + * -- GETTER -- Whether there is failure during the authentication. Take note that if no + * authentication was taken place (for example no signature), then this will be `false` + * + * @return + */ + private final boolean failure; + + /** + * -- GETTER -- Whether the request is authenticated + * + * @return + */ + private final boolean authenticated; + + private AuthenticationStatus(boolean failure, boolean authenticated, String reason) { + this.failure = failure; + this.authenticated = authenticated; + this.reason = reason; + } + + public static AuthenticationStatus failure(String reason) { + return new AuthenticationStatus(true, false, reason); + } + + @Override + public String toString() { + return "AuthenticationStatus{" + + "reason='" + + reason + + '\'' + + ", failure=" + + failure + + ", authenticated=" + + authenticated + + '}'; + } + + @Override + public boolean equals(Object o) { + if (this == o) return true; + if (o == null || getClass() != o.getClass()) return false; + + AuthenticationStatus that = (AuthenticationStatus) o; + + if (failure != that.failure) return false; + if (authenticated != that.authenticated) return false; + return Objects.equals(reason, that.reason); + } + + @Override + public int hashCode() { + int result = reason != null ? reason.hashCode() : 0; + result = 31 * result + (failure ? 1 : 0); + result = 31 * result + (authenticated ? 1 : 0); + return result; + } + } + + interface SignatureAuthenticator { + boolean authenticate(String optionsString, String signature); + } + + static class HmacSignatureAuthenticator implements SignatureAuthenticator { + private final Mac mac; + + HmacSignatureAuthenticator(byte[] secret) { + mac = getMac(secret); + } + + private Mac getMac(byte[] secret) { + SecretKeySpec signingKey = new SecretKeySpec(secret, "HMACSHA1"); + try { + Mac mac = Mac.getInstance("HMACSHA1"); + mac.init(signingKey); + + return mac; + } catch (GeneralSecurityException e) { + logger.warn("Failed to initialize HMAC for x-trace options: " + e.getMessage(), e); + return null; + } + } + + @Override + public boolean authenticate(String optionsString, String signature) { + byte[] rawHmac = mac.doFinal(optionsString.getBytes()); + String expectedSignature = HexUtils.bytesToHex(rawHmac).toLowerCase(); + return expectedSignature.equals(signature); + } + } +} diff --git a/libs/sampling/src/test/java/com/solarwinds/joboe/sampling/MetadataTest.java b/libs/sampling/src/test/java/com/solarwinds/joboe/sampling/MetadataTest.java new file mode 100644 index 00000000..e8a59382 --- /dev/null +++ b/libs/sampling/src/test/java/com/solarwinds/joboe/sampling/MetadataTest.java @@ -0,0 +1,204 @@ +/* + * © SolarWinds Worldwide, LLC. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.solarwinds.joboe.sampling; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertNotEquals; +import static org.junit.jupiter.api.Assertions.assertTrue; +import static org.mockito.Mockito.atLeastOnce; +import static org.mockito.Mockito.mockStatic; + +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.ExtendWith; +import org.mockito.ArgumentCaptor; +import org.mockito.Captor; +import org.mockito.MockedStatic; +import org.mockito.junit.jupiter.MockitoExtension; + +@ExtendWith(MockitoExtension.class) +public class MetadataTest { + + @Captor private ArgumentCaptor> listenerArgumentCaptor; + + @Test + public void testHexEncode() throws Exception { + // Make sure we can encode and decode hex strings + Metadata md1 = new Metadata(); + md1.randomize(); + + String hex1 = md1.toHexString(); + + Metadata md2 = new Metadata(); + md2.fromHexString(hex1); + + assertEquals(md1, md2); + } + + @Test + public void testRandomization() { + + // Make sure IDs are unique: + Metadata md1 = new Metadata(); + md1.randomize(); + + Metadata md2 = new Metadata(); + md2.randomize(); + + assertNotEquals(md1.toHexString(), md2.toHexString()); + + String hex2 = md2.toHexString(); + + md2.randomizeOpID(); + assertNotEquals(md2.toHexString(), hex2); + + Metadata md3 = new Metadata(md2); + assertEquals(md2, md3); + + // Make sure flag is set properly + Metadata md4 = new Metadata(); + md4.randomize(true); + assertTrue(md4.isSampled()); + + Metadata md5 = new Metadata(); + md5.randomize(false); + assertFalse(md5.isSampled()); + } + + @Test + public void testCompatibility() { + // should not accept trace id from different version + String v1Id = "01-4bf92f3577b34da6a3ce929d0e0e4736-00f067aa0ba902b7-01"; + String v0Id = "00-4bf92f3577b34da6a3ce929d0e0e4736-00f067aa0ba902b7-01"; + Metadata.setup(SamplingConfiguration.builder().build()); + + assertFalse(Metadata.isCompatible(v1Id)); + assertTrue(Metadata.isCompatible(v0Id)); + } + + @Test + public void testSampled() throws SamplingException { + String sampledId = "00-4bf92f3577b34da6a3ce929d0e0e4736-00f067aa0ba902b7-01"; + String notSampledId = "00-4bf92f3577b34da6a3ce929d0e0e4736-00f067aa0ba902b7-00"; + Metadata.setup(SamplingConfiguration.builder().build()); + + assertTrue(new Metadata(sampledId).isSampled()); + assertFalse(new Metadata(notSampledId).isSampled()); + + Metadata md = new Metadata(); + md.setSampled(true); + assertTrue(md.isSampled()); + md.setSampled(false); + assertFalse(md.isSampled()); + } + + @Test + public void testInit() { + + // Test initialization + Metadata md = new Metadata(); + assertFalse(md.isValid()); + + md.randomizeOpID(); + assertFalse(md.isValid()); + + md.randomizeTaskID(); + assertTrue(md.isValid()); + } + + @Test + public void testTtlChange() { + MockedStatic settingsManagerMock = mockStatic(SettingsManager.class); + assertEquals(Metadata.DEFAULT_TTL, Metadata.getTtl()); + Metadata.setup(SamplingConfiguration.builder().build()); + settingsManagerMock.verify( + () -> SettingsManager.registerListener(listenerArgumentCaptor.capture()), atLeastOnce()); + + int newTtl = 10; + listenerArgumentCaptor + .getAllValues() + .forEach( + integerSettingsArgChangeListener -> integerSettingsArgChangeListener.onChange(newTtl)); + assertEquals(newTtl * 1000, Metadata.getTtl()); // sec to millisec + + // revert to default + listenerArgumentCaptor + .getAllValues() + .forEach( + integerSettingsArgChangeListener -> integerSettingsArgChangeListener.onChange(null)); + assertEquals(Metadata.DEFAULT_TTL, Metadata.getTtl()); + settingsManagerMock.close(); + } + + @Test + public void testMaxEventsChange() { + MockedStatic settingsManagerMock = mockStatic(SettingsManager.class); + Metadata.setup(SamplingConfiguration.builder().build()); + assertEquals(Metadata.DEFAULT_MAX_EVENTS, Metadata.getMaxEvents()); + settingsManagerMock.verify( + () -> SettingsManager.registerListener(listenerArgumentCaptor.capture()), atLeastOnce()); + + int newMaxEvents = 100; + listenerArgumentCaptor + .getAllValues() + .forEach( + integerSettingsArgChangeListener -> + integerSettingsArgChangeListener.onChange(newMaxEvents)); + assertEquals(newMaxEvents, Metadata.getMaxEvents()); + + // revert to default + listenerArgumentCaptor + .getAllValues() + .forEach( + integerSettingsArgChangeListener -> integerSettingsArgChangeListener.onChange(null)); + + assertEquals(Metadata.DEFAULT_MAX_EVENTS, Metadata.getMaxEvents()); + settingsManagerMock.close(); + } + + @Test + public void testMaxBacktracesChange() { + MockedStatic settingsManagerMock = mockStatic(SettingsManager.class); + Metadata.setup(SamplingConfiguration.builder().build()); + assertEquals(Metadata.DEFAULT_MAX_BACKTRACES, Metadata.getMaxBacktraces()); + settingsManagerMock.verify( + () -> SettingsManager.registerListener(listenerArgumentCaptor.capture()), atLeastOnce()); + + int newMaxBacktraces = 100; + listenerArgumentCaptor + .getAllValues() + .forEach( + integerSettingsArgChangeListener -> + integerSettingsArgChangeListener.onChange(newMaxBacktraces)); + assertEquals(newMaxBacktraces, Metadata.getMaxBacktraces()); + + // revert to default + listenerArgumentCaptor + .getAllValues() + .forEach( + integerSettingsArgChangeListener -> integerSettingsArgChangeListener.onChange(null)); + assertEquals(Metadata.DEFAULT_MAX_BACKTRACES, Metadata.getMaxBacktraces()); + settingsManagerMock.close(); + } + + @Test + public static String getXTraceid(int version, boolean sampled) { + Metadata metadata = new Metadata(); + metadata.randomize(sampled); + return metadata.toHexString(version); + } +} diff --git a/libs/sampling/src/test/java/com/solarwinds/joboe/sampling/SettingsArgTest.java b/libs/sampling/src/test/java/com/solarwinds/joboe/sampling/SettingsArgTest.java new file mode 100644 index 00000000..629e579d --- /dev/null +++ b/libs/sampling/src/test/java/com/solarwinds/joboe/sampling/SettingsArgTest.java @@ -0,0 +1,109 @@ +/* + * © SolarWinds Worldwide, LLC. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.solarwinds.joboe.sampling; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertNull; +import static org.junit.jupiter.api.Assertions.assertTrue; + +import java.nio.ByteBuffer; +import java.nio.ByteOrder; +import java.util.Arrays; +import org.junit.jupiter.api.Test; + +public class SettingsArgTest { + + @Test + public void testDoubleSettingsArg() { + SettingsArg arg = new SettingsArg.DoubleSettingsArg("test-double"); + assertEquals("test-double", arg.getKey()); + ByteBuffer buffer = ByteBuffer.allocate(8).order(ByteOrder.LITTLE_ENDIAN); + buffer.putDouble(1.23); + buffer.flip(); + assertEquals(1.23, arg.readValue(buffer)); + } + + @Test + public void testIntegerSettingsArg() { + SettingsArg arg = new SettingsArg.IntegerSettingsArg("test-int"); + assertEquals("test-int", arg.getKey()); + ByteBuffer buffer = ByteBuffer.allocate(4).order(ByteOrder.LITTLE_ENDIAN); + buffer.putInt(3); + buffer.flip(); + assertEquals(3, (int) arg.readValue(buffer)); + } + + @Test + public void testBooleanSettingsArg() { + SettingsArg arg = new SettingsArg.BooleanSettingsArg("test-boolean"); + assertEquals("test-boolean", arg.getKey()); + ByteBuffer buffer = ByteBuffer.allocate(4).order(ByteOrder.LITTLE_ENDIAN); + buffer.putInt(1); // boolean uses int in bytebuffer (binary) + buffer.flip(); + assertEquals(true, arg.readValue(buffer)); + + buffer = ByteBuffer.allocate(4).order(ByteOrder.LITTLE_ENDIAN); + buffer.putInt(0); // boolean uses int in bytebuffer (binary) + buffer.flip(); + assertEquals(false, arg.readValue(buffer)); + } + + @Test + public void testDoubleSettingsArgReadValueWithObject() { + SettingsArg arg = new SettingsArg.DoubleSettingsArg("test-double"); + assertEquals(3.0, arg.readValue(3), 0); + assertEquals(3.45, arg.readValue(3.45), 0); + assertNull(arg.readValue("3.0")); + } + + @Test + public void testIntegerSettingsArgReadValueWithObject() { + SettingsArg arg = new SettingsArg.IntegerSettingsArg("test-int"); + assertEquals(3, arg.readValue(3)); + assertEquals(3, arg.readValue(3.0)); + assertNull(arg.readValue("3.0")); + } + + @Test + public void testBooleanSettingsArgReadValueWithObject() { + SettingsArg arg = new SettingsArg.BooleanSettingsArg("test-boolean"); + assertTrue(arg.readValue(true)); + assertFalse(arg.readValue(false)); + assertNull(arg.readValue("3.0")); + } + + @Test + public void testByteSettingsArgReadValueWithObject() { + SettingsArg arg = new SettingsArg.ByteArraySettingsArg("test-bytes"); + assertTrue(Arrays.equals(new byte[1], arg.readValue(new byte[1]))); + assertTrue(Arrays.equals("bytes".getBytes(), arg.readValue("bytes"))); + assertNull(arg.readValue(4)); + } + + @Test + public void testInvalidArg() { + SettingsArg arg = new SettingsArg.DoubleSettingsArg("test-invalid"); + assertEquals("test-invalid", arg.getKey()); + ByteBuffer buffer = + ByteBuffer.allocate(4) + .order(ByteOrder.LITTLE_ENDIAN); // 4 bytes will trigger buffer underflow + buffer.putInt(3); + buffer.flip(); + assertNull(arg.readValue(buffer)); // should just give null + } +} diff --git a/libs/sampling/src/test/java/com/solarwinds/joboe/sampling/SettingsManagerTest.java b/libs/sampling/src/test/java/com/solarwinds/joboe/sampling/SettingsManagerTest.java new file mode 100644 index 00000000..5ab99778 --- /dev/null +++ b/libs/sampling/src/test/java/com/solarwinds/joboe/sampling/SettingsManagerTest.java @@ -0,0 +1,169 @@ +/* + * © SolarWinds Worldwide, LLC. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.solarwinds.joboe.sampling; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertNull; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.when; + +import java.util.Collections; +import java.util.HashMap; +import java.util.Map; +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.TimeUnit; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.ExtendWith; +import org.mockito.ArgumentCaptor; +import org.mockito.Captor; +import org.mockito.Mock; +import org.mockito.junit.jupiter.MockitoExtension; + +@ExtendWith(MockitoExtension.class) +public class SettingsManagerTest { + + @Captor private ArgumentCaptor settingsListenerArgumentCaptor; + + @Mock private SettingsFetcher settingsFetcherMock; + + @Test + public void testArgChangeListener() { + TestArgChangeListener bucketCapacityListener = + new TestArgChangeListener(SettingsArg.BUCKET_CAPACITY); + TestArgChangeListener metricsFlushIntervalListener = + new TestArgChangeListener(SettingsArg.METRIC_FLUSH_INTERVAL); + SettingsManager.registerListener(bucketCapacityListener); + + SettingsManager.registerListener(metricsFlushIntervalListener); + SettingsManager.initializeFetcher(settingsFetcherMock); + verify(settingsFetcherMock).registerListener(settingsListenerArgumentCaptor.capture()); + + // test settings args + Map, Object> args = new HashMap, Object>(); + args.put(SettingsArg.BUCKET_CAPACITY, 1.0); + args.put(SettingsArg.METRIC_FLUSH_INTERVAL, 5); + + settingsListenerArgumentCaptor + .getValue() + .onSettingsRetrieved( + SettingsStub.builder() + .withFlags(TracingMode.ALWAYS) + .withSampleRate(TraceDecisionUtil.SAMPLE_RESOLUTION) + .withSettingsArgs(args) + .build()); + assertEquals(1.0, bucketCapacityListener.newValue); + assertEquals((Integer) 5, metricsFlushIntervalListener.newValue); + + // test settings args changes + args.clear(); + args.put(SettingsArg.BUCKET_CAPACITY, 2.0); + args.put(SettingsArg.METRIC_FLUSH_INTERVAL, 6); + + settingsListenerArgumentCaptor + .getValue() + .onSettingsRetrieved( + SettingsStub.builder() + .withFlags(TracingMode.ALWAYS) + .withSampleRate(TraceDecisionUtil.SAMPLE_RESOLUTION) + .withSettingsArgs(args) + .build()); + + assertEquals(2.0, bucketCapacityListener.newValue); + assertEquals((Integer) 6, metricsFlushIntervalListener.newValue); + + // test settings args set to null from non null + settingsListenerArgumentCaptor + .getValue() + .onSettingsRetrieved( + SettingsStub.builder() + .withFlags(TracingMode.ALWAYS) + .withSampleRate(TraceDecisionUtil.SAMPLE_RESOLUTION) + .withSettingsArgs(Collections.emptyMap()) + .build()); + + assertNull(bucketCapacityListener.newValue); + assertNull(metricsFlushIntervalListener.newValue); + + // one of the test args changed from null to some value + args.clear(); + args.put(SettingsArg.BUCKET_CAPACITY, 3.0); + settingsListenerArgumentCaptor + .getValue() + .onSettingsRetrieved( + SettingsStub.builder() + .withFlags(TracingMode.ALWAYS) + .withSampleRate(TraceDecisionUtil.SAMPLE_RESOLUTION) + .withSettingsArgs(args) + .build()); + + assertEquals(3.0, bucketCapacityListener.newValue); // the value has not been changed + assertNull(metricsFlushIntervalListener.newValue); // value has been changed + bucketCapacityListener.newValue = null; // reset listener + + // test values unchanged + assertNull(bucketCapacityListener.newValue); // the value has not been changed + assertNull(metricsFlushIntervalListener.newValue); // the value has not been changed + + SettingsManager.removeListener(bucketCapacityListener); + SettingsManager.removeListener(metricsFlushIntervalListener); + } + + @Test + public void testGetSettings() { + final CountDownLatch countDownLatch = new CountDownLatch(1); + when(settingsFetcherMock.getSettings()) + .thenReturn( + SettingsStub.builder() + .withFlags(TracingMode.ALWAYS) + .withSampleRate(TraceDecisionUtil.SAMPLE_RESOLUTION) + .build()); + when(settingsFetcherMock.isSettingsAvailableLatch()).thenReturn(countDownLatch); + + // simulate a 5 secs delay for fetching + new Thread() { + @Override + public void run() { + try { + TimeUnit.SECONDS.sleep(5); + countDownLatch.countDown(); + } catch (InterruptedException e) { + e.printStackTrace(); + } + } + }.start(); + + SettingsManager.initializeFetcher(settingsFetcherMock); + + assert (SettingsManager.getSettings(2, TimeUnit.SECONDS) + == null); // should give null, as 2 sec < 5 sec + assert (SettingsManager.getSettings(4, TimeUnit.SECONDS) + != null); // should not be null as 6 sec > 5 sec + } + + private static class TestArgChangeListener extends SettingsArgChangeListener { + private T newValue; + + public TestArgChangeListener(SettingsArg type) { + super(type); + } + + @Override + public void onChange(T newValue) { + this.newValue = newValue; + } + } +} diff --git a/libs/sampling/src/test/java/com/solarwinds/joboe/sampling/SettingsStub.java b/libs/sampling/src/test/java/com/solarwinds/joboe/sampling/SettingsStub.java new file mode 100644 index 00000000..c6638ff7 --- /dev/null +++ b/libs/sampling/src/test/java/com/solarwinds/joboe/sampling/SettingsStub.java @@ -0,0 +1,165 @@ +/* + * © SolarWinds Worldwide, LLC. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.solarwinds.joboe.sampling; + +import java.util.HashMap; +import java.util.Map; + +public class SettingsStub extends Settings { + private final short settingsType; + private short flags = 0; + private int sampleRate = 1000000; + private final Map, Object> args; + + private SettingsStub( + short settingsType, short flags, Integer sampleRate, Map, Object> args) { + this.settingsType = settingsType; + this.flags = flags; + if (sampleRate != null) { + this.sampleRate = sampleRate; + } + this.args = args; + } + + @Override + public long getValue() { + return sampleRate; + } + + @Override + public long getTimestamp() { + return 0; + } + + @Override + public short getFlags() { + return flags; + } + + @Override + @SuppressWarnings("unchecked") + public T getArgValue(SettingsArg arg) { + return (T) args.get(arg); + } + + @Override + public short getType() { + return settingsType; + } + + @Override + public long getTtl() { + return 0; + } + + public static SettingsStubBuilder builder() { + return new SettingsStubBuilder(); + } + + public static class SettingsStubBuilder { + private short flags = 0; + private Integer sampleRate = null; + private final Map, Object> args = new HashMap, Object>(); + + public static final double DEFAULT_TOKEN_BUCKET_RATE = 8.0; + public static final double DEFAULT_TOKEN_BUCKET_CAPACITY = 16.0; + private short settingsType = Settings.OBOE_SETTINGS_TYPE_DEFAULT_SAMPLE_RATE; + + public SettingsStubBuilder() { + addDefaultArgs(); + } + + private void addDefaultArgs() { + args.put(SettingsArg.BUCKET_RATE, DEFAULT_TOKEN_BUCKET_RATE); + args.put(SettingsArg.BUCKET_CAPACITY, DEFAULT_TOKEN_BUCKET_CAPACITY); + args.put(SettingsArg.RELAXED_BUCKET_RATE, DEFAULT_TOKEN_BUCKET_RATE); + args.put(SettingsArg.RELAXED_BUCKET_CAPACITY, DEFAULT_TOKEN_BUCKET_CAPACITY); + args.put(SettingsArg.STRICT_BUCKET_RATE, DEFAULT_TOKEN_BUCKET_RATE); + args.put(SettingsArg.STRICT_BUCKET_CAPACITY, DEFAULT_TOKEN_BUCKET_CAPACITY); + } + + public SettingsStubBuilder withFlags( + boolean isStart, + boolean isThrough, + boolean isThroughAlways, + boolean isTriggerTraceEnabled, + boolean isOverride) { + flags = getFlags(isStart, isThrough, isThroughAlways, isTriggerTraceEnabled, isOverride); + return this; + } + + public SettingsStubBuilder withFlags(TracingMode tracingMode) { + flags |= tracingMode.toFlags(); + return this; + } + + public SettingsStubBuilder withSampleRate(int sampleRate) { + this.sampleRate = sampleRate; + return this; + } + + public SettingsStubBuilder withSettingsArg(SettingsArg arg, T value) { + args.put(arg, value); + return this; + } + + public SettingsStubBuilder withSettingsArgs(Map, ?> args) { + this.args.clear(); + this.args.putAll(args); + return this; + } + + public SettingsStubBuilder withSettingsType(short settingsType) { + this.settingsType = settingsType; + return this; + } + + public SettingsStub build() { + return new SettingsStub(settingsType, flags, sampleRate, args); + } + + private static short getFlags( + boolean isStart, + boolean isThrough, + boolean isThroughAlways, + boolean isTriggerTraceEnabled, + boolean isOverride) { + byte flags = 0; + if (isOverride) { + flags |= Settings.OBOE_SETTINGS_FLAG_OVERRIDE; + } + + if (isStart) { + flags |= Settings.OBOE_SETTINGS_FLAG_SAMPLE_START; + } + + if (isThrough) { + flags |= Settings.OBOE_SETTINGS_FLAG_SAMPLE_THROUGH; + } + + if (isThroughAlways) { + flags |= Settings.OBOE_SETTINGS_FLAG_SAMPLE_THROUGH_ALWAYS; + } + + if (isTriggerTraceEnabled) { + flags |= Settings.OBOE_SETTINGS_FLAG_TRIGGER_TRACE_ENABLED; + } + + return flags; + } + } +} diff --git a/libs/sampling/src/test/java/com/solarwinds/joboe/sampling/TokenBucketTest.java b/libs/sampling/src/test/java/com/solarwinds/joboe/sampling/TokenBucketTest.java new file mode 100644 index 00000000..5f63ef83 --- /dev/null +++ b/libs/sampling/src/test/java/com/solarwinds/joboe/sampling/TokenBucketTest.java @@ -0,0 +1,135 @@ +/* + * © SolarWinds Worldwide, LLC. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.solarwinds.joboe.sampling; + +import static org.junit.jupiter.api.Assertions.*; + +import org.junit.jupiter.api.Test; + +public class TokenBucketTest { + @Test + public void testCapacity1() throws Exception { + + TokenBucket bucket = new TokenBucket(100.0, 0.0); + + for (int i = 0; i < 100; i++) { + assertTrue(bucket.consume()); + } + assertFalse(bucket.consume()); + } + + @Test + public void testCapacity2() { + // default capacity + TokenBucket bucket = new TokenBucket(0, 0); + + assertFalse(bucket.consume()); + } + + @Test + public void testReset1() { + TokenBucket bucket = new TokenBucket(0.0, 0.0); + + assertFalse(bucket.consume()); + + // now set it to higher capacity, it should not affect available tokens + bucket.setCapacity(100); + + assertFalse(bucket.consume()); + } + + @Test + public void testReset2() { + TokenBucket bucket = new TokenBucket(100.0, 0.0); + + assertTrue(bucket.consume()); + + // now set it to lower capacity, the existing available token should be lowered accordingly + bucket.setCapacity(0); + + assertFalse(bucket.consume()); + } + + @Test + public void testReset3() { + TokenBucket bucket = new TokenBucket(1.0, 0.0); + + assertTrue(bucket.consume()); + + bucket.setCapacity(1.0); // same capacity, does not affect the available tokens + + assertFalse(bucket.consume()); + } + + @Test + public void testRate1() throws InterruptedException { + TokenBucket bucket = new TokenBucket(1000.0, 1.0); + + final int maxTry = + 2000; // avoid infinity loop if there's any problem on the TokenBucket instrumentation + int counter = 0; + while (bucket.consume() && counter < maxTry) { // should quickly deplete the available tokens + counter++; + } + + if (counter + == maxTry) { // should not reach this unless TokenBucket is not functioning correctly + fail("TokenBucket is expected to be exhausted but it was not!"); + } + + // now wait for a while for the bucket to be replenished + Thread.sleep(1500); + assertTrue(bucket.consume()); + } + + @Test + public void testRate2() throws InterruptedException { + TokenBucket bucket = new TokenBucket(10.0, 1000.0); + + final int maxTry = 1000; + int counter = 0; + while (bucket.consume() && counter < maxTry) { + counter++; + Thread.sleep(2); // replenish rate is faster than deplete rate + } + + if (counter < maxTry) { // should reach this unless TokenBucket is not functioning correctly + fail( + "TokenBucket is NOT expected to be exhausted but it actually ran out of available token!"); + } + } + + @Test + public void testRate3() throws InterruptedException { + TokenBucket bucket = new TokenBucket(10.0, 10.0); + + for (int i = 0; i < 100; i++) { + assertTrue(bucket.consume()); + Thread.sleep(100); // roughly 1000/100 = 10 request per second which is the same as rate + } + } + + @Test + public void testConsume() { + TokenBucket bucket = new TokenBucket(3.00, 0.00); + + assertTrue(bucket.consume()); // OK, 2 left + assertFalse(bucket.consume(3)); // only 2 left, not enough token + assertTrue(bucket.consume(2)); // OK, 0 left + assertFalse(bucket.consume()); // none left + } +} diff --git a/libs/sampling/src/test/java/com/solarwinds/joboe/sampling/TraceDecisionUtilTest.java b/libs/sampling/src/test/java/com/solarwinds/joboe/sampling/TraceDecisionUtilTest.java new file mode 100644 index 00000000..c1118fc1 --- /dev/null +++ b/libs/sampling/src/test/java/com/solarwinds/joboe/sampling/TraceDecisionUtilTest.java @@ -0,0 +1,1403 @@ +/* + * © SolarWinds Worldwide, LLC. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.solarwinds.joboe.sampling; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertNotNull; +import static org.junit.jupiter.api.Assertions.assertTrue; +import static org.mockito.Mockito.when; + +import java.util.AbstractMap; +import java.util.ArrayList; +import java.util.Collections; +import java.util.HashMap; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; +import java.util.concurrent.Callable; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; +import java.util.concurrent.TimeUnit; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.ExtendWith; +import org.mockito.Mock; +import org.mockito.junit.jupiter.MockitoExtension; + +@ExtendWith(MockitoExtension.class) +public class TraceDecisionUtilTest { + private static final String TEST_LAYER = "test"; + + private static final String X_TRACE_ID_SAMPLED = + MetadataTest.getXTraceid(Metadata.CURRENT_VERSION, true); + private static final String X_TRACE_ID_NOT_SAMPLED = + MetadataTest.getXTraceid(Metadata.CURRENT_VERSION, false); + private static final String X_TRACE_ID_INCOMPATIBLE = + MetadataTest.getXTraceid(Metadata.CURRENT_VERSION + 1, true); + private static final String X_TRACE_ID_ALL_ZEROS = new Metadata().toHexString(); + private static final String X_TRACE_ID_INCORRECT_FORMAT = "XYZ"; + + private static final XtraceOptions TRIGGER_TRACE_OPTIONS = + XtraceOptions.getXTraceOptions("trigger-trace", null); + + @Mock private SettingsFetcher settingsFetcherMock; + + @BeforeEach + void setup() { + SettingsManager.initialize(settingsFetcherMock, SamplingConfiguration.builder().build()); + TraceDecisionUtil.reset(); + } + + @Test + public void testShouldTraceRequest() throws Exception { + // tracing mode NEVER + when(settingsFetcherMock.getSettings()) + .thenReturn(SettingsStub.builder().withFlags(false, false, false, false, true).build()); + + assertFalse(TraceDecisionUtil.shouldTraceRequest(TEST_LAYER, null, null, null).isSampled()); + assertFalse(TraceDecisionUtil.shouldTraceRequest(TEST_LAYER, null, null, null).isSampled()); + assertFalse( + TraceDecisionUtil.shouldTraceRequest(TEST_LAYER, X_TRACE_ID_SAMPLED, null, null) + .isSampled()); + assertFalse( + TraceDecisionUtil.shouldTraceRequest(TEST_LAYER, X_TRACE_ID_NOT_SAMPLED, null, null) + .isSampled()); + assertFalse( + TraceDecisionUtil.shouldTraceRequest(TEST_LAYER, X_TRACE_ID_INCOMPATIBLE, null, null) + .isSampled()); + assertFalse( + TraceDecisionUtil.shouldTraceRequest(TEST_LAYER, X_TRACE_ID_ALL_ZEROS, null, null) + .isSampled()); + assertFalse( + TraceDecisionUtil.shouldTraceRequest(TEST_LAYER, X_TRACE_ID_INCORRECT_FORMAT, null, null) + .isSampled()); + + // tracing mode ALWAYS + when(settingsFetcherMock.getSettings()) + .thenReturn(SettingsStub.builder().withFlags(true, false, true, true, true).build()); + + assertTrue(TraceDecisionUtil.shouldTraceRequest(TEST_LAYER, null, null, null).isSampled()); + assertTrue( + TraceDecisionUtil.shouldTraceRequest(TEST_LAYER, X_TRACE_ID_SAMPLED, null, null) + .isSampled()); + assertTrue( + TraceDecisionUtil.shouldTraceRequest(TEST_LAYER, null, TRIGGER_TRACE_OPTIONS, null) + .isSampled()); + assertFalse( + TraceDecisionUtil.shouldTraceRequest(TEST_LAYER, X_TRACE_ID_NOT_SAMPLED, null, null) + .isSampled()); // if upstream decides to not sample this, then it should not continue + assertTrue( + TraceDecisionUtil.shouldTraceRequest(TEST_LAYER, X_TRACE_ID_INCOMPATIBLE, null, null) + .isSampled()); + assertTrue( + TraceDecisionUtil.shouldTraceRequest(TEST_LAYER, X_TRACE_ID_ALL_ZEROS, null, null) + .isSampled()); + assertTrue( + TraceDecisionUtil.shouldTraceRequest(TEST_LAYER, X_TRACE_ID_INCORRECT_FORMAT, null, null) + .isSampled()); + + // tracing mode THROUGH_ALWAYS + when(settingsFetcherMock.getSettings()) + .thenReturn(SettingsStub.builder().withFlags(false, false, true, false, true).build()); + + assertFalse(TraceDecisionUtil.shouldTraceRequest(TEST_LAYER, null, null, null).isSampled()); + assertFalse( + TraceDecisionUtil.shouldTraceRequest(TEST_LAYER, null, TRIGGER_TRACE_OPTIONS, null) + .isSampled()); + assertTrue( + TraceDecisionUtil.shouldTraceRequest(TEST_LAYER, X_TRACE_ID_SAMPLED, null, null) + .isSampled()); + assertFalse( + TraceDecisionUtil.shouldTraceRequest(TEST_LAYER, X_TRACE_ID_NOT_SAMPLED, null, null) + .isSampled()); // if upstream decides to not sample this, then it should not continue + assertFalse( + TraceDecisionUtil.shouldTraceRequest(TEST_LAYER, X_TRACE_ID_INCOMPATIBLE, null, null) + .isSampled()); // incompatible x-trace header, so x-trace id is ignored + assertFalse( + TraceDecisionUtil.shouldTraceRequest(TEST_LAYER, X_TRACE_ID_ALL_ZEROS, null, null) + .isSampled()); // invalid x-trace header, so x-trace id is ignored + assertFalse( + TraceDecisionUtil.shouldTraceRequest(TEST_LAYER, X_TRACE_ID_INCORRECT_FORMAT, null, null) + .isSampled()); // incorrect format x-trace header, so x-trace id is ignored + } + + @Test + public void testNoSettings() throws Exception { + when(settingsFetcherMock.getSettings()) + .thenReturn(SettingsStub.builder().withFlags(false, false, false, false, true).build()); + + // do not trace in any situation if settings not available + // https://tracelytics.atlassian.net/browse/TVI-1588 + assertFalse( + TraceDecisionUtil.shouldTraceRequest("NotFoundLayer", null, null, null).isSampled()); + assertFalse( + TraceDecisionUtil.shouldTraceRequest("NotFoundLayer", X_TRACE_ID_SAMPLED, null, null) + .isSampled()); + } + + @Test + public void testPrecedence() throws Exception { + when(settingsFetcherMock.getSettings()) + .thenReturn(SettingsStub.builder().withFlags(false, false, false, false, true).build()); + + // case 1: set remote TracingMode = ENABLED with no override, no local settings + // local universal : null/null + // remote : ENABLED/100% (override OFF) + when(settingsFetcherMock.getSettings()) + .thenReturn( + SettingsStub.builder() + .withFlags(true, false, true, true, false) + .withSettingsType(Settings.OBOE_SETTINGS_TYPE_LAYER_SAMPLE_RATE) + .build()); + assertTrue(TraceDecisionUtil.shouldTraceRequest(TEST_LAYER, null, null, null).isSampled()); + assertTrue( + TraceDecisionUtil.shouldTraceRequest(TEST_LAYER, X_TRACE_ID_SAMPLED, null, null) + .isSampled()); + + // case 2: set remote TracingMode = ENABLED with override, no local settings + // local universal : null/null + // remote : ENABLED/100% + when(settingsFetcherMock.getSettings()) + .thenReturn( + SettingsStub.builder() + .withFlags(true, false, true, true, true) + .withSettingsType(Settings.OBOE_SETTINGS_TYPE_LAYER_SAMPLE_RATE) + .build()); + assertTrue(TraceDecisionUtil.shouldTraceRequest(TEST_LAYER, null, null, null).isSampled()); + assertTrue( + TraceDecisionUtil.shouldTraceRequest(TEST_LAYER, X_TRACE_ID_SAMPLED, null, null) + .isSampled()); + + // case 3: set local TracingMode = ENABLED with no sample rate + // local universal : ENABLED/null + // remote : ENABLED/100% + SettingsManager.initialize( + settingsFetcherMock, + SamplingConfiguration.builder().tracingMode(TracingMode.ENABLED).build()); + when(settingsFetcherMock.getSettings()) + .thenReturn( + SettingsStub.builder() + .withFlags(true, false, true, true, true) + .withSettingsType(Settings.OBOE_SETTINGS_TYPE_LAYER_SAMPLE_RATE) + .build()); + + // should still trace + assertTrue(TraceDecisionUtil.shouldTraceRequest(TEST_LAYER, null, null, null).isSampled()); + assertTrue( + TraceDecisionUtil.shouldTraceRequest(TEST_LAYER, X_TRACE_ID_SAMPLED, null, null) + .isSampled()); + + // case 4: set no local TracingMode, with sample rate = 0% + // local universal : null/0% + // remote : ENABLED/100% + SettingsManager.initialize( + settingsFetcherMock, SamplingConfiguration.builder().sampleRate(0).build()); + when(settingsFetcherMock.getSettings()) + .thenReturn( + SettingsStub.builder() + .withFlags(true, false, true, true, true) + .withSettingsType(Settings.OBOE_SETTINGS_TYPE_LAYER_SAMPLE_RATE) + .build()); + // should not start trace + assertFalse(TraceDecisionUtil.shouldTraceRequest(TEST_LAYER, null, null, null).isSampled()); + // but it should continue trace + assertTrue( + TraceDecisionUtil.shouldTraceRequest(TEST_LAYER, X_TRACE_ID_SAMPLED, null, null) + .isSampled()); + + // case 5: set local TracingMode = DISABLED, with no sample Rate + // local universal : DISABLED/null + // remote : ENABLED/100% + SettingsManager.initialize( + settingsFetcherMock, + SamplingConfiguration.builder().tracingMode(TracingMode.DISABLED).build()); + when(settingsFetcherMock.getSettings()) + .thenReturn( + SettingsStub.builder() + .withFlags(true, false, true, true, true) + .withSettingsType(Settings.OBOE_SETTINGS_TYPE_LAYER_SAMPLE_RATE) + .build()); + // should not trace + assertFalse(TraceDecisionUtil.shouldTraceRequest(TEST_LAYER, null, null, null).isSampled()); + assertFalse( + TraceDecisionUtil.shouldTraceRequest(TEST_LAYER, X_TRACE_ID_SAMPLED, null, null) + .isSampled()); + + // case 6: add URL settings with Sample rate = 1000000 + // local URL : ENBALED/100% + // local universal : DISABLED/null + // remote : ENABLED/100% + TraceConfigs testingUrlSampleRateConfigs = + buildUrlConfigs(url -> true, TracingMode.ALWAYS, 1000000); + SettingsManager.initialize( + settingsFetcherMock, + SamplingConfiguration.builder() + .internalTransactionSettings(testingUrlSampleRateConfigs) + .tracingMode(TracingMode.DISABLED) + .build()); + + when(settingsFetcherMock.getSettings()) + .thenReturn( + SettingsStub.builder() + .withFlags(true, false, true, true, true) + .withSettingsType(Settings.OBOE_SETTINGS_TYPE_LAYER_SAMPLE_RATE) + .build()); + + assertTrue( + TraceDecisionUtil.shouldTraceRequest( + TEST_LAYER, null, null, Collections.singletonList("http://something.html")) + .isSampled()); + assertTrue( + TraceDecisionUtil.shouldTraceRequest( + TEST_LAYER, + X_TRACE_ID_SAMPLED, + null, + Collections.singletonList("http://something.html")) + .isSampled()); + + // case 7: add transaction settings with Trace modes with no sample rate + // local transaction settings (*.png, *.jpg) : DISABLED/0% + // local transaction settings (*.trace.*) : ENABLED/100% + // local universal : DISABLED/null + // remote : ENABLED/100% + Map urlTraceConfigsByMatcher = new LinkedHashMap<>(); + urlTraceConfigsByMatcher.put( + url -> url.endsWith("png") || url.endsWith("jpg"), + buildTraceConfig(TracingMode.DISABLED, 0)); + urlTraceConfigsByMatcher.put( + url -> url.contains("trace"), buildTraceConfig(TracingMode.ENABLED, null)); + SettingsManager.initialize( + settingsFetcherMock, + SamplingConfiguration.builder() + .internalTransactionSettings(new TraceConfigs(urlTraceConfigsByMatcher)) + .tracingMode(TracingMode.DISABLED) + .build()); + + when(settingsFetcherMock.getSettings()) + .thenReturn( + SettingsStub.builder() + .withFlags(true, false, true, true, true) + .withSettingsType(Settings.OBOE_SETTINGS_TYPE_LAYER_SAMPLE_RATE) + .build()); + + assertFalse( + TraceDecisionUtil.shouldTraceRequest( + TEST_LAYER, null, null, Collections.singletonList("http://something.png")) + .isSampled()); + assertEquals( + SampleRateSource.FILE, + TraceDecisionUtil.shouldTraceRequest( + TEST_LAYER, null, null, Collections.singletonList("http://something.png")) + .getTraceConfig() + .getSampleRateSource()); + assertTrue( + TraceDecisionUtil.shouldTraceRequest( + TEST_LAYER, null, null, Collections.singletonList("http://trace-this")) + .isSampled()); + assertEquals( + SampleRateSource.OBOE, + TraceDecisionUtil.shouldTraceRequest( + TEST_LAYER, null, null, Collections.singletonList("http://trace-this")) + .getTraceConfig() + .getSampleRateSource()); // source is from OBOE as rate is NOT defined in local config + } + + @Test + public void testUrlConfigs() throws Exception { + // Add SRv1 always, override, sampling rate 1000000 + when(settingsFetcherMock.getSettings()) + .thenReturn( + SettingsStub.builder() + .withFlags(true, false, true, true, true) + .withSampleRate(1000000) + .withSettingsType(Settings.OBOE_SETTINGS_TYPE_LAYER_SAMPLE_RATE) + .build()); + + assertEquals(1000000, TraceDecisionUtil.getRemoteTraceConfig().getSampleRate()); + assertTrue( + TraceDecisionUtil.shouldTraceRequest( + TEST_LAYER, null, null, Collections.singletonList("http://something.html")) + .isSampled()); + assertTrue( + TraceDecisionUtil.shouldTraceRequest( + TEST_LAYER, + X_TRACE_ID_SAMPLED, + null, + Collections.singletonList("http://something.html")) + .isSampled()); + assertTrue( + TraceDecisionUtil.shouldTraceRequest( + TEST_LAYER, null, null, Collections.singletonList("http://something.html")) + .isReportMetrics()); + + // Add local URL rate, should override the SRv1 rate if pattern matches + TraceConfigs testingUrlSampleRateConfigs = + buildUrlConfigs(url -> url.endsWith(".html"), TracingMode.ALWAYS, 0); + SettingsManager.initialize( + settingsFetcherMock, + SamplingConfiguration.builder() + .internalTransactionSettings(testingUrlSampleRateConfigs) + .build()); + + // pattern match, should all have sample rate 0% for new traces, but continuing/AVW trace should + // still go on + assertFalse( + TraceDecisionUtil.shouldTraceRequest( + TEST_LAYER, null, null, Collections.singletonList("http://something.html")) + .isSampled()); + assertTrue( + TraceDecisionUtil.shouldTraceRequest( + TEST_LAYER, + X_TRACE_ID_SAMPLED, + null, + Collections.singletonList("http://something.html")) + .isSampled()); + assertTrue( + TraceDecisionUtil.shouldTraceRequest( + TEST_LAYER, null, null, Collections.singletonList("http://something.html")) + .isReportMetrics()); // metrics should still be reported even for 0% rate + + // pattern not match, take the Srv1 override with sample rate 100% + assertTrue( + TraceDecisionUtil.shouldTraceRequest( + TEST_LAYER, null, null, Collections.singletonList("http://something.xxx")) + .isSampled()); + assertTrue( + TraceDecisionUtil.shouldTraceRequest( + TEST_LAYER, + X_TRACE_ID_SAMPLED, + null, + Collections.singletonList("http://something.xxx")) + .isSampled()); + assertTrue( + TraceDecisionUtil.shouldTraceRequest( + TEST_LAYER, null, null, Collections.singletonList("http://something.xxx")) + .isReportMetrics()); + + // Add local URL rate, should override the SRv1 rate if pattern matches + testingUrlSampleRateConfigs = + buildUrlConfigs(url -> url.endsWith(".html"), TracingMode.NEVER, 0); + SettingsManager.initialize( + settingsFetcherMock, + SamplingConfiguration.builder() + .internalTransactionSettings(testingUrlSampleRateConfigs) + .build()); + + // pattern match, should block all traffic even for continuing traces since the url tracingMode + // is never + assertFalse( + TraceDecisionUtil.shouldTraceRequest( + TEST_LAYER, null, null, Collections.singletonList("http://something.html")) + .isSampled()); + assertFalse( + TraceDecisionUtil.shouldTraceRequest( + TEST_LAYER, + X_TRACE_ID_SAMPLED, + null, + Collections.singletonList("http://something.html")) + .isSampled()); + assertFalse( + TraceDecisionUtil.shouldTraceRequest( + TEST_LAYER, + X_TRACE_ID_SAMPLED, + null, + Collections.singletonList("http://something.html")) + .isReportMetrics()); // trace mode never disables metrics too + + // pattern not match, take the Srv1 override with sample rate 100% + assertTrue( + TraceDecisionUtil.shouldTraceRequest( + TEST_LAYER, null, null, Collections.singletonList("http://something.xxx")) + .isSampled()); + assertTrue( + TraceDecisionUtil.shouldTraceRequest( + TEST_LAYER, + X_TRACE_ID_SAMPLED, + null, + Collections.singletonList("http://something.xxx")) + .isSampled()); + assertTrue( + TraceDecisionUtil.shouldTraceRequest( + TEST_LAYER, + X_TRACE_ID_SAMPLED, + null, + Collections.singletonList("http://something.xxx")) + .isReportMetrics()); + + Map urlTraceConfigsByMatcher = new LinkedHashMap<>(); + urlTraceConfigsByMatcher.put( + url -> url.endsWith("png") || url.endsWith("jpg"), + buildTraceConfig(TracingMode.DISABLED, 0)); + urlTraceConfigsByMatcher.put( + url -> url.contains("trace"), buildTraceConfig(TracingMode.ENABLED, null)); + SettingsManager.initialize( + settingsFetcherMock, + SamplingConfiguration.builder() + .internalTransactionSettings(new TraceConfigs(urlTraceConfigsByMatcher)) + .build()); + TraceDecision traceDecision; + + // pattern match on "disabled", should block all traffic even for continuing traces since the + // transaction settings tracingMode is disabled + traceDecision = + TraceDecisionUtil.shouldTraceRequest( + TEST_LAYER, null, null, Collections.singletonList("http://something.png")); + assertFalse(traceDecision.isSampled()); + assertEquals( + 0, + traceDecision + .getTraceConfig() + .getSampleRate()); // rate is coming from the local transaction settings + assertEquals( + SampleRateSource.FILE, + traceDecision + .getTraceConfig() + .getSampleRateSource()); // rate is coming from the local transaction settings + assertFalse( + TraceDecisionUtil.shouldTraceRequest( + TEST_LAYER, + X_TRACE_ID_SAMPLED, + null, + Collections.singletonList("http://something.png")) + .isSampled()); + assertFalse( + TraceDecisionUtil.shouldTraceRequest( + TEST_LAYER, + X_TRACE_ID_SAMPLED, + null, + Collections.singletonList("http://something.png")) + .isReportMetrics()); // trace mode never disables metrics too + + // pattern match on "enabled", take the flags from local config but sample rate from remote + // settings + traceDecision = + TraceDecisionUtil.shouldTraceRequest( + TEST_LAYER, null, null, Collections.singletonList("http://trace.com")); + assertTrue(traceDecision.isSampled()); + assertEquals( + 1000000, + traceDecision.getTraceConfig().getSampleRate()); // rate is coming from the remote settings + assertEquals( + SampleRateSource.OBOE, + traceDecision + .getTraceConfig() + .getSampleRateSource()); // rate is coming from the remote settings + assertTrue( + TraceDecisionUtil.shouldTraceRequest( + TEST_LAYER, X_TRACE_ID_SAMPLED, null, Collections.singletonList("http://trace.com")) + .isSampled()); + assertTrue( + TraceDecisionUtil.shouldTraceRequest( + TEST_LAYER, X_TRACE_ID_SAMPLED, null, Collections.singletonList("http://trace.com")) + .isReportMetrics()); + + // pattern not match, take the Srv1 override with sample rate 100% + traceDecision = + TraceDecisionUtil.shouldTraceRequest( + TEST_LAYER, null, null, Collections.singletonList("http://something.xxx")); + assertTrue(traceDecision.isSampled()); + assertEquals( + 1000000, + traceDecision.getTraceConfig().getSampleRate()); // rate is coming from the remote settings + assertEquals( + SampleRateSource.OBOE, + traceDecision + .getTraceConfig() + .getSampleRateSource()); // rate is coming from the remote settings + assertTrue( + TraceDecisionUtil.shouldTraceRequest( + TEST_LAYER, + X_TRACE_ID_SAMPLED, + null, + Collections.singletonList("http://something.xxx")) + .isSampled()); + assertTrue( + TraceDecisionUtil.shouldTraceRequest( + TEST_LAYER, + X_TRACE_ID_SAMPLED, + null, + Collections.singletonList("http://something.xxx")) + .isReportMetrics()); + } + + @Test + public void testThroughput() throws Exception { + when(settingsFetcherMock.getSettings()) + .thenReturn(SettingsStub.builder().withFlags(true, false, true, true, false).build()); + + int data; + TraceDecisionUtil.consumeMetricsData(TraceDecisionUtil.MetricType.THROUGHPUT); // clear it + + TraceDecisionUtil.shouldTraceRequest("LayerA", null, null, null); + TraceDecisionUtil.shouldTraceRequest("LayerA", null, null, null); + TraceDecisionUtil.shouldTraceRequest( + "LayerC", X_TRACE_ID_SAMPLED, null, null); // Continue trace, should also count + + data = TraceDecisionUtil.consumeMetricsData(TraceDecisionUtil.MetricType.THROUGHPUT); + + assertEquals(3, data); + + data = + TraceDecisionUtil.consumeMetricsData( + TraceDecisionUtil.MetricType + .THROUGHPUT); // consumed once already, so this time should return 0 + assertEquals(0, data); + } + + @Test + public void testThroughputConcurrency() throws Exception { + when(settingsFetcherMock.getSettings()) + .thenReturn(SettingsStub.builder().withFlags(true, false, true, true, false).build()); + + TraceDecisionUtil.consumeMetricsData(TraceDecisionUtil.MetricType.THROUGHPUT); // clear it + + final int THREAD_COUNT = 1000; + final int RUN_PER_THREAD = 100; + + ExecutorService executorService = Executors.newFixedThreadPool(THREAD_COUNT); + List> tasks = new ArrayList>(); + for (int i = 0; i < THREAD_COUNT; i++) { + tasks.add( + () -> { + for (int i1 = 0; i1 < RUN_PER_THREAD; i1++) { + TraceDecisionUtil.shouldTraceRequest("LayerA", null, null, null); + } + return null; + }); + } + + executorService.invokeAll(tasks); + + int data = TraceDecisionUtil.consumeMetricsData(TraceDecisionUtil.MetricType.THROUGHPUT); + assertEquals(THREAD_COUNT * RUN_PER_THREAD, data); + + // try to do increment and clear at once + tasks.clear(); + for (int i = 0; i < THREAD_COUNT; i++) { + tasks.add( + () -> { + TraceDecisionUtil.shouldTraceRequest("LayerA", null, null, null); + int data1 = + TraceDecisionUtil.consumeMetricsData(TraceDecisionUtil.MetricType.THROUGHPUT); + return data1; + }); + } + + executorService.invokeAll( + tasks); // do not really need to assert, we just want to make sure no exceptions is + // triggered. The number could be a bit off and it's acceptable + executorService.shutdown(); + executorService.awaitTermination(10, TimeUnit.SECONDS); + } + + @Test + public void testTokenBucketExhaustion() throws Exception { + TraceDecisionUtil.consumeMetricsData( + TraceDecisionUtil.MetricType.TOKEN_BUCKET_EXHAUSTION); // clear it + SettingsManager.initialize(settingsFetcherMock, SamplingConfiguration.builder().build()); + + when(settingsFetcherMock.getSettings()) + .thenReturn( + SettingsStub.builder() + .withFlags(true, false, true, true, false) + .withSampleRate(1000000) + .withSettingsArg(SettingsArg.BUCKET_CAPACITY, 0.0) + .withSettingsArg(SettingsArg.BUCKET_RATE, 0.0) + .build()); + TraceDecisionUtil.shouldTraceRequest("LayerA", null, null, null); // exhaustion +1 + TraceDecisionUtil.shouldTraceRequest("LayerA", null, null, null); // exhaustion +1 + TraceDecisionUtil.shouldTraceRequest( + "LayerA", + X_TRACE_ID_SAMPLED, + null, + null); // no change in exhaustion, as Continue trace does not have bucket restriction + // LayerA should have 2 token bucket exhaustion count + + when(settingsFetcherMock.getSettings()) + .thenReturn( + SettingsStub.builder() + .withFlags(true, false, true, true, false) + .withSampleRate(0) + .withSettingsArg(SettingsArg.BUCKET_CAPACITY, 0.0) + .withSettingsArg(SettingsArg.BUCKET_RATE, 0.0) + .build()); + TraceDecisionUtil.shouldTraceRequest( + "LayerB", null, null, null); // no change in exhaustion, sample rate at 0 + TraceDecisionUtil.shouldTraceRequest( + "LayerB", null, null, null); // no change in exhaustion, sample rate at 0 + TraceDecisionUtil.shouldTraceRequest( + "LayerB", + X_TRACE_ID_SAMPLED, + null, + null); // no change in exhaustion, as Continue trace does not have bucket restriction + // LayerB should not appear in the map as it has no exhaustion count + + when(settingsFetcherMock.getSettings()) + .thenReturn( + SettingsStub.builder() + .withFlags(true, false, true, true, false) + .withSampleRate(1000000) + .withSettingsArg(SettingsArg.BUCKET_CAPACITY, 1.0) + .withSettingsArg(SettingsArg.BUCKET_RATE, 0.0) + .build()); + TraceDecisionUtil.shouldTraceRequest( + "LayerC", null, null, null); // exhaustion +1, sharing same token bucket as LayerA + TraceDecisionUtil.shouldTraceRequest( + "LayerC", null, null, null); // exhaustion +1, tokens all used up already + TraceDecisionUtil.shouldTraceRequest( + "LayerC", + X_TRACE_ID_SAMPLED, + null, + null); // no change in exhaustion, as Continue trace does not have bucket restriction + // LayerC should have 1 token bucket exhaustion count + + int data = + TraceDecisionUtil.consumeMetricsData(TraceDecisionUtil.MetricType.TOKEN_BUCKET_EXHAUSTION); + assertEquals(4, data); + + data = + TraceDecisionUtil.consumeMetricsData( + TraceDecisionUtil.MetricType + .TOKEN_BUCKET_EXHAUSTION); // consumed once already, so this time should return + // empty map + assertEquals(0, data); + + // test signed/trigger trace requests + when(settingsFetcherMock.getSettings()) + .thenReturn( + SettingsStub.builder() + .withFlags(true, false, true, true, false) + .withSampleRate(1000000) + .withSettingsArg(SettingsArg.BUCKET_CAPACITY, 0.0) + .withSettingsArg(SettingsArg.BUCKET_RATE, 0.0) + .withSettingsArg(SettingsArg.STRICT_BUCKET_CAPACITY, 1.0) + .withSettingsArg(SettingsArg.STRICT_BUCKET_RATE, 0.0) + .withSettingsArg(SettingsArg.RELAXED_BUCKET_CAPACITY, 2.0) + .withSettingsArg(SettingsArg.RELAXED_BUCKET_RATE, 0.0) + .build()); + TraceDecisionUtil.shouldTraceRequest("LayerA", null, null, null); // exhaustion +1 + TraceDecisionUtil.shouldTraceRequest("LayerA", null, null, null); // exhaustion +1 + TraceDecisionUtil.shouldTraceRequest("LayerA", null, null, null); // exhaustion +1 + + TraceDecisionUtil.shouldTraceRequest("LayerA", null, TRIGGER_TRACE_OPTIONS, null); // ok + TraceDecisionUtil.shouldTraceRequest( + "LayerA", null, TRIGGER_TRACE_OPTIONS, null); // exhaustion +1 + TraceDecisionUtil.shouldTraceRequest( + "LayerA", null, TRIGGER_TRACE_OPTIONS, null); // exhaustion +1 + + XtraceOptions goodSignatureWithTriggerTraceOptions = + new XtraceOptions( + Collections.singletonMap(XtraceOption.TRIGGER_TRACE, true), + Collections.emptyList(), + XtraceOptions.AuthenticationStatus.OK); + TraceDecisionUtil.shouldTraceRequest( + "LayerA", null, goodSignatureWithTriggerTraceOptions, null); // ok + TraceDecisionUtil.shouldTraceRequest( + "LayerA", null, goodSignatureWithTriggerTraceOptions, null); // ok + TraceDecisionUtil.shouldTraceRequest( + "LayerA", null, goodSignatureWithTriggerTraceOptions, null); // exhaustion +1 + + data = + TraceDecisionUtil.consumeMetricsData( + TraceDecisionUtil.MetricType + .TOKEN_BUCKET_EXHAUSTION); // consumed once already, so this time should return + // empty map + assertEquals(6, data); + } + + @Test + public void testTokenBucket() throws Exception { + when(settingsFetcherMock.getSettings()) + .thenReturn(SettingsStub.builder().withFlags(true, false, true, true, false).build()); + + String bucketLayer = "test"; + + // bucket capacity at 100, rate at 100 trace per sec + when(settingsFetcherMock.getSettings()) + .thenReturn( + SettingsStub.builder() + .withFlags(true, true, true, true, true) + .withSampleRate(1000000) + .withSettingsArg(SettingsArg.BUCKET_CAPACITY, 30.0) + .withSettingsArg(SettingsArg.BUCKET_RATE, 0.0) + .build()); + + TraceConfig config = + TraceDecisionUtil.shouldTraceRequest( + bucketLayer, null, null, Collections.singletonList("http://something.html")) + .getTraceConfig(); // tracing with token + assertNotNull(config); + assertEquals(30.0, config.getBucketCapacity(TokenBucketType.REGULAR)); + assertEquals(0.0, config.getBucketRate(TokenBucketType.REGULAR)); + assertTrue( + TraceDecisionUtil.shouldTraceRequest( + bucketLayer, + X_TRACE_ID_SAMPLED, + null, + Collections.singletonList("http://something.html")) + .isSampled()); // continue trace not restricted by token bucket + + // bucket capacity at 0, rate at 100 trace per sec + when(settingsFetcherMock.getSettings()) + .thenReturn( + SettingsStub.builder() + .withFlags(true, true, true, true, true) + .withSampleRate(1000000) + .withSettingsArg(SettingsArg.BUCKET_CAPACITY, 0.0) + .withSettingsArg(SettingsArg.BUCKET_RATE, 100.0) + .build()); + + TimeUnit.SECONDS.sleep(1); + assertFalse( + TraceDecisionUtil.shouldTraceRequest( + bucketLayer, null, null, Collections.singletonList("http://something.html")) + .isSampled()); // no new trace as capacity is at zero + assertTrue( + TraceDecisionUtil.shouldTraceRequest( + bucketLayer, + X_TRACE_ID_SAMPLED, + null, + Collections.singletonList("http://something.html")) + .isSampled()); // continue trace not restricted by token bucket + + // bucket capacity at 50, rate at 0 trace per sec + when(settingsFetcherMock.getSettings()) + .thenReturn( + SettingsStub.builder() + .withFlags(true, true, true, true, true) + .withSampleRate(1000000) + .withSettingsArg(SettingsArg.BUCKET_CAPACITY, 50.0) + .withSettingsArg(SettingsArg.BUCKET_RATE, 0.0) + .build()); + TimeUnit.SECONDS.sleep(1); + assertFalse( + TraceDecisionUtil.shouldTraceRequest( + bucketLayer, null, null, Collections.singletonList("http://something.html")) + .isSampled()); // not tracing, sharing the same bucket instance, it has capacity 50 now + // but zero replenish rate and 0 left-over token from previous capacity + // which is zero + + // bucket capacity at 50, rate at 100 trace per sec, should trace again + when(settingsFetcherMock.getSettings()) + .thenReturn( + SettingsStub.builder() + .withFlags(true, true, true, true, true) + .withSampleRate(1000000) + .withSettingsArg(SettingsArg.BUCKET_CAPACITY, 50.0) + .withSettingsArg(SettingsArg.BUCKET_RATE, 100.0) + .build()); + TimeUnit.SECONDS.sleep(1); + assertTrue( + TraceDecisionUtil.shouldTraceRequest( + bucketLayer, null, null, Collections.singletonList("http://something.html")) + .isSampled()); // not tracing, sharing the same bucket instance, it has capacity 50 now + // but zero replenish rate and 0 left-over token from previous capacity + // which is zero + } + + @Test + public void testGetTokenBucket() throws Exception { + assertTrue( + TraceDecisionUtil.getTokenBucket(TokenBucketType.REGULAR, 1.0, 0.0) + .consume()); // first time returns true 1 token consumed + assertFalse( + TraceDecisionUtil.getTokenBucket(TokenBucketType.REGULAR, 1.0, 0.0) + .consume()); // second time returns false, no token remains + + // try RELAXED bucket type + assertTrue( + TraceDecisionUtil.getTokenBucket(TokenBucketType.RELAXED, 1.0, 0.0) + .consume()); // first time returns true 1 token consumed + assertFalse( + TraceDecisionUtil.getTokenBucket(TokenBucketType.RELAXED, 1.0, 0.0) + .consume()); // second time returns false, no token remains + + // try STRICT bucket type + assertTrue( + TraceDecisionUtil.getTokenBucket(TokenBucketType.STRICT, 1.0, 0.0) + .consume()); // first time returns true 1 token consumed + assertFalse( + TraceDecisionUtil.getTokenBucket(TokenBucketType.STRICT, 1.0, 0.0) + .consume()); // second time returns false, no token remains + + // try the REGULAR type again + assertFalse( + TraceDecisionUtil.getTokenBucket(TokenBucketType.REGULAR, 1.0, 0.0) + .consume()); // still no token remains + } + + @Test + public void testTriggerTraceTraceDecision() throws Exception { + TraceDecision result; + // trigger trace enabled is set to true, tracing rate 100% + when(settingsFetcherMock.getSettings()) + .thenReturn( + SettingsStub.builder() + .withFlags(true, false, true, true, false) + .withSampleRate(1000000) + .build()); + result = TraceDecisionUtil.shouldTraceRequest("LayerA", null, TRIGGER_TRACE_OPTIONS, null); + assertTrue(result.isSampled()); // tracing, trigger trace flagged and enabled + assertTrue(result.isReportMetrics()); // metric should be reported regardless of rate + + // trigger trace enabled is set to true, tracing rate 0 + when(settingsFetcherMock.getSettings()) + .thenReturn( + SettingsStub.builder() + .withFlags(true, false, true, true, false) + .withSampleRate(0) + .build()); + result = TraceDecisionUtil.shouldTraceRequest("LayerA", null, TRIGGER_TRACE_OPTIONS, null); + assertTrue(result.isSampled()); // tracing, trigger trace flagged and enabled + assertTrue(result.isReportMetrics()); // metric should be reported regardless of rate + + // trigger trace enabled is set to false, tracing rate 0 + when(settingsFetcherMock.getSettings()) + .thenReturn( + SettingsStub.builder() + .withFlags(true, false, true, false, false) + .withSampleRate(0) + .build()); + result = TraceDecisionUtil.shouldTraceRequest("LayerA", null, TRIGGER_TRACE_OPTIONS, null); + assertFalse(result.isSampled()); // not tracing, trigger trace flagged but disabled + assertTrue(result.isReportMetrics()); // metric should be reported regardless of rate + + // trigger trace enabled is set to false, tracing rate 100% + when(settingsFetcherMock.getSettings()) + .thenReturn( + SettingsStub.builder() + .withFlags(true, false, true, false, false) + .withSampleRate(1000000) + .build()); + result = TraceDecisionUtil.shouldTraceRequest("LayerA", null, TRIGGER_TRACE_OPTIONS, null); + assertFalse(result.isSampled()); // not tracing, trigger trace flagged but disabled + assertTrue(result.isReportMetrics()); // metric should be reported regardless of rate + + // trigger trace enabled is set to false, tracing mode disabled + when(settingsFetcherMock.getSettings()) + .thenReturn( + SettingsStub.builder() + .withFlags(false, false, false, false, false) + .withSampleRate(0) + .build()); + result = TraceDecisionUtil.shouldTraceRequest("LayerA", null, TRIGGER_TRACE_OPTIONS, null); + assertFalse( + result.isSampled()); // not tracing, tracing mode disabled (so is the trigger trace option) + assertFalse(result.isReportMetrics()); // no metrics, tracing mode disabled + + // trigger trace enabled is set to true, tracing rate 100% - bad signature + when(settingsFetcherMock.getSettings()) + .thenReturn( + SettingsStub.builder() + .withFlags(true, false, true, true, false) + .withSampleRate(1000000) + .build()); + XtraceOptions badSignatureOptions = + new XtraceOptions( + Collections.emptyMap(), + Collections.emptyList(), + XtraceOptions.AuthenticationStatus.failure("bad-signature")); + result = TraceDecisionUtil.shouldTraceRequest("LayerA", null, badSignatureOptions, null); + assertFalse(result.isSampled()); // bad signature, no tracing + assertTrue( + result + .isReportMetrics()); // metric should still be reported as bad signature does not affect + // metrics reporting + + // trigger trace enabled is set to disable, trace mode disabled - bad signature + when(settingsFetcherMock.getSettings()) + .thenReturn( + SettingsStub.builder() + .withFlags(false, false, false, false, false) + .withSampleRate(0) + .build()); + result = TraceDecisionUtil.shouldTraceRequest("LayerA", null, badSignatureOptions, null); + assertFalse(result.isSampled()); // bad signature, no tracing + assertFalse( + result.isReportMetrics()); // metric should not be reported due to trace mode disabled + } + + @Test + public void testTraceCount() throws Exception { + when(settingsFetcherMock.getSettings()) + .thenReturn(SettingsStub.builder().withFlags(true, false, true, true, false).build()); + + int data; + TraceDecisionUtil.consumeMetricsData(TraceDecisionUtil.MetricType.TRACE_COUNT); // clear it + + when(settingsFetcherMock.getSettings()) + .thenReturn( + SettingsStub.builder() + .withFlags(true, false, true, true, false) + .withSampleRate(1000000) + .build()); // ALWAYS sample rate = 100% + TraceDecisionUtil.shouldTraceRequest( + "LayerA", null, null, null); // new trace at 100%, should count + TraceDecisionUtil.shouldTraceRequest( + "LayerA", X_TRACE_ID_SAMPLED, null, null); // Continue trace, should count + when(settingsFetcherMock.getSettings()) + .thenReturn( + SettingsStub.builder() + .withFlags(true, false, true, true, false) + .withSampleRate(0) + .build()); // ALWAYS. sample rate = 0% + TraceDecisionUtil.shouldTraceRequest( + "LayerB", null, null, null); // new trace at 0%, should not count + TraceDecisionUtil.shouldTraceRequest( + "LayerB", X_TRACE_ID_SAMPLED, null, null); // Continue trace, should count + when(settingsFetcherMock.getSettings()) + .thenReturn( + SettingsStub.builder() + .withFlags(false, false, true, false, false) + .withSampleRate(1000000) + .build()); // THROUGH. sample rate = 100% (not used anyway for THROUGH traces) + TraceDecisionUtil.shouldTraceRequest( + "LayerC", null, null, null); // new trace at THROUGH mode, should not count + TraceDecisionUtil.shouldTraceRequest( + "LayerC", X_TRACE_ID_SAMPLED, null, null); // Continue trace, should count + + data = TraceDecisionUtil.consumeMetricsData(TraceDecisionUtil.MetricType.TRACE_COUNT); + + assertEquals(4, data); + } + + @Test + public void testSampleCount() throws Exception { + when(settingsFetcherMock.getSettings()) + .thenReturn(SettingsStub.builder().withFlags(true, false, true, true, false).build()); + + int data; + TraceDecisionUtil.consumeMetricsData(TraceDecisionUtil.MetricType.SAMPLE_COUNT); // clear it + + when(settingsFetcherMock.getSettings()) + .thenReturn( + SettingsStub.builder() + .withFlags(true, false, true, true, false) + .withSampleRate(1000000) + .build()); // ALWAYS sample rate = 100% + TraceDecisionUtil.shouldTraceRequest("LayerA", null, null, null); // new trace at 100%, sampled + TraceDecisionUtil.shouldTraceRequest( + "LayerA", X_TRACE_ID_SAMPLED, null, null); // Continue trace, no sampling + when(settingsFetcherMock.getSettings()) + .thenReturn( + SettingsStub.builder() + .withFlags(true, false, true, true, false) + .withSampleRate(500000) + .build()); // ALWAYS. sample rate = 50% + TraceDecisionUtil.shouldTraceRequest("LayerB", null, null, null); // new trace at 0%, sampled + TraceDecisionUtil.shouldTraceRequest( + "LayerB", X_TRACE_ID_SAMPLED, null, null); // Continue trace, no sampling + when(settingsFetcherMock.getSettings()) + .thenReturn( + SettingsStub.builder() + .withFlags(false, false, true, false, false) + .withSampleRate(1000000) + .build()); // THROUGH. sample rate = 100% (not used anyway for THROUGH traces) + TraceDecisionUtil.shouldTraceRequest( + "LayerC", null, null, null); // new trace at THROUGH mode, no sampling + TraceDecisionUtil.shouldTraceRequest( + "LayerC", X_TRACE_ID_SAMPLED, null, null); // Continue trace, no sampling + + data = TraceDecisionUtil.consumeMetricsData(TraceDecisionUtil.MetricType.SAMPLE_COUNT); + + assertEquals(2, data); + } + + @Test + public void testThroughTraceCount() throws Exception { + when(settingsFetcherMock.getSettings()) + .thenReturn(SettingsStub.builder().withFlags(true, false, true, true, false).build()); + + int data; + TraceDecisionUtil.consumeMetricsData( + TraceDecisionUtil.MetricType.THROUGH_TRACE_COUNT); // clear it + + when(settingsFetcherMock.getSettings()) + .thenReturn( + SettingsStub.builder() + .withFlags(true, false, true, true, false) + .withSampleRate(1000000) + .build()); // ALWAYS sample rate = 100% + TraceDecisionUtil.shouldTraceRequest("LayerA", null, null, null); // not through trace + TraceDecisionUtil.shouldTraceRequest( + "LayerA", X_TRACE_ID_SAMPLED, null, null); // THROUGH_TRACE_COUNT +1 + when(settingsFetcherMock.getSettings()) + .thenReturn( + SettingsStub.builder() + .withFlags(true, false, true, true, false) + .withSampleRate(500000) + .build()); // ALWAYS. sample rate = 50% + TraceDecisionUtil.shouldTraceRequest("LayerB", null, null, null); // not through trace + TraceDecisionUtil.shouldTraceRequest( + "LayerB", X_TRACE_ID_SAMPLED, null, null); // THROUGH_TRACE_COUNT +1 + when(settingsFetcherMock.getSettings()) + .thenReturn( + SettingsStub.builder() + .withFlags(false, false, true, false, false) + .withSampleRate(1000000) + .build()); // THROUGH. sample rate = 100% (not used anyway for THROUGH traces) + TraceDecisionUtil.shouldTraceRequest("LayerC", null, null, null); // not through trace + TraceDecisionUtil.shouldTraceRequest( + "LayerC", X_TRACE_ID_SAMPLED, null, null); // THROUGH_TRACE_COUNT +1 + when(settingsFetcherMock.getSettings()) + .thenReturn( + SettingsStub.builder() + .withFlags(false, false, false, false, false) + .withSampleRate(1000000) + .build()); // NEVER. sample rate = 100% (not used anyway as no traces get through) + TraceDecisionUtil.shouldTraceRequest("LayerD", null, null, null); // not through trace + TraceDecisionUtil.shouldTraceRequest( + "LayerD", X_TRACE_ID_SAMPLED, null, null); // through flag off + + data = TraceDecisionUtil.consumeMetricsData(TraceDecisionUtil.MetricType.THROUGH_TRACE_COUNT); + + assertEquals(3, data); + } + + @Test + public void testTriggerTraceCount() throws Exception { + when(settingsFetcherMock.getSettings()) + .thenReturn(SettingsStub.builder().withFlags(true, false, true, true, false).build()); + + int data; + TraceDecisionUtil.consumeMetricsData( + TraceDecisionUtil.MetricType.TRIGGERED_TRACE_COUNT); // clear it + TraceDecisionUtil.consumeMetricsData(TraceDecisionUtil.MetricType.TRACE_COUNT); // clear it + TraceDecisionUtil.consumeMetricsData(TraceDecisionUtil.MetricType.THROUGHPUT); // clear it + + when(settingsFetcherMock.getSettings()) + .thenReturn( + SettingsStub.builder() + .withFlags(true, false, true, true, false) + .withSampleRate(TraceDecisionUtil.SAMPLE_RESOLUTION) + .build()); + XtraceOptions badSignatureOptions = + new XtraceOptions( + Collections.emptyMap(), + Collections.emptyList(), + XtraceOptions.AuthenticationStatus.failure("bad-signature")); + XtraceOptions goodSignatureWithTriggerTraceOptions = + new XtraceOptions( + Collections.singletonMap(XtraceOption.TRIGGER_TRACE, true), + Collections.emptyList(), + XtraceOptions.AuthenticationStatus.OK); + + // not a trigger trace + TraceDecisionUtil.shouldTraceRequest( + "LayerA", null, null, null); // tracing, sample rate at 100% + // trigger trace but no signature + TraceDecisionUtil.shouldTraceRequest( + "LayerA", null, TRIGGER_TRACE_OPTIONS, null); // tracing, trigger trace requested + // bad signature + TraceDecisionUtil.shouldTraceRequest( + "LayerA", null, badSignatureOptions, null); // not tracing, bad signature + // good signature with trigger trace + TraceDecisionUtil.shouldTraceRequest( + "LayerA", + null, + goodSignatureWithTriggerTraceOptions, + null); // tracing, trigger trace requested + + // check THROUGHPUT + data = TraceDecisionUtil.consumeMetricsData(TraceDecisionUtil.MetricType.THROUGHPUT); + // check the Layer tag + assertEquals(4, data); + + // check TRACE_COUNT + data = TraceDecisionUtil.consumeMetricsData(TraceDecisionUtil.MetricType.TRACE_COUNT); + // check the Layer tag + assertEquals(3, data); + + // check TRIGGERED_TRACE_COUNT, only 2 of them are traced and flagged as trigger trace + data = TraceDecisionUtil.consumeMetricsData(TraceDecisionUtil.MetricType.TRIGGERED_TRACE_COUNT); + assertEquals(2, data); + } + + @Test + public void testLastSampleRate() throws Exception { + when(settingsFetcherMock.getSettings()) + .thenReturn(SettingsStub.builder().withFlags(true, false, true, true, false).build()); + + // Add local URL rate, should override the SRv1 rate if pattern matches + TraceConfigs testingUrlSampleRateConfigs = + buildUrlConfigs(url -> url.endsWith("html"), TracingMode.NEVER, 0); + SettingsManager.initialize( + settingsFetcherMock, + SamplingConfiguration.builder() + .internalTransactionSettings(testingUrlSampleRateConfigs) + .build()); + TraceDecisionUtil.consumeLastTraceConfigs(); // clear it + + when(settingsFetcherMock.getSettings()) + .thenReturn( + SettingsStub.builder() + .withFlags(true, false, true, true, false) + .withSampleRate(1000000) + .withSettingsType(Settings.OBOE_SETTINGS_TYPE_LAYER_SAMPLE_RATE) + .build()); // ALWAYS sample rate = 100% + TraceDecisionUtil.shouldTraceRequest("LayerA", null, null, null); // 100%, ALWAYS => record 100% + TraceDecisionUtil.shouldTraceRequest( + "LayerA", + null, + null, + Collections.singletonList("something.html")); // URL overrides, NEVER => do not record + when(settingsFetcherMock.getSettings()) + .thenReturn( + SettingsStub.builder() + .withFlags(true, false, true, true, false) + .withSampleRate(0) + .withSettingsType(Settings.OBOE_SETTINGS_TYPE_LAYER_SAMPLE_RATE) + .build()); // ALWAYS. sample rate = 0% + TraceDecisionUtil.shouldTraceRequest("LayerB", null, null, null); // 0%, ALWAYS => record 0% + when(settingsFetcherMock.getSettings()) + .thenReturn( + SettingsStub.builder() + .withFlags(false, false, true, false, false) + .withSampleRate(1000000) + .withSettingsType(Settings.OBOE_SETTINGS_TYPE_LAYER_SAMPLE_RATE) + .build()); // THROUGH. sample rate = 100% (not used anyway for THROUGH traces) + TraceDecisionUtil.shouldTraceRequest( + "LayerC", X_TRACE_ID_SAMPLED, null, null); // 100%, THROUGH => do not record + TraceDecisionUtil.shouldTraceRequest( + "LayerC", null, null, null); // 100%, THROUGH => do not record + when(settingsFetcherMock.getSettings()) + .thenReturn( + SettingsStub.builder() + .withFlags(true, false, true, true, false) + .withSampleRate(0) + .withSettingsType(Settings.OBOE_SETTINGS_TYPE_LAYER_SAMPLE_RATE) + .build()); // ALWAYS. sample rate = 0% + + Map data = TraceDecisionUtil.consumeLastTraceConfigs(); + + assertEquals(2, data.size()); + assertEquals(1000000, data.get("LayerA").getSampleRate()); + assertEquals(SampleRateSource.OBOE, data.get("LayerA").getSampleRateSource()); + assertEquals(0, data.get("LayerB").getSampleRate()); + assertEquals(SampleRateSource.OBOE, data.get("LayerB").getSampleRateSource()); + assertFalse(data.containsKey("LayerC")); + + // case 2: add local settings sampleRate= 500000 (50%), now it should override all the oboe + // settings + SettingsManager.initialize( + settingsFetcherMock, + SamplingConfiguration.builder() + .internalTransactionSettings(testingUrlSampleRateConfigs) + .sampleRate(500000) + .tracingMode(TracingMode.ALWAYS) + .build()); + + TraceDecisionUtil.shouldTraceRequest( + "LayerA", null, null, null); // 50% (local settings), ALWAYS => record 50% + TraceDecisionUtil.shouldTraceRequest( + "LayerA", + null, + null, + Collections.singletonList("something.html")); // URL overrides, NEVER => do not record + TraceDecisionUtil.shouldTraceRequest( + "LayerB", null, null, null); // 50% (local settings), ALWAYS => record 50% + TraceDecisionUtil.shouldTraceRequest( + "LayerC", X_TRACE_ID_SAMPLED, null, null); // 50% (local settings), ALWAYS => record 50% + + data = TraceDecisionUtil.consumeLastTraceConfigs(); + + assertEquals(3, data.size()); + assertEquals(500000, data.get("LayerA").getSampleRate()); + assertEquals(SampleRateSource.FILE, data.get("LayerA").getSampleRateSource()); + assertEquals(500000, data.get("LayerB").getSampleRate()); + assertEquals(SampleRateSource.FILE, data.get("LayerB").getSampleRateSource()); + assertEquals(500000, data.get("LayerC").getSampleRate()); + assertEquals(SampleRateSource.FILE, data.get("LayerC").getSampleRateSource()); + } + + @Test + public void testGetRemoteSampleRate() { + Settings settings; + TraceConfig remoteSampleRate; + + // test remote settings with no args + settings = + SettingsStub.builder() + .withFlags(TracingMode.ALWAYS) + .withSampleRate(0) + .withSettingsArgs(Collections.emptyMap()) + .build(); + when(settingsFetcherMock.getSettings()).thenReturn(settings); + remoteSampleRate = TraceDecisionUtil.getRemoteTraceConfig(); + assertEquals( + 0.0, remoteSampleRate.getBucketCapacity(TokenBucketType.REGULAR)); // should default to 0 + assertEquals( + 0.0, remoteSampleRate.getBucketRate(TokenBucketType.REGULAR)); // should default to 0 + + // test remote settings that give empty value for "BucketCapacity" and "BucketRate" + Map, Object> args = new HashMap, Object>(); + args.put(SettingsArg.BUCKET_CAPACITY, null); + args.put(SettingsArg.BUCKET_RATE, null); + settings = + SettingsStub.builder() + .withFlags(TracingMode.ALWAYS) + .withSampleRate(0) + .withSettingsArgs(args) + .build(); + when(settingsFetcherMock.getSettings()).thenReturn(settings); + remoteSampleRate = TraceDecisionUtil.getRemoteTraceConfig(); + assertEquals( + 0.0, remoteSampleRate.getBucketCapacity(TokenBucketType.REGULAR)); // should default to 0 + assertEquals( + 0.0, remoteSampleRate.getBucketRate(TokenBucketType.REGULAR)); // should default to 0 + + // test remote settings that give negative values for "BucketCapacity" and "BucketRate" + args = new HashMap, Object>(); + args.put(SettingsArg.BUCKET_CAPACITY, -1.0); + args.put(SettingsArg.BUCKET_RATE, -2.0); + settings = + SettingsStub.builder() + .withFlags(TracingMode.ALWAYS) + .withSampleRate(0) + .withSettingsArgs(args) + .build(); + when(settingsFetcherMock.getSettings()).thenReturn(settings); + remoteSampleRate = TraceDecisionUtil.getRemoteTraceConfig(); + assertEquals( + 0.0, remoteSampleRate.getBucketCapacity(TokenBucketType.REGULAR)); // should default to 0 + assertEquals( + 0.0, remoteSampleRate.getBucketRate(TokenBucketType.REGULAR)); // should default to 0 + + // test remote settings that give valid values for "BucketCapacity" and "BucketRate" + args = new HashMap, Object>(); + args.put(SettingsArg.BUCKET_CAPACITY, 1.0); + args.put(SettingsArg.BUCKET_RATE, 2.0); + settings = + SettingsStub.builder() + .withFlags(TracingMode.ALWAYS) + .withSampleRate(0) + .withSettingsArgs(args) + .build(); + when(settingsFetcherMock.getSettings()).thenReturn(settings); + remoteSampleRate = TraceDecisionUtil.getRemoteTraceConfig(); + assertEquals( + 1.0, remoteSampleRate.getBucketCapacity(TokenBucketType.REGULAR)); // should default to 0 + assertEquals( + 2.0, remoteSampleRate.getBucketRate(TokenBucketType.REGULAR)); // should default to 0 + } + + /** + * Test getting local config with different trace mode and {@link + * SamplingConfiguration#isTriggerTraceEnabled()} values + */ + @Test + public void testTriggerTraceConfig() { + TraceConfig remoteConfigEnabled = + new TraceConfig( + TraceDecisionUtil.SAMPLE_RESOLUTION, + SampleRateSource.OBOE_DEFAULT, + (short) + (Settings.OBOE_SETTINGS_FLAG_SAMPLE_START + | Settings.OBOE_SETTINGS_FLAG_SAMPLE_THROUGH_ALWAYS + | Settings.OBOE_SETTINGS_FLAG_TRIGGER_TRACE_ENABLED + | Settings.OBOE_SETTINGS_FLAG_OVERRIDE)); + TraceConfig remoteConfigDisabled = + new TraceConfig(0, SampleRateSource.OBOE_DEFAULT, Settings.OBOE_SETTINGS_FLAG_OVERRIDE); + TraceConfig localConfigDefault = new TraceConfig(null, SampleRateSource.DEFAULT, null); + TraceConfig localConfigEnabled = + new TraceConfig(null, SampleRateSource.FILE, TracingMode.ENABLED.toFlags()); + TraceConfig localConfigDisabled = + new TraceConfig(0, SampleRateSource.FILE, TracingMode.DISABLED.toFlags()); + TraceConfig localConfigSampleRateConfigured = + new TraceConfig(TraceDecisionUtil.SAMPLE_RESOLUTION, SampleRateSource.FILE, null); + + TraceConfig result; + // Remote tracing enabled, Local tracing default, trigger trace disabled - trigger trace should + // be disabled + result = TraceDecisionUtil.computeTraceConfig(remoteConfigEnabled, localConfigDefault, false); + assertFalse(result.hasSampleTriggerTraceFlag()); + + // Remote tracing enabled, Local tracing enabled, trigger trace disabled - trigger trace should + // be disabled + result = TraceDecisionUtil.computeTraceConfig(remoteConfigEnabled, localConfigDisabled, false); + assertFalse(result.hasSampleTriggerTraceFlag()); + + // Remote tracing enabled, Local tracing sample rate configured, trigger trace disabled - + // trigger trace should be disabled + result = + TraceDecisionUtil.computeTraceConfig( + remoteConfigEnabled, localConfigSampleRateConfigured, false); + assertFalse(result.hasSampleTriggerTraceFlag()); + + // Remote tracing enabled, Local tracing default, trigger trace enabled - trigger trace should + // be enabled + result = TraceDecisionUtil.computeTraceConfig(remoteConfigEnabled, localConfigDefault, true); + assertTrue(result.hasSampleTriggerTraceFlag()); + + // Remote tracing enabled, Local tracing disabled, trigger trace enabled - trigger trace should + // be disabled - local trace mode disable wins + result = TraceDecisionUtil.computeTraceConfig(remoteConfigEnabled, localConfigDisabled, true); + assertFalse(result.hasSampleTriggerTraceFlag()); + + // Remote tracing disabled, Local tracing enabled, trigger trace enabled - trigger trace should + // be disabled - remote trace mode disable wins + result = TraceDecisionUtil.computeTraceConfig(remoteConfigDisabled, localConfigEnabled, true); + assertFalse(result.hasSampleTriggerTraceFlag()); + } + + @Test + public void testBadSignature() { + when(settingsFetcherMock.getSettings()) + .thenReturn( + SettingsStub.builder() + .withFlags(true, false, true, true, false) + .withSampleRate(1000000) + .build()); + TraceDecision traceDecision; + traceDecision = TraceDecisionUtil.shouldTraceRequest(TEST_LAYER, null, null, null); + assertTrue(traceDecision.isSampled()); + assertTrue(traceDecision.isReportMetrics()); + + XtraceOptions badSignatureOptions = + new XtraceOptions( + Collections.emptyMap(), + Collections.emptyList(), + XtraceOptions.AuthenticationStatus.failure("bad-signature")); + traceDecision = + TraceDecisionUtil.shouldTraceRequest(TEST_LAYER, null, badSignatureOptions, null); + assertFalse(traceDecision.isSampled()); + assertTrue(traceDecision.isReportMetrics()); + } + + private static Map.Entry getLayerTag(String layer) { + return getTag("Layer", layer); + } + + private static Map.Entry getTag(String key, Object value) { + return new AbstractMap.SimpleEntry(key, value); + } + + private TraceConfigs buildUrlConfigs( + ResourceMatcher matcher, TracingMode tracingMode, Integer sampleRate) { + Map result = new LinkedHashMap(); + TraceConfig traceConfig = buildTraceConfig(tracingMode, sampleRate); + result.put(matcher, traceConfig); + return new TraceConfigs(result); + } + + private TraceConfig buildTraceConfig(TracingMode tracingMode, Integer sampleRate) { + // The core test case should test directly on a defined TraceConfig. The logic that builds a + // TraceConfig is Agent specific + return new TraceConfig(sampleRate, SampleRateSource.FILE, tracingMode.toFlags()); + } +} diff --git a/libs/sampling/src/test/java/com/solarwinds/joboe/sampling/XtraceOptionTest.java b/libs/sampling/src/test/java/com/solarwinds/joboe/sampling/XtraceOptionTest.java new file mode 100644 index 00000000..86d6092f --- /dev/null +++ b/libs/sampling/src/test/java/com/solarwinds/joboe/sampling/XtraceOptionTest.java @@ -0,0 +1,32 @@ +/* + * © SolarWinds Worldwide, LLC. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.solarwinds.joboe.sampling; + +import static org.junit.jupiter.api.Assertions.*; + +import org.junit.jupiter.api.Test; + +public class XtraceOptionTest { + @Test + public void testFromKey() { + assertEquals( + XtraceOption.TRIGGER_TRACE, XtraceOption.fromKey(XtraceOption.TRIGGER_TRACE.getKey())); + assertTrue(XtraceOption.fromKey(XtraceOption.CUSTOM_KV_PREFIX + "abc").isCustomKv()); + assertNull(XtraceOption.fromKey("unknown")); + assertNull(XtraceOption.fromKey("trigger trace")); + } +} diff --git a/libs/sampling/src/test/java/com/solarwinds/joboe/sampling/XtraceOptionsResponseTest.java b/libs/sampling/src/test/java/com/solarwinds/joboe/sampling/XtraceOptionsResponseTest.java new file mode 100644 index 00000000..3dc727d2 --- /dev/null +++ b/libs/sampling/src/test/java/com/solarwinds/joboe/sampling/XtraceOptionsResponseTest.java @@ -0,0 +1,225 @@ +/* + * © SolarWinds Worldwide, LLC. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.solarwinds.joboe.sampling; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertNull; + +import java.util.Collections; +import org.junit.jupiter.api.Test; + +public class XtraceOptionsResponseTest { + @Test + public void testResponse() { + TraceConfig traceConfig = + new TraceConfig( + TraceDecisionUtil.SAMPLE_RESOLUTION, + SampleRateSource.OBOE_DEFAULT, + TracingMode.ALWAYS.toFlags()); + XtraceOptions options; + + XTraceOptionsResponse response; + // no x-trace options + response = + XTraceOptionsResponse.computeResponse( + XtraceOptions.getXTraceOptions(null, null), + new TraceDecision(true, true, traceConfig, TraceDecisionUtil.RequestType.REGULAR), + true); + assertNull(response); + + // empty x-trace options + response = + XTraceOptionsResponse.computeResponse( + XtraceOptions.getXTraceOptions("", null), + new TraceDecision(true, true, traceConfig, TraceDecisionUtil.RequestType.REGULAR), + true); + assertEquals("trigger-trace=not-requested", response.toString()); + + // trigger trace (unauthenticated) + options = + new XtraceOptions( + Collections.singletonMap(XtraceOption.TRIGGER_TRACE, true), + Collections.emptyList(), + XtraceOptions.AuthenticationStatus.NOT_AUTHENTICATED); + response = + XTraceOptionsResponse.computeResponse( + options, + new TraceDecision( + true, + true, + traceConfig, + TraceDecisionUtil.RequestType.UNAUTHENTICATED_TRIGGER_TRACE), + true); + assertEquals("trigger-trace=ok", response.toString()); + + // trigger trace (authenticated) + options = + new XtraceOptions( + Collections.singletonMap(XtraceOption.TRIGGER_TRACE, true), + Collections.emptyList(), + XtraceOptions.AuthenticationStatus.OK); + response = + XTraceOptionsResponse.computeResponse( + options, + new TraceDecision( + true, true, traceConfig, TraceDecisionUtil.RequestType.AUTHENTICATED_TRIGGER_TRACE), + true); + assertEquals("auth=ok;trigger-trace=ok", response.toString()); + + // trigger trace no remote settings + options = + new XtraceOptions( + Collections.singletonMap(XtraceOption.TRIGGER_TRACE, true), + Collections.emptyList(), + XtraceOptions.AuthenticationStatus.NOT_AUTHENTICATED); + response = + XTraceOptionsResponse.computeResponse( + options, + new TraceDecision( + false, false, null, TraceDecisionUtil.RequestType.UNAUTHENTICATED_TRIGGER_TRACE), + true); + assertEquals("trigger-trace=settings-not-available", response.toString()); + + // trigger trace bucket exhausted + options = + new XtraceOptions( + Collections.singletonMap(XtraceOption.TRIGGER_TRACE, true), + Collections.emptyList(), + XtraceOptions.AuthenticationStatus.NOT_AUTHENTICATED); + response = + XTraceOptionsResponse.computeResponse( + options, + new TraceDecision( + false, + false, + true, + traceConfig, + TraceDecisionUtil.RequestType.UNAUTHENTICATED_TRIGGER_TRACE), + true); + assertEquals("trigger-trace=rate-exceeded", response.toString()); + + // trigger trace trace mode = disabled + options = + new XtraceOptions( + Collections.singletonMap(XtraceOption.TRIGGER_TRACE, true), + Collections.emptyList(), + XtraceOptions.AuthenticationStatus.NOT_AUTHENTICATED); + TraceConfig tracingDisabledConfig = + new TraceConfig(0, SampleRateSource.FILE, TracingMode.NEVER.toFlags()); + response = + XTraceOptionsResponse.computeResponse( + options, + new TraceDecision( + false, + false, + tracingDisabledConfig, + TraceDecisionUtil.RequestType.UNAUTHENTICATED_TRIGGER_TRACE), + true); + assertEquals("trigger-trace=tracing-disabled", response.toString()); + + // trigger trace feature is disabled + options = + new XtraceOptions( + Collections.singletonMap(XtraceOption.TRIGGER_TRACE, true), + Collections.emptyList(), + XtraceOptions.AuthenticationStatus.NOT_AUTHENTICATED); + TraceConfig featureDisabledConfig = + new TraceConfig( + TraceDecisionUtil.SAMPLE_RESOLUTION, + SampleRateSource.FILE, + (short) + (TracingMode.ENABLED.toFlags() + & ~Settings.OBOE_SETTINGS_FLAG_TRIGGER_TRACE_ENABLED)); + response = + XTraceOptionsResponse.computeResponse( + options, + new TraceDecision( + false, + true, + featureDisabledConfig, + TraceDecisionUtil.RequestType.UNAUTHENTICATED_TRIGGER_TRACE), + true); + assertEquals("trigger-trace=trigger-tracing-disabled", response.toString()); + } + + @Test + public void testExceptionResponse() { + TraceConfig traceConfig = + new TraceConfig( + TraceDecisionUtil.SAMPLE_RESOLUTION, + SampleRateSource.OBOE_DEFAULT, + TracingMode.ALWAYS.toFlags()); + + XTraceOptionsResponse response; + // unknown X-Trace-Options + response = + XTraceOptionsResponse.computeResponse( + XtraceOptions.getXTraceOptions( + "unknown1=1;unknown2;" + XtraceOption.SW_KEYS.getKey() + "=3", null), + new TraceDecision(true, true, traceConfig, TraceDecisionUtil.RequestType.REGULAR), + true); + assertEquals("trigger-trace=not-requested;ignored=unknown1,unknown2", response.toString()); + + // invalid trigger-trace (has value) + response = + XTraceOptionsResponse.computeResponse( + XtraceOptions.getXTraceOptions( + XtraceOption.TRIGGER_TRACE.getKey() + "=0;" + XtraceOption.SW_KEYS.getKey() + "=3", + null), + new TraceDecision(true, true, traceConfig, TraceDecisionUtil.RequestType.REGULAR), + true); + assertEquals("trigger-trace=not-requested;ignored=trigger-trace", response.toString()); + } + + @Test + public void testBadSignatureResponse() { + TraceConfig traceConfig = + new TraceConfig( + TraceDecisionUtil.SAMPLE_RESOLUTION, + SampleRateSource.OBOE_DEFAULT, + TracingMode.ALWAYS.toFlags()); + + XTraceOptionsResponse response; + // bad timestamp + XtraceOptions badTimestampOptions = + new XtraceOptions( + Collections.emptyMap(), + Collections.emptyList(), + XtraceOptions.AuthenticationStatus.failure("bad-timestamp")); + response = + XTraceOptionsResponse.computeResponse( + badTimestampOptions, + new TraceDecision( + false, true, traceConfig, TraceDecisionUtil.RequestType.BAD_SIGNATURE), + true); + assertEquals("auth=bad-timestamp", response.toString()); + + // bad signature + XtraceOptions badSignatureOptions = + new XtraceOptions( + Collections.emptyMap(), + Collections.emptyList(), + XtraceOptions.AuthenticationStatus.failure("bad-signature")); + response = + XTraceOptionsResponse.computeResponse( + badSignatureOptions, + new TraceDecision( + false, true, traceConfig, TraceDecisionUtil.RequestType.BAD_SIGNATURE), + true); + assertEquals("auth=bad-signature", response.toString()); + } +} diff --git a/libs/sampling/src/test/java/com/solarwinds/joboe/sampling/XtraceOptionsTest.java b/libs/sampling/src/test/java/com/solarwinds/joboe/sampling/XtraceOptionsTest.java new file mode 100644 index 00000000..d816d69f --- /dev/null +++ b/libs/sampling/src/test/java/com/solarwinds/joboe/sampling/XtraceOptionsTest.java @@ -0,0 +1,385 @@ +/* + * © SolarWinds Worldwide, LLC. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.solarwinds.joboe.sampling; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertNull; +import static org.junit.jupiter.api.Assertions.assertTrue; + +import java.io.File; +import java.io.IOException; +import java.nio.file.Files; +import java.nio.file.Paths; +import java.util.Collections; +import java.util.HashMap; +import java.util.Iterator; +import org.junit.jupiter.api.Test; + +public class XtraceOptionsTest { + + @Test + @SuppressWarnings("unchecked") + public void testGetXTraceOptions() throws Exception { + assertNull(XtraceOptions.getXTraceOptions(null, null)); + + String swKeys = "lo:se"; + XtraceOptions options = + XtraceOptions.getXTraceOptions( + XtraceOption.TRIGGER_TRACE.getKey() + + XtraceOptions.ENTRY_SEPARATOR + + XtraceOption.SW_KEYS.getKey() + + XtraceOptions.KEY_VALUE_SEPARATOR + + swKeys + + XtraceOptions.ENTRY_SEPARATOR + + XtraceOption.CUSTOM_KV_PREFIX + + "tag1" + + XtraceOptions.KEY_VALUE_SEPARATOR + + "v1" + + XtraceOptions.ENTRY_SEPARATOR + + XtraceOption.CUSTOM_KV_PREFIX + + "tag2" + + XtraceOptions.KEY_VALUE_SEPARATOR + + "v2", + null); + assertEquals( + XtraceOptions.AuthenticationStatus.NOT_AUTHENTICATED, options.getAuthenticationStatus()); + + HashMap, String> expectedCustomKvs = + new HashMap, String>(); + expectedCustomKvs.put( + (XtraceOption) XtraceOption.fromKey(XtraceOption.CUSTOM_KV_PREFIX + "tag1"), "v1"); + expectedCustomKvs.put( + (XtraceOption) XtraceOption.fromKey(XtraceOption.CUSTOM_KV_PREFIX + "tag2"), "v2"); + + assertEquals(swKeys, options.getOptionValue(XtraceOption.SW_KEYS)); + assertEquals(Boolean.TRUE, options.getOptionValue(XtraceOption.TRIGGER_TRACE)); + assertEquals(expectedCustomKvs, options.getCustomKvs()); + assertEquals(Collections.emptyList(), options.getExceptions()); + + // no trigger-trace option + options = + XtraceOptions.getXTraceOptions( + XtraceOption.SW_KEYS.getKey() + + XtraceOptions.KEY_VALUE_SEPARATOR + + swKeys + + XtraceOptions.ENTRY_SEPARATOR + + XtraceOption.CUSTOM_KV_PREFIX + + "tag1" + + XtraceOptions.KEY_VALUE_SEPARATOR + + "v1" + + XtraceOptions.ENTRY_SEPARATOR + + XtraceOption.CUSTOM_KV_PREFIX + + "tag2" + + XtraceOptions.KEY_VALUE_SEPARATOR + + "v2", + null); + assertEquals( + XtraceOptions.AuthenticationStatus.NOT_AUTHENTICATED, options.getAuthenticationStatus()); + assertEquals(Collections.emptyList(), options.getExceptions()); + assertEquals(swKeys, options.getOptionValue(XtraceOption.SW_KEYS)); + assertEquals(expectedCustomKvs, options.getCustomKvs()); + } + + @Test + public void testFormatting() { + XtraceOptions options; + String swKeys = "lo:se"; + // leading and trailing whitespace + options = + XtraceOptions.getXTraceOptions( + " " + + XtraceOption.TRIGGER_TRACE.getKey() + + XtraceOptions.ENTRY_SEPARATOR + + XtraceOption.SW_KEYS.getKey() + + XtraceOptions.KEY_VALUE_SEPARATOR + + swKeys + + " ", + null); + assertEquals(swKeys, options.getOptionValue(XtraceOption.SW_KEYS)); + assertEquals(Boolean.TRUE, options.getOptionValue(XtraceOption.TRIGGER_TRACE)); + + // space in between kv pairs are trimmed + // leading and trailing whitespace + options = + XtraceOptions.getXTraceOptions( + XtraceOption.TRIGGER_TRACE.getKey() + + " " + + XtraceOptions.ENTRY_SEPARATOR + + " " + + XtraceOption.SW_KEYS.getKey() + + " " + + XtraceOptions.KEY_VALUE_SEPARATOR + + " " + + swKeys, + null); + assertEquals(swKeys, options.getOptionValue(XtraceOption.SW_KEYS)); + assertEquals(Boolean.TRUE, options.getOptionValue(XtraceOption.TRIGGER_TRACE)); + + // space in key is considered invalid + options = XtraceOptions.getXTraceOptions("trigger trace", null); + assertEquals(Boolean.FALSE, options.getOptionValue(XtraceOption.TRIGGER_TRACE)); + assertEquals( + "trigger trace", + ((XtraceOptions.UnknownXTraceOptionException) options.getExceptions().get(0)) + .getInvalidOptionKey()); + + // key/value separator (=) in value is okay + String customKey = XtraceOption.CUSTOM_KV_PREFIX + "1"; + String customValue = "foo" + XtraceOptions.KEY_VALUE_SEPARATOR + "5"; + options = + XtraceOptions.getXTraceOptions( + customKey + XtraceOptions.KEY_VALUE_SEPARATOR + customValue, null); + assertEquals(0, options.getExceptions().size()); + assertEquals(1, options.getCustomKvs().size()); + assertEquals(customKey, options.getCustomKvs().keySet().iterator().next().getKey()); + assertEquals(customValue, options.getCustomKvs().values().iterator().next()); + } + + @Test + @SuppressWarnings("unchecked") + public void testDuplicatedOption() { + XtraceOptions options = + XtraceOptions.getXTraceOptions( + XtraceOption.SW_KEYS.getKey() + + XtraceOptions.KEY_VALUE_SEPARATOR + + "p1" + + XtraceOptions.ENTRY_SEPARATOR + + XtraceOption.SW_KEYS.getKey() + + XtraceOptions.KEY_VALUE_SEPARATOR + + "p2" + + XtraceOptions.ENTRY_SEPARATOR + + XtraceOption.CUSTOM_KV_PREFIX + + "tag1" + + XtraceOptions.KEY_VALUE_SEPARATOR + + "v1" + + XtraceOptions.ENTRY_SEPARATOR + + XtraceOption.CUSTOM_KV_PREFIX + + "tag1" + + XtraceOptions.KEY_VALUE_SEPARATOR + + "v2", + null); + HashMap, String> expectedCustomKvs = + new HashMap, String>(); + expectedCustomKvs.put( + (XtraceOption) XtraceOption.fromKey(XtraceOption.CUSTOM_KV_PREFIX + "tag1"), + "v1"); // take the first value only + + assertEquals("p1", options.getOptionValue(XtraceOption.SW_KEYS)); + assertEquals(expectedCustomKvs, options.getCustomKvs()); + } + + @Test + public void testGetXTraceOptionsExceptions() throws Exception { + XtraceOptions options = + XtraceOptions.getXTraceOptions( + XtraceOption.TRIGGER_TRACE.getKey() + + XtraceOptions.ENTRY_SEPARATOR + + "unknown-tag1" + + XtraceOptions.KEY_VALUE_SEPARATOR + + "v1" + + XtraceOptions.ENTRY_SEPARATOR + + "unknown-tag2", + null); + + assertEquals(Boolean.TRUE, options.getOptionValue(XtraceOption.TRIGGER_TRACE)); + assertEquals(2, options.getExceptions().size()); + assertEquals( + "unknown-tag1", + ((XtraceOptions.UnknownXTraceOptionException) options.getExceptions().get(0)) + .getInvalidOptionKey()); + assertEquals( + "unknown-tag2", + ((XtraceOptions.UnknownXTraceOptionException) options.getExceptions().get(1)) + .getInvalidOptionKey()); + + // test invalid format + options = + XtraceOptions.getXTraceOptions( + XtraceOption.TRIGGER_TRACE.getKey() + + XtraceOptions.KEY_VALUE_SEPARATOR + + "1" + + XtraceOptions.ENTRY_SEPARATOR + + XtraceOption.CUSTOM_KV_PREFIX + + "1", + null); // trigger trace should not have value, custom kv should have a value + assertEquals(Boolean.FALSE, options.getOptionValue(XtraceOption.TRIGGER_TRACE)); + assertEquals( + XtraceOption.TRIGGER_TRACE.getKey(), + ((XtraceOptions.InvalidFormatXTraceOptionException) options.getExceptions().get(0)) + .getInvalidOptionKey()); + assertEquals( + XtraceOption.CUSTOM_KV_PREFIX + "1", + ((XtraceOptions.InvalidFormatXTraceOptionException) options.getExceptions().get(1)) + .getInvalidOptionKey()); + + // test invalid value + options = + XtraceOptions.getXTraceOptions( + XtraceOption.TS.getKey() + XtraceOptions.KEY_VALUE_SEPARATOR + "abc", + null); // ts should be a long + assertEquals( + XtraceOption.TS.getKey(), + ((XtraceOptions.InvalidValueXTraceOptionException) options.getExceptions().get(0)) + .getInvalidOptionKey()); + + // parse some options either though others are bad + options = + XtraceOptions.getXTraceOptions("trigger-trace;custom-foo=' bar;bar' ;custom-bar=foo", null); + assertEquals(Boolean.TRUE, options.getOptionValue(XtraceOption.TRIGGER_TRACE)); + assertEquals(2, options.getCustomKvs().size()); + Iterator> customKeyIterator = options.getCustomKvs().keySet().iterator(); + Iterator customValueIterator = options.getCustomKvs().values().iterator(); + assertEquals("custom-foo", customKeyIterator.next().getKey()); + assertEquals("' bar", customValueIterator.next()); + + assertEquals("custom-bar", customKeyIterator.next().getKey()); + assertEquals("foo", customValueIterator.next()); + + assertEquals( + "bar'", + ((XtraceOptions.UnknownXTraceOptionException) options.getExceptions().get(0)) + .getInvalidOptionKey()); + + options = + XtraceOptions.getXTraceOptions( + ";trigger-trace;custom-something=value_thing;sw-keys=02973r70:9wqj21,0d9j1;1;2;=custom-key=val?;=", + null); + assertEquals(Boolean.TRUE, options.getOptionValue(XtraceOption.TRIGGER_TRACE)); + assertEquals("02973r70:9wqj21,0d9j1", options.getOptionValue(XtraceOption.SW_KEYS)); + assertEquals(1, options.getCustomKvs().size()); + customKeyIterator = options.getCustomKvs().keySet().iterator(); + customValueIterator = options.getCustomKvs().values().iterator(); + assertEquals("custom-something", customKeyIterator.next().getKey()); + assertEquals("value_thing", customValueIterator.next()); + assertEquals( + 2, + options + .getExceptions() + .size()); // should only flag exception for 1 and 2, the last two entry starts with '=' + // will be ignored + assertEquals( + "1", + ((XtraceOptions.UnknownXTraceOptionException) options.getExceptions().get(0)) + .getInvalidOptionKey()); + assertEquals( + "2", + ((XtraceOptions.UnknownXTraceOptionException) options.getExceptions().get(1)) + .getInvalidOptionKey()); + + // skip sequel ; + options = + XtraceOptions.getXTraceOptions( + "custom-something=value_thing;sw-keys=02973r70;;;;custom-key=val", null); + assertEquals("02973r70", options.getOptionValue(XtraceOption.SW_KEYS)); + assertEquals(2, options.getCustomKvs().size()); + customKeyIterator = options.getCustomKvs().keySet().iterator(); + customValueIterator = options.getCustomKvs().values().iterator(); + assertEquals("custom-something", customKeyIterator.next().getKey()); + assertEquals("value_thing", customValueIterator.next()); + assertEquals("custom-key", customKeyIterator.next().getKey()); + assertEquals("val", customValueIterator.next()); + + // case sensitive + options = XtraceOptions.getXTraceOptions("Trigger-Trace;Custom-something=value_thing", null); + assertEquals(Boolean.FALSE, options.getOptionValue(XtraceOption.TRIGGER_TRACE)); + assertEquals(2, options.getExceptions().size()); + assertEquals( + "Trigger-Trace", + ((XtraceOptions.UnknownXTraceOptionException) options.getExceptions().get(0)) + .getInvalidOptionKey()); + assertEquals( + "Custom-something", + ((XtraceOptions.UnknownXTraceOptionException) options.getExceptions().get(1)) + .getInvalidOptionKey()); + + // no X-Trace-Options but has signature + options = XtraceOptions.getXTraceOptions(null, "abc"); + assertNull(options); + } + + @Test + public void testHmacAuthenticator() throws Exception { + byte[] content = + Files.readAllBytes(Paths.get(new File("src/test/resources/hmac-signature.txt").getPath())); + XtraceOptions.HmacSignatureAuthenticator authenticator = + new XtraceOptions.HmacSignatureAuthenticator(content); + + assertTrue( + authenticator.authenticate( + "trigger-trace;sw-keys=lo:se,check-id:123;ts=1564597681", + "26e33ce58c52afc507c5c1e9feff4ac5562c9e1c")); + assertFalse( + authenticator.authenticate( + "trigger-trace;sw-keys=lo:se,check-id:123;ts=1564597681", + "2c1c398c3e6be898f47f74bf74f035903b48baaa")); + } + + @Test + public void testAuthenticate() throws IOException { + byte[] content = + Files.readAllBytes(Paths.get(new File("src/test/resources/hmac-signature.txt").getPath())); + XtraceOptions.HmacSignatureAuthenticator authenticator = + new XtraceOptions.HmacSignatureAuthenticator(content); + + // missing ts + assertEquals( + XtraceOptions.AuthenticationStatus.failure("bad-timestamp"), + XtraceOptions.authenticate( + "trigger-trace;sw-keys=lo:se,check-id:123", + null, + "2c1c398c3e6be898f47f74bf74f035903b48b59c", + authenticator)); + + long outOfRangeTimestamp = + System.currentTimeMillis() / 1000 - (XtraceOptions.TIMESTAMP_MAX_DELTA + 1); + // timestamp out of range + assertEquals( + XtraceOptions.AuthenticationStatus.failure("bad-timestamp"), + XtraceOptions.authenticate( + "trigger-trace;sw-keys=lo:se,check-id:123;ts=" + outOfRangeTimestamp, + outOfRangeTimestamp, + "2c1c398c3e6be898f47f74bf74f035903b48b59c", + authenticator)); + + // no signature + assertEquals( + XtraceOptions.AuthenticationStatus.NOT_AUTHENTICATED, + XtraceOptions.authenticate( + "trigger-trace;sw-keys=lo:se,check-id:123", null, null, authenticator)); + + // valid signature - using a mock up authenticator here to bypass the signature check - which is + // verify in testHmacAuthenticator + long goodTimestamp = System.currentTimeMillis() / 1000; + assertEquals( + XtraceOptions.AuthenticationStatus.OK, + XtraceOptions.authenticate( + "trigger-trace;sw-keys=lo:se,check-id:123;ts=" + goodTimestamp, + goodTimestamp, + "2c1c398c3e6be898f47f74bf74f035903b48b59c", + ((optionsString, signature) -> true))); + + // authenticator not ready + assertEquals( + XtraceOptions.AuthenticationStatus.failure("authenticator-unavailable"), + XtraceOptions.authenticate( + "trigger-trace;sw-keys=lo:se,check-id:123;ts=" + goodTimestamp, + goodTimestamp, + "2c1c398c3e6be898f47f74bf74f035903b48b59c", + null)); + } +} diff --git a/libs/sampling/src/test/resources/hmac-signature.txt b/libs/sampling/src/test/resources/hmac-signature.txt new file mode 100644 index 00000000..abac1060 --- /dev/null +++ b/libs/sampling/src/test/resources/hmac-signature.txt @@ -0,0 +1 @@ +8mZ98ZnZhhggcsUmdMbS \ No newline at end of file diff --git a/libs/shared/build.gradle.kts b/libs/shared/build.gradle.kts index e2c4002f..b27581a9 100644 --- a/libs/shared/build.gradle.kts +++ b/libs/shared/build.gradle.kts @@ -23,14 +23,12 @@ plugins { dependencies { compileOnly(project(":bootstrap")) - compileOnly("org.projectlombok:lombok") - annotationProcessor("org.projectlombok:lombok") - compileOnly("com.solarwinds.joboe:config") - compileOnly("com.solarwinds.joboe:logging") + compileOnly(project(":libs:config")) + compileOnly(project(":libs:logging")) compileOnly("io.opentelemetry.semconv:opentelemetry-semconv-incubating") - compileOnly("com.solarwinds.joboe:sampling") + compileOnly(project(":libs:sampling")) compileOnly("com.google.auto.service:auto-service") annotationProcessor("com.google.auto.service:auto-service") @@ -53,7 +51,9 @@ dependencies { compileOnly("io.opentelemetry:opentelemetry-sdk-extension-incubator") testImplementation("org.json:json") - testImplementation("com.solarwinds.joboe:sampling") + testImplementation(project(":libs:config")) + testImplementation(project(":libs:logging")) + testImplementation(project(":libs:sampling")) testImplementation("io.opentelemetry:opentelemetry-exporter-otlp") testImplementation("io.opentelemetry:opentelemetry-api-incubator") @@ -75,6 +75,11 @@ buildConfig { buildConfigField("String", "BUILD_DATETIME", "\"$formattedDate\"") } +tasks.named("compileJava") { + // Disable AutoService verify check to prevent rawtypes warnings for generic service provider interfaces + options.compilerArgs.add("-Averify=false") +} + swoJava { minJavaVersionSupported.set(JavaVersion.VERSION_1_8) } diff --git a/libs/shared/src/main/java/com/solarwinds/opentelemetry/extensions/SamplingUtil.java b/libs/shared/src/main/java/com/solarwinds/opentelemetry/extensions/SamplingUtil.java index 2a541d99..1db06f30 100644 --- a/libs/shared/src/main/java/com/solarwinds/opentelemetry/extensions/SamplingUtil.java +++ b/libs/shared/src/main/java/com/solarwinds/opentelemetry/extensions/SamplingUtil.java @@ -18,8 +18,8 @@ import com.solarwinds.joboe.sampling.TraceDecision; import com.solarwinds.joboe.sampling.TraceDecisionUtil; -import com.solarwinds.joboe.sampling.XTraceOption; -import com.solarwinds.joboe.sampling.XTraceOptions; +import com.solarwinds.joboe.sampling.XtraceOption; +import com.solarwinds.joboe.sampling.XtraceOptions; import io.opentelemetry.api.common.AttributesBuilder; import java.util.regex.Pattern; @@ -47,7 +47,7 @@ public static boolean isValidSwTraceState(String swVal) { public static void addXtraceOptionsToAttribute( TraceDecision traceDecision, - XTraceOptions xtraceOptions, + XtraceOptions xtraceOptions, AttributesBuilder attributesBuilder) { if (xtraceOptions != null) { xtraceOptions @@ -62,7 +62,7 @@ public static void addXtraceOptionsToAttribute( attributesBuilder.put("TriggeredTrace", true); } - String swKeys = xtraceOptions.getOptionValue(XTraceOption.SW_KEYS); + String swKeys = xtraceOptions.getOptionValue(XtraceOption.SW_KEYS); if (swKeys != null) { attributesBuilder.put("SWKeys", swKeys); } diff --git a/libs/shared/src/main/java/com/solarwinds/opentelemetry/extensions/SharedNames.java b/libs/shared/src/main/java/com/solarwinds/opentelemetry/extensions/SharedNames.java index 76baa558..a85911e9 100644 --- a/libs/shared/src/main/java/com/solarwinds/opentelemetry/extensions/SharedNames.java +++ b/libs/shared/src/main/java/com/solarwinds/opentelemetry/extensions/SharedNames.java @@ -19,11 +19,11 @@ public final class SharedNames { private SharedNames() {} - public static String COMPONENT_NAME = "solarwinds"; + public static final String COMPONENT_NAME = "solarwinds"; - public static String TRANSACTION_NAME_KEY = "sw.transaction"; + public static final String TRANSACTION_NAME_KEY = "sw.transaction"; - public static String SPAN_STACKTRACE_FILTER_CLASS = + public static final String SPAN_STACKTRACE_FILTER_CLASS = "com.solarwinds.opentelemetry.extensions.SpanStacktraceFilter"; // This is visible to customer via span layer and can be used to configure transaction diff --git a/libs/shared/src/main/java/com/solarwinds/opentelemetry/extensions/SolarwindsContextPropagator.java b/libs/shared/src/main/java/com/solarwinds/opentelemetry/extensions/SolarwindsContextPropagator.java index 3522d0d5..b449f046 100644 --- a/libs/shared/src/main/java/com/solarwinds/opentelemetry/extensions/SolarwindsContextPropagator.java +++ b/libs/shared/src/main/java/com/solarwinds/opentelemetry/extensions/SolarwindsContextPropagator.java @@ -18,7 +18,7 @@ import static com.solarwinds.opentelemetry.extensions.SamplingUtil.SW_XTRACE_OPTIONS_RESP_KEY; -import com.solarwinds.joboe.sampling.XTraceOptions; +import com.solarwinds.joboe.sampling.XtraceOptions; import io.opentelemetry.api.trace.Span; import io.opentelemetry.api.trace.SpanContext; import io.opentelemetry.api.trace.TraceState; @@ -162,8 +162,8 @@ private String updateTraceState(TraceState traceState, String swTraceStateValue) public Context extract(Context context, C carrier, TextMapGetter getter) { final String traceOptions = getter.get(carrier, X_TRACE_OPTIONS); final String traceOptionsSignature = getter.get(carrier, X_TRACE_OPTIONS_SIGNATURE); - final XTraceOptions xTraceOptions = - XTraceOptions.getXTraceOptions(traceOptions, traceOptionsSignature); + final XtraceOptions xTraceOptions = + XtraceOptions.getXTraceOptions(traceOptions, traceOptionsSignature); if (xTraceOptions != null) { context = context.with(TriggerTraceContextKey.KEY, xTraceOptions); context = context.with(TriggerTraceContextKey.XTRACE_OPTIONS, traceOptions); diff --git a/libs/shared/src/main/java/com/solarwinds/opentelemetry/extensions/SolarwindsSampler.java b/libs/shared/src/main/java/com/solarwinds/opentelemetry/extensions/SolarwindsSampler.java index 28a7e318..6fca103d 100644 --- a/libs/shared/src/main/java/com/solarwinds/opentelemetry/extensions/SolarwindsSampler.java +++ b/libs/shared/src/main/java/com/solarwinds/opentelemetry/extensions/SolarwindsSampler.java @@ -24,8 +24,8 @@ import com.solarwinds.joboe.logging.Logger; import com.solarwinds.joboe.logging.LoggerFactory; import com.solarwinds.joboe.sampling.TraceDecision; -import com.solarwinds.joboe.sampling.XTraceOptions; import com.solarwinds.joboe.sampling.XTraceOptionsResponse; +import com.solarwinds.joboe.sampling.XtraceOptions; import com.solarwinds.opentelemetry.core.Constants; import com.solarwinds.opentelemetry.core.Util; import io.opentelemetry.api.common.AttributeKey; @@ -104,7 +104,7 @@ public SamplingResult shouldSample( final SamplingResult samplingResult; final AttributesBuilder additionalAttributesBuilder = Attributes.builder(); - final XTraceOptions xTraceOptions = parentContext.get(TriggerTraceContextKey.KEY); + final XtraceOptions xTraceOptions = parentContext.get(TriggerTraceContextKey.KEY); String xtraceOptionsResponseStr = null; List signals = @@ -201,7 +201,7 @@ public String getDescription() { } SamplingResult toOtSamplingResult( - TraceDecision traceDecision, XTraceOptions xtraceOptions, boolean genesis) { + TraceDecision traceDecision, XtraceOptions xtraceOptions, boolean genesis) { SamplingResult result = NOT_TRACED; if (traceDecision.isSampled()) { diff --git a/libs/shared/src/main/java/com/solarwinds/opentelemetry/extensions/TransactionNameManager.java b/libs/shared/src/main/java/com/solarwinds/opentelemetry/extensions/TransactionNameManager.java index c1d5dfe2..1ecb9165 100644 --- a/libs/shared/src/main/java/com/solarwinds/opentelemetry/extensions/TransactionNameManager.java +++ b/libs/shared/src/main/java/com/solarwinds/opentelemetry/extensions/TransactionNameManager.java @@ -48,13 +48,13 @@ public class TransactionNameManager { public static final int MAX_TRANSACTION_NAME_LENGTH = 255; public static final String TRANSACTION_NAME_ELLIPSIS = "..."; - private static String[] customTransactionNamePattern = null; + private static volatile String[] customTransactionNamePattern = null; static final Cache URL_TRANSACTION_NAME_CACHE = Caffeine.newBuilder().maximumSize(1000).expireAfterWrite(Duration.ofMinutes(20)).build(); private static final Set EXISTING_TRANSACTION_NAMES = new HashSet<>(); - private static boolean limitExceeded; - private static int maxNameCount = DEFAULT_MAX_NAME_COUNT; + private static volatile boolean limitExceeded; + private static volatile int maxNameCount = DEFAULT_MAX_NAME_COUNT; private static NamingScheme namingScheme = new DefaultNamingScheme(null); diff --git a/libs/shared/src/main/java/com/solarwinds/opentelemetry/extensions/TriggerTraceContextKey.java b/libs/shared/src/main/java/com/solarwinds/opentelemetry/extensions/TriggerTraceContextKey.java index ffee8abf..8c113159 100644 --- a/libs/shared/src/main/java/com/solarwinds/opentelemetry/extensions/TriggerTraceContextKey.java +++ b/libs/shared/src/main/java/com/solarwinds/opentelemetry/extensions/TriggerTraceContextKey.java @@ -16,11 +16,11 @@ package com.solarwinds.opentelemetry.extensions; -import com.solarwinds.joboe.sampling.XTraceOptions; +import com.solarwinds.joboe.sampling.XtraceOptions; import io.opentelemetry.context.ContextKey; final class TriggerTraceContextKey { - public static final ContextKey KEY = ContextKey.named("sw-trigger-trace-key"); + public static final ContextKey KEY = ContextKey.named("sw-trigger-trace-key"); public static final ContextKey XTRACE_OPTIONS = ContextKey.named("xtrace-options"); public static final ContextKey XTRACE_OPTIONS_SIGNATURE = ContextKey.named("xtrace-options-signature"); diff --git a/libs/shared/src/main/java/com/solarwinds/opentelemetry/extensions/config/parser/json/LogSettingParser.java b/libs/shared/src/main/java/com/solarwinds/opentelemetry/extensions/config/parser/json/LogSettingParser.java index 5fa31a1c..ca6991fe 100644 --- a/libs/shared/src/main/java/com/solarwinds/opentelemetry/extensions/config/parser/json/LogSettingParser.java +++ b/libs/shared/src/main/java/com/solarwinds/opentelemetry/extensions/config/parser/json/LogSettingParser.java @@ -116,13 +116,6 @@ private LogSetting convertJsonValue(String stringValue) throws InvalidConfigExce try { logFilePath = Paths.get(locationString); - // if (!logFilePath.isAbsolute()) { //then use the agent - // directory as the base - // if (ResourceDirectory.getAgentDirectory() != null) { - // logFilePath = - // Paths.get(ResourceDirectory.getAgentDirectory(), logFilePath.toString()); - // } - // } maxSize = fileObject.has(FILE_MAX_SIZE_KEY) ? fileObject.getInt(FILE_MAX_SIZE_KEY) : null; diff --git a/libs/shared/src/test/java/com/solarwinds/opentelemetry/extensions/SamplingUtilTest.java b/libs/shared/src/test/java/com/solarwinds/opentelemetry/extensions/SamplingUtilTest.java index c8f2b153..02e39cce 100644 --- a/libs/shared/src/test/java/com/solarwinds/opentelemetry/extensions/SamplingUtilTest.java +++ b/libs/shared/src/test/java/com/solarwinds/opentelemetry/extensions/SamplingUtilTest.java @@ -25,7 +25,7 @@ import com.solarwinds.joboe.sampling.TraceDecision; import com.solarwinds.joboe.sampling.TraceDecisionUtil; -import com.solarwinds.joboe.sampling.XTraceOptions; +import com.solarwinds.joboe.sampling.XtraceOptions; import io.opentelemetry.api.common.Attributes; import io.opentelemetry.api.common.AttributesBuilder; import org.junit.jupiter.api.Test; @@ -41,7 +41,7 @@ class SamplingUtilTest { @Test void verifyThatTriggeredTraceAttributeIsAddedForAuthenticatedTriggerTrace() { AttributesBuilder builder = Attributes.builder(); - XTraceOptions xTraceOptions = XTraceOptions.getXTraceOptions("trigger-trace", null); + XtraceOptions xTraceOptions = XtraceOptions.getXTraceOptions("trigger-trace", null); when(traceDecisionMock.getRequestType()) .thenReturn(TraceDecisionUtil.RequestType.AUTHENTICATED_TRIGGER_TRACE); @@ -52,7 +52,7 @@ void verifyThatTriggeredTraceAttributeIsAddedForAuthenticatedTriggerTrace() { @Test void verifyThatTriggeredTraceAttributeIsAddedForUnauthenticatedTriggerTrace() { AttributesBuilder builder = Attributes.builder(); - XTraceOptions xTraceOptions = XTraceOptions.getXTraceOptions("trigger-trace", null); + XtraceOptions xTraceOptions = XtraceOptions.getXTraceOptions("trigger-trace", null); when(traceDecisionMock.getRequestType()) .thenReturn(TraceDecisionUtil.RequestType.UNAUTHENTICATED_TRIGGER_TRACE); @@ -63,7 +63,7 @@ void verifyThatTriggeredTraceAttributeIsAddedForUnauthenticatedTriggerTrace() { @Test void verifyThatCustomKvAttributesAreAdded() { AttributesBuilder builder = Attributes.builder(); - XTraceOptions xTraceOptions = XTraceOptions.getXTraceOptions("custom-chubi=chubby;", null); + XtraceOptions xTraceOptions = XtraceOptions.getXTraceOptions("custom-chubi=chubby;", null); when(traceDecisionMock.getRequestType()) .thenReturn(TraceDecisionUtil.RequestType.UNAUTHENTICATED_TRIGGER_TRACE); @@ -74,8 +74,8 @@ void verifyThatCustomKvAttributesAreAdded() { @Test void verifyThatSwKeysAttributeIsAdded() { AttributesBuilder builder = Attributes.builder(); - XTraceOptions xTraceOptions = - XTraceOptions.getXTraceOptions("sw-keys=lo:se,check-id:123", null); + XtraceOptions xTraceOptions = + XtraceOptions.getXTraceOptions("sw-keys=lo:se,check-id:123", null); when(traceDecisionMock.getRequestType()) .thenReturn(TraceDecisionUtil.RequestType.AUTHENTICATED_TRIGGER_TRACE); diff --git a/libs/shared/src/test/java/com/solarwinds/opentelemetry/extensions/SolarwindsContextPropagatorTest.java b/libs/shared/src/test/java/com/solarwinds/opentelemetry/extensions/SolarwindsContextPropagatorTest.java index 5db83027..c601bca6 100644 --- a/libs/shared/src/test/java/com/solarwinds/opentelemetry/extensions/SolarwindsContextPropagatorTest.java +++ b/libs/shared/src/test/java/com/solarwinds/opentelemetry/extensions/SolarwindsContextPropagatorTest.java @@ -30,7 +30,7 @@ import static org.mockito.Mockito.verify; import static org.mockito.Mockito.when; -import com.solarwinds.joboe.sampling.XTraceOptions; +import com.solarwinds.joboe.sampling.XtraceOptions; import com.solarwinds.opentelemetry.extensions.stubs.TextMapGetterStub; import io.opentelemetry.api.trace.Span; import io.opentelemetry.api.trace.SpanContext; @@ -64,8 +64,9 @@ class SolarwindsContextPropagatorTest { @Captor private ArgumentCaptor stringArgumentCaptor; - private final String traceId = "80ad68e98d3449dfc54098b38fc466ec"; - private final String spanId = "a2d8376f3cab2837"; + private static final String traceId = "80ad68e98d3449dfc54098b38fc466ec"; + + private static final String spanId = "a2d8376f3cab2837"; private final SpanContext spanContext = SpanContext.create( @@ -172,7 +173,7 @@ void verifyThatXtraceOptionsIsExtractedAndPutIntoContext() { Context newContext = solarwindsContextPropagator.extract(Context.current(), carrier, textMapGetterStub); - XTraceOptions xTraceOptions = newContext.get(TriggerTraceContextKey.KEY); + XtraceOptions xTraceOptions = newContext.get(TriggerTraceContextKey.KEY); assertEquals(String.format("%s=%s;", key, value), newContext.get(XTRACE_OPTIONS)); assertNotNull(xTraceOptions); diff --git a/libs/shared/src/test/java/com/solarwinds/opentelemetry/extensions/SolarwindsSamplerTest.java b/libs/shared/src/test/java/com/solarwinds/opentelemetry/extensions/SolarwindsSamplerTest.java index fb7bd6a7..95542141 100644 --- a/libs/shared/src/test/java/com/solarwinds/opentelemetry/extensions/SolarwindsSamplerTest.java +++ b/libs/shared/src/test/java/com/solarwinds/opentelemetry/extensions/SolarwindsSamplerTest.java @@ -27,7 +27,7 @@ import com.solarwinds.joboe.sampling.TraceConfig; import com.solarwinds.joboe.sampling.TraceDecision; import com.solarwinds.joboe.sampling.TraceDecisionUtil; -import com.solarwinds.joboe.sampling.XTraceOptions; +import com.solarwinds.joboe.sampling.XtraceOptions; import com.solarwinds.opentelemetry.core.Util; import io.opentelemetry.api.common.Attributes; import io.opentelemetry.api.trace.Span; @@ -63,7 +63,7 @@ class SolarwindsSamplerTest { @Mock private TraceDecision traceDecisionMock; - @Mock private XTraceOptions xTraceOptionsMock; + @Mock private XtraceOptions xtraceOptionsMock; @Mock private TraceConfig traceConfigMock; @@ -90,7 +90,7 @@ void returnSamplingResultGivenTraceDecisionIsSampled() { when(traceDecisionMock.getRequestType()).thenReturn(TraceDecisionUtil.RequestType.REGULAR); when(traceDecisionMock.isReportMetrics()).thenReturn(true); - tested.toOtSamplingResult(traceDecisionMock, xTraceOptionsMock, false); + tested.toOtSamplingResult(traceDecisionMock, xtraceOptionsMock, false); verify(traceDecisionMock, atLeastOnce()).getTraceConfig(); verify(traceConfigMock, atLeastOnce()).getSampleRate(); @@ -101,7 +101,7 @@ void returnSamplingResultGivenTraceDecisionIsMetricsOnly() { when(traceDecisionMock.isSampled()).thenReturn(false); when(traceDecisionMock.isReportMetrics()).thenReturn(true); - SamplingResult actual = tested.toOtSamplingResult(traceDecisionMock, xTraceOptionsMock, false); + SamplingResult actual = tested.toOtSamplingResult(traceDecisionMock, xtraceOptionsMock, false); assertEquals(SolarwindsSampler.METRICS_ONLY, actual); } @@ -110,7 +110,7 @@ void returnSamplingResultGivenTraceDecisionIsNotSample() { when(traceDecisionMock.isSampled()).thenReturn(false); when(traceDecisionMock.isReportMetrics()).thenReturn(false); - SamplingResult actual = tested.toOtSamplingResult(traceDecisionMock, xTraceOptionsMock, false); + SamplingResult actual = tested.toOtSamplingResult(traceDecisionMock, xtraceOptionsMock, false); assertEquals(SolarwindsSampler.NOT_TRACED, actual); } diff --git a/long-running-test-arch/xk6/go.sum b/long-running-test-arch/xk6/go.sum new file mode 100644 index 00000000..110ba890 --- /dev/null +++ b/long-running-test-arch/xk6/go.sum @@ -0,0 +1,45 @@ +github.com/cenkalti/backoff/v4 v4.3.0/go.mod h1:Y3VNntkOUPxTVeUxJ/G5vcM//AlwfmyYozVcomhLiZE= +github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/fatih/color v1.18.0/go.mod h1:4FelSpRwEGDpQ12mAdzqdOukCy4u8WUtOY6lkT/6HfU= +github.com/go-logr/logr v1.2.2/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A= +github.com/go-logr/logr v1.4.2/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY= +github.com/go-logr/stdr v1.2.2/go.mod h1:mMo/vtBO5dYbehREoey6XUKy/eSumjCCveDpRre4VKE= +github.com/golang/protobuf v1.5.4/go.mod h1:lnTiLA8Wa4RWRcIUkrtSVa5nRhsEGBg48fD6rSs7xps= +github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= +github.com/grpc-ecosystem/grpc-gateway/v2 v2.25.1/go.mod h1:RBRO7fro65R6tjKzYgLAFo0t1QEXY1Dp+i/bvpRiqiQ= +github.com/josharian/intern v1.0.0/go.mod h1:5DoeVV0s6jJacbCEi61lwdGj/aVlrQvzHFFd8Hwg//Y= +github.com/mailru/easyjson v0.9.0/go.mod h1:1+xMtQp2MRNVL/V1bOzuP3aP8VNwRW55fQUto+XFtTU= +github.com/mattn/go-colorable v0.1.14/go.mod h1:6LmQG8QLFO4G5z1gPvYEzlUgJ2wF+stgPZH1UqBm1s8= +github.com/mattn/go-isatty v0.0.20/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y= +github.com/mstoykov/atlas v0.0.0-20220811071828-388f114305dd/go.mod h1:9vRHVuLCjoFfE3GT06X0spdOAO+Zzo4AMjdIwUHBvAk= +github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/sirupsen/logrus v1.9.3/go.mod h1:naHLuLoDiP4jHNo9R0sCBMtWGeIprob74mVsIT4qYEQ= +github.com/spf13/afero v1.12.0/go.mod h1:ZTlWwG4/ahT8W7T0WQ5uYmjI9duaLQGy3Q2OAl4sk/4= +github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= +github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= +go.k6.io/k6 v0.56.0/go.mod h1:cPEOdGLfMi+rrwBXxcFhBucFi2P4KRZc6XmnwVgBRHM= +go.opentelemetry.io/auto/sdk v1.1.0/go.mod h1:3wSPjt5PWp2RhlCcmmOial7AvC4DQqZb7a7wCow3W8A= +go.opentelemetry.io/otel v1.33.0/go.mod h1:SUUkR6csvUQl+yjReHu5uM3EtVV7MBm5FHKRlNx4I8I= +go.opentelemetry.io/otel/exporters/otlp/otlpmetric/otlpmetricgrpc v1.33.0/go.mod h1:ZiGDq7xwDMKmWDrN1XsXAj0iC7hns+2DhxBFSncNHSE= +go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.33.0/go.mod h1:cpgtDBaqD/6ok/UG0jT15/uKjAY8mRA53diogHBg3UI= +go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.33.0/go.mod h1:57gTHJSE5S1tqg+EKsLPlTWhpHMsWlVmer+LA926XiA= +go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp v1.33.0/go.mod h1:wAy0T/dUbs468uOlkT31xjvqQgEVXv58BRFWEgn5v/0= +go.opentelemetry.io/otel/metric v1.33.0/go.mod h1:L9+Fyctbp6HFTddIxClbQkjtubW6O9QS3Ann/M82u6M= +go.opentelemetry.io/otel/sdk v1.33.0/go.mod h1:A1Q5oi7/9XaMlIWzPSxLRWOI8nG3FnzHJNbiENQuihM= +go.opentelemetry.io/otel/sdk/metric v1.33.0/go.mod h1:dL5ykHZmm1B1nVRk9dDjChwDmt81MjVp3gLkQRwKf/Q= +go.opentelemetry.io/otel/trace v1.33.0/go.mod h1:uIcdVUZMpTAmz0tI1z04GoVSezK37CbGV4fr1f2nBck= +go.opentelemetry.io/proto/otlp v1.5.0/go.mod h1:keN8WnHxOy8PG0rQZjJJ5A2ebUoafqWp0eVQ4yIXvJ4= +golang.org/x/net v0.40.0/go.mod h1:y0hY0exeL2Pku80/zKK7tpntoX23cqL3Oa6njdgRtds= +golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.33.0/go.mod h1:BJP2sWEmIv4KK5OTEluFJCKSidICx8ciO85XgH3Ak8k= +golang.org/x/text v0.25.0/go.mod h1:WEdwpYrmk1qmdHvhkSTNPm3app7v4rsT8F2UD6+VHIA= +golang.org/x/time v0.9.0/go.mod h1:3BpzKBy/shNhVucY/MWOyx10tF3SFh9QdLuxbVysPQM= +google.golang.org/genproto/googleapis/api v0.0.0-20250106144421-5f5ef82da422/go.mod h1:b6h1vNKhxaSoEI+5jc3PJUCustfli/mRab7295pY7rw= +google.golang.org/genproto/googleapis/rpc v0.0.0-20250106144421-5f5ef82da422/go.mod h1:3ENsm/5D1mzDyhpzeRi1NR784I0BcofWBoSc5QqqMK4= +google.golang.org/grpc v1.69.4/go.mod h1:vyjdE6jLBI76dgpDojsFGNaHlxdjXN9ghpnd2o7JGZ4= +google.golang.org/protobuf v1.36.2/go.mod h1:9fA7Ob0pmnwhb644+1+CVWFRbNajQ6iRojtC/QF5bRE= +gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/guregu/null.v3 v3.5.0/go.mod h1:E4tX2Qe3h7QdL+uZ3a0vqvYwKQsRSQKM5V4YltdgH9Y= +gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= diff --git a/settings.gradle.kts b/settings.gradle.kts index 84dac03f..fe5b8038 100644 --- a/settings.gradle.kts +++ b/settings.gradle.kts @@ -45,4 +45,8 @@ include("testing") include("testing:agent-for-testing") include("testing:agent-test-extension") include("dependencyManagement") +include("libs:core") +include("libs:config") +include("libs:logging") +include("libs:sampling") diff --git a/spotbugs-exclude.xml b/spotbugs-exclude.xml new file mode 100644 index 00000000..19d103e8 --- /dev/null +++ b/spotbugs-exclude.xml @@ -0,0 +1,65 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/testing/agent-for-testing/build.gradle.kts b/testing/agent-for-testing/build.gradle.kts index adae5eab..289e5631 100644 --- a/testing/agent-for-testing/build.gradle.kts +++ b/testing/agent-for-testing/build.gradle.kts @@ -45,12 +45,11 @@ dependencies { javaagentLibs(project(":instrumentation:hibernate:hibernate-shared")) bootstrapLibs(project(":bootstrap")) - bootstrapLibs("com.solarwinds.joboe:core") - bootstrapLibs("com.solarwinds.joboe:metrics") + bootstrapLibs(project(":libs:core")) - bootstrapLibs("com.solarwinds.joboe:config") - bootstrapLibs("com.solarwinds.joboe:sampling") - bootstrapLibs("com.solarwinds.joboe:logging") + bootstrapLibs(project(":libs:config")) + bootstrapLibs(project(":libs:sampling")) + bootstrapLibs(project(":libs:logging")) bootstrapLibs("org.json:json") upstreamAgent("io.opentelemetry.javaagent:opentelemetry-agent-for-testing")