diff --git a/agent/agent-tooling/src/main/java/com/microsoft/applicationinsights/agent/internal/init/SecondEntryPoint.java b/agent/agent-tooling/src/main/java/com/microsoft/applicationinsights/agent/internal/init/SecondEntryPoint.java index ac2fe85bf7..6cc887b40a 100644 --- a/agent/agent-tooling/src/main/java/com/microsoft/applicationinsights/agent/internal/init/SecondEntryPoint.java +++ b/agent/agent-tooling/src/main/java/com/microsoft/applicationinsights/agent/internal/init/SecondEntryPoint.java @@ -264,6 +264,19 @@ public void customize(AutoConfigurationCustomizer autoConfiguration) { return props; }) .addPropertiesCustomizer(new AiConfigCustomizer()) + .addPropertiesCustomizer( + otelConfig -> { + Map props = new HashMap<>(); + if (isAksAttach()) { + String metricsExporter = otelConfig.getString("otel.metrics.exporter"); + String amle = + otelConfig.getString("applicationinsights.metrics.to.loganalytics.enabled"); + props.put( + "otel.metrics.exporter", + conditionallyAddAzureMonitorExporter(metricsExporter, amle)); + } + return props; + }) .addSpanExporterCustomizer( (spanExporter, configProperties) -> { if (spanExporter instanceof AzureMonitorSpanExporterProvider.MarkerSpanExporter) { @@ -798,4 +811,46 @@ private static CompletableResultCode flushAll( }); return overallResult; } + + private static boolean isAksAttach() { + return !Strings.isNullOrEmpty(System.getenv("AKS_ARM_NAMESPACE_ID")); + } + + // visible for tests + // Per spec: when amle=true, ensure azure_monitor is included; otherwise respect user's setting + // https://github.com/aep-health-and-standards/Telemetry-Collection-Spec/blob/main/ApplicationInsights/AutoAttach_Env_Vars.md#metrics-exporter + static String conditionallyAddAzureMonitorExporter(String metricsExporter, String amle) { + + // Default to azure_monitor when not set + if (Strings.isNullOrEmpty(metricsExporter)) { + // Note: this won't really happen since we default otel.metrics.exporter + // already in the PropertiesSupplier above which runs before this + return AzureMonitorExporterProviderKeys.EXPORTER_NAME; + } + + // When amle=true, ensure azure_monitor is included + if ("true".equals(amle)) { + if ("none".equals(metricsExporter)) { + return AzureMonitorExporterProviderKeys.EXPORTER_NAME; + } + if (!containsAzureMonitor(metricsExporter)) { + return metricsExporter + "," + AzureMonitorExporterProviderKeys.EXPORTER_NAME; + } + } + + return metricsExporter; + } + + // visible for tests + static boolean containsAzureMonitor(String metricsExporter) { + if (metricsExporter == null) { + return false; + } + for (String exporter : metricsExporter.split(",")) { + if (AzureMonitorExporterProviderKeys.EXPORTER_NAME.equals(exporter.trim())) { + return true; + } + } + return false; + } } diff --git a/agent/agent-tooling/src/test/java/com/microsoft/applicationinsights/agent/internal/init/SecondEntryPointTest.java b/agent/agent-tooling/src/test/java/com/microsoft/applicationinsights/agent/internal/init/SecondEntryPointTest.java new file mode 100644 index 0000000000..b47774fe5a --- /dev/null +++ b/agent/agent-tooling/src/test/java/com/microsoft/applicationinsights/agent/internal/init/SecondEntryPointTest.java @@ -0,0 +1,48 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +package com.microsoft.applicationinsights.agent.internal.init; + +import static org.assertj.core.api.Assertions.assertThat; + +import java.util.stream.Stream; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.Arguments; +import org.junit.jupiter.params.provider.MethodSource; + +class SecondEntryPointTest { + + // Test cases matching the spec table in + // https://github.com/aep-health-and-standards/Telemetry-Collection-Spec/blob/main/ApplicationInsights/AutoAttach_Env_Vars.md#metrics-exporter + // + // OTEL_METRICS_EXPORTER | AMLE | azure_monitor included + // ----------------------|--------|------------------------ + static Stream metricsExporterSpecTable() { + return Stream.of( + // AMLE unset + Arguments.of(null, null, true), + Arguments.of("none", null, false), + Arguments.of("azure_monitor", null, true), + Arguments.of("otlp,azure_monitor", null, true), + Arguments.of("otlp", null, false), + // AMLE=true (always include azure_monitor) + Arguments.of(null, "true", true), + Arguments.of("none", "true", true), + Arguments.of("azure_monitor", "true", true), + Arguments.of("otlp,azure_monitor", "true", true), + Arguments.of("otlp", "true", true), + // AMLE=false (same as unset) + Arguments.of(null, "false", true), + Arguments.of("none", "false", false), + Arguments.of("azure_monitor", "false", true), + Arguments.of("otlp,azure_monitor", "false", true), + Arguments.of("otlp", "false", false)); + } + + @ParameterizedTest(name = "exporter={0}, amle={1} -> included={2}") + @MethodSource("metricsExporterSpecTable") + void testUpdateMetricsExporter(String exporter, String amle, boolean expectAzureMonitor) { + String result = SecondEntryPoint.conditionallyAddAzureMonitorExporter(exporter, amle); + assertThat(SecondEntryPoint.containsAzureMonitor(result)).isEqualTo(expectAzureMonitor); + } +} diff --git a/smoke-tests/apps/OtlpMetrics/src/smokeTest/java/com/microsoft/applicationinsights/smoketest/OtlpLogAnalyticsOnAksTest.java b/smoke-tests/apps/OtlpMetrics/src/smokeTest/java/com/microsoft/applicationinsights/smoketest/OtlpLogAnalyticsOnAksTest.java new file mode 100644 index 0000000000..246cbfb687 --- /dev/null +++ b/smoke-tests/apps/OtlpMetrics/src/smokeTest/java/com/microsoft/applicationinsights/smoketest/OtlpLogAnalyticsOnAksTest.java @@ -0,0 +1,231 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +package com.microsoft.applicationinsights.smoketest; + +import static com.microsoft.applicationinsights.smoketest.EnvironmentValue.TOMCAT_8_JAVA_11; +import static com.microsoft.applicationinsights.smoketest.EnvironmentValue.TOMCAT_8_JAVA_11_OPENJ9; +import static com.microsoft.applicationinsights.smoketest.EnvironmentValue.TOMCAT_8_JAVA_17; +import static com.microsoft.applicationinsights.smoketest.EnvironmentValue.TOMCAT_8_JAVA_17_OPENJ9; +import static com.microsoft.applicationinsights.smoketest.EnvironmentValue.TOMCAT_8_JAVA_21; +import static com.microsoft.applicationinsights.smoketest.EnvironmentValue.TOMCAT_8_JAVA_21_OPENJ9; +import static com.microsoft.applicationinsights.smoketest.EnvironmentValue.TOMCAT_8_JAVA_25; +import static com.microsoft.applicationinsights.smoketest.EnvironmentValue.TOMCAT_8_JAVA_25_OPENJ9; +import static com.microsoft.applicationinsights.smoketest.EnvironmentValue.TOMCAT_8_JAVA_8; +import static com.microsoft.applicationinsights.smoketest.EnvironmentValue.TOMCAT_8_JAVA_8_OPENJ9; +import static java.util.concurrent.TimeUnit.SECONDS; +import static org.assertj.core.api.Assertions.assertThat; +import static org.awaitility.Awaitility.await; +import static org.mockserver.model.HttpRequest.request; +import static org.springframework.boot.test.context.SpringBootTest.WebEnvironment.RANDOM_PORT; + +import com.microsoft.applicationinsights.smoketest.schemav2.Data; +import com.microsoft.applicationinsights.smoketest.schemav2.Envelope; +import com.microsoft.applicationinsights.smoketest.schemav2.MetricData; +import com.microsoft.applicationinsights.smoketest.schemav2.RequestData; +import io.opentelemetry.proto.metrics.v1.Metric; +import java.util.List; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.RegisterExtension; +import org.mockserver.model.HttpRequest; +import org.springframework.boot.test.context.SpringBootTest; + +@SpringBootTest( + classes = {OtlpApplication.class}, + webEnvironment = RANDOM_PORT) +@UseAgent +abstract class OtlpLogAnalyticsOnAksTest { + + @RegisterExtension + static final SmokeTestExtension testing = + SmokeTestExtension.builder() + .setEnvVar("APPLICATIONINSIGHTS_METRICS_TO_LOGANALYTICS_ENABLED", "true") + .setEnvVar("AKS_ARM_NAMESPACE_ID", "dummy-aks-namespace") + .useOtlpEndpointOnly() + .build(); + + @Test + @TargetUri("/ping") + public void testOtlpTelemetry() throws Exception { + // verify request sent to breeze endpoint + List rdList = testing.mockedIngestion.waitForItems("RequestData", 1); + Envelope rdEnvelope = rdList.get(0); + RequestData rd = (RequestData) ((Data) rdEnvelope.getData()).getBaseData(); + assertThat(rd.getName()).isEqualTo("GET /OtlpMetrics/ping"); + + // verify custom histogram metric sent to Application Insights endpoint + List metricList = + testing.mockedIngestion.waitForItems( + "MetricData", OtlpLogAnalyticsOnAksTest::isHistogramMetric, 1); + Envelope metricEnvelope = metricList.get(0); + MetricData metricData = (MetricData) ((Data) metricEnvelope.getData()).getBaseData(); + assertThat(metricData.getMetrics().get(0).getName()).isEqualTo("histogram-test-otlp-exporter"); + + // verify stable otel metric sent to Application Insights endpoint + List stableOtelMetrics = + testing.mockedIngestion.waitForItems( + "MetricData", OtlpLogAnalyticsOnAksTest::isStableOtelMetric, 1); + Envelope stableOtelMetricEnvelope = stableOtelMetrics.get(0); + assertThat( + ((MetricData) ((Data) stableOtelMetricEnvelope.getData()).getBaseData()) + .getMetrics() + .get(0) + .getName()) + .isEqualTo("http.server.request.duration"); + + // verify pre-aggregated standard metric sent to Application Insights endpoint + List standardMetrics = + testing.mockedIngestion.waitForStandardMetricItems("requests/duration", 1); + Envelope standardMetricEnvelope = standardMetrics.get(0); + MetricData standardMetricData = + (MetricData) ((Data) standardMetricEnvelope.getData()).getBaseData(); + assertThat(standardMetricData.getMetrics().get(0).getName()) + .isEqualTo("http.server.request.duration"); + assertThat(standardMetricData.getProperties().get("_MS.IsAutocollected")).isEqualTo("True"); + + // verify Statsbeat sent to the breeze endpoint + verifyStatsbeatSentToBreezeEndpoint(); + + // verify custom histogram metric 'histogram-test-otlp-exporter' and otel metric + // 'http.server.request.duration' sent to OTLP endpoint + // verify Statsbeat doesn't get sent to OTLP endpoint + verifyMetricsSentToOtlpEndpoint(); + } + + @SuppressWarnings("PreferJavaTimeOverload") // legacy time API required for backward compatibility + private void verifyMetricsSentToOtlpEndpoint() { + await() + .atMost(60, SECONDS) + .untilAsserted( + () -> { + HttpRequest[] requests = + testing + .mockedOtlpIngestion + .getCollectorServer() + .retrieveRecordedRequests(request()); + + // verify metrics + List metrics = + testing.mockedOtlpIngestion.extractMetricsFromRequests(requests); + assertThat(metrics) + .extracting(Metric::getName) + .contains("histogram-test-otlp-exporter", "http.server.request.duration") + .doesNotContain("Attach", "Feature"); // statsbeat + }); + } + + private static boolean isHistogramMetric(Envelope envelope) { + if (envelope.getData().getBaseType().equals("MetricData")) { + MetricData data = (MetricData) ((Data) envelope.getData()).getBaseData(); + return data.getMetrics().get(0).getName().equals("histogram-test-otlp-exporter"); + } + return false; + } + + private static boolean isStableOtelMetric(Envelope envelope) { + if (envelope.getData().getBaseType().equals("MetricData")) { + MetricData data = (MetricData) ((Data) envelope.getData()).getBaseData(); + return data.getMetrics().get(0).getName().equals("http.server.request.duration") + && data.getProperties().get("http.response.status_code") != null; + } + return false; + } + + private void verifyStatsbeatSentToBreezeEndpoint() throws Exception { + List statsbeatMetricList = + testing.mockedIngestion.waitForItems( + "MetricData", OtlpLogAnalyticsOnAksTest::isAttachStatsbeat, 1); + Envelope statsbeatEnvelope = statsbeatMetricList.get(0); + MetricData statsbeatMetricData = + (MetricData) ((Data) statsbeatEnvelope.getData()).getBaseData(); + assertThat(statsbeatMetricData.getMetrics().get(0).getName()).isEqualTo("Attach"); + assertThat(statsbeatMetricData.getProperties().get("rp")).isNotNull(); + assertThat(statsbeatMetricData.getProperties().get("attach")).isEqualTo("StandaloneAuto"); + + List features = + testing.mockedIngestion.waitForItems( + "MetricData", OtlpLogAnalyticsOnAksTest::isFeatureStatsbeat, 2); + Envelope featureEnvelope = features.get(0); + MetricData featureMetricData = (MetricData) ((Data) featureEnvelope.getData()).getBaseData(); + assertThat(featureMetricData.getMetrics().get(0).getName()).isEqualTo("Feature"); + assertThat(featureMetricData.getProperties().get("type")).isNotEmpty(); + + List requestSuccessCounts = + testing.mockedIngestion.waitForItems( + "MetricData", OtlpLogAnalyticsOnAksTest::isRequestSuccessCount, 1); + Envelope rscEnvelope = requestSuccessCounts.get(0); + MetricData rscMetricData = (MetricData) ((Data) rscEnvelope.getData()).getBaseData(); + assertThat(rscMetricData.getMetrics().get(0).getName()).isEqualTo("Request_Success_Count"); + assertThat(rscMetricData.getProperties().get("endpoint")).isEqualTo("breeze"); + + List requestDurations = + testing.mockedIngestion.waitForItems( + "MetricData", OtlpLogAnalyticsOnAksTest::isRequestDuration, 1); + Envelope rdEnvelope = requestDurations.get(0); + MetricData rdMetricData = (MetricData) ((Data) rdEnvelope.getData()).getBaseData(); + assertThat(rdMetricData.getMetrics().get(0).getName()).isEqualTo("Request_Duration"); + assertThat(rdMetricData.getProperties().get("endpoint")).isEqualTo("breeze"); + } + + private static boolean isAttachStatsbeat(Envelope envelope) { + if (envelope.getData().getBaseType().equals("MetricData")) { + MetricData data = (MetricData) ((Data) envelope.getData()).getBaseData(); + return data.getMetrics().get(0).getName().equals("Attach"); + } + return false; + } + + private static boolean isFeatureStatsbeat(Envelope envelope) { + if (envelope.getData().getBaseType().equals("MetricData")) { + MetricData data = (MetricData) ((Data) envelope.getData()).getBaseData(); + return data.getMetrics().get(0).getName().equals("Feature"); + } + return false; + } + + private static boolean isRequestSuccessCount(Envelope envelope) { + if (envelope.getData().getBaseType().equals("MetricData")) { + MetricData data = (MetricData) ((Data) envelope.getData()).getBaseData(); + return data.getMetrics().get(0).getName().equals("Request_Success_Count"); + } + return false; + } + + private static boolean isRequestDuration(Envelope envelope) { + if (envelope.getData().getBaseType().equals("MetricData")) { + MetricData data = (MetricData) ((Data) envelope.getData()).getBaseData(); + return data.getMetrics().get(0).getName().equals("Request_Duration"); + } + return false; + } + + @Environment(TOMCAT_8_JAVA_8) + static class Tomcat8Java8Test extends OtlpLogAnalyticsOnAksTest {} + + @Environment(TOMCAT_8_JAVA_8_OPENJ9) + static class Tomcat8Java8OpenJ9Test extends OtlpLogAnalyticsOnAksTest {} + + @Environment(TOMCAT_8_JAVA_11) + static class Tomcat8Java11Test extends OtlpLogAnalyticsOnAksTest {} + + @Environment(TOMCAT_8_JAVA_11_OPENJ9) + static class Tomcat8Java11OpenJ9Test extends OtlpLogAnalyticsOnAksTest {} + + @Environment(TOMCAT_8_JAVA_17) + static class Tomcat8Java17Test extends OtlpLogAnalyticsOnAksTest {} + + @Environment(TOMCAT_8_JAVA_17_OPENJ9) + static class Tomcat8Java17OpenJ9Test extends OtlpLogAnalyticsOnAksTest {} + + @Environment(TOMCAT_8_JAVA_21) + static class Tomcat8Java21Test extends OtlpLogAnalyticsOnAksTest {} + + @Environment(TOMCAT_8_JAVA_21_OPENJ9) + static class Tomcat8Java21OpenJ9Test extends OtlpLogAnalyticsOnAksTest {} + + @Environment(TOMCAT_8_JAVA_25) + static class Tomcat8Java23Test extends OtlpLogAnalyticsOnAksTest {} + + @Environment(TOMCAT_8_JAVA_25_OPENJ9) + static class Tomcat8Java23OpenJ9Test extends OtlpLogAnalyticsOnAksTest {} +} diff --git a/smoke-tests/framework/src/main/java/com/microsoft/applicationinsights/smoketest/SmokeTestExtension.java b/smoke-tests/framework/src/main/java/com/microsoft/applicationinsights/smoketest/SmokeTestExtension.java index ad7245cf5a..c3337724ea 100644 --- a/smoke-tests/framework/src/main/java/com/microsoft/applicationinsights/smoketest/SmokeTestExtension.java +++ b/smoke-tests/framework/src/main/java/com/microsoft/applicationinsights/smoketest/SmokeTestExtension.java @@ -114,6 +114,7 @@ public class SmokeTestExtension private final List jvmArgs; private final boolean useDefaultHttpPort; private final boolean useOtlpEndpoint; + private final boolean useOtlpEndpointOnly; public static SmokeTestExtension create() { return builder().build(); @@ -139,7 +140,8 @@ public static SmokeTestExtensionBuilder builder() { Map envVars, List jvmArgs, boolean useDefaultHttpPort, - boolean useOtlpEndpoint) { + boolean useOtlpEndpoint, + boolean useOtlpEndpointOnly) { this.skipHealthCheck = skipHealthCheck; this.readOnly = readOnly; this.dependencyContainer = dependencyContainer; @@ -169,6 +171,7 @@ public static SmokeTestExtensionBuilder builder() { this.jvmArgs = jvmArgs; this.useDefaultHttpPort = useDefaultHttpPort; this.useOtlpEndpoint = useOtlpEndpoint; + this.useOtlpEndpointOnly = useOtlpEndpointOnly; mockedIngestion = new MockedAppInsightsIngestionServer(useOld3xAgent); } @@ -220,7 +223,7 @@ private void prepareEnvironment(Environment environment) throws Exception { mockedIngestion.startServer(); mockedIngestion.setRequestLoggingEnabled(true); mockedIngestion.setQuickPulseRequestLoggingEnabled(true); - if (useOtlpEndpoint) { + if (useOtlpEndpoint || useOtlpEndpointOnly) { mockedOtlpIngestion.startServer(); } network = Network.newNetwork(); @@ -430,6 +433,12 @@ private void startTestApplicationContainer() throws Exception { envVars.put("OTEL_EXPORTER_OTLP_PROTOCOL", "http/protobuf"); } + if (useOtlpEndpointOnly) { + envVars.put("OTEL_METRICS_EXPORTER", "otlp"); + envVars.put("OTEL_EXPORTER_OTLP_METRICS_ENDPOINT", FAKE_OTLP_INGESTION_ENDPOINT); + envVars.put("OTEL_EXPORTER_OTLP_PROTOCOL", "http/protobuf"); + } + GenericContainer container; if (REMOTE_DEBUG || useDefaultHttpPort) { FixedHostPortGenericContainer fixedPortContainer = @@ -479,6 +488,13 @@ private void startTestApplicationContainer() throws Exception { javaToolOptions.add("-Dotel.exporter.otlp.metrics.endpoint=" + FAKE_OTLP_INGESTION_ENDPOINT); javaToolOptions.add("-Dotel.exporter.otlp.protocol=http/protobuf"); } + if (useOtlpEndpointOnly) { + // TODO (trask) don't use azure_monitor exporter for smoke test health check + javaToolOptions.add("-Dotel.metrics.exporter=otlp"); + javaToolOptions.add("-Dotel.exporter.otlp.metrics.endpoint=" + FAKE_OTLP_INGESTION_ENDPOINT); + javaToolOptions.add("-Dotel.exporter.otlp.protocol=http/protobuf"); + } + if (REMOTE_DEBUG) { javaToolOptions.add( "-agentlib:jdwp=transport=dt_socket,address=0.0.0.0:5005,server=y,suspend=y"); @@ -579,7 +595,7 @@ public void afterAll(ExtensionContext context) throws Exception { mockedIngestion.stopServer(); mockedIngestion.setRequestLoggingEnabled(false); mockedIngestion.setQuickPulseRequestLoggingEnabled(false); - if (useOtlpEndpoint) { + if (useOtlpEndpoint || useOtlpEndpointOnly) { mockedOtlpIngestion.stopServer(); } } diff --git a/smoke-tests/framework/src/main/java/com/microsoft/applicationinsights/smoketest/SmokeTestExtensionBuilder.java b/smoke-tests/framework/src/main/java/com/microsoft/applicationinsights/smoketest/SmokeTestExtensionBuilder.java index 610141bb78..e223c64aa0 100644 --- a/smoke-tests/framework/src/main/java/com/microsoft/applicationinsights/smoketest/SmokeTestExtensionBuilder.java +++ b/smoke-tests/framework/src/main/java/com/microsoft/applicationinsights/smoketest/SmokeTestExtensionBuilder.java @@ -29,6 +29,7 @@ public class SmokeTestExtensionBuilder { private final List jvmArgs = new ArrayList<>(); private boolean useDefaultHttpPort; private boolean useOtlpEndpoint; + private boolean useOtlpEndpointOnly; public SmokeTestExtensionBuilder setDependencyContainer( String envVarName, GenericContainer container) { @@ -108,6 +109,11 @@ public SmokeTestExtensionBuilder useOtlpEndpoint() { return this; } + public SmokeTestExtensionBuilder useOtlpEndpointOnly() { + this.useOtlpEndpointOnly = true; + return this; + } + public SmokeTestExtension build() { return new SmokeTestExtension( dependencyContainer, @@ -125,6 +131,7 @@ public SmokeTestExtension build() { envVars, jvmArgs, useDefaultHttpPort, - useOtlpEndpoint); + useOtlpEndpoint, + useOtlpEndpointOnly); } }