Extract metrics from communication

communication/build.gradle.kts

@@ -20,7 +20,7 @@ dependencies {
   api(libs.okio)
   api(libs.okhttp)
   api(libs.moshi)
-  implementation(libs.dogstatsd)
+  implementation(project(":products:metrics:lib"))
 
   testImplementation(project(":utils:test-utils"))
   testImplementation(libs.bundles.junit5)
@@ -49,9 +49,9 @@ val excludedClassesCoverage by extra(
     "datadog.communication.http.OkHttpUtils.GZipByteBufferRequestBody",
     "datadog.communication.http.OkHttpUtils.GZipRequestBodyDecorator",
     "datadog.communication.http.OkHttpUtils.JsonRequestBody",
-    "datadog.communication.monitor.DDAgentStatsDConnection",
-    "datadog.communication.monitor.DDAgentStatsDConnection.*",
-    "datadog.communication.monitor.LoggingStatsDClient",
+    // "datadog.communication.monitor.DDAgentStatsDConnection",
+    // "datadog.communication.monitor.DDAgentStatsDConnection.*",
+    // "datadog.communication.monitor.LoggingStatsDClient",
     "datadog.communication.BackendApiFactory",
     "datadog.communication.BackendApiFactory.Intake",
     "datadog.communication.EvpProxyApi",

communication/src/main/java/datadog/communication/ddagent/DDAgentFeaturesDiscovery.java

@@ -12,9 +12,9 @@
 import com.squareup.moshi.Types;
 import datadog.common.container.ContainerInfo;
 import datadog.communication.http.OkHttpUtils;
-import datadog.communication.monitor.DDAgentStatsDClientManager;
-import datadog.communication.monitor.Monitoring;
-import datadog.communication.monitor.Recording;
+import datadog.metrics.api.Monitoring;
+import datadog.metrics.api.Recording;
+import datadog.metrics.statsd.DDAgentStatsDClientManager;
 import datadog.trace.api.BaseHash;
 import datadog.trace.api.telemetry.LogCollector;
 import datadog.trace.util.Strings;

communication/src/main/java/datadog/communication/ddagent/SharedCommunicationObjects.java

@@ -6,7 +6,7 @@
 import datadog.common.container.ContainerInfo;
 import datadog.common.socket.SocketUtils;
 import datadog.communication.http.OkHttpUtils;
-import datadog.communication.monitor.Monitoring;
+import datadog.metrics.api.Monitoring;
 import datadog.remoteconfig.ConfigurationPoller;
 import datadog.remoteconfig.DefaultConfigurationPoller;
 import datadog.trace.api.Config;

communication/src/test/groovy/datadog/communication/ddagent/DDAgentFeaturesDiscoveryTest.groovy

@@ -1,7 +1,7 @@
 package datadog.communication.ddagent
 
 import datadog.common.container.ContainerInfo
-import datadog.communication.monitor.Monitoring
+import datadog.metrics.api.Monitoring
 import datadog.trace.test.util.DDSpecification
 import datadog.trace.util.Strings
 import okhttp3.Call

communication/src/test/groovy/datadog/communication/ddagent/SharedCommunicationsObjectsSpecification.groovy

@@ -1,6 +1,6 @@
 package datadog.communication.ddagent
 
-import datadog.communication.monitor.Monitoring
+import datadog.metrics.api.Monitoring
 import datadog.trace.api.Config
 import datadog.trace.test.util.DDSpecification
 import okhttp3.HttpUrl

dd-java-agent/agent-bootstrap/build.gradle

@@ -23,6 +23,7 @@ dependencies {
   api project(':dd-java-agent:agent-debugger:debugger-bootstrap')
   api project(':components:environment')
   api project(':components:json')
+  api project(':products:metrics:agent')
   api libs.instrument.java
   api libs.slf4j
   // ^ Generally a bad idea for libraries, but we're shadowing.

dd-java-agent/agent-bootstrap/src/main/java/datadog/trace/bootstrap/Agent.java

@@ -23,9 +23,9 @@
 import datadog.environment.SystemProperties;
 import datadog.instrument.classinject.ClassInjector;
 import datadog.instrument.utils.ClassLoaderValue;
+import datadog.metrics.statsd.StatsDClientManager;
 import datadog.trace.api.Config;
 import datadog.trace.api.Platform;
-import datadog.trace.api.StatsDClientManager;
 import datadog.trace.api.WithGlobalTracer;
 import datadog.trace.api.appsec.AppSecEventTracker;
 import datadog.trace.api.config.AppSecConfig;
@@ -663,6 +663,7 @@ public InstallDatadogTracerCallback(
         throw new UndeclaredThrowableException(e);
       }
 
+      installDatadogMeter(initTelemetry);
       installDatadogTracer(initTelemetry, scoClass, sco);
       maybeInstallLogsIntake(scoClass, sco);
       maybeStartIast(instrumentation);
@@ -795,6 +796,35 @@ private static synchronized void startDatadogAgent(
     }
   }
 
+  private static synchronized void installDatadogMeter(InitializationTelemetry initTelemetry) {
+    if (AGENT_CLASSLOADER == null) {
+      throw new IllegalStateException("Datadog agent should have been started already");
+    }
+
+    StaticEventLogger.begin("AgentMeter");
+    try {
+      AGENT_CLASSLOADER.loadClass("datadog.trace.agent.tooling.MeterInstaller");
+    } catch (ClassNotFoundException e) {
+      log.error("Error loading DatadogMeter", e);
+    }
+
+    try {
+      // Install AgentMeter, StatsDClient and Monitoring
+      final Class<?> tracerInstallerClass =
+          AGENT_CLASSLOADER.loadClass("datadog.trace.agent.tooling.MeterInstaller");
+      final Method installMeterMethod = tracerInstallerClass.getMethod("installMeter");
+      installMeterMethod.invoke(null);
+    } catch (final FatalAgentMisconfigurationError ex) {
+      throw ex;
+    } catch (final Throwable ex) {
+      log.error("Throwable thrown while installing the Datadog meter", ex);
+
+      initTelemetry.onFatalError(ex);
+    }
+
+    StaticEventLogger.end("AgentMeter");
+  }
+
   private static synchronized void installDatadogTracer(
       InitializationTelemetry initTelemetry, Class<?> scoClass, Object sco) {
     if (AGENT_CLASSLOADER == null) {

dd-java-agent/agent-crashtracking/build.gradle

@@ -14,6 +14,7 @@ dependencies {
   implementation libs.slf4j
   implementation project(':communication')
   implementation project(':internal-api')
+  implementation project(':products:metrics:lib')
   implementation project(':utils:container-utils')
   implementation project(':utils:version-utils')
   implementation project(path: ':dd-java-agent:ddprof-lib', configuration: 'shadow')

dd-java-agent/agent-crashtracking/src/main/java/datadog/crashtracking/OOMENotifier.java

@@ -1,8 +1,8 @@
 package datadog.crashtracking;
 
-import static datadog.communication.monitor.DDAgentStatsDClientManager.statsDClientManager;
+import static datadog.metrics.statsd.DDAgentStatsDClientManager.statsDClientManager;
 
-import datadog.trace.api.StatsDClient;
+import datadog.metrics.statsd.StatsDClient;
 import de.thetaphi.forbiddenapis.SuppressForbidden;
 import java.util.concurrent.locks.LockSupport;
 import org.slf4j.Logger;

dd-java-agent/agent-debugger/build.gradle

@@ -35,6 +35,7 @@ dependencies {
   implementation libs.bundles.asm
   implementation project(':internal-api')
   implementation project(':communication')
+  implementation project(':products:metrics:lib')
   compileOnly project(':dd-java-agent:agent-tooling')
   compileOnly project(':dd-java-agent:agent-builder')
   compileOnly project(':dd-trace-core')

dd-java-agent/agent-debugger/src/main/java/com/datadog/debugger/agent/StatsdMetricForwarder.java

@@ -2,9 +2,9 @@
 
 import com.datadog.debugger.sink.ProbeStatusSink;
 import com.timgroup.statsd.StatsDClientErrorHandler;
-import datadog.communication.monitor.DDAgentStatsDClientManager;
+import datadog.metrics.statsd.DDAgentStatsDClientManager;
+import datadog.metrics.statsd.StatsDClient;
 import datadog.trace.api.Config;
-import datadog.trace.api.StatsDClient;
 import datadog.trace.bootstrap.debugger.DebuggerContext;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;

dd-java-agent/agent-debugger/src/main/java/com/datadog/debugger/util/DebuggerMetrics.java

@@ -1,8 +1,8 @@
 package com.datadog.debugger.util;
 
-import datadog.communication.monitor.DDAgentStatsDClientManager;
+import datadog.metrics.statsd.DDAgentStatsDClientManager;
+import datadog.metrics.statsd.StatsDClient;
 import datadog.trace.api.Config;
-import datadog.trace.api.StatsDClient;
 
 /** implements a StatsD client for internal debugger agent metrics */
 public class DebuggerMetrics implements StatsDClient {

dd-java-agent/agent-jmxfetch/src/main/java/datadog/trace/agent/jmxfetch/AgentStatsdReporter.java

@@ -1,6 +1,6 @@
 package datadog.trace.agent.jmxfetch;
 
-import datadog.trace.api.StatsDClient;
+import datadog.metrics.statsd.StatsDClient;
 import datadog.trace.api.flare.TracerFlare;
 import java.io.IOException;
 import java.text.NumberFormat;

dd-java-agent/agent-jmxfetch/src/main/java/datadog/trace/agent/jmxfetch/JMXFetch.java

@@ -5,10 +5,10 @@
 import static org.datadog.jmxfetch.AppConfig.ACTION_COLLECT;
 
 import datadog.environment.SystemProperties;
+import datadog.metrics.statsd.StatsDClient;
+import datadog.metrics.statsd.StatsDClientManager;
 import datadog.trace.api.Config;
 import datadog.trace.api.GlobalTracer;
-import datadog.trace.api.StatsDClient;
-import datadog.trace.api.StatsDClientManager;
 import datadog.trace.api.flare.TracerFlare;
 import datadog.trace.api.telemetry.LogCollector;
 import de.thetaphi.forbiddenapis.SuppressForbidden;

dd-java-agent/agent-tooling/build.gradle

@@ -63,6 +63,7 @@ dependencies {
   api libs.bytebuddyagent
   implementation group: 'net.java.dev.jna', name: 'jna', version: '5.8.0'
   implementation group: 'net.java.dev.jna', name: 'jna-platform', version: '5.8.0'
+  implementation project(':products:metrics:lib')
 
   api project(':dd-trace-core')
   implementation project(':dd-java-agent:agent-crashtracking')

dd-java-agent/agent-tooling/src/main/java/datadog/trace/agent/tooling/MeterInstaller.java

@@ -0,0 +1,86 @@
+package datadog.trace.agent.tooling;
+
+import static datadog.metrics.statsd.DDAgentStatsDClientManager.statsDClientManager;
+import static java.util.concurrent.TimeUnit.SECONDS;
+
+import datadog.metrics.agent.AgentMeter;
+import datadog.metrics.api.DDSketchHistograms;
+import datadog.metrics.api.Monitoring;
+import datadog.metrics.api.MonitoringImpl;
+import datadog.metrics.statsd.StatsDClient;
+import datadog.trace.api.Config;
+import datadog.trace.api.config.GeneralConfig;
+import datadog.trace.core.DDTraceCoreInfo;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Map;
+
+public class MeterInstaller {
+  private static final String LANG_STATSD_TAG = "lang";
+  private static final String LANG_VERSION_STATSD_TAG = "lang_version";
+  private static final String LANG_INTERPRETER_STATSD_TAG = "lang_interpreter";
+  private static final String LANG_INTERPRETER_VENDOR_STATSD_TAG = "lang_interpreter_vendor";
+  private static final String TRACER_VERSION_STATSD_TAG = "tracer_version";
+
+  public static void installMeter() {
+    Config config = Config.get();
+    StatsDClient statsDClient = createStatsDClient(config);
+    Monitoring monitoring =
+        config.isHealthMetricsEnabled()
+            ? new MonitoringImpl(statsDClient, 10, SECONDS)
+            : MonitoringImpl.DISABLED;
+    AgentMeter.registerIfAbsent(statsDClient, monitoring, DDSketchHistograms.INSTANCE);
+  }
+
+  private static StatsDClient createStatsDClient(Config config) {
+    if (!config.isHealthMetricsEnabled()) {
+      return StatsDClient.NO_OP;
+    } else {
+      String host = config.getHealthMetricsStatsdHost();
+      if (host == null) {
+        host = config.getJmxFetchStatsdHost();
+      }
+      Integer port = config.getHealthMetricsStatsdPort();
+      if (port == null) {
+        port = config.getJmxFetchStatsdPort();
+      }
+
+      return statsDClientManager()
+          .statsDClient(
+              host,
+              port,
+              config.getDogStatsDNamedPipe(),
+              // use replace to stop string being changed to 'ddtrot.dd.tracer' in dd-trace-ot
+              "datadog:tracer".replace(':', '.'),
+              generateConstantTags(config));
+    }
+  }
+
+  private static String[] generateConstantTags(final Config config) {
+    final List<String> constantTags = new ArrayList<>();
+
+    constantTags.add(statsdTag(LANG_STATSD_TAG, "java"));
+    constantTags.add(statsdTag(LANG_VERSION_STATSD_TAG, DDTraceCoreInfo.JAVA_VERSION));
+    constantTags.add(statsdTag(LANG_INTERPRETER_STATSD_TAG, DDTraceCoreInfo.JAVA_VM_NAME));
+    constantTags.add(statsdTag(LANG_INTERPRETER_VENDOR_STATSD_TAG, DDTraceCoreInfo.JAVA_VM_VENDOR));
+    constantTags.add(statsdTag(TRACER_VERSION_STATSD_TAG, DDTraceCoreInfo.VERSION));
+    constantTags.add(statsdTag("service", config.getServiceName()));
+
+    final Map<String, String> mergedSpanTags = config.getMergedSpanTags();
+    final String version = mergedSpanTags.get(GeneralConfig.VERSION);
+    if (version != null && !version.isEmpty()) {
+      constantTags.add(statsdTag("version", version));
+    }
+
+    final String env = mergedSpanTags.get(GeneralConfig.ENV);
+    if (env != null && !env.isEmpty()) {
+      constantTags.add(statsdTag("env", env));
+    }
+
+    return constantTags.toArray(new String[0]);
+  }
+
+  private static String statsdTag(final String tagPrefix, final String tagValue) {
+    return tagPrefix + ":" + tagValue;
+  }
+}

dd-java-agent/agent-tooling/src/main/java/datadog/trace/agent/tooling/nativeimage/TracerActivation.java

@@ -2,11 +2,11 @@
 
 import com.datadog.profiling.controller.openjdk.JFREventContextIntegration;
 import datadog.communication.ddagent.SharedCommunicationObjects;
-import datadog.communication.monitor.DDAgentStatsDClientManager;
+import datadog.metrics.statsd.DDAgentStatsDClientManager;
+import datadog.metrics.statsd.StatsDClientManager;
 import datadog.trace.agent.jmxfetch.JMXFetch;
 import datadog.trace.agent.tooling.ProfilerInstaller;
 import datadog.trace.agent.tooling.TracerInstaller;
-import datadog.trace.api.StatsDClientManager;
 import datadog.trace.bootstrap.instrumentation.api.ProfilingContextIntegration;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;

dd-java-agent/appsec/appsec-test-fixtures/src/main/groovy/com/datadog/appsec/AppSecHttpServerTest.groovy

@@ -1,7 +1,7 @@
 package com.datadog.appsec
 
 import datadog.communication.ddagent.SharedCommunicationObjects
-import datadog.communication.monitor.Monitoring
+import datadog.metrics.api.Monitoring
 import datadog.trace.agent.test.base.WithHttpServer
 import datadog.trace.api.Config
 import datadog.trace.api.appsec.AppSecEventTracker
@@ -24,7 +24,7 @@ abstract class AppSecHttpServerTest<SERVER> extends WithHttpServer<SERVER> {
     def config = Config.get()
     sco.createRemaining(config)
     assert sco.configurationPoller(config) == null
-    assert sco.monitoring instanceof Monitoring.DisabledMonitoring
+    assert sco.monitoring == Monitoring.DISABLED
     AppSecEventTracker.install()
 
     AppSecSystem.start(ss, sco)

dd-java-agent/appsec/appsec-test-fixtures/src/main/groovy/com/datadog/appsec/AppSecInactiveHttpServerTest.groovy

@@ -1,7 +1,7 @@
 package com.datadog.appsec
 
 import datadog.communication.ddagent.SharedCommunicationObjects
-import datadog.communication.monitor.Monitoring
+import datadog.metrics.api.Monitoring
 import datadog.trace.agent.test.base.HttpServerTest.ServerEndpoint
 import datadog.trace.agent.test.base.WithHttpServer
 import datadog.trace.api.Config
@@ -44,7 +44,7 @@ abstract class AppSecInactiveHttpServerTest extends WithHttpServer {
     def config = Config.get()
     sco.createRemaining(config)
     assert sco.configurationPoller(config) == null
-    assert sco.monitoring instanceof Monitoring.DisabledMonitoring
+    assert sco.monitoring == Monitoring.DISABLED
 
     AppSecSystem.start(ss, sco)
     assert !AppSecSystem.active

dd-java-agent/appsec/build.gradle

@@ -14,6 +14,7 @@ dependencies {
   api libs.slf4j
   implementation project(':internal-api')
   implementation project(':communication')
+  implementation project(':products:metrics:api')
   implementation project(':telemetry')
   implementation group: 'io.sqreen', name: 'libsqreen', version: '17.3.0'
   implementation libs.moshi

dd-java-agent/appsec/src/jmh/java/datadog/appsec/benchmark/AppSecBenchmark.java

@@ -7,7 +7,7 @@
 import com.datadog.appsec.AppSecSystem;
 import datadog.communication.ddagent.DDAgentFeaturesDiscovery;
 import datadog.communication.ddagent.SharedCommunicationObjects;
-import datadog.communication.monitor.Monitoring;
+import datadog.metrics.api.Monitoring;
 import datadog.trace.api.gateway.BlockResponseFunction;
 import datadog.trace.api.gateway.CallbackProvider;
 import datadog.trace.api.gateway.Flow;