From 07e5e9a59f452df5553082fa43fb34765da7be7f Mon Sep 17 00:00:00 2001 From: Piotr Wolski Date: Fri, 14 Nov 2025 09:21:46 -0700 Subject: [PATCH 01/12] Track schema registry usage --- .gitignore | 4 + .../RecordingDatastreamsPayloadWriter.groovy | 17 + .../confluent-schema-registry-7.0/README.md | 141 +++++++++ .../build.gradle | 24 ++ .../ConfluentSchemaRegistryModule.java | 45 +++ .../KafkaAvroDeserializerInstrumentation.java | 175 +++++++++++ .../KafkaAvroSerializerInstrumentation.java | 200 ++++++++++++ .../SchemaRegistryClientInstrumentation.java | 128 ++++++++ .../SchemaRegistryContext.java | 61 ++++ .../SchemaRegistryMetrics.java | 240 ++++++++++++++ ...fluentSchemaRegistryDataStreamsTest.groovy | 294 ++++++++++++++++++ .../groovy/ConfluentSchemaRegistryTest.groovy | 204 ++++++++++++ .../KafkaProducerInstrumentation.java | 15 + .../DefaultDataStreamsMonitoring.java | 28 ++ .../MsgPackDatastreamsPayloadWriter.java | 65 +++- .../trace/core/datastreams/StatsBucket.java | 99 ++++++ .../AgentDataStreamsMonitoring.java | 12 + .../NoopDataStreamsMonitoring.java | 4 + .../api/datastreams/SchemaRegistryUsage.java | 60 ++++ settings.gradle.kts | 1 + 20 files changed, 1816 insertions(+), 1 deletion(-) create mode 100644 dd-java-agent/instrumentation/confluent-schema-registry/confluent-schema-registry-7.0/README.md create mode 100644 dd-java-agent/instrumentation/confluent-schema-registry/confluent-schema-registry-7.0/build.gradle create mode 100644 dd-java-agent/instrumentation/confluent-schema-registry/confluent-schema-registry-7.0/src/main/java/datadog/trace/instrumentation/confluentschemaregistry/ConfluentSchemaRegistryModule.java create mode 100644 dd-java-agent/instrumentation/confluent-schema-registry/confluent-schema-registry-7.0/src/main/java/datadog/trace/instrumentation/confluentschemaregistry/KafkaAvroDeserializerInstrumentation.java create mode 100644 dd-java-agent/instrumentation/confluent-schema-registry/confluent-schema-registry-7.0/src/main/java/datadog/trace/instrumentation/confluentschemaregistry/KafkaAvroSerializerInstrumentation.java create mode 100644 dd-java-agent/instrumentation/confluent-schema-registry/confluent-schema-registry-7.0/src/main/java/datadog/trace/instrumentation/confluentschemaregistry/SchemaRegistryClientInstrumentation.java create mode 100644 dd-java-agent/instrumentation/confluent-schema-registry/confluent-schema-registry-7.0/src/main/java/datadog/trace/instrumentation/confluentschemaregistry/SchemaRegistryContext.java create mode 100644 dd-java-agent/instrumentation/confluent-schema-registry/confluent-schema-registry-7.0/src/main/java/datadog/trace/instrumentation/confluentschemaregistry/SchemaRegistryMetrics.java create mode 100644 dd-java-agent/instrumentation/confluent-schema-registry/confluent-schema-registry-7.0/src/test/groovy/ConfluentSchemaRegistryDataStreamsTest.groovy create mode 100644 dd-java-agent/instrumentation/confluent-schema-registry/confluent-schema-registry-7.0/src/test/groovy/ConfluentSchemaRegistryTest.groovy create mode 100644 internal-api/src/main/java/datadog/trace/api/datastreams/SchemaRegistryUsage.java diff --git a/.gitignore b/.gitignore index c0dcdb32d1b..686905af989 100644 --- a/.gitignore +++ b/.gitignore @@ -46,6 +46,10 @@ out/ ###################### .vscode +# Cursor # +########## +.cursor + # Others # ########## /logs/* diff --git a/dd-java-agent/instrumentation-testing/src/main/groovy/datadog/trace/agent/test/datastreams/RecordingDatastreamsPayloadWriter.groovy b/dd-java-agent/instrumentation-testing/src/main/groovy/datadog/trace/agent/test/datastreams/RecordingDatastreamsPayloadWriter.groovy index b963a0a08bc..154c99edd50 100644 --- a/dd-java-agent/instrumentation-testing/src/main/groovy/datadog/trace/agent/test/datastreams/RecordingDatastreamsPayloadWriter.groovy +++ b/dd-java-agent/instrumentation-testing/src/main/groovy/datadog/trace/agent/test/datastreams/RecordingDatastreamsPayloadWriter.groovy @@ -19,6 +19,9 @@ class RecordingDatastreamsPayloadWriter implements DatastreamsPayloadWriter { @SuppressWarnings('UnusedPrivateField') private final Set backlogs = [] + @SuppressWarnings('UnusedPrivateField') + private final List schemaRegistryUsages = [] + private final Set serviceNameOverrides = [] @Override @@ -33,6 +36,11 @@ class RecordingDatastreamsPayloadWriter implements DatastreamsPayloadWriter { this.@backlogs.add(backlog.getKey()) } } + if (bucket.schemaRegistryUsages != null) { + for (Map.Entry usage : bucket.schemaRegistryUsages) { + this.@schemaRegistryUsages.add(usage.getKey()) + } + } } } @@ -52,10 +60,15 @@ class RecordingDatastreamsPayloadWriter implements DatastreamsPayloadWriter { Collections.unmodifiableList(new ArrayList<>(this.@backlogs)) } + synchronized List getSchemaRegistryUsages() { + Collections.unmodifiableList(new ArrayList<>(this.@schemaRegistryUsages)) + } + synchronized void clear() { this.@payloads.clear() this.@groups.clear() this.@backlogs.clear() + this.@schemaRegistryUsages.clear() } void waitForPayloads(int count, long timeout = TimeUnit.SECONDS.toMillis(3)) { @@ -70,6 +83,10 @@ class RecordingDatastreamsPayloadWriter implements DatastreamsPayloadWriter { waitFor(count, timeout, this.@backlogs) } + void waitForSchemaRegistryUsages(int count, long timeout = TimeUnit.SECONDS.toMillis(3)) { + waitFor(count, timeout, this.@schemaRegistryUsages) + } + private static void waitFor(int count, long timeout, Collection collection) { long deadline = System.currentTimeMillis() + timeout while (System.currentTimeMillis() < deadline) { diff --git a/dd-java-agent/instrumentation/confluent-schema-registry/confluent-schema-registry-7.0/README.md b/dd-java-agent/instrumentation/confluent-schema-registry/confluent-schema-registry-7.0/README.md new file mode 100644 index 00000000000..cba91edc90c --- /dev/null +++ b/dd-java-agent/instrumentation/confluent-schema-registry/confluent-schema-registry-7.0/README.md @@ -0,0 +1,141 @@ +# Confluent Schema Registry Instrumentation + +This instrumentation module provides detailed observability for Confluent Schema Registry operations in Kafka applications. + +## Features + +This instrumentation captures: + +### Producer Operations +- **Schema Registration**: Tracks when schemas are registered with the Schema Registry + - Subject name + - Schema ID assigned + - Success/failure status + - Compatibility check results +- **Serialization**: Logs every message serialization with: + - Topic name + - Key schema ID (if applicable) + - Value schema ID + - Success/failure status + +### Consumer Operations +- **Deserialization**: Tracks every message deserialization with: + - Topic name + - Key schema ID (if present in message) + - Value schema ID (extracted from Confluent wire format) + - Success/failure status + +### Schema Registry Client Operations +- **Schema Registration** (`register()` method) + - Successful registrations with schema ID + - Compatibility failures with error details +- **Compatibility Checks** (`testCompatibility()` method) + - Pass/fail status + - Error messages for incompatible schemas +- **Schema Retrieval** (`getSchemaById()` method) + - Schema ID lookups during deserialization + +## Metrics Collected + +The `SchemaRegistryMetrics` class tracks: + +- `schemaRegistrationSuccess` - Count of successful schema registrations +- `schemaRegistrationFailure` - Count of failed schema registrations (compatibility issues) +- `schemaCompatibilitySuccess` - Count of successful compatibility checks +- `schemaCompatibilityFailure` - Count of failed compatibility checks +- `serializationSuccess` - Count of successful message serializations +- `serializationFailure` - Count of failed serializations +- `deserializationSuccess` - Count of successful message deserializations +- `deserializationFailure` - Count of failed deserializations + +## Log Output Examples + +### Successful Producer Operation +``` +[Schema Registry] Schema registered successfully - Subject: myTopic-value, Schema ID: 123, Is Key: false, Topic: myTopic +[Schema Registry] Produce to topic 'myTopic', schema for key: none, schema for value: 123, serializing: VALUE +``` + +### Failed Schema Registration (Incompatibility) +``` +[Schema Registry] Schema registration FAILED - Subject: myTopic-value, Is Key: false, Topic: myTopic, Error: Schema being registered is incompatible with an earlier schema +[Schema Registry] Schema compatibility check FAILED - Subject: myTopic-value, Error: Schema being registered is incompatible with an earlier schema +[Schema Registry] Serialization FAILED for topic 'myTopic', VALUE - Error: Schema being registered is incompatible with an earlier schema +``` + +### Consumer Operation +``` +[Schema Registry] Retrieved schema from registry - Schema ID: 123, Type: Schema +[Schema Registry] Consume from topic 'myTopic', schema for key: none, schema for value: 123, deserializing: VALUE +``` + +## Supported Serialization Formats + +- **Avro** (via `KafkaAvroSerializer`/`KafkaAvroDeserializer`) +- **Protobuf** (via `KafkaProtobufSerializer`/`KafkaProtobufDeserializer`) + +## Implementation Details + +### Instrumented Classes + +1. **CachedSchemaRegistryClient** - The main Schema Registry client + - `register(String subject, Schema schema)` - Schema registration + - `testCompatibility(String subject, Schema schema)` - Compatibility testing + - `getSchemaById(int id)` - Schema retrieval + +2. **AbstractKafkaSchemaSerDe and subclasses** - Serializers + - `serialize(String topic, Object data)` - Message serialization + - `serialize(String topic, Headers headers, Object data)` - With headers (Kafka 2.1+) + +3. **AbstractKafkaSchemaSerDe and subclasses** - Deserializers + - `deserialize(String topic, byte[] data)` - Message deserialization + - `deserialize(String topic, Headers headers, byte[] data)` - With headers (Kafka 2.1+) + +### Context Management + +The `SchemaRegistryContext` class uses ThreadLocal storage to pass context between: +- Serializer → Schema Registry Client (for logging topic information) +- Deserializer → Schema Registry Client (for logging topic information) + +This allows the instrumentation to correlate schema operations with the topics they're associated with. + +## Usage + +This instrumentation is automatically activated when: +1. Confluent Schema Registry client (version 7.0.0+) is present on the classpath +2. The Datadog Java agent is attached to the JVM + +No configuration or code changes are required. + +## Metrics Access + +To access metrics programmatically: + +```java +import datadog.trace.instrumentation.confluentschemaregistry.SchemaRegistryMetrics; + +// Get current counts +long registrationFailures = SchemaRegistryMetrics.getSchemaRegistrationFailureCount(); +long compatibilityFailures = SchemaRegistryMetrics.getSchemaCompatibilityFailureCount(); +long serializationFailures = SchemaRegistryMetrics.getSerializationFailureCount(); + +// Print summary +SchemaRegistryMetrics.printSummary(); +``` + +## Monitoring Schema Compatibility Issues + +The primary use case for this instrumentation is to detect and monitor schema compatibility issues that cause production failures. By tracking `schemaRegistrationFailure` and `schemaCompatibilityFailure` metrics, you can: + +1. **Alert on schema compatibility failures** before they impact production +2. **Track the rate of schema-related errors** per topic +3. **Identify problematic schema changes** that break compatibility +4. **Monitor serialization/deserialization failure rates** as a proxy for schema issues + +## Future Enhancements + +Potential additions: +- JSON Schema serializer support (currently excluded due to dependency issues) +- Schema evolution tracking +- Schema version diff logging +- Integration with Datadog APM for schema-related span tags diff --git a/dd-java-agent/instrumentation/confluent-schema-registry/confluent-schema-registry-7.0/build.gradle b/dd-java-agent/instrumentation/confluent-schema-registry/confluent-schema-registry-7.0/build.gradle new file mode 100644 index 00000000000..89a2b2821f0 --- /dev/null +++ b/dd-java-agent/instrumentation/confluent-schema-registry/confluent-schema-registry-7.0/build.gradle @@ -0,0 +1,24 @@ +apply from: "$rootDir/gradle/java.gradle" + +muzzle { + pass { + group = "io.confluent" + module = "kafka-schema-registry-client" + versions = "[7.0.0,)" + assertInverse = true + } +} + +dependencies { + compileOnly group: 'io.confluent', name: 'kafka-schema-registry-client', version: '7.0.0' + compileOnly group: 'io.confluent', name: 'kafka-avro-serializer', version: '7.0.0' + compileOnly group: 'io.confluent', name: 'kafka-protobuf-serializer', version: '7.0.0' + compileOnly group: 'org.apache.kafka', name: 'kafka-clients', version: '3.0.0' + + testImplementation group: 'io.confluent', name: 'kafka-schema-registry-client', version: '7.5.2' + testImplementation group: 'io.confluent', name: 'kafka-avro-serializer', version: '7.5.2' + testImplementation group: 'io.confluent', name: 'kafka-protobuf-serializer', version: '7.5.1' + testImplementation group: 'org.apache.kafka', name: 'kafka-clients', version: '3.5.0' + testImplementation group: 'org.apache.avro', name: 'avro', version: '1.11.0' +} + diff --git a/dd-java-agent/instrumentation/confluent-schema-registry/confluent-schema-registry-7.0/src/main/java/datadog/trace/instrumentation/confluentschemaregistry/ConfluentSchemaRegistryModule.java b/dd-java-agent/instrumentation/confluent-schema-registry/confluent-schema-registry-7.0/src/main/java/datadog/trace/instrumentation/confluentschemaregistry/ConfluentSchemaRegistryModule.java new file mode 100644 index 00000000000..47f10064646 --- /dev/null +++ b/dd-java-agent/instrumentation/confluent-schema-registry/confluent-schema-registry-7.0/src/main/java/datadog/trace/instrumentation/confluentschemaregistry/ConfluentSchemaRegistryModule.java @@ -0,0 +1,45 @@ +package datadog.trace.instrumentation.confluentschemaregistry; + +import static datadog.trace.agent.tooling.bytebuddy.matcher.ClassLoaderMatchers.hasClassNamed; +import static java.util.Arrays.asList; + +import com.google.auto.service.AutoService; +import datadog.trace.agent.tooling.Instrumenter; +import datadog.trace.agent.tooling.InstrumenterModule; +import java.util.List; +import net.bytebuddy.matcher.ElementMatcher; + +/** + * Instrumentation module for Confluent Schema Registry to capture schema operations including + * registration, compatibility checks, serialization, and deserialization. + */ +@AutoService(InstrumenterModule.class) +public class ConfluentSchemaRegistryModule extends InstrumenterModule.Tracing { + + public ConfluentSchemaRegistryModule() { + super("confluent-schema-registry"); + } + + @Override + public String[] helperClassNames() { + return new String[] { + packageName + ".SchemaRegistryMetrics", packageName + ".SchemaRegistryContext", + }; + } + + @Override + @SuppressWarnings("unchecked") + public List typeInstrumentations() { + return (List) + (List) + asList( + new SchemaRegistryClientInstrumentation(), + new KafkaAvroSerializerInstrumentation(), + new KafkaAvroDeserializerInstrumentation()); + } + + @Override + public ElementMatcher.Junction classLoaderMatcher() { + return hasClassNamed("io.confluent.kafka.schemaregistry.client.SchemaRegistryClient"); + } +} diff --git a/dd-java-agent/instrumentation/confluent-schema-registry/confluent-schema-registry-7.0/src/main/java/datadog/trace/instrumentation/confluentschemaregistry/KafkaAvroDeserializerInstrumentation.java b/dd-java-agent/instrumentation/confluent-schema-registry/confluent-schema-registry-7.0/src/main/java/datadog/trace/instrumentation/confluentschemaregistry/KafkaAvroDeserializerInstrumentation.java new file mode 100644 index 00000000000..19b3aa44c18 --- /dev/null +++ b/dd-java-agent/instrumentation/confluent-schema-registry/confluent-schema-registry-7.0/src/main/java/datadog/trace/instrumentation/confluentschemaregistry/KafkaAvroDeserializerInstrumentation.java @@ -0,0 +1,175 @@ +package datadog.trace.instrumentation.confluentschemaregistry; + +import static datadog.trace.agent.tooling.bytebuddy.matcher.NameMatchers.named; +import static net.bytebuddy.matcher.ElementMatchers.isMethod; +import static net.bytebuddy.matcher.ElementMatchers.isPublic; +import static net.bytebuddy.matcher.ElementMatchers.takesArgument; +import static net.bytebuddy.matcher.ElementMatchers.takesArguments; + +import datadog.trace.agent.tooling.Instrumenter; +import datadog.trace.agent.tooling.Instrumenter.MethodTransformer; +import net.bytebuddy.asm.Advice; +import net.bytebuddy.description.type.TypeDescription; +import net.bytebuddy.matcher.ElementMatcher; + +/** + * Instruments AbstractKafkaSchemaSerDe (base class for Avro, Protobuf, and JSON deserializers) to + * capture deserialization operations. + */ +public class KafkaAvroDeserializerInstrumentation + implements Instrumenter.ForTypeHierarchy, Instrumenter.HasMethodAdvice { + + @Override + public String hierarchyMarkerType() { + return "io.confluent.kafka.serializers.AbstractKafkaSchemaSerDe"; + } + + @Override + public ElementMatcher hierarchyMatcher() { + return named("io.confluent.kafka.serializers.AbstractKafkaSchemaSerDe") + .or(named("io.confluent.kafka.serializers.AbstractKafkaAvroDeserializer")) + .or(named("io.confluent.kafka.serializers.KafkaAvroDeserializer")) + .or(named("io.confluent.kafka.serializers.protobuf.KafkaProtobufDeserializer")); + } + + @Override + public void methodAdvice(MethodTransformer transformer) { + // Instrument deserialize(String topic, byte[] data) + transformer.applyAdvice( + isMethod() + .and(named("deserialize")) + .and(isPublic()) + .and(takesArguments(2)) + .and(takesArgument(0, String.class)) + .and(takesArgument(1, byte[].class)), + getClass().getName() + "$DeserializeAdvice"); + + // Instrument deserialize(String topic, Headers headers, byte[] data) for Kafka 2.1+ + transformer.applyAdvice( + isMethod() + .and(named("deserialize")) + .and(isPublic()) + .and(takesArguments(3)) + .and(takesArgument(0, String.class)) + .and(takesArgument(2, byte[].class)), + getClass().getName() + "$DeserializeWithHeadersAdvice"); + } + + public static class DeserializeAdvice { + @Advice.OnMethodEnter(suppress = Throwable.class) + public static void onEnter( + @Advice.This Object deserializer, + @Advice.Argument(0) String topic, + @Advice.Argument(1) byte[] data) { + // Set the topic in context + SchemaRegistryContext.setTopic(topic); + + // Determine if this is a key or value deserializer + String className = deserializer.getClass().getSimpleName(); + boolean isKey = + className.contains("Key") || deserializer.getClass().getName().contains("Key"); + SchemaRegistryContext.setIsKey(isKey); + + // Extract schema ID from the data if present (Confluent wire format) + if (data != null && data.length >= 5) { + // Confluent wire format: [magic_byte][4-byte schema id][data] + if (data[0] == 0) { + int schemaId = + ((data[1] & 0xFF) << 24) + | ((data[2] & 0xFF) << 16) + | ((data[3] & 0xFF) << 8) + | (data[4] & 0xFF); + + if (isKey) { + SchemaRegistryContext.setKeySchemaId(schemaId); + } else { + SchemaRegistryContext.setValueSchemaId(schemaId); + } + } + } + } + + @Advice.OnMethodExit(onThrowable = Throwable.class, suppress = Throwable.class) + public static void onExit( + @Advice.This Object deserializer, + @Advice.Argument(0) String topic, + @Advice.Return Object result, + @Advice.Thrown Throwable throwable) { + + Boolean isKey = SchemaRegistryContext.getIsKey(); + Integer keySchemaId = SchemaRegistryContext.getKeySchemaId(); + Integer valueSchemaId = SchemaRegistryContext.getValueSchemaId(); + + if (throwable != null) { + SchemaRegistryMetrics.recordDeserializationFailure( + topic, throwable.getMessage(), isKey != null ? isKey : false); + } else if (result != null) { + // Successful deserialization + SchemaRegistryMetrics.recordDeserialization( + topic, keySchemaId, valueSchemaId, isKey != null ? isKey : false); + } + + // Clear context after deserialization + SchemaRegistryContext.clear(); + } + } + + public static class DeserializeWithHeadersAdvice { + @Advice.OnMethodEnter(suppress = Throwable.class) + public static void onEnter( + @Advice.This Object deserializer, + @Advice.Argument(0) String topic, + @Advice.Argument(2) byte[] data) { + // Set the topic in context + SchemaRegistryContext.setTopic(topic); + + // Determine if this is a key or value deserializer + String className = deserializer.getClass().getSimpleName(); + boolean isKey = + className.contains("Key") || deserializer.getClass().getName().contains("Key"); + SchemaRegistryContext.setIsKey(isKey); + + // Extract schema ID from the data if present (Confluent wire format) + if (data != null && data.length >= 5) { + // Confluent wire format: [magic_byte][4-byte schema id][data] + if (data[0] == 0) { + int schemaId = + ((data[1] & 0xFF) << 24) + | ((data[2] & 0xFF) << 16) + | ((data[3] & 0xFF) << 8) + | (data[4] & 0xFF); + + if (isKey) { + SchemaRegistryContext.setKeySchemaId(schemaId); + } else { + SchemaRegistryContext.setValueSchemaId(schemaId); + } + } + } + } + + @Advice.OnMethodExit(onThrowable = Throwable.class, suppress = Throwable.class) + public static void onExit( + @Advice.This Object deserializer, + @Advice.Argument(0) String topic, + @Advice.Return Object result, + @Advice.Thrown Throwable throwable) { + + Boolean isKey = SchemaRegistryContext.getIsKey(); + Integer keySchemaId = SchemaRegistryContext.getKeySchemaId(); + Integer valueSchemaId = SchemaRegistryContext.getValueSchemaId(); + + if (throwable != null) { + SchemaRegistryMetrics.recordDeserializationFailure( + topic, throwable.getMessage(), isKey != null ? isKey : false); + } else if (result != null) { + // Successful deserialization + SchemaRegistryMetrics.recordDeserialization( + topic, keySchemaId, valueSchemaId, isKey != null ? isKey : false); + } + + // Clear context after deserialization + SchemaRegistryContext.clear(); + } + } +} diff --git a/dd-java-agent/instrumentation/confluent-schema-registry/confluent-schema-registry-7.0/src/main/java/datadog/trace/instrumentation/confluentschemaregistry/KafkaAvroSerializerInstrumentation.java b/dd-java-agent/instrumentation/confluent-schema-registry/confluent-schema-registry-7.0/src/main/java/datadog/trace/instrumentation/confluentschemaregistry/KafkaAvroSerializerInstrumentation.java new file mode 100644 index 00000000000..1ab0fbff2ee --- /dev/null +++ b/dd-java-agent/instrumentation/confluent-schema-registry/confluent-schema-registry-7.0/src/main/java/datadog/trace/instrumentation/confluentschemaregistry/KafkaAvroSerializerInstrumentation.java @@ -0,0 +1,200 @@ +package datadog.trace.instrumentation.confluentschemaregistry; + +import static datadog.trace.agent.tooling.bytebuddy.matcher.NameMatchers.named; +import static net.bytebuddy.matcher.ElementMatchers.isMethod; +import static net.bytebuddy.matcher.ElementMatchers.isPublic; +import static net.bytebuddy.matcher.ElementMatchers.returns; +import static net.bytebuddy.matcher.ElementMatchers.takesArgument; +import static net.bytebuddy.matcher.ElementMatchers.takesArguments; + +import datadog.trace.agent.tooling.Instrumenter; +import datadog.trace.agent.tooling.Instrumenter.MethodTransformer; +import net.bytebuddy.asm.Advice; +import net.bytebuddy.description.type.TypeDescription; +import net.bytebuddy.matcher.ElementMatcher; + +/** + * Instruments AbstractKafkaSchemaSerDe (base class for Avro, Protobuf, and JSON serializers) to + * capture serialization operations. + */ +public class KafkaAvroSerializerInstrumentation + implements Instrumenter.ForTypeHierarchy, Instrumenter.HasMethodAdvice { + + @Override + public String hierarchyMarkerType() { + return "io.confluent.kafka.serializers.AbstractKafkaSchemaSerDe"; + } + + @Override + public ElementMatcher hierarchyMatcher() { + return named("io.confluent.kafka.serializers.AbstractKafkaSchemaSerDe") + .or(named("io.confluent.kafka.serializers.AbstractKafkaAvroSerializer")) + .or(named("io.confluent.kafka.serializers.KafkaAvroSerializer")) + .or(named("io.confluent.kafka.serializers.protobuf.KafkaProtobufSerializer")); + } + + @Override + public void methodAdvice(MethodTransformer transformer) { + // Instrument serialize(String topic, Object data) + transformer.applyAdvice( + isMethod() + .and(named("serialize")) + .and(isPublic()) + .and(takesArguments(2)) + .and(takesArgument(0, String.class)) + .and(returns(byte[].class)), + getClass().getName() + "$SerializeAdvice"); + + // Instrument serialize(String topic, Headers headers, Object data) for Kafka 2.1+ + transformer.applyAdvice( + isMethod() + .and(named("serialize")) + .and(isPublic()) + .and(takesArguments(3)) + .and(takesArgument(0, String.class)) + .and(returns(byte[].class)), + getClass().getName() + "$SerializeWithHeadersAdvice"); + } + + public static class SerializeAdvice { + @Advice.OnMethodEnter(suppress = Throwable.class) + public static void onEnter(@Advice.This Object serializer, @Advice.Argument(0) String topic) { + // Set the topic in context so that the schema registry client can use it + SchemaRegistryContext.setTopic(topic); + + // Determine if this is a key or value serializer + String className = serializer.getClass().getSimpleName(); + boolean isKey = className.contains("Key") || serializer.getClass().getName().contains("Key"); + SchemaRegistryContext.setIsKey(isKey); + } + + @Advice.OnMethodExit(onThrowable = Throwable.class, suppress = Throwable.class) + public static void onExit( + @Advice.This Object serializer, + @Advice.Argument(0) String topic, + @Advice.Return byte[] result, + @Advice.Thrown Throwable throwable) { + + try { + Boolean isKey = SchemaRegistryContext.getIsKey(); + + if (throwable != null) { + SchemaRegistryMetrics.recordSerializationFailure( + topic, throwable.getMessage(), isKey != null ? isKey : false); + } else if (result != null) { + // Extract schema ID from the serialized bytes (Confluent wire format) + Integer schemaId = null; + try { + // Confluent wire format: [magic_byte][4-byte schema id][data] + if (result.length >= 5 && result[0] == 0) { + schemaId = + ((result[1] & 0xFF) << 24) + | ((result[2] & 0xFF) << 16) + | ((result[3] & 0xFF) << 8) + | (result[4] & 0xFF); + } + } catch (Throwable ignored) { + // Suppress any errors in schema ID extraction + } + + // Store in context for correlation + if (isKey != null && isKey) { + SchemaRegistryContext.setKeySchemaId(schemaId); + } else { + SchemaRegistryContext.setValueSchemaId(schemaId); + } + + // Get both schema IDs for logging + Integer keySchemaId = SchemaRegistryContext.getKeySchemaId(); + Integer valueSchemaId = SchemaRegistryContext.getValueSchemaId(); + + // Successful serialization + SchemaRegistryMetrics.recordSerialization( + topic, keySchemaId, valueSchemaId, isKey != null ? isKey : false); + } + } catch (Throwable t) { + // Don't let instrumentation errors break the application + // but try to log them if possible + try { + SchemaRegistryMetrics.recordSerializationFailure(topic, "Instrumentation error", false); + } catch (Throwable ignored) { + // Really suppress everything + } + } finally { + // Clear context after serialization + SchemaRegistryContext.clear(); + } + } + } + + public static class SerializeWithHeadersAdvice { + @Advice.OnMethodEnter(suppress = Throwable.class) + public static void onEnter(@Advice.This Object serializer, @Advice.Argument(0) String topic) { + // Set the topic in context so that the schema registry client can use it + SchemaRegistryContext.setTopic(topic); + + // Determine if this is a key or value serializer + String className = serializer.getClass().getSimpleName(); + boolean isKey = className.contains("Key") || serializer.getClass().getName().contains("Key"); + SchemaRegistryContext.setIsKey(isKey); + } + + @Advice.OnMethodExit(onThrowable = Throwable.class, suppress = Throwable.class) + public static void onExit( + @Advice.This Object serializer, + @Advice.Argument(0) String topic, + @Advice.Return byte[] result, + @Advice.Thrown Throwable throwable) { + + try { + Boolean isKey = SchemaRegistryContext.getIsKey(); + + if (throwable != null) { + SchemaRegistryMetrics.recordSerializationFailure( + topic, throwable.getMessage(), isKey != null ? isKey : false); + } else if (result != null) { + // Extract schema ID from the serialized bytes (Confluent wire format) + Integer schemaId = null; + try { + // Confluent wire format: [magic_byte][4-byte schema id][data] + if (result.length >= 5 && result[0] == 0) { + schemaId = + ((result[1] & 0xFF) << 24) + | ((result[2] & 0xFF) << 16) + | ((result[3] & 0xFF) << 8) + | (result[4] & 0xFF); + } + } catch (Throwable ignored) { + // Suppress any errors in schema ID extraction + } + + // Store in context for correlation + if (isKey != null && isKey) { + SchemaRegistryContext.setKeySchemaId(schemaId); + } else { + SchemaRegistryContext.setValueSchemaId(schemaId); + } + + // Get both schema IDs for logging + Integer keySchemaId = SchemaRegistryContext.getKeySchemaId(); + Integer valueSchemaId = SchemaRegistryContext.getValueSchemaId(); + + // Successful serialization + SchemaRegistryMetrics.recordSerialization( + topic, keySchemaId, valueSchemaId, isKey != null ? isKey : false); + } + } catch (Throwable t) { + // Don't let instrumentation errors break the application + // but try to log them if possible + try { + SchemaRegistryMetrics.recordSerializationFailure(topic, "Instrumentation error", false); + } catch (Throwable ignored) { + // Really suppress everything + } + } finally { + // Clear context after serialization + SchemaRegistryContext.clear(); + } + } + } +} diff --git a/dd-java-agent/instrumentation/confluent-schema-registry/confluent-schema-registry-7.0/src/main/java/datadog/trace/instrumentation/confluentschemaregistry/SchemaRegistryClientInstrumentation.java b/dd-java-agent/instrumentation/confluent-schema-registry/confluent-schema-registry-7.0/src/main/java/datadog/trace/instrumentation/confluentschemaregistry/SchemaRegistryClientInstrumentation.java new file mode 100644 index 00000000000..6768eba2423 --- /dev/null +++ b/dd-java-agent/instrumentation/confluent-schema-registry/confluent-schema-registry-7.0/src/main/java/datadog/trace/instrumentation/confluentschemaregistry/SchemaRegistryClientInstrumentation.java @@ -0,0 +1,128 @@ +package datadog.trace.instrumentation.confluentschemaregistry; + +import static datadog.trace.agent.tooling.bytebuddy.matcher.NameMatchers.named; +import static net.bytebuddy.matcher.ElementMatchers.isMethod; +import static net.bytebuddy.matcher.ElementMatchers.isPublic; +import static net.bytebuddy.matcher.ElementMatchers.returns; +import static net.bytebuddy.matcher.ElementMatchers.takesArgument; +import static net.bytebuddy.matcher.ElementMatchers.takesArguments; + +import datadog.trace.agent.tooling.Instrumenter; +import datadog.trace.agent.tooling.Instrumenter.MethodTransformer; +import net.bytebuddy.asm.Advice; + +/** + * Instruments the CachedSchemaRegistryClient to capture schema registration and compatibility check + * operations. + */ +public class SchemaRegistryClientInstrumentation + implements Instrumenter.ForSingleType, Instrumenter.HasMethodAdvice { + + @Override + public String instrumentedType() { + return "io.confluent.kafka.schemaregistry.client.CachedSchemaRegistryClient"; + } + + @Override + public void methodAdvice(MethodTransformer transformer) { + // Instrument register(String subject, Schema schema) + transformer.applyAdvice( + isMethod() + .and(named("register")) + .and(isPublic()) + .and(takesArguments(2)) + .and(takesArgument(0, String.class)) + .and(returns(int.class)), + getClass().getName() + "$RegisterAdvice"); + + // Instrument testCompatibility(String subject, Schema schema) + transformer.applyAdvice( + isMethod() + .and(named("testCompatibility")) + .and(isPublic()) + .and(takesArguments(2)) + .and(takesArgument(0, String.class)) + .and(returns(boolean.class)), + getClass().getName() + "$TestCompatibilityAdvice"); + + // Instrument getSchemaById(int id) + transformer.applyAdvice( + isMethod() + .and(named("getSchemaById")) + .and(isPublic()) + .and(takesArguments(1)) + .and(takesArgument(0, int.class)), + getClass().getName() + "$GetSchemaByIdAdvice"); + } + + public static class RegisterAdvice { + @Advice.OnMethodEnter(suppress = Throwable.class) + public static void onEnter(@Advice.Argument(0) String subject) { + // Track that we're attempting registration + } + + @Advice.OnMethodExit(onThrowable = Throwable.class, suppress = Throwable.class) + public static void onExit( + @Advice.Argument(0) String subject, + @Advice.Return int schemaId, + @Advice.Thrown Throwable throwable) { + + String topic = SchemaRegistryContext.getTopic(); + Boolean isKey = SchemaRegistryContext.getIsKey(); + + if (throwable != null) { + // Registration failed - likely due to compatibility issues + String errorMessage = throwable.getMessage(); + SchemaRegistryMetrics.recordSchemaRegistrationFailure( + subject, errorMessage, isKey != null ? isKey : false, topic); + + // Also log that this is a compatibility failure + if (errorMessage != null + && (errorMessage.contains("incompatible") || errorMessage.contains("compatibility"))) { + SchemaRegistryMetrics.recordCompatibilityCheck(subject, false, errorMessage); + } + } else { + // Registration successful + SchemaRegistryMetrics.recordSchemaRegistration( + subject, schemaId, isKey != null ? isKey : false, topic); + + // Store the schema ID in context + if (isKey != null && isKey) { + SchemaRegistryContext.setKeySchemaId(schemaId); + } else { + SchemaRegistryContext.setValueSchemaId(schemaId); + } + } + } + } + + public static class TestCompatibilityAdvice { + @Advice.OnMethodExit(onThrowable = Throwable.class, suppress = Throwable.class) + public static void onExit( + @Advice.Argument(0) String subject, + @Advice.Return boolean compatible, + @Advice.Thrown Throwable throwable) { + + if (throwable != null) { + SchemaRegistryMetrics.recordCompatibilityCheck(subject, false, throwable.getMessage()); + } else { + SchemaRegistryMetrics.recordCompatibilityCheck( + subject, compatible, compatible ? null : "Schema is not compatible"); + } + } + } + + public static class GetSchemaByIdAdvice { + @Advice.OnMethodExit(onThrowable = Throwable.class, suppress = Throwable.class) + public static void onExit( + @Advice.Argument(0) int schemaId, + @Advice.Return Object schema, + @Advice.Thrown Throwable throwable) { + + if (throwable == null && schema != null) { + String schemaType = schema.getClass().getSimpleName(); + SchemaRegistryMetrics.recordSchemaRetrieval(schemaId, schemaType); + } + } + } +} diff --git a/dd-java-agent/instrumentation/confluent-schema-registry/confluent-schema-registry-7.0/src/main/java/datadog/trace/instrumentation/confluentschemaregistry/SchemaRegistryContext.java b/dd-java-agent/instrumentation/confluent-schema-registry/confluent-schema-registry-7.0/src/main/java/datadog/trace/instrumentation/confluentschemaregistry/SchemaRegistryContext.java new file mode 100644 index 00000000000..c0d122e8b06 --- /dev/null +++ b/dd-java-agent/instrumentation/confluent-schema-registry/confluent-schema-registry-7.0/src/main/java/datadog/trace/instrumentation/confluentschemaregistry/SchemaRegistryContext.java @@ -0,0 +1,61 @@ +package datadog.trace.instrumentation.confluentschemaregistry; + +/** + * Thread-local context for passing topic, schema, and cluster information between serialization and + * registry calls. + */ +public class SchemaRegistryContext { + private static final ThreadLocal currentTopic = new ThreadLocal<>(); + private static final ThreadLocal clusterId = new ThreadLocal<>(); + private static final ThreadLocal isKey = new ThreadLocal<>(); + private static final ThreadLocal keySchemaId = new ThreadLocal<>(); + private static final ThreadLocal valueSchemaId = new ThreadLocal<>(); + + public static void setTopic(String topic) { + currentTopic.set(topic); + } + + public static String getTopic() { + return currentTopic.get(); + } + + public static void setClusterId(String cluster) { + clusterId.set(cluster); + } + + public static String getClusterId() { + return clusterId.get(); + } + + public static void setIsKey(boolean key) { + isKey.set(key); + } + + public static Boolean getIsKey() { + return isKey.get(); + } + + public static void setKeySchemaId(Integer schemaId) { + keySchemaId.set(schemaId); + } + + public static Integer getKeySchemaId() { + return keySchemaId.get(); + } + + public static void setValueSchemaId(Integer schemaId) { + valueSchemaId.set(schemaId); + } + + public static Integer getValueSchemaId() { + return valueSchemaId.get(); + } + + public static void clear() { + currentTopic.remove(); + clusterId.remove(); + isKey.remove(); + keySchemaId.remove(); + valueSchemaId.remove(); + } +} diff --git a/dd-java-agent/instrumentation/confluent-schema-registry/confluent-schema-registry-7.0/src/main/java/datadog/trace/instrumentation/confluentschemaregistry/SchemaRegistryMetrics.java b/dd-java-agent/instrumentation/confluent-schema-registry/confluent-schema-registry-7.0/src/main/java/datadog/trace/instrumentation/confluentschemaregistry/SchemaRegistryMetrics.java new file mode 100644 index 00000000000..260eea52ba4 --- /dev/null +++ b/dd-java-agent/instrumentation/confluent-schema-registry/confluent-schema-registry-7.0/src/main/java/datadog/trace/instrumentation/confluentschemaregistry/SchemaRegistryMetrics.java @@ -0,0 +1,240 @@ +package datadog.trace.instrumentation.confluentschemaregistry; + +import datadog.trace.bootstrap.instrumentation.api.AgentTracer; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.atomic.AtomicLong; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +/** + * Collects and logs metrics about Schema Registry operations including schema registrations, + * compatibility checks, and serialization/deserialization operations. Also reports to Data Streams + * Monitoring. + */ +public class SchemaRegistryMetrics { + private static final Logger log = LoggerFactory.getLogger(SchemaRegistryMetrics.class); + + // Counters for different operations + private static final AtomicLong schemaRegistrationSuccess = new AtomicLong(0); + private static final AtomicLong schemaRegistrationFailure = new AtomicLong(0); + private static final AtomicLong schemaCompatibilitySuccess = new AtomicLong(0); + private static final AtomicLong schemaCompatibilityFailure = new AtomicLong(0); + private static final AtomicLong serializationSuccess = new AtomicLong(0); + private static final AtomicLong serializationFailure = new AtomicLong(0); + private static final AtomicLong deserializationSuccess = new AtomicLong(0); + private static final AtomicLong deserializationFailure = new AtomicLong(0); + + // Track schema IDs per topic for context + private static final ConcurrentHashMap topicKeySchemaIds = + new ConcurrentHashMap<>(); + private static final ConcurrentHashMap topicValueSchemaIds = + new ConcurrentHashMap<>(); + + public static void recordSchemaRegistration( + String subject, int schemaId, boolean isKey, String topic) { + schemaRegistrationSuccess.incrementAndGet(); + if (topic != null) { + if (isKey) { + topicKeySchemaIds.put(topic, schemaId); + } else { + topicValueSchemaIds.put(topic, schemaId); + } + } + log.info( + "[Schema Registry] Schema registered successfully - Subject: {}, Schema ID: {}, Is Key: {}, Topic: {}", + subject, + schemaId, + isKey, + topic); + } + + public static void recordSchemaRegistrationFailure( + String subject, String errorMessage, boolean isKey, String topic) { + schemaRegistrationFailure.incrementAndGet(); + log.error( + "[Schema Registry] Schema registration FAILED - Subject: {}, Is Key: {}, Topic: {}, Error: {}", + subject, + isKey, + topic, + errorMessage); + } + + public static void recordCompatibilityCheck( + String subject, boolean compatible, String errorMessage) { + if (compatible) { + schemaCompatibilitySuccess.incrementAndGet(); + log.info("[Schema Registry] Schema compatibility check PASSED - Subject: {}", subject); + } else { + schemaCompatibilityFailure.incrementAndGet(); + log.error( + "[Schema Registry] Schema compatibility check FAILED - Subject: {}, Error: {}", + subject, + errorMessage); + } + } + + public static void recordSerialization( + String topic, Integer keySchemaId, Integer valueSchemaId, boolean isKey) { + serializationSuccess.incrementAndGet(); + + String clusterId = SchemaRegistryContext.getClusterId(); + log.info("[Schema Registry] DEBUG: Retrieved clusterId from context: '{}'", clusterId); + log.info( + "[Schema Registry] Produce to topic '{}', cluster: {}, schema for key: {}, schema for value: {}, serializing: {}", + topic, + clusterId != null ? clusterId : "unknown", + keySchemaId != null ? keySchemaId : "none", + valueSchemaId != null ? valueSchemaId : "none", + isKey ? "KEY" : "VALUE"); + + // Report to Data Streams Monitoring + Integer schemaId = isKey ? keySchemaId : valueSchemaId; + if (schemaId != null && topic != null) { + try { + AgentTracer.get() + .getDataStreamsMonitoring() + .setSchemaRegistryUsage(topic, clusterId, schemaId, true, isKey); + } catch (Throwable t) { + // Don't fail the application if DSM reporting fails + log.debug("Failed to report schema registry usage to DSM", t); + } + } + } + + public static void recordSerializationFailure(String topic, String errorMessage, boolean isKey) { + serializationFailure.incrementAndGet(); + + String clusterId = SchemaRegistryContext.getClusterId(); + log.error( + "[Schema Registry] Serialization FAILED for topic '{}', cluster: {}, {} - Error: {}", + topic, + clusterId != null ? clusterId : "unknown", + isKey ? "KEY" : "VALUE", + errorMessage); + + // Report to Data Streams Monitoring (use -1 as schema ID for failures) + if (topic != null) { + try { + AgentTracer.get() + .getDataStreamsMonitoring() + .setSchemaRegistryUsage(topic, clusterId, -1, false, isKey); + } catch (Throwable t) { + // Don't fail the application if DSM reporting fails + log.debug("Failed to report schema registry failure to DSM", t); + } + } + } + + public static void recordDeserialization( + String topic, Integer keySchemaId, Integer valueSchemaId, boolean isKey) { + deserializationSuccess.incrementAndGet(); + + String clusterId = SchemaRegistryContext.getClusterId(); + log.info( + "[Schema Registry] Consume from topic '{}', cluster: {}, schema for key: {}, schema for value: {}, deserializing: {}", + topic, + clusterId != null ? clusterId : "unknown", + keySchemaId != null ? keySchemaId : "none", + valueSchemaId != null ? valueSchemaId : "none", + isKey ? "KEY" : "VALUE"); + + // Report to Data Streams Monitoring + Integer schemaId = isKey ? keySchemaId : valueSchemaId; + if (schemaId != null && topic != null) { + try { + AgentTracer.get() + .getDataStreamsMonitoring() + .setSchemaRegistryUsage(topic, clusterId, schemaId, true, isKey); + } catch (Throwable t) { + // Don't fail the application if DSM reporting fails + log.debug("Failed to report schema registry usage to DSM", t); + } + } + } + + public static void recordDeserializationFailure( + String topic, String errorMessage, boolean isKey) { + deserializationFailure.incrementAndGet(); + + String clusterId = SchemaRegistryContext.getClusterId(); + log.error( + "[Schema Registry] Deserialization FAILED for topic '{}', cluster: {}, {} - Error: {}", + topic, + clusterId != null ? clusterId : "unknown", + isKey ? "KEY" : "VALUE", + errorMessage); + + // Report to Data Streams Monitoring (use -1 as schema ID for failures) + if (topic != null) { + try { + AgentTracer.get() + .getDataStreamsMonitoring() + .setSchemaRegistryUsage(topic, clusterId, -1, false, isKey); + } catch (Throwable t) { + // Don't fail the application if DSM reporting fails + log.debug("Failed to report schema registry failure to DSM", t); + } + } + } + + public static void recordSchemaRetrieval(int schemaId, String schemaType) { + log.debug( + "[Schema Registry] Retrieved schema from registry - Schema ID: {}, Type: {}", + schemaId, + schemaType); + } + + // Methods to get current counts for metrics reporting + public static long getSchemaRegistrationSuccessCount() { + return schemaRegistrationSuccess.get(); + } + + public static long getSchemaRegistrationFailureCount() { + return schemaRegistrationFailure.get(); + } + + public static long getSchemaCompatibilitySuccessCount() { + return schemaCompatibilitySuccess.get(); + } + + public static long getSchemaCompatibilityFailureCount() { + return schemaCompatibilityFailure.get(); + } + + public static long getSerializationSuccessCount() { + return serializationSuccess.get(); + } + + public static long getSerializationFailureCount() { + return serializationFailure.get(); + } + + public static long getDeserializationSuccessCount() { + return deserializationSuccess.get(); + } + + public static long getDeserializationFailureCount() { + return deserializationFailure.get(); + } + + public static void printSummary() { + log.info("========== Schema Registry Metrics Summary =========="); + log.info( + "Schema Registrations - Success: {}, Failure: {}", + schemaRegistrationSuccess.get(), + schemaRegistrationFailure.get()); + log.info( + "Compatibility Checks - Success: {}, Failure: {}", + schemaCompatibilitySuccess.get(), + schemaCompatibilityFailure.get()); + log.info( + "Serializations - Success: {}, Failure: {}", + serializationSuccess.get(), + serializationFailure.get()); + log.info( + "Deserializations - Success: {}, Failure: {}", + deserializationSuccess.get(), + deserializationFailure.get()); + log.info("====================================================="); + } +} diff --git a/dd-java-agent/instrumentation/confluent-schema-registry/confluent-schema-registry-7.0/src/test/groovy/ConfluentSchemaRegistryDataStreamsTest.groovy b/dd-java-agent/instrumentation/confluent-schema-registry/confluent-schema-registry-7.0/src/test/groovy/ConfluentSchemaRegistryDataStreamsTest.groovy new file mode 100644 index 00000000000..2f72923eca3 --- /dev/null +++ b/dd-java-agent/instrumentation/confluent-schema-registry/confluent-schema-registry-7.0/src/test/groovy/ConfluentSchemaRegistryDataStreamsTest.groovy @@ -0,0 +1,294 @@ +import datadog.trace.agent.test.InstrumentationSpecification +import io.confluent.kafka.serializers.KafkaAvroSerializer +import io.confluent.kafka.schemaregistry.client.MockSchemaRegistryClient +import io.confluent.kafka.schemaregistry.client.SchemaRegistryClient +import org.apache.avro.Schema +import org.apache.avro.generic.GenericData +import org.apache.avro.generic.GenericRecord +import org.apache.kafka.common.serialization.Serializer +import spock.lang.Shared + +import java.util.concurrent.TimeUnit + +/** + * Tests that Schema Registry usage is tracked in Data Streams Monitoring. + * Tests both successful and unsuccessful serialization/deserialization operations. + */ +class ConfluentSchemaRegistryDataStreamsTest extends InstrumentationSpecification { + @Shared + SchemaRegistryClient schemaRegistryClient + + @Shared + Schema testSchema + + @Override + protected boolean isDataStreamsEnabled() { + return true + } + + @Override + protected long dataStreamsBucketDuration() { + return TimeUnit.SECONDS.toNanos(1) + } + + void setup() { + schemaRegistryClient = new MockSchemaRegistryClient() + testSchema = new Schema.Parser().parse(""" + { + "type": "record", + "name": "TestRecord", + "fields": [ + {"name": "field1", "type": "string"}, + {"name": "field2", "type": "int"} + ] + } + """) + } + + def "test successful producer serialization tracks schema registry usage"() { + setup: + def topicName = "test-topic-producer" + def testClusterId = "test-cluster-producer" + def serializer = new KafkaAvroSerializer(schemaRegistryClient) + def config = [ + "schema.registry.url": "mock://test-url", + "auto.register.schemas": "true" + ] + serializer.configure(config, false) // false = value serializer + + // Create a test record + GenericRecord record = new GenericData.Record(testSchema) + record.put("field1", "test-value") + record.put("field2", 42) + + when: "we serialize a message with cluster ID set" + datadog.trace.instrumentation.confluentschemaregistry.SchemaRegistryContext.setClusterId(testClusterId) + byte[] serialized = serializer.serialize(topicName, record) + + and: "we wait for DSM to flush" + Thread.sleep(1200) // Wait for bucket duration + buffer + TEST_DATA_STREAMS_MONITORING.report() + TEST_DATA_STREAMS_WRITER.waitForSchemaRegistryUsages(1, TimeUnit.SECONDS.toMillis(5)) + + then: "the serialization was successful" + serialized != null + serialized.length > 0 + + and: "schema registry usage was tracked" + def usages = TEST_DATA_STREAMS_WRITER.schemaRegistryUsages + usages.size() >= 1 + + and: "the usage contains the correct information" + def usage = usages.find { u -> u.topic == topicName } + usage != null + usage.schemaId > 0 // Valid schema ID + usage.isSuccess() == true // Successful operation + usage.isKey() == false // Value serializer + usage.clusterId == testClusterId // Cluster ID is included + + cleanup: + serializer.close() + datadog.trace.instrumentation.confluentschemaregistry.SchemaRegistryContext.clear() + } + + def "test successful producer with key and value serializers"() { + setup: + def topicName = "test-topic-key-value" + def testClusterId = "test-cluster-key-value" + def keySerializer = new KafkaAvroSerializer(schemaRegistryClient) + def valueSerializer = new KafkaAvroSerializer(schemaRegistryClient) + def config = [ + "schema.registry.url": "mock://test-url", + "auto.register.schemas": "true" + ] + keySerializer.configure(config, true) // true = key serializer + valueSerializer.configure(config, false) // false = value serializer + + // Create test records + GenericRecord keyRecord = new GenericData.Record(testSchema) + keyRecord.put("field1", "key-value") + keyRecord.put("field2", 1) + + GenericRecord valueRecord = new GenericData.Record(testSchema) + valueRecord.put("field1", "value-value") + valueRecord.put("field2", 2) + + when: "we serialize both key and value with cluster ID" + datadog.trace.instrumentation.confluentschemaregistry.SchemaRegistryContext.setClusterId(testClusterId) + byte[] keyBytes = keySerializer.serialize(topicName, keyRecord) + datadog.trace.instrumentation.confluentschemaregistry.SchemaRegistryContext.setClusterId(testClusterId) + byte[] valueBytes = valueSerializer.serialize(topicName, valueRecord) + + and: "we wait for DSM to flush" + Thread.sleep(1200) // Wait for bucket duration + buffer + TEST_DATA_STREAMS_MONITORING.report() + TEST_DATA_STREAMS_WRITER.waitForSchemaRegistryUsages(2, TimeUnit.SECONDS.toMillis(5)) + + then: "both serializations were successful" + keyBytes != null && keyBytes.length > 0 + valueBytes != null && valueBytes.length > 0 + + and: "both usages were tracked" + def usages = TEST_DATA_STREAMS_WRITER.schemaRegistryUsages + usages.size() >= 2 + + and: "we have both key and value tracked" + def keyUsage = usages.find { u -> u.isKey() && u.topic == topicName } + def valueUsage = usages.find { u -> !u.isKey() && u.topic == topicName } + + keyUsage != null + keyUsage.schemaId > 0 + keyUsage.isSuccess() == true + keyUsage.clusterId == testClusterId + + valueUsage != null + valueUsage.schemaId > 0 + valueUsage.isSuccess() == true + valueUsage.clusterId == testClusterId + + cleanup: + keySerializer.close() + valueSerializer.close() + datadog.trace.instrumentation.confluentschemaregistry.SchemaRegistryContext.clear() + } + + def "test serialization failure is tracked"() { + setup: + def topicName = "test-topic-failure" + + // Create a custom serializer that will fail + Serializer failingSerializer = new KafkaAvroSerializer(schemaRegistryClient) { + @Override + byte[] serialize(String topic, Object data) { + throw new RuntimeException("Intentional serialization failure") + } + } + def config = [ + "schema.registry.url": "mock://test-url" + ] + failingSerializer.configure(config, false) + + GenericRecord record = new GenericData.Record(testSchema) + record.put("field1", "test") + record.put("field2", 123) + + when: "we try to serialize and it fails" + try { + failingSerializer.serialize(topicName, record) + } catch (RuntimeException e) { + // Expected + } + + and: "we wait for DSM to flush" + Thread.sleep(1200) + TEST_DATA_STREAMS_MONITORING.report() + TEST_DATA_STREAMS_WRITER.waitForSchemaRegistryUsages(1, TimeUnit.SECONDS.toMillis(5)) + + then: "the failure was tracked" + def usages = TEST_DATA_STREAMS_WRITER.schemaRegistryUsages + def failureUsage = usages.find { u -> u.topic == topicName } + + failureUsage != null + failureUsage.schemaId == -1 // Failure indicator + failureUsage.isSuccess() == false + + cleanup: + failingSerializer.close() + } + + def "test schema IDs are correctly extracted from serialized messages"() { + setup: + def topicName = "test-topic-schema-id" + def testClusterId = "test-cluster-schema-id" + def serializer = new KafkaAvroSerializer(schemaRegistryClient) + def config = [ + "schema.registry.url": "mock://test-url", + "auto.register.schemas": "true" + ] + serializer.configure(config, false) + + // Create multiple records with the same schema + def records = (1..3).collect { i -> + GenericRecord record = new GenericData.Record(testSchema) + record.put("field1", "value-$i") + record.put("field2", i) + record + } + + when: "we serialize multiple messages with cluster ID" + datadog.trace.instrumentation.confluentschemaregistry.SchemaRegistryContext.setClusterId(testClusterId) + def serializedMessages = records.collect { record -> + serializer.serialize(topicName, record) + } + + and: "we wait for DSM to flush" + Thread.sleep(1200) + TEST_DATA_STREAMS_MONITORING.report() + TEST_DATA_STREAMS_WRITER.waitForSchemaRegistryUsages(3, TimeUnit.SECONDS.toMillis(5)) + + then: "all messages were serialized" + serializedMessages.every { m -> m != null && m.length > 0 } + + and: "all usages have the same schema ID (cached schema)" + def usages = TEST_DATA_STREAMS_WRITER.schemaRegistryUsages + .findAll { u -> u.topic == topicName } + usages.size() >= 3 + + def schemaIds = usages.collect { u -> u.schemaId }.unique() + schemaIds.size() == 1 // All use the same schema ID + schemaIds[0] > 0 // Valid schema ID + + and: "cluster ID is present" + usages.every { u -> u.clusterId == testClusterId } + + cleanup: + serializer.close() + datadog.trace.instrumentation.confluentschemaregistry.SchemaRegistryContext.clear() + } + + def "test schema registry usage metrics are aggregated by topic"() { + setup: + def topic1 = "test-topic-1" + def topic2 = "test-topic-2" + def testClusterId = "test-cluster-multi-topic" + def serializer = new KafkaAvroSerializer(schemaRegistryClient) + def config = [ + "schema.registry.url": "mock://test-url", + "auto.register.schemas": "true" + ] + serializer.configure(config, false) + + GenericRecord record = new GenericData.Record(testSchema) + record.put("field1", "test") + record.put("field2", 1) + + when: "we produce to multiple topics with cluster ID" + datadog.trace.instrumentation.confluentschemaregistry.SchemaRegistryContext.setClusterId(testClusterId) + serializer.serialize(topic1, record) + datadog.trace.instrumentation.confluentschemaregistry.SchemaRegistryContext.setClusterId(testClusterId) + serializer.serialize(topic2, record) + datadog.trace.instrumentation.confluentschemaregistry.SchemaRegistryContext.setClusterId(testClusterId) + serializer.serialize(topic1, record) // Second message to topic1 + + and: "we wait for DSM to flush" + Thread.sleep(1200) + TEST_DATA_STREAMS_MONITORING.report() + TEST_DATA_STREAMS_WRITER.waitForSchemaRegistryUsages(3, TimeUnit.SECONDS.toMillis(5)) + + then: "usages are tracked per topic" + def usages = TEST_DATA_STREAMS_WRITER.schemaRegistryUsages + def topic1Usages = usages.findAll { u -> u.topic == topic1 } + def topic2Usages = usages.findAll { u -> u.topic == topic2 } + + topic1Usages.size() >= 2 + topic2Usages.size() >= 1 + + and: "all are successful" + usages.every { u -> u.isSuccess() } + + cleanup: + serializer.close() + datadog.trace.instrumentation.confluentschemaregistry.SchemaRegistryContext.clear() + } +} + diff --git a/dd-java-agent/instrumentation/confluent-schema-registry/confluent-schema-registry-7.0/src/test/groovy/ConfluentSchemaRegistryTest.groovy b/dd-java-agent/instrumentation/confluent-schema-registry/confluent-schema-registry-7.0/src/test/groovy/ConfluentSchemaRegistryTest.groovy new file mode 100644 index 00000000000..061931b99a5 --- /dev/null +++ b/dd-java-agent/instrumentation/confluent-schema-registry/confluent-schema-registry-7.0/src/test/groovy/ConfluentSchemaRegistryTest.groovy @@ -0,0 +1,204 @@ +import spock.lang.Specification +import io.confluent.kafka.schemaregistry.client.MockSchemaRegistryClient +import io.confluent.kafka.schemaregistry.client.SchemaRegistryClient +import io.confluent.kafka.serializers.KafkaAvroDeserializer +import io.confluent.kafka.serializers.KafkaAvroSerializer +import org.apache.avro.Schema +import org.apache.avro.generic.GenericData +import org.apache.avro.generic.GenericRecord + +/** + * Simple test to verify Schema Registry serializer/deserializer behavior. + * This test verifies that: + * 1. Confluent wire format includes schema ID in bytes + * 2. Schema ID can be extracted from serialized data + * + * To test with instrumentation, run your app with the agent and check logs. + */ +class ConfluentSchemaRegistryTest extends Specification { + + def "test schema ID is extracted from serialized bytes"() { + setup: + SchemaRegistryClient schemaRegistry = new MockSchemaRegistryClient() + String topic = "test-topic" + + // Define a simple Avro schema + String schemaStr = ''' + { + "type": "record", + "name": "TestRecord", + "fields": [ + {"name": "id", "type": "string"}, + {"name": "value", "type": "int"} + ] + } + ''' + Schema schema = new Schema.Parser().parse(schemaStr) + + // Configure serializer + def props = [ + "schema.registry.url": "mock://test" + ] + KafkaAvroSerializer serializer = new KafkaAvroSerializer(schemaRegistry, props) + + // Create a record + GenericRecord record = new GenericData.Record(schema) + record.put("id", "test-id-123") + record.put("value", 42) + + when: + byte[] serialized = serializer.serialize(topic, record) + + then: + // Verify serialization succeeded + serialized != null + serialized.length > 5 + + // Verify Confluent wire format: magic byte (0x00) + 4-byte schema ID + serialized[0] == 0 + + // Extract schema ID from bytes (big-endian) + int schemaId = ((serialized[1] & 0xFF) << 24) | + ((serialized[2] & 0xFF) << 16) | + ((serialized[3] & 0xFF) << 8) | + (serialized[4] & 0xFF) + + println "\n========== Confluent Wire Format Test ==========" + println "Topic: ${topic}" + println "Schema ID from bytes: ${schemaId}" + def first10 = serialized.length >= 10 ? serialized[0..9] : serialized + println "First 10 bytes: ${first10.collect { String.format('%02X', it & 0xFF) }.join(' ')}" + println "This is the schema ID our instrumentation should extract!" + println "================================================\n" + + // Verify schema ID is positive (valid) + schemaId > 0 + + cleanup: + serializer?.close() + } + + def "test deserialization captures schema ID"() { + setup: + SchemaRegistryClient schemaRegistry = new MockSchemaRegistryClient() + String topic = "test-topic" + + String schemaStr = ''' + { + "type": "record", + "name": "TestRecord", + "fields": [ + {"name": "name", "type": "string"} + ] + } + ''' + Schema schema = new Schema.Parser().parse(schemaStr) + + def props = ["schema.registry.url": "mock://test"] + KafkaAvroSerializer serializer = new KafkaAvroSerializer(schemaRegistry, props) + KafkaAvroDeserializer deserializer = new KafkaAvroDeserializer(schemaRegistry, props) + + GenericRecord record = new GenericData.Record(schema) + record.put("name", "TestName") + + when: + byte[] serialized = serializer.serialize(topic, record) + Object deserialized = deserializer.deserialize(topic, serialized) + + then: + deserialized != null + deserialized instanceof GenericRecord + ((GenericRecord) deserialized).get("name").toString() == "TestName" + + println "\n========== Deserialization Test ==========" + println "Topic: ${topic}" + println "Deserialized record: ${deserialized}" + println "=========================================\n" + + cleanup: + serializer?.close() + deserializer?.close() + } + + def "test schema registration is tracked"() { + setup: + SchemaRegistryClient schemaRegistry = new MockSchemaRegistryClient() + String subject = "test-subject" + String schemaStr = ''' + { + "type": "record", + "name": "User", + "fields": [ + {"name": "username", "type": "string"} + ] + } + ''' + Schema schema = new Schema.Parser().parse(schemaStr) + + when: + int schemaId = schemaRegistry.register(subject, schema) + + then: + schemaId > 0 + + println "\n========== Schema Registration Test ==========" + println "Subject: ${subject}" + println "Registered schema ID: ${schemaId}" + println "============================================\n" + } + + def "test end-to-end with real KafkaAvroSerializer"() { + setup: + SchemaRegistryClient schemaRegistry = new MockSchemaRegistryClient() + String topic = "products" + + String productSchema = ''' + { + "type": "record", + "name": "Product", + "namespace": "com.example", + "fields": [ + {"name": "id", "type": "string"}, + {"name": "name", "type": "string"}, + {"name": "price", "type": "double"} + ] + } + ''' + Schema schema = new Schema.Parser().parse(productSchema) + + def props = ["schema.registry.url": "mock://test"] + KafkaAvroSerializer serializer = new KafkaAvroSerializer(schemaRegistry, props) + + GenericRecord product = new GenericData.Record(schema) + product.put("id", "PROD-001") + product.put("name", "Test Product") + product.put("price", 29.99) + + when: + byte[] serialized = serializer.serialize(topic, product) + + then: + serialized != null + println "\n========== End-to-End Test ==========" + println "Topic: ${topic}" + println "Serialized ${serialized.length} bytes" + def first10 = serialized.length >= 10 ? serialized[0..9] : serialized + println "First 10 bytes: ${first10.collect { String.format('%02X', it & 0xFF) }.join(' ')}" + + // Extract and print schema ID + if (serialized.length >= 5 && serialized[0] == 0) { + int schemaId = ((serialized[1] & 0xFF) << 24) | + ((serialized[2] & 0xFF) << 16) | + ((serialized[3] & 0xFF) << 8) | + (serialized[4] & 0xFF) + println "Schema ID from wire format: ${schemaId}" + println "\nWhen running with DD agent, you should see:" + println "[Schema Registry] Produce to topic 'products', schema for key: none, schema for value: ${schemaId}, serializing: VALUE" + } + println "======================================\n" + + cleanup: + serializer?.close() + } +} + diff --git a/dd-java-agent/instrumentation/kafka/kafka-clients-0.11/src/main/java/datadog/trace/instrumentation/kafka_clients/KafkaProducerInstrumentation.java b/dd-java-agent/instrumentation/kafka/kafka-clients-0.11/src/main/java/datadog/trace/instrumentation/kafka_clients/KafkaProducerInstrumentation.java index ba66a01b6da..5e2d0c7a2fa 100644 --- a/dd-java-agent/instrumentation/kafka/kafka-clients-0.11/src/main/java/datadog/trace/instrumentation/kafka_clients/KafkaProducerInstrumentation.java +++ b/dd-java-agent/instrumentation/kafka/kafka-clients-0.11/src/main/java/datadog/trace/instrumentation/kafka_clients/KafkaProducerInstrumentation.java @@ -118,6 +118,21 @@ public static AgentScope onEnter( @Advice.Argument(value = 1, readOnly = false) Callback callback) { String clusterId = InstrumentationContext.get(Metadata.class, String.class).get(metadata); + // Set cluster ID in SchemaRegistryContext for Schema Registry instrumentation + // Use reflection to avoid compile-time dependency on the schema registry module + if (clusterId != null) { + try { + Class contextClass = + Class.forName( + "datadog.trace.instrumentation.confluentschemaregistry.SchemaRegistryContext", + false, + metadata.getClass().getClassLoader()); + contextClass.getMethod("setClusterId", String.class).invoke(null, clusterId); + } catch (Throwable ignored) { + // Ignore if SchemaRegistryContext is not available + } + } + final AgentSpan parent = activeSpan(); final AgentSpan span = startSpan(KAFKA_PRODUCE); PRODUCER_DECORATE.afterStart(span); diff --git a/dd-trace-core/src/main/java/datadog/trace/core/datastreams/DefaultDataStreamsMonitoring.java b/dd-trace-core/src/main/java/datadog/trace/core/datastreams/DefaultDataStreamsMonitoring.java index c86b9402081..5151e9fbb4b 100644 --- a/dd-trace-core/src/main/java/datadog/trace/core/datastreams/DefaultDataStreamsMonitoring.java +++ b/dd-trace-core/src/main/java/datadog/trace/core/datastreams/DefaultDataStreamsMonitoring.java @@ -17,6 +17,7 @@ import datadog.trace.api.Config; import datadog.trace.api.TraceConfig; import datadog.trace.api.datastreams.*; +import datadog.trace.api.datastreams.SchemaRegistryUsage; import datadog.trace.api.experimental.DataStreamsContextCarrier; import datadog.trace.api.time.TimeSource; import datadog.trace.bootstrap.instrumentation.api.AgentSpan; @@ -217,6 +218,28 @@ public void trackBacklog(DataStreamsTags tags, long value) { inbox.offer(new Backlog(tags, value, timeSource.getCurrentTimeNanos(), getThreadServiceName())); } + @Override + public void setSchemaRegistryUsage( + String topic, String clusterId, int schemaId, boolean isSuccess, boolean isKey) { + log.info( + "[DSM Schema Registry] Recording usage: topic={}, clusterId={}, schemaId={}, success={}, isKey={}", + topic, + clusterId, + schemaId, + isSuccess, + isKey); + + inbox.offer( + new SchemaRegistryUsage( + topic, + clusterId, + schemaId, + isSuccess, + isKey, + timeSource.getCurrentTimeNanos(), + getThreadServiceName())); + } + @Override public void setCheckpoint(AgentSpan span, DataStreamsContext context) { PathwayContext pathwayContext = span.context().getPathwayContext(); @@ -358,6 +381,11 @@ public void run() { StatsBucket statsBucket = getStatsBucket(backlog.getTimestampNanos(), backlog.getServiceNameOverride()); statsBucket.addBacklog(backlog); + } else if (payload instanceof SchemaRegistryUsage) { + SchemaRegistryUsage usage = (SchemaRegistryUsage) payload; + StatsBucket statsBucket = + getStatsBucket(usage.getTimestampNanos(), usage.getServiceNameOverride()); + statsBucket.addSchemaRegistryUsage(usage); } } } catch (Exception e) { diff --git a/dd-trace-core/src/main/java/datadog/trace/core/datastreams/MsgPackDatastreamsPayloadWriter.java b/dd-trace-core/src/main/java/datadog/trace/core/datastreams/MsgPackDatastreamsPayloadWriter.java index 0df8e7291d7..34f2d134ecb 100644 --- a/dd-trace-core/src/main/java/datadog/trace/core/datastreams/MsgPackDatastreamsPayloadWriter.java +++ b/dd-trace-core/src/main/java/datadog/trace/core/datastreams/MsgPackDatastreamsPayloadWriter.java @@ -15,8 +15,11 @@ import java.util.Collection; import java.util.List; import java.util.Map; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; public class MsgPackDatastreamsPayloadWriter implements DatastreamsPayloadWriter { + private static final Logger log = LoggerFactory.getLogger(MsgPackDatastreamsPayloadWriter.class); private static final byte[] ENV = "Env".getBytes(ISO_8859_1); private static final byte[] VERSION = "Version".getBytes(ISO_8859_1); private static final byte[] PRIMARY_TAG = "PrimaryTag".getBytes(ISO_8859_1); @@ -35,6 +38,13 @@ public class MsgPackDatastreamsPayloadWriter implements DatastreamsPayloadWriter private static final byte[] PARENT_HASH = "ParentHash".getBytes(ISO_8859_1); private static final byte[] BACKLOG_VALUE = "Value".getBytes(ISO_8859_1); private static final byte[] BACKLOG_TAGS = "Tags".getBytes(ISO_8859_1); + private static final byte[] SCHEMA_REGISTRY_USAGES = "SchemaRegistryUsages".getBytes(ISO_8859_1); + private static final byte[] TOPIC = "Topic".getBytes(ISO_8859_1); + private static final byte[] KAFKA_CLUSTER_ID = "KafkaClusterId".getBytes(ISO_8859_1); + private static final byte[] SCHEMA_ID = "SchemaId".getBytes(ISO_8859_1); + private static final byte[] COUNT = "Count".getBytes(ISO_8859_1); + private static final byte[] IS_SUCCESS = "IsSuccess".getBytes(ISO_8859_1); + private static final byte[] IS_KEY = "IsKey".getBytes(ISO_8859_1); private static final byte[] PRODUCTS_MASK = "ProductMask".getBytes(ISO_8859_1); private static final byte[] PROCESS_TAGS = "ProcessTags".getBytes(ISO_8859_1); @@ -121,7 +131,11 @@ public void writePayload(Collection data, String serviceNameOverrid for (StatsBucket bucket : data) { boolean hasBacklogs = !bucket.getBacklogs().isEmpty(); - writer.startMap(3 + (hasBacklogs ? 1 : 0)); + boolean hasSchemaRegistryUsages = !bucket.getSchemaRegistryUsages().isEmpty(); + int mapSize = 3; + if (hasBacklogs) mapSize++; + if (hasSchemaRegistryUsages) mapSize++; + writer.startMap(mapSize); /* 1 */ writer.writeUTF8(START); @@ -139,6 +153,11 @@ public void writePayload(Collection data, String serviceNameOverrid /* 4 */ writeBacklogs(bucket.getBacklogs(), writer); } + + if (hasSchemaRegistryUsages) { + /* 5 */ + writeSchemaRegistryUsages(bucket.getSchemaRegistryUsages(), writer); + } } /* 8 */ @@ -207,6 +226,50 @@ private void writeBacklogs( } } + private void writeSchemaRegistryUsages( + Collection> usages, + Writable packer) { + if (!usages.isEmpty()) { + log.info("[DSM Schema Registry] Flushing {} schema registry usage entries", usages.size()); + } + + packer.writeUTF8(SCHEMA_REGISTRY_USAGES); + packer.startArray(usages.size()); + for (Map.Entry entry : usages) { + StatsBucket.SchemaRegistryKey key = entry.getKey(); + StatsBucket.SchemaRegistryCount value = entry.getValue(); + + log.info( + "[DSM Schema Registry] Flushing entry: topic={}, clusterId={}, schemaId={}, success={}, isKey={}, count={}", + key.getTopic(), + key.getClusterId(), + key.getSchemaId(), + key.isSuccess(), + key.isKey(), + value.getCount()); + + packer.startMap(5); + + packer.writeUTF8(TOPIC); + packer.writeString(key.getTopic(), null); + + packer.writeUTF8(KAFKA_CLUSTER_ID); + packer.writeString(key.getClusterId(), null); + + packer.writeUTF8(SCHEMA_ID); + packer.writeInt(key.getSchemaId()); + + packer.writeUTF8(IS_SUCCESS); + packer.writeBoolean(key.isSuccess()); + + packer.writeUTF8(IS_KEY); + packer.writeBoolean(key.isKey()); + + packer.writeUTF8(COUNT); + packer.writeLong(value.getCount()); + } + } + private void writeDataStreamsTags(DataStreamsTags tags, Writable packer) { packer.startArray(tags.nonNullSize()); diff --git a/dd-trace-core/src/main/java/datadog/trace/core/datastreams/StatsBucket.java b/dd-trace-core/src/main/java/datadog/trace/core/datastreams/StatsBucket.java index c61550d0e3e..22e66f9a39e 100644 --- a/dd-trace-core/src/main/java/datadog/trace/core/datastreams/StatsBucket.java +++ b/dd-trace-core/src/main/java/datadog/trace/core/datastreams/StatsBucket.java @@ -2,6 +2,7 @@ import datadog.trace.api.datastreams.Backlog; import datadog.trace.api.datastreams.DataStreamsTags; +import datadog.trace.api.datastreams.SchemaRegistryUsage; import datadog.trace.api.datastreams.StatsPoint; import java.util.Collection; import java.util.HashMap; @@ -12,6 +13,7 @@ public class StatsBucket { private final long bucketDurationNanos; private final Map hashToGroup = new HashMap<>(); private final Map backlogs = new HashMap<>(); + private final Map schemaRegistryUsages = new HashMap<>(); public StatsBucket(long startTimeNanos, long bucketDurationNanos) { this.startTimeNanos = startTimeNanos; @@ -40,6 +42,18 @@ public void addBacklog(Backlog backlog) { (k, v) -> (v == null) ? backlog.getValue() : Math.max(v, backlog.getValue())); } + public void addSchemaRegistryUsage(SchemaRegistryUsage usage) { + SchemaRegistryKey key = + new SchemaRegistryKey( + usage.getTopic(), + usage.getClusterId(), + usage.getSchemaId(), + usage.isSuccess(), + usage.isKey()); + schemaRegistryUsages.compute( + key, (k, v) -> (v == null) ? new SchemaRegistryCount(1) : v.increment()); + } + public long getStartTimeNanos() { return startTimeNanos; } @@ -55,4 +69,89 @@ public Collection getGroups() { public Collection> getBacklogs() { return backlogs.entrySet(); } + + public Collection> getSchemaRegistryUsages() { + return schemaRegistryUsages.entrySet(); + } + + /** + * Key for aggregating schema registry usage by topic, cluster, schema ID, success, and key/value + * type. + */ + public static class SchemaRegistryKey { + private final String topic; + private final String clusterId; + private final int schemaId; + private final boolean isSuccess; + private final boolean isKey; + + public SchemaRegistryKey( + String topic, String clusterId, int schemaId, boolean isSuccess, boolean isKey) { + this.topic = topic; + this.clusterId = clusterId; + this.schemaId = schemaId; + this.isSuccess = isSuccess; + this.isKey = isKey; + } + + public String getTopic() { + return topic; + } + + public String getClusterId() { + return clusterId; + } + + public int getSchemaId() { + return schemaId; + } + + public boolean isSuccess() { + return isSuccess; + } + + public boolean isKey() { + return isKey; + } + + @Override + public boolean equals(Object o) { + if (this == o) return true; + if (o == null || getClass() != o.getClass()) return false; + SchemaRegistryKey that = (SchemaRegistryKey) o; + return schemaId == that.schemaId + && isSuccess == that.isSuccess + && isKey == that.isKey + && java.util.Objects.equals(topic, that.topic) + && java.util.Objects.equals(clusterId, that.clusterId); + } + + @Override + public int hashCode() { + int result = topic != null ? topic.hashCode() : 0; + result = 31 * result + (clusterId != null ? clusterId.hashCode() : 0); + result = 31 * result + schemaId; + result = 31 * result + (isSuccess ? 1 : 0); + result = 31 * result + (isKey ? 1 : 0); + return result; + } + } + + /** Count of schema registry usages. */ + public static class SchemaRegistryCount { + private long count; + + public SchemaRegistryCount(long count) { + this.count = count; + } + + public SchemaRegistryCount increment() { + count++; + return this; + } + + public long getCount() { + return count; + } + } } diff --git a/internal-api/src/main/java/datadog/trace/api/datastreams/AgentDataStreamsMonitoring.java b/internal-api/src/main/java/datadog/trace/api/datastreams/AgentDataStreamsMonitoring.java index b7c51bd36ec..225eefdc21c 100644 --- a/internal-api/src/main/java/datadog/trace/api/datastreams/AgentDataStreamsMonitoring.java +++ b/internal-api/src/main/java/datadog/trace/api/datastreams/AgentDataStreamsMonitoring.java @@ -8,6 +8,18 @@ public interface AgentDataStreamsMonitoring extends DataStreamsCheckpointer { void trackBacklog(DataStreamsTags tags, long value); + /** + * Tracks Schema Registry usage for Data Streams Monitoring. + * + * @param topic Kafka topic name + * @param clusterId Kafka cluster ID (important: schema IDs are only unique per cluster) + * @param schemaId Schema ID from Schema Registry + * @param isSuccess Whether the schema operation succeeded + * @param isKey Whether this is for the key (true) or value (false) + */ + void setSchemaRegistryUsage( + String topic, String clusterId, int schemaId, boolean isSuccess, boolean isKey); + /** * Sets data streams checkpoint, used for both produce and consume operations. * diff --git a/internal-api/src/main/java/datadog/trace/api/datastreams/NoopDataStreamsMonitoring.java b/internal-api/src/main/java/datadog/trace/api/datastreams/NoopDataStreamsMonitoring.java index f5cdcb0c82f..47109dee4c3 100644 --- a/internal-api/src/main/java/datadog/trace/api/datastreams/NoopDataStreamsMonitoring.java +++ b/internal-api/src/main/java/datadog/trace/api/datastreams/NoopDataStreamsMonitoring.java @@ -11,6 +11,10 @@ public class NoopDataStreamsMonitoring implements AgentDataStreamsMonitoring { @Override public void trackBacklog(DataStreamsTags tags, long value) {} + @Override + public void setSchemaRegistryUsage( + String topic, String clusterId, int schemaId, boolean isSuccess, boolean isKey) {} + @Override public void setCheckpoint(AgentSpan span, DataStreamsContext context) {} diff --git a/internal-api/src/main/java/datadog/trace/api/datastreams/SchemaRegistryUsage.java b/internal-api/src/main/java/datadog/trace/api/datastreams/SchemaRegistryUsage.java new file mode 100644 index 00000000000..2ea7ad7d0da --- /dev/null +++ b/internal-api/src/main/java/datadog/trace/api/datastreams/SchemaRegistryUsage.java @@ -0,0 +1,60 @@ +package datadog.trace.api.datastreams; + +/** + * SchemaRegistryUsage tracks usage of Confluent Schema Registry for data streams. This allows + * monitoring schema compatibility checks, registrations, and failures. + */ +public class SchemaRegistryUsage implements InboxItem { + private final String topic; + private final String clusterId; + private final int schemaId; + private final boolean isSuccess; + private final boolean isKey; + private final long timestampNanos; + private final String serviceNameOverride; + + public SchemaRegistryUsage( + String topic, + String clusterId, + int schemaId, + boolean isSuccess, + boolean isKey, + long timestampNanos, + String serviceNameOverride) { + this.topic = topic; + this.clusterId = clusterId; + this.schemaId = schemaId; + this.isSuccess = isSuccess; + this.isKey = isKey; + this.timestampNanos = timestampNanos; + this.serviceNameOverride = serviceNameOverride; + } + + public String getTopic() { + return topic; + } + + public String getClusterId() { + return clusterId; + } + + public int getSchemaId() { + return schemaId; + } + + public boolean isSuccess() { + return isSuccess; + } + + public boolean isKey() { + return isKey; + } + + public long getTimestampNanos() { + return timestampNanos; + } + + public String getServiceNameOverride() { + return serviceNameOverride; + } +} diff --git a/settings.gradle.kts b/settings.gradle.kts index 92aba9c108e..26b42860e4a 100644 --- a/settings.gradle.kts +++ b/settings.gradle.kts @@ -301,6 +301,7 @@ include( ":dd-java-agent:instrumentation:commons-lang:commons-lang-2.1", ":dd-java-agent:instrumentation:commons-lang:commons-lang-3.5", ":dd-java-agent:instrumentation:commons-text-1.0", + ":dd-java-agent:instrumentation:confluent-schema-registry:confluent-schema-registry-7.0", ":dd-java-agent:instrumentation:couchbase:couchbase-2.0", ":dd-java-agent:instrumentation:couchbase:couchbase-2.6", ":dd-java-agent:instrumentation:couchbase:couchbase-3.1", From b139559a4ce4f49d1efe3889c89181d712f91852 Mon Sep 17 00:00:00 2001 From: Piotr Wolski Date: Fri, 14 Nov 2025 12:30:33 -0700 Subject: [PATCH 02/12] fix cluster ID tagging for consumer --- .../SchemaRegistryContext.java | 9 +++++ ...fluentSchemaRegistryDataStreamsTest.groovy | 8 ++--- .../KafkaConsumerInfoInstrumentation.java | 36 ++++++++++++++++++- .../MsgPackDatastreamsPayloadWriter.java | 6 ++-- 4 files changed, 51 insertions(+), 8 deletions(-) diff --git a/dd-java-agent/instrumentation/confluent-schema-registry/confluent-schema-registry-7.0/src/main/java/datadog/trace/instrumentation/confluentschemaregistry/SchemaRegistryContext.java b/dd-java-agent/instrumentation/confluent-schema-registry/confluent-schema-registry-7.0/src/main/java/datadog/trace/instrumentation/confluentschemaregistry/SchemaRegistryContext.java index c0d122e8b06..bac93ffc165 100644 --- a/dd-java-agent/instrumentation/confluent-schema-registry/confluent-schema-registry-7.0/src/main/java/datadog/trace/instrumentation/confluentschemaregistry/SchemaRegistryContext.java +++ b/dd-java-agent/instrumentation/confluent-schema-registry/confluent-schema-registry-7.0/src/main/java/datadog/trace/instrumentation/confluentschemaregistry/SchemaRegistryContext.java @@ -52,6 +52,15 @@ public static Integer getValueSchemaId() { } public static void clear() { + currentTopic.remove(); + // Don't clear clusterId - it should persist for the entire poll() batch + // clusterId.remove(); + isKey.remove(); + keySchemaId.remove(); + valueSchemaId.remove(); + } + + public static void clearAll() { currentTopic.remove(); clusterId.remove(); isKey.remove(); diff --git a/dd-java-agent/instrumentation/confluent-schema-registry/confluent-schema-registry-7.0/src/test/groovy/ConfluentSchemaRegistryDataStreamsTest.groovy b/dd-java-agent/instrumentation/confluent-schema-registry/confluent-schema-registry-7.0/src/test/groovy/ConfluentSchemaRegistryDataStreamsTest.groovy index 2f72923eca3..cf031318cc8 100644 --- a/dd-java-agent/instrumentation/confluent-schema-registry/confluent-schema-registry-7.0/src/test/groovy/ConfluentSchemaRegistryDataStreamsTest.groovy +++ b/dd-java-agent/instrumentation/confluent-schema-registry/confluent-schema-registry-7.0/src/test/groovy/ConfluentSchemaRegistryDataStreamsTest.groovy @@ -88,7 +88,7 @@ class ConfluentSchemaRegistryDataStreamsTest extends InstrumentationSpecificatio cleanup: serializer.close() - datadog.trace.instrumentation.confluentschemaregistry.SchemaRegistryContext.clear() + datadog.trace.instrumentation.confluentschemaregistry.SchemaRegistryContext.clearAll() } def "test successful producer with key and value serializers"() { @@ -149,7 +149,7 @@ class ConfluentSchemaRegistryDataStreamsTest extends InstrumentationSpecificatio cleanup: keySerializer.close() valueSerializer.close() - datadog.trace.instrumentation.confluentschemaregistry.SchemaRegistryContext.clear() + datadog.trace.instrumentation.confluentschemaregistry.SchemaRegistryContext.clearAll() } def "test serialization failure is tracked"() { @@ -243,7 +243,7 @@ class ConfluentSchemaRegistryDataStreamsTest extends InstrumentationSpecificatio cleanup: serializer.close() - datadog.trace.instrumentation.confluentschemaregistry.SchemaRegistryContext.clear() + datadog.trace.instrumentation.confluentschemaregistry.SchemaRegistryContext.clearAll() } def "test schema registry usage metrics are aggregated by topic"() { @@ -288,7 +288,7 @@ class ConfluentSchemaRegistryDataStreamsTest extends InstrumentationSpecificatio cleanup: serializer.close() - datadog.trace.instrumentation.confluentschemaregistry.SchemaRegistryContext.clear() + datadog.trace.instrumentation.confluentschemaregistry.SchemaRegistryContext.clearAll() } } diff --git a/dd-java-agent/instrumentation/kafka/kafka-clients-0.11/src/main/java/datadog/trace/instrumentation/kafka_clients/KafkaConsumerInfoInstrumentation.java b/dd-java-agent/instrumentation/kafka/kafka-clients-0.11/src/main/java/datadog/trace/instrumentation/kafka_clients/KafkaConsumerInfoInstrumentation.java index 33056572e87..50b1ad03c08 100644 --- a/dd-java-agent/instrumentation/kafka/kafka-clients-0.11/src/main/java/datadog/trace/instrumentation/kafka_clients/KafkaConsumerInfoInstrumentation.java +++ b/dd-java-agent/instrumentation/kafka/kafka-clients-0.11/src/main/java/datadog/trace/instrumentation/kafka_clients/KafkaConsumerInfoInstrumentation.java @@ -204,7 +204,41 @@ public static void muzzleCheck(ConsumerRecord record) { */ public static class RecordsAdvice { @Advice.OnMethodEnter(suppress = Throwable.class) - public static AgentScope onEnter() { + public static AgentScope onEnter(@Advice.This KafkaConsumer consumer) { + // Set cluster ID in SchemaRegistryContext before deserialization + // Use reflection to avoid compile-time dependency on the schema registry module + KafkaConsumerInfo kafkaConsumerInfo = + InstrumentationContext.get(KafkaConsumer.class, KafkaConsumerInfo.class).get(consumer); + System.out.println("[DEBUG Consumer] kafkaConsumerInfo: " + kafkaConsumerInfo); + if (kafkaConsumerInfo != null) { + // Extract cluster ID directly inline to avoid muzzle issues + String clusterId = null; + if (Config.get().isDataStreamsEnabled()) { + Metadata consumerMetadata = kafkaConsumerInfo.getClientMetadata(); + System.out.println("[DEBUG Consumer] metadata: " + consumerMetadata); + if (consumerMetadata != null) { + clusterId = + InstrumentationContext.get(Metadata.class, String.class).get(consumerMetadata); + } + } + System.out.println("[DEBUG Consumer] Extracted clusterId: " + clusterId); + if (clusterId != null) { + try { + Class contextClass = + Class.forName( + "datadog.trace.instrumentation.confluentschemaregistry.SchemaRegistryContext", + false, + consumer.getClass().getClassLoader()); + contextClass.getMethod("setClusterId", String.class).invoke(null, clusterId); + System.out.println( + "[DEBUG Consumer] Set clusterId in SchemaRegistryContext: " + clusterId); + } catch (Throwable t) { + // Ignore if SchemaRegistryContext is not available + System.out.println("[DEBUG Consumer] Failed to set clusterId: " + t.getMessage()); + } + } + } + if (traceConfig().isDataStreamsEnabled()) { final AgentSpan span = startSpan(KAFKA_POLL); return activateSpan(span); diff --git a/dd-trace-core/src/main/java/datadog/trace/core/datastreams/MsgPackDatastreamsPayloadWriter.java b/dd-trace-core/src/main/java/datadog/trace/core/datastreams/MsgPackDatastreamsPayloadWriter.java index 34f2d134ecb..40797d83d57 100644 --- a/dd-trace-core/src/main/java/datadog/trace/core/datastreams/MsgPackDatastreamsPayloadWriter.java +++ b/dd-trace-core/src/main/java/datadog/trace/core/datastreams/MsgPackDatastreamsPayloadWriter.java @@ -248,13 +248,13 @@ private void writeSchemaRegistryUsages( key.isKey(), value.getCount()); - packer.startMap(5); + packer.startMap(6); // 6 fields: Topic, KafkaClusterId, SchemaId, IsSuccess, IsKey, Count packer.writeUTF8(TOPIC); - packer.writeString(key.getTopic(), null); + packer.writeString(key.getTopic() != null ? key.getTopic() : "", null); packer.writeUTF8(KAFKA_CLUSTER_ID); - packer.writeString(key.getClusterId(), null); + packer.writeString(key.getClusterId() != null ? key.getClusterId() : "", null); packer.writeUTF8(SCHEMA_ID); packer.writeInt(key.getSchemaId()); From b362939cf63fc4299711dfb59318857b46f07c6a Mon Sep 17 00:00:00 2001 From: Piotr Wolski Date: Mon, 17 Nov 2025 13:17:49 -0700 Subject: [PATCH 03/12] Easier instrumentation --- .../confluent-schema-registry-7.0/README.md | 133 +++------- .../ConfluentSchemaRegistryModule.java | 45 ---- .../KafkaAvroDeserializerInstrumentation.java | 175 ------------- .../KafkaAvroSerializerInstrumentation.java | 200 --------------- .../KafkaDeserializerInstrumentation.java | 93 +++++++ .../KafkaSerializerInstrumentation.java | 90 +++++++ .../SchemaRegistryClientInstrumentation.java | 128 ---------- .../SchemaRegistryContext.java | 70 ----- .../SchemaRegistryMetrics.java | 240 ------------------ .../KafkaConsumerInfoInstrumentation.java | 34 --- .../KafkaProducerInstrumentation.java | 15 -- .../api/datastreams/SchemaRegistryUsage.java | 1 + 12 files changed, 217 insertions(+), 1007 deletions(-) delete mode 100644 dd-java-agent/instrumentation/confluent-schema-registry/confluent-schema-registry-7.0/src/main/java/datadog/trace/instrumentation/confluentschemaregistry/ConfluentSchemaRegistryModule.java delete mode 100644 dd-java-agent/instrumentation/confluent-schema-registry/confluent-schema-registry-7.0/src/main/java/datadog/trace/instrumentation/confluentschemaregistry/KafkaAvroDeserializerInstrumentation.java delete mode 100644 dd-java-agent/instrumentation/confluent-schema-registry/confluent-schema-registry-7.0/src/main/java/datadog/trace/instrumentation/confluentschemaregistry/KafkaAvroSerializerInstrumentation.java create mode 100644 dd-java-agent/instrumentation/confluent-schema-registry/confluent-schema-registry-7.0/src/main/java/datadog/trace/instrumentation/confluentschemaregistry/KafkaDeserializerInstrumentation.java create mode 100644 dd-java-agent/instrumentation/confluent-schema-registry/confluent-schema-registry-7.0/src/main/java/datadog/trace/instrumentation/confluentschemaregistry/KafkaSerializerInstrumentation.java delete mode 100644 dd-java-agent/instrumentation/confluent-schema-registry/confluent-schema-registry-7.0/src/main/java/datadog/trace/instrumentation/confluentschemaregistry/SchemaRegistryClientInstrumentation.java delete mode 100644 dd-java-agent/instrumentation/confluent-schema-registry/confluent-schema-registry-7.0/src/main/java/datadog/trace/instrumentation/confluentschemaregistry/SchemaRegistryContext.java delete mode 100644 dd-java-agent/instrumentation/confluent-schema-registry/confluent-schema-registry-7.0/src/main/java/datadog/trace/instrumentation/confluentschemaregistry/SchemaRegistryMetrics.java diff --git a/dd-java-agent/instrumentation/confluent-schema-registry/confluent-schema-registry-7.0/README.md b/dd-java-agent/instrumentation/confluent-schema-registry/confluent-schema-registry-7.0/README.md index cba91edc90c..0e36cb4c9b1 100644 --- a/dd-java-agent/instrumentation/confluent-schema-registry/confluent-schema-registry-7.0/README.md +++ b/dd-java-agent/instrumentation/confluent-schema-registry/confluent-schema-registry-7.0/README.md @@ -1,141 +1,74 @@ # Confluent Schema Registry Instrumentation -This instrumentation module provides detailed observability for Confluent Schema Registry operations in Kafka applications. +This instrumentation module provides Data Streams Monitoring (DSM) support for Confluent Schema Registry operations in Kafka applications. ## Features -This instrumentation captures: +This instrumentation captures schema registry usage for both serialization and deserialization operations, reporting them to Datadog's Data Streams Monitoring. ### Producer Operations -- **Schema Registration**: Tracks when schemas are registered with the Schema Registry - - Subject name - - Schema ID assigned - - Success/failure status - - Compatibility check results -- **Serialization**: Logs every message serialization with: +- **Serialization**: Tracks message serialization with: - Topic name - - Key schema ID (if applicable) - - Value schema ID + - Schema ID (extracted from serialized bytes) + - Key vs value serialization - Success/failure status ### Consumer Operations -- **Deserialization**: Tracks every message deserialization with: +- **Deserialization**: Tracks message deserialization with: - Topic name - - Key schema ID (if present in message) - - Value schema ID (extracted from Confluent wire format) + - Schema ID (extracted from Confluent wire format) + - Key vs value deserialization - Success/failure status -### Schema Registry Client Operations -- **Schema Registration** (`register()` method) - - Successful registrations with schema ID - - Compatibility failures with error details -- **Compatibility Checks** (`testCompatibility()` method) - - Pass/fail status - - Error messages for incompatible schemas -- **Schema Retrieval** (`getSchemaById()` method) - - Schema ID lookups during deserialization - -## Metrics Collected - -The `SchemaRegistryMetrics` class tracks: - -- `schemaRegistrationSuccess` - Count of successful schema registrations -- `schemaRegistrationFailure` - Count of failed schema registrations (compatibility issues) -- `schemaCompatibilitySuccess` - Count of successful compatibility checks -- `schemaCompatibilityFailure` - Count of failed compatibility checks -- `serializationSuccess` - Count of successful message serializations -- `serializationFailure` - Count of failed serializations -- `deserializationSuccess` - Count of successful message deserializations -- `deserializationFailure` - Count of failed deserializations - -## Log Output Examples - -### Successful Producer Operation -``` -[Schema Registry] Schema registered successfully - Subject: myTopic-value, Schema ID: 123, Is Key: false, Topic: myTopic -[Schema Registry] Produce to topic 'myTopic', schema for key: none, schema for value: 123, serializing: VALUE -``` - -### Failed Schema Registration (Incompatibility) -``` -[Schema Registry] Schema registration FAILED - Subject: myTopic-value, Is Key: false, Topic: myTopic, Error: Schema being registered is incompatible with an earlier schema -[Schema Registry] Schema compatibility check FAILED - Subject: myTopic-value, Error: Schema being registered is incompatible with an earlier schema -[Schema Registry] Serialization FAILED for topic 'myTopic', VALUE - Error: Schema being registered is incompatible with an earlier schema -``` - -### Consumer Operation -``` -[Schema Registry] Retrieved schema from registry - Schema ID: 123, Type: Schema -[Schema Registry] Consume from topic 'myTopic', schema for key: none, schema for value: 123, deserializing: VALUE -``` - ## Supported Serialization Formats - **Avro** (via `KafkaAvroSerializer`/`KafkaAvroDeserializer`) +- **JSON Schema** (via `KafkaJsonSchemaSerializer`/`KafkaJsonSchemaDeserializer`) - **Protobuf** (via `KafkaProtobufSerializer`/`KafkaProtobufDeserializer`) ## Implementation Details ### Instrumented Classes -1. **CachedSchemaRegistryClient** - The main Schema Registry client - - `register(String subject, Schema schema)` - Schema registration - - `testCompatibility(String subject, Schema schema)` - Compatibility testing - - `getSchemaById(int id)` - Schema retrieval - -2. **AbstractKafkaSchemaSerDe and subclasses** - Serializers - - `serialize(String topic, Object data)` - Message serialization - - `serialize(String topic, Headers headers, Object data)` - With headers (Kafka 2.1+) +1. **KafkaAvroSerializer** - Avro message serialization +2. **KafkaJsonSchemaSerializer** - JSON Schema message serialization +3. **KafkaProtobufSerializer** - Protobuf message serialization +4. **KafkaAvroDeserializer** - Avro message deserialization +5. **KafkaJsonSchemaDeserializer** - JSON Schema message deserialization +6. **KafkaProtobufDeserializer** - Protobuf message deserialization -3. **AbstractKafkaSchemaSerDe and subclasses** - Deserializers - - `deserialize(String topic, byte[] data)` - Message deserialization - - `deserialize(String topic, Headers headers, byte[] data)` - With headers (Kafka 2.1+) +### Schema ID Extraction -### Context Management +Schema IDs are extracted directly from the Confluent wire format: +- **Format**: `[magic_byte][4-byte schema id][data]` +- The magic byte (0x00) indicates Confluent wire format +- Schema ID is a 4-byte big-endian integer -The `SchemaRegistryContext` class uses ThreadLocal storage to pass context between: -- Serializer → Schema Registry Client (for logging topic information) -- Deserializer → Schema Registry Client (for logging topic information) +### Key vs Value Detection -This allows the instrumentation to correlate schema operations with the topics they're associated with. +The instrumentation determines whether a serializer/deserializer is for keys or values by calling the `isKey()` method available on all Confluent serializers/deserializers. ## Usage This instrumentation is automatically activated when: 1. Confluent Schema Registry client (version 7.0.0+) is present on the classpath 2. The Datadog Java agent is attached to the JVM +3. Data Streams Monitoring is enabled No configuration or code changes are required. -## Metrics Access +## Data Streams Monitoring Integration -To access metrics programmatically: +Schema registry usage is reported directly to Datadog's Data Streams Monitoring via: ```java -import datadog.trace.instrumentation.confluentschemaregistry.SchemaRegistryMetrics; - -// Get current counts -long registrationFailures = SchemaRegistryMetrics.getSchemaRegistrationFailureCount(); -long compatibilityFailures = SchemaRegistryMetrics.getSchemaCompatibilityFailureCount(); -long serializationFailures = SchemaRegistryMetrics.getSerializationFailureCount(); - -// Print summary -SchemaRegistryMetrics.printSummary(); +AgentTracer.get() + .getDataStreamsMonitoring() + .setSchemaRegistryUsage(topic, clusterId, schemaId, isError, isKey); ``` -## Monitoring Schema Compatibility Issues - -The primary use case for this instrumentation is to detect and monitor schema compatibility issues that cause production failures. By tracking `schemaRegistrationFailure` and `schemaCompatibilityFailure` metrics, you can: - -1. **Alert on schema compatibility failures** before they impact production -2. **Track the rate of schema-related errors** per topic -3. **Identify problematic schema changes** that break compatibility -4. **Monitor serialization/deserialization failure rates** as a proxy for schema issues - -## Future Enhancements - -Potential additions: -- JSON Schema serializer support (currently excluded due to dependency issues) -- Schema evolution tracking -- Schema version diff logging -- Integration with Datadog APM for schema-related span tags +This allows tracking of: +- Schema usage patterns across topics +- Schema registry operation success rates +- Key vs value schema usage +- Schema evolution over time diff --git a/dd-java-agent/instrumentation/confluent-schema-registry/confluent-schema-registry-7.0/src/main/java/datadog/trace/instrumentation/confluentschemaregistry/ConfluentSchemaRegistryModule.java b/dd-java-agent/instrumentation/confluent-schema-registry/confluent-schema-registry-7.0/src/main/java/datadog/trace/instrumentation/confluentschemaregistry/ConfluentSchemaRegistryModule.java deleted file mode 100644 index 47f10064646..00000000000 --- a/dd-java-agent/instrumentation/confluent-schema-registry/confluent-schema-registry-7.0/src/main/java/datadog/trace/instrumentation/confluentschemaregistry/ConfluentSchemaRegistryModule.java +++ /dev/null @@ -1,45 +0,0 @@ -package datadog.trace.instrumentation.confluentschemaregistry; - -import static datadog.trace.agent.tooling.bytebuddy.matcher.ClassLoaderMatchers.hasClassNamed; -import static java.util.Arrays.asList; - -import com.google.auto.service.AutoService; -import datadog.trace.agent.tooling.Instrumenter; -import datadog.trace.agent.tooling.InstrumenterModule; -import java.util.List; -import net.bytebuddy.matcher.ElementMatcher; - -/** - * Instrumentation module for Confluent Schema Registry to capture schema operations including - * registration, compatibility checks, serialization, and deserialization. - */ -@AutoService(InstrumenterModule.class) -public class ConfluentSchemaRegistryModule extends InstrumenterModule.Tracing { - - public ConfluentSchemaRegistryModule() { - super("confluent-schema-registry"); - } - - @Override - public String[] helperClassNames() { - return new String[] { - packageName + ".SchemaRegistryMetrics", packageName + ".SchemaRegistryContext", - }; - } - - @Override - @SuppressWarnings("unchecked") - public List typeInstrumentations() { - return (List) - (List) - asList( - new SchemaRegistryClientInstrumentation(), - new KafkaAvroSerializerInstrumentation(), - new KafkaAvroDeserializerInstrumentation()); - } - - @Override - public ElementMatcher.Junction classLoaderMatcher() { - return hasClassNamed("io.confluent.kafka.schemaregistry.client.SchemaRegistryClient"); - } -} diff --git a/dd-java-agent/instrumentation/confluent-schema-registry/confluent-schema-registry-7.0/src/main/java/datadog/trace/instrumentation/confluentschemaregistry/KafkaAvroDeserializerInstrumentation.java b/dd-java-agent/instrumentation/confluent-schema-registry/confluent-schema-registry-7.0/src/main/java/datadog/trace/instrumentation/confluentschemaregistry/KafkaAvroDeserializerInstrumentation.java deleted file mode 100644 index 19b3aa44c18..00000000000 --- a/dd-java-agent/instrumentation/confluent-schema-registry/confluent-schema-registry-7.0/src/main/java/datadog/trace/instrumentation/confluentschemaregistry/KafkaAvroDeserializerInstrumentation.java +++ /dev/null @@ -1,175 +0,0 @@ -package datadog.trace.instrumentation.confluentschemaregistry; - -import static datadog.trace.agent.tooling.bytebuddy.matcher.NameMatchers.named; -import static net.bytebuddy.matcher.ElementMatchers.isMethod; -import static net.bytebuddy.matcher.ElementMatchers.isPublic; -import static net.bytebuddy.matcher.ElementMatchers.takesArgument; -import static net.bytebuddy.matcher.ElementMatchers.takesArguments; - -import datadog.trace.agent.tooling.Instrumenter; -import datadog.trace.agent.tooling.Instrumenter.MethodTransformer; -import net.bytebuddy.asm.Advice; -import net.bytebuddy.description.type.TypeDescription; -import net.bytebuddy.matcher.ElementMatcher; - -/** - * Instruments AbstractKafkaSchemaSerDe (base class for Avro, Protobuf, and JSON deserializers) to - * capture deserialization operations. - */ -public class KafkaAvroDeserializerInstrumentation - implements Instrumenter.ForTypeHierarchy, Instrumenter.HasMethodAdvice { - - @Override - public String hierarchyMarkerType() { - return "io.confluent.kafka.serializers.AbstractKafkaSchemaSerDe"; - } - - @Override - public ElementMatcher hierarchyMatcher() { - return named("io.confluent.kafka.serializers.AbstractKafkaSchemaSerDe") - .or(named("io.confluent.kafka.serializers.AbstractKafkaAvroDeserializer")) - .or(named("io.confluent.kafka.serializers.KafkaAvroDeserializer")) - .or(named("io.confluent.kafka.serializers.protobuf.KafkaProtobufDeserializer")); - } - - @Override - public void methodAdvice(MethodTransformer transformer) { - // Instrument deserialize(String topic, byte[] data) - transformer.applyAdvice( - isMethod() - .and(named("deserialize")) - .and(isPublic()) - .and(takesArguments(2)) - .and(takesArgument(0, String.class)) - .and(takesArgument(1, byte[].class)), - getClass().getName() + "$DeserializeAdvice"); - - // Instrument deserialize(String topic, Headers headers, byte[] data) for Kafka 2.1+ - transformer.applyAdvice( - isMethod() - .and(named("deserialize")) - .and(isPublic()) - .and(takesArguments(3)) - .and(takesArgument(0, String.class)) - .and(takesArgument(2, byte[].class)), - getClass().getName() + "$DeserializeWithHeadersAdvice"); - } - - public static class DeserializeAdvice { - @Advice.OnMethodEnter(suppress = Throwable.class) - public static void onEnter( - @Advice.This Object deserializer, - @Advice.Argument(0) String topic, - @Advice.Argument(1) byte[] data) { - // Set the topic in context - SchemaRegistryContext.setTopic(topic); - - // Determine if this is a key or value deserializer - String className = deserializer.getClass().getSimpleName(); - boolean isKey = - className.contains("Key") || deserializer.getClass().getName().contains("Key"); - SchemaRegistryContext.setIsKey(isKey); - - // Extract schema ID from the data if present (Confluent wire format) - if (data != null && data.length >= 5) { - // Confluent wire format: [magic_byte][4-byte schema id][data] - if (data[0] == 0) { - int schemaId = - ((data[1] & 0xFF) << 24) - | ((data[2] & 0xFF) << 16) - | ((data[3] & 0xFF) << 8) - | (data[4] & 0xFF); - - if (isKey) { - SchemaRegistryContext.setKeySchemaId(schemaId); - } else { - SchemaRegistryContext.setValueSchemaId(schemaId); - } - } - } - } - - @Advice.OnMethodExit(onThrowable = Throwable.class, suppress = Throwable.class) - public static void onExit( - @Advice.This Object deserializer, - @Advice.Argument(0) String topic, - @Advice.Return Object result, - @Advice.Thrown Throwable throwable) { - - Boolean isKey = SchemaRegistryContext.getIsKey(); - Integer keySchemaId = SchemaRegistryContext.getKeySchemaId(); - Integer valueSchemaId = SchemaRegistryContext.getValueSchemaId(); - - if (throwable != null) { - SchemaRegistryMetrics.recordDeserializationFailure( - topic, throwable.getMessage(), isKey != null ? isKey : false); - } else if (result != null) { - // Successful deserialization - SchemaRegistryMetrics.recordDeserialization( - topic, keySchemaId, valueSchemaId, isKey != null ? isKey : false); - } - - // Clear context after deserialization - SchemaRegistryContext.clear(); - } - } - - public static class DeserializeWithHeadersAdvice { - @Advice.OnMethodEnter(suppress = Throwable.class) - public static void onEnter( - @Advice.This Object deserializer, - @Advice.Argument(0) String topic, - @Advice.Argument(2) byte[] data) { - // Set the topic in context - SchemaRegistryContext.setTopic(topic); - - // Determine if this is a key or value deserializer - String className = deserializer.getClass().getSimpleName(); - boolean isKey = - className.contains("Key") || deserializer.getClass().getName().contains("Key"); - SchemaRegistryContext.setIsKey(isKey); - - // Extract schema ID from the data if present (Confluent wire format) - if (data != null && data.length >= 5) { - // Confluent wire format: [magic_byte][4-byte schema id][data] - if (data[0] == 0) { - int schemaId = - ((data[1] & 0xFF) << 24) - | ((data[2] & 0xFF) << 16) - | ((data[3] & 0xFF) << 8) - | (data[4] & 0xFF); - - if (isKey) { - SchemaRegistryContext.setKeySchemaId(schemaId); - } else { - SchemaRegistryContext.setValueSchemaId(schemaId); - } - } - } - } - - @Advice.OnMethodExit(onThrowable = Throwable.class, suppress = Throwable.class) - public static void onExit( - @Advice.This Object deserializer, - @Advice.Argument(0) String topic, - @Advice.Return Object result, - @Advice.Thrown Throwable throwable) { - - Boolean isKey = SchemaRegistryContext.getIsKey(); - Integer keySchemaId = SchemaRegistryContext.getKeySchemaId(); - Integer valueSchemaId = SchemaRegistryContext.getValueSchemaId(); - - if (throwable != null) { - SchemaRegistryMetrics.recordDeserializationFailure( - topic, throwable.getMessage(), isKey != null ? isKey : false); - } else if (result != null) { - // Successful deserialization - SchemaRegistryMetrics.recordDeserialization( - topic, keySchemaId, valueSchemaId, isKey != null ? isKey : false); - } - - // Clear context after deserialization - SchemaRegistryContext.clear(); - } - } -} diff --git a/dd-java-agent/instrumentation/confluent-schema-registry/confluent-schema-registry-7.0/src/main/java/datadog/trace/instrumentation/confluentschemaregistry/KafkaAvroSerializerInstrumentation.java b/dd-java-agent/instrumentation/confluent-schema-registry/confluent-schema-registry-7.0/src/main/java/datadog/trace/instrumentation/confluentschemaregistry/KafkaAvroSerializerInstrumentation.java deleted file mode 100644 index 1ab0fbff2ee..00000000000 --- a/dd-java-agent/instrumentation/confluent-schema-registry/confluent-schema-registry-7.0/src/main/java/datadog/trace/instrumentation/confluentschemaregistry/KafkaAvroSerializerInstrumentation.java +++ /dev/null @@ -1,200 +0,0 @@ -package datadog.trace.instrumentation.confluentschemaregistry; - -import static datadog.trace.agent.tooling.bytebuddy.matcher.NameMatchers.named; -import static net.bytebuddy.matcher.ElementMatchers.isMethod; -import static net.bytebuddy.matcher.ElementMatchers.isPublic; -import static net.bytebuddy.matcher.ElementMatchers.returns; -import static net.bytebuddy.matcher.ElementMatchers.takesArgument; -import static net.bytebuddy.matcher.ElementMatchers.takesArguments; - -import datadog.trace.agent.tooling.Instrumenter; -import datadog.trace.agent.tooling.Instrumenter.MethodTransformer; -import net.bytebuddy.asm.Advice; -import net.bytebuddy.description.type.TypeDescription; -import net.bytebuddy.matcher.ElementMatcher; - -/** - * Instruments AbstractKafkaSchemaSerDe (base class for Avro, Protobuf, and JSON serializers) to - * capture serialization operations. - */ -public class KafkaAvroSerializerInstrumentation - implements Instrumenter.ForTypeHierarchy, Instrumenter.HasMethodAdvice { - - @Override - public String hierarchyMarkerType() { - return "io.confluent.kafka.serializers.AbstractKafkaSchemaSerDe"; - } - - @Override - public ElementMatcher hierarchyMatcher() { - return named("io.confluent.kafka.serializers.AbstractKafkaSchemaSerDe") - .or(named("io.confluent.kafka.serializers.AbstractKafkaAvroSerializer")) - .or(named("io.confluent.kafka.serializers.KafkaAvroSerializer")) - .or(named("io.confluent.kafka.serializers.protobuf.KafkaProtobufSerializer")); - } - - @Override - public void methodAdvice(MethodTransformer transformer) { - // Instrument serialize(String topic, Object data) - transformer.applyAdvice( - isMethod() - .and(named("serialize")) - .and(isPublic()) - .and(takesArguments(2)) - .and(takesArgument(0, String.class)) - .and(returns(byte[].class)), - getClass().getName() + "$SerializeAdvice"); - - // Instrument serialize(String topic, Headers headers, Object data) for Kafka 2.1+ - transformer.applyAdvice( - isMethod() - .and(named("serialize")) - .and(isPublic()) - .and(takesArguments(3)) - .and(takesArgument(0, String.class)) - .and(returns(byte[].class)), - getClass().getName() + "$SerializeWithHeadersAdvice"); - } - - public static class SerializeAdvice { - @Advice.OnMethodEnter(suppress = Throwable.class) - public static void onEnter(@Advice.This Object serializer, @Advice.Argument(0) String topic) { - // Set the topic in context so that the schema registry client can use it - SchemaRegistryContext.setTopic(topic); - - // Determine if this is a key or value serializer - String className = serializer.getClass().getSimpleName(); - boolean isKey = className.contains("Key") || serializer.getClass().getName().contains("Key"); - SchemaRegistryContext.setIsKey(isKey); - } - - @Advice.OnMethodExit(onThrowable = Throwable.class, suppress = Throwable.class) - public static void onExit( - @Advice.This Object serializer, - @Advice.Argument(0) String topic, - @Advice.Return byte[] result, - @Advice.Thrown Throwable throwable) { - - try { - Boolean isKey = SchemaRegistryContext.getIsKey(); - - if (throwable != null) { - SchemaRegistryMetrics.recordSerializationFailure( - topic, throwable.getMessage(), isKey != null ? isKey : false); - } else if (result != null) { - // Extract schema ID from the serialized bytes (Confluent wire format) - Integer schemaId = null; - try { - // Confluent wire format: [magic_byte][4-byte schema id][data] - if (result.length >= 5 && result[0] == 0) { - schemaId = - ((result[1] & 0xFF) << 24) - | ((result[2] & 0xFF) << 16) - | ((result[3] & 0xFF) << 8) - | (result[4] & 0xFF); - } - } catch (Throwable ignored) { - // Suppress any errors in schema ID extraction - } - - // Store in context for correlation - if (isKey != null && isKey) { - SchemaRegistryContext.setKeySchemaId(schemaId); - } else { - SchemaRegistryContext.setValueSchemaId(schemaId); - } - - // Get both schema IDs for logging - Integer keySchemaId = SchemaRegistryContext.getKeySchemaId(); - Integer valueSchemaId = SchemaRegistryContext.getValueSchemaId(); - - // Successful serialization - SchemaRegistryMetrics.recordSerialization( - topic, keySchemaId, valueSchemaId, isKey != null ? isKey : false); - } - } catch (Throwable t) { - // Don't let instrumentation errors break the application - // but try to log them if possible - try { - SchemaRegistryMetrics.recordSerializationFailure(topic, "Instrumentation error", false); - } catch (Throwable ignored) { - // Really suppress everything - } - } finally { - // Clear context after serialization - SchemaRegistryContext.clear(); - } - } - } - - public static class SerializeWithHeadersAdvice { - @Advice.OnMethodEnter(suppress = Throwable.class) - public static void onEnter(@Advice.This Object serializer, @Advice.Argument(0) String topic) { - // Set the topic in context so that the schema registry client can use it - SchemaRegistryContext.setTopic(topic); - - // Determine if this is a key or value serializer - String className = serializer.getClass().getSimpleName(); - boolean isKey = className.contains("Key") || serializer.getClass().getName().contains("Key"); - SchemaRegistryContext.setIsKey(isKey); - } - - @Advice.OnMethodExit(onThrowable = Throwable.class, suppress = Throwable.class) - public static void onExit( - @Advice.This Object serializer, - @Advice.Argument(0) String topic, - @Advice.Return byte[] result, - @Advice.Thrown Throwable throwable) { - - try { - Boolean isKey = SchemaRegistryContext.getIsKey(); - - if (throwable != null) { - SchemaRegistryMetrics.recordSerializationFailure( - topic, throwable.getMessage(), isKey != null ? isKey : false); - } else if (result != null) { - // Extract schema ID from the serialized bytes (Confluent wire format) - Integer schemaId = null; - try { - // Confluent wire format: [magic_byte][4-byte schema id][data] - if (result.length >= 5 && result[0] == 0) { - schemaId = - ((result[1] & 0xFF) << 24) - | ((result[2] & 0xFF) << 16) - | ((result[3] & 0xFF) << 8) - | (result[4] & 0xFF); - } - } catch (Throwable ignored) { - // Suppress any errors in schema ID extraction - } - - // Store in context for correlation - if (isKey != null && isKey) { - SchemaRegistryContext.setKeySchemaId(schemaId); - } else { - SchemaRegistryContext.setValueSchemaId(schemaId); - } - - // Get both schema IDs for logging - Integer keySchemaId = SchemaRegistryContext.getKeySchemaId(); - Integer valueSchemaId = SchemaRegistryContext.getValueSchemaId(); - - // Successful serialization - SchemaRegistryMetrics.recordSerialization( - topic, keySchemaId, valueSchemaId, isKey != null ? isKey : false); - } - } catch (Throwable t) { - // Don't let instrumentation errors break the application - // but try to log them if possible - try { - SchemaRegistryMetrics.recordSerializationFailure(topic, "Instrumentation error", false); - } catch (Throwable ignored) { - // Really suppress everything - } - } finally { - // Clear context after serialization - SchemaRegistryContext.clear(); - } - } - } -} diff --git a/dd-java-agent/instrumentation/confluent-schema-registry/confluent-schema-registry-7.0/src/main/java/datadog/trace/instrumentation/confluentschemaregistry/KafkaDeserializerInstrumentation.java b/dd-java-agent/instrumentation/confluent-schema-registry/confluent-schema-registry-7.0/src/main/java/datadog/trace/instrumentation/confluentschemaregistry/KafkaDeserializerInstrumentation.java new file mode 100644 index 00000000000..91ff69128d6 --- /dev/null +++ b/dd-java-agent/instrumentation/confluent-schema-registry/confluent-schema-registry-7.0/src/main/java/datadog/trace/instrumentation/confluentschemaregistry/KafkaDeserializerInstrumentation.java @@ -0,0 +1,93 @@ +package datadog.trace.instrumentation.confluentschemaregistry; + +import static datadog.trace.agent.tooling.bytebuddy.matcher.NameMatchers.named; +import static net.bytebuddy.matcher.ElementMatchers.isMethod; +import static net.bytebuddy.matcher.ElementMatchers.isPublic; +import static net.bytebuddy.matcher.ElementMatchers.takesArgument; +import static net.bytebuddy.matcher.ElementMatchers.takesArguments; + +import datadog.trace.agent.tooling.Instrumenter; +import datadog.trace.agent.tooling.Instrumenter.MethodTransformer; +import datadog.trace.bootstrap.instrumentation.api.AgentTracer; +import net.bytebuddy.asm.Advice; + +/** + * Instruments Confluent Schema Registry deserializers (Avro, Protobuf, and JSON) to capture + * deserialization operations. + */ +public class KafkaDeserializerInstrumentation + implements Instrumenter.ForKnownTypes, Instrumenter.HasMethodAdvice { + + @Override + public String[] knownMatchingTypes() { + return new String[] { + "io.confluent.kafka.serializers.KafkaAvroDeserializer", + "io.confluent.kafka.serializers.json.KafkaJsonSchemaDeserializer", + "io.confluent.kafka.serializers.protobuf.KafkaProtobufDeserializer" + }; + } + + @Override + public void methodAdvice(MethodTransformer transformer) { + // Instrument deserialize(String topic, Headers headers, byte[] data) + // The 2-arg version calls this one, so we only need to instrument this to avoid duplicates + transformer.applyAdvice( + isMethod() + .and(named("deserialize")) + .and(isPublic()) + .and(takesArguments(3)) + .and(takesArgument(0, String.class)) + .and(takesArgument(2, byte[].class)), + getClass().getName() + "$DeserializeAdvice"); + } + + public static class DeserializeAdvice { + @Advice.OnMethodExit(onThrowable = Throwable.class, suppress = Throwable.class) + public static void onExit( + @Advice.This org.apache.kafka.common.serialization.Deserializer deserializer, + @Advice.Argument(0) String topic, + @Advice.Argument(2) byte[] data, + @Advice.Return Object result, + @Advice.Thrown Throwable throwable) { + + // Get isKey from the deserializer object + boolean isKey = false; + if (deserializer instanceof io.confluent.kafka.serializers.KafkaAvroDeserializer) { + isKey = ((io.confluent.kafka.serializers.KafkaAvroDeserializer) deserializer).isKey(); + } else if (deserializer + instanceof io.confluent.kafka.serializers.json.KafkaJsonSchemaDeserializer) { + isKey = + ((io.confluent.kafka.serializers.json.KafkaJsonSchemaDeserializer) deserializer) + .isKey(); + } else if (deserializer + instanceof io.confluent.kafka.serializers.protobuf.KafkaProtobufDeserializer) { + isKey = + ((io.confluent.kafka.serializers.protobuf.KafkaProtobufDeserializer) deserializer) + .isKey(); + } + + boolean isError = throwable != null; + int schemaId = -1; + + // Extract schema ID from the input bytes if successful + if (!isError && data != null && data.length >= 5 && data[0] == 0) { + try { + // Confluent wire format: [magic_byte][4-byte schema id][data] + schemaId = + ((data[1] & 0xFF) << 24) + | ((data[2] & 0xFF) << 16) + | ((data[3] & 0xFF) << 8) + | (data[4] & 0xFF); + } catch (Throwable ignored) { + // If extraction fails, keep schemaId as -1 + } + } + + // Record the schema registry usage + AgentTracer.get() + .getDataStreamsMonitoring() + .setSchemaRegistryUsage(topic, null, schemaId, isError, isKey); + } + } +} + diff --git a/dd-java-agent/instrumentation/confluent-schema-registry/confluent-schema-registry-7.0/src/main/java/datadog/trace/instrumentation/confluentschemaregistry/KafkaSerializerInstrumentation.java b/dd-java-agent/instrumentation/confluent-schema-registry/confluent-schema-registry-7.0/src/main/java/datadog/trace/instrumentation/confluentschemaregistry/KafkaSerializerInstrumentation.java new file mode 100644 index 00000000000..5795fd0e3de --- /dev/null +++ b/dd-java-agent/instrumentation/confluent-schema-registry/confluent-schema-registry-7.0/src/main/java/datadog/trace/instrumentation/confluentschemaregistry/KafkaSerializerInstrumentation.java @@ -0,0 +1,90 @@ +package datadog.trace.instrumentation.confluentschemaregistry; + +import static datadog.trace.agent.tooling.bytebuddy.matcher.NameMatchers.named; +import static net.bytebuddy.matcher.ElementMatchers.isMethod; +import static net.bytebuddy.matcher.ElementMatchers.isPublic; +import static net.bytebuddy.matcher.ElementMatchers.returns; +import static net.bytebuddy.matcher.ElementMatchers.takesArgument; +import static net.bytebuddy.matcher.ElementMatchers.takesArguments; + +import datadog.trace.agent.tooling.Instrumenter; +import datadog.trace.agent.tooling.Instrumenter.MethodTransformer; +import datadog.trace.bootstrap.instrumentation.api.AgentTracer; +import net.bytebuddy.asm.Advice; + +/** + * Instruments Confluent Schema Registry serializers (Avro, Protobuf, and JSON) to capture + * serialization operations. + */ +public class KafkaSerializerInstrumentation + implements Instrumenter.ForKnownTypes, Instrumenter.HasMethodAdvice { + + @Override + public String[] knownMatchingTypes() { + return new String[] { + "io.confluent.kafka.serializers.KafkaAvroSerializer", + "io.confluent.kafka.serializers.json.KafkaJsonSchemaSerializer", + "io.confluent.kafka.serializers.protobuf.KafkaProtobufSerializer" + }; + } + + @Override + public void methodAdvice(MethodTransformer transformer) { + // Instrument both serialize(String topic, Object data) + // and serialize(String topic, Headers headers, Object data) for Kafka 2.1+ + transformer.applyAdvice( + isMethod() + .and(named("serialize")) + .and(isPublic()) + .and(takesArgument(0, String.class)) + .and(returns(byte[].class)), + getClass().getName() + "$SerializeAdvice"); + } + + public static class SerializeAdvice { + @Advice.OnMethodExit(onThrowable = Throwable.class, suppress = Throwable.class) + public static void onExit( + @Advice.This org.apache.kafka.common.serialization.Serializer serializer, + @Advice.Argument(0) String topic, + @Advice.Return byte[] result, + @Advice.Thrown Throwable throwable) { + + // Get isKey from the serializer object + boolean isKey = false; + if (serializer instanceof io.confluent.kafka.serializers.KafkaAvroSerializer) { + isKey = ((io.confluent.kafka.serializers.KafkaAvroSerializer) serializer).isKey(); + } else if (serializer + instanceof io.confluent.kafka.serializer KafkaJsonSchemaSerializer) { + isKey = + ((io.confluent.kafka.serializers.KafkaJsonSchemaSerializer) serializer).isKey(); + } else if (serializer + instanceof io.confluent.kafka.serializers.protobuf.KafkaProtobufSerializer) { + isKey = + ((io.confluent.kafka.serializers.protobuf.KafkaProtobufSerializer) serializer).isKey(); + } + + boolean isError = throwable != null; + int schemaId = -1; + + // Extract schema ID from the serialized bytes if successful + if (!isError && result != null && result.length >= 5 && result[0] == 0) { + try { + // Confluent wire format: [magic_byte][4-byte schema id][data] + schemaId = + ((result[1] & 0xFF) << 24) + | ((result[2] & 0xFF) << 16) + | ((result[3] & 0xFF) << 8) + | (result[4] & 0xFF); + } catch (Throwable ignored) { + // If extraction fails, keep schemaId as -1 + } + } + + // Record the schema registry usage + AgentTracer.get() + .getDataStreamsMonitoring() + .setSchemaRegistryUsage(topic, null, schemaId, isError, isKey); + } + } +} + diff --git a/dd-java-agent/instrumentation/confluent-schema-registry/confluent-schema-registry-7.0/src/main/java/datadog/trace/instrumentation/confluentschemaregistry/SchemaRegistryClientInstrumentation.java b/dd-java-agent/instrumentation/confluent-schema-registry/confluent-schema-registry-7.0/src/main/java/datadog/trace/instrumentation/confluentschemaregistry/SchemaRegistryClientInstrumentation.java deleted file mode 100644 index 6768eba2423..00000000000 --- a/dd-java-agent/instrumentation/confluent-schema-registry/confluent-schema-registry-7.0/src/main/java/datadog/trace/instrumentation/confluentschemaregistry/SchemaRegistryClientInstrumentation.java +++ /dev/null @@ -1,128 +0,0 @@ -package datadog.trace.instrumentation.confluentschemaregistry; - -import static datadog.trace.agent.tooling.bytebuddy.matcher.NameMatchers.named; -import static net.bytebuddy.matcher.ElementMatchers.isMethod; -import static net.bytebuddy.matcher.ElementMatchers.isPublic; -import static net.bytebuddy.matcher.ElementMatchers.returns; -import static net.bytebuddy.matcher.ElementMatchers.takesArgument; -import static net.bytebuddy.matcher.ElementMatchers.takesArguments; - -import datadog.trace.agent.tooling.Instrumenter; -import datadog.trace.agent.tooling.Instrumenter.MethodTransformer; -import net.bytebuddy.asm.Advice; - -/** - * Instruments the CachedSchemaRegistryClient to capture schema registration and compatibility check - * operations. - */ -public class SchemaRegistryClientInstrumentation - implements Instrumenter.ForSingleType, Instrumenter.HasMethodAdvice { - - @Override - public String instrumentedType() { - return "io.confluent.kafka.schemaregistry.client.CachedSchemaRegistryClient"; - } - - @Override - public void methodAdvice(MethodTransformer transformer) { - // Instrument register(String subject, Schema schema) - transformer.applyAdvice( - isMethod() - .and(named("register")) - .and(isPublic()) - .and(takesArguments(2)) - .and(takesArgument(0, String.class)) - .and(returns(int.class)), - getClass().getName() + "$RegisterAdvice"); - - // Instrument testCompatibility(String subject, Schema schema) - transformer.applyAdvice( - isMethod() - .and(named("testCompatibility")) - .and(isPublic()) - .and(takesArguments(2)) - .and(takesArgument(0, String.class)) - .and(returns(boolean.class)), - getClass().getName() + "$TestCompatibilityAdvice"); - - // Instrument getSchemaById(int id) - transformer.applyAdvice( - isMethod() - .and(named("getSchemaById")) - .and(isPublic()) - .and(takesArguments(1)) - .and(takesArgument(0, int.class)), - getClass().getName() + "$GetSchemaByIdAdvice"); - } - - public static class RegisterAdvice { - @Advice.OnMethodEnter(suppress = Throwable.class) - public static void onEnter(@Advice.Argument(0) String subject) { - // Track that we're attempting registration - } - - @Advice.OnMethodExit(onThrowable = Throwable.class, suppress = Throwable.class) - public static void onExit( - @Advice.Argument(0) String subject, - @Advice.Return int schemaId, - @Advice.Thrown Throwable throwable) { - - String topic = SchemaRegistryContext.getTopic(); - Boolean isKey = SchemaRegistryContext.getIsKey(); - - if (throwable != null) { - // Registration failed - likely due to compatibility issues - String errorMessage = throwable.getMessage(); - SchemaRegistryMetrics.recordSchemaRegistrationFailure( - subject, errorMessage, isKey != null ? isKey : false, topic); - - // Also log that this is a compatibility failure - if (errorMessage != null - && (errorMessage.contains("incompatible") || errorMessage.contains("compatibility"))) { - SchemaRegistryMetrics.recordCompatibilityCheck(subject, false, errorMessage); - } - } else { - // Registration successful - SchemaRegistryMetrics.recordSchemaRegistration( - subject, schemaId, isKey != null ? isKey : false, topic); - - // Store the schema ID in context - if (isKey != null && isKey) { - SchemaRegistryContext.setKeySchemaId(schemaId); - } else { - SchemaRegistryContext.setValueSchemaId(schemaId); - } - } - } - } - - public static class TestCompatibilityAdvice { - @Advice.OnMethodExit(onThrowable = Throwable.class, suppress = Throwable.class) - public static void onExit( - @Advice.Argument(0) String subject, - @Advice.Return boolean compatible, - @Advice.Thrown Throwable throwable) { - - if (throwable != null) { - SchemaRegistryMetrics.recordCompatibilityCheck(subject, false, throwable.getMessage()); - } else { - SchemaRegistryMetrics.recordCompatibilityCheck( - subject, compatible, compatible ? null : "Schema is not compatible"); - } - } - } - - public static class GetSchemaByIdAdvice { - @Advice.OnMethodExit(onThrowable = Throwable.class, suppress = Throwable.class) - public static void onExit( - @Advice.Argument(0) int schemaId, - @Advice.Return Object schema, - @Advice.Thrown Throwable throwable) { - - if (throwable == null && schema != null) { - String schemaType = schema.getClass().getSimpleName(); - SchemaRegistryMetrics.recordSchemaRetrieval(schemaId, schemaType); - } - } - } -} diff --git a/dd-java-agent/instrumentation/confluent-schema-registry/confluent-schema-registry-7.0/src/main/java/datadog/trace/instrumentation/confluentschemaregistry/SchemaRegistryContext.java b/dd-java-agent/instrumentation/confluent-schema-registry/confluent-schema-registry-7.0/src/main/java/datadog/trace/instrumentation/confluentschemaregistry/SchemaRegistryContext.java deleted file mode 100644 index bac93ffc165..00000000000 --- a/dd-java-agent/instrumentation/confluent-schema-registry/confluent-schema-registry-7.0/src/main/java/datadog/trace/instrumentation/confluentschemaregistry/SchemaRegistryContext.java +++ /dev/null @@ -1,70 +0,0 @@ -package datadog.trace.instrumentation.confluentschemaregistry; - -/** - * Thread-local context for passing topic, schema, and cluster information between serialization and - * registry calls. - */ -public class SchemaRegistryContext { - private static final ThreadLocal currentTopic = new ThreadLocal<>(); - private static final ThreadLocal clusterId = new ThreadLocal<>(); - private static final ThreadLocal isKey = new ThreadLocal<>(); - private static final ThreadLocal keySchemaId = new ThreadLocal<>(); - private static final ThreadLocal valueSchemaId = new ThreadLocal<>(); - - public static void setTopic(String topic) { - currentTopic.set(topic); - } - - public static String getTopic() { - return currentTopic.get(); - } - - public static void setClusterId(String cluster) { - clusterId.set(cluster); - } - - public static String getClusterId() { - return clusterId.get(); - } - - public static void setIsKey(boolean key) { - isKey.set(key); - } - - public static Boolean getIsKey() { - return isKey.get(); - } - - public static void setKeySchemaId(Integer schemaId) { - keySchemaId.set(schemaId); - } - - public static Integer getKeySchemaId() { - return keySchemaId.get(); - } - - public static void setValueSchemaId(Integer schemaId) { - valueSchemaId.set(schemaId); - } - - public static Integer getValueSchemaId() { - return valueSchemaId.get(); - } - - public static void clear() { - currentTopic.remove(); - // Don't clear clusterId - it should persist for the entire poll() batch - // clusterId.remove(); - isKey.remove(); - keySchemaId.remove(); - valueSchemaId.remove(); - } - - public static void clearAll() { - currentTopic.remove(); - clusterId.remove(); - isKey.remove(); - keySchemaId.remove(); - valueSchemaId.remove(); - } -} diff --git a/dd-java-agent/instrumentation/confluent-schema-registry/confluent-schema-registry-7.0/src/main/java/datadog/trace/instrumentation/confluentschemaregistry/SchemaRegistryMetrics.java b/dd-java-agent/instrumentation/confluent-schema-registry/confluent-schema-registry-7.0/src/main/java/datadog/trace/instrumentation/confluentschemaregistry/SchemaRegistryMetrics.java deleted file mode 100644 index 260eea52ba4..00000000000 --- a/dd-java-agent/instrumentation/confluent-schema-registry/confluent-schema-registry-7.0/src/main/java/datadog/trace/instrumentation/confluentschemaregistry/SchemaRegistryMetrics.java +++ /dev/null @@ -1,240 +0,0 @@ -package datadog.trace.instrumentation.confluentschemaregistry; - -import datadog.trace.bootstrap.instrumentation.api.AgentTracer; -import java.util.concurrent.ConcurrentHashMap; -import java.util.concurrent.atomic.AtomicLong; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -/** - * Collects and logs metrics about Schema Registry operations including schema registrations, - * compatibility checks, and serialization/deserialization operations. Also reports to Data Streams - * Monitoring. - */ -public class SchemaRegistryMetrics { - private static final Logger log = LoggerFactory.getLogger(SchemaRegistryMetrics.class); - - // Counters for different operations - private static final AtomicLong schemaRegistrationSuccess = new AtomicLong(0); - private static final AtomicLong schemaRegistrationFailure = new AtomicLong(0); - private static final AtomicLong schemaCompatibilitySuccess = new AtomicLong(0); - private static final AtomicLong schemaCompatibilityFailure = new AtomicLong(0); - private static final AtomicLong serializationSuccess = new AtomicLong(0); - private static final AtomicLong serializationFailure = new AtomicLong(0); - private static final AtomicLong deserializationSuccess = new AtomicLong(0); - private static final AtomicLong deserializationFailure = new AtomicLong(0); - - // Track schema IDs per topic for context - private static final ConcurrentHashMap topicKeySchemaIds = - new ConcurrentHashMap<>(); - private static final ConcurrentHashMap topicValueSchemaIds = - new ConcurrentHashMap<>(); - - public static void recordSchemaRegistration( - String subject, int schemaId, boolean isKey, String topic) { - schemaRegistrationSuccess.incrementAndGet(); - if (topic != null) { - if (isKey) { - topicKeySchemaIds.put(topic, schemaId); - } else { - topicValueSchemaIds.put(topic, schemaId); - } - } - log.info( - "[Schema Registry] Schema registered successfully - Subject: {}, Schema ID: {}, Is Key: {}, Topic: {}", - subject, - schemaId, - isKey, - topic); - } - - public static void recordSchemaRegistrationFailure( - String subject, String errorMessage, boolean isKey, String topic) { - schemaRegistrationFailure.incrementAndGet(); - log.error( - "[Schema Registry] Schema registration FAILED - Subject: {}, Is Key: {}, Topic: {}, Error: {}", - subject, - isKey, - topic, - errorMessage); - } - - public static void recordCompatibilityCheck( - String subject, boolean compatible, String errorMessage) { - if (compatible) { - schemaCompatibilitySuccess.incrementAndGet(); - log.info("[Schema Registry] Schema compatibility check PASSED - Subject: {}", subject); - } else { - schemaCompatibilityFailure.incrementAndGet(); - log.error( - "[Schema Registry] Schema compatibility check FAILED - Subject: {}, Error: {}", - subject, - errorMessage); - } - } - - public static void recordSerialization( - String topic, Integer keySchemaId, Integer valueSchemaId, boolean isKey) { - serializationSuccess.incrementAndGet(); - - String clusterId = SchemaRegistryContext.getClusterId(); - log.info("[Schema Registry] DEBUG: Retrieved clusterId from context: '{}'", clusterId); - log.info( - "[Schema Registry] Produce to topic '{}', cluster: {}, schema for key: {}, schema for value: {}, serializing: {}", - topic, - clusterId != null ? clusterId : "unknown", - keySchemaId != null ? keySchemaId : "none", - valueSchemaId != null ? valueSchemaId : "none", - isKey ? "KEY" : "VALUE"); - - // Report to Data Streams Monitoring - Integer schemaId = isKey ? keySchemaId : valueSchemaId; - if (schemaId != null && topic != null) { - try { - AgentTracer.get() - .getDataStreamsMonitoring() - .setSchemaRegistryUsage(topic, clusterId, schemaId, true, isKey); - } catch (Throwable t) { - // Don't fail the application if DSM reporting fails - log.debug("Failed to report schema registry usage to DSM", t); - } - } - } - - public static void recordSerializationFailure(String topic, String errorMessage, boolean isKey) { - serializationFailure.incrementAndGet(); - - String clusterId = SchemaRegistryContext.getClusterId(); - log.error( - "[Schema Registry] Serialization FAILED for topic '{}', cluster: {}, {} - Error: {}", - topic, - clusterId != null ? clusterId : "unknown", - isKey ? "KEY" : "VALUE", - errorMessage); - - // Report to Data Streams Monitoring (use -1 as schema ID for failures) - if (topic != null) { - try { - AgentTracer.get() - .getDataStreamsMonitoring() - .setSchemaRegistryUsage(topic, clusterId, -1, false, isKey); - } catch (Throwable t) { - // Don't fail the application if DSM reporting fails - log.debug("Failed to report schema registry failure to DSM", t); - } - } - } - - public static void recordDeserialization( - String topic, Integer keySchemaId, Integer valueSchemaId, boolean isKey) { - deserializationSuccess.incrementAndGet(); - - String clusterId = SchemaRegistryContext.getClusterId(); - log.info( - "[Schema Registry] Consume from topic '{}', cluster: {}, schema for key: {}, schema for value: {}, deserializing: {}", - topic, - clusterId != null ? clusterId : "unknown", - keySchemaId != null ? keySchemaId : "none", - valueSchemaId != null ? valueSchemaId : "none", - isKey ? "KEY" : "VALUE"); - - // Report to Data Streams Monitoring - Integer schemaId = isKey ? keySchemaId : valueSchemaId; - if (schemaId != null && topic != null) { - try { - AgentTracer.get() - .getDataStreamsMonitoring() - .setSchemaRegistryUsage(topic, clusterId, schemaId, true, isKey); - } catch (Throwable t) { - // Don't fail the application if DSM reporting fails - log.debug("Failed to report schema registry usage to DSM", t); - } - } - } - - public static void recordDeserializationFailure( - String topic, String errorMessage, boolean isKey) { - deserializationFailure.incrementAndGet(); - - String clusterId = SchemaRegistryContext.getClusterId(); - log.error( - "[Schema Registry] Deserialization FAILED for topic '{}', cluster: {}, {} - Error: {}", - topic, - clusterId != null ? clusterId : "unknown", - isKey ? "KEY" : "VALUE", - errorMessage); - - // Report to Data Streams Monitoring (use -1 as schema ID for failures) - if (topic != null) { - try { - AgentTracer.get() - .getDataStreamsMonitoring() - .setSchemaRegistryUsage(topic, clusterId, -1, false, isKey); - } catch (Throwable t) { - // Don't fail the application if DSM reporting fails - log.debug("Failed to report schema registry failure to DSM", t); - } - } - } - - public static void recordSchemaRetrieval(int schemaId, String schemaType) { - log.debug( - "[Schema Registry] Retrieved schema from registry - Schema ID: {}, Type: {}", - schemaId, - schemaType); - } - - // Methods to get current counts for metrics reporting - public static long getSchemaRegistrationSuccessCount() { - return schemaRegistrationSuccess.get(); - } - - public static long getSchemaRegistrationFailureCount() { - return schemaRegistrationFailure.get(); - } - - public static long getSchemaCompatibilitySuccessCount() { - return schemaCompatibilitySuccess.get(); - } - - public static long getSchemaCompatibilityFailureCount() { - return schemaCompatibilityFailure.get(); - } - - public static long getSerializationSuccessCount() { - return serializationSuccess.get(); - } - - public static long getSerializationFailureCount() { - return serializationFailure.get(); - } - - public static long getDeserializationSuccessCount() { - return deserializationSuccess.get(); - } - - public static long getDeserializationFailureCount() { - return deserializationFailure.get(); - } - - public static void printSummary() { - log.info("========== Schema Registry Metrics Summary =========="); - log.info( - "Schema Registrations - Success: {}, Failure: {}", - schemaRegistrationSuccess.get(), - schemaRegistrationFailure.get()); - log.info( - "Compatibility Checks - Success: {}, Failure: {}", - schemaCompatibilitySuccess.get(), - schemaCompatibilityFailure.get()); - log.info( - "Serializations - Success: {}, Failure: {}", - serializationSuccess.get(), - serializationFailure.get()); - log.info( - "Deserializations - Success: {}, Failure: {}", - deserializationSuccess.get(), - deserializationFailure.get()); - log.info("====================================================="); - } -} diff --git a/dd-java-agent/instrumentation/kafka/kafka-clients-0.11/src/main/java/datadog/trace/instrumentation/kafka_clients/KafkaConsumerInfoInstrumentation.java b/dd-java-agent/instrumentation/kafka/kafka-clients-0.11/src/main/java/datadog/trace/instrumentation/kafka_clients/KafkaConsumerInfoInstrumentation.java index 50b1ad03c08..c4bc7858b6f 100644 --- a/dd-java-agent/instrumentation/kafka/kafka-clients-0.11/src/main/java/datadog/trace/instrumentation/kafka_clients/KafkaConsumerInfoInstrumentation.java +++ b/dd-java-agent/instrumentation/kafka/kafka-clients-0.11/src/main/java/datadog/trace/instrumentation/kafka_clients/KafkaConsumerInfoInstrumentation.java @@ -205,40 +205,6 @@ public static void muzzleCheck(ConsumerRecord record) { public static class RecordsAdvice { @Advice.OnMethodEnter(suppress = Throwable.class) public static AgentScope onEnter(@Advice.This KafkaConsumer consumer) { - // Set cluster ID in SchemaRegistryContext before deserialization - // Use reflection to avoid compile-time dependency on the schema registry module - KafkaConsumerInfo kafkaConsumerInfo = - InstrumentationContext.get(KafkaConsumer.class, KafkaConsumerInfo.class).get(consumer); - System.out.println("[DEBUG Consumer] kafkaConsumerInfo: " + kafkaConsumerInfo); - if (kafkaConsumerInfo != null) { - // Extract cluster ID directly inline to avoid muzzle issues - String clusterId = null; - if (Config.get().isDataStreamsEnabled()) { - Metadata consumerMetadata = kafkaConsumerInfo.getClientMetadata(); - System.out.println("[DEBUG Consumer] metadata: " + consumerMetadata); - if (consumerMetadata != null) { - clusterId = - InstrumentationContext.get(Metadata.class, String.class).get(consumerMetadata); - } - } - System.out.println("[DEBUG Consumer] Extracted clusterId: " + clusterId); - if (clusterId != null) { - try { - Class contextClass = - Class.forName( - "datadog.trace.instrumentation.confluentschemaregistry.SchemaRegistryContext", - false, - consumer.getClass().getClassLoader()); - contextClass.getMethod("setClusterId", String.class).invoke(null, clusterId); - System.out.println( - "[DEBUG Consumer] Set clusterId in SchemaRegistryContext: " + clusterId); - } catch (Throwable t) { - // Ignore if SchemaRegistryContext is not available - System.out.println("[DEBUG Consumer] Failed to set clusterId: " + t.getMessage()); - } - } - } - if (traceConfig().isDataStreamsEnabled()) { final AgentSpan span = startSpan(KAFKA_POLL); return activateSpan(span); diff --git a/dd-java-agent/instrumentation/kafka/kafka-clients-0.11/src/main/java/datadog/trace/instrumentation/kafka_clients/KafkaProducerInstrumentation.java b/dd-java-agent/instrumentation/kafka/kafka-clients-0.11/src/main/java/datadog/trace/instrumentation/kafka_clients/KafkaProducerInstrumentation.java index 5e2d0c7a2fa..ba66a01b6da 100644 --- a/dd-java-agent/instrumentation/kafka/kafka-clients-0.11/src/main/java/datadog/trace/instrumentation/kafka_clients/KafkaProducerInstrumentation.java +++ b/dd-java-agent/instrumentation/kafka/kafka-clients-0.11/src/main/java/datadog/trace/instrumentation/kafka_clients/KafkaProducerInstrumentation.java @@ -118,21 +118,6 @@ public static AgentScope onEnter( @Advice.Argument(value = 1, readOnly = false) Callback callback) { String clusterId = InstrumentationContext.get(Metadata.class, String.class).get(metadata); - // Set cluster ID in SchemaRegistryContext for Schema Registry instrumentation - // Use reflection to avoid compile-time dependency on the schema registry module - if (clusterId != null) { - try { - Class contextClass = - Class.forName( - "datadog.trace.instrumentation.confluentschemaregistry.SchemaRegistryContext", - false, - metadata.getClass().getClassLoader()); - contextClass.getMethod("setClusterId", String.class).invoke(null, clusterId); - } catch (Throwable ignored) { - // Ignore if SchemaRegistryContext is not available - } - } - final AgentSpan parent = activeSpan(); final AgentSpan span = startSpan(KAFKA_PRODUCE); PRODUCER_DECORATE.afterStart(span); diff --git a/internal-api/src/main/java/datadog/trace/api/datastreams/SchemaRegistryUsage.java b/internal-api/src/main/java/datadog/trace/api/datastreams/SchemaRegistryUsage.java index 2ea7ad7d0da..08fe5d61fe9 100644 --- a/internal-api/src/main/java/datadog/trace/api/datastreams/SchemaRegistryUsage.java +++ b/internal-api/src/main/java/datadog/trace/api/datastreams/SchemaRegistryUsage.java @@ -58,3 +58,4 @@ public String getServiceNameOverride() { return serviceNameOverride; } } + From 7d26fe3d0c73e3d7b57651b6308366c62a9184f5 Mon Sep 17 00:00:00 2001 From: Piotr Wolski Date: Mon, 17 Nov 2025 14:01:36 -0700 Subject: [PATCH 04/12] fix deserialize instrumentation --- .../KafkaDeserializerInstrumentation.java | 61 +++++++++++++------ .../KafkaSerializerInstrumentation.java | 58 +++++++++++++----- .../api/datastreams/SchemaRegistryUsage.java | 1 - 3 files changed, 87 insertions(+), 33 deletions(-) diff --git a/dd-java-agent/instrumentation/confluent-schema-registry/confluent-schema-registry-7.0/src/main/java/datadog/trace/instrumentation/confluentschemaregistry/KafkaDeserializerInstrumentation.java b/dd-java-agent/instrumentation/confluent-schema-registry/confluent-schema-registry-7.0/src/main/java/datadog/trace/instrumentation/confluentschemaregistry/KafkaDeserializerInstrumentation.java index 91ff69128d6..85d8ff5ee5f 100644 --- a/dd-java-agent/instrumentation/confluent-schema-registry/confluent-schema-registry-7.0/src/main/java/datadog/trace/instrumentation/confluentschemaregistry/KafkaDeserializerInstrumentation.java +++ b/dd-java-agent/instrumentation/confluent-schema-registry/confluent-schema-registry-7.0/src/main/java/datadog/trace/instrumentation/confluentschemaregistry/KafkaDeserializerInstrumentation.java @@ -8,16 +8,24 @@ import datadog.trace.agent.tooling.Instrumenter; import datadog.trace.agent.tooling.Instrumenter.MethodTransformer; +import datadog.trace.agent.tooling.InstrumenterModule; +import datadog.trace.bootstrap.InstrumentationContext; import datadog.trace.bootstrap.instrumentation.api.AgentTracer; +import java.util.HashMap; +import java.util.Map; import net.bytebuddy.asm.Advice; /** * Instruments Confluent Schema Registry deserializers (Avro, Protobuf, and JSON) to capture * deserialization operations. */ -public class KafkaDeserializerInstrumentation +public class KafkaDeserializerInstrumentation extends InstrumenterModule.Tracing implements Instrumenter.ForKnownTypes, Instrumenter.HasMethodAdvice { + public KafkaDeserializerInstrumentation() { + super("confluent-schema-registry", "kafka"); + } + @Override public String[] knownMatchingTypes() { return new String[] { @@ -27,8 +35,25 @@ public String[] knownMatchingTypes() { }; } + @Override + public Map contextStore() { + Map contextStores = new HashMap<>(); + contextStores.put( + "org.apache.kafka.common.serialization.Deserializer", Boolean.class.getName()); + return contextStores; + } + @Override public void methodAdvice(MethodTransformer transformer) { + // Instrument configure to capture isKey value + transformer.applyAdvice( + isMethod() + .and(named("configure")) + .and(isPublic()) + .and(takesArguments(2)) + .and(takesArgument(1, boolean.class)), + getClass().getName() + "$ConfigureAdvice"); + // Instrument deserialize(String topic, Headers headers, byte[] data) // The 2-arg version calls this one, so we only need to instrument this to avoid duplicates transformer.applyAdvice( @@ -41,6 +66,18 @@ public void methodAdvice(MethodTransformer transformer) { getClass().getName() + "$DeserializeAdvice"); } + public static class ConfigureAdvice { + @Advice.OnMethodExit(suppress = Throwable.class) + public static void onExit( + @Advice.This org.apache.kafka.common.serialization.Deserializer deserializer, + @Advice.Argument(1) boolean isKey) { + // Store the isKey value in InstrumentationContext for later use + InstrumentationContext.get( + org.apache.kafka.common.serialization.Deserializer.class, Boolean.class) + .put(deserializer, isKey); + } + } + public static class DeserializeAdvice { @Advice.OnMethodExit(onThrowable = Throwable.class, suppress = Throwable.class) public static void onExit( @@ -50,21 +87,12 @@ public static void onExit( @Advice.Return Object result, @Advice.Thrown Throwable throwable) { - // Get isKey from the deserializer object - boolean isKey = false; - if (deserializer instanceof io.confluent.kafka.serializers.KafkaAvroDeserializer) { - isKey = ((io.confluent.kafka.serializers.KafkaAvroDeserializer) deserializer).isKey(); - } else if (deserializer - instanceof io.confluent.kafka.serializers.json.KafkaJsonSchemaDeserializer) { - isKey = - ((io.confluent.kafka.serializers.json.KafkaJsonSchemaDeserializer) deserializer) - .isKey(); - } else if (deserializer - instanceof io.confluent.kafka.serializers.protobuf.KafkaProtobufDeserializer) { - isKey = - ((io.confluent.kafka.serializers.protobuf.KafkaProtobufDeserializer) deserializer) - .isKey(); - } + // Get isKey from InstrumentationContext (stored during configure) + Boolean isKeyObj = + InstrumentationContext.get( + org.apache.kafka.common.serialization.Deserializer.class, Boolean.class) + .get(deserializer); + boolean isKey = isKeyObj != null ? isKeyObj : false; boolean isError = throwable != null; int schemaId = -1; @@ -90,4 +118,3 @@ public static void onExit( } } } - diff --git a/dd-java-agent/instrumentation/confluent-schema-registry/confluent-schema-registry-7.0/src/main/java/datadog/trace/instrumentation/confluentschemaregistry/KafkaSerializerInstrumentation.java b/dd-java-agent/instrumentation/confluent-schema-registry/confluent-schema-registry-7.0/src/main/java/datadog/trace/instrumentation/confluentschemaregistry/KafkaSerializerInstrumentation.java index 5795fd0e3de..a201e35c5d3 100644 --- a/dd-java-agent/instrumentation/confluent-schema-registry/confluent-schema-registry-7.0/src/main/java/datadog/trace/instrumentation/confluentschemaregistry/KafkaSerializerInstrumentation.java +++ b/dd-java-agent/instrumentation/confluent-schema-registry/confluent-schema-registry-7.0/src/main/java/datadog/trace/instrumentation/confluentschemaregistry/KafkaSerializerInstrumentation.java @@ -9,16 +9,24 @@ import datadog.trace.agent.tooling.Instrumenter; import datadog.trace.agent.tooling.Instrumenter.MethodTransformer; +import datadog.trace.agent.tooling.InstrumenterModule; +import datadog.trace.bootstrap.InstrumentationContext; import datadog.trace.bootstrap.instrumentation.api.AgentTracer; +import java.util.HashMap; +import java.util.Map; import net.bytebuddy.asm.Advice; /** * Instruments Confluent Schema Registry serializers (Avro, Protobuf, and JSON) to capture * serialization operations. */ -public class KafkaSerializerInstrumentation +public class KafkaSerializerInstrumentation extends InstrumenterModule.Tracing implements Instrumenter.ForKnownTypes, Instrumenter.HasMethodAdvice { + public KafkaSerializerInstrumentation() { + super("confluent-schema-registry", "kafka"); + } + @Override public String[] knownMatchingTypes() { return new String[] { @@ -28,8 +36,24 @@ public String[] knownMatchingTypes() { }; } + @Override + public Map contextStore() { + Map contextStores = new HashMap<>(); + contextStores.put("org.apache.kafka.common.serialization.Serializer", Boolean.class.getName()); + return contextStores; + } + @Override public void methodAdvice(MethodTransformer transformer) { + // Instrument configure to capture isKey value + transformer.applyAdvice( + isMethod() + .and(named("configure")) + .and(isPublic()) + .and(takesArguments(2)) + .and(takesArgument(1, boolean.class)), + getClass().getName() + "$ConfigureAdvice"); + // Instrument both serialize(String topic, Object data) // and serialize(String topic, Headers headers, Object data) for Kafka 2.1+ transformer.applyAdvice( @@ -41,6 +65,18 @@ public void methodAdvice(MethodTransformer transformer) { getClass().getName() + "$SerializeAdvice"); } + public static class ConfigureAdvice { + @Advice.OnMethodExit(suppress = Throwable.class) + public static void onExit( + @Advice.This org.apache.kafka.common.serialization.Serializer serializer, + @Advice.Argument(1) boolean isKey) { + // Store the isKey value in InstrumentationContext for later use + InstrumentationContext.get( + org.apache.kafka.common.serialization.Serializer.class, Boolean.class) + .put(serializer, isKey); + } + } + public static class SerializeAdvice { @Advice.OnMethodExit(onThrowable = Throwable.class, suppress = Throwable.class) public static void onExit( @@ -49,19 +85,12 @@ public static void onExit( @Advice.Return byte[] result, @Advice.Thrown Throwable throwable) { - // Get isKey from the serializer object - boolean isKey = false; - if (serializer instanceof io.confluent.kafka.serializers.KafkaAvroSerializer) { - isKey = ((io.confluent.kafka.serializers.KafkaAvroSerializer) serializer).isKey(); - } else if (serializer - instanceof io.confluent.kafka.serializer KafkaJsonSchemaSerializer) { - isKey = - ((io.confluent.kafka.serializers.KafkaJsonSchemaSerializer) serializer).isKey(); - } else if (serializer - instanceof io.confluent.kafka.serializers.protobuf.KafkaProtobufSerializer) { - isKey = - ((io.confluent.kafka.serializers.protobuf.KafkaProtobufSerializer) serializer).isKey(); - } + // Get isKey from InstrumentationContext (stored during configure) + Boolean isKeyObj = + InstrumentationContext.get( + org.apache.kafka.common.serialization.Serializer.class, Boolean.class) + .get(serializer); + boolean isKey = isKeyObj != null ? isKeyObj : false; boolean isError = throwable != null; int schemaId = -1; @@ -87,4 +116,3 @@ public static void onExit( } } } - diff --git a/internal-api/src/main/java/datadog/trace/api/datastreams/SchemaRegistryUsage.java b/internal-api/src/main/java/datadog/trace/api/datastreams/SchemaRegistryUsage.java index 08fe5d61fe9..2ea7ad7d0da 100644 --- a/internal-api/src/main/java/datadog/trace/api/datastreams/SchemaRegistryUsage.java +++ b/internal-api/src/main/java/datadog/trace/api/datastreams/SchemaRegistryUsage.java @@ -58,4 +58,3 @@ public String getServiceNameOverride() { return serviceNameOverride; } } - From 867cfa9652d7d3e30c72f35c648f9619b6d6e1c0 Mon Sep 17 00:00:00 2001 From: Piotr Wolski Date: Mon, 17 Nov 2025 16:05:21 -0700 Subject: [PATCH 05/12] Add back Kafka cluster ID tagging --- .../confluent-schema-registry-7.0/README.md | 74 ----- .../ClusterIdHolder.java | 22 ++ .../KafkaDeserializerInstrumentation.java | 16 +- .../KafkaSerializerInstrumentation.java | 16 +- ...fluentSchemaRegistryDataStreamsTest.groovy | 263 +++--------------- .../groovy/ConfluentSchemaRegistryTest.groovy | 204 -------------- .../KafkaConsumerInfoInstrumentation.java | 35 +++ .../KafkaProducerInstrumentation.java | 26 ++ .../DefaultDataStreamsMonitoring.java | 13 +- .../MsgPackDatastreamsPayloadWriter.java | 10 +- .../trace/core/datastreams/StatsBucket.java | 24 +- .../AgentDataStreamsMonitoring.java | 8 +- .../NoopDataStreamsMonitoring.java | 7 +- .../api/datastreams/SchemaRegistryUsage.java | 7 + 14 files changed, 211 insertions(+), 514 deletions(-) delete mode 100644 dd-java-agent/instrumentation/confluent-schema-registry/confluent-schema-registry-7.0/README.md create mode 100644 dd-java-agent/instrumentation/confluent-schema-registry/confluent-schema-registry-7.0/src/main/java/datadog/trace/instrumentation/confluentschemaregistry/ClusterIdHolder.java delete mode 100644 dd-java-agent/instrumentation/confluent-schema-registry/confluent-schema-registry-7.0/src/test/groovy/ConfluentSchemaRegistryTest.groovy diff --git a/dd-java-agent/instrumentation/confluent-schema-registry/confluent-schema-registry-7.0/README.md b/dd-java-agent/instrumentation/confluent-schema-registry/confluent-schema-registry-7.0/README.md deleted file mode 100644 index 0e36cb4c9b1..00000000000 --- a/dd-java-agent/instrumentation/confluent-schema-registry/confluent-schema-registry-7.0/README.md +++ /dev/null @@ -1,74 +0,0 @@ -# Confluent Schema Registry Instrumentation - -This instrumentation module provides Data Streams Monitoring (DSM) support for Confluent Schema Registry operations in Kafka applications. - -## Features - -This instrumentation captures schema registry usage for both serialization and deserialization operations, reporting them to Datadog's Data Streams Monitoring. - -### Producer Operations -- **Serialization**: Tracks message serialization with: - - Topic name - - Schema ID (extracted from serialized bytes) - - Key vs value serialization - - Success/failure status - -### Consumer Operations -- **Deserialization**: Tracks message deserialization with: - - Topic name - - Schema ID (extracted from Confluent wire format) - - Key vs value deserialization - - Success/failure status - -## Supported Serialization Formats - -- **Avro** (via `KafkaAvroSerializer`/`KafkaAvroDeserializer`) -- **JSON Schema** (via `KafkaJsonSchemaSerializer`/`KafkaJsonSchemaDeserializer`) -- **Protobuf** (via `KafkaProtobufSerializer`/`KafkaProtobufDeserializer`) - -## Implementation Details - -### Instrumented Classes - -1. **KafkaAvroSerializer** - Avro message serialization -2. **KafkaJsonSchemaSerializer** - JSON Schema message serialization -3. **KafkaProtobufSerializer** - Protobuf message serialization -4. **KafkaAvroDeserializer** - Avro message deserialization -5. **KafkaJsonSchemaDeserializer** - JSON Schema message deserialization -6. **KafkaProtobufDeserializer** - Protobuf message deserialization - -### Schema ID Extraction - -Schema IDs are extracted directly from the Confluent wire format: -- **Format**: `[magic_byte][4-byte schema id][data]` -- The magic byte (0x00) indicates Confluent wire format -- Schema ID is a 4-byte big-endian integer - -### Key vs Value Detection - -The instrumentation determines whether a serializer/deserializer is for keys or values by calling the `isKey()` method available on all Confluent serializers/deserializers. - -## Usage - -This instrumentation is automatically activated when: -1. Confluent Schema Registry client (version 7.0.0+) is present on the classpath -2. The Datadog Java agent is attached to the JVM -3. Data Streams Monitoring is enabled - -No configuration or code changes are required. - -## Data Streams Monitoring Integration - -Schema registry usage is reported directly to Datadog's Data Streams Monitoring via: - -```java -AgentTracer.get() - .getDataStreamsMonitoring() - .setSchemaRegistryUsage(topic, clusterId, schemaId, isError, isKey); -``` - -This allows tracking of: -- Schema usage patterns across topics -- Schema registry operation success rates -- Key vs value schema usage -- Schema evolution over time diff --git a/dd-java-agent/instrumentation/confluent-schema-registry/confluent-schema-registry-7.0/src/main/java/datadog/trace/instrumentation/confluentschemaregistry/ClusterIdHolder.java b/dd-java-agent/instrumentation/confluent-schema-registry/confluent-schema-registry-7.0/src/main/java/datadog/trace/instrumentation/confluentschemaregistry/ClusterIdHolder.java new file mode 100644 index 00000000000..8a5e27f2f76 --- /dev/null +++ b/dd-java-agent/instrumentation/confluent-schema-registry/confluent-schema-registry-7.0/src/main/java/datadog/trace/instrumentation/confluentschemaregistry/ClusterIdHolder.java @@ -0,0 +1,22 @@ +package datadog.trace.instrumentation.confluentschemaregistry; + +/** + * Thread-local holder for Kafka cluster ID to be used during schema registry operations. The Kafka + * producer/consumer instrumentation sets this before serialization/deserialization, and the schema + * registry serializer/deserializer instrumentation reads it. + */ +public class ClusterIdHolder { + private static final ThreadLocal CLUSTER_ID = new ThreadLocal<>(); + + public static void set(String clusterId) { + CLUSTER_ID.set(clusterId); + } + + public static String get() { + return CLUSTER_ID.get(); + } + + public static void clear() { + CLUSTER_ID.remove(); + } +} diff --git a/dd-java-agent/instrumentation/confluent-schema-registry/confluent-schema-registry-7.0/src/main/java/datadog/trace/instrumentation/confluentschemaregistry/KafkaDeserializerInstrumentation.java b/dd-java-agent/instrumentation/confluent-schema-registry/confluent-schema-registry-7.0/src/main/java/datadog/trace/instrumentation/confluentschemaregistry/KafkaDeserializerInstrumentation.java index 85d8ff5ee5f..ee91aa8bdca 100644 --- a/dd-java-agent/instrumentation/confluent-schema-registry/confluent-schema-registry-7.0/src/main/java/datadog/trace/instrumentation/confluentschemaregistry/KafkaDeserializerInstrumentation.java +++ b/dd-java-agent/instrumentation/confluent-schema-registry/confluent-schema-registry-7.0/src/main/java/datadog/trace/instrumentation/confluentschemaregistry/KafkaDeserializerInstrumentation.java @@ -6,6 +6,7 @@ import static net.bytebuddy.matcher.ElementMatchers.takesArgument; import static net.bytebuddy.matcher.ElementMatchers.takesArguments; +import com.google.auto.service.AutoService; import datadog.trace.agent.tooling.Instrumenter; import datadog.trace.agent.tooling.Instrumenter.MethodTransformer; import datadog.trace.agent.tooling.InstrumenterModule; @@ -19,6 +20,7 @@ * Instruments Confluent Schema Registry deserializers (Avro, Protobuf, and JSON) to capture * deserialization operations. */ +@AutoService(InstrumenterModule.class) public class KafkaDeserializerInstrumentation extends InstrumenterModule.Tracing implements Instrumenter.ForKnownTypes, Instrumenter.HasMethodAdvice { @@ -35,6 +37,11 @@ public String[] knownMatchingTypes() { }; } + @Override + public String[] helperClassNames() { + return new String[] {"datadog.trace.instrumentation.confluentschemaregistry.ClusterIdHolder"}; + } + @Override public Map contextStore() { Map contextStores = new HashMap<>(); @@ -94,11 +101,14 @@ public static void onExit( .get(deserializer); boolean isKey = isKeyObj != null ? isKeyObj : false; - boolean isError = throwable != null; + // Get cluster ID from thread-local (set by Kafka consumer instrumentation) + String clusterId = ClusterIdHolder.get(); + + boolean isSuccess = throwable == null; int schemaId = -1; // Extract schema ID from the input bytes if successful - if (!isError && data != null && data.length >= 5 && data[0] == 0) { + if (isSuccess && data != null && data.length >= 5 && data[0] == 0) { try { // Confluent wire format: [magic_byte][4-byte schema id][data] schemaId = @@ -114,7 +124,7 @@ public static void onExit( // Record the schema registry usage AgentTracer.get() .getDataStreamsMonitoring() - .setSchemaRegistryUsage(topic, null, schemaId, isError, isKey); + .setSchemaRegistryUsage(topic, clusterId, schemaId, isSuccess, isKey, "deserialize"); } } } diff --git a/dd-java-agent/instrumentation/confluent-schema-registry/confluent-schema-registry-7.0/src/main/java/datadog/trace/instrumentation/confluentschemaregistry/KafkaSerializerInstrumentation.java b/dd-java-agent/instrumentation/confluent-schema-registry/confluent-schema-registry-7.0/src/main/java/datadog/trace/instrumentation/confluentschemaregistry/KafkaSerializerInstrumentation.java index a201e35c5d3..1643c1ce20c 100644 --- a/dd-java-agent/instrumentation/confluent-schema-registry/confluent-schema-registry-7.0/src/main/java/datadog/trace/instrumentation/confluentschemaregistry/KafkaSerializerInstrumentation.java +++ b/dd-java-agent/instrumentation/confluent-schema-registry/confluent-schema-registry-7.0/src/main/java/datadog/trace/instrumentation/confluentschemaregistry/KafkaSerializerInstrumentation.java @@ -7,6 +7,7 @@ import static net.bytebuddy.matcher.ElementMatchers.takesArgument; import static net.bytebuddy.matcher.ElementMatchers.takesArguments; +import com.google.auto.service.AutoService; import datadog.trace.agent.tooling.Instrumenter; import datadog.trace.agent.tooling.Instrumenter.MethodTransformer; import datadog.trace.agent.tooling.InstrumenterModule; @@ -20,6 +21,7 @@ * Instruments Confluent Schema Registry serializers (Avro, Protobuf, and JSON) to capture * serialization operations. */ +@AutoService(InstrumenterModule.class) public class KafkaSerializerInstrumentation extends InstrumenterModule.Tracing implements Instrumenter.ForKnownTypes, Instrumenter.HasMethodAdvice { @@ -36,6 +38,11 @@ public String[] knownMatchingTypes() { }; } + @Override + public String[] helperClassNames() { + return new String[] {"datadog.trace.instrumentation.confluentschemaregistry.ClusterIdHolder"}; + } + @Override public Map contextStore() { Map contextStores = new HashMap<>(); @@ -92,11 +99,14 @@ public static void onExit( .get(serializer); boolean isKey = isKeyObj != null ? isKeyObj : false; - boolean isError = throwable != null; + // Get cluster ID from thread-local (set by Kafka producer instrumentation) + String clusterId = ClusterIdHolder.get(); + + boolean isSuccess = throwable == null; int schemaId = -1; // Extract schema ID from the serialized bytes if successful - if (!isError && result != null && result.length >= 5 && result[0] == 0) { + if (isSuccess && result != null && result.length >= 5 && result[0] == 0) { try { // Confluent wire format: [magic_byte][4-byte schema id][data] schemaId = @@ -112,7 +122,7 @@ public static void onExit( // Record the schema registry usage AgentTracer.get() .getDataStreamsMonitoring() - .setSchemaRegistryUsage(topic, null, schemaId, isError, isKey); + .setSchemaRegistryUsage(topic, clusterId, schemaId, isSuccess, isKey, "serialize"); } } } diff --git a/dd-java-agent/instrumentation/confluent-schema-registry/confluent-schema-registry-7.0/src/test/groovy/ConfluentSchemaRegistryDataStreamsTest.groovy b/dd-java-agent/instrumentation/confluent-schema-registry/confluent-schema-registry-7.0/src/test/groovy/ConfluentSchemaRegistryDataStreamsTest.groovy index cf031318cc8..7c6e24e0717 100644 --- a/dd-java-agent/instrumentation/confluent-schema-registry/confluent-schema-registry-7.0/src/test/groovy/ConfluentSchemaRegistryDataStreamsTest.groovy +++ b/dd-java-agent/instrumentation/confluent-schema-registry/confluent-schema-registry-7.0/src/test/groovy/ConfluentSchemaRegistryDataStreamsTest.groovy @@ -1,18 +1,18 @@ import datadog.trace.agent.test.InstrumentationSpecification +import io.confluent.kafka.serializers.KafkaAvroDeserializer import io.confluent.kafka.serializers.KafkaAvroSerializer import io.confluent.kafka.schemaregistry.client.MockSchemaRegistryClient import io.confluent.kafka.schemaregistry.client.SchemaRegistryClient import org.apache.avro.Schema import org.apache.avro.generic.GenericData import org.apache.avro.generic.GenericRecord -import org.apache.kafka.common.serialization.Serializer import spock.lang.Shared import java.util.concurrent.TimeUnit /** - * Tests that Schema Registry usage is tracked in Data Streams Monitoring. - * Tests both successful and unsuccessful serialization/deserialization operations. + * Tests that Schema Registry usage is tracked in Data Streams Monitoring + * for both serialization and deserialization operations. */ class ConfluentSchemaRegistryDataStreamsTest extends InstrumentationSpecification { @Shared @@ -45,250 +45,77 @@ class ConfluentSchemaRegistryDataStreamsTest extends InstrumentationSpecificatio """) } - def "test successful producer serialization tracks schema registry usage"() { + def "test schema registry tracks both serialize and deserialize operations"() { setup: - def topicName = "test-topic-producer" - def testClusterId = "test-cluster-producer" - def serializer = new KafkaAvroSerializer(schemaRegistryClient) + def topicName = "test-topic" + def testClusterId = "test-cluster" def config = [ "schema.registry.url": "mock://test-url", "auto.register.schemas": "true" ] + + // Create serializer and deserializer + def serializer = new KafkaAvroSerializer(schemaRegistryClient) serializer.configure(config, false) // false = value serializer + def deserializer = new KafkaAvroDeserializer(schemaRegistryClient) + deserializer.configure(config, false) // false = value deserializer + // Create a test record GenericRecord record = new GenericData.Record(testSchema) record.put("field1", "test-value") record.put("field2", 42) - when: "we serialize a message with cluster ID set" - datadog.trace.instrumentation.confluentschemaregistry.SchemaRegistryContext.setClusterId(testClusterId) + when: "we produce a message (serialize)" + datadog.trace.instrumentation.confluentschemaregistry.ClusterIdHolder.set(testClusterId) byte[] serialized = serializer.serialize(topicName, record) - and: "we wait for DSM to flush" - Thread.sleep(1200) // Wait for bucket duration + buffer - TEST_DATA_STREAMS_MONITORING.report() - TEST_DATA_STREAMS_WRITER.waitForSchemaRegistryUsages(1, TimeUnit.SECONDS.toMillis(5)) - - then: "the serialization was successful" - serialized != null - serialized.length > 0 - - and: "schema registry usage was tracked" - def usages = TEST_DATA_STREAMS_WRITER.schemaRegistryUsages - usages.size() >= 1 - - and: "the usage contains the correct information" - def usage = usages.find { u -> u.topic == topicName } - usage != null - usage.schemaId > 0 // Valid schema ID - usage.isSuccess() == true // Successful operation - usage.isKey() == false // Value serializer - usage.clusterId == testClusterId // Cluster ID is included - - cleanup: - serializer.close() - datadog.trace.instrumentation.confluentschemaregistry.SchemaRegistryContext.clearAll() - } - - def "test successful producer with key and value serializers"() { - setup: - def topicName = "test-topic-key-value" - def testClusterId = "test-cluster-key-value" - def keySerializer = new KafkaAvroSerializer(schemaRegistryClient) - def valueSerializer = new KafkaAvroSerializer(schemaRegistryClient) - def config = [ - "schema.registry.url": "mock://test-url", - "auto.register.schemas": "true" - ] - keySerializer.configure(config, true) // true = key serializer - valueSerializer.configure(config, false) // false = value serializer - - // Create test records - GenericRecord keyRecord = new GenericData.Record(testSchema) - keyRecord.put("field1", "key-value") - keyRecord.put("field2", 1) - - GenericRecord valueRecord = new GenericData.Record(testSchema) - valueRecord.put("field1", "value-value") - valueRecord.put("field2", 2) - - when: "we serialize both key and value with cluster ID" - datadog.trace.instrumentation.confluentschemaregistry.SchemaRegistryContext.setClusterId(testClusterId) - byte[] keyBytes = keySerializer.serialize(topicName, keyRecord) - datadog.trace.instrumentation.confluentschemaregistry.SchemaRegistryContext.setClusterId(testClusterId) - byte[] valueBytes = valueSerializer.serialize(topicName, valueRecord) + and: "we consume the message (deserialize)" + datadog.trace.instrumentation.confluentschemaregistry.ClusterIdHolder.set(testClusterId) + def deserialized = deserializer.deserialize(topicName, serialized) and: "we wait for DSM to flush" Thread.sleep(1200) // Wait for bucket duration + buffer TEST_DATA_STREAMS_MONITORING.report() TEST_DATA_STREAMS_WRITER.waitForSchemaRegistryUsages(2, TimeUnit.SECONDS.toMillis(5)) - then: "both serializations were successful" - keyBytes != null && keyBytes.length > 0 - valueBytes != null && valueBytes.length > 0 + then: "the message was serialized and deserialized successfully" + serialized != null + serialized.length > 0 + deserialized != null + deserialized.get("field1").toString() == "test-value" + deserialized.get("field2") == 42 - and: "both usages were tracked" + and: "two schema registry usages were tracked" def usages = TEST_DATA_STREAMS_WRITER.schemaRegistryUsages usages.size() >= 2 - and: "we have both key and value tracked" - def keyUsage = usages.find { u -> u.isKey() && u.topic == topicName } - def valueUsage = usages.find { u -> !u.isKey() && u.topic == topicName } - - keyUsage != null - keyUsage.schemaId > 0 - keyUsage.isSuccess() == true - keyUsage.clusterId == testClusterId - - valueUsage != null - valueUsage.schemaId > 0 - valueUsage.isSuccess() == true - valueUsage.clusterId == testClusterId - - cleanup: - keySerializer.close() - valueSerializer.close() - datadog.trace.instrumentation.confluentschemaregistry.SchemaRegistryContext.clearAll() - } - - def "test serialization failure is tracked"() { - setup: - def topicName = "test-topic-failure" - - // Create a custom serializer that will fail - Serializer failingSerializer = new KafkaAvroSerializer(schemaRegistryClient) { - @Override - byte[] serialize(String topic, Object data) { - throw new RuntimeException("Intentional serialization failure") - } - } - def config = [ - "schema.registry.url": "mock://test-url" - ] - failingSerializer.configure(config, false) - - GenericRecord record = new GenericData.Record(testSchema) - record.put("field1", "test") - record.put("field2", 123) - - when: "we try to serialize and it fails" - try { - failingSerializer.serialize(topicName, record) - } catch (RuntimeException e) { - // Expected - } - - and: "we wait for DSM to flush" - Thread.sleep(1200) - TEST_DATA_STREAMS_MONITORING.report() - TEST_DATA_STREAMS_WRITER.waitForSchemaRegistryUsages(1, TimeUnit.SECONDS.toMillis(5)) - - then: "the failure was tracked" - def usages = TEST_DATA_STREAMS_WRITER.schemaRegistryUsages - def failureUsage = usages.find { u -> u.topic == topicName } - - failureUsage != null - failureUsage.schemaId == -1 // Failure indicator - failureUsage.isSuccess() == false - - cleanup: - failingSerializer.close() - } - - def "test schema IDs are correctly extracted from serialized messages"() { - setup: - def topicName = "test-topic-schema-id" - def testClusterId = "test-cluster-schema-id" - def serializer = new KafkaAvroSerializer(schemaRegistryClient) - def config = [ - "schema.registry.url": "mock://test-url", - "auto.register.schemas": "true" - ] - serializer.configure(config, false) - - // Create multiple records with the same schema - def records = (1..3).collect { i -> - GenericRecord record = new GenericData.Record(testSchema) - record.put("field1", "value-$i") - record.put("field2", i) - record + and: "one is a serialize operation" + def serializeUsage = usages.find { u -> + u.topic == topicName && u.operation == "serialize" } - - when: "we serialize multiple messages with cluster ID" - datadog.trace.instrumentation.confluentschemaregistry.SchemaRegistryContext.setClusterId(testClusterId) - def serializedMessages = records.collect { record -> - serializer.serialize(topicName, record) + serializeUsage != null + serializeUsage.schemaId > 0 // Valid schema ID + serializeUsage.isSuccess() == true + serializeUsage.isKey() == false + serializeUsage.clusterId == testClusterId + + and: "one is a deserialize operation" + def deserializeUsage = usages.find { u -> + u.topic == topicName && u.operation == "deserialize" } + deserializeUsage != null + deserializeUsage.schemaId > 0 // Valid schema ID + deserializeUsage.isSuccess() == true + deserializeUsage.isKey() == false + deserializeUsage.clusterId == testClusterId - and: "we wait for DSM to flush" - Thread.sleep(1200) - TEST_DATA_STREAMS_MONITORING.report() - TEST_DATA_STREAMS_WRITER.waitForSchemaRegistryUsages(3, TimeUnit.SECONDS.toMillis(5)) - - then: "all messages were serialized" - serializedMessages.every { m -> m != null && m.length > 0 } - - and: "all usages have the same schema ID (cached schema)" - def usages = TEST_DATA_STREAMS_WRITER.schemaRegistryUsages - .findAll { u -> u.topic == topicName } - usages.size() >= 3 - - def schemaIds = usages.collect { u -> u.schemaId }.unique() - schemaIds.size() == 1 // All use the same schema ID - schemaIds[0] > 0 // Valid schema ID - - and: "cluster ID is present" - usages.every { u -> u.clusterId == testClusterId } + and: "both operations used the same schema ID" + serializeUsage.schemaId == deserializeUsage.schemaId cleanup: serializer.close() - datadog.trace.instrumentation.confluentschemaregistry.SchemaRegistryContext.clearAll() - } - - def "test schema registry usage metrics are aggregated by topic"() { - setup: - def topic1 = "test-topic-1" - def topic2 = "test-topic-2" - def testClusterId = "test-cluster-multi-topic" - def serializer = new KafkaAvroSerializer(schemaRegistryClient) - def config = [ - "schema.registry.url": "mock://test-url", - "auto.register.schemas": "true" - ] - serializer.configure(config, false) - - GenericRecord record = new GenericData.Record(testSchema) - record.put("field1", "test") - record.put("field2", 1) - - when: "we produce to multiple topics with cluster ID" - datadog.trace.instrumentation.confluentschemaregistry.SchemaRegistryContext.setClusterId(testClusterId) - serializer.serialize(topic1, record) - datadog.trace.instrumentation.confluentschemaregistry.SchemaRegistryContext.setClusterId(testClusterId) - serializer.serialize(topic2, record) - datadog.trace.instrumentation.confluentschemaregistry.SchemaRegistryContext.setClusterId(testClusterId) - serializer.serialize(topic1, record) // Second message to topic1 - - and: "we wait for DSM to flush" - Thread.sleep(1200) - TEST_DATA_STREAMS_MONITORING.report() - TEST_DATA_STREAMS_WRITER.waitForSchemaRegistryUsages(3, TimeUnit.SECONDS.toMillis(5)) - - then: "usages are tracked per topic" - def usages = TEST_DATA_STREAMS_WRITER.schemaRegistryUsages - def topic1Usages = usages.findAll { u -> u.topic == topic1 } - def topic2Usages = usages.findAll { u -> u.topic == topic2 } - - topic1Usages.size() >= 2 - topic2Usages.size() >= 1 - - and: "all are successful" - usages.every { u -> u.isSuccess() } - - cleanup: - serializer.close() - datadog.trace.instrumentation.confluentschemaregistry.SchemaRegistryContext.clearAll() + deserializer.close() + datadog.trace.instrumentation.confluentschemaregistry.ClusterIdHolder.clear() } } - diff --git a/dd-java-agent/instrumentation/confluent-schema-registry/confluent-schema-registry-7.0/src/test/groovy/ConfluentSchemaRegistryTest.groovy b/dd-java-agent/instrumentation/confluent-schema-registry/confluent-schema-registry-7.0/src/test/groovy/ConfluentSchemaRegistryTest.groovy deleted file mode 100644 index 061931b99a5..00000000000 --- a/dd-java-agent/instrumentation/confluent-schema-registry/confluent-schema-registry-7.0/src/test/groovy/ConfluentSchemaRegistryTest.groovy +++ /dev/null @@ -1,204 +0,0 @@ -import spock.lang.Specification -import io.confluent.kafka.schemaregistry.client.MockSchemaRegistryClient -import io.confluent.kafka.schemaregistry.client.SchemaRegistryClient -import io.confluent.kafka.serializers.KafkaAvroDeserializer -import io.confluent.kafka.serializers.KafkaAvroSerializer -import org.apache.avro.Schema -import org.apache.avro.generic.GenericData -import org.apache.avro.generic.GenericRecord - -/** - * Simple test to verify Schema Registry serializer/deserializer behavior. - * This test verifies that: - * 1. Confluent wire format includes schema ID in bytes - * 2. Schema ID can be extracted from serialized data - * - * To test with instrumentation, run your app with the agent and check logs. - */ -class ConfluentSchemaRegistryTest extends Specification { - - def "test schema ID is extracted from serialized bytes"() { - setup: - SchemaRegistryClient schemaRegistry = new MockSchemaRegistryClient() - String topic = "test-topic" - - // Define a simple Avro schema - String schemaStr = ''' - { - "type": "record", - "name": "TestRecord", - "fields": [ - {"name": "id", "type": "string"}, - {"name": "value", "type": "int"} - ] - } - ''' - Schema schema = new Schema.Parser().parse(schemaStr) - - // Configure serializer - def props = [ - "schema.registry.url": "mock://test" - ] - KafkaAvroSerializer serializer = new KafkaAvroSerializer(schemaRegistry, props) - - // Create a record - GenericRecord record = new GenericData.Record(schema) - record.put("id", "test-id-123") - record.put("value", 42) - - when: - byte[] serialized = serializer.serialize(topic, record) - - then: - // Verify serialization succeeded - serialized != null - serialized.length > 5 - - // Verify Confluent wire format: magic byte (0x00) + 4-byte schema ID - serialized[0] == 0 - - // Extract schema ID from bytes (big-endian) - int schemaId = ((serialized[1] & 0xFF) << 24) | - ((serialized[2] & 0xFF) << 16) | - ((serialized[3] & 0xFF) << 8) | - (serialized[4] & 0xFF) - - println "\n========== Confluent Wire Format Test ==========" - println "Topic: ${topic}" - println "Schema ID from bytes: ${schemaId}" - def first10 = serialized.length >= 10 ? serialized[0..9] : serialized - println "First 10 bytes: ${first10.collect { String.format('%02X', it & 0xFF) }.join(' ')}" - println "This is the schema ID our instrumentation should extract!" - println "================================================\n" - - // Verify schema ID is positive (valid) - schemaId > 0 - - cleanup: - serializer?.close() - } - - def "test deserialization captures schema ID"() { - setup: - SchemaRegistryClient schemaRegistry = new MockSchemaRegistryClient() - String topic = "test-topic" - - String schemaStr = ''' - { - "type": "record", - "name": "TestRecord", - "fields": [ - {"name": "name", "type": "string"} - ] - } - ''' - Schema schema = new Schema.Parser().parse(schemaStr) - - def props = ["schema.registry.url": "mock://test"] - KafkaAvroSerializer serializer = new KafkaAvroSerializer(schemaRegistry, props) - KafkaAvroDeserializer deserializer = new KafkaAvroDeserializer(schemaRegistry, props) - - GenericRecord record = new GenericData.Record(schema) - record.put("name", "TestName") - - when: - byte[] serialized = serializer.serialize(topic, record) - Object deserialized = deserializer.deserialize(topic, serialized) - - then: - deserialized != null - deserialized instanceof GenericRecord - ((GenericRecord) deserialized).get("name").toString() == "TestName" - - println "\n========== Deserialization Test ==========" - println "Topic: ${topic}" - println "Deserialized record: ${deserialized}" - println "=========================================\n" - - cleanup: - serializer?.close() - deserializer?.close() - } - - def "test schema registration is tracked"() { - setup: - SchemaRegistryClient schemaRegistry = new MockSchemaRegistryClient() - String subject = "test-subject" - String schemaStr = ''' - { - "type": "record", - "name": "User", - "fields": [ - {"name": "username", "type": "string"} - ] - } - ''' - Schema schema = new Schema.Parser().parse(schemaStr) - - when: - int schemaId = schemaRegistry.register(subject, schema) - - then: - schemaId > 0 - - println "\n========== Schema Registration Test ==========" - println "Subject: ${subject}" - println "Registered schema ID: ${schemaId}" - println "============================================\n" - } - - def "test end-to-end with real KafkaAvroSerializer"() { - setup: - SchemaRegistryClient schemaRegistry = new MockSchemaRegistryClient() - String topic = "products" - - String productSchema = ''' - { - "type": "record", - "name": "Product", - "namespace": "com.example", - "fields": [ - {"name": "id", "type": "string"}, - {"name": "name", "type": "string"}, - {"name": "price", "type": "double"} - ] - } - ''' - Schema schema = new Schema.Parser().parse(productSchema) - - def props = ["schema.registry.url": "mock://test"] - KafkaAvroSerializer serializer = new KafkaAvroSerializer(schemaRegistry, props) - - GenericRecord product = new GenericData.Record(schema) - product.put("id", "PROD-001") - product.put("name", "Test Product") - product.put("price", 29.99) - - when: - byte[] serialized = serializer.serialize(topic, product) - - then: - serialized != null - println "\n========== End-to-End Test ==========" - println "Topic: ${topic}" - println "Serialized ${serialized.length} bytes" - def first10 = serialized.length >= 10 ? serialized[0..9] : serialized - println "First 10 bytes: ${first10.collect { String.format('%02X', it & 0xFF) }.join(' ')}" - - // Extract and print schema ID - if (serialized.length >= 5 && serialized[0] == 0) { - int schemaId = ((serialized[1] & 0xFF) << 24) | - ((serialized[2] & 0xFF) << 16) | - ((serialized[3] & 0xFF) << 8) | - (serialized[4] & 0xFF) - println "Schema ID from wire format: ${schemaId}" - println "\nWhen running with DD agent, you should see:" - println "[Schema Registry] Produce to topic 'products', schema for key: none, schema for value: ${schemaId}, serializing: VALUE" - } - println "======================================\n" - - cleanup: - serializer?.close() - } -} - diff --git a/dd-java-agent/instrumentation/kafka/kafka-clients-0.11/src/main/java/datadog/trace/instrumentation/kafka_clients/KafkaConsumerInfoInstrumentation.java b/dd-java-agent/instrumentation/kafka/kafka-clients-0.11/src/main/java/datadog/trace/instrumentation/kafka_clients/KafkaConsumerInfoInstrumentation.java index c4bc7858b6f..2854b58c1f7 100644 --- a/dd-java-agent/instrumentation/kafka/kafka-clients-0.11/src/main/java/datadog/trace/instrumentation/kafka_clients/KafkaConsumerInfoInstrumentation.java +++ b/dd-java-agent/instrumentation/kafka/kafka-clients-0.11/src/main/java/datadog/trace/instrumentation/kafka_clients/KafkaConsumerInfoInstrumentation.java @@ -205,6 +205,29 @@ public static void muzzleCheck(ConsumerRecord record) { public static class RecordsAdvice { @Advice.OnMethodEnter(suppress = Throwable.class) public static AgentScope onEnter(@Advice.This KafkaConsumer consumer) { + // Set cluster ID in ClusterIdHolder for Schema Registry instrumentation + KafkaConsumerInfo kafkaConsumerInfo = + InstrumentationContext.get(KafkaConsumer.class, KafkaConsumerInfo.class).get(consumer); + if (kafkaConsumerInfo != null && Config.get().isDataStreamsEnabled()) { + Metadata consumerMetadata = kafkaConsumerInfo.getClientMetadata(); + if (consumerMetadata != null) { + String clusterId = + InstrumentationContext.get(Metadata.class, String.class).get(consumerMetadata); + if (clusterId != null) { + try { + Class holderClass = + Class.forName( + "datadog.trace.instrumentation.confluentschemaregistry.ClusterIdHolder", + false, + consumer.getClass().getClassLoader()); + holderClass.getMethod("set", String.class).invoke(null, clusterId); + } catch (Throwable ignored) { + // Ignore if Schema Registry instrumentation is not available + } + } + } + } + if (traceConfig().isDataStreamsEnabled()) { final AgentSpan span = startSpan(KAFKA_POLL); return activateSpan(span); @@ -227,6 +250,18 @@ public static void captureGroup( } recordsCount = records.count(); } + // Clear cluster ID from Schema Registry instrumentation + try { + Class holderClass = + Class.forName( + "datadog.trace.instrumentation.confluentschemaregistry.ClusterIdHolder", + false, + consumer.getClass().getClassLoader()); + holderClass.getMethod("clear").invoke(null); + } catch (Throwable ignored) { + // Ignore if Schema Registry instrumentation is not available + } + if (scope == null) { return; } diff --git a/dd-java-agent/instrumentation/kafka/kafka-clients-0.11/src/main/java/datadog/trace/instrumentation/kafka_clients/KafkaProducerInstrumentation.java b/dd-java-agent/instrumentation/kafka/kafka-clients-0.11/src/main/java/datadog/trace/instrumentation/kafka_clients/KafkaProducerInstrumentation.java index ba66a01b6da..8888b3d1bf2 100644 --- a/dd-java-agent/instrumentation/kafka/kafka-clients-0.11/src/main/java/datadog/trace/instrumentation/kafka_clients/KafkaProducerInstrumentation.java +++ b/dd-java-agent/instrumentation/kafka/kafka-clients-0.11/src/main/java/datadog/trace/instrumentation/kafka_clients/KafkaProducerInstrumentation.java @@ -118,6 +118,20 @@ public static AgentScope onEnter( @Advice.Argument(value = 1, readOnly = false) Callback callback) { String clusterId = InstrumentationContext.get(Metadata.class, String.class).get(metadata); + // Set cluster ID for Schema Registry instrumentation + if (clusterId != null) { + try { + Class holderClass = + Class.forName( + "datadog.trace.instrumentation.confluentschemaregistry.ClusterIdHolder", + false, + metadata.getClass().getClassLoader()); + holderClass.getMethod("set", String.class).invoke(null, clusterId); + } catch (Throwable ignored) { + // Ignore if Schema Registry instrumentation is not available + } + } + final AgentSpan parent = activeSpan(); final AgentSpan span = startSpan(KAFKA_PRODUCE); PRODUCER_DECORATE.afterStart(span); @@ -184,6 +198,18 @@ record = @Advice.OnMethodExit(onThrowable = Throwable.class, suppress = Throwable.class) public static void stopSpan( @Advice.Enter final AgentScope scope, @Advice.Thrown final Throwable throwable) { + // Clear cluster ID from Schema Registry instrumentation + try { + Class holderClass = + Class.forName( + "datadog.trace.instrumentation.confluentschemaregistry.ClusterIdHolder", + false, + scope.getClass().getClassLoader()); + holderClass.getMethod("clear").invoke(null); + } catch (Throwable ignored) { + // Ignore if Schema Registry instrumentation is not available + } + PRODUCER_DECORATE.onError(scope, throwable); PRODUCER_DECORATE.beforeFinish(scope); scope.close(); diff --git a/dd-trace-core/src/main/java/datadog/trace/core/datastreams/DefaultDataStreamsMonitoring.java b/dd-trace-core/src/main/java/datadog/trace/core/datastreams/DefaultDataStreamsMonitoring.java index 5151e9fbb4b..199c0e834ac 100644 --- a/dd-trace-core/src/main/java/datadog/trace/core/datastreams/DefaultDataStreamsMonitoring.java +++ b/dd-trace-core/src/main/java/datadog/trace/core/datastreams/DefaultDataStreamsMonitoring.java @@ -220,14 +220,20 @@ public void trackBacklog(DataStreamsTags tags, long value) { @Override public void setSchemaRegistryUsage( - String topic, String clusterId, int schemaId, boolean isSuccess, boolean isKey) { + String topic, + String clusterId, + int schemaId, + boolean isSuccess, + boolean isKey, + String operation) { log.info( - "[DSM Schema Registry] Recording usage: topic={}, clusterId={}, schemaId={}, success={}, isKey={}", + "[DSM Schema Registry] Recording usage: topic={}, clusterId={}, schemaId={}, success={}, isKey={}, operation={}", topic, clusterId, schemaId, isSuccess, - isKey); + isKey, + operation); inbox.offer( new SchemaRegistryUsage( @@ -236,6 +242,7 @@ public void setSchemaRegistryUsage( schemaId, isSuccess, isKey, + operation, timeSource.getCurrentTimeNanos(), getThreadServiceName())); } diff --git a/dd-trace-core/src/main/java/datadog/trace/core/datastreams/MsgPackDatastreamsPayloadWriter.java b/dd-trace-core/src/main/java/datadog/trace/core/datastreams/MsgPackDatastreamsPayloadWriter.java index 40797d83d57..e27149353b8 100644 --- a/dd-trace-core/src/main/java/datadog/trace/core/datastreams/MsgPackDatastreamsPayloadWriter.java +++ b/dd-trace-core/src/main/java/datadog/trace/core/datastreams/MsgPackDatastreamsPayloadWriter.java @@ -45,6 +45,7 @@ public class MsgPackDatastreamsPayloadWriter implements DatastreamsPayloadWriter private static final byte[] COUNT = "Count".getBytes(ISO_8859_1); private static final byte[] IS_SUCCESS = "IsSuccess".getBytes(ISO_8859_1); private static final byte[] IS_KEY = "IsKey".getBytes(ISO_8859_1); + private static final byte[] OPERATION = "Operation".getBytes(ISO_8859_1); private static final byte[] PRODUCTS_MASK = "ProductMask".getBytes(ISO_8859_1); private static final byte[] PROCESS_TAGS = "ProcessTags".getBytes(ISO_8859_1); @@ -240,15 +241,17 @@ private void writeSchemaRegistryUsages( StatsBucket.SchemaRegistryCount value = entry.getValue(); log.info( - "[DSM Schema Registry] Flushing entry: topic={}, clusterId={}, schemaId={}, success={}, isKey={}, count={}", + "[DSM Schema Registry] Flushing entry: topic={}, clusterId={}, schemaId={}, success={}, isKey={}, operation={}, count={}", key.getTopic(), key.getClusterId(), key.getSchemaId(), key.isSuccess(), key.isKey(), + key.getOperation(), value.getCount()); - packer.startMap(6); // 6 fields: Topic, KafkaClusterId, SchemaId, IsSuccess, IsKey, Count + packer.startMap( + 7); // 7 fields: Topic, KafkaClusterId, SchemaId, IsSuccess, IsKey, Operation, Count packer.writeUTF8(TOPIC); packer.writeString(key.getTopic() != null ? key.getTopic() : "", null); @@ -265,6 +268,9 @@ private void writeSchemaRegistryUsages( packer.writeUTF8(IS_KEY); packer.writeBoolean(key.isKey()); + packer.writeUTF8(OPERATION); + packer.writeString(key.getOperation() != null ? key.getOperation() : "", null); + packer.writeUTF8(COUNT); packer.writeLong(value.getCount()); } diff --git a/dd-trace-core/src/main/java/datadog/trace/core/datastreams/StatsBucket.java b/dd-trace-core/src/main/java/datadog/trace/core/datastreams/StatsBucket.java index 22e66f9a39e..71fc4af1ba0 100644 --- a/dd-trace-core/src/main/java/datadog/trace/core/datastreams/StatsBucket.java +++ b/dd-trace-core/src/main/java/datadog/trace/core/datastreams/StatsBucket.java @@ -49,7 +49,8 @@ public void addSchemaRegistryUsage(SchemaRegistryUsage usage) { usage.getClusterId(), usage.getSchemaId(), usage.isSuccess(), - usage.isKey()); + usage.isKey(), + usage.getOperation()); schemaRegistryUsages.compute( key, (k, v) -> (v == null) ? new SchemaRegistryCount(1) : v.increment()); } @@ -75,8 +76,8 @@ public Collection> getSchemaRe } /** - * Key for aggregating schema registry usage by topic, cluster, schema ID, success, and key/value - * type. + * Key for aggregating schema registry usage by topic, cluster, schema ID, success, key/value + * type, and operation. */ public static class SchemaRegistryKey { private final String topic; @@ -84,14 +85,21 @@ public static class SchemaRegistryKey { private final int schemaId; private final boolean isSuccess; private final boolean isKey; + private final String operation; public SchemaRegistryKey( - String topic, String clusterId, int schemaId, boolean isSuccess, boolean isKey) { + String topic, + String clusterId, + int schemaId, + boolean isSuccess, + boolean isKey, + String operation) { this.topic = topic; this.clusterId = clusterId; this.schemaId = schemaId; this.isSuccess = isSuccess; this.isKey = isKey; + this.operation = operation; } public String getTopic() { @@ -114,6 +122,10 @@ public boolean isKey() { return isKey; } + public String getOperation() { + return operation; + } + @Override public boolean equals(Object o) { if (this == o) return true; @@ -123,7 +135,8 @@ public boolean equals(Object o) { && isSuccess == that.isSuccess && isKey == that.isKey && java.util.Objects.equals(topic, that.topic) - && java.util.Objects.equals(clusterId, that.clusterId); + && java.util.Objects.equals(clusterId, that.clusterId) + && java.util.Objects.equals(operation, that.operation); } @Override @@ -133,6 +146,7 @@ public int hashCode() { result = 31 * result + schemaId; result = 31 * result + (isSuccess ? 1 : 0); result = 31 * result + (isKey ? 1 : 0); + result = 31 * result + (operation != null ? operation.hashCode() : 0); return result; } } diff --git a/internal-api/src/main/java/datadog/trace/api/datastreams/AgentDataStreamsMonitoring.java b/internal-api/src/main/java/datadog/trace/api/datastreams/AgentDataStreamsMonitoring.java index 225eefdc21c..9c2651f0e9c 100644 --- a/internal-api/src/main/java/datadog/trace/api/datastreams/AgentDataStreamsMonitoring.java +++ b/internal-api/src/main/java/datadog/trace/api/datastreams/AgentDataStreamsMonitoring.java @@ -16,9 +16,15 @@ public interface AgentDataStreamsMonitoring extends DataStreamsCheckpointer { * @param schemaId Schema ID from Schema Registry * @param isSuccess Whether the schema operation succeeded * @param isKey Whether this is for the key (true) or value (false) + * @param operation The operation type: "serialize" or "deserialize" */ void setSchemaRegistryUsage( - String topic, String clusterId, int schemaId, boolean isSuccess, boolean isKey); + String topic, + String clusterId, + int schemaId, + boolean isSuccess, + boolean isKey, + String operation); /** * Sets data streams checkpoint, used for both produce and consume operations. diff --git a/internal-api/src/main/java/datadog/trace/api/datastreams/NoopDataStreamsMonitoring.java b/internal-api/src/main/java/datadog/trace/api/datastreams/NoopDataStreamsMonitoring.java index 47109dee4c3..8ba4cda65d2 100644 --- a/internal-api/src/main/java/datadog/trace/api/datastreams/NoopDataStreamsMonitoring.java +++ b/internal-api/src/main/java/datadog/trace/api/datastreams/NoopDataStreamsMonitoring.java @@ -13,7 +13,12 @@ public void trackBacklog(DataStreamsTags tags, long value) {} @Override public void setSchemaRegistryUsage( - String topic, String clusterId, int schemaId, boolean isSuccess, boolean isKey) {} + String topic, + String clusterId, + int schemaId, + boolean isSuccess, + boolean isKey, + String operation) {} @Override public void setCheckpoint(AgentSpan span, DataStreamsContext context) {} diff --git a/internal-api/src/main/java/datadog/trace/api/datastreams/SchemaRegistryUsage.java b/internal-api/src/main/java/datadog/trace/api/datastreams/SchemaRegistryUsage.java index 2ea7ad7d0da..814f8a05f09 100644 --- a/internal-api/src/main/java/datadog/trace/api/datastreams/SchemaRegistryUsage.java +++ b/internal-api/src/main/java/datadog/trace/api/datastreams/SchemaRegistryUsage.java @@ -10,6 +10,7 @@ public class SchemaRegistryUsage implements InboxItem { private final int schemaId; private final boolean isSuccess; private final boolean isKey; + private final String operation; private final long timestampNanos; private final String serviceNameOverride; @@ -19,6 +20,7 @@ public SchemaRegistryUsage( int schemaId, boolean isSuccess, boolean isKey, + String operation, long timestampNanos, String serviceNameOverride) { this.topic = topic; @@ -26,6 +28,7 @@ public SchemaRegistryUsage( this.schemaId = schemaId; this.isSuccess = isSuccess; this.isKey = isKey; + this.operation = operation; this.timestampNanos = timestampNanos; this.serviceNameOverride = serviceNameOverride; } @@ -50,6 +53,10 @@ public boolean isKey() { return isKey; } + public String getOperation() { + return operation; + } + public long getTimestampNanos() { return timestampNanos; } From 1043c9803b77b8a61a4633a8819d5caa4e611943 Mon Sep 17 00:00:00 2001 From: Piotr Wolski Date: Mon, 17 Nov 2025 19:47:32 -0700 Subject: [PATCH 06/12] Make jacoco pass --- .../RecordingDatastreamsPayloadWriter.groovy | 6 +- .../MsgPackDatastreamsPayloadWriter.java | 28 +--- .../trace/core/datastreams/StatsBucket.java | 35 +---- .../DefaultDataStreamsMonitoringTest.groovy | 146 ++++++++++++++++++ 4 files changed, 162 insertions(+), 53 deletions(-) diff --git a/dd-java-agent/instrumentation-testing/src/main/groovy/datadog/trace/agent/test/datastreams/RecordingDatastreamsPayloadWriter.groovy b/dd-java-agent/instrumentation-testing/src/main/groovy/datadog/trace/agent/test/datastreams/RecordingDatastreamsPayloadWriter.groovy index 154c99edd50..9feb48f3a98 100644 --- a/dd-java-agent/instrumentation-testing/src/main/groovy/datadog/trace/agent/test/datastreams/RecordingDatastreamsPayloadWriter.groovy +++ b/dd-java-agent/instrumentation-testing/src/main/groovy/datadog/trace/agent/test/datastreams/RecordingDatastreamsPayloadWriter.groovy @@ -20,7 +20,7 @@ class RecordingDatastreamsPayloadWriter implements DatastreamsPayloadWriter { private final Set backlogs = [] @SuppressWarnings('UnusedPrivateField') - private final List schemaRegistryUsages = [] + private final List schemaRegistryUsages = [] private final Set serviceNameOverrides = [] @@ -37,7 +37,7 @@ class RecordingDatastreamsPayloadWriter implements DatastreamsPayloadWriter { } } if (bucket.schemaRegistryUsages != null) { - for (Map.Entry usage : bucket.schemaRegistryUsages) { + for (Map.Entry usage : bucket.schemaRegistryUsages) { this.@schemaRegistryUsages.add(usage.getKey()) } } @@ -60,7 +60,7 @@ class RecordingDatastreamsPayloadWriter implements DatastreamsPayloadWriter { Collections.unmodifiableList(new ArrayList<>(this.@backlogs)) } - synchronized List getSchemaRegistryUsages() { + synchronized List getSchemaRegistryUsages() { Collections.unmodifiableList(new ArrayList<>(this.@schemaRegistryUsages)) } diff --git a/dd-trace-core/src/main/java/datadog/trace/core/datastreams/MsgPackDatastreamsPayloadWriter.java b/dd-trace-core/src/main/java/datadog/trace/core/datastreams/MsgPackDatastreamsPayloadWriter.java index e27149353b8..fd9972da853 100644 --- a/dd-trace-core/src/main/java/datadog/trace/core/datastreams/MsgPackDatastreamsPayloadWriter.java +++ b/dd-trace-core/src/main/java/datadog/trace/core/datastreams/MsgPackDatastreamsPayloadWriter.java @@ -15,11 +15,8 @@ import java.util.Collection; import java.util.List; import java.util.Map; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; public class MsgPackDatastreamsPayloadWriter implements DatastreamsPayloadWriter { - private static final Logger log = LoggerFactory.getLogger(MsgPackDatastreamsPayloadWriter.class); private static final byte[] ENV = "Env".getBytes(ISO_8859_1); private static final byte[] VERSION = "Version".getBytes(ISO_8859_1); private static final byte[] PRIMARY_TAG = "PrimaryTag".getBytes(ISO_8859_1); @@ -228,27 +225,12 @@ private void writeBacklogs( } private void writeSchemaRegistryUsages( - Collection> usages, - Writable packer) { - if (!usages.isEmpty()) { - log.info("[DSM Schema Registry] Flushing {} schema registry usage entries", usages.size()); - } - + Collection> usages, Writable packer) { packer.writeUTF8(SCHEMA_REGISTRY_USAGES); packer.startArray(usages.size()); - for (Map.Entry entry : usages) { - StatsBucket.SchemaRegistryKey key = entry.getKey(); - StatsBucket.SchemaRegistryCount value = entry.getValue(); - - log.info( - "[DSM Schema Registry] Flushing entry: topic={}, clusterId={}, schemaId={}, success={}, isKey={}, operation={}, count={}", - key.getTopic(), - key.getClusterId(), - key.getSchemaId(), - key.isSuccess(), - key.isKey(), - key.getOperation(), - value.getCount()); + for (Map.Entry entry : usages) { + StatsBucket.SchemaKey key = entry.getKey(); + long count = entry.getValue(); packer.startMap( 7); // 7 fields: Topic, KafkaClusterId, SchemaId, IsSuccess, IsKey, Operation, Count @@ -272,7 +254,7 @@ private void writeSchemaRegistryUsages( packer.writeString(key.getOperation() != null ? key.getOperation() : "", null); packer.writeUTF8(COUNT); - packer.writeLong(value.getCount()); + packer.writeLong(count); } } diff --git a/dd-trace-core/src/main/java/datadog/trace/core/datastreams/StatsBucket.java b/dd-trace-core/src/main/java/datadog/trace/core/datastreams/StatsBucket.java index 71fc4af1ba0..1ddbffd94a4 100644 --- a/dd-trace-core/src/main/java/datadog/trace/core/datastreams/StatsBucket.java +++ b/dd-trace-core/src/main/java/datadog/trace/core/datastreams/StatsBucket.java @@ -13,7 +13,7 @@ public class StatsBucket { private final long bucketDurationNanos; private final Map hashToGroup = new HashMap<>(); private final Map backlogs = new HashMap<>(); - private final Map schemaRegistryUsages = new HashMap<>(); + private final Map schemaRegistryUsages = new HashMap<>(); public StatsBucket(long startTimeNanos, long bucketDurationNanos) { this.startTimeNanos = startTimeNanos; @@ -43,16 +43,15 @@ public void addBacklog(Backlog backlog) { } public void addSchemaRegistryUsage(SchemaRegistryUsage usage) { - SchemaRegistryKey key = - new SchemaRegistryKey( + SchemaKey key = + new SchemaKey( usage.getTopic(), usage.getClusterId(), usage.getSchemaId(), usage.isSuccess(), usage.isKey(), usage.getOperation()); - schemaRegistryUsages.compute( - key, (k, v) -> (v == null) ? new SchemaRegistryCount(1) : v.increment()); + schemaRegistryUsages.merge(key, 1L, Long::sum); } public long getStartTimeNanos() { @@ -71,7 +70,7 @@ public Collection> getBacklogs() { return backlogs.entrySet(); } - public Collection> getSchemaRegistryUsages() { + public Collection> getSchemaRegistryUsages() { return schemaRegistryUsages.entrySet(); } @@ -79,7 +78,7 @@ public Collection> getSchemaRe * Key for aggregating schema registry usage by topic, cluster, schema ID, success, key/value * type, and operation. */ - public static class SchemaRegistryKey { + public static class SchemaKey { private final String topic; private final String clusterId; private final int schemaId; @@ -87,7 +86,7 @@ public static class SchemaRegistryKey { private final boolean isKey; private final String operation; - public SchemaRegistryKey( + public SchemaKey( String topic, String clusterId, int schemaId, @@ -130,7 +129,7 @@ public String getOperation() { public boolean equals(Object o) { if (this == o) return true; if (o == null || getClass() != o.getClass()) return false; - SchemaRegistryKey that = (SchemaRegistryKey) o; + SchemaKey that = (SchemaKey) o; return schemaId == that.schemaId && isSuccess == that.isSuccess && isKey == that.isKey @@ -150,22 +149,4 @@ public int hashCode() { return result; } } - - /** Count of schema registry usages. */ - public static class SchemaRegistryCount { - private long count; - - public SchemaRegistryCount(long count) { - this.count = count; - } - - public SchemaRegistryCount increment() { - count++; - return this; - } - - public long getCount() { - return count; - } - } } diff --git a/dd-trace-core/src/test/groovy/datadog/trace/core/datastreams/DefaultDataStreamsMonitoringTest.groovy b/dd-trace-core/src/test/groovy/datadog/trace/core/datastreams/DefaultDataStreamsMonitoringTest.groovy index 1a2a4c680c8..15123b6bb99 100644 --- a/dd-trace-core/src/test/groovy/datadog/trace/core/datastreams/DefaultDataStreamsMonitoringTest.groovy +++ b/dd-trace-core/src/test/groovy/datadog/trace/core/datastreams/DefaultDataStreamsMonitoringTest.groovy @@ -4,6 +4,7 @@ import datadog.communication.ddagent.DDAgentFeaturesDiscovery import datadog.trace.api.Config import datadog.trace.api.TraceConfig import datadog.trace.api.datastreams.DataStreamsTags +import datadog.trace.api.datastreams.SchemaRegistryUsage import datadog.trace.api.datastreams.StatsPoint import datadog.trace.api.experimental.DataStreamsContextCarrier import datadog.trace.api.time.ControllableTimeSource @@ -872,6 +873,151 @@ class DefaultDataStreamsMonitoringTest extends DDCoreSpecification { payloadWriter.close() dataStreams.close() } + + def "Schema registry usages are aggregated by operation"() { + given: + def conditions = new PollingConditions(timeout: 2) + def features = Stub(DDAgentFeaturesDiscovery) { + supportsDataStreams() >> true + } + def timeSource = new ControllableTimeSource() + def payloadWriter = new CapturingPayloadWriter() + def sink = Mock(Sink) + def traceConfig = Mock(TraceConfig) { + isDataStreamsEnabled() >> true + } + + when: + def dataStreams = new DefaultDataStreamsMonitoring(sink, features, timeSource, { traceConfig }, payloadWriter, DEFAULT_BUCKET_DURATION_NANOS) + dataStreams.start() + + // Record serialize and deserialize operations + dataStreams.setSchemaRegistryUsage("test-topic", "test-cluster", 123, true, false, "serialize") + dataStreams.setSchemaRegistryUsage("test-topic", "test-cluster", 123, true, false, "serialize") // duplicate serialize + dataStreams.setSchemaRegistryUsage("test-topic", "test-cluster", 123, true, false, "deserialize") + dataStreams.setSchemaRegistryUsage("test-topic", "test-cluster", 456, true, true, "serialize") // different schema/key + + timeSource.advance(DEFAULT_BUCKET_DURATION_NANOS) + dataStreams.report() + + then: + conditions.eventually { + assert dataStreams.inbox.isEmpty() + assert dataStreams.thread.state != Thread.State.RUNNABLE + assert payloadWriter.buckets.size() == 1 + } + + with(payloadWriter.buckets.get(0)) { + schemaRegistryUsages.size() == 3 // 3 unique combinations + + // Find serialize operation for schema 123 (should have count 2) + def serializeUsage = schemaRegistryUsages.find { e -> + e.key.schemaId == 123 && e.key.operation == "serialize" && !e.key.isKey + } + serializeUsage != null + serializeUsage.value == 2L // Aggregated 2 serialize operations + + // Find deserialize operation for schema 123 (should have count 1) + def deserializeUsage = schemaRegistryUsages.find { e -> + e.key.schemaId == 123 && e.key.operation == "deserialize" && !e.key.isKey + } + deserializeUsage != null + deserializeUsage.value == 1L + + // Find serialize operation for schema 456 with isKey=true (should have count 1) + def keySerializeUsage = schemaRegistryUsages.find { e -> + e.key.schemaId == 456 && e.key.operation == "serialize" && e.key.isKey + } + keySerializeUsage != null + keySerializeUsage.value == 1L + } + + cleanup: + payloadWriter.close() + dataStreams.close() + } + + def "SchemaKey equals and hashCode work correctly"() { + given: + def key1 = new StatsBucket.SchemaKey("topic1", "cluster1", 123, true, false, "serialize") + def key2 = new StatsBucket.SchemaKey("topic1", "cluster1", 123, true, false, "serialize") + def key3 = new StatsBucket.SchemaKey("topic2", "cluster1", 123, true, false, "serialize") // different topic + def key4 = new StatsBucket.SchemaKey("topic1", "cluster2", 123, true, false, "serialize") // different cluster + def key5 = new StatsBucket.SchemaKey("topic1", "cluster1", 456, true, false, "serialize") // different schema + def key6 = new StatsBucket.SchemaKey("topic1", "cluster1", 123, false, false, "serialize") // different success + def key7 = new StatsBucket.SchemaKey("topic1", "cluster1", 123, true, true, "serialize") // different isKey + def key8 = new StatsBucket.SchemaKey("topic1", "cluster1", 123, true, false, "deserialize") // different operation + + expect: + // Reflexive + key1.equals(key1) + key1.hashCode() == key1.hashCode() + + // Symmetric + key1.equals(key2) + key2.equals(key1) + key1.hashCode() == key2.hashCode() + + // Different topic + !key1.equals(key3) + !key3.equals(key1) + + // Different cluster + !key1.equals(key4) + !key4.equals(key1) + + // Different schema ID + !key1.equals(key5) + !key5.equals(key1) + + // Different success + !key1.equals(key6) + !key6.equals(key1) + + // Different isKey + !key1.equals(key7) + !key7.equals(key1) + + // Different operation + !key1.equals(key8) + !key8.equals(key1) + + // Null check + !key1.equals(null) + + // Different class + !key1.equals("not a schema key") + } + + def "StatsBucket aggregates schema registry usages correctly"() { + given: + def bucket = new StatsBucket(1000L, 10000L) + def usage1 = new SchemaRegistryUsage("topic1", "cluster1", 123, true, false, "serialize", 1000L, null) + def usage2 = new SchemaRegistryUsage("topic1", "cluster1", 123, true, false, "serialize", 2000L, null) + def usage3 = new SchemaRegistryUsage("topic1", "cluster1", 123, true, false, "deserialize", 3000L, null) + + when: + bucket.addSchemaRegistryUsage(usage1) + bucket.addSchemaRegistryUsage(usage2) // should increment count for same key + bucket.addSchemaRegistryUsage(usage3) // different operation, new key + + def usages = bucket.getSchemaRegistryUsages() + def usageMap = usages.collectEntries { [(it.key): it.value] } + + then: + usages.size() == 2 + + // Check serialize count + def serializeKey = new StatsBucket.SchemaKey("topic1", "cluster1", 123, true, false, "serialize") + usageMap[serializeKey] == 2L + + // Check deserialize count + def deserializeKey = new StatsBucket.SchemaKey("topic1", "cluster1", 123, true, false, "deserialize") + usageMap[deserializeKey] == 1L + + // Check that different operations create different keys + serializeKey != deserializeKey + } } class CapturingPayloadWriter implements DatastreamsPayloadWriter { From e45acd901da89fcd3a46821fb5b76a97d41e703f Mon Sep 17 00:00:00 2001 From: Piotr Wolski Date: Mon, 17 Nov 2025 20:53:05 -0700 Subject: [PATCH 07/12] exclude schema usage from test coverage --- internal-api/build.gradle.kts | 1 + 1 file changed, 1 insertion(+) diff --git a/internal-api/build.gradle.kts b/internal-api/build.gradle.kts index f05ab8f0b2a..cac64359668 100644 --- a/internal-api/build.gradle.kts +++ b/internal-api/build.gradle.kts @@ -68,6 +68,7 @@ val excludedClassesCoverage by extra( "datadog.trace.api.datastreams.InboxItem", "datadog.trace.api.datastreams.NoopDataStreamsMonitoring", "datadog.trace.api.datastreams.NoopPathwayContext", + "datadog.trace.api.datastreams.SchemaRegistryUsage", "datadog.trace.api.datastreams.StatsPoint", // Debugger "datadog.trace.api.debugger.DebuggerConfigUpdate", From 324c0b93a0e2aa1f58564006121290cd71dc7ea9 Mon Sep 17 00:00:00 2001 From: Piotr Wolski Date: Tue, 18 Nov 2025 10:08:20 -0700 Subject: [PATCH 08/12] Don't use reflection --- .../build.gradle | 2 ++ .../KafkaDeserializerInstrumentation.java | 22 ++++++++---------- .../KafkaSerializerInstrumentation.java | 21 ++++++++--------- ...fluentSchemaRegistryDataStreamsTest.groovy | 7 +++--- .../KafkaConsumerInfoInstrumentation.java | 23 +++---------------- .../KafkaProducerInstrumentation.java | 23 +++---------------- .../kafka_common}/ClusterIdHolder.java | 2 +- .../DefaultDataStreamsMonitoring.java | 9 -------- .../MsgPackDatastreamsPayloadWriter.java | 8 +++++-- 9 files changed, 37 insertions(+), 80 deletions(-) rename dd-java-agent/instrumentation/{confluent-schema-registry/confluent-schema-registry-7.0/src/main/java/datadog/trace/instrumentation/confluentschemaregistry => kafka/kafka-common/src/main/java/datadog/trace/instrumentation/kafka_common}/ClusterIdHolder.java (90%) diff --git a/dd-java-agent/instrumentation/confluent-schema-registry/confluent-schema-registry-7.0/build.gradle b/dd-java-agent/instrumentation/confluent-schema-registry/confluent-schema-registry-7.0/build.gradle index 89a2b2821f0..8e0a847fb6d 100644 --- a/dd-java-agent/instrumentation/confluent-schema-registry/confluent-schema-registry-7.0/build.gradle +++ b/dd-java-agent/instrumentation/confluent-schema-registry/confluent-schema-registry-7.0/build.gradle @@ -10,11 +10,13 @@ muzzle { } dependencies { + compileOnly project(':dd-java-agent:instrumentation:kafka:kafka-common') compileOnly group: 'io.confluent', name: 'kafka-schema-registry-client', version: '7.0.0' compileOnly group: 'io.confluent', name: 'kafka-avro-serializer', version: '7.0.0' compileOnly group: 'io.confluent', name: 'kafka-protobuf-serializer', version: '7.0.0' compileOnly group: 'org.apache.kafka', name: 'kafka-clients', version: '3.0.0' + testImplementation project(':dd-java-agent:instrumentation:kafka:kafka-common') testImplementation group: 'io.confluent', name: 'kafka-schema-registry-client', version: '7.5.2' testImplementation group: 'io.confluent', name: 'kafka-avro-serializer', version: '7.5.2' testImplementation group: 'io.confluent', name: 'kafka-protobuf-serializer', version: '7.5.1' diff --git a/dd-java-agent/instrumentation/confluent-schema-registry/confluent-schema-registry-7.0/src/main/java/datadog/trace/instrumentation/confluentschemaregistry/KafkaDeserializerInstrumentation.java b/dd-java-agent/instrumentation/confluent-schema-registry/confluent-schema-registry-7.0/src/main/java/datadog/trace/instrumentation/confluentschemaregistry/KafkaDeserializerInstrumentation.java index ee91aa8bdca..6870b2cb5af 100644 --- a/dd-java-agent/instrumentation/confluent-schema-registry/confluent-schema-registry-7.0/src/main/java/datadog/trace/instrumentation/confluentschemaregistry/KafkaDeserializerInstrumentation.java +++ b/dd-java-agent/instrumentation/confluent-schema-registry/confluent-schema-registry-7.0/src/main/java/datadog/trace/instrumentation/confluentschemaregistry/KafkaDeserializerInstrumentation.java @@ -12,9 +12,11 @@ import datadog.trace.agent.tooling.InstrumenterModule; import datadog.trace.bootstrap.InstrumentationContext; import datadog.trace.bootstrap.instrumentation.api.AgentTracer; +import datadog.trace.instrumentation.kafka_common.ClusterIdHolder; import java.util.HashMap; import java.util.Map; import net.bytebuddy.asm.Advice; +import org.apache.kafka.common.serialization.Deserializer; /** * Instruments Confluent Schema Registry deserializers (Avro, Protobuf, and JSON) to capture @@ -39,14 +41,13 @@ public String[] knownMatchingTypes() { @Override public String[] helperClassNames() { - return new String[] {"datadog.trace.instrumentation.confluentschemaregistry.ClusterIdHolder"}; + return new String[] {"datadog.trace.instrumentation.kafka_common.ClusterIdHolder"}; } @Override public Map contextStore() { Map contextStores = new HashMap<>(); - contextStores.put( - "org.apache.kafka.common.serialization.Deserializer", Boolean.class.getName()); + contextStores.put(Deserializer.class.getName(), Boolean.class.getName()); return contextStores; } @@ -76,19 +77,16 @@ public void methodAdvice(MethodTransformer transformer) { public static class ConfigureAdvice { @Advice.OnMethodExit(suppress = Throwable.class) public static void onExit( - @Advice.This org.apache.kafka.common.serialization.Deserializer deserializer, - @Advice.Argument(1) boolean isKey) { + @Advice.This Deserializer deserializer, @Advice.Argument(1) boolean isKey) { // Store the isKey value in InstrumentationContext for later use - InstrumentationContext.get( - org.apache.kafka.common.serialization.Deserializer.class, Boolean.class) - .put(deserializer, isKey); + InstrumentationContext.get(Deserializer.class, Boolean.class).put(deserializer, isKey); } } public static class DeserializeAdvice { @Advice.OnMethodExit(onThrowable = Throwable.class, suppress = Throwable.class) public static void onExit( - @Advice.This org.apache.kafka.common.serialization.Deserializer deserializer, + @Advice.This Deserializer deserializer, @Advice.Argument(0) String topic, @Advice.Argument(2) byte[] data, @Advice.Return Object result, @@ -96,10 +94,8 @@ public static void onExit( // Get isKey from InstrumentationContext (stored during configure) Boolean isKeyObj = - InstrumentationContext.get( - org.apache.kafka.common.serialization.Deserializer.class, Boolean.class) - .get(deserializer); - boolean isKey = isKeyObj != null ? isKeyObj : false; + InstrumentationContext.get(Deserializer.class, Boolean.class).get(deserializer); + boolean isKey = isKeyObj != null && isKeyObj; // Get cluster ID from thread-local (set by Kafka consumer instrumentation) String clusterId = ClusterIdHolder.get(); diff --git a/dd-java-agent/instrumentation/confluent-schema-registry/confluent-schema-registry-7.0/src/main/java/datadog/trace/instrumentation/confluentschemaregistry/KafkaSerializerInstrumentation.java b/dd-java-agent/instrumentation/confluent-schema-registry/confluent-schema-registry-7.0/src/main/java/datadog/trace/instrumentation/confluentschemaregistry/KafkaSerializerInstrumentation.java index 1643c1ce20c..0619071b5a6 100644 --- a/dd-java-agent/instrumentation/confluent-schema-registry/confluent-schema-registry-7.0/src/main/java/datadog/trace/instrumentation/confluentschemaregistry/KafkaSerializerInstrumentation.java +++ b/dd-java-agent/instrumentation/confluent-schema-registry/confluent-schema-registry-7.0/src/main/java/datadog/trace/instrumentation/confluentschemaregistry/KafkaSerializerInstrumentation.java @@ -13,9 +13,11 @@ import datadog.trace.agent.tooling.InstrumenterModule; import datadog.trace.bootstrap.InstrumentationContext; import datadog.trace.bootstrap.instrumentation.api.AgentTracer; +import datadog.trace.instrumentation.kafka_common.ClusterIdHolder; import java.util.HashMap; import java.util.Map; import net.bytebuddy.asm.Advice; +import org.apache.kafka.common.serialization.Serializer; /** * Instruments Confluent Schema Registry serializers (Avro, Protobuf, and JSON) to capture @@ -40,13 +42,13 @@ public String[] knownMatchingTypes() { @Override public String[] helperClassNames() { - return new String[] {"datadog.trace.instrumentation.confluentschemaregistry.ClusterIdHolder"}; + return new String[] {"datadog.trace.instrumentation.kafka_common.ClusterIdHolder"}; } @Override public Map contextStore() { Map contextStores = new HashMap<>(); - contextStores.put("org.apache.kafka.common.serialization.Serializer", Boolean.class.getName()); + contextStores.put(Serializer.class.getName(), Boolean.class.getName()); return contextStores; } @@ -75,29 +77,24 @@ public void methodAdvice(MethodTransformer transformer) { public static class ConfigureAdvice { @Advice.OnMethodExit(suppress = Throwable.class) public static void onExit( - @Advice.This org.apache.kafka.common.serialization.Serializer serializer, - @Advice.Argument(1) boolean isKey) { + @Advice.This Serializer serializer, @Advice.Argument(1) boolean isKey) { // Store the isKey value in InstrumentationContext for later use - InstrumentationContext.get( - org.apache.kafka.common.serialization.Serializer.class, Boolean.class) - .put(serializer, isKey); + InstrumentationContext.get(Serializer.class, Boolean.class).put(serializer, isKey); } } public static class SerializeAdvice { @Advice.OnMethodExit(onThrowable = Throwable.class, suppress = Throwable.class) public static void onExit( - @Advice.This org.apache.kafka.common.serialization.Serializer serializer, + @Advice.This Serializer serializer, @Advice.Argument(0) String topic, @Advice.Return byte[] result, @Advice.Thrown Throwable throwable) { // Get isKey from InstrumentationContext (stored during configure) Boolean isKeyObj = - InstrumentationContext.get( - org.apache.kafka.common.serialization.Serializer.class, Boolean.class) - .get(serializer); - boolean isKey = isKeyObj != null ? isKeyObj : false; + InstrumentationContext.get(Serializer.class, Boolean.class).get(serializer); + boolean isKey = isKeyObj != null && isKeyObj; // Get cluster ID from thread-local (set by Kafka producer instrumentation) String clusterId = ClusterIdHolder.get(); diff --git a/dd-java-agent/instrumentation/confluent-schema-registry/confluent-schema-registry-7.0/src/test/groovy/ConfluentSchemaRegistryDataStreamsTest.groovy b/dd-java-agent/instrumentation/confluent-schema-registry/confluent-schema-registry-7.0/src/test/groovy/ConfluentSchemaRegistryDataStreamsTest.groovy index 7c6e24e0717..25806cc42a4 100644 --- a/dd-java-agent/instrumentation/confluent-schema-registry/confluent-schema-registry-7.0/src/test/groovy/ConfluentSchemaRegistryDataStreamsTest.groovy +++ b/dd-java-agent/instrumentation/confluent-schema-registry/confluent-schema-registry-7.0/src/test/groovy/ConfluentSchemaRegistryDataStreamsTest.groovy @@ -1,4 +1,5 @@ import datadog.trace.agent.test.InstrumentationSpecification +import datadog.trace.instrumentation.kafka_common.ClusterIdHolder import io.confluent.kafka.serializers.KafkaAvroDeserializer import io.confluent.kafka.serializers.KafkaAvroSerializer import io.confluent.kafka.schemaregistry.client.MockSchemaRegistryClient @@ -67,11 +68,11 @@ class ConfluentSchemaRegistryDataStreamsTest extends InstrumentationSpecificatio record.put("field2", 42) when: "we produce a message (serialize)" - datadog.trace.instrumentation.confluentschemaregistry.ClusterIdHolder.set(testClusterId) + ClusterIdHolder.set(testClusterId) byte[] serialized = serializer.serialize(topicName, record) and: "we consume the message (deserialize)" - datadog.trace.instrumentation.confluentschemaregistry.ClusterIdHolder.set(testClusterId) + ClusterIdHolder.set(testClusterId) def deserialized = deserializer.deserialize(topicName, serialized) and: "we wait for DSM to flush" @@ -116,6 +117,6 @@ class ConfluentSchemaRegistryDataStreamsTest extends InstrumentationSpecificatio cleanup: serializer.close() deserializer.close() - datadog.trace.instrumentation.confluentschemaregistry.ClusterIdHolder.clear() + ClusterIdHolder.clear() } } diff --git a/dd-java-agent/instrumentation/kafka/kafka-clients-0.11/src/main/java/datadog/trace/instrumentation/kafka_clients/KafkaConsumerInfoInstrumentation.java b/dd-java-agent/instrumentation/kafka/kafka-clients-0.11/src/main/java/datadog/trace/instrumentation/kafka_clients/KafkaConsumerInfoInstrumentation.java index 2854b58c1f7..f0df784ed34 100644 --- a/dd-java-agent/instrumentation/kafka/kafka-clients-0.11/src/main/java/datadog/trace/instrumentation/kafka_clients/KafkaConsumerInfoInstrumentation.java +++ b/dd-java-agent/instrumentation/kafka/kafka-clients-0.11/src/main/java/datadog/trace/instrumentation/kafka_clients/KafkaConsumerInfoInstrumentation.java @@ -22,6 +22,7 @@ import datadog.trace.bootstrap.InstrumentationContext; import datadog.trace.bootstrap.instrumentation.api.AgentScope; import datadog.trace.bootstrap.instrumentation.api.AgentSpan; +import datadog.trace.instrumentation.kafka_common.ClusterIdHolder; import java.util.HashMap; import java.util.List; import java.util.Map; @@ -214,16 +215,7 @@ public static AgentScope onEnter(@Advice.This KafkaConsumer consumer) { String clusterId = InstrumentationContext.get(Metadata.class, String.class).get(consumerMetadata); if (clusterId != null) { - try { - Class holderClass = - Class.forName( - "datadog.trace.instrumentation.confluentschemaregistry.ClusterIdHolder", - false, - consumer.getClass().getClassLoader()); - holderClass.getMethod("set", String.class).invoke(null, clusterId); - } catch (Throwable ignored) { - // Ignore if Schema Registry instrumentation is not available - } + ClusterIdHolder.set(clusterId); } } } @@ -251,16 +243,7 @@ public static void captureGroup( recordsCount = records.count(); } // Clear cluster ID from Schema Registry instrumentation - try { - Class holderClass = - Class.forName( - "datadog.trace.instrumentation.confluentschemaregistry.ClusterIdHolder", - false, - consumer.getClass().getClassLoader()); - holderClass.getMethod("clear").invoke(null); - } catch (Throwable ignored) { - // Ignore if Schema Registry instrumentation is not available - } + ClusterIdHolder.clear(); if (scope == null) { return; diff --git a/dd-java-agent/instrumentation/kafka/kafka-clients-0.11/src/main/java/datadog/trace/instrumentation/kafka_clients/KafkaProducerInstrumentation.java b/dd-java-agent/instrumentation/kafka/kafka-clients-0.11/src/main/java/datadog/trace/instrumentation/kafka_clients/KafkaProducerInstrumentation.java index 8888b3d1bf2..b1618657106 100644 --- a/dd-java-agent/instrumentation/kafka/kafka-clients-0.11/src/main/java/datadog/trace/instrumentation/kafka_clients/KafkaProducerInstrumentation.java +++ b/dd-java-agent/instrumentation/kafka/kafka-clients-0.11/src/main/java/datadog/trace/instrumentation/kafka_clients/KafkaProducerInstrumentation.java @@ -35,6 +35,7 @@ import datadog.trace.bootstrap.instrumentation.api.AgentSpan; import datadog.trace.bootstrap.instrumentation.api.AgentTracer; import datadog.trace.bootstrap.instrumentation.api.InstrumentationTags; +import datadog.trace.instrumentation.kafka_common.ClusterIdHolder; import java.util.Map; import net.bytebuddy.asm.Advice; import net.bytebuddy.matcher.ElementMatcher; @@ -120,16 +121,7 @@ public static AgentScope onEnter( // Set cluster ID for Schema Registry instrumentation if (clusterId != null) { - try { - Class holderClass = - Class.forName( - "datadog.trace.instrumentation.confluentschemaregistry.ClusterIdHolder", - false, - metadata.getClass().getClassLoader()); - holderClass.getMethod("set", String.class).invoke(null, clusterId); - } catch (Throwable ignored) { - // Ignore if Schema Registry instrumentation is not available - } + ClusterIdHolder.set(clusterId); } final AgentSpan parent = activeSpan(); @@ -199,16 +191,7 @@ record = public static void stopSpan( @Advice.Enter final AgentScope scope, @Advice.Thrown final Throwable throwable) { // Clear cluster ID from Schema Registry instrumentation - try { - Class holderClass = - Class.forName( - "datadog.trace.instrumentation.confluentschemaregistry.ClusterIdHolder", - false, - scope.getClass().getClassLoader()); - holderClass.getMethod("clear").invoke(null); - } catch (Throwable ignored) { - // Ignore if Schema Registry instrumentation is not available - } + ClusterIdHolder.clear(); PRODUCER_DECORATE.onError(scope, throwable); PRODUCER_DECORATE.beforeFinish(scope); diff --git a/dd-java-agent/instrumentation/confluent-schema-registry/confluent-schema-registry-7.0/src/main/java/datadog/trace/instrumentation/confluentschemaregistry/ClusterIdHolder.java b/dd-java-agent/instrumentation/kafka/kafka-common/src/main/java/datadog/trace/instrumentation/kafka_common/ClusterIdHolder.java similarity index 90% rename from dd-java-agent/instrumentation/confluent-schema-registry/confluent-schema-registry-7.0/src/main/java/datadog/trace/instrumentation/confluentschemaregistry/ClusterIdHolder.java rename to dd-java-agent/instrumentation/kafka/kafka-common/src/main/java/datadog/trace/instrumentation/kafka_common/ClusterIdHolder.java index 8a5e27f2f76..aeb0eee55c4 100644 --- a/dd-java-agent/instrumentation/confluent-schema-registry/confluent-schema-registry-7.0/src/main/java/datadog/trace/instrumentation/confluentschemaregistry/ClusterIdHolder.java +++ b/dd-java-agent/instrumentation/kafka/kafka-common/src/main/java/datadog/trace/instrumentation/kafka_common/ClusterIdHolder.java @@ -1,4 +1,4 @@ -package datadog.trace.instrumentation.confluentschemaregistry; +package datadog.trace.instrumentation.kafka_common; /** * Thread-local holder for Kafka cluster ID to be used during schema registry operations. The Kafka diff --git a/dd-trace-core/src/main/java/datadog/trace/core/datastreams/DefaultDataStreamsMonitoring.java b/dd-trace-core/src/main/java/datadog/trace/core/datastreams/DefaultDataStreamsMonitoring.java index 199c0e834ac..5359082d522 100644 --- a/dd-trace-core/src/main/java/datadog/trace/core/datastreams/DefaultDataStreamsMonitoring.java +++ b/dd-trace-core/src/main/java/datadog/trace/core/datastreams/DefaultDataStreamsMonitoring.java @@ -226,15 +226,6 @@ public void setSchemaRegistryUsage( boolean isSuccess, boolean isKey, String operation) { - log.info( - "[DSM Schema Registry] Recording usage: topic={}, clusterId={}, schemaId={}, success={}, isKey={}, operation={}", - topic, - clusterId, - schemaId, - isSuccess, - isKey, - operation); - inbox.offer( new SchemaRegistryUsage( topic, diff --git a/dd-trace-core/src/main/java/datadog/trace/core/datastreams/MsgPackDatastreamsPayloadWriter.java b/dd-trace-core/src/main/java/datadog/trace/core/datastreams/MsgPackDatastreamsPayloadWriter.java index fd9972da853..277820af4e9 100644 --- a/dd-trace-core/src/main/java/datadog/trace/core/datastreams/MsgPackDatastreamsPayloadWriter.java +++ b/dd-trace-core/src/main/java/datadog/trace/core/datastreams/MsgPackDatastreamsPayloadWriter.java @@ -131,8 +131,12 @@ public void writePayload(Collection data, String serviceNameOverrid boolean hasBacklogs = !bucket.getBacklogs().isEmpty(); boolean hasSchemaRegistryUsages = !bucket.getSchemaRegistryUsages().isEmpty(); int mapSize = 3; - if (hasBacklogs) mapSize++; - if (hasSchemaRegistryUsages) mapSize++; + if (hasBacklogs) { + mapSize++; + } + if (hasSchemaRegistryUsages) { + mapSize++; + } writer.startMap(mapSize); /* 1 */ From 77b92a52f18870a4054ff7f6feacf6735eab33f7 Mon Sep 17 00:00:00 2001 From: Piotr Wolski Date: Tue, 18 Nov 2025 10:28:34 -0700 Subject: [PATCH 09/12] add Kafka cluster ID instrumentation to Kafka 3.8 --- .../kafka_clients38/ProducerAdvice.java | 10 ++++++++++ .../kafka_clients38/RecordsAdvice.java | 20 ++++++++++++++++++- 2 files changed, 29 insertions(+), 1 deletion(-) diff --git a/dd-java-agent/instrumentation/kafka/kafka-clients-3.8/src/main/java17/datadog/trace/instrumentation/kafka_clients38/ProducerAdvice.java b/dd-java-agent/instrumentation/kafka/kafka-clients-3.8/src/main/java17/datadog/trace/instrumentation/kafka_clients38/ProducerAdvice.java index 8d86dca9ea8..be6bdc2500b 100644 --- a/dd-java-agent/instrumentation/kafka/kafka-clients-3.8/src/main/java17/datadog/trace/instrumentation/kafka_clients38/ProducerAdvice.java +++ b/dd-java-agent/instrumentation/kafka/kafka-clients-3.8/src/main/java17/datadog/trace/instrumentation/kafka_clients38/ProducerAdvice.java @@ -22,6 +22,7 @@ import datadog.trace.bootstrap.instrumentation.api.AgentScope; import datadog.trace.bootstrap.instrumentation.api.AgentSpan; import datadog.trace.bootstrap.instrumentation.api.InstrumentationTags; +import datadog.trace.instrumentation.kafka_common.ClusterIdHolder; import net.bytebuddy.asm.Advice; import org.apache.kafka.clients.Metadata; import org.apache.kafka.clients.producer.Callback; @@ -39,6 +40,12 @@ public static AgentScope onEnter( @Advice.Argument(value = 0, readOnly = false) ProducerRecord record, @Advice.Argument(value = 1, readOnly = false) Callback callback) { String clusterId = InstrumentationContext.get(Metadata.class, String.class).get(metadata); + + // Set cluster ID for Schema Registry instrumentation + if (clusterId != null) { + ClusterIdHolder.set(clusterId); + } + final AgentSpan parent = activeSpan(); final AgentSpan span = startSpan(KAFKA_PRODUCE); PRODUCER_DECORATE.afterStart(span); @@ -106,6 +113,9 @@ record = @Advice.OnMethodExit(onThrowable = Throwable.class, suppress = Throwable.class) public static void stopSpan( @Advice.Enter final AgentScope scope, @Advice.Thrown final Throwable throwable) { + // Clear cluster ID from Schema Registry instrumentation + ClusterIdHolder.clear(); + PRODUCER_DECORATE.onError(scope, throwable); PRODUCER_DECORATE.beforeFinish(scope); scope.close(); diff --git a/dd-java-agent/instrumentation/kafka/kafka-clients-3.8/src/main/java17/datadog/trace/instrumentation/kafka_clients38/RecordsAdvice.java b/dd-java-agent/instrumentation/kafka/kafka-clients-3.8/src/main/java17/datadog/trace/instrumentation/kafka_clients38/RecordsAdvice.java index 1afd4db7cb0..12e7089a562 100644 --- a/dd-java-agent/instrumentation/kafka/kafka-clients-3.8/src/main/java17/datadog/trace/instrumentation/kafka_clients38/RecordsAdvice.java +++ b/dd-java-agent/instrumentation/kafka/kafka-clients-3.8/src/main/java17/datadog/trace/instrumentation/kafka_clients38/RecordsAdvice.java @@ -6,10 +6,13 @@ import static datadog.trace.bootstrap.instrumentation.api.InstrumentationTags.KAFKA_RECORDS_COUNT; import static datadog.trace.instrumentation.kafka_clients38.KafkaDecorator.KAFKA_POLL; +import datadog.trace.api.Config; import datadog.trace.bootstrap.InstrumentationContext; import datadog.trace.bootstrap.instrumentation.api.AgentScope; import datadog.trace.bootstrap.instrumentation.api.AgentSpan; +import datadog.trace.instrumentation.kafka_common.ClusterIdHolder; import net.bytebuddy.asm.Advice; +import org.apache.kafka.clients.Metadata; import org.apache.kafka.clients.consumer.ConsumerRecords; import org.apache.kafka.clients.consumer.internals.ConsumerDelegate; @@ -20,7 +23,19 @@ */ public class RecordsAdvice { @Advice.OnMethodEnter(suppress = Throwable.class) - public static AgentScope onEnter() { + public static AgentScope onEnter(@Advice.This ConsumerDelegate consumer) { + // Set cluster ID in ClusterIdHolder for Schema Registry instrumentation + KafkaConsumerInfo kafkaConsumerInfo = + InstrumentationContext.get(ConsumerDelegate.class, KafkaConsumerInfo.class).get(consumer); + if (kafkaConsumerInfo != null && Config.get().isDataStreamsEnabled()) { + String clusterId = + KafkaConsumerInstrumentationHelper.extractClusterId( + kafkaConsumerInfo, InstrumentationContext.get(Metadata.class, String.class)); + if (clusterId != null) { + ClusterIdHolder.set(clusterId); + } + } + if (traceConfig().isDataStreamsEnabled()) { final AgentSpan span = startSpan(KAFKA_POLL); return activateSpan(span); @@ -45,6 +60,9 @@ public static void captureGroup( } recordsCount = records.count(); } + // Clear cluster ID from Schema Registry instrumentation + ClusterIdHolder.clear(); + if (scope == null) { return; } From c67fe808cf54c61624646b3a9074882403a650a6 Mon Sep 17 00:00:00 2001 From: Piotr Wolski Date: Tue, 18 Nov 2025 13:02:37 -0700 Subject: [PATCH 10/12] Fix instrumentation crash --- .../KafkaDeserializerInstrumentation.java | 2 +- .../KafkaSerializerInstrumentation.java | 2 +- .../kafka_clients/KafkaConsumerInfoInstrumentation.java | 4 +++- .../kafka_clients/KafkaProducerInstrumentation.java | 1 + .../kafka_clients38/KafkaConsumerInfoInstrumentation.java | 4 +++- .../kafka_clients38/KafkaProducerInstrumentation.java | 1 + .../LegacyKafkaConsumerInfoInstrumentation.java | 4 +++- 7 files changed, 13 insertions(+), 5 deletions(-) diff --git a/dd-java-agent/instrumentation/confluent-schema-registry/confluent-schema-registry-7.0/src/main/java/datadog/trace/instrumentation/confluentschemaregistry/KafkaDeserializerInstrumentation.java b/dd-java-agent/instrumentation/confluent-schema-registry/confluent-schema-registry-7.0/src/main/java/datadog/trace/instrumentation/confluentschemaregistry/KafkaDeserializerInstrumentation.java index 6870b2cb5af..a61fa4d46f4 100644 --- a/dd-java-agent/instrumentation/confluent-schema-registry/confluent-schema-registry-7.0/src/main/java/datadog/trace/instrumentation/confluentschemaregistry/KafkaDeserializerInstrumentation.java +++ b/dd-java-agent/instrumentation/confluent-schema-registry/confluent-schema-registry-7.0/src/main/java/datadog/trace/instrumentation/confluentschemaregistry/KafkaDeserializerInstrumentation.java @@ -47,7 +47,7 @@ public String[] helperClassNames() { @Override public Map contextStore() { Map contextStores = new HashMap<>(); - contextStores.put(Deserializer.class.getName(), Boolean.class.getName()); + contextStores.put("org.apache.kafka.common.serialization.Deserializer", "java.lang.Boolean"); return contextStores; } diff --git a/dd-java-agent/instrumentation/confluent-schema-registry/confluent-schema-registry-7.0/src/main/java/datadog/trace/instrumentation/confluentschemaregistry/KafkaSerializerInstrumentation.java b/dd-java-agent/instrumentation/confluent-schema-registry/confluent-schema-registry-7.0/src/main/java/datadog/trace/instrumentation/confluentschemaregistry/KafkaSerializerInstrumentation.java index 0619071b5a6..9b6c904a12b 100644 --- a/dd-java-agent/instrumentation/confluent-schema-registry/confluent-schema-registry-7.0/src/main/java/datadog/trace/instrumentation/confluentschemaregistry/KafkaSerializerInstrumentation.java +++ b/dd-java-agent/instrumentation/confluent-schema-registry/confluent-schema-registry-7.0/src/main/java/datadog/trace/instrumentation/confluentschemaregistry/KafkaSerializerInstrumentation.java @@ -48,7 +48,7 @@ public String[] helperClassNames() { @Override public Map contextStore() { Map contextStores = new HashMap<>(); - contextStores.put(Serializer.class.getName(), Boolean.class.getName()); + contextStores.put("org.apache.kafka.common.serialization.Serializer", "java.lang.Boolean"); return contextStores; } diff --git a/dd-java-agent/instrumentation/kafka/kafka-clients-0.11/src/main/java/datadog/trace/instrumentation/kafka_clients/KafkaConsumerInfoInstrumentation.java b/dd-java-agent/instrumentation/kafka/kafka-clients-0.11/src/main/java/datadog/trace/instrumentation/kafka_clients/KafkaConsumerInfoInstrumentation.java index f0df784ed34..ada3a013d74 100644 --- a/dd-java-agent/instrumentation/kafka/kafka-clients-0.11/src/main/java/datadog/trace/instrumentation/kafka_clients/KafkaConsumerInfoInstrumentation.java +++ b/dd-java-agent/instrumentation/kafka/kafka-clients-0.11/src/main/java/datadog/trace/instrumentation/kafka_clients/KafkaConsumerInfoInstrumentation.java @@ -79,7 +79,9 @@ public String instrumentedType() { @Override public String[] helperClassNames() { return new String[] { - packageName + ".KafkaDecorator", packageName + ".KafkaConsumerInfo", + packageName + ".KafkaDecorator", + packageName + ".KafkaConsumerInfo", + "datadog.trace.instrumentation.kafka_common.ClusterIdHolder", }; } diff --git a/dd-java-agent/instrumentation/kafka/kafka-clients-0.11/src/main/java/datadog/trace/instrumentation/kafka_clients/KafkaProducerInstrumentation.java b/dd-java-agent/instrumentation/kafka/kafka-clients-0.11/src/main/java/datadog/trace/instrumentation/kafka_clients/KafkaProducerInstrumentation.java index b1618657106..2dc6059f7fc 100644 --- a/dd-java-agent/instrumentation/kafka/kafka-clients-0.11/src/main/java/datadog/trace/instrumentation/kafka_clients/KafkaProducerInstrumentation.java +++ b/dd-java-agent/instrumentation/kafka/kafka-clients-0.11/src/main/java/datadog/trace/instrumentation/kafka_clients/KafkaProducerInstrumentation.java @@ -79,6 +79,7 @@ public String[] helperClassNames() { packageName + ".NoopTextMapInjectAdapter", packageName + ".KafkaProducerCallback", "datadog.trace.instrumentation.kafka_common.StreamingContext", + "datadog.trace.instrumentation.kafka_common.ClusterIdHolder", packageName + ".AvroSchemaExtractor", }; } diff --git a/dd-java-agent/instrumentation/kafka/kafka-clients-3.8/src/main/java/datadog/trace/instrumentation/kafka_clients38/KafkaConsumerInfoInstrumentation.java b/dd-java-agent/instrumentation/kafka/kafka-clients-3.8/src/main/java/datadog/trace/instrumentation/kafka_clients38/KafkaConsumerInfoInstrumentation.java index 576982b9da4..334aa6c9aac 100644 --- a/dd-java-agent/instrumentation/kafka/kafka-clients-3.8/src/main/java/datadog/trace/instrumentation/kafka_clients38/KafkaConsumerInfoInstrumentation.java +++ b/dd-java-agent/instrumentation/kafka/kafka-clients-3.8/src/main/java/datadog/trace/instrumentation/kafka_clients38/KafkaConsumerInfoInstrumentation.java @@ -67,7 +67,9 @@ public ElementMatcher hierarchyMatcher() { @Override public String[] helperClassNames() { return new String[] { - packageName + ".KafkaDecorator", packageName + ".KafkaConsumerInfo", + packageName + ".KafkaDecorator", + packageName + ".KafkaConsumerInfo", + "datadog.trace.instrumentation.kafka_common.ClusterIdHolder", }; } diff --git a/dd-java-agent/instrumentation/kafka/kafka-clients-3.8/src/main/java/datadog/trace/instrumentation/kafka_clients38/KafkaProducerInstrumentation.java b/dd-java-agent/instrumentation/kafka/kafka-clients-3.8/src/main/java/datadog/trace/instrumentation/kafka_clients38/KafkaProducerInstrumentation.java index bb0af11f3db..a06469fb494 100644 --- a/dd-java-agent/instrumentation/kafka/kafka-clients-3.8/src/main/java/datadog/trace/instrumentation/kafka_clients38/KafkaProducerInstrumentation.java +++ b/dd-java-agent/instrumentation/kafka/kafka-clients-3.8/src/main/java/datadog/trace/instrumentation/kafka_clients38/KafkaProducerInstrumentation.java @@ -41,6 +41,7 @@ public String[] helperClassNames() { packageName + ".NoopTextMapInjectAdapter", packageName + ".KafkaProducerCallback", "datadog.trace.instrumentation.kafka_common.StreamingContext", + "datadog.trace.instrumentation.kafka_common.ClusterIdHolder", packageName + ".AvroSchemaExtractor", }; } diff --git a/dd-java-agent/instrumentation/kafka/kafka-clients-3.8/src/main/java/datadog/trace/instrumentation/kafka_clients38/LegacyKafkaConsumerInfoInstrumentation.java b/dd-java-agent/instrumentation/kafka/kafka-clients-3.8/src/main/java/datadog/trace/instrumentation/kafka_clients38/LegacyKafkaConsumerInfoInstrumentation.java index 73aa4534816..4b415b0bc97 100644 --- a/dd-java-agent/instrumentation/kafka/kafka-clients-3.8/src/main/java/datadog/trace/instrumentation/kafka_clients38/LegacyKafkaConsumerInfoInstrumentation.java +++ b/dd-java-agent/instrumentation/kafka/kafka-clients-3.8/src/main/java/datadog/trace/instrumentation/kafka_clients38/LegacyKafkaConsumerInfoInstrumentation.java @@ -67,7 +67,9 @@ public ElementMatcher hierarchyMatcher() { @Override public String[] helperClassNames() { return new String[] { - packageName + ".KafkaDecorator", packageName + ".KafkaConsumerInfo", + packageName + ".KafkaDecorator", + packageName + ".KafkaConsumerInfo", + "datadog.trace.instrumentation.kafka_common.ClusterIdHolder", }; } From a74e7f97fc68497d7a855f5270f11580753e686b Mon Sep 17 00:00:00 2001 From: Piotr Wolski Date: Tue, 18 Nov 2025 14:22:13 -0700 Subject: [PATCH 11/12] rename setSchemaRegistryUsage --- .../KafkaDeserializerInstrumentation.java | 2 +- .../KafkaSerializerInstrumentation.java | 2 +- .../core/datastreams/DefaultDataStreamsMonitoring.java | 2 +- .../datastreams/DefaultDataStreamsMonitoringTest.groovy | 8 ++++---- .../trace/api/datastreams/AgentDataStreamsMonitoring.java | 2 +- .../trace/api/datastreams/NoopDataStreamsMonitoring.java | 2 +- 6 files changed, 9 insertions(+), 9 deletions(-) diff --git a/dd-java-agent/instrumentation/confluent-schema-registry/confluent-schema-registry-7.0/src/main/java/datadog/trace/instrumentation/confluentschemaregistry/KafkaDeserializerInstrumentation.java b/dd-java-agent/instrumentation/confluent-schema-registry/confluent-schema-registry-7.0/src/main/java/datadog/trace/instrumentation/confluentschemaregistry/KafkaDeserializerInstrumentation.java index a61fa4d46f4..10ecbde4991 100644 --- a/dd-java-agent/instrumentation/confluent-schema-registry/confluent-schema-registry-7.0/src/main/java/datadog/trace/instrumentation/confluentschemaregistry/KafkaDeserializerInstrumentation.java +++ b/dd-java-agent/instrumentation/confluent-schema-registry/confluent-schema-registry-7.0/src/main/java/datadog/trace/instrumentation/confluentschemaregistry/KafkaDeserializerInstrumentation.java @@ -120,7 +120,7 @@ public static void onExit( // Record the schema registry usage AgentTracer.get() .getDataStreamsMonitoring() - .setSchemaRegistryUsage(topic, clusterId, schemaId, isSuccess, isKey, "deserialize"); + .reportSchemaRegistryUsage(topic, clusterId, schemaId, isSuccess, isKey, "deserialize"); } } } diff --git a/dd-java-agent/instrumentation/confluent-schema-registry/confluent-schema-registry-7.0/src/main/java/datadog/trace/instrumentation/confluentschemaregistry/KafkaSerializerInstrumentation.java b/dd-java-agent/instrumentation/confluent-schema-registry/confluent-schema-registry-7.0/src/main/java/datadog/trace/instrumentation/confluentschemaregistry/KafkaSerializerInstrumentation.java index 9b6c904a12b..c85015315ca 100644 --- a/dd-java-agent/instrumentation/confluent-schema-registry/confluent-schema-registry-7.0/src/main/java/datadog/trace/instrumentation/confluentschemaregistry/KafkaSerializerInstrumentation.java +++ b/dd-java-agent/instrumentation/confluent-schema-registry/confluent-schema-registry-7.0/src/main/java/datadog/trace/instrumentation/confluentschemaregistry/KafkaSerializerInstrumentation.java @@ -119,7 +119,7 @@ public static void onExit( // Record the schema registry usage AgentTracer.get() .getDataStreamsMonitoring() - .setSchemaRegistryUsage(topic, clusterId, schemaId, isSuccess, isKey, "serialize"); + .reportSchemaRegistryUsage(topic, clusterId, schemaId, isSuccess, isKey, "serialize"); } } } diff --git a/dd-trace-core/src/main/java/datadog/trace/core/datastreams/DefaultDataStreamsMonitoring.java b/dd-trace-core/src/main/java/datadog/trace/core/datastreams/DefaultDataStreamsMonitoring.java index 5359082d522..931d3583afe 100644 --- a/dd-trace-core/src/main/java/datadog/trace/core/datastreams/DefaultDataStreamsMonitoring.java +++ b/dd-trace-core/src/main/java/datadog/trace/core/datastreams/DefaultDataStreamsMonitoring.java @@ -219,7 +219,7 @@ public void trackBacklog(DataStreamsTags tags, long value) { } @Override - public void setSchemaRegistryUsage( + public void reportSchemaRegistryUsage( String topic, String clusterId, int schemaId, diff --git a/dd-trace-core/src/test/groovy/datadog/trace/core/datastreams/DefaultDataStreamsMonitoringTest.groovy b/dd-trace-core/src/test/groovy/datadog/trace/core/datastreams/DefaultDataStreamsMonitoringTest.groovy index 15123b6bb99..1119e540ac9 100644 --- a/dd-trace-core/src/test/groovy/datadog/trace/core/datastreams/DefaultDataStreamsMonitoringTest.groovy +++ b/dd-trace-core/src/test/groovy/datadog/trace/core/datastreams/DefaultDataStreamsMonitoringTest.groovy @@ -892,10 +892,10 @@ class DefaultDataStreamsMonitoringTest extends DDCoreSpecification { dataStreams.start() // Record serialize and deserialize operations - dataStreams.setSchemaRegistryUsage("test-topic", "test-cluster", 123, true, false, "serialize") - dataStreams.setSchemaRegistryUsage("test-topic", "test-cluster", 123, true, false, "serialize") // duplicate serialize - dataStreams.setSchemaRegistryUsage("test-topic", "test-cluster", 123, true, false, "deserialize") - dataStreams.setSchemaRegistryUsage("test-topic", "test-cluster", 456, true, true, "serialize") // different schema/key + dataStreams.reportSchemaRegistryUsage("test-topic", "test-cluster", 123, true, false, "serialize") + dataStreams.reportSchemaRegistryUsage("test-topic", "test-cluster", 123, true, false, "serialize") // duplicate serialize + dataStreams.reportSchemaRegistryUsage("test-topic", "test-cluster", 123, true, false, "deserialize") + dataStreams.reportSchemaRegistryUsage("test-topic", "test-cluster", 456, true, true, "serialize") // different schema/key timeSource.advance(DEFAULT_BUCKET_DURATION_NANOS) dataStreams.report() diff --git a/internal-api/src/main/java/datadog/trace/api/datastreams/AgentDataStreamsMonitoring.java b/internal-api/src/main/java/datadog/trace/api/datastreams/AgentDataStreamsMonitoring.java index 9c2651f0e9c..c11fab3fc28 100644 --- a/internal-api/src/main/java/datadog/trace/api/datastreams/AgentDataStreamsMonitoring.java +++ b/internal-api/src/main/java/datadog/trace/api/datastreams/AgentDataStreamsMonitoring.java @@ -18,7 +18,7 @@ public interface AgentDataStreamsMonitoring extends DataStreamsCheckpointer { * @param isKey Whether this is for the key (true) or value (false) * @param operation The operation type: "serialize" or "deserialize" */ - void setSchemaRegistryUsage( + void reportSchemaRegistryUsage( String topic, String clusterId, int schemaId, diff --git a/internal-api/src/main/java/datadog/trace/api/datastreams/NoopDataStreamsMonitoring.java b/internal-api/src/main/java/datadog/trace/api/datastreams/NoopDataStreamsMonitoring.java index 8ba4cda65d2..bd8e19fe6dc 100644 --- a/internal-api/src/main/java/datadog/trace/api/datastreams/NoopDataStreamsMonitoring.java +++ b/internal-api/src/main/java/datadog/trace/api/datastreams/NoopDataStreamsMonitoring.java @@ -12,7 +12,7 @@ public class NoopDataStreamsMonitoring implements AgentDataStreamsMonitoring { public void trackBacklog(DataStreamsTags tags, long value) {} @Override - public void setSchemaRegistryUsage( + public void reportSchemaRegistryUsage( String topic, String clusterId, int schemaId, From 026f824db12895bfd6392c32e8dd3f2dc3d91dfd Mon Sep 17 00:00:00 2001 From: Piotr Wolski Date: Wed, 19 Nov 2025 08:49:43 -0700 Subject: [PATCH 12/12] Add helper for extracting schema ID --- .../KafkaDeserializerInstrumentation.java | 21 ++++------------- .../KafkaSerializerInstrumentation.java | 21 ++++------------- .../SchemaIdExtractor.java | 23 +++++++++++++++++++ .../test/groovy/KafkaClientTestBase.groovy | 4 ++++ .../test/groovy/KafkaClientTestBase.groovy | 4 ++++ .../kafka_common/ClusterIdHolder.java | 2 +- 6 files changed, 42 insertions(+), 33 deletions(-) create mode 100644 dd-java-agent/instrumentation/confluent-schema-registry/confluent-schema-registry-7.0/src/main/java/datadog/trace/instrumentation/confluentschemaregistry/SchemaIdExtractor.java diff --git a/dd-java-agent/instrumentation/confluent-schema-registry/confluent-schema-registry-7.0/src/main/java/datadog/trace/instrumentation/confluentschemaregistry/KafkaDeserializerInstrumentation.java b/dd-java-agent/instrumentation/confluent-schema-registry/confluent-schema-registry-7.0/src/main/java/datadog/trace/instrumentation/confluentschemaregistry/KafkaDeserializerInstrumentation.java index 10ecbde4991..825bd0db51b 100644 --- a/dd-java-agent/instrumentation/confluent-schema-registry/confluent-schema-registry-7.0/src/main/java/datadog/trace/instrumentation/confluentschemaregistry/KafkaDeserializerInstrumentation.java +++ b/dd-java-agent/instrumentation/confluent-schema-registry/confluent-schema-registry-7.0/src/main/java/datadog/trace/instrumentation/confluentschemaregistry/KafkaDeserializerInstrumentation.java @@ -41,7 +41,10 @@ public String[] knownMatchingTypes() { @Override public String[] helperClassNames() { - return new String[] {"datadog.trace.instrumentation.kafka_common.ClusterIdHolder"}; + return new String[] { + "datadog.trace.instrumentation.kafka_common.ClusterIdHolder", + packageName + ".SchemaIdExtractor" + }; } @Override @@ -101,21 +104,7 @@ public static void onExit( String clusterId = ClusterIdHolder.get(); boolean isSuccess = throwable == null; - int schemaId = -1; - - // Extract schema ID from the input bytes if successful - if (isSuccess && data != null && data.length >= 5 && data[0] == 0) { - try { - // Confluent wire format: [magic_byte][4-byte schema id][data] - schemaId = - ((data[1] & 0xFF) << 24) - | ((data[2] & 0xFF) << 16) - | ((data[3] & 0xFF) << 8) - | (data[4] & 0xFF); - } catch (Throwable ignored) { - // If extraction fails, keep schemaId as -1 - } - } + int schemaId = isSuccess ? SchemaIdExtractor.extractSchemaId(data) : -1; // Record the schema registry usage AgentTracer.get() diff --git a/dd-java-agent/instrumentation/confluent-schema-registry/confluent-schema-registry-7.0/src/main/java/datadog/trace/instrumentation/confluentschemaregistry/KafkaSerializerInstrumentation.java b/dd-java-agent/instrumentation/confluent-schema-registry/confluent-schema-registry-7.0/src/main/java/datadog/trace/instrumentation/confluentschemaregistry/KafkaSerializerInstrumentation.java index c85015315ca..7ad15382669 100644 --- a/dd-java-agent/instrumentation/confluent-schema-registry/confluent-schema-registry-7.0/src/main/java/datadog/trace/instrumentation/confluentschemaregistry/KafkaSerializerInstrumentation.java +++ b/dd-java-agent/instrumentation/confluent-schema-registry/confluent-schema-registry-7.0/src/main/java/datadog/trace/instrumentation/confluentschemaregistry/KafkaSerializerInstrumentation.java @@ -42,7 +42,10 @@ public String[] knownMatchingTypes() { @Override public String[] helperClassNames() { - return new String[] {"datadog.trace.instrumentation.kafka_common.ClusterIdHolder"}; + return new String[] { + "datadog.trace.instrumentation.kafka_common.ClusterIdHolder", + packageName + ".SchemaIdExtractor" + }; } @Override @@ -100,21 +103,7 @@ public static void onExit( String clusterId = ClusterIdHolder.get(); boolean isSuccess = throwable == null; - int schemaId = -1; - - // Extract schema ID from the serialized bytes if successful - if (isSuccess && result != null && result.length >= 5 && result[0] == 0) { - try { - // Confluent wire format: [magic_byte][4-byte schema id][data] - schemaId = - ((result[1] & 0xFF) << 24) - | ((result[2] & 0xFF) << 16) - | ((result[3] & 0xFF) << 8) - | (result[4] & 0xFF); - } catch (Throwable ignored) { - // If extraction fails, keep schemaId as -1 - } - } + int schemaId = isSuccess ? SchemaIdExtractor.extractSchemaId(result) : -1; // Record the schema registry usage AgentTracer.get() diff --git a/dd-java-agent/instrumentation/confluent-schema-registry/confluent-schema-registry-7.0/src/main/java/datadog/trace/instrumentation/confluentschemaregistry/SchemaIdExtractor.java b/dd-java-agent/instrumentation/confluent-schema-registry/confluent-schema-registry-7.0/src/main/java/datadog/trace/instrumentation/confluentschemaregistry/SchemaIdExtractor.java new file mode 100644 index 00000000000..786ff25bc21 --- /dev/null +++ b/dd-java-agent/instrumentation/confluent-schema-registry/confluent-schema-registry-7.0/src/main/java/datadog/trace/instrumentation/confluentschemaregistry/SchemaIdExtractor.java @@ -0,0 +1,23 @@ +package datadog.trace.instrumentation.confluentschemaregistry; + +/** + * Helper class to extract schema ID from Confluent Schema Registry wire format. Wire format: + * [magic_byte][4-byte schema id][data] + */ +public class SchemaIdExtractor { + public static int extractSchemaId(byte[] data) { + if (data == null || data.length < 5 || data[0] != 0) { + return -1; + } + + try { + // Confluent wire format: [magic_byte][4-byte schema id][data] + return ((data[1] & 0xFF) << 24) + | ((data[2] & 0xFF) << 16) + | ((data[3] & 0xFF) << 8) + | (data[4] & 0xFF); + } catch (Throwable ignored) { + return -1; + } + } +} diff --git a/dd-java-agent/instrumentation/kafka/kafka-clients-0.11/src/test/groovy/KafkaClientTestBase.groovy b/dd-java-agent/instrumentation/kafka/kafka-clients-0.11/src/test/groovy/KafkaClientTestBase.groovy index 9870627fda3..686c550a8b6 100644 --- a/dd-java-agent/instrumentation/kafka/kafka-clients-0.11/src/test/groovy/KafkaClientTestBase.groovy +++ b/dd-java-agent/instrumentation/kafka/kafka-clients-0.11/src/test/groovy/KafkaClientTestBase.groovy @@ -1,4 +1,5 @@ import datadog.trace.api.datastreams.DataStreamsTags +import datadog.trace.instrumentation.kafka_common.ClusterIdHolder import static datadog.trace.agent.test.utils.TraceUtils.basicSpan import static datadog.trace.agent.test.utils.TraceUtils.runUnderTrace @@ -250,6 +251,9 @@ abstract class KafkaClientTestBase extends VersionedNamingTestBase { received.value() == greeting received.key() == null + // verify ClusterIdHolder was properly cleaned up after produce and consume + ClusterIdHolder.get() == null + int nTraces = isDataStreamsEnabled() ? 3 : 2 int produceTraceIdx = nTraces - 1 TEST_WRITER.waitForTraces(nTraces) diff --git a/dd-java-agent/instrumentation/kafka/kafka-clients-3.8/src/test/groovy/KafkaClientTestBase.groovy b/dd-java-agent/instrumentation/kafka/kafka-clients-3.8/src/test/groovy/KafkaClientTestBase.groovy index 78d1d8f70de..c0a2c9c2625 100644 --- a/dd-java-agent/instrumentation/kafka/kafka-clients-3.8/src/test/groovy/KafkaClientTestBase.groovy +++ b/dd-java-agent/instrumentation/kafka/kafka-clients-3.8/src/test/groovy/KafkaClientTestBase.groovy @@ -8,6 +8,7 @@ import datadog.trace.bootstrap.instrumentation.api.Tags import datadog.trace.common.writer.ListWriter import datadog.trace.core.DDSpan import datadog.trace.core.datastreams.StatsGroup +import datadog.trace.instrumentation.kafka_common.ClusterIdHolder import org.apache.kafka.clients.consumer.ConsumerConfig import org.apache.kafka.clients.consumer.ConsumerRecord import org.apache.kafka.clients.consumer.KafkaConsumer @@ -217,6 +218,9 @@ abstract class KafkaClientTestBase extends VersionedNamingTestBase { def received = records.poll(10, TimeUnit.SECONDS) received.value() == greeting received.key() == null + + // verify ClusterIdHolder was properly cleaned up after produce and consume + ClusterIdHolder.get() == null int nTraces = isDataStreamsEnabled() ? 3 : 2 int produceTraceIdx = nTraces - 1 TEST_WRITER.waitForTraces(nTraces) diff --git a/dd-java-agent/instrumentation/kafka/kafka-common/src/main/java/datadog/trace/instrumentation/kafka_common/ClusterIdHolder.java b/dd-java-agent/instrumentation/kafka/kafka-common/src/main/java/datadog/trace/instrumentation/kafka_common/ClusterIdHolder.java index aeb0eee55c4..fa0b90eebce 100644 --- a/dd-java-agent/instrumentation/kafka/kafka-common/src/main/java/datadog/trace/instrumentation/kafka_common/ClusterIdHolder.java +++ b/dd-java-agent/instrumentation/kafka/kafka-common/src/main/java/datadog/trace/instrumentation/kafka_common/ClusterIdHolder.java @@ -17,6 +17,6 @@ public static String get() { } public static void clear() { - CLUSTER_ID.remove(); + CLUSTER_ID.set(null); } }