Skip to content

Lara.kulkarni/logs for testing spark spans #10729

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Draft
larakulkarni1 wants to merge 13 commits into
base: master
Choose a base branch
from
Draft

Lara.kulkarni/logs for testing spark spans #10729

Changes from all commits
Commits
Show all changes
13 commits
Select commit Hold shift + click to select a range
1189586
updating to get rid of wrong spark spans
larakulkarni1 Feb 20, 2026
5aeb249
fixed format
larakulkarni1 Feb 20, 2026
fb1450e
updating logs to test
larakulkarni1 Feb 23, 2026
1d3d147
fixing an error + adding right logs
larakulkarni1 Feb 24, 2026
f0be2cf
ran spotless
larakulkarni1 Feb 24, 2026
de2d1b6
added a log
larakulkarni1 Feb 24, 2026
9714fb0
added logs
larakulkarni1 Feb 27, 2026
7646838
added logs
larakulkarni1 Feb 27, 2026
5177a4f
Merge branch 'master' into lara.kulkarni/updatingTheSparkSPans
larakulkarni1 Mar 2, 2026
2efadac
spotless cleaning
larakulkarni1 Mar 2, 2026
40a7988
added log
larakulkarni1 Mar 3, 2026
cf226ce
updated with logs
larakulkarni1 Mar 4, 2026
fb6e47f
Merge branch 'master' into lara.kulkarni/logsForTestingSparkSpans
larakulkarni1 Mar 5, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
40 changes: 38 additions & 2 deletions ...ommon/src/main/java/datadog/trace/instrumentation/spark/AbstractDatadogSparkListener.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -233,6 +233,9 @@ public void setupOpenLineage(DDTraceId traceId) {
public synchronized void onApplicationStart(SparkListenerApplicationStart applicationStart) {
this.applicationStart = applicationStart;

if (isRunningOnDatabricks) {
log.info("databricksClusterName{}", databricksClusterName);
}
if (openLineageSparkListener == null) {
openLineageSparkListener =
InstanceStore.of(SparkListenerInterface.class).get("openLineageListener");
Expand All @@ -246,6 +249,7 @@ public synchronized void onApplicationStart(SparkListenerApplicationStart applic
.orElse(predeterminedTraceIdContext.getTraceId()));
}
notifyOl(x -> openLineageSparkListener.onApplicationStart(x), applicationStart);
log.info("end of application start");
}

private void initApplicationSpanIfNotInitialized() {
Expand All @@ -254,7 +258,7 @@ private void initApplicationSpanIfNotInitialized() {
}

log.debug("Starting tracer application span.");

log.info("Not a databricks span");
AgentTracer.SpanBuilder builder = buildSparkSpan("spark.application", null);

if (applicationStart != null) {
Expand Down Expand Up @@ -308,6 +312,9 @@ private void captureOpenlineageContextIfPresent(

@Override
public void onApplicationEnd(SparkListenerApplicationEnd applicationEnd) {
if (isRunningOnDatabricks) {
log.info("On Application End");
}
log.info(
"Received spark application end event, finish trace on this event: {}",
finishTraceOnApplicationEnd);
Expand All @@ -325,17 +332,46 @@ public synchronized void finishApplication(
log.info("Finishing spark application trace");

if (applicationEnded) {
log.info(
"finishApplicationEnded: isRunningOnDatabricks={}, databricksClusterName={}, applicationSpan={}, jobCount={}",
isRunningOnDatabricks,
databricksClusterName,
applicationSpan != null ? "exists" : "null",
jobCount);
return;
}
applicationEnded = true;

if (applicationSpan == null && jobCount > 0) {
log.info(
"finishApplication: isRunningOnDatabricks={}, databricksClusterName={}, applicationSpan={}, jobCount={}",
isRunningOnDatabricks,
databricksClusterName,
applicationSpan != null ? "exists" : "null",
jobCount);

if ((applicationSpan == null && jobCount > 0) || isRunningOnDatabricks) {
// If the application span is not initialized, but spark jobs have been executed, all those
// spark jobs were databricks or streaming. In this case we don't send the application span
log.info(
"finishApplicationInDatabricksCheck: isRunningOnDatabricks={}, databricksClusterName={}, applicationSpan={}, jobCount={}",
isRunningOnDatabricks,
databricksClusterName,
applicationSpan != null ? "exists" : "null",
jobCount);
return;
}
initApplicationSpanIfNotInitialized();

// if (applicationSpan == null) {
// // On Databricks or streaming environments, the application span is not created.
// // Flush any remaining traces and return.
// log.info(
// "No application span created, skipping. isRunningOnDatabricks={}",
// isRunningOnDatabricks);
// tracer.flush();
// return;
// }

if (throwable != null) {
applicationSpan.addThrowable(throwable);
} else if (exitCode != 0) {
Expand Down