feat: distributed tracing

.gitignore

@@ -17,3 +17,4 @@ test_integrations/phoenix_app/db
 
 test_integrations/*/_build
 test_integrations/*/deps
+test_integrations/*/test-results/

lib/sentry.ex

@@ -14,7 +14,8 @@ defmodule Sentry do
 
     * Automatically for Plug/Phoenix applications — see the
       [*Setup with Plug and Phoenix* guide](setup-with-plug-and-phoenix.html), and the
-      `Sentry.PlugCapture` and `Sentry.PlugContext` modules.
+      `Sentry.PlugCapture`, `Sentry.PlugContext`, `Sentry.Plug.LiveViewContext`, and
+      `Sentry.Phoenix.LiveViewTracing`.
 
     * Through integrations for various ecosystem tools, like [Oban](oban-integration.html)
       or [Quantum](quantum-integration.html).

lib/sentry/opentelemetry/propagator.ex

@@ -0,0 +1,153 @@
+if Sentry.OpenTelemetry.VersionChecker.tracing_compatible?() do
+  defmodule Sentry.OpenTelemetry.Propagator do
+    @moduledoc """
+    OpenTelemetry propagator for Sentry distributed tracing.
+
+    This propagator implements the `sentry-trace` and `sentry-baggage` header propagation
+    to enable distributed tracing across service boundaries. It follows the W3C Trace Context.
+
+    ## Usage
+
+    Add the propagator to your OpenTelemetry configuration:
+
+        config :opentelemetry,
+          text_map_propagators: [
+            :trace_context,
+            :baggage,
+            Sentry.OpenTelemetry.Propagator
+          ]
+
+    This will enable automatic propagation of Sentry trace context in HTTP headers.
+    """
+
+    import Bitwise
+
+    require Record
+    require OpenTelemetry.Tracer, as: Tracer
+
+    @behaviour :otel_propagator_text_map
+
+    @fields Record.extract(:span_ctx, from_lib: "opentelemetry_api/include/opentelemetry.hrl")
+    Record.defrecordp(:span_ctx, @fields)
+
+    @sentry_trace_key "sentry-trace"
+    @sentry_baggage_key "baggage"
+    @sentry_trace_ctx_key :"sentry-trace"
+    @sentry_baggage_ctx_key :"sentry-baggage"
+
+    @impl true
+    def fields(_opts) do
+      [@sentry_trace_key, @sentry_baggage_key]
+    end
+
+    @impl true
+    def inject(ctx, carrier, setter, _opts) do
+      case Tracer.current_span_ctx(ctx) do
+        span_ctx(trace_id: tid, span_id: sid, trace_flags: flags) when tid != 0 and sid != 0 ->
+          sentry_trace_header = encode_sentry_trace({tid, sid, flags})
+          carrier = setter.(@sentry_trace_key, sentry_trace_header, carrier)
+
+          # Inject baggage if it exists in the context
+          # Note: :otel_ctx.get_value/2 returns the key itself if value not found
+          baggage_value = :otel_ctx.get_value(ctx, @sentry_baggage_ctx_key, :not_found)
+
+          if is_binary(baggage_value) and baggage_value != :not_found do
+            setter.(@sentry_baggage_key, baggage_value, carrier)
+          else
+            carrier
+          end
+
+        _ ->
+          carrier
+      end
+    end
+
+    @impl true
+    def extract(ctx, carrier, _keys_fun, getter, _opts) do
+      case getter.(@sentry_trace_key, carrier) do
+        :undefined ->
+          ctx
+
+        header when is_binary(header) ->
+          case decode_sentry_trace(header) do
+            {:ok, {trace_hex, span_hex, sampled}} ->
+              ctx =
+                ctx
+                |> :otel_ctx.set_value(@sentry_trace_ctx_key, {trace_hex, span_hex, sampled})
+                |> maybe_set_baggage(getter.(@sentry_baggage_key, carrier))
+
+              trace_id = hex_to_int(trace_hex)
+              span_id = hex_to_int(span_hex)
+
+              # Always use sampled (1) to simulate a sampled trace on the OTel side
+              trace_flags = 1
+
+              remote_span_ctx =
+                :otel_tracer.from_remote_span(trace_id, span_id, trace_flags)
+
+              Tracer.set_current_span(ctx, remote_span_ctx)
+
+            {:error, _reason} ->
+              # Invalid header format, skip propagation
+              ctx
+          end
+
+        _ ->
+          ctx
+      end
+    end
+
+    # Encode trace ID, span ID, and sampled flag to sentry-trace header format
+    # Format: {trace_id}-{span_id}-{sampled}
+    defp encode_sentry_trace({trace_id_int, span_id_int, trace_flags}) do
+      sampled = if (trace_flags &&& 1) == 1, do: "1", else: "0"
+      int_to_hex(trace_id_int, 16) <> "-" <> int_to_hex(span_id_int, 8) <> "-" <> sampled
+    end
+
+    # Decode sentry-trace header
+    # Format: {trace_id}-{span_id}-{sampled} or {trace_id}-{span_id}
+    defp decode_sentry_trace(
+           <<trace_hex::binary-size(32), "-", span_hex::binary-size(16), "-",
+             sampled::binary-size(1)>>
+         ) do
+      {:ok, {trace_hex, span_hex, sampled == "1"}}
+    end
+
+    defp decode_sentry_trace(<<trace_hex::binary-size(32), "-", span_hex::binary-size(16)>>) do
+      {:ok, {trace_hex, span_hex, false}}
+    end
+
+    defp decode_sentry_trace(_invalid) do
+      {:error, :invalid_format}
+    end
+
+    defp maybe_set_baggage(ctx, :undefined), do: ctx
+    defp maybe_set_baggage(ctx, ""), do: ctx
+    defp maybe_set_baggage(ctx, nil), do: ctx
+
+    defp maybe_set_baggage(ctx, baggage) when is_binary(baggage) do
+      :otel_ctx.set_value(ctx, @sentry_baggage_ctx_key, baggage)
+    end
+
+    # Convert hex string to integer
+    defp hex_to_int(hex) do
+      hex
+      |> Base.decode16!(case: :mixed)
+      |> :binary.decode_unsigned()
+    end
+
+    # Convert integer to hex string with padding
+    defp int_to_hex(value, num_bytes) do
+      value
+      |> :binary.encode_unsigned()
+      |> bin_pad_left(num_bytes)
+      |> Base.encode16(case: :lower)
+    end
+
+    # Pad binary to specified number of bytes
+    defp bin_pad_left(bin, total_bytes) do
+      missing = total_bytes - byte_size(bin)
+      if missing > 0, do: :binary.copy(<<0>>, missing) <> bin, else: bin
+    end
+  end
+end

lib/sentry/opentelemetry/span_processor.ex

@@ -27,8 +27,20 @@ if Sentry.OpenTelemetry.VersionChecker.tracing_compatible?() do
 
       SpanStorage.store_span(span_record)
 
-      if span_record.parent_span_id == nil do
-        child_span_records = SpanStorage.get_child_spans(span_record.span_id)
+      # Check if this is a root span (no parent) or a transaction root (HTTP server request span)
+      # HTTP server request spans should be treated as transaction roots even when they have
+      # an external parent span ID (from distributed tracing)
+      is_transaction_root =
+        span_record.parent_span_id == nil or
+          is_http_server_request_span?(span_record) or
+          is_live_view_server_span?(span_record)
+
+      if is_transaction_root do
+        child_span_records =
+          span_record.span_id
+          |> SpanStorage.get_child_spans()
+          |> maybe_add_remote_children(span_record)
+
         transaction = build_transaction(span_record, child_span_records)
 
         result =
@@ -47,8 +59,18 @@ if Sentry.OpenTelemetry.VersionChecker.tracing_compatible?() do
               {:error, :invalid_span}
           end
 
+        # Clean up: remove the transaction root span and all its children
+        # Note: For distributed tracing, the transaction root span may have been stored
+        # as a child span (with a remote parent_span_id). In that case, we need to also
+        # remove it from the child spans, not just look for it as a root span.
         :ok = SpanStorage.remove_root_span(span_record.span_id)
 
+        if span_record.parent_span_id != nil do
+          # This span was stored as a child because it has a remote parent (distributed tracing).
+          # We need to explicitly remove it from the child spans storage.
+          :ok = SpanStorage.remove_child_span(span_record.parent_span_id, span_record.span_id)
+        end
+
         result
       else
         true
@@ -60,6 +82,75 @@ if Sentry.OpenTelemetry.VersionChecker.tracing_compatible?() do
       :ok
     end
 
+    # Helper function to detect if a span represents an HTTP server request
+    # that should be treated as a transaction root for distributed tracing
+    defp is_http_server_request_span?(%{kind: kind, attributes: attributes}) do
+      kind == :server and
+        Map.has_key?(attributes, to_string(HTTPAttributes.http_request_method()))
+    end
+
+    defp is_live_view_server_span?(%{kind: :server, origin: origin, name: name})
+         when origin in ["opentelemetry_phoenix", :opentelemetry_phoenix] do
+      String.ends_with?(name, ".mount") or
+        String.contains?(name, ".handle_params") or
+        String.contains?(name, ".handle_event")
+    end
+
+    defp is_live_view_server_span?(_span_record), do: false
+
+    defp maybe_add_remote_children(child_span_records, %{parent_span_id: nil}) do
+      child_span_records
+    end
+
+    defp maybe_add_remote_children(child_span_records, span_record) do
+      if is_live_view_server_span?(span_record) do
+        existing_ids = MapSet.new(child_span_records, & &1.span_id)
+
+        adopted_children =
+          span_record.parent_span_id
+          |> SpanStorage.get_child_spans()
+          |> Enum.filter(&eligible_for_adoption?(&1, span_record, existing_ids))
+          |> Enum.map(&%{&1 | parent_span_id: span_record.span_id})
+
+        Enum.each(adopted_children, fn child ->
+          :ok = SpanStorage.remove_child_span(span_record.parent_span_id, child.span_id)
+        end)
+
+        child_span_records ++ adopted_children
+      else
+        child_span_records
+      end
+    end
+
+    defp eligible_for_adoption?(child, span_record, existing_ids) do
+      not MapSet.member?(existing_ids, child.span_id) and
+        child.parent_span_id == span_record.parent_span_id and
+        child.trace_id == span_record.trace_id and
+        child.kind != :server and
+        occurs_within_span?(child, span_record)
+    end
+
+    defp occurs_within_span?(child, parent) do
+      with {:ok, parent_start} <- parse_datetime(parent.start_time),
+           {:ok, parent_end} <- parse_datetime(parent.end_time),
+           {:ok, child_start} <- parse_datetime(child.start_time),
+           {:ok, child_end} <- parse_datetime(child.end_time) do
+        DateTime.compare(child_start, parent_start) != :lt and
+          DateTime.compare(child_end, parent_end) != :gt
+      else
+        _ -> true
+      end
+    end
+
+    defp parse_datetime(nil), do: :error
+
+    defp parse_datetime(timestamp) do
+      case DateTime.from_iso8601(timestamp) do
+        {:ok, datetime, _offset} -> {:ok, datetime}
+        {:error, _} -> :error
+      end
+    end
+
     defp build_transaction(root_span_record, child_span_records) do
       root_span = build_span(root_span_record)
       child_spans = Enum.map(child_span_records, &build_span(&1))

lib/sentry/opentelemetry/span_storage.ex

@@ -121,6 +121,16 @@ if Sentry.OpenTelemetry.VersionChecker.tracing_compatible?() do
       :ok
     end
 
+    @spec remove_child_span(String.t(), String.t(), keyword()) :: :ok
+    def remove_child_span(parent_span_id, span_id, opts \\ []) do
+      table_name = Keyword.get(opts, :table_name, default_table_name())
+      key = {:child_span, parent_span_id, span_id}
+
+      :ets.delete(table_name, key)
+
+      :ok
+    end
+
     defp schedule_cleanup(interval) do
       Process.send_after(self(), :cleanup_stale_spans, interval)
     end

lib/sentry/phoenix/live_view_tracing.ex

@@ -0,0 +1,67 @@
+if Code.ensure_loaded?(Phoenix.LiveView) and
+     Sentry.OpenTelemetry.VersionChecker.tracing_compatible?() do
+  defmodule Sentry.Phoenix.LiveViewTracing do
+    @moduledoc """
+    LiveView hook that attaches the propagated OpenTelemetry context saved by `Sentry.Plug.LiveViewContext`.
+
+    Configure your router with `on_mount {Sentry.Phoenix.LiveViewTracing, :attach}` so that the LiveView
+    process can deserialize the carrier written to the session, attach it to the process, and inherit the
+    incoming trace ID for spans emitted by `opentelemetry_phoenix`.
+
+    If the session key is missing (for example, when a LiveView spawns another LiveView), the hook falls
+    back to `OpentelemetryProcessPropagator.fetch_parent_ctx/0` when the dependency is available.
+    """
+
+    alias OpenTelemetry.Ctx
+    alias Sentry.OpenTelemetry.Propagator
+
+    @session_key "__sentry_live_view_context__"
+    @context_token_key :sentry_live_view_tracing_token
+
+    @doc """
+    Attach the propagated context to a LiveView process.
+    """
+    @spec on_mount(atom(), map(), map(), Phoenix.LiveView.Socket.t()) ::
+            {:cont, Phoenix.LiveView.Socket.t()}
+    def on_mount(:attach, _params, session, socket) do
+      {:cont, maybe_attach_context(socket, session)}
+    end
+
+    defp maybe_attach_context(socket, session) do
+      case Map.get(session, @session_key) do
+        carrier when is_map(carrier) and carrier != %{} ->
+          attach_context(socket, extract_context(carrier))
+
+        _ ->
+          attach_context(socket, fetch_parent_context())
+      end
+    end
+
+    defp extract_context(carrier) do
+      ctx = Ctx.get_current()
+      keys_fun = fn _ -> Map.keys(carrier) end
+      getter = fn key, _ -> Map.get(carrier, key, :undefined) end
+
+      Propagator.extract(ctx, carrier, keys_fun, getter, [])
+    end
+
+    defp fetch_parent_context do
+      module = :OpentelemetryProcessPropagator
+
+      if Code.ensure_loaded?(module) do
+        apply(module, :fetch_parent_ctx, [])
+      else
+        :undefined
+      end
+    end
+
+    defp attach_context(socket, ctx) when ctx in [:undefined, nil] do
+      socket
+    end
+
+    defp attach_context(socket, ctx) do
+      token = Ctx.attach(ctx)
+      %{socket | private: Map.put(socket.private, @context_token_key, token)}
+    end
+  end
+end