Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Add Probability Sampler for Activity #702

Merged
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Jump to
Jump to file
Failed to load files.
Diff view
Diff view
Original file line number Diff line number Diff line change
Expand Up @@ -77,9 +77,11 @@ public void Start(string url)
}

string requestContent;
using (var childSpan = source.StartActivity("ReadStream", ActivityKind.Consumer))
using (var reader = new StreamReader(context.Request.InputStream, context.Request.ContentEncoding))
{
requestContent = reader.ReadToEnd();
childSpan.AddEvent(new ActivityEvent("StreamReader.ReadToEnd"));
}

activity?.AddTag("request.content", requestContent);
Expand Down
16 changes: 4 additions & 12 deletions src/OpenTelemetry/Trace/ActivitySampler.cs
Original file line number Diff line number Diff line change
Expand Up @@ -13,7 +13,6 @@
// See the License for the specific language governing permissions and
// limitations under the License.
// </copyright>
using System.Collections.Generic;
using System.Diagnostics;

namespace OpenTelemetry.Trace
Expand All @@ -31,18 +30,11 @@ public abstract class ActivitySampler
/// <summary>
/// Checks whether activity needs to be created and tracked.
/// </summary>
/// <param name="parentContext">Parent activity context. Typically taken from the wire.</param>
/// <param name="traceId">Trace ID of a activity to be created.</param>
/// <param name="spanId">Span ID of a activity to be created.</param>
/// <param name="name"> Name (DisplayName) of the activity to be created. Note, that the name of the activity is settable.
/// So this name can be changed later and Sampler implementation should assume that.
/// Typical example of a name change is when <see cref="Activity"/> representing incoming http request
/// has a name of url path and then being updated with route name when routing complete.
/// <param name="samplingParameters">
/// The <see cref="ActivitySamplingParameters"/> used by the <see cref="ActivitySampler"/>
/// to decide if the <see cref="Activity"/> to be created is going to be sampled or not.
/// </param>
/// <param name="activityKind">The kind of the Activity.</param>
/// <param name="tags">Initial set of Tags for the Activity being constructed.</param>
/// <param name="links">Links associated with the activity.</param>
/// <returns>Sampling decision on whether activity needs to be sampled or not.</returns>
public abstract SamplingResult ShouldSample(in ActivityContext parentContext, in ActivityTraceId traceId, in ActivitySpanId spanId, string name, ActivityKind activityKind, IEnumerable<KeyValuePair<string, string>> tags, IEnumerable<ActivityLink> links);
public abstract SamplingResult ShouldSample(in ActivitySamplingParameters samplingParameters);
}
}
85 changes: 85 additions & 0 deletions src/OpenTelemetry/Trace/ActivitySamplingParameters.cs
Original file line number Diff line number Diff line change
@@ -0,0 +1,85 @@
// <copyright file="ActivitySamplingParameters.cs" company="OpenTelemetry Authors">
// Copyright The OpenTelemetry Authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
// </copyright>
using System.Collections.Generic;
using System.Diagnostics;

namespace OpenTelemetry.Trace
{
/// <summary>
/// Sampling parameters passed to an <see cref="ActivitySampler"/> for it to make a sampling decision.
/// </summary>
public readonly struct ActivitySamplingParameters
{
/// <summary>
/// Initializes a new instance of the <see cref="ActivitySamplingParameters"/> struct.
/// </summary>
/// <param name="parentContext">Parent activity context. Typically taken from the wire.</param>
/// <param name="traceId">Trace ID of a activity to be created.</param>
/// <param name="name">The name (DisplayName) of the activity to be created. Note, that the name of the activity is settable.
/// So this name can be changed later and Sampler implementation should assume that.
/// Typical example of a name change is when <see cref="Activity"/> representing incoming http request
/// has a name of url path and then being updated with route name when routing complete.
/// </param>
/// <param name="kind">The kind of the Activity to be created.</param>
/// <param name="tags">Initial set of Tags for the Activity being constructed.</param>
/// <param name="links">Links associated with the activity.</param>
public ActivitySamplingParameters(
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

when we fit sampling into GetRequestedDataUsingParentId (we current do only GetRequestedDataUsingContext), the sampling parameter will not have ActivityContext, but only the parentid.
(just posting a note for later revisit)

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Except by the TraceId this becomes ActivityCreationOptions<T> depending on how the other issues are handled we may be able to use it directly.

ActivityContext parentContext,
ActivityTraceId traceId,
string name,
ActivityKind kind,
IEnumerable<KeyValuePair<string, string>> tags = null, // TODO: Empty
IEnumerable<ActivityLink> links = null)
{
this.ParentContext = parentContext;
this.TraceId = traceId;
this.Name = name;
this.Kind = kind;
this.Tags = tags;
this.Links = links;
}

/// <summary>
/// Gets the parent activity context.
/// </summary>
public ActivityContext ParentContext { get; }

/// <summary>
/// Gets the trace ID of parent activity or a new generated one for root span/activity.
/// </summary>
public ActivityTraceId TraceId { get; }

/// <summary>
/// Gets the name to be given to the span/activity.
/// </summary>
public string Name { get; }

/// <summary>
/// Gets the kind of span/activity to be created.
/// </summary>
public ActivityKind Kind { get; }

/// <summary>
/// Gets the tags to be associated to the span/activity to be created.
/// </summary>
public IEnumerable<KeyValuePair<string, string>> Tags { get; }

/// <summary>
/// Gets the links to be added to the activity to be created.
/// </summary>
public IEnumerable<ActivityLink> Links { get; }
}
}
40 changes: 32 additions & 8 deletions src/OpenTelemetry/Trace/Configuration/OpenTelemetrySdk.cs
Original file line number Diff line number Diff line change
Expand Up @@ -84,14 +84,8 @@ public static IDisposable EnableOpenTelemetry(Action<OpenTelemetryBuilder> confi
// This prevents Activity from being created at all.
GetRequestedDataUsingContext = (ref ActivityCreationOptions<ActivityContext> options) =>
{
var shouldSample = sampler.ShouldSample(
options.Parent,
options.Parent.TraceId,
spanId: default, // Passing default SpanId here. The actual SpanId is not known before actual Activity creation
options.Name,
options.Kind,
options.Tags,
options.Links);
BuildSamplingParameters(options, out var samplingParameters);
var shouldSample = sampler.ShouldSample(samplingParameters);
if (shouldSample.IsSampled)
{
return ActivityDataRequest.AllDataAndRecorded;
Expand All @@ -109,5 +103,35 @@ public static IDisposable EnableOpenTelemetry(Action<OpenTelemetryBuilder> confi

return listener;
}

internal static void BuildSamplingParameters(
in ActivityCreationOptions<ActivityContext> options, out ActivitySamplingParameters samplingParameters)
{
ActivityContext parentContext = options.Parent;
if (parentContext == default)
{
// Check if there is already a parent for the current activity.
var parentActivity = Activity.Current;
if (parentActivity != null)
{
parentContext = parentActivity.Context;
}
}
Comment on lines +111 to +119
Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

PR dotnet/runtime#37185 removes the need for this part of the code.

Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I am not sure if you'll merge this OT PR as is till getting the .NET change. but if you do, please be aware that parentActivity.Context is possible to throw if the parent was created with defaulted trace id. my fix in .NET is taking care of this issue too to guarantee Activity.Context not throw under any condition.

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Thanks for the heads up @tarekgh - I think that for now it is better to go as it is so we can keep experimenting with the ActivitySource this code will be removed as soon as we have access to dotnet/runtime#37185


// This is not going to be the final traceId of the Activity (if one is created), however, it is
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I am not sure if this is correct (except when starting a brand new activity without parent)
If there is an active parent (explicit or Activity.Current), then the traceid for the parent will be used as traceid for the activity to be created.

If there was no parent, then the ActivityTraceId.CreateRandom(); you pass to Sampling will be different from actual TraceId of the Activity created.

Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

The unit test you have validates this. Can we state that TraceId is different only for the Root one scenario, and for everything else, the traceid is the correct one of the activity to be created.

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Agreed: the behavior here is a bit tricky. For the ProbabilityActivitySampler itself, it works because as you said it respects the sampled flag. However, in principle, someone may want to write a sampler that does its selection based on the actual bits of the traceId, and the one received by the sampler would not be the actual traceId of the root span. In other words per OTel spec, it is possible to write a sampler that selects based on traceId bits, ignoring the sampled flag, and still sampling complete traces.

Copy link
Contributor Author

@pjanotti pjanotti May 28, 2020

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Ideally, we should be able to pass the already generated traceId to the Activity to be created.

@tarekgh I remember that you had already stated this issue in other comment threads. Seeing it concretely here makes me wonder if we could force the Activity to be created to use the same one passed to the sampler.

Copy link
Contributor

@tarekgh tarekgh May 28, 2020

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Sorry I am not clear about the ask here. In the current APIs, you can create the parent context and pass it to ActivitySource.StartActivity, and we'll honor this context and the new Activity will be created with the trace Id passed in this parent context. I am seeing you have full control over how you want the new Activity will be created with which trace id.

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

@cijothomas @tarekgh Putting this together with https://github.com/open-telemetry/opentelemetry-dotnet/pull/702/files#r431881590 I think that we should consider if all work that is done here (BuildSamplingParameters method) should be done before calling the first GetRequestedDataUsingContext directly in System.Diagnostics.

The main downside that I see is that since the current Activity is only created if sampled we may need to add it to the AsyncLocal for the "current context" otherwise this will be creating a bunch of traceIds if there are nested activities not being sampled.

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Actually there is a bug with the code as it is: if the real "logical root" is not sampled it is possible that one of its "logical children" becomes sampled and it will show up as an incorrect "root". The proposal above should fix that.

Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Partially: it solves the handling of the parent in the case that the root span is sampled (see https://github.com/open-telemetry/opentelemetry-dotnet/pull/702/files#r432778065). However, it doesn't solve the case of not sampling the root/parent since Activity.Current is going to be null in such cases and the original traceId for the (not sampled) root span lost.

In order to make each step clear and easy to review I suggest the PR on dotnet/runtime (dotnet/runtime#37185) to be merged as it is since it fixes a specific issue on its own and we address the other part separately.

/cc @cijothomas @tarekgh

// needed in order for the sampling to work. This differs from other OTel SDKs in which it is
// the Sampler always receives the actual traceId of a root span/activity.
ActivityTraceId traceId = parentContext.TraceId != default
? parentContext.TraceId
: ActivityTraceId.CreateRandom();
pjanotti marked this conversation as resolved.
Show resolved Hide resolved

samplingParameters = new ActivitySamplingParameters(
parentContext,
traceId,
options.Name,
options.Kind,
options.Tags,
options.Links);
}
}
}
4 changes: 1 addition & 3 deletions src/OpenTelemetry/Trace/Samplers/AlwaysOffActivitySampler.cs
Original file line number Diff line number Diff line change
Expand Up @@ -13,8 +13,6 @@
// See the License for the specific language governing permissions and
// limitations under the License.
// </copyright>
using System.Collections.Generic;
using System.Diagnostics;

namespace OpenTelemetry.Trace.Samplers
{
Expand All @@ -27,7 +25,7 @@ public sealed class AlwaysOffActivitySampler : ActivitySampler
public override string Description { get; } = nameof(AlwaysOffActivitySampler);

/// <inheritdoc />
public override SamplingResult ShouldSample(in ActivityContext parentContext, in ActivityTraceId traceId, in ActivitySpanId spanId, string name, ActivityKind activityKind, IEnumerable<KeyValuePair<string, string>> tags, IEnumerable<ActivityLink> links)
public override SamplingResult ShouldSample(in ActivitySamplingParameters samplingParameters)
{
return new SamplingResult(false);
}
Expand Down
4 changes: 1 addition & 3 deletions src/OpenTelemetry/Trace/Samplers/AlwaysOnActivitySampler.cs
Original file line number Diff line number Diff line change
Expand Up @@ -13,8 +13,6 @@
// See the License for the specific language governing permissions and
// limitations under the License.
// </copyright>
using System.Collections.Generic;
using System.Diagnostics;

namespace OpenTelemetry.Trace.Samplers
{
Expand All @@ -28,7 +26,7 @@ public sealed class AlwaysOnActivitySampler : ActivitySampler
public override string Description { get; } = nameof(AlwaysOnActivitySampler);

/// <inheritdoc />
public override SamplingResult ShouldSample(in ActivityContext parentContext, in ActivityTraceId traceId, in ActivitySpanId spanId, string name, ActivityKind activityKind, IEnumerable<KeyValuePair<string, string>> tags, IEnumerable<ActivityLink> links)
public override SamplingResult ShouldSample(in ActivitySamplingParameters samplingParameters)
{
return new SamplingResult(true);
}
Expand Down
118 changes: 118 additions & 0 deletions src/OpenTelemetry/Trace/Samplers/ProbabilityActivitySampler.cs
Original file line number Diff line number Diff line change
@@ -0,0 +1,118 @@
// <copyright file="ProbabilityActivitySampler.cs" company="OpenTelemetry Authors">
// Copyright The OpenTelemetry Authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
// </copyright>
using System;
using System.Diagnostics;
using System.Globalization;

namespace OpenTelemetry.Trace.Samplers
{
/// <summary>
/// Sampler implementation which will take a sample if parent Activity or any linked Activity is sampled.
/// Otherwise, samples traces according to the specified probability.
/// </summary>
public sealed class ProbabilityActivitySampler : ActivitySampler
{
private readonly long idUpperBound;
private readonly double probability;

/// <summary>
/// Initializes a new instance of the <see cref="ProbabilityActivitySampler"/> class.
/// </summary>
/// <param name="probability">The desired probability of sampling. This must be between 0.0 and 1.0.
/// Higher the value, higher is the probability of a given Activity to be sampled in.
/// </param>
public ProbabilityActivitySampler(double probability)
{
if (probability < 0.0 || probability > 1.0)
{
throw new ArgumentOutOfRangeException(nameof(probability), "Probability must be in range [0.0, 1.0]");
}

this.probability = probability;

// The expected description is like ProbabilityActivitySampler{0.000100}
this.Description = "ProbabilityActivitySampler{" + this.probability.ToString("F6", CultureInfo.InvariantCulture) + "}";

// Special case the limits, to avoid any possible issues with lack of precision across
// double/long boundaries. For probability == 0.0, we use Long.MIN_VALUE as this guarantees
// that we will never sample a trace, even in the case where the id == Long.MIN_VALUE, since
// Math.Abs(Long.MIN_VALUE) == Long.MIN_VALUE.
if (this.probability == 0.0)
{
this.idUpperBound = long.MinValue;
}
else if (this.probability == 1.0)
{
this.idUpperBound = long.MaxValue;
}
else
{
this.idUpperBound = (long)(probability * long.MaxValue);
}
}

/// <inheritdoc />
public override string Description { get; }

/// <inheritdoc />
public override SamplingResult ShouldSample(in ActivitySamplingParameters samplingParameters)
{
// If the parent is sampled keep the sampling decision.
var parentContext = samplingParameters.ParentContext;
if ((parentContext.TraceFlags & ActivityTraceFlags.Recorded) != 0)
{
return new SamplingResult(true);
}

if (samplingParameters.Links != null)
{
// If any parent link is sampled keep the sampling decision.
foreach (var parentLink in samplingParameters.Links)
{
if ((parentLink.Context.TraceFlags & ActivityTraceFlags.Recorded) != 0)
{
return new SamplingResult(true);
}
}
}

// Always sample if we are within probability range. This is true even for child activities (that
// may have had a different sampling decision made) to allow for different sampling policies,
// and dynamic increases to sampling probabilities for debugging purposes.
// Note use of '<' for comparison. This ensures that we never sample for probability == 0.0,
// while allowing for a (very) small chance of *not* sampling if the id == Long.MAX_VALUE.
// This is considered a reasonable trade-off for the simplicity/performance requirements (this
// code is executed in-line for every Activity creation).
Span<byte> traceIdBytes = stackalloc byte[16];
samplingParameters.TraceId.CopyTo(traceIdBytes);
return Math.Abs(this.GetLowerLong(traceIdBytes)) < this.idUpperBound ? new SamplingResult(true) : new SamplingResult(false);
}

private long GetLowerLong(ReadOnlySpan<byte> bytes)
{
long result = 0;
for (var i = 0; i < 8; i++)
{
result <<= 8;
#pragma warning disable CS0675 // Bitwise-or operator used on a sign-extended operand
result |= bytes[i] & 0xff;
#pragma warning restore CS0675 // Bitwise-or operator used on a sign-extended operand
}

return result;
}
}
}