Skip to content

Commit

Permalink
fix: acquire lock before adding dropped spans
Browse files Browse the repository at this point in the history
Spans from the same transaction share the dropped spans map.
If they end concurrency a race condition could happen when updating
the map.
Add a RW lock to prevent that.
  • Loading branch information
kruskall committed Jul 11, 2023
1 parent 326e364 commit 7fe3968
Show file tree
Hide file tree
Showing 2 changed files with 25 additions and 14 deletions.
8 changes: 5 additions & 3 deletions modelwriter.go
Original file line number Diff line number Diff line change
Expand Up @@ -276,9 +276,11 @@ func normalizeOutcome(outcome string) string {
}
}

func buildDroppedSpansStats(dss droppedSpanTimingsMap) []model.DroppedSpansStats {
out := make([]model.DroppedSpansStats, 0, len(dss))
for k, timing := range dss {
func buildDroppedSpansStats(dss *droppedSpanTimingsMap) []model.DroppedSpansStats {
dss.mu.RLock()
defer dss.mu.RUnlock()
out := make([]model.DroppedSpansStats, 0, len(dss.m))
for k, timing := range dss.m {
out = append(out, model.DroppedSpansStats{
DestinationServiceResource: k.destination,
ServiceTargetType: k.serviceTargetType,
Expand Down
31 changes: 20 additions & 11 deletions transaction.go
Original file line number Diff line number Diff line change
Expand Up @@ -49,8 +49,10 @@ func (t *Tracer) StartTransactionOptions(name, transactionType string, opts Tran
Context: Context{
captureBodyMask: CaptureBodyTransactions,
},
spanTimings: make(spanTimingsMap),
droppedSpansStats: make(droppedSpanTimingsMap, maxDroppedSpanStats),
spanTimings: make(spanTimingsMap),
droppedSpansStats: &droppedSpanTimingsMap{
m: make(map[droppedSpanTimingsKey]spanTiming, maxDroppedSpanStats),
},
}
var seed int64
if err := binary.Read(cryptorand.Reader, binary.LittleEndian, &seed); err != nil {
Expand Down Expand Up @@ -409,7 +411,7 @@ type TransactionData struct {
spansDropped int
childrenTimer childrenTimer
spanTimings spanTimingsMap
droppedSpansStats droppedSpanTimingsMap
droppedSpansStats *droppedSpanTimingsMap
rand *rand.Rand // for ID generation

compressedSpan compressedSpan
Expand Down Expand Up @@ -439,28 +441,35 @@ type droppedSpanTimingsKey struct {
}

// droppedSpanTimingsMap records span timings for groups of dropped spans.
type droppedSpanTimingsMap map[droppedSpanTimingsKey]spanTiming
type droppedSpanTimingsMap struct {
m map[droppedSpanTimingsKey]spanTiming
mu sync.RWMutex
}

// add accumulates the timing for a {destination, outcome} pair, silently drops
// any pairs that would cause the map to exceed the maxDroppedSpanStats.
func (m droppedSpanTimingsMap) add(targetType, targetName, dst, outcome string, count int, d time.Duration) {
func (m *droppedSpanTimingsMap) add(targetType, targetName, dst, outcome string, count int, d time.Duration) {
k := droppedSpanTimingsKey{
serviceTargetType: targetType,
serviceTargetName: targetName,
destination: dst,
outcome: outcome,
}
timing, ok := m[k]
if ok || maxDroppedSpanStats > len(m) {
m.mu.Lock()
defer m.mu.Unlock()
timing, ok := m.m[k]
if ok || maxDroppedSpanStats > len(m.m) {
timing.count += uint64(count)
timing.duration += int64(d)
m[k] = timing
m.m[k] = timing
}
}

// reset resets m back to its initial zero state.
func (m droppedSpanTimingsMap) reset() {
for k := range m {
delete(m, k)
func (m *droppedSpanTimingsMap) reset() {
m.mu.Lock()
defer m.mu.Unlock()
for k := range m.m {
delete(m.m, k)
}
}

0 comments on commit 7fe3968

Please sign in to comment.