fix: optimize the usage of ApiTraceGraph in the ingestion pipeline #195
There are no files selected for viewing
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,31 @@ | ||
| package org.hypertrace.traceenricher.trace.util; | ||
|
|
||
| import java.time.Duration; | ||
| import java.time.Instant; | ||
| import java.util.concurrent.TimeUnit; | ||
| import org.hypertrace.core.datamodel.StructuredTrace; | ||
| import org.slf4j.Logger; | ||
| import org.slf4j.LoggerFactory; | ||
|
|
||
| public class ApiTraceGraphBuilder { | ||
| private static final Logger LOG = LoggerFactory.getLogger(ApiTraceGraphBuilder.class); | ||
|
|
||
| private static ThreadLocal<ApiTraceGraph> cachedGraph = new ThreadLocal<>(); | ||
| private static ThreadLocal<StructuredTrace> cachedTrace = new ThreadLocal<>(); | ||
|
|
||
| public static ApiTraceGraph buildGraph(StructuredTrace trace) { | ||
| if (!GraphBuilderUtil.isSameStructuredTrace(cachedTrace.get(), trace)) { | ||
| Instant start = Instant.now(); | ||
| ApiTraceGraph graph = new ApiTraceGraph(trace); | ||
| LOG.debug( | ||
| "Time taken in building ApiTraceGraph:{} for tenantId:{}", | ||
| Duration.between(start, Instant.now()).toMillis(), | ||
| TimeUnit.MILLISECONDS, | ||
| trace.getCustomerId()); | ||
| cachedTrace.set(StructuredTrace.newBuilder(trace).build()); | ||
| cachedGraph.set(graph); | ||
| return graph; | ||
| } | ||
| return cachedGraph.get(); | ||
| } | ||
| } | ||
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,40 @@ | ||
| package org.hypertrace.traceenricher.trace.util; | ||
|
|
||
| import org.hypertrace.core.datamodel.StructuredTrace; | ||
| import org.slf4j.Logger; | ||
| import org.slf4j.LoggerFactory; | ||
|
|
||
| public class GraphBuilderUtil { | ||
|
Contributor
There was a problem hiding this comment. Is it possible to add unit tests for this util? |
||
| private static final Logger LOG = LoggerFactory.getLogger(GraphBuilderUtil.class); | ||
|
|
||
| /** | ||
| * optimistic method of comparing two trace for considering rebuilding of entire graph structure. | ||
| */ | ||
| static boolean isSameStructuredTrace(StructuredTrace cachedTrace, StructuredTrace trace) { | ||
|
Contributor
There was a problem hiding this comment. the method name could be misleading, we are trying to check if the components in the Trace are same as previous or not. Trace might have changed due to event enrichment or trace enrichment. |
||
| if (cachedTrace == null || trace == null) { | ||
| LOG.debug("Cached and Input trace are not same. Reason: one of the input is null"); | ||
| return false; | ||
| } | ||
| // is processed and cached are same trace? | ||
| if (!cachedTrace.getCustomerId().equals(trace.getCustomerId()) | ||
| || !cachedTrace.getTraceId().equals(trace.getTraceId())) { | ||
| LOG.debug( | ||
| "Cached and Input trace are not same. Reason: doesn't not match either traceId or tenantId"); | ||
| return false; | ||
| } | ||
|
|
||
| // trace internally changed (full trace comparision is costly, so we are doing only with | ||
| // required fields) | ||
| if (cachedTrace.getEntityList().size() != trace.getEntityList().size() | ||
| || cachedTrace.getEventList().size() != trace.getEventList().size() | ||
| || cachedTrace.getEntityEdgeList().size() != trace.getEntityEdgeList().size() | ||
| || cachedTrace.getEntityEventEdgeList().size() != trace.getEntityEventEdgeList().size() | ||
| || cachedTrace.getEventEdgeList().size() != trace.getEventEdgeList().size()) { | ||
| LOG.debug( | ||
| "Cached and Input trace are not same. Reason: they are having different size either for event or entities"); | ||
| return false; | ||
| } | ||
|
|
||
| return true; | ||
| } | ||
| } | ||
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,62 @@ | ||
| package org.hypertrace.traceenricher.trace.util; | ||
|
|
||
| import static org.mockito.Mockito.mock; | ||
| import static org.mockito.Mockito.mockConstruction; | ||
| import static org.mockito.Mockito.mockStatic; | ||
| import static org.mockito.Mockito.when; | ||
|
|
||
| import java.nio.ByteBuffer; | ||
| import java.util.List; | ||
| import org.hypertrace.core.datamodel.Edge; | ||
| import org.hypertrace.core.datamodel.Entity; | ||
| import org.hypertrace.core.datamodel.Event; | ||
| import org.hypertrace.core.datamodel.StructuredTrace; | ||
| import org.junit.jupiter.api.Assertions; | ||
| import org.junit.jupiter.api.Test; | ||
| import org.mockito.MockedConstruction; | ||
| import org.mockito.MockedStatic; | ||
|
|
||
| public class ApiTraceGraphBuilderTest { | ||
|
|
||
| @Test | ||
| void testBuildGraph() { | ||
|
|
||
| Entity entity = mock(Entity.class); | ||
| Event parent = mock(Event.class); | ||
| Event child = mock(Event.class); | ||
| Edge eventEdge = mock(Edge.class); | ||
|
|
||
| StructuredTrace underTestTrace = mock(StructuredTrace.class); | ||
| when(underTestTrace.getCustomerId()).thenReturn("__defaultTenant"); | ||
| when(underTestTrace.getTraceId()).thenReturn(ByteBuffer.wrap("2ebbc19b6428510f".getBytes())); | ||
| when(underTestTrace.getEntityList()).thenReturn(List.of(entity)); | ||
| when(underTestTrace.getEventList()).thenReturn(List.of(parent, child)); | ||
| when(underTestTrace.getEntityEdgeList()).thenReturn(List.of()); | ||
| when(underTestTrace.getEntityEventEdgeList()).thenReturn(List.of()); | ||
| when(underTestTrace.getEventEdgeList()).thenReturn(List.of(eventEdge)); | ||
|
|
||
| // structure trace builder | ||
| try (MockedStatic<StructuredTrace> builderMockedStatic = mockStatic(StructuredTrace.class)) { | ||
| StructuredTrace.Builder builder = mock(StructuredTrace.Builder.class); | ||
| when(builder.build()).thenReturn(underTestTrace); | ||
|
|
||
| builderMockedStatic | ||
| .when(() -> StructuredTrace.newBuilder(underTestTrace)) | ||
| .thenReturn(builder); | ||
|
|
||
| // make two calls, and check that first call create cache entries, and second call uses same | ||
| try (MockedConstruction<ApiTraceGraph> mockedConstruction = | ||
| mockConstruction(ApiTraceGraph.class)) { | ||
| // first call | ||
| ApiTraceGraph actual = ApiTraceGraphBuilder.buildGraph(underTestTrace); | ||
| Assertions.assertNotNull(actual); | ||
| Assertions.assertEquals(1, mockedConstruction.constructed().size()); | ||
|
Contributor
There was a problem hiding this comment. what is this
Contributor
Author
There was a problem hiding this comment. The number of times |
||
|
|
||
| // second call | ||
| ApiTraceGraph second = ApiTraceGraphBuilder.buildGraph(underTestTrace); | ||
| Assertions.assertEquals(actual, second); | ||
| Assertions.assertEquals(1, mockedConstruction.constructed().size()); | ||
|
Contributor
There was a problem hiding this comment. I think, its also important to test that the cached graph is not returned when a different trace is passed?
Contributor
Author
There was a problem hiding this comment. The first call is testing that we are going to if block where the cache is prepared. And |
||
| } | ||
| } | ||
| } | ||
| } | ||
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,88 @@ | ||
| package org.hypertrace.traceenricher.trace.util; | ||
|
|
||
| import static org.mockito.Mockito.mock; | ||
| import static org.mockito.Mockito.when; | ||
|
|
||
| import java.nio.ByteBuffer; | ||
| import java.util.List; | ||
| import org.hypertrace.core.datamodel.Edge; | ||
| import org.hypertrace.core.datamodel.Entity; | ||
| import org.hypertrace.core.datamodel.Event; | ||
| import org.hypertrace.core.datamodel.StructuredTrace; | ||
| import org.junit.jupiter.api.Assertions; | ||
| import org.junit.jupiter.api.Test; | ||
|
|
||
| public class GraphBuilderUtilTest { | ||
|
|
||
| @Test | ||
| public void testSameTraceForNullInputs() { | ||
| StructuredTrace cachedTrace = mock(StructuredTrace.class); | ||
| StructuredTrace underTestTrace = mock(StructuredTrace.class); | ||
|
|
||
| boolean result = GraphBuilderUtil.isSameStructuredTrace(null, null); | ||
| Assertions.assertFalse(result); | ||
|
|
||
| result = GraphBuilderUtil.isSameStructuredTrace(null, underTestTrace); | ||
| Assertions.assertFalse(result); | ||
|
|
||
| result = GraphBuilderUtil.isSameStructuredTrace(cachedTrace, null); | ||
| Assertions.assertFalse(result); | ||
| } | ||
|
|
||
| @Test | ||
| public void testSameTraceForTenantAndTraceCondition() { | ||
| // different tenant id | ||
| StructuredTrace cachedTrace = mock(StructuredTrace.class); | ||
| when(cachedTrace.getCustomerId()).thenReturn("__defaultTenant"); | ||
| when(cachedTrace.getTraceId()).thenReturn(ByteBuffer.wrap("2ebbc19b6428510f".getBytes())); | ||
|
|
||
| StructuredTrace underTestTrace = mock(StructuredTrace.class); | ||
| when(underTestTrace.getCustomerId()).thenReturn("__defaultTenantUnderTest"); | ||
| when(underTestTrace.getTraceId()).thenReturn(ByteBuffer.wrap("2ebbc19b6428510f".getBytes())); | ||
|
|
||
| boolean result = GraphBuilderUtil.isSameStructuredTrace(cachedTrace, underTestTrace); | ||
| Assertions.assertFalse(result); | ||
|
|
||
| // different trace ids | ||
| cachedTrace = mock(StructuredTrace.class); | ||
| when(cachedTrace.getCustomerId()).thenReturn("__defaultTenant"); | ||
| when(cachedTrace.getTraceId()).thenReturn(ByteBuffer.wrap("2ebbc19b6428510f".getBytes())); | ||
|
|
||
| underTestTrace = mock(StructuredTrace.class); | ||
| when(underTestTrace.getCustomerId()).thenReturn("__defaultTenant"); | ||
| when(underTestTrace.getTraceId()).thenReturn(ByteBuffer.wrap("2ebbc19b6428511f".getBytes())); | ||
|
|
||
| result = GraphBuilderUtil.isSameStructuredTrace(cachedTrace, underTestTrace); | ||
| Assertions.assertFalse(result); | ||
| } | ||
|
|
||
| @Test | ||
| public void testSameTraceForSizeCondition() { | ||
| Entity entity = mock(Entity.class); | ||
| Event parent = mock(Event.class); | ||
| Event child = mock(Event.class); | ||
| Edge eventEdge = mock(Edge.class); | ||
|
|
||
| // same size | ||
| StructuredTrace cachedTrace = mock(StructuredTrace.class); | ||
| when(cachedTrace.getCustomerId()).thenReturn("__defaultTenant"); | ||
| when(cachedTrace.getTraceId()).thenReturn(ByteBuffer.wrap("2ebbc19b6428510f".getBytes())); | ||
| when(cachedTrace.getEntityList()).thenReturn(List.of(entity)); | ||
| when(cachedTrace.getEventList()).thenReturn(List.of(parent, child)); | ||
| when(cachedTrace.getEntityEdgeList()).thenReturn(List.of()); | ||
| when(cachedTrace.getEntityEventEdgeList()).thenReturn(List.of()); | ||
| when(cachedTrace.getEventEdgeList()).thenReturn(List.of(eventEdge)); | ||
|
|
||
| StructuredTrace underTestTrace = mock(StructuredTrace.class); | ||
| when(underTestTrace.getCustomerId()).thenReturn("__defaultTenant"); | ||
| when(underTestTrace.getTraceId()).thenReturn(ByteBuffer.wrap("2ebbc19b6428510f".getBytes())); | ||
| when(underTestTrace.getEntityList()).thenReturn(List.of(entity)); | ||
| when(underTestTrace.getEventList()).thenReturn(List.of(parent, child)); | ||
| when(underTestTrace.getEntityEdgeList()).thenReturn(List.of()); | ||
| when(underTestTrace.getEntityEventEdgeList()).thenReturn(List.of()); | ||
| when(underTestTrace.getEventEdgeList()).thenReturn(List.of(eventEdge)); | ||
|
|
||
| boolean result = GraphBuilderUtil.isSameStructuredTrace(cachedTrace, underTestTrace); | ||
| Assertions.assertTrue(result); | ||
|
Contributor
There was a problem hiding this comment. Test the negative case as well?
Contributor
Author
There was a problem hiding this comment. The remains are covered with the other two tests, and this I have created as an end to cover up all. But, if you want, I can add one to the list item.
Contributor
There was a problem hiding this comment. I think it can be taken up subsequently |
||
| } | ||
| } | ||
Oops, something went wrong.
Add this suggestion to a batch that can be applied as a single commit.
This suggestion is invalid because no changes were made to the code.
Suggestions cannot be applied while the pull request is closed.
Suggestions cannot be applied while viewing a subset of changes.
Only one suggestion per line can be applied in a batch.
Add this suggestion to a batch that can be applied as a single commit.
Applying suggestions on deleted lines is not supported.
You must change the existing code in this line in order to create a valid suggestion.
Outdated suggestions cannot be applied.
This suggestion has been applied or marked resolved.
Suggestions cannot be applied from pending reviews.
Suggestions cannot be applied on multi-line comments.
Suggestions cannot be applied while the pull request is queued to merge.
Suggestion cannot be applied right now. Please check back later.
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
unit tests?