Skip to content

refactor: extract backend read write and identify #219

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 1 commit into from
Jul 11, 2021
Merged

refactor: extract backend read write and identify #219

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 2 additions & 2 deletions hypertrace-trace-enricher/hypertrace-trace-enricher-impl/build.gradle.kts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -19,9 +19,9 @@ dependencies {
implementation("org.hypertrace.core.datamodel:data-model:0.1.16")
implementation("org.hypertrace.entity.service:entity-service-client:0.6.8")
implementation("org.hypertrace.core.serviceframework:platform-metrics:0.1.26")
implementation("org.hypertrace.core.grpcutils:grpc-client-utils:0.4.0")
implementation("org.hypertrace.core.grpcutils:grpc-client-utils:0.5.2")
implementation("org.hypertrace.config.service:spaces-config-service-api:0.1.0")
implementation("org.hypertrace.core.grpcutils:grpc-context-utils:0.4.0")
implementation("org.hypertrace.core.grpcutils:grpc-context-utils:0.5.2")

implementation("com.typesafe:config:1.4.1")
implementation("org.apache.httpcomponents:httpclient:4.5.13")
Expand Down
26 changes: 16 additions & 10 deletions .../hypertrace/traceenricher/enrichment/enrichers/backend/AbstractBackendEntityEnricher.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -19,6 +19,7 @@
import org.hypertrace.core.datamodel.shared.SpanAttributeUtils;
import org.hypertrace.core.datamodel.shared.StructuredTraceGraph;
import org.hypertrace.core.datamodel.shared.trace.AttributeValueCreator;
import org.hypertrace.core.grpcutils.context.RequestContext;
import org.hypertrace.entity.constants.v1.BackendAttribute;
import org.hypertrace.entity.data.service.client.EdsClient;
import org.hypertrace.entity.data.service.v1.AttributeValue;
Expand Down Expand Up @@ -95,7 +96,7 @@ public void enrichTrace(StructuredTrace trace) {
EnrichedSpanUtils.isExitSpan(event)
&& SpanAttributeUtils.isLeafSpan(structuredTraceGraph, event))
// resolve backend entity
.map(event -> Pair.of(event, resolve(event, structuredTraceGraph)))
.map(event -> Pair.of(event, resolve(event, trace, structuredTraceGraph)))
.filter(pair -> pair.getRight().isPresent())
// check if backend entity is valid
.filter(pair -> isValidBackendEntity(pair.getLeft(), pair.getRight().get()))
Expand Down Expand Up @@ -235,16 +236,15 @@ private Map<String, org.hypertrace.core.datamodel.AttributeValue> getAttributesT

@Nullable
private Entity createBackendIfMissing(Entity backendEntity) {
RequestContext requestContext = RequestContext.forTenantId(backendEntity.getTenantId());
try {
Optional<Entity> backendFromCache =
entityCache
.getBackendIdAttrsToEntityCache()
.get(
Pair.of(
backendEntity.getTenantId(), backendEntity.getIdentifyingAttributesMap()));
.get(requestContext.buildContextualKey(backendEntity.getIdentifyingAttributesMap()));
return backendFromCache.orElseGet(
() -> {
Entity result = edsClient.upsert(backendEntity);
Entity result = this.upsertBackend(backendEntity);
LOGGER.info("Created backend:{}", result);
return result;
});
Expand All @@ -259,7 +259,8 @@ private org.hypertrace.core.datamodel.AttributeValue createAttributeValue(Attrib
}

@VisibleForTesting
public Optional<BackendInfo> resolve(Event event, StructuredTraceGraph structuredTraceGraph) {
public Optional<BackendInfo> resolve(
Event event, StructuredTrace trace, StructuredTraceGraph structuredTraceGraph) {
for (BackendProvider backendProvider : getBackendProviders()) {
backendProvider.init(event);

Expand All @@ -280,7 +281,7 @@ public Optional<BackendInfo> resolve(Event event, StructuredTraceGraph structure
}

String backendUri = maybeBackendUri.get();
final Builder entityBuilder = getBackendEntityBuilder(type, backendUri, event);
final Builder entityBuilder = getBackendEntityBuilder(type, backendUri, event, trace);
backendProvider.getEntityAttributes(event).forEach(entityBuilder::putAttributes);

Map<String, org.hypertrace.core.datamodel.AttributeValue> enrichedAttributes =
Expand All @@ -303,14 +304,15 @@ public Optional<BackendInfo> resolve(Event event, StructuredTraceGraph structure
return Optional.empty();
}

private Builder getBackendEntityBuilder(BackendType type, String backendURI, Event event) {
protected Builder getBackendEntityBuilder(
BackendType type, String backendURI, Event event, StructuredTrace trace) {
Copy link
Contributor

@kotharironak kotharironak Jul 10, 2021

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Currently, the newly passed argument -StructuredTrace trace is not used. I am assuming it requires a follow-up, right?

Copy link
Contributor Author

@aaron-steinfeld aaron-steinfeld Jul 10, 2021 •
edited
Loading

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

It's not currently used in the abstract version, no - but as we migrate attribute retrieval from utils to the reader pattern, the reader always requires the trace.

String[] hostAndPort = backendURI.split(COLON);
String host = hostAndPort[0];
String port = (hostAndPort.length == 2) ? hostAndPort[1] : DEFAULT_PORT;
String port = hostAndPort.length == 2 ? hostAndPort[1] : DEFAULT_PORT;
String fqn = fqnResolver.resolve(host, event);
String entityName = port.equals(DEFAULT_PORT) ? fqn : COLON_JOINER.join(fqn, port);
final Builder entityBuilder =
org.hypertrace.entity.data.service.v1.Entity.newBuilder()
Entity.newBuilder()
.setEntityType(EntityType.BACKEND.name())
.setTenantId(event.getCustomerId())
.setEntityName(entityName)
Expand All @@ -328,4 +330,8 @@ private Builder getBackendEntityBuilder(BackendType type, String backendURI, Eve
EnricherUtil.createAttributeValue(HexUtils.getHex(event.getEventId())));
return entityBuilder;
}

protected Entity upsertBackend(Entity backendEntity) {
return edsClient.upsert(backendEntity);
}
}
32 changes: 16 additions & 16 deletions ...pl/src/main/java/org/hypertrace/traceenricher/enrichment/enrichers/cache/EntityCache.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -9,6 +9,7 @@
import java.util.concurrent.TimeUnit;
import javax.annotation.Nonnull;
import org.apache.commons.lang3.tuple.Pair;
import org.hypertrace.core.grpcutils.context.ContextualKey;
import org.hypertrace.entity.constants.v1.CommonAttribute;
import org.hypertrace.entity.data.service.client.EdsClient;
import org.hypertrace.entity.data.service.v1.AttributeValue;
Expand Down Expand Up @@ -89,25 +90,12 @@ public List<Entity> load(@Nonnull Pair<String, String> key) {
* Cache of Backend identifying attributes to Entity Key: Map of identifying attributes Value:
* Optional Backend entity
*/
private final LoadingCache<Pair<String, Map<String, AttributeValue>>, Optional<Entity>>
private final LoadingCache<ContextualKey<Map<String, AttributeValue>>, Optional<Entity>>
backendIdAttrsToEntityCache =
CacheBuilder.newBuilder()
.maximumSize(10000)
.expireAfterWrite(5, TimeUnit.MINUTES)
.build(
new CacheLoader<>() {
@Override
public Optional<Entity> load(
@Nonnull Pair<String, Map<String, AttributeValue>> pair) {
ByTypeAndIdentifyingAttributes request =
ByTypeAndIdentifyingAttributes.newBuilder()
.setEntityType(EntityType.BACKEND.name())
.putAllIdentifyingAttributes(pair.getRight())
.build();
return Optional.ofNullable(
edsClient.getByTypeAndIdentifyingAttributes(pair.getLeft(), request));
}
});
.build(CacheLoader.from(this::loadBackendFromIdentifyingAttributes));

public EntityCache(EdsClient edsClient) {
this.edsClient = edsClient;
Expand All @@ -125,8 +113,20 @@ public LoadingCache<Pair<String, String>, List<Entity>> getNameToNamespaceEntity
return namespaceCache;
}

public LoadingCache<Pair<String, Map<String, AttributeValue>>, Optional<Entity>>
public LoadingCache<ContextualKey<Map<String, AttributeValue>>, Optional<Entity>>
getBackendIdAttrsToEntityCache() {
return backendIdAttrsToEntityCache;
}

protected Optional<Entity> loadBackendFromIdentifyingAttributes(
ContextualKey<Map<String, AttributeValue>> key) {
ByTypeAndIdentifyingAttributes request =
ByTypeAndIdentifyingAttributes.newBuilder()
.setEntityType(EntityType.BACKEND.name())
.putAllIdentifyingAttributes(key.getData())
.build();
return Optional.ofNullable(
edsClient.getByTypeAndIdentifyingAttributes(
key.getContext().getTenantId().orElseThrow(), request));
}
}
15 changes: 12 additions & 3 deletions ...ace/traceenricher/enrichment/enrichers/backend/provider/CassandraBackendProviderTest.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -16,6 +16,7 @@
import org.hypertrace.core.datamodel.EventRefType;
import org.hypertrace.core.datamodel.MetricValue;
import org.hypertrace.core.datamodel.Metrics;
import org.hypertrace.core.datamodel.StructuredTrace;
import org.hypertrace.core.datamodel.shared.StructuredTraceGraph;
import org.hypertrace.core.datamodel.shared.trace.AttributeValueCreator;
import org.hypertrace.entity.data.service.v1.Entity;
Expand All @@ -32,19 +33,23 @@ public class CassandraBackendProviderTest {

private AbstractBackendEntityEnricher backendEntityEnricher;
private StructuredTraceGraph structuredTraceGraph;
private StructuredTrace structuredTrace;

@BeforeEach
public void setup() {
backendEntityEnricher = new MockBackendEntityEnricher();
backendEntityEnricher.init(ConfigFactory.empty(), mock(ClientRegistry.class));

structuredTraceGraph = mock(StructuredTraceGraph.class);
structuredTrace = mock(StructuredTrace.class);
}

@Test
public void testBackendResolution() {
BackendInfo backendInfo =
backendEntityEnricher.resolve(getCassandraEvent(), structuredTraceGraph).get();
backendEntityEnricher
.resolve(getCassandraEvent(), structuredTrace, structuredTraceGraph)
.get();
Entity entity = backendInfo.getEntity();
Assertions.assertEquals("localhost:9000", entity.getEntityName());
Map<String, AttributeValue> attributes = backendInfo.getAttributes();
Expand All @@ -60,7 +65,9 @@ public void testBackendResolution() {
@Test
public void testBackendResolutionForOTEvent() {
BackendInfo backendInfo =
backendEntityEnricher.resolve(getCassandraOTEvent(), structuredTraceGraph).get();
backendEntityEnricher
.resolve(getCassandraOTEvent(), structuredTrace, structuredTraceGraph)
.get();
Entity entity = backendInfo.getEntity();
Assertions.assertEquals("localhost:9000", entity.getEntityName());
Map<String, AttributeValue> attributes = backendInfo.getAttributes();
Expand All @@ -70,7 +77,9 @@ public void testBackendResolutionForOTEvent() {
@Test
public void testBackendResolutionWithoutConnectionString() {
BackendInfo backendInfo =
backendEntityEnricher.resolve(getCassandraOTelEvent(), structuredTraceGraph).get();
backendEntityEnricher
.resolve(getCassandraOTelEvent(), structuredTrace, structuredTraceGraph)
.get();
Entity entity = backendInfo.getEntity();
Assertions.assertEquals("test:9000", entity.getEntityName());
Map<String, AttributeValue> attributes = backendInfo.getAttributes();
Expand Down
9 changes: 7 additions & 2 deletions ...e/traceenricher/enrichment/enrichers/backend/provider/ClientSpanEndpointProviderTest.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -17,6 +17,7 @@
import org.hypertrace.core.datamodel.EventRefType;
import org.hypertrace.core.datamodel.MetricValue;
import org.hypertrace.core.datamodel.Metrics;
import org.hypertrace.core.datamodel.StructuredTrace;
import org.hypertrace.core.datamodel.shared.StructuredTraceGraph;
import org.hypertrace.entity.constants.v1.BackendAttribute;
import org.hypertrace.entity.constants.v1.ServiceAttribute;
Expand All @@ -37,12 +38,14 @@ public class ClientSpanEndpointProviderTest {

private AbstractBackendEntityEnricher backendEntityEnricher;
private StructuredTraceGraph structuredTraceGraph;
private StructuredTrace structuredTrace;

@BeforeEach
public void setup() {
backendEntityEnricher = new MockBackendEntityEnricher();
backendEntityEnricher.init(ConfigFactory.empty(), mock(ClientRegistry.class));

structuredTrace = mock(StructuredTrace.class);
structuredTraceGraph = mock(StructuredTraceGraph.class);
}

Expand Down Expand Up @@ -115,7 +118,8 @@ public void checkFallbackBackendEntityGeneratedFromClientExitSpan() {
.build();

when(structuredTraceGraph.getParentEvent(e)).thenReturn(parentEvent);
Entity backendEntity = backendEntityEnricher.resolve(e, structuredTraceGraph).get().getEntity();
Entity backendEntity =
backendEntityEnricher.resolve(e, structuredTrace, structuredTraceGraph).get().getEntity();
assertEquals(
"redis",
backendEntity
Expand Down Expand Up @@ -197,7 +201,8 @@ public void checkFallbackBackendEntityGeneratedFromClientExitSpan() {
// Since the event serviceName is same as parentServiceName, no new service entity should be
// created.
when(structuredTraceGraph.getParentEvent(e)).thenReturn(parentEvent);
Assertions.assertTrue(backendEntityEnricher.resolve(e, structuredTraceGraph).isEmpty());
Assertions.assertTrue(
backendEntityEnricher.resolve(e, structuredTrace, structuredTraceGraph).isEmpty());
}

static class MockBackendEntityEnricher extends AbstractBackendEntityEnricher {
Expand Down
15 changes: 12 additions & 3 deletions ...traceenricher/enrichment/enrichers/backend/provider/ElasticsearchBackendProviderTest.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -16,6 +16,7 @@
import org.hypertrace.core.datamodel.EventRefType;
import org.hypertrace.core.datamodel.MetricValue;
import org.hypertrace.core.datamodel.Metrics;
import org.hypertrace.core.datamodel.StructuredTrace;
import org.hypertrace.core.datamodel.shared.StructuredTraceGraph;
import org.hypertrace.core.datamodel.shared.trace.AttributeValueCreator;
import org.hypertrace.entity.data.service.v1.Entity;
Expand All @@ -31,19 +32,23 @@
public class ElasticsearchBackendProviderTest {
private AbstractBackendEntityEnricher backendEntityEnricher;
private StructuredTraceGraph structuredTraceGraph;
private StructuredTrace structuredTrace;

@BeforeEach
public void setup() {
backendEntityEnricher = new MockBackendEntityEnricher();
backendEntityEnricher.init(ConfigFactory.empty(), mock(ClientRegistry.class));

structuredTrace = mock(StructuredTrace.class);
structuredTraceGraph = mock(StructuredTraceGraph.class);
}

@Test
public void testBackendResolution() {
BackendInfo backendInfo =
backendEntityEnricher.resolve(getElasticsearchEvent(), structuredTraceGraph).get();
backendEntityEnricher
.resolve(getElasticsearchEvent(), structuredTrace, structuredTraceGraph)
.get();
Entity entity = backendInfo.getEntity();
Assertions.assertEquals("test-index", entity.getEntityName());
Map<String, AttributeValue> attributes = backendInfo.getAttributes();
Expand All @@ -59,7 +64,9 @@ public void testBackendResolution() {
@Test
public void testBackendResolutionForOTEvent() {
BackendInfo backendInfo =
backendEntityEnricher.resolve(getElasticsearchOTEvent(), structuredTraceGraph).get();
backendEntityEnricher
.resolve(getElasticsearchOTEvent(), structuredTrace, structuredTraceGraph)
.get();
Entity entity = backendInfo.getEntity();
Assertions.assertEquals("test", entity.getEntityName());
Map<String, AttributeValue> attributes = backendInfo.getAttributes();
Expand All @@ -75,7 +82,9 @@ public void testBackendResolutionForOTEvent() {
@Test
public void testBackendResolutionForOTelEvent() {
BackendInfo backendInfo =
backendEntityEnricher.resolve(getElasticsearchOTelEvent(), structuredTraceGraph).get();
backendEntityEnricher
.resolve(getElasticsearchOTelEvent(), structuredTrace, structuredTraceGraph)
.get();
Entity entity = backendInfo.getEntity();
Assertions.assertEquals("test:2000", entity.getEntityName());
Map<String, AttributeValue> attributes = backendInfo.getAttributes();
Expand Down
6 changes: 5 additions & 1 deletion ...pertrace/traceenricher/enrichment/enrichers/backend/provider/GrpcBackendProviderTest.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -17,6 +17,7 @@
import org.hypertrace.core.datamodel.EventRefType;
import org.hypertrace.core.datamodel.MetricValue;
import org.hypertrace.core.datamodel.Metrics;
import org.hypertrace.core.datamodel.StructuredTrace;
import org.hypertrace.core.datamodel.shared.StructuredTraceGraph;
import org.hypertrace.core.datamodel.shared.trace.AttributeValueCreator;
import org.hypertrace.core.span.constants.v1.Grpc;
Expand All @@ -35,12 +36,14 @@
public class GrpcBackendProviderTest {
private AbstractBackendEntityEnricher backendEntityEnricher;
private StructuredTraceGraph structuredTraceGraph;
private StructuredTrace structuredTrace;

@BeforeEach
public void setup() {
backendEntityEnricher = new MockBackendEntityEnricher();
backendEntityEnricher.init(ConfigFactory.empty(), mock(ClientRegistry.class));

structuredTrace = mock(StructuredTrace.class);
structuredTraceGraph = mock(StructuredTraceGraph.class);
}

Expand Down Expand Up @@ -110,7 +113,8 @@ public void checkBackendEntityGeneratedFromGrpcEvent() {
.setRefType(EventRefType.CHILD_OF)
.build()))
.build();
final BackendInfo backendInfo = backendEntityEnricher.resolve(e, structuredTraceGraph).get();
final BackendInfo backendInfo =
backendEntityEnricher.resolve(e, structuredTrace, structuredTraceGraph).get();
final Entity backendEntity = backendInfo.getEntity();
assertEquals("productcatalogservice:3550", backendEntity.getEntityName());
assertEquals(3, backendEntity.getIdentifyingAttributesCount());
Expand Down
Loading