From 61546f0f318e4048e74a444d189837e9d2ee7116 Mon Sep 17 00:00:00 2001 From: aman-bansal Date: Tue, 22 Nov 2022 21:13:12 +0530 Subject: [PATCH 1/4] chore | adding the excluded domains for creating backends --- .../AbstractBackendEntityEnricher.java | 28 ++++++++++--------- .../traceenricher/util/DomainUtil.java | 18 ++++++++++++ 2 files changed, 33 insertions(+), 13 deletions(-) create mode 100644 hypertrace-trace-enricher/hypertrace-trace-enricher-impl/src/main/java/org/hypertrace/traceenricher/util/DomainUtil.java diff --git a/hypertrace-trace-enricher/hypertrace-trace-enricher-impl/src/main/java/org/hypertrace/traceenricher/enrichment/enrichers/backend/AbstractBackendEntityEnricher.java b/hypertrace-trace-enricher/hypertrace-trace-enricher-impl/src/main/java/org/hypertrace/traceenricher/enrichment/enrichers/backend/AbstractBackendEntityEnricher.java index d35814bea..3a765cd74 100644 --- a/hypertrace-trace-enricher/hypertrace-trace-enricher-impl/src/main/java/org/hypertrace/traceenricher/enrichment/enrichers/backend/AbstractBackendEntityEnricher.java +++ b/hypertrace-trace-enricher/hypertrace-trace-enricher-impl/src/main/java/org/hypertrace/traceenricher/enrichment/enrichers/backend/AbstractBackendEntityEnricher.java @@ -37,6 +37,7 @@ import org.hypertrace.traceenricher.enrichment.enrichers.backend.provider.BackendProvider; import org.hypertrace.traceenricher.enrichment.enrichers.cache.EntityCache; import org.hypertrace.traceenricher.enrichment.enrichers.resolver.backend.BackendInfo; +import org.hypertrace.traceenricher.util.DomainUtil; import org.hypertrace.traceenricher.util.EnricherUtil; import org.hypertrace.traceenricher.util.EntityAvroConverter; import org.slf4j.Logger; @@ -64,10 +65,12 @@ public abstract class AbstractBackendEntityEnricher extends AbstractTraceEnriche EnrichedSpanConstants.getValue(Backend.BACKEND_OPERATION); private static final String BACKEND_DESTINATION_ATTR = EnrichedSpanConstants.getValue(Backend.BACKEND_DESTINATION); + private static final String BACKEND_EXCLUDED_DOMAINS = "backend.excluded.domains"; private EdsClient edsClient; private EntityCache entityCache; private FqnResolver fqnResolver; + private List backendExcludedDomains; @Override public void init(Config enricherConfig, ClientRegistry clientRegistry) { @@ -76,6 +79,10 @@ public void init(Config enricherConfig, ClientRegistry clientRegistry) { this.entityCache = clientRegistry.getEntityCache(); setup(enricherConfig, clientRegistry); this.fqnResolver = getFqnResolver(); + this.backendExcludedDomains = + enricherConfig.hasPath(BACKEND_EXCLUDED_DOMAINS) + ? enricherConfig.getStringList(BACKEND_EXCLUDED_DOMAINS) + : Collections.emptyList(); } public abstract void setup(Config enricherConfig, ClientRegistry clientRegistry); @@ -122,18 +129,6 @@ protected boolean canResolveBackend(StructuredTraceGraph structuredTraceGraph, E return true; } - /** - * Method to check if backend uri is valid or not. This will enable any custom logic to be - * inserted in the implementing classes. - * - * @param backendURI backend URI information - * @return true if backend uri resolution is allowed - */ - protected boolean isValidBackendUri(String backendURI) { - // by default allow the backend uri to proceed - return true; - } - /** Checks if the candidateEntity is indeed a backend Entity */ private boolean isValidBackendEntity( StructuredTrace trace, Event backendSpan, BackendInfo candidateInfo) { @@ -323,7 +318,7 @@ public Optional resolve( } String backendUri = maybeBackendUri.get(); - if (!isValidBackendUri(backendUri)) { + if (isDomainExcluded(event, backendUri)) { return Optional.empty(); } @@ -349,6 +344,13 @@ public Optional resolve( return Optional.empty(); } + private boolean isDomainExcluded(Event event, String backendUri) { + String[] hostAndPort = backendUri.split(COLON); + String host = fqnResolver.resolve(hostAndPort[0], event); + String domain = DomainUtil.getPrimaryDomain(host); + return backendExcludedDomains.contains(domain); + } + private Builder getEntityBuilder( StructuredTrace trace, Event event, BackendType type, String backendUri) { try { diff --git a/hypertrace-trace-enricher/hypertrace-trace-enricher-impl/src/main/java/org/hypertrace/traceenricher/util/DomainUtil.java b/hypertrace-trace-enricher/hypertrace-trace-enricher-impl/src/main/java/org/hypertrace/traceenricher/util/DomainUtil.java new file mode 100644 index 000000000..69fd0c5e0 --- /dev/null +++ b/hypertrace-trace-enricher/hypertrace-trace-enricher-impl/src/main/java/org/hypertrace/traceenricher/util/DomainUtil.java @@ -0,0 +1,18 @@ +package org.hypertrace.traceenricher.util; + +import com.google.common.net.InternetDomainName; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +public class DomainUtil { + private static final Logger LOGGER = LoggerFactory.getLogger(DomainUtil.class); + + public static String getPrimaryDomain(String url) { + try { + return InternetDomainName.from(url).topPrivateDomain().toString(); + } catch (Exception exception) { + LOGGER.error("Error while extracting the primary domain from the url {} ", url, exception); + return url; + } + } +} From 0aebde9348dfc653b561b2c6b52be0d9d3758284 Mon Sep 17 00:00:00 2001 From: aman-bansal Date: Tue, 22 Nov 2022 21:15:36 +0530 Subject: [PATCH 2/4] name fix --- .../org/hypertrace/traceenricher/util/DomainUtil.java | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/hypertrace-trace-enricher/hypertrace-trace-enricher-impl/src/main/java/org/hypertrace/traceenricher/util/DomainUtil.java b/hypertrace-trace-enricher/hypertrace-trace-enricher-impl/src/main/java/org/hypertrace/traceenricher/util/DomainUtil.java index 69fd0c5e0..8250a9c75 100644 --- a/hypertrace-trace-enricher/hypertrace-trace-enricher-impl/src/main/java/org/hypertrace/traceenricher/util/DomainUtil.java +++ b/hypertrace-trace-enricher/hypertrace-trace-enricher-impl/src/main/java/org/hypertrace/traceenricher/util/DomainUtil.java @@ -7,12 +7,12 @@ public class DomainUtil { private static final Logger LOGGER = LoggerFactory.getLogger(DomainUtil.class); - public static String getPrimaryDomain(String url) { + public static String getPrimaryDomain(String host) { try { - return InternetDomainName.from(url).topPrivateDomain().toString(); + return InternetDomainName.from(host).topPrivateDomain().toString(); } catch (Exception exception) { - LOGGER.error("Error while extracting the primary domain from the url {} ", url, exception); - return url; + LOGGER.error("Error while extracting the primary domain from the host {} ", host, exception); + return host; } } } From 314c9e29d0f6e37869204c06e94c56986229593c Mon Sep 17 00:00:00 2001 From: aman-bansal Date: Tue, 22 Nov 2022 21:16:35 +0530 Subject: [PATCH 3/4] adding test --- .../traceenricher/util/DomainUtilTest.java | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) create mode 100644 hypertrace-trace-enricher/hypertrace-trace-enricher-impl/src/test/java/org/hypertrace/traceenricher/util/DomainUtilTest.java diff --git a/hypertrace-trace-enricher/hypertrace-trace-enricher-impl/src/test/java/org/hypertrace/traceenricher/util/DomainUtilTest.java b/hypertrace-trace-enricher/hypertrace-trace-enricher-impl/src/test/java/org/hypertrace/traceenricher/util/DomainUtilTest.java new file mode 100644 index 000000000..243419522 --- /dev/null +++ b/hypertrace-trace-enricher/hypertrace-trace-enricher-impl/src/test/java/org/hypertrace/traceenricher/util/DomainUtilTest.java @@ -0,0 +1,17 @@ +package org.hypertrace.traceenricher.util; + +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.Test; + +public class DomainUtilTest { + + @Test + public void testGetDomainHost() { + Assertions.assertEquals(DomainUtil.getPrimaryDomain("www.xyz.com"), "xyz.com"); + Assertions.assertEquals(DomainUtil.getPrimaryDomain("www.abc.xyz.com"), "xyz.com"); + Assertions.assertEquals(DomainUtil.getPrimaryDomain("abc.xyz.com"), "xyz.com"); + Assertions.assertEquals(DomainUtil.getPrimaryDomain("xyz.com"), "xyz.com"); + + Assertions.assertEquals(DomainUtil.getPrimaryDomain("10.0.0.0"), "10.0.0.0"); + } +} From eed83da4228e4514f3f638b24531bad7f7d5f9ef Mon Sep 17 00:00:00 2001 From: aman-bansal Date: Wed, 23 Nov 2022 12:47:06 +0530 Subject: [PATCH 4/4] removing domain util --- .../backend/AbstractBackendEntityEnricher.java | 4 ++-- .../traceenricher/util/DomainUtil.java | 18 ------------------ .../traceenricher/util/DomainUtilTest.java | 17 ----------------- .../http/HttpSemanticConventionUtils.java | 10 ++++++++++ .../http/HttpSemanticConventionUtilsTest.java | 11 +++++++++++ 5 files changed, 23 insertions(+), 37 deletions(-) delete mode 100644 hypertrace-trace-enricher/hypertrace-trace-enricher-impl/src/main/java/org/hypertrace/traceenricher/util/DomainUtil.java delete mode 100644 hypertrace-trace-enricher/hypertrace-trace-enricher-impl/src/test/java/org/hypertrace/traceenricher/util/DomainUtilTest.java diff --git a/hypertrace-trace-enricher/hypertrace-trace-enricher-impl/src/main/java/org/hypertrace/traceenricher/enrichment/enrichers/backend/AbstractBackendEntityEnricher.java b/hypertrace-trace-enricher/hypertrace-trace-enricher-impl/src/main/java/org/hypertrace/traceenricher/enrichment/enrichers/backend/AbstractBackendEntityEnricher.java index 3a765cd74..19c1eccac 100644 --- a/hypertrace-trace-enricher/hypertrace-trace-enricher-impl/src/main/java/org/hypertrace/traceenricher/enrichment/enrichers/backend/AbstractBackendEntityEnricher.java +++ b/hypertrace-trace-enricher/hypertrace-trace-enricher-impl/src/main/java/org/hypertrace/traceenricher/enrichment/enrichers/backend/AbstractBackendEntityEnricher.java @@ -27,6 +27,7 @@ import org.hypertrace.entity.data.service.v1.Entity.Builder; import org.hypertrace.entity.service.constants.EntityConstants; import org.hypertrace.entity.v1.entitytype.EntityType; +import org.hypertrace.semantic.convention.utils.http.HttpSemanticConventionUtils; import org.hypertrace.semantic.convention.utils.span.SpanSemanticConventionUtils; import org.hypertrace.traceenricher.enrichedspan.constants.BackendType; import org.hypertrace.traceenricher.enrichedspan.constants.EnrichedSpanConstants; @@ -37,7 +38,6 @@ import org.hypertrace.traceenricher.enrichment.enrichers.backend.provider.BackendProvider; import org.hypertrace.traceenricher.enrichment.enrichers.cache.EntityCache; import org.hypertrace.traceenricher.enrichment.enrichers.resolver.backend.BackendInfo; -import org.hypertrace.traceenricher.util.DomainUtil; import org.hypertrace.traceenricher.util.EnricherUtil; import org.hypertrace.traceenricher.util.EntityAvroConverter; import org.slf4j.Logger; @@ -347,7 +347,7 @@ public Optional resolve( private boolean isDomainExcluded(Event event, String backendUri) { String[] hostAndPort = backendUri.split(COLON); String host = fqnResolver.resolve(hostAndPort[0], event); - String domain = DomainUtil.getPrimaryDomain(host); + String domain = HttpSemanticConventionUtils.getPrimaryDomain(host); return backendExcludedDomains.contains(domain); } diff --git a/hypertrace-trace-enricher/hypertrace-trace-enricher-impl/src/main/java/org/hypertrace/traceenricher/util/DomainUtil.java b/hypertrace-trace-enricher/hypertrace-trace-enricher-impl/src/main/java/org/hypertrace/traceenricher/util/DomainUtil.java deleted file mode 100644 index 8250a9c75..000000000 --- a/hypertrace-trace-enricher/hypertrace-trace-enricher-impl/src/main/java/org/hypertrace/traceenricher/util/DomainUtil.java +++ /dev/null @@ -1,18 +0,0 @@ -package org.hypertrace.traceenricher.util; - -import com.google.common.net.InternetDomainName; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -public class DomainUtil { - private static final Logger LOGGER = LoggerFactory.getLogger(DomainUtil.class); - - public static String getPrimaryDomain(String host) { - try { - return InternetDomainName.from(host).topPrivateDomain().toString(); - } catch (Exception exception) { - LOGGER.error("Error while extracting the primary domain from the host {} ", host, exception); - return host; - } - } -} diff --git a/hypertrace-trace-enricher/hypertrace-trace-enricher-impl/src/test/java/org/hypertrace/traceenricher/util/DomainUtilTest.java b/hypertrace-trace-enricher/hypertrace-trace-enricher-impl/src/test/java/org/hypertrace/traceenricher/util/DomainUtilTest.java deleted file mode 100644 index 243419522..000000000 --- a/hypertrace-trace-enricher/hypertrace-trace-enricher-impl/src/test/java/org/hypertrace/traceenricher/util/DomainUtilTest.java +++ /dev/null @@ -1,17 +0,0 @@ -package org.hypertrace.traceenricher.util; - -import org.junit.jupiter.api.Assertions; -import org.junit.jupiter.api.Test; - -public class DomainUtilTest { - - @Test - public void testGetDomainHost() { - Assertions.assertEquals(DomainUtil.getPrimaryDomain("www.xyz.com"), "xyz.com"); - Assertions.assertEquals(DomainUtil.getPrimaryDomain("www.abc.xyz.com"), "xyz.com"); - Assertions.assertEquals(DomainUtil.getPrimaryDomain("abc.xyz.com"), "xyz.com"); - Assertions.assertEquals(DomainUtil.getPrimaryDomain("xyz.com"), "xyz.com"); - - Assertions.assertEquals(DomainUtil.getPrimaryDomain("10.0.0.0"), "10.0.0.0"); - } -} diff --git a/semantic-convention-utils/src/main/java/org/hypertrace/semantic/convention/utils/http/HttpSemanticConventionUtils.java b/semantic-convention-utils/src/main/java/org/hypertrace/semantic/convention/utils/http/HttpSemanticConventionUtils.java index 5440c0cd7..c7a677a7b 100644 --- a/semantic-convention-utils/src/main/java/org/hypertrace/semantic/convention/utils/http/HttpSemanticConventionUtils.java +++ b/semantic-convention-utils/src/main/java/org/hypertrace/semantic/convention/utils/http/HttpSemanticConventionUtils.java @@ -44,6 +44,7 @@ import com.google.common.base.Splitter; import com.google.common.collect.Lists; import com.google.common.collect.Sets; +import com.google.common.net.InternetDomainName; import io.micrometer.core.instrument.util.StringUtils; import java.net.HttpCookie; import java.net.MalformedURLException; @@ -230,6 +231,15 @@ public static Optional getEnvironmentForSpan(Event event) { event, OTelDeploymentSemanticConventions.DEPLOYMENT_ENVIRONMENT.getValue())); } + public static String getPrimaryDomain(String host) { + try { + return InternetDomainName.from(host).topPrivateDomain().toString(); + } catch (Exception exception) { + LOGGER.error("Error while extracting the primary domain from the host {} ", host, exception); + return host; + } + } + /** * OTel mandates one of the following set to be present for http server span - http.scheme, * http.host, http.target - http.scheme, http.server_name, net.host.port, http.target - diff --git a/semantic-convention-utils/src/test/java/org/hypertrace/semantic/convention/utils/http/HttpSemanticConventionUtilsTest.java b/semantic-convention-utils/src/test/java/org/hypertrace/semantic/convention/utils/http/HttpSemanticConventionUtilsTest.java index f2e76b437..9e7d3f8e5 100644 --- a/semantic-convention-utils/src/test/java/org/hypertrace/semantic/convention/utils/http/HttpSemanticConventionUtilsTest.java +++ b/semantic-convention-utils/src/test/java/org/hypertrace/semantic/convention/utils/http/HttpSemanticConventionUtilsTest.java @@ -591,4 +591,15 @@ public void testGetPathFromUrl() { "/api/v1/gatekeeper/check?url=%2Fpixel%2Factivities%3Fadvertisable%3DTRHRT&method=GET&service=pixel"); Assertions.assertEquals(path.get(), "/api/v1/gatekeeper/check"); } + + @Test + public void testGetPrimaryDomain() { + Assertions.assertEquals(HttpSemanticConventionUtils.getPrimaryDomain("www.xyz.com"), "xyz.com"); + Assertions.assertEquals( + HttpSemanticConventionUtils.getPrimaryDomain("www.abc.xyz.com"), "xyz.com"); + Assertions.assertEquals(HttpSemanticConventionUtils.getPrimaryDomain("abc.xyz.com"), "xyz.com"); + Assertions.assertEquals(HttpSemanticConventionUtils.getPrimaryDomain("xyz.com"), "xyz.com"); + + Assertions.assertEquals(HttpSemanticConventionUtils.getPrimaryDomain("10.0.0.0"), "10.0.0.0"); + } }