Skip to content

gcp-o11y: Remove monitored resource detection for logging #10020

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 2 commits into from
Apr 6, 2023
Merged

gcp-o11y: Remove monitored resource detection for logging #10020

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
0715700
removed populating monitored resource to k8s_conatiner by default for…
DNVindhya Apr 5, 2023
08cd175
remove kubernetes resource detection logic from observability
DNVindhya Apr 5, 2023
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 1 addition & 3 deletions gcp-observability/src/main/java/io/grpc/gcp/observability/GcpObservability.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -90,12 +90,10 @@ public final class GcpObservability implements AutoCloseable {
*/
public static synchronized GcpObservability grpcInit() throws IOException {
if (instance == null) {
GlobalLocationTags globalLocationTags = new GlobalLocationTags();
ObservabilityConfigImpl observabilityConfig = ObservabilityConfigImpl.getInstance();
TraceLoggingHelper traceLoggingHelper = new TraceLoggingHelper(
observabilityConfig.getProjectId());
Sink sink = new GcpLogSink(observabilityConfig.getProjectId(),
globalLocationTags.getLocationTags(), observabilityConfig,
Sink sink = new GcpLogSink(observabilityConfig.getProjectId(), observabilityConfig,
SERVICES_TO_EXCLUDE, traceLoggingHelper);
LogHelper helper = new LogHelper(sink);
ConfigFilterHelper configFilterHelper = ConfigFilterHelper.getInstance(observabilityConfig);
Expand Down
148 changes: 0 additions & 148 deletions gcp-observability/src/main/java/io/grpc/gcp/observability/GlobalLocationTags.java
View file
Open in desktop

This file was deleted.

107 changes: 0 additions & 107 deletions gcp-observability/src/main/java/io/grpc/gcp/observability/MetadataConfig.java
View file
Open in desktop

This file was deleted.

41 changes: 5 additions & 36 deletions gcp-observability/src/main/java/io/grpc/gcp/observability/logging/GcpLogSink.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -20,7 +20,6 @@

import com.google.api.gax.batching.BatchingSettings;
import com.google.api.gax.batching.FlowController;
import com.google.cloud.MonitoredResource;
import com.google.cloud.logging.LogEntry;
import com.google.cloud.logging.Logging;
import com.google.cloud.logging.LoggingOptions;
Expand All @@ -30,7 +29,6 @@
import com.google.common.annotations.VisibleForTesting;
import com.google.common.base.Strings;
import com.google.common.collect.ImmutableMap;
import com.google.common.collect.ImmutableSet;
import com.google.protobuf.util.JsonFormat;
import io.grpc.Internal;
import io.grpc.gcp.observability.ObservabilityConfig;
Expand All @@ -42,8 +40,6 @@
import java.util.Collection;
import java.util.Collections;
import java.util.Map;
import java.util.Objects;
import java.util.Set;
import java.util.logging.Level;
import java.util.logging.Logger;
import org.threeten.bp.Duration;
Expand All @@ -58,13 +54,8 @@ public class GcpLogSink implements Sink {
private static final String DEFAULT_LOG_NAME =
"microservices.googleapis.com%2Fobservability%2Fgrpc";
private static final Severity DEFAULT_LOG_LEVEL = Severity.DEBUG;
private static final String K8S_MONITORED_RESOURCE_TYPE = "k8s_container";
private static final Set<String> kubernetesResourceLabelSet
= ImmutableSet.of("project_id", "location", "cluster_name", "namespace_name",
"pod_name", "container_name");
private final String projectId;
private final Map<String, String> customTags;
private final MonitoredResource kubernetesResource;
/** Lazily initialize cloud logging client to avoid circular initialization. Because cloud
* logging APIs also uses gRPC. */
private volatile Logging gcpLoggingClient;
Expand All @@ -75,10 +66,10 @@ public class GcpLogSink implements Sink {


@VisibleForTesting
GcpLogSink(Logging loggingClient, String projectId, Map<String, String> locationTags,
GcpLogSink(Logging loggingClient, String projectId,
ObservabilityConfig config, Collection<String> servicesToExclude,
TraceLoggingHelper traceLoggingHelper) {
this(projectId, locationTags, config, servicesToExclude, traceLoggingHelper);
this(projectId, config, servicesToExclude, traceLoggingHelper);
this.gcpLoggingClient = loggingClient;
}

Expand All @@ -88,12 +79,11 @@ public class GcpLogSink implements Sink {
* @param projectId GCP project id to write logs
* @param servicesToExclude service names for which log entries should not be generated
*/
public GcpLogSink(String projectId, Map<String, String> locationTags,
public GcpLogSink(String projectId,
ObservabilityConfig config, Collection<String> servicesToExclude,
TraceLoggingHelper traceLoggingHelper) {
this.projectId = projectId;
this.customTags = getCustomTags(config.getCustomTags(), locationTags, projectId);
this.kubernetesResource = getResource(locationTags);
this.customTags = getCustomTags(config.getCustomTags());
this.servicesToExclude = checkNotNull(servicesToExclude, "servicesToExclude");
this.isTraceEnabled = config.isEnableCloudTracing();
this.traceLoggingHelper = traceLoggingHelper;
Expand Down Expand Up @@ -126,7 +116,6 @@ public void write(GrpcLogRecord logProto, SpanContext spanContext) {
LogEntry.newBuilder(JsonPayload.of(logProtoMap))
.setSeverity(DEFAULT_LOG_LEVEL)
.setLogName(DEFAULT_LOG_NAME)
.setResource(kubernetesResource)
.setTimestamp(Instant.now());

if (!customTags.isEmpty()) {
Expand Down Expand Up @@ -178,34 +167,14 @@ Logging createLoggingClient() {
}

@VisibleForTesting
static Map<String, String> getCustomTags(Map<String, String> customTags,
Map<String, String> locationTags, String projectId) {
static Map<String, String> getCustomTags(Map<String, String> customTags) {
Copy link
Contributor

@sanjaypujare sanjaypujare Apr 6, 2023 •
edited
Loading

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

This function effectively does very little/nothing - handle null on line 172 may be. Can you just replace calls with equivalent logic (handle null)?

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

The function also creates an immutable map of customTags. On further looking realized null check is not required, as ObservabilityConfigImpl will always return Collections.emptyMap() when custom tags are not set in the configuration.

So I have two options:

  1. Remove null check and keep the function as-is to create an immutable map.
  2. Make the method inline and remove null check i.e
this.customTags = config.getCustomTags();

I prefer Option 1, as it gives us an immutable map. Lmk what you think.

Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Okay sounds good- option 1 is ok

ImmutableMap.Builder<String, String> tagsBuilder = ImmutableMap.builder();
String sourceProjectId = locationTags.get("project_id");
if (!Strings.isNullOrEmpty(projectId)
&& !Strings.isNullOrEmpty(sourceProjectId)
&& !Objects.equals(sourceProjectId, projectId)) {
tagsBuilder.put("source_project_id", sourceProjectId);
}
if (customTags != null) {
tagsBuilder.putAll(customTags);
}
return tagsBuilder.buildOrThrow();
}

@VisibleForTesting
static MonitoredResource getResource(Map<String, String> resourceTags) {
MonitoredResource.Builder builder = MonitoredResource.newBuilder(K8S_MONITORED_RESOURCE_TYPE);
if ((resourceTags != null) && !resourceTags.isEmpty()) {
for (Map.Entry<String, String> entry : resourceTags.entrySet()) {
String resourceKey = entry.getKey();
if (kubernetesResourceLabelSet.contains(resourceKey)) {
builder.addLabel(resourceKey, entry.getValue());
}
}
}
return builder.build();
}

@SuppressWarnings("unchecked")
private Map<String, Object> protoToMapConverter(GrpcLogRecord logProto)
Expand Down
Loading