Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

impl(GCS+gRPC): metrics exporter vs. connection options #14252

Merged
merged 1 commit into from
May 16, 2024
Merged

impl(GCS+gRPC): metrics exporter vs. connection options #14252

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
43 changes: 21 additions & 22 deletions google/cloud/storage/internal/grpc/metrics_exporter_options.cc
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -41,25 +41,8 @@ auto ByName(opentelemetry::sdk::resource::ResourceAttributes const& attributes,

Options MetricsExporterOptions(
Project const& project,
opentelemetry::sdk::resource::Resource const& resource,
Options const& options) {
opentelemetry::sdk::resource::Resource const& resource) {
namespace sc = ::opentelemetry::sdk::resource::SemanticConventions;
auto result =
Options{}
.set<otel_internal::ServiceTimeSeriesOption>(true)
.set<otel_internal::MetricPrefixOption>("storage.googleapis.com/");

// We need a UUID because there may be multiple monitored resources within the
// same process, and we need these monitored resources to be globally unique
// or GCM may complain about the publication rate. There is no information
// available that can make this unique enough, all the clients in a service
// may be using the same project and bucket (not that buckets are available).
//
// This is fairly expensive, it requires initializing a new PRNG, and fetching
// entropy from the OS. Outside tests, this function will be called a handful
// of times, so the performance is not that important.
auto uuid =
google::cloud::internal::InvocationIdGenerator().MakeInvocationId();

auto const& attributes = resource.GetAttributes();
auto monitored_resource = google::api::MonitoredResource{};
Expand All @@ -71,12 +54,29 @@ Options MetricsExporterOptions(
labels["cloud_platform"] = ByName(attributes, sc::kCloudPlatform, "unknown");
labels["host_id"] =
ByName(attributes, "faas.id", ByName(attributes, sc::kHostId, "unknown"));
labels["instance_id"] = std::move(uuid);

// We need a UUID because there may be multiple monitored resources within the
// same process, and we need these monitored resources to be globally unique
// or GCM may complain about the publication rate. There is no information
// available that can make this unique enough, all the clients in a service
// may be using the same project and bucket (not that buckets are available).
//
// This is fairly expensive, it requires initializing a new PRNG, and fetching
// entropy from the OS. Outside tests, this function will be called a handful
// of times, so the performance is not that important.
labels["instance_id"] =
google::cloud::internal::InvocationIdGenerator().MakeInvocationId();
labels["api"] = "GRPC";

result.set<otel_internal::MonitoredResourceOption>(
std::move(monitored_resource));
return Options{}
.set<otel_internal::ServiceTimeSeriesOption>(true)
.set<otel_internal::MetricPrefixOption>("storage.googleapis.com/")
.set<otel_internal::MonitoredResourceOption>(
std::move(monitored_resource));
}

Options MetricsExporterConnectionOptions(Options const& options) {
auto result = Options{};
auto ep_canonical = std::string{"storage.googleapis.com"};
auto ep_private = std::string{"private.googleapis.com"};
auto ep_restricted = std::string{"restricted.googleapis.com"};
Expand All @@ -95,7 +95,6 @@ Options MetricsExporterOptions(
if (matches(ep_canonical)) return result;
if (matches(ep_private)) result.set<EndpointOption>(ep_private);
if (matches(ep_restricted)) result.set<EndpointOption>(ep_restricted);

return result;
}

Expand Down
12 changes: 6 additions & 6 deletions google/cloud/storage/internal/grpc/metrics_exporter_options.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -27,14 +27,14 @@ namespace cloud {
namespace storage_internal {
GOOGLE_CLOUD_CPP_INLINE_NAMESPACE_BEGIN

/**
* Returns the monitoring options given the (fully populated) options for
* Storage.
*/
/// Returns the monitoring exporter options given the project and resource.
Options MetricsExporterOptions(
Project const& project,
opentelemetry::sdk::resource::Resource const& resource,
Options const& options);
opentelemetry::sdk::resource::Resource const& resource);

/// Returns the monitoring exporter connection options given the fully populated
/// storage options.
Options MetricsExporterConnectionOptions(Options const& options);

#endif // GOOGLE_CLOUD_CPP_HAVE_OPENTELEMETRY

Expand Down
90 changes: 42 additions & 48 deletions google/cloud/storage/internal/grpc/metrics_exporter_options_test.cc
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -40,11 +40,7 @@ using ::google::protobuf::TextFormat;
using ::testing::Contains;
using ::testing::Pair;

auto TestResource() {
return opentelemetry::sdk::resource::Resource::Create({});
}

TEST(MetricsExporterOptions, DefaultEndpoint) {
TEST(MetricsExporterConnectionOptions, DefaultEndpoint) {
// In all these cases the default monitoring endpoint is either the best
// choice, or the least bad choice.
struct TestCase {
Expand Down Expand Up @@ -83,74 +79,49 @@ TEST(MetricsExporterOptions, DefaultEndpoint) {
expected_ud = t.universe_domain;
}
options = DefaultOptionsGrpc(std::move(options));
auto const actual =
MetricsExporterOptions(Project("test-only"), TestResource(), options);
auto const actual = MetricsExporterConnectionOptions(options);
EXPECT_FALSE(actual.has<EndpointOption>());
EXPECT_EQ(actual.get<internal::UniverseDomainOption>(), expected_ud);
EXPECT_TRUE(actual.has<otel_internal::MonitoredResourceOption>());
EXPECT_TRUE(actual.get<otel_internal::ServiceTimeSeriesOption>());
EXPECT_EQ(actual.get<otel_internal::MetricPrefixOption>(),
"storage.googleapis.com/");
}
}

TEST(MetricsExporterOptions, PrivateDefaultUD) {
TEST(MetricsExporterConnectionOptions, PrivateDefaultUD) {
for (std::string prefix : {"", "google-c2p:///"}) {
SCOPED_TRACE("Testing with prefix = " + prefix);
auto actual = MetricsExporterOptions(
Project("test-only"), TestResource(),
auto actual = MetricsExporterConnectionOptions(
Options{}.set<EndpointOption>(prefix + "private.googleapis.com"));
EXPECT_THAT(actual.get<EndpointOption>(), "private.googleapis.com");
EXPECT_TRUE(actual.has<otel_internal::MonitoredResourceOption>());
EXPECT_TRUE(actual.get<otel_internal::ServiceTimeSeriesOption>());
EXPECT_EQ(actual.get<otel_internal::MetricPrefixOption>(),
"storage.googleapis.com/");
}
}

TEST(MetricsExporterOptions, PrivateUD) {
TEST(MetricsExporterConnectionOptions, PrivateUD) {
for (std::string prefix : {"", "google-c2p:///"}) {
SCOPED_TRACE("Testing with prefix = " + prefix);
auto actual = MetricsExporterOptions(
Project("test-only"), TestResource(),
auto actual = MetricsExporterConnectionOptions(
Options{}
.set<EndpointOption>(prefix + "private.ud.net")
.set<internal::UniverseDomainOption>("ud.net"));
EXPECT_THAT(actual.get<EndpointOption>(), "private.ud.net");
EXPECT_TRUE(actual.has<otel_internal::MonitoredResourceOption>());
EXPECT_TRUE(actual.get<otel_internal::ServiceTimeSeriesOption>());
EXPECT_EQ(actual.get<otel_internal::MetricPrefixOption>(),
"storage.googleapis.com/");
}
}

TEST(MetricsExporterOptions, RestrictedDefaultUD) {
TEST(MetricsExporterConnectionOptions, RestrictedDefaultUD) {
for (std::string prefix : {"", "google-c2p:///"}) {
SCOPED_TRACE("Testing with prefix = " + prefix);
auto actual = MetricsExporterOptions(
Project("test-only"), TestResource(),
auto actual = MetricsExporterConnectionOptions(
Options{}.set<EndpointOption>(prefix + "restricted.googleapis.com"));
EXPECT_THAT(actual.get<EndpointOption>(), "restricted.googleapis.com");
EXPECT_TRUE(actual.has<otel_internal::MonitoredResourceOption>());
EXPECT_TRUE(actual.get<otel_internal::ServiceTimeSeriesOption>());
EXPECT_EQ(actual.get<otel_internal::MetricPrefixOption>(),
"storage.googleapis.com/");
}
}

TEST(MetricsExporterOptions, RestrictedUD) {
TEST(MetricsExporterConnectionOptions, RestrictedUD) {
for (std::string prefix : {"", "google-c2p:///"}) {
SCOPED_TRACE("Testing with prefix = " + prefix);
auto actual = MetricsExporterOptions(
Project("test-only"), TestResource(),
auto actual = MetricsExporterConnectionOptions(
Options{}
.set<EndpointOption>(prefix + "restricted.ud.net")
.set<internal::UniverseDomainOption>("ud.net"));
EXPECT_THAT(actual.get<EndpointOption>(), "restricted.ud.net");
EXPECT_TRUE(actual.has<otel_internal::MonitoredResourceOption>());
EXPECT_TRUE(actual.get<otel_internal::ServiceTimeSeriesOption>());
EXPECT_EQ(actual.get<otel_internal::MetricPrefixOption>(),
"storage.googleapis.com/");
}
}

Expand All @@ -160,15 +131,13 @@ MATCHER(MatchesInstanceId, "looks like an instance id") {
}

TEST(MetricsExporterOptions, MonitoredResource) {
auto actual =
MetricsExporterOptions(Project("test-project"),
opentelemetry::sdk::resource::Resource::Create({
{"cloud.availability_zone", "us-central1-c"},
{"cloud.region", "us-central1"},
{"cloud.platform", "gcp"},
{"host.id", "test-host-id"},
}),
Options{});
auto actual = MetricsExporterOptions(
Project("test-project"), opentelemetry::sdk::resource::Resource::Create({
{"cloud.availability_zone", "us-central1-c"},
{"cloud.region", "us-central1"},
{"cloud.platform", "gcp"},
{"host.id", "test-host-id"},
}));

EXPECT_TRUE(actual.has<otel_internal::MonitoredResourceOption>());
auto mr = actual.get<otel_internal::MonitoredResourceOption>();
Expand All @@ -190,6 +159,31 @@ TEST(MetricsExporterOptions, MonitoredResource) {
EXPECT_THAT(mr, IsProtoEqual(expected));
}

TEST(MetricsExporterOptions, DefaultMonitoredResource) {
auto actual = MetricsExporterOptions(
Project("test-project"),
opentelemetry::sdk::resource::Resource::Create({}));

EXPECT_TRUE(actual.has<otel_internal::MonitoredResourceOption>());
auto mr = actual.get<otel_internal::MonitoredResourceOption>();
auto labels = mr.labels();
// The `instance_id` label has unpredictable values,
EXPECT_THAT(labels, Contains(Pair("instance_id", MatchesInstanceId())));
mr.mutable_labels()->erase("instance_id");

auto constexpr kExpected = R"pb(
type: "storage_client"
labels { key: "project_id" value: "test-project" }
labels { key: "location" value: "global" }
labels { key: "cloud_platform" value: "unknown" }
labels { key: "host_id" value: "unknown" }
labels { key: "api" value: "GRPC" }
)pb";
auto expected = google::api::MonitoredResource{};
ASSERT_TRUE(TextFormat::ParseFromString(kExpected, &expected));
EXPECT_THAT(mr, IsProtoEqual(expected));
}

} // namespace
GOOGLE_CLOUD_CPP_INLINE_NAMESPACE_END
} // namespace storage_internal
Expand Down
Loading