Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Google Cloud Spanner Receiver gracefully handles unreadable databases #26732

Merged
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
34 commits
Select commit Hold shift + click to select a range
f953470
re-added changes again
spidercensus Oct 6, 2023
935f358
added changelog
spidercensus Oct 6, 2023
8289c77
Merge branch 'main' into spidercensus/fix-googlecloudspanner
spidercensus Oct 6, 2023
99795f9
removed whitespace from issues field in chlog. would have been nice t…
spidercensus Oct 9, 2023
9ecf8bd
Merge branch 'main' into spidercensus/fix-googlecloudspanner
spidercensus Oct 10, 2023
d3a7691
put quotes around chlog entries.
spidercensus Oct 11, 2023
a5425c1
updated go.sum
spidercensus Oct 11, 2023
8c0ae22
Merge branch 'main' into spidercensus/fix-googlecloudspanner
spidercensus Oct 11, 2023
e78a7b9
updated go.mod and go.sum again
spidercensus Oct 11, 2023
4a42460
updated chlog again.
spidercensus Oct 11, 2023
48d5709
updated to go 1.21
spidercensus Oct 11, 2023
a074ce6
Merge branch 'main' into spidercensus/fix-googlecloudspanner
spidercensus Oct 11, 2023
0f37b36
Merge branch 'main' into spidercensus/fix-googlecloudspanner
spidercensus Oct 11, 2023
c3f3649
Merge branch 'main' into spidercensus/fix-googlecloudspanner
spidercensus Oct 11, 2023
306d1a6
updated go.sum after running go mod tidy -compat=1.20
spidercensus Oct 12, 2023
80f9272
Merge branch 'main' into spidercensus/fix-googlecloudspanner
spidercensus Oct 12, 2023
a1c579d
ran go mod tidy again with go version 1.20 EVEN THOUGH I ran with com…
spidercensus Oct 14, 2023
8138873
Merge branch 'main' into spidercensus/fix-googlecloudspanner
spidercensus Oct 24, 2023
81bead8
Merge branch 'main' into spidercensus/fix-googlecloudspanner
spidercensus Oct 26, 2023
799733d
fixed code without fixing tests
spidercensus Oct 26, 2023
8da11ad
updated tests
spidercensus Oct 26, 2023
161467b
Merge branch 'main' into spidercensus/fix-googlecloudspanner
spidercensus Oct 27, 2023
21bab8a
ran `go mod tidy -compat=1.20` again
spidercensus Oct 27, 2023
1333be8
Merge branch 'main' into spidercensus/fix-googlecloudspanner
spidercensus Oct 27, 2023
98daa7c
Merge branch 'main' into spidercensus/fix-googlecloudspanner
spidercensus Oct 30, 2023
68f063c
Merge branch 'main' into spidercensus/fix-googlecloudspanner
spidercensus Oct 30, 2023
95ec7d0
Merge branch 'main' into spidercensus/fix-googlecloudspanner
spidercensus Oct 31, 2023
a7b97c4
Merge branch 'main' into spidercensus/fix-googlecloudspanner
spidercensus Oct 31, 2023
c17fb99
added go.sum. again.
spidercensus Oct 31, 2023
497862f
Merge branch 'main' into spidercensus/fix-googlecloudspanner
spidercensus Dec 4, 2023
4e3f828
Merge branch 'main' into spidercensus/fix-googlecloudspanner
spidercensus Dec 4, 2023
843acba
updated go.mod and go.sum again according to go1.20 run
spidercensus Dec 4, 2023
2857fd5
removed coverage output files from google cloud spanner receiver.
spidercensus Dec 4, 2023
42baa21
Merge remote-tracking branch 'origin/main' into spidercensus/fix-goog…
mx-psi Dec 5, 2023
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Jump to
Jump to file
Failed to load files.
Diff view
Diff view
27 changes: 27 additions & 0 deletions .chloggen/spidercensus_fix-googlecloudspanner.yaml
Original file line number Diff line number Diff line change
@@ -0,0 +1,27 @@
# Use this changelog template to create an entry for release notes.

# One of 'breaking', 'deprecation', 'new_component', 'enhancement', 'bug_fix'
change_type: bug_fix

# The name of the component, or a single word describing the area of concern, (e.g. filelogreceiver)
component: googlecloudspannerreceiver

# A brief description of the change. Surround your text with quotes ("") if it needs to start with a backtick (`).
note: "Google Cloud Spanner Receiver currently generates an exception and exits if it attempts to read data from a database that doesn't exist. However it's normal for a single receiver to poll multiple databases, so this is not graceful failure. This PR makes a change to gracefully generate an error in case of an unreadable missing database and then continue reading other databases.."

# Mandatory: One or more tracking issues related to the change. You can use the PR number here if no issue exists.
issues: [26732]

# (Optional) One or more lines of additional information to render under the primary note.
# These lines will be padded with 2 spaces and then inserted directly into the document.
# Use pipe (|) for multiline entries.
subtext:

# If your change doesn't affect end users or the exported elements of any package,
# you should instead start your pull request title with [chore] or use the "Skip Changelog" label.
# Optional: The change log or logs in which this entry should be included.
# e.g. '[user]' or '[user, api]'
# Include 'user' if the change is relevant to end users.
# Include 'api' if there is a change to a library API.
# Default: '[user]'
change_logs: [user]
2 changes: 1 addition & 1 deletion receiver/googlecloudspannerreceiver/go.mod
Original file line number Diff line number Diff line change
Expand Up @@ -12,6 +12,7 @@ require (
go.opentelemetry.io/collector/consumer v0.90.2-0.20231201205146-6e2fdc755b34
go.opentelemetry.io/collector/pdata v1.0.1-0.20231201205146-6e2fdc755b34
go.opentelemetry.io/collector/receiver v0.90.2-0.20231201205146-6e2fdc755b34
go.uber.org/multierr v1.11.0
go.uber.org/zap v1.26.0
google.golang.org/api v0.151.0
google.golang.org/grpc v1.59.0
Expand Down Expand Up @@ -56,7 +57,6 @@ require (
go.opentelemetry.io/otel v1.21.0 // indirect
go.opentelemetry.io/otel/metric v1.21.0 // indirect
go.opentelemetry.io/otel/trace v1.21.0 // indirect
go.uber.org/multierr v1.11.0 // indirect
golang.org/x/crypto v0.15.0 // indirect
golang.org/x/net v0.18.0 // indirect
golang.org/x/oauth2 v0.14.0 // indirect
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -7,6 +7,7 @@ import (
"context"
"fmt"

"go.uber.org/multierr"
"go.uber.org/zap"

"github.com/open-telemetry/opentelemetry-collector-contrib/receiver/googlecloudspannerreceiver/internal/datasource"
Expand Down Expand Up @@ -61,26 +62,41 @@ func (databaseReader *DatabaseReader) Name() string {
}

func (databaseReader *DatabaseReader) Shutdown() {
databaseReader.logger.Debug("Closing connection to database",
zap.String("database", databaseReader.database.DatabaseID().ID()))
databaseReader.logger.Debug(
"Closing connection to database",
zap.String("database", databaseReader.database.DatabaseID().ID()),
)
databaseReader.database.Client().Close()
}

func (databaseReader *DatabaseReader) Read(ctx context.Context) ([]*metadata.MetricsDataPoint, error) {
databaseReader.logger.Debug("Executing read method for database",
zap.String("database", databaseReader.database.DatabaseID().ID()))
databaseReader.logger.Debug(
"Executing read method for database",
zap.String("database", databaseReader.database.DatabaseID().ID()),
)

var result []*metadata.MetricsDataPoint
var (
result []*metadata.MetricsDataPoint
err error
)

for _, reader := range databaseReader.readers {
dataPoints, err := reader.Read(ctx)
if err != nil {
return nil, fmt.Errorf("cannot read data for data points databaseReader %q because of an error: %w",
reader.Name(), err)
}

dataPoints, readErr := reader.Read(ctx)
result = append(result, dataPoints...)
if readErr != nil {
err = multierr.Append(
err,
fmt.Errorf("cannot read data for data points databaseReader %q because of an error: %w", reader.Name(), readErr),
)
}
}
if err != nil {
databaseReader.logger.Warn(
"Errors encountered while reading database",
zap.String("database", databaseReader.database.DatabaseID().ID()),
zap.Int("error_count", len(multierr.Errors(err))),
)
}

return result, nil
return result, err
}
Original file line number Diff line number Diff line change
Expand Up @@ -7,6 +7,7 @@ import (
"context"
"strings"

"go.uber.org/multierr"
"go.uber.org/zap"

"github.com/open-telemetry/opentelemetry-collector-contrib/receiver/googlecloudspannerreceiver/internal/metadata"
Expand All @@ -33,18 +34,21 @@ func (projectReader *ProjectReader) Shutdown() {
}

func (projectReader *ProjectReader) Read(ctx context.Context) ([]*metadata.MetricsDataPoint, error) {
var result []*metadata.MetricsDataPoint
var (
result []*metadata.MetricsDataPoint
err error
)

for _, databaseReader := range projectReader.databaseReaders {
dataPoints, err := databaseReader.Read(ctx)
if err != nil {
return nil, err
dataPoints, readErr := databaseReader.Read(ctx)
if readErr == nil {
result = append(result, dataPoints...)
} else {
err = multierr.Append(err, readErr)
}

result = append(result, dataPoints...)
}

return result, nil
return result, err
}

func (projectReader *ProjectReader) Name() string {
Expand Down
24 changes: 18 additions & 6 deletions receiver/googlecloudspannerreceiver/receiver.go
Original file line number Diff line number Diff line change
Expand Up @@ -11,6 +11,8 @@ import (
"go.opentelemetry.io/collector/component"
"go.opentelemetry.io/collector/pdata/pmetric"
"go.opentelemetry.io/collector/receiver"
"go.opentelemetry.io/collector/receiver/scrapererror"
"go.uber.org/multierr"
"go.uber.org/zap"

"github.com/open-telemetry/opentelemetry-collector-contrib/receiver/googlecloudspannerreceiver/internal/datasource"
Expand Down Expand Up @@ -41,18 +43,28 @@ func newGoogleCloudSpannerReceiver(logger *zap.Logger, config *Config) *googleCl
}

func (r *googleCloudSpannerReceiver) Scrape(ctx context.Context) (pmetric.Metrics, error) {
var allMetricsDataPoints []*metadata.MetricsDataPoint
var (
allMetricsDataPoints []*metadata.MetricsDataPoint
err error
)

for _, projectReader := range r.projectReaders {
dataPoints, err := projectReader.Read(ctx)
if err != nil {
return pmetric.Metrics{}, err
dataPoints, readErr := projectReader.Read(ctx)
allMetricsDataPoints = append(allMetricsDataPoints, dataPoints...)
if readErr != nil {
err = multierr.Append(err, readErr)
}
}

allMetricsDataPoints = append(allMetricsDataPoints, dataPoints...)
metrics, buildErr := r.metricsBuilder.Build(allMetricsDataPoints)
if buildErr != nil {
err = multierr.Append(err, buildErr)
}

return r.metricsBuilder.Build(allMetricsDataPoints)
if err != nil && metrics.DataPointCount() > 0 {
err = scrapererror.NewPartialScrapeError(err, len(multierr.Errors(err)))
}
return metrics, err
}

func (r *googleCloudSpannerReceiver) Start(ctx context.Context, _ component.Host) error {
Expand Down
2 changes: 1 addition & 1 deletion receiver/googlecloudspannerreceiver/receiver_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -52,7 +52,7 @@ func newMetricsBuilder(throwErrorOnShutdown bool) metadata.MetricsBuilder {
}

func (b *metricsBuilder) Build([]*metadata.MetricsDataPoint) (pmetric.Metrics, error) {
return pmetric.Metrics{}, nil
return pmetric.NewMetrics(), nil
}

func (b *metricsBuilder) Shutdown() error {
Expand Down