diff --git a/exporter/collector/README.md b/exporter/collector/README.md index 99bcb6c9a..bfb3dedde 100644 --- a/exporter/collector/README.md +++ b/exporter/collector/README.md @@ -322,8 +322,8 @@ By default, the exporter sends telemetry to the project specified by `project` i ### Multi-Project quota usage -The `gcp.project.id` label can be combined with the `destination_project_quota` option to attribute quota usage to the project parsed by the label. This feature is currently only available -for traces and metrics. The Collector's default service account will need `roles/serviceusage.serviceUsageConsumer` IAM permissions in the destination quota project. +The `gcp.project.id` label can be combined with the `destination_project_quota` option to attribute quota usage to the project parsed by the label. This feature is available +for traces, metrics, and logs. The Collector's default service account will need `roles/serviceusage.serviceUsageConsumer` IAM permissions in the destination quota project. Note that this option will not work if a quota project is already defined in your Collector's GCP credentials. In this case, the telemetry will fail to export with a "project not found" error. This can be done by manually editing your [ADC file](https://cloud.google.com/docs/authentication/application-default-credentials#personal) (if it exists) to remove the `quota_project_id` entry line. diff --git a/exporter/collector/integrationtest/testcases/testcases_logs.go b/exporter/collector/integrationtest/testcases/testcases_logs.go index a444c2f62..d15999033 100644 --- a/exporter/collector/integrationtest/testcases/testcases_logs.go +++ b/exporter/collector/integrationtest/testcases/testcases_logs.go @@ -32,6 +32,14 @@ var LogsTestCases = []TestCase{ OTLPInputFixturePath: "testdata/fixtures/logs/logs_multi_project.json", ExpectFixturePath: "testdata/fixtures/logs/logs_multi_project_expected.json", }, + { + Name: "Multi-project logs with destination_project_quota enabled", + OTLPInputFixturePath: "testdata/fixtures/logs/logs_multi_project.json", + ExpectFixturePath: "testdata/fixtures/logs/logs_multi_project_destination_quota_expected.json", + ConfigureCollector: func(cfg *collector.Config) { + cfg.DestinationProjectQuota = true + }, + }, { Name: "Logs with scope information", OTLPInputFixturePath: "testdata/fixtures/logs/logs_apache_error_scope.json", diff --git a/exporter/collector/integrationtest/testdata/fixtures/logs/logs_multi_project_destination_quota_expected.json b/exporter/collector/integrationtest/testdata/fixtures/logs/logs_multi_project_destination_quota_expected.json new file mode 100644 index 000000000..b9c1fe32c --- /dev/null +++ b/exporter/collector/integrationtest/testdata/fixtures/logs/logs_multi_project_destination_quota_expected.json @@ -0,0 +1,780 @@ +{ + "writeLogEntriesRequests": [ + { + "entries": [ + { + "logName": "projects/fakeprojectid/logs/multi-project", + "resource": { + "type": "gce_instance", + "labels": { + "instance_id": "", + "zone": "" + } + }, + "textPayload": "127.0.0.1 - - [26/Apr/2022:22:53:36 +0800] \"GET / HTTP/1.1\" 200 1247", + "timestamp": "1970-01-01T00:00:00Z", + "httpRequest": { + "requestMethod": "GET", + "requestUrl": "/", + "status": 200, + "responseSize": "1247", + "remoteIp": "127.0.0.1", + "protocol": "HTTP/1.1" + }, + "labels": { + "log.file.name": "test.log" + } + }, + { + "logName": "projects/fakeprojectid/logs/multi-project", + "resource": { + "type": "gce_instance", + "labels": { + "instance_id": "", + "zone": "" + } + }, + "textPayload": "127.0.0.1 - - [26/Apr/2022:22:53:37 +0800] \"GET /lamp.png HTTP/1.1\" 200 51164", + "timestamp": "1970-01-01T00:00:00Z", + "httpRequest": { + "requestMethod": "GET", + "requestUrl": "/lamp.png", + "status": 200, + "responseSize": "51164", + "remoteIp": "127.0.0.1", + "protocol": "HTTP/1.1" + }, + "labels": { + "log.file.name": "test.log" + } + }, + { + "logName": "projects/fakeprojectid/logs/multi-project", + "resource": { + "type": "gce_instance", + "labels": { + "instance_id": "", + "zone": "" + } + }, + "textPayload": "127.0.0.1 - - [26/Apr/2022:22:53:37 +0800] \"GET /favicon.ico HTTP/1.1\" 200 3990", + "timestamp": "1970-01-01T00:00:00Z", + "httpRequest": { + "requestMethod": "GET", + "requestUrl": "/favicon.ico", + "status": 200, + "responseSize": "3990", + "remoteIp": "127.0.0.1", + "protocol": "HTTP/1.1" + }, + "labels": { + "log.file.name": "test.log" + } + }, + { + "logName": "projects/fakeprojectid/logs/multi-project", + "resource": { + "type": "gce_instance", + "labels": { + "instance_id": "", + "zone": "" + } + }, + "textPayload": "127.0.0.1 - - [26/Apr/2022:22:53:51 +0800] \"GET / HTTP/1.1\" 200 1247", + "timestamp": "1970-01-01T00:00:00Z", + "httpRequest": { + "requestMethod": "GET", + "requestUrl": "/", + "status": 200, + "responseSize": "1247", + "remoteIp": "127.0.0.1", + "protocol": "HTTP/1.1" + }, + "labels": { + "log.file.name": "test.log" + } + }, + { + "logName": "projects/fakeprojectid/logs/multi-project", + "resource": { + "type": "gce_instance", + "labels": { + "instance_id": "", + "zone": "" + } + }, + "textPayload": "127.0.0.1 - - [26/Apr/2022:22:53:52 +0800] \"GET / HTTP/1.1\" 200 1247", + "timestamp": "1970-01-01T00:00:00Z", + "httpRequest": { + "requestMethod": "GET", + "requestUrl": "/", + "status": 200, + "responseSize": "1247", + "remoteIp": "127.0.0.1", + "protocol": "HTTP/1.1" + }, + "labels": { + "log.file.name": "test.log" + } + }, + { + "logName": "projects/fakeprojectid/logs/multi-project", + "resource": { + "type": "gce_instance", + "labels": { + "instance_id": "", + "zone": "" + } + }, + "textPayload": "127.0.0.1 - - [26/Apr/2022:22:53:53 +0800] \"GET / HTTP/1.1\" 200 1247", + "timestamp": "1970-01-01T00:00:00Z", + "httpRequest": { + "requestMethod": "GET", + "requestUrl": "/", + "status": 200, + "responseSize": "1247", + "remoteIp": "127.0.0.1", + "protocol": "HTTP/1.1" + }, + "labels": { + "log.file.name": "test.log" + } + }, + { + "logName": "projects/fakeprojectid/logs/multi-project", + "resource": { + "type": "gce_instance", + "labels": { + "instance_id": "", + "zone": "" + } + }, + "textPayload": "127.0.0.1 - - [26/Apr/2022:22:53:53 +0800] \"GET / HTTP/1.1\" 200 1247", + "timestamp": "1970-01-01T00:00:00Z", + "httpRequest": { + "requestMethod": "GET", + "requestUrl": "/", + "status": 200, + "responseSize": "1247", + "remoteIp": "127.0.0.1", + "protocol": "HTTP/1.1" + }, + "labels": { + "log.file.name": "test.log" + } + }, + { + "logName": "projects/fakeprojectid/logs/multi-project", + "resource": { + "type": "gce_instance", + "labels": { + "instance_id": "", + "zone": "" + } + }, + "textPayload": "127.0.0.1 - - [26/Apr/2022:22:53:53 +0800] \"GET / HTTP/1.1\" 200 1247", + "timestamp": "1970-01-01T00:00:00Z", + "httpRequest": { + "requestMethod": "GET", + "requestUrl": "/", + "status": 200, + "responseSize": "1247", + "remoteIp": "127.0.0.1", + "protocol": "HTTP/1.1" + }, + "labels": { + "log.file.name": "test.log" + } + }, + { + "logName": "projects/fakeprojectid/logs/multi-project", + "resource": { + "type": "gce_instance", + "labels": { + "instance_id": "", + "zone": "" + } + }, + "textPayload": "127.0.0.1 - - [26/Apr/2022:22:53:53 +0800] \"GET / HTTP/1.1\" 200 1247", + "timestamp": "1970-01-01T00:00:00Z", + "httpRequest": { + "requestMethod": "GET", + "requestUrl": "/", + "status": 200, + "responseSize": "1247", + "remoteIp": "127.0.0.1", + "protocol": "HTTP/1.1" + }, + "labels": { + "log.file.name": "test.log" + } + }, + { + "logName": "projects/fakeprojectid/logs/multi-project", + "resource": { + "type": "gce_instance", + "labels": { + "instance_id": "", + "zone": "" + } + }, + "textPayload": "127.0.0.1 - - [26/Apr/2022:22:53:54 +0800] \"GET / HTTP/1.1\" 200 1247", + "timestamp": "1970-01-01T00:00:00Z", + "httpRequest": { + "requestMethod": "GET", + "requestUrl": "/", + "status": 200, + "responseSize": "1247", + "remoteIp": "127.0.0.1", + "protocol": "HTTP/1.1" + }, + "labels": { + "log.file.name": "test.log" + } + }, + { + "logName": "projects/fakeprojectid/logs/multi-project", + "resource": { + "type": "gce_instance", + "labels": { + "instance_id": "", + "zone": "" + } + }, + "textPayload": "127.0.0.1 - - [26/Apr/2022:22:53:54 +0800] \"GET / HTTP/1.1\" 200 1247", + "timestamp": "1970-01-01T00:00:00Z", + "httpRequest": { + "requestMethod": "GET", + "requestUrl": "/", + "status": 200, + "responseSize": "1247", + "remoteIp": "127.0.0.1", + "protocol": "HTTP/1.1" + }, + "labels": { + "log.file.name": "test.log" + } + }, + { + "logName": "projects/fakeprojectid/logs/multi-project", + "resource": { + "type": "gce_instance", + "labels": { + "instance_id": "", + "zone": "" + } + }, + "textPayload": "127.0.0.1 - - [26/Apr/2022:22:53:54 +0800] \"GET / HTTP/1.1\" 200 1247", + "timestamp": "1970-01-01T00:00:00Z", + "httpRequest": { + "requestMethod": "GET", + "requestUrl": "/", + "status": 200, + "responseSize": "1247", + "remoteIp": "127.0.0.1", + "protocol": "HTTP/1.1" + }, + "labels": { + "log.file.name": "test.log" + } + }, + { + "logName": "projects/fakeprojectid/logs/multi-project", + "resource": { + "type": "gce_instance", + "labels": { + "instance_id": "", + "zone": "" + } + }, + "textPayload": "127.0.0.1 - - [26/Apr/2022:22:53:54 +0800] \"GET / HTTP/1.1\" 200 1247", + "timestamp": "1970-01-01T00:00:00Z", + "httpRequest": { + "requestMethod": "GET", + "requestUrl": "/", + "status": 200, + "responseSize": "1247", + "remoteIp": "127.0.0.1", + "protocol": "HTTP/1.1" + }, + "labels": { + "log.file.name": "test.log" + } + }, + { + "logName": "projects/fakeprojectid/logs/multi-project", + "resource": { + "type": "gce_instance", + "labels": { + "instance_id": "", + "zone": "" + } + }, + "textPayload": "127.0.0.1 - - [26/Apr/2022:22:53:54 +0800] \"GET / HTTP/1.1\" 200 1247", + "timestamp": "1970-01-01T00:00:00Z", + "httpRequest": { + "requestMethod": "GET", + "requestUrl": "/", + "status": 200, + "responseSize": "1247", + "remoteIp": "127.0.0.1", + "protocol": "HTTP/1.1" + }, + "labels": { + "log.file.name": "test.log" + } + }, + { + "logName": "projects/fakeprojectid/logs/multi-project", + "resource": { + "type": "gce_instance", + "labels": { + "instance_id": "", + "zone": "" + } + }, + "textPayload": "127.0.0.1 - - [26/Apr/2022:22:53:54 +0800] \"GET / HTTP/1.1\" 200 1247", + "timestamp": "1970-01-01T00:00:00Z", + "httpRequest": { + "requestMethod": "GET", + "requestUrl": "/", + "status": 200, + "responseSize": "1247", + "remoteIp": "127.0.0.1", + "protocol": "HTTP/1.1" + }, + "labels": { + "log.file.name": "test.log" + } + }, + { + "logName": "projects/fakeprojectid/logs/multi-project", + "resource": { + "type": "gce_instance", + "labels": { + "instance_id": "", + "zone": "" + } + }, + "textPayload": "127.0.0.2 - - [26/Apr/2022:22:54:38 +0800] \"GET / HTTP/1.1\" 200 4429", + "timestamp": "1970-01-01T00:00:00Z", + "httpRequest": { + "requestMethod": "GET", + "requestUrl": "/", + "status": 200, + "responseSize": "4429", + "remoteIp": "127.0.0.2", + "protocol": "HTTP/1.1" + }, + "labels": { + "log.file.name": "test.log" + } + }, + { + "logName": "projects/fakeprojectid/logs/multi-project", + "resource": { + "type": "gce_instance", + "labels": { + "instance_id": "", + "zone": "" + } + }, + "textPayload": "127.0.0.1 - - [26/Apr/2022:22:54:43 +0800] \"-\" 408 -", + "timestamp": "1970-01-01T00:00:00Z", + "labels": { + "log.file.name": "test.log" + } + } + ], + "partialSuccess": true + }, + { + "entries": [ + { + "logName": "projects/fake-other-project/logs/multi-project", + "resource": { + "type": "gce_instance", + "labels": { + "instance_id": "", + "zone": "" + } + }, + "textPayload": "127.0.0.1 - - [26/Apr/2022:22:53:36 +0800] \"GET / HTTP/1.1\" 200 1247", + "timestamp": "1970-01-01T00:00:00Z", + "httpRequest": { + "requestMethod": "GET", + "requestUrl": "/", + "status": 200, + "responseSize": "1247", + "remoteIp": "127.0.0.1", + "protocol": "HTTP/1.1" + }, + "labels": { + "log.file.name": "test.log" + } + }, + { + "logName": "projects/fake-other-project/logs/multi-project", + "resource": { + "type": "gce_instance", + "labels": { + "instance_id": "", + "zone": "" + } + }, + "textPayload": "127.0.0.1 - - [26/Apr/2022:22:53:37 +0800] \"GET /lamp.png HTTP/1.1\" 200 51164", + "timestamp": "1970-01-01T00:00:00Z", + "httpRequest": { + "requestMethod": "GET", + "requestUrl": "/lamp.png", + "status": 200, + "responseSize": "51164", + "remoteIp": "127.0.0.1", + "protocol": "HTTP/1.1" + }, + "labels": { + "log.file.name": "test.log" + } + }, + { + "logName": "projects/fake-other-project/logs/multi-project", + "resource": { + "type": "gce_instance", + "labels": { + "instance_id": "", + "zone": "" + } + }, + "textPayload": "127.0.0.1 - - [26/Apr/2022:22:53:37 +0800] \"GET /favicon.ico HTTP/1.1\" 200 3990", + "timestamp": "1970-01-01T00:00:00Z", + "httpRequest": { + "requestMethod": "GET", + "requestUrl": "/favicon.ico", + "status": 200, + "responseSize": "3990", + "remoteIp": "127.0.0.1", + "protocol": "HTTP/1.1" + }, + "labels": { + "log.file.name": "test.log" + } + }, + { + "logName": "projects/fake-other-project/logs/multi-project", + "resource": { + "type": "gce_instance", + "labels": { + "instance_id": "", + "zone": "" + } + }, + "textPayload": "127.0.0.1 - - [26/Apr/2022:22:53:51 +0800] \"GET / HTTP/1.1\" 200 1247", + "timestamp": "1970-01-01T00:00:00Z", + "httpRequest": { + "requestMethod": "GET", + "requestUrl": "/", + "status": 200, + "responseSize": "1247", + "remoteIp": "127.0.0.1", + "protocol": "HTTP/1.1" + }, + "labels": { + "log.file.name": "test.log" + } + }, + { + "logName": "projects/fake-other-project/logs/multi-project", + "resource": { + "type": "gce_instance", + "labels": { + "instance_id": "", + "zone": "" + } + }, + "textPayload": "127.0.0.1 - - [26/Apr/2022:22:53:52 +0800] \"GET / HTTP/1.1\" 200 1247", + "timestamp": "1970-01-01T00:00:00Z", + "httpRequest": { + "requestMethod": "GET", + "requestUrl": "/", + "status": 200, + "responseSize": "1247", + "remoteIp": "127.0.0.1", + "protocol": "HTTP/1.1" + }, + "labels": { + "log.file.name": "test.log" + } + }, + { + "logName": "projects/fake-other-project/logs/multi-project", + "resource": { + "type": "gce_instance", + "labels": { + "instance_id": "", + "zone": "" + } + }, + "textPayload": "127.0.0.1 - - [26/Apr/2022:22:53:53 +0800] \"GET / HTTP/1.1\" 200 1247", + "timestamp": "1970-01-01T00:00:00Z", + "httpRequest": { + "requestMethod": "GET", + "requestUrl": "/", + "status": 200, + "responseSize": "1247", + "remoteIp": "127.0.0.1", + "protocol": "HTTP/1.1" + }, + "labels": { + "log.file.name": "test.log" + } + }, + { + "logName": "projects/fake-other-project/logs/multi-project", + "resource": { + "type": "gce_instance", + "labels": { + "instance_id": "", + "zone": "" + } + }, + "textPayload": "127.0.0.1 - - [26/Apr/2022:22:53:53 +0800] \"GET / HTTP/1.1\" 200 1247", + "timestamp": "1970-01-01T00:00:00Z", + "httpRequest": { + "requestMethod": "GET", + "requestUrl": "/", + "status": 200, + "responseSize": "1247", + "remoteIp": "127.0.0.1", + "protocol": "HTTP/1.1" + }, + "labels": { + "log.file.name": "test.log" + } + }, + { + "logName": "projects/fake-other-project/logs/multi-project", + "resource": { + "type": "gce_instance", + "labels": { + "instance_id": "", + "zone": "" + } + }, + "textPayload": "127.0.0.1 - - [26/Apr/2022:22:53:53 +0800] \"GET / HTTP/1.1\" 200 1247", + "timestamp": "1970-01-01T00:00:00Z", + "httpRequest": { + "requestMethod": "GET", + "requestUrl": "/", + "status": 200, + "responseSize": "1247", + "remoteIp": "127.0.0.1", + "protocol": "HTTP/1.1" + }, + "labels": { + "log.file.name": "test.log" + } + }, + { + "logName": "projects/fake-other-project/logs/multi-project", + "resource": { + "type": "gce_instance", + "labels": { + "instance_id": "", + "zone": "" + } + }, + "textPayload": "127.0.0.1 - - [26/Apr/2022:22:53:53 +0800] \"GET / HTTP/1.1\" 200 1247", + "timestamp": "1970-01-01T00:00:00Z", + "httpRequest": { + "requestMethod": "GET", + "requestUrl": "/", + "status": 200, + "responseSize": "1247", + "remoteIp": "127.0.0.1", + "protocol": "HTTP/1.1" + }, + "labels": { + "log.file.name": "test.log" + } + }, + { + "logName": "projects/fake-other-project/logs/multi-project", + "resource": { + "type": "gce_instance", + "labels": { + "instance_id": "", + "zone": "" + } + }, + "textPayload": "127.0.0.1 - - [26/Apr/2022:22:53:54 +0800] \"GET / HTTP/1.1\" 200 1247", + "timestamp": "1970-01-01T00:00:00Z", + "httpRequest": { + "requestMethod": "GET", + "requestUrl": "/", + "status": 200, + "responseSize": "1247", + "remoteIp": "127.0.0.1", + "protocol": "HTTP/1.1" + }, + "labels": { + "log.file.name": "test.log" + } + }, + { + "logName": "projects/fake-other-project/logs/multi-project", + "resource": { + "type": "gce_instance", + "labels": { + "instance_id": "", + "zone": "" + } + }, + "textPayload": "127.0.0.1 - - [26/Apr/2022:22:53:54 +0800] \"GET / HTTP/1.1\" 200 1247", + "timestamp": "1970-01-01T00:00:00Z", + "httpRequest": { + "requestMethod": "GET", + "requestUrl": "/", + "status": 200, + "responseSize": "1247", + "remoteIp": "127.0.0.1", + "protocol": "HTTP/1.1" + }, + "labels": { + "log.file.name": "test.log" + } + }, + { + "logName": "projects/fake-other-project/logs/multi-project", + "resource": { + "type": "gce_instance", + "labels": { + "instance_id": "", + "zone": "" + } + }, + "textPayload": "127.0.0.1 - - [26/Apr/2022:22:53:54 +0800] \"GET / HTTP/1.1\" 200 1247", + "timestamp": "1970-01-01T00:00:00Z", + "httpRequest": { + "requestMethod": "GET", + "requestUrl": "/", + "status": 200, + "responseSize": "1247", + "remoteIp": "127.0.0.1", + "protocol": "HTTP/1.1" + }, + "labels": { + "log.file.name": "test.log" + } + }, + { + "logName": "projects/fake-other-project/logs/multi-project", + "resource": { + "type": "gce_instance", + "labels": { + "instance_id": "", + "zone": "" + } + }, + "textPayload": "127.0.0.1 - - [26/Apr/2022:22:53:54 +0800] \"GET / HTTP/1.1\" 200 1247", + "timestamp": "1970-01-01T00:00:00Z", + "httpRequest": { + "requestMethod": "GET", + "requestUrl": "/", + "status": 200, + "responseSize": "1247", + "remoteIp": "127.0.0.1", + "protocol": "HTTP/1.1" + }, + "labels": { + "log.file.name": "test.log" + } + }, + { + "logName": "projects/fake-other-project/logs/multi-project", + "resource": { + "type": "gce_instance", + "labels": { + "instance_id": "", + "zone": "" + } + }, + "textPayload": "127.0.0.1 - - [26/Apr/2022:22:53:54 +0800] \"GET / HTTP/1.1\" 200 1247", + "timestamp": "1970-01-01T00:00:00Z", + "httpRequest": { + "requestMethod": "GET", + "requestUrl": "/", + "status": 200, + "responseSize": "1247", + "remoteIp": "127.0.0.1", + "protocol": "HTTP/1.1" + }, + "labels": { + "log.file.name": "test.log" + } + }, + { + "logName": "projects/fake-other-project/logs/multi-project", + "resource": { + "type": "gce_instance", + "labels": { + "instance_id": "", + "zone": "" + } + }, + "textPayload": "127.0.0.1 - - [26/Apr/2022:22:53:54 +0800] \"GET / HTTP/1.1\" 200 1247", + "timestamp": "1970-01-01T00:00:00Z", + "httpRequest": { + "requestMethod": "GET", + "requestUrl": "/", + "status": 200, + "responseSize": "1247", + "remoteIp": "127.0.0.1", + "protocol": "HTTP/1.1" + }, + "labels": { + "log.file.name": "test.log" + } + }, + { + "logName": "projects/fake-other-project/logs/multi-project", + "resource": { + "type": "gce_instance", + "labels": { + "instance_id": "", + "zone": "" + } + }, + "textPayload": "127.0.0.2 - - [26/Apr/2022:22:54:38 +0800] \"GET / HTTP/1.1\" 200 4429", + "timestamp": "1970-01-01T00:00:00Z", + "httpRequest": { + "requestMethod": "GET", + "requestUrl": "/", + "status": 200, + "responseSize": "4429", + "remoteIp": "127.0.0.2", + "protocol": "HTTP/1.1" + }, + "labels": { + "log.file.name": "test.log" + } + }, + { + "logName": "projects/fake-other-project/logs/multi-project", + "resource": { + "type": "gce_instance", + "labels": { + "instance_id": "", + "zone": "" + } + }, + "textPayload": "127.0.0.1 - - [26/Apr/2022:22:54:43 +0800] \"-\" 408 -", + "timestamp": "1970-01-01T00:00:00Z", + "labels": { + "log.file.name": "test.log" + } + } + ], + "partialSuccess": true + } + ] +} diff --git a/exporter/collector/logs.go b/exporter/collector/logs.go index d98b7b32f..61cc0ea24 100644 --- a/exporter/collector/logs.go +++ b/exporter/collector/logs.go @@ -31,6 +31,7 @@ import ( "google.golang.org/genproto/googleapis/api/monitoredres" "google.golang.org/grpc" "google.golang.org/grpc/encoding/gzip" + "google.golang.org/grpc/metadata" "google.golang.org/protobuf/proto" "github.com/googleapis/gax-go/v2" @@ -171,42 +172,49 @@ func (l *LogsExporter) Shutdown(ctx context.Context) error { } func (l *LogsExporter) PushLogs(ctx context.Context, ld plog.Logs) error { - entries, err := l.mapper.createEntries(ld) + projectEntries, err := l.mapper.createEntries(ld) if err != nil { return err } errors := []error{} - entry := 0 - currentBatchSize := 0 - // Send entries in WriteRequest chunks - // TODO(damemi): Add integration test for batch request processing - for len(entries) > 0 { - // default to max int so that when we are at index=len we skip the size check to avoid panic - // (index=len is the break condition when we reassign entries=entries[len:]) - entrySize := defaultMaxRequestSize - if entry < len(entries) { - entrySize = proto.Size(entries[entry]) - } + for project, entries := range projectEntries { + entry := 0 + currentBatchSize := 0 + // Send entries in WriteRequest chunks + // TODO(damemi): Add integration test for batch request processing + for len(entries) > 0 { + // default to max int so that when we are at index=len we skip the size check to avoid panic + // (index=len is the break condition when we reassign entries=entries[len:]) + entrySize := defaultMaxRequestSize + if entry < len(entries) { + entrySize = proto.Size(entries[entry]) + } - // this block gets skipped if we are out of entries to check - if currentBatchSize+entrySize < defaultMaxRequestSize { - // if adding the current entry to the current batch doesn't go over the request size, - // increase the index and account for the new request size, then continue - currentBatchSize += entrySize - entry++ - continue - } + // this block gets skipped if we are out of entries to check + if currentBatchSize+entrySize < defaultMaxRequestSize { + // if adding the current entry to the current batch doesn't go over the request size, + // increase the index and account for the new request size, then continue + currentBatchSize += entrySize + entry++ + continue + } - // if the current entry goes over the request size (or we have gone over every entry, i.e. index=len), - // write the list up to but not including the current entry's index - _, err := l.writeLogEntries(ctx, entries[:entry]) - if err != nil { - errors = append(errors, err) - } + // override destination project quota for this write request, if applicable + if l.cfg.DestinationProjectQuota { + ctx = metadata.NewOutgoingContext(ctx, metadata.New(map[string]string{"x-goog-user-project": strings.TrimPrefix(project, "projects/")})) + } + + // if the current entry goes over the request size (or we have gone over every entry, i.e. index=len), + // write the list up to but not including the current entry's index + _, err := l.writeLogEntries(ctx, entries[:entry]) + if err != nil { + errors = append(errors, err) + } - entries = entries[entry:] - entry = 0 + entries = entries[entry:] + entry = 0 + } } if len(errors) > 0 { @@ -215,9 +223,12 @@ func (l *LogsExporter) PushLogs(ctx context.Context, ld plog.Logs) error { return nil } -func (l logMapper) createEntries(ld plog.Logs) ([]*logpb.LogEntry, error) { +func (l logMapper) createEntries(ld plog.Logs) (map[string][]*logpb.LogEntry, error) { + // if destination_project_quota is enabled, projectMapKey will be the name of the project for each batch of entries + // otherwise, we can mix project entries for more efficient batching and store all entries in a single list + projectMapKey := "" errors := []error{} - entries := make([]*logpb.LogEntry, 0) + entries := make(map[string][]*logpb.LogEntry) for i := 0; i < ld.ResourceLogs().Len(); i++ { rl := ld.ResourceLogs().At(i) mr := defaultResourceToMonitoredResource(rl.Resource()) @@ -269,7 +280,13 @@ func (l logMapper) createEntries(ld plog.Logs) ([]*logpb.LogEntry, error) { errors = append(errors, err) continue } - entries = append(entries, internalLogEntry) + if l.cfg.DestinationProjectQuota { + projectMapKey = projectID + } + if _, ok := entries[projectMapKey]; !ok { + entries[projectMapKey] = make([]*logpb.LogEntry, 0) + } + entries[projectMapKey] = append(entries[projectMapKey], internalLogEntry) } } }