[exporter/Logzioexporter] logs support

CHANGELOG.md

@@ -45,6 +45,7 @@
 - `datadogexporter`: Deprecate `Sanitize` method of `Config` struct (#8829)
 - `observiqexporter`: Deprecate the observiq exporter (#10977)
 - `honeycombexporter`: Deprecate honeycomb exporter (#10318)
+- `logzioexporter`: Announcing `custom_endpoint`, `drain_interval`, `queue_capacity`, `queue_max_length` configuration options will be deprecated in upcoming releases (#10821)
 
 ### 🚀 New components 🚀
 
@@ -54,6 +55,7 @@
 
 ### 💡 Enhancements 💡
 
+- `logzioexporter`: Add support for logs pipeline and support for exporterhelper (#10821)
 - `tailsamplingprocessor`: Add trace_state policy (#10852)
 - `mongodbatlasreceiver` Add support for receiving alerts (#10854)
 - `cmd/mdatagen`: Allow attribute values of any types (#9245)
@@ -108,6 +110,9 @@
 - `couchbasereceiver`: Fully removed unimplemented Couchbase receiver (#10482)
 - `hostmetricsreciever`: Fix Load Scraper to normalize 1m, 5m, and 15m averages independently (#8267)
 
+### 🚩 Deprecations 🚩
+
+
 ### 🚀 New components 🚀
 
 - `flinkmetricsreceiver`: Add implementation of Flink Metric Receiver (#10121)

cmd/configschema/go.mod

@@ -78,7 +78,6 @@ require (
 	github.com/aws/aws-sdk-go-v2/service/sso v1.4.2 // indirect
 	github.com/aws/aws-sdk-go-v2/service/sts v1.7.2 // indirect
 	github.com/aws/smithy-go v1.11.0 // indirect
-	github.com/beeker1121/goque v2.1.0+incompatible // indirect
 	github.com/beorn7/perks v1.0.1 // indirect
 	github.com/blang/semver v3.5.1+incompatible // indirect
 	github.com/bmatcuk/doublestar/v3 v3.0.0 // indirect
@@ -215,8 +214,6 @@ require (
 	github.com/leoluk/perflib_exporter v0.1.0 // indirect
 	github.com/lib/pq v1.10.6 // indirect
 	github.com/linode/linodego v1.5.0 // indirect
-	github.com/logzio/jaeger-logzio v1.0.6 // indirect
-	github.com/logzio/logzio-go v1.0.6 // indirect
 	github.com/lufia/plan9stats v0.0.0-20211012122336-39d0f177ccd0 // indirect
 	github.com/magiconair/properties v1.8.6 // indirect
 	github.com/mailru/easyjson v0.7.7 // indirect
@@ -243,7 +240,6 @@ require (
 	github.com/mwitkow/go-conntrack v0.0.0-20190716064945-2f068394615f // indirect
 	github.com/nginxinc/nginx-prometheus-exporter v0.8.1-0.20201110005315-f5a5f8086c19 // indirect
 	github.com/observiq/ctimefmt v1.0.0 // indirect
-	github.com/olivere/elastic v6.2.37+incompatible // indirect
 	github.com/open-telemetry/opentelemetry-collector-contrib/exporter/alibabacloudlogserviceexporter v0.54.0 // indirect
 	github.com/open-telemetry/opentelemetry-collector-contrib/exporter/awscloudwatchlogsexporter v0.54.0 // indirect
 	github.com/open-telemetry/opentelemetry-collector-contrib/exporter/awsemfexporter v0.54.0 // indirect
@@ -459,7 +455,6 @@ require (
 	github.com/stretchr/objx v0.4.0 // indirect
 	github.com/subosito/gotenv v1.2.0 // indirect
 	github.com/syndtr/gocapability v0.0.0-20200815063812-42c35b437635 // indirect
-	github.com/syndtr/goleveldb v1.0.0 // indirect
 	github.com/tencentcloud/tencentcloud-sdk-go/tencentcloud/common v1.0.429 // indirect
 	github.com/tg123/go-htpasswd v1.2.0 // indirect
 	github.com/tidwall/gjson v1.12.1 // indirect

exporter/logzioexporter/README.md

@@ -1,39 +1,115 @@
 # Logzio Exporter
 
-| Status                   |            |
-| ------------------------ |------------|
-| Stability                | [stable]   |
-| Supported pipeline types | traces     |
-| Distributions            | [contrib]  |
-
-This exporter supports sending trace data to [Logz.io](https://www.logz.io)
-
-The following configuration options are supported:
-
-* `account_token` (Required): Your logz.io account token for your tracing account.
-* `metrics_token` (Optional): This is deprecated, but may be used for the OpenSearch/ElasticSearch based Metrics backend.
-* `region` (Optional): Your logz.io account [region code](https://docs.logz.io/user-guide/accounts/account-region.html#available-regions). Defaults to `us`. Required only if your logz.io region is different than US.
-* `custom_endpoint` (Optional): Custom endpoint, mostly used for dev or testing. This will override the region parameter.
-* `drain_interval` (Optional): Queue drain interval in seconds. Defaults to `3`.
-* `queue_capacity` (Optional): Queue capacity in bytes. Defaults to `20 * 1024 * 1024` ~ 20mb.
-* `queue_max_length` (Optional): Max number of items allowed in the queue. Defaults to `500000`.
-
-Example:
+| Status                   |                       |
+| ------------------------ | --------------------- |
+| Stability                | traces [stable]       |
+|                          | logs [beta]           |
+| Supported pipeline types | traces, logs |
+| Distributions            |  [contrib]   |
+
+This exporter supports sending trace and log data to [Logz.io](https://www.logz.io)
+
+### The following configuration options are supported:
+Logz.io exporter is utilizing opentelemetry [exporter helper](https://github.com/open-telemetry/opentelemetry-collector/blob/main/exporter/exporterhelper/README.md) for `retry_on_failure`,`sending_queue` and `timeout` settings
+- `account_token` (Required): Your logz.io account token for your tracing or logs account.
+- `region` Your logz.io account [region code](https://docs.logz.io/user-guide/accounts/account-region.html#available-regions). Defaults to `us`. Required only if your logz.io region is different than US.
+- `endpoint` Custom endpoint, mostly used for dev or testing. This will override the region parameter.
+- `retry_on_failure` 
+    - `enabled` (default = true)
+    - `initial_interval`: Time to wait after the first failure before retrying; ignored if `enabled` is `false`  (default = 5s)
+    - `max_interval`: Is the upper bound on backoff; ignored if `enabled` is `false` (default = 30s)
+    - `max_elapsed_time`: Is the maximum amount of time spent trying to send a batch; ignored if `enabled` is `false` (default = 300s)
+- `sending_queue`
+    - `enabled` (default = true)
+    - `num_consumers`: Number of consumers that dequeue batches; ignored if `enabled` is `false` (default = 10)
+    - `queue_size`: Maximum number of batches kept in memory before dropping; ignored if `enabled` is `false`
+      User should calculate this as `num_seconds * requests_per_second` where:
+        - `num_seconds` is the number of seconds to buffer in case of a backend outage
+        - `requests_per_second` is the average number of requests per seconds.
+        - default = 5000
+- `timeout`: Time to wait per individual attempt to send data to a backend. default = 30s
+
+#### Tracing example:
+* We recommend using `batch` processor. Batching helps better compress the data and reduce the number of outgoing connections required to transmit the data.
 
 ```yaml
+receivers:
+  otlp:
+    protocols:
+      grpc:
+        endpoint: "0.0.0.0:4317"
+      http:
+        endpoint: "0.0.0.0:4318"
+  jaeger:
+    protocols:
+      thrift_compact:
+        endpoint: "0.0.0.0:6831"
+      thrift_binary:
+        endpoint: "0.0.0.0:6832"
+      grpc:
+        endpoint: "0.0.0.0:14250"
+      thrift_http:
+        endpoint: "0.0.0.0:14268"
+processors:
+  batch:
+    send_batch_size: 10000
+    timeout: 1s
 exporters:
-  logzio:
+  logzio/traces:
     account_token: "LOGZIOtraceTOKEN"
-    metrics_token: "LOGZIOmetricsTOKEN"
-    region: "eu"
-    drain_interval: 3
-    queue_capacity: 5000000
-    queue_max_length: 500000
+    region: "us"
+service:
+  pipelines:
+    traces:
+      receivers: [ otlp,jaeger ]
+      processors: [ batch ]
+      exporters: [ logzio/traces ]
+  telemetry:
+    logs:
+      level: "debug"
 ```
-In order to use the Prometheus backend you must use the standard prometheusremotewrite exporter as well. The following [regions](https://docs.logz.io/user-guide/accounts/account-region.html#supported-regions-for-prometheus-metrics) are supported and configured as follows. The Logz.io Listener URL for for your region, configured to use port 8052 for http traffic, or port 8053 for https traffic.
+#### Logs example:
+* We recommend using `batch` processor. Batching helps better compress the data and reduce the number of outgoing connections required to transmit the data.
+* We recommend adding `type` attribute to classify your log records
+* We recommend adding `resourcedetection` processor to add metadata to your log records
 
+```yaml
+receivers:
+  filelog:
+    include: [ "/private/var/log/*.log" ] # MacOs system logs
+    include_file_name: false
+    include_file_path: true 
+    operators:
+      - type: move
+        from: attributes["log.file.path"]
+        to: attributes["log_file_path"]
+    attributes:
+      type: <<your-logzio-type>>
+processors:
+  batch:
+    send_batch_size: 10000
+    timeout: 1s
+  resourcedetection/system:
+    detectors: [ "system" ]
+    system:
+      hostname_sources: [ "os" ]
+exporters:
+  logzio/logs:
+    account_token: "LOGZIOlogsTOKEN"
+    region: "us"
+service:
+  pipelines:
+    logs:
+      receivers: [filelog]
+      processors: [ resourcedetection/system, batch ]
+      exporters: [logzio/logs]
+  telemetry:
+    logs:
+      level: "debug"
+```
+#### Metrics:
+In order to use the Prometheus backend you must use the standard prometheusremotewrite exporter as well. The following [regions](https://docs.logz.io/user-guide/accounts/account-region.html#supported-regions-for-prometheus-metrics) are supported and configured as follows. The Logz.io Listener URL for for your region, configured to use port 8052 for http traffic, or port 8053 for https traffic.
 Example:
-
 ```yaml
 exporters:
   prometheusremotewrite:
@@ -60,7 +136,7 @@ receivers:
           - targets: [ "0.0.0.0:8889" ]
 
 exporters:
-  logzio:
+  logzio/traces:
     account_token: "LOGZIOtraceTOKEN"
     region: "us"
 
@@ -69,11 +145,17 @@ exporters:
     headers:
       Authorization: "Bearer LOGZIOprometheusTOKEN"
 
+processors:
+  batch:
+    send_batch_size: 10000
+    timeout: 1s
+
 service:
   pipelines:
     traces:
       receivers: [jaeger]
-      exporters: [logzio]
+      processors: [batch]
+      exporters: [logzio/traces]
 
     metrics:
       receivers: [prometheus]
@@ -84,5 +166,7 @@ service:
       level: debug #activate debug mode
 ```
 
+
+[beta]:https://github.com/open-telemetry/opentelemetry-collector#beta
 [stable]:https://github.com/open-telemetry/opentelemetry-collector#stable
-[contrib]:https://github.com/open-telemetry/opentelemetry-collector-releases/tree/main/distributions/otelcol-contrib
+[contrib]:https://github.com/open-telemetry/opentelemetry-collector-releases/tree/main/distributions/otelcol-contrib

exporter/logzioexporter/config.go

@@ -17,24 +17,52 @@ package logzioexporter // import "github.com/open-telemetry/opentelemetry-collec
 import (
 	"errors"
 
+	"github.com/hashicorp/go-hclog"
 	"go.opentelemetry.io/collector/config"
+	"go.opentelemetry.io/collector/config/confighttp"
+	"go.opentelemetry.io/collector/exporter/exporterhelper"
 )
 
 // Config contains Logz.io specific configuration such as Account TracesToken, Region, etc.
 type Config struct {
-	config.ExporterSettings `mapstructure:",squash"`
-	TracesToken             string `mapstructure:"account_token"`    // Your Logz.io Account Token, can be found at https://app.logz.io/#/dashboard/settings/general
-	MetricsToken            string `mapstructure:"metrics_token"`    // Your Logz.io Metrics Token, can be found at https://docs.logz.io/user-guide/accounts/finding-your-metrics-account-token/
-	Region                  string `mapstructure:"region"`           // Your Logz.io 2-letter region code, can be found at https://docs.logz.io/user-guide/accounts/account-region.html#available-regions
-	CustomEndpoint          string `mapstructure:"custom_endpoint"`  // Custom endpoint to ship traces to. Use only for dev and tests.
-	DrainInterval           int    `mapstructure:"drain_interval"`   // Queue drain interval in seconds. Defaults to `3`.
-	QueueCapacity           int64  `mapstructure:"queue_capacity"`   // Queue capacity in bytes. Defaults to `20 * 1024 * 1024` ~ 20mb.
-	QueueMaxLength          int    `mapstructure:"queue_max_length"` // Max number of items allowed in the queue. Defaults to `500000`.
+	config.ExporterSettings       `mapstructure:",squash"`          // squash ensures fields are correctly decoded in embedded struct
+	confighttp.HTTPClientSettings `mapstructure:",squash"`          // confighttp client settings https://pkg.go.dev/go.opentelemetry.io/collector/config/confighttp#HTTPClientSettings
+	exporterhelper.QueueSettings  `mapstructure:"sending_queue"`    // exporter helper queue settings https://pkg.go.dev/go.opentelemetry.io/collector/exporter/exporterhelper#QueueSettings
+	exporterhelper.RetrySettings  `mapstructure:"retry_on_failure"` // exporter helper retry settings https://pkg.go.dev/go.opentelemetry.io/collector/exporter/exporterhelper#RetrySettings
+	Token                         string                            `mapstructure:"account_token"`    // Your Logz.io Account Token, can be found at https://app.logz.io/#/dashboard/settings/general
+	Region                        string                            `mapstructure:"region"`           // Your Logz.io 2-letter region code, can be found at https://docs.logz.io/user-guide/accounts/account-region.html#available-regions
+	CustomEndpoint                string                            `mapstructure:"custom_endpoint"`  // **Deprecation** Custom endpoint to ship traces to. Use only for dev and tests.
+	DrainInterval                 int                               `mapstructure:"drain_interval"`   // **Deprecation** Queue drain interval in seconds. Defaults to `3`.
+	QueueCapacity                 int64                             `mapstructure:"queue_capacity"`   // **Deprecation** Queue capacity in bytes. Defaults to `20 * 1024 * 1024` ~ 20mb.
+	QueueMaxLength                int                               `mapstructure:"queue_max_length"` // **Deprecation** Max number of items allowed in the queue. Defaults to `500000`.
 }
 
-func (c *Config) validate() error {
-	if c.TracesToken == "" {
+func (c *Config) Validate() error {
+	if c.Token == "" {
 		return errors.New("`account_token` not specified")
 	}
 	return nil
 }
+
+// CheckAndWarnDeprecatedOptions Is checking for soon deprecated configuration options (queue_max_length, queue_capacity, drain_interval, custom_endpoint) log a warning message and map to the relevant updated option
+func (c *Config) checkAndWarnDeprecatedOptions(logger hclog.Logger) {
+	if c.QueueCapacity != 0 {
+		logger.Warn("You are using  the deprecated`queue_capacity` option that will be removed in the next release; use exporter helper configuration instead: https://github.com/open-telemetry/opentelemetry-collector/blob/main/exporter/exporterhelper/README.md")
+	}
+	// Warn and map queue_max_length -> QueueSettings.QueueSize
+	if c.QueueMaxLength != 0 {
+		logger.Warn("You are using the deprecated `queue_max_length` option that will be removed in the next release; use exporter helper `queue_size` configuration instead: https://github.com/open-telemetry/opentelemetry-collector/blob/main/exporter/exporterhelper/README.md")
+		c.QueueSettings.Enabled = true
+		logger.Warn("Mapping `queue_max_length` -> `QueueSettings.QueueSize`")
+		c.QueueSettings.QueueSize = c.QueueMaxLength
+	}
+	if c.DrainInterval != 0 {
+		logger.Warn("You are using the deprecated `drain_interval` option that will be removed in the next release; use batch processor `timeout` configuration instead: https://github.com/open-telemetry/opentelemetry-collector/tree/main/processor/batchprocessor#batch-processor")
+	}
+	// Warn and map CustomEndpoint -> Endpoint
+	if c.CustomEndpoint != "" {
+		logger.Warn("You are using the deprecated `custom_endpoint` option that will be removed in the next release; please use `endpoint` configuration instead: https://github.com/open-telemetry/opentelemetry-collector/tree/main/config/confighttp")
+		logger.Warn("Mapping `custom_endpoint` -> `Endpoint`")
+		c.HTTPClientSettings.Endpoint = c.CustomEndpoint
+	}
+}