LOG-902: Support Http output

Makefile

@@ -253,13 +253,15 @@ test-functional-fluentd:
 
 .PHONY: test-functional-vector
 test-functional-vector:
+	RELATED_IMAGE_FLUENTD=$(IMAGE_LOGGING_FLUENTD) \
 	RELATED_IMAGE_VECTOR=$(IMAGE_LOGGING_VECTOR) \
 	RELATED_IMAGE_LOG_FILE_METRIC_EXPORTER=$(IMAGE_LOGFILEMETRICEXPORTER) \
 	go test --tags=vector -race \
 		./test/functional/outputs/elasticsearch/... \
 		./test/functional/outputs/kafka/... \
 		./test/functional/outputs/cloudwatch/... \
 		./test/functional/outputs/loki/... \
+		./test/functional/outputs/http/... \
 		./test/functional/normalization \
 		-ginkgo.noColor -timeout=40m -ginkgo.slowSpecThreshold=45.0
 

apis/logging/v1/cluster_log_forwarder_types.go

@@ -162,7 +162,7 @@ type OutputSpec struct {
 
 	// Type of output plugin.
 	//
-	// +kubebuilder:validation:Enum:=syslog;fluentdForward;elasticsearch;kafka;cloudwatch;loki;googleCloudLogging;splunk
+	// +kubebuilder:validation:Enum:=syslog;fluentdForward;elasticsearch;kafka;cloudwatch;loki;googleCloudLogging;splunk;http
 	// +required
 	Type string `json:"type"`
 

apis/logging/v1/output_types.go

@@ -13,6 +13,7 @@ const (
 	OutputTypeLoki               = "loki"
 	OutputTypeGoogleCloudLogging = "googleCloudLogging"
 	OutputTypeSplunk             = "splunk"
+	OutputTypeHttp               = "http"
 )
 
 // OutputTypeSpec is a union of optional additional configuration specific to an
@@ -30,10 +31,12 @@ type OutputTypeSpec struct {
 	Cloudwatch *Cloudwatch `json:"cloudwatch,omitempty"`
 	// +optional
 	Loki *Loki `json:"loki,omitempty"`
-	//+optional
+	// +optional
 	GoogleCloudLogging *GoogleCloudLogging `json:"googleCloudLogging,omitempty"`
 	// +optional
 	Splunk *Splunk `json:"splunk,omitempty"`
+	// +optional
+	Http *Http `json:"http,omitempty"`
 }
 
 // Cloudwatch provides configuration for the output type `cloudwatch`
@@ -282,3 +285,24 @@ type Splunk struct {
 	// +optional
 	Fields []string `json:"fields,omitempty"`
 }
+
+// Http provided configuration for sending json encoded logs to a generic http endpoint.
+type Http struct {
+	// Headers specify optional headers to be sent with the request
+	// +optional
+	Headers map[string]string `json:"headers,omitempty"`
+
+	// Timeout specifies the Http request timeout in seconds. If not set, 10secs is used.
+	// +optional
+	Timeout string `json:"timeout,omitempty"`
+
+	// Method specifies the Http method to be used for sending logs. If not set, 'POST' is used.
+	// +kubebuilder:validation:Enum:=GET;HEAD;POST;PUT;DELETE;OPTIONS;TRACE;PATCH
+	// +optional
+	Method string `json:"method,omitempty"`
+
+	// Compression specifies the compression type to be used. If not set, defaults to 'none' which does not compress
+	// +kubebuilder:validation:Enum:=gzip;none
+	// +optional
+	Compression string `json:"compression,omitempty"`
+}

bundle/manifests/logging.openshift.io_clusterlogforwarders.yaml

@@ -215,6 +215,42 @@ spec:
                         projectId:
                           type: string
                       type: object
+                    http:
+                      description: Http provided configuration for sending json encoded
+                        logs to a generic http endpoint.
+                      properties:
+                        compression:
+                          description: Compression specifies the compression type
+                            to be used. If not set, defaults to 'none' which does
+                            not compress
+                          enum:
+                          - gzip
+                          - none
+                          type: string
+                        headers:
+                          additionalProperties:
+                            type: string
+                          description: Headers specify optional headers to be sent
+                            with the request
+                          type: object
+                        method:
+                          description: Method specifies the Http method to be used
+                            for sending logs. If not set, 'POST' is used.
+                          enum:
+                          - GET
+                          - HEAD
+                          - POST
+                          - PUT
+                          - DELETE
+                          - OPTIONS
+                          - TRACE
+                          - PATCH
+                          type: string
+                        timeout:
+                          description: Timeout specifies the Http request timeout
+                            in seconds. If not set, 10secs is used.
+                          type: string
+                      type: object
                     kafka:
                       description: 'Kafka provides optional extra properties for `type:
                         kafka`'
@@ -399,6 +435,7 @@ spec:
                       - loki
                       - googleCloudLogging
                       - splunk
+                      - http
                       type: string
                     url:
                       description: "URL to send log records to. \n An absolute URL,

config/crd/bases/logging.openshift.io_clusterlogforwarders.yaml

@@ -211,6 +211,42 @@ spec:
                         projectId:
                           type: string
                       type: object
+                    http:
+                      description: Http provided configuration for sending json encoded
+                        logs to a generic http endpoint.
+                      properties:
+                        compression:
+                          description: Compression specifies the compression type
+                            to be used. If not set, defaults to 'none' which does
+                            not compress
+                          enum:
+                          - gzip
+                          - none
+                          type: string
+                        headers:
+                          additionalProperties:
+                            type: string
+                          description: Headers specify optional headers to be sent
+                            with the request
+                          type: object
+                        method:
+                          description: Method specifies the Http method to be used
+                            for sending logs. If not set, 'POST' is used.
+                          enum:
+                          - GET
+                          - HEAD
+                          - POST
+                          - PUT
+                          - DELETE
+                          - OPTIONS
+                          - TRACE
+                          - PATCH
+                          type: string
+                        timeout:
+                          description: Timeout specifies the Http request timeout
+                            in seconds. If not set, 10secs is used.
+                          type: string
+                      type: object
                     kafka:
                       description: 'Kafka provides optional extra properties for `type:
                         kafka`'
@@ -395,6 +431,7 @@ spec:
                       - loki
                       - googleCloudLogging
                       - splunk
+                      - http
                       type: string
                     url:
                       description: "URL to send log records to. \n An absolute URL,

docs/features/collection.adoc

@@ -42,6 +42,7 @@ logstash 7.10.1|✓|
 |Loki|REST over HTTP(S)|Loki 2.3.0|✓|✓
 |Splunk|HEC|v9.0.0||✓
 |Syslog|RFC3164,RFC5424|rsyslog 8.39.0|✓|
+|Http|Http 1.1|vector 0.21, fluentd 1.14.6||✓
 |======
 
 === Authorization and Authentication
@@ -50,11 +51,11 @@ logstash 7.10.1|✓|
 [options="header"]
 |======
 |Feature|Output Type|Fluentd|Vector
-|Token|loki, splunk||
-|Certificates|elasticsearch, kafka, fluentd forward, splunk|✓|✓
+|Token|loki, splunk, http||
+|Certificates|elasticsearch, kafka, fluentd forward, splunk, http|✓|✓
 |Cloud service keys|cloudwatch|✓|✓
 |Cloud service keys| google||✓
-|Username / Password|elasticsearch, kafka||
+|Username / Password|elasticsearch, kafka, http||
 |Security Token Service (STS)|cloudwatch|✓|✓
 |SASL|kafka|✓|✓
 |======

internal/generator/vector/output/http/basic.go

@@ -0,0 +1,19 @@
+package http
+
+import (
+	"github.com/openshift/cluster-logging-operator/internal/generator"
+)
+
+type BasicAuthConf generator.ConfLiteral
+
+func (t BasicAuthConf) Name() string {
+	return "httpBasicAuthConf"
+}
+
+func (t BasicAuthConf) Template() string {
+	return `
+{{define "httpBasicAuthConf" -}}
+# {{.Desc}}
+[sinks.{{.ComponentID}}.auth]
+{{- end}}`
+}

internal/generator/vector/output/http/bearer_token.go

@@ -0,0 +1,17 @@
+package http
+
+import "github.com/openshift/cluster-logging-operator/internal/generator/vector/output/security"
+
+type BearerToken security.BearerToken
+
+func (bt BearerToken) Name() string {
+	return "httpBearerTokenTemplate"
+}
+
+func (bt BearerToken) Template() string {
+	return `{{define "` + bt.Name() + `" -}}
+strategy = "bearer"
+token = "{{.Token}}"
+{{end}}
+`
+}

internal/generator/vector/output/http/ca_file.go

@@ -0,0 +1,18 @@
+package http
+
+import (
+	"github.com/openshift/cluster-logging-operator/internal/generator/vector/output/security"
+)
+
+type CAFile security.CAFile
+
+func (ca CAFile) Name() string {
+	return "httpCAFileTemplate"
+}
+
+func (ca CAFile) Template() string {
+	return `{{define "` + ca.Name() + `" -}}
+ca_file = {{.CAFilePath}}
+{{- end}}
+`
+}

internal/generator/vector/output/http/cert_key.go

@@ -0,0 +1,18 @@
+package http
+
+import (
+	"github.com/openshift/cluster-logging-operator/internal/generator/vector/output/security"
+)
+
+type TLSKeyCert security.TLSCertKey
+
+func (kc TLSKeyCert) Name() string {
+	return "httpCertKeyTemplate"
+}
+
+func (kc TLSKeyCert) Template() string {
+	return `{{define "` + kc.Name() + `" -}}
+key_file = {{.KeyPath}}
+crt_file = {{.CertPath}}
+{{- end}}`
+}