diff --git a/.github/workflows/publish-version-4.3.yaml b/.github/workflows/publish-version-4.3.yaml new file mode 100644 index 000000000..26f40f26a --- /dev/null +++ b/.github/workflows/publish-version-4.3.yaml @@ -0,0 +1,86 @@ +name: Publish version 4.3 + +env: + doc_versionnumber: "4.3" + +on: + push: + branches: + - release-4.3 + workflow_dispatch: + +jobs: + build: + name: Build + runs-on: ubuntu-latest + + permissions: + contents: write + pages: write + id-token: write + + concurrency: + group: "pages" + cancel-in-progress: false + + environment: + name: github-pages-test + url: ${{ steps.deployment.outputs.page_url }} + + steps: + - name: Checkout code + uses: actions/checkout@f43a0e5ff2bd294095638e18286ca9a3d1956744 # v3.6.0 + with: + ref: release-4.3 + submodules: 'recursive' + + - name: Set up Pages + id: pages + uses: actions/configure-pages@f156874f8191504dae5b037505266ed5dda6c382 # v3.0.6 + + - name: Set up Hugo + uses: peaceiris/actions-hugo@16361eb4acea8698b220b76c0d4e84e1fd22c61d # v2.6.0 + with: + hugo-version: '0.110.0' + extended: true + + - name: Set up Node + uses: actions/setup-node@5e21ff4d9bc1a8cf6de233a3057d20ec6b3fb69d # v3.8.1 + with: + node-version: 16 + + - name: Install dependencies + run: | + cd themes/docsy + npm install + + - name: Set up PostCSS + run: npm install --save-dev autoprefixer postcss-cli postcss + + - name: Build + run: hugo --environment production --baseURL ${{ steps.pages.outputs.base_url }}/${{ env.doc_versionnumber }}/ + + # - name: Upload artifact + # uses: actions/upload-pages-artifact@64bcae551a7b18bcb9a09042ddf1960979799187 # v1.0.8 + # with: + # path: ./public/ + + - name: Checkout code to update + uses: actions/checkout@f43a0e5ff2bd294095638e18286ca9a3d1956744 # v3.6.0 + with: + ref: 'gh-pages-test' + path: 'tmp/gh-pages' + # - name: Display file structure + # run: ls -R + - name: Copy built site to GH pages + run: | + rm -rf tmp/gh-pages/${{ env.doc_versionnumber }} + mkdir -p tmp/gh-pages/${{ env.doc_versionnumber }} + mv public/* tmp/gh-pages/${{ env.doc_versionnumber }} + - name: Commit & Push changes + uses: actions-js/push@master + with: + github_token: ${{ secrets.GITHUB_TOKEN }} + message: 'Publish updated docs for ${{ env.doc_versionnumber }}, ${{ github.event.repository.pushed_at}}' + branch: 'gh-pages-test' + directory: 'tmp/gh-pages' diff --git a/config/_default/config.toml b/config/_default/config.toml index 786b873d3..d46366b96 100644 --- a/config/_default/config.toml +++ b/config/_default/config.toml @@ -69,7 +69,7 @@ twitter = "calisti12" # Used in the "version-banner" partial to display a version number for the # current doc set. - version = "4.3.0" + version = "4.4.0" version_menu = "Releases" version_menu_canonicallinks = true version_menu_pagelinks = true @@ -169,9 +169,13 @@ twitter = "calisti12" ####################### # Add your release versions here [[params.versions]] - version = "latest (4.3.0)" + version = "latest (4.4.0)" githubbranch = "master" url = "" +[[params.versions]] + version = "4.3" + githubbranch = "release-4.3" + url = "/4.3/" [[params.versions]] version = "4.2" githubbranch = "release-4.2" diff --git a/content/docs/configuration/crds/v1beta1/fluentbit_types.md b/content/docs/configuration/crds/v1beta1/fluentbit_types.md index eb47fd0d6..f85d92ee9 100644 --- a/content/docs/configuration/crds/v1beta1/fluentbit_types.md +++ b/content/docs/configuration/crds/v1beta1/fluentbit_types.md @@ -202,6 +202,10 @@ Default: - Default: - +### bufferVolumeResources (corev1.ResourceRequirements, optional) {#fluentbitspec-buffervolumeresources} + +Default: - + ### customConfigSecret (string, optional) {#fluentbitspec-customconfigsecret} Default: - @@ -607,7 +611,7 @@ Default: "0" ### Kube_URL (string, optional) {#filterkubernetes-kube_url} -API Server end-point (default: `https://kubernetes.default.svc:443`) +API Server end-point. Default: `https://kubernetes.default.svc:443` diff --git a/content/docs/configuration/crds/v1beta1/fluentd_types.md b/content/docs/configuration/crds/v1beta1/fluentd_types.md index e77be3d1c..8346a9ff5 100644 --- a/content/docs/configuration/crds/v1beta1/fluentd_types.md +++ b/content/docs/configuration/crds/v1beta1/fluentd_types.md @@ -132,6 +132,10 @@ Default: - Default: - +### bufferVolumeResources (corev1.ResourceRequirements, optional) {#fluentdspec-buffervolumeresources} + +Default: - + ### security (*Security, optional) {#fluentdspec-security} Default: - @@ -299,6 +303,10 @@ Container image to use for the drain watch sidecar Default: - +### labels (map[string]string, optional) {#fluentddrainconfig-labels} + +Labels to use for the drain watch sidecar on top of labels added by the operator by default. Default values can be overwritten. + ### deleteVolume (bool, optional) {#fluentddrainconfig-deletevolume} Should persistent volume claims be deleted after draining is done diff --git a/content/docs/configuration/crds/v1beta1/logging_types.md b/content/docs/configuration/crds/v1beta1/logging_types.md index 4c852b161..e32103877 100644 --- a/content/docs/configuration/crds/v1beta1/logging_types.md +++ b/content/docs/configuration/crds/v1beta1/logging_types.md @@ -32,6 +32,12 @@ Override generated config. This is a *raw* configuration string for troubleshoot Default: - +### configCheck (ConfigCheck, optional) {#loggingspec-configcheck} + +ConfigCheck settings that apply to both fluentd and syslog-ng + +Default: - + ### fluentbit (*FluentbitSpec, optional) {#loggingspec-fluentbit} FluentbitAgent daemonset configuration. Deprecated, will be removed with next major version Migrate to the standalone NodeAgent resource @@ -84,7 +90,7 @@ Default: - Cluster domain name to be used when templating URLs to services . -Default: "cluster.local" +Default: "cluster.local." ### controlNamespace (string, required) {#loggingspec-controlnamespace} @@ -111,6 +117,27 @@ EnableRecreateWorkloadOnImmutableFieldChange enables the operator to recreate th Default: - +## ConfigCheck + +### strategy (ConfigCheckStrategy, optional) {#configcheck-strategy} + +Select the config check strategy to use. `DryRun`: parse and validate configuration `StartWithTimeout`: start with given configuration and exit after specified timeout. + +Default: `DryRun` + +### timeoutSeconds (int, optional) {#configcheck-timeoutseconds} + +Configure timeout in seconds if strategy is StartWithTimeout + +Default: - + +### labels (map[string]string, optional) {#configcheck-labels} + +Labels to use for the configcheck pods on top of labels added by the operator by default. Default values can be overwritten. + +Default: - + + ## LoggingStatus LoggingStatus defines the observed state of Logging diff --git a/content/docs/configuration/crds/v1beta1/loggingroute_types.md b/content/docs/configuration/crds/v1beta1/loggingroute_types.md new file mode 100644 index 000000000..a2d865c01 --- /dev/null +++ b/content/docs/configuration/crds/v1beta1/loggingroute_types.md @@ -0,0 +1,106 @@ +--- +title: LoggingRouteSpec +weight: 200 +generated_file: true +--- + +## LoggingRouteSpec + +LoggingRouteSpec defines the desired state of LoggingRoute + +### source (string, required) {#loggingroutespec-source} + +Source identifies the logging that this policy applies to + +Default: - + +### targets (metav1.LabelSelector, required) {#loggingroutespec-targets} + +Targets refers to the list of logging resources specified by a label selector to forward logs to. Filtering of namespaces will happen based on the watchNamespaces and watchNamespaceSelector fields of the target logging resource + +Default: - + + +## LoggingRouteStatus + +LoggingRouteStatus defines the actual state of the LoggingRoute + +### tenants ([]Tenant, optional) {#loggingroutestatus-tenants} + +Enumerate all loggings with all the destination namespaces expanded + +Default: - + +### problems ([]string, optional) {#loggingroutestatus-problems} + +Enumerate problems that prohibits this route to take effect and populate the tenants field + +Default: - + +### problemsCount (int, optional) {#loggingroutestatus-problemscount} + +Summarize the number of problems for the CLI output + +Default: - + +### notices ([]string, optional) {#loggingroutestatus-notices} + +Enumerate non-blocker issues the user should pay attention to + +Default: - + +### noticesCount (int, optional) {#loggingroutestatus-noticescount} + +Summarize the number of notices for the CLI output + +Default: - + + +## Tenant + +### name (string, required) {#tenant-name} + +Default: - + +### namespaces ([]string, optional) {#tenant-namespaces} + +Default: - + + +## LoggingRoute + +LoggingRoute (experimental) +Connects a log collector with log aggregators from other logging domains and routes relevant logs based on watch namespaces + +### (metav1.TypeMeta, required) {#loggingroute-} + +Default: - + +### metadata (metav1.ObjectMeta, optional) {#loggingroute-metadata} + +Default: - + +### spec (LoggingRouteSpec, optional) {#loggingroute-spec} + +Default: - + +### status (LoggingRouteStatus, optional) {#loggingroute-status} + +Default: - + + +## LoggingRouteList + +### (metav1.TypeMeta, required) {#loggingroutelist-} + +Default: - + +### metadata (metav1.ListMeta, optional) {#loggingroutelist-metadata} + +Default: - + +### items ([]LoggingRoute, required) {#loggingroutelist-items} + +Default: - + + diff --git a/content/docs/configuration/crds/v1beta1/syslogng_output_types.md b/content/docs/configuration/crds/v1beta1/syslogng_output_types.md index 6f4a1879a..557709ef5 100644 --- a/content/docs/configuration/crds/v1beta1/syslogng_output_types.md +++ b/content/docs/configuration/crds/v1beta1/syslogng_output_types.md @@ -28,6 +28,14 @@ Default: - Default: - +### redis (*output.RedisOutput, optional) {#syslogngoutputspec-redis} + +Default: - + +### mongodb (*output.MongoDB, optional) {#syslogngoutputspec-mongodb} + +Default: - + ### sumologic-http (*output.SumologicHTTPOutput, optional) {#syslogngoutputspec-sumologic-http} Default: - @@ -40,10 +48,22 @@ Default: - Default: - +### elasticsearch (*output.ElasticsearchOutput, optional) {#syslogngoutputspec-elasticsearch} + +Default: - + ### logscale (*output.LogScaleOutput, optional) {#syslogngoutputspec-logscale} Default: - +### splunk_hec_event (*output.SplunkHECOutput, optional) {#syslogngoutputspec-splunk_hec_event} + +Default: - + +### loki (*output.LokiOutput, optional) {#syslogngoutputspec-loki} + +Default: - + ## SyslogNGOutput diff --git a/content/docs/configuration/crds/v1beta1/syslogng_types.md b/content/docs/configuration/crds/v1beta1/syslogng_types.md index dc7a8d08a..18a4b959b 100644 --- a/content/docs/configuration/crds/v1beta1/syslogng_types.md +++ b/content/docs/configuration/crds/v1beta1/syslogng_types.md @@ -106,7 +106,13 @@ Default: - ### stats (*Stats, optional) {#globaloptions-stats} -TODO switch to this by default +See the [AxoSyslog Core documentation](https://axoflow.com/docs/axosyslog-core/chapter-global-options/reference-options/#global-option-stats) + +Default: - + +### log_level (*string, optional) {#globaloptions-log_level} + +See the [AxoSyslog Core documentation](https://axoflow.com/docs/axosyslog-core/chapter-global-options/reference-options/#global-options-log-level) Default: - diff --git a/content/docs/configuration/plugins/outputs/datadog.md b/content/docs/configuration/plugins/outputs/datadog.md index adc5a0272..252cf7309 100644 --- a/content/docs/configuration/plugins/outputs/datadog.md +++ b/content/docs/configuration/plugins/outputs/datadog.md @@ -13,7 +13,8 @@ More info at [https://github.com/DataDog/fluent-plugin-datadog](https://github.c ```yaml spec: datadog: - api_key '' + api_key: + value: '' # for referencing a secret see https://kube-logging.dev/docs/configuration/plugins/outputs/secret/ dd_source: '' dd_tags: ',' dd_sourcecategory: '' diff --git a/content/docs/configuration/plugins/outputs/elasticsearch.md b/content/docs/configuration/plugins/outputs/elasticsearch.md index 45f9d724f..bf4287188 100644 --- a/content/docs/configuration/plugins/outputs/elasticsearch.md +++ b/content/docs/configuration/plugins/outputs/elasticsearch.md @@ -573,4 +573,8 @@ Specify whether overwriting data stream ilm policy or not. Default: - +### use_legacy_template (*bool, optional) {#elasticsearch-use_legacy_template} +If set to true, the output uses the [legacy index template format](https://www.elastic.co/guide/en/elasticsearch/reference/7.13/indices-templates-v1.html). Otherwise, it uses the [composable index template](https://www.elastic.co/guide/en/elasticsearch/reference/7.13/index-templates.html) format. + +Default: true diff --git a/content/docs/configuration/plugins/outputs/kinesis_firehose.md b/content/docs/configuration/plugins/outputs/kinesis_firehose.md index 07287b1c4..2cfc9b78a 100644 --- a/content/docs/configuration/plugins/outputs/kinesis_firehose.md +++ b/content/docs/configuration/plugins/outputs/kinesis_firehose.md @@ -19,101 +19,101 @@ generated_file: true ``` ## Configuration -## KinesisStream +## KinesisFirehose -Send your logs to a Kinesis Stream +Send your logs to a Kinesis Firehose -### delivery_stream_name (string, required) {#kinesisstream-delivery_stream_name} +### delivery_stream_name (string, required) {#kinesisfirehose-delivery_stream_name} Name of the delivery stream to put data. Default: - -### append_new_line (*bool, optional) {#kinesisstream-append_new_line} +### append_new_line (*bool, optional) {#kinesisfirehose-append_new_line} If it is enabled, the plugin adds new line character (\n) to each serialized record. Before appending \n, plugin calls chomp and removes separator from the end of each record as chomp_record is true. Therefore, you don't need to enable chomp_record option when you use kinesis_firehose output with default configuration (append_new_line is true). If you want to set append_new_line false, you can choose chomp_record false (default) or true (compatible format with plugin v2). (Default:true) Default: - -### aws_key_id (*secret.Secret, optional) {#kinesisstream-aws_key_id} +### aws_key_id (*secret.Secret, optional) {#kinesisfirehose-aws_key_id} AWS access key id. This parameter is required when your agent is not running on EC2 instance with an IAM Role. Default: - -### aws_sec_key (*secret.Secret, optional) {#kinesisstream-aws_sec_key} +### aws_sec_key (*secret.Secret, optional) {#kinesisfirehose-aws_sec_key} AWS secret key. This parameter is required when your agent is not running on EC2 instance with an IAM Role. Default: - -### aws_ses_token (*secret.Secret, optional) {#kinesisstream-aws_ses_token} +### aws_ses_token (*secret.Secret, optional) {#kinesisfirehose-aws_ses_token} AWS session token. This parameter is optional, but can be provided if using MFA or temporary credentials when your agent is not running on EC2 instance with an IAM Role. Default: - -### aws_iam_retries (int, optional) {#kinesisstream-aws_iam_retries} +### aws_iam_retries (int, optional) {#kinesisfirehose-aws_iam_retries} The number of attempts to make (with exponential backoff) when loading instance profile credentials from the EC2 metadata service using an IAM role. Defaults to 5 retries. Default: - -### assume_role_credentials (*KinesisFirehoseAssumeRoleCredentials, optional) {#kinesisstream-assume_role_credentials} +### assume_role_credentials (*KinesisFirehoseAssumeRoleCredentials, optional) {#kinesisfirehose-assume_role_credentials} Typically, you can use AssumeRole for cross-account access or federation. Default: - -### process_credentials (*KinesisFirehoseProcessCredentials, optional) {#kinesisstream-process_credentials} +### process_credentials (*KinesisFirehoseProcessCredentials, optional) {#kinesisfirehose-process_credentials} This loads AWS access credentials from an external process. Default: - -### region (string, optional) {#kinesisstream-region} +### region (string, optional) {#kinesisfirehose-region} AWS region of your stream. It should be in form like us-east-1, us-west-2. Default nil, which means try to find from environment variable AWS_REGION. Default: - -### retries_on_batch_request (int, optional) {#kinesisstream-retries_on_batch_request} +### retries_on_batch_request (int, optional) {#kinesisfirehose-retries_on_batch_request} The plugin will put multiple records to Amazon Kinesis Data Streams in batches using PutRecords. A set of records in a batch may fail for reasons documented in the Kinesis Service API Reference for PutRecords. Failed records will be retried retries_on_batch_request times Default: - -### reset_backoff_if_success (bool, optional) {#kinesisstream-reset_backoff_if_success} +### reset_backoff_if_success (bool, optional) {#kinesisfirehose-reset_backoff_if_success} Boolean, default true. If enabled, when after retrying, the next retrying checks the number of succeeded records on the former batch request and reset exponential backoff if there is any success. Because batch request could be composed by requests across shards, simple exponential backoff for the batch request wouldn't work some cases. Default: - -### batch_request_max_count (int, optional) {#kinesisstream-batch_request_max_count} +### batch_request_max_count (int, optional) {#kinesisfirehose-batch_request_max_count} Integer, default 500. The number of max count of making batch request from record chunk. It can't exceed the default value because it's API limit. Default: - -### batch_request_max_size (int, optional) {#kinesisstream-batch_request_max_size} +### batch_request_max_size (int, optional) {#kinesisfirehose-batch_request_max_size} Integer. The number of max size of making batch request from record chunk. It can't exceed the default value because it's API limit. Default: - -### format (*Format, optional) {#kinesisstream-format} +### format (*Format, optional) {#kinesisfirehose-format} [Format](../format/) Default: - -### buffer (*Buffer, optional) {#kinesisstream-buffer} +### buffer (*Buffer, optional) {#kinesisfirehose-buffer} [Buffer](../buffer/) Default: - -### slow_flush_log_threshold (string, optional) {#kinesisstream-slow_flush_log_threshold} +### slow_flush_log_threshold (string, optional) {#kinesisfirehose-slow_flush_log_threshold} The threshold for chunk flush performance check. Parameter type is float, not time, default: 20.0 (seconds) If chunk flush takes longer time than this threshold, fluentd logs warning message and increases metric fluentd_output_status_slow_flush_count. @@ -124,31 +124,31 @@ Default: - assume_role_credentials -### role_arn (string, required) {#assume role-credentials-role_arn} +### role_arn (string, required) {#assume role credentials-role_arn} The Amazon Resource Name (ARN) of the role to assume Default: - -### role_session_name (string, required) {#assume role-credentials-role_session_name} +### role_session_name (string, required) {#assume-role-credentials-role_session_name} An identifier for the assumed role session Default: - -### policy (string, optional) {#assume role-credentials-policy} +### policy (string, optional) {#assume-role-credentials-policy} An IAM policy in JSON format Default: - -### duration_seconds (string, optional) {#assume role-credentials-duration_seconds} +### duration_seconds (string, optional) {#assume-role-credentials-duration_seconds} The duration, in seconds, of the role session (900-3600) Default: - -### external_id (string, optional) {#assume role-credentials-external_id} +### external_id (string, optional) {#assume-role-credentials-external_id} A unique identifier that is used by third parties when assuming roles in their customers' accounts. diff --git a/content/docs/configuration/plugins/outputs/loki.md b/content/docs/configuration/plugins/outputs/loki.md index 38ed78500..937b84fe6 100644 --- a/content/docs/configuration/plugins/outputs/loki.md +++ b/content/docs/configuration/plugins/outputs/loki.md @@ -115,6 +115,12 @@ Configure Kubernetes metadata in a Prometheus like format Default: false +### include_thread_label (*bool, optional) {#output-config-include_thread_label} + +whether to include the fluentd_thread label when multiple threads are used for flushing. + +Default: true + ### buffer (*Buffer, optional) {#output-config-buffer} [Buffer](../buffer/) diff --git a/content/docs/configuration/plugins/outputs/opensearch.md b/content/docs/configuration/plugins/outputs/opensearch.md index db53756ed..c4ee0408e 100644 --- a/content/docs/configuration/plugins/outputs/opensearch.md +++ b/content/docs/configuration/plugins/outputs/opensearch.md @@ -505,7 +505,7 @@ Default: - ### use_legacy_template (*bool, optional) {#opensearch-use_legacy_template} -use_legacy_template (default: true) +Specify wether to use legacy template or not. Default: true diff --git a/content/docs/configuration/plugins/syslog-ng-filters/match.md b/content/docs/configuration/plugins/syslog-ng-filters/match.md index 92004a7f4..854d95d7d 100644 --- a/content/docs/configuration/plugins/syslog-ng-filters/match.md +++ b/content/docs/configuration/plugins/syslog-ng-filters/match.md @@ -42,9 +42,9 @@ Default: - Default: - -## [Regexp Directive](https://axoflow.com/docs/axosyslog-core/chapter-manipulating-messages/customizing-message-format/reference-template-functions/#template-function-list) {#Regexp-Directive} +## Regexp Directive {#Regexp-Directive} -Specify filtering rule. +Specify filtering rule. For details, see the [AxoSyslog Core documentation](https://axoflow.com/docs/axosyslog-core/chapter-manipulating-messages/customizing-message-format/reference-template-functions/#template-function-list) ### pattern (string, required) {#regexp-directive-pattern} diff --git a/content/docs/configuration/plugins/syslog-ng-filters/parser.md b/content/docs/configuration/plugins/syslog-ng-filters/parser.md index 4371c16a0..32d088a38 100644 --- a/content/docs/configuration/plugins/syslog-ng-filters/parser.md +++ b/content/docs/configuration/plugins/syslog-ng-filters/parser.md @@ -45,6 +45,9 @@ Default: - Default: - +### metrics-probe (*MetricsProbe, optional) {#parser-metrics-probe} + +Counts the messages that pass through the flow, and creates labeled stats counters based on the fields of the passing messages. For details, see the [documentation of the AxoSyslog syslog-ng distribution](https://axoflow.com/docs/axosyslog-core/chapter-parsers/metrics-probe/). ## [Regexp parser](https://axoflow.com/docs/axosyslog-core/chapter-parsers/parser-regexp/) @@ -79,3 +82,39 @@ Default: - Flags to influence the behavior of the [syslog-parser()](https://axoflow.com/docs/axosyslog-core/chapter-parsers/parser-syslog/parser-syslog-options/). For details, see the [syslog-parser() documentation of the AxoSyslog syslog-ng distribution](https://axoflow.com/docs/axosyslog-core/chapter-parsers/parser-syslog/parser-syslog-options/#flags). Default: - + +## MetricsProbe + +Counts the messages that pass through the flow, and creates labeled stats counters based on the fields of the passing messages. For details, see the [documentation of the AxoSyslog syslog-ng distribution](https://axoflow.com/docs/axosyslog-core/chapter-parsers/metrics-probe/). + +{{< highlight yaml>}}SyslogNGFlow +apiVersion: logging.banzaicloud.io/v1beta1 +kind: SyslogNGFlow +metadata: + name: flow-mertrics-probe + namespace: default +spec: + filters: + - parser: + metrics-probe: + key: "flow_events" + labels: + namespace: "${json.kubernetes.namespace_name}"{{< /highlight >}} + +### key (string, optional) {#metricsprobe-key} + +The name of the counter to create. Note that the value of this option is always prefixed with `syslogng_`, so for example `key("my-custom-key")` becomes `syslogng_my-custom-key`. + +Default: - + +### labels (ArrowMap, optional) {#metricsprobe-labels} + +The labels used to create separate counters, based on the fields of the messages processed by `metrics-probe()`. The keys of the map are the name of the label, and the values are syslog-ng templates. + +Default: - + +### level (int, optional) {#metricsprobe-level} + +Sets the stats level of the generated metrics (default 0). + +Default: 0 diff --git a/content/docs/configuration/plugins/syslog-ng-outputs/elasticsearch.md b/content/docs/configuration/plugins/syslog-ng-outputs/elasticsearch.md new file mode 100644 index 000000000..3fbf6451d --- /dev/null +++ b/content/docs/configuration/plugins/syslog-ng-outputs/elasticsearch.md @@ -0,0 +1,49 @@ +--- +title: Elasticsearch +weight: 200 +generated_file: true +--- + +## Overview + +Based on the [ElasticSearch destination of AxoSyslog core](https://axoflow.com/docs/axosyslog-core/chapter-destinations/configuring-destinations-elasticsearch-http/). + +## Configuration + +### (HTTPOutput, required) {#elasticsearchoutput-} + +Default: - + +### index (string, optional) {#elasticsearchoutput-index} + +Name of the data stream, index, or index alias to perform the action on. + +Default: - + +### type (*string, optional) {#elasticsearchoutput-type} + +The document type associated with the operation. Elasticsearch indices now support a single document type: `_doc` + +### custom_id (string, optional) {#elasticsearchoutput-custom_id} + +The document ID. If no ID is specified, a document ID is automatically generated. + +Default: - + +### logstash_prefix (string, optional) {#elasticsearchoutput-logstash_prefix} + +Set the prefix for logs in logstash format. If set, then Index field will be ignored. + +Default: - + +### logstash_prefix_separator (string, optional) {#elasticsearchoutput-logstash_prefix_separator} + +Set the separator between LogstashPrefix and LogStashDateformat. + +Default: `-` + +### logstash_suffix (string, optional) {#elasticsearchoutput-logstash_suffix} + +Set the suffix for logs in logstash format. + +Default: `${YEAR}.${MONTH}.${DAY}` diff --git a/content/docs/configuration/plugins/syslog-ng-outputs/file.md b/content/docs/configuration/plugins/syslog-ng-outputs/file.md index 75cd4934b..c3f49dabd 100644 --- a/content/docs/configuration/plugins/syslog-ng-outputs/file.md +++ b/content/docs/configuration/plugins/syslog-ng-outputs/file.md @@ -15,6 +15,8 @@ The `file` output stores log records in a plain text file. For details on the available options of the output, see the [documentation of the AxoSyslog syslog-ng distribution](https://axoflow.com/docs/axosyslog-core/chapter-destinations/configuring-destinations-file/). +## Configuration + ### path (string, required) {#fileoutput-path} Store file path diff --git a/content/docs/configuration/plugins/syslog-ng-outputs/http.md b/content/docs/configuration/plugins/syslog-ng-outputs/http.md index 1067ae32e..933165eec 100644 --- a/content/docs/configuration/plugins/syslog-ng-outputs/http.md +++ b/content/docs/configuration/plugins/syslog-ng-outputs/http.md @@ -4,13 +4,11 @@ weight: 200 generated_file: true --- -# Sending messages over HTTP ## Overview For details on the available options of the output, see the [documentation of the AxoSyslog syslog-ng distribution](https://axoflow.com/docs/axosyslog-core/chapter-destinations/configuring-destinations-http-nonjava/). ## Configuration -## HTTPOutput ### url (string, optional) {#httpoutput-url} @@ -50,7 +48,7 @@ Default: - ### body (string, optional) {#httpoutput-body} -The body of the HTTP request, for example, body("${ISODATE} ${MESSAGE}"). You can use strings, macros, and template functions in the body. If not set, it will contain the message received from the source by default. +The body of the HTTP request, for example, `body("${ISODATE} ${MESSAGE}")`. You can use strings, macros, and template functions in the body. If not set, it will contain the message received from the source by default. Default: - @@ -74,7 +72,7 @@ Default: - ### method (string, optional) {#httpoutput-method} -Specifies the HTTP method to use when sending the message to the server. POST | PUT +Specifies the HTTP method to use when sending the message to the server. `POST | PUT` Default: - @@ -104,12 +102,32 @@ Default: - ### workers (int, optional) {#httpoutput-workers} -Description: Specifies the number of worker threads (at least 1) that syslog-ng OSE uses to send messages to the server. Increasing the number of worker threads can drastically improve the performance of the destination. +Specifies the number of worker threads (at least 1) that syslog-ng OSE uses to send messages to the server. Increasing the number of worker threads can drastically improve the performance of the destination. Default: - ### persist_name (string, optional) {#httpoutput-persist_name} +If you receive the following error message during AxoSyslog startup, set the `persist-name()` option of the duplicate drivers: `Error checking the uniqueness of the persist names, please override it with persist-name option. Shutting down.` See the [AxoSyslog Core documentation](https://axoflow.com/docs/axosyslog-core/chapter-destinations/configuring-destinations-http-nonjava/reference-destination-http-nonjava/#persist-name) for more information. + +Default: - + +### log-fifo-size (int, optional) {#httpoutput-log-fifo-size} + +The number of messages that the output queue can store. + +Default: - + +### timeout (int, optional) {#httpoutput-timeout} + +Sets the maximum number of messages sent to the destination per second. Use this output-rate-limiting functionality only when using disk-buffer as well to avoid the risk of losing messages. Specifying 0 or a lower value sets the output limit to unlimited. + +Default: - + +### response-action (filter.RawArrowMap, optional) {#httpoutput-response-action} + +Specifies what AxoSyslog does with the log message, based on the response code received from the HTTP server. See the [AxoSyslog Core documentation](https://axoflow.com/docs/axosyslog-core/chapter-destinations/configuring-destinations-http-nonjava/reference-destination-http-nonjava/#response-action) for more information. + Default: - @@ -117,19 +135,19 @@ Default: - ### batch-lines (int, optional) {#batch-batch-lines} -Description: Specifies how many lines are flushed to a destination in one batch. The syslog-ng OSE application waits for this number of lines to accumulate and sends them off in a single batch. Increasing this number increases throughput as more messages are sent in a single batch, but also increases message latency. For example, if you set batch-lines() to 100, syslog-ng OSE waits for 100 messages. +Description: Specifies how many lines are flushed to a destination in one batch. The syslog-ng OSE application waits for this number of lines to accumulate and sends them off in a single batch. Increasing this number increases throughput as more messages are sent in a single batch, but also increases message latency. For example, if you set `batch-lines()` to 100, syslog-ng OSE waits for 100 messages. Default: - ### batch-bytes (int, optional) {#batch-batch-bytes} -Description: Sets the maximum size of payload in a batch. If the size of the messages reaches this value, syslog-ng OSE sends the batch to the destination even if the number of messages is less than the value of the batch-lines() option. Note that if the batch-timeout() option is enabled and the queue becomes empty, syslog-ng OSE flushes the messages only if batch-timeout() expires, or the batch reaches the limit set in batch-bytes(). +Description: Sets the maximum size of payload in a batch. If the size of the messages reaches this value, syslog-ng OSE sends the batch to the destination even if the number of messages is less than the value of the `batch-lines()` option. Note that if the `batch-timeout()` option is enabled and the queue becomes empty, syslog-ng OSE flushes the messages only if `batch-timeout()` expires, or the batch reaches the limit set in `batch-bytes()`. Default: - ### batch-timeout (int, optional) {#batch-batch-timeout} -Description: Specifies the time syslog-ng OSE waits for lines to accumulate in the output buffer. The syslog-ng OSE application sends batches to the destinations evenly. The timer starts when the first message arrives to the buffer, so if only few messages arrive, syslog-ng OSE sends messages to the destination at most once every batch-timeout() milliseconds. +Description: Specifies the time syslog-ng OSE waits for lines to accumulate in the output buffer. The syslog-ng OSE application sends batches to the destinations evenly. The timer starts when the first message arrives to the buffer, so if only few messages arrive, syslog-ng OSE sends messages to the destination at most once every `batch-timeout()` milliseconds. Default: - diff --git a/content/docs/configuration/plugins/syslog-ng-outputs/logscale.md b/content/docs/configuration/plugins/syslog-ng-outputs/logscale.md index 0edc24082..6ff1fdf34 100644 --- a/content/docs/configuration/plugins/syslog-ng-outputs/logscale.md +++ b/content/docs/configuration/plugins/syslog-ng-outputs/logscale.md @@ -4,11 +4,9 @@ weight: 200 generated_file: true --- -# logscale -## Overview +Based on the [LogScale destination of AxoSyslog core](https://axoflow.com/docs/axosyslog-core/chapter-destinations/crowdstrike-falcon/). ## Configuration -## LogScaleOutput ### url (*secret.Secret, optional) {#logscaleoutput-url} diff --git a/content/docs/configuration/plugins/syslog-ng-outputs/loki.md b/content/docs/configuration/plugins/syslog-ng-outputs/loki.md new file mode 100644 index 000000000..152d6462b --- /dev/null +++ b/content/docs/configuration/plugins/syslog-ng-outputs/loki.md @@ -0,0 +1,101 @@ +--- +title: Loki +weight: 200 +generated_file: true +--- + +Sends messages to Grafana Loki over gRPC, based on the [Loki destination of AxoSyslog Core](https://axoflow.com/docs/axosyslog-core/chapter-destinations/syslog-ng-with-loki/). + +For example: + +{{< highlight yaml >}} +apiVersion: logging.banzaicloud.io/v1beta1 +kind: SyslogNGOutput +metadata: + name: loki +spec: + loki: + url: "loki.loki:8000" + labels: + "app": "$PROGRAM" + "host": "$HOST" + workers: 16 + batch-timeout: 10000 + batch-lines: 1000 +{{< /highlight >}} + +## Configuration + +### labels (filter.ArrowMap, optional) {#lokioutput-labels} + +Using the Labels map, Kubernetes label to Loki label mapping can be configured. Example: `{"app" : "$PROGRAM"}` + +Default: - + +### url (string, optional) {#lokioutput-url} + +Specifies the hostname or IP address and optionally the port number of the web service that can receive log data via HTTP. Use a colon (:) after the address to specify the port number of the server. For example: `http://127.0.0.1:8000` + +Default: - + +### time_reopen (int, optional) {#lokioutput-time_reopen} + +The time to wait in seconds before a dead connection is reestablished. + +Default: 60 + +### disk_buffer (*DiskBuffer, optional) {#lokioutput-disk_buffer} + +This option enables putting outgoing messages into the disk buffer of the destination to avoid message loss in case of a system failure on the destination side. For details, see the [Syslog-ng DiskBuffer options](../disk_buffer/). + +Default: false + +### batch-lines (int, optional) {#lokioutput-batch-lines} + +Description: Specifies how many lines are flushed to a destination in one batch. The syslog-ng OSE application waits for this number of lines to accumulate and sends them off in a single batch. Increasing this number increases throughput as more messages are sent in a single batch, but also increases message latency. For example, if you set batch-lines() to 100, syslog-ng OSE waits for 100 messages. + +Default: - + +### batch-timeout (int, optional) {#lokioutput-batch-timeout} + +Description: Specifies the time syslog-ng OSE waits for lines to accumulate in the output buffer. The syslog-ng OSE application sends batches to the destinations evenly. The timer starts when the first message arrives to the buffer, so if only few messages arrive, syslog-ng OSE sends messages to the destination at most once every batch-timeout() milliseconds. + +Default: - + +### retries (int, optional) {#lokioutput-retries} + +The number of times syslog-ng OSE attempts to send a message to this destination. If syslog-ng OSE could not send a message, it will try again until the number of attempts reaches retries, then drops the message. + +Default: - + +### workers (int, optional) {#lokioutput-workers} + +Specifies the number of worker threads (at least 1) that syslog-ng OSE uses to send messages to the server. Increasing the number of worker threads can drastically improve the performance of the destination. + +Default: - + +### persist_name (string, optional) {#lokioutput-persist_name} + +If you receive the following error message during AxoSyslog startup, set the persist-name() option of the duplicate drivers: `Error checking the uniqueness of the persist names, please override it with persist-name option. Shutting down.` See [syslog-ng docs](https://axoflow.com/docs/axosyslog-core/chapter-destinations/configuring-destinations-http-nonjava/reference-destination-http-nonjava/#persist-name) for more information. + +Default: - + +### log-fifo-size (int, optional) {#lokioutput-log-fifo-size} + +The number of messages that the output queue can store. + +Default: - + +### timestamp (string, optional) {#lokioutput-timestamp} + +The timestamp that will be applied to the outgoing messages (possible values: current|received|msg default: current). Loki does not accept events, in which the timestamp is not monotonically increasing. + +Default: - + +### template (string, optional) {#lokioutput-template} + +Template for customizing the log message format. + +Default: - + + diff --git a/content/docs/configuration/plugins/syslog-ng-outputs/mongodb.md b/content/docs/configuration/plugins/syslog-ng-outputs/mongodb.md new file mode 100644 index 000000000..0fdf212ff --- /dev/null +++ b/content/docs/configuration/plugins/syslog-ng-outputs/mongodb.md @@ -0,0 +1,156 @@ +--- +title: MongoDB +weight: 200 +generated_file: true +--- + +Based on the [MongoDB destination of AxoSyslog core](https://axoflow.com/docs/axosyslog-core/chapter-destinations/configuring-destinations-mongodb/). + +## Example + +{{< highlight yaml >}} +apiVersion: logging.banzaicloud.io/v1beta1 +kind: SyslogNGOutput +metadata: + name: mongodb + namespace: default +spec: + mongodb: + collection: syslog + uri: mongodb://127.0.0.1:27017/syslog?wtimeoutMS=60000&socketTimeoutMS=60000&connectTimeoutMS=60000 + value_pairs: scope("selected-macros" "nv-pairs") + {{}} + +## Configuration + +### collection (string, required) {#mongodb-collection} + +The name of the MongoDB collection where the log messages are stored (collections are similar to SQL tables). Note that the name of the collection must not start with a dollar sign ($), and that it may contain dot (.) characters. + +Default: - + +### compaction (bool, required) {#mongodb-compaction} + +If set to yes, syslog-ng OSE cannot lose logs in case of reload/restart, unreachable destination or syslog-ng OSE crash. This solution provides a slower, but reliable disk-buffer option. + +Default: - + +### dir (string, optional) {#mongodb-dir} + +Defines the folder where the disk-buffer files are stored. + +Default: - + +### disk_buffer (*DiskBuffer, optional) {#mongodb-disk_buffer} + +This option enables putting outgoing messages into the disk buffer of the destination to avoid message loss in case of a system failure on the destination side. For details, see the [Syslog-ng DiskBuffer options](../disk_buffer/). + +Default: false + +### uri (string, optional) {#mongodb-uri} + +Defines the folder where the disk-buffer files are stored. + +Default: "mongodb://127.0.0.1:27017/syslog?wtimeoutMS=60000&socketTimeoutMS=60000&connectTimeoutMS=60000" + +### value_pairs (ValuePairs, optional) {#mongodb-value_pairs} + +Creates structured name-value pairs from the data and metadata of the log message. + +Default: "scope("selected-macros" "nv-pairs")" + +### (Batch, required) {#mongodb-} + +Batching parameters + +Default: - + +### (Bulk, required) {#mongodb-} + +Bulk operation related options + +Default: - + +### log-fifo-size (int, optional) {#mongodb-log-fifo-size} + +The number of messages that the output queue can store. + +Default: - + +### persist_name (string, optional) {#mongodb-persist_name} + +If you receive the following error message during AxoSyslog startup, set the persist-name() option of the duplicate drivers: `Error checking the uniqueness of the persist names, please override it with persist-name option. Shutting down.` See [syslog-ng docs](https://axoflow.com/docs/axosyslog-core/chapter-destinations/configuring-destinations-http-nonjava/reference-destination-http-nonjava/#persist-name) for more information. + +Default: - + +### retries (int, optional) {#mongodb-retries} + +The number of times syslog-ng OSE attempts to send a message to this destination. If syslog-ng OSE could not send a message, it will try again until the number of attempts reaches retries, then drops the message. + +Default: - + +### time_reopen (int, optional) {#mongodb-time_reopen} + +The time to wait in seconds before a dead connection is reestablished. + +Default: 60 + +### write_concern (RawString, optional) {#mongodb-write_concern} + +Description: Sets the write concern mode of the MongoDB operations, for both bulk and single mode. See the [AxoSyslog Core documentation](https://axoflow.com/docs/axosyslog-core/chapter-destinations/configuring-destinations-mongodb/reference-destination-mongodb/#mongodb-option-write-concern) + +Default: - + + +## Bulk + +Bulk operation related options +See [AxoSyslog Core documentation](https://axoflow.com/docs/axosyslog-core/chapter-destinations/configuring-destinations-mongodb/reference-destination-mongodb/#mongodb-option-bulk) + +### bulk (*bool, optional) {#bulk-bulk} + +Enables bulk insert mode. If disabled, each messages is inserted individually. + +Default: yes + +### bulk_bypass_validation (*bool, optional) {#bulk-bulk_bypass_validation} + +If set to yes, it disables MongoDB bulk operations validation mode. + +Default: no + +### bulk_unordered (*bool, optional) {#bulk-bulk_unordered} + +Description: Enables unordered bulk operations mode. + +Default: no + + +## ValuePairs + +TODO move this to a common module once it is used in more places + +### scope (RawString, optional) {#valuepairs-scope} + +Default: - + +### exclude (RawString, optional) {#valuepairs-exclude} + +Default: - + +### key (RawString, optional) {#valuepairs-key} + +Default: - + +### pair (RawString, optional) {#valuepairs-pair} + +Default: - + + +## RawString + +### raw_string (string, optional) {#rawstring-raw_string} + +Default: - + + diff --git a/content/docs/configuration/plugins/syslog-ng-outputs/redis.md b/content/docs/configuration/plugins/syslog-ng-outputs/redis.md new file mode 100644 index 000000000..2d8212b12 --- /dev/null +++ b/content/docs/configuration/plugins/syslog-ng-outputs/redis.md @@ -0,0 +1,114 @@ +--- +title: Redis +weight: 200 +generated_file: true +--- + +Based on the [Redis destination of AxoSyslog core](https://axoflow.com/docs/axosyslog-core/chapter-destinations/configuring-destinations-redis/). + +## Example + +{{< highlight yaml >}} +apiVersion: logging.banzaicloud.io/v1beta1 +kind: SyslogNGOutput +metadata: + name: redis + namespace: default +spec: + redis: + host: 127.0.0.1 + port: 6379 + retries: 3 + throttle: 0 + time-reopen: 60 + workers: 1 + {{}} + +## Configuration + +### host (string, optional) {#redisoutput-host} + +The hostname or IP address of the Redis server. + +Default: 127.0.0.1 + +### auth (*secret.Secret, optional) {#redisoutput-auth} + +The password used for authentication on a password-protected Redis server. + +Default: - + +### port (int, optional) {#redisoutput-port} + +The port number of the Redis server. + +Default: 6379 + +### retries (int, optional) {#redisoutput-retries} + +If syslog-ng OSE cannot send a message, it will try again until the number of attempts reaches retries(). + +Default: 3 + +### throttle (int, optional) {#redisoutput-throttle} + +Sets the maximum number of messages sent to the destination per second. Use this output-rate-limiting functionality only when using disk-buffer as well to avoid the risk of losing messages. Specifying 0 or a lower value sets the output limit to unlimited. + +Default: 0 + +### time-reopen (int, optional) {#redisoutput-time-reopen} + +The time to wait in seconds before a dead connection is reestablished. + +Default: 60 + +### workers (int, optional) {#redisoutput-workers} + +Specifies the number of worker threads (at least 1) that syslog-ng OSE uses to send messages to the server. Increasing the number of worker threads can drastically improve the performance of the destination. + +Default: 1 + +### command_and_arguments ([]string, optional) {#redisoutput-command_and_arguments} + +The Redis command to execute, for example, LPUSH, INCR, or HINCRBY. Using the HINCRBY command with an increment value of 1 allows you to create various statistics. For example, the command("HINCRBY" "${HOST}/programs" "${PROGRAM}" "1") command counts the number of log messages on each host for each program. + +Default: "" + +### command (StringList, optional) {#redisoutput-command} + +Internal rendered form of the CommandAndArguments field + +Default: - + +### (Batch, required) {#redisoutput-} + +Batching parameters + +Default: - + +### log-fifo-size (int, optional) {#redisoutput-log-fifo-size} + +The number of messages that the output queue can store. + +Default: - + +### disk_buffer (*DiskBuffer, optional) {#redisoutput-disk_buffer} + +This option enables putting outgoing messages into the disk buffer of the destination to avoid message loss in case of a system failure on the destination side. For details, see the [syslog-ng DiskBuffer options](../disk_buffer/). + +Default: false + +### persist_name (string, optional) {#redisoutput-persist_name} + +Persistname + +Default: - + + +## StringList + +### string-list ([]string, optional) {#stringlist-string-list} + +Default: - + + diff --git a/content/docs/configuration/plugins/syslog-ng-outputs/s3.md b/content/docs/configuration/plugins/syslog-ng-outputs/s3.md new file mode 100644 index 000000000..a310e7f73 --- /dev/null +++ b/content/docs/configuration/plugins/syslog-ng-outputs/s3.md @@ -0,0 +1,159 @@ +--- +title: Sending messages from a local network to a S3 (compatible) server +linktitle: S3 +weight: 200 +generated_file: true +--- + +## Example +{{< highlight yaml >}} +apiVersion: logging.banzaicloud.io/v1beta1 +kind: SyslogNGOutput +metadata: +name: s3 +spec: +s3: + url: "https://some-s3-compatible-endpoint:8088" + bucket: "s3bucket-name" + access_key: + valueFrom: + secretKeyRef: + name: s3 + key: access-key + secret_key: + valueFrom: + secretKeyRef: + name: s3 + key: secret-key + object_key: "${HOST}/my-logs" +{{}} + +## S3Output + +### url (string, optional) {#s3output-url} + +The hostname or IP address of the S3 server. + +Default: - + +### bucket (string, optional) {#s3output-bucket} + +The bucket name of the S3 server. + +Default: - + +### access_key (*secret.Secret, optional) {#s3output-access_key} + +The access_key for the S3 server. + +Default: - + +### secret_key (*secret.Secret, optional) {#s3output-secret_key} + +The secret_key for the S3 server. + +Default: - + +### object_key (string, optional) {#s3output-object_key} + +The object_key for the S3 server. + +Default: - + +### object_key_timestamp (RawString, optional) {#s3output-object_key_timestamp} + +Set object_key_timestamp + +Default: - + +### template (RawString, optional) {#s3output-template} + +Template + +Default: - + +### compression (*bool, optional) {#s3output-compression} + +Enable or disable compression. + +Default: false + +### compresslevel (int, optional) {#s3output-compresslevel} + +Set the compression level (1-9). + +Default: 9 + +### chunk_size (int, optional) {#s3output-chunk_size} + +Set the chunk size. + +Default: 5MiB + +### max_object_size (int, optional) {#s3output-max_object_size} + +Set the maximum object size size. + +Default: 5120GiB + +### upload_threads (int, optional) {#s3output-upload_threads} + +Set the number of upload threads. + +Default: 8 + +### max_pending_uploads (int, optional) {#s3output-max_pending_uploads} + +Set the maximum number of pending uploads. + +Default: 32 + +### flush_grace_period (int, optional) {#s3output-flush_grace_period} + +Set the number of seconds for flush period. + +Default: 60 + +### region (string, optional) {#s3output-region} + +Set the region option. + +Default: - + +### storage_class (string, optional) {#s3output-storage_class} + +Set the storage_class option. + +Default: - + +### canned_acl (string, optional) {#s3output-canned_acl} + +Set the canned_acl option. + +Default: - + +### log-fifo-size (int, optional) {#s3output-log-fifo-size} + +The number of messages that the output queue can store. + +Default: - + +### persist_name (string, optional) {#s3output-persist_name} + +Persistname + +Default: - + +### retries (int, optional) {#s3output-retries} + +The number of times syslog-ng OSE attempts to send a message to this destination. If syslog-ng OSE could not send a message, it will try again until the number of attempts reaches retries, then drops the message. + +Default: - + +### throttle (int, optional) {#s3output-throttle} + +Sets the maximum number of messages sent to the destination per second. Use this output-rate-limiting functionality only when using disk-buffer as well to avoid the risk of losing messages. Specifying 0 or a lower value sets the output limit to unlimited. + +Default: 0 + + diff --git a/content/docs/configuration/plugins/syslog-ng-outputs/splunk_hec.md b/content/docs/configuration/plugins/syslog-ng-outputs/splunk_hec.md new file mode 100644 index 000000000..57d678005 --- /dev/null +++ b/content/docs/configuration/plugins/syslog-ng-outputs/splunk_hec.md @@ -0,0 +1,99 @@ +--- +title: SplunkHEC +weight: 200 +generated_file: true +--- + +Based on the [Splunk destination of AxoSyslog core](https://axoflow.com/docs/axosyslog-core/chapter-destinations/syslog-ng-with-splunk/). + +## Configuration + +### (HTTPOutput, required) {#splunkhecoutput-} + +Default: - + +### token (secret.Secret, optional) {#splunkhecoutput-token} + +The token that syslog-ng OSE uses to authenticate on the event collector. + +Default: - + +### event (string, optional) {#splunkhecoutput-event} + +event() accepts a template, which declares the content of the log message sent to Splunk. Default value: ${MSG} + +Default: - + +### index (string, optional) {#splunkhecoutput-index} + +Splunk index where the messages will be stored. + +Default: - + +### source (string, optional) {#splunkhecoutput-source} + +Sets the source field. + +Default: - + +### sourcetype (string, optional) {#splunkhecoutput-sourcetype} + +Sets the sourcetype field. + +Default: - + +### host (string, optional) {#splunkhecoutput-host} + +Sets the host field. + +Default: - + +### time (string, optional) {#splunkhecoutput-time} + +Sets the time field. + +Default: - + +### default_index (string, optional) {#splunkhecoutput-default_index} + +Fallback option for index field. See [syslog-ng docs](https://axoflow.com/docs/axosyslog-core/chapter-destinations/syslog-ng-with-splunk/) + +Default: - + +### default_source (string, optional) {#splunkhecoutput-default_source} + +Fallback option for source field. + +Default: - + +### default_sourcetype (string, optional) {#splunkhecoutput-default_sourcetype} + +Fallback option for sourcetype field. + +Default: - + +### fields (string, optional) {#splunkhecoutput-fields} + +Additional indexing metadata for Splunk. + +Default: - + +### extra_headers ([]string, optional) {#splunkhecoutput-extra_headers} + +Additional HTTP request headers. + +Default: - + +### extra_queries ([]string, optional) {#splunkhecoutput-extra_queries} + +Additional HTTP request query options. + +Default: - + +### content_type (string, optional) {#splunkhecoutput-content_type} + +Additional HTTP request content-type option. + +Default: - + + diff --git a/content/docs/configuration/plugins/syslog-ng-outputs/tls.md b/content/docs/configuration/plugins/syslog-ng-outputs/tls.md index 525d4e4a8..0fa62e7c5 100644 --- a/content/docs/configuration/plugins/syslog-ng-outputs/tls.md +++ b/content/docs/configuration/plugins/syslog-ng-outputs/tls.md @@ -4,46 +4,43 @@ weight: 200 generated_file: true --- -# TLS config for syslog-ng outputs -## Overview - More info at https://www.syslog-ng.com/technical-documents/doc/syslog-ng-open-source-edition/3.37/administration-guide/32#kanchor2338 +For details on how TLS configuration works in syslog-ng, see the [AxoSyslog Core documentation](https://axoflow.com/docs/axosyslog-core/chapter-encrypted-transport-tls/tlsoptions/). ## Configuration -## TLS ### ca_dir (*secret.Secret, optional) {#tls-ca_dir} -The name of a directory that contains a set of trusted CA certificates in PEM format. [more information](https://www.syslog-ng.com/technical-documents/doc/syslog-ng-open-source-edition/3.37/administration-guide/73#kanchor3142) +The name of a directory that contains a set of trusted CA certificates in PEM format. For details, see the [AxoSyslog Core documentation](https://axoflow.com/docs/axosyslog-core/chapter-encrypted-transport-tls/tlsoptions/#ca-dir) Default: - ### ca_file (*secret.Secret, optional) {#tls-ca_file} -The name of a file that contains a set of trusted CA certificates in PEM format. (Optional) [more information](https://www.syslog-ng.com/technical-documents/doc/syslog-ng-open-source-edition/3.37/administration-guide/73#kanchor3144) +The name of a file that contains a set of trusted CA certificates in PEM format. (Optional) For details, see the [AxoSyslog Core documentation](https://axoflow.com/docs/axosyslog-core/chapter-encrypted-transport-tls/tlsoptions/#ca-file) Default: - ### key_file (*secret.Secret, optional) {#tls-key_file} -The name of a file that contains an unencrypted private key in PEM format, suitable as a TLS key. [more information](https://www.syslog-ng.com/technical-documents/doc/syslog-ng-open-source-edition/3.37/administration-guide/73#kanchor3163) +The name of a file that contains an unencrypted private key in PEM format, suitable as a TLS key. For details, see the [AxoSyslog Core documentation](https://axoflow.com/docs/axosyslog-core/chapter-encrypted-transport-tls/tlsoptions/#key-file) Default: - ### cert_file (*secret.Secret, optional) {#tls-cert_file} -Name of a file, that contains an X.509 certificate (or a certificate chain) in PEM format, suitable as a TLS certificate, matching the private key set in the key-file() option. [more information](https://www.syslog-ng.com/technical-documents/doc/syslog-ng-open-source-edition/3.37/administration-guide/73#kanchor3146) +Name of a file, that contains an X.509 certificate (or a certificate chain) in PEM format, suitable as a TLS certificate, matching the private key set in the key-file() option. For details, see the [AxoSyslog Core documentation](https://axoflow.com/docs/axosyslog-core/chapter-encrypted-transport-tls/tlsoptions/#cert-file) Default: - -### peer_verify (string, optional) {#tls-peer_verify} +### peer_verify (*bool, optional) {#tls-peer_verify} -Verification method of the peer. [more information](https://www.syslog-ng.com/technical-documents/doc/syslog-ng-open-source-edition/3.37/administration-guide/73#tls-options-peer-verify) +Verification method of the peer. For details, see the [AxoSyslog Core documentation](https://axoflow.com/docs/axosyslog-core/chapter-encrypted-transport-tls/tlsoptions/#tls-options-peer-verify) Default: - ### use-system-cert-store (*bool, optional) {#tls-use-system-cert-store} -Use the certificate store of the system for verifying HTTPS certificates. [more information](https://curl.se/docs/sslcerts.html) +Use the certificate store of the system for verifying HTTPS certificates. For details, see the [AxoSyslog Core documentation](https://curl.se/docs/sslcerts.html) Default: - diff --git a/content/docs/examples/es-nginx.md b/content/docs/examples/es-nginx.md index b9a8cf243..10cff82f1 100644 --- a/content/docs/examples/es-nginx.md +++ b/content/docs/examples/es-nginx.md @@ -167,9 +167,11 @@ To validate that the deployment was successful, complete the following steps. 1. Check fluentd logs: ```bash - kubectl exec -ti -n logging default-logging-simple-fluentd-0 -- tail -f /fluentd/log/out + kubectl logs -f -n logging default-logging-simple-fluentd-0 -c fluentd ``` + {{< include-headless "note-fluentd-logs.md" >}} + 1. Use the following command to retrieve the password of the `elastic` user: ```bash diff --git a/content/docs/examples/example-s3.md b/content/docs/examples/example-s3.md index a64bb5178..9c0e124f2 100644 --- a/content/docs/examples/example-s3.md +++ b/content/docs/examples/example-s3.md @@ -133,9 +133,11 @@ Install the Logging operator. Check fluentd logs (errors with AWS credentials should be visible here): ```bash -kubectl exec -ti -n logging default-logging-simple-fluentd-0 -- tail -f /fluentd/log/out +kubectl logs -f -n logging default-logging-simple-fluentd-0 -c fluentd ``` +{{< include-headless "note-fluentd-logs.md" >}} + Check the output. The logs will be available in the bucket on a `path` like: ```bash /logs/default.default-logging-simple-fluentbit-lsdp5.fluent-bit/2019/09/11/201909111432_0.gz diff --git a/content/docs/examples/multitenancy.md b/content/docs/examples/multitenancy.md new file mode 100644 index 000000000..7035b97ca --- /dev/null +++ b/content/docs/examples/multitenancy.md @@ -0,0 +1,63 @@ +--- +title: Nodegroup-based multitenancy +weight: 600 +--- + +Nodegroup-based multitenancy allows you to have multiple tenants (for example, different developer teams or customer environments) on the same cluster who can configure their own logging resources within their assigned namespaces residing on different node groups. +These resources are isolated from the resources of the other tenants so the configuration issues and performance characteristics of one tenant doesn't affect the others. + +## Sample setup + +The following procedure creates two tenants (A and B) and their respective namespaces on a two-node cluster. + +1. If you don't already have a cluster, create one with your provider. For a quick test, you can use a local cluster, for example, using minikube: + + ```bash + minikube start --nodes=2 + ``` + +1. Set labels on the nodes that correspond to your tenants, for example, `tenant-a` and `tenant-b`. + + ```bash + kubectl label node minikube tenant=tenant-a + kubectl label node minikube-m02 tenant=tenant-b + ``` + +1. Install the logging operator + + ```bash + helm install logging-operator oci://ghcr.io/kube-logging/helm-charts/logging-operator + ``` + +1. Apply the sample resources from the [project repository](https://github.com/kube-logging/logging-operator/tree/master/config/samples/mulitenant-hard/logging). These create namespaces, flows, and sample outputs for the two tenants. + +1. (Optional) Install a sample log generator application to the respective namespaces of your tenants. For example: + + ```bash + helm upgrade --install --namespace a --create-namespace --set "nodeSelector.tenant=tenant-a" log-generator oci://ghcr.io/kube-logging/helm-charts/log-generator + helm upgrade --install --namespace b --create-namespace --set "nodeSelector.tenant=tenant-b" log-generator oci://ghcr.io/kube-logging/helm-charts/log-generator + ``` + +1. Check that your pods are up and running by running `kubectl get pods -A` + + If you have followed the examples, the output should look like: + + ```bash + NAMESPACE NAME READY STATUS RESTARTS AGE + a-control a-fluentbit-4tqzg 1/1 Running 0 9m29s + a-control a-fluentd-0 2/2 Running 0 4m48s + a log-generator-6cfb45c684-q6fl6 1/1 Running 0 3m25s + b-control b-fluentbit-qmf58 1/1 Running 0 9m20s + b-control b-fluentd-0 2/2 Running 0 9m16s + b log-generator-7b95b6fdc5-cshh7 1/1 Running 0 8m49s + default logging-operator-bbd66bb7d-qvsmg 1/1 Running 0 35m + infra test-receiver-7c45f9cd77-whvlv 1/1 Running 0 53m + ``` + +1. Check logs coming from both tenants `kubectl logs -f -n infra svc/test-receiver` + + Expected output should show logs from both tenants + ```bash + [0] tenant_a: [[1695999280.157810965, {}], {"log"=>"15.238.250.48 - - [29/Sep/2023:14:54:38 +0000] "PUT /pro... + [0] tenant_b: [[1695999280.160868923, {}], {"log"=>"252.201.89.36 - - [29/Sep/2023:14:54:33 +0000] "POST /bl... + ``` diff --git a/content/docs/image-versions.md b/content/docs/image-versions.md index 86ffa6c45..96e97949a 100644 --- a/content/docs/image-versions.md +++ b/content/docs/image-versions.md @@ -5,6 +5,29 @@ weight: 750 Logging operator uses the following image versions. +## Logging operator version 4.4 + +| Image repository | GitHub repository | Version | +| -------- | --- | -- | +| ghcr.io/kube-logging/node-exporter | https://github.com/kube-logging/node-exporter-image | v0.7.1 | +| ghcr.io/kube-logging/config-reloader | https://github.com/kube-logging/config-reloader | v0.0.5 | +| ghcr.io/kube-logging/fluentd-drain-watch | https://github.com/kube-logging/fluentd-drain-watch | v0.2.1 | +| k8s.gcr.io/pause | | 3.2 | +| docker.io/busybox | https://github.com/docker-library/busybox | latest | +| ghcr.io/axoflow/axosyslog | https://github.com/axoflow/axosyslog-docker/ | 4.4.0 | +| docker.io/fluent/fluent-bit | https://github.com/fluent/fluent-bit-docker-image | 2.1.8 | +| ghcr.io/kube-logging/fluentd | https://github.com/kube-logging/fluentd-images | v1.15-ruby3 | +| ghcr.io/axoflow/axosyslog-metrics-exporter | https://github.com/axoflow/axosyslog-metrics-exporter | 0.0.2 | +| ghcr.io/kube-logging/syslogng-reload | https://github.com/kube-logging/syslogng-reload-image | v1.3.1 | +| ghcr.io/kube-logging/eventrouter | https://github.com/kube-logging/eventrouter | v0.4.0 | + +### The following images are now replaced + +| Image used in 4.3 | New image in 4.4 | +| -------- | --- | -- | +| banzaicloud/eventrouter | ghcr.io/kube-logging/eventrouter | +| ghcr.io/kube-logging/syslog-ng-exporter | ghcr.io/axoflow/axosyslog-metrics-exporter | + ## Logging operator version 4.3 | Image repository | GitHub repository | Version | diff --git a/content/docs/logging-infrastructure/fluentd.md b/content/docs/logging-infrastructure/fluentd.md index 0cd14085d..77fa5fbf7 100644 --- a/content/docs/logging-infrastructure/fluentd.md +++ b/content/docs/logging-infrastructure/fluentd.md @@ -52,10 +52,9 @@ spec: The following snippet redirects Fluentd's stdout to a file and configures rotation settings. -This is important to avoid Fluentd getting into a ripple effect when there is an error and the error message gets back to the system as a log message, which generates another error, and so on. - -Default settings configured by the operator: +This mechanism was used prior to version 4.4 to avoid Fluent-bit rereading Fluentd's logs and causing an exponentially growing amount of redundant logs. +Example configuration used by the operator in version 4.3 and earlier (keep 10 files, 10M each): ```yaml spec: fluentd: @@ -66,14 +65,7 @@ spec: size: 10485760 ``` -Disabling it and write to stdout (not recommended): - -```yaml -spec: - fluentd: - fluentOutLogrotate: - enabled: false -``` +Fluentd logs are now excluded using the `fluentbit.io/exclude: "true"` annotation. ## Scaling diff --git a/content/docs/operation/troubleshooting/fluentd.md b/content/docs/operation/troubleshooting/fluentd.md index 21fb01c27..fbb9446d0 100644 --- a/content/docs/operation/troubleshooting/fluentd.md +++ b/content/docs/operation/troubleshooting/fluentd.md @@ -103,14 +103,20 @@ Use the following command to change the log level of Fluentd. `kubectl edit loggings.logging.banzaicloud.io logging-demo` ```yaml -fluentd: - logLevel: debug +spec: + fluentd: + logLevel: debug ``` ## Get Fluentd logs The following command displays the logs of the Fluentd container. -`kubectl exec -it logging-demo-fluentd-0 cat /fluentd/log/out` + +```bash +kubectl logs -f logging-demo-fluentd-0 -c fluentd +``` + +{{< include-headless "note-fluentd-logs.md" >}} > Tip: If the logs include the `error="can't create buffer file ...` error message, Fluentd can’t create the buffer file at the specified location. This can mean for example that the disk is full, the filesystem is read-only, or some other permission error. Check the buffer-related settings of your [Fluentd configuration]({{< relref "/docs/configuration/crds/v1beta1/fluentd_types.md" >}}). diff --git a/content/docs/whats-new/_index.md b/content/docs/whats-new/_index.md new file mode 100644 index 000000000..0cd64d5b8 --- /dev/null +++ b/content/docs/whats-new/_index.md @@ -0,0 +1,115 @@ +--- +title: What's new +weight: 50 +--- + +## Version 4.4 + +The following are the highlights and main changes of Logging operator 4.4. For a complete list of changes and bugfixes, see the [Logging operator 4.4 releases page](https://github.com/kube-logging/logging-operator/releases/tag/4.4.0). + +### New syslog-ng features + +When using syslog-ng as the log aggregator, you can now use the following new outputs: + +- [ElasticSearch]({{< relref "/docs/configuration/plugins/syslog-ng-outputs/elasticsearch.md" >}}) +- [Grafana Loki]({{< relref "/docs/configuration/plugins/syslog-ng-outputs/loki.md" >}}) +- [MongoDB]({{< relref "/docs/configuration/plugins/syslog-ng-outputs/mongodb.md" >}}) +- [Redis]({{< relref "/docs/configuration/plugins/syslog-ng-outputs/redis.md" >}}) +- [Amazon S3]({{< relref "/docs/configuration/plugins/syslog-ng-outputs/s3.md" >}}) +- [Splunk HEC]({{< relref "/docs/configuration/plugins/syslog-ng-outputs/splunk_hec.md" >}}) +- The [HTTP]({{< relref "/docs/configuration/plugins/syslog-ng-outputs/http.md" >}}) output now supports the `log-fifo-size`, `response-action`, and `timeout` fields. + +You can now use the `metrics-probe()` parser of syslog-ng in syslogNGFLow and SyslogNGClusterFlow. For details, see {{% xref "/docs/configuration/plugins/syslog-ng-filters/parser.md#metricsprobe" %}}. + +### Multitenancy with namespace-based routing + +Logging operator now supports namespace based routing for efficient aggregator-level multi-tenancy. + +In the project repository you can: +- find an [overview about multitenancy](https://github.com/kube-logging/logging-operator/blob/master/docs/multi-tenancy.md). +- find more detailed information about the new [LoggingRoute](https://github.com/kube-logging/logging-operator/blob/master/docs/logging-route.md) resource that enables this new behaviour. +- find a [simple example](https://github.com/kube-logging/logging-operator/tree/master/config/samples/multitenant-routing) to demonstrate the new behaviour + +On a side note, nodegroup level isolation for hard multitenancy is also supported, see the {{% xref "docs/examples/multitenancy.md" %}} example. + +### Forwarder logs + +Fluent-bit now doesn't process the logs of the Fluentd and syslog-ng forwarders by default to avoid infinitely growing message loops. With this change, you can access Fluentd and syslog-ng logs simply by running `kubectl logs ` + +In a future Logging operator version the logs of the aggregators will also be available for routing to external outputs. + +### Timeout-based configuration checks + +You can now use timeout-based configuration strategies for both syslog-ng and Fluentd. For example: + +```yaml +apiVersion: logging.banzaicloud.io/v1beta1 +kind: Logging +metadata: + name: all-to-file +spec: + configCheck: + strategy: StartWithTimeout + timeoutSeconds: 5 + syslogNG: {} + controlNamespace: default +``` + +```yaml +apiVersion: logging.banzaicloud.io/v1beta1 +kind: Logging +metadata: + name: all-to-file +spec: + configCheck: + strategy: StartWithTimeout + timeoutSeconds: 5 + fluentd: {} + controlNamespace: default +``` + +### Istio support + +By default, Logging operator adds a label to avoid injecting the Istio sidecar into jobs/individual pods that run to completion. Configuration checkers and Fluentd drainer pods now have `sidecar.istio.io/inject` set to `false` by default. You can configure Fluentd drainer labels in the Logging spec. + +For non-istio users, these changes make no difference, as this label is only used in Istio context. For Istio users, these defaults make Logging operator work out of the box. + +### Improved buffer metrics + +The buffer metrics sidecar configuration has been rewritten to add a new metric and improve performance by avoiding unnecessary cardinality. + +The name of the metric has been changed as well, but the original metric was kept in place to avoid breaking existing clients. + +**Metrics currently supported by the sidecar** + +Old +``` ++# HELP node_buffer_size_bytes Disk space used [deprecated] ++# TYPE node_buffer_size_bytes gauge ++node_buffer_size_bytes{entity="/buffers"} 32253 +``` + +New +``` ++# HELP logging_buffer_files File count ++# TYPE logging_buffer_files gauge ++logging_buffer_files{entity="/buffers",host="all-to-file-fluentd-0"} 2 ++# HELP logging_buffer_size_bytes Disk space used ++# TYPE logging_buffer_size_bytes gauge ++logging_buffer_size_bytes{entity="/buffers",host="all-to-file-fluentd-0"} 32253 +``` + +## Other improvements + +- You can now configure the resources of the buffer metrics sidecar. +- You can now rerun failed configuration checks if there is no configcheck pod. +- The [Fluentd ElasticSearch output]({{< relref "/docs/configuration/plugins/outputs/elasticsearch.md" >}}) now supports the [composable index template](https://www.elastic.co/guide/en/elasticsearch/reference/7.13/index-templates.html) format. To use it, set the `use_legacy_template` option to `false`. +- The metrics for the syslog-ng forwarder are now exported using [axosyslog-metrics-exporter](https://github.com/axoflow/axosyslog-metrics-exporter). + +### Image and dependency updates + +For the list of images used in Logging operator, see {{% xref "/docs/image-versions.md" %}}. + +Fluentd images with versions `v1.14` and `v1.15` are now EOL due to the fact they are based on ruby 2.7 which is EOL as well. + +The currently supported image is [v1.15-ruby3](https://github.com/kube-logging/fluentd-images/tree/main/v1.15-ruby3) and build configuration for [v1.15-staging](https://github.com/kube-logging/fluentd-images/tree/main/v1.15-staging) is available for staging experimental changes. diff --git a/content/headless/note-fluentd-logs.md b/content/headless/note-fluentd-logs.md new file mode 100644 index 000000000..649be7938 --- /dev/null +++ b/content/headless/note-fluentd-logs.md @@ -0,0 +1,2 @@ +> Fluentd logs were written to the container filesystem up until Logging operator version 4.3, which has been changed to stdout with 4.4. + See {{% xref "/docs/logging-infrastructure/fluentd.md#fluentoutlogrotate" %}} why this was changed and how you can re-enable it if needed.