From da38df9d68d8da39a8a1d751787c66273ff068ef Mon Sep 17 00:00:00 2001 From: Robert Fekete Date: Fri, 15 Sep 2023 16:56:39 +0200 Subject: [PATCH 01/30] Adds 4.3 docs to the menu Signed-off-by: Robert Fekete --- .github/workflows/publish-version-4.3.yaml | 86 ++++++++++++++++++++++ config/_default/config.toml | 8 +- 2 files changed, 92 insertions(+), 2 deletions(-) create mode 100644 .github/workflows/publish-version-4.3.yaml diff --git a/.github/workflows/publish-version-4.3.yaml b/.github/workflows/publish-version-4.3.yaml new file mode 100644 index 000000000..26f40f26a --- /dev/null +++ b/.github/workflows/publish-version-4.3.yaml @@ -0,0 +1,86 @@ +name: Publish version 4.3 + +env: + doc_versionnumber: "4.3" + +on: + push: + branches: + - release-4.3 + workflow_dispatch: + +jobs: + build: + name: Build + runs-on: ubuntu-latest + + permissions: + contents: write + pages: write + id-token: write + + concurrency: + group: "pages" + cancel-in-progress: false + + environment: + name: github-pages-test + url: ${{ steps.deployment.outputs.page_url }} + + steps: + - name: Checkout code + uses: actions/checkout@f43a0e5ff2bd294095638e18286ca9a3d1956744 # v3.6.0 + with: + ref: release-4.3 + submodules: 'recursive' + + - name: Set up Pages + id: pages + uses: actions/configure-pages@f156874f8191504dae5b037505266ed5dda6c382 # v3.0.6 + + - name: Set up Hugo + uses: peaceiris/actions-hugo@16361eb4acea8698b220b76c0d4e84e1fd22c61d # v2.6.0 + with: + hugo-version: '0.110.0' + extended: true + + - name: Set up Node + uses: actions/setup-node@5e21ff4d9bc1a8cf6de233a3057d20ec6b3fb69d # v3.8.1 + with: + node-version: 16 + + - name: Install dependencies + run: | + cd themes/docsy + npm install + + - name: Set up PostCSS + run: npm install --save-dev autoprefixer postcss-cli postcss + + - name: Build + run: hugo --environment production --baseURL ${{ steps.pages.outputs.base_url }}/${{ env.doc_versionnumber }}/ + + # - name: Upload artifact + # uses: actions/upload-pages-artifact@64bcae551a7b18bcb9a09042ddf1960979799187 # v1.0.8 + # with: + # path: ./public/ + + - name: Checkout code to update + uses: actions/checkout@f43a0e5ff2bd294095638e18286ca9a3d1956744 # v3.6.0 + with: + ref: 'gh-pages-test' + path: 'tmp/gh-pages' + # - name: Display file structure + # run: ls -R + - name: Copy built site to GH pages + run: | + rm -rf tmp/gh-pages/${{ env.doc_versionnumber }} + mkdir -p tmp/gh-pages/${{ env.doc_versionnumber }} + mv public/* tmp/gh-pages/${{ env.doc_versionnumber }} + - name: Commit & Push changes + uses: actions-js/push@master + with: + github_token: ${{ secrets.GITHUB_TOKEN }} + message: 'Publish updated docs for ${{ env.doc_versionnumber }}, ${{ github.event.repository.pushed_at}}' + branch: 'gh-pages-test' + directory: 'tmp/gh-pages' diff --git a/config/_default/config.toml b/config/_default/config.toml index fe3fcf21c..b8e140560 100644 --- a/config/_default/config.toml +++ b/config/_default/config.toml @@ -69,7 +69,7 @@ twitter = "calisti12" # Used in the "version-banner" partial to display a version number for the # current doc set. - version = "4.3.0" + version = "4.4.0" version_menu = "Releases" version_menu_canonicallinks = true version_menu_pagelinks = true @@ -169,9 +169,13 @@ twitter = "calisti12" ####################### # Add your release versions here [[params.versions]] - version = "latest (4.3.0)" + version = "latest (4.4.0)" githubbranch = "master" url = "" +[[params.versions]] + version = "4.3" + githubbranch = "release-4.3" + url = "/4.3/" [[params.versions]] version = "4.2" githubbranch = "release-4.2" From b58fecd75229b74c47608f5ace72b01eb76762f6 Mon Sep 17 00:00:00 2001 From: Robert Fekete Date: Fri, 15 Sep 2023 17:30:21 +0200 Subject: [PATCH 02/30] Updates generated syslog-ng output docs Signed-off-by: Robert Fekete --- .../syslog-ng-outputs/elasticsearch.md | 49 ++++++ .../plugins/syslog-ng-outputs/file.md | 2 + .../plugins/syslog-ng-outputs/http.md | 34 +++- .../plugins/syslog-ng-outputs/logscale.md | 4 +- .../plugins/syslog-ng-outputs/mongodb.md | 156 ++++++++++++++++++ .../plugins/syslog-ng-outputs/redis.md | 118 +++++++++++++ .../plugins/syslog-ng-outputs/splunk_hec.md | 99 +++++++++++ .../plugins/syslog-ng-outputs/tls.md | 19 +-- 8 files changed, 459 insertions(+), 22 deletions(-) create mode 100644 content/docs/configuration/plugins/syslog-ng-outputs/elasticsearch.md create mode 100644 content/docs/configuration/plugins/syslog-ng-outputs/mongodb.md create mode 100644 content/docs/configuration/plugins/syslog-ng-outputs/redis.md create mode 100644 content/docs/configuration/plugins/syslog-ng-outputs/splunk_hec.md diff --git a/content/docs/configuration/plugins/syslog-ng-outputs/elasticsearch.md b/content/docs/configuration/plugins/syslog-ng-outputs/elasticsearch.md new file mode 100644 index 000000000..3fbf6451d --- /dev/null +++ b/content/docs/configuration/plugins/syslog-ng-outputs/elasticsearch.md @@ -0,0 +1,49 @@ +--- +title: Elasticsearch +weight: 200 +generated_file: true +--- + +## Overview + +Based on the [ElasticSearch destination of AxoSyslog core](https://axoflow.com/docs/axosyslog-core/chapter-destinations/configuring-destinations-elasticsearch-http/). + +## Configuration + +### (HTTPOutput, required) {#elasticsearchoutput-} + +Default: - + +### index (string, optional) {#elasticsearchoutput-index} + +Name of the data stream, index, or index alias to perform the action on. + +Default: - + +### type (*string, optional) {#elasticsearchoutput-type} + +The document type associated with the operation. Elasticsearch indices now support a single document type: `_doc` + +### custom_id (string, optional) {#elasticsearchoutput-custom_id} + +The document ID. If no ID is specified, a document ID is automatically generated. + +Default: - + +### logstash_prefix (string, optional) {#elasticsearchoutput-logstash_prefix} + +Set the prefix for logs in logstash format. If set, then Index field will be ignored. + +Default: - + +### logstash_prefix_separator (string, optional) {#elasticsearchoutput-logstash_prefix_separator} + +Set the separator between LogstashPrefix and LogStashDateformat. + +Default: `-` + +### logstash_suffix (string, optional) {#elasticsearchoutput-logstash_suffix} + +Set the suffix for logs in logstash format. + +Default: `${YEAR}.${MONTH}.${DAY}` diff --git a/content/docs/configuration/plugins/syslog-ng-outputs/file.md b/content/docs/configuration/plugins/syslog-ng-outputs/file.md index 75cd4934b..c3f49dabd 100644 --- a/content/docs/configuration/plugins/syslog-ng-outputs/file.md +++ b/content/docs/configuration/plugins/syslog-ng-outputs/file.md @@ -15,6 +15,8 @@ The `file` output stores log records in a plain text file. For details on the available options of the output, see the [documentation of the AxoSyslog syslog-ng distribution](https://axoflow.com/docs/axosyslog-core/chapter-destinations/configuring-destinations-file/). +## Configuration + ### path (string, required) {#fileoutput-path} Store file path diff --git a/content/docs/configuration/plugins/syslog-ng-outputs/http.md b/content/docs/configuration/plugins/syslog-ng-outputs/http.md index 1067ae32e..933165eec 100644 --- a/content/docs/configuration/plugins/syslog-ng-outputs/http.md +++ b/content/docs/configuration/plugins/syslog-ng-outputs/http.md @@ -4,13 +4,11 @@ weight: 200 generated_file: true --- -# Sending messages over HTTP ## Overview For details on the available options of the output, see the [documentation of the AxoSyslog syslog-ng distribution](https://axoflow.com/docs/axosyslog-core/chapter-destinations/configuring-destinations-http-nonjava/). ## Configuration -## HTTPOutput ### url (string, optional) {#httpoutput-url} @@ -50,7 +48,7 @@ Default: - ### body (string, optional) {#httpoutput-body} -The body of the HTTP request, for example, body("${ISODATE} ${MESSAGE}"). You can use strings, macros, and template functions in the body. If not set, it will contain the message received from the source by default. +The body of the HTTP request, for example, `body("${ISODATE} ${MESSAGE}")`. You can use strings, macros, and template functions in the body. If not set, it will contain the message received from the source by default. Default: - @@ -74,7 +72,7 @@ Default: - ### method (string, optional) {#httpoutput-method} -Specifies the HTTP method to use when sending the message to the server. POST | PUT +Specifies the HTTP method to use when sending the message to the server. `POST | PUT` Default: - @@ -104,12 +102,32 @@ Default: - ### workers (int, optional) {#httpoutput-workers} -Description: Specifies the number of worker threads (at least 1) that syslog-ng OSE uses to send messages to the server. Increasing the number of worker threads can drastically improve the performance of the destination. +Specifies the number of worker threads (at least 1) that syslog-ng OSE uses to send messages to the server. Increasing the number of worker threads can drastically improve the performance of the destination. Default: - ### persist_name (string, optional) {#httpoutput-persist_name} +If you receive the following error message during AxoSyslog startup, set the `persist-name()` option of the duplicate drivers: `Error checking the uniqueness of the persist names, please override it with persist-name option. Shutting down.` See the [AxoSyslog Core documentation](https://axoflow.com/docs/axosyslog-core/chapter-destinations/configuring-destinations-http-nonjava/reference-destination-http-nonjava/#persist-name) for more information. + +Default: - + +### log-fifo-size (int, optional) {#httpoutput-log-fifo-size} + +The number of messages that the output queue can store. + +Default: - + +### timeout (int, optional) {#httpoutput-timeout} + +Sets the maximum number of messages sent to the destination per second. Use this output-rate-limiting functionality only when using disk-buffer as well to avoid the risk of losing messages. Specifying 0 or a lower value sets the output limit to unlimited. + +Default: - + +### response-action (filter.RawArrowMap, optional) {#httpoutput-response-action} + +Specifies what AxoSyslog does with the log message, based on the response code received from the HTTP server. See the [AxoSyslog Core documentation](https://axoflow.com/docs/axosyslog-core/chapter-destinations/configuring-destinations-http-nonjava/reference-destination-http-nonjava/#response-action) for more information. + Default: - @@ -117,19 +135,19 @@ Default: - ### batch-lines (int, optional) {#batch-batch-lines} -Description: Specifies how many lines are flushed to a destination in one batch. The syslog-ng OSE application waits for this number of lines to accumulate and sends them off in a single batch. Increasing this number increases throughput as more messages are sent in a single batch, but also increases message latency. For example, if you set batch-lines() to 100, syslog-ng OSE waits for 100 messages. +Description: Specifies how many lines are flushed to a destination in one batch. The syslog-ng OSE application waits for this number of lines to accumulate and sends them off in a single batch. Increasing this number increases throughput as more messages are sent in a single batch, but also increases message latency. For example, if you set `batch-lines()` to 100, syslog-ng OSE waits for 100 messages. Default: - ### batch-bytes (int, optional) {#batch-batch-bytes} -Description: Sets the maximum size of payload in a batch. If the size of the messages reaches this value, syslog-ng OSE sends the batch to the destination even if the number of messages is less than the value of the batch-lines() option. Note that if the batch-timeout() option is enabled and the queue becomes empty, syslog-ng OSE flushes the messages only if batch-timeout() expires, or the batch reaches the limit set in batch-bytes(). +Description: Sets the maximum size of payload in a batch. If the size of the messages reaches this value, syslog-ng OSE sends the batch to the destination even if the number of messages is less than the value of the `batch-lines()` option. Note that if the `batch-timeout()` option is enabled and the queue becomes empty, syslog-ng OSE flushes the messages only if `batch-timeout()` expires, or the batch reaches the limit set in `batch-bytes()`. Default: - ### batch-timeout (int, optional) {#batch-batch-timeout} -Description: Specifies the time syslog-ng OSE waits for lines to accumulate in the output buffer. The syslog-ng OSE application sends batches to the destinations evenly. The timer starts when the first message arrives to the buffer, so if only few messages arrive, syslog-ng OSE sends messages to the destination at most once every batch-timeout() milliseconds. +Description: Specifies the time syslog-ng OSE waits for lines to accumulate in the output buffer. The syslog-ng OSE application sends batches to the destinations evenly. The timer starts when the first message arrives to the buffer, so if only few messages arrive, syslog-ng OSE sends messages to the destination at most once every `batch-timeout()` milliseconds. Default: - diff --git a/content/docs/configuration/plugins/syslog-ng-outputs/logscale.md b/content/docs/configuration/plugins/syslog-ng-outputs/logscale.md index 0edc24082..6ff1fdf34 100644 --- a/content/docs/configuration/plugins/syslog-ng-outputs/logscale.md +++ b/content/docs/configuration/plugins/syslog-ng-outputs/logscale.md @@ -4,11 +4,9 @@ weight: 200 generated_file: true --- -# logscale -## Overview +Based on the [LogScale destination of AxoSyslog core](https://axoflow.com/docs/axosyslog-core/chapter-destinations/crowdstrike-falcon/). ## Configuration -## LogScaleOutput ### url (*secret.Secret, optional) {#logscaleoutput-url} diff --git a/content/docs/configuration/plugins/syslog-ng-outputs/mongodb.md b/content/docs/configuration/plugins/syslog-ng-outputs/mongodb.md new file mode 100644 index 000000000..0fdf212ff --- /dev/null +++ b/content/docs/configuration/plugins/syslog-ng-outputs/mongodb.md @@ -0,0 +1,156 @@ +--- +title: MongoDB +weight: 200 +generated_file: true +--- + +Based on the [MongoDB destination of AxoSyslog core](https://axoflow.com/docs/axosyslog-core/chapter-destinations/configuring-destinations-mongodb/). + +## Example + +{{< highlight yaml >}} +apiVersion: logging.banzaicloud.io/v1beta1 +kind: SyslogNGOutput +metadata: + name: mongodb + namespace: default +spec: + mongodb: + collection: syslog + uri: mongodb://127.0.0.1:27017/syslog?wtimeoutMS=60000&socketTimeoutMS=60000&connectTimeoutMS=60000 + value_pairs: scope("selected-macros" "nv-pairs") + {{}} + +## Configuration + +### collection (string, required) {#mongodb-collection} + +The name of the MongoDB collection where the log messages are stored (collections are similar to SQL tables). Note that the name of the collection must not start with a dollar sign ($), and that it may contain dot (.) characters. + +Default: - + +### compaction (bool, required) {#mongodb-compaction} + +If set to yes, syslog-ng OSE cannot lose logs in case of reload/restart, unreachable destination or syslog-ng OSE crash. This solution provides a slower, but reliable disk-buffer option. + +Default: - + +### dir (string, optional) {#mongodb-dir} + +Defines the folder where the disk-buffer files are stored. + +Default: - + +### disk_buffer (*DiskBuffer, optional) {#mongodb-disk_buffer} + +This option enables putting outgoing messages into the disk buffer of the destination to avoid message loss in case of a system failure on the destination side. For details, see the [Syslog-ng DiskBuffer options](../disk_buffer/). + +Default: false + +### uri (string, optional) {#mongodb-uri} + +Defines the folder where the disk-buffer files are stored. + +Default: "mongodb://127.0.0.1:27017/syslog?wtimeoutMS=60000&socketTimeoutMS=60000&connectTimeoutMS=60000" + +### value_pairs (ValuePairs, optional) {#mongodb-value_pairs} + +Creates structured name-value pairs from the data and metadata of the log message. + +Default: "scope("selected-macros" "nv-pairs")" + +### (Batch, required) {#mongodb-} + +Batching parameters + +Default: - + +### (Bulk, required) {#mongodb-} + +Bulk operation related options + +Default: - + +### log-fifo-size (int, optional) {#mongodb-log-fifo-size} + +The number of messages that the output queue can store. + +Default: - + +### persist_name (string, optional) {#mongodb-persist_name} + +If you receive the following error message during AxoSyslog startup, set the persist-name() option of the duplicate drivers: `Error checking the uniqueness of the persist names, please override it with persist-name option. Shutting down.` See [syslog-ng docs](https://axoflow.com/docs/axosyslog-core/chapter-destinations/configuring-destinations-http-nonjava/reference-destination-http-nonjava/#persist-name) for more information. + +Default: - + +### retries (int, optional) {#mongodb-retries} + +The number of times syslog-ng OSE attempts to send a message to this destination. If syslog-ng OSE could not send a message, it will try again until the number of attempts reaches retries, then drops the message. + +Default: - + +### time_reopen (int, optional) {#mongodb-time_reopen} + +The time to wait in seconds before a dead connection is reestablished. + +Default: 60 + +### write_concern (RawString, optional) {#mongodb-write_concern} + +Description: Sets the write concern mode of the MongoDB operations, for both bulk and single mode. See the [AxoSyslog Core documentation](https://axoflow.com/docs/axosyslog-core/chapter-destinations/configuring-destinations-mongodb/reference-destination-mongodb/#mongodb-option-write-concern) + +Default: - + + +## Bulk + +Bulk operation related options +See [AxoSyslog Core documentation](https://axoflow.com/docs/axosyslog-core/chapter-destinations/configuring-destinations-mongodb/reference-destination-mongodb/#mongodb-option-bulk) + +### bulk (*bool, optional) {#bulk-bulk} + +Enables bulk insert mode. If disabled, each messages is inserted individually. + +Default: yes + +### bulk_bypass_validation (*bool, optional) {#bulk-bulk_bypass_validation} + +If set to yes, it disables MongoDB bulk operations validation mode. + +Default: no + +### bulk_unordered (*bool, optional) {#bulk-bulk_unordered} + +Description: Enables unordered bulk operations mode. + +Default: no + + +## ValuePairs + +TODO move this to a common module once it is used in more places + +### scope (RawString, optional) {#valuepairs-scope} + +Default: - + +### exclude (RawString, optional) {#valuepairs-exclude} + +Default: - + +### key (RawString, optional) {#valuepairs-key} + +Default: - + +### pair (RawString, optional) {#valuepairs-pair} + +Default: - + + +## RawString + +### raw_string (string, optional) {#rawstring-raw_string} + +Default: - + + diff --git a/content/docs/configuration/plugins/syslog-ng-outputs/redis.md b/content/docs/configuration/plugins/syslog-ng-outputs/redis.md new file mode 100644 index 000000000..aa532566e --- /dev/null +++ b/content/docs/configuration/plugins/syslog-ng-outputs/redis.md @@ -0,0 +1,118 @@ +--- +title: Redis +weight: 200 +generated_file: true +--- + +Based on the [Redis destination of AxoSyslog core](https://axoflow.com/docs/axosyslog-core/chapter-destinations/configuring-destinations-redis/). + +## Example + + {{< highlight yaml >}} + apiVersion: logging.banzaicloud.io/v1beta1 + kind: SyslogNGOutput + metadata: + + name: redis + namespace: default + + spec: + + redis: + host: 127.0.0.1 + port: 6379 + retries: 3 + throttle: 0 + time-reopen: 60 + workers: 1 + + {{}} + +## Configuration + +### host (string, optional) {#redisoutput-host} + +The hostname or IP address of the Redis server. + +Default: 127.0.0.1 + +### auth (*secret.Secret, optional) {#redisoutput-auth} + +The password used for authentication on a password-protected Redis server. + +Default: - + +### port (int, optional) {#redisoutput-port} + +The port number of the Redis server. + +Default: 6379 + +### retries (int, optional) {#redisoutput-retries} + +If syslog-ng OSE cannot send a message, it will try again until the number of attempts reaches retries(). + +Default: 3 + +### throttle (int, optional) {#redisoutput-throttle} + +Sets the maximum number of messages sent to the destination per second. Use this output-rate-limiting functionality only when using disk-buffer as well to avoid the risk of losing messages. Specifying 0 or a lower value sets the output limit to unlimited. + +Default: 0 + +### time-reopen (int, optional) {#redisoutput-time-reopen} + +The time to wait in seconds before a dead connection is reestablished. + +Default: 60 + +### workers (int, optional) {#redisoutput-workers} + +Specifies the number of worker threads (at least 1) that syslog-ng OSE uses to send messages to the server. Increasing the number of worker threads can drastically improve the performance of the destination. + +Default: 1 + +### command_and_arguments ([]string, optional) {#redisoutput-command_and_arguments} + +The Redis command to execute, for example, LPUSH, INCR, or HINCRBY. Using the HINCRBY command with an increment value of 1 allows you to create various statistics. For example, the command("HINCRBY" "${HOST}/programs" "${PROGRAM}" "1") command counts the number of log messages on each host for each program. + +Default: "" + +### command (StringList, optional) {#redisoutput-command} + +Internal rendered form of the CommandAndArguments field + +Default: - + +### (Batch, required) {#redisoutput-} + +Batching parameters + +Default: - + +### log-fifo-size (int, optional) {#redisoutput-log-fifo-size} + +The number of messages that the output queue can store. + +Default: - + +### disk_buffer (*DiskBuffer, optional) {#redisoutput-disk_buffer} + +This option enables putting outgoing messages into the disk buffer of the destination to avoid message loss in case of a system failure on the destination side. For details, see the [syslog-ng DiskBuffer options](../disk_buffer/). + +Default: false + +### persist_name (string, optional) {#redisoutput-persist_name} + +Persistname + +Default: - + + +## StringList + +### string-list ([]string, optional) {#stringlist-string-list} + +Default: - + + diff --git a/content/docs/configuration/plugins/syslog-ng-outputs/splunk_hec.md b/content/docs/configuration/plugins/syslog-ng-outputs/splunk_hec.md new file mode 100644 index 000000000..57d678005 --- /dev/null +++ b/content/docs/configuration/plugins/syslog-ng-outputs/splunk_hec.md @@ -0,0 +1,99 @@ +--- +title: SplunkHEC +weight: 200 +generated_file: true +--- + +Based on the [Splunk destination of AxoSyslog core](https://axoflow.com/docs/axosyslog-core/chapter-destinations/syslog-ng-with-splunk/). + +## Configuration + +### (HTTPOutput, required) {#splunkhecoutput-} + +Default: - + +### token (secret.Secret, optional) {#splunkhecoutput-token} + +The token that syslog-ng OSE uses to authenticate on the event collector. + +Default: - + +### event (string, optional) {#splunkhecoutput-event} + +event() accepts a template, which declares the content of the log message sent to Splunk. Default value: ${MSG} + +Default: - + +### index (string, optional) {#splunkhecoutput-index} + +Splunk index where the messages will be stored. + +Default: - + +### source (string, optional) {#splunkhecoutput-source} + +Sets the source field. + +Default: - + +### sourcetype (string, optional) {#splunkhecoutput-sourcetype} + +Sets the sourcetype field. + +Default: - + +### host (string, optional) {#splunkhecoutput-host} + +Sets the host field. + +Default: - + +### time (string, optional) {#splunkhecoutput-time} + +Sets the time field. + +Default: - + +### default_index (string, optional) {#splunkhecoutput-default_index} + +Fallback option for index field. See [syslog-ng docs](https://axoflow.com/docs/axosyslog-core/chapter-destinations/syslog-ng-with-splunk/) + +Default: - + +### default_source (string, optional) {#splunkhecoutput-default_source} + +Fallback option for source field. + +Default: - + +### default_sourcetype (string, optional) {#splunkhecoutput-default_sourcetype} + +Fallback option for sourcetype field. + +Default: - + +### fields (string, optional) {#splunkhecoutput-fields} + +Additional indexing metadata for Splunk. + +Default: - + +### extra_headers ([]string, optional) {#splunkhecoutput-extra_headers} + +Additional HTTP request headers. + +Default: - + +### extra_queries ([]string, optional) {#splunkhecoutput-extra_queries} + +Additional HTTP request query options. + +Default: - + +### content_type (string, optional) {#splunkhecoutput-content_type} + +Additional HTTP request content-type option. + +Default: - + + diff --git a/content/docs/configuration/plugins/syslog-ng-outputs/tls.md b/content/docs/configuration/plugins/syslog-ng-outputs/tls.md index 525d4e4a8..0fa62e7c5 100644 --- a/content/docs/configuration/plugins/syslog-ng-outputs/tls.md +++ b/content/docs/configuration/plugins/syslog-ng-outputs/tls.md @@ -4,46 +4,43 @@ weight: 200 generated_file: true --- -# TLS config for syslog-ng outputs -## Overview - More info at https://www.syslog-ng.com/technical-documents/doc/syslog-ng-open-source-edition/3.37/administration-guide/32#kanchor2338 +For details on how TLS configuration works in syslog-ng, see the [AxoSyslog Core documentation](https://axoflow.com/docs/axosyslog-core/chapter-encrypted-transport-tls/tlsoptions/). ## Configuration -## TLS ### ca_dir (*secret.Secret, optional) {#tls-ca_dir} -The name of a directory that contains a set of trusted CA certificates in PEM format. [more information](https://www.syslog-ng.com/technical-documents/doc/syslog-ng-open-source-edition/3.37/administration-guide/73#kanchor3142) +The name of a directory that contains a set of trusted CA certificates in PEM format. For details, see the [AxoSyslog Core documentation](https://axoflow.com/docs/axosyslog-core/chapter-encrypted-transport-tls/tlsoptions/#ca-dir) Default: - ### ca_file (*secret.Secret, optional) {#tls-ca_file} -The name of a file that contains a set of trusted CA certificates in PEM format. (Optional) [more information](https://www.syslog-ng.com/technical-documents/doc/syslog-ng-open-source-edition/3.37/administration-guide/73#kanchor3144) +The name of a file that contains a set of trusted CA certificates in PEM format. (Optional) For details, see the [AxoSyslog Core documentation](https://axoflow.com/docs/axosyslog-core/chapter-encrypted-transport-tls/tlsoptions/#ca-file) Default: - ### key_file (*secret.Secret, optional) {#tls-key_file} -The name of a file that contains an unencrypted private key in PEM format, suitable as a TLS key. [more information](https://www.syslog-ng.com/technical-documents/doc/syslog-ng-open-source-edition/3.37/administration-guide/73#kanchor3163) +The name of a file that contains an unencrypted private key in PEM format, suitable as a TLS key. For details, see the [AxoSyslog Core documentation](https://axoflow.com/docs/axosyslog-core/chapter-encrypted-transport-tls/tlsoptions/#key-file) Default: - ### cert_file (*secret.Secret, optional) {#tls-cert_file} -Name of a file, that contains an X.509 certificate (or a certificate chain) in PEM format, suitable as a TLS certificate, matching the private key set in the key-file() option. [more information](https://www.syslog-ng.com/technical-documents/doc/syslog-ng-open-source-edition/3.37/administration-guide/73#kanchor3146) +Name of a file, that contains an X.509 certificate (or a certificate chain) in PEM format, suitable as a TLS certificate, matching the private key set in the key-file() option. For details, see the [AxoSyslog Core documentation](https://axoflow.com/docs/axosyslog-core/chapter-encrypted-transport-tls/tlsoptions/#cert-file) Default: - -### peer_verify (string, optional) {#tls-peer_verify} +### peer_verify (*bool, optional) {#tls-peer_verify} -Verification method of the peer. [more information](https://www.syslog-ng.com/technical-documents/doc/syslog-ng-open-source-edition/3.37/administration-guide/73#tls-options-peer-verify) +Verification method of the peer. For details, see the [AxoSyslog Core documentation](https://axoflow.com/docs/axosyslog-core/chapter-encrypted-transport-tls/tlsoptions/#tls-options-peer-verify) Default: - ### use-system-cert-store (*bool, optional) {#tls-use-system-cert-store} -Use the certificate store of the system for verifying HTTPS certificates. [more information](https://curl.se/docs/sslcerts.html) +Use the certificate store of the system for verifying HTTPS certificates. For details, see the [AxoSyslog Core documentation](https://curl.se/docs/sslcerts.html) Default: - From 7f82761dcc0f7013bce3e8febb07640b5919da9f Mon Sep 17 00:00:00 2001 From: Robert Fekete Date: Fri, 15 Sep 2023 17:40:42 +0200 Subject: [PATCH 03/30] Updates generated fluentd output docs Signed-off-by: Robert Fekete --- content/docs/configuration/plugins/outputs/datadog.md | 3 ++- content/docs/configuration/plugins/outputs/elasticsearch.md | 6 ++++++ content/docs/configuration/plugins/outputs/loki.md | 6 ++++++ content/docs/configuration/plugins/outputs/opensearch.md | 2 +- .../docs/configuration/plugins/syslog-ng-filters/match.md | 4 ++-- 5 files changed, 17 insertions(+), 4 deletions(-) diff --git a/content/docs/configuration/plugins/outputs/datadog.md b/content/docs/configuration/plugins/outputs/datadog.md index adc5a0272..252cf7309 100644 --- a/content/docs/configuration/plugins/outputs/datadog.md +++ b/content/docs/configuration/plugins/outputs/datadog.md @@ -13,7 +13,8 @@ More info at [https://github.com/DataDog/fluent-plugin-datadog](https://github.c ```yaml spec: datadog: - api_key '' + api_key: + value: '' # for referencing a secret see https://kube-logging.dev/docs/configuration/plugins/outputs/secret/ dd_source: '' dd_tags: ',' dd_sourcecategory: '' diff --git a/content/docs/configuration/plugins/outputs/elasticsearch.md b/content/docs/configuration/plugins/outputs/elasticsearch.md index 45f9d724f..8ac159e0c 100644 --- a/content/docs/configuration/plugins/outputs/elasticsearch.md +++ b/content/docs/configuration/plugins/outputs/elasticsearch.md @@ -573,4 +573,10 @@ Specify whether overwriting data stream ilm policy or not. Default: - +### use_legacy_template (*bool, optional) {#elasticsearch-use_legacy_template} + +Specify wether to use legacy template or not. + +Default: true + diff --git a/content/docs/configuration/plugins/outputs/loki.md b/content/docs/configuration/plugins/outputs/loki.md index 38ed78500..937b84fe6 100644 --- a/content/docs/configuration/plugins/outputs/loki.md +++ b/content/docs/configuration/plugins/outputs/loki.md @@ -115,6 +115,12 @@ Configure Kubernetes metadata in a Prometheus like format Default: false +### include_thread_label (*bool, optional) {#output-config-include_thread_label} + +whether to include the fluentd_thread label when multiple threads are used for flushing. + +Default: true + ### buffer (*Buffer, optional) {#output-config-buffer} [Buffer](../buffer/) diff --git a/content/docs/configuration/plugins/outputs/opensearch.md b/content/docs/configuration/plugins/outputs/opensearch.md index db53756ed..c4ee0408e 100644 --- a/content/docs/configuration/plugins/outputs/opensearch.md +++ b/content/docs/configuration/plugins/outputs/opensearch.md @@ -505,7 +505,7 @@ Default: - ### use_legacy_template (*bool, optional) {#opensearch-use_legacy_template} -use_legacy_template (default: true) +Specify wether to use legacy template or not. Default: true diff --git a/content/docs/configuration/plugins/syslog-ng-filters/match.md b/content/docs/configuration/plugins/syslog-ng-filters/match.md index 92004a7f4..854d95d7d 100644 --- a/content/docs/configuration/plugins/syslog-ng-filters/match.md +++ b/content/docs/configuration/plugins/syslog-ng-filters/match.md @@ -42,9 +42,9 @@ Default: - Default: - -## [Regexp Directive](https://axoflow.com/docs/axosyslog-core/chapter-manipulating-messages/customizing-message-format/reference-template-functions/#template-function-list) {#Regexp-Directive} +## Regexp Directive {#Regexp-Directive} -Specify filtering rule. +Specify filtering rule. For details, see the [AxoSyslog Core documentation](https://axoflow.com/docs/axosyslog-core/chapter-manipulating-messages/customizing-message-format/reference-template-functions/#template-function-list) ### pattern (string, required) {#regexp-directive-pattern} From 06bef6c6233e35054b5fc06ebacc4ce313567ee1 Mon Sep 17 00:00:00 2001 From: Robert Fekete Date: Fri, 15 Sep 2023 17:45:16 +0200 Subject: [PATCH 04/30] Updates the generated CRD docs Signed-off-by: Robert Fekete --- .../crds/v1beta1/fluentbit_types.md | 6 +++- .../crds/v1beta1/fluentd_types.md | 8 +++++ .../crds/v1beta1/logging_types.md | 29 ++++++++++++++++++- .../crds/v1beta1/syslogng_output_types.md | 16 ++++++++++ .../crds/v1beta1/syslogng_types.md | 8 ++++- 5 files changed, 64 insertions(+), 3 deletions(-) diff --git a/content/docs/configuration/crds/v1beta1/fluentbit_types.md b/content/docs/configuration/crds/v1beta1/fluentbit_types.md index eb47fd0d6..f85d92ee9 100644 --- a/content/docs/configuration/crds/v1beta1/fluentbit_types.md +++ b/content/docs/configuration/crds/v1beta1/fluentbit_types.md @@ -202,6 +202,10 @@ Default: - Default: - +### bufferVolumeResources (corev1.ResourceRequirements, optional) {#fluentbitspec-buffervolumeresources} + +Default: - + ### customConfigSecret (string, optional) {#fluentbitspec-customconfigsecret} Default: - @@ -607,7 +611,7 @@ Default: "0" ### Kube_URL (string, optional) {#filterkubernetes-kube_url} -API Server end-point (default: `https://kubernetes.default.svc:443`) +API Server end-point. Default: `https://kubernetes.default.svc:443` diff --git a/content/docs/configuration/crds/v1beta1/fluentd_types.md b/content/docs/configuration/crds/v1beta1/fluentd_types.md index e77be3d1c..8346a9ff5 100644 --- a/content/docs/configuration/crds/v1beta1/fluentd_types.md +++ b/content/docs/configuration/crds/v1beta1/fluentd_types.md @@ -132,6 +132,10 @@ Default: - Default: - +### bufferVolumeResources (corev1.ResourceRequirements, optional) {#fluentdspec-buffervolumeresources} + +Default: - + ### security (*Security, optional) {#fluentdspec-security} Default: - @@ -299,6 +303,10 @@ Container image to use for the drain watch sidecar Default: - +### labels (map[string]string, optional) {#fluentddrainconfig-labels} + +Labels to use for the drain watch sidecar on top of labels added by the operator by default. Default values can be overwritten. + ### deleteVolume (bool, optional) {#fluentddrainconfig-deletevolume} Should persistent volume claims be deleted after draining is done diff --git a/content/docs/configuration/crds/v1beta1/logging_types.md b/content/docs/configuration/crds/v1beta1/logging_types.md index 4c852b161..e32103877 100644 --- a/content/docs/configuration/crds/v1beta1/logging_types.md +++ b/content/docs/configuration/crds/v1beta1/logging_types.md @@ -32,6 +32,12 @@ Override generated config. This is a *raw* configuration string for troubleshoot Default: - +### configCheck (ConfigCheck, optional) {#loggingspec-configcheck} + +ConfigCheck settings that apply to both fluentd and syslog-ng + +Default: - + ### fluentbit (*FluentbitSpec, optional) {#loggingspec-fluentbit} FluentbitAgent daemonset configuration. Deprecated, will be removed with next major version Migrate to the standalone NodeAgent resource @@ -84,7 +90,7 @@ Default: - Cluster domain name to be used when templating URLs to services . -Default: "cluster.local" +Default: "cluster.local." ### controlNamespace (string, required) {#loggingspec-controlnamespace} @@ -111,6 +117,27 @@ EnableRecreateWorkloadOnImmutableFieldChange enables the operator to recreate th Default: - +## ConfigCheck + +### strategy (ConfigCheckStrategy, optional) {#configcheck-strategy} + +Select the config check strategy to use. `DryRun`: parse and validate configuration `StartWithTimeout`: start with given configuration and exit after specified timeout. + +Default: `DryRun` + +### timeoutSeconds (int, optional) {#configcheck-timeoutseconds} + +Configure timeout in seconds if strategy is StartWithTimeout + +Default: - + +### labels (map[string]string, optional) {#configcheck-labels} + +Labels to use for the configcheck pods on top of labels added by the operator by default. Default values can be overwritten. + +Default: - + + ## LoggingStatus LoggingStatus defines the observed state of Logging diff --git a/content/docs/configuration/crds/v1beta1/syslogng_output_types.md b/content/docs/configuration/crds/v1beta1/syslogng_output_types.md index 6f4a1879a..e9128cfff 100644 --- a/content/docs/configuration/crds/v1beta1/syslogng_output_types.md +++ b/content/docs/configuration/crds/v1beta1/syslogng_output_types.md @@ -28,6 +28,14 @@ Default: - Default: - +### redis (*output.RedisOutput, optional) {#syslogngoutputspec-redis} + +Default: - + +### mongodb (*output.MongoDB, optional) {#syslogngoutputspec-mongodb} + +Default: - + ### sumologic-http (*output.SumologicHTTPOutput, optional) {#syslogngoutputspec-sumologic-http} Default: - @@ -40,10 +48,18 @@ Default: - Default: - +### elasticsearch (*output.ElasticsearchOutput, optional) {#syslogngoutputspec-elasticsearch} + +Default: - + ### logscale (*output.LogScaleOutput, optional) {#syslogngoutputspec-logscale} Default: - +### splunk_hec_event (*output.SplunkHECOutput, optional) {#syslogngoutputspec-splunk_hec_event} + +Default: - + ## SyslogNGOutput diff --git a/content/docs/configuration/crds/v1beta1/syslogng_types.md b/content/docs/configuration/crds/v1beta1/syslogng_types.md index dc7a8d08a..18a4b959b 100644 --- a/content/docs/configuration/crds/v1beta1/syslogng_types.md +++ b/content/docs/configuration/crds/v1beta1/syslogng_types.md @@ -106,7 +106,13 @@ Default: - ### stats (*Stats, optional) {#globaloptions-stats} -TODO switch to this by default +See the [AxoSyslog Core documentation](https://axoflow.com/docs/axosyslog-core/chapter-global-options/reference-options/#global-option-stats) + +Default: - + +### log_level (*string, optional) {#globaloptions-log_level} + +See the [AxoSyslog Core documentation](https://axoflow.com/docs/axosyslog-core/chapter-global-options/reference-options/#global-options-log-level) Default: - From e0fce88fa112ee8df950e53ef57c94ab54701500 Mon Sep 17 00:00:00 2001 From: Robert Fekete Date: Mon, 18 Sep 2023 13:12:02 +0200 Subject: [PATCH 05/30] Update used images for 4.4 Signed-off-by: Robert Fekete --- content/docs/image-versions.md | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) diff --git a/content/docs/image-versions.md b/content/docs/image-versions.md index 86ffa6c45..73d1009c5 100644 --- a/content/docs/image-versions.md +++ b/content/docs/image-versions.md @@ -5,6 +5,23 @@ weight: 750 Logging operator uses the following image versions. +## Logging operator version 4.4 + +| Image repository | GitHub repository | Version | +| -------- | --- | -- | +| ghcr.io/kube-logging/node-exporter | https://github.com/kube-logging/node-exporter-image | v0.7.1 | +| ghcr.io/kube-logging/config-reloader | https://github.com/kube-logging/config-reloader | v0.0.5 | +| ghcr.io/kube-logging/fluentd-drain-watch | https://github.com/kube-logging/fluentd-drain-watch | v0.2.1 | +| k8s.gcr.io/pause | | 3.2 | +| docker.io/busybox | https://github.com/docker-library/busybox | latest | +| ghcr.io/axoflow/axosyslog | https://github.com/axoflow/axosyslog-docker/ | 4.3.1 | +| docker.io/fluent/fluent-bit | https://github.com/fluent/fluent-bit-docker-image | 2.1.8 | +| ghcr.io/kube-logging/fluentd | https://github.com/kube-logging/fluentd-images | v1.15-ruby3 | +| ghcr.io/kube-logging/syslog-ng-exporter | https://github.com/kube-logging/syslog_ng_exporter | replaced with ghcr.io/axoflow/axosyslog-metrics-exporter | +| ghcr.io/axoflow/axosyslog-metrics-exporter | https://github.com/axoflow/axosyslog-metrics-exporter | 0.0.2 | +| ghcr.io/kube-logging/syslogng-reload | https://github.com/kube-logging/syslogng-reload-image | v1.3.1 | +| banzaicloud/eventrouter | https://github.com/kube-logging/event-router | v0.4.0 | + ## Logging operator version 4.3 | Image repository | GitHub repository | Version | From a1b2f8fb77a23959b26001f6db5b6050313f9a0a Mon Sep 17 00:00:00 2001 From: Robert Fekete Date: Mon, 18 Sep 2023 14:17:45 +0200 Subject: [PATCH 06/30] Adds mertricsprobe to syslog-ng parser plugin docs Signed-off-by: Robert Fekete --- .../plugins/syslog-ng-filters/parser.md | 39 +++++++++++++++++++ 1 file changed, 39 insertions(+) diff --git a/content/docs/configuration/plugins/syslog-ng-filters/parser.md b/content/docs/configuration/plugins/syslog-ng-filters/parser.md index 4371c16a0..32d088a38 100644 --- a/content/docs/configuration/plugins/syslog-ng-filters/parser.md +++ b/content/docs/configuration/plugins/syslog-ng-filters/parser.md @@ -45,6 +45,9 @@ Default: - Default: - +### metrics-probe (*MetricsProbe, optional) {#parser-metrics-probe} + +Counts the messages that pass through the flow, and creates labeled stats counters based on the fields of the passing messages. For details, see the [documentation of the AxoSyslog syslog-ng distribution](https://axoflow.com/docs/axosyslog-core/chapter-parsers/metrics-probe/). ## [Regexp parser](https://axoflow.com/docs/axosyslog-core/chapter-parsers/parser-regexp/) @@ -79,3 +82,39 @@ Default: - Flags to influence the behavior of the [syslog-parser()](https://axoflow.com/docs/axosyslog-core/chapter-parsers/parser-syslog/parser-syslog-options/). For details, see the [syslog-parser() documentation of the AxoSyslog syslog-ng distribution](https://axoflow.com/docs/axosyslog-core/chapter-parsers/parser-syslog/parser-syslog-options/#flags). Default: - + +## MetricsProbe + +Counts the messages that pass through the flow, and creates labeled stats counters based on the fields of the passing messages. For details, see the [documentation of the AxoSyslog syslog-ng distribution](https://axoflow.com/docs/axosyslog-core/chapter-parsers/metrics-probe/). + +{{< highlight yaml>}}SyslogNGFlow +apiVersion: logging.banzaicloud.io/v1beta1 +kind: SyslogNGFlow +metadata: + name: flow-mertrics-probe + namespace: default +spec: + filters: + - parser: + metrics-probe: + key: "flow_events" + labels: + namespace: "${json.kubernetes.namespace_name}"{{< /highlight >}} + +### key (string, optional) {#metricsprobe-key} + +The name of the counter to create. Note that the value of this option is always prefixed with `syslogng_`, so for example `key("my-custom-key")` becomes `syslogng_my-custom-key`. + +Default: - + +### labels (ArrowMap, optional) {#metricsprobe-labels} + +The labels used to create separate counters, based on the fields of the messages processed by `metrics-probe()`. The keys of the map are the name of the label, and the values are syslog-ng templates. + +Default: - + +### level (int, optional) {#metricsprobe-level} + +Sets the stats level of the generated metrics (default 0). + +Default: 0 From b714efc7f714f47ee4f39e7a43c1cb1be95a6ab1 Mon Sep 17 00:00:00 2001 From: Robert Fekete Date: Mon, 18 Sep 2023 16:01:22 +0200 Subject: [PATCH 07/30] Adds draft whatsnew page Signed-off-by: Robert Fekete --- content/docs/whats-new/_index.md | 75 ++++++++++++++++++++++++++++++++ 1 file changed, 75 insertions(+) create mode 100644 content/docs/whats-new/_index.md diff --git a/content/docs/whats-new/_index.md b/content/docs/whats-new/_index.md new file mode 100644 index 000000000..206d6fdff --- /dev/null +++ b/content/docs/whats-new/_index.md @@ -0,0 +1,75 @@ +--- +title: What's new +weight: 50 +--- + +## Version 4.4 + +The following are the highlights and main changes of Logging operator 4.4. For a complete list of changes and bugfixes, see the [Logging operator 4.4 releases page](https://github.com/kube-logging/logging-operator/releases/tag/4.4.0). + +### New syslog-ng features + +When using syslog-ng as the log aggregator, you can now use the following new outputs: + +- [ElasticSearch]({{< relref "/docs/configuration/plugins/syslog-ng-outputs/elasticsearch.md" >}}) +- [MongoDB]({{< relref "/docs/configuration/plugins/syslog-ng-outputs/mongodb.md" >}}) +- [Redis]({{< relref "/docs/configuration/plugins/syslog-ng-outputs/redis.md" >}}) +- [Splunk HEC]({{< relref "/docs/configuration/plugins/syslog-ng-outputs/splunk_hec.md" >}}) +- The [HTTP]({{< relref "/docs/configuration/plugins/syslog-ng-outputs/http.md" >}}) output now supports the `log-fifo-size`, `response-action`, and `timeout` fields. + +You can now use the `metrics-probe()` parser of syslog-ng in syslogNGFLow and SyslogNGClusterFlow. For details, see {{% xref "/docs/configuration/plugins/syslog-ng-filters/parser.md#metricsprobe" %}}. + +### Multitenancy and namespace-based routing + +Logging operator now supports hard multitenancy and namespace-based routing. For an example configuration, see the [sample configuration files](https://github.com/kube-logging/logging-operator/tree/master/config/samples/mulitenant-hard). + +### Forwarder logs + +Logging operator now doesn't forward the logs of the Fluentd and syslog-ng forwarders to the configured outputs to avoid infinite message loops. The logs of Fluentd and syslog-ng are now sent to the standard output of their pods, so you can access them by running `kubectl logs ` + +### Timeout-based configuration checks + +You can now use timeout-based configuration strategies for both syslog-ng and Fluentd. For example: + +```yaml +apiVersion: logging.banzaicloud.io/v1beta1 +kind: Logging +metadata: + name: all-to-file +spec: + configCheck: + strategy: StartWithTimeout + timeoutSeconds: 5 + syslogNG: {} + controlNamespace: default +``` + +```yaml +apiVersion: logging.banzaicloud.io/v1beta1 +kind: Logging +metadata: + name: all-to-file +spec: + configCheck: + strategy: StartWithTimeout + timeoutSeconds: 5 + fluentd: {} + controlNamespace: default +``` + +### Istio support + +By default, Logging operator now doesn't inject the istio sidecar into jobs/individual pods that run to completion. Configuration checkers and Fluentd drainer pods now have `sidecar.istio.io/inject` set to `false` by default. You can configure Fluentd drainer labels in the Logging spec. + +For non-istio users, these changes make no difference, as this label is only used in Istio context. For Istio users, these defaults make Logging operator work out of the box. + +## Other improvements + +- You can now configure the resources of the buffer metrics sidecar. +- You can now rerun failed configuration checks if there is no configcheck pod. +- The [Fluentd ElasticSearch output]({{< relref "/docs/configuration/plugins/outputs/elasticsearch.md" >}}) now supports the [composable index template](https://www.elastic.co/guide/en/elasticsearch/reference/7.13/index-templates.html) format. To use it, set the `use_legacy_template` option to `false`. +- The metrics for the syslog-ng forwarder are now exported using [axosyslog-metrics-exporter](https://github.com/axoflow/axosyslog-metrics-exporter). + +### Image and dependency updates + +For the list of images used in Logging operator, see {{% xref "/docs/image-versions.md" %}}. From 95d2310701ee66fbde18a5a6f7c4b46ac2265c9c Mon Sep 17 00:00:00 2001 From: Robert Fekete Date: Mon, 18 Sep 2023 16:02:04 +0200 Subject: [PATCH 08/30] Minimal docs for elasticsearch composable index template Signed-off-by: Robert Fekete --- content/docs/configuration/plugins/outputs/elasticsearch.md | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/content/docs/configuration/plugins/outputs/elasticsearch.md b/content/docs/configuration/plugins/outputs/elasticsearch.md index 8ac159e0c..bf4287188 100644 --- a/content/docs/configuration/plugins/outputs/elasticsearch.md +++ b/content/docs/configuration/plugins/outputs/elasticsearch.md @@ -575,8 +575,6 @@ Default: - ### use_legacy_template (*bool, optional) {#elasticsearch-use_legacy_template} -Specify wether to use legacy template or not. +If set to true, the output uses the [legacy index template format](https://www.elastic.co/guide/en/elasticsearch/reference/7.13/indices-templates-v1.html). Otherwise, it uses the [composable index template](https://www.elastic.co/guide/en/elasticsearch/reference/7.13/index-templates.html) format. Default: true - - From 3a9ff07df8075d4919f7dfb6d5af74ed99590133 Mon Sep 17 00:00:00 2001 From: Robert Fekete Date: Tue, 19 Sep 2023 15:47:59 +0200 Subject: [PATCH 09/30] Update content/docs/whats-new/_index.md Co-authored-by: Peter Wilcsinszky Signed-off-by: Robert Fekete --- content/docs/whats-new/_index.md | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/content/docs/whats-new/_index.md b/content/docs/whats-new/_index.md index 206d6fdff..b8a6e1957 100644 --- a/content/docs/whats-new/_index.md +++ b/content/docs/whats-new/_index.md @@ -25,7 +25,9 @@ Logging operator now supports hard multitenancy and namespace-based routing. For ### Forwarder logs -Logging operator now doesn't forward the logs of the Fluentd and syslog-ng forwarders to the configured outputs to avoid infinite message loops. The logs of Fluentd and syslog-ng are now sent to the standard output of their pods, so you can access them by running `kubectl logs ` +Fluent-bit now doesn't process the logs of the Fluentd and syslog-ng forwarders by default to avoid infinitely growing message loops. With this Fluentd and syslog-ng logs can simply be accessed by running `kubectl logs ` + +In a future version of the logging operator the logs of the aggregators will also be available for routing to external outputs. ### Timeout-based configuration checks From ed4dbbc13076753efba6aa120a130bb6047f48f6 Mon Sep 17 00:00:00 2001 From: Robert Fekete Date: Tue, 19 Sep 2023 15:48:17 +0200 Subject: [PATCH 10/30] Update content/docs/whats-new/_index.md Co-authored-by: Peter Wilcsinszky Signed-off-by: Robert Fekete --- content/docs/whats-new/_index.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/content/docs/whats-new/_index.md b/content/docs/whats-new/_index.md index b8a6e1957..ac6771a0f 100644 --- a/content/docs/whats-new/_index.md +++ b/content/docs/whats-new/_index.md @@ -61,7 +61,7 @@ spec: ### Istio support -By default, Logging operator now doesn't inject the istio sidecar into jobs/individual pods that run to completion. Configuration checkers and Fluentd drainer pods now have `sidecar.istio.io/inject` set to `false` by default. You can configure Fluentd drainer labels in the Logging spec. +By default, Logging operator adds a label to avoid injecting the Istio sidecar into jobs/individual pods that run to completion. Configuration checkers and Fluentd drainer pods now have `sidecar.istio.io/inject` set to `false` by default. You can configure Fluentd drainer labels in the Logging spec. For non-istio users, these changes make no difference, as this label is only used in Istio context. For Istio users, these defaults make Logging operator work out of the box. From dc93105a78fc108981687bfacfc8232ffa424eaf Mon Sep 17 00:00:00 2001 From: Peter Wilcsinszky Date: Thu, 21 Sep 2023 09:33:16 +0200 Subject: [PATCH 11/30] Fluentd logs for troubleshooting Signed-off-by: Peter Wilcsinszky --- content/docs/examples/es-nginx.md | 4 +++- content/docs/examples/example-s3.md | 4 +++- content/docs/logging-infrastructure/fluentd.md | 14 +++----------- content/docs/operation/troubleshooting/fluentd.md | 9 ++++++--- content/headless/note-fluentd-logs.md | 2 ++ 5 files changed, 17 insertions(+), 16 deletions(-) create mode 100644 content/headless/note-fluentd-logs.md diff --git a/content/docs/examples/es-nginx.md b/content/docs/examples/es-nginx.md index b9a8cf243..10cff82f1 100644 --- a/content/docs/examples/es-nginx.md +++ b/content/docs/examples/es-nginx.md @@ -167,9 +167,11 @@ To validate that the deployment was successful, complete the following steps. 1. Check fluentd logs: ```bash - kubectl exec -ti -n logging default-logging-simple-fluentd-0 -- tail -f /fluentd/log/out + kubectl logs -f -n logging default-logging-simple-fluentd-0 -c fluentd ``` + {{< include-headless "note-fluentd-logs.md" >}} + 1. Use the following command to retrieve the password of the `elastic` user: ```bash diff --git a/content/docs/examples/example-s3.md b/content/docs/examples/example-s3.md index a64bb5178..9c0e124f2 100644 --- a/content/docs/examples/example-s3.md +++ b/content/docs/examples/example-s3.md @@ -133,9 +133,11 @@ Install the Logging operator. Check fluentd logs (errors with AWS credentials should be visible here): ```bash -kubectl exec -ti -n logging default-logging-simple-fluentd-0 -- tail -f /fluentd/log/out +kubectl logs -f -n logging default-logging-simple-fluentd-0 -c fluentd ``` +{{< include-headless "note-fluentd-logs.md" >}} + Check the output. The logs will be available in the bucket on a `path` like: ```bash /logs/default.default-logging-simple-fluentbit-lsdp5.fluent-bit/2019/09/11/201909111432_0.gz diff --git a/content/docs/logging-infrastructure/fluentd.md b/content/docs/logging-infrastructure/fluentd.md index 0cd14085d..77fa5fbf7 100644 --- a/content/docs/logging-infrastructure/fluentd.md +++ b/content/docs/logging-infrastructure/fluentd.md @@ -52,10 +52,9 @@ spec: The following snippet redirects Fluentd's stdout to a file and configures rotation settings. -This is important to avoid Fluentd getting into a ripple effect when there is an error and the error message gets back to the system as a log message, which generates another error, and so on. - -Default settings configured by the operator: +This mechanism was used prior to version 4.4 to avoid Fluent-bit rereading Fluentd's logs and causing an exponentially growing amount of redundant logs. +Example configuration used by the operator in version 4.3 and earlier (keep 10 files, 10M each): ```yaml spec: fluentd: @@ -66,14 +65,7 @@ spec: size: 10485760 ``` -Disabling it and write to stdout (not recommended): - -```yaml -spec: - fluentd: - fluentOutLogrotate: - enabled: false -``` +Fluentd logs are now excluded using the `fluentbit.io/exclude: "true"` annotation. ## Scaling diff --git a/content/docs/operation/troubleshooting/fluentd.md b/content/docs/operation/troubleshooting/fluentd.md index 21fb01c27..8119180cf 100644 --- a/content/docs/operation/troubleshooting/fluentd.md +++ b/content/docs/operation/troubleshooting/fluentd.md @@ -103,14 +103,17 @@ Use the following command to change the log level of Fluentd. `kubectl edit loggings.logging.banzaicloud.io logging-demo` ```yaml -fluentd: - logLevel: debug +spec: + fluentd: + logLevel: debug ``` ## Get Fluentd logs The following command displays the logs of the Fluentd container. -`kubectl exec -it logging-demo-fluentd-0 cat /fluentd/log/out` +`kubectl logs -f logging-demo-fluentd-0 -c fluentd` + +{{< include-headless "note-fluentd-logs.md" >}} > Tip: If the logs include the `error="can't create buffer file ...` error message, Fluentd can’t create the buffer file at the specified location. This can mean for example that the disk is full, the filesystem is read-only, or some other permission error. Check the buffer-related settings of your [Fluentd configuration]({{< relref "/docs/configuration/crds/v1beta1/fluentd_types.md" >}}). diff --git a/content/headless/note-fluentd-logs.md b/content/headless/note-fluentd-logs.md new file mode 100644 index 000000000..42cf368b9 --- /dev/null +++ b/content/headless/note-fluentd-logs.md @@ -0,0 +1,2 @@ +> Fluentd logs were written to the container filesystem up until version 4.3, which has been changed to stdout with 4.4. + See {{% xref "/docs/logging-infrastructure/fluentd.md#fluentoutlogrotate" %}} why this was changed and how you can re-enable it if needed. From 9efd1d02f4ed09fc81b48328dbc26c0095bee37f Mon Sep 17 00:00:00 2001 From: Peter Wilcsinszky Date: Thu, 21 Sep 2023 09:42:48 +0200 Subject: [PATCH 12/30] Note on EOL fluentd images Signed-off-by: Peter Wilcsinszky --- content/docs/whats-new/_index.md | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/content/docs/whats-new/_index.md b/content/docs/whats-new/_index.md index ac6771a0f..d74fe1b79 100644 --- a/content/docs/whats-new/_index.md +++ b/content/docs/whats-new/_index.md @@ -75,3 +75,7 @@ For non-istio users, these changes make no difference, as this label is only use ### Image and dependency updates For the list of images used in Logging operator, see {{% xref "/docs/image-versions.md" %}}. + +Fluentd images with versions `v1.14` and `v1.15` are now EOL due to the fact they are based on ruby 2.7 which is EOL as well. + +The currently supported image is [v1.15-ruby3](https://github.com/kube-logging/fluentd-images/tree/main/v1.15-ruby3) and build configuration for [https://github.com/kube-logging/fluentd-images/tree/main/v1.15-staging](v1.15-staging) is available for staging experimental changes. From fb0c20224e552011b4a9a689a6a3de72d298064d Mon Sep 17 00:00:00 2001 From: Robert Fekete Date: Thu, 21 Sep 2023 09:47:10 +0200 Subject: [PATCH 13/30] Style fixes Signed-off-by: Robert Fekete --- content/docs/operation/troubleshooting/fluentd.md | 5 ++++- content/headless/note-fluentd-logs.md | 2 +- 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/content/docs/operation/troubleshooting/fluentd.md b/content/docs/operation/troubleshooting/fluentd.md index 8119180cf..fbb9446d0 100644 --- a/content/docs/operation/troubleshooting/fluentd.md +++ b/content/docs/operation/troubleshooting/fluentd.md @@ -111,7 +111,10 @@ spec: ## Get Fluentd logs The following command displays the logs of the Fluentd container. -`kubectl logs -f logging-demo-fluentd-0 -c fluentd` + +```bash +kubectl logs -f logging-demo-fluentd-0 -c fluentd +``` {{< include-headless "note-fluentd-logs.md" >}} diff --git a/content/headless/note-fluentd-logs.md b/content/headless/note-fluentd-logs.md index 42cf368b9..649be7938 100644 --- a/content/headless/note-fluentd-logs.md +++ b/content/headless/note-fluentd-logs.md @@ -1,2 +1,2 @@ -> Fluentd logs were written to the container filesystem up until version 4.3, which has been changed to stdout with 4.4. +> Fluentd logs were written to the container filesystem up until Logging operator version 4.3, which has been changed to stdout with 4.4. See {{% xref "/docs/logging-infrastructure/fluentd.md#fluentoutlogrotate" %}} why this was changed and how you can re-enable it if needed. From f6e58cd014999ba41b2e204d317e863433156e36 Mon Sep 17 00:00:00 2001 From: Robert Fekete Date: Thu, 21 Sep 2023 09:47:39 +0200 Subject: [PATCH 14/30] Include kinesis firehose output fixes Signed-off-by: Robert Fekete --- .../plugins/outputs/kinesis_firehose.md | 46 +++++++++---------- 1 file changed, 23 insertions(+), 23 deletions(-) diff --git a/content/docs/configuration/plugins/outputs/kinesis_firehose.md b/content/docs/configuration/plugins/outputs/kinesis_firehose.md index 07287b1c4..2cfc9b78a 100644 --- a/content/docs/configuration/plugins/outputs/kinesis_firehose.md +++ b/content/docs/configuration/plugins/outputs/kinesis_firehose.md @@ -19,101 +19,101 @@ generated_file: true ``` ## Configuration -## KinesisStream +## KinesisFirehose -Send your logs to a Kinesis Stream +Send your logs to a Kinesis Firehose -### delivery_stream_name (string, required) {#kinesisstream-delivery_stream_name} +### delivery_stream_name (string, required) {#kinesisfirehose-delivery_stream_name} Name of the delivery stream to put data. Default: - -### append_new_line (*bool, optional) {#kinesisstream-append_new_line} +### append_new_line (*bool, optional) {#kinesisfirehose-append_new_line} If it is enabled, the plugin adds new line character (\n) to each serialized record. Before appending \n, plugin calls chomp and removes separator from the end of each record as chomp_record is true. Therefore, you don't need to enable chomp_record option when you use kinesis_firehose output with default configuration (append_new_line is true). If you want to set append_new_line false, you can choose chomp_record false (default) or true (compatible format with plugin v2). (Default:true) Default: - -### aws_key_id (*secret.Secret, optional) {#kinesisstream-aws_key_id} +### aws_key_id (*secret.Secret, optional) {#kinesisfirehose-aws_key_id} AWS access key id. This parameter is required when your agent is not running on EC2 instance with an IAM Role. Default: - -### aws_sec_key (*secret.Secret, optional) {#kinesisstream-aws_sec_key} +### aws_sec_key (*secret.Secret, optional) {#kinesisfirehose-aws_sec_key} AWS secret key. This parameter is required when your agent is not running on EC2 instance with an IAM Role. Default: - -### aws_ses_token (*secret.Secret, optional) {#kinesisstream-aws_ses_token} +### aws_ses_token (*secret.Secret, optional) {#kinesisfirehose-aws_ses_token} AWS session token. This parameter is optional, but can be provided if using MFA or temporary credentials when your agent is not running on EC2 instance with an IAM Role. Default: - -### aws_iam_retries (int, optional) {#kinesisstream-aws_iam_retries} +### aws_iam_retries (int, optional) {#kinesisfirehose-aws_iam_retries} The number of attempts to make (with exponential backoff) when loading instance profile credentials from the EC2 metadata service using an IAM role. Defaults to 5 retries. Default: - -### assume_role_credentials (*KinesisFirehoseAssumeRoleCredentials, optional) {#kinesisstream-assume_role_credentials} +### assume_role_credentials (*KinesisFirehoseAssumeRoleCredentials, optional) {#kinesisfirehose-assume_role_credentials} Typically, you can use AssumeRole for cross-account access or federation. Default: - -### process_credentials (*KinesisFirehoseProcessCredentials, optional) {#kinesisstream-process_credentials} +### process_credentials (*KinesisFirehoseProcessCredentials, optional) {#kinesisfirehose-process_credentials} This loads AWS access credentials from an external process. Default: - -### region (string, optional) {#kinesisstream-region} +### region (string, optional) {#kinesisfirehose-region} AWS region of your stream. It should be in form like us-east-1, us-west-2. Default nil, which means try to find from environment variable AWS_REGION. Default: - -### retries_on_batch_request (int, optional) {#kinesisstream-retries_on_batch_request} +### retries_on_batch_request (int, optional) {#kinesisfirehose-retries_on_batch_request} The plugin will put multiple records to Amazon Kinesis Data Streams in batches using PutRecords. A set of records in a batch may fail for reasons documented in the Kinesis Service API Reference for PutRecords. Failed records will be retried retries_on_batch_request times Default: - -### reset_backoff_if_success (bool, optional) {#kinesisstream-reset_backoff_if_success} +### reset_backoff_if_success (bool, optional) {#kinesisfirehose-reset_backoff_if_success} Boolean, default true. If enabled, when after retrying, the next retrying checks the number of succeeded records on the former batch request and reset exponential backoff if there is any success. Because batch request could be composed by requests across shards, simple exponential backoff for the batch request wouldn't work some cases. Default: - -### batch_request_max_count (int, optional) {#kinesisstream-batch_request_max_count} +### batch_request_max_count (int, optional) {#kinesisfirehose-batch_request_max_count} Integer, default 500. The number of max count of making batch request from record chunk. It can't exceed the default value because it's API limit. Default: - -### batch_request_max_size (int, optional) {#kinesisstream-batch_request_max_size} +### batch_request_max_size (int, optional) {#kinesisfirehose-batch_request_max_size} Integer. The number of max size of making batch request from record chunk. It can't exceed the default value because it's API limit. Default: - -### format (*Format, optional) {#kinesisstream-format} +### format (*Format, optional) {#kinesisfirehose-format} [Format](../format/) Default: - -### buffer (*Buffer, optional) {#kinesisstream-buffer} +### buffer (*Buffer, optional) {#kinesisfirehose-buffer} [Buffer](../buffer/) Default: - -### slow_flush_log_threshold (string, optional) {#kinesisstream-slow_flush_log_threshold} +### slow_flush_log_threshold (string, optional) {#kinesisfirehose-slow_flush_log_threshold} The threshold for chunk flush performance check. Parameter type is float, not time, default: 20.0 (seconds) If chunk flush takes longer time than this threshold, fluentd logs warning message and increases metric fluentd_output_status_slow_flush_count. @@ -124,31 +124,31 @@ Default: - assume_role_credentials -### role_arn (string, required) {#assume role-credentials-role_arn} +### role_arn (string, required) {#assume role credentials-role_arn} The Amazon Resource Name (ARN) of the role to assume Default: - -### role_session_name (string, required) {#assume role-credentials-role_session_name} +### role_session_name (string, required) {#assume-role-credentials-role_session_name} An identifier for the assumed role session Default: - -### policy (string, optional) {#assume role-credentials-policy} +### policy (string, optional) {#assume-role-credentials-policy} An IAM policy in JSON format Default: - -### duration_seconds (string, optional) {#assume role-credentials-duration_seconds} +### duration_seconds (string, optional) {#assume-role-credentials-duration_seconds} The duration, in seconds, of the role session (900-3600) Default: - -### external_id (string, optional) {#assume role-credentials-external_id} +### external_id (string, optional) {#assume-role-credentials-external_id} A unique identifier that is used by third parties when assuming roles in their customers' accounts. From 79bd1aa1f702ea95670cf6b210c879c6d0c5774a Mon Sep 17 00:00:00 2001 From: Robert Fekete Date: Thu, 21 Sep 2023 09:53:08 +0200 Subject: [PATCH 15/30] Adds loki output to syslog-ng outputs Signed-off-by: Robert Fekete --- .../crds/v1beta1/syslogng_output_types.md | 4 + .../plugins/syslog-ng-outputs/loki.md | 83 +++++++++++++++++++ 2 files changed, 87 insertions(+) create mode 100644 content/docs/configuration/plugins/syslog-ng-outputs/loki.md diff --git a/content/docs/configuration/crds/v1beta1/syslogng_output_types.md b/content/docs/configuration/crds/v1beta1/syslogng_output_types.md index e9128cfff..557709ef5 100644 --- a/content/docs/configuration/crds/v1beta1/syslogng_output_types.md +++ b/content/docs/configuration/crds/v1beta1/syslogng_output_types.md @@ -60,6 +60,10 @@ Default: - Default: - +### loki (*output.LokiOutput, optional) {#syslogngoutputspec-loki} + +Default: - + ## SyslogNGOutput diff --git a/content/docs/configuration/plugins/syslog-ng-outputs/loki.md b/content/docs/configuration/plugins/syslog-ng-outputs/loki.md new file mode 100644 index 000000000..332a2bdd3 --- /dev/null +++ b/content/docs/configuration/plugins/syslog-ng-outputs/loki.md @@ -0,0 +1,83 @@ +--- +title: Loki +weight: 200 +generated_file: true +--- + +Sends messages to Grafana Loki over gRPC, based on the [Loki destination of AxoSyslog Core](https://axoflow.com/docs/axosyslog-core/chapter-destinations/syslog-ng-with-loki/). + +## Configuration + +### labels (filter.ArrowMap, optional) {#lokioutput-labels} + +Using the Labels map, Kubernetes label to Loki label mapping can be configured. Example: `{"app" : "$PROGRAM"}` + +Default: - + +### url (string, optional) {#lokioutput-url} + +Specifies the hostname or IP address and optionally the port number of the web service that can receive log data via HTTP. Use a colon (:) after the address to specify the port number of the server. For example: `http://127.0.0.1:8000` + +Default: - + +### time_reopen (int, optional) {#lokioutput-time_reopen} + +The time to wait in seconds before a dead connection is reestablished. + +Default: 60 + +### disk_buffer (*DiskBuffer, optional) {#lokioutput-disk_buffer} + +This option enables putting outgoing messages into the disk buffer of the destination to avoid message loss in case of a system failure on the destination side. For details, see the [Syslog-ng DiskBuffer options](../disk_buffer/). + +Default: false + +### batch-lines (int, optional) {#lokioutput-batch-lines} + +Description: Specifies how many lines are flushed to a destination in one batch. The syslog-ng OSE application waits for this number of lines to accumulate and sends them off in a single batch. Increasing this number increases throughput as more messages are sent in a single batch, but also increases message latency. For example, if you set batch-lines() to 100, syslog-ng OSE waits for 100 messages. + +Default: - + +### batch-timeout (int, optional) {#lokioutput-batch-timeout} + +Description: Specifies the time syslog-ng OSE waits for lines to accumulate in the output buffer. The syslog-ng OSE application sends batches to the destinations evenly. The timer starts when the first message arrives to the buffer, so if only few messages arrive, syslog-ng OSE sends messages to the destination at most once every batch-timeout() milliseconds. + +Default: - + +### retries (int, optional) {#lokioutput-retries} + +The number of times syslog-ng OSE attempts to send a message to this destination. If syslog-ng OSE could not send a message, it will try again until the number of attempts reaches retries, then drops the message. + +Default: - + +### workers (int, optional) {#lokioutput-workers} + +Specifies the number of worker threads (at least 1) that syslog-ng OSE uses to send messages to the server. Increasing the number of worker threads can drastically improve the performance of the destination. + +Default: - + +### persist_name (string, optional) {#lokioutput-persist_name} + +If you receive the following error message during AxoSyslog startup, set the persist-name() option of the duplicate drivers: `Error checking the uniqueness of the persist names, please override it with persist-name option. Shutting down.` See [syslog-ng docs](https://axoflow.com/docs/axosyslog-core/chapter-destinations/configuring-destinations-http-nonjava/reference-destination-http-nonjava/#persist-name) for more information. + +Default: - + +### log-fifo-size (int, optional) {#lokioutput-log-fifo-size} + +The number of messages that the output queue can store. + +Default: - + +### timestamp (string, optional) {#lokioutput-timestamp} + +The timestamp that will be applied to the outgoing messages (possible values: current|received|msg default: current). Loki does not accept events, in which the timestamp is not monotonically increasing. + +Default: - + +### template (string, optional) {#lokioutput-template} + +Template for customizing the log message format. + +Default: - + + From 3434d1a1545c9c7fb83526c7e31f1a4e8969c31c Mon Sep 17 00:00:00 2001 From: Robert Fekete Date: Thu, 21 Sep 2023 10:18:34 +0200 Subject: [PATCH 16/30] Minor whatsnew corrections Signed-off-by: Robert Fekete --- content/docs/whats-new/_index.md | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/content/docs/whats-new/_index.md b/content/docs/whats-new/_index.md index d74fe1b79..0865ecf47 100644 --- a/content/docs/whats-new/_index.md +++ b/content/docs/whats-new/_index.md @@ -12,6 +12,7 @@ The following are the highlights and main changes of Logging operator 4.4. For a When using syslog-ng as the log aggregator, you can now use the following new outputs: - [ElasticSearch]({{< relref "/docs/configuration/plugins/syslog-ng-outputs/elasticsearch.md" >}}) +- [Grafana Loki]({{< relref "/docs/configuration/plugins/syslog-ng-outputs/loki.md" >}}) - [MongoDB]({{< relref "/docs/configuration/plugins/syslog-ng-outputs/mongodb.md" >}}) - [Redis]({{< relref "/docs/configuration/plugins/syslog-ng-outputs/redis.md" >}}) - [Splunk HEC]({{< relref "/docs/configuration/plugins/syslog-ng-outputs/splunk_hec.md" >}}) @@ -25,9 +26,9 @@ Logging operator now supports hard multitenancy and namespace-based routing. For ### Forwarder logs -Fluent-bit now doesn't process the logs of the Fluentd and syslog-ng forwarders by default to avoid infinitely growing message loops. With this Fluentd and syslog-ng logs can simply be accessed by running `kubectl logs ` +Fluent-bit now doesn't process the logs of the Fluentd and syslog-ng forwarders by default to avoid infinitely growing message loops. With this change, you can access Fluentd and syslog-ng logs simply by running `kubectl logs ` -In a future version of the logging operator the logs of the aggregators will also be available for routing to external outputs. +In a future Logging operator version the logs of the aggregators will also be available for routing to external outputs. ### Timeout-based configuration checks From 527937ab0bf08fe7305bc4e62d6297211fcf4acd Mon Sep 17 00:00:00 2001 From: Robert Fekete Date: Thu, 21 Sep 2023 10:30:50 +0200 Subject: [PATCH 17/30] Link fix Signed-off-by: Robert Fekete --- content/docs/whats-new/_index.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/content/docs/whats-new/_index.md b/content/docs/whats-new/_index.md index 0865ecf47..24f5e44f2 100644 --- a/content/docs/whats-new/_index.md +++ b/content/docs/whats-new/_index.md @@ -79,4 +79,4 @@ For the list of images used in Logging operator, see {{% xref "/docs/image-versi Fluentd images with versions `v1.14` and `v1.15` are now EOL due to the fact they are based on ruby 2.7 which is EOL as well. -The currently supported image is [v1.15-ruby3](https://github.com/kube-logging/fluentd-images/tree/main/v1.15-ruby3) and build configuration for [https://github.com/kube-logging/fluentd-images/tree/main/v1.15-staging](v1.15-staging) is available for staging experimental changes. +The currently supported image is [v1.15-ruby3](https://github.com/kube-logging/fluentd-images/tree/main/v1.15-ruby3) and build configuration for [v1.15-staging](https://github.com/kube-logging/fluentd-images/tree/main/v1.15-staging) is available for staging experimental changes. From 6a6df799a6034efd86ca8c35387b89ee328ba9cb Mon Sep 17 00:00:00 2001 From: Robert Fekete Date: Fri, 22 Sep 2023 14:33:08 +0200 Subject: [PATCH 18/30] Multitenancy basic draft Signed-off-by: Robert Fekete --- content/docs/examples/multitenancy.md | 56 +++++++++++++++++++++++++++ 1 file changed, 56 insertions(+) create mode 100644 content/docs/examples/multitenancy.md diff --git a/content/docs/examples/multitenancy.md b/content/docs/examples/multitenancy.md new file mode 100644 index 000000000..996752e14 --- /dev/null +++ b/content/docs/examples/multitenancy.md @@ -0,0 +1,56 @@ +--- +title: Namespace-based multitenancy +weight: 350 +--- + +Namespace-based multitenancy allows you have multiple tenants (for example, different developer teams) on the same cluster who can configure their own logging resources within their assigned namespaces. + +- These resources are separate from the resources of the other tenants, and +- the configuration checks of Logging operator ensure that errors in the resources of a namespace don't affect other tenants. For example, an error cannot bring down the logging of the entire cluster. + + +## Sample setup + +The following procedure creates two tenants (A and B) and their respective namespaces on a two-node cluster. + +1. If you don't already have a cluster, create one with your provider. For a quick test, you can use a local cluster, for example, using minikube: + + ```bash + minikube start --nodes=2 + ``` + +1. Set labels on the nodes that correspond to your tenants, for example, `tenant-a` and `tenant-b`. + + ```shell + kubectl label node minikube tenant=tenant-a + kubectl label node minikube-m02 tenant=tenant-b + ``` + +1. Apply the sample resources from the [project repository](https://github.com/kube-logging/logging-operator/tree/master/config/samples/mulitenant-hard/logging). These create namespaces, flows, and sample outputs for the two tenants. + + + ```bash + kubectl apply -f https://github.com/kube-logging/logging-operator/tree/master/config/samples/mulitenant-hard/logging + ``` + + +1. (Optional) Install a sample log generator application to the respective namespaces of your tenants. For example: + + ```bash + helm upgrade --install --namespace a --create-namespace --set "nodeSelector.tenant=tenant-a" log-generator oci://ghcr.io/kube-logging/helm-charts/log-generator + helm upgrade --install --namespace b --create-namespace --set "nodeSelector.tenant=tenant-b" log-generator oci://ghcr.io/kube-logging/helm-charts/log-generator + ``` + +1. Check that your pods are up and running by running `kubectl get pods` + + If you have followed the examples, the output should look like: + + ```bash + NAMESPACE NAME READY STATUS RESTARTS AGE IP NODE NOMINATED NODE READINESS GATES + a-control a-fluentbit-2997s 1/1 Running 0 9m15s 10.244.0.5 minikube + a-control a-fluentd-0 2/2 Running 0 9m15s 10.244.0.6 minikube + a log-generator-6cfb45c684-kbzk4 1/1 Running 0 11m 10.244.0.3 minikube + b-control b-fluentbit-9bvbn 1/1 Running 0 7m30s 10.244.1.7 minikube-m02 + b-control b-fluentd-0 2/2 Running 0 7m29s 10.244.1.8 minikube-m02 + b log-generator-7b95b6fdc5-62bnr 1/1 Running 0 11m 10.244.1.3 minikube-m02 + ``` From 942000369b9d7ec0513032293647d716c88b6f34 Mon Sep 17 00:00:00 2001 From: Robert Fekete Date: Fri, 22 Sep 2023 14:58:29 +0200 Subject: [PATCH 19/30] Update content/docs/image-versions.md Co-authored-by: Peter Wilcsinszky Signed-off-by: Robert Fekete --- content/docs/image-versions.md | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/content/docs/image-versions.md b/content/docs/image-versions.md index 73d1009c5..082f9d7e9 100644 --- a/content/docs/image-versions.md +++ b/content/docs/image-versions.md @@ -22,6 +22,13 @@ Logging operator uses the following image versions. | ghcr.io/kube-logging/syslogng-reload | https://github.com/kube-logging/syslogng-reload-image | v1.3.1 | | banzaicloud/eventrouter | https://github.com/kube-logging/event-router | v0.4.0 | +The following images are new replaced + +| Image used in 4.3 | New image in 4.4 | +| -------- | --- | -- | +| banzaicloud/eventrouter | ghcr.io/kube-logging/eventrouter | +| ghcr.io/kube-logging/syslog-ng-exporter | ghcr.io/axoflow/axosyslog-metrics-exporter | + ## Logging operator version 4.3 | Image repository | GitHub repository | Version | From 35c59a6702cf089f3a62fb8b362e51935552ed7b Mon Sep 17 00:00:00 2001 From: Robert Fekete Date: Fri, 22 Sep 2023 14:58:46 +0200 Subject: [PATCH 20/30] Update content/docs/image-versions.md Co-authored-by: Peter Wilcsinszky Signed-off-by: Robert Fekete --- content/docs/image-versions.md | 1 - 1 file changed, 1 deletion(-) diff --git a/content/docs/image-versions.md b/content/docs/image-versions.md index 082f9d7e9..23223393e 100644 --- a/content/docs/image-versions.md +++ b/content/docs/image-versions.md @@ -17,7 +17,6 @@ Logging operator uses the following image versions. | ghcr.io/axoflow/axosyslog | https://github.com/axoflow/axosyslog-docker/ | 4.3.1 | | docker.io/fluent/fluent-bit | https://github.com/fluent/fluent-bit-docker-image | 2.1.8 | | ghcr.io/kube-logging/fluentd | https://github.com/kube-logging/fluentd-images | v1.15-ruby3 | -| ghcr.io/kube-logging/syslog-ng-exporter | https://github.com/kube-logging/syslog_ng_exporter | replaced with ghcr.io/axoflow/axosyslog-metrics-exporter | | ghcr.io/axoflow/axosyslog-metrics-exporter | https://github.com/axoflow/axosyslog-metrics-exporter | 0.0.2 | | ghcr.io/kube-logging/syslogng-reload | https://github.com/kube-logging/syslogng-reload-image | v1.3.1 | | banzaicloud/eventrouter | https://github.com/kube-logging/event-router | v0.4.0 | From 413499efe6ecf81032755c572242d4120cf6e061 Mon Sep 17 00:00:00 2001 From: Robert Fekete Date: Fri, 22 Sep 2023 14:58:58 +0200 Subject: [PATCH 21/30] Update content/docs/image-versions.md Co-authored-by: Peter Wilcsinszky Signed-off-by: Robert Fekete --- content/docs/image-versions.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/content/docs/image-versions.md b/content/docs/image-versions.md index 23223393e..8fe1f6e35 100644 --- a/content/docs/image-versions.md +++ b/content/docs/image-versions.md @@ -19,7 +19,7 @@ Logging operator uses the following image versions. | ghcr.io/kube-logging/fluentd | https://github.com/kube-logging/fluentd-images | v1.15-ruby3 | | ghcr.io/axoflow/axosyslog-metrics-exporter | https://github.com/axoflow/axosyslog-metrics-exporter | 0.0.2 | | ghcr.io/kube-logging/syslogng-reload | https://github.com/kube-logging/syslogng-reload-image | v1.3.1 | -| banzaicloud/eventrouter | https://github.com/kube-logging/event-router | v0.4.0 | +| ghcr.io/kube-logging/eventrouter | https://github.com/kube-logging/eventrouter | v0.4.0 | The following images are new replaced From 4bc5bb3250a576e0be9806a1d82722629f2bfef1 Mon Sep 17 00:00:00 2001 From: Robert Fekete Date: Fri, 22 Sep 2023 15:02:30 +0200 Subject: [PATCH 22/30] Indentation fixes in the redis output docs --- .../plugins/syslog-ng-outputs/redis.md | 32 ++++++++----------- 1 file changed, 14 insertions(+), 18 deletions(-) diff --git a/content/docs/configuration/plugins/syslog-ng-outputs/redis.md b/content/docs/configuration/plugins/syslog-ng-outputs/redis.md index aa532566e..2d8212b12 100644 --- a/content/docs/configuration/plugins/syslog-ng-outputs/redis.md +++ b/content/docs/configuration/plugins/syslog-ng-outputs/redis.md @@ -8,24 +8,20 @@ Based on the [Redis destination of AxoSyslog core](https://axoflow.com/docs/axos ## Example - {{< highlight yaml >}} - apiVersion: logging.banzaicloud.io/v1beta1 - kind: SyslogNGOutput - metadata: - - name: redis - namespace: default - - spec: - - redis: - host: 127.0.0.1 - port: 6379 - retries: 3 - throttle: 0 - time-reopen: 60 - workers: 1 - +{{< highlight yaml >}} +apiVersion: logging.banzaicloud.io/v1beta1 +kind: SyslogNGOutput +metadata: + name: redis + namespace: default +spec: + redis: + host: 127.0.0.1 + port: 6379 + retries: 3 + throttle: 0 + time-reopen: 60 + workers: 1 {{}} ## Configuration From bb7b201d3d40d15fe6660ed575cad3ca9098a8f8 Mon Sep 17 00:00:00 2001 From: Robert Fekete Date: Fri, 22 Sep 2023 15:02:50 +0200 Subject: [PATCH 23/30] Move multitenancy example to the end of the list --- content/docs/examples/multitenancy.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/content/docs/examples/multitenancy.md b/content/docs/examples/multitenancy.md index 996752e14..120b2f0ab 100644 --- a/content/docs/examples/multitenancy.md +++ b/content/docs/examples/multitenancy.md @@ -1,6 +1,6 @@ --- title: Namespace-based multitenancy -weight: 350 +weight: 600 --- Namespace-based multitenancy allows you have multiple tenants (for example, different developer teams) on the same cluster who can configure their own logging resources within their assigned namespaces. From 9b88263c359579e9193b58f1989d83b3bda2719b Mon Sep 17 00:00:00 2001 From: Peter Wilcsinszky Date: Fri, 29 Sep 2023 16:59:04 +0200 Subject: [PATCH 24/30] nodegroup multitenancy example fixes Signed-off-by: Peter Wilcsinszky --- content/docs/examples/multitenancy.md | 45 ++++++++++++++++----------- 1 file changed, 26 insertions(+), 19 deletions(-) diff --git a/content/docs/examples/multitenancy.md b/content/docs/examples/multitenancy.md index 120b2f0ab..7035b97ca 100644 --- a/content/docs/examples/multitenancy.md +++ b/content/docs/examples/multitenancy.md @@ -1,13 +1,10 @@ --- -title: Namespace-based multitenancy +title: Nodegroup-based multitenancy weight: 600 --- -Namespace-based multitenancy allows you have multiple tenants (for example, different developer teams) on the same cluster who can configure their own logging resources within their assigned namespaces. - -- These resources are separate from the resources of the other tenants, and -- the configuration checks of Logging operator ensure that errors in the resources of a namespace don't affect other tenants. For example, an error cannot bring down the logging of the entire cluster. - +Nodegroup-based multitenancy allows you to have multiple tenants (for example, different developer teams or customer environments) on the same cluster who can configure their own logging resources within their assigned namespaces residing on different node groups. +These resources are isolated from the resources of the other tenants so the configuration issues and performance characteristics of one tenant doesn't affect the others. ## Sample setup @@ -21,18 +18,18 @@ The following procedure creates two tenants (A and B) and their respective names 1. Set labels on the nodes that correspond to your tenants, for example, `tenant-a` and `tenant-b`. - ```shell + ```bash kubectl label node minikube tenant=tenant-a kubectl label node minikube-m02 tenant=tenant-b ``` -1. Apply the sample resources from the [project repository](https://github.com/kube-logging/logging-operator/tree/master/config/samples/mulitenant-hard/logging). These create namespaces, flows, and sample outputs for the two tenants. - +1. Install the logging operator ```bash - kubectl apply -f https://github.com/kube-logging/logging-operator/tree/master/config/samples/mulitenant-hard/logging + helm install logging-operator oci://ghcr.io/kube-logging/helm-charts/logging-operator ``` - + +1. Apply the sample resources from the [project repository](https://github.com/kube-logging/logging-operator/tree/master/config/samples/mulitenant-hard/logging). These create namespaces, flows, and sample outputs for the two tenants. 1. (Optional) Install a sample log generator application to the respective namespaces of your tenants. For example: @@ -41,16 +38,26 @@ The following procedure creates two tenants (A and B) and their respective names helm upgrade --install --namespace b --create-namespace --set "nodeSelector.tenant=tenant-b" log-generator oci://ghcr.io/kube-logging/helm-charts/log-generator ``` -1. Check that your pods are up and running by running `kubectl get pods` +1. Check that your pods are up and running by running `kubectl get pods -A` If you have followed the examples, the output should look like: ```bash - NAMESPACE NAME READY STATUS RESTARTS AGE IP NODE NOMINATED NODE READINESS GATES - a-control a-fluentbit-2997s 1/1 Running 0 9m15s 10.244.0.5 minikube - a-control a-fluentd-0 2/2 Running 0 9m15s 10.244.0.6 minikube - a log-generator-6cfb45c684-kbzk4 1/1 Running 0 11m 10.244.0.3 minikube - b-control b-fluentbit-9bvbn 1/1 Running 0 7m30s 10.244.1.7 minikube-m02 - b-control b-fluentd-0 2/2 Running 0 7m29s 10.244.1.8 minikube-m02 - b log-generator-7b95b6fdc5-62bnr 1/1 Running 0 11m 10.244.1.3 minikube-m02 + NAMESPACE NAME READY STATUS RESTARTS AGE + a-control a-fluentbit-4tqzg 1/1 Running 0 9m29s + a-control a-fluentd-0 2/2 Running 0 4m48s + a log-generator-6cfb45c684-q6fl6 1/1 Running 0 3m25s + b-control b-fluentbit-qmf58 1/1 Running 0 9m20s + b-control b-fluentd-0 2/2 Running 0 9m16s + b log-generator-7b95b6fdc5-cshh7 1/1 Running 0 8m49s + default logging-operator-bbd66bb7d-qvsmg 1/1 Running 0 35m + infra test-receiver-7c45f9cd77-whvlv 1/1 Running 0 53m + ``` + +1. Check logs coming from both tenants `kubectl logs -f -n infra svc/test-receiver` + + Expected output should show logs from both tenants + ```bash + [0] tenant_a: [[1695999280.157810965, {}], {"log"=>"15.238.250.48 - - [29/Sep/2023:14:54:38 +0000] "PUT /pro... + [0] tenant_b: [[1695999280.160868923, {}], {"log"=>"252.201.89.36 - - [29/Sep/2023:14:54:33 +0000] "POST /bl... ``` From c275f78cf371242abcd90e8d84b6093df532e48d Mon Sep 17 00:00:00 2001 From: Peter Wilcsinszky Date: Fri, 29 Sep 2023 17:18:45 +0200 Subject: [PATCH 25/30] Update whatsnew with latest multitenant stuff (links to project for now) Signed-off-by: Peter Wilcsinszky --- content/docs/whats-new/_index.md | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/content/docs/whats-new/_index.md b/content/docs/whats-new/_index.md index 24f5e44f2..a739910b0 100644 --- a/content/docs/whats-new/_index.md +++ b/content/docs/whats-new/_index.md @@ -20,9 +20,16 @@ When using syslog-ng as the log aggregator, you can now use the following new ou You can now use the `metrics-probe()` parser of syslog-ng in syslogNGFLow and SyslogNGClusterFlow. For details, see {{% xref "/docs/configuration/plugins/syslog-ng-filters/parser.md#metricsprobe" %}}. -### Multitenancy and namespace-based routing +### Multitenancy with namespace-based routing -Logging operator now supports hard multitenancy and namespace-based routing. For an example configuration, see the [sample configuration files](https://github.com/kube-logging/logging-operator/tree/master/config/samples/mulitenant-hard). +Logging operator now supports namespace based routing for efficient aggregator-level multi-tenancy. + +In the project repository you can: +- find an [overview about multitenancy](https://github.com/kube-logging/logging-operator/blob/master/docs/multi-tenancy.md). +- find more detailed information about the new [LoggingRoute](https://github.com/kube-logging/logging-operator/blob/master/docs/logging-route.md) resource that enables this new behaviour. +- find a [simple example](https://github.com/kube-logging/logging-operator/tree/master/config/samples/multitenant-routing) to demonstrate the new behaviour + +On a side note, nodegroup level isolation for hard multitenancy is also supported, see the {{% xref "docs/examples/multitenancy.md" %}} example. ### Forwarder logs From e70f10c56c3b46428d28918269dc27a0efc6fe6e Mon Sep 17 00:00:00 2001 From: Robert Fekete Date: Sat, 30 Sep 2023 09:25:21 +0200 Subject: [PATCH 26/30] Adds generated LoggingRoute spec docs Signed-off-by: Robert Fekete --- .../crds/v1beta1/loggingroute_types.md | 106 ++++++++++++++++++ 1 file changed, 106 insertions(+) create mode 100644 content/docs/configuration/crds/v1beta1/loggingroute_types.md diff --git a/content/docs/configuration/crds/v1beta1/loggingroute_types.md b/content/docs/configuration/crds/v1beta1/loggingroute_types.md new file mode 100644 index 000000000..a2d865c01 --- /dev/null +++ b/content/docs/configuration/crds/v1beta1/loggingroute_types.md @@ -0,0 +1,106 @@ +--- +title: LoggingRouteSpec +weight: 200 +generated_file: true +--- + +## LoggingRouteSpec + +LoggingRouteSpec defines the desired state of LoggingRoute + +### source (string, required) {#loggingroutespec-source} + +Source identifies the logging that this policy applies to + +Default: - + +### targets (metav1.LabelSelector, required) {#loggingroutespec-targets} + +Targets refers to the list of logging resources specified by a label selector to forward logs to. Filtering of namespaces will happen based on the watchNamespaces and watchNamespaceSelector fields of the target logging resource + +Default: - + + +## LoggingRouteStatus + +LoggingRouteStatus defines the actual state of the LoggingRoute + +### tenants ([]Tenant, optional) {#loggingroutestatus-tenants} + +Enumerate all loggings with all the destination namespaces expanded + +Default: - + +### problems ([]string, optional) {#loggingroutestatus-problems} + +Enumerate problems that prohibits this route to take effect and populate the tenants field + +Default: - + +### problemsCount (int, optional) {#loggingroutestatus-problemscount} + +Summarize the number of problems for the CLI output + +Default: - + +### notices ([]string, optional) {#loggingroutestatus-notices} + +Enumerate non-blocker issues the user should pay attention to + +Default: - + +### noticesCount (int, optional) {#loggingroutestatus-noticescount} + +Summarize the number of notices for the CLI output + +Default: - + + +## Tenant + +### name (string, required) {#tenant-name} + +Default: - + +### namespaces ([]string, optional) {#tenant-namespaces} + +Default: - + + +## LoggingRoute + +LoggingRoute (experimental) +Connects a log collector with log aggregators from other logging domains and routes relevant logs based on watch namespaces + +### (metav1.TypeMeta, required) {#loggingroute-} + +Default: - + +### metadata (metav1.ObjectMeta, optional) {#loggingroute-metadata} + +Default: - + +### spec (LoggingRouteSpec, optional) {#loggingroute-spec} + +Default: - + +### status (LoggingRouteStatus, optional) {#loggingroute-status} + +Default: - + + +## LoggingRouteList + +### (metav1.TypeMeta, required) {#loggingroutelist-} + +Default: - + +### metadata (metav1.ListMeta, optional) {#loggingroutelist-metadata} + +Default: - + +### items ([]LoggingRoute, required) {#loggingroutelist-items} + +Default: - + + From ee0dee9d1e9cf0181a8b93d72806ceb76dcd04d5 Mon Sep 17 00:00:00 2001 From: Robert Fekete Date: Sat, 30 Sep 2023 09:33:07 +0200 Subject: [PATCH 27/30] Adds generated S3 docs Signed-off-by: Robert Fekete --- .../plugins/syslog-ng-outputs/s3.md | 159 ++++++++++++++++++ content/docs/whats-new/_index.md | 1 + 2 files changed, 160 insertions(+) create mode 100644 content/docs/configuration/plugins/syslog-ng-outputs/s3.md diff --git a/content/docs/configuration/plugins/syslog-ng-outputs/s3.md b/content/docs/configuration/plugins/syslog-ng-outputs/s3.md new file mode 100644 index 000000000..a310e7f73 --- /dev/null +++ b/content/docs/configuration/plugins/syslog-ng-outputs/s3.md @@ -0,0 +1,159 @@ +--- +title: Sending messages from a local network to a S3 (compatible) server +linktitle: S3 +weight: 200 +generated_file: true +--- + +## Example +{{< highlight yaml >}} +apiVersion: logging.banzaicloud.io/v1beta1 +kind: SyslogNGOutput +metadata: +name: s3 +spec: +s3: + url: "https://some-s3-compatible-endpoint:8088" + bucket: "s3bucket-name" + access_key: + valueFrom: + secretKeyRef: + name: s3 + key: access-key + secret_key: + valueFrom: + secretKeyRef: + name: s3 + key: secret-key + object_key: "${HOST}/my-logs" +{{}} + +## S3Output + +### url (string, optional) {#s3output-url} + +The hostname or IP address of the S3 server. + +Default: - + +### bucket (string, optional) {#s3output-bucket} + +The bucket name of the S3 server. + +Default: - + +### access_key (*secret.Secret, optional) {#s3output-access_key} + +The access_key for the S3 server. + +Default: - + +### secret_key (*secret.Secret, optional) {#s3output-secret_key} + +The secret_key for the S3 server. + +Default: - + +### object_key (string, optional) {#s3output-object_key} + +The object_key for the S3 server. + +Default: - + +### object_key_timestamp (RawString, optional) {#s3output-object_key_timestamp} + +Set object_key_timestamp + +Default: - + +### template (RawString, optional) {#s3output-template} + +Template + +Default: - + +### compression (*bool, optional) {#s3output-compression} + +Enable or disable compression. + +Default: false + +### compresslevel (int, optional) {#s3output-compresslevel} + +Set the compression level (1-9). + +Default: 9 + +### chunk_size (int, optional) {#s3output-chunk_size} + +Set the chunk size. + +Default: 5MiB + +### max_object_size (int, optional) {#s3output-max_object_size} + +Set the maximum object size size. + +Default: 5120GiB + +### upload_threads (int, optional) {#s3output-upload_threads} + +Set the number of upload threads. + +Default: 8 + +### max_pending_uploads (int, optional) {#s3output-max_pending_uploads} + +Set the maximum number of pending uploads. + +Default: 32 + +### flush_grace_period (int, optional) {#s3output-flush_grace_period} + +Set the number of seconds for flush period. + +Default: 60 + +### region (string, optional) {#s3output-region} + +Set the region option. + +Default: - + +### storage_class (string, optional) {#s3output-storage_class} + +Set the storage_class option. + +Default: - + +### canned_acl (string, optional) {#s3output-canned_acl} + +Set the canned_acl option. + +Default: - + +### log-fifo-size (int, optional) {#s3output-log-fifo-size} + +The number of messages that the output queue can store. + +Default: - + +### persist_name (string, optional) {#s3output-persist_name} + +Persistname + +Default: - + +### retries (int, optional) {#s3output-retries} + +The number of times syslog-ng OSE attempts to send a message to this destination. If syslog-ng OSE could not send a message, it will try again until the number of attempts reaches retries, then drops the message. + +Default: - + +### throttle (int, optional) {#s3output-throttle} + +Sets the maximum number of messages sent to the destination per second. Use this output-rate-limiting functionality only when using disk-buffer as well to avoid the risk of losing messages. Specifying 0 or a lower value sets the output limit to unlimited. + +Default: 0 + + diff --git a/content/docs/whats-new/_index.md b/content/docs/whats-new/_index.md index a739910b0..d006e7c3c 100644 --- a/content/docs/whats-new/_index.md +++ b/content/docs/whats-new/_index.md @@ -15,6 +15,7 @@ When using syslog-ng as the log aggregator, you can now use the following new ou - [Grafana Loki]({{< relref "/docs/configuration/plugins/syslog-ng-outputs/loki.md" >}}) - [MongoDB]({{< relref "/docs/configuration/plugins/syslog-ng-outputs/mongodb.md" >}}) - [Redis]({{< relref "/docs/configuration/plugins/syslog-ng-outputs/redis.md" >}}) +- [Amazon S3]({{< relref "/docs/configuration/plugins/syslog-ng-outputs/s3.md" >}}) - [Splunk HEC]({{< relref "/docs/configuration/plugins/syslog-ng-outputs/splunk_hec.md" >}}) - The [HTTP]({{< relref "/docs/configuration/plugins/syslog-ng-outputs/http.md" >}}) output now supports the `log-fifo-size`, `response-action`, and `timeout` fields. From 745538f39f55de6a7463e81c40601cb5736478f6 Mon Sep 17 00:00:00 2001 From: Robert Fekete Date: Sat, 30 Sep 2023 09:34:00 +0200 Subject: [PATCH 28/30] Clarify menu name Signed-off-by: Robert Fekete --- content/docs/configuration/plugins/outputs/_index.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/content/docs/configuration/plugins/outputs/_index.md b/content/docs/configuration/plugins/outputs/_index.md index 4c2a3e695..a110fbe9e 100644 --- a/content/docs/configuration/plugins/outputs/_index.md +++ b/content/docs/configuration/plugins/outputs/_index.md @@ -1,4 +1,4 @@ --- -title: Outputs +title: Fluentd Outputs --- From b3576a211fdcca8233b04beb2f8d87d73bfc5265 Mon Sep 17 00:00:00 2001 From: Peter Wilcsinszky Date: Mon, 2 Oct 2023 08:53:02 +0200 Subject: [PATCH 29/30] buffer metrics update Signed-off-by: Peter Wilcsinszky --- content/docs/image-versions.md | 4 ++-- content/docs/whats-new/_index.md | 25 +++++++++++++++++++++++++ 2 files changed, 27 insertions(+), 2 deletions(-) diff --git a/content/docs/image-versions.md b/content/docs/image-versions.md index 8fe1f6e35..96e97949a 100644 --- a/content/docs/image-versions.md +++ b/content/docs/image-versions.md @@ -14,14 +14,14 @@ Logging operator uses the following image versions. | ghcr.io/kube-logging/fluentd-drain-watch | https://github.com/kube-logging/fluentd-drain-watch | v0.2.1 | | k8s.gcr.io/pause | | 3.2 | | docker.io/busybox | https://github.com/docker-library/busybox | latest | -| ghcr.io/axoflow/axosyslog | https://github.com/axoflow/axosyslog-docker/ | 4.3.1 | +| ghcr.io/axoflow/axosyslog | https://github.com/axoflow/axosyslog-docker/ | 4.4.0 | | docker.io/fluent/fluent-bit | https://github.com/fluent/fluent-bit-docker-image | 2.1.8 | | ghcr.io/kube-logging/fluentd | https://github.com/kube-logging/fluentd-images | v1.15-ruby3 | | ghcr.io/axoflow/axosyslog-metrics-exporter | https://github.com/axoflow/axosyslog-metrics-exporter | 0.0.2 | | ghcr.io/kube-logging/syslogng-reload | https://github.com/kube-logging/syslogng-reload-image | v1.3.1 | | ghcr.io/kube-logging/eventrouter | https://github.com/kube-logging/eventrouter | v0.4.0 | -The following images are new replaced +### The following images are now replaced | Image used in 4.3 | New image in 4.4 | | -------- | --- | -- | diff --git a/content/docs/whats-new/_index.md b/content/docs/whats-new/_index.md index d006e7c3c..0cd64d5b8 100644 --- a/content/docs/whats-new/_index.md +++ b/content/docs/whats-new/_index.md @@ -74,6 +74,31 @@ By default, Logging operator adds a label to avoid injecting the Istio sidecar i For non-istio users, these changes make no difference, as this label is only used in Istio context. For Istio users, these defaults make Logging operator work out of the box. +### Improved buffer metrics + +The buffer metrics sidecar configuration has been rewritten to add a new metric and improve performance by avoiding unnecessary cardinality. + +The name of the metric has been changed as well, but the original metric was kept in place to avoid breaking existing clients. + +**Metrics currently supported by the sidecar** + +Old +``` ++# HELP node_buffer_size_bytes Disk space used [deprecated] ++# TYPE node_buffer_size_bytes gauge ++node_buffer_size_bytes{entity="/buffers"} 32253 +``` + +New +``` ++# HELP logging_buffer_files File count ++# TYPE logging_buffer_files gauge ++logging_buffer_files{entity="/buffers",host="all-to-file-fluentd-0"} 2 ++# HELP logging_buffer_size_bytes Disk space used ++# TYPE logging_buffer_size_bytes gauge ++logging_buffer_size_bytes{entity="/buffers",host="all-to-file-fluentd-0"} 32253 +``` + ## Other improvements - You can now configure the resources of the buffer metrics sidecar. From 62ca8d5b21b52d720067b1a5fa3b6012a2e74fd7 Mon Sep 17 00:00:00 2001 From: Robert Fekete Date: Mon, 2 Oct 2023 09:00:50 +0200 Subject: [PATCH 30/30] [4.4][loki] Adds an example config to the syslog-ng loki output --- .../plugins/syslog-ng-outputs/loki.md | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) diff --git a/content/docs/configuration/plugins/syslog-ng-outputs/loki.md b/content/docs/configuration/plugins/syslog-ng-outputs/loki.md index 332a2bdd3..152d6462b 100644 --- a/content/docs/configuration/plugins/syslog-ng-outputs/loki.md +++ b/content/docs/configuration/plugins/syslog-ng-outputs/loki.md @@ -6,6 +6,24 @@ generated_file: true Sends messages to Grafana Loki over gRPC, based on the [Loki destination of AxoSyslog Core](https://axoflow.com/docs/axosyslog-core/chapter-destinations/syslog-ng-with-loki/). +For example: + +{{< highlight yaml >}} +apiVersion: logging.banzaicloud.io/v1beta1 +kind: SyslogNGOutput +metadata: + name: loki +spec: + loki: + url: "loki.loki:8000" + labels: + "app": "$PROGRAM" + "host": "$HOST" + workers: 16 + batch-timeout: 10000 + batch-lines: 1000 +{{< /highlight >}} + ## Configuration ### labels (filter.ArrowMap, optional) {#lokioutput-labels}