update fluent bit: #870

TrivadisPF · May 2, 2024 · ddabf44 · ddabf44
1 parent cd2d3a8
commit ddabf44
Show file tree

Hide file tree

Showing 4 changed files with 99 additions and 2 deletions.
diff --git a/documentation/configuration.md b/documentation/configuration.md
@@ -550,6 +550,8 @@ For each service there might be some other settings, such as controlling the num
 | `FLUENTD_s3_bucket_file_type`                                  	| `json` 	| 1.17.0 	| Name of the S3 bucket to use if Minio is enabled.                                                                                                            	|
 | [**_Fluent Bit_**](./services/fluent-bit.md)  &nbsp;&nbsp;&nbsp;&nbsp;![x86-64](./images/x86-64.png)  ![arm](./images/arm.png)                              	|         	|       	|                                                                                                                                                                                     	|     
 | `FLUENT_BIT_enable`                                  	| `false` 	| 1.18.0 	| Generate Fluent Bit service                                                                                                                                                        	|
+| `FLUENT_BIT_volume_map_config`                                  	| `false` 	| 1.18.0 	| Map a fluent-bit config file into the service?                                                                                                                                                        	|
+| `FLUENT_BIT_config_filename`                                  	| `fluent-bit.yaml` 	| 1.18.0 	| The name of the fluent-bit config file. You have to enable `FLUENT_BIT_volume_map_config` so that it becomes effective. You can also use a `.conf` file (the traditional way for configuration of Fluent Bit).                                                                                                                                                     	|
 | [**_FileBeat_**](./services/filebeat.md)  &nbsp;&nbsp;&nbsp;&nbsp;![x86-64](./images/x86-64.png)  ![arm](./images/arm.png)                              	|         	|       	|                                                                                                                                                                                     	|     
 | `FILEBEAT_enable`                                  	| `false` 	| 1.17.0 	| Generate FileBeat service                                                                                                                                                        	|        
 | `FILEBEAT_conf_file_name`                                  	| `` 	| 1.17.0 	| Name of a custom config file to map into the FileBeat container.                                                                                                                                                      	|

diff --git a/modern-data-platform-stack/generator-config/templates/docker-compose.yml.j2 b/modern-data-platform-stack/generator-config/templates/docker-compose.yml.j2
@@ -9116,7 +9116,7 @@ services:
     volumes:
       - ./data-transfer:/data-transfer
     {% if FLUENT_BIT_volume_map_config | default(false) %}
-      - ./custom-conf/fluent-bit/:/fluent-bit/etc/
+      - ./custom-conf/fluent-bit/{{FLUENT_BIT_config_filename}}:/fluent-bit/etc/{{FLUENT_BIT_config_filename}}
     {% endif -%}   {#  FLUENT_BIT_volume_map_config #}
     {%if use_timezone | default(false) %}
       - "./etc/timezone:/etc/timezone:ro"
@@ -9125,6 +9125,7 @@ services:
     {%if logging_driver is defined and logging_driver and logging_driver in ('fluentd','loki','syslog','splunk') | default(false) %}
     <<: *logging
     {% endif -%}   {#  logging_driver is defined ... #}
+    command: ["/fluent-bit/bin/fluent-bit", "-c", "/fluent-bit/etc/{{FLUENT_BIT_config_filename}}"]
     restart: {{container_restart_policy}}
 {% endif %}   {#  FLUENT_BIT_enable  #}
 

diff --git a/modern-data-platform-stack/generator-config/vars/config.yml b/modern-data-platform-stack/generator-config/vars/config.yml
@@ -928,6 +928,7 @@
       #
       FLUENT_BIT_enable: false
       FLUENT_BIT_volume_map_config: false
+      FLUENT_BIT_config_filename: 'fluent-bit.yaml'
 
       #
       # ===== FileBeat ========
@@ -2132,7 +2133,7 @@
       # ===== NATS ========
       #
       NATS_enable: false
-      
+
       #
       #=====  MinIO Object Storage ========
       #

diff --git a/modern-data-platform-stack/static-data/custom-conf/fluent-bit/fluent-bit.conf.template b/modern-data-platform-stack/static-data/custom-conf/fluent-bit/fluent-bit.conf.template
@@ -0,0 +1,93 @@
+[SERVICE]
+    # Flush
+    # =====
+    # set an interval of seconds before to flush records to a destination
+    flush        1
+
+    # Daemon
+    # ======
+    # instruct Fluent Bit to run in foreground or background mode.
+    daemon       Off
+
+    # Log_Level
+    # =========
+    # Set the verbosity level of the service, values can be:
+    #
+    # - error
+    # - warning
+    # - info
+    # - debug
+    # - trace
+    #
+    # by default 'info' is set, that means it includes 'error' and 'warning'.
+    log_level    info
+
+    # Parsers File
+    # ============
+    # specify an optional 'Parsers' configuration file
+    parsers_file parsers.conf
+
+    # Plugins File
+    # ============
+    # specify an optional 'Plugins' configuration file to load external plugins.
+    plugins_file plugins.conf
+
+    # HTTP Server
+    # ===========
+    # Enable/Disable the built-in HTTP Server for metrics
+    http_server  Off
+    http_listen  0.0.0.0
+    http_port    2020
+
+    # Storage
+    # =======
+    # Fluent Bit can use memory and filesystem buffering based mechanisms
+    #
+    # - https://docs.fluentbit.io/manual/administration/buffering-and-storage
+    #
+    # storage metrics
+    # ---------------
+    # publish storage pipeline metrics in '/api/v1/storage'. The metrics are
+    # exported only if the 'http_server' option is enabled.
+    #
+    storage.metrics on
+
+    # storage.path
+    # ------------
+    # absolute file system path to store filesystem data buffers (chunks).
+    #
+    # storage.path /tmp/storage
+
+    # storage.sync
+    # ------------
+    # configure the synchronization mode used to store the data into the
+    # filesystem. It can take the values normal or full.
+    #
+    # storage.sync normal
+
+    # storage.checksum
+    # ----------------
+    # enable the data integrity check when writing and reading data from the
+    # filesystem. The storage layer uses the CRC32 algorithm.
+    #
+    # storage.checksum off
+
+    # storage.backlog.mem_limit
+    # -------------------------
+    # if storage.path is set, Fluent Bit will look for data chunks that were
+    # not delivered and are still in the storage layer, these are called
+    # backlog data. This option configure a hint of maximum value of memory
+    # to use when processing these records.
+    #
+    # storage.backlog.mem_limit 5M
+
+[INPUT]
+    name cpu
+    tag  cpu.local
+
+    # Read interval (sec) Default: 1
+    interval_sec 1
+
+[OUTPUT]
+    name  stdout
+    match *