Gangams/logs 50k eps per node

.trivyignore

@@ -35,4 +35,4 @@ CVE-2023-5678
 #golang MEDIUM
 CVE-2023-48795
 CVE-2024-24786
-CVE-2023-45288
+CVE-2020-8559

build/common/installer/scripts/fluent-bit-conf-customizer.rb

@@ -25,11 +25,22 @@
 
 @default_service_interval = "15"
 @default_mem_buf_limit = "10"
+@default_high_log_scale_service_interval = "1"
+@default_high_log_scale_max_storage_chunks_up = "500" # Each chunk size is ~2MB
+@default_high_log_scale_max_storage_type = "filesystem" # filesystem = memory + filesystem in fluent-bit
 
 def is_number?(value)
   true if Integer(value) rescue false
 end
 
+def is_high_log_scale_mode?
+  isHighLogScaleMode = false
+  if !ENV["IS_HIGH_LOG_SCALE_MODE"].nil? && !ENV["IS_HIGH_LOG_SCALE_MODE"].empty? && ENV["IS_HIGH_LOG_SCALE_MODE"].to_s.downcase == "true"
+    isHighLogScaleMode = true
+  end
+  return isHighLogScaleMode
+end
+
 def substituteMultiline(multilineLogging, stacktraceLanguages, new_contents)
     if !multilineLogging.nil? && multilineLogging.to_s.downcase == "true"
       if !stacktraceLanguages.nil? && !stacktraceLanguages.empty?
@@ -48,6 +59,15 @@ def substituteMultiline(multilineLogging, stacktraceLanguages, new_contents)
     return new_contents
 end
 
+def substituteStorageTotalLimitSize(new_contents)
+   if is_high_log_scale_mode?
+      new_contents = new_contents.gsub("${STORAGE_TOTAL_LIMIT_SIZE_MB}", "storage.total_limit_size        1000M")
+   else
+      new_contents = new_contents.gsub("\n    ${STORAGE_TOTAL_LIMIT_SIZE_MB}\n", "\n")
+   end
+   return new_contents
+end
+
 def substituteResourceOptimization(resourceOptimizationEnabled, new_contents)
   #Update the config file only in two conditions: 1. Linux and resource optimization is enabled 2. Windows and using aad msi auth and not using geneva logs integration
   if (!@isWindows && !resourceOptimizationEnabled.nil? && resourceOptimizationEnabled.to_s.downcase == "true") || (@isWindows && @using_aad_msi_auth && !@geneva_logs_integration)
@@ -79,8 +99,18 @@ def substituteFluentBitPlaceHolders
     windowsFluentBitDisabled = ENV["AZMON_WINDOWS_FLUENT_BIT_DISABLED"]
     kubernetesMetadataCollection = ENV["AZMON_KUBERNETES_METADATA_ENABLED"]
     annotationBasedLogFiltering = ENV["AZMON_ANNOTATION_BASED_LOG_FILTERING"]
+    storageMaxChunksUp = ENV["FBIT_STORAGE_MAX_CHUNKS_UP"]
+    storageType = ENV["FBIT_STORAGE_TYPE"]
+    enableFbitThreading = ENV["ENABLE_FBIT_THREADING"]
+
 
-    serviceInterval = (!interval.nil? && is_number?(interval) && interval.to_i > 0) ? interval : @default_service_interval
+    serviceInterval = @default_service_interval
+    if is_high_log_scale_mode?
+      serviceInterval = @default_high_log_scale_service_interval
+      puts "Since high log scale mode configured hence using Flush interval: #{serviceInterval}"
+    elsif (!interval.nil? && is_number?(interval) && interval.to_i > 0)
+      serviceInterval = interval
+    end
     serviceIntervalSetting = "Flush         " + serviceInterval
 
     tailBufferChunkSize = (!bufferChunkSize.nil? && is_number?(bufferChunkSize) && bufferChunkSize.to_i > 0) ? bufferChunkSize : nil
@@ -115,6 +145,33 @@ def substituteFluentBitPlaceHolders
       new_contents = new_contents.gsub("\n    ${TAIL_IGNORE_OLDER}\n", "\n")
     end
 
+    if is_high_log_scale_mode?
+      puts "Since high log scale mode configured hence using threaded on for tail plugin"
+      new_contents = new_contents.gsub("${TAIL_THREADED}", "threaded on")
+    elsif (!enableFbitThreading.nil? && !enableFbitThreading.empty? && enableFbitThreading.to_s.downcase == "true" )
+      new_contents = new_contents.gsub("${TAIL_THREADED}", "threaded on")
+    else
+      new_contents = new_contents.gsub("\n    ${TAIL_THREADED}\n", "\n")
+    end
+
+    if is_high_log_scale_mode?
+      new_contents = new_contents.gsub("${STORAGE_TYPE}", "storage.type " + @default_high_log_scale_max_storage_type)
+      puts "Since high log scale mode configured hence using storage.type: #{@default_high_log_scale_max_storage_type} for tail plugin"
+    elsif !storageType.nil? && !storageType.empty?
+      new_contents = new_contents.gsub("${STORAGE_TYPE}", "storage.type " + storageType)
+    else
+      new_contents = new_contents.gsub("\n    ${STORAGE_TYPE}\n", "\n")
+    end
+
+    if is_high_log_scale_mode?
+      new_contents = new_contents.gsub("${MAX_STORAGE_CHUNKS_UP}", "storage.max_chunks_up " + @default_high_log_scale_max_storage_chunks_up)
+      puts "Since high log scale mode configured hence using storage.max_chunks_up: #{@default_high_log_scale_max_storage_chunks_up} for tail plugin"
+    elsif !storageMaxChunksUp.nil? && !storageMaxChunksUp.empty?
+      new_contents = new_contents.gsub("${MAX_STORAGE_CHUNKS_UP}", "storage.max_chunks_up " + storageMaxChunksUp)
+    else
+      new_contents = new_contents.gsub("\n    ${MAX_STORAGE_CHUNKS_UP}\n", "\n")
+    end
+
     if !kubernetesMetadataCollection.nil? && kubernetesMetadataCollection.to_s.downcase == "true"
       new_contents = new_contents.gsub("#${KubernetesFilterEnabled}", "")
     end
@@ -135,6 +192,7 @@ def substituteFluentBitPlaceHolders
 
     puts "config::Starting to substitute the placeholders in fluent-bit-common.conf file for log collection"
     text = File.read(@fluent_bit_common_conf_path)
+    text = substituteStorageTotalLimitSize(text)
     new_contents = substituteMultiline(multilineLogging, stacktraceLanguages, text)
     File.open(@fluent_bit_common_conf_path, "w") { |file| file.puts new_contents }
     puts "config::Successfully substituted the placeholders in fluent-bit-common.conf file"

build/common/installer/scripts/tomlparser-agent-config.rb

@@ -1,6 +1,5 @@
 #!/usr/local/bin/ruby
 
-
 @os_type = ENV["OS_TYPE"]
 require "tomlrb"
 
@@ -68,6 +67,9 @@
 @outputForwardWorkers = 10
 @outputForwardRetryLimit = 10
 @requireAckResponse = "false"
+@fbitStorageMaxChunksUp = 0
+@fbitStorageType = ""
+@enableFluentBitThreading = false
 
 # configmap settings related to mdsd
 @mdsdMonitoringMaxEventRate = 0
@@ -101,6 +103,8 @@
 @waittime_port_25226 = 45
 @waittime_port_25228 = 120
 @waittime_port_25229 = 45
+@waittime_port_13000 = 45 # default waittime for AMACA data port
+@waittime_port_12563 = 45 # default waittime for AMACA config port
 
 def is_number?(value)
   true if Integer(value) rescue false
@@ -238,6 +242,23 @@ def populateSettingValuesFromConfigMap(parsedConfig)
           @enableFbitInternalMetrics = true
           puts "Using config map value: enable_internal_metrics = #{@enableFbitInternalMetrics}"
         end
+
+        fbitStorageMaxChunksUp = fbit_config[:storage_max_chunks_up]
+        if is_valid_number?(fbitStorageMaxChunksUp)
+            @fbitStorageMaxChunksUp = fbitStorageMaxChunksUp.to_i
+            puts "Using config map value: fbitStorageMaxChunksUp  = #{@fbitStorageMaxChunksUp}"
+        end
+        fbitStorageType = fbit_config[:storage_type]
+        if !fbitStorageType.nil? && !fbitStorageType.empty? && ["memory", "filesystem"].include?(fbitStorageType)
+            @fbitStorageType = fbitStorageType
+            puts "Using config map value: fbitStorageType  = #{@fbitStorageType}"
+        end
+
+        enableFluentBitThreading = fbit_config[:enable_threading]
+        if !enableFluentBitThreading.nil? && enableFluentBitThreading.strip.casecmp("true") == 0
+          @enableFluentBitThreading = enableFluentBitThreading
+          puts "Using config map value: enableFluentBitThreading  = #{@enableFluentBitThreading}"
+        end
       end
 
       # fbit forward plugins geneva settings per tenant
@@ -391,6 +412,16 @@ def populateSettingValuesFromConfigMap(parsedConfig)
           @waittime_port_25229 = waittime.to_i
           puts "Using config map value: WAITTIME_PORT_25229 = #{@waittime_port_25229}"
         end
+        waittime = network_listener_waittime_config[:tcp_port_13000]
+        if is_valid_waittime?(waittime, @waittime_port_13000)
+          @waittime_port_13000 = waittime.to_i
+          puts "Using config map value: WAITTIME_PORT_13000 = #{@waittime_port_13000}"
+        end
+        waittime = network_listener_waittime_config[:tcp_port_12563]
+        if is_valid_waittime?(waittime, @waittime_port_12563)
+          @waittime_port_12563 = waittime.to_i
+          puts "Using config map value: WAITTIME_PORT_12563 = #{@waittime_port_12563}"
+        end
       end
     end
   rescue => errorStr
@@ -440,6 +471,18 @@ def populateSettingValuesFromConfigMap(parsedConfig)
     file.write("export FBIT_TAIL_IGNORE_OLDER=#{@fbitTailIgnoreOlder}\n")
   end
 
+  if @fbitStorageMaxChunksUp > 0
+    file.write("export FBIT_STORAGE_MAX_CHUNKS_UP=#{@fbitStorageMaxChunksUp}\n")
+  end
+
+  if !@fbitStorageType.nil? && !@fbitStorageType.empty?
+    file.write("export FBIT_STORAGE_TYPE=#{@fbitStorageType}\n")
+  end
+
+  if @enableFluentBitThreading
+    file.write("export ENABLE_FBIT_THREADING=#{@enableFluentBitThreading}\n")
+  end
+
   if @storageTotalLimitSizeMB > 0
     file.write("export STORAGE_TOTAL_LIMIT_SIZE_MB=#{@storageTotalLimitSizeMB.to_s + "M"}\n")
   end
@@ -509,6 +552,8 @@ def populateSettingValuesFromConfigMap(parsedConfig)
   file.write("export WAITTIME_PORT_25226=#{@waittime_port_25226}\n")
   file.write("export WAITTIME_PORT_25228=#{@waittime_port_25228}\n")
   file.write("export WAITTIME_PORT_25229=#{@waittime_port_25229}\n")
+  file.write("export WAITTIME_PORT_13000=#{@waittime_port_13000}\n")
+  file.write("export WAITTIME_PORT_12563=#{@waittime_port_12563}\n")
 
   # Close file after writing all environment variables
   file.close
@@ -548,6 +593,20 @@ def get_command_windows(env_variable_name, env_variable_value)
       commands = get_command_windows("FBIT_TAIL_IGNORE_OLDER", @fbitTailIgnoreOlder)
       file.write(commands)
     end
+
+    if @fbitStorageMaxChunksUp > 0
+      commands = get_command_windows("FBIT_STORAGE_MAX_CHUNKS_UP", @fbitStorageMaxChunksUp)
+      file.write(commands)
+    end
+    if !@fbitStorageType.nil? && !@fbitStorageType.empty?
+      commands = get_command_windows("FBIT_STORAGE_TYPE", @fbitStorageType)
+    end
+
+    if @enableFluentBitThreading
+      commands = get_command_windows("ENABLE_FBIT_THREADING", @enableFluentBitThreading)
+      file.write(commands)
+    end
+
     if @promFbitChunkSize > 0
       commands = get_command_windows("AZMON_FBIT_CHUNK_SIZE", @promFbitChunkSize.to_s + "m")
       file.write(commands)

build/common/installer/scripts/tomlparser-common-agent-config.rb

@@ -10,6 +10,7 @@
 
 @disableTelemetry = false
 @logEnableKubernetesMetadataCacheTTLSeconds = 60
+@enableHighLogScaleMode = false
 
 def is_windows?
   return !@os_type.nil? && !@os_type.empty? && @os_type.strip.casecmp("windows") == 0
@@ -72,6 +73,14 @@ def populateSettingValuesFromConfigMap(parsedConfig)
         end
       end
     end
+    if !parsedConfig.nil? && !parsedConfig[:agent_settings].nil?
+      high_log_scale = parsedConfig[:agent_settings][:high_log_scale]
+      if !high_log_scale.nil? && !high_log_scale[:enabled].nil?
+        @enableHighLogScaleMode = high_log_scale[:enabled]
+        puts "Using config map value: enabled = #{@enableHighLogScaleMode} for high log scale config"
+      end
+    end
+
   rescue => errorStr
     puts "config::error:Exception while reading config settings for agent configuration setting - #{errorStr}, using defaults"
   end
@@ -102,6 +111,12 @@ def get_command_windows(env_variable_name, env_variable_value)
       commands = get_command_windows("DISABLE_TELEMETRY", @disableTelemetry)
       file.write(commands)
     end
+
+    if @enableHighLogScaleMode
+      commands = get_command_windows("ENABLE_HIGH_LOG_SCALE_MODE", @enableHighLogScaleMode)
+      file.write(commands)
+    end
+
     commands = get_command_windows("AZMON_KUBERNETES_METADATA_CACHE_TTL_SECONDS", @logEnableKubernetesMetadataCacheTTLSeconds)
     file.write(commands)
     # Close file after writing all environment variables
@@ -118,6 +133,9 @@ def get_command_windows(env_variable_name, env_variable_value)
     if @disableTelemetry
       file.write("export DISABLE_TELEMETRY=#{@disableTelemetry}\n")
     end
+    if @enableHighLogScaleMode
+      file.write("export ENABLE_HIGH_LOG_SCALE_MODE=#{@enableHighLogScaleMode}\n")
+    end
     file.write("export AZMON_KUBERNETES_METADATA_CACHE_TTL_SECONDS=#{@logEnableKubernetesMetadataCacheTTLSeconds}\n")
     # Close file after writing all environment variables
     file.close

build/linux/installer/conf/PA.json

@@ -0,0 +1,11 @@
+{
+    "ServiceArguments": {
+        "Version": "1.0"
+    },
+    "EndpointConfigurations": [
+        {
+            "Destination": "GigLA",
+            "EndpointPort": 13000
+        }
+    ]
+}

build/linux/installer/conf/fluent-bit-common.conf

@@ -26,6 +26,20 @@
     Skip_Long_Lines On
     Ignore_Older 2m
 
+[INPUT]
+    Name tail
+    Alias ama-logs_amaca_log_tail
+    Tag oms.container.log.flbplugin.amaca.*
+    Path /var/opt/microsoft/linuxmonagent/amaca/log/*.log
+    Read_from_Head true
+    DB /var/opt/microsoft/docker-cimprov/state/amaca-ai.db
+    DB.Sync Off
+    Parser docker
+    Mem_Buf_Limit 1m
+    Path_Key filepath
+    Skip_Long_Lines On
+    Ignore_Older 2m
+
 [INPUT]
     Name tail
     Alias ama-logs_termination_log_tail
@@ -77,13 +91,20 @@
     Match oms.container.log.flbplugin.mdsd.qos.*
     regex log ^MaODSRequest,https.*_LogManagement_LINUX_SYSLOGS_BLOB
 
+[FILTER]
+    Name grep
+    Alias ama-logs_amaca_log_tail
+    Match oms.container.log.flbplugin.amaca.*
+    Exclude log Information
+
 [OUTPUT]
     Name                            oms
     Alias                           oms_output
     EnableTelemetry                 true
     Retry_Limit                     10
     TelemetryPushIntervalSeconds    300
     Match                           oms.container.*
+    ${STORAGE_TOTAL_LIMIT_SIZE_MB}
 
 [OUTPUT]
     Name                            oms

build/linux/installer/conf/fluent-bit.conf

@@ -3,6 +3,7 @@
     ${SERVICE_FLUSH_INTERVAL}
     HTTP_Server   Off
     Daemon        Off
+    ${MAX_STORAGE_CHUNKS_UP}
     storage.path  /var/opt/microsoft/docker-cimprov/state/flbstore/
     storage.sync  normal
     storage.checksum off
@@ -24,6 +25,7 @@
     DB /var/log/omsagent-fblogs.db
     DB.Sync Off
     Parser docker
+    ${STORAGE_TYPE}
     ${TAIL_MEM_BUF_LIMIT}
     ${TAIL_BUFFER_CHUNK_SIZE}
     ${TAIL_BUFFER_MAX_SIZE}
@@ -33,6 +35,7 @@
     Skip_Long_Lines On
     ${TAIL_IGNORE_OLDER}
     Exclude_Path ${AZMON_CLUSTER_LOG_TAIL_EXCLUDE_PATH}
+    ${TAIL_THREADED}
 
 #NOTE: Multiline should be the first filter https://docs.fluentbit.io/manual/pipeline/filters/multiline-stacktrace
 #${MultilineEnabled}[FILTER]

build/linux/installer/datafiles/base_container.data

@@ -45,6 +45,8 @@ MAINTAINER:              'Microsoft Corporation'
 
 /opt/tomlparser-geneva-config.rb;                                       build/common/installer/scripts/tomlparser-geneva-config.rb;     755; root; root
 
+/etc/opt/microsoft/azuremonitoragent/amacoreagent/PA.json;              build/linux/installer/conf/PA.json;                             755; root; root
+
 /opt/fluent-bit-geneva-conf-customizer.rb;                            build/common/installer/scripts/fluent-bit-geneva-conf-customizer.rb;     755; root; root
 /etc/opt/microsoft/docker-cimprov/fluent-bit-input.conf;			    build/linux/installer/conf/fluent-bit-input.conf;                644; root; root
 /etc/opt/microsoft/docker-cimprov/fluent-bit-common.conf;			    build/linux/installer/conf/fluent-bit-common.conf;                644; root; root
@@ -181,6 +183,10 @@ MAINTAINER:              'Microsoft Corporation'
 /var/opt/microsoft/docker-cimprov/state/ContainerInventory; 755; root; root
 /var/opt/microsoft/docker-cimprov/log;                  755; root; root
 /var/opt/microsoft/linuxmonagent/log;                   755; root; root
+/var/opt/microsoft/linuxmonagent/amaca;                 755; root; root
+/var/opt/microsoft/linuxmonagent/amaca/log;             755; root; root
+
+/etc/opt/microsoft/azuremonitoragent/amacoreagent;      755; root; root
 
 /opt/fluent-bit;                          755; root; root;sysdir
 /opt/fluent-bit/bin;                      755; root; root;sysdir