Revert "Gangams/logs 50k eps per node (#1235)"

This reverts commit 69f3e8e.
microsoft · May 10, 2024 · 7067e87 · 7067e87
1 parent 69f3e8e
commit 7067e87
Show file tree

Hide file tree

Showing 23 changed files with 266 additions and 567 deletions.
diff --git a/build/common/installer/scripts/fluent-bit-conf-customizer.rb b/build/common/installer/scripts/fluent-bit-conf-customizer.rb
@@ -25,23 +25,11 @@
 
 @default_service_interval = "15"
 @default_mem_buf_limit = "10"
-@default_high_log_scale_service_interval = "1"
-@default_high_log_scale_max_storage_chunks_up = "500" # Each chunk size is ~2MB
-@default_high_log_scale_max_storage_type = "filesystem" # filesystem = memory + filesystem in fluent-bit
-@default_high_log_scale_max_storage_total_limit_size = "10G"
 
 def is_number?(value)
   true if Integer(value) rescue false
 end
 
-def is_high_log_scale_mode?
-  isHighLogScaleMode = false
-  if !ENV["IS_HIGH_LOG_SCALE_MODE"].nil? && !ENV["IS_HIGH_LOG_SCALE_MODE"].empty? && ENV["IS_HIGH_LOG_SCALE_MODE"].to_s.downcase == "true"
-    isHighLogScaleMode = true
-  end
-  return isHighLogScaleMode
-end
-
 def substituteMultiline(multilineLogging, stacktraceLanguages, new_contents)
     if !multilineLogging.nil? && multilineLogging.to_s.downcase == "true"
       if !stacktraceLanguages.nil? && !stacktraceLanguages.empty?
@@ -60,15 +48,6 @@ def substituteMultiline(multilineLogging, stacktraceLanguages, new_contents)
     return new_contents
 end
 
-def substituteStorageTotalLimitSize(new_contents)
-   if is_high_log_scale_mode?
-      new_contents = new_contents.gsub("#${AZMON_STORAGE_TOTAL_LIMIT_SIZE_MB}", "storage.total_limit_size  " + @default_high_log_scale_max_storage_total_limit_size)
-   else
-      new_contents = new_contents.gsub("\n    #${AZMON_STORAGE_TOTAL_LIMIT_SIZE_MB}\n", "\n")
-   end
-   return new_contents
-end
-
 def substituteResourceOptimization(resourceOptimizationEnabled, new_contents)
   #Update the config file only in two conditions: 1. Linux and resource optimization is enabled 2. Windows and using aad msi auth and not using geneva logs integration
   if (!@isWindows && !resourceOptimizationEnabled.nil? && resourceOptimizationEnabled.to_s.downcase == "true") || (@isWindows && @using_aad_msi_auth && !@geneva_logs_integration)
@@ -84,40 +63,6 @@ def substituteResourceOptimization(resourceOptimizationEnabled, new_contents)
   return new_contents
 end
 
-def substituteHighLogScaleConfig(enableFbitThreading, storageType, storageMaxChunksUp, new_contents)
-  begin
-      if is_high_log_scale_mode? || (!enableFbitThreading.nil? && !enableFbitThreading.empty? && enableFbitThreading.to_s.downcase == "true" )
-        new_contents = new_contents.gsub("#${AZMON_TAIL_THREADED}", "threaded on")
-        puts "using threaded on for tail plugin"
-      else
-        new_contents = new_contents.gsub("\n    #${AZMON_TAIL_THREADED}\n", "\n")
-      end
-
-      if is_high_log_scale_mode?
-        new_contents = new_contents.gsub("#${AZMON_STORAGE_TYPE}", "storage.type " + @default_high_log_scale_max_storage_type)
-        puts "using storage.type: #{@default_high_log_scale_max_storage_type} for tail plugin"
-      elsif !storageType.nil? && !storageType.empty?
-        new_contents = new_contents.gsub("#${AZMON_STORAGE_TYPE}", "storage.type " + storageType)
-        puts "using storage.type: #{storageType} for tail plugin"
-      else
-        new_contents = new_contents.gsub("\n    #${AZMON_STORAGE_TYPE}\n", "\n")
-      end
-
-      if is_high_log_scale_mode?
-        new_contents = new_contents.gsub("#${AZMON_MAX_STORAGE_CHUNKS_UP}", "storage.max_chunks_up " + @default_high_log_scale_max_storage_chunks_up)
-        puts "using storage.max_chunks_up: #{@default_high_log_scale_max_storage_chunks_up} for tail plugin"
-      elsif !storageMaxChunksUp.nil? && !storageMaxChunksUp.empty?
-        new_contents = new_contents.gsub("#${AZMON_MAX_STORAGE_CHUNKS_UP}", "storage.max_chunks_up " + storageMaxChunksUp)
-        puts "using storage.max_chunks_up: #{storageMaxChunksUp} for tail plugin"
-      else
-        new_contents = new_contents.gsub("\n    #${AZMON_MAX_STORAGE_CHUNKS_UP}\n", "\n")
-      end
-  rescue => err
-     puts "config::substituteHighLogScaleConfig failed with an error: #{err}"
-  end
-  return new_contents
-end
-
 def substituteFluentBitPlaceHolders
   begin
     # Replace the fluentbit config file with custom values if present
@@ -134,18 +79,8 @@ def substituteFluentBitPlaceHolders
     windowsFluentBitDisabled = ENV["AZMON_WINDOWS_FLUENT_BIT_DISABLED"]
     kubernetesMetadataCollection = ENV["AZMON_KUBERNETES_METADATA_ENABLED"]
     annotationBasedLogFiltering = ENV["AZMON_ANNOTATION_BASED_LOG_FILTERING"]
-    storageMaxChunksUp = ENV["FBIT_STORAGE_MAX_CHUNKS_UP"]
-    storageType = ENV["FBIT_STORAGE_TYPE"]
-    enableFbitThreading = ENV["ENABLE_FBIT_THREADING"]
-
 
-    serviceInterval = @default_service_interval
-    if is_high_log_scale_mode?
-      serviceInterval = @default_high_log_scale_service_interval
-      puts " using Flush interval: #{serviceInterval}"
-    elsif (!interval.nil? && is_number?(interval) && interval.to_i > 0)
-      serviceInterval = interval
-    end
+    serviceInterval = (!interval.nil? && is_number?(interval) && interval.to_i > 0) ? interval : @default_service_interval
     serviceIntervalSetting = "Flush         " + serviceInterval
 
     tailBufferChunkSize = (!bufferChunkSize.nil? && is_number?(bufferChunkSize) && bufferChunkSize.to_i > 0) ? bufferChunkSize : nil
@@ -180,8 +115,6 @@ def substituteFluentBitPlaceHolders
       new_contents = new_contents.gsub("\n    ${TAIL_IGNORE_OLDER}\n", "\n")
     end
 
-    new_contents = substituteHighLogScaleConfig(enableFbitThreading, storageType,  storageMaxChunksUp, new_contents)
-
     if !kubernetesMetadataCollection.nil? && kubernetesMetadataCollection.to_s.downcase == "true"
       new_contents = new_contents.gsub("#${KubernetesFilterEnabled}", "")
     end
@@ -202,7 +135,6 @@ def substituteFluentBitPlaceHolders
 
     puts "config::Starting to substitute the placeholders in fluent-bit-common.conf file for log collection"
     text = File.read(@fluent_bit_common_conf_path)
-    text = substituteStorageTotalLimitSize(text)
     new_contents = substituteMultiline(multilineLogging, stacktraceLanguages, text)
     File.open(@fluent_bit_common_conf_path, "w") { |file| file.puts new_contents }
     puts "config::Successfully substituted the placeholders in fluent-bit-common.conf file"

diff --git a/build/common/installer/scripts/tomlparser-agent-config.rb b/build/common/installer/scripts/tomlparser-agent-config.rb
@@ -1,5 +1,6 @@
 #!/usr/local/bin/ruby
 
+
 @os_type = ENV["OS_TYPE"]
 require "tomlrb"
 
@@ -67,9 +68,6 @@
 @outputForwardWorkers = 10
 @outputForwardRetryLimit = 10
 @requireAckResponse = "false"
-@fbitStorageMaxChunksUp = 0
-@fbitStorageType = ""
-@enableFluentBitThreading = false
 
 # configmap settings related to mdsd
 @mdsdMonitoringMaxEventRate = 0
@@ -103,8 +101,6 @@
 @waittime_port_25226 = 45
 @waittime_port_25228 = 120
 @waittime_port_25229 = 45
-@waittime_port_13000 = 45 # default waittime for AMACA data port
-@waittime_port_12563 = 45 # default waittime for AMACA config port
 
 def is_number?(value)
   true if Integer(value) rescue false
@@ -242,23 +238,6 @@ def populateSettingValuesFromConfigMap(parsedConfig)
           @enableFbitInternalMetrics = true
           puts "Using config map value: enable_internal_metrics = #{@enableFbitInternalMetrics}"
         end
-
-        fbitStorageMaxChunksUp = fbit_config[:storage_max_chunks_up]
-        if is_valid_number?(fbitStorageMaxChunksUp)
-            @fbitStorageMaxChunksUp = fbitStorageMaxChunksUp.to_i
-            puts "Using config map value: fbitStorageMaxChunksUp  = #{@fbitStorageMaxChunksUp}"
-        end
-        fbitStorageType = fbit_config[:storage_type]
-        if !fbitStorageType.nil? && !fbitStorageType.empty? && ["memory", "filesystem"].include?(fbitStorageType)
-            @fbitStorageType = fbitStorageType
-            puts "Using config map value: fbitStorageType  = #{@fbitStorageType}"
-        end
-
-        enableFluentBitThreading = fbit_config[:enable_threading]
-        if !enableFluentBitThreading.nil? && enableFluentBitThreading.strip.casecmp("true") == 0
-          @enableFluentBitThreading = enableFluentBitThreading
-          puts "Using config map value: enableFluentBitThreading  = #{@enableFluentBitThreading}"
-        end
       end
 
       # fbit forward plugins geneva settings per tenant
@@ -412,16 +391,6 @@ def populateSettingValuesFromConfigMap(parsedConfig)
           @waittime_port_25229 = waittime.to_i
           puts "Using config map value: WAITTIME_PORT_25229 = #{@waittime_port_25229}"
         end
-        waittime = network_listener_waittime_config[:tcp_port_13000]
-        if is_valid_waittime?(waittime, @waittime_port_13000)
-          @waittime_port_13000 = waittime.to_i
-          puts "Using config map value: WAITTIME_PORT_13000 = #{@waittime_port_13000}"
-        end
-        waittime = network_listener_waittime_config[:tcp_port_12563]
-        if is_valid_waittime?(waittime, @waittime_port_12563)
-          @waittime_port_12563 = waittime.to_i
-          puts "Using config map value: WAITTIME_PORT_12563 = #{@waittime_port_12563}"
-        end
       end
     end
   rescue => errorStr
@@ -471,18 +440,6 @@ def populateSettingValuesFromConfigMap(parsedConfig)
     file.write("export FBIT_TAIL_IGNORE_OLDER=#{@fbitTailIgnoreOlder}\n")
   end
 
-  if @fbitStorageMaxChunksUp > 0
-    file.write("export FBIT_STORAGE_MAX_CHUNKS_UP=#{@fbitStorageMaxChunksUp}\n")
-  end
-
-  if !@fbitStorageType.nil? && !@fbitStorageType.empty?
-    file.write("export FBIT_STORAGE_TYPE=#{@fbitStorageType}\n")
-  end
-
-  if @enableFluentBitThreading
-    file.write("export ENABLE_FBIT_THREADING=#{@enableFluentBitThreading}\n")
-  end
-
   if @storageTotalLimitSizeMB > 0
     file.write("export STORAGE_TOTAL_LIMIT_SIZE_MB=#{@storageTotalLimitSizeMB.to_s + "M"}\n")
   end
@@ -552,8 +509,6 @@ def populateSettingValuesFromConfigMap(parsedConfig)
   file.write("export WAITTIME_PORT_25226=#{@waittime_port_25226}\n")
   file.write("export WAITTIME_PORT_25228=#{@waittime_port_25228}\n")
   file.write("export WAITTIME_PORT_25229=#{@waittime_port_25229}\n")
-  file.write("export WAITTIME_PORT_13000=#{@waittime_port_13000}\n")
-  file.write("export WAITTIME_PORT_12563=#{@waittime_port_12563}\n")
 
   # Close file after writing all environment variables
   file.close
@@ -593,20 +548,6 @@ def get_command_windows(env_variable_name, env_variable_value)
       commands = get_command_windows("FBIT_TAIL_IGNORE_OLDER", @fbitTailIgnoreOlder)
       file.write(commands)
     end
-
-    if @fbitStorageMaxChunksUp > 0
-      commands = get_command_windows("FBIT_STORAGE_MAX_CHUNKS_UP", @fbitStorageMaxChunksUp)
-      file.write(commands)
-    end
-    if !@fbitStorageType.nil? && !@fbitStorageType.empty?
-      commands = get_command_windows("FBIT_STORAGE_TYPE", @fbitStorageType)
-    end
-
-    if @enableFluentBitThreading
-      commands = get_command_windows("ENABLE_FBIT_THREADING", @enableFluentBitThreading)
-      file.write(commands)
-    end
-
     if @promFbitChunkSize > 0
       commands = get_command_windows("AZMON_FBIT_CHUNK_SIZE", @promFbitChunkSize.to_s + "m")
       file.write(commands)

diff --git a/build/common/installer/scripts/tomlparser-common-agent-config.rb b/build/common/installer/scripts/tomlparser-common-agent-config.rb
@@ -10,7 +10,6 @@
 
 @disableTelemetry = false
 @logEnableKubernetesMetadataCacheTTLSeconds = 60
-@enableHighLogScaleMode = false
 
 def is_windows?
   return !@os_type.nil? && !@os_type.empty? && @os_type.strip.casecmp("windows") == 0
@@ -73,14 +72,6 @@ def populateSettingValuesFromConfigMap(parsedConfig)
         end
       end
     end
-    if !parsedConfig.nil? && !parsedConfig[:agent_settings].nil?
-      high_log_scale = parsedConfig[:agent_settings][:high_log_scale]
-      if !high_log_scale.nil? && !high_log_scale[:enabled].nil?
-        @enableHighLogScaleMode = high_log_scale[:enabled]
-        puts "Using config map value: enabled = #{@enableHighLogScaleMode} for high log scale config"
-      end
-    end
-
   rescue => errorStr
     puts "config::error:Exception while reading config settings for agent configuration setting - #{errorStr}, using defaults"
   end
@@ -111,12 +102,6 @@ def get_command_windows(env_variable_name, env_variable_value)
       commands = get_command_windows("DISABLE_TELEMETRY", @disableTelemetry)
       file.write(commands)
     end
-
-    if @enableHighLogScaleMode
-      commands = get_command_windows("ENABLE_HIGH_LOG_SCALE_MODE", @enableHighLogScaleMode)
-      file.write(commands)
-    end
-
     commands = get_command_windows("AZMON_KUBERNETES_METADATA_CACHE_TTL_SECONDS", @logEnableKubernetesMetadataCacheTTLSeconds)
     file.write(commands)
     # Close file after writing all environment variables
@@ -133,9 +118,6 @@ def get_command_windows(env_variable_name, env_variable_value)
     if @disableTelemetry
       file.write("export DISABLE_TELEMETRY=#{@disableTelemetry}\n")
     end
-    if @enableHighLogScaleMode
-      file.write("export ENABLE_HIGH_LOG_SCALE_MODE=#{@enableHighLogScaleMode}\n")
-    end
     file.write("export AZMON_KUBERNETES_METADATA_CACHE_TTL_SECONDS=#{@logEnableKubernetesMetadataCacheTTLSeconds}\n")
     # Close file after writing all environment variables
     file.close

diff --git a/build/linux/installer/conf/PA.json b/build/linux/installer/conf/PA.json
diff --git a/build/linux/installer/conf/fluent-bit-common.conf b/build/linux/installer/conf/fluent-bit-common.conf
@@ -26,20 +26,6 @@
     Skip_Long_Lines On
     Ignore_Older 2m
 
-[INPUT]
-    Name tail
-    Alias ama-logs_amaca_log_tail
-    Tag oms.container.log.flbplugin.amaca.*
-    Path /var/opt/microsoft/linuxmonagent/amaca/log/*.log
-    Read_from_Head true
-    DB /var/opt/microsoft/docker-cimprov/state/amaca-ai.db
-    DB.Sync Off
-    Parser docker
-    Mem_Buf_Limit 1m
-    Path_Key filepath
-    Skip_Long_Lines On
-    Ignore_Older 2m
-
 [INPUT]
     Name tail
     Alias ama-logs_termination_log_tail
@@ -91,17 +77,10 @@
     Match oms.container.log.flbplugin.mdsd.qos.*
     regex log ^MaODSRequest,https.*_LogManagement_LINUX_SYSLOGS_BLOB
 
-[FILTER]
-    Name grep
-    Alias ama-logs_amaca_log_tail
-    Match oms.container.log.flbplugin.amaca.*
-    Exclude log Information
-
 [OUTPUT]
     Name                            oms
     Alias                           oms_output
     EnableTelemetry                 true
     Retry_Limit                     10
     TelemetryPushIntervalSeconds    300
     Match                           oms.container.*
-    #${AZMON_STORAGE_TOTAL_LIMIT_SIZE_MB}
diff --git a/build/linux/installer/conf/fluent-bit.conf b/build/linux/installer/conf/fluent-bit.conf
@@ -3,7 +3,6 @@
     ${SERVICE_FLUSH_INTERVAL}
     HTTP_Server   Off
     Daemon        Off
-    #${AZMON_MAX_STORAGE_CHUNKS_UP}
     storage.path  /var/opt/microsoft/docker-cimprov/state/flbstore/
     storage.sync  normal
     storage.checksum off
@@ -25,7 +24,6 @@
     DB /var/log/omsagent-fblogs.db
     DB.Sync Off
     Parser docker
-    #${AZMON_STORAGE_TYPE}
     ${TAIL_MEM_BUF_LIMIT}
     ${TAIL_BUFFER_CHUNK_SIZE}
     ${TAIL_BUFFER_MAX_SIZE}
@@ -35,7 +33,6 @@
     Skip_Long_Lines On
     ${TAIL_IGNORE_OLDER}
     Exclude_Path ${AZMON_CLUSTER_LOG_TAIL_EXCLUDE_PATH}
-    #${AZMON_TAIL_THREADED}
 
 #NOTE: Multiline should be the first filter https://docs.fluentbit.io/manual/pipeline/filters/multiline-stacktrace
 #${MultilineEnabled}[FILTER]

diff --git a/build/linux/installer/datafiles/base_container.data b/build/linux/installer/datafiles/base_container.data
@@ -45,8 +45,6 @@ MAINTAINER:              'Microsoft Corporation'
 
 /opt/tomlparser-geneva-config.rb;                                       build/common/installer/scripts/tomlparser-geneva-config.rb;     755; root; root
 
-/etc/opt/microsoft/azuremonitoragent/amacoreagent/PA.json;              build/linux/installer/conf/PA.json;                             755; root; root
-
 /opt/fluent-bit-geneva-conf-customizer.rb;                            build/common/installer/scripts/fluent-bit-geneva-conf-customizer.rb;     755; root; root
 /etc/opt/microsoft/docker-cimprov/fluent-bit-input.conf;			    build/linux/installer/conf/fluent-bit-input.conf;                644; root; root
 /etc/opt/microsoft/docker-cimprov/fluent-bit-common.conf;			    build/linux/installer/conf/fluent-bit-common.conf;                644; root; root
@@ -183,10 +181,6 @@ MAINTAINER:              'Microsoft Corporation'
 /var/opt/microsoft/docker-cimprov/state/ContainerInventory; 755; root; root
 /var/opt/microsoft/docker-cimprov/log;                  755; root; root
 /var/opt/microsoft/linuxmonagent/log;                   755; root; root
-/var/opt/microsoft/linuxmonagent/amaca;                 755; root; root
-/var/opt/microsoft/linuxmonagent/amaca/log;             755; root; root
-
-/etc/opt/microsoft/azuremonitoragent/amacoreagent;      755; root; root
 
 /opt/fluent-bit;                          755; root; root;sysdir
 /opt/fluent-bit/bin;                      755; root; root;sysdir