Fixes #17175 - Added support for memory monitoring.

A world will be terminated if memory limit is exceeded.
This will cause the running process to be killed.
Since we are monitoring our processes, a new process will
be spawned.

Relies on Dynflow/dynflow#211

bin/dynflow-executor

@@ -2,45 +2,70 @@
 
 require 'optparse'
 
-options = { foreman_root: Dir.pwd }
+class ArgvParser
+  attr_reader :options, :command
 
-opts = OptionParser.new do |opts|
-  opts.banner = <<BANNER
+  def initialize(argv, file)
+    @options = { foreman_root: Dir.pwd }
+
+    opts = OptionParser.new do |opts|
+      opts.banner = banner(file)
+
+      opts.on('-h', '--help', 'Show this message') do
+        puts opts
+        exit 1
+      end
+      opts.on('-f', '--foreman-root=PATH', "Path to Foreman Rails root path. By default '#{@options[:foreman_root]}'") do |path|
+        @options[:foreman_root] = path
+      end
+      opts.on('-c', '--executors-count=COUNT', 'Number of parallel executors to spawn. Overrides EXECUTORS_COUNT environment varaible.') do |count|
+        @options[:executors_count] = count.to_i
+      end
+      opts.on('-m', '--memory-limit=SIZE', 'Limits the amount of memory an executor can consume. Overrides EXECUTOR_MEMORY_LIMIT environment varaible. You can use kb, mb, gb') do |size|
+        @options[:memory_limit] = size
+      end
+      opts.on('--executor-memory-init-delay=SECONDS', 'Start memory polling after SECONDS. Overrides EXECUTOR_MEMORY_MONITOR_DELAY environment varaible.') do |seconds|
+        @options[:memory_init_delay] = seconds.to_i
+      end
+      opts.on('--executor-memory-polling-interval=SECONDS', 'Check for memory useage every SECONDS sec. Overrides EXECUTOR_MEMORY_MONITOR_INTERVAL environment varaible.') do |seconds|
+        @options[:memory_polling_interval] = seconds.to_i
+      end
+    end
+
+    args = opts.parse!(argv)
+    @command = args.first || 'run'
+  end
+
+  def banner(file)
+    banner = <<BANNER
 Run Dynflow executor for Foreman tasks.
 
-Usage: #{File.basename($PROGRAM_NAME)} [options] ACTION"
+Usage: #{File.basename(file)} [options] ACTION"
 
 ACTION can be one of:
 
-  * start   - start the executor on background. It creates these files
-              in tmp/pid directory:
+* start   - start the executor on background. It creates these files
+          in tmp/pid directory:
 
-                * dynflow_executor_monitor.pid - pid of monitor ensuring
-                                                 the executor keeps running
-                * dynflow_executor.pid         - pid of the executor itself
-                * dynflow_executor.output      - stdout of the executor
-  * stop    - stops the running executor
-  * restart - restarts the running executor
-  * run     - run the executor in foreground
+            * dynflow_executor_monitor.pid - pid of monitor ensuring
+                                             the executor keeps running
+            * dynflow_executor.pid         - pid of the executor itself
+            * dynflow_executor.output      - stdout of the executor
+* stop    - stops the running executor
+* restart - restarts the running executor
+* run     - run the executor in foreground
 
 BANNER
-
-  opts.on('-h', '--help', 'Show this message') do
-    puts opts
-    exit 1
-  end
-  opts.on('-f', '--foreman-root=PATH', "Path to Foreman Rails root path. By default '#{options[:foreman_root]}'") do |path|
-    options[:foreman_root] = path
-  end
-  opts.on('-c', '--executors-count=COUNT', 'Number of parallel executors to spawn. Overrides EXECUTORS_COUNT environment varaible.') do |count|
-    options[:executors_count] = count.to_i
+    banner
   end
 end
 
-args = opts.parse!(ARGV)
-command = args.first || 'run'
+# run the script if it's executed explicitly
+if $PROGRAM_NAME == __FILE__
+  parser = ArgvParser.new(ARGV, $PROGRAM_NAME)
 
-app_file = File.expand_path('./config/application', options[:foreman_root])
-require app_file
+  app_file = File.expand_path('./config/application', parser.options[:foreman_root])
+  require app_file
 
-ForemanTasks::Dynflow::Daemon.new.run_background(command, options)
+  ForemanTasks::Dynflow::Daemon.new.run_background(parser.command, parser.options)
+end

deploy/foreman-tasks.sysconfig

@@ -12,3 +12,15 @@ RUBY_GC_MALLOC_LIMIT_MAX=16000100
 RUBY_GC_MALLOC_LIMIT_GROWTH_FACTOR=1.1
 RUBY_GC_OLDMALLOC_LIMIT=16000100
 RUBY_GC_OLDMALLOC_LIMIT_MAX=16000100
+
+#Set the number of executors you want to run
+#EXECUTORS_COUNT=1
+
+#Set memory limit for executor process, before it's restarted automatically
+#EXECUTOR_MEMORY_LIMIT=2gb
+
+#Set delay before first memory polling to let executor initialize (in sec)
+#EXECUTOR_MEMORY_MONITOR_DELAY=7200 #default: 2 hours
+
+#Set memory polling interval, process memory will be checked every N seconds.
+#EXECUTOR_MEMORY_MONITOR_INTERVAL=60

foreman-tasks.gemspec

@@ -29,9 +29,10 @@ DESC
   s.extra_rdoc_files = Dir['README*', 'LICENSE']
 
   s.add_dependency "foreman-tasks-core"
-  s.add_dependency "dynflow", '~> 0.8.17'
+  s.add_dependency "dynflow", '~> 0.8.23'
   s.add_dependency "sequel" # for Dynflow process persistence
   s.add_dependency "sinatra" # for Dynflow web console
   s.add_dependency "daemons" # for running remote executor
   s.add_dependency "parse-cron", '~> 0.1.4'
+  s.add_dependency "get_process_mem" # for memory polling
 end

lib/foreman_tasks/dynflow.rb

@@ -9,8 +9,9 @@ class Dynflow
     require 'foreman_tasks/dynflow/daemon'
     require 'foreman_tasks/dynflow/console_authorizer'
 
-    def initialize
+    def initialize(world_class = nil)
       @required = false
+      @world_class = world_class
     end
 
     def config
@@ -37,7 +38,7 @@ def initialize!
       if config.lazy_initialization && defined?(PhusionPassenger)
         config.dynflow_logger.warn('ForemanTasks: lazy loading with PhusionPassenger might lead to unexpected results')
       end
-      config.initialize_world.tap do |world|
+      init_world.tap do |world|
         @world = world
 
         unless config.remote?
@@ -119,5 +120,12 @@ def eager_load_actions!
     def loaded_paths
       @loaded_paths ||= Set.new
     end
+
+    private
+
+    def init_world
+      return config.initialize_world(@world_class) if @world_class
+      config.initialize_world
+    end
   end
 end

lib/foreman_tasks/dynflow/configuration.rb

@@ -59,6 +59,8 @@ def run_on_init_hooks(world)
 
     def initialize_world(world_class = ::Dynflow::World)
       world_class.new(world_config)
+    rescue ArgumentError
+      world_class.new
     end
 
     # No matter what config.remote says, when the process is marked as executor,

lib/foreman_tasks/dynflow/daemon.rb

@@ -1,32 +1,59 @@
 require 'fileutils'
+require 'daemons'
+require 'get_process_mem'
+require 'dynflow/watchers/memory_consumption_watcher'
 
 module ForemanTasks
   class Dynflow::Daemon
+    attr_reader :memory_info_provider_class, :dynflow_memory_watcher_class,
+                :daemons_class
+
+    # make Daemon dependency injection ready for testing purposes
+    def initialize(
+      memory_info_provider_class = GetProcessMem,
+      dynflow_memory_watcher_class = ::Dynflow::Watchers::MemoryConsumptionWatcher,
+      daemons_class = ::Daemons
+    )
+      @memory_info_provider_class = memory_info_provider_class
+      @dynflow_memory_watcher_class = dynflow_memory_watcher_class
+      @daemons_class = daemons_class
+    end
+
     # load the Rails environment and initialize the executor
     # in this thread.
-    def run(foreman_root = Dir.pwd)
+    def run(foreman_root = Dir.pwd, options = {})
       STDERR.puts('Starting Rails environment')
       foreman_env_file = File.expand_path('./config/environment.rb', foreman_root)
       unless File.exist?(foreman_env_file)
         raise "#{foreman_root} doesn't seem to be a foreman root directory"
       end
+
+      Foreman::Logging.logger('foreman-tasks').info("Starting dynflow with the following options: #{options}")
+
       ForemanTasks.dynflow.executor!
+
+      if options[:memory_limit] > 0
+        ForemanTasks.dynflow.config.on_init do |world|
+          memory_watcher = initialize_memory_watcher(world, options[:memory_limit], options)
+          world.terminated.on_completion do
+            STDERR.puts("World has been terminated")
+            memory_watcher = nil # the object can be disposed
+          end
+        end
+      end
+
       require foreman_env_file
-      STDERR.puts('Everything ready')
+      STDERR.puts("Everything ready for world: #{(ForemanTasks.dynflow.initialized? ? ForemanTasks.dynflow.world : nil)}")
+      memory_info_provider = memory_info_provider_class.new
+      # Let it have 10M more than it has.
+      STDERR.puts("Memory consumption: #{memory_info_provider.bytes}")
       sleep
     ensure
       STDERR.puts('Exiting')
     end
 
     # run the executor as a daemon
     def run_background(command = 'start', options = {})
-      default_options = { foreman_root: Dir.pwd,
-                          process_name: 'dynflow_executor',
-                          pid_dir: "#{Rails.root}/tmp/pids",
-                          log_dir: File.join(Rails.root, 'log'),
-                          wait_attempts: 300,
-                          wait_sleep: 1,
-                          executors_count: (ENV['EXECUTORS_COUNT'] || 1).to_i }
       options = default_options.merge(options)
       FileUtils.mkdir_p(options[:pid_dir])
       begin
@@ -42,18 +69,13 @@ def run_background(command = 'start', options = {})
       STDERR.puts("Dynflow Executor: #{command} in progress")
 
       options[:executors_count].times do
-        Daemons.run_proc(options[:process_name],
-                         :multiple => true,
-                         :dir => options[:pid_dir],
-                         :log_dir => options[:log_dir],
-                         :dir_mode => :normal,
-                         :monitor => true,
-                         :log_output => true,
-                         :log_output_syslog => true,
-                         :ARGV => [command]) do |*_args|
+        daemons_class.run_proc(
+          options[:process_name],
+          daemons_options(command, options)
+        ) do |*_args|
           begin
             ::Logging.reopen
-            run(options[:foreman_root])
+            run(options[:foreman_root], options)
           rescue => e
             STDERR.puts e.message
             Foreman::Logging.exception('Failed running foreman-tasks daemon', e)
@@ -68,5 +90,59 @@ def run_background(command = 'start', options = {})
     def world
       ForemanTasks.dynflow.world
     end
+
+    private
+
+    def daemons_options(command, options)
+      {
+        :multiple => true,
+        :dir => options[:pid_dir],
+        :log_dir => options[:log_dir],
+        :dir_mode => :normal,
+        :monitor => true,
+        :log_output => true,
+        :log_output_syslog => true,
+        :monitor_interval => [options[:memory_polling_interval] / 2, 30].min,
+        :ARGV => [command]
+      }
+    end
+
+    def default_options
+      {
+        foreman_root: Dir.pwd,
+        process_name: 'dynflow_executor',
+        pid_dir: "#{Rails.root}/tmp/pids",
+        log_dir: File.join(Rails.root, 'log'),
+        wait_attempts: 300,
+        wait_sleep: 1,
+        executors_count: (ENV['EXECUTORS_COUNT'] || 1).to_i,
+        memory_limit: begin
+                        ENV['EXECUTOR_MEMORY_LIMIT'].to_gb.gigabytes
+                      rescue RuntimeError
+                        ENV['EXECUTOR_MEMORY_LIMIT'].to_i
+                      end,
+        memory_init_delay: (ENV['EXECUTOR_MEMORY_MONITOR_DELAY'] || 7200).to_i, # 2 hours
+        memory_polling_interval: (ENV['EXECUTOR_MEMORY_MONITOR_INTERVAL'] || 60).to_i
+      }
+    end
+
+    def initialize_memory_watcher(world, memory_limit, options)
+      watcher_options = {}
+      watcher_options[:polling_interval] = options[:memory_polling_interval]
+      watcher_options[:initial_wait] = options[:memory_init_delay]
+      watcher_options[:memory_checked_callback] = ->(current_memory, memory_limit) { log_memory_within_limit(current_memory, memory_limit) }
+      watcher_options[:memory_limit_exceeded_callback] = ->(current_memory, memory_limit) { log_memory_limit_exceeded(current_memory, memory_limit) }
+      dynflow_memory_watcher_class.new(world, memory_limit, watcher_options)
+    end
+
+    def log_memory_limit_exceeded(current_memory, memory_limit)
+      message = "Memory level exceeded, registered #{current_memory} bytes, which is greater than #{memory_limit} limit."
+      Foreman::Logging.logger('foreman-tasks').error(message)
+    end
+
+    def log_memory_within_limit(current_memory, memory_limit)
+      message = "Memory level OK, registered #{current_memory} bytes, which is less than #{memory_limit} limit."
+      Foreman::Logging.logger('foreman-tasks').debug(message)
+    end
   end
 end

test/unit/config/environment.rb

@@ -0,0 +1 @@
+# dummy appllication.rb - for unit testing dynflow-executor