[webui] calculate the worker status directly in the API

src/api/app/controllers/status_controller.rb

@@ -7,15 +7,15 @@ class PermissionDeniedError < APIException
   end
 
   def list_messages
-    @messages = StatusMessage.alive.limit(params[:limit]).order("created_at DESC").includes(:user)
+    @messages = StatusMessage.alive.limit(params[:limit]).order('created_at DESC').includes(:user)
     @count = @messages.size
-    render xml: render_to_string(partial: "messages")
+    render xml: render_to_string(partial: 'messages')
   end
 
   def show_message
     @messages = [StatusMessage.find(params[:id])]
     @count = 1
-    render xml: render_to_string(partial: "messages")
+    render xml: render_to_string(partial: 'messages')
   end
 
   class CreatingMessagesError < APIException
@@ -54,39 +54,15 @@ def save_new_message(msg)
   def delete_message
     # check permissions
     unless permissions.status_message_create
-      raise PermissionDeniedError.new "message cannot be deleted, you have not sufficient permissions"
+      raise PermissionDeniedError.new 'message cannot be deleted, you have not sufficient permissions'
     end
 
     StatusMessage.find(params[:id]).delete
     render_ok
   end
 
   def workerstatus
-    begin
-      data = Rails.cache.read('workerstatus')
-    rescue Zlib::GzipFile::Error
-      data = nil
-    end
-    data=ActiveXML::Node.new(data || update_workerstatus_cache)
-    prjs=Hash.new
-    data.each_building do |b|
-      prjs[b.project] = 1
-    end
-    names = Hash.new
-    # now try to find those we have a match for (the rest are hidden from you
-    Project.where(name: prjs.keys).pluck(:name).each do |n|
-      names[n] = 1
-    end
-    data.each_building do |b|
-      # no prj -> we are not allowed
-      unless names.has_key? b.project
-        logger.debug "workerstatus2clean: hiding #{b.project} for user #{User.current.login}"
-        b.set_attribute('project', '---')
-        b.set_attribute('repository', '---')
-        b.set_attribute('package', '---')
-      end
-    end
-    send_data data.dump_xml
+    send_data WorkerStatus.hidden.dump_xml
   end
 
   def history
@@ -103,84 +79,6 @@ def history
     @values = StatusHistory.where("time >= ? AND \`key\` = ?", starttime, params[:key]).pluck(:time, :value).collect { |time, value| [time.to_i, value.to_f] }
   end
 
-  def save_value_line(e, prefix)
-    line = StatusHistory.new
-    line.time = @mytime
-    line.key = "#{prefix}_#{e['arch']}"
-    line.value = e['jobs']
-    line.save
-  end
-
-  def update_workerstatus_cache
-    # do not add hiding in here - this is purely for statistics
-    ret=backend_get('/build/_workerstatus')
-    data=Xmlhash.parse(ret)
-
-    @mytime = Time.now.to_i
-    Rails.cache.write('workerstatus', ret, expires_in: 3.minutes)
-    Rails.cache.write('workerhash', data, expires_in: 3.minutes)
-    StatusHistory.transaction do
-      data.elements('blocked') do |e|
-        save_value_line(e, 'blocked')
-      end
-      data.elements('waiting') do |e|
-        save_value_line(e, 'waiting')
-      end
-      data.elements('partition') do |p|
-        p.elements('daemon') do |daemon|
-          parse_daemon_infos(daemon)
-        end
-      end
-      parse_worker_infos(data)
-    end
-    ret
-  end
-
-  def parse_daemon_infos(daemon)
-    return unless daemon['type'] == 'scheduler'
-    arch = daemon['arch']
-    # FIXME2.5: The current architecture model is a gross hack, not connected at all
-    #           to the backend config.
-    a=Architecture.find_by_name(arch)
-    if a
-      a.available=true
-      a.save
-    end
-    queue = daemon.get('queue')
-    return unless queue
-    StatusHistory.create :time => @mytime, :key => "squeue_high_#{arch}", :value => queue['high'].to_i
-    StatusHistory.create :time => @mytime, :key => "squeue_next_#{arch}", :value => queue['next'].to_i
-    StatusHistory.create :time => @mytime, :key => "squeue_med_#{arch}", :value => queue['med'].to_i
-    StatusHistory.create :time => @mytime, :key => "squeue_low_#{arch}", :value => queue['low'].to_i
-  end
-
-  def parse_worker_infos(data)
-    allworkers = Hash.new
-    workers = Hash.new
-    %w{building idle}.each do |state|
-      data.elements(state) do |e|
-        id=e['workerid']
-        if workers.has_key? id
-          logger.debug 'building+idle worker'
-          next
-        end
-        workers[id] = 1
-        key = state + '_' + e['hostarch']
-        allworkers["building_#{e['hostarch']}"] ||= 0
-        allworkers["idle_#{e['hostarch']}"] ||= 0
-        allworkers[key] = allworkers[key] + 1
-      end
-    end
-
-    allworkers.each do |key, value|
-      line = StatusHistory.new
-      line.time = @mytime
-      line.key = key
-      line.value = value
-      line.save
-    end
-  end
-
   # move to models?
   def role_from_cache(role_id)
     @rolecache[role_id] || (@rolecache[role_id] = Role.find(role_id).title)
@@ -238,7 +136,7 @@ def bsrequest
                            package: action.source_package,
                            expand: 1, rev: action.source_rev)
     @result = PackageBuildStatus.new(spkg).result(target_project: tproj, srcmd5: dir['srcmd5'])
-    render xml: render_to_string(partial: "bsrequest")
+    render xml: render_to_string(partial: 'bsrequest')
   end
 
   class NotFoundError < APIException

src/api/app/models/worker_status.rb

@@ -0,0 +1,105 @@
+class WorkerStatus
+
+  def self.hidden
+    mydata = Rails.cache.read('workerstatus')
+    ws = ActiveXML::Node.new(mydata || ActiveXML.backend.direct_http('/build/_workerstatus'))
+    prjs=Hash.new
+    ws.each_building do |b|
+      prjs[b.project] = 1
+    end
+    names = Hash.new
+    # now try to find those we have a match for (the rest are hidden from you
+    Project.where(name: prjs.keys).pluck(:name).each do |n|
+      names[n] = 1
+    end
+    ws.each_building do |b|
+      # no prj -> we are not allowed
+      unless names.has_key? b.project
+        Rails.logger.debug "workerstatus2clean: hiding #{b.project} for user #{User.current.login}"
+        b.set_attribute('project', '---')
+        b.set_attribute('repository', '---')
+        b.set_attribute('package', '---')
+      end
+    end
+    ws
+  end
+
+  def update_workerstatus_cache
+    # do not add hiding in here - this is purely for statistics
+    ret=ActiveXML.backend.direct_http('/build/_workerstatus')
+    wdata=Xmlhash.parse(ret)
+    @mytime = Time.now.to_i
+    Rails.cache.write('workerstatus', ret, expires_in: 3.minutes)
+    StatusHistory.transaction do
+      wdata.elements('blocked') do |e|
+        save_value_line(e, 'blocked')
+      end
+      wdata.elements('waiting') do |e|
+        save_value_line(e, 'waiting')
+      end
+      wdata.elements('partition') do |p|
+        p.elements('daemon') do |daemon|
+          parse_daemon_infos(daemon)
+        end
+      end
+      parse_worker_infos(wdata)
+    end
+    ret
+  end
+
+  private
+
+  def parse_daemon_infos(daemon)
+    return unless daemon['type'] == 'scheduler'
+    arch = daemon['arch']
+    # FIXME2.5: The current architecture model is a gross hack, not connected at all
+    #           to the backend config.
+    a=Architecture.find_by_name(arch)
+    if a
+      a.available=true
+      a.save
+    end
+    queue = daemon.get('queue')
+    return unless queue
+    StatusHistory.create :time => @mytime, :key => "squeue_high_#{arch}", :value => queue['high'].to_i
+    StatusHistory.create :time => @mytime, :key => "squeue_next_#{arch}", :value => queue['next'].to_i
+    StatusHistory.create :time => @mytime, :key => "squeue_med_#{arch}", :value => queue['med'].to_i
+    StatusHistory.create :time => @mytime, :key => "squeue_low_#{arch}", :value => queue['low'].to_i
+  end
+
+  def parse_worker_infos(wdata)
+    allworkers = Hash.new
+    workers = Hash.new
+    %w{building idle}.each do |state|
+      wdata.elements(state) do |e|
+        id=e['workerid']
+        if workers.has_key? id
+          Rails.logger.debug 'building+idle worker'
+          next
+        end
+        workers[id] = 1
+        key = state + '_' + e['hostarch']
+        allworkers["building_#{e['hostarch']}"] ||= 0
+        allworkers["idle_#{e['hostarch']}"] ||= 0
+        allworkers[key] = allworkers[key] + 1
+      end
+    end
+
+    allworkers.each do |key, value|
+      line = StatusHistory.new
+      line.time = @mytime
+      line.key = key
+      line.value = value
+      line.save
+    end
+  end
+
+  def save_value_line(e, prefix)
+    line = StatusHistory.new
+    line.time = @mytime
+    line.key = "#{prefix}_#{e['arch']}"
+    line.value = e['jobs']
+    line.save
+  end
+
+end

src/api/config/clock.rb

@@ -8,10 +8,8 @@
 include Clockwork
 
 every(30.seconds, 'status.refresh') do
-  Rails.logger.debug "Refresh worker status"
-  c = StatusController.new
   # this should be fast, so don't delay
-  c.update_workerstatus_cache
+  WorkerStatus.new.update_workerstatus_cache
 end
 
 every(1.hour, 'refresh issues') do
@@ -29,7 +27,7 @@
 
 every(1.day, 'optimize history', thread: true) do 
   ActiveRecord::Base.connection_pool.with_connection do |sql|
-    sql.execute "optimize table status_histories;"
+    sql.execute 'optimize table status_histories;'
   end
 end
 

src/api/config/environments/test.rb

@@ -18,7 +18,7 @@
   # ActionMailer::Base.deliveries array.
   config.action_mailer.delivery_method = :test
 
-  config.cache_store = :memory_store
+  config.cache_store = :null_store
 
   config.active_support.deprecation = :log
 

src/api/config/initializers/session_store.rb

@@ -1,2 +1,6 @@
 # cookies are too small and active record sessions cause too much load
-Rails.application.config.session_store ActionDispatch::Session::CacheStore, :expire_after => 1.day
+if Rails.env.test?
+  Rails.application.config.session_store ActionDispatch::Session::CookieStore
+else
+  Rails.application.config.session_store ActionDispatch::Session::CacheStore, :expire_after => 1.day
+end

src/api/webui/app/controllers/webui/main_controller.rb

@@ -22,9 +22,7 @@ def systemstatus
     if @spider_bot
       @workerstatus = Xmlhash::XMLHash.new
     else
-      @workerstatus = Rails.cache.fetch('frontpage_workerstatus', :expires_in => 15.minutes, :shared => true) do
-        Webui::Workerstatus.find(:all).to_hash
-      end
+      @workerstatus = WorkerStatus.hidden.to_hash
     end
     @waiting_packages = 0
     @workerstatus.elements('waiting') {|waiting| @waiting_packages += waiting['jobs'].to_i}

src/api/webui/app/controllers/webui/monitor_controller.rb

@@ -5,8 +5,7 @@ class Webui::MonitorController < Webui::WebuiController
   before_filter :fetch_workerstatus, :only => [:old, :filtered_list, :update_building]
 
   def fetch_workerstatus
-     @workerstatus = Webui::Workerstatus.find(:all).to_hash
-     Rails.cache.write('frontpage_workerstatus', @workerstatus, :expires_in => 15.minutes)
+     @workerstatus = WorkerStatus.hidden.to_hash
   end
   private :fetch_workerstatus
 
@@ -25,11 +24,11 @@ def index
 
       workers = Hash.new
       workers_list = Array.new
-      @workerstatus.elements("building") do |b|
-        workers_list << [b["workerid"], b["hostarch"]]
+      @workerstatus.elements('building') do |b|
+        workers_list << [b['workerid'], b['hostarch']]
       end
-      @workerstatus.elements("idle") do |b|
-        workers_list << [b["workerid"], b["hostarch"]]
+      @workerstatus.elements('idle') do |b|
+        workers_list << [b['workerid'], b['hostarch']]
       end
       workers_list.each do |bid, barch|
         hostname, subid = bid.gsub(%r{[:]}, '/').split('/')
@@ -48,14 +47,14 @@ def update_building
     check_ajax
     workers = Hash.new
     max_time = 4 * 3600
-    @workerstatus.elements("idle") do |b|
-      id=b["workerid"].gsub(%r{[:./]}, '_')
+    @workerstatus.elements('idle') do |b|
+      id=b['workerid'].gsub(%r{[:./]}, '_')
       workers[id] = Hash.new
     end
 
-    @workerstatus.elements("building") do |b|
-      id=b["workerid"].gsub(%r{[:./]}, '_')
-      delta = (Time.now - Time.at(b["starttime"].to_i)).round
+    @workerstatus.elements('building') do |b|
+      id=b['workerid'].gsub(%r{[:./]}, '_')
+      delta = (Time.now - Time.at(b['starttime'].to_i)).round
       if delta < 5
 	delta = 5
       end
@@ -66,8 +65,8 @@ def update_building
       if (delta > 100)
 	delta = 100
       end
-      workers[id] = { "delta" => delta, "project" => b["project"], "repository" => b["repository"], 
-	"package" => b["package"], "arch" => b["arch"], "starttime" => b["starttime"]}
+      workers[id] = { 'delta' => delta, 'project' => b['project'], 'repository' => b['repository'],
+	'package' => b['package'], 'arch' => b['arch'], 'starttime' => b['starttime']}
     end
     # logger.debug workers.inspect
     render :json => workers
@@ -81,10 +80,10 @@ def events
     arch = params[:arch]
     range = params[:range]
     %w{waiting blocked squeue_high squeue_med}.each do |prefix|
-      data[prefix] = frontend.gethistory(prefix + "_" + arch, range, !discard_cache?).map {|time,value| [time*1000,value]}
+      data[prefix] = frontend.gethistory(prefix + '_' + arch, range, !discard_cache?).map {|time,value| [time*1000,value]}
     end
     %w{idle building}.each do |prefix|
-      data[prefix] = frontend.gethistory(prefix + "_" + map_to_workers(arch), range, !discard_cache?).map {|time,value| [time*1000,value]}
+      data[prefix] = frontend.gethistory(prefix + '_' + map_to_workers(arch), range, !discard_cache?).map {|time,value| [time*1000,value]}
     end
     low = Hash.new
     frontend.gethistory("squeue_low_#{arch}", range).each do |time,value|
@@ -95,10 +94,10 @@ def events
       clow = low[time] || 0
       comb << [1000*time, clow + value]
     end
-    data["squeue_low"] = comb
-    max = Webui::MonitorController.addarrays(data["squeue_high"], data["squeue_med"]).map{|time,value| value}.max || 0
-    data["events_max"] = max * 2
-    data["jobs_max"] =  maximumvalue(data["waiting"]) * 2
+    data['squeue_low'] = comb
+    max = Webui::MonitorController.addarrays(data['squeue_high'], data['squeue_med']).map{|time,value| value}.max || 0
+    data['events_max'] = max * 2
+    data['jobs_max'] =  maximumvalue(data['waiting']) * 2
     render :json => data
   end