From 5fa1ae4eb1d6077809c52aa3c96f9af11dd30fc4 Mon Sep 17 00:00:00 2001 From: Stephan Kulow Date: Wed, 9 Oct 2013 15:57:19 +0200 Subject: [PATCH] [webui] calculate the worker status directly in the API --- src/api/app/controllers/status_controller.rb | 114 +----------------- src/api/app/models/worker_status.rb | 105 ++++++++++++++++ src/api/config/clock.rb | 6 +- src/api/config/environments/test.rb | 2 +- src/api/config/initializers/session_store.rb | 6 +- .../app/controllers/webui/main_controller.rb | 4 +- .../controllers/webui/monitor_controller.rb | 37 +++--- .../app/controllers/webui/user_controller.rb | 4 +- .../app/controllers/webui/webui_controller.rb | 51 ++++---- .../webui/app/models/webui/workerstatus.rb | 2 - .../webui/config/initializers/activexml.rb | 4 - 11 files changed, 165 insertions(+), 170 deletions(-) create mode 100644 src/api/app/models/worker_status.rb delete mode 100644 src/api/webui/app/models/webui/workerstatus.rb diff --git a/src/api/app/controllers/status_controller.rb b/src/api/app/controllers/status_controller.rb index 00821f4cf84..529555367a9 100644 --- a/src/api/app/controllers/status_controller.rb +++ b/src/api/app/controllers/status_controller.rb @@ -7,15 +7,15 @@ class PermissionDeniedError < APIException end def list_messages - @messages = StatusMessage.alive.limit(params[:limit]).order("created_at DESC").includes(:user) + @messages = StatusMessage.alive.limit(params[:limit]).order('created_at DESC').includes(:user) @count = @messages.size - render xml: render_to_string(partial: "messages") + render xml: render_to_string(partial: 'messages') end def show_message @messages = [StatusMessage.find(params[:id])] @count = 1 - render xml: render_to_string(partial: "messages") + render xml: render_to_string(partial: 'messages') end class CreatingMessagesError < APIException @@ -54,7 +54,7 @@ def save_new_message(msg) def delete_message # check permissions unless permissions.status_message_create - raise PermissionDeniedError.new "message cannot be deleted, you have not sufficient permissions" + raise PermissionDeniedError.new 'message cannot be deleted, you have not sufficient permissions' end StatusMessage.find(params[:id]).delete @@ -62,31 +62,7 @@ def delete_message end def workerstatus - begin - data = Rails.cache.read('workerstatus') - rescue Zlib::GzipFile::Error - data = nil - end - data=ActiveXML::Node.new(data || update_workerstatus_cache) - prjs=Hash.new - data.each_building do |b| - prjs[b.project] = 1 - end - names = Hash.new - # now try to find those we have a match for (the rest are hidden from you - Project.where(name: prjs.keys).pluck(:name).each do |n| - names[n] = 1 - end - data.each_building do |b| - # no prj -> we are not allowed - unless names.has_key? b.project - logger.debug "workerstatus2clean: hiding #{b.project} for user #{User.current.login}" - b.set_attribute('project', '---') - b.set_attribute('repository', '---') - b.set_attribute('package', '---') - end - end - send_data data.dump_xml + send_data WorkerStatus.hidden.dump_xml end def history @@ -103,84 +79,6 @@ def history @values = StatusHistory.where("time >= ? AND \`key\` = ?", starttime, params[:key]).pluck(:time, :value).collect { |time, value| [time.to_i, value.to_f] } end - def save_value_line(e, prefix) - line = StatusHistory.new - line.time = @mytime - line.key = "#{prefix}_#{e['arch']}" - line.value = e['jobs'] - line.save - end - - def update_workerstatus_cache - # do not add hiding in here - this is purely for statistics - ret=backend_get('/build/_workerstatus') - data=Xmlhash.parse(ret) - - @mytime = Time.now.to_i - Rails.cache.write('workerstatus', ret, expires_in: 3.minutes) - Rails.cache.write('workerhash', data, expires_in: 3.minutes) - StatusHistory.transaction do - data.elements('blocked') do |e| - save_value_line(e, 'blocked') - end - data.elements('waiting') do |e| - save_value_line(e, 'waiting') - end - data.elements('partition') do |p| - p.elements('daemon') do |daemon| - parse_daemon_infos(daemon) - end - end - parse_worker_infos(data) - end - ret - end - - def parse_daemon_infos(daemon) - return unless daemon['type'] == 'scheduler' - arch = daemon['arch'] - # FIXME2.5: The current architecture model is a gross hack, not connected at all - # to the backend config. - a=Architecture.find_by_name(arch) - if a - a.available=true - a.save - end - queue = daemon.get('queue') - return unless queue - StatusHistory.create :time => @mytime, :key => "squeue_high_#{arch}", :value => queue['high'].to_i - StatusHistory.create :time => @mytime, :key => "squeue_next_#{arch}", :value => queue['next'].to_i - StatusHistory.create :time => @mytime, :key => "squeue_med_#{arch}", :value => queue['med'].to_i - StatusHistory.create :time => @mytime, :key => "squeue_low_#{arch}", :value => queue['low'].to_i - end - - def parse_worker_infos(data) - allworkers = Hash.new - workers = Hash.new - %w{building idle}.each do |state| - data.elements(state) do |e| - id=e['workerid'] - if workers.has_key? id - logger.debug 'building+idle worker' - next - end - workers[id] = 1 - key = state + '_' + e['hostarch'] - allworkers["building_#{e['hostarch']}"] ||= 0 - allworkers["idle_#{e['hostarch']}"] ||= 0 - allworkers[key] = allworkers[key] + 1 - end - end - - allworkers.each do |key, value| - line = StatusHistory.new - line.time = @mytime - line.key = key - line.value = value - line.save - end - end - # move to models? def role_from_cache(role_id) @rolecache[role_id] || (@rolecache[role_id] = Role.find(role_id).title) @@ -238,7 +136,7 @@ def bsrequest package: action.source_package, expand: 1, rev: action.source_rev) @result = PackageBuildStatus.new(spkg).result(target_project: tproj, srcmd5: dir['srcmd5']) - render xml: render_to_string(partial: "bsrequest") + render xml: render_to_string(partial: 'bsrequest') end class NotFoundError < APIException diff --git a/src/api/app/models/worker_status.rb b/src/api/app/models/worker_status.rb new file mode 100644 index 00000000000..77707afc9d9 --- /dev/null +++ b/src/api/app/models/worker_status.rb @@ -0,0 +1,105 @@ +class WorkerStatus + + def self.hidden + mydata = Rails.cache.read('workerstatus') + ws = ActiveXML::Node.new(mydata || ActiveXML.backend.direct_http('/build/_workerstatus')) + prjs=Hash.new + ws.each_building do |b| + prjs[b.project] = 1 + end + names = Hash.new + # now try to find those we have a match for (the rest are hidden from you + Project.where(name: prjs.keys).pluck(:name).each do |n| + names[n] = 1 + end + ws.each_building do |b| + # no prj -> we are not allowed + unless names.has_key? b.project + Rails.logger.debug "workerstatus2clean: hiding #{b.project} for user #{User.current.login}" + b.set_attribute('project', '---') + b.set_attribute('repository', '---') + b.set_attribute('package', '---') + end + end + ws + end + + def update_workerstatus_cache + # do not add hiding in here - this is purely for statistics + ret=ActiveXML.backend.direct_http('/build/_workerstatus') + wdata=Xmlhash.parse(ret) + @mytime = Time.now.to_i + Rails.cache.write('workerstatus', ret, expires_in: 3.minutes) + StatusHistory.transaction do + wdata.elements('blocked') do |e| + save_value_line(e, 'blocked') + end + wdata.elements('waiting') do |e| + save_value_line(e, 'waiting') + end + wdata.elements('partition') do |p| + p.elements('daemon') do |daemon| + parse_daemon_infos(daemon) + end + end + parse_worker_infos(wdata) + end + ret + end + + private + + def parse_daemon_infos(daemon) + return unless daemon['type'] == 'scheduler' + arch = daemon['arch'] + # FIXME2.5: The current architecture model is a gross hack, not connected at all + # to the backend config. + a=Architecture.find_by_name(arch) + if a + a.available=true + a.save + end + queue = daemon.get('queue') + return unless queue + StatusHistory.create :time => @mytime, :key => "squeue_high_#{arch}", :value => queue['high'].to_i + StatusHistory.create :time => @mytime, :key => "squeue_next_#{arch}", :value => queue['next'].to_i + StatusHistory.create :time => @mytime, :key => "squeue_med_#{arch}", :value => queue['med'].to_i + StatusHistory.create :time => @mytime, :key => "squeue_low_#{arch}", :value => queue['low'].to_i + end + + def parse_worker_infos(wdata) + allworkers = Hash.new + workers = Hash.new + %w{building idle}.each do |state| + wdata.elements(state) do |e| + id=e['workerid'] + if workers.has_key? id + Rails.logger.debug 'building+idle worker' + next + end + workers[id] = 1 + key = state + '_' + e['hostarch'] + allworkers["building_#{e['hostarch']}"] ||= 0 + allworkers["idle_#{e['hostarch']}"] ||= 0 + allworkers[key] = allworkers[key] + 1 + end + end + + allworkers.each do |key, value| + line = StatusHistory.new + line.time = @mytime + line.key = key + line.value = value + line.save + end + end + + def save_value_line(e, prefix) + line = StatusHistory.new + line.time = @mytime + line.key = "#{prefix}_#{e['arch']}" + line.value = e['jobs'] + line.save + end + +end \ No newline at end of file diff --git a/src/api/config/clock.rb b/src/api/config/clock.rb index cd66e6953eb..5199666e9dd 100644 --- a/src/api/config/clock.rb +++ b/src/api/config/clock.rb @@ -8,10 +8,8 @@ include Clockwork every(30.seconds, 'status.refresh') do - Rails.logger.debug "Refresh worker status" - c = StatusController.new # this should be fast, so don't delay - c.update_workerstatus_cache + WorkerStatus.new.update_workerstatus_cache end every(1.hour, 'refresh issues') do @@ -29,7 +27,7 @@ every(1.day, 'optimize history', thread: true) do ActiveRecord::Base.connection_pool.with_connection do |sql| - sql.execute "optimize table status_histories;" + sql.execute 'optimize table status_histories;' end end diff --git a/src/api/config/environments/test.rb b/src/api/config/environments/test.rb index caf6d06e979..50bbcd1213c 100644 --- a/src/api/config/environments/test.rb +++ b/src/api/config/environments/test.rb @@ -18,7 +18,7 @@ # ActionMailer::Base.deliveries array. config.action_mailer.delivery_method = :test - config.cache_store = :memory_store + config.cache_store = :null_store config.active_support.deprecation = :log diff --git a/src/api/config/initializers/session_store.rb b/src/api/config/initializers/session_store.rb index 0b1794c6fe4..5d2620ab5fa 100644 --- a/src/api/config/initializers/session_store.rb +++ b/src/api/config/initializers/session_store.rb @@ -1,2 +1,6 @@ # cookies are too small and active record sessions cause too much load -Rails.application.config.session_store ActionDispatch::Session::CacheStore, :expire_after => 1.day +if Rails.env.test? + Rails.application.config.session_store ActionDispatch::Session::CookieStore +else + Rails.application.config.session_store ActionDispatch::Session::CacheStore, :expire_after => 1.day +end diff --git a/src/api/webui/app/controllers/webui/main_controller.rb b/src/api/webui/app/controllers/webui/main_controller.rb index 9022cb8bf67..bf21d4e7921 100644 --- a/src/api/webui/app/controllers/webui/main_controller.rb +++ b/src/api/webui/app/controllers/webui/main_controller.rb @@ -22,9 +22,7 @@ def systemstatus if @spider_bot @workerstatus = Xmlhash::XMLHash.new else - @workerstatus = Rails.cache.fetch('frontpage_workerstatus', :expires_in => 15.minutes, :shared => true) do - Webui::Workerstatus.find(:all).to_hash - end + @workerstatus = WorkerStatus.hidden.to_hash end @waiting_packages = 0 @workerstatus.elements('waiting') {|waiting| @waiting_packages += waiting['jobs'].to_i} diff --git a/src/api/webui/app/controllers/webui/monitor_controller.rb b/src/api/webui/app/controllers/webui/monitor_controller.rb index 5564a16fbe0..9719e131fa8 100644 --- a/src/api/webui/app/controllers/webui/monitor_controller.rb +++ b/src/api/webui/app/controllers/webui/monitor_controller.rb @@ -5,8 +5,7 @@ class Webui::MonitorController < Webui::WebuiController before_filter :fetch_workerstatus, :only => [:old, :filtered_list, :update_building] def fetch_workerstatus - @workerstatus = Webui::Workerstatus.find(:all).to_hash - Rails.cache.write('frontpage_workerstatus', @workerstatus, :expires_in => 15.minutes) + @workerstatus = WorkerStatus.hidden.to_hash end private :fetch_workerstatus @@ -25,11 +24,11 @@ def index workers = Hash.new workers_list = Array.new - @workerstatus.elements("building") do |b| - workers_list << [b["workerid"], b["hostarch"]] + @workerstatus.elements('building') do |b| + workers_list << [b['workerid'], b['hostarch']] end - @workerstatus.elements("idle") do |b| - workers_list << [b["workerid"], b["hostarch"]] + @workerstatus.elements('idle') do |b| + workers_list << [b['workerid'], b['hostarch']] end workers_list.each do |bid, barch| hostname, subid = bid.gsub(%r{[:]}, '/').split('/') @@ -48,14 +47,14 @@ def update_building check_ajax workers = Hash.new max_time = 4 * 3600 - @workerstatus.elements("idle") do |b| - id=b["workerid"].gsub(%r{[:./]}, '_') + @workerstatus.elements('idle') do |b| + id=b['workerid'].gsub(%r{[:./]}, '_') workers[id] = Hash.new end - @workerstatus.elements("building") do |b| - id=b["workerid"].gsub(%r{[:./]}, '_') - delta = (Time.now - Time.at(b["starttime"].to_i)).round + @workerstatus.elements('building') do |b| + id=b['workerid'].gsub(%r{[:./]}, '_') + delta = (Time.now - Time.at(b['starttime'].to_i)).round if delta < 5 delta = 5 end @@ -66,8 +65,8 @@ def update_building if (delta > 100) delta = 100 end - workers[id] = { "delta" => delta, "project" => b["project"], "repository" => b["repository"], - "package" => b["package"], "arch" => b["arch"], "starttime" => b["starttime"]} + workers[id] = { 'delta' => delta, 'project' => b['project'], 'repository' => b['repository'], + 'package' => b['package'], 'arch' => b['arch'], 'starttime' => b['starttime']} end # logger.debug workers.inspect render :json => workers @@ -81,10 +80,10 @@ def events arch = params[:arch] range = params[:range] %w{waiting blocked squeue_high squeue_med}.each do |prefix| - data[prefix] = frontend.gethistory(prefix + "_" + arch, range, !discard_cache?).map {|time,value| [time*1000,value]} + data[prefix] = frontend.gethistory(prefix + '_' + arch, range, !discard_cache?).map {|time,value| [time*1000,value]} end %w{idle building}.each do |prefix| - data[prefix] = frontend.gethistory(prefix + "_" + map_to_workers(arch), range, !discard_cache?).map {|time,value| [time*1000,value]} + data[prefix] = frontend.gethistory(prefix + '_' + map_to_workers(arch), range, !discard_cache?).map {|time,value| [time*1000,value]} end low = Hash.new frontend.gethistory("squeue_low_#{arch}", range).each do |time,value| @@ -95,10 +94,10 @@ def events clow = low[time] || 0 comb << [1000*time, clow + value] end - data["squeue_low"] = comb - max = Webui::MonitorController.addarrays(data["squeue_high"], data["squeue_med"]).map{|time,value| value}.max || 0 - data["events_max"] = max * 2 - data["jobs_max"] = maximumvalue(data["waiting"]) * 2 + data['squeue_low'] = comb + max = Webui::MonitorController.addarrays(data['squeue_high'], data['squeue_med']).map{|time,value| value}.max || 0 + data['events_max'] = max * 2 + data['jobs_max'] = maximumvalue(data['waiting']) * 2 render :json => data end diff --git a/src/api/webui/app/controllers/webui/user_controller.rb b/src/api/webui/app/controllers/webui/user_controller.rb index 93682f1ca64..d3124684283 100644 --- a/src/api/webui/app/controllers/webui/user_controller.rb +++ b/src/api/webui/app/controllers/webui/user_controller.rb @@ -14,11 +14,11 @@ def logout logger.info "Logging out: #{session[:login]}" reset_session @user = nil - @return_to_path = "/" + @return_to_path = root_path if CONFIG['proxy_auth_mode'] == :on redirect_to CONFIG['proxy_auth_logout_page'] else - redirect_to '/' + redirect_to root_path end Person.free_cache session[:login] end diff --git a/src/api/webui/app/controllers/webui/webui_controller.rb b/src/api/webui/app/controllers/webui/webui_controller.rb index 97042b7cf9f..2769e84432e 100644 --- a/src/api/webui/app/controllers/webui/webui_controller.rb +++ b/src/api/webui/app/controllers/webui/webui_controller.rb @@ -7,7 +7,7 @@ module Webui class WebuiController < ActionController::Base Rails.cache.set_domain if Rails.cache.respond_to?('set_domain'); - before_filter :check_mobile_views + #before_filter :check_mobile_views before_filter :instantiate_controller_and_action_names before_filter :set_return_to, :reset_activexml, :authenticate before_filter :check_user @@ -26,17 +26,17 @@ class WebuiController < ActionController::Base # HTTPPaymentRequired, UnauthorizedError or Forbidden # here so the exception handler catches it but what the heck... rescue_from ActiveXML::Transport::ForbiddenError do |exception| - if exception.code == "unregistered_ichain_user" - render template: "user/request_ichain" and return - elsif exception.code == "unregistered_user" - render file: Rails.root.join("public/403"), formats: [:html], status: 402, layout: false and return - elsif exception.code == "unconfirmed_user" - render file: Rails.root.join("public/402"), formats: [:html], status: 402, layout: false + if exception.code == 'unregistered_ichain_user' + render template: 'user/request_ichain' and return + elsif exception.code == 'unregistered_user' + render file: Rails.root.join('public/403'), formats: [:html], status: 402, layout: false and return + elsif exception.code == 'unconfirmed_user' + render file: Rails.root.join('public/402'), formats: [:html], status: 402, layout: false else if @user - render file: Rails.root.join("public/403"), formats: [:html], status: :forbidden, layout: false + render file: Rails.root.join('public/403'), formats: [:html], status: :forbidden, layout: false else - render file: Rails.root.join("public/401"), formats: [:html], status: :unauthorized, layout: false + render file: Rails.root.join('public/401'), formats: [:html], status: :unauthorized, layout: false end end end @@ -57,7 +57,7 @@ def initialize( _xml, _errors ) class MissingParameterError < Exception; end rescue_from MissingParameterError do |exception| logger.debug "#{exception.class.name} #{exception.message} #{exception.backtrace.join('\n')}" - render file: Rails.root.join("public/404"), status: 404, layout: false, formats: [:html] + render file: Rails.root.join('public/404'), status: 404, layout: false, formats: [:html] end protected @@ -67,8 +67,8 @@ def set_return_to @return_to_host = params['return_to_host'] else # we have a proxy in front of us - @return_to_host = CONFIG['external_webui_protocol'] || "http" - @return_to_host += "://" + @return_to_host = CONFIG['external_webui_protocol'] || 'http' + @return_to_host += '://' @return_to_host += CONFIG['external_webui_host'] || request.host end @return_to_path = params['return_to_path'] || request.env['ORIGINAL_FULLPATH'] @@ -78,7 +78,7 @@ def set_return_to def require_login if !session[:login] render :text => 'Please login' and return if request.xhr? - flash[:error] = "Please login to access the requested page." + flash[:error] = 'Please login to access the requested page.' mode = :off mode = CONFIG['proxy_auth_mode'] unless CONFIG['proxy_auth_mode'].blank? if (mode == :off) @@ -117,8 +117,8 @@ def authenticate_proxy session[:login] = proxy_user session[:email] = proxy_email # Set the headers for direct connection to the api, TODO: is this thread safe? - ActiveXML::api.set_additional_header( "X-Username", proxy_user ) - ActiveXML::api.set_additional_header( "X-Email", proxy_email ) if proxy_email + ActiveXML::api.set_additional_header( 'X-Username', proxy_user ) + ActiveXML::api.set_additional_header( 'X-Email', proxy_email ) if proxy_email else session[:login] = nil session[:email] = nil @@ -141,9 +141,9 @@ def valid_project_name? name end def valid_package_name_read? name - return true if name == "_project" - return true if name == "_product" - return true if name == "_deltas" + return true if name == '_project' + return true if name == '_product' + return true if name == '_deltas' return true if name =~ /^_product:[-+\w\.:]*$/ return true if name =~ /^_patchinfo:[-+\w\.:]*$/ name =~ /^[[:alnum:]][-+\w\.:]*$/ @@ -177,8 +177,8 @@ def valid_group_name? name def reset_activexml transport = ActiveXML::api - transport.delete_additional_header "X-Username" - transport.delete_additional_header "X-Email" + transport.delete_additional_header 'X-Username' + transport.delete_additional_header 'X-Email' transport.delete_additional_header 'Authorization' end @@ -274,7 +274,7 @@ def map_to_workers(arch) def put_body_to_tempfile(xmlbody) file = Tempfile.new('xml').path - file = File.open(file + ".xml", "w") + file = File.open(file + '.xml', 'w') file.write(xmlbody) file.close return file.path @@ -311,7 +311,7 @@ def validate_xhtml begin document = Nokogiri::XML::Document.parse(xmlbody, nil, nil, Nokogiri::XML::ParseOptions::STRICT) rescue Nokogiri::XML::SyntaxError => e - errors << ("[%s:%s]" % [e.line, e.column]) + e.inspect + errors << ('[%s:%s]' % [e.line, e.column]) + e.inspect errors << put_body_to_tempfile(xmlbody) end @@ -321,7 +321,7 @@ def validate_xhtml document = nil errors << put_body_to_tempfile(xmlbody) ses.each do |err| - errors << ("[%s:%s]" % [err.line, err.column]) + err.inspect + errors << ('[%s:%s]' % [err.line, err.column]) + err.inspect end end end @@ -329,7 +329,7 @@ def validate_xhtml unless document self.instance_variable_set(:@_response_body, nil) logger.debug "XML Errors #{errors.inspect} #{xmlbody}" - render :template => "webui/xml_errors", :locals => { :oldbody => xmlbody, :errors => errors }, :status => 400 + render :template => 'webui/xml_errors', :locals => { :oldbody => xmlbody, :errors => errors }, :status => 400 end end @@ -344,7 +344,7 @@ def require_configuration # Before filter to check if current user is administrator def require_admin if !@user || !@user.is_admin? - flash[:error] = "Requires admin privileges" + flash[:error] = 'Requires admin privileges' redirect_back_or_to :controller => 'main', :action => 'index' and return end end @@ -394,7 +394,6 @@ def mobile_request? end def check_mobile_views - Rails.logger.debug "check_mobile" #prepend_view_path(Rails.root.join('app', 'mobile_views')) if mobile_request? end diff --git a/src/api/webui/app/models/webui/workerstatus.rb b/src/api/webui/app/models/webui/workerstatus.rb deleted file mode 100644 index 0aeb3ebfed8..00000000000 --- a/src/api/webui/app/models/webui/workerstatus.rb +++ /dev/null @@ -1,2 +0,0 @@ -class Webui::Workerstatus < Webui::Node -end diff --git a/src/api/webui/config/initializers/activexml.rb b/src/api/webui/config/initializers/activexml.rb index 1ea4f82343f..ee2f4068129 100644 --- a/src/api/webui/config/initializers/activexml.rb +++ b/src/api/webui/config/initializers/activexml.rb @@ -95,10 +95,6 @@ def add(d) map.connect :patchinfo, 'rest:///source/:project/:package/_patchinfo', :issues => 'rest:///source/:project/:package/?view=issues' - # Monitor - map.connect :workerstatus, 'rest:///status/workerstatus', - :all => 'rest:///status/workerstatus' - # Statistics map.connect :latestadded, 'rest:///statistics/latest_added?:limit', :specific => 'rest:///statistics/added_timestamp/:project/:package'