[api] replace linked_packages with backend_packages

Also saves md5sum for all packages not just links

src/api/Rakefile

@@ -11,15 +11,13 @@ require(File.join(File.dirname(__FILE__), 'config', 'boot'))
 require 'workers/update_issues.rb'
 task(:updateissues => :environment) { Delayed::Job.enqueue UpdateIssuesJob.new }
 
-require 'workers/update_package_meta_job.rb'
-task(:updatepackagemeta => :environment) { u=UpdatePackageMetaJob.new; u.perform }
+task(:updatepackagemeta => :environment) { UpdatePackageMetaJob.new.perform }
 
 require 'workers/write_configuration.rb'
-task(:writeconfiguration => :environment) { i=WriteConfigurationJob.new; i.perform }
+task(:writeconfiguration => :environment) { WriteConfigurationJob.new.perform }
 
 require 'workers/import_requests.rb'
-#task(:importrequests => :environment) { Delayed::Job.enqueue ImportRequestsDelayedJob.new }
-task(:importrequests => :environment) { i=ImportRequestsDelayedJob.new; i.perform }
+task(:importrequests => :environment) { ImportRequestsDelayedJob.new.perform }
 
 if Rails.env.test?
   begin

src/api/app/controllers/request_controller.rb

@@ -1,5 +1,5 @@
 require 'base64'
-require 'event/request'
+require 'event'
 
 include MaintenanceHelper
 

src/api/app/controllers/source_controller.rb

@@ -2,7 +2,7 @@
 include ValidationHelper
 
 require 'builder/xchar'
-require 'event/package'
+require 'event'
 
 class SourceController < ApplicationController
 

src/api/app/controllers/status_controller.rb

@@ -1,4 +1,4 @@
-require 'status_helper'
+require_dependency 'status_helper'
 
 class StatusController < ApplicationController
 
@@ -84,9 +84,9 @@ def workerstatus
       # no prj -> we are not allowed
       unless names.has_key? b.project
         logger.debug "workerstatus2clean: hiding #{b.project} for user #{User.current.login}"
-        b.set_attribute('project', "---")
-        b.set_attribute('repository', "---")
-        b.set_attribute('package', "---")
+        b.set_attribute('project', '---')
+        b.set_attribute('repository', '---')
+        b.set_attribute('package', '---')
       end
     end
     send_data data.dump_xml
@@ -222,13 +222,8 @@ def find_relationships_for_packages(packages)
 
   def project
     dbproj = Project.get_by_name(params[:project])
-    key ='project_status_xml2_%s' % dbproj.name
-    xml = Rails.cache.fetch(key, :expires_in => 10.seconds) do
-      @packages = dbproj.complex_status(backend)
-      find_relationships_for_packages(@packages)
-      render_to_string
-    end
-    render :text => xml
+    @packages = ProjectStatusHelper.calc_status(dbproj)
+    find_relationships_for_packages(@packages)
   end
 
   def bsrequest

src/api/app/helpers/maintenance_helper.rb

@@ -344,7 +344,7 @@ def do_branch params
         next unless p[:package].class == Package # only for local packages
 
         pkg = p[:package]
-        if pkg.package_kinds.find_by_kind 'link'
+        if pkg.is_of_kind? 'link'
           # is the package itself a local link ?
           link = Suse::Backend.get( "/source/#{p[:package].project.name}/#{p[:package].name}/_link")
           ret = ActiveXML::Node.new(link.body)
@@ -566,7 +566,7 @@ def release_package(sourcePackage, targetProjectName, targetPackageName,
     else
       tpkg = Package.new(:name => targetPackageName, :title => sourcePackage.title, :description => sourcePackage.description)
       targetProject.packages << tpkg
-      if sourcePackage.package_kinds.find_by_kind 'patchinfo'
+      if sourcePackage.is_of_kind? 'patchinfo'
         # publish patchinfos only
         tpkg.flags.create( :flag => 'publish', :status => "enable" )
       end
@@ -661,7 +661,7 @@ def release_package(sourcePackage, targetProjectName, targetPackageName,
     end
 
     # create or update main package linking to incident package
-    unless sourcePackage.package_kinds.find_by_kind 'patchinfo'
+    unless sourcePackage.is_of_kind? 'patchinfo'
       basePackageName = targetPackageName.gsub(/\.[^\.]*$/, '')
 
       # only if package does not contain a _patchinfo file

src/api/app/helpers/status_helper.rb

@@ -11,7 +11,7 @@ class LinkInfo
 end
 
 class PackInfo
-  attr_accessor :devel_project, :devel_package
+  attr_accessor :devel_project, :devel_package, :bp
   attr_accessor :srcmd5, :verifymd5, :changesmd5, :maxmtime, :error, :link
   attr_reader :name, :project, :key, :package_id
   attr_accessor :develpack
@@ -122,59 +122,24 @@ def fails
 
 class ProjectStatusHelper
 
-  def self.get_xml(uri)
-    key = Digest::MD5.hexdigest(uri)
-    d = Rails.cache.fetch(key, :expires_in => 2.hours) do
-      Suse::Backend.get(uri).body
-    end
-    Xmlhash.parse(d)
-  end
-
   def self.check_md5(proj, packages, mypackages)
-    uri = '/getprojpack?project=%s&withsrcmd5=1&ignoredisable=1' % CGI.escape(proj)
-    packages.each do |package|
-      uri += "&package=" + CGI.escape(package.name)
-    end
-    data = get_xml(uri)
-
-    data.get('project').elements('package') do |p|
-
-      packname = p['name']
-      key = proj + "/" + packname
-      next unless mypackages.has_key?(key)
-      mypackages[key].srcmd5 = p['srcmd5']
-      if p['verifymd5']
-        mypackages[key].verifymd5 = p['verifymd5']
+    # remap
+    packages = Project.find_by_name(proj).packages.where(name: packages.map { |p| p.name })
+    BackendPackage.where(package_id: packages).each do |bp|
+      key = proj + "/" + bp.package.name
+      obj = mypackages[key]
+      next unless obj
+      obj.bp = bp
+      obj.srcmd5 = obj.bp.srcmd5
+      obj.verifymd5 = obj.bp.verifymd5
+      obj.error = obj.bp.error
+      if obj.bp.links_to
+        obj.link.project = obj.bp.links_to.project.name
+        obj.link.package = obj.bp.links_to.name
       end
-      p.elements('linked') do |l|
-        mypackages[key].link.project = l['project']
-        mypackages[key].link.package = l['package']
-        break # the first link will do
-      end
-      p.elements('error') do |e|
-        mypackages[key].error = e
-        break
-      end
-      cmd5, mtime = Rails.cache.fetch("change-data-%s" % p['srcmd5']) do
-        begin
-          directory = Directory.hashed(project: proj, package: packname, expand: 1)
-        rescue ActiveXML::Transport::Error
-          directory = nil
-        end
-        changesfile="%s.changes" % packname
-        md5 = ''
-        mtime = 0
-        directory.elements('entry') do |e|
-          if e['name'] == changesfile
-            md5 = e['md5']
-          end
-          mtime = [mtime, Integer(e['mtime'])].max
-        end if directory
-        [md5, mtime]
-      end
-      mypackages[key].changesmd5 = cmd5 unless cmd5.empty?
-      mypackages[key].maxmtime = mtime unless mtime == 0
-    end if data
+      obj.changesmd5 = obj.bp.changesmd5
+      obj.maxmtime = obj.bp.maxmtime
+    end
   end
 
   def self.update_projpack(proj, mypackages)

src/api/app/indices/package_index.rb

@@ -4,9 +4,9 @@
   has :db_project_id, :as => :project_id
   has attribs.attrib_type_id, :as => :attrib_type_ids
   has package_issues.issue_id, :as => :issue_ids
-  has "(SELECT count(*) FROM linked_packages WHERE links_to_id = packages.id)", :as => :linked_count, :type => :integer
+  has "(SELECT count(*) FROM backend_packages WHERE links_to_id = packages.id)", :as => :linked_count, :type => :integer
   has activity_index
-  has "EXISTS (SELECT * FROM linked_packages WHERE package_id = packages.id)", :as => :links_to_other, :type => :boolean
+  has "EXISTS (SELECT * FROM backend_packages WHERE package_id = packages.id)", :as => :links_to_other, :type => :boolean
   has "EXISTS (SELECT * FROM packages p WHERE p.develpackage_id = packages.id)", :as => :is_devel, :type => :boolean
   has updated_at
 end

src/api/app/jobs/check_package_event.rb

@@ -0,0 +1,27 @@
+class CheckPackageEvent
+
+  attr_accessor :event
+  attr_accessor :checked_pkgs
+
+  def initialize(event)
+    self.event = event
+    self.checked_pkgs = {}
+  end
+
+  def update_pkg(pkg)
+    return if self.checked_pkgs.has_key? pkg.id
+    pkg.update_backendinfo
+    self.checked_pkgs[pkg.id] = 1
+    BackendPackage.where(links_to_id: pkg.id).each do |p|
+      update_pkg(Package.find(p.package_id))
+    end
+  end
+
+  def perform
+    pl = event.payload
+    pkg = Package.find_by_project_and_name(pl['project'], pl['package'])
+    return unless pkg # there is nothing we can do
+    # dig into recursion
+    update_pkg(pkg)
+  end
+end

src/api/app/jobs/update_package_meta_job.rb

@@ -0,0 +1,36 @@
+class UpdatePackageMetaJob
+
+  def scan_links
+    names = Package.distinct(:name).order(:name).pluck(:name)
+    while !names.empty? do
+      slice = names.slice!(0, 30)
+      path = "/search/package/id?match=("
+      path += slice.map { |name| "linkinfo/@package='#{CGI.escape(name)}'" }.join("+or+")
+      path += ")"
+      answer = Xmlhash.parse(Suse::Backend.get(path).body)
+      answer.elements('package') do |p|
+        pkg = Package.find_by_project_and_name(p['project'], p['name'])
+        # if there is a linkinfo for a package not in database, there can not be a linked_package either
+        next unless pkg
+        pkg.update_backendinfo
+      end
+
+    end
+  end
+
+  def perform
+    # first we scan the links so that commits happening
+    # while the delayed job runs can update our work
+    scan_links
+
+    Package.order(:name).each do |pkg|
+      next unless Package.exists?(pkg)
+      begin
+        pkg.update_backendinfo
+      rescue ActiveXML::Transport::Error
+      end
+    end
+  end
+
+end
+

src/api/app/models/backend_info.rb

@@ -8,10 +8,6 @@ def self.set_value(key, value)
     v.save!
   end
 
-  def self.lastevents_nr=(nr)
-    self.set_value('lastevents_nr', nr.to_s)
-  end
-
   def self.lastnotification_nr=(nr)
     self.set_value('lastnotification_nr', nr.to_s)
   end
@@ -26,62 +22,8 @@ def self.get_integer(key)
     Integer(nr[0])
   end
 
-  def self.lastevents_nr
-    self.get_integer('lastevents_nr')
-  end
-
   def self.lastnotification_nr
     self.get_integer('lastnotification_nr')
   end
 
-  # long running task - if we're out of sync with backend
-  def scan_all_links
-    start_time = Time.now
-    names = Package.distinct(:name).order(:name).pluck(:name)
-    while !names.empty? do
-      slice = names.slice!(0, 30)
-      path = "/search/package/id?match=("
-      path += slice.map { |name| "linkinfo/@package='#{CGI.escape(name)}'" }.join("+or+")
-      path += ")"
-      answer = Xmlhash.parse(Suse::Backend.get(path).body)
-      answer.elements('package') do |p|
-        pkg = Package.find_by_project_and_name(p['project'], p['name'])
-        # if there is a linkinfo for a package not in database, there can not be a linked_package either
-        next unless pkg
-        pkg.update_linkinfo
-      end
-
-    end
-    # every link we haven't seen in this loop, is no link anymore
-    LinkedPackage.where("updated_at < ?", start_time).delete_all
-  end
-
-  def update_last_events
-    # pick first admin so we can see all projects - as this function is called from delayed job
-    # TODO: add an admin user without password exactly for delayed_jobs?
-    User.current ||= User.get_default_admin
-
-    # it's possible that we see the same event more often but the
-    # alternative is waiting for the *next* event, which would
-    # make this function hang for 15 minutes and we want to call
-    # it often
-    event = BackendInfo.lastevents_nr
-    lastevents = Xmlhash.parse(Suse::Backend.get("/lastevents?start=#{event}").body)
-
-    if lastevents['sync'] == 'lost'
-      # we're doomed!
-      BackendInfo.lastevents_nr = Integer(lastevents['next']) - 1
-      BackendInfo.first.scan_all_links
-      return
-    end
-
-    lastevents.elements('event') do |e|
-      next if e['type'] != 'package'
-      pkg = Package.find_by_project_and_name(e['project'], e['package'])
-      next unless pkg
-      pkg.update_linkinfo
-    end
-    BackendInfo.lastevents_nr = Integer(lastevents['next']) - 1
-  end
-
 end

src/api/app/models/linked_package.rb → src/api/app/models/backend_package.rb

@@ -1,4 +1,4 @@
-class LinkedPackage < ActiveRecord::Base
+class BackendPackage < ActiveRecord::Base
   # a package can have one target _link (or not)
   self.primary_key = 'package_id'
   belongs_to :links_to, class_name: "Package"

src/api/app/models/bs_request.rb

@@ -1,5 +1,5 @@
 require 'xmlhash'
-require 'event/request'
+require 'event'
 require 'opensuse/backend'
 
 class BsRequest < ActiveRecord::Base

src/api/app/models/bs_request_action.rb

@@ -754,7 +754,7 @@ def create_expand_package(packages, opts = {})
       if self.is_maintenance_incident?
         newTargets << tprj
         newAction.target_releaseproject = releaseproject.name if releaseproject
-      elsif self.is_maintenance_release? and pkg.package_kinds.find_by_kind 'channel'
+      elsif self.is_maintenance_release? and pkg.is_of_kind? 'channel'
         newAction.action_type = :submit
         newAction.target_project = tprj
         newAction.target_package = tpkg
@@ -773,14 +773,14 @@ def create_expand_package(packages, opts = {})
       end
     end
     if self.is_maintenance_release? and !found_patchinfo and !opts[:ignore_build_state]
-      raise MissingPatchinfo.new "maintenance release request without patchinfo would release no binaries"
+      raise MissingPatchinfo.new 'maintenance release request without patchinfo would release no binaries'
     end
 
     # new packages (eg patchinfos) go to all target projects by default in maintenance requests
     newTargets.uniq!
     newPackages.each do |pkg|
       releaseTargets=nil
-      if pkg.package_kinds.find_by_kind 'patchinfo'
+      if pkg.is_of_kind? 'patchinfo'
         releaseTargets = Patchinfo.new.fetch_release_targets(pkg)
       end
       newTargets.each do |p|

src/api/app/models/bs_request_action_maintenance_incident.rb

@@ -17,7 +17,7 @@ class NoMaintenanceReleaseTarget < APIException
   end
 
   def get_releaseproject(pkg, tprj)
-    return nil if pkg.package_kinds.find_by_kind 'patchinfo'
+    return nil if pkg.is_of_kind? 'patchinfo'
     releaseproject = nil
     if self.target_releaseproject
       releaseproject = Project.get_by_name self.target_releaseproject
@@ -62,7 +62,7 @@ def merge_into_maintenance_incident(incidentProject, base, releaseproject=nil, r
         next
       end
       # patchinfos are handled as new packages
-      if pkg.package_kinds.find_by_kind 'patchinfo'
+      if pkg.is_of_kind? 'patchinfo'
         if Package.exists_by_project_and_name(incidentProject.name, pkg.name, follow_project_links: false)
           new_pkg = Package.get_by_project_and_name(incidentProject.name, pkg.name, use_source: false, follow_project_links: false)
         else