Merge pull request #4528 from bgeuken/update_2_9

Update 2.9 branch

dist/clouduploader.rb

@@ -24,16 +24,22 @@
 FileUtils.ln_s(image_path, File.join(Dir.pwd, filename))
 
 def get_ec2_credentials(data)
-  # Credentials are stored in  ~/.aws/credentials
-  out, err, status = Open3.capture3(
+  command = [
     'aws',
     'sts',
     'assume-role',
     "--role-arn=#{data['arn']}",
     "--external-id=#{data['external_id']}",
     '--role-session-name=obs',
     "--duration-seconds=#{THIRTY_MINUTES}"
-  )
+  ]
+
+  if data['vpc_subnet_id']
+    command << "--vpc-subnet-id=#{data['vpc_subnet_id']}"
+  end
+
+  # Credentials are stored in  ~/.aws/credentials
+  out, err, status = Open3.capture3(command)
 
   if status.success?
     STDOUT.write("Successfully authenticated.\n")

dist/ec2utils.conf.example

@@ -0,0 +1,166 @@
+# One can configure as many accounts as one chooses. Configured accounts
+# start with the "account-" prefix. Account information is retrieved by any
+# of the ec2utils tools by combining the prefix "account-" with the name
+# specified with the --account command line option
+#
+# For example the information for the "account-servers" configuration is
+# used by specifying --account servers on the command line of any of the
+# tools
+[account-servers]
+access_key_id =
+secret_access_key =
+ssh_key_name =
+ssh_private_key =
+subnet_id_us-east-1 = subnet-123456
+
+# The aki IDs are published by Amazon on the following page:
+# http://docs.aws.amazon.com/AWSEC2/latest/UserGuide/UserProvidedKernels.html#AmazonKernelImageIDs
+# The ami, instance_type, and user are needed for those utilities that require
+# a running instance in Amazon EC2 to perform their operations.
+[region-ap-northeast-1]
+ami = ami-383c1956
+instance_type = t2.micro
+aki_i386 = aki-136bf512
+aki_x86_64 = aki-176bf516
+g2_aki_x86_64 = aki-f1ad9bf0
+user = ec2-user
+# Allow a region to overwrite the account key
+# ssk_key_name =
+# ssh_private_key =
+
+[region-ap-northeast-2]
+ami = ami-249b554a
+instance_type = t2.micro
+aki_i386 = aki-01a66b6f
+aki_x86_64 = aki-01a66b6f
+g2_aki_x86_64 = aki-0ea66b60
+user = ec2-user
+# Allow a region to overwrite the account key
+# ssk_key_name =
+# ssh_private_key =
+
+[region-ap-southeast-1]
+ami = ami-c9b572aa
+instance_type = t2.micro
+aki_i386 = aki-ae3973fc
+aki_x86_64 = aki-503e7402
+g2_aki_x86_64 = aki-ca755498
+user = ec2-user
+# Allow a region to overwrite the account key
+# ssk_key_name =
+# ssh_private_key =
+
+[region-ap-southeast-2]
+ami = ami-48d38c2b
+instance_type = t2.micro
+aki_i386 = aki-cd62fff7
+aki_x86_64 = aki-c362fff9
+g2_aki_x86_64 = aki-8faec3b5
+user = ec2-user
+# Allow a region to overwrite the account key
+# ssk_key_name =
+# ssh_private_key =
+
+[region-cn-north-1]
+ami = ami-bcc45885
+instance_type = t2.micro
+aki_i386 = aki-908f1da9
+aki_x86_64 = aki-9e8f1da7
+g2_aki_x86_64 = aki-9c4ad8a5
+user = ec2-user
+# Allow a region to overwrite the account key
+# ssk_key_name =
+# ssh_private_key =
+
+[region-eu-central-1]
+ami = ami-bc5b48d0
+instance_type = t2.micro
+aki_i386 = aki-3e4c7a23
+aki_x86_64 = aki-184c7a05
+g2_aki_x86_64 = aki-e23f09ff
+user = ec2-user
+# Allow a region to overwrite the account key
+# ssk_key_name =
+# ssh_private_key =
+
+[region-eu-west-1]
+ami = ami-bff32ccc
+instance_type = t2.micro
+aki_i386 = aki-68a3451f
+aki_x86_64 = aki-52a34525
+g2_aki_x86_64 = aki-fce8478b
+user = ec2-user
+# Allow a region to overwrite the account key
+# ssk_key_name =
+# ssh_private_key =
+
+[region-sa-east-1]
+ami = ami-6817af04
+instance_type = t2.micro
+aki_i386 = aki-5b53f446
+aki_x86_64 = aki-5553f448
+g2_aki_x86_64 = aki-b99024a4
+user = ec2-user
+# Allow a region to overwrite the account key
+# ssk_key_name =
+# ssh_private_key =
+
+[region-us-east-1]
+#Amazon Linux
+#ami = ami-4b814f22
+# SLES 12
+ami = ami-60b6c60a
+instance_type = t2.micro
+aki_i386 = aki-8f9dcae6
+aki_x86_64 = aki-919dcaf8
+g2_aki_x86_64 = aki-f4bc469c
+user = ec2-user
+backing-store = mag,ssd
+# Allow a region to overwrite the account key
+# ssk_key_name =
+# ssh_private_key =
+
+[region-us-east-2]
+ami = ami-71ca9114
+instance_type = t2.micro
+aki_i386 = aki-da055ebf
+aki_x86_64 = aki-d83a61bd
+user = ec2-user
+backing-store = ssd
+# Allow a region to overwrite the account key
+# ssk_key_name =
+# ssh_private_key =
+
+[region-us-gov-west-1]
+ami = ami-c2b5d7e1
+instance_type = t2.micro
+aki_i386 = aki-1fe98d3c
+aki_x86_64 = aki-1de98d3e
+g2_aki_x86_64 = aki-07026424
+user = ec2-user
+# Allow a region to overwrite the account key
+# ssk_key_name =
+# ssh_private_key =
+
+[region-us-west-1]
+ami = ami-d5ea86b5
+instance_type = t2.micro
+aki_i386 = aki-8e0531cb
+aki_x86_64 = aki-880531cd
+g2_aki_x86_64 = aki-f9786dbc
+user = ec2-user
+# Allow a region to overwrite the account key
+# ssk_key_name =
+# ssh_private_key =
+
+[region-us-west-2]
+ami = ami-f0091d91
+instance_type = t2.micro
+aki_i386 = aki-f08f11c0
+aki_x86_64 = aki-fc8f11cc
+g2_aki_x86_64 = aki-5f125d6f
+user = ec2-user
+# Allow a region to overwrite the account key
+# ssk_key_name =
+# ssh_private_key =
+

dist/obs-server.spec

@@ -281,6 +281,17 @@ This package contains test cases for testing a installed appliances.
  * checks if database setup worked correctly
  * checks if required service came up properly
 
+%package -n obs-cloud-uploader
+Summary:       The Open Build Service -- Image Cloud Uploader
+Requires:      obs-server
+Requires:      python3-boto3
+Requires:      python3-ec2uploadimg
+Requires:      python3-ec2utilsbase
+Group:         Productivity/Networking/Web/Utilities
+
+%description -n obs-cloud-uploader
+This package contains all the necessary tools for upload images to the cloud.
+
 #--------------------------------------------------------------------------------
 %prep
 
@@ -339,6 +350,10 @@ if ! test -L %{buildroot}/usr/lib/obs/server/build; then
   exit 1
 fi
 
+install -m 755 $RPM_BUILD_DIR/open-build-service-%version/dist/clouduploader.rb $RPM_BUILD_ROOT/%{_bindir}/clouduploader
+mkdir -p $RPM_BUILD_ROOT/etc/obs/cloudupload
+install -m 644 $RPM_BUILD_DIR/open-build-service-%version/dist/ec2utils.conf.example $RPM_BUILD_ROOT/etc/obs/cloudupload/.ec2utils.conf
+
 %check
 %if 0%{?disable_obs_test_suite}
 echo "WARNING:"
@@ -405,23 +420,33 @@ getent passwd obsrun >/dev/null || \
 exit 0
 
 %preun
-%stop_on_removal obssrcserver obsrepserver obsdispatcher obsscheduler obspublisher obswarden obssigner obsdodup obsservicedispatch obsservice obsclouduploadworker obsclouduploadserver
+%stop_on_removal obssrcserver obsrepserver obsdispatcher obsscheduler obspublisher obswarden obssigner obsdodup obsservicedispatch obsservice
 
 %service_del_preun obsdeltastore
 
 %preun -n obs-worker
 %stop_on_removal obsworker
 
+%preun -n obs-cloud-uploader
+%stop_on_removal obsclouduploadworker obsclouduploadserver
+
 %post
 %if 0%{?suse_version} >= 1315
-%reload_on_update obssrcserver obsrepserver obsdispatcher obspublisher obswarden obssigner obsdodup obsservicedispatch obsservice obsclouduploadworker obsclouduploadserver
+%reload_on_update obssrcserver obsrepserver obsdispatcher obspublisher obswarden obssigner obsdodup obsservicedispatch obsservice
 %else
-%restart_on_update obssrcserver obsrepserver obsdispatcher obspublisher obswarden obssigner obsdodup obsservicedispatch obsservice obsclouduploadworker obsclouduploadserver
+%restart_on_update obssrcserver obsrepserver obsdispatcher obspublisher obswarden obssigner obsdodup obsservicedispatch obsservice
 %endif
 # systemd kills the init script executing the reload first on reload....
 %restart_on_update obsscheduler
 %service_add_post obsdeltastore
 
+%post -n obs-cloud-uploader
+%if 0%{?suse_version} >= 1315
+%reload_on_update obsservice obsclouduploadworker obsclouduploadserver
+%else
+%restart_on_update obsservice obsclouduploadworker obsclouduploadserver
+%endif
+
 %posttrans
 [ -d /srv/obs ] || install -d -o obsrun -g obsrun /srv/obs
 # this changes from directory to symlink. rpm can not handle this itself.
@@ -502,8 +527,6 @@ chown %{apache_user}:%{apache_group} /srv/www/obs/api/log/production.log
 %{_unitdir}/obsdeltastore.service
 /etc/init.d/obsservicedispatch
 /etc/init.d/obssigner
-/etc/init.d/obsclouduploadworker
-/etc/init.d/obsclouduploadserver
 /usr/sbin/obs_admin
 /usr/sbin/obs_serverstatus
 /usr/sbin/rcobsdispatcher
@@ -516,8 +539,6 @@ chown %{apache_user}:%{apache_group} /srv/www/obs/api/log/production.log
 /usr/sbin/rcobsdeltastore
 /usr/sbin/rcobsservicedispatch
 /usr/sbin/rcobssigner
-/usr/sbin/rcobsclouduploadworker
-/usr/sbin/rcobsclouduploadserver
 /usr/lib/obs/server/plugins
 /usr/lib/obs/server/BSDispatcher
 /usr/lib/obs/server/BSRepServer
@@ -549,8 +570,6 @@ chown %{apache_user}:%{apache_group} /srv/www/obs/api/log/production.log
 /usr/lib/obs/server/bs_worker
 /usr/lib/obs/server/bs_signer
 /usr/lib/obs/server/bs_warden
-/usr/lib/obs/server/bs_clouduploadserver
-/usr/lib/obs/server/bs_clouduploadworker
 /usr/lib/obs/server/worker
 /usr/lib/obs/server/worker-deltagen.spec
 %config(noreplace) /usr/lib/obs/server/BSConfig.pm
@@ -698,6 +717,18 @@ usermod -a -G docker obsservicerun
 %dir /usr/lib/obs/tests/appliance
 /usr/lib/obs/tests/appliance/*
 
+%files -n obs-cloud-uploader
+%defattr(-,root,root)
+/etc/init.d/obsclouduploadworker
+/etc/init.d/obsclouduploadserver
+/usr/sbin/rcobsclouduploadworker
+/usr/sbin/rcobsclouduploadserver
+/usr/lib/obs/server/bs_clouduploadserver
+/usr/lib/obs/server/bs_clouduploadworker
+%{_bindir}/clouduploader
+%dir /etc/obs
+%dir /etc/obs/cloudupload
+%config(noreplace) /etc/obs/cloudupload/.ec2utils.conf
 
 %package -n obs-container-registry
 Summary:        The Open Build Service -- container registry

src/api/app/controllers/webui/cloud/upload_jobs_controller.rb

@@ -13,7 +13,7 @@ def index
       end
 
       def new
-        xml_object = OpenStruct.new(params.slice(:project, :package, :repository, :arch, :filename))
+        xml_object = OpenStruct.new(params.slice(:project, :package, :repository, :arch, :filename, :vpc_subnet_id))
         @upload_job = ::Cloud::Backend::UploadJob.new(xml_object: xml_object)
         @ec2_regions = ::Cloud::Ec2::Configuration::REGIONS
       end
@@ -59,7 +59,7 @@ def validate_configuration_presence
 
       def permitted_params
         params.require(:cloud_backend_upload_job).permit(
-          :project, :package, :repository, :arch, :filename, :region, :ami_name, :target
+          :project, :package, :repository, :arch, :filename, :region, :ami_name, :target, :vpc_subnet_id
         )
       end
 

src/api/app/jobs/send_event_emails_job.rb

@@ -1,23 +1,25 @@
 class SendEventEmailsJob < ApplicationJob
   queue_as :mailers
 
-  def perform(event_id)
-    event = Event::Base.find(event_id)
-    subscribers = event.subscribers
+  def perform
+    Event::Base.where(mails_sent: false).order(created_at: :asc).limit(1000).each do |event|
+      subscribers = event.subscribers
 
-    if subscribers.empty?
-      event.update_attributes(mails_sent: true)
-      return
-    end
+      if subscribers.empty?
+        event.update_attributes(mails_sent: true)
+        next
+      end
 
-    begin
-      create_rss_notifications(event)
-      EventMailer.event(subscribers, event).deliver_now
-    rescue StandardError => e
-      Airbrake.notify(e, event_id: event.id)
-    ensure
-      event.update_attributes(mails_sent: true)
+      begin
+        create_rss_notifications(event)
+        EventMailer.event(subscribers, event).deliver_now
+      rescue StandardError => e
+        Airbrake.notify(e, event_id: event.id)
+      ensure
+        event.update_attributes(mails_sent: true)
+      end
     end
+    true
   end
 
   private

src/api/app/models/bs_request_action.rb

@@ -624,7 +624,9 @@ def create_expand_package(packages, opts = {})
 
       # Will this be a new package ?
       unless missing_ok_link
-        unless e && tprj && tprj.exists_package?(tpkg, follow_project_links: true, allow_remote_packages: false)
+        # check if the main package container exists in target.
+        # take into account that an additional local link with spec file might got added
+        unless e && tprj && tprj.exists_package?(ltpkg, follow_project_links: true, allow_remote_packages: false)
           if is_maintenance_release?
             pkg.project.repositories.includes(:release_targets).each do |repo|
               repo.release_targets.each do |rt|

src/api/app/models/cloud/backend/upload_job.rb

@@ -18,6 +18,7 @@ class UploadJob
                      :repository,
                      :arch,
                      :filename,
+                     :vpc_subnet_id,
                      :size,
                      :backend_response
       alias_method :id, :name

src/api/app/models/cloud/ec2/configuration.rb

@@ -30,6 +30,10 @@ class Configuration < ApplicationRecord
       def self.table_name_prefix
         'cloud_ec2_'
       end
+
+      def upload_parameters
+        attributes.except('id', 'created_at', 'updated_at')
+      end
     end
   end
 end