Remove staging capabilities

- Remove ability for CC to perform staging based on values of new_stager_percent and new_stager_email - Remove secure user setup - Remove dependency on vcap_staging gem Change-Id: Ica2667127dcf77d71ad2414da6ec643afaca4e18
cloudfoundry-attic · Sep 6, 2012 · 176a030 · 176a030
1 parent a1ce3d3
commit 176a030
Show file tree

Hide file tree

Showing 11 changed files with 4 additions and 448 deletions.
diff --git a/cloud_controller/Gemfile b/cloud_controller/Gemfile
@@ -8,7 +8,6 @@ gem 'nats', :require => 'nats/client'
 # VCAP common components
 gem 'vcap_common', :require => ['vcap/common', 'vcap/component'], :git => 'https://github.com/cloudfoundry/vcap-common.git', :ref => 'fd6b6d91'
 gem 'vcap_logging', :require => ['vcap/logging'], :git => 'https://github.com/cloudfoundry/common.git', :ref => 'e36886a1'
-gem 'vcap_staging', '~> 0.1.63', :git => 'https://github.com/cloudfoundry/vcap-staging.git', :ref => '1740e4da'
 gem 'cf-uaa-client', '~> 1.0', :git => 'https://github.com/cloudfoundry/uaa.git', :ref => 'dad29c90'
 
 # For queuing staging tasks

diff --git a/cloud_controller/Gemfile.lock b/cloud_controller/Gemfile.lock
@@ -52,18 +52,6 @@ GIT
       thin
       yajl-ruby (~> 0.8.3)
 
-GIT
-  remote: https://github.com/cloudfoundry/vcap-staging.git
-  revision: 1740e4daef7cb988f16375a4ec7b69392c5705ac
-  ref: 1740e4da
-  specs:
-    vcap_staging (0.1.63)
-      nokogiri (>= 1.4.4)
-      rake
-      rspec
-      uuidtools (~> 2.1.2)
-      yajl-ruby (>= 0.7.9)
-
 GEM
   remote: http://rubygems.org/
   specs:
@@ -233,5 +221,4 @@ DEPENDENCIES
   uuidtools (~> 2.1.2)
   vcap_common!
   vcap_logging!
-  vcap_staging (~> 0.1.63)!
   yajl-ruby (~> 0.8.3)
diff --git a/cloud_controller/app/controllers/apps_controller.rb b/cloud_controller/app/controllers/apps_controller.rb
@@ -133,13 +133,7 @@ def start_update
     error_on_lock_mismatch(@app)
     @app.lock_version += 1
     manager = AppManager.new(@app)
-    if @app.needs_staging?
-      if user.uses_new_stager?
-        stage_app(@app)
-      else
-        manager.stage
-      end
-    end
+    stage_app(@app) if @app.needs_staging?
     manager.stop_all
     manager.started
     render :nothing => true, :status => 204
@@ -181,7 +175,7 @@ def check_update
   def files
     # XXX - Yuck. This will have to do until we update VMC with a real
     #       way to fetch staging logs.
-    if user.uses_new_stager? && (params[:path] == 'logs/staging.log')
+    if params[:path] == 'logs/staging.log'
       log = StagingTaskLog.fetch_fibered(@app.id)
       if log
         render :text => log.task_log
@@ -328,13 +322,7 @@ def update_app_from_params(app)
     # Process any changes that require action on out part here.
     manager = AppManager.new(app)
 
-    if app.needs_staging?
-      if user.uses_new_stager?
-        stage_app(app)
-      else
-        manager.stage
-      end
-    end
+    stage_app(app) if app.needs_staging?
 
     if changed.include?('state')
       if app.stopped?

diff --git a/cloud_controller/app/models/app_manager.rb b/cloud_controller/app/models/app_manager.rb
@@ -1,165 +1,21 @@
 class AppManager
   attr_reader :app
 
-  DEFAULT_MAX_CONCURRENT_STAGERS = 10
-  DEFAULT_MAX_STAGING_RUNTIME    = 60
-
   class << self
 
-    def max_running
-      AppConfig[:staging][:max_concurrent_stagers] || DEFAULT_MAX_CONCURRENT_STAGERS
-    end
-
-    def max_runtime
-      AppConfig[:staging][:max_staging_runtime] || DEFAULT_MAX_STAGING_RUNTIME
-    end
-
     def pending
       @pending ||= []
     end
 
     def running
       @running ||= {}
     end
-
-    def secure_staging_dir(user, dir)
-      CloudController.logger.debug("Securing directory '#{dir}'", :tags => [:staging])
-      system("chown -R #{user[:user]} #{dir}")
-      system("chgrp -R #{user[:group]} #{dir}")
-      system("chmod -R o-rwx #{dir}")
-      system("chmod -R g-rwx #{dir}")
-    end
-
-    def queue_staging_job(job)
-      pending << job
-      VCAP::Component.varz[:pending_stage_cmds] = pending.length
-      process_queue
-    end
-
-    def staging_job_expired(job)
-      CloudController.logger.warn("Killing long running staging process: #{job.inspect}", :tags => [:staging])
-      job.delete(:expire_timer)
-      # Forcefully take out long running stager
-      `kill -9 #{job[:pid]}`
-      complete_running(job)
-    end
-
-    def mark_running(job, pid)
-      job[:pid] = pid
-      job[:start] = Time.now
-      job[:expire_timer] = EM.add_timer(max_runtime) { AppManager.staging_job_expired(job) }
-      running[pid] = job
-
-      VCAP::Component.varz[:running_stage_cmds] = running.length
-    end
-
-    def complete_running(job)
-      EM.cancel_timer(job[:expire_timer]) if job[:expire_timer]
-      if job[:user]
-        CloudController.logger.debug("Checking in user #{job[:user]}", :tags => [:staging, :secure])
-        `sudo -u '##{job[:user][:uid]}' pkill -9 -U #{job[:user][:uid]} 2>&1`
-        SecureUserManager.instance.return_secure_user(job[:user])
-        job[:user] = nil
-      end
-      running.delete job[:pid]
-      VCAP::Component.varz[:running_stage_cmds] = running.length
-      process_queue
-    end
-
-    def process_queue
-      return if running.length >= max_running
-      return if pending.empty?
-      job = pending.shift
-      VCAP::Component.varz[:pending_stage_cmds] = pending.length
-
-      CloudController.logger.debug("Starting staging command: #{job[:cmd]} #{job[:config]}", :tags => [:staging])
-
-      if AppConfig[:staging][:secure]
-        job[:user] = user = SecureUserManager.instance.grab_secure_user
-        CloudController.logger.debug("Checked out user #{user.inspect}", :tags => [:staging, :secure])
-
-        job[:config]["secure_user"] = {
-          "uid" => Integer(user[:uid]),
-          "gid" => Integer(user[:gid]),
-        }
-        CloudController.logger.debug("Command config changed to '#{job[:config]}'", :tags => [:staging, :secure])
-
-        AppManager.secure_staging_dir(job[:user], job[:staging_dir])
-        AppManager.secure_staging_dir(job[:user], job[:exploded_dir])
-        AppManager.secure_staging_dir(job[:user], job[:work_dir])
-      end
-
-      plugin_config_file = File.join(job[:work_dir],"plugin_config")
-      StagingPlugin::Config.to_file(job[:config], plugin_config_file)
-      job[:cmd] = "#{job[:cmd]} #{plugin_config_file}"
-
-      Bundler.with_clean_env do
-
-        pid = EM.system(job[:cmd]) do |output, status|
-
-          if status.exitstatus != 0
-            CloudController.logger.debug("Staging command FAILED with #{status.exitstatus}: #{output}", :tags => [:staging])
-          else
-            CloudController.logger.debug('Staging command SUCCEEDED', :tags => [:staging])
-          end
-
-          Fiber.new do
-            # Finalize staging here if all is well.
-            manager = AppManager.new(job[:app])
-            begin
-              if manager.app_still_exists? # Will reload app
-                # Save the app even if staging failed to display the log to the user
-                manager.package_staged_app(job[:staging_dir])
-                job[:app].package_state = 'FAILED' if status.exitstatus != 0
-                manager.save_staged_app_state
-              end
-            rescue => e
-              CloudController.logger.warn("Exception after return from staging: #{e}", :tags => [:staging])
-              CloudController.logger.error(e, :tags => [:staging])
-            ensure
-              FileUtils.rm_rf(job[:staging_dir])
-              FileUtils.rm_rf(job[:exploded_dir])
-              FileUtils.rm_rf(job[:work_dir]) if job[:work_dir]
-            end
-          end.resume
-
-          # Clean up running reference
-          complete_running(job)
-
-        end
-        mark_running(job, pid)
-
-      end
-    end
-
   end
 
   def initialize(app)
     @app = app
   end
 
-  def run_staging_command(script, exploded_dir, staging_dir, env_json)
-    work_dir = Dir.mktmpdir
-    plugin_config = {
-      "source_dir"   => exploded_dir,
-      "dest_dir"     => staging_dir,
-      "environment"  => env_json
-    }
-
-    job = {
-      :app => @app,
-      :cmd => "#{script}",
-      :config => plugin_config,
-      :staging_dir => staging_dir,
-      :exploded_dir => exploded_dir,
-      :work_dir => work_dir
-    }
-
-    CloudController.logger.debug("Queueing staging command #{job[:cmd]} #{job[:config]}", :tags => [:staging])
-
-    AppManager.queue_staging_job(job)
-  end
-
   def health_manager_message_received(payload)
     CloudController.logger.debug("[HealthManager] Received #{payload[:op]} request for app #{app.id} - #{app.name}")
 
@@ -294,33 +150,6 @@ def save_staged_app_state
     end
   end
 
-  def stage
-    return if app.package_hash.blank? || app.staged?
-
-    CloudController.logger.debug "app: #{app.id} Staging starting"
-
-    app_source_dir = Dir.mktmpdir
-    app.explode_into(app_source_dir)
-    output_dir = Dir.mktmpdir
-    # Call the selected staging script without changing directories.
-    run_plugin_path = Rails.root.join('script', 'run_plugin.rb')
-    staging_script  = "#{CloudController.current_ruby} #{run_plugin_path} #{app.framework}"
-    # Perform staging command
-    run_staging_command(staging_script, app_source_dir, output_dir,
-      app.staging_task_properties)
-
-    once_app_is_staged do
-      CloudController.logger.debug "app: #{app.id} Staging complete"
-    end
-
-  rescue => e
-    CloudController.logger.error("Failed on exception! #{e}", :tags => [:staging])
-    CloudController.logger.error(e, :tags => [:staging])
-    app.package_state = 'FAILED'
-    save_staged_app_state
-    raise e # re-raise here to propogate to the API call.
-  end
-
   # Returns an array of hashes containing 'index', 'state', 'since'(timestamp),
   # 'debug_ip', and 'debug_port' for all instances running, or trying to run,
   # the app.
@@ -564,20 +393,4 @@ def fiber_sleep(secs)
     EM.add_timer(secs) { f.resume }
     Fiber.yield
   end
-
-  # Update the SHA1 stored for the app, repack the new bits, and mark the app as staged.
-  # repack does the right thing but needs a Fiber context, which will not be present here
-  def package_staged_app(staging_dir)
-    tmpdir = Dir.mktmpdir # we create the working directory ourselves so we can clean it up.
-    staged_file = AppPackage.repack_app_in(staging_dir, tmpdir, :tgz)
-
-    app.update_staged_package(staged_file)
-
-    app.package_state = 'STAGED'
-  rescue
-    app.package_state = 'FAILED'
-  ensure
-    FileUtils.rm_rf(tmpdir)
-    FileUtils.rm_rf(File.dirname(staged_file)) if staged_file
-  end
 end
diff --git a/cloud_controller/app/models/user.rb b/cloud_controller/app/models/user.rb
@@ -116,15 +116,4 @@ def no_more_apps?
       count
     end
   end
-
-
-  def uses_new_stager?(cfg=AppConfig)
-    stg = cfg[:staging]
-    if (stg[:new_stager_percent] && ((self.id % 100) < stg[:new_stager_percent])) \
-       || (stg[:new_stager_email_regexp] && stg[:new_stager_email_regexp].match(self.email))
-      true
-    else
-      false
-    end
-  end
 end
diff --git a/cloud_controller/config/appconfig.rb b/cloud_controller/config/appconfig.rb
@@ -3,7 +3,6 @@
 # mode by checking that flag. This code runs too early to know for sure if
 # we are starting in production mode.
 require 'vcap/common'
-require 'vcap/staging/plugin/common'
 require 'openssl'
 
 config_file = ENV['CLOUD_CONTROLLER_CONFIG']
@@ -62,12 +61,10 @@
              :rails_environment => 'development',
              :local_route  => '127.0.0.1',
              :allow_external_app_uris => false,
-             :staging => { :max_concurrent_stagers => 10,
-                           :max_staging_runtime => 60 },
+             :staging => { :max_staging_runtime => 60 },
              :external_port => 9022,
              :directories => { :droplets          => '/var/vcap/shared/droplets',
                                :resources         => '/var/vcap/shared/resources',
-                               :staging_cache     => '/var/vcap.local/staging',
                                :tmpdir            => '/var/vcap/data/cloud_controller/tmp'},
              :mbus => 'nats://localhost:4222/',
              :logging => { :level => 'debug' },
@@ -224,15 +221,6 @@
   exit 1
 end
 
-if AppConfig[:staging][:new_stager_email_regexp]
-  AppConfig[:staging][:new_stager_email_regexp] = Regexp.new(AppConfig[:staging][:new_stager_email_regexp])
-end
-
-if (AppConfig[:staging][:new_stager_percent] || AppConfig[:staging][:new_stager_email_regexp]) && !AppConfig[:redis]
-  $stderr.puts "You must supply a redis config to use the new stager"
-  exit 1
-end
-
 AppConfig[:staging][:queue] ||= "staging"
 
 unless AppConfig[:directories][:staging_manifests]

diff --git a/cloud_controller/config/initializers/staging.rb b/cloud_controller/config/initializers/staging.rb