Permalink
Browse files

Merge branch 'master' into error-normalization

Must add to cloudfoundry/errors:
["StackInvalid",   HTTP::BAD_REQUEST, 250001, "The stack is invalid: %s"],
["StackNameTaken", HTTP::BAD_REQUEST, 250002, "The stack name is taken: %s"],
["StackNotFound",  HTTP::NOT_FOUND,   250003, "The stack could not be found: %s"]

Conflicts:
	lib/cloud_controller/errors.rb
  • Loading branch information...
2 parents 791fbcc + 5520dac commit 6242f89e489e92d19b7919892ebc1f6fa6ce34e4 @mariash mariash committed Mar 7, 2013
Showing with 1,403 additions and 605 deletions.
  1. +5 −0 config/stacks.yml
  2. +38 −0 db/migrations/20130219194917_create_stacks_table.rb
  3. +5 −4 lib/cloud_controller/api/app.rb
  4. +11 −14 lib/cloud_controller/api/app_bits.rb
  5. +2 −2 lib/cloud_controller/api/space_summary.rb
  6. +20 −0 lib/cloud_controller/api/stack.rb
  7. +1 −1 lib/cloud_controller/app_package.rb
  8. +1 −0 lib/cloud_controller/app_stager.rb
  9. +16 −10 lib/cloud_controller/config.rb
  10. +5 −1 lib/cloud_controller/dea/dea_client.rb
  11. +46 −51 lib/cloud_controller/dea/dea_pool.rb
  12. 0 lib/cloud_controller/errors.rb
  13. +1 −1 lib/cloud_controller/legacy_api/legacy_bulk.rb
  14. +1 −0 lib/cloud_controller/models.rb
  15. +13 −10 lib/cloud_controller/models/app.rb
  16. +81 −0 lib/cloud_controller/models/stack.rb
  17. +3 −5 lib/cloud_controller/rest_controller/object_serialization.rb
  18. +9 −4 lib/cloud_controller/runner.rb
  19. +0 −1 lib/sequel_plugins/vcap_validations.rb
  20. +90 −66 spec/api/app_bits_spec.rb
  21. +5 −2 spec/api/app_spec.rb
  22. +20 −13 spec/api/helpers/invalid_resource.rb
  23. +34 −0 spec/api/helpers/rejects_changes.rb
  24. +3 −3 spec/api/legacy_bulk_spec.rb
  25. +1 −1 spec/api/legacy_staging_spec.rb
  26. +2 −2 spec/api/space_summary_spec.rb
  27. +17 −2 spec/api/spec_helper.rb
  28. +16 −0 spec/api/stacks_spec.rb
  29. +63 −12 spec/app_package_spec.rb
  30. +311 −293 spec/app_stager_spec.rb
  31. +30 −0 spec/config_spec.rb
  32. +8 −1 spec/dea/dea_client_spec.rb
  33. +96 −83 spec/dea/dea_pool_spec.rb
  34. +2 −0 spec/fixtures/config/invalid_stacks.yml
  35. +57 −0 spec/fixtures/config/minimal_config.yml
  36. +7 −0 spec/fixtures/config/stacks.yml
  37. +16 −0 spec/integration/info_spec.rb
  38. +51 −3 spec/models/app_spec.rb
  39. +113 −0 spec/models/stack_spec.rb
  40. +116 −15 spec/spec_helper.rb
  41. +6 −0 spec/support/fakes/blueprints.rb
  42. +14 −5 spec/support/fakes/service_gateway_client_fake.rb
  43. +67 −0 spec/support/integration_setup.rb
View
@@ -0,0 +1,5 @@
+# lucid64 stack is just a default stack that was made
+# in the create_stacks_table migration
+default: "lucid64"
+
+stacks: []
@@ -0,0 +1,38 @@
+# Copyright (c) 2009-2013 VMware, Inc.
+require "securerandom"
+
+Sequel.migration do
+ change do
+ create_table :stacks do
+ VCAP::Migration.common(self)
+
+ String :name, :null => false, :case_insenstive => true
+ String :description, :null => false
+
+ index :name, :unique => true
+ end
+
+ # Create single stack to be able to
+ # set existing apps to use it; otherwise,
+ # we cannot add non_null constraint on apps.stack_id.
+ lucid64_stack_id = self[:stacks].insert(
+ :guid => SecureRandom.uuid,
+ :name => "lucid64",
+ :description => "Ubuntu 10.04 on x86-64",
+ :created_at => Time.now,
+ )
+
+ alter_table :apps do
+ add_column :stack_id, Integer
+ add_foreign_key [:stack_id], :stacks, :name => :fk_apps_stack_id
+ end
+
+ self[:apps].update(
+ :stack_id => lucid64_stack_id,
+ )
+
+ alter_table :apps do
+ set_column_not_null :stack_id
+ end
+ end
+end
@@ -13,20 +13,21 @@ module VCAP::CloudController
define_attributes do
attribute :name, String
attribute :production, Message::Boolean, :default => false
+
to_one :space
to_one :runtime, :optional_in => :create
to_one :framework, :optional_in => :create
+ to_one :stack, :optional_in => :create
+
attribute :environment_json, Hash, :default => {}
attribute :memory, Integer, :default => 256
attribute :instances, Integer, :default => 1
attribute :disk_quota, Integer, :default => 1024
- # TODO: renable exclude_in => :create for state, but not until it is
- # coordinated with ilia and ramnivas
- attribute :state, String, :default => "STOPPED" # , :exclude_in => :create
+ attribute :state, String, :default => "STOPPED"
attribute :command, String, :default => nil
attribute :console, Message::Boolean, :default => false
- attribute :debug, String, :default => nil
+ attribute :debug, String, :default => nil
# a URL pointing to a git repository
# note that this will not match private git URLs, i.e. git@github.com:foo/bar.git
@@ -14,38 +14,36 @@ module VCAP::CloudController
def upload(id)
app = find_id_and_validate_access(:update, id)
- if config[:nginx][:use_nginx]
- mandatory_params = ["application_path", "resources"]
- else
- mandatory_params = ["application", "resources"]
- end
- mandatory_params.each do |k|
- raise Errors::AppBitsUploadInvalid.new("missing :#{k}") unless params[k]
+ unless params["resources"]
+ raise Errors::AppBitsUploadInvalid.new("missing :resources")
end
resources = json_param("resources")
unless resources.kind_of?(Array)
- raise Errors::AppBitsUploadInvalid.new("resources is not an Array")
+ raise Errors::AppBitsUploadInvalid.new("invalid :resources")
end
# TODO: validate upload path
if config[:nginx][:use_nginx]
- path = params["application_path"]
- uploaded_file = Struct.new(:path).new(path)
+ if path = params["application_path"]
+ uploaded_file = Struct.new(:path).new(path)
+ end
else
application = params["application"]
if application.kind_of?(Hash) && application[:tempfile]
uploaded_file = application[:tempfile]
- else
- raise Errors::AppBitsUploadInvalid.new("bad :application")
end
end
- sha1 = AppPackage.to_zip(app.guid, uploaded_file, resources)
+ sha1 = AppPackage.to_zip(app.guid, resources, uploaded_file)
app.package_hash = sha1
app.save
HTTP::CREATED
+ rescue VCAP::CloudController::Errors::AppBitsUploadInvalid, VCAP::CloudController::Errors::AppPackageInvalid
+ app.package_state = "FAILED"
+ app.save
+ raise
end
def download(id)
@@ -78,7 +76,6 @@ def json_param(name)
end
put "#{path_id}/bits", :upload
-
get "#{path_id}/download", :download
end
end
@@ -49,8 +49,8 @@ def app_summary(app)
:urls => app.routes.map(&:fqdn),
:routes => app.routes.map(&:as_summary_json),
:service_count => app.service_bindings_dataset.count,
- :framework_name => app.framework.name,
- :runtime_name => app.runtime.name,
+ :framework => { :name => app.framework.name },
+ :runtime => { :name => app.runtime.name },
:running_instances => 0,
}.merge(app.to_hash)
end
@@ -0,0 +1,20 @@
+# Copyright (c) 2009-2012 VMware, Inc.
+
+module VCAP::CloudController
+ rest_controller :Stack do
+ disable_default_routes
+
+ permissions_required do
+ read Permissions::CFAdmin
+ read Permissions::Authenticated
+ end
+
+ define_attributes do
+ attribute :name, String
+ attribute :description, String
+ end
+
+ get path, :enumerate
+ get path_id, :read
+ end
+end
@@ -19,7 +19,7 @@ def configure(config = {})
# Collects the necessary files and returns the sha1 of the resulting
# app package.
- def to_zip(guid, uploaded_file, resources)
+ def to_zip(guid, resources, uploaded_file)
validate_package_size(uploaded_file, resources)
tmpdir = Dir.mktmpdir
@@ -21,6 +21,7 @@ def configure(config, message_bus, redis_client = nil)
end
def stage_app(app, options={}, &completion_callback)
+ raise Errors::AppPackageInvalid.new("The app package hash is empty") if app.package_hash.nil? || app.package_hash.empty?
task = AppStagerTask.new(@config, @message_bus, @redis_client, app)
task.stage(options, &completion_callback)
end
@@ -44,9 +44,11 @@ class VCAP::CloudController::Config < VCAP::Config
optional(:tmpdir) => String,
optional(:droplets) => String,
optional(:staging_manifests) => String,
+ optional(:stacks) => String,
},
optional(:runtimes_file) => String,
+ optional(:stacks_file) => String,
:db => {
:database => String, # db connection string for sequel
@@ -137,14 +139,6 @@ def self.from_file(file_name)
merge_defaults(config)
end
- def self.merge_defaults(config)
- config[:directories] ||= {}
- config[:directories][:staging_manifests] ||=
- File.join(config_dir, "frameworks")
- config[:runtimes_file] ||= File.join(config_dir, "runtimes.yml")
- config
- end
-
def self.configure(config)
mbus = VCAP::CloudController::MessageBus.new(config)
VCAP::CloudController::MessageBus.instance = mbus
@@ -156,8 +150,9 @@ def self.configure(config)
VCAP::CloudController::AppStager.configure(config, mbus)
VCAP::CloudController::LegacyStaging.configure(config)
- VCAP::CloudController::DeaPool.configure(config, mbus)
- VCAP::CloudController::DeaClient.configure(config, mbus)
+ dea_pool = VCAP::CloudController::DeaPool.new(config, mbus)
+ VCAP::CloudController::DeaClient.configure(config, mbus, dea_pool)
+
VCAP::CloudController::HealthManagerClient.configure(mbus)
VCAP::CloudController::LegacyBulk.configure(config, mbus)
@@ -167,4 +162,15 @@ def self.configure(config)
def self.config_dir
@config_dir ||= File.expand_path("../../../config", __FILE__)
end
+
+ private
+
+ def self.merge_defaults(config)
+ config[:runtimes_file] ||= File.join(config_dir, "runtimes.yml")
+ config[:stacks_file] ||= File.join(config_dir, "stacks.yml")
+
+ config[:directories] ||= {}
+ config[:directories][:staging_manifests] ||= File.join(config_dir, "frameworks")
+ config
+ end
end
@@ -24,12 +24,16 @@ class << self
attr_reader :config, :message_bus, :dea_pool
- def configure(config, message_bus, dea_pool = DeaPool)
+ def configure(config, message_bus, dea_pool)
@config = config
@message_bus = message_bus
@dea_pool = dea_pool
end
+ def run
+ @dea_pool.register_subscriptions
+ end
+
def start(app)
start_instances_in_range(app, (0...app.instances))
app.routes_changed = false
@@ -3,73 +3,68 @@
require "vcap/stager/client"
module VCAP::CloudController
- module DeaPool
+ class DeaPool
DEA_ADVERTISEMENT_EXPIRATION = 10
- class << self
- attr_reader :config, :message_bus
+ attr_reader :config, :message_bus
- def configure(config, message_bus)
- @config = config
- @message_bus = message_bus
- @deas = {}
+ def initialize(config, message_bus)
+ @config = config
+ @message_bus = message_bus
+ @deas = {}
+ end
+
+ def register_subscriptions
+ message_bus.subscribe("dea.advertise") do |msg|
+ logger.debug2("Received dea.advertise: #{msg}")
+ process_advertise_message(msg)
end
+ end
- def register_subscriptions
- message_bus.subscribe("dea.advertise") do |msg|
- process_advertise_message(msg)
- end
+ def process_advertise_message(msg)
+ mutex.synchronize do
+ @deas[msg[:id]] = {
+ :advertisement => msg,
+ :last_update => Time.now
+ }
end
+ end
- def find_dea(mem, runtime)
- mutex.synchronize do
- deas.keys.shuffle.each do |id|
- dea = lookup_dea_unless_expired(id)
- next unless dea
- return id if dea_meets_needs?(dea, mem, runtime)
+ def find_dea(mem, runtime)
+ mutex.synchronize do
+ @deas.keys.shuffle.each do |id|
+ dea = @deas[id]
+ if dea_expired?(dea)
+ @deas.delete(id)
+ elsif dea_meets_needs?(dea, mem, runtime)
+ return id
end
- nil
end
+ nil
end
+ end
- private
-
- attr_reader :deas
+ private
- def process_advertise_message(msg)
- logger.debug2 "dea advertisement #{msg}"
- refresh_dea_stats(msg[:id], msg)
- end
-
- def refresh_dea_stats(id, advertisement)
- mutex.synchronize do
- deas[id] = { :advertisement => advertisement,
- :last_update => Time.now }
- end
- end
-
- def lookup_dea_unless_expired(id)
- dea = deas[id]
- if Time.now.to_i - dea[:last_update].to_i > DEA_ADVERTISEMENT_EXPIRATION
- deas.delete(id)
- dea = nil
- end
- dea
- end
+ def dea_expired?(dea)
+ (Time.now.to_i - dea[:last_update].to_i) > DEA_ADVERTISEMENT_EXPIRATION
+ end
- def dea_meets_needs?(dea, mem, runtime)
- stats = dea[:advertisement]
- stats[:available_memory] >= mem && (stats[:runtimes].nil? || stats[:runtimes].member?(runtime))
+ def dea_meets_needs?(dea, mem, runtime)
+ stats = dea[:advertisement]
+ if stats[:available_memory] >= mem
+ (stats[:runtimes].nil? || stats[:runtimes].member?(runtime))
+ else
+ false
end
+ end
- MUTEX = Mutex.new
- def mutex
- MUTEX
- end
+ def mutex
+ @mutes ||= Mutex.new
+ end
- def logger
- @logger ||= Steno.logger("cc.dea.pool")
- end
+ def logger
+ @logger ||= Steno.logger("cc.dea.pool")
end
end
end
No changes.
@@ -82,7 +82,7 @@ def bulk_apps
apps = {}
Models::App.where { |app|
app.id > last_id
- }.limit(batch_size).each do |app|
+ }.order(:id).limit(batch_size).each do |app|
hash = {}
export_attributes = [
:instances,
@@ -94,6 +94,7 @@ class InvalidRelation < StandardError; end
require "cloud_controller/models/service_instance"
require "cloud_controller/models/service_plan"
require "cloud_controller/models/space"
+require "cloud_controller/models/stack"
require "cloud_controller/models/user"
require "cloud_controller/models/quota_definition"
Oops, something went wrong.

0 comments on commit 6242f89

Please sign in to comment.