Add support for multiple platforms in OpenShift.

broker-util/oo-admin-ctl-cartridge

@@ -8,7 +8,22 @@ PATH = "#{ENV['OPENSHIFT_BROKER_DIR'] || '/var/www/openshift/broker'}/config/env
 class Command
   def import_node(options)
     env!(options)
-    carts = OpenShift::ApplicationContainerProxy.send(options.node ? :instance : :find_one, *[options.node].compact).get_available_cartridges
+
+    carts = []
+
+    if options.node
+      carts = OpenShift::ApplicationContainerProxy.instance(options.node)
+    else
+      node_platforms = Rails.configuration.openshift[:node_platforms]
+
+      node_platforms.each do |platform|
+        node_carts = OpenShift::ApplicationContainerProxy.find_one(nil, platform).get_available_cartridges
+        node_carts.each do |cart|
+          carts << cart unless carts.any? {|existing_cart| existing_cart.name == cart.name }
+        end
+      end
+    end
+
     types = CartridgeType.update_from(carts, nil, options.force)
     update_types(options, types)
   rescue OpenShift::NodeException

broker/conf/broker.conf

@@ -159,3 +159,8 @@ ALLOW_ALIAS_IN_DOMAIN="false"
 # WARNING: do not include private credentials in any URL; they would be visible in every app's cloned repository.
 DEFAULT_APP_TEMPLATES=
 
+# List the node platforms available on the cloud
+# Comma-separated list of node platforms (e.g.:linux,windows).
+# All node platforms must be present in the node network.
+# In the case of conflicting cartridges, priority is given to the platforms with a lower index.
+# NODE_PLATFORMS=linux

broker/config/environments/development.rb

@@ -103,7 +103,8 @@
     :allow_multiple_haproxy_on_node => conf.get_bool('ALLOW_MULTIPLE_HAPROXY_ON_NODE', "true"),
     :syslog_enabled => conf.get_bool('SYSLOG_ENABLED', 'false'),
     :app_template_for => OpenShift::Controller::Configuration.parse_url_hash(conf.get('DEFAULT_APP_TEMPLATES', nil)),
-    :default_max_teams => (conf.get("DEFAULT_MAX_TEAMS", "0")).to_i
+    :default_max_teams => (conf.get("DEFAULT_MAX_TEAMS", "0")).to_i,
+    :node_platforms => OpenShift::Controller::Configuration.parse_list(conf.get('NODE_PLATFORMS', 'linux')).map { |platform| platform.downcase }
   }
 
   config.auth = {

broker/config/environments/production.rb

@@ -92,7 +92,8 @@
     :allow_multiple_haproxy_on_node => conf.get_bool('ALLOW_MULTIPLE_HAPROXY_ON_NODE', "false"),
     :syslog_enabled => conf.get_bool('SYSLOG_ENABLED', 'false'),
     :app_template_for => OpenShift::Controller::Configuration.parse_url_hash(conf.get('DEFAULT_APP_TEMPLATES', nil)),
-    :default_max_teams => (conf.get("DEFAULT_MAX_TEAMS", "0")).to_i
+    :default_max_teams => (conf.get("DEFAULT_MAX_TEAMS", "0")).to_i,
+    :node_platforms => OpenShift::Controller::Configuration.parse_list(conf.get('NODE_PLATFORMS', 'linux')).map { |platform| platform.downcase }    
   }
 
   config.auth = {

broker/config/environments/test.rb

@@ -101,7 +101,8 @@
     :allow_multiple_haproxy_on_node => conf.get_bool('ALLOW_MULTIPLE_HAPROXY_ON_NODE', "true"),
     :syslog_enabled => conf.get_bool('SYSLOG_ENABLED', 'false'),
     :app_template_for => OpenShift::Controller::Configuration.parse_url_hash(conf.get('DEFAULT_APP_TEMPLATES', nil)),
-    :default_max_teams => (conf.get("DEFAULT_MAX_TEAMS", "0")).to_i
+    :default_max_teams => (conf.get("DEFAULT_MAX_TEAMS", "0")).to_i,
+    :node_platforms => OpenShift::Controller::Configuration.parse_list(conf.get('NODE_PLATFORMS', 'linux')).map { |platform| platform.downcase }
   }
 
   config.auth = {

broker/test/functional/application_controller_test.rb

@@ -758,6 +758,15 @@ def assert_invalid_manifest
       Categories:
       - web_proxy
       MANIFEST
+
+    php_cart = CartridgeCache.find_cartridge(php_version)
+
+    mock_cart = mock
+    mock_cart.expects(:platform).at_least_once.with.returns('linux')
+
+    CartridgeCache.expects(:find_cartridge).at_least_once.with('mock-mock-0.1', anything).returns(mock_cart)
+    CartridgeCache.expects(:find_cartridge).at_least_once.with(php_version, anything).returns(php_cart)
+
     @app_name = "app#{@random}"
     post :create, {"name" => @app_name, "cartridge" => [php_version, {"url" => "manifest://test"}], "domain_id" => @domain.namespace, "scale" => true}
     assert_response :success
@@ -784,6 +793,7 @@ def assert_invalid_manifest
       Categories:
       - web_proxy
       MANIFEST
+
     @app_name = "app#{@random}"
     post :create, {"name" => @app_name, "cartridge" => [php_version, haproxy_version, {"url" => "manifest://test"}], "domain_id" => @domain.namespace, "scale" => true}
     assert_response :unprocessable_entity

common/lib/openshift-origin-common/models/cartridge.rb

@@ -93,7 +93,7 @@ class Cartridge < OpenShift::Model
                   :provides, :requires, :conflicts, :suggests, :native_requires,
                   :path, :license_url, :categories, :website, :suggests_feature,
                   :help_topics, :cart_data_def, :additional_control_actions, :versions, :cartridge_vendor,
-                  :endpoints, :cartridge_version, :obsolete
+                  :endpoints, :cartridge_version, :obsolete, :platform
 
     # Profile information
     attr_accessor :components, :group_overrides,
@@ -148,6 +148,10 @@ def components=(data)
       @components.each {|comp| @_component_name_map[comp.name] = comp }
     end
 
+    def scaling_required?
+      @components.any? { |comp| comp.scaling.required }
+    end
+
     def get_component(comp_name)
       @_component_name_map[comp_name]
     end
@@ -179,6 +183,7 @@ def from_descriptor(spec_hash={})
       self.cart_data_def = spec_hash["Cart-Data"] || {}
       self.additional_control_actions = spec_hash["Additional-Control-Actions"] || []
       self.cartridge_version = spec_hash["Cartridge-Version"] || "0.0.0"
+      self.platform = (spec_hash["Platform"] || "linux").downcase
 
       self.provides = [self.provides] if self.provides.class == String
       self.requires = [self.requires] if self.requires.class == String

common/lib/openshift-origin-common/models/scaling.rb

@@ -1,12 +1,13 @@
 module OpenShift
   class Scaling < OpenShift::Model
-    attr_accessor :min, :max, :min_managed, :multiplier
+    attr_accessor :min, :max, :required, :min_managed, :multiplier
 
     def initialize
       self.min = 1
       self.max = -1
       self.min_managed = 1
       self.multiplier = 1
+      self.required = false
     end
 
 # def multiplier
@@ -25,6 +26,7 @@ def from_descriptor(spec_hash = {})
       self.max = spec_hash["Max"].to_i || -1
       self.min_managed = spec_hash["Min-Managed"].to_i || 1
       self.multiplier = spec_hash["Multiplier"].nil? ? 1 : spec_hash["Multiplier"].to_i
+      self.required = spec_hash["Required"]
       self
     end
 
@@ -33,7 +35,8 @@ def to_descriptor
         "Min" => self.min,
         "Max" => self.max,
         "Min-Managed" => self.min_managed,
-        "Multiplier" => self.multiplier
+        "Multiplier" => self.multiplier,
+        "Required" => self.required,
       }
     end
   end

controller/app/models/application.rb

@@ -518,24 +518,30 @@ def implicit_application_overrides(specs=nil)
 
     # Overrides from the basic cartridge definitions.
     overrides = []
+
     specs.each do |spec|
       # Cartridges can contribute overrides
       spec.cartridge.group_overrides.each do |override|
         if o = GroupOverride.resolve_from(specs, override)
-          overrides << o.implicit
+          platforms = o.components.map {|component| CartridgeCache.find_cartridge(component.cartridge_name, self).platform }.uniq
+
+          # only allow grouping if we're working with a single platform
+          if platforms.size == 1
+            overrides << o.implicit
+          end
         end
       end
 
       # Each component has an implicit override based on its scaling
       comp = spec.component
       overrides <<
-        if comp.is_sparse?
-          GroupOverride.new([spec]).implicit
-        elsif spec.cartridge.is_external?
-          GroupOverride.new([spec], 0, 0).implicit
-        else
-          GroupOverride.new([spec], comp.scaling.min, comp.scaling.max).implicit
-        end
+          if comp.is_sparse?
+            GroupOverride.new([spec]).implicit
+          elsif spec.cartridge.is_external?
+            GroupOverride.new([spec], 0, 0).implicit
+          else
+            GroupOverride.new([spec], comp.scaling.min, comp.scaling.max).implicit
+          end
     end
 
     # Overrides that are implicit to applications of this type
@@ -714,6 +720,11 @@ def add_cartridges(cartridges, group_overrides=[], init_git_url=nil, user_env_va
         end
       end
 
+      # Validate that we are not trying to create a non-scalable app with carts that need to be scalable
+      if cart.scaling_required? && !self.scalable
+        raise OpenShift::UserException.new("#{cart.name} must be embedded in a scalable app.", 109)
+      end
+
       # Validate that the cartridges support scalable if necessary
       if self.scalable && !(cart.is_plugin? || cart.is_service? || cart.is_web_framework? || cart.is_external?)
         raise OpenShift::UserException.new("#{cart.name} cannot be embedded in scalable app '#{name}'.", 109, 'cartridge')
@@ -1745,6 +1756,13 @@ def self.run_in_application_lock(application, &block)
     end
   end
 
+  # Determines if the application's cartridges are colocatable
+  # == Returns:
+  # True if the app is hosted on a single platform, false otherwise
+  def is_collocatable?()
+    self.component_instances.map {|comp| comp.get_cartridge.platform }.uniq.size == 1
+  end
+
   def update_requirements(cartridges, replacements, overrides, init_git_url=nil, user_env_vars=nil)
     current = group_instances_with_overrides
     connections, updated = elaborate(cartridges, overrides)
@@ -1808,7 +1826,10 @@ def calculate_gear_create_ops(ginst_id, gear_ids, deploy_gear_id, comp_specs, co
         app_dns_gear_id = gear_id.to_s
       end
 
-      init_gear_op = InitGearOp.new(group_instance_id: ginst_id, gear_id: gear_id,
+      cartridge = CartridgeCache.find_cartridge(comp_specs.first.cartridge_name, self)
+      platform = cartridge ? cartridge.platform : nil
+
+      init_gear_op = InitGearOp.new(group_instance_id: ginst_id, platform: platform, gear_id: gear_id,
                                     gear_size: gear_size, addtl_fs_gb: additional_filesystem_gb,
                                     comp_specs: gear_comp_specs[gear_id], host_singletons: host_singletons,
                                     app_dns: app_dns, pre_save: (not self.persisted?))
@@ -2033,7 +2054,11 @@ def calculate_add_component_ops(gear_comp_specs, comp_spec_gears, group_instance
         end
 
         git_url = nil
-        git_url = init_git_url if gear_id == deploy_gear_id && cartridge.is_deployable?
+
+        if gear_id == deploy_gear_id and needs_git_url?(cartridge)
+          git_url = init_git_url
+        end
+
         add_component_op = AddCompOp.new(gear_id: gear_id, comp_spec: comp_spec, init_git_url: git_url, prereq: new_component_op_id + [prereq_id])
         ops << add_component_op
         component_ops[comp_spec][:adds] << add_component_op
@@ -2958,4 +2983,11 @@ def compute_upgrades(replacements)
         h
       end
     end
+
+    def needs_git_url?(cartridge)
+      # we need the template git url for the component if the cartridge is deployable
+      # or if we have a standalone web cartridge
+      is_standalone_web_proxy = (cartridge.is_web_proxy? and !self.is_collocatable?)
+      (cartridge.is_deployable? or is_standalone_web_proxy)
+    end
 end

controller/app/models/cartridge_type.rb

@@ -181,6 +181,10 @@ def has_feature?(feature)
     provides.include?(feature)
   end
 
+  def scaling_required?
+    self.components.any? { |comp| comp.scaling.required }
+  end
+
   def ===(other)
     return true if other == self
     if other.is_a?(String)

controller/app/models/gear.rb

@@ -94,9 +94,8 @@ def self.valid_gear_size?(gear_size)
 
   def reserve_uid(gear_size = nil)
     gear_size = group_instance.gear_size unless gear_size
-
     opts = { :node_profile => gear_size, :least_preferred_servers => non_ha_server_identities,
-             :restricted_servers => restricted_server_identities, :gear => self }
+             :restricted_servers => restricted_server_identities, :gear => self, :platform => group_instance.platform}
     @container = OpenShift::ApplicationContainerProxy.find_available(opts)
     reserved_uid = @container.reserve_uid
     Application.where({"_id" => application._id, "gears.uuid" => self.uuid}).update({"$set" => {"gears.$.server_identity" => @container.id, "gears.$.uid" => reserved_uid}})

controller/app/models/group_instance.rb

@@ -17,6 +17,8 @@ class GroupInstance
   include Mongoid::Document
   embedded_in :application, class_name: Application.name
 
+  field :platform, type: String, default: "linux"
+
   field :gear_size, type: String
   field :addtl_fs_gb, type: Integer
 
@@ -30,9 +32,14 @@ class GroupInstance
   #   @Note used when this gear is hosting the web_proxy component
   def initialize(attrs = nil, options = nil)
     custom_id = attrs[:custom_id]
+    custom_platform = attrs[:custom_platform]
+
     attrs.delete(:custom_id)
+    attrs.delete(:custom_platform)
+
     super(attrs, options)
     self._id = custom_id unless custom_id.nil?
+    self.platform = custom_platform unless custom_platform.nil?
   end
 
   def gears

controller/app/pending_ops_models/init_gear_op.rb

@@ -2,6 +2,7 @@ class InitGearOp < PendingAppOp
 
   field :gear_id, type: String
   field :group_instance_id, type: String
+  field :platform, type: String
   field :host_singletons, type: Boolean, default: false
   field :app_dns, type: Boolean, default: false
 
@@ -21,7 +22,7 @@ def execute
         begin
           get_group_instance
         rescue Mongoid::Errors::DocumentNotFound
-          application.group_instances << GroupInstance.new(custom_id: group_instance_id, addtl_fs_gb: addtl_fs_gb, gear_size: gear_size)
+          application.group_instances << GroupInstance.new(custom_id: group_instance_id, addtl_fs_gb: addtl_fs_gb, gear_size: gear_size, custom_platform: platform)
           get_group_instance
         end
 

controller/lib/openshift/application_container_proxy.rb

@@ -41,8 +41,8 @@ def self.find_available(opts=nil)
       @proxy_provider.new(server_id, district)
     end
 
-    def self.find_one(node_profile=nil)
-      server_id = @proxy_provider.find_one_impl(node_profile)
+    def self.find_one(node_profile=nil, platform='linux')
+      server_id = @proxy_provider.find_one_impl(node_profile, platform)
       raise OpenShift::NodeUnavailableException.new("No nodes available", 140) if server_id.blank?
       @proxy_provider.new(server_id)
     end

node-util/bin/oo-gear-registry

@@ -58,7 +58,7 @@ command :read do |c|
 
     if requested_gears
       requested_gears.each_value do |gear|
-        puts "#{gear.uuid},#{gear.namespace},#{gear.dns},#{gear.proxy_hostname},#{gear.proxy_port}"
+        puts "#{gear.uuid},#{gear.namespace},#{gear.dns},#{gear.proxy_hostname},#{gear.proxy_port},#{gear.platform}"
       end
     end
   end