Allow custom auth tokens

 - Fix initial advertisement of plans from mpgw to ccng
 - Update Appdirect marketplace to process updated offering schema
 - Bump base gem
   git shortlog 48a675d..f13368a717
   Harshawardhan Gadgil (2):
      Refactor async services gateway
      Merge "Refactor async services gateway"

   Tang Rui (2):
      [services-ng] Still stop warden container even if failed to run stop script to stop
        service process
      [services-ng] Destroy warden container when failed to start service instance

Change-Id: I7a5c1313069f8747dc7d9082d6739808d6a881bb

atmos/Gemfile

@@ -13,7 +13,7 @@ gem "xml-simple"
 
 gem 'vcap_common', :require => ['vcap/common', 'vcap/component'], :git => 'git://github.com/cloudfoundry/vcap-common.git', :ref => 'fd6b6d91'
 gem 'vcap_logging', :require => ['vcap/logging'], :git => 'git://github.com/cloudfoundry/common.git', :ref => 'b96ec1192'
-gem 'vcap_services_base', :git => 'git://github.com/cloudfoundry/vcap-services-base.git', :ref => '48a675d'
+gem 'vcap_services_base', :git => 'git://github.com/cloudfoundry/vcap-services-base.git', :ref => 'f13368a717'
 gem 'warden-client', :require => ['warden/client'], :git => 'git://github.com/cloudfoundry/warden.git', :ref => 'fe6cb51'
 gem 'warden-protocol', :require => ['warden/protocol'], :git => 'git://github.com/cloudfoundry/warden.git', :ref => 'fe6cb51'
 

atmos/Gemfile.lock

@@ -32,8 +32,8 @@ GIT
 
 GIT
   remote: git://github.com/cloudfoundry/vcap-services-base.git
-  revision: 48a675d08d22e1d38f30de1fde84534fcd22d60a
-  ref: 48a675d
+  revision: f13368a7175f34e67b718f4aad59e07d454c7581
+  ref: f13368a717
   specs:
     vcap_services_base (0.1.16)
       curb (~> 0.7.16)

couchdb/Gemfile

@@ -14,7 +14,7 @@ gem "couchrest"
 
 gem 'vcap_common', :require => ['vcap/common', 'vcap/component'], :git => 'git://github.com/cloudfoundry/vcap-common.git', :ref => 'fd6b6d91'
 gem 'vcap_logging', :require => ['vcap/logging'], :git => 'git://github.com/cloudfoundry/common.git', :ref => 'b96ec1192'
-gem 'vcap_services_base', :git => 'git://github.com/cloudfoundry/vcap-services-base.git', :ref => '48a675d'
+gem 'vcap_services_base', :git => 'git://github.com/cloudfoundry/vcap-services-base.git', :ref => 'f13368a717'
 gem 'warden-client', :require => ['warden/client'], :git => 'git://github.com/cloudfoundry/warden.git', :ref => 'fe6cb51'
 gem 'warden-protocol', :require => ['warden/protocol'], :git => 'git://github.com/cloudfoundry/warden.git', :ref => 'fe6cb51'
 

couchdb/Gemfile.lock

@@ -32,8 +32,8 @@ GIT
 
 GIT
   remote: git://github.com/cloudfoundry/vcap-services-base.git
-  revision: 48a675d08d22e1d38f30de1fde84534fcd22d60a
-  ref: 48a675d
+  revision: f13368a7175f34e67b718f4aad59e07d454c7581
+  ref: f13368a717
   specs:
     vcap_services_base (0.1.16)
       curb (~> 0.7.16)

echo/Gemfile

@@ -13,7 +13,7 @@ gem "thin"
 
 gem 'vcap_common', :require => ['vcap/common', 'vcap/component'], :git => 'git://github.com/cloudfoundry/vcap-common.git', :ref => 'fd6b6d91'
 gem 'vcap_logging', :require => ['vcap/logging'], :git => 'git://github.com/cloudfoundry/common.git', :ref => 'b96ec1192'
-gem 'vcap_services_base', :git => 'git://github.com/cloudfoundry/vcap-services-base.git', :ref => '48a675d'
+gem 'vcap_services_base', :git => 'git://github.com/cloudfoundry/vcap-services-base.git', :ref => 'f13368a717'
 gem 'warden-client', :require => ['warden/client'], :git => 'git://github.com/cloudfoundry/warden.git', :ref => 'fe6cb51'
 gem 'warden-protocol', :require => ['warden/protocol'], :git => 'git://github.com/cloudfoundry/warden.git', :ref => 'fe6cb51'
 

echo/Gemfile.lock

@@ -32,8 +32,8 @@ GIT
 
 GIT
   remote: git://github.com/cloudfoundry/vcap-services-base.git
-  revision: 48a675d08d22e1d38f30de1fde84534fcd22d60a
-  ref: 48a675d
+  revision: f13368a7175f34e67b718f4aad59e07d454c7581
+  ref: f13368a717
   specs:
     vcap_services_base (0.1.16)
       curb (~> 0.7.16)

elasticsearch/Gemfile

@@ -12,7 +12,7 @@ gem "rest-client"
 
 gem 'vcap_common', :require => ['vcap/common', 'vcap/component'], :git => 'git://github.com/cloudfoundry/vcap-common.git', :ref => 'fd6b6d91'
 gem 'vcap_logging', :require => ['vcap/logging'], :git => 'git://github.com/cloudfoundry/common.git', :ref => 'b96ec1192'
-gem 'vcap_services_base', :git => 'git://github.com/cloudfoundry/vcap-services-base.git', :ref => '48a675d'
+gem 'vcap_services_base', :git => 'git://github.com/cloudfoundry/vcap-services-base.git', :ref => 'f13368a717'
 gem 'warden-client', :require => ['warden/client'], :git => 'git://github.com/cloudfoundry/warden.git', :ref => 'fe6cb51'
 gem 'warden-protocol', :require => ['warden/protocol'], :git => 'git://github.com/cloudfoundry/warden.git', :ref => 'fe6cb51'
 

elasticsearch/Gemfile.lock

@@ -32,8 +32,8 @@ GIT
 
 GIT
   remote: git://github.com/cloudfoundry/vcap-services-base.git
-  revision: 48a675d08d22e1d38f30de1fde84534fcd22d60a
-  ref: 48a675d
+  revision: f13368a7175f34e67b718f4aad59e07d454c7581
+  ref: f13368a717
   specs:
     vcap_services_base (0.1.16)
       curb (~> 0.7.16)

filesystem/Gemfile

@@ -12,7 +12,7 @@ gem "thin"
 
 gem 'vcap_common', :require => ['vcap/common', 'vcap/component'], :git => 'git://github.com/cloudfoundry/vcap-common.git', :ref => 'fd6b6d91'
 gem 'vcap_logging', :require => ['vcap/logging'], :git => 'git://github.com/cloudfoundry/common.git', :ref => 'b96ec1192'
-gem 'vcap_services_base', :git => 'git://github.com/cloudfoundry/vcap-services-base.git', :ref => '48a675d'
+gem 'vcap_services_base', :git => 'git://github.com/cloudfoundry/vcap-services-base.git', :ref => 'f13368a717'
 gem 'warden-client', :require => ['warden/client'], :git => 'git://github.com/cloudfoundry/warden.git', :ref => 'fe6cb51'
 gem 'warden-protocol', :require => ['warden/protocol'], :git => 'git://github.com/cloudfoundry/warden.git', :ref => 'fe6cb51'
 

filesystem/Gemfile.lock

@@ -32,8 +32,8 @@ GIT
 
 GIT
   remote: git://github.com/cloudfoundry/vcap-services-base.git
-  revision: 48a675d08d22e1d38f30de1fde84534fcd22d60a
-  ref: 48a675d
+  revision: f13368a7175f34e67b718f4aad59e07d454c7581
+  ref: f13368a717
   specs:
     vcap_services_base (0.1.16)
       curb (~> 0.7.16)

marketplace/Gemfile

@@ -11,7 +11,7 @@ gem "cf-uaa-client"
 
 gem 'vcap_common', :require => ['vcap/common', 'vcap/component'], :git => 'git://github.com/cloudfoundry/vcap-common.git', :ref => 'fd6b6d91'
 gem 'vcap_logging', :require => ['vcap/logging'], :git => 'git://github.com/cloudfoundry/common.git', :ref => 'b96ec1192'
-gem 'vcap_services_base', :git => 'git://github.com/cloudfoundry/vcap-services-base.git', :ref => '48a675d'
+gem 'vcap_services_base', :git => 'git://github.com/cloudfoundry/vcap-services-base.git', :ref => 'f13368a717'
 gem 'warden-client', :require => ['warden/client'], :git => 'git://github.com/cloudfoundry/warden.git', :ref => 'fe6cb51'
 gem 'warden-protocol', :require => ['warden/protocol'], :git => 'git://github.com/cloudfoundry/warden.git', :ref => 'fe6cb51'
 

marketplace/Gemfile.lock

@@ -32,8 +32,8 @@ GIT
 
 GIT
   remote: git://github.com/cloudfoundry/vcap-services-base.git
-  revision: 48a675d08d22e1d38f30de1fde84534fcd22d60a
-  ref: 48a675d
+  revision: f13368a7175f34e67b718f4aad59e07d454c7581
+  ref: f13368a717
   specs:
     vcap_services_base (0.1.16)
       curb (~> 0.7.16)

marketplace/lib/base/cc_client_v2.rb

@@ -63,14 +63,6 @@ def get_registered_services_from_cc
 
               if @service_auth_tokens.has_key?(key.to_sym)
                 entity = s["entity"]
-                svc = {
-                  "label"       => entity["label"],
-                  "description" => entity["description"],
-                  "provider"    => entity["provider"],
-                  "version"     => entity["version"],
-                  "url"         => entity["url"],
-                  "info_url"    => entity["info_url"]
-                }
 
                 plans = {}
                 entity["service_plans"].each { |p|
@@ -81,10 +73,19 @@ def get_registered_services_from_cc
                   }
                 }
 
+                svc = {
+                  "id"          => entity["label"],
+                  "description" => entity["description"],
+                  "provider"    => entity["provider"],
+                  "version"     => entity["version"],
+                  "url"         => entity["url"],
+                  "info_url"    => entity["info_url"],
+                  "plans"       => plans
+                }
+
                 registered_services[key] = {
                   "guid"    => s["metadata"]["guid"],
                   "service" => svc,
-                  "plans"   => plans,
                 }
 
                 @logger.debug("Found #{key} = #{registered_services[key].inspect}")

marketplace/lib/base/marketplace_async_gateway_v2.rb

@@ -5,17 +5,15 @@
 require 'timeout'
 
 require 'vcap/component'
+require 'base_async_gateway'
 require_relative 'cc_client_v2'
 
 module VCAP
   module Services
     module Marketplace
-      class MarketplaceAsyncServiceGatewayV2 < VCAP::Services::AsynchronousServiceGateway
+      class MarketplaceAsyncServiceGatewayV2 < VCAP::Services::BaseAsynchronousServiceGateway
 
-        REQ_OPTS    = %w(mbus external_uri cloud_controller_uri).map {|o| o.to_sym}
-
-        set :raise_errors, Proc.new {false}
-        set :show_exceptions, false
+        REQ_OPTS          = %w(cc_api_version mbus external_uri cloud_controller_uri service_auth_tokens).map {|o| o.to_sym}
 
         def initialize(opts)
           super(opts)
@@ -56,6 +54,7 @@ def setup(opts)
           @handle_fetched        = true # set to true in order to compatible with base asycn gateway.
 
           @refresh_interval      = opts[:refresh_interval] || 300
+          @service_auth_tokens   = opts[:service_auth_tokens]
 
           @marketplace_client    = load_marketplace(opts)
 
@@ -224,7 +223,7 @@ def advertise_services(active=true)
             req, plans = @marketplace_client.generate_ccng_advertise_request(svc, active)
             guid = (@catalog_in_ccdb[label])["guid"]
 
-            plans_to_add, plans_to_update = process_plans(plans, @catalog_in_ccdb[label]["plans"])
+            plans_to_add, plans_to_update = process_plans(plans, @catalog_in_ccdb[label]["service"]["plans"])
 
             @logger.debug("Refresh offering: #{req.inspect}")
             advertise_service_to_cc(req, guid, plans_to_add, plans_to_update)
@@ -248,7 +247,8 @@ def advertise_services(active=true)
           new_offerings = marketplace_offerings - active_offerings
           new_offerings.each do |label|
             svc = @catalog_in_marketplace[label]
-            req, plans_to_add = @marketplace_client.generate_ccng_advertise_request(svc, active)
+            req, plans = @marketplace_client.generate_ccng_advertise_request(svc, active)
+            plans_to_add = plans.values
 
             @logger.debug("Add new offering: #{req.inspect}")
             advertise_service_to_cc(req, nil, plans_to_add, {}) # nil guid => new service, so add all plans
@@ -269,8 +269,6 @@ def advertise_service_to_cc(req, service_guid, plans_to_add, plans_to_update)
           result
         end
 
-
-
         ############  Varz Processing ##############
 
         def snapshot_and_reset_stats
@@ -363,6 +361,13 @@ def on_exit(stop_event_loop=true)
         #
         helpers do
 
+          def is_a_valid_auth_token(token)
+            configured_tokens = @service_auth_tokens.values
+            valid = configured_tokens.include?(token)
+            @logger.error("Requested token: #{token} is invalid") if !valid
+            valid
+          end
+
           def reply_error(resp='{}')
             async_reply_raw(500, {'Content-Type' => Rack::Mime.mime_type('.json')}, resp)
           end
@@ -374,7 +379,38 @@ def fmt_error(e)
 
         #################### Handlers ###################
 
+        # Validate incoming request
+        def validate_incoming_request
+          unless request.media_type == Rack::Mime.mime_type('.json')
+            error_msg = ServiceError.new(ServiceError::INVALID_CONTENT).to_hash
+            abort_request(error_msg)
+          end
+
+          # Check custom service auth tokens
+          unless auth_token && is_a_valid_auth_token(auth_token)
+            error_msg = ServiceError.new(ServiceError::NOT_AUTHORIZED).to_hash
+            @logger.error("Failed auth token check, auth token: #{auth_token} is invalid: #{error_msg.inspect}")
+            abort_request(error_msg)
+          end
+
+          content_type :json
+        end
+
+        error [JsonMessage::ValidationError, JsonMessage::ParseError] do
+          error_msg = ServiceError.new(ServiceError::MALFORMATTED_REQ).to_hash
+          abort_request(error_msg)
+        end
+
+        not_found do
+          error_msg = ServiceError.new(ServiceError::NOT_FOUND, request.path_info).to_hash
+          abort_request(error_msg)
+        end
+
         # Helpers for unit testing
+        get "/" do
+          return {"marketplace" => @marketplace_client.name, "offerings" => @catalog_in_marketplace}.to_json
+        end
+
         post "/marketplace/set/:key/:value" do
           @logger.info("TEST HELPER ENDPOINT - set: key=#{params[:key]}, value=#{params[:value]}")
           Fiber.new {
@@ -389,10 +425,6 @@ def fmt_error(e)
           async_mode
         end
 
-        get "/" do
-          return {"marketplace" => @marketplace_client.name, "offerings" => @catalog_in_marketplace}.to_json
-        end
-
         # Provision a marketplace service
         post "/gateway/v1/configurations" do
           @logger.info("Got request_body=#{request_body}")

marketplace/lib/marketplaces/appdirect/appdirect_helper.rb

@@ -45,12 +45,11 @@ def load_catalog
                 data = JSON.parse(response.body) #VCAP::Services::AppDirect::AppDirectCatalogResponse.decode(raw)
                 catalog = []
                 data.each do |service|
-                  # Add checks for specific categories which determine whether the addon should be listed on cc
-                  @logger.debug("Got service '#{service["id"]}' from AppDirect")
-                  if (@whitelist.nil? || @whitelist.include?(service["id"]))
+                  if (@whitelist.nil? || @whitelist.include?(service["name"]))
+                    @logger.info("Accepting whitelisted service: #{service["name"]}")
                     catalog << service
                   else
-                    @logger.warn("Ignoring service Offering: #{service["id"]} since it is not whitelisted")
+                    @logger.warn("Ignoring service Offering: #{service["name"]} since it is not whitelisted")
                   end
                 end
                 @logger.info("Got #{catalog.size} services from AppDirect")