diff --git a/script/e2e-kube.sh b/script/e2e-kube.sh index af1aab0b0..b5439ba9e 100755 --- a/script/e2e-kube.sh +++ b/script/e2e-kube.sh @@ -138,12 +138,10 @@ if [[ ("${ESP_ROLLOUT_STRATEGY}" == "managed") && ("${BACKEND}" == "bookstore") create_service "${ESP_SERVICE}" "${SERVICE_IDL}" # Need to wait for ServiceControl to detect new rollout - # Need to run some traffic in order for ServiceControl to send the new rollout. # Here wait for 200 seconds. for l in {1..20} do echo "Wait for the new config to propagate: ${l}" - check_http_service "${HOST}/shelves" 200 sleep 10 done diff --git a/src/api_manager/api_manager_impl.cc b/src/api_manager/api_manager_impl.cc index cb35e8354..84c7ddf1b 100644 --- a/src/api_manager/api_manager_impl.cc +++ b/src/api_manager/api_manager_impl.cc @@ -211,7 +211,8 @@ utils::Status ApiManagerImpl::Init() { if (status.ok()) { AddAndDeployConfigs(std::move(configs), true); } - })); + }, + [this]() { DetectRolloutIDChange(); })); if (global_context_->server_config()->has_service_config_rollout()) { config_manager_->set_current_rollout_id(global_context_->server_config() @@ -223,6 +224,13 @@ utils::Status ApiManagerImpl::Init() { return utils::Status::OK; } +void ApiManagerImpl::DetectRolloutIDChange() { + if (!service_context_map_.empty()) { + const auto &it = service_context_map_.begin(); + it->second->service_control()->SendEmptyReport(); + } +} + utils::Status ApiManagerImpl::Close() { if (global_context_->cloud_trace_aggregator()) { global_context_->cloud_trace_aggregator()->SendAndClearTraces(); diff --git a/src/api_manager/api_manager_impl.h b/src/api_manager/api_manager_impl.h index e1c4aec8a..a64eb69a5 100644 --- a/src/api_manager/api_manager_impl.h +++ b/src/api_manager/api_manager_impl.h @@ -94,6 +94,9 @@ class ApiManagerImpl : public ApiManager { utils::Status AddAndDeployConfigs( std::vector> &&configs, bool initialize); + // Send empty report to detect rollout ID change + void DetectRolloutIDChange(); + // The check work flow. std::shared_ptr check_workflow_; diff --git a/src/api_manager/config_manager.cc b/src/api_manager/config_manager.cc index ddc827a4c..874346886 100644 --- a/src/api_manager/config_manager.cc +++ b/src/api_manager/config_manager.cc @@ -25,23 +25,29 @@ const int kFetchThrottleWindowInS = 300; const char kRolloutStrategyManaged[] = "managed"; +// The default periodical interval to detect rollout changes. Unit: seconds. +const int kDetectRolloutChangeIntervalInS = 60; + } // namespace ConfigManager::ConfigManager( std::shared_ptr global_context, - RolloutApplyFunction rollout_apply_function) + RolloutApplyFunction rollout_apply_function, + std::function detect_rollout_func) : global_context_(global_context), rollout_apply_function_(rollout_apply_function), fetch_throttle_window_in_s_(kFetchThrottleWindowInS) { + int detect_rollout_interval_s = kDetectRolloutChangeIntervalInS; if (global_context_->server_config() && global_context_->server_config()->has_service_management_config()) { + const auto& cfg = + global_context_->server_config()->service_management_config(); // update fetch_throttle_window - if (global_context_->server_config() - ->service_management_config() - .fetch_throttle_window_s() > 0) { - fetch_throttle_window_in_s_ = global_context_->server_config() - ->service_management_config() - .fetch_throttle_window_s(); + if (cfg.fetch_throttle_window_s() > 0) { + fetch_throttle_window_in_s_ = cfg.fetch_throttle_window_s(); + } + if (cfg.detect_rollout_interval_s() > 0) { + detect_rollout_interval_s = cfg.detect_rollout_interval_s(); } } static std::random_device random_device; @@ -52,12 +58,20 @@ ConfigManager::ConfigManager( 0, fetch_throttle_window_in_s_ * 1000)); service_management_fetch_.reset(new ServiceManagementFetch(global_context)); + + if (detect_rollout_func) { + detect_rollout_change_timer_ = global_context_->env()->StartPeriodicTimer( + std::chrono::seconds(detect_rollout_interval_s), detect_rollout_func); + } } ConfigManager::~ConfigManager() { if (fetch_timer_) { fetch_timer_->Stop(); } + if (detect_rollout_change_timer_) { + detect_rollout_change_timer_->Stop(); + } }; void ConfigManager::SetLatestRolloutId( diff --git a/src/api_manager/config_manager.h b/src/api_manager/config_manager.h index cf2dd4b5e..8c12a46c4 100644 --- a/src/api_manager/config_manager.h +++ b/src/api_manager/config_manager.h @@ -72,7 +72,8 @@ class ConfigManager { // rollout_apply_function when it successfully downloads the latest successful // rollout ConfigManager(std::shared_ptr global_context, - RolloutApplyFunction rollout_apply_function); + RolloutApplyFunction rollout_apply_function, + std::function detect_rollout_func); virtual ~ConfigManager(); public: @@ -122,6 +123,9 @@ class ConfigManager { // The random objects to throttle the timer std::default_random_engine random_generator_; std::unique_ptr> random_dist_; + + // Periodic timer to send empty report to detect latest rollout change. + std::unique_ptr detect_rollout_change_timer_; }; } // namespace api_manager diff --git a/src/api_manager/config_manager_test.cc b/src/api_manager/config_manager_test.cc index 75757776c..87fc92e19 100644 --- a/src/api_manager/config_manager_test.cc +++ b/src/api_manager/config_manager_test.cc @@ -225,7 +225,8 @@ TEST_F(ConfigManagerServiceNameConfigIdTest, VerifyTimerIntervalDistribution) { [this, &sequence](const utils::Status&, const std::vector>&) { sequence++; - })); + }, + nullptr)); config_manager->set_current_rollout_id("2017-05-01r0"); // Default is 5 minute interval. Use 5 slot for each minute. @@ -283,7 +284,8 @@ TEST_F(ConfigManagerServiceNameConfigIdTest, RolloutSingleServiceConfig) { EXPECT_EQ(kServiceConfig1, list[0].first); EXPECT_EQ(100, list[0].second); sequence++; - })); + }, + nullptr)); config_manager->SetLatestRolloutId("2017-05-01r0", std::chrono::system_clock::now()); @@ -306,7 +308,8 @@ TEST_F(ConfigManagerServiceNameConfigIdTest, RolloutIDNotChanged) { [this, &sequence](const utils::Status& status, const std::vector>& list) { sequence++; - })); + }, + nullptr)); // set rollout_id to 2017-05-01r0 which is same as kRolloutsResponse1 config_manager->set_current_rollout_id("2017-05-01r0"); @@ -327,7 +330,8 @@ TEST_F(ConfigManagerServiceNameConfigIdTest, RepeatedTrigger) { EXPECT_EQ(kServiceConfig1, list[0].first); EXPECT_EQ(100, list[0].second); sequence++; - })); + }, + nullptr)); config_manager->set_current_rollout_id("2017-05-01r0"); auto now = std::chrono::system_clock::now(); @@ -416,7 +420,8 @@ TEST_F(ConfigManagerServiceNameConfigIdTest, RolloutMultipleServiceConfig) { EXPECT_EQ(kServiceConfig2, list[1].first); EXPECT_EQ(20, list[1].second); sequence++; - })); + }, + nullptr)); config_manager->SetLatestRolloutId("2017-05-01r0", std::chrono::system_clock::now()); @@ -487,7 +492,8 @@ TEST_F(ConfigManagerServiceNameConfigIdTest, [this, &sequence](const utils::Status& status, const std::vector>& list) { sequence++; - })); + }, + nullptr)); config_manager->SetLatestRolloutId("2017-05-01r0", std::chrono::system_clock::now()); @@ -553,7 +559,8 @@ TEST_F(ConfigManagerServiceNameConfigIdTest, RolloutSingleServiceConfigUpdate) { EXPECT_EQ(100, list[0].second); sequence++; - })); + }, + nullptr)); config_manager->SetLatestRolloutId("2017-05-01r0", std::chrono::system_clock::now()); @@ -600,7 +607,8 @@ TEST_F(ConfigManagerServiceNameConfigIdTest, EXPECT_EQ(100, list[0].second); sequence++; - })); + }, + nullptr)); config_manager->SetLatestRolloutId("2017-05-01r0", std::chrono::system_clock::now()); diff --git a/src/api_manager/proto/server_config.proto b/src/api_manager/proto/server_config.proto index 26c21cb0e..de573f0f6 100644 --- a/src/api_manager/proto/server_config.proto +++ b/src/api_manager/proto/server_config.proto @@ -236,6 +236,10 @@ message ServiceManagementConfig { // at the same time to exceed its quota. // If not specified, or 0, default is 300 (5 minutes). int32 fetch_throttle_window_s = 2; + + // The periodical timer interval in seconds to detect rollout changes. + // If not specified, or 0, default is 60s (1 minute). + int32 detect_rollout_interval_s = 3; } // Maps service configuration files to their corresponding traffic percentage. diff --git a/src/api_manager/service_control/aggregated.cc b/src/api_manager/service_control/aggregated.cc index cb5503a6f..a0defa32d 100644 --- a/src/api_manager/service_control/aggregated.cc +++ b/src/api_manager/service_control/aggregated.cc @@ -305,6 +305,16 @@ Status Aggregated::Close() { return Status::OK; } +void Aggregated::SendEmptyReport() { + ReportRequest request; + ReportResponse* response = new ReportResponse; + Call(request, response, + [this, response](const ::google::protobuf::util::Status&) { + delete response; + }, + nullptr); +} + Status Aggregated::Report(const ReportRequestInfo& info) { if (!client_) { return Status(Code::INTERNAL, "Missing service control client"); diff --git a/src/api_manager/service_control/aggregated.h b/src/api_manager/service_control/aggregated.h index a2e77a1a2..a4c2882cc 100644 --- a/src/api_manager/service_control/aggregated.h +++ b/src/api_manager/service_control/aggregated.h @@ -49,6 +49,8 @@ class Aggregated : public Interface { virtual ~Aggregated(); + void SendEmptyReport() override; + virtual utils::Status Report(const ReportRequestInfo& info); virtual void Check( diff --git a/src/api_manager/service_control/interface.h b/src/api_manager/service_control/interface.h index 279b956d7..3b094a90c 100644 --- a/src/api_manager/service_control/interface.h +++ b/src/api_manager/service_control/interface.h @@ -58,6 +58,9 @@ class Interface { // is enabled. HTTP request errors carry Nginx error code. virtual utils::Status Report(const ReportRequestInfo& info) = 0; + // Send an empty report to get the latest rollout_id. + virtual void SendEmptyReport() = 0; + // Sends ServiceControl Check asynchronously. // on_done() function will be called once it is completed. // utils::Status in the on_done callback: diff --git a/src/nginx/t/BUILD b/src/nginx/t/BUILD index edde3c063..e2f144445 100644 --- a/src/nginx/t/BUILD +++ b/src/nginx/t/BUILD @@ -25,7 +25,7 @@ ################################################################################ # load("@io_bazel_rules_perl//perl:perl.bzl", "perl_library") -load("//:nginx.bzl", "nginx_test", "nginx_suite") +load("//:nginx.bzl", "nginx_suite", "nginx_test") perl_library( name = "perl_library", @@ -211,6 +211,7 @@ nginx_suite( "check_with_token.t", "config_extra_field.t", "config_missing.t", + "config_rollouts_by_timer.t", "config_rollouts_managed.t", "cors.t", "cors_disabled.t", diff --git a/src/nginx/t/config_rollouts_by_timer.t b/src/nginx/t/config_rollouts_by_timer.t new file mode 100644 index 000000000..32b0771bc --- /dev/null +++ b/src/nginx/t/config_rollouts_by_timer.t @@ -0,0 +1,265 @@ +# Copyright (C) Extensible Service Proxy Authors +# All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions +# are met: +# 1. Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# 2. Redistributions in binary form must reproduce the above copyright +# notice, this list of conditions and the following disclaimer in the +# documentation and/or other materials provided with the distribution. +# +# THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND +# ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE +# ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE +# FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL +# DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS +# OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) +# HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT +# LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY +# OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF +# SUCH DAMAGE. +# +################################################################################ +# +use strict; +use warnings; +use JSON::PP; +use Data::Dumper; + +################################################################################ + +use src::nginx::t::ApiManager; # Must be first (sets up import path to + # the Nginx test module) +use src::nginx::t::HttpServer; +use src::nginx::t::ServiceControl; +use Test::Nginx; # Imports Nginx's test module +use Test::More; # And the test framework + +################################################################################ + +# Port assignments +my $NginxPort = ApiManager::pick_port(); +my $BackendPort = ApiManager::pick_port(); +my $ServiceControlPort = ApiManager::pick_port(); +my $ServiceManagementPort = ApiManager::pick_port(); + +my $t = Test::Nginx->new()->has(qw/http proxy/)->plan(7); + +# Save service configuration that disables the report cache. +# Report request will be sent for each client request +$t->write_file_expand('server.pb.txt', <<"EOF"); +service_control_config { + report_aggregator_config { + cache_entries: 0 + flush_interval_ms: 1000 + } + check_aggregator_config { + cache_entries: 0 + flush_interval_ms: 1000 + } +} +service_management_config { + url: "http://127.0.0.1:${ServiceManagementPort}" + fetch_throttle_window_s: 1 + detect_rollout_interval_s: 1 +} +service_config_rollout { + traffic_percentages { + key: "%%TESTDIR%%/service.1.pb.txt" + value: 80 + } + traffic_percentages { + key: "%%TESTDIR%%/service.2.pb.txt" + value: 20 + } + rollout_id: "2017-05-16r0" +} +rollout_strategy: "managed" +EOF + +# Save service name in the service configuration protocol buffer file. +$t->write_file( 'service.1.pb.txt', + ApiManager::get_bookstore_service_config . <<"EOF"); +control { + environment: "http://127.0.0.1:${ServiceControlPort}" +} +EOF + +my $service_config_2 = ApiManager::get_bookstore_service_config; +my $find = 'id: "2016-08-25r1"'; +my $replace = 'id: "2016-08-25r2"'; +$service_config_2 =~ s/$find/$replace/g; + +# Save service name in the service configuration protocol buffer file. +$t->write_file( 'service.2.pb.txt', $service_config_2 . <<"EOF"); +control { + environment: "http://127.0.0.1:${ServiceControlPort}" +} +EOF + +ApiManager::write_file_expand( $t, 'nginx.conf', <<"EOF"); +%%TEST_GLOBALS%% +daemon off; +events { + worker_connections 32; +} +http { + %%TEST_GLOBALS_HTTP%% + server_tokens off; + server { + listen 127.0.0.1:${NginxPort}; + server_name localhost; + location / { + endpoints { + server_config server.pb.txt; + %%TEST_CONFIG%% + on; + } + proxy_pass http://127.0.0.1:${BackendPort}; + } + location /endpoints_status { + endpoints_status; + access_log off; + } + } +} +EOF + +my $report_done = 'report_done'; +my $rollout_done = 'rollout_done'; + +$t->run_daemon( \&servicecontrol, $t, $ServiceControlPort, 'servicecontrol.log', + $report_done); +$t->run_daemon( \&servicemanagement, $t, $ServiceManagementPort, + 'servicemanagement.log', $rollout_done); + +is( $t->waitforsocket("127.0.0.1:${ServiceControlPort}"), 1, + 'Service control socket ready.' ); +is( $t->waitforsocket("127.0.0.1:${ServiceManagementPort}"), 1, + 'Service management socket ready.' ); +$t->run(); + +################################################################################ + +# By not sending any traffic, this test relies on detect_rollout_timer +# to detect rollout changes. Its interval is 1s, it will send an empty report every second +# The report will return the latest rollout id. If it is changed, it will trigger a new +# config fetch. + +# waiting for the first report request is done +is($t->waitforfile($t->{_testdir}."/".${report_done} . ".0"), 1, 'Report1 body file ready.'); + +# waiting for the rolllout update +is($t->waitforfile("$t->{_testdir}/${rollout_done}"), 1, 'Rollout body file ready.'); + +my $max_retry = 5; +my ($response, $response_headers, $response_body, $endpoints_status); +# verify the the final /endpoints_status +do { + # wait for the refresh + sleep(1); + + $response = ApiManager::http_get($NginxPort, '/endpoints_status' ); + ( $response_headers, $response_body ) = split /\r\n\r\n/, $response, 2; + $endpoints_status = decode_json( $response_body ); +} while($max_retry-- > 0 && $endpoints_status->{processes}[0]->{espStatus}[0]-> + {serviceConfigRollouts}->{rolloutId} ne '2016-08-25r1'); + +like( $response_headers, qr/HTTP\/1\.1 200 OK/, 'Returned HTTP 200.' ); +is($endpoints_status->{processes}[0]->{espStatus}[0]->{serviceConfigRollouts}-> + {rolloutId}, '2016-08-25r1', + "Rollout was updated from the service management API" ); +is($endpoints_status->{processes}[0]->{espStatus}[0]->{serviceConfigRollouts}-> + {percentages}->{'2016-08-25r3'}, '100', "Rollout 2016-08-25r3 is 100%" ); + +$t->stop_daemons(); + + +################################################################################ + +sub servicecontrol { + my ( $t, $port, $file, $done) = @_; + my $server = HttpServer->new( $port, $t->testdir() . '/' . $file ) + or die "Can't create test server socket: $!\n"; + local $SIG{PIPE} = 'IGNORE'; + + my $index = 0; + my $report_response = ServiceControl::convert_proto(<<'EOF', 'report_response', 'binary'); +{ + "serviceRolloutId": "2016-08-25r1" +} +EOF + + $server->on_sub('POST', '/v1/services/endpoints-test.cloudendpointsapis.com:report', sub { + my ($headers, $body, $client) = @_; + print $client <<'EOF' . $report_response; +HTTP/1.1 200 OK +Connection: close + +EOF + $t->write_file($done.".".$index, ':report done'); + $index++; + }); + + $server->run(); +} + +sub servicemanagement { + my ( $t, $port, $file, $done) = @_; + my $server = HttpServer->new( $port, $t->testdir() . '/' . $file ) + or die "Can't create test server socket: $!\n"; + local $SIG{PIPE} = 'IGNORE'; + + $server->on_sub('GET', '/v1/services/endpoints-test.cloudendpointsapis.com/rollouts?filter=status=SUCCESS', sub { + my ($headers, $body, $client) = @_; + + print $client <<'EOF' ; +HTTP/1.1 200 OK +Connection: close + +{ + "rollouts": [ + { + "rolloutId": "2016-08-25r1", + "createTime": "2017-05-01T22:40:09.884Z", + "createdBy": "test_user@google.com", + "status": "SUCCESS", + "trafficPercentStrategy": { + "percentages": { + "2016-08-25r3": 100 + } + }, + "serviceName": "endpoints-test.cloudendpointsapis.com" + } + ] +} +EOF + }); + + my $service_config_3 = ApiManager::get_bookstore_service_config; + my $find = 'id: "2016-08-25r1"'; + my $replace = 'id: "2016-08-25r3"'; + $service_config_3 =~ s/$find/$replace/g; + + $server->on_sub('GET', '/v1/services/endpoints-test.cloudendpointsapis.com/configs/2016-08-25r3', sub { + my ($headers, $body, $client) = @_; + print $client <<"EOF" ; +HTTP/1.1 200 OK +Connection: close + +control { + environment: "http://127.0.0.1:${ServiceControlPort}" +} +EOF + print $client $service_config_3; + + $t->write_file($done, ':rollout done'); + }); + + $server->run(); +} + +################################################################################