metadata: add fetch failure status timeout

src/api_manager/auth/service_account_token.h

@@ -54,6 +54,16 @@ class ServiceAccountToken {
     access_token_.set_token(token, expiration);
   }
 
+  //  Set the last failed fetch time
+  void set_last_failed_fetch_time(std::chrono::system_clock::time_point time) {
+    last_failed_fetch_time_ = time;
+  }
+
+  // Get the last failed fetch time
+  std::chrono::system_clock::time_point last_failed_fetch_time() const {
+    return last_failed_fetch_time_;
+  }
+
   // The access token from metadata server response in JSON format:
   // {
   //   "access_token":" ... ",
@@ -154,6 +164,9 @@ class ServiceAccountToken {
 
   // Fetching state
   FetchState state_;
+
+  // The time of last failed fetch
+  std::chrono::system_clock::time_point last_failed_fetch_time_;
 };
 
 }  // namespace auth

src/api_manager/fetch_metadata.cc

@@ -21,6 +21,7 @@
 
 using ::google::api_manager::utils::Status;
 using ::google::protobuf::util::error::Code;
+using std::chrono::system_clock;
 
 namespace google {
 namespace api_manager {
@@ -39,7 +40,8 @@ const char kMetadataInstanceIdentityToken[] =
 // The maximum lifetime of a cache token. Unit: seconds.
 // Token expired in 1 hour, reduce 100 seconds for grace buffer.
 const int kInstanceIdentityTokenExpiration = 3500;
-
+// Time window (in seconds) of failure status after a failed fetch.
+const int kFailureStatusWindow = 5;
 // Initial metadata fetch timeout (1s)
 const int kMetadataFetchTimeout = 1000;
 // Maximum number of retries to fetch token from metadata
@@ -117,9 +119,14 @@ void GlobalFetchServiceAccountToken(
       }
       break;
     case auth::ServiceAccountToken::FAILED:
-      // permanent failure
-      continuation(Status(Code::INTERNAL, kFailedTokenFetch));
-      return;
+      // If the current time doesn't get out of the time window of failure
+      // status, it will return kFailedTokenFetch directly.
+      if (system_clock::now() - token->last_failed_fetch_time() <
+          std::chrono::seconds(kFailureStatusWindow)) {
+        continuation(Status(Code::INTERNAL, kFailedTokenFetch));
+        return;
+      }
+      break;
     case auth::ServiceAccountToken::NONE:
     default:
       env->LogDebug("Need to fetch service account token");
@@ -133,6 +140,7 @@ void GlobalFetchServiceAccountToken(
                   // fetch failed
                   if (!status.ok()) {
                     env->LogDebug("Failed to fetch service account token");
+                    token->set_last_failed_fetch_time(system_clock::now());
                     token->set_state(auth::ServiceAccountToken::FAILED);
                     continuation(Status(Code::INTERNAL, kFailedTokenFetch));
                     return;

src/nginx/t/BUILD

@@ -220,6 +220,7 @@ nginx_suite(
         "init_service_configs_single.t",
         "metadata.t",
         "metadata_fail.t",
+        "metadata_fetch_fail.t",
         "metadata_timeout.t",
         "metadata_token.t",
         "multiple_apis.t",

src/nginx/t/metadata_fetch_fail.t

@@ -0,0 +1,155 @@
+# Copyright (C) Extensible Service Proxy Authors
+# All rights reserved.
+#
+# Redistribution and use in source and binary forms, with or without
+# modification, are permitted provided that the following conditions
+# are met:
+# 1. Redistributions of source code must retain the above copyright
+#    notice, this list of conditions and the following disclaimer.
+# 2. Redistributions in binary form must reproduce the above copyright
+#    notice, this list of conditions and the following disclaimer in the
+#    documentation and/or other materials provided with the distribution.
+#
+# THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
+# ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+# ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
+# FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+# DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
+# OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+# HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+# LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+# OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+# SUCH DAMAGE.
+#
+################################################################################
+#
+use strict;
+use warnings;
+
+################################################################################
+
+use src::nginx::t::ApiManager; # Must be first (sets up import path to the Nginx test module)
+use src::nginx::t::HttpServer;
+use src::nginx::t::ServiceControl;
+use Test::Nginx; # Imports Nginx's test module
+use Test::More;  # And the test framework
+
+################################################################################
+
+# Port assignments
+my $NginxPort = ApiManager::pick_port();
+my $BackendPort = ApiManager::pick_port();
+my $ServiceControlPort = ApiManager::pick_port();
+my $MetadataPort = ApiManager::pick_port();
+
+my $t = Test::Nginx->new()->has(qw/http proxy/)->plan(8);
+
+my $config = ApiManager::get_bookstore_service_config . <<"EOF";
+control {
+  environment: "http://127.0.0.1:${ServiceControlPort}"
+}
+EOF
+
+$t->write_file('service.pb.txt', $config);
+ApiManager::write_file_expand($t, 'nginx.conf', <<"EOF");
+%%TEST_GLOBALS%%
+daemon off;
+events { worker_connections 32; }
+http {
+  %%TEST_GLOBALS_HTTP%%
+  server_tokens off;
+  endpoints {
+    metadata_server http://127.0.0.1:${MetadataPort};
+  }
+  server {
+    listen 127.0.0.1:${NginxPort};
+    server_name localhost;
+    location / {
+      endpoints {
+        api service.pb.txt;
+        %%TEST_CONFIG%%
+        on;
+      }
+      proxy_pass http://127.0.0.1:${BackendPort};
+    }
+  }
+}
+EOF
+
+
+sub test_metadata {
+    my ($sleepLength, $wantReqHeader, $wantReqBody) = @_;
+    my $backend_log = 'backend.log';
+
+    $t->run_daemon(\&backends, $t, $BackendPort, $backend_log);
+    $t->run_daemon(\&metadata, $t, $MetadataPort, 'metadata.log');
+    is($t->waitforsocket("127.0.0.1:${MetadataPort}"), 1, 'Metadata socket ready.');
+
+    $t->run();
+
+    ################################################################################
+
+    my $shelves1 = ApiManager::http_get($NginxPort, '/shelves');
+
+    my ($shelves_headers1, $shelves_body1) = split /\r\n\r\n/, $shelves1, 2;
+    like($shelves_headers1, qr/HTTP\/1\.1 500 Internal Server Error/, '/shelves returned HTTP 500.');
+    like($shelves_body1, qr/Failed to fetch service account token/, 'Returned Failure Status');
+
+    ################################################################################
+    # if no sleep, the service account token still doesn't get out of last failed
+    # fetch so it will fail.
+    sleep $sleepLength;
+
+    my $shelves2 = ApiManager::http_get($NginxPort, '/shelves');
+
+    $t->stop();
+    $t->stop_daemons();
+
+    my ($shelves_headers2, $shelves_body2) = split /\r\n\r\n/, $shelves2, 2;
+    like($shelves_headers2, $wantReqHeader, 'Got expected request header');
+    like($shelves_body2, $wantReqBody, 'Got expected request body');
+}
+# Fail the first request by failed fetch and do the second request after 2s, which
+# also get failed since the failed fetch status doesn't expire.
+test_metadata(2, qr/HTTP\/1\.1 500 Internal Server Error/, qr/Failed to fetch service account token/);
+# Fail the first request by failed fetch and do the second request after sleeping
+# 6s , which will get the token.
+test_metadata(6, qr/HTTP\/1.1 401 Unauthorized/, qr/Method doesn't allow unregistered callers/);
+
+################################################################################
+
+sub metadata {
+    my ($t, $port, $file) = @_;
+    my $server = HttpServer->new($port, $t->testdir() . '/' . $file)
+        or die "Can't create test server socket: $!\n";
+    my $request_count = 0;
+
+    local $SIG{PIPE} = 'IGNORE';
+    $server->on_sub('GET', '/computeMetadata/v1/instance/service-accounts/default/token', sub {
+        my ($headers, $body, $client) = @_;
+        $request_count++;
+
+        # The retry time is 5 and it would be 6 times for the first failed fetch.
+        if ($request_count < 7) {
+            return;
+        }
+
+        # Only the 7th request will get the token.
+        print $client <<'EOF';
+HTTP/1.1 200 OK
+Metadata-Flavor: Google
+Content-Type: application/json
+
+{
+ "access_token":"ya29.7gFRTEGmovWacYDnQIpC9X9Qp8cH0sgQyWVrZaB1Eg1WoAhQMSG4L2rtaHk1",
+ "expires_in":200,
+ "token_type":"Bearer"
+}
+EOF
+    });
+
+    $server->run();
+}
+
+################################################################################