Merge remote-tracking branch 'origin/master' into retry-filter

* origin/master: (38 commits)
  test: add tests for corner-cases around sending requests before run() starts or after run() ends. (envoyproxy#4114)
  perf: reduce the memory usage of LC Trie construction (envoyproxy#4117)
  test: moving redundant code in websocket_integration_test to utilities (envoyproxy#4127)
  test: make YamlLoadFromStringFail less picky about error msg. (envoyproxy#4141)
  rbac: add rbac network filter. (envoyproxy#4083)
  fuzz: route lookup and header finalization fuzzer. (envoyproxy#4116)
  Set content-type and content-length (envoyproxy#4113)
  fault: use FractionalPercent for percent (envoyproxy#3978)
  test: Fix inverted exact match logic in IntegrationTcpClient::waitForData() (envoyproxy#4134)
  Added cluster_name to load assignment config for static cluster (envoyproxy#4123)
  ssl: refactor ContextConfig to use TlsCertificateConfig (envoyproxy#4115)
  syscall: refactor OsSysCalls for deeper errno latching (envoyproxy#4111)
  thrift_proxy: fix oneway bugs (envoyproxy#4025)
  Do not crash when converting YAML to JSON fails (envoyproxy#4110)
  config: allow unknown fields flag (take 2) (envoyproxy#4096)
  Use a jittered backoff strategy for handling HdsDelegate stream/connection failures (envoyproxy#4108)
  bazel: use GCS remote cache (envoyproxy#4050)
  Add thread local cache of overload action states (envoyproxy#4090)
  Added TCP healthcheck capabilities to the HdsDelegate (envoyproxy#4079)
  secret: add secret provider interface and use it for TlsCertificates (envoyproxy#4086)
  ...

Signed-off-by: Snow Pettersen <snowp@squareup.com>

.circleci/config.yml

@@ -9,6 +9,8 @@ jobs:
        - image: *envoy-build-image
      resource_class: xlarge
      working_directory: /source
+     environment:
+       BAZEL_REMOTE_CACHE: https://storage.googleapis.com/envoy-circleci-bazel-cache/
      steps:
        - run: rm -rf /home/circleci/project/.git # CircleCI git caching is likely broken
        - checkout
@@ -21,6 +23,8 @@ jobs:
        - image: *envoy-build-image
      resource_class: xlarge
      working_directory: /source
+     environment:
+       BAZEL_REMOTE_CACHE: https://storage.googleapis.com/envoy-circleci-bazel-cache/
      steps:
        - run: rm -rf /home/circleci/project/.git # CircleCI git caching is likely broken
        - run: echo $CIRCLE_SHA1
@@ -33,6 +37,8 @@ jobs:
        - image: *envoy-build-image
      resource_class: xlarge
      working_directory: /source
+     environment:
+       BAZEL_REMOTE_CACHE: https://storage.googleapis.com/envoy-circleci-bazel-cache/
      steps:
        - run: rm -rf /home/circleci/project/.git # CircleCI git caching is likely broken
        - checkout
@@ -64,6 +70,8 @@ jobs:
        - run: ci/filter_example_mirror.sh
    ipv6_tests:
      machine: true
+     environment:
+       BAZEL_REMOTE_CACHE: https://storage.googleapis.com/envoy-circleci-bazel-cache/
      steps:
      - run: rm -rf /home/circleci/project/.git # CircleCI git caching is likely broken
      - checkout
@@ -136,6 +144,8 @@ jobs:
    mac:
      macos:
        xcode: "9.3.0"
+     environment:
+       BAZEL_REMOTE_CACHE: https://storage.googleapis.com/envoy-circleci-bazel-cache/
      steps:
        - run: sudo ntpdate -vu time.apple.com
        - run: rm -rf /home/circleci/project/.git # CircleCI git caching is likely broken

DEPRECATED.md

@@ -23,6 +23,9 @@ A logged warning is expected for each deprecated item that is in deprecation win
   is deprecated. Please use the new `upgrade_configs` in the
   [HttpConnectionManager](https://github.com/envoyproxy/envoy/blob/master/api/envoy/config/filter/network/http_connection_manager/v2/http_connection_manager.proto)
   instead.
+* Use of the integer `percent` field in [FaultDelay](https://github.com/envoyproxy/envoy/blob/master/api/envoy/config/filter/fault/v2/fault.proto)
+  and in [FaultAbort](https://github.com/envoyproxy/envoy/blob/master/api/envoy/config/filter/http/fault/v2/fault.proto) is deprecated in favor
+  of the new `FractionalPercent` based `percentage` field.
 * Setting hosts via `hosts` field in `Cluster` is deprecated. Use `load_assignment` instead.
 * Use of `response_headers_to_*` and `request_headers_to_add` are deprecated at the `RouteAction`
   level. Please use the configuration options at the `Route` level.

api/docs/BUILD

@@ -47,6 +47,7 @@ proto_library(
         "//envoy/config/filter/network/http_connection_manager/v2:http_connection_manager",
         "//envoy/config/filter/network/mongo_proxy/v2:mongo_proxy",
         "//envoy/config/filter/network/rate_limit/v2:rate_limit",
+        "//envoy/config/filter/network/rbac/v2:rbac",
         "//envoy/config/filter/network/redis_proxy/v2:redis_proxy",
         "//envoy/config/filter/network/tcp_proxy/v2:tcp_proxy",
         "//envoy/config/grpc_credential/v2alpha:file_based_metadata",

api/envoy/api/v2/cds.proto

@@ -46,7 +46,7 @@ service ClusterDiscoveryService {
 // [#protodoc-title: Clusters]
 
 // Configuration for a single upstream cluster.
-// [#comment:next free field: 35]
+// [#comment:next free field: 36]
 message Cluster {
   // Supplies the name of the cluster which must be unique across all clusters.
   // The cluster name is used when emitting
@@ -226,6 +226,12 @@ message Cluster {
   // connections to happen over plain text.
   core.Http2ProtocolOptions http2_protocol_options = 14;
 
+  // The extension_protocol_options field is used to provide extension-specific protocol options
+  // for upstream connections. The key should match the extension filter name, such as
+  // "envoy.filters.network.thrift_proxy". See the extension's documentation for details on
+  // specific options.
+  map<string, google.protobuf.Struct> extension_protocol_options = 35;
+
   reserved 15;
 
   // If the DNS refresh rate is specified and the cluster type is either

api/envoy/api/v2/endpoint/BUILD

@@ -34,6 +34,7 @@ api_proto_library_internal(
     srcs = ["load_report.proto"],
     visibility = ["//envoy/api/v2:friends"],
     deps = [
+        "//envoy/api/v2/core:address",
         "//envoy/api/v2/core:base",
     ],
 )
@@ -42,6 +43,7 @@ api_go_proto_library(
     name = "load_report",
     proto = ":load_report",
     deps = [
+        "//envoy/api/v2/core:address_go_proto",
         "//envoy/api/v2/core:base_go_proto",
     ],
 )

api/envoy/api/v2/endpoint/load_report.proto

@@ -2,6 +2,7 @@ syntax = "proto3";
 
 package envoy.api.v2.endpoint;
 
+import "envoy/api/v2/core/address.proto";
 import "envoy/api/v2/core/base.proto";
 
 import "google/protobuf/duration.proto";
@@ -19,10 +20,47 @@ message UpstreamLocalityStats {
   // collected from. Zone and region names could be empty if unknown.
   core.Locality locality = 1;
 
-  // The total number of requests sent by this Envoy since the last report. A
+  // The total number of requests sent by this Envoy since the last report. This
+  // information is aggregated over all the upstream Endpoints. total_requests
+  // can be inferred from:
+  //
+  // .. code-block:: none
+  //
+  //   total_requests = total_successful_requests + total_requests_in_progress +
+  //     total_error_requests
+  //
+  // The total number of requests successfully completed by the endpoints in the
+  // locality.
+  uint64 total_successful_requests = 2;
+
+  // The total number of unfinished requests
+  uint64 total_requests_in_progress = 3;
+
+  // The total number of requests that failed due to errors at the endpoint,
+  // aggregated over all endpoints in the locality.
+  uint64 total_error_requests = 4;
+
+  // Stats for multi-dimensional load balancing.
+  repeated EndpointLoadMetricStats load_metric_stats = 5;
+
+  // Endpoint granularity stats information for this locality. This information
+  // is populated if the Server requests it by setting
+  // :ref:`LoadStatsResponse.report_endpoint_granularity<envoy_api_field_load_stats.LoadStatsResponse.report_endpoint_granularity>`.
+  repeated UpstreamEndpointStats upstream_endpoint_stats = 7;
+
+  // [#not-implemented-hide:] The priority of the endpoint group these metrics
+  // were collected from.
+  uint32 priority = 6;
+}
+
+message UpstreamEndpointStats {
+  // Upstream host address.
+  core.Address address = 1;
+
+  // The total number of requests successfully completed by the endpoint. A
   // single HTTP or gRPC request or stream is counted as one request. A TCP
   // connection is also treated as one request. There is no explicit
-  // total_requests field below for a locality, but it may be inferred from:
+  // total_requests field below for an endpoint, but it may be inferred from:
   //
   // .. code-block:: none
   //
@@ -35,7 +73,7 @@ message UpstreamLocalityStats {
   // the grpc-status values are those not covered by total_error_requests below.
   uint64 total_successful_requests = 2;
 
-  // The total number of unfinished requests
+  // The total number of unfinished requests for this endpoint.
   uint64 total_requests_in_progress = 3;
 
   // The total number of requests that failed due to errors at the endpoint.
@@ -52,10 +90,6 @@ message UpstreamLocalityStats {
 
   // Stats for multi-dimensional load balancing.
   repeated EndpointLoadMetricStats load_metric_stats = 5;
-
-  // [#not-implemented-hide:] The priority of the endpoint group these metrics
-  // were collected from.
-  uint32 priority = 6;
 }
 
 // [#not-implemented-hide:] Not configuration. TBD how to doc proto APIs.

api/envoy/config/filter/fault/v2/BUILD

@@ -9,4 +9,5 @@ api_proto_library_internal(
         "//envoy/config/filter/http/fault/v2:__pkg__",
         "//envoy/config/filter/network/mongo_proxy/v2:__pkg__",
     ],
+    deps = ["//envoy/type:percent"],
 )

api/envoy/config/filter/fault/v2/fault.proto

@@ -3,6 +3,8 @@ syntax = "proto3";
 package envoy.config.filter.fault.v2;
 option go_package = "v2";
 
+import "envoy/type/percent.proto";
+
 import "google/protobuf/duration.proto";
 
 import "validate/validate.proto";
@@ -24,7 +26,11 @@ message FaultDelay {
 
   // An integer between 0-100 indicating the percentage of operations/connection requests
   // on which the delay will be injected.
-  uint32 percent = 2 [(validate.rules).uint32.lte = 100];
+  //
+  // .. attention::
+  //
+  //   Use of integer `percent` value is deprecated. Use fractional `percentage` field instead.
+  uint32 percent = 2 [(validate.rules).uint32.lte = 100, deprecated = true];
 
   oneof fault_delay_secifier {
     option (validate.required) = true;
@@ -37,4 +43,7 @@ message FaultDelay {
     google.protobuf.Duration fixed_delay = 3
         [(validate.rules).duration.gt = {}, (gogoproto.stdduration) = true];
   }
+
+  // The percentage of operations/connection requests on which the delay will be injected.
+  envoy.type.FractionalPercent percentage = 4;
 }

api/envoy/config/filter/http/ext_authz/v2alpha/ext_authz.proto

@@ -42,24 +42,31 @@ message ExtAuthz {
 // whether the request is authorized or not.
 //
 // A successful check allows the authorization service adding or overriding headers from the
-// original request before dispatching it to the upstream. This is done by including the headers in
-// the response sent back from the authorization service to the filter. Note that `Status`,
-// `Method`, `Path` and `Content Length` response headers are automatically removed from this
-// response by the filter. If other headers need be deleted, they should be specified in
-// `response_headers_to_remove` field.
+// original request before dispatching it to the upstream. This is done by configuring which headers
+// in the authorization response should be sent to the upstream. See *allowed_authorization_headers*
+// bellow.
 //
 // A failed check will cause this filter to close the HTTP request normally with 403 (Forbidden),
-// unless a different status code has been indicated by the authorization service via response
-// headers. The HTTP service also allows the authorization filter to also pass data from the
-// response body to the downstream client in case of a denied request.
+// unless a different status code has been indicated by the authorization server via response
+// headers. If other headers in the authorization response need to be sent to client, this can also
+// be done by specifying them in *allowed_authorization_headers*.
 message HttpService {
   // Sets the HTTP server URI which the authorization requests must be sent to.
   envoy.api.v2.core.HttpUri server_uri = 1;
 
-  // Sets an optional prefix to the value of authorization request header `path`.
+  // Sets an optional prefix to the value of authorization request header *Path*.
   string path_prefix = 2;
 
-  // Sets a list of headers that should be not be sent *from the authorization server* to the
-  // upstream.
-  repeated string response_headers_to_remove = 3;
+  reserved 3;
+
+  // Sets a list of headers that can be sent from the authorization server to the upstream service,
+  // or to the downstream client when present in the authorization response. Note that a matched
+  // request header will have its value overridden by the ones sent from the authorization server.
+  repeated string allowed_authorization_headers = 4;
+
+  // Sets a list of headers that should be sent *from the filter* to the authorization server
+  // when they are also present in the client request. Note that *Content-Length*, *Authority*,
+  // *Method* and *Path* are always dispatched to the authorization server by default. The message
+  // will not contain body data and the *Content-Length* will be set to zero.
+  repeated string allowed_request_headers = 5;
 }

api/envoy/config/filter/http/fault/v2/BUILD

@@ -8,5 +8,6 @@ api_proto_library_internal(
     deps = [
         "//envoy/api/v2/route",
         "//envoy/config/filter/fault/v2:fault",
+        "//envoy/type:percent",
     ],
 )

api/envoy/config/filter/http/fault/v2/fault.proto

@@ -3,8 +3,9 @@ syntax = "proto3";
 package envoy.config.filter.http.fault.v2;
 option go_package = "v2";
 
-import "envoy/config/filter/fault/v2/fault.proto";
 import "envoy/api/v2/route/route.proto";
+import "envoy/config/filter/fault/v2/fault.proto";
+import "envoy/type/percent.proto";
 
 import "validate/validate.proto";
 
@@ -14,14 +15,22 @@ import "validate/validate.proto";
 message FaultAbort {
   // An integer between 0-100 indicating the percentage of requests/operations/connections
   // that will be aborted with the error code provided.
-  uint32 percent = 1 [(validate.rules).uint32.lte = 100];
+  //
+  // .. attention::
+  //
+  //   Use of integer `percent` value is deprecated. Use fractional `percentage` field instead.
+  uint32 percent = 1 [(validate.rules).uint32.lte = 100, deprecated = true];
 
   oneof error_type {
     option (validate.required) = true;
 
     // HTTP status code to use to abort the HTTP request.
     uint32 http_status = 2 [(validate.rules).uint32 = {gte: 200, lt: 600}];
   }
+
+  // The percentage of requests/operations/connections that will be aborted with the error code
+  // provided.
+  envoy.type.FractionalPercent percentage = 3;
 }
 
 message HTTPFault {

api/envoy/config/filter/http/rbac/v2/rbac.proto

@@ -19,7 +19,7 @@ message RBAC {
 
   // Shadow rules are not enforced by the filter (i.e., returning a 403)
   // but will emit stats and logs and can be used for rule testing.
-  // If absent, no shadow RBAC policy with be applied.
+  // If absent, no shadow RBAC policy will be applied.
   config.rbac.v2alpha.RBAC shadow_rules = 2;
 }
 

api/envoy/config/filter/network/rbac/v2/BUILD

@@ -0,0 +1,9 @@
+load("//bazel:api_build_system.bzl", "api_proto_library_internal")
+
+licenses(["notice"])  # Apache 2
+
+api_proto_library_internal(
+    name = "rbac",
+    srcs = ["rbac.proto"],
+    deps = ["//envoy/config/rbac/v2alpha:rbac"],
+)

api/envoy/config/filter/network/rbac/v2/rbac.proto

@@ -0,0 +1,30 @@
+syntax = "proto3";
+
+package envoy.config.filter.network.rbac.v2;
+option go_package = "v2";
+
+import "envoy/config/rbac/v2alpha/rbac.proto";
+
+import "validate/validate.proto";
+import "gogoproto/gogo.proto";
+
+// [#protodoc-title: RBAC]
+// Role-Based Access Control :ref:`configuration overview <config_network_filters_rbac>`.
+
+// RBAC network filter config.
+//
+// Header and Metadata should not be used in rules/shadow_rules in RBAC network filter as
+// this information is only available in :ref:`RBAC http filter <config_http_filters_rbac>`.
+message RBAC {
+  // Specify the RBAC rules to be applied globally.
+  // If absent, no enforcing RBAC policy will be applied.
+  config.rbac.v2alpha.RBAC rules = 1;
+
+  // Shadow rules are not enforced by the filter but will emit stats and logs
+  // and can be used for rule testing.
+  // If absent, no shadow RBAC policy will be applied.
+  config.rbac.v2alpha.RBAC shadow_rules = 2;
+
+  // The prefix to use when emitting statistics.
+  string stat_prefix = 3 [(validate.rules).string.min_bytes = 1];
+}

api/envoy/config/rbac/v2alpha/rbac.proto

@@ -103,7 +103,8 @@ message Permission {
     // When any is set, it matches any action.
     bool any = 3 [(validate.rules).bool.const = true];
 
-    // A header (or psuedo-header such as :path or :method) on the incoming HTTP request.
+    // A header (or psuedo-header such as :path or :method) on the incoming HTTP request. Only
+    // available for HTTP request.
     envoy.api.v2.route.HeaderMatcher header = 4;
 
     // A CIDR block that describes the destination IP.
@@ -112,7 +113,8 @@ message Permission {
     // A port number that describes the destination port connecting to.
     uint32 destination_port = 6 [(validate.rules).uint32.lte = 65535];
 
-    // Metadata that describes additional information about the action.
+    // Metadata that describes additional information about the action. Only available for HTTP
+    // request.
     envoy.type.matcher.MetadataMatcher metadata = 7;
 
     // Negates matching the provided permission. For instance, if the value of `not_rule` would
@@ -156,10 +158,12 @@ message Principal {
     // A CIDR block that describes the downstream IP.
     envoy.api.v2.core.CidrRange source_ip = 5;
 
-    // A header (or psuedo-header such as :path or :method) on the incoming HTTP request.
+    // A header (or psuedo-header such as :path or :method) on the incoming HTTP request. Only
+    // available for HTTP request.
     envoy.api.v2.route.HeaderMatcher header = 6;
 
-    // Metadata that describes additional information about the principal.
+    // Metadata that describes additional information about the principal. Only available for HTTP
+    // request.
     envoy.type.matcher.MetadataMatcher metadata = 7;
 
     // Negates matching the provided principal. For instance, if the value of `not_id` would match,