From 3781be0e624470a4ee613c4ffc9816976f5dd701 Mon Sep 17 00:00:00 2001 From: Kuat Date: Fri, 11 Oct 2019 14:10:54 -0700 Subject: [PATCH] mixer: add hash policy for maglev (#2456) * add hash policy for mixer Signed-off-by: Kuat Yessenov * fix test Signed-off-by: Kuat Yessenov * name the workspace Signed-off-by: Kuat Yessenov * update upstream Signed-off-by: Kuat Yessenov * fix tests Signed-off-by: Kuat Yessenov --- WORKSPACE | 7 ++-- include/istio/control/http/BUILD | 2 +- src/envoy/http/authn/BUILD | 6 ++-- .../http_filter_integration_test.cc | 35 ++++++++++++++++++- src/envoy/http/mixer/filter.cc | 4 +-- src/envoy/http/mixer/report_data.h | 8 +++-- src/envoy/utils/BUILD | 2 +- src/envoy/utils/grpc_transport.cc | 16 ++++++--- src/istio/control/http/BUILD | 2 +- test/integration/int_client.cc | 6 ++-- test/integration/int_server.cc | 6 ++-- 11 files changed, 71 insertions(+), 23 deletions(-) diff --git a/WORKSPACE b/WORKSPACE index 95e0c7419e8d..213a1f5845ed 100644 --- a/WORKSPACE +++ b/WORKSPACE @@ -14,6 +14,7 @@ # ################################################################################ # +workspace(name = "io_istio_proxy") # http_archive is not a native function since bazel 0.19 load("@bazel_tools//tools/build_defs/repo:http.bzl", "http_archive") @@ -35,10 +36,10 @@ bind( # 1. Determine SHA256 `wget https://github.com/envoyproxy/envoy-wasm/archive/$COMMIT.tar.gz && sha256sum $COMMIT.tar.gz` # 2. Update .bazelrc and .bazelversion files. # -# envoy-wasm commit date: 10/09/2019 -ENVOY_SHA = "ec03328688895d99c9f5ae4fd7f8459ef3e95212" +# envoy-wasm commit date: 10/11/2019 +ENVOY_SHA = "29b71643c999af4e31ba86d41740edbef71c73fe" -ENVOY_SHA256 = "4abf05fd040b56af630b624068d462035bd986bccf775f821b755fd3c0df8083" +ENVOY_SHA256 = "2a29684ebc13736eb012e0aaaedc08860ddfdb07dec55f37c7e9a100a4bcd7f3" LOCAL_ENVOY_PROJECT = "/PATH/TO/ENVOY" diff --git a/include/istio/control/http/BUILD b/include/istio/control/http/BUILD index 532a48e82fce..f0fd444b7624 100644 --- a/include/istio/control/http/BUILD +++ b/include/istio/control/http/BUILD @@ -23,5 +23,5 @@ cc_library( "request_handler.h", ], visibility = ["//visibility:public"], - deps = ["//src/istio/authn:context_proto_cc"], + deps = ["//src/istio/authn:context_proto_cc_proto"], ) diff --git a/src/envoy/http/authn/BUILD b/src/envoy/http/authn/BUILD index f32672751567..65cd947b66f8 100644 --- a/src/envoy/http/authn/BUILD +++ b/src/envoy/http/authn/BUILD @@ -46,7 +46,7 @@ envoy_cc_library( "//src/envoy/http/jwt_auth:jwt_lib", "//src/envoy/utils:filter_names_lib", "//src/envoy/utils:utils_lib", - "//src/istio/authn:context_proto_cc", + "//src/istio/authn:context_proto_cc_proto", ], ) @@ -66,7 +66,7 @@ envoy_cc_library( "//src/envoy/utils:authn_lib", "//src/envoy/utils:filter_names_lib", "//src/envoy/utils:utils_lib", - "//src/istio/authn:context_proto_cc", + "//src/istio/authn:context_proto_cc_proto", "@envoy//source/exe:envoy_common_lib", ], ) @@ -76,7 +76,7 @@ envoy_cc_test_library( hdrs = ["test_utils.h"], repository = "@envoy", deps = [ - "//src/istio/authn:context_proto_cc", + "//src/istio/authn:context_proto_cc_proto", ], ) diff --git a/src/envoy/http/jwt_auth/integration_test/http_filter_integration_test.cc b/src/envoy/http/jwt_auth/integration_test/http_filter_integration_test.cc index 22429dff6f80..cfd1947f2681 100644 --- a/src/envoy/http/jwt_auth/integration_test/http_filter_integration_test.cc +++ b/src/envoy/http/jwt_auth/integration_test/http_filter_integration_test.cc @@ -13,6 +13,9 @@ * limitations under the License. */ +#include +#include + #include "test/integration/http_integration.h" #include "test/integration/utility.h" @@ -51,7 +54,37 @@ class JwtVerificationFilterIntegrationTest 0, FakeHttpConnection::Type::HTTP1, version_, timeSystem())); registerPort("upstream_1", fake_upstreams_.back()->localAddress()->ip()->port()); - createTestServer(ConfigPath(), {"http"}); + + // upstream envoy hardcodes workspace name, so this code is duplicated + const std::string path = ConfigPath(); + const std::string json_path = + TestEnvironment::runfilesPath(path, "io_istio_proxy"); + std::string out_json_string = + TestEnvironment::readFileToStringForTest(json_path); + + // Substitute ports. + for (const auto& it : port_map_) { + const std::regex port_regex("\\{\\{ " + it.first + " \\}\\}"); + out_json_string = std::regex_replace(out_json_string, port_regex, + std::to_string(it.second)); + } + + // Substitute paths and other common things. + out_json_string = TestEnvironment::substitute(out_json_string, version_); + + const std::string extension = + absl::EndsWith(path, ".yaml") ? ".yaml" : ".json"; + const std::string out_json_path = + TestEnvironment::temporaryPath(path + ".with.ports" + extension); + TestEnvironment::createParentPath(out_json_path); + { + std::ofstream out_json_file(out_json_path); + out_json_file << out_json_string; + } + + test_server_ = + createIntegrationTestServer(out_json_path, nullptr, timeSystem()); + registerTestServerPorts({"http"}); } /** diff --git a/src/envoy/http/mixer/filter.cc b/src/envoy/http/mixer/filter.cc index 778bbe547f1a..ce32f9a52f76 100644 --- a/src/envoy/http/mixer/filter.cc +++ b/src/envoy/http/mixer/filter.cc @@ -58,7 +58,7 @@ void Filter::ReadPerRouteConfig( FilterHeadersStatus Filter::decodeHeaders(HeaderMap& headers, bool) { ENVOY_LOG(debug, "Called Mixer::Filter : {}", __func__); - request_total_size_ += headers.byteSize(); + request_total_size_ += headers.refreshByteSize(); ::istio::control::http::Controller::PerRouteConfig config; auto route = decoder_callbacks_->route(); @@ -103,7 +103,7 @@ FilterDataStatus Filter::decodeData(Buffer::Instance& data, bool end_stream) { FilterTrailersStatus Filter::decodeTrailers(HeaderMap& trailers) { ENVOY_LOG(debug, "Called Mixer::Filter : {}", __func__); - request_total_size_ += trailers.byteSize(); + request_total_size_ += trailers.refreshByteSize(); if (state_ == Calling) { return FilterTrailersStatus::StopIteration; } diff --git a/src/envoy/http/mixer/report_data.h b/src/envoy/http/mixer/report_data.h index 84b18f2468e6..2d12fee6b2b6 100644 --- a/src/envoy/http/mixer/report_data.h +++ b/src/envoy/http/mixer/report_data.h @@ -72,10 +72,14 @@ class ReportData : public ::istio::control::http::ReportData, response_total_size_(info.bytesSent()), request_total_size_(request_total_size) { if (response_headers != nullptr) { - response_total_size_ += response_headers->byteSize(); + response_total_size_ += + (response_headers->byteSize() ? response_headers->byteSize().value() + : response_headers->byteSizeInternal()); } if (response_trailers != nullptr) { - response_total_size_ += response_trailers->byteSize(); + response_total_size_ += (response_trailers->byteSize() + ? response_trailers->byteSize().value() + : response_trailers->byteSizeInternal()); } } diff --git a/src/envoy/utils/BUILD b/src/envoy/utils/BUILD index 84a2f1b69600..81eb30104460 100644 --- a/src/envoy/utils/BUILD +++ b/src/envoy/utils/BUILD @@ -35,7 +35,7 @@ envoy_cc_library( ":filter_names_lib", ":utils_lib", "//include/istio/utils:attribute_names_header", - "//src/istio/authn:context_proto_cc", + "//src/istio/authn:context_proto_cc_proto", "//src/istio/utils:attribute_names_lib", "//src/istio/utils:utils_lib", "@envoy//source/exe:envoy_common_lib", diff --git a/src/envoy/utils/grpc_transport.cc b/src/envoy/utils/grpc_transport.cc index f80917bf3c93..243530e2d80c 100644 --- a/src/envoy/utils/grpc_transport.cc +++ b/src/envoy/utils/grpc_transport.cc @@ -38,12 +38,20 @@ GrpcTransport::GrpcTransport( : async_client_(std::move(async_client)), response_(response), serialized_forward_attributes_(serialized_forward_attributes), - on_done_(on_done), - request_(async_client_->send( - descriptor(), request, *this, parent_span, - absl::optional(kGrpcRequestTimeoutMs))) { + on_done_(on_done) { ENVOY_LOG(debug, "Sending {} request: {}", descriptor().name(), request.DebugString()); + Envoy::Http::AsyncClient::RequestOptions options; + options.setTimeout(kGrpcRequestTimeoutMs); + Protobuf::RepeatedPtrField + hash_policy; + hash_policy.Add()->mutable_header()->set_header_name( + kIstioAttributeHeader.get()); + hash_policy.Add()->mutable_header()->set_header_name( + Envoy::Http::Headers::get().Host.get()); + options.setHashPolicy(hash_policy); + request_ = + async_client_->send(descriptor(), request, *this, parent_span, options); } template diff --git a/src/istio/control/http/BUILD b/src/istio/control/http/BUILD index ae91194784ff..09e1a1e00f96 100644 --- a/src/istio/control/http/BUILD +++ b/src/istio/control/http/BUILD @@ -32,7 +32,7 @@ cc_library( deps = [ "//include/istio/control/http:headers_lib", "//include/istio/utils:attribute_names_header", - "//src/istio/authn:context_proto_cc", + "//src/istio/authn:context_proto_cc_proto", "//src/istio/control:common_lib", "//src/istio/utils:attribute_names_lib", "//src/istio/utils:utils_lib", diff --git a/test/integration/int_client.cc b/test/integration/int_client.cc index bf94f69ec8d1..f7f5d65b0c8d 100644 --- a/test/integration/int_client.cc +++ b/test/integration/int_client.cc @@ -254,7 +254,8 @@ class Http1ClientConnection : public ClientConnection { dispatcher), stats_(), network_connection_(std::move(network_connection)), - http_connection_(*network_connection_, stats_, *this), + http_connection_(*network_connection_, stats_, *this, + Envoy::Http::DEFAULT_MAX_HEADERS_COUNT), read_filter_{std::make_shared(client.name(), id, http_connection_)} { network_connection_->addReadFilter(read_filter_); @@ -297,7 +298,8 @@ class Http2ClientConnection : public ClientConnection { settings_(), network_connection_(std::move(network_connection)), http_connection_(*network_connection_, *this, stats_, settings_, - max_request_headers_kb), + max_request_headers_kb, + Envoy::Http::DEFAULT_MAX_HEADERS_COUNT), read_filter_{std::make_shared(client.name(), id, http_connection_)} { network_connection_->addReadFilter(read_filter_); diff --git a/test/integration/int_server.cc b/test/integration/int_server.cc index 22a24314a2e8..29110787705f 100644 --- a/test/integration/int_server.cc +++ b/test/integration/int_server.cc @@ -311,7 +311,7 @@ ServerConnection::ServerConnection( http_connection_ = std::make_unique( network_connection, scope, *this, Envoy::Http::Http1Settings(), - max_request_headers_kb); + max_request_headers_kb, Envoy::Http::DEFAULT_MAX_HEADERS_COUNT); break; case Envoy::Http::CodecClient::Type::HTTP2: { Envoy::Http::Http2Settings settings; @@ -320,7 +320,7 @@ ServerConnection::ServerConnection( http_connection_ = std::make_unique( network_connection, *this, scope, settings, - max_request_headers_kb); + max_request_headers_kb, Envoy::Http::DEFAULT_MAX_HEADERS_COUNT); } break; default: ENVOY_LOG(error, @@ -330,7 +330,7 @@ ServerConnection::ServerConnection( http_connection_ = std::make_unique( network_connection, scope, *this, Envoy::Http::Http1Settings(), - max_request_headers_kb); + max_request_headers_kb, Envoy::Http::DEFAULT_MAX_HEADERS_COUNT); break; } }