Merge branch 'release/v1.26-backportedfork' into 1.26.8-forward

api/envoy/config/tap/v3/common.proto

@@ -4,6 +4,7 @@ package envoy.config.tap.v3;
 
 import "envoy/config/common/matcher/v3/matcher.proto";
 import "envoy/config/core/v3/base.proto";
+import "envoy/config/core/v3/extension.proto";
 import "envoy/config/core/v3/grpc_service.proto";
 import "envoy/config/route/v3/route_components.proto";
 
@@ -183,7 +184,7 @@ message OutputConfig {
 }
 
 // Tap output sink configuration.
-// [#next-free-field: 6]
+// [#next-free-field: 7]
 message OutputSink {
   option (udpa.annotations.versioning).previous_message_type =
       "envoy.service.tap.v2alpha.OutputSink";
@@ -259,6 +260,9 @@ message OutputSink {
     //   been configured to receive tap configuration from some other source (e.g., static
     //   file, XDS, etc.) configuring the buffered admin output type will fail.
     BufferedAdminSink buffered_admin = 5;
+
+    // Tap output filter will be defined by an extension type
+    core.v3.TypedExtensionConfig custom_sink = 6;
   }
 }
 

api/envoy/extensions/filters/http/ext_proc/v3/ext_proc.proto

@@ -29,8 +29,6 @@ option (xds.annotations.v3.file_status).work_in_progress = true;
 // **Current Implementation Status:**
 // All options and processing modes are implemented except for the following:
 //
-// * Request and response attributes are not sent and not processed.
-// * Dynamic metadata in responses from the external processor is ignored.
 // * "async mode" is not implemented.
 
 // The filter communicates with an external gRPC service called an "external processor"
@@ -95,7 +93,7 @@ option (xds.annotations.v3.file_status).work_in_progress = true;
 // messages, and the server must reply with
 // :ref:`ProcessingResponse <envoy_v3_api_msg_service.ext_proc.v3.ProcessingResponse>`.
 
-// [#next-free-field: 11]
+// [#next-free-field: 17]
 message ExternalProcessor {
   // Configuration for the gRPC service that the filter will communicate with.
   // The filter supports both the "Envoy" and "Google" gRPC clients.
@@ -168,6 +166,37 @@ message ExternalProcessor {
   // :ref:`override_message_timeout <envoy_v3_api_field_service.ext_proc.v3.ProcessingResponse.override_message_timeout>`
   // If not specified, by default it is 0, which will effectively disable the ``override_message_timeout`` API.
   google.protobuf.Duration max_message_timeout = 10;
+
+  reserved 11, 12, 13, 14, 15;
+
+  // Options related to the sending and receiving of dynamic metadata
+  MetadataOptions metadata_options = 16;
+}
+
+// The MetadataOptions structure defines options for the sending and receiving of
+// dynamic metadata. Specifically, which namespaces to send to the server, whether
+// metadata returned by the server may be written, and how that metadata may be written.
+message MetadataOptions {
+  message MetadataNamespaces {
+    // Specifies a list of metadata namespaces whose values, if present,
+    // will be passed to the ext_proc service as an opaque *protobuf::Struct*.
+    repeated string untyped = 1;
+
+    // Specifies a list of metadata namespaces whose values, if present,
+    // will be passed to the ext_proc service as a *protobuf::Any*. This allows
+    // envoy and the external processing server to share the protobuf message
+    // definition for safe parsing.
+    repeated string typed = 2;
+  }
+
+  // Describes which typed or untyped dynamic metadata namespaces to forward to
+  // the external processing server.
+  MetadataNamespaces forwarding_namespaces = 1;
+
+  // Describes which typed or untyped dynamic metadata namespaces to accept from
+  // the external processing server. Set to empty or leave unset to disallow writing
+  // any received dynamic metadata. Receiving of typed metadata is not supported.
+  MetadataNamespaces receiving_namespaces = 2;
 }
 
 // Extra settings that may be added to per-route configuration for a
@@ -188,7 +217,7 @@ message ExtProcPerRoute {
 }
 
 // Overrides that may be set on a per-route basis
-// [#next-free-field: 6]
+// [#next-free-field: 7]
 message ExtProcOverrides {
   // Set a different processing mode for this route than the default.
   ProcessingMode processing_mode = 1;
@@ -209,4 +238,7 @@ message ExtProcOverrides {
 
   // Set a different gRPC service for this route than the default.
   config.core.v3.GrpcService grpc_service = 5;
+
+  // Options related to the sending and receiving of dynamic metadata
+  MetadataOptions metadata_options = 6;
 }

api/envoy/service/ext_proc/v3/external_processor.proto

@@ -59,7 +59,7 @@ service ExternalProcessor {
 
 // This represents the different types of messages that Envoy can send
 // to an external processing server.
-// [#next-free-field: 8]
+// [#next-free-field: 9]
 message ProcessingRequest {
   // Specify whether the filter that sent this request is running in synchronous
   // or asynchronous mode. The choice of synchronous or asynchronous mode
@@ -118,6 +118,9 @@ message ProcessingRequest {
     // in the filter configuration.
     HttpTrailers response_trailers = 7;
   }
+
+  // Dynamic metadata associated with the request.
+  config.core.v3.Metadata metadata_context = 8;
 }
 
 // For every ProcessingRequest received by the server with the ``async_mode`` field
@@ -161,9 +164,9 @@ message ProcessingResponse {
     ImmediateResponse immediate_response = 7;
   }
 
-  // [#not-implemented-hide:]
-  // Optional metadata that will be emitted as dynamic metadata to be consumed by the next
-  // filter. This metadata will be placed in the namespace ``envoy.filters.http.ext_proc``.
+  // Optional metadata that will be emitted as dynamic metadata to be consumed by
+  // following filters. This metadata will be placed in the namespace(s) specified by the top-level
+  // field name(s) of the struct.
   google.protobuf.Struct dynamic_metadata = 8;
 
   // Override how parts of the HTTP request and response are processed

changelogs/current.yaml

@@ -11,3 +11,41 @@ new_features:
     Added an off-by-default runtime flag
     ``envoy.reloadable_features.google_grpc_disable_tls_13`` to disable TLSv1.3
     usage by gRPC SDK for ``google_grpc`` services.
+
+    fixed a bug where handshake may fail when both private key provider and cert validation are set.
+- area: buffer
+  change: |
+    Fixed a bug (https://github.com/envoyproxy/envoy/issues/28760) that the internal listener causes an undefined
+    behavior due to the unintended release of the buffer memory.
+- area: http
+  change: |
+    Fixed recursion when HTTP connection is disconnected due to a high number of premature resets.
+- area: proxy protocol
+  change: |
+    fixed a crash when Envoy is configured for PROXY protocol on both a listener and cluster, and the listener receives
+    a PROXY protocol header with address type LOCAL (typically used for health checks).
+- area: proxy_protocol
+  change: |
+    Fix crash due to uncaught exception when the operating system does not support an address type (such as IPv6) that is
+    received in a proxy protocol header. Connections will instead be dropped/reset.
+- area: proxy_protocol
+  change: |
+    Fixed a bug where TLVs with non utf8 characters were inserted as protobuf values into filter metadata circumventing
+    ext_authz checks when ``failure_mode_allow`` is set to ``true``.
+- area: http
+  change: |
+    Fixed crash when HTTP request idle and per try timeouts occurs within backoff interval.
+- area: url matching
+  change: |
+    Fixed excessive CPU utilization when using regex URL template matcher.
+
+removed_config_or_runtime:
+# *Normally occurs at the end of the* :ref:`deprecation period <deprecated>`
+
+new_features:
+- area: tap
+  change: |
+    added :ref:`custom_sink <envoy_v3_api_field_config.tap.v3.OutputSink.custom_sink>` type to enable writing tap data
+    out to a custom sink extension.
+
+deprecated:

source/common/http/filter_manager.cc

@@ -867,6 +867,13 @@ FilterManager::commonDecodePrefix(ActiveStreamDecoderFilter* filter,
 }
 
 void DownstreamFilterManager::onLocalReply(StreamFilterBase::LocalReplyData& data) {
+  if (Runtime::runtimeFeatureEnabled(
+          "envoy.reloadable_features.on_local_reply_createfilterchain")) {
+    // To ensure we have filters over which we can iterate and call onLocalReply.
+    // If the filter chain already exists this will be a no-op.
+    createFilterChain();
+  }
+
   state_.under_on_local_reply_ = true;
   filter_manager_callbacks_.onLocalReply(data.code_);
 

source/common/runtime/runtime_features.cc

@@ -43,6 +43,8 @@ RUNTIME_GUARD(envoy_reloadable_features_enable_update_listener_socket_options);
 RUNTIME_GUARD(envoy_reloadable_features_expand_agnostic_stream_lifetime);
 RUNTIME_GUARD(envoy_reloadable_features_finish_reading_on_decode_trailers);
 RUNTIME_GUARD(envoy_reloadable_features_fix_hash_key);
+RUNTIME_GUARD(envoy_reloadable_features_ext_authz_http_send_original_xff);
+RUNTIME_GUARD(envoy_reloadable_features_ext_proc_disable_response_processing_on_local_reply);
 RUNTIME_GUARD(envoy_reloadable_features_format_ports_as_numbers);
 RUNTIME_GUARD(envoy_reloadable_features_handle_uppercase_scheme);
 RUNTIME_GUARD(envoy_reloadable_features_http2_decode_metadata_with_quiche);
@@ -59,6 +61,7 @@ RUNTIME_GUARD(envoy_reloadable_features_no_extension_lookup_by_name);
 RUNTIME_GUARD(envoy_reloadable_features_no_full_scan_certs_on_sni_mismatch);
 RUNTIME_GUARD(envoy_reloadable_features_oauth_header_passthrough_fix);
 RUNTIME_GUARD(envoy_reloadable_features_oauth_use_url_encoding);
+RUNTIME_GUARD(envoy_reloadable_features_on_local_reply_createfilterchain);
 RUNTIME_GUARD(envoy_reloadable_features_original_dst_rely_on_idle_timeout);
 RUNTIME_GUARD(envoy_reloadable_features_prohibit_route_refresh_after_response_headers_sent);
 RUNTIME_GUARD(envoy_reloadable_features_quic_defer_logging_to_ack_listener);

source/extensions/common/tap/BUILD

@@ -28,6 +28,7 @@ envoy_cc_library(
         ":tap_interface",
         "//source/common/common:assert_lib",
         "//source/common/common:hex_lib",
+        "//source/common/config:utility_lib",
         "//source/extensions/common/matcher:matcher_lib",
         "@envoy_api//envoy/config/tap/v3:pkg_cc_proto",
         "@envoy_api//envoy/data/tap/v3:pkg_cc_proto",

source/extensions/common/tap/tap.h

@@ -76,6 +76,27 @@ class Sink {
 };
 
 using SinkPtr = std::unique_ptr<Sink>;
+using SinkContext =
+    absl::variant<std::reference_wrapper<Server::Configuration::FactoryContext>,
+                  std::reference_wrapper<Server::Configuration::TransportSocketFactoryContext>>;
+
+/**
+ * Abstract tap sink factory. Produces a factory that can instantiate SinkPtr objects
+ */
+class TapSinkFactory : public Config::TypedFactory {
+public:
+  ~TapSinkFactory() override = default;
+  std::string category() const override { return "envoy.tap.sinks"; }
+
+  /**
+   * Create a Sink that can be used for writing out data produced by the tap filter.
+   * @param config supplies the protobuf configuration for the sink factory
+   * @param cluster_manager is a ClusterManager from the HTTP/transport socket context
+   */
+  virtual SinkPtr createSinkPtr(const Protobuf::Message& config, SinkContext context) PURE;
+};
+
+using TapSinkFactoryPtr = std::unique_ptr<TapSinkFactory>;
 
 /**
  * Generic configuration for a tap extension (filter, transport socket, etc.).

source/extensions/common/tap/tap_config_base.cc

@@ -3,9 +3,11 @@
 #include "envoy/config/tap/v3/common.pb.h"
 #include "envoy/data/tap/v3/common.pb.h"
 #include "envoy/data/tap/v3/wrapper.pb.h"
+#include "envoy/server/transport_socket_config.h"
 
 #include "source/common/common/assert.h"
 #include "source/common/common/fmt.h"
+#include "source/common/config/utility.h"
 #include "source/common/protobuf/utility.h"
 #include "source/extensions/common/matcher/matcher.h"
 
@@ -45,12 +47,13 @@ bool Utility::addBufferToProtoBytes(envoy::data::tap::v3::Body& output_body,
 }
 
 TapConfigBaseImpl::TapConfigBaseImpl(const envoy::config::tap::v3::TapConfig& proto_config,
-                                     Common::Tap::Sink* admin_streamer)
+                                     Common::Tap::Sink* admin_streamer, SinkContext context)
     : max_buffered_rx_bytes_(PROTOBUF_GET_WRAPPED_OR_DEFAULT(
           proto_config.output_config(), max_buffered_rx_bytes, DefaultMaxBufferedBytes)),
       max_buffered_tx_bytes_(PROTOBUF_GET_WRAPPED_OR_DEFAULT(
           proto_config.output_config(), max_buffered_tx_bytes, DefaultMaxBufferedBytes)),
       streaming_(proto_config.output_config().streaming()) {
+
   using ProtoOutputSink = envoy::config::tap::v3::OutputSink;
   auto& sinks = proto_config.output_config().sinks();
   ASSERT(sinks.size() == 1);
@@ -86,6 +89,33 @@ TapConfigBaseImpl::TapConfigBaseImpl(const envoy::config::tap::v3::TapConfig& pr
     sink_ = std::make_unique<FilePerTapSink>(sinks[0].file_per_tap());
     sink_to_use_ = sink_.get();
     break;
+  case ProtoOutputSink::OutputSinkTypeCase::kCustomSink: {
+    TapSinkFactory& tap_sink_factory =
+        Envoy::Config::Utility::getAndCheckFactory<TapSinkFactory>(sinks[0].custom_sink());
+
+    // extract message validation visitor from the context and use it to define config
+    ProtobufTypes::MessagePtr config;
+    using TsfContextRef =
+        std::reference_wrapper<Server::Configuration::TransportSocketFactoryContext>;
+    using HttpContextRef = std::reference_wrapper<Server::Configuration::FactoryContext>;
+    if (absl::holds_alternative<TsfContextRef>(context)) {
+      Server::Configuration::TransportSocketFactoryContext& tsf_context =
+          absl::get<TsfContextRef>(context).get();
+      config = Config::Utility::translateAnyToFactoryConfig(sinks[0].custom_sink().typed_config(),
+                                                            tsf_context.messageValidationVisitor(),
+                                                            tap_sink_factory);
+    } else {
+      Server::Configuration::FactoryContext& http_context =
+          absl::get<HttpContextRef>(context).get();
+      config = Config::Utility::translateAnyToFactoryConfig(
+          sinks[0].custom_sink().typed_config(),
+          http_context.messageValidationContext().staticValidationVisitor(), tap_sink_factory);
+    }
+
+    sink_ = tap_sink_factory.createSinkPtr(*config, context);
+    sink_to_use_ = sink_.get();
+    break;
+  }
   case envoy::config::tap::v3::OutputSink::OutputSinkTypeCase::kStreamingGrpc:
     PANIC("not implemented");
   case envoy::config::tap::v3::OutputSink::OutputSinkTypeCase::OUTPUT_SINK_TYPE_NOT_SET:

source/extensions/common/tap/tap_config_base.h

@@ -103,7 +103,7 @@ class TapConfigBaseImpl : public virtual TapConfig {
 
 protected:
   TapConfigBaseImpl(const envoy::config::tap::v3::TapConfig& proto_config,
-                    Common::Tap::Sink* admin_streamer);
+                    Common::Tap::Sink* admin_streamer, SinkContext context);
 
 private:
   // This is the default setting for both RX/TX max buffered bytes. (This means that per tap, the

source/extensions/filters/common/expr/context.cc

@@ -152,7 +152,13 @@ absl::optional<CelValue> ResponseWrapper::operator[](CelValue key) const {
   }
   auto value = key.StringOrDie().value();
   if (value == Code) {
-    auto code = info_.responseCode();
+    absl::optional<uint32_t> code;
+    uint32_t maybecode;
+    if (info_.responseCode().has_value()) {
+      code.emplace(info_.responseCode().value());
+    } else if (headers_.value_ != nullptr && absl::SimpleAtoi(headers_.value_->getStatusValue(), &maybecode)) {
+      code.emplace(maybecode);
+    }
     if (code.has_value()) {
       return CelValue::CreateInt64(code.value());
     }

source/extensions/filters/common/expr/evaluator.cc

@@ -83,7 +83,7 @@ ActivationPtr createActivation(const StreamInfo::StreamInfo& info,
                                             response_trailers);
 }
 
-BuilderPtr createBuilder(Protobuf::Arena* arena) {
+Extensions::Filters::Common::Expr::BuilderPtr createBuilder(Protobuf::Arena* arena) {
   google::api::expr::runtime::InterpreterOptions options;
 
   // Security-oriented defaults

source/extensions/filters/common/expr/evaluator.h

@@ -60,7 +60,7 @@ ActivationPtr createActivation(const StreamInfo::StreamInfo& info,
 // Creates an expression builder. The optional arena is used to enable constant folding
 // for intermediate evaluation results.
 // Throws an exception if fails to construct an expression builder.
-BuilderPtr createBuilder(Protobuf::Arena* arena);
+Filters::Common::Expr::BuilderPtr createBuilder(Protobuf::Arena* arena);
 
 // Creates an interpretable expression from a protobuf representation.
 // Throws an exception if fails to construct a runtime expression.

source/extensions/filters/http/ext_proc/BUILD

@@ -19,6 +19,12 @@ envoy_cc_library(
         "ext_proc.h",
         "processor_state.h",
     ],
+    copts = select({
+        "//bazel:windows_x86_64": [],
+        "//conditions:default": [
+            "-DUSE_CEL_PARSER",
+        ],
+    }),
     deps = [
         ":client_interface",
         ":mutation_utils_lib",
@@ -30,23 +36,40 @@ envoy_cc_library(
         "//source/common/runtime:runtime_features_lib",
         "//source/extensions/filters/common/mutation_rules:mutation_rules_lib",
         "//source/extensions/filters/http/common:pass_through_filter_lib",
+        "//source/extensions/filters/common/expr:evaluator_lib",
         "@com_google_absl//absl/status",
         "@com_google_absl//absl/strings:str_format",
+        "@com_google_cel_cpp//eval/public:builtin_func_registrar",
+        "@com_google_cel_cpp//eval/public:cel_expr_builder_factory",
         "@envoy_api//envoy/config/common/mutation_rules/v3:pkg_cc_proto",
         "@envoy_api//envoy/config/core/v3:pkg_cc_proto",
         "@envoy_api//envoy/extensions/filters/http/ext_proc/v3:pkg_cc_proto",
         "@envoy_api//envoy/service/ext_proc/v3:pkg_cc_proto",
-    ],
+    ] + select(
+        {
+            "//bazel:windows_x86_64": [],
+            "//conditions:default": [
+                "@com_google_cel_cpp//parser",
+            ],
+        },
+    ),
 )
 
 envoy_cc_extension(
     name = "config",
     srcs = ["config.cc"],
     hdrs = ["config.h"],
+    copts = select({
+        "//bazel:windows_x86_64": [],
+        "//conditions:default": [
+            "-DUSE_CEL_PARSER",
+        ],
+    }),
     deps = [
         ":client_lib",
         ":ext_proc",
         "//source/extensions/filters/http/common:factory_base_lib",
+        "//source/extensions/filters/common/expr:evaluator_lib",
         "@envoy_api//envoy/extensions/filters/http/ext_proc/v3:pkg_cc_proto",
     ],
 )