diff --git a/WORKSPACE b/WORKSPACE index 3aa555ce5..d7646b17b 100644 --- a/WORKSPACE +++ b/WORKSPACE @@ -43,6 +43,7 @@ git_repository( "@//patches:0004-thread_local-reset-slot-in-worker-threads-first.patch", "@//patches:0005-http-header-expose-attribute.patch", "@//patches:0006-test-integration-Defer-fake-upstream-read-enable-un.patch", + "@//patches:0007-config-add-grpc-mux-stream-event-callback.patch", "@//patches:0008-repo-Make-yq-dependency-optional-for-CI-config-parsi.patch", ], # // clang-format off: Envoy's format check: Only repository_locations.bzl may contains URL references diff --git a/cilium/BUILD b/cilium/BUILD index ca9e1269e..1af2056e8 100644 --- a/cilium/BUILD +++ b/cilium/BUILD @@ -28,6 +28,17 @@ envoy_cc_library( ], ) +envoy_cc_library( + name = "versioned_lib", + hdrs = ["versioned.h"], + repository = "@envoy", + deps = [ + "@com_google_absl//absl/container:flat_hash_map", + "@com_google_absl//absl/container:flat_hash_set", + "@envoy//source/common/common:assert_lib", + ], +) + envoy_cc_library( name = "network_policy_lib", srcs = [ @@ -45,6 +56,7 @@ envoy_cc_library( "//cilium:conntrack_lib", "//cilium:grpc_subscription_lib", "//cilium:ipcache_lib", + "//cilium:versioned_lib", "//cilium/api:npds_cc_proto", "@envoy//envoy/singleton:manager_interface", "@envoy//source/common/common:logger_lib", diff --git a/cilium/api/bpf_metadata.proto b/cilium/api/bpf_metadata.proto index 9e660991c..ad74c69f4 100644 --- a/cilium/api/bpf_metadata.proto +++ b/cilium/api/bpf_metadata.proto @@ -84,6 +84,6 @@ message BpfMetadata { // Cache is garbage collected at interval 10 times the ttl (default 30 ms). google.protobuf.Duration cache_gc_interval = 15; - // Configuration for the source of NPDS updates. Currently this field is not supported. - envoy.config.core.v3.ConfigSource npds_config = 16; + // Configuration for the source of Cilium xDS updates. + envoy.config.core.v3.ConfigSource config_source = 16; } diff --git a/cilium/api/npds.proto b/cilium/api/npds.proto index 1a43692e9..b5a1e37ac 100644 --- a/cilium/api/npds.proto +++ b/cilium/api/npds.proto @@ -17,6 +17,7 @@ import "validate/validate.proto"; // [#protodoc-title: Network policy management and NPDS] // Each resource name is a network policy identifier. +// Deprecated: This service will be removed when Cilium 1.20 is the oldest supported release. service NetworkPolicyDiscoveryService { option (envoy.annotations.resource).type = "cilium.NetworkPolicy"; @@ -33,6 +34,32 @@ service NetworkPolicyDiscoveryService { } } +// Policy and selector resource names are exact-match identifiers in delta NPDS. +service NetworkPolicyResourceDiscoveryService { + option (envoy.annotations.resource).type = "cilium.NetworkPolicyResource"; + + rpc DeltaNetworkPolicyResources(stream envoy.service.discovery.v3.DeltaDiscoveryRequest) + returns (stream envoy.service.discovery.v3.DeltaDiscoveryResponse) { + } +} + +// A delta NPDS resource that carries either an endpoint policy or a shared selector. +message NetworkPolicyResource { + oneof resource { + NetworkPolicy policy = 1; + Selector selector = 2; + } +} + +// A shared set of remote identities referenced by selector resource name. +// Unlike the old state-of-the-world remote identity lists, an empty selector +// matches nothing. +message Selector { + // The set of numeric remote security IDs selected by this selector. + // If empty, this selector selects no remote identities. + repeated uint32 remote_identities = 1; +} + // A network policy that is enforced by a filter on the network flows to/from // associated hosts. message NetworkPolicy { @@ -153,6 +180,12 @@ message PortNetworkPolicyRule { // Optional. If not specified, any remote host is matched by this predicate. repeated uint32 remote_policies = 7; + // Optional selector resource names that can be resolved to shared remote + // policy sets in delta NPDS. + // Selector references are matched by exact selector resource name. + // Optional. If not specified, any remote host is matched by this predicate. + repeated string selectors = 11; + // Optional downstream TLS context. If present, the incoming connection must // be a TLS connection. TLSContext downstream_tls_context = 3; diff --git a/cilium/api/nphds.proto b/cilium/api/nphds.proto index 82a6dbb2a..bb5628510 100644 --- a/cilium/api/nphds.proto +++ b/cilium/api/nphds.proto @@ -26,6 +26,10 @@ service NetworkPolicyHostsDiscoveryService { body: "*" }; } + + rpc DeltaNetworkPolicyHosts(stream envoy.service.discovery.v3.DeltaDiscoveryRequest) + returns (stream envoy.service.discovery.v3.DeltaDiscoveryResponse) { + } } // The mapping of a network policy identifier to the IP addresses of all the diff --git a/cilium/bpf_metadata.cc b/cilium/bpf_metadata.cc index b7161e1c4..f08812bd9 100644 --- a/cilium/bpf_metadata.cc +++ b/cilium/bpf_metadata.cc @@ -198,30 +198,6 @@ SINGLETON_MANAGER_REGISTRATION(cilium_bpf_conntrack); SINGLETON_MANAGER_REGISTRATION(cilium_host_map); SINGLETON_MANAGER_REGISTRATION(cilium_network_policy); -namespace { - -std::shared_ptr -createHostMap(Server::Configuration::ListenerFactoryContext& context, - envoy::config::core::v3::ConfigSource& npds_config) { - return context.serverFactoryContext().singletonManager().getTyped( - SINGLETON_MANAGER_REGISTERED_NAME(cilium_host_map), [&context, npds_config] { - auto map = std::make_shared(context.serverFactoryContext()); - map->startSubscription(context.serverFactoryContext(), npds_config); - return map; - }); -} - -std::shared_ptr -createPolicyMap(Server::Configuration::FactoryContext& context, - envoy::config::core::v3::ConfigSource& npds_config) { - return context.serverFactoryContext().singletonManager().getTyped( - SINGLETON_MANAGER_REGISTERED_NAME(cilium_network_policy), [&context, npds_config] { - return std::make_shared(context, npds_config, true); - }); -} - -} // namespace - Config::Config(const ::cilium::BpfMetadata& config, Server::Configuration::ListenerFactoryContext& context) : so_linger_(config.has_original_source_so_linger_time() @@ -239,8 +215,8 @@ Config::Config(const ::cilium::BpfMetadata& config, ipcache_entry_ttl_( PROTOBUF_GET_MS_OR_DEFAULT(config, cache_entry_ttl, DEFAULT_CACHE_ENTRY_TTL_MS)), random_(context.serverFactoryContext().api().randomGenerator()), - npds_config_(config.has_npds_config() ? config.npds_config() - : Cilium::CILIUM_XDS_API_CONFIG) { + config_source_(config.has_config_source() ? config.config_source() + : Cilium::CILIUM_XDS_API_CONFIG) { if (is_l7lb_ && is_ingress_) { throw EnvoyException("cilium.bpf_metadata: is_l7lb may not be set with is_ingress"); } @@ -259,7 +235,15 @@ Config::Config(const ::cilium::BpfMetadata& config, config.ipv6_source_address())); } if (config.use_nphds()) { - hosts_ = createHostMap(context, npds_config_); + hosts_ = + context.serverFactoryContext().singletonManager().getTyped( + SINGLETON_MANAGER_REGISTERED_NAME(cilium_host_map), + [&context, config_source = config_source_] { + auto map = std::make_shared(context.serverFactoryContext()); + map->startSubscription(context.serverFactoryContext(), config_source); + return map; + }); + hosts_->setConfigSource(config_source_); } // Note: all instances use the bpf root of the first filter with non-empty @@ -296,7 +280,14 @@ Config::Config(const ::cilium::BpfMetadata& config, // instances! // Only created if either ipcache_ or hosts_ map exists if (ipcache_ || hosts_) { - npmap_ = createPolicyMap(context, npds_config_); + npmap_ = + context.serverFactoryContext().singletonManager().getTyped( + SINGLETON_MANAGER_REGISTERED_NAME(cilium_network_policy), + [&context, config_source = config_source_] { + return std::make_shared(context, config_source, true); + }); + // update desired config source on the map + npmap_->setConfigSource(config_source_); } } diff --git a/cilium/bpf_metadata.h b/cilium/bpf_metadata.h index 0346a83d3..753578a5c 100644 --- a/cilium/bpf_metadata.h +++ b/cilium/bpf_metadata.h @@ -173,7 +173,7 @@ class Config : public Cilium::PolicyResolver, std::string l7lb_policy_name_; std::chrono::milliseconds ipcache_entry_ttl_; Random::RandomGenerator& random_; - envoy::config::core::v3::ConfigSource npds_config_; + envoy::config::core::v3::ConfigSource config_source_; std::shared_ptr npmap_; Cilium::CtMapSharedPtr ct_maps_; diff --git a/cilium/grpc_subscription.cc b/cilium/grpc_subscription.cc index d14ab2c96..57013a4de 100644 --- a/cilium/grpc_subscription.cc +++ b/cilium/grpc_subscription.cc @@ -2,22 +2,22 @@ #include -#include +#include #include #include #include #include #include "envoy/annotations/resource.pb.h" +#include "envoy/common/callback.h" #include "envoy/common/exception.h" -#include "envoy/common/random_generator.h" #include "envoy/config/core/v3/config_source.pb.h" #include "envoy/config/custom_config_validators.h" +#include "envoy/config/grpc_mux.h" #include "envoy/config/subscription.h" #include "envoy/config/subscription_factory.h" -#include "envoy/event/dispatcher.h" #include "envoy/grpc/async_client.h" -#include "envoy/local_info/local_info.h" +#include "envoy/server/factory_context.h" #include "envoy/stats/scope.h" #include "envoy/upstream/cluster_manager.h" @@ -27,9 +27,12 @@ #include "source/common/grpc/common.h" #include "source/common/protobuf/protobuf.h" // IWYU pragma: keep #include "source/extensions/config_subscription/grpc/grpc_mux_context.h" +#include "source/extensions/config_subscription/grpc/grpc_mux_impl.h" #include "source/extensions/config_subscription/grpc/grpc_subscription_impl.h" +#include "source/extensions/config_subscription/grpc/new_grpc_mux_impl.h" #include "absl/container/flat_hash_map.h" +#include "absl/container/flat_hash_set.h" #include "absl/status/statusor.h" #include "absl/strings/match.h" #include "absl/strings/string_view.h" @@ -40,6 +43,31 @@ namespace Cilium { namespace { +class StreamEventSubscription : public Config::Subscription { +public: + StreamEventSubscription(std::unique_ptr subscription, + Common::CallbackHandlePtr stream_event_handle) + : subscription_(std::move(subscription)), + stream_event_handle_(std::move(stream_event_handle)) {} + + void start(const absl::flat_hash_set& resource_names) override { + subscription_->start(resource_names); + } + + void + updateResourceInterest(const absl::flat_hash_set& update_to_these_names) override { + subscription_->updateResourceInterest(update_to_these_names); + } + + void requestOnDemandUpdate(const absl::flat_hash_set& add_these_names) override { + subscription_->requestOnDemandUpdate(add_these_names); + } + +private: + std::unique_ptr subscription_; + Common::CallbackHandlePtr stream_event_handle_; +}; + // service RPC method fully qualified names. struct Service { std::string sotw_grpc_method_; @@ -59,6 +87,7 @@ TypeUrlToServiceMap* buildTypeUrlToServiceMap() { // https://www.mail-archive.com/protobuf@googlegroups.com/msg04540.html. for (absl::string_view name : { "cilium.NetworkPolicyDiscoveryService", + "cilium.NetworkPolicyResourceDiscoveryService", "cilium.NetworkPolicyHostsDiscoveryService", }) { const auto* service_desc = @@ -120,57 +149,87 @@ const Protobuf::MethodDescriptor& sotwGrpcMethod(absl::string_view type_url) { it->second.sotw_grpc_method_); } -std::unique_ptr +std::unique_ptr subscribe(const absl::string_view type_url, - const envoy::config::core::v3::ConfigSource& npds_config, - const LocalInfo::LocalInfo& local_info, Upstream::ClusterManager& cm, - Event::Dispatcher& dispatcher, Random::RandomGenerator& random, Stats::Scope& scope, + const envoy::config::core::v3::ConfigSource& config_source, + Server::Configuration::CommonFactoryContext& context, Stats::Scope& scope, Config::SubscriptionCallbacks& callbacks, Config::OpaqueResourceDecoderSharedPtr resource_decoder, - std::chrono::milliseconds init_fetch_timeout) { - auto& api_config_source = npds_config.api_config_source(); - THROW_IF_NOT_OK(Config::Utility::checkApiConfigSourceSubscriptionBackingCluster( - cm.primaryClusters(), api_config_source)); - + Config::GrpcMuxStreamEventCallback on_stream_event) { + auto initial_fetch_timeout = Config::Utility::configSourceInitialFetchTimeout(config_source); Config::SubscriptionStats stats = Config::Utility::generateStats(scope); Envoy::Config::SubscriptionOptions options; - // No-op custom validators - Envoy::Config::CustomConfigValidatorsPtr nop_config_validators = - std::make_unique(); - auto factory_or_error = Config::Utility::factoryForGrpcApiConfigSource( - cm.grpcAsyncClientManager(), api_config_source, scope, true, 0, false); - THROW_IF_NOT_OK_REF(factory_or_error.status()); - - absl::StatusOr rate_limit_settings_or_error = - Config::Utility::parseRateLimitSettings(api_config_source); - THROW_IF_NOT_OK_REF(rate_limit_settings_or_error.status()); - - Config::GrpcMuxContext grpc_mux_context{ - /*async_client_=*/THROW_OR_RETURN_VALUE( - factory_or_error.value()->createUncachedRawAsyncClient(), Grpc::RawAsyncClientPtr), - /*failover_async_client_=*/nullptr, - /*dispatcher_=*/dispatcher, - /*service_method_=*/sotwGrpcMethod(type_url), - /*local_info_=*/local_info, - /*rate_limit_settings_=*/rate_limit_settings_or_error.value(), - /*scope_=*/scope, - /*config_validators_=*/std::move(nop_config_validators), - /*xds_resources_delegate_=*/absl::nullopt, - /*xds_config_tracker_=*/absl::nullopt, - /*backoff_strategy_=*/ - std::make_unique( - Config::SubscriptionFactory::RetryInitialDelayMs, - Config::SubscriptionFactory::RetryMaxDelayMs, random), - /*target_xds_authority_=*/"", - /*eds_resources_cache_=*/nullptr, // EDS cache is only used for ADS. - /*skip_subsequent_node_=*/api_config_source.set_node_on_first_message_only(), - }; - - return std::make_unique( - std::make_shared(grpc_mux_context), callbacks, resource_decoder, stats, type_url, - dispatcher, init_fetch_timeout, - /*is_aggregated*/ false, options); + std::shared_ptr grpc_mux; + bool is_aggregated = + config_source.config_source_specifier_case() == envoy::config::core::v3::ConfigSource::kAds; + if (is_aggregated) { + grpc_mux = std::static_pointer_cast(context.xdsManager().adsMux()); + } else { + auto& api_config_source = config_source.api_config_source(); + THROW_IF_NOT_OK(Config::Utility::checkApiConfigSourceSubscriptionBackingCluster( + context.clusterManager().primaryClusters(), api_config_source)); + + // No-op custom validators + Envoy::Config::CustomConfigValidatorsPtr nop_config_validators = + std::make_unique(); + auto factory_or_error = Config::Utility::factoryForGrpcApiConfigSource( + context.clusterManager().grpcAsyncClientManager(), api_config_source, scope, true, 0, + false); + THROW_IF_NOT_OK_REF(factory_or_error.status()); + + absl::StatusOr rate_limit_settings_or_error = + Config::Utility::parseRateLimitSettings(api_config_source); + THROW_IF_NOT_OK_REF(rate_limit_settings_or_error.status()); + + const auto& api_type = api_config_source.api_type(); + bool use_delta = api_type == envoy::config::core::v3::ApiConfigSource::DELTA_GRPC || + api_type == envoy::config::core::v3::ApiConfigSource::AGGREGATED_DELTA_GRPC; + const auto& service_method = use_delta ? deltaGrpcMethod(type_url) : sotwGrpcMethod(type_url); + + Config::GrpcMuxContext grpc_mux_context{ + THROW_OR_RETURN_VALUE(factory_or_error.value()->createUncachedRawAsyncClient(), + Grpc::RawAsyncClientPtr), + /*failover_async_client_=*/nullptr, + context.mainThreadDispatcher(), + service_method, + context.localInfo(), + rate_limit_settings_or_error.value(), + scope, + std::move(nop_config_validators), + /*xds_resources_delegate_=*/absl::nullopt, + /*xds_config_tracker_=*/absl::nullopt, + std::make_unique( + Config::SubscriptionFactory::RetryInitialDelayMs, + Config::SubscriptionFactory::RetryMaxDelayMs, context.api().randomGenerator()), + /*target_xds_authority_=*/"", + /*eds_resources_cache_=*/nullptr, // EDS cache is only used for ADS. + /*skip_subsequent_node_=*/api_config_source.set_node_on_first_message_only(), + }; + + grpc_mux = use_delta ? std::static_pointer_cast( + std::make_shared(grpc_mux_context)) + : std::static_pointer_cast( + std::make_shared(grpc_mux_context)); + } + + Common::CallbackHandlePtr stream_event_handle; + if (on_stream_event) { + auto stream_event_callback = std::move(on_stream_event); + stream_event_handle = grpc_mux->addStreamEventCallback(stream_event_callback); + if (grpc_mux->grpcStreamConnected()) { + stream_event_callback(Config::GrpcMuxStreamEvent::Established); + } + } + + auto subscription = std::make_unique( + grpc_mux, callbacks, resource_decoder, stats, type_url, context.mainThreadDispatcher(), + initial_fetch_timeout, is_aggregated, options); + if (stream_event_handle) { + return std::make_unique(std::move(subscription), + std::move(stream_event_handle)); + } + return subscription; } } // namespace Cilium diff --git a/cilium/grpc_subscription.h b/cilium/grpc_subscription.h index eeaa87198..1e8e85f08 100644 --- a/cilium/grpc_subscription.h +++ b/cilium/grpc_subscription.h @@ -1,56 +1,24 @@ #pragma once -#include #include -#include "envoy/common/random_generator.h" #include "envoy/config/core/v3/config_source.pb.h" +#include "envoy/config/grpc_mux.h" #include "envoy/config/subscription.h" -#include "envoy/event/dispatcher.h" -#include "envoy/local_info/local_info.h" +#include "envoy/ssl/context_manager.h" #include "envoy/stats/scope.h" -#include "envoy/upstream/cluster_manager.h" - -#include "source/extensions/config_subscription/grpc/grpc_mux_context.h" -#include "source/extensions/config_subscription/grpc/grpc_mux_impl.h" -#include "source/extensions/config_subscription/grpc/grpc_subscription_impl.h" #include "absl/strings/string_view.h" namespace Envoy { namespace Cilium { -// GrpcMux wrapper to get access to control plane identifier -class GrpcMuxImpl : public Config::GrpcMuxImpl { -public: - GrpcMuxImpl(Config::GrpcMuxContext& grpc_mux_context) : Config::GrpcMuxImpl(grpc_mux_context) {} - - ~GrpcMuxImpl() override = default; - - void onStreamEstablished() override { - new_stream_ = true; - Config::GrpcMuxImpl::onStreamEstablished(); - } - - // isNewStream returns true for the first call after a new stream has been established - bool isNewStream() { - bool new_stream = new_stream_; - new_stream_ = false; - return new_stream; - } - -private: - bool new_stream_ = true; -}; - -std::unique_ptr +std::unique_ptr subscribe(const absl::string_view type_url, - const envoy::config::core::v3::ConfigSource& npds_config, - const LocalInfo::LocalInfo& local_info, Upstream::ClusterManager& cm, - Event::Dispatcher& dispatcher, Random::RandomGenerator& random, Stats::Scope& scope, + const envoy::config::core::v3::ConfigSource& config_source, + Server::Configuration::CommonFactoryContext& context, Stats::Scope& scope, Config::SubscriptionCallbacks& callbacks, Config::OpaqueResourceDecoderSharedPtr resource_decoder, - std::chrono::milliseconds init_fetch_timeout = std::chrono::milliseconds(0)); - + Config::GrpcMuxStreamEventCallback on_stream_event = {}); } // namespace Cilium } // namespace Envoy diff --git a/cilium/host_map.cc b/cilium/host_map.cc index eabfa34bf..77a0fd8cc 100644 --- a/cilium/host_map.cc +++ b/cilium/host_map.cc @@ -4,15 +4,18 @@ #include #include +#include #include #include #include #include +#include #include #include #include "envoy/common/exception.h" #include "envoy/config/core/v3/config_source.pb.h" +#include "envoy/config/grpc_mux.h" #include "envoy/config/subscription.h" #include "envoy/event/dispatcher.h" #include "envoy/server/factory_context.h" @@ -21,9 +24,11 @@ #include "envoy/thread_local/thread_local.h" #include "envoy/thread_local/thread_local_object.h" +#include "source/common/common/assert.h" #include "source/common/common/logger.h" #include "source/common/common/macros.h" +#include "absl/container/flat_hash_set.h" #include "absl/numeric/int128.h" #include "absl/status/status.h" #include "absl/strings/str_cat.h" @@ -58,9 +63,18 @@ unsigned int checkPrefix(T addr, bool have_prefix, unsigned int plen, absl::stri } // namespace struct ThreadLocalHostMapInitializer : public PolicyHostMap::ThreadLocalHostMap { -protected: +public: friend class PolicyHostMap; // PolicyHostMap can insert(); + ThreadLocalHostMapInitializer() = default; + + explicit ThreadLocalHostMapInitializer(const PolicyHostMap::ThreadLocalHostMap* host_map) { + if (host_map != nullptr) { + static_cast(*this) = *host_map; + } + } + +protected: // find the map of the given prefix length, insert in the decreasing order if // it does not exist template @@ -159,6 +173,29 @@ struct ThreadLocalHostMapInitializer : public PolicyHostMap::ThreadLocalHostMap fmt::format("NetworkPolicyHosts: Invalid host entry \'{}\' for policy {}", host, policy)); } } + + template + void prunePolicyMapVec(MapVec& maps, const absl::flat_hash_set& nids) { + for (auto vec_it = maps.begin(); vec_it != maps.end();) { + auto& map = vec_it->second; + for (auto map_it = map.begin(); map_it != map.end();) { + auto it = map_it++; + if (nids.contains(it->second)) { + map.erase(it); + } + } + if (map.empty()) { + vec_it = maps.erase(vec_it); + } else { + ++vec_it; + } + } + } + + void remove(const absl::flat_hash_set removed_nids) { + prunePolicyMapVec(ipv4_to_policy_, removed_nids); + prunePolicyMapVec(ipv6_to_policy_, removed_nids); + } }; uint64_t PolicyHostMap::instance_id_ = 0; @@ -197,23 +234,156 @@ PolicyHostMap::PolicyHostMap(Server::Configuration::CommonFactoryContext& contex void PolicyHostMap::startSubscription(Server::Configuration::CommonFactoryContext& context, const envoy::config::core::v3::ConfigSource& npds_config) { - if (npds_config.config_source_specifier_case() == envoy::config::core::v3::ConfigSource::kAds) { - auto ads_mux = context.xdsManager().adsMux(); - subscription_ = THROW_OR_RETURN_VALUE( - context.clusterManager().subscriptionFactory().subscriptionOverAdsGrpcMux( - ads_mux, npds_config, NetworkPolicyHostsTypeUrl, *scope_, *this, - std::make_shared(), {}), - Config::SubscriptionPtr); - } else { - subscription_ = subscribe(NetworkPolicyHostsTypeUrl, npds_config, context.localInfo(), - context.clusterManager(), context.mainThreadDispatcher(), - context.api().randomGenerator(), *scope_, *this, - std::make_shared()); + context_ = &context; + desired_config_source_ = npds_config; + subscribe(); +} + +void PolicyHostMap::setConfigSource( + const envoy::config::core::v3::ConfigSource& config_source) const { + auto* self = const_cast(this); + self->desired_config_source_ = config_source; + if (self->context_ != nullptr) { + self->maybeRecreateSubscriptionInDesiredMode(/*transport_closed=*/false); + } +} + +bool PolicyHostMap::subscriptionUseDeltaXds() const { + if (!config_source_.has_api_config_source()) { + return false; } + const auto& api_type = config_source_.api_config_source().api_type(); + return api_type == envoy::config::core::v3::ApiConfigSource::DELTA_GRPC || + api_type == envoy::config::core::v3::ApiConfigSource::AGGREGATED_DELTA_GRPC; +} +void PolicyHostMap::subscribe() { + ASSERT(context_ != nullptr); + subscription_connected_ = false; + config_source_ = desired_config_source_; + ++subscription_id_; + + auto on_stream_event = [weak_this = weak_from_this(), + id = subscription_id_](Config::GrpcMuxStreamEvent event) { + if (auto shared_this = weak_this.lock()) { + shared_this->onSubscriptionStreamEvent(id, event); + } + }; + + subscription_ = + Cilium::subscribe(NetworkPolicyHostsTypeUrl, config_source_, *context_, *scope_, *this, + std::make_shared(), std::move(on_stream_event)); subscription_->start({}); } +void PolicyHostMap::onSubscriptionStreamEvent(uint64_t subscription_id, + Config::GrpcMuxStreamEvent event) { + if (subscription_id != subscription_id_) { + return; + } + + switch (event) { + case Config::GrpcMuxStreamEvent::Established: + subscription_connected_ = true; + break; + case Config::GrpcMuxStreamEvent::Closed: + if (!subscription_connected_) { + return; + } + subscription_connected_ = false; + + if (context_ == nullptr) { + return; + } + + context_->mainThreadDispatcher().post( + [weak_this = weak_from_this(), subscription_id = subscription_id_]() { + if (auto shared_this = weak_this.lock()) { + if (subscription_id != shared_this->subscription_id_) { + return; + } + shared_this->maybeRecreateSubscriptionInDesiredMode(/*transport_closed=*/true); + } + }); + break; + } +} + +void PolicyHostMap::maybeRecreateSubscriptionInDesiredMode(bool transport_closed) { + if (subscription_ && (subscription_connected_ || !transport_closed)) { + if (subscription_connected_ && subscriptionUseDeltaXds()) { + return; + } + if (Protobuf::util::MessageDifferencer::Equals(config_source_, desired_config_source_)) { + return; + } + } + + subscribe(); +} + +absl::Status +PolicyHostMap::onConfigUpdate(const std::vector& added_resources, + const Protobuf::RepeatedPtrField& removed_resources, + const std::string& system_version_info) { + const bool is_new_stream = subscription_id_ != accepted_subscription_id_; + ENVOY_LOG( + debug, + "PolicyHostMap::onConfigUpdate({}), {} added_resources, {} removed_resources, version: {}, " + "subscription_id: {}, accepted_subscription_id: {}, is_new_stream: {}", + name_, added_resources.size(), removed_resources.size(), system_version_info, + subscription_id_, accepted_subscription_id_, is_new_stream); + + auto newmap = + std::make_shared(is_new_stream ? nullptr : getHostMap()); + + absl::flat_hash_set to_remove; + to_remove.reserve(added_resources.size() + removed_resources.size()); + + for (const auto& name : removed_resources) { + uint64_t nid = 0; + auto [ptr, ec] = std::from_chars(name.data(), name.data() + name.size(), nid); + if (ec != std::errc{} || ptr != name.data() + name.size()) { + throw EnvoyException(fmt::format("Invalid removed resource name '{}'", name)); + } + ENVOY_LOG(trace, + "Removing NetworkPolicyHosts for policy {} in delta onConfigUpdate() version {}", nid, + system_version_info); + to_remove.insert(nid); + } + for (const auto& resource : added_resources) { + const auto& config = dynamic_cast(resource.get().resource()); + to_remove.insert(config.policy()); + } + newmap->remove(to_remove); + + for (const auto& resource : added_resources) { + const auto& config = dynamic_cast(resource.get().resource()); + ENVOY_LOG(trace, + "Received NetworkPolicyHosts for policy {} in delta onConfigUpdate() version {}", + config.policy(), system_version_info); + newmap->insert(config); + } + + // Force 'this' to be not deleted for as long as the lambda stays + // alive. Note that generally capturing a shared pointer is + // dangerous as it may happen that there is a circular reference + // from 'this' to itself via the lambda capture, leading to 'this' + // never being released. It should happen in this case, though. + std::shared_ptr shared_this = shared_from_this(); + + // Assign the new map to all threads. + tls_->set([shared_this, newmap](Event::Dispatcher&) -> ThreadLocal::ThreadLocalObjectSharedPtr { + UNREFERENCED_PARAMETER(shared_this); + ENVOY_LOG(trace, "PolicyHostMap: Assigning new map"); + return newmap; + }); + logmaps("delta onConfigUpdate"); + accepted_subscription_id_ = subscription_id_; + stats_.update_success_.inc(); + return absl::OkStatus(); +} + absl::Status PolicyHostMap::onConfigUpdate(const std::vector& resources, const std::string& version_info) { diff --git a/cilium/host_map.h b/cilium/host_map.h index cce99431d..629336d70 100644 --- a/cilium/host_map.h +++ b/cilium/host_map.h @@ -15,6 +15,7 @@ #include "envoy/common/exception.h" #include "envoy/config/core/v3/config_source.pb.h" +#include "envoy/config/grpc_mux.h" #include "envoy/config/subscription.h" #include "envoy/network/address.h" #include "envoy/protobuf/message_validator.h" @@ -26,7 +27,6 @@ #include "envoy/thread_local/thread_local_object.h" #include "source/common/common/logger.h" -#include "source/common/common/macros.h" #include "source/common/network/utility.h" #include "source/common/protobuf/message_validator_impl.h" #include "source/common/protobuf/protobuf.h" @@ -116,7 +116,9 @@ class PolicyHostMap : public Singleton::Instance, } void startSubscription(Server::Configuration::CommonFactoryContext& context, - const envoy::config::core::v3::ConfigSource& npds_config); + const envoy::config::core::v3::ConfigSource& config_source); + + void setConfigSource(const envoy::config::core::v3::ConfigSource& config_source) const; // This is used for testing with a file-based subscription void startSubscription(std::unique_ptr&& subscription) { @@ -229,22 +231,27 @@ class PolicyHostMap : public Singleton::Instance, const std::string& version_info) override; absl::Status onConfigUpdate(const std::vector& added_resources, const Protobuf::RepeatedPtrField& removed_resources, - const std::string& system_version_info) override { - // NOT IMPLEMENTED YET. - UNREFERENCED_PARAMETER(added_resources); - UNREFERENCED_PARAMETER(removed_resources); - UNREFERENCED_PARAMETER(system_version_info); - return absl::OkStatus(); - } + const std::string& system_version_info) override; void onConfigUpdateFailed(Envoy::Config::ConfigUpdateFailureReason, const EnvoyException* e) override; private: + bool subscriptionUseDeltaXds() const; + void subscribe(); + void onSubscriptionStreamEvent(uint64_t subscription_id, Config::GrpcMuxStreamEvent event); + void maybeRecreateSubscriptionInDesiredMode(bool transport_closed); + ThreadLocal::SlotPtr tls_; std::string name_; Stats::ScopeSharedPtr scope_; Stats::ScopeSharedPtr stats_scope_; std::unique_ptr subscription_; + Server::Configuration::CommonFactoryContext* context_{nullptr}; + envoy::config::core::v3::ConfigSource desired_config_source_; + envoy::config::core::v3::ConfigSource config_source_; + uint64_t subscription_id_{0}; + uint64_t accepted_subscription_id_{0}; + bool subscription_connected_{false}; static uint64_t instance_id_; PolicyHostsStats stats_; }; diff --git a/cilium/network_policy.cc b/cilium/network_policy.cc index dfea381aa..b2aee1844 100644 --- a/cilium/network_policy.cc +++ b/cilium/network_policy.cc @@ -6,9 +6,10 @@ #include #include +#include +#include #include #include -#include #include #include #include @@ -22,12 +23,15 @@ #include "envoy/config/core/v3/address.pb.h" #include "envoy/config/core/v3/base.pb.h" #include "envoy/config/core/v3/config_source.pb.h" +#include "envoy/config/grpc_mux.h" #include "envoy/config/subscription.h" #include "envoy/event/dispatcher_thread_deletable.h" #include "envoy/http/header_map.h" #include "envoy/init/manager.h" #include "envoy/network/address.h" +#include "envoy/server/config_tracker.h" #include "envoy/server/factory_context.h" +#include "envoy/server/transport_socket_config.h" #include "envoy/ssl/context.h" #include "envoy/ssl/context_config.h" #include "envoy/stats/scope.h" @@ -46,29 +50,48 @@ #include "source/common/network/utility.h" #include "source/common/protobuf/protobuf.h" #include "source/common/protobuf/utility.h" -#include "source/extensions/config_subscription/grpc/grpc_subscription_impl.h" #include "source/server/transport_socket_config_impl.h" +#include "absl/container/btree_map.h" #include "absl/container/btree_set.h" +#include "absl/container/flat_hash_map.h" #include "absl/container/flat_hash_set.h" #include "absl/status/status.h" #include "absl/strings/ascii.h" #include "absl/strings/match.h" #include "absl/strings/str_replace.h" #include "absl/strings/string_view.h" +#include "absl/types/variant.h" #include "cilium/accesslog.h" #include "cilium/api/npds.pb.h" #include "cilium/grpc_subscription.h" #include "cilium/ipcache.h" #include "cilium/secret_watcher.h" +#include "cilium/versioned.h" namespace { static constexpr absl::string_view NetworkPolicyTypeUrl = "type.googleapis.com/cilium.NetworkPolicy"; +static constexpr absl::string_view NetworkPolicyResourceTypeUrl = + "type.googleapis.com/cilium.NetworkPolicyResource"; } // namespace +namespace Envoy { +namespace Cilium { + +// Supported verdict kinds +using RuleVerdict = enum { + None = 0, + Pass = 1, + Allow = 2, + Deny = 3, +}; + +} // namespace Cilium +} // namespace Envoy + namespace fmt { template <> struct formatter { @@ -87,6 +110,9 @@ template <> struct formatter { case Envoy::Cilium::RuleVerdict::Deny: name = "DENY"; break; + case Envoy::Cilium::RuleVerdict::Pass: + name = "PASS"; + break; default: name = "UNKNOWN"; break; @@ -100,7 +126,515 @@ template <> struct formatter { namespace Envoy { namespace Cilium { +// A specific version of a selector used in a policy. Each update yields a new instance. +class SelectorInstance : public VersionedNode, + public absl::flat_hash_set {}; + +// Read-only series of specific selector insteances. +class NamedSelectorReadable : public VersionedReadable { +public: + explicit NamedSelectorReadable(const std::string& name) : name_(name) {} + + const std::string& name() const { return name_; } + +private: + std::string name_; +}; + +// Stable handle on a read-only selector for accessing specific versions of the selector. +using SelectorHandle = std::shared_ptr; + +// Writable series of selector versions for main-thread updates +class NamedSelectorValue : public VersionedValue { +public: + explicit NamedSelectorValue(const std::string& name) + : VersionedValue(name) {} +}; + +// Map of named selectors, keyed with xDS resource name +class SelectorMap : public VersionedMap { +public: + using VersionedMap::VersionedMap; +}; + +class PolicyInstanceImpl; + +// Stable policy map, usually keyed with endpoint IP (IPv4 and IPv6). +using PolicyMapSnapshot = + absl::flat_hash_map>; + +// variant wrapper for supported resource map keys for delta policy updates +// Delta xDS refers to removed resources by resource name, so we must have a map to +// locate the policy/selector to be removed. +class ResourceKey { +public: + struct PolicyResourceEntry { + std::shared_ptr policy; + }; + + struct PolicyEndpointIpEntry {}; + + struct SelectorResourceEntry { + SelectorHandle handle; + }; + + static ResourceKey policyResource(const std::shared_ptr& policy) { + return ResourceKey(PolicyResourceEntry{policy}); + } + + static ResourceKey policyEndpointIp() { return ResourceKey(PolicyEndpointIpEntry{}); } + + static ResourceKey selectorResource(const SelectorHandle& handle) { + return ResourceKey(SelectorResourceEntry{handle}); + } + + const PolicyResourceEntry* policyResourceEntry() const { + return absl::get_if(&value_); + } + + const SelectorResourceEntry* selectorResourceEntry() const { + return absl::get_if(&value_); + } + + bool isPolicyEndpointIpEntry() const { + return absl::holds_alternative(value_); + } + +private: + explicit ResourceKey(const PolicyResourceEntry& value) : value_(value) {} + explicit ResourceKey(const PolicyEndpointIpEntry& value) : value_(value) {} + explicit ResourceKey(const SelectorResourceEntry& value) : value_(value) {} + + absl::variant value_; +}; + +// Map of Delta xDS resources for name collision and duplicate name detection. +class ResourceMap : public absl::flat_hash_map { +public: + using absl::flat_hash_map::flat_hash_map; + + const ResourceKey* findEntry(const std::string& key) const { + auto it = find(key); + return it != end() ? &it->second : nullptr; + } + + std::string findPolicyResourceName(const std::shared_ptr& policy) const; + + void replaceWith(std::vector>&& entries) { + clear(); + reserve(entries.size()); + for (auto& [key, value] : entries) { + insert_or_assign(std::move(key), std::move(value)); + } + } + + void erasePolicyResource(PolicyMapSnapshot& policy_map, const std::string& resource_name, + const std::shared_ptr& policy); +}; + +// ResourceMapOverlay lets delta updates stage tentative resource-map removals and insertions on top +// of the current ResourceMap while validation is still in progress. This preserves transactional +// behavior without copying the full map: failed updates can be discarded cheaply, and successful +// ones are applied to the real map only after the whole update has been accepted. +class ResourceMapOverlay { +public: + ResourceMapOverlay() = default; + explicit ResourceMapOverlay(const ResourceMap& base) : base_(&base) {} + + const ResourceKey* findEntry(const std::string& key) const { + auto upsert_it = upserts_.find(key); + if (upsert_it != upserts_.end()) { + return &upsert_it->second; + } + if (removed_.contains(key)) { + return nullptr; + } + return base_ ? base_->findEntry(key) : nullptr; + } + + std::string findPolicyResourceName(const std::shared_ptr& policy) const; + + std::string describeExistingResourceKey(const std::string& key, + const PolicyMapSnapshot& policy_map) const; + + SelectorHandle getSelectorHandleOrThrow(const std::string& selector) const { + const auto* entry = findEntry(selector); + if (entry == nullptr) { + throw EnvoyException(fmt::format( + "NetworkPolicyResource rule references missing selector resource '{}'", selector)); + } + const auto* selector_entry = entry->selectorResourceEntry(); + if (selector_entry == nullptr || selector_entry->handle == nullptr) { + throw EnvoyException(fmt::format( + "NetworkPolicyResource rule references non-selector resource '{}'", selector)); + } + return selector_entry->handle; + } + + bool emplace(std::string key, ResourceKey value) { + if (findEntry(key)) { + return false; + } + removed_.erase(key); + return upserts_.emplace(std::move(key), std::move(value)).second; + } + + void insertOrAssign(std::string key, ResourceKey value) { + removed_.erase(key); + upserts_.insert_or_assign(std::move(key), std::move(value)); + } + + void erase(const std::string& key) { + upserts_.erase(key); + if (base_ && base_->find(key) != base_->end()) { + removed_.insert(key); + } else { + removed_.erase(key); + } + } + + bool eraseSelectorResourceIfPresent(const std::string& key) { + const auto* entry = findEntry(key); + if (entry == nullptr || entry->selectorResourceEntry() == nullptr) { + return false; + } + erase(key); + return true; + } + + bool erasePolicyResourceIfPresent(PolicyMapSnapshot& policy_map, + const std::string& resource_name) { + const auto* entry = findEntry(resource_name); + if (entry == nullptr) { + return false; + } + const auto* policy_entry = entry->policyResourceEntry(); + if (policy_entry == nullptr) { + return false; + } + erasePolicyResource(policy_map, resource_name, policy_entry->policy); + return true; + } + + void erasePolicyResource(PolicyMapSnapshot& policy_map, const std::string& resource_name, + const std::shared_ptr& policy); + + void applyTo(ResourceMap& map) && { + if (!upserts_.empty()) { + map.reserve(map.size() + upserts_.size()); + } + for (const auto& key : removed_) { + map.erase(key); + } + for (auto& [key, value] : upserts_) { + map.insert_or_assign(std::move(key), std::move(value)); + } + } + +private: + const ResourceMap* base_{}; + absl::flat_hash_set removed_; + absl::flat_hash_map upserts_; +}; + +// helper for validating resource names. +void validateResourceName(absl::string_view resource_name, absl::string_view subject) { + if (resource_name.empty()) { + throw EnvoyException(fmt::format("{} must not be empty", subject)); + } + if (std::ranges::any_of(resource_name, [](unsigned char c) { return absl::ascii_isspace(c); })) { + throw EnvoyException( + fmt::format("{} '{}' must not contain whitespace", subject, resource_name)); + } +} + +// PolicyStreamState is shared by all policies created from one accepted NPDS stream generation. +// Same-stream selector-only updates publish a newer selector version into this object so existing +// policies follow immediately. When the NPDS stream restarts, new policies get a fresh state +// object while old policies keep the old one until the old policy map has quiesced and been +// retired. This allows the new stream to reuse selector resource names so that the xDS server +// need not keep selector resource names in stable storage accross restarts. +class PolicyStreamState { +public: + explicit PolicyStreamState(uint64_t stream_generation, SelectorVersion version = versionMin) + : stream_generation_(stream_generation), version_(version) {} + + uint64_t streamGeneration() const { return stream_generation_; } + + SelectorVersion version() const { return version_.load(std::memory_order_acquire); } + + void publishVersion(SelectorVersion version) { + version_.store(version, std::memory_order_release); + } + +private: + const uint64_t stream_generation_; + std::atomic version_; +}; +using PolicyStreamStateSharedPtr = std::shared_ptr; +using PolicyStreamStateConstSharedPtr = std::shared_ptr; + +class NetworkPolicyMapImpl : public Envoy::Config::SubscriptionCallbacks, + public Envoy::Event::DispatcherThreadDeletable, + public Logger::Loggable, + public std::enable_shared_from_this { +public: + friend class PortNetworkPolicyRule; + NetworkPolicyMapImpl(Server::Configuration::FactoryContext& context, + const envoy::config::core::v3::ConfigSource& config_source); + ~NetworkPolicyMapImpl() override; + + void subscribe(); + + // This is used for testing with a file-based subscription + void subscribe(std::unique_ptr&& subscription) { + subscription_ = std::move(subscription); + config_source_ = desired_config_source_; + subscription_connected_ = false; + } + + const envoy::config::core::v3::ConfigSource& getConfigSource() const { return config_source_; } + + // Config::SubscriptionCallbacks + absl::Status onConfigUpdate(const std::vector& resources, + const std::string& version_info) override; + absl::Status onConfigUpdate(const std::vector& added_resources, + const Protobuf::RepeatedPtrField& removed_resources, + const std::string& system_version_info) override; + void onConfigUpdateFailed(Envoy::Config::ConfigUpdateFailureReason, + const EnvoyException* e) override; + + Server::Configuration::TransportSocketFactoryContext& transportFactoryContext() const { + return *transport_factory_context_; + } + + Regex::Engine& regexEngine() const { return context_.regexEngine(); } + + void tlsWrapperMissingPolicyInc() const { stats_.tls_wrapper_missing_policy_.inc(); } + +protected: + uint64_t streamGeneration() const { return subscription_stream_generation_; } + void resetStreamForTest() { subscription_stream_generation_++; } + + bool useDeltaXds() const { + if (!desired_config_source_.has_api_config_source()) { + return false; + } + const auto& api_type = desired_config_source_.api_config_source().api_type(); + return api_type == envoy::config::core::v3::ApiConfigSource::DELTA_GRPC || + api_type == envoy::config::core::v3::ApiConfigSource::AGGREGATED_DELTA_GRPC; + } + + bool subscriptionUseDeltaXds() const { + if (!config_source_.has_api_config_source()) { + return false; + } + const auto& api_type = config_source_.api_config_source().api_type(); + return api_type == envoy::config::core::v3::ApiConfigSource::DELTA_GRPC || + api_type == envoy::config::core::v3::ApiConfigSource::AGGREGATED_DELTA_GRPC; + } + + void setConfigSource(const envoy::config::core::v3::ConfigSource& config_source) { + desired_config_source_ = config_source; + maybeRecreateSubscriptionInDesiredMode(/*transport_closed=*/false); + } + + // run the given function after all the threads have scheduled + void runAfterAllThreads(std::function cb) const { + // We can guarantee the callback 'cb' runs in the main thread after all worker threads have + // entered their event loop, and thus relinquished all state, such as policy lookup results that + // were stored in their call stack, by posting and empty function to their event queues and + // waiting until all of them have returned, as managed by 'runOnAllWorkerThreads'. + context_.threadLocal().runOnAllWorkerThreads([]() {}, cb); + } + + void reopenIpcache(); + + std::shared_ptr + createOrReusePolicy(const std::string& resource_name, const cilium::NetworkPolicy& config, + const PolicyStreamStateConstSharedPtr& policy_stream_state, + const ResourceMap& resource_map, + const ResourceMapOverlay* pending_resource_map); + + SelectorHandle createOrReuseSelector(const std::string& resource_name, + const cilium::Selector& config, uint64_t update_version); + + void installNewPolicyMap(PolicyMapSnapshot&& new_policy_map, + Init::ManagerImpl& version_init_manager, std::string&& version_name, + const PolicyStreamStateSharedPtr& policy_stream_state); + +private: + void onSubscriptionStreamEvent(uint64_t subscription_id, Config::GrpcMuxStreamEvent event) { + // skip stale notifications for earlier subscriptions + if (subscription_id != subscription_id_) { + return; + } + switch (event) { + case Config::GrpcMuxStreamEvent::Established: + ++subscription_stream_generation_; + subscription_connected_ = true; + break; + case Config::GrpcMuxStreamEvent::Closed: + if (!subscription_connected_) { + return; + } + subscription_connected_ = false; + + // Test code executes synchronously + if (subscription_factory_for_test_) { + maybeRecreateSubscriptionInDesiredMode(/*transport_closed=*/true); + return; + } + + // The close callback runs on the subscription object's own stack, so defer any possible + // recreation until after it unwinds to avoid destroying the current subscription + // mid-callback. + context_.mainThreadDispatcher().post( + [weak_this = weak_from_this(), subscription_id = subscription_id_]() { + if (auto shared_this = weak_this.lock()) { + // skip stale callbacks for earlier subscriptions + if (subscription_id != shared_this->subscription_id_) { + return; + } + shared_this->maybeRecreateSubscriptionInDesiredMode(/*transport_closed=*/true); + } + }); + break; + } + } + + void startSubscription() { + ASSERT(subscription_ != nullptr); + subscription_->start({}); + } + + void maybeRecreateSubscriptionInDesiredMode(bool transport_closed) { + // only ever skip subscribe if we have a subscription already, and it is already connected in + // delta or desired mode, or still connecting in desired mode. + if (subscription_ && (subscription_connected_ || !transport_closed)) { + if (subscription_connected_ && subscriptionUseDeltaXds()) { + // Keep delta on a connected subscription until transport closes. + return; + } + if (Protobuf::util::MessageDifferencer::Equals(config_source_, desired_config_source_)) { + // Let the current subscription keep going when it is already in the desired mode. + return; + } + } + + // Recreate the subscription in the latest desired mode. + subscribe(); + } + + // Helpers for atomic swap of the policy map pointer. + // + // store() is only used for the initialization of the map during construction. + // exchange() is used to atomically swap in a new map, the old map pointer is returned. + // Once a map is stored or swapped in to the atomic pointer by the main thread, it may be "loaded" + // from the atomic pointer by any thread. This is why the load returns a const pointer. + // + // For the loaded pointer to be safe to use, we must use acquire/release memory ordering: + // - when a pointer stored or swapped in, 'std::memory_order_release' informs the compiler to make + // sure it is not reordering any write operations into the map to happen after the pointer is + // written, and emits CPU instructions to also make the CPU out-of-order-execution logic to not + // reorder any write operations to happen after the pointer itself is written. This guarantees + // that the map is not modified after the point when the worker threads can observe the new + // pointer value, i.e., the map is actaully immutable (const) from that point forward. + // - when the pointer is read (by a worker thread) 'std::memory_order_acquire' in the load + // operation informs the compiler to emit CPU instructions to make the CPU + // out-of-order-execution logic to not reorder any reads from the new map to happen before the + // pointer itself is read, so that no values from the map are read before the map was "released" + // by the store or exchange operation. + // + // Typically it is easier to think about the release part of the acquire/release semantics, as at + // the point of the store or exchange operation the compiler and the CPU know the location of the + // map in memory before and after the pointer is stored, so that without + // 'std::memory_order_release' there is an understandable risk of such write after release + // happening. On the acquire side it seems less likely that the compiler or the CPU could know the + // new map pointer value in advance and even try to reorder any read operations to happen before + // the pointer is actually read. But consider the typical case where the pointer value is actually + // not changing between consecutice load operations. The compiler or the CPU could speculate that + // to be the case and read some values from the old memory location. 'std::memory_order_acquire' + // tells the compiler (which then "tells" the CPU) that this can not be done, and all reads must + // actually happen after the pointer value is loaded, be it a new one or the same as before. + // + const PolicyMapSnapshot* load() const { return map_ptr_.load(std::memory_order_acquire); } + void store(const PolicyMapSnapshot* map) { map_ptr_.store(map, std::memory_order_release); } + const PolicyMapSnapshot* exchange(const PolicyMapSnapshot* map) { + return map_ptr_.exchange(map, std::memory_order_release); + } + + const PolicyInstance* getPolicyInstanceImpl(const std::string& endpoint_policy_name) const; + PolicyInstanceConstSharedPtr + getPolicyInstanceSharedImpl(const std::string& endpoint_policy_name) const; + uint64_t policySelectorStreamGenerationForTestImpl(const PolicyInstance& policy) const; + SelectorVersion policySelectorVersionForTestImpl(const PolicyInstance& policy) const; + void removeInitManager(); + + static uint64_t instance_id_; + + bool subscription_connected_{false}; + uint64_t subscription_id_{0}; + bool subscription_should_start_{false}; + + void scheduleSelectorDeferredDeletion(DeferredDeletion&& deferred); + void scheduleSelectorGCAndDeferredDeletion(uint64_t published_version, + const PolicyMapSnapshot* old_policy_map = nullptr); + void startManagedSubscriptionForTest() { + subscription_should_start_ = true; + subscribe(); + } + void setSubscriptionFactoryForTest(NetworkPolicyMap::SubscriptionFactoryForTest factory) { + subscription_factory_for_test_ = std::move(factory); + } + void onSubscriptionConnectedForTest() { + onSubscriptionStreamEvent(subscription_id_, Config::GrpcMuxStreamEvent::Established); + } + void onSubscriptionTransportCloseForTest() { + onSubscriptionStreamEvent(subscription_id_, Config::GrpcMuxStreamEvent::Closed); + } + bool subscriptionConnectedForTest() const { return subscription_connected_; } + + Server::Configuration::ServerFactoryContext& context_; + + std::atomic map_ptr_; + SelectorMap selector_map_; + // Policies hold a shared per-stream state object. A freshly installed stream stores its actual + // gRPC stream generation here, so same-stream selector-only updates advance existing policies + // immediately while old policies remain pinned to the latest selector version reached by their + // own stream. + PolicyStreamStateSharedPtr policy_stream_state_{std::make_shared(0)}; + ResourceMap resource_map_; + Stats::ScopeSharedPtr npds_stats_scope_; + Stats::ScopeSharedPtr policy_stats_scope_; + + envoy::config::core::v3::ConfigSource desired_config_source_; + envoy::config::core::v3::ConfigSource config_source_; + + // init target which starts gRPC subscription + Init::TargetImpl init_target_; + std::shared_ptr + transport_factory_context_; + // Between policy updates, keep a dormant init manager installed so unexpected late init-target + // registrations do not hit the listener's already-initialized manager. If it accumulates targets + // while parked, log and rotate it out before making it active again. + std::unique_ptr parked_init_manager_; + + std::unique_ptr subscription_; + static uint64_t subscription_stream_generation_; + NetworkPolicyMap::SubscriptionFactoryForTest subscription_factory_for_test_; + + ProtobufTypes::MessagePtr dumpNetworkPolicyConfigs(const Matchers::StringMatcher& name_matcher); + Server::ConfigTracker::EntryOwnerPtr config_tracker_entry_; + +protected: + friend class NetworkPolicyMap; + + PolicyStats stats_; +}; + uint64_t NetworkPolicyMapImpl::instance_id_ = 0; +uint64_t NetworkPolicyMapImpl::subscription_stream_generation_ = 1; IpAddressPair::IpAddressPair(const cilium::NetworkPolicy& proto) { for (const auto& ip_addr : proto.endpoint_ips()) { @@ -124,7 +658,8 @@ class HeaderMatch : public Logger::Loggable { : name_(config.name()), value_(config.value()), match_action_(config.match_action()), mismatch_action_(config.mismatch_action()) { if (!config.value_sds_secret().empty()) { - secret_ = std::make_unique(parent, config.value_sds_secret()); + secret_ = std::make_unique( + parent.transportFactoryContext(), parent.getConfigSource(), config.value_sds_secret()); } } @@ -459,34 +994,56 @@ SniPattern::SniPattern(const Regex::Engine& engine, absl::string_view sni) { class PortNetworkPolicyRule : public Logger::Loggable { public: PortNetworkPolicyRule() - : name_("default allow rule"), deny_(false), proxy_id_(0), precedence_(0), - tier_last_precedence_(0), mutable_remotes_(false), l7_proto_("") {} + : name_("default allow rule"), verdict_(RuleVerdict::Allow), proxy_id_(0), precedence_(0), + tier_last_precedence_(0), pass_index_(0), l7_proto_("") {} PortNetworkPolicyRule(const NetworkPolicyMapImpl& parent, - const cilium::PortNetworkPolicyRule& rule, bool shared_resource) - : name_(rule.name()), deny_(rule.deny()), proxy_id_(uint16_t(rule.proxy_id())), - precedence_(rule.precedence()), tier_last_precedence_(rule.pass_precedence()), - mutable_remotes_(!shared_resource), l7_proto_(rule.l7_proto()) { + const cilium::PortNetworkPolicyRule& rule, + const ResourceMapOverlay* resource_map) + : name_(rule.name()), + verdict_(rule.pass_precedence() ? RuleVerdict::Pass + : (rule.deny() ? RuleVerdict::Deny : RuleVerdict::Allow)), + proxy_id_(uint16_t(rule.proxy_id())), precedence_(rule.precedence()), + tier_last_precedence_(rule.pass_precedence()), pass_index_(0), l7_proto_(rule.l7_proto()) { if (tier_last_precedence_ > precedence_) { throw EnvoyException( fmt::format("PortNetworkPolicyRule: pass_precedence {} must be lower than precedence {}", tier_last_precedence_, precedence_)); } - for (const auto& remote : rule.remote_policies()) { - ENVOY_LOG(trace, "Cilium L7 PortNetworkPolicyRule(): {} remote {} by rule: {}", - deny_ ? "Denying" : "Allowing", remote, name_); - remotes_.emplace(remote); + if (resource_map) { + if (rule.remote_policies_size()) { + throw EnvoyException( + "NetworkPolicyResource rule must use selectors instead of remote_policies"); + } + selectors_.reserve(rule.selectors_size()); + for (const auto& selector : rule.selectors()) { + ENVOY_LOG(trace, "Cilium L7 PortNetworkPolicyRule(): {} selector {} by rule: {}", verdict_, + selector, name_); + selectors_.emplace_back(resource_map->getSelectorHandleOrThrow(selector)); + } + } else { + if (rule.selectors_size()) { + throw EnvoyException("NetworkPolicy rule must not use selectors"); + } + for (const auto remote : rule.remote_policies()) { + ENVOY_LOG(trace, "Cilium L7 PortNetworkPolicyRule(): {} remote {} by rule: {}", verdict_, + remote, name_); + remotes_.emplace(remote); + } } if (rule.has_downstream_tls_context()) { auto config = rule.downstream_tls_context(); - server_context_ = std::make_unique(parent, config); + server_context_ = std::make_unique(parent.transportFactoryContext(), + parent.getConfigSource(), config); } if (rule.has_upstream_tls_context()) { auto config = rule.upstream_tls_context(); - client_context_ = std::make_unique(parent, config); + client_context_ = std::make_unique(parent.transportFactoryContext(), + parent.getConfigSource(), config); } for (const auto& sni : rule.server_names()) { - ENVOY_LOG(trace, "Cilium L7 PortNetworkPolicyRule(): Allowing SNI {} by rule {}", sni, name_); + ENVOY_LOG(trace, "Cilium L7 PortNetworkPolicyRule(): {} SNI {} by rule {}", verdict_, sni, + name_); allowed_snis_.emplace_back(parent.regexEngine(), sni); } if (rule.has_http_rules()) { @@ -509,29 +1066,41 @@ class PortNetworkPolicyRule : public Logger::Loggable { } } - // inheritpassprecedence bumps up the precedence of a rule in a lower tier to the precedence - // range reserved right after the precedence of the given pass rule. - void inheritPassPrecedence(const PortNetworkPolicyRule& pass_rule) { - precedence_ -= pass_rule.tier_last_precedence_; - precedence_ += pass_rule.precedence_; - } + bool isRemoteWildcard() const { return remotes_.empty() && selectors_.empty(); } + + bool matchesRemoteId(uint32_t remote_id, const SelectorVersion selector_version) const { + if (isRemoteWildcard()) { + return true; + } + if (!remotes_.empty()) { + return remotes_.contains(remote_id); + } - bool isRemoteWildcard() const { return remotes_.empty(); } + for (const auto& selector : selectors_) { + const auto resolved_selector = selector->get(selector_version); + if (resolved_selector && resolved_selector->contains(remote_id)) { + return true; + } + } + return false; + } - RuleVerdict getVerdict(uint16_t proxy_id, uint32_t remote_id) const { + RuleVerdict getVerdict(uint16_t proxy_id, uint32_t remote_id, + const SelectorVersion selector_version) const { // proxy_id must match if we have any. - if (proxy_id_ != 0 && proxy_id != proxy_id_) { + if (proxy_id_ && proxy_id != proxy_id_) { return RuleVerdict::None; } // Remote ID must match if we have any. - if (!isRemoteWildcard() && !remotes_.contains(remote_id)) { + if (!matchesRemoteId(remote_id, selector_version)) { return RuleVerdict::None; // no verdict } - // Allow rules allow by default when remotes_ is empty, deny rules do not - return deny_ ? RuleVerdict::Deny : RuleVerdict::Allow; + ASSERT(verdict_ != RuleVerdict::None, "rule must have a verdict"); + return verdict_; } - RuleVerdict getVerdict(uint16_t proxy_id, uint32_t remote_id, absl::string_view sni) const { + RuleVerdict getVerdict(uint16_t proxy_id, uint32_t remote_id, absl::string_view sni, + const SelectorVersion selector_version) const { // sni must match if we have any if (!allowed_snis_.empty() && (sni.empty() || std::ranges::none_of(allowed_snis_, [&](const auto& pattern) { @@ -539,13 +1108,14 @@ class PortNetworkPolicyRule : public Logger::Loggable { }))) { return RuleVerdict::None; } - return getVerdict(proxy_id, remote_id); + return getVerdict(proxy_id, remote_id, selector_version); } RuleVerdict getVerdict(uint16_t proxy_id, uint32_t remote_id, Envoy::Http::RequestHeaderMap& headers, - Cilium::AccessLog::Entry& log_entry) const { - auto verdict = getVerdict(proxy_id, remote_id); + Cilium::AccessLog::Entry& log_entry, + const SelectorVersion selector_version) const { + auto verdict = getVerdict(proxy_id, remote_id, selector_version); if (!hasHttpRules() || verdict != RuleVerdict::Allow) { return verdict; } @@ -569,8 +1139,9 @@ class PortNetworkPolicyRule : public Logger::Loggable { return (header_matched) ? RuleVerdict::Allow : RuleVerdict::None; } - RuleVerdict useProxylib(uint16_t proxy_id, uint32_t remote_id, std::string& l7_proto) const { - auto verdict = getVerdict(proxy_id, remote_id); + RuleVerdict useProxylib(uint16_t proxy_id, uint32_t remote_id, std::string& l7_proto, + const SelectorVersion selector_version) const { + auto verdict = getVerdict(proxy_id, remote_id, selector_version); if (verdict != RuleVerdict::Allow) { return verdict; } @@ -585,8 +1156,9 @@ class PortNetworkPolicyRule : public Logger::Loggable { // Envoy Metadata matcher, called after deny has already been checked for RuleVerdict getVerdict(uint16_t proxy_id, uint32_t remote_id, - const envoy::config::core::v3::Metadata& metadata) const { - auto verdict = getVerdict(proxy_id, remote_id); + const envoy::config::core::v3::Metadata& metadata, + const SelectorVersion selector_version) const { + auto verdict = getVerdict(proxy_id, remote_id, selector_version); if (verdict != RuleVerdict::Allow) { return verdict; } @@ -637,14 +1209,24 @@ class PortNetworkPolicyRule : public Logger::Loggable { } void toString(int indent, std::string& res) const { - res.append(indent - 2, ' ').append("- remotes: ["); - res.append(fmt::format("{}", fmt::join(remotes_, ","))); + if (!selectors_.empty()) { + res.append(indent - 2, ' ').append("- selectors: ["); + std::vector quoted_selectors; + quoted_selectors.reserve(selectors_.size()); + for (const auto& selector : selectors_) { + quoted_selectors.emplace_back(fmt::format("\"{}\"", selector->name())); + } + res.append(fmt::format("{}", fmt::join(quoted_selectors, ","))); + } else { + res.append(indent - 2, ' ').append("- remotes: ["); + res.append(fmt::format("{}", fmt::join(remotes_, ","))); + } res.append("]\n"); if (!name_.empty()) { res.append(indent, ' ').append("name: \"").append(name_).append("\"\n"); } - if (deny_) { + if (verdict_ == RuleVerdict::Deny) { res.append(indent, ' ').append("deny: true\n"); } if (precedence_) { @@ -654,7 +1236,7 @@ class PortNetworkPolicyRule : public Logger::Loggable { res.append(indent, ' ') .append(fmt::format("tier_last_precedence: {}\n", tier_last_precedence_)); } - if (proxy_id_ != 0) { + if (proxy_id_) { res.append(indent, ' ').append(fmt::format("proxy_id: {}\n", proxy_id_)); } @@ -700,12 +1282,13 @@ class PortNetworkPolicyRule : public Logger::Loggable { DownstreamTLSContextSharedPtr server_context_; UpstreamTLSContextSharedPtr client_context_; bool has_headermatches_{false}; - bool deny_; - uint16_t proxy_id_; + const RuleVerdict verdict_; + const uint16_t proxy_id_; uint32_t precedence_; - uint32_t tier_last_precedence_; + const uint32_t tier_last_precedence_; + uint32_t pass_index_; absl::btree_set remotes_; - bool mutable_remotes_; + std::vector selectors_; std::vector allowed_snis_; // All SNIs allowed if empty. std::shared_ptr> @@ -748,7 +1331,7 @@ class PortNetworkPolicyRules : public Logger::Loggable { if (rule->has_headermatches_) { can_short_circuit_ = false; } - if (rule->tier_last_precedence_ != 0) { + if (rule->tier_last_precedence_) { has_pass_rules_ = true; } } @@ -757,16 +1340,17 @@ class PortNetworkPolicyRules : public Logger::Loggable { // append merges 'rules' to 'rules_' by placing the new 'rules' to the end of 'rules_'. // First call marks 'rules_' as initialized. Of further calls, if either is empty, - // we must add a default allow rule to retain the semantics of an empty rules. + // we must add a default allow rule to retain the semantics of empty rules. void append(const NetworkPolicyMapImpl& parent, const Protobuf::RepeatedPtrField& rules, - bool shared_resource) { + const ResourceMapOverlay* selector_resource_map) { if (initialized_ && rules.empty() != rules_.empty()) { // add an explicit allow-all rule to keep the combined semantics addDefaultAllowRule(); } for (const auto& it : rules) { - rules_.emplace_back(std::make_shared(parent, it, shared_resource)); + rules_.emplace_back( + std::make_shared(parent, it, selector_resource_map)); updateFor(rules_.back()); } initialized_ = true; @@ -777,47 +1361,81 @@ class PortNetworkPolicyRules : public Logger::Loggable { // we must add a default allow rule to retain the semantics of an empty rules. void prepend(const NetworkPolicyMapImpl& parent, const Protobuf::RepeatedPtrField& rules, - bool shared_resource) { + const ResourceMapOverlay* resource_map) { if (initialized_ && rules.empty() != rules_.empty()) { // add an explicit allow-all rule to keep the combined semantics rules_.emplace(rules_.begin(), std::make_shared()); } for (const auto& it : rules) { rules_.emplace(rules_.begin(), - std::make_shared(parent, it, shared_resource)); + std::make_shared(parent, it, resource_map)); updateFor(rules_.front()); } initialized_ = true; } - // appendNonPassRules merges non-pass rules from 'rules' to 'rules_' by placing the new rules to - // the end of 'rules_'. First call marks 'rules_' as initialized. Of further calls, if either is - // empty, we must add a default allow rule to retain the semantics of an empty rules. - void appendNonPassRules(const std::vector& rules) { + // appendRules merges all rules from 'rules' to the end of 'rules_'. + // First call marks 'rules_' as initialized. Of further calls, if either is empty, + // we must add a default allow rule to retain the semantics of the combined rules. + void appendRules(const std::vector& rules) { if (initialized_ && rules.empty() != rules_.empty()) { - // add an explicit allow-all rule to keep the combined semantics addDefaultAllowRule(); } for (auto& rule : rules) { - if (rule->tier_last_precedence_ == 0) { - rules_.insert(rules_.end(), rule); - updateFor(rule); - } + rules_.insert(rules_.end(), rule); + updateFor(rule); } initialized_ = true; } - // sort by descending precedence, retaining the original order within each precedence level + // Sort by descending precedence. Within the same precedence, deny rules come first, + // then allow rules, and pass rules last. This lets runtime pass handling jump + // immediately, as any same-precedence allow/deny verdict has already been seen. void sort() { - // sortRules(rules_); std::stable_sort(rules_.begin(), rules_.end(), [](const PortNetworkPolicyRuleConstSharedPtr& a, const PortNetworkPolicyRuleConstSharedPtr& b) { return (a->precedence_ > b->precedence_) || - (a->precedence_ == b->precedence_ && (a->deny_ && !b->deny_)); + (a->precedence_ == b->precedence_ && a->verdict_ > b->verdict_); }); } + void prepareRuntimePasses() { + if (!has_pass_rules_) { + return; + } + + uint32_t pass_precedence = 0; + for (uint32_t idx = 0; idx < rules_.size(); idx++) { + if (rules_[idx]->tier_last_precedence_ == 0) { + continue; + } + + if (rules_[idx].use_count() > 1) { + // Pass continuation index is specific to this ordered rule set, so shared pass + // rules must be cloned before storing the computed continuation on the rule. + rules_[idx] = std::make_shared(*rules_[idx]); + } + auto& rule = const_cast(*rules_[idx]); + + if (pass_precedence && rule.precedence_ < pass_precedence) { + pass_precedence = 0; + } + + if (pass_precedence && rule.tier_last_precedence_ != pass_precedence) { + throw EnvoyException(fmt::format("PortNetworkPolicy: Inconsistent pass precedence {} != {}", + rule.tier_last_precedence_, pass_precedence)); + } + pass_precedence = rule.tier_last_precedence_; + + uint32_t pass_index = idx + 1; + while (pass_index < rules_.size() && rules_[pass_index]->precedence_ >= pass_precedence) { + pass_index++; + } + rule.pass_index_ = pass_index; + } + } + bool empty() const { return rules_.empty(); } template RuleVerdict forEachRule(bool can_short_circuit, F&& func) const { @@ -835,21 +1453,30 @@ class PortNetworkPolicyRules : public Logger::Loggable { return RuleVerdict::Allow; } - for (const auto& rule : rules_) { + for (uint32_t idx = 0; idx < rules_.size();) { + const auto& rule = rules_[idx]; + // lower precedence rules are skipped if there is a verdict if (verdict != RuleVerdict::None && rule->precedence_ < verdict_precedence) { break; } auto rule_verdict = func(*rule); - if (rule_verdict != RuleVerdict::None) { + if (rule_verdict == RuleVerdict::Pass) { + ASSERT(rule->pass_index_, "matching pass rule must have a continuation index"); + if (verdict == RuleVerdict::None || verdict_precedence < rule->precedence_) { + idx = rule->pass_index_; + continue; + } + } else if (rule_verdict != RuleVerdict::None) { verdict = rule_verdict; verdict_precedence = rule->precedence_; // Short-circuit on the first deny or on first allow if no rules have HeaderMatches if (rule_verdict == RuleVerdict::Deny || can_short_circuit) { - break; + return verdict; } } + idx++; } return verdict; } @@ -875,9 +1502,18 @@ class PortNetworkPolicyRules : public Logger::Loggable { return RuleVerdict::Allow; } - for (const auto& rule : rules_) { + for (uint32_t idx = 0; idx < rules_.size();) { + const auto& rule = rules_[idx]; + auto rule_verdict = get_verdict(*rule); switch (rule_verdict) { + case RuleVerdict::Pass: + ASSERT(rule->pass_index_, "matching pass rule must have a continuation index"); + if (verdict == RuleVerdict::None || verdict_precedence < rule->precedence_) { + idx = rule->pass_index_; + continue; + } + break; case RuleVerdict::Deny: // return higher precedence allow verdict if any. if (verdict != RuleVerdict::None && verdict_precedence > rule->precedence_) { @@ -898,15 +1534,17 @@ class PortNetworkPolicyRules : public Logger::Loggable { case RuleVerdict::None: break; } + idx++; } return verdict; } RuleVerdict getVerdict(uint16_t proxy_id, uint32_t remote_id, Envoy::Http::RequestHeaderMap& headers, - Cilium::AccessLog::Entry& log_entry) const { + Cilium::AccessLog::Entry& log_entry, + const SelectorVersion selector_version) const { auto verdict = forEachRule(can_short_circuit_, [&](const auto& rule) { - return rule.getVerdict(proxy_id, remote_id, headers, log_entry); + return rule.getVerdict(proxy_id, remote_id, headers, log_entry, selector_version); }); ENVOY_LOG(trace, @@ -915,24 +1553,30 @@ class PortNetworkPolicyRules : public Logger::Loggable { return verdict; } - RuleVerdict getVerdict(uint16_t proxy_id, uint32_t remote_id, absl::string_view sni) const { - auto verdict = forEachRule( - true, [&](const auto& rule) { return rule.getVerdict(proxy_id, remote_id, sni); }); + RuleVerdict getVerdict(uint16_t proxy_id, uint32_t remote_id, absl::string_view sni, + const SelectorVersion selector_version) const { + auto verdict = forEachRule(true, [&](const auto& rule) { + return rule.getVerdict(proxy_id, remote_id, sni, selector_version); + }); ENVOY_LOG(trace, "Cilium L7 PortNetworkPolicyRules(proxy_id: {}, remote_id: {}, sni: {}): {}", proxy_id, remote_id, sni, verdict); return verdict; } - RuleVerdict useProxylib(uint16_t proxy_id, uint32_t remote_id, std::string& l7_proto) const { - return forEachRule( - true, [&](const auto& rule) { return rule.useProxylib(proxy_id, remote_id, l7_proto); }); + RuleVerdict useProxylib(uint16_t proxy_id, uint32_t remote_id, std::string& l7_proto, + const SelectorVersion selector_version) const { + return forEachRule(true, [&](const auto& rule) { + return rule.useProxylib(proxy_id, remote_id, l7_proto, selector_version); + }); } RuleVerdict getVerdict(uint16_t proxy_id, uint32_t remote_id, - const envoy::config::core::v3::Metadata& metadata) const { - auto verdict = forEachRule( - true, [&](const auto& rule) { return rule.getVerdict(proxy_id, remote_id, metadata); }); + const envoy::config::core::v3::Metadata& metadata, + const SelectorVersion selector_version) const { + auto verdict = forEachRule(true, [&](const auto& rule) { + return rule.getVerdict(proxy_id, remote_id, metadata, selector_version); + }); ENVOY_LOG(trace, "Cilium L7 PortNetworkPolicyRules(proxy_id: {}, remote_id: {}, metadata: {}): {}", @@ -942,26 +1586,30 @@ class PortNetworkPolicyRules : public Logger::Loggable { } RuleVerdict getServerTlsContext(uint16_t proxy_id, uint32_t remote_id, absl::string_view sni, - Ssl::ContextSharedPtr& tls_ctx, - const Ssl::ContextConfig*& config) const { + Ssl::ContextSharedPtr& tls_ctx, const Ssl::ContextConfig*& config, + const SelectorVersion selector_version) const { tls_ctx = nullptr; return forEachRulePred( - [&](const auto& rule) { return rule.getVerdict(proxy_id, remote_id, sni); }, + [&](const auto& rule) { + return rule.getVerdict(proxy_id, remote_id, sni, selector_version); + }, [&](const auto& rule) { return rule.getServerTlsContext(tls_ctx, config); }); } RuleVerdict getClientTlsContext(uint16_t proxy_id, uint32_t remote_id, absl::string_view sni, - Ssl::ContextSharedPtr& tls_ctx, - const Ssl::ContextConfig*& config) const { + Ssl::ContextSharedPtr& tls_ctx, const Ssl::ContextConfig*& config, + const SelectorVersion selector_version) const { tls_ctx = nullptr; return forEachRulePred( - [&](const auto& rule) { return rule.getVerdict(proxy_id, remote_id, sni); }, + [&](const auto& rule) { + return rule.getVerdict(proxy_id, remote_id, sni, selector_version); + }, [&](const auto& rule) { return rule.getClientTlsContext(tls_ctx, config); }); } void toString(int indent, std::string& res) const { res.append(indent - 2, ' ').append("- rules:\n"); - for (auto& rule : rules_) { + for (const auto& rule : rules_) { rule->toString(indent + 2, res); } if (!can_short_circuit_) { @@ -978,6 +1626,11 @@ class PortNetworkPolicyRules : public Logger::Loggable { return false; } + bool hasOnlyPassRules() const { + return !rules_.empty() && + std::ranges::all_of(rules_, [](const auto& rule) { return rule->pass_index_ != 0; }); + } + // ordered set of rules as a sorted vector std::vector rules_; // Allowed if empty. bool can_short_circuit_{true}; @@ -985,9 +1638,33 @@ class PortNetworkPolicyRules : public Logger::Loggable { bool initialized_{false}; }; +// PortRangeCompare is used for as std::less replacement for port range keys. +// +// All port ranges in the map have non-overlapping keys, which allows total ordering needed for +// ordered map containers. When inserting new ranges, any range overlap will be flagged as a +// "duplicate" entry, as overlapping keys are considered equal (as neither is strictly less than the +// other given this comparison predicate). +// On lookups we'll set both ends of the port range to the same port number, which will find the one +// range that it overlaps with, if one exists. +using PortRange = std::pair; +struct PortRangeCompare { + bool operator()(const PortRange& a, const PortRange& b) const { + // return true if range 'a.first - a.second' is below range 'b.first - b.second'. + return a.second < b.first; + } +}; + +// PolicySnapshot is keyed by port ranges, and contains a list of PortNetworkPolicyRules's +// applicable to this range. A list is needed as rules may come from multiple sources (e.g., +// resulting from use of named ports and numbered ports in Cilium NetworkPolicy at the same time). +class PolicySnapshot : public absl::btree_map { +public: + using absl::btree_map::btree_map; +}; + namespace { -const PortNetworkPolicyRules* findPortRules(const PolicyMap& map, uint16_t port) { +const PortNetworkPolicyRules* findPortRules(const PolicySnapshot& map, uint16_t port) { // Look up with an exact port first, then fall back to the wildcard port (0). If policy is found // with the exact port, then the returned policy also contains all the wildcard port rules, so we // do not need to perform a separate wildcard port policy lookup. If no policy is defined for the @@ -1007,13 +1684,14 @@ const PortNetworkPolicyRules* findPortRules(const PolicyMap& map, uint16_t port) } // namespace -PortPolicy::PortPolicy(const PolicyMap& map, uint16_t port) - : map_(map), port_rules_(findPortRules(map_, port)), - has_http_rules_(port_rules_ && port_rules_->hasHttpRules()) {} +PortPolicy::PortPolicy(const PolicySnapshot& map, uint16_t port, SelectorVersion selector_version) + : port_rules_(findPortRules(map, port)), + has_http_rules_(port_rules_ && port_rules_->hasHttpRules()), + selector_version_(selector_version) {} bool PortPolicy::useProxylib(uint16_t proxy_id, uint32_t remote_id, std::string& l7_proto) const { if (port_rules_) { - auto verdict = port_rules_->useProxylib(proxy_id, remote_id, l7_proto); + auto verdict = port_rules_->useProxylib(proxy_id, remote_id, l7_proto, selector_version_); if (verdict == RuleVerdict::Allow) { return true; } @@ -1033,14 +1711,15 @@ bool PortPolicy::allowed(uint16_t proxy_id, uint32_t remote_id, if (!port_rules_) { return false; } - return port_rules_->getVerdict(proxy_id, remote_id, headers, log_entry) == RuleVerdict::Allow; + return port_rules_->getVerdict(proxy_id, remote_id, headers, log_entry, selector_version_) == + RuleVerdict::Allow; } bool PortPolicy::allowed(uint16_t proxy_id, uint32_t remote_id, absl::string_view sni) const { if (!port_rules_) { return false; } - return port_rules_->getVerdict(proxy_id, remote_id, sni) == RuleVerdict::Allow; + return port_rules_->getVerdict(proxy_id, remote_id, sni, selector_version_) == RuleVerdict::Allow; } bool PortPolicy::allowed(uint16_t proxy_id, uint32_t remote_id, @@ -1048,7 +1727,8 @@ bool PortPolicy::allowed(uint16_t proxy_id, uint32_t remote_id, if (!port_rules_) { return false; } - return port_rules_->getVerdict(proxy_id, remote_id, metadata) == RuleVerdict::Allow; + return port_rules_->getVerdict(proxy_id, remote_id, metadata, selector_version_) == + RuleVerdict::Allow; } Ssl::ContextSharedPtr PortPolicy::getServerTlsContext(uint16_t proxy_id, uint32_t remote_id, @@ -1060,7 +1740,8 @@ Ssl::ContextSharedPtr PortPolicy::getServerTlsContext(uint16_t proxy_id, uint32_ config = nullptr; raw_socket_allowed = false; if (port_rules_) { - auto verdict = port_rules_->getServerTlsContext(proxy_id, remote_id, sni, tls_ctx, config); + auto verdict = port_rules_->getServerTlsContext(proxy_id, remote_id, sni, tls_ctx, config, + selector_version_); raw_socket_allowed = verdict == RuleVerdict::Allow && tls_ctx == nullptr && config == nullptr; } return tls_ctx; @@ -1075,7 +1756,8 @@ Ssl::ContextSharedPtr PortPolicy::getClientTlsContext(uint16_t proxy_id, uint32_ config = nullptr; raw_socket_allowed = false; if (port_rules_) { - auto verdict = port_rules_->getClientTlsContext(proxy_id, remote_id, sni, tls_ctx, config); + auto verdict = port_rules_->getClientTlsContext(proxy_id, remote_id, sni, tls_ctx, config, + selector_version_); raw_socket_allowed = verdict == RuleVerdict::Allow && tls_ctx == nullptr && config == nullptr; } return tls_ctx; @@ -1087,467 +1769,27 @@ bool inline rangesOverlap(const PortRange& a, const PortRange& b) { // !(a.second < b.first || a.first > b.second) return a.second >= b.first && a.first <= b.second; } - -template -absl::btree_set intersection(const absl::btree_set& a, const absl::btree_set& b) { - absl::btree_set result; - std::set_intersection(a.begin(), a.end(), b.begin(), b.end(), - std::inserter(result, result.begin())); - return result; -} } // namespace -// ShadowedRemotes maintains state for shadowed remote identities within a tier. When a higher -// precedence rule has a verdict for a given remote identity, that identity becomes "shadowed" and -// is removed from the set of remote identities of the remaining rules of the tier. For pass -// verdicts this shadowing is immediate, for allow/deny verdicts the shadowing takes place for the -// next precedence level, so that rules on the same precedence level do not shadow each other. -class ShadowedRemotes { +class PortNetworkPolicy : public Logger::Loggable { public: - // reset is used to re-initialize state for a new port range - void reset(uint32_t first_precedence) { - shadowed_pass_remotes_.clear(); - shadowed_nonpass_remotes_.clear(); - current_precedence_nonpass_remotes_.clear(); - shadow_all_lower_precedence_ = false; - current_precedence_has_wildcard_ = false; - previous_precedence_ = first_precedence; - } - - // resetForNewTier is used to re-initialize state for each new tier - void resetForNewTier(uint32_t first_precedence) { - shadowed_pass_remotes_.clear(); - // shadowed_nonpass_remotes_ are kept for the new tier - // shadow_all_lower_precedence_ is not reset for the new tier - current_precedence_nonpass_remotes_.clear(); - // current_precedence_has_wildcard_ = false; - previous_precedence_ = first_precedence; - } - - // shadowRemotes marks 'remotes' as shadowed and returns 'true' if any of them were not already - // shadowed. - bool shadowPassRemotes(const absl::btree_set& pass_remotes) { - bool any_unshadowed = false; - for (auto remote : pass_remotes) { - if (!shadowed_pass_remotes_.contains(remote)) { - if (!shadowed_nonpass_remotes_.contains(remote)) { - any_unshadowed = true; - } - shadowed_pass_remotes_.insert(remote); - } - } - return any_unshadowed; - } - - // filterShadowPassRemotes filters out any already shadowed remotes from the pass rule and marks - // the remaining remotes as shadowed. Returns 'true' if any remotes remain. - bool filterShadowPassRemotes(PortNetworkPolicyRule& rule) { - absl::erase_if(rule.remotes_, [&](const auto& x) { - return shadowed_pass_remotes_.contains(x) || shadowed_nonpass_remotes_.contains(x); - }); - if (rule.remotes_.empty()) { - return false; - } - shadowed_pass_remotes_.insert(rule.remotes_.begin(), rule.remotes_.end()); - return true; - } - - // shadowRule collects the remotes from 'rule' for shadowing once the first rule on the next - // (lower) precedence level is processed. No shadowing between rules on the same precedence - // level. Returns 'true' if the rule itself should be skipped. - bool shadowNonpassRule(PortNetworkPolicyRule& rule) { - // Same-precedence allow/deny rules do not shadow each other. - // Only after leaving a precedence level do its verdict identities shadow - // lower-precedence rules. - if (rule.precedence_ != previous_precedence_) { - shadowed_nonpass_remotes_.insert(current_precedence_nonpass_remotes_.begin(), - current_precedence_nonpass_remotes_.end()); - current_precedence_nonpass_remotes_.clear(); - if (current_precedence_has_wildcard_) { - shadow_all_lower_precedence_ = true; - } - // current_precedence_has_wildcard_ = false; - previous_precedence_ = rule.precedence_; - } - - if (shadow_all_lower_precedence_) { - return true; - } - - if (rule.isRemoteWildcard()) { - current_precedence_has_wildcard_ = true; - } else { - if (rule.mutable_remotes_) { - // Check if this rule is (partially) shadowed by nonpass rules on any higher tier. - if (!shadowed_nonpass_remotes_.empty()) { - absl::erase_if(rule.remotes_, - [&](const auto& x) { return shadowed_nonpass_remotes_.contains(x); }); - if (rule.remotes_.empty()) { - return true; - } - } - - // Check if this rule is (partially) shadowed by pass rules on this tier. - if (!shadowed_pass_remotes_.empty()) { - absl::erase_if(rule.remotes_, - [&](const auto& x) { return shadowed_pass_remotes_.contains(x); }); - if (rule.remotes_.empty()) { - return true; - } - } - } else { - // rule.remotes_ can not be modified, check if it is completely shadowed - if (std::ranges::all_of(rule.remotes_, [&](const auto& x) { - return shadowed_nonpass_remotes_.contains(x) || shadowed_pass_remotes_.contains(x); - })) { - return true; - } - } - - // Defer shadowing identities until this precedence level is complete, - // so same-precedence rules do not shadow each other. - current_precedence_nonpass_remotes_.insert(rule.remotes_.begin(), rule.remotes_.end()); - } - - return false; - } - -private: - absl::btree_set shadowed_pass_remotes_; - absl::btree_set shadowed_nonpass_remotes_; - absl::btree_set current_precedence_nonpass_remotes_; - uint32_t previous_precedence_{0}; - bool shadow_all_lower_precedence_{false}; - bool current_precedence_has_wildcard_{false}; -}; - -class Passes { -public: - // reset state for a new port range - void reset(uint32_t first_precedence) { - wildcard_it_ = wildcard_pass_rules_.begin(); - pass_rules_.clear(); - pass_rules_tier_index_.clear(); - - tier_pass_rules_.clear(); - tier_wildcard_pass_ = false; - pass_precedence_ = 0; - - shadowing_.reset(first_precedence); - } - - void resetForNextTier(uint32_t first_precedence) { - // Move the collected pass rules to be considered for lower tiers. - pass_rules_tier_index_.push_back(pass_rules_.size()); - pass_rules_.insert(pass_rules_.end(), std::make_move_iterator(tier_pass_rules_.begin()), - std::make_move_iterator(tier_pass_rules_.end())); - - tier_pass_rules_.clear(); - tier_wildcard_pass_ = false; - pass_precedence_ = 0; - - shadowing_.resetForNewTier(first_precedence); - } - - void inheritHigherTierWildcardPassRules(uint32_t precedence) { - // Inherit *higher tier* pass rules from the wildcard port. - // Needed since the wildcard port may have more tiers. - for (; wildcard_it_ != wildcard_pass_rules_.end() && - (*wildcard_it_)->tier_last_precedence_ > precedence; - wildcard_it_++) { - const auto& wildcard_rule = *wildcard_it_; - // Add index entry if this starts a new pass tier. - if (pass_rules_.empty() || - wildcard_rule->tier_last_precedence_ != pass_rules_.back()->tier_last_precedence_) { - pass_rules_tier_index_.push_back(pass_rules_.size()); - } - pass_rules_.insert(pass_rules_.end(), wildcard_rule); - } - } - - void inheritCurrentTierWildcardPassRules(uint32_t precedence) { - // Inherit *current tier* higher or equal precedence pass rules from wildcard port. - for (; - wildcard_it_ != wildcard_pass_rules_.end() && (*wildcard_it_)->precedence_ >= precedence && - (*wildcard_it_)->tier_last_precedence_ <= precedence; - wildcard_it_++) { - const auto& wildcard_rule = *wildcard_it_; - - ensurePassPrecedence(wildcard_rule->tier_last_precedence_); - - if (tier_wildcard_pass_) { - continue; - } - - // Insert to tier_pass_rules_ if any unshadowed remotes remain. - if (wildcard_rule->isRemoteWildcard()) { - tier_wildcard_pass_ = true; - } else if (!shadowing_.shadowPassRemotes(wildcard_rule->remotes_)) { - // Only insert if some remotes are not already shadowed. - // We do not remove already-shadowed remotes from wildcard_rule because - // wildcard pass entries are shared and deep-copying here is expensive. - continue; - } - tier_pass_rules_.emplace_back(wildcard_rule); - } - } - - // addPassRule adds state from a rule with a pass verdict. - void addPassRule(const PortNetworkPolicyRuleConstSharedPtr& rule) { - ensurePassPrecedence(rule->tier_last_precedence_); - - auto& mutable_rule = const_cast(*rule); - - if (!tier_wildcard_pass_) { - if (mutable_rule.isRemoteWildcard()) { - tier_wildcard_pass_ = true; - } else if (!shadowing_.filterShadowPassRemotes(mutable_rule)) { - return; - } - if (!tier_pass_rules_.empty() && tier_pass_rules_.back()->precedence_ == rule->precedence_) { - // Same-precedence pass rule already exists; merge remotes. - tier_pass_rules_.back()->remotes_.insert(mutable_rule.remotes_.begin(), - mutable_rule.remotes_.end()); - } else { - tier_pass_rules_.emplace_back(std::const_pointer_cast(rule)); - } - } - } - - bool - promoteRuleFromHigherTierPasses(std::vector& rules, - std::vector::iterator& it) { - // Mutable reference to the rule for in-place updates below. - auto& rule = const_cast(**it); - - bool promoted = false; - - // Check if this rule needs to be (partially) promoted due to higher-tier passes: - // - pick highest-precedence pass from each higher tier for each remote ID - // - apply in reverse order of tiers - // - if all remotes are covered, promotion can happen fully in-place. - int tier_end = pass_rules_.size(); - for (int tier_start : pass_rules_tier_index_ | std::views::reverse) { - // Skip pass rules on same or lower tiers. - if (pass_rules_[tier_start]->tier_last_precedence_ < rule.precedence_) { - continue; - } - - for (int idx = tier_start; idx < tier_end; idx++) { - auto& pass_rule = pass_rules_[idx]; - // Whole rule is promoted in-place if pass is wildcard or sets are equal. - if (pass_rule->isRemoteWildcard() || rule.remotes_ == pass_rule->remotes_) { - rule.inheritPassPrecedence(*pass_rule); - promoted = true; - break; // Later pass verdicts on this tier have no effect. - } - - // Pass rule is not wildcard and sets differ. - // If mutable_rule is wildcard, keep original and add promoted clone. - if (rule.isRemoteWildcard()) { - auto new_rule = std::make_shared(rule); - new_rule->remotes_ = pass_rule->remotes_; - new_rule->inheritPassPrecedence(*pass_rule); - it = rules.insert(it, new_rule); - it++; - promoted = true; - continue; // Later pass verdicts may specify other remote sets. - } - - // Neither side is wildcard; split by set intersection. - auto remotes = intersection(pass_rule->remotes_, rule.remotes_); - if (!remotes.empty()) { - auto new_rule = std::make_shared(rule); - new_rule->remotes_ = remotes; - new_rule->inheritPassPrecedence(*pass_rule); - it = rules.insert(it, new_rule); - it++; - promoted = true; - for (const auto& remote : remotes) { - rule.remotes_.erase(remote); - } - } - } - // Update for previous tier, if any. - tier_end = tier_start; - } - - return promoted; - } - - // storeWildcardPassRules stores the current pass rules as wildcard port pass rules to be - // considered when processing non-wildcard port rules. - void storeWildcardPassRules(const PortRange& port_range) { - if (!wildcard_pass_rules_.empty()) { - throw EnvoyException(fmt::format("PortNetworkPolicy: Wildcard port range {}-{}, but " - "wildcard pass rules has already been set", - port_range.first, port_range.second)); - } - wildcard_pass_rules_ = pass_rules_; - - // store also the pass rules for the last tier, as they are not yet included in pass_rules_. - if (pass_precedence_ != 0 && !tier_pass_rules_.empty()) { - wildcard_pass_rules_.insert(wildcard_pass_rules_.end(), - std::make_move_iterator(tier_pass_rules_.begin()), - std::make_move_iterator(tier_pass_rules_.end())); - } - } - - // Applies pass verdicts for rules on a given port range. - // Returns true if resulting rules should be kept, false if the rules became empty. - // - // - a pass verdict rule applies on a given port (range) (can be the wildcard port), - // and a set of remote IDs (L3). If the remote ID set is empty, then it applies to all peers. - // (there is no "wildcard identity" (e.g., '0') in the set of the remote IDs) - // - a pass verdict rule (like a deny verdict rule) has no L7 rule components - // - each pass verdict rule has a specific precedence and pass_precedence, and the function is - // to bypass the remaining lower precedence rules upto the pass_precedence, and to promote the - // priority of the intersecting remote ID set lower precedence rules to immediately follow - // the precedence of the pass verdict rule. - // - Precedence promotion is needed to make a verdicts found via actual port vs. wildcard port - // comparable. This allows a higher precedence allow rule to take precedence over a lower - // precedence deny rule, even it the allow originally had a lower precedence, but was - // "passed-to" from a higher precedece pass rule. - // - // We pre-process the rules accordingly here so that the policy lookup at enforcement time - // does not need to consider pass verdicts at all. The key insights to consider are: - // - rules are already split up to non-overlapping port ranges, so we only need - // to consider the remote ID sets (and the wildcard port) - // - if the lower precedence rule remote ID set is covered by the pass rule remote ID set, - // then we can simply promote the precedence (and re-sort afterwards) - // - if the pass rule applies to all remote IDs (empty set == wildcard), then it covers all - // possible sets of remote IDs - // - if not wildcard, but the sets are the same, then the pass verdicts "covers" the rule - // in question - // - otherwise the rule needs to be split into two: - // - one with the intersection of the remote IDs of the two rules, with precedence promotion - // - other with the remaining remote IDs, left with the original precedence - // - this includes the case where the lower precedence rule applies to all identities - // (empty ID set) - void apply(const PortRange& port_range, PortNetworkPolicyRules& rules) { - if (rules.rules_.empty() && !wildcard_pass_rules_.empty()) { - // add the default allow rule so that the wildcard port pass can apply to it. - rules.addDefaultAllowRule(); - } - if (!rules.rules_.empty() && (!wildcard_pass_rules_.empty() || rules.has_pass_rules_)) { - bool must_sort = false; - - // reset state for the new range's rules - reset(rules.rules_.front()->precedence_); - - bool keep = false; // assume rule is dropped - for (auto it = rules.rules_.begin(); it != rules.rules_.end(); - it = keep ? (keep = false, ++it) : rules.rules_.erase(it)) { - auto& rule = *it; - - // Check if we have reached the next tier. - if (pass_precedence_ != 0 && rule->precedence_ < pass_precedence_) { - resetForNextTier(rule->precedence_); - } - - // Skip remaining rules on this tier? - if (tier_wildcard_pass_) { - continue; - } - - // Inherit wildcard-port pass rules affecting this rule. - inheritHigherTierWildcardPassRules(rule->precedence_); - inheritCurrentTierWildcardPassRules(rule->precedence_); - - // skip remaining rules on this tier? - if (tier_wildcard_pass_) { - continue; - } - - // Is this a pass verdict rule? - if (rule->tier_last_precedence_ != 0) { - addPassRule(rule); - // Pass rules are not kept - continue; - } - - // Is the rule shadowed? (If not then updates shadowed state) - if (shadowing_.shadowNonpassRule(const_cast(*rule))) { - continue; - } - - // Apply passes to the rule and insert. - if (promoteRuleFromHigherTierPasses(rules.rules_, it)) { - must_sort = true; - } - // keep rule in place - keep = true; - } - - // Have to sort if precedences have been updated in-place. - if (must_sort) { - rules.sort(); - - // remove shadowed rules due to promoted precedences - shadowing_.reset(rules.rules_.front()->precedence_); - for (auto it = rules.rules_.begin(); it != rules.rules_.end(); - it = keep ? ++it : rules.rules_.erase(it)) { - keep = !shadowing_.shadowNonpassRule(const_cast(**it)); - } - } - - // Store wildcard port passes for consideration for non-wildcard ports. - if (port_range.first == 0) { - storeWildcardPassRules(port_range); - } - - // Mark ranges with all rules removed for clean-up. - if (rules.empty()) { - // Empty rule set would always allow. Mark for removal. - empty_ranges_.push_back(port_range); - } - } - } - - std::vector& emptyRanges() { return empty_ranges_; } - -private: - void ensurePassPrecedence(uint32_t tier_last_precedence) { - // pass_precedence_ is non-zero when a pass verdict has been seen and - // defines the end of the current tier. All pass verdicts on a specific - // tier must have the same pass_precedence so tier boundaries stay unambiguous. - if (tier_last_precedence == 0 || - (pass_precedence_ != 0 && tier_last_precedence != pass_precedence_)) { - throw EnvoyException(fmt::format("PortNetworkPolicy: Inconsistent pass precedence {} != {}", - tier_last_precedence, pass_precedence_)); - } - pass_precedence_ = tier_last_precedence; - } - - std::vector empty_ranges_; - std::vector wildcard_pass_rules_; - std::vector::iterator wildcard_it_; - ShadowedRemotes shadowing_; - std::vector pass_rules_; - std::vector pass_rules_tier_index_; - uint32_t pass_precedence_{0}; - std::vector tier_pass_rules_; - bool tier_wildcard_pass_{false}; -}; - -class PortNetworkPolicy : public Logger::Loggable { -public: - PortNetworkPolicy(const NetworkPolicyMapImpl& parent, - const Protobuf::RepeatedPtrField& rules) { - for (const auto& rule : rules) { - // Only TCP supported for HTTP - if (rule.protocol() == envoy::config::core::v3::SocketAddress::TCP) { - // Port may be zero, which matches any port. - uint16_t port = rule.port(); - // End port may be zero, which means no range - uint16_t end_port = rule.end_port(); - if (end_port < port) { - if (end_port != 0) { - throw EnvoyException(fmt::format( - "PortNetworkPolicy: Invalid port range, end port is less than start port {}-{}", - port, end_port)); - } - end_port = port; + PortNetworkPolicy(const NetworkPolicyMapImpl& parent, + const Protobuf::RepeatedPtrField& rules, + const ResourceMapOverlay* resource_map) { + for (const auto& rule : rules) { + // Only TCP supported for HTTP + if (rule.protocol() == envoy::config::core::v3::SocketAddress::TCP) { + // Port may be zero, which matches any port. + uint16_t port = rule.port(); + // End port may be zero, which means no range + uint16_t end_port = rule.end_port(); + if (end_port < port) { + if (end_port) { + throw EnvoyException(fmt::format( + "PortNetworkPolicy: Invalid port range, end port is less than start port {}-{}", + port, end_port)); + } + end_port = port; } if (port == 0) { @@ -1678,7 +1920,6 @@ class PortNetworkPolicy : public Logger::Loggable { RELEASE_ASSERT(it != rules_.end(), "first overlapping entry not found"); } // Add rules to all the overlapping entries - bool shared_resource = rule_range.first == 0; // wildcard port rules are shared bool singular = rule_range.first == rule_range.second; for (; it != rules_.end() && rangesOverlap(it->first, rule_range); it++) { auto range = it->first; @@ -1693,10 +1934,10 @@ class PortNetworkPolicy : public Logger::Loggable { // so the relative order of rules from this batch is reversed. This // is harmless: equal-precedence rules are evaluated as alternatives // (stable sort only affects presentation/debug ordering). - rules.prepend(parent, rule.rules(), shared_resource); + rules.prepend(parent, rule.rules(), resource_map); } else { // Rules with a non-trivial range go to the back of the list - rules.append(parent, rule.rules(), shared_resource); + rules.append(parent, rule.rules(), resource_map); } } } else { @@ -1714,7 +1955,7 @@ class PortNetworkPolicy : public Logger::Loggable { if (!wildcard_rules) { break; } - rules.appendNonPassRules(wildcard_rules->rules_); + rules.appendRules(wildcard_rules->rules_); } bool have_passes = false; @@ -1730,24 +1971,27 @@ class PortNetworkPolicy : public Logger::Loggable { } } - // Apply pass verdicts, if any. if (have_passes) { - Passes passes; - - // This loop always iterates the wildcard port first, if rules for it exist. for (auto& [port_range, rules] : rules_) { - passes.apply(port_range, rules); + (void)port_range; + rules.prepareRuntimePasses(); } - // Delete port ranges that only contained pass rules. - // Otherwise the policy would always accept. - for (auto port_range : passes.emptyRanges()) { - rules_.erase(port_range); + // Pass-only ranges do not yield a final verdict, but keeping them would make + // higher-level L7 checks treat the range as "no L7 rules" and allow by default. + for (auto it = rules_.begin(); it != rules_.end();) { + if (it->second.hasOnlyPassRules()) { + it = rules_.erase(it); + } else { + ++it; + } } } } - const PortPolicy findPortPolicy(uint16_t port) const { return PortPolicy(rules_, port); } + const PortPolicy findPortPolicy(uint16_t port, const SelectorVersion selector_version) const { + return PortPolicy(rules_, port, selector_version); + } void toString(int indent, std::string& res) const { if (rules_.empty()) { @@ -1762,7 +2006,7 @@ class PortNetworkPolicy : public Logger::Loggable { } } - PolicyMap rules_; + PolicySnapshot rules_; bool has_http_rules_ = false; }; @@ -1770,11 +2014,15 @@ class PortNetworkPolicy : public Logger::Loggable { // methods. class PolicyInstanceImpl : public PolicyInstance { public: + friend class NetworkPolicyMapImpl; PolicyInstanceImpl(const NetworkPolicyMapImpl& parent, uint64_t hash, - const cilium::NetworkPolicy& proto) + const cilium::NetworkPolicy& proto, + const PolicyStreamStateConstSharedPtr& policy_stream_state, + const ResourceMapOverlay* resource_map) : endpoint_id_(proto.endpoint_id()), hash_(hash), policy_proto_(proto), endpoint_ips_(proto), - parent_(parent), ingress_(parent, policy_proto_.ingress_per_port_policies()), - egress_(parent, policy_proto_.egress_per_port_policies()) {} + parent_(parent), policy_stream_state_(policy_stream_state), + ingress_(parent, policy_proto_.ingress_per_port_policies(), resource_map), + egress_(parent, policy_proto_.egress_per_port_policies(), resource_map) {} bool allowed(bool ingress, uint16_t proxy_id, uint32_t remote_id, uint16_t port, Envoy::Http::RequestHeaderMap& headers, @@ -1793,7 +2041,9 @@ class PolicyInstanceImpl : public PolicyInstance { } const PortPolicy findPortPolicy(bool ingress, uint16_t port) const override { - return ingress ? ingress_.findPortPolicy(port) : egress_.findPortPolicy(port); + const auto selector_version = policy_stream_state_->version(); + return ingress ? ingress_.findPortPolicy(port, selector_version) + : egress_.findPortPolicy(port, selector_version); } bool useProxylib(bool ingress, uint16_t proxy_id, uint32_t remote_id, uint16_t port, @@ -1825,29 +2075,164 @@ class PolicyInstanceImpl : public PolicyInstance { private: const NetworkPolicyMapImpl& parent_; + const PolicyStreamStateConstSharedPtr policy_stream_state_; const PortNetworkPolicy ingress_; const PortNetworkPolicy egress_; }; +template std::string endpointIpsForLog(const EndpointIps& endpoint_ips) { + std::string formatted = "["; + bool first = true; + for (const auto& endpoint_ip : endpoint_ips) { + if (!first) { + formatted += ", "; + } + formatted += endpoint_ip; + first = false; + } + formatted += "]"; + return formatted; +} + +std::string describePolicyResourceForLog(absl::string_view resource_name, + const std::shared_ptr& policy) { + ASSERT(policy != nullptr, "policy resource description requires a policy"); + return fmt::format("policy resource '{}' (endpoint_id {}, endpoint_ips {})", resource_name, + policy->endpoint_id_, endpointIpsForLog(policy->policy_proto_.endpoint_ips())); +} + +std::string describePolicyResourceForLog(absl::string_view resource_name, + const cilium::NetworkPolicy& policy) { + return fmt::format("policy resource '{}' (endpoint_id {}, endpoint_ips {})", resource_name, + policy.endpoint_id(), endpointIpsForLog(policy.endpoint_ips())); +} + +std::string +ResourceMap::findPolicyResourceName(const std::shared_ptr& policy) const { + if (policy == nullptr) { + return {}; + } + for (const auto& [resource_name, resource_key] : *this) { + const auto* policy_entry = resource_key.policyResourceEntry(); + if (policy_entry != nullptr && policy_entry->policy == policy) { + return resource_name; + } + } + return {}; +} + +std::string ResourceMapOverlay::findPolicyResourceName( + const std::shared_ptr& policy) const { + if (policy == nullptr) { + return {}; + } + for (const auto& [resource_name, resource_key] : upserts_) { + const auto* policy_entry = resource_key.policyResourceEntry(); + if (policy_entry != nullptr && policy_entry->policy == policy) { + return resource_name; + } + } + if (base_ == nullptr) { + return {}; + } + for (const auto& [resource_name, resource_key] : *base_) { + if (removed_.contains(resource_name)) { + continue; + } + const auto* policy_entry = resource_key.policyResourceEntry(); + if (policy_entry != nullptr && policy_entry->policy == policy) { + return resource_name; + } + } + return {}; +} + +std::string +ResourceMapOverlay::describeExistingResourceKey(const std::string& key, + const PolicyMapSnapshot& policy_map) const { + const auto* entry = findEntry(key); + if (entry == nullptr) { + return fmt::format("resource key '{}'", key); + } + if (entry->selectorResourceEntry() != nullptr) { + return fmt::format("selector resource '{}'", key); + } + if (const auto* policy_entry = entry->policyResourceEntry(); + policy_entry != nullptr && policy_entry->policy != nullptr) { + return describePolicyResourceForLog(key, policy_entry->policy); + } + if (!entry->isPolicyEndpointIpEntry()) { + return fmt::format("resource key '{}'", key); + } + + auto policy_it = policy_map.find(key); + if (policy_it == policy_map.end()) { + return fmt::format("endpoint IP alias '{}'", key); + } + + const auto& policy = policy_it->second; + const auto resource_name = findPolicyResourceName(policy); + if (!resource_name.empty()) { + return fmt::format("endpoint IP alias '{}' owned by {}", key, + describePolicyResourceForLog(resource_name, policy)); + } + + return fmt::format("endpoint IP alias '{}' owned by endpoint_id {} with endpoint_ips {}", key, + policy->endpoint_id_, endpointIpsForLog(policy->policy_proto_.endpoint_ips())); +} + +void ResourceMap::erasePolicyResource(PolicyMapSnapshot& policy_map, + const std::string& resource_name, + const std::shared_ptr& policy) { + ASSERT(policy != nullptr, "policy resource key must carry a policy"); + for (const auto& endpoint_ip : policy->policy_proto_.endpoint_ips()) { + policy_map.erase(endpoint_ip); + erase(endpoint_ip); + } + erase(resource_name); +} + +void ResourceMapOverlay::erasePolicyResource( + PolicyMapSnapshot& policy_map, const std::string& resource_name, + const std::shared_ptr& policy) { + ASSERT(policy != nullptr, "policy resource key must carry a policy"); + for (const auto& endpoint_ip : policy->policy_proto_.endpoint_ips()) { + policy_map.erase(endpoint_ip); + erase(endpoint_ip); + } + erase(resource_name); +} + +namespace { + +bool policyUsesSelectors(const cilium::NetworkPolicy& policy) { + for (const auto& port_policy : policy.ingress_per_port_policies()) { + if (std::ranges::any_of(port_policy.rules(), + [](const auto& rule) { return rule.selectors_size() > 0; })) { + return true; + } + } + for (const auto& port_policy : policy.egress_per_port_policies()) { + if (std::ranges::any_of(port_policy.rules(), + [](const auto& rule) { return rule.selectors_size() > 0; })) { + return true; + } + } + return false; +} + +} // namespace + // Common base constructor // This is used directly for testing with a file-based subscription NetworkPolicyMap::NetworkPolicyMap(Server::Configuration::FactoryContext& context, - const envoy::config::core::v3::ConfigSource& npds_config, + const envoy::config::core::v3::ConfigSource& config_source, bool subscribe) : context_(context.serverFactoryContext()) { - impl_ = std::make_unique(context, npds_config); - - if (context_.admin().has_value()) { - ENVOY_LOG(debug, "Registering NetworkPolicies to config tracker"); - config_tracker_entry_ = context_.admin()->getConfigTracker().add( - "networkpolicies", [this](const Matchers::StringMatcher& name_matcher) { - return dumpNetworkPolicyConfigs(name_matcher); - }); - RELEASE_ASSERT(config_tracker_entry_, ""); - } + impl_ = std::make_unique(context, config_source); if (subscribe) { - getImpl().startSubscription(npds_config); + impl_->subscribe(); } } @@ -1869,18 +2254,81 @@ NetworkPolicyMap::~NetworkPolicyMap() { ENVOY_LOG(debug, "Cilium L7 NetworkPolicyMap: posting NetworkPolicyMapImpl deletion to main thread"); - context_.mainThreadDispatcher().deleteInDispatcherThread( - Event::DispatcherThreadDeletableConstPtr(impl_.release())); + context_.mainThreadDispatcher().post([impl = std::move(impl_)]() mutable { impl.reset(); }); +} + +bool NetworkPolicyMap::exists(const std::string& endpoint_policy_name) const { + return impl_->getPolicyInstanceImpl(endpoint_policy_name); } -NetworkPolicyMapImpl::NetworkPolicyMapImpl(Server::Configuration::FactoryContext& context, - const envoy::config::core::v3::ConfigSource& npds_config) +bool NetworkPolicyMap::useDeltaXds() const { return impl_->useDeltaXds(); } + +void NetworkPolicyMap::setConfigSource( + const envoy::config::core::v3::ConfigSource& config_source) const { + impl_->setConfigSource(config_source); +} + +void NetworkPolicyMap::startSubscriptionForTest( + std::unique_ptr&& subscription) { + impl_->subscribe(std::move(subscription)); +} + +void NetworkPolicyMap::startManagedSubscriptionForTest() { + impl_->startManagedSubscriptionForTest(); +} + +void NetworkPolicyMap::setSubscriptionFactoryForTest(SubscriptionFactoryForTest factory) { + impl_->setSubscriptionFactoryForTest(std::move(factory)); +} + +void NetworkPolicyMap::onSubscriptionConnectedForTest() { impl_->onSubscriptionConnectedForTest(); } + +void NetworkPolicyMap::onSubscriptionTransportCloseForTest() { + impl_->onSubscriptionTransportCloseForTest(); +} + +bool NetworkPolicyMap::subscriptionUseDeltaXdsForTest() const { + return impl_->subscriptionUseDeltaXds(); +} + +bool NetworkPolicyMap::subscriptionConnectedForTest() const { + return impl_->subscriptionConnectedForTest(); +} + +Envoy::Config::SubscriptionCallbacks& NetworkPolicyMap::subscriptionCallbacksForTest() const { + return *impl_; +} + +PolicyStats& NetworkPolicyMap::statsForTest() const { return impl_->stats_; } + +void NetworkPolicyMap::resetStreamForTest() { impl_->resetStreamForTest(); } + +PolicyInstanceConstSharedPtr +NetworkPolicyMap::getPolicyInstanceSharedForTest(const std::string& endpoint_policy_name) const { + return impl_->getPolicyInstanceSharedImpl(endpoint_policy_name); +} + +uint64_t +NetworkPolicyMap::policySelectorStreamGenerationForTest(const PolicyInstance& policy) const { + return impl_->policySelectorStreamGenerationForTestImpl(policy); +} + +SelectorVersion NetworkPolicyMap::policySelectorVersionForTest(const PolicyInstance& policy) const { + return impl_->policySelectorVersionForTestImpl(policy); +} + +NetworkPolicyMapImpl::NetworkPolicyMapImpl( + Server::Configuration::FactoryContext& context, + const envoy::config::core::v3::ConfigSource& config_source) : context_(context.serverFactoryContext()), map_ptr_(nullptr), npds_stats_scope_(context_.serverScope().createScope("cilium.npds.")), policy_stats_scope_(context_.serverScope().createScope("cilium.policy.")), - init_target_(fmt::format("Cilium Network Policy subscription start"), + desired_config_source_(config_source), config_source_(config_source), + init_target_(fmt::format("Cilium NetworkPolicy subscription start"), [this]() { - subscription_->start({}); + // Production subscription is allowed to start from now on. + subscription_should_start_ = true; + startSubscription(); // Allow listener init to continue before network policy updates are received init_target_.ready(); }), @@ -1889,15 +2337,24 @@ NetworkPolicyMapImpl::NetworkPolicyMapImpl(Server::Configuration::FactoryContext context_, *npds_stats_scope_, context_.messageValidationContext().dynamicValidationVisitor())), parked_init_manager_(std::make_unique("Cilium NetworkPolicyMap parked")), - npds_config_(npds_config), - stats_{ALL_CILIUM_POLICY_STATS(POOL_COUNTER(*policy_stats_scope_))} { + stats_{ALL_CILIUM_POLICY_COUNTERS(POOL_COUNTER(*policy_stats_scope_)) + ALL_CILIUM_POLICY_GAUGES(POOL_GAUGE(*policy_stats_scope_))} { // Use listener init manager for subscription initialization context.initManager().add(init_target_); transport_factory_context_->setInitManager(*parked_init_manager_); // Allocate an initial policy map so that the map pointer is never a nullptr - store(new RawPolicyMap()); + store(new PolicyMapSnapshot()); ENVOY_LOG(trace, "NetworkPolicyMapImpl({}) created.", instance_id_); + + if (context_.admin().has_value()) { + ENVOY_LOG(debug, "Registering NetworkPolicies to config tracker"); + config_tracker_entry_ = context_.admin()->getConfigTracker().add( + "networkpolicies", [this](const Matchers::StringMatcher& name_matcher) { + return dumpNetworkPolicyConfigs(name_matcher); + }); + RELEASE_ASSERT(config_tracker_entry_, ""); + } } // NetworkPolicyMapImpl destructor must only be called from the main thread. @@ -1907,41 +2364,128 @@ NetworkPolicyMapImpl::~NetworkPolicyMapImpl() { delete load(); } -void NetworkPolicyMapImpl::startSubscription( - const envoy::config::core::v3::ConfigSource& npds_config) { - if (npds_config.config_source_specifier_case() == envoy::config::core::v3::ConfigSource::kAds) { - auto ads_mux = context_.xdsManager().adsMux(); - subscription_ = THROW_OR_RETURN_VALUE( - context_.clusterManager().subscriptionFactory().subscriptionOverAdsGrpcMux( - ads_mux, npds_config, NetworkPolicyTypeUrl, *npds_stats_scope_, *this, - std::make_shared(), {}), - Config::SubscriptionPtr); - } else { - subscription_ = subscribe(NetworkPolicyTypeUrl, npds_config, context_.localInfo(), - context_.clusterManager(), context_.mainThreadDispatcher(), - context_.api().randomGenerator(), *npds_stats_scope_, *this, - std::make_shared()); +void NetworkPolicyMapImpl::subscribe() { + subscription_connected_ = false; + config_source_ = desired_config_source_; + ++subscription_id_; + + if (subscription_factory_for_test_) { + subscription_ = subscription_factory_for_test_(subscriptionUseDeltaXds()); + if (subscription_should_start_) { + startSubscription(); + } + return; + } + + auto on_stream_event = [weak_this = weak_from_this(), + id = subscription_id_](Config::GrpcMuxStreamEvent event) { + if (auto shared_this = weak_this.lock()) { + shared_this->onSubscriptionStreamEvent(id, event); + } + }; + + absl::string_view type_url = NetworkPolicyTypeUrl; + Config::OpaqueResourceDecoderSharedPtr decoder = std::make_shared(); + if (useDeltaXds()) { + type_url = NetworkPolicyResourceTypeUrl; + decoder = std::make_shared(); } - subscription_->start({}); + subscription_ = Cilium::subscribe(type_url, config_source_, context_, *npds_stats_scope_, *this, + decoder, std::move(on_stream_event)); + + if (subscription_should_start_) { + startSubscription(); + } +} + +void NetworkPolicyMapImpl::reopenIpcache() { + // Get ipcache singleton only if it was successfully created previously. + // Cilium agent re-creates IP cache on restart, and the first accepted update on + // the new stream must reopen it before workers enforce refreshed identities. + IpCacheSharedPtr ipcache = IpCache::getIpCache(context_); + if (ipcache) { + ENVOY_LOG(info, "Reopening ipcache on new stream"); + ipcache->open(); + } } -void NetworkPolicyMapImpl::tlsWrapperMissingPolicyInc() const { - stats_.tls_wrapper_missing_policy_.inc(); +std::shared_ptr NetworkPolicyMapImpl::createOrReusePolicy( + const std::string& resource_name, const cilium::NetworkPolicy& config, + const PolicyStreamStateConstSharedPtr& policy_stream_state, const ResourceMap& resource_map, + const ResourceMapOverlay* pending_resource_map) { + const uint64_t new_hash = MessageUtil::hash(config); + auto it = resource_map.find(resource_name); + if (it != resource_map.cend()) { + const auto* old_policy_entry = it->second.policyResourceEntry(); + if (old_policy_entry == nullptr) { + return std::make_shared(*this, new_hash, config, + policy_stream_state, pending_resource_map); + } + const auto& old_policy = old_policy_entry->policy; + if (old_policy && old_policy->hash_ == new_hash && + Protobuf::util::MessageDifferencer::Equals(old_policy->policy_proto_, config) && + !(pending_resource_map && policyUsesSelectors(config))) { + ENVOY_LOG(trace, "New policy is equal to old one, not updating."); + return old_policy; + } + } + + return std::make_shared(*this, new_hash, config, policy_stream_state, + pending_resource_map); } -bool NetworkPolicyMapImpl::isNewStream() { - auto sub = dynamic_cast(subscription_.get()); - if (!sub) { - ENVOY_LOG(error, "Cilium NetworkPolicyMapImpl: Cannot get GrpcSubscriptionImpl"); - return false; +SelectorHandle NetworkPolicyMapImpl::createOrReuseSelector(const std::string& resource_name, + const cilium::Selector& config, + uint64_t update_version) { + // Compare against the selector visible in the currently prepared update version, not just the + // last published one. Under the single-update-in-flight VersionedMap contract, any selector + // visible in 'update_version' is also the indefinite selector value for that candidate update. + auto selector_value = selector_map_.find(resource_name); + if (selector_value) { + const auto* old_selector = selector_value->get(update_version); + if (old_selector && + old_selector->size() == static_cast(config.remote_identities_size()) && + std::ranges::all_of(config.remote_identities(), [&](const auto remote_identity) { + return old_selector->contains(remote_identity); + })) { + return selector_value; + } } - auto mux = dynamic_cast(sub->grpcMux().get()); - if (!mux) { - ENVOY_LOG(error, "Cilium NetworkPolicyMapImpl: Cannot get GrpcMuxImpl"); - return false; + + // otherwise create a new one and insert it to the selector map. + + auto selector = new SelectorInstance(); + selector->reserve(config.remote_identities_size()); + for (const auto remote_identity : config.remote_identities()) { + selector->emplace(remote_identity); } - return mux->isNewStream(); + return selector_map_.insert(resource_name, selector); +} + +void NetworkPolicyMapImpl::installNewPolicyMap( + PolicyMapSnapshot&& new_policy_map, Init::ManagerImpl& version_init_manager, + std::string&& version_name, const PolicyStreamStateSharedPtr& policy_stream_state) { + // Initialize SDS secrets. We do not wait for the completion. + version_init_manager.initialize(Init::WatcherImpl(std::move(version_name), []() {})); + + auto new_policy_map_ptr = std::make_unique(std::move(new_policy_map)); + // Publish selector data before publishing the new policy map. New policies created above already + // point at 'policy_stream_state', so any worker that can observe the swapped-in policy map must + // also be able to observe the selector version those policies expect to use. + auto new_version = selector_map_.publishNextVersion(); + if (new_version > 0) { + policy_stream_state->publishVersion(new_version); + } + policy_stream_state_ = policy_stream_state; + stats_.policy_stream_generation_.set(policy_stream_state_->streamGeneration()); + + // old version can be GC'd once all worker threads have quiesced + const auto* old_policy_map = exchange(new_policy_map_ptr.release()); + + // Delete the old map and first-phase GC old selector versions once all worker threads have + // entered their event queues, as this is proof that they no longer refer to the old map. + scheduleSelectorGCAndDeferredDeletion(new_version, old_policy_map); } // removeInitManager must be called at the end of each policy update @@ -1984,6 +2528,35 @@ void NetworkPolicyMapImpl::removeInitManager() { transport_factory_context_->setInitManager(*parked_init_manager_); } +void NetworkPolicyMapImpl::scheduleSelectorDeferredDeletion( + DeferredDeletion&& deferred) { + if (deferred.empty()) { + return; + } + auto deferred_owner = std::make_shared>(std::move(deferred)); + // The callback exists only to keep the deferred-deletion batch alive until all workers have + // quiesced once more. The batch deletes its nodes from the closure destructor. + runAfterAllThreads([deferred_owner]() {}); +} + +void NetworkPolicyMapImpl::scheduleSelectorGCAndDeferredDeletion( + uint64_t published_version, const PolicyMapSnapshot* old_policy_map) { + if (published_version == 0 && old_policy_map == nullptr) { + return; + } + runAfterAllThreads([shared_this = shared_from_this(), published_version, old_policy_map]() { + // Clean-up in the main thread after all worker threads have scheduled. + // Delete the old policy map before selector GC. Old policies are the only remaining users of + // old-stream selector versions; once the old map is gone after this quiescence point, those + // selector versions may be unlinked and deferred for deletion. + delete old_policy_map; + if (published_version == 0) { + return; + } + shared_this->scheduleSelectorDeferredDeletion(shared_this->selector_map_.gc(published_version)); + }); +} + // onConfigUpdate parses the new network policy resources, allocates a new policy map and atomically // swaps it in place of the old policy map. Throws if any of the 'resources' can not be // parsed. Otherwise an OK status is returned without pausing NPDS gRPC stream, causing a new @@ -1991,23 +2564,23 @@ void NetworkPolicyMapImpl::removeInitManager() { absl::Status NetworkPolicyMapImpl::onConfigUpdate( const std::vector& resources, const std::string& version_info) { - ENVOY_LOG(debug, "NetworkPolicyMapImpl::onConfigUpdate({}), {} resources, version: {}", - instance_id_, resources.size(), version_info); + subscription_connected_ = true; + auto stream_generation = streamGeneration(); + // policy_stream_state_ gets updated on first successful update, + // so 'is_new_stream' remains 'true' as long as the stream has not had a successful update yet. + const bool is_new_stream = stream_generation != policy_stream_state_->streamGeneration(); + ENVOY_LOG(debug, "NetworkPolicyMapImpl::onConfigUpdate({}), {} resources, version: {}, stream {}", + instance_id_, resources.size(), version_info, stream_generation); stats_.updates_total_.inc(); // Reopen IPcache for every new stream. Cilium agent re-creates IP cache on restart, // and that is also when the old stream terminates and a new one is created. // New security identities (e.g., for FQDN policies) only get inserted to the new IP cache, // so open it before the workers get a chance to enforce policy on the new IDs. - if (isNewStream()) { - ENVOY_LOG(info, "New NetworkPolicy stream"); + if (is_new_stream) { + ENVOY_LOG(info, "New NetworkPolicy stream {}", stream_generation); - // Get ipcache singleton only if it was successfully created previously - IpCacheSharedPtr ipcache = IpCache::getIpCache(context_); - if (ipcache != nullptr) { - ENVOY_LOG(info, "Reopening ipcache on new stream"); - ipcache->open(); - } + reopenIpcache(); } std::string version_name = fmt::format("NetworkPolicyMap version {}", version_info); @@ -2017,69 +2590,365 @@ absl::Status NetworkPolicyMapImpl::onConfigUpdate( // SDS secrets will use this! transport_factory_context_->setInitManager(version_init_manager); - const auto* old_map = load(); - RawPolicyMap new_map; - { + const auto policy_stream_state = + is_new_stream + ? std::make_shared(stream_generation, selector_map_.getVersion()) + : policy_stream_state_; + PolicyMapSnapshot new_policy_map; + std::vector> resource_entries; + try { + for (const auto& resource : resources) { + const auto& config = dynamic_cast(resource.get().resource()); + const std::string& resource_name = resource.get().name(); + validateResourceName(resource_name, "NetworkPolicy resource name"); + if (config.endpoint_ips().empty()) { + throw EnvoyException("NetworkPolicy has no endpoint ips"); + } + ENVOY_LOG(debug, + "Received NetworkPolicy for endpoint {}, endpoint_ip {} in onConfigUpdate() " + "version {}", + config.endpoint_id(), config.endpoint_ips()[0], version_info); + + auto policy = + createOrReusePolicy(resource_name, config, policy_stream_state, resource_map_, nullptr); + if (!resource_name.empty()) { + resource_entries.emplace_back(resource_name, ResourceKey::policyResource(policy)); + } + for (const auto& endpoint_ip : config.endpoint_ips()) { + ENVOY_LOG(trace, "Cilium updating or keeping network policy for endpoint {}", endpoint_ip); + // new_policy_map is not exception safe, policy must be computed separately! + new_policy_map.insert_or_assign(endpoint_ip, policy); + resource_entries.emplace_back(endpoint_ip, ResourceKey::policyEndpointIp()); + } + } + } catch (const EnvoyException& e) { + ENVOY_LOG(warn, "NetworkPolicy update for version {} failed: {}", version_info, e.what()); + stats_.updates_rejected_.inc(); + removeInitManager(); + throw; // re-throw + } + + stats_.update_success_.inc(); + + installNewPolicyMap(std::move(new_policy_map), version_init_manager, std::move(version_name), + policy_stream_state); + resource_map_.replaceWith(std::move(resource_entries)); + + return absl::OkStatus(); +} + +absl::Status NetworkPolicyMapImpl::onConfigUpdate( + const std::vector& added_resources, + const Protobuf::RepeatedPtrField& removed_resources, + const std::string& system_version_info) { + subscription_connected_ = true; + auto stream_generation = streamGeneration(); + // policy_stream_state_ gets updated on first successful update, + // so 'is_new_stream' remains 'true' as long as the stream has not had a successful update yet. + const bool is_new_stream = stream_generation != policy_stream_state_->streamGeneration(); + + // first find if this is a selector-only update + bool updates_policies = false; + bool updates_selectors = false; + for (const auto& removed_resource : removed_resources) { + validateResourceName(removed_resource, "NetworkPolicyResource removed resource name"); + auto resource_it = resource_map_.find(removed_resource); + if (resource_it == resource_map_.end()) { + continue; + } + if (resource_it->second.selectorResourceEntry()) { + updates_selectors = true; + } else { + updates_policies = true; + } + } + for (const auto& resource : added_resources) { + const auto& typed_resource = + dynamic_cast(resource.get().resource()); + const std::string& resource_name = resource.get().name(); + if (resource_name.empty()) { + throw EnvoyException("NetworkPolicyResource has no name"); + } + validateResourceName(resource_name, "NetworkPolicyResource added resource name"); + switch (typed_resource.resource_case()) { + case cilium::NetworkPolicyResource::kPolicy: + updates_policies = true; + break; + case cilium::NetworkPolicyResource::kSelector: + updates_selectors = true; + break; + case cilium::NetworkPolicyResource::RESOURCE_NOT_SET: + break; + } + } + + ENVOY_LOG(debug, + "NetworkPolicyMapImpl::onConfigUpdate({}), {} added resources, {} removed resources, " + "version: {}, stream {}, updates_selectors: {}, updates_policies: {}", + instance_id_, added_resources.size(), removed_resources.size(), system_version_info, + stream_generation, updates_selectors, updates_policies); + stats_.updates_total_.inc(); + + // Reopen IPcache for every new stream. Cilium agent re-creates IP cache on restart, + // and that is also when the old stream terminates and a new one is created. + // New security identities (e.g., for FQDN policies) only get inserted to the new IP cache, + // so open it before the workers get a chance to enforce policy on the new IDs. + if (is_new_stream) { + ENVOY_LOG(info, "New NetworkPolicyResource stream {}", stream_generation); + reopenIpcache(); + } + + if (!is_new_stream && updates_selectors && !updates_policies) { + ResourceMapOverlay pending_resource_map(resource_map_); + try { - for (const auto& resource : resources) { - const auto& config = dynamic_cast(resource.get().resource()); - ENVOY_LOG(debug, - "Received Network Policy for endpoint {}, endpoint_ip {} in onConfigUpdate() " - "version {}", - config.endpoint_id(), config.endpoint_ips()[0], version_info); - if (config.endpoint_ips().empty()) { - throw EnvoyException("Network Policy has no endpoint ips"); + const auto selector_update_version = selector_map_.prepareNextVersion(); + + for (const auto& resource : removed_resources) { + ENVOY_LOG(trace, "Cilium removing NetworkPolicyResource selector {}", resource); + const auto* resource_entry = pending_resource_map.findEntry(resource); + if (resource_entry == nullptr) { + ENVOY_LOG(debug, + "NetworkPolicyResource removed selector name '{}' not found from resource map", + resource); + continue; } - - // First find the old config to figure out if an update is needed. - const uint64_t new_hash = MessageUtil::hash(config); - auto it = old_map->find(config.endpoint_ips()[0]); - if (it != old_map->cend()) { - const auto& old_policy = it->second; - if (old_policy && old_policy->hash_ == new_hash && - Protobuf::util::MessageDifferencer::Equals(old_policy->policy_proto_, config)) { - ENVOY_LOG(trace, "New policy is equal to old one, not updating."); - for (const auto& endpoint_ip : config.endpoint_ips()) { - ENVOY_LOG(trace, "Cilium keeping network policy for endpoint {}", endpoint_ip); - new_map.emplace(endpoint_ip, old_policy); - } - continue; - } + if (resource_entry->isPolicyEndpointIpEntry()) { + throw EnvoyException(fmt::format("NetworkPolicyResource removed selector name " + "'{}' is a policy endpoint IP alias, " + "not a resource name", + resource)); + } + if (resource_entry->policyResourceEntry()) { + throw EnvoyException(fmt::format( + "NetworkPolicyResource removed selector name '{}' refers to a policy resource", + resource)); } + selector_map_.clear(resource); + pending_resource_map.erase(resource); + } - // May throw - auto new_policy = std::make_shared(*this, new_hash, config); + for (const auto& resource : added_resources) { + const auto& typed_resource = + dynamic_cast(resource.get().resource()); + if (typed_resource.resource_case() != cilium::NetworkPolicyResource::kSelector) { + continue; + } + const std::string& resource_name = resource.get().name(); + pending_resource_map.eraseSelectorResourceIfPresent(resource_name); + } - for (const auto& endpoint_ip : config.endpoint_ips()) { - ENVOY_LOG(trace, "Cilium updating network policy for endpoint {}", endpoint_ip); - // new_map is not exception safe, new_policy must be computed separately! - new_map.emplace(endpoint_ip, new_policy); + for (const auto& resource : added_resources) { + const auto& typed_resource = + dynamic_cast(resource.get().resource()); + const std::string& resource_name = resource.get().name(); + + switch (typed_resource.resource_case()) { + case cilium::NetworkPolicyResource::kSelector: { + ENVOY_LOG(debug, + "Received NetworkPolicyResource selector {} in onConfigUpdate() " + "version {}", + resource_name, system_version_info); + auto selector_handle = createOrReuseSelector(resource_name, typed_resource.selector(), + selector_update_version); + if (!pending_resource_map.emplace(resource_name, + ResourceKey::selectorResource(selector_handle))) { + throw EnvoyException(fmt::format( + "NetworkPolicyResource selector update for version {} has duplicate resource key " + "'{}' on an old stream: " + "incoming selector resource '{}' collides with existing {}", + system_version_info, resource_name, resource_name, + pending_resource_map.describeExistingResourceKey(resource_name, *load()))); + } + break; + } + case cilium::NetworkPolicyResource::kPolicy: + IS_ENVOY_BUG("Selector-only NetworkPolicyResource update unexpectedly included a policy"); + break; + case cilium::NetworkPolicyResource::RESOURCE_NOT_SET: + throw EnvoyException("NetworkPolicyResource has no payload"); } } } catch (const EnvoyException& e) { - ENVOY_LOG(warn, "NetworkPolicy update for version {} failed: {}", version_info, e.what()); + ENVOY_LOG(warn, "NetworkPolicyResource update for version {} failed: {}", system_version_info, + e.what()); stats_.updates_rejected_.inc(); - - removeInitManager(); + scheduleSelectorDeferredDeletion(selector_map_.revert()); throw; // re-throw } - removeInitManager(); - // Initialize SDS secrets. We do not wait for the completion. - version_init_manager.initialize(Init::WatcherImpl(version_name, []() {})); + // Same-stream selector-only updates become visible to existing policies by first publishing the + // selector version itself and only then publishing that version number through the shared + // stream state. Reversing this order would let workers observe a selector version that has not + // yet been published in the selector map. + auto new_version = selector_map_.publishNextVersion(); + if (new_version > 0) { + policy_stream_state_->publishVersion(new_version); + scheduleSelectorGCAndDeferredDeletion(new_version); + } + std::move(pending_resource_map).applyTo(resource_map_); + return absl::OkStatus(); + } + + std::string version_name = fmt::format("NetworkPolicyMap version {}", system_version_info); + Init::ManagerImpl version_init_manager(version_name); + transport_factory_context_->setInitManager(version_init_manager); + + const auto* old_policy_map = load(); + PolicyMapSnapshot new_policy_map = is_new_stream ? PolicyMapSnapshot{} : *old_policy_map; + ResourceMapOverlay pending_resource_map = + is_new_stream ? ResourceMapOverlay() : ResourceMapOverlay(resource_map_); + const auto policy_stream_state = + is_new_stream + ? std::make_shared(stream_generation, selector_map_.getVersion()) + : policy_stream_state_; + try { + const auto selector_update_version = selector_map_.prepareNextVersion(); + + for (const auto& removed_resource : removed_resources) { + ENVOY_LOG(trace, "Cilium removing NetworkPolicyResource {}", removed_resource); + const auto* resource_entry = pending_resource_map.findEntry(removed_resource); + if (resource_entry == nullptr) { + continue; + } + if (resource_entry->selectorResourceEntry()) { + selector_map_.clear(removed_resource); + pending_resource_map.erase(removed_resource); + continue; + } + if (pending_resource_map.erasePolicyResourceIfPresent(new_policy_map, removed_resource)) { + continue; + } + throw EnvoyException( + fmt::format("NetworkPolicyResource removed resource '{}' is a policy endpoint IP alias, " + "not a resource name", + removed_resource)); + } + + for (const auto& resource : added_resources) { + const auto& typed_resource = + dynamic_cast(resource.get().resource()); + const std::string& resource_name = resource.get().name(); + const auto* resource_entry = pending_resource_map.findEntry(resource_name); + if (resource_entry == nullptr) { + continue; + } + + switch (typed_resource.resource_case()) { + case cilium::NetworkPolicyResource::kSelector: + pending_resource_map.eraseSelectorResourceIfPresent(resource_name); + break; + case cilium::NetworkPolicyResource::kPolicy: + pending_resource_map.erasePolicyResourceIfPresent(new_policy_map, resource_name); + break; + case cilium::NetworkPolicyResource::RESOURCE_NOT_SET: + break; + } + } + + for (const auto& resource : added_resources) { + const auto& typed_resource = + dynamic_cast(resource.get().resource()); + const std::string& resource_name = resource.get().name(); - // Swap the new map in, new_map goes out of scope right after to eliminate accidental - // modification. - old_map = exchange(new RawPolicyMap(std::move(new_map))); + if (typed_resource.resource_case() != cilium::NetworkPolicyResource::kSelector) { + continue; + } + + ENVOY_LOG(debug, + "Received NetworkPolicyResource selector {} in onConfigUpdate() " + "version {}", + resource_name, system_version_info); + auto selector_handle = + createOrReuseSelector(resource_name, typed_resource.selector(), selector_update_version); + if (!pending_resource_map.emplace(resource_name, + ResourceKey::selectorResource(selector_handle))) { + throw EnvoyException(fmt::format( + "NetworkPolicyResource update for version {} has duplicate resource key '{}' on {} " + "stream: " + "incoming selector resource '{}' collides with existing {}", + system_version_info, resource_name, is_new_stream ? "a new" : "an old", resource_name, + pending_resource_map.describeExistingResourceKey(resource_name, new_policy_map))); + } + } + + for (const auto& resource : added_resources) { + const auto& typed_resource = + dynamic_cast(resource.get().resource()); + const std::string& resource_name = resource.get().name(); + + switch (typed_resource.resource_case()) { + case cilium::NetworkPolicyResource::kSelector: + break; + case cilium::NetworkPolicyResource::kPolicy: { + const auto& config = typed_resource.policy(); + if (config.endpoint_ips().empty()) { + throw EnvoyException("NetworkPolicyResource has no endpoint ips"); + } + if (config.endpoint_id() == 0) { + throw EnvoyException("NetworkPolicyResource endpoint_id must be non-zero"); + } + ENVOY_LOG(debug, + "Received NetworkPolicyResource {} for endpoint {}, endpoint_ip {} in " + "onConfigUpdate() version {}", + resource_name, config.endpoint_id(), config.endpoint_ips()[0], + system_version_info); + + auto policy = createOrReusePolicy(resource_name, config, policy_stream_state, resource_map_, + &pending_resource_map); + if (!pending_resource_map.emplace(resource_name, ResourceKey::policyResource(policy))) { + throw EnvoyException(fmt::format( + "NetworkPolicyResource update for version {} has duplicate resource key '{}' on {} " + "stream: " + "incoming {} collides with existing {}", + system_version_info, resource_name, is_new_stream ? "a new" : "an old", + describePolicyResourceForLog(resource_name, config), + pending_resource_map.describeExistingResourceKey(resource_name, new_policy_map))); + } + for (const auto& endpoint_ip : config.endpoint_ips()) { + ENVOY_LOG(trace, "Cilium updating network policy for endpoint {}", endpoint_ip); + if (!pending_resource_map.emplace(endpoint_ip, ResourceKey::policyEndpointIp())) { + throw EnvoyException(fmt::format( + "NetworkPolicyResource update for version {} has duplicate resource key '{}' on {} " + "stream: " + "incoming {} collides with existing {}", + system_version_info, endpoint_ip, is_new_stream ? "a new" : "an old", + describePolicyResourceForLog(resource_name, config), + pending_resource_map.describeExistingResourceKey(endpoint_ip, new_policy_map))); + } + if (!new_policy_map.emplace(endpoint_ip, policy).second) { + throw EnvoyException(fmt::format( + "NetworkPolicyResource update for version {} has duplicate resource key '{}' on {} " + "stream: " + "incoming {} collides with existing {}", + system_version_info, endpoint_ip, is_new_stream ? "a new" : "an old", + describePolicyResourceForLog(resource_name, config), + pending_resource_map.describeExistingResourceKey(endpoint_ip, new_policy_map))); + } + } + break; + } + case cilium::NetworkPolicyResource::RESOURCE_NOT_SET: + throw EnvoyException("NetworkPolicyResource has no payload"); + } + } + } catch (const EnvoyException& e) { + ENVOY_LOG(warn, "NetworkPolicyResource update for version {} failed: {}", system_version_info, + e.what()); + stats_.updates_rejected_.inc(); + removeInitManager(); + scheduleSelectorDeferredDeletion(selector_map_.revert()); + throw; // re-throw + } + removeInitManager(); + installNewPolicyMap(std::move(new_policy_map), version_init_manager, std::move(version_name), + policy_stream_state); + // do not carry over any resources from an old stream + if (is_new_stream) { + resource_map_.clear(); } + std::move(pending_resource_map).applyTo(resource_map_); - // Delete the old map once all worker threads have entered their event queues, as this - // is proof that they no longer refer to the old map. - runAfterAllThreads([old_map]() { - // Clean-up in the main thread after all threads have scheduled - delete old_map; - }); - stats_.update_success_.inc(); return absl::OkStatus(); } @@ -2087,29 +2956,17 @@ void NetworkPolicyMapImpl::onConfigUpdateFailed(Envoy::Config::ConfigUpdateFailu const EnvoyException*) { // We need to allow server startup to continue, even if we have a bad // config. - ENVOY_LOG(debug, "Network Policy Update failed, keeping existing policy."); -} - -void NetworkPolicyMapImpl::runAfterAllThreads(std::function cb) const { - // We can guarantee the callback 'cb' runs in the main thread after all worker threads have - // entered their event loop, and thus relinquished all state, such as policy lookup results that - // were stored in their call stack, by posting and empty function to their event queues and - // waiting until all of them have returned, as managed by 'runOnAllWorkerThreads'. - // - // For now we rely on the implementation dependent fact that the reference returned by - // context_.threadLocal() actually is a ThreadLocal::Instance reference, where - // runOnAllWorkerThreads() is exposed. Without this cast we'd need to use a dummy thread local - // variable that would take a thread local slot for no other purpose than to avoid this type cast. - dynamic_cast(context_.threadLocal()).runOnAllWorkerThreads([]() {}, cb); + ENVOY_LOG(debug, "NetworkPolicy update on stream {} failed, keeping existing policy.", + streamGeneration()); } ProtobufTypes::MessagePtr -NetworkPolicyMap::dumpNetworkPolicyConfigs(const Matchers::StringMatcher& name_matcher) { +NetworkPolicyMapImpl::dumpNetworkPolicyConfigs(const Matchers::StringMatcher& name_matcher) { ENVOY_LOG(debug, "Writing NetworkPolicies to NetworkPoliciesConfigDump"); std::vector policy_endpoint_ids; auto config_dump = std::make_unique(); - for (const auto& item : *getImpl().load()) { + for (const auto& item : *load()) { // filter duplicates (policies are stored per endpoint ip) if (std::find(policy_endpoint_ids.begin(), policy_endpoint_ids.end(), item.second->policy_proto_.endpoint_id()) != policy_endpoint_ids.end()) { @@ -2144,7 +3001,7 @@ class AllowAllEgressPolicyInstanceImpl : public PolicyInstance { } const PortPolicy findPortPolicy(bool ingress, uint16_t) const override { - return ingress ? PortPolicy(empty_map_, 0) : PortPolicy(empty_map_, 1); + return ingress ? PortPolicy(empty_map_, 0, versionMin) : PortPolicy(empty_map_, 1, versionMin); } bool useProxylib(bool, uint16_t, uint32_t, uint16_t, std::string&) const override { @@ -2160,16 +3017,17 @@ class AllowAllEgressPolicyInstanceImpl : public PolicyInstance { void tlsWrapperMissingPolicyInc() const override {} private: - PolicyMap empty_map_; + PolicySnapshot empty_map_; static const std::string empty_string; static const IpAddressPair empty_ips; }; const std::string AllowAllEgressPolicyInstanceImpl::empty_string = ""; const IpAddressPair AllowAllEgressPolicyInstanceImpl::empty_ips{}; -AllowAllEgressPolicyInstanceImpl NetworkPolicyMap::AllowAllEgressPolicy; - -PolicyInstance& NetworkPolicyMap::getAllowAllEgressPolicy() { return AllowAllEgressPolicy; } +PolicyInstance& NetworkPolicyMap::getAllowAllEgressPolicy() { + static AllowAllEgressPolicyInstanceImpl allow_all_egress_policy; + return allow_all_egress_policy; +} // Deny-all policy class DenyAllPolicyInstanceImpl : public PolicyInstance { @@ -2186,7 +3044,7 @@ class DenyAllPolicyInstanceImpl : public PolicyInstance { } const PortPolicy findPortPolicy(bool, uint16_t) const override { - return PortPolicy(empty_map_, 0); + return PortPolicy(empty_map_, 0, versionMin); } bool useProxylib(bool, uint16_t, uint32_t, uint16_t, std::string&) const override { @@ -2202,16 +3060,17 @@ class DenyAllPolicyInstanceImpl : public PolicyInstance { void tlsWrapperMissingPolicyInc() const override {} private: - PolicyMap empty_map_; + PolicySnapshot empty_map_; static const std::string empty_string; static const IpAddressPair empty_ips; }; const std::string DenyAllPolicyInstanceImpl::empty_string = ""; const IpAddressPair DenyAllPolicyInstanceImpl::empty_ips{}; -DenyAllPolicyInstanceImpl NetworkPolicyMap::DenyAllPolicy; - -PolicyInstance& NetworkPolicyMap::getDenyAllPolicy() { return DenyAllPolicy; } +PolicyInstance& NetworkPolicyMap::getDenyAllPolicy() { + static DenyAllPolicyInstanceImpl deny_all_policy; + return deny_all_policy; +} const PolicyInstance* NetworkPolicyMapImpl::getPolicyInstanceImpl(const std::string& endpoint_ip) const { @@ -2223,6 +3082,32 @@ NetworkPolicyMapImpl::getPolicyInstanceImpl(const std::string& endpoint_ip) cons return nullptr; } +PolicyInstanceConstSharedPtr +NetworkPolicyMapImpl::getPolicyInstanceSharedImpl(const std::string& endpoint_ip) const { + const auto* map = load(); + auto it = map->find(endpoint_ip); + if (it != map->end()) { + return it->second; + } + return nullptr; +} + +uint64_t NetworkPolicyMapImpl::policySelectorStreamGenerationForTestImpl( + const PolicyInstance& policy) const { + if (const auto* policy_impl = dynamic_cast(&policy)) { + return policy_impl->policy_stream_state_->streamGeneration(); + } + return 0; +} + +SelectorVersion +NetworkPolicyMapImpl::policySelectorVersionForTestImpl(const PolicyInstance& policy) const { + if (const auto* policy_impl = dynamic_cast(&policy)) { + return policy_impl->policy_stream_state_->version(); + } + return versionMin; +} + // getPolicyInstance return a const reference to a policy in the policy map for the given // 'endpoint_ip'. If there is no policy for the given IP, a default policy is returned, // controlled by the 'default_allow_egress' argument as follows: @@ -2236,10 +3121,8 @@ NetworkPolicyMapImpl::getPolicyInstanceImpl(const std::string& endpoint_ip) cons // for Cilium Ingress when there is no egress policy enforcement for the Ingress traffic. const PolicyInstance& NetworkPolicyMap::getPolicyInstance(const std::string& endpoint_ip, bool default_allow_egress) const { - const auto* policy = getImpl().getPolicyInstanceImpl(endpoint_ip); - return policy != nullptr ? *policy - : default_allow_egress ? *static_cast(&AllowAllEgressPolicy) - : *static_cast(&DenyAllPolicy); + const auto* policy = impl_->getPolicyInstanceImpl(endpoint_ip); + return policy ? *policy : default_allow_egress ? getAllowAllEgressPolicy() : getDenyAllPolicy(); } } // namespace Cilium diff --git a/cilium/network_policy.h b/cilium/network_policy.h index fd60fe2fd..55eecc942 100644 --- a/cilium/network_policy.h +++ b/cilium/network_policy.h @@ -2,48 +2,34 @@ #include -#include #include #include #include #include -#include -#include #include "envoy/common/exception.h" -#include "envoy/common/matchers.h" #include "envoy/common/pure.h" #include "envoy/common/regex.h" #include "envoy/config/core/v3/base.pb.h" #include "envoy/config/core/v3/config_source.pb.h" #include "envoy/config/subscription.h" -#include "envoy/event/dispatcher_thread_deletable.h" #include "envoy/http/header_map.h" #include "envoy/network/address.h" #include "envoy/protobuf/message_validator.h" -#include "envoy/server/config_tracker.h" #include "envoy/server/factory_context.h" -#include "envoy/server/transport_socket_config.h" #include "envoy/singleton/instance.h" #include "envoy/ssl/context.h" #include "envoy/ssl/context_config.h" -#include "envoy/stats/scope.h" #include "envoy/stats/stats_macros.h" // IWYU pragma: keep #include "source/common/common/assert.h" #include "source/common/common/logger.h" #include "source/common/common/macros.h" #include "source/common/common/thread.h" -#include "source/common/init/manager_impl.h" -#include "source/common/init/target_impl.h" #include "source/common/protobuf/message_validator_impl.h" #include "source/common/protobuf/protobuf.h" #include "source/common/protobuf/utility.h" -#include "source/server/transport_socket_config_impl.h" -#include "absl/container/btree_map.h" -#include "absl/container/flat_hash_map.h" -#include "absl/status/status.h" #include "absl/strings/ascii.h" #include "absl/strings/string_view.h" #include "cilium/accesslog.h" @@ -54,35 +40,9 @@ namespace Envoy { namespace Cilium { -// PortRangeCompare is used for as std::less replacement for port range keys. -// -// All port ranges in the map have non-overlapping keys, which allows total ordering needed for -// ordered map containers. When inserting new ranges, any range overlap will be flagged as a -// "duplicate" entry, as overlapping keys are considered equal (as neither is strictly less than the -// other given this comparison predicate). -// On lookups we'll set both ends of the port range to the same port number, which will find the one -// range that it overlaps with, if one exists. -using PortRange = std::pair; -struct PortRangeCompare { - bool operator()(const PortRange& a, const PortRange& b) const { - // return true if range 'a.first - a.second' is below range 'b.first - b.second'. - return a.second < b.first; - } -}; - class PortNetworkPolicyRules; - -// PolicyMap is keyed by port ranges, and contains a list of PortNetworkPolicyRules's applicable -// to this range. A list is needed as rules may come from multiple sources (e.g., resulting from -// use of named ports and numbered ports in Cilium Network Policy at the same time). -using PolicyMap = absl::btree_map; - -// Supported message types -using RuleVerdict = enum { - None = 0, - Allow = 1, - Deny = 2, -}; +class PolicySnapshot; +using SelectorVersion = uint64_t; // PortPolicy holds a reference to a set of rules in a policy map that apply to the given port. // Methods then iterate through the set to determine if policy allows or denies. This is needed to @@ -93,7 +53,7 @@ class PortPolicy : public Logger::Loggable { friend class PortNetworkPolicy; friend class DenyAllPolicyInstanceImpl; friend class AllowAllEgressPolicyInstanceImpl; - PortPolicy(const PolicyMap& map, uint16_t port); + PortPolicy(const PolicySnapshot& map, uint16_t port, SelectorVersion selector_version); public: // If hasHttpRules() returns false, then HTTP policy enforcement can be skipped, @@ -129,7 +89,6 @@ class PortPolicy : public Logger::Loggable { bool& raw_socket_allowed) const; private: - const PolicyMap& map_; // using raw pointers by design: // - pointer to distinguish between no rules and empty rules // - not using shared pointer to not allow a worker thread to hold the last reference to policy @@ -140,6 +99,7 @@ class PortPolicy : public Logger::Loggable { // rules. const PortNetworkPolicyRules* port_rules_; const bool has_http_rules_; + const SelectorVersion selector_version_; }; class IpAddressPair { @@ -187,8 +147,6 @@ class PolicyInstance { }; using PolicyInstanceConstSharedPtr = std::shared_ptr; -class PolicyInstanceImpl; - class NetworkPolicyDecoder : public Envoy::Config::OpaqueResourceDecoder { public: NetworkPolicyDecoder() : validation_visitor_(ProtobufMessage::getNullValidationVisitor()) {} @@ -213,175 +171,102 @@ class NetworkPolicyDecoder : public Envoy::Config::OpaqueResourceDecoder { ProtobufMessage::ValidationVisitor& validation_visitor_; }; +// cilium::NetworkPolicyResource does not carry a resource name, but relies on the +// DeltaDiscoveryRespons Resource wrapper to have the name. Hence can not use +// Envoy::Config::OpaqueResourceDecoderImpl +class NetworkPolicyResourceDecoder : public Envoy::Config::OpaqueResourceDecoder { +public: + NetworkPolicyResourceDecoder() + : validation_visitor_(ProtobufMessage::getNullValidationVisitor()) {} + + // Config::OpaqueResourceDecoder + ProtobufTypes::MessagePtr decodeResource(const Protobuf::Any& resource) override { + auto typed_message = std::make_unique(); + // If the Any is a synthetic empty message (e.g. because the resource field + // was not set in Resource, this might be empty, so we shouldn't decode. + if (!resource.type_url().empty()) { + MessageUtil::anyConvertAndValidate(resource, *typed_message, + validation_visitor_); + } + return typed_message; + } + + std::string resourceName(const Protobuf::Message&) override { + throw EnvoyException( + "NetworkPolicyResource does not carry a name and must be wrapped in Resource"); + } + +private: + ProtobufMessage::ValidationVisitor& validation_visitor_; +}; + /** * All Cilium L7 filter stats. @see stats_macros.h */ // clang-format off -#define ALL_CILIUM_POLICY_STATS(COUNTER) \ - COUNTER(updates_total) \ - COUNTER(updates_rejected) \ - COUNTER(tls_wrapper_missing_policy) \ +#define ALL_CILIUM_POLICY_COUNTERS(COUNTER) \ + COUNTER(updates_total) \ + COUNTER(updates_rejected) \ + COUNTER(tls_wrapper_missing_policy) \ COUNTER(update_success) + +#define ALL_CILIUM_POLICY_GAUGES(GAUGE) \ + GAUGE(policy_stream_generation, NeverImport) // clang-format on /** * Struct definition for all policy stats. @see stats_macros.h */ struct PolicyStats { - ALL_CILIUM_POLICY_STATS(GENERATE_COUNTER_STRUCT) + ALL_CILIUM_POLICY_COUNTERS(GENERATE_COUNTER_STRUCT) + ALL_CILIUM_POLICY_GAUGES(GENERATE_GAUGE_STRUCT) }; -using RawPolicyMap = absl::flat_hash_map>; - -class NetworkPolicyMapImpl : public Envoy::Config::SubscriptionCallbacks, - public Envoy::Event::DispatcherThreadDeletable, - public Logger::Loggable { -public: - NetworkPolicyMapImpl(Server::Configuration::FactoryContext& context, - const envoy::config::core::v3::ConfigSource& npds_config); - ~NetworkPolicyMapImpl() override; - - void startSubscription(const envoy::config::core::v3::ConfigSource& npds_config); - - // This is used for testing with a file-based subscription - void startSubscription(std::unique_ptr&& subscription) { - subscription_ = std::move(subscription); - } - - const envoy::config::core::v3::ConfigSource& getConfigSource() const { return npds_config_; } - - // run the given function after all the threads have scheduled - void runAfterAllThreads(std::function) const; - - // Config::SubscriptionCallbacks - absl::Status onConfigUpdate(const std::vector& resources, - const std::string& version_info) override; - absl::Status onConfigUpdate(const std::vector& added_resources, - const Protobuf::RepeatedPtrField& removed_resources, - const std::string& system_version_info) override { - // NOT IMPLEMENTED YET. - UNREFERENCED_PARAMETER(added_resources); - UNREFERENCED_PARAMETER(removed_resources); - UNREFERENCED_PARAMETER(system_version_info); - return absl::OkStatus(); - } - void onConfigUpdateFailed(Envoy::Config::ConfigUpdateFailureReason, - const EnvoyException* e) override; - - Server::Configuration::TransportSocketFactoryContext& transportFactoryContext() const { - return *transport_factory_context_; - } - - Regex::Engine& regexEngine() const { return context_.regexEngine(); } - - void tlsWrapperMissingPolicyInc() const; - -private: - // Helpers for atomic swap of the policy map pointer. - // - // store() is only used for the initialization of the map during construction. - // exchange() is used to atomically swap in a new map, the old map pointer is returned. - // Once a map is stored or swapped in to the atomic pointer by the main thread, it may be "loaded" - // from the atomic pointer by any thread. This is why the load returns a const pointer. - // - // For the loaded pointer to be safe to use, we must use acquire/release memory ordering: - // - when a pointer stored or swapped in, 'std::memory_order_release' informs the compiler to make - // sure it is not reordering any write operations into the map to happen after the pointer is - // written, and emits CPU instructions to also make the CPU out-of-order-execution logic to not - // reorder any write operations to happen after the pointer itself is written. This guarantees - // that the map is not modified after the point when the worker threads can observe the new - // pointer value, i.e., the map is actaully immutable (const) from that point forward. - // - when the pointer is read (by a worker thread) 'std::memory_order_acquire' in the load - // operation informs the compiler to emit CPU instructions to make the CPU - // out-of-order-execution logic to not reorder any reads from the new map to happen before the - // pointer itself is read, so that no values from the map are read before the map was "released" - // by the store or exchange operation. - // - // Typically it is easier to think about the release part of the acquire/release semantics, as at - // the point of the store or exchange operation the compiler and the CPU know the location of the - // map in memory before and after the pointer is stored, so that without - // 'std::memory_order_release' there is an understandable risk of such write after release - // happening. On the acquire side it seems less likely that the compiler or the CPU could know the - // new map pointer value in advance and even try to reorder any read operations to happen before - // the pointer is actually read. But consider the typical case where the pointer value is actually - // not changing between consecutice load operations. The compiler or the CPU could speculate that - // to be the case and read some values from the old memory location. 'std::memory_order_acquire' - // tells the compiler (which then "tells" the CPU) that this can not be done, and all reads must - // actually happen after the pointer value is loaded, be it a new one or the same as before. - // - const RawPolicyMap* load() const { return map_ptr_.load(std::memory_order_acquire); } - void store(const RawPolicyMap* map) { map_ptr_.store(map, std::memory_order_release); } - const RawPolicyMap* exchange(const RawPolicyMap* map) { - return map_ptr_.exchange(map, std::memory_order_release); - } - - const PolicyInstance* getPolicyInstanceImpl(const std::string& endpoint_policy_name) const; - - void removeInitManager(); - - bool isNewStream(); - - static uint64_t instance_id_; - - Server::Configuration::ServerFactoryContext& context_; - std::atomic map_ptr_; - Stats::ScopeSharedPtr npds_stats_scope_; - Stats::ScopeSharedPtr policy_stats_scope_; - - // init target which starts gRPC subscription - Init::TargetImpl init_target_; - std::shared_ptr - transport_factory_context_; - // Between policy updates, keep a dormant init manager installed so unexpected late init-target - // registrations do not hit the listener's already-initialized manager. If it accumulates targets - // while parked, log and rotate it out before making it active again. - std::unique_ptr parked_init_manager_; - - std::unique_ptr subscription_; - envoy::config::core::v3::ConfigSource npds_config_; - -protected: - friend class NetworkPolicyMap; - friend class CiliumNetworkPolicyTest; - - PolicyStats stats_; -}; - -class DenyAllPolicyInstanceImpl; -class AllowAllEgressPolicyInstanceImpl; +class NetworkPolicyMapImpl; class NetworkPolicyMap : public Singleton::Instance, public Logger::Loggable { public: NetworkPolicyMap(Server::Configuration::FactoryContext& context, - const envoy::config::core::v3::ConfigSource& npds_config, + const envoy::config::core::v3::ConfigSource& config_source, bool subscribe = false); + ~NetworkPolicyMap() override; - // This is used for testing with a file-based subscription - void startSubscription(std::unique_ptr&& subscription) { - getImpl().startSubscription(std::move(subscription)); - } + bool exists(const std::string& endpoint_policy_name) const; + bool useDeltaXds() const; + + void setConfigSource(const envoy::config::core::v3::ConfigSource& config_source) const; const PolicyInstance& getPolicyInstance(const std::string& endpoint_policy_name, bool allow_egress) const; - static DenyAllPolicyInstanceImpl DenyAllPolicy; static PolicyInstance& getDenyAllPolicy(); - static AllowAllEgressPolicyInstanceImpl AllowAllEgressPolicy; static PolicyInstance& getAllowAllEgressPolicy(); - bool exists(const std::string& endpoint_policy_name) const { - return getImpl().getPolicyInstanceImpl(endpoint_policy_name) != nullptr; - } + using SubscriptionFactoryForTest = + std::function(bool use_delta_xds)>; - NetworkPolicyMapImpl& getImpl() const { return *impl_; } +protected: + friend class CiliumNetworkPolicyTest; + friend struct TestHelper; + PolicyStats& statsForTest() const; + void resetStreamForTest(); + PolicyInstanceConstSharedPtr + getPolicyInstanceSharedForTest(const std::string& endpoint_policy_name) const; + uint64_t policySelectorStreamGenerationForTest(const PolicyInstance& policy) const; + SelectorVersion policySelectorVersionForTest(const PolicyInstance& policy) const; + void startSubscriptionForTest(std::unique_ptr&& subscription); + void startManagedSubscriptionForTest(); + void setSubscriptionFactoryForTest(SubscriptionFactoryForTest factory); + void onSubscriptionConnectedForTest(); + void onSubscriptionTransportCloseForTest(); + bool subscriptionUseDeltaXdsForTest() const; + bool subscriptionConnectedForTest() const; + Envoy::Config::SubscriptionCallbacks& subscriptionCallbacksForTest() const; private: Server::Configuration::ServerFactoryContext& context_; - std::unique_ptr impl_; - - ProtobufTypes::MessagePtr dumpNetworkPolicyConfigs(const Matchers::StringMatcher& name_matcher); - Server::ConfigTracker::EntryOwnerPtr config_tracker_entry_; + std::shared_ptr impl_; }; using NetworkPolicyMapSharedPtr = std::shared_ptr; diff --git a/cilium/secret_watcher.cc b/cilium/secret_watcher.cc index 1961a7f56..b3b4093fc 100644 --- a/cilium/secret_watcher.cc +++ b/cilium/secret_watcher.cc @@ -23,7 +23,6 @@ #include "absl/status/status.h" #include "absl/synchronization/mutex.h" #include "cilium/api/npds.pb.h" -#include "cilium/network_policy.h" namespace Envoy { namespace Cilium { @@ -38,24 +37,28 @@ getCiliumSDSConfig(const std::string&, const envoy::config::core::v3::ConfigSour return config_source; } +GetSdsConfigFunc getSDSConfig = &getCiliumSDSConfig; + Secret::GenericSecretConfigProviderSharedPtr secretProvider(Server::Configuration::TransportSocketFactoryContext& context, - const std::string& sds_name, const NetworkPolicyMapImpl& parent) { - const envoy::config::core::v3::ConfigSource& config_source = - getSDSConfig(sds_name, parent.getConfigSource()); + const envoy::config::core::v3::ConfigSource& config_source, + const std::string& sds_name) { + const envoy::config::core::v3::ConfigSource& sds_config_source = + getSDSConfig(sds_name, config_source); return context.serverFactoryContext().secretManager().findOrCreateGenericSecretProvider( - config_source, sds_name, context.serverFactoryContext(), context.initManager()); + sds_config_source, sds_name, context.serverFactoryContext(), context.initManager()); } } // namespace -GetSdsConfigFunc getSDSConfig = &getCiliumSDSConfig; void setSDSConfigFunc(GetSdsConfigFunc func) { getSDSConfig = func; } void resetSDSConfigFunc() { getSDSConfig = &getCiliumSDSConfig; } -SecretWatcher::SecretWatcher(const NetworkPolicyMapImpl& parent, const std::string& sds_name) - : parent_(parent), name_(sds_name), - secret_provider_(secretProvider(parent.transportFactoryContext(), sds_name, parent)), +SecretWatcher::SecretWatcher(Server::Configuration::TransportSocketFactoryContext& context, + const envoy::config::core::v3::ConfigSource& config_source, + const std::string& sds_name) + : context_(context), name_(sds_name), + secret_provider_(secretProvider(context, config_source, sds_name)), update_secret_(readAndWatchSecret()) {} SecretWatcher::~SecretWatcher() { @@ -74,7 +77,7 @@ Envoy::Common::CallbackHandlePtr SecretWatcher::readAndWatchSecret() { absl::Status SecretWatcher::store() { const auto* secret = secret_provider_->secret(); if (secret != nullptr) { - Api::Api& api = parent_.transportFactoryContext().serverFactoryContext().api(); + Api::Api& api = context_.serverFactoryContext().api(); auto string_or_error = Config::DataSource::read(secret->secret(), true, api); if (!string_or_error.ok()) { return string_or_error.status(); @@ -82,8 +85,9 @@ absl::Status SecretWatcher::store() { std::string* p = new std::string(string_or_error.value()); std::string* old = ptr_.exchange(p, std::memory_order_release); if (old != nullptr) { - // Delete old value after all threads have scheduled - parent_.runAfterAllThreads([old]() { delete old; }); + // Delete old value after all worker threads have scheduled + context_.serverFactoryContext().threadLocal().runOnAllWorkerThreads([]() {}, + [old]() { delete old; }); } } return absl::OkStatus(); @@ -91,21 +95,22 @@ absl::Status SecretWatcher::store() { const std::string* SecretWatcher::load() const { return ptr_.load(std::memory_order_acquire); } -TLSContext::TLSContext(const NetworkPolicyMapImpl& parent, const std::string& name) - : manager_(parent.transportFactoryContext().serverFactoryContext().sslContextManager()), - scope_(parent.transportFactoryContext().serverFactoryContext().serverScope()), +TLSContext::TLSContext(Server::Configuration::TransportSocketFactoryContext& context, + const std::string& name) + : manager_(context.serverFactoryContext().sslContextManager()), + scope_(context.serverFactoryContext().serverScope()), init_target_(fmt::format("TLS Context {} secret", name), []() {}) {} namespace { void setCommonConfig(const cilium::TLSContext config, - envoy::extensions::transport_sockets::tls::v3::CommonTlsContext* tls_context, - const NetworkPolicyMapImpl& parent) { + const envoy::config::core::v3::ConfigSource& config_source, + envoy::extensions::transport_sockets::tls::v3::CommonTlsContext* tls_context) { if (!config.validation_context_sds_secret().empty()) { auto sds_secret = tls_context->mutable_validation_context_sds_secret_config(); sds_secret->set_name(config.validation_context_sds_secret()); - auto* config_source = sds_secret->mutable_sds_config(); - *config_source = getSDSConfig(config.validation_context_sds_secret(), parent.getConfigSource()); + auto* mutable_config_source = sds_secret->mutable_sds_config(); + *mutable_config_source = getSDSConfig(sds_secret->name(), config_source); } else if (!config.trusted_ca().empty()) { auto validation_context = tls_context->mutable_validation_context(); auto trusted_ca = validation_context->mutable_trusted_ca(); @@ -114,8 +119,8 @@ void setCommonConfig(const cilium::TLSContext config, if (!config.tls_sds_secret().empty()) { auto sds_secret = tls_context->add_tls_certificate_sds_secret_configs(); sds_secret->set_name(config.tls_sds_secret()); - auto* config_source = sds_secret->mutable_sds_config(); - *config_source = getSDSConfig(config.tls_sds_secret(), parent.getConfigSource()); + auto* mutable_config_source = sds_secret->mutable_sds_config(); + *mutable_config_source = getSDSConfig(sds_secret->name(), config_source); } else if (!config.certificate_chain().empty()) { auto tls_certificate = tls_context->add_tls_certificates(); auto certificate_chain = tls_certificate->mutable_certificate_chain(); @@ -137,9 +142,10 @@ void setCommonConfig(const cilium::TLSContext config, } // namespace -DownstreamTLSContext::DownstreamTLSContext(const NetworkPolicyMapImpl& parent, - const cilium::TLSContext config) - : TLSContext(parent, "server") { +DownstreamTLSContext::DownstreamTLSContext( + Server::Configuration::TransportSocketFactoryContext& context, + const envoy::config::core::v3::ConfigSource& config_source, const cilium::TLSContext config) + : TLSContext(context, "server") { // Server config always needs the TLS certificate to present to the client if (config.tls_sds_secret().empty() && config.certificate_chain().empty()) { throw EnvoyException("Downstream TLS Context: missing certificate chain"); @@ -153,13 +159,13 @@ DownstreamTLSContext::DownstreamTLSContext(const NetworkPolicyMapImpl& parent, auto require_tls_certificate = context_config.mutable_require_client_certificate(); require_tls_certificate->set_value(true); } - setCommonConfig(config, tls_context, parent); + setCommonConfig(config, config_source, tls_context); for (int i = 0; i < config.server_names_size(); i++) { server_names_.emplace_back(config.server_names(i)); } auto server_config_or_error = Extensions::TransportSockets::Tls::ServerContextConfigImpl::create( - context_config, parent.transportFactoryContext(), server_names_, false); + context_config, context, server_names_, false); // NOLINTNEXTLINE(performance-unnecessary-copy-initialization) THROW_IF_NOT_OK(server_config_or_error.status()); server_config_ = std::move(server_config_or_error.value()); @@ -182,13 +188,14 @@ DownstreamTLSContext::DownstreamTLSContext(const NetworkPolicyMapImpl& parent, if (server_config_->isReady()) { static_cast(create_server_context()); } else { - parent.transportFactoryContext().initManager().add(init_target_); + context.initManager().add(init_target_); } } -UpstreamTLSContext::UpstreamTLSContext(const NetworkPolicyMapImpl& parent, - cilium::TLSContext config) - : TLSContext(parent, "client") { +UpstreamTLSContext::UpstreamTLSContext( + Server::Configuration::TransportSocketFactoryContext& context, + const envoy::config::core::v3::ConfigSource& config_source, cilium::TLSContext config) + : TLSContext(context, "client") { // Client context always needs the trusted CA for server certificate validation // TODO: Default to system default trusted CAs? if (config.validation_context_sds_secret().empty() && config.trusted_ca().empty()) { @@ -197,7 +204,7 @@ UpstreamTLSContext::UpstreamTLSContext(const NetworkPolicyMapImpl& parent, envoy::extensions::transport_sockets::tls::v3::UpstreamTlsContext context_config; auto tls_context = context_config.mutable_common_tls_context(); - setCommonConfig(config, tls_context, parent); + setCommonConfig(config, config_source, tls_context); if (config.server_names_size() > 0) { if (config.server_names_size() > 1) { @@ -205,8 +212,8 @@ UpstreamTLSContext::UpstreamTLSContext(const NetworkPolicyMapImpl& parent, } context_config.set_sni(config.server_names(0)); } - auto client_config_or_error = Extensions::TransportSockets::Tls::ClientContextConfigImpl::create( - context_config, parent.transportFactoryContext()); + auto client_config_or_error = + Extensions::TransportSockets::Tls::ClientContextConfigImpl::create(context_config, context); // NOLINTNEXTLINE(performance-unnecessary-copy-initialization) THROW_IF_NOT_OK(client_config_or_error.status()); @@ -229,7 +236,7 @@ UpstreamTLSContext::UpstreamTLSContext(const NetworkPolicyMapImpl& parent, if (client_config_->isReady()) { static_cast(create_client_context()); } else { - parent.transportFactoryContext().initManager().add(init_target_); + context.initManager().add(init_target_); } } diff --git a/cilium/secret_watcher.h b/cilium/secret_watcher.h index 8c0756a1e..ffa90b91c 100644 --- a/cilium/secret_watcher.h +++ b/cilium/secret_watcher.h @@ -12,6 +12,7 @@ #include "envoy/ssl/context.h" #include "envoy/ssl/context_config.h" #include "envoy/ssl/context_manager.h" +#include "envoy/ssl/private_key/private_key.h" #include "envoy/stats/scope.h" #include "source/common/common/logger.h" @@ -21,24 +22,21 @@ #include "absl/status/status.h" #include "absl/synchronization/mutex.h" #include "cilium/api/npds.pb.h" -#include "cilium/network_policy.h" namespace Envoy { namespace Cilium { -// Cilium XDS API config source. Used for all Cilium XDS. -extern const envoy::config::core::v3::ConfigSource CILIUM_XDS_API_CONFIG; - // Facility for SDS config override for testing using GetSdsConfigFunc = std::function; -extern GetSdsConfigFunc getSDSConfig; void setSDSConfigFunc(GetSdsConfigFunc); void resetSDSConfigFunc(); class SecretWatcher : public Logger::Loggable { public: - SecretWatcher(const NetworkPolicyMapImpl& parent, const std::string& sds_name); + SecretWatcher(Server::Configuration::TransportSocketFactoryContext& context, + const envoy::config::core::v3::ConfigSource& config_source, + const std::string& sds_name); ~SecretWatcher(); const std::string& name() const { return name_; } @@ -49,7 +47,7 @@ class SecretWatcher : public Logger::Loggable { absl::Status store(); const std::string* load() const; - const NetworkPolicyMapImpl& parent_; + Server::Configuration::TransportSocketFactoryContext& context_; const std::string name_; std::atomic ptr_{nullptr}; Secret::GenericSecretConfigProviderSharedPtr secret_provider_; @@ -63,7 +61,8 @@ class TLSContext : public Logger::Loggable { TLSContext() = delete; protected: - TLSContext(const NetworkPolicyMapImpl& parent, const std::string& name); + TLSContext(Server::Configuration::TransportSocketFactoryContext& context, + const std::string& name); Envoy::Ssl::ContextManager& manager_; Stats::Scope& scope_; @@ -73,7 +72,9 @@ class TLSContext : public Logger::Loggable { class DownstreamTLSContext : protected TLSContext { public: - DownstreamTLSContext(const NetworkPolicyMapImpl& parent, const cilium::TLSContext config); + DownstreamTLSContext(Server::Configuration::TransportSocketFactoryContext& context, + const envoy::config::core::v3::ConfigSource& config_source, + const cilium::TLSContext config); ~DownstreamTLSContext() { manager_.removeContext(server_context_); } const Ssl::ContextConfig& getTlsContextConfig() const { return *server_config_; } @@ -92,7 +93,9 @@ using DownstreamTLSContextSharedPtr = std::shared_ptr; class UpstreamTLSContext : protected TLSContext { public: - UpstreamTLSContext(const NetworkPolicyMapImpl& parent, cilium::TLSContext config); + UpstreamTLSContext(Server::Configuration::TransportSocketFactoryContext& context, + const envoy::config::core::v3::ConfigSource& config_source, + cilium::TLSContext config); ~UpstreamTLSContext() { manager_.removeContext(client_context_); } const Ssl::ContextConfig& getTlsContextConfig() const { return *client_config_; } diff --git a/cilium/versioned.h b/cilium/versioned.h new file mode 100644 index 000000000..9ba0bfa40 --- /dev/null +++ b/cilium/versioned.h @@ -0,0 +1,518 @@ +#pragma once + +// NOLINT(namespace-envoy) + +#include +#include +#include +#include +#include +#include + +#include "source/common/common/assert.h" + +#include "absl/container/flat_hash_map.h" +#include "absl/container/flat_hash_set.h" + +constexpr uint64_t versionMin = 0; +constexpr uint64_t versionMax = std::numeric_limits::max() - 1; +constexpr uint64_t versionNotRemoved = std::numeric_limits::max(); + +// Versioned.h provides a lock-free reader / single-writer versioned-value container, +// based on earlier implementations in OVS (C) and Cilium (Go). +// +// API contract and intended use: +// +// 1. Object model +// - T is a CRTP node type inheriting from VersionedNode. +// - VersionedValue stores a linked list of historical T nodes, newest first. +// - VersionedReadable exposes only the worker-safe read API: get(version). +// - VersionedHandle is a shared_ptr, so handles intentionally +// expose only the readable API to worker/runtime code. Main-thread mutation goes through +// VersionedMap, which keeps mutable shared_ptr internally. +// - The optional readable base R allows a handle to expose extra read-only metadata (for +// example selector names) without exposing VersionedValue mutators. +// +// 2. Threading model +// - There is exactly one mutator thread: the main thread. +// - Worker threads may concurrently call VersionedReadable::get(version) and may traverse nodes +// reached from a handle without additional locking. +// - Any future cross-thread mutation must still preserve the exclusive-writer rule, e.g. by +// holding an exclusive lock around all mutation. +// - VersionedMap itself is main-thread-only. Its name directory (map_), dirty set, and +// transactional state must not be accessed from worker threads. +// +// 3. VersionedMap transaction flow +// - prepareNextVersion() starts a new unpublished update and returns the candidate version to +// use for any main-thread lookups against in-flight state. +// - insert(key, value) reuses the existing stable handle if the key is still present in the +// main-thread directory; otherwise it creates a fresh handle. +// - clear(key) marks the key's current value invisible in the candidate version but does not +// erase the key from the main-thread directory immediately. +// - publishNextVersion() publishes all pending changes, removes keys whose handle has no visible +// value in the published version, and returns the new published version, or 0 if there was +// nothing to publish. +// - revert() discards the unpublished update, restores visibility of older nodes as needed, and +// removes keys whose handle has no value in the published version. +// - find(key) returns the stable readable handle parked under the key in the main-thread +// directory. It does not itself answer whether the key is visible in any specific version; use +// handle->get(version) for that. +// +// 4. Stable-handle semantics +// - Updating an active key writes a new version onto the same stable handle. +// - clear() followed by re-add of the same key in the same unpublished update also reuses that +// same handle; the caller should compare against the candidate version returned from +// prepareNextVersion(), not against the currently published version. +// - Once a key has been fully removed across a published version boundary, the name disappears +// from VersionedMap. A later same-name add creates a fresh stable handle. +// - This is why old published policies can keep using an old cleared handle while refreshed +// policies bind a new one after a later same-name re-add. +// +// 5. Reader semantics +// - get(version) walks from the current head and returns the first node visible in the requested +// version. +// - The walk stops at the first node that was added before the requested version but is no +// longer visible there; all remaining nodes are older and therefore already invisible in that +// version. +// - Querying an older published version after newer publishes is logically stale and may produce +// outdated results, but it must remain memory-safe. +// +// 6. Deferred deletion / grace periods +// - Unpublished nodes removed by revert() were never visible to workers, but are still unlinked +// into a DeferredDeletion batch so that any concurrent traversal that had already stepped into +// them cannot race with delete. +// - gc(published_version) is only the first GC phase. It must be called only after all workers +// have quiesced for that published version, unlinks nodes that are no longer visible to +// anyone, and returns them in a DeferredDeletion batch. +// - The returned DeferredDeletion batch must stay alive until all workers have quiesced once +// more. Destroying the batch performs the actual node deletion. +// - The production pattern is therefore: publish version N, wait for worker quiescence, call +// gc(N), then keep the returned DeferredDeletion alive for one more quiescence round. +// +// 7. High-level memory-model rationale +// - The main thread publishes new list links with 'release' stores to the list node pointers. +// - Workers traverse with 'acquire' loads from the list node pointers, so they either observe +// the old reachable chain or the newly published/unlinked chain, but not torn pointer state. +// - published_version_ is stored with 'release' in publishNextVersion() and loaded with +// 'acquire' by readers, so readers that observe a new published version also observe the +// prior main-thread mutations that made that version reachable. +// - remove_version_ uses 'relaxed' atomic loads/stores because it controls logical visibility, +// not structural reachability. Structural coherence and memory safety come from the +// 'release'/'acquire' ordering on list node pointers plus the extra deferred-deletion grace +// period. +// - Many mutation-side loads/stores are 'relaxed' because mutation is single-threaded on the +// main thread. +// +// 8. DeferredDeletion implementation detail +// - DeferredDeletion reuses VersionedNode::next_ to chain together unlinked nodes awaiting +// destruction. This keeps deferred deletion self-contained without an extra node container. +// - Public code must not rely on any semantics of detached invisible tails beyond memory safety; +// only get(version) is the supported read API. +// +// 9. Test coverage in tests/versioned_test.cc +// - Value-level tests cover visibility rules, shadowing, clear(), revert(), first-phase GC, and +// deferred deletion ownership/move semantics. +// - Map-level tests cover version numbering, publish-without-change, insert/update/clear/revert, +// same-handle reuse for active updates and same-update clear+read, fresh-handle behavior +// after published removal, and dirty-handle retention across multiple future GC runs. +// - Stable validator tests verify consistent published snapshots and post-unlink/pre-deletion +// states. +// - A multithreaded chaos test exercises one writer with multiple busy-loop readers, repeated +// publish/revert/GC cycles, stale-version reads, and traversal safety under deferred deletion. +// +template class VersionedReadable; +template > class VersionedValue; +template class DeferredDeletion; +template > struct VersionedTestAccess; + +// VersionedNode is CRTP type on T, T must inherit from VersionedNode +template class VersionedNode { +public: + template + VersionedNode() + : next_(nullptr), add_version_(versionNotRemoved), remove_version_(versionNotRemoved) {} + +private: + template friend class VersionedReadable; + template friend class VersionedValue; + template friend class DeferredDeletion; + template friend struct VersionedTestAccess; + + bool isUnpublished() const { return add_version_ == versionNotRemoved; } + + void setAddVersion(uint64_t version) { + ASSERT(version < versionNotRemoved); + add_version_ = version; // set before published + } + + void removeInVersion(uint64_t version) { + remove_version_.store(version, std::memory_order_relaxed); + } + + bool isVisibleInVersion(uint64_t version) const { + return add_version_ <= version && version < remove_version_.load(std::memory_order_relaxed); + } + + bool isAddedAfterVersion(uint64_t version) const { return add_version_ > version; } + + bool isAddedBeforeVersion(uint64_t version) const { return add_version_ < version; } + + bool isEventuallyInvisible() const { + return remove_version_.load(std::memory_order_relaxed) < versionNotRemoved; + } + + bool isVisibleOnlyBeforeVersion(uint64_t version) const { + return remove_version_.load(std::memory_order_relaxed) <= version; + } + + bool isRemovedAfterVersion(uint64_t version) const { + const auto remove_version = remove_version_.load(std::memory_order_relaxed); + return remove_version < versionNotRemoved && version < remove_version; + } + + VersionedNode* getNext() const { return next_.load(std::memory_order_acquire); } + + void setNext(VersionedNode* next) { return next_.store(next, std::memory_order_release); } + + VersionedNode* getNextProtected() const { return next_.load(std::memory_order_relaxed); } + std::atomic*>* getNextP() { return &next_; } + + T* value() { return static_cast(this); } + const T* value() const { return static_cast(this); } + + std::atomic*> next_; + + uint64_t add_version_; // Version object was added in. + std::atomic remove_version_; // Version object is removed in. +}; + +template class VersionedReadable { +public: + const T* get(uint64_t version) const { + for (auto node = head_.load(std::memory_order_acquire); node; node = node->getNext()) { + if (node->isVisibleInVersion(version)) { + return node->value(); + } + // stop traveral on first non-visible version that was added before this version, all + // remaining nodes are already invisible. + if (node->isAddedBeforeVersion(version)) { + break; + } + } + return nullptr; // not found + } + +protected: + VersionedReadable() : head_(nullptr) {} + + template friend class VersionedValue; + template friend class DeferredDeletion; + template friend struct VersionedTestAccess; + + std::atomic*> head_; +}; + +// T is the CRTP node type stored in the version chain. R is the read-only interface exposed +// through VersionedHandle: it defaults to VersionedReadable, but callers may provide a richer +// readable base when handles need extra read-side API without exposing the main-thread mutators. +template class VersionedValue : public R { +public: + using Readable = R; + using R::head_; + + VersionedValue() = default; + + template >> + explicit VersionedValue(Args&&... args) : R(std::forward(args)...) {} + + ~VersionedValue() { + VersionedNode* next; + for (auto node = head_.load(std::memory_order_relaxed); node; node = next) { + next = node->getNextProtected(); + delete node->value(); + } + } + + // make all versions invisible starting at "version" + void clear(uint64_t version) { + for (auto node = head_.load(std::memory_order_relaxed); node; node = node->getNextProtected()) { + if (!node->isEventuallyInvisible()) { + node->removeInVersion(version); + } + } + } + + void set(uint64_t version, VersionedNode* node) { + // publish a new version that is visible starting at "version" + ASSERT(node->isUnpublished()); // node not pushed into a list yet + node->setAddVersion(version); + node->setNext(head_.load(std::memory_order_relaxed)); + head_.store(node, std::memory_order_release); + + // make all other versions invisible at "version", starting from the first old node. + for (node = node->getNextProtected(); node; node = node->getNextProtected()) { + if (node->isVisibleInVersion(version)) { + node->removeInVersion(version); + } + } + } + + bool empty() const { return head_.load(std::memory_order_relaxed) == nullptr; } + + // Unlink unpublished versions added after 'version' and append them to 'deferred' for deletion + // after an additional grace period. Removed nodes that were still visible in 'version' are + // restored. + void revert(uint64_t version, DeferredDeletion& deferred) { + auto prev = &head_; + VersionedNode* next; + for (auto node = head_.load(std::memory_order_relaxed); node; node = next) { + next = node->getNextProtected(); + if (node->isAddedAfterVersion(version)) { + // unlink node by pointing prev to next + prev->store(next, std::memory_order_release); + deferred.push(node); + // prev stays + continue; + } + if (node->isRemovedAfterVersion(version)) { + // visibility was limited, restore + node->removeInVersion(versionNotRemoved); + } + prev = node->getNextP(); + } + } + + // Unlink all versions not visible to anyone after the "version" was published and all readers + // have quiesced, but do not delete them yet. Unlinked nodes are appended to 'deferred' and are + // only deleted after one more worker-thread quiescence round. Any nodes not yet visible at + // "version" must not be unlinked. + // Returns true if there are no future version removals left in this handle after this first + // phase GC run. + bool gcForVersion(uint64_t version, DeferredDeletion& deferred) { + ASSERT(version < versionNotRemoved); + auto prev = &head_; + VersionedNode* next; + bool has_future_gc_work = false; + for (auto node = head_.load(std::memory_order_relaxed); node; node = next) { + next = node->getNextProtected(); + if (node->isVisibleOnlyBeforeVersion(version)) { + // unlink node by pointing prev to next + prev->store(next, std::memory_order_release); + deferred.push(node); + // prev stays + continue; + } + if (node->isEventuallyInvisible()) { + has_future_gc_work = true; + } + prev = node->getNextP(); + } + return !has_future_gc_work; + } + +protected: + friend class DeferredDeletion; + template friend struct VersionedTestAccess; +}; + +template class DeferredDeletion : protected VersionedValue { +public: + // Bring `head_` from the templated base class into this scope so the rest of the code can use + // normal unqualified member access. Without this, dependent-base lookup would require + // `this->head_` everywhere. + using VersionedValue::head_; + + DeferredDeletion() = default; + DeferredDeletion(const DeferredDeletion&) = delete; + DeferredDeletion& operator=(const DeferredDeletion&) = delete; + DeferredDeletion(DeferredDeletion&& other) noexcept { + head_.store(other.head_.exchange(nullptr, std::memory_order_relaxed), + std::memory_order_relaxed); + tail_ = std::exchange(other.tail_, nullptr); + } + DeferredDeletion& operator=(DeferredDeletion&&) noexcept = delete; + ~DeferredDeletion() = default; + + bool empty() const { return head_.load(std::memory_order_relaxed) == nullptr; } + +private: + template friend class VersionedValue; + + void push(VersionedNode* node) { + node->setNext(nullptr); + if (tail_) { + tail_->setNext(node); + } else { + // Only the main thread is accessing the deferred-deletion head, hence relaxed. + head_.store(node, std::memory_order_relaxed); + } + tail_ = node; + } + + VersionedNode* tail_{nullptr}; +}; + +template > +using VersionedHandle = std::shared_ptr; + +// VersionedMap is main-thread / exclusive-writer only: map_, pending_keys_, dirty_values_, and +// all mutation methods are accessed only by the single publishing thread. Worker threads only use +// the published Handle objects and then call the read-only handle API. +template > class VersionedMap { +public: + static_assert(std::is_base_of_v); + static_assert(std::is_base_of_v, typename V::Readable>); + using Handle = VersionedHandle; + + uint64_t getVersion() const { return published_version_.load(std::memory_order_acquire); } + + Handle find(const K& key) const { + auto it = map_.find(key); + if (it != map_.cend()) { + return it->second; + } + return nullptr; + } + + uint64_t prepareNextVersion() { + ASSERT(pending_keys_.empty()); + return ++next_version_; + } + + Handle insert(const K& key, T* value) { + ASSERT(next_version_ > published_version_.load(std::memory_order_relaxed)); + auto [it, inserted] = map_.try_emplace(key); + if (inserted) { + if constexpr (std::is_constructible_v) { + it->second = std::make_shared(key); + } else { + it->second = std::make_shared(); + } + } + auto& handle = it->second; + handle->set(next_version_, value); + pending_keys_.emplace(key); + return handle; + } + + void clear(const K& key) { + ASSERT(next_version_ > published_version_.load(std::memory_order_relaxed)); + auto it = map_.find(key); + if (it != map_.cend()) { + it->second->clear(next_version_); + pending_keys_.emplace(key); + } + } + + // returns the newly published version, or 0 if there was nothing to publish + uint64_t publishNextVersion() { + auto version = published_version_.load(std::memory_order_relaxed); + if (next_version_ <= version || pending_keys_.empty()) { + // no changes, revert back to the published version + next_version_ = version; + pending_keys_.clear(); + return 0; + } + for (const auto& key : pending_keys_) { + auto it = map_.find(key); + ASSERT(it != map_.end()); + dirty_values_.emplace(it->second); + if (it->second->get(next_version_) == nullptr) { + map_.erase(it); + } + } + published_version_.store(next_version_, std::memory_order_release); + pending_keys_.clear(); + return next_version_; + } + + DeferredDeletion revert() { + auto version = published_version_.load(std::memory_order_relaxed); + DeferredDeletion deferred; + for (const auto& key : pending_keys_) { + auto it = map_.find(key); + ASSERT(it != map_.end()); + it->second->revert(version, deferred); + if (it->second->get(version) == nullptr) { + map_.erase(it); + } + } + pending_keys_.clear(); + next_version_ = version; + return deferred; + } + + // First phase GC after all readers have quiesced for 'published_version'. This unlinks nodes that + // are no longer visible and returns them in a deferred-deletion batch that must survive until all + // readers have quiesced once more before it is destroyed. + DeferredDeletion gc(uint64_t published_version) { + DeferredDeletion deferred; + for (auto it = dirty_values_.begin(); it != dirty_values_.end();) { + const auto& handle = *it; + if (handle->gcForVersion(published_version, deferred)) { + dirty_values_.erase(it++); + continue; + } + ++it; + } + return deferred; + } + +private: + friend struct VersionedTestAccess; + using MutableHandle = std::shared_ptr; + + // Persistent published state. + std::atomic published_version_{versionMin}; + absl::flat_hash_map map_; + absl::flat_hash_set dirty_values_; + + // Transactional state for the currently prepared unpublished update. + // Valid only after prepareNextVersion() and until publishNextVersion() or revert(). + // `next_version_` is the candidate version being built, and `pending_keys_` + // contains the keys touched in that candidate update. The container is reused + // across updates to avoid repeated allocation churn. + uint64_t next_version_{versionMin}; + absl::flat_hash_set pending_keys_; +}; + +template struct VersionedTestAccess { + using Map = VersionedMap; + using Handle = typename Map::Handle; + using MutableHandle = typename Map::MutableHandle; + using Node = VersionedNode; + + static const absl::flat_hash_map& entries(const Map& map) { return map.map_; } + + static const absl::flat_hash_set& dirtyValues(const Map& map) { + return map.dirty_values_; + } + + static const Node* head(const Handle& handle) { + if (handle == nullptr) { + return nullptr; + } + auto value = std::static_pointer_cast(handle); + return value->head_.load(std::memory_order_acquire); + } + + static const Node* next(const Node* node) { + return node ? node->next_.load(std::memory_order_acquire) : nullptr; + } + + static const T* value(const Node* node) { return node ? node->value() : nullptr; } + + static uint64_t addVersion(const Node* node) { return node->add_version_; } + + static uint64_t removeVersion(const Node* node) { + return node->remove_version_.load(std::memory_order_relaxed); + } + + static bool isVisibleInVersion(const Node* node, uint64_t version) { + return node && node->isVisibleInVersion(version); + } + + static bool isAddedAfterVersion(const Node* node, uint64_t version) { + return node && node->isAddedAfterVersion(version); + } +}; diff --git a/go/cilium/api/bpf_metadata.pb.go b/go/cilium/api/bpf_metadata.pb.go index d8cc555ac..f54900960 100644 --- a/go/cilium/api/bpf_metadata.pb.go +++ b/go/cilium/api/bpf_metadata.pb.go @@ -87,8 +87,8 @@ type BpfMetadata struct { CacheEntryTtl *durationpb.Duration `protobuf:"bytes,14,opt,name=cache_entry_ttl,json=cacheEntryTtl,proto3" json:"cache_entry_ttl,omitempty"` // Cache is garbage collected at interval 10 times the ttl (default 30 ms). CacheGcInterval *durationpb.Duration `protobuf:"bytes,15,opt,name=cache_gc_interval,json=cacheGcInterval,proto3" json:"cache_gc_interval,omitempty"` - // Configuration for the source of NPDS updates. Currently this field is not supported. - NpdsConfig *v3.ConfigSource `protobuf:"bytes,16,opt,name=npds_config,json=npdsConfig,proto3" json:"npds_config,omitempty"` + // Configuration for the source of Cilium xDS updates. + ConfigSource *v3.ConfigSource `protobuf:"bytes,16,opt,name=config_source,json=configSource,proto3" json:"config_source,omitempty"` unknownFields protoimpl.UnknownFields sizeCache protoimpl.SizeCache } @@ -228,9 +228,9 @@ func (x *BpfMetadata) GetCacheGcInterval() *durationpb.Duration { return nil } -func (x *BpfMetadata) GetNpdsConfig() *v3.ConfigSource { +func (x *BpfMetadata) GetConfigSource() *v3.ConfigSource { if x != nil { - return x.NpdsConfig + return x.ConfigSource } return nil } @@ -239,7 +239,7 @@ var File_cilium_api_bpf_metadata_proto protoreflect.FileDescriptor const file_cilium_api_bpf_metadata_proto_rawDesc = "" + "\n" + - "\x1dcilium/api/bpf_metadata.proto\x12\x06cilium\x1a(envoy/config/core/v3/config_source.proto\x1a\x1egoogle/protobuf/duration.proto\x1a\x17validate/validate.proto\"\xd9\x06\n" + + "\x1dcilium/api/bpf_metadata.proto\x12\x06cilium\x1a(envoy/config/core/v3/config_source.proto\x1a\x1egoogle/protobuf/duration.proto\x1a\x17validate/validate.proto\"\xdd\x06\n" + "\vBpfMetadata\x12\x19\n" + "\bbpf_root\x18\x01 \x01(\tR\abpfRoot\x12\x1d\n" + "\n" + @@ -257,9 +257,8 @@ const file_cilium_api_bpf_metadata_proto_rawDesc = "" + "\fipcache_name\x18\f \x01(\tR\vipcacheName\x12\x1b\n" + "\tuse_nphds\x18\r \x01(\bR\buseNphds\x12A\n" + "\x0fcache_entry_ttl\x18\x0e \x01(\v2\x19.google.protobuf.DurationR\rcacheEntryTtl\x12E\n" + - "\x11cache_gc_interval\x18\x0f \x01(\v2\x19.google.protobuf.DurationR\x0fcacheGcInterval\x12C\n" + - "\vnpds_config\x18\x10 \x01(\v2\".envoy.config.core.v3.ConfigSourceR\n" + - "npdsConfigB!\n" + + "\x11cache_gc_interval\x18\x0f \x01(\v2\x19.google.protobuf.DurationR\x0fcacheGcInterval\x12G\n" + + "\rconfig_source\x18\x10 \x01(\v2\".envoy.config.core.v3.ConfigSourceR\fconfigSourceB!\n" + "\x1f_original_source_so_linger_timeB.Z,github.com/cilium/proxy/go/cilium/api;ciliumb\x06proto3" var ( @@ -284,7 +283,7 @@ var file_cilium_api_bpf_metadata_proto_depIdxs = []int32{ 1, // 0: cilium.BpfMetadata.policy_update_warning_limit:type_name -> google.protobuf.Duration 1, // 1: cilium.BpfMetadata.cache_entry_ttl:type_name -> google.protobuf.Duration 1, // 2: cilium.BpfMetadata.cache_gc_interval:type_name -> google.protobuf.Duration - 2, // 3: cilium.BpfMetadata.npds_config:type_name -> envoy.config.core.v3.ConfigSource + 2, // 3: cilium.BpfMetadata.config_source:type_name -> envoy.config.core.v3.ConfigSource 4, // [4:4] is the sub-list for method output_type 4, // [4:4] is the sub-list for method input_type 4, // [4:4] is the sub-list for extension type_name diff --git a/go/cilium/api/bpf_metadata.pb.validate.go b/go/cilium/api/bpf_metadata.pb.validate.go index bc30162c6..46c7b7ada 100644 --- a/go/cilium/api/bpf_metadata.pb.validate.go +++ b/go/cilium/api/bpf_metadata.pb.validate.go @@ -176,11 +176,11 @@ func (m *BpfMetadata) validate(all bool) error { } if all { - switch v := interface{}(m.GetNpdsConfig()).(type) { + switch v := interface{}(m.GetConfigSource()).(type) { case interface{ ValidateAll() error }: if err := v.ValidateAll(); err != nil { errors = append(errors, BpfMetadataValidationError{ - field: "NpdsConfig", + field: "ConfigSource", reason: "embedded message failed validation", cause: err, }) @@ -188,16 +188,16 @@ func (m *BpfMetadata) validate(all bool) error { case interface{ Validate() error }: if err := v.Validate(); err != nil { errors = append(errors, BpfMetadataValidationError{ - field: "NpdsConfig", + field: "ConfigSource", reason: "embedded message failed validation", cause: err, }) } } - } else if v, ok := interface{}(m.GetNpdsConfig()).(interface{ Validate() error }); ok { + } else if v, ok := interface{}(m.GetConfigSource()).(interface{ Validate() error }); ok { if err := v.Validate(); err != nil { return BpfMetadataValidationError{ - field: "NpdsConfig", + field: "ConfigSource", reason: "embedded message failed validation", cause: err, } diff --git a/go/cilium/api/npds.pb.go b/go/cilium/api/npds.pb.go index 2b09745e8..a4760071c 100644 --- a/go/cilium/api/npds.pb.go +++ b/go/cilium/api/npds.pb.go @@ -75,7 +75,7 @@ func (x HeaderMatch_MatchAction) Number() protoreflect.EnumNumber { // Deprecated: Use HeaderMatch_MatchAction.Descriptor instead. func (HeaderMatch_MatchAction) EnumDescriptor() ([]byte, []int) { - return file_cilium_api_npds_proto_rawDescGZIP(), []int{5, 0} + return file_cilium_api_npds_proto_rawDescGZIP(), []int{7, 0} } type HeaderMatch_MismatchAction int32 @@ -130,7 +130,139 @@ func (x HeaderMatch_MismatchAction) Number() protoreflect.EnumNumber { // Deprecated: Use HeaderMatch_MismatchAction.Descriptor instead. func (HeaderMatch_MismatchAction) EnumDescriptor() ([]byte, []int) { - return file_cilium_api_npds_proto_rawDescGZIP(), []int{5, 1} + return file_cilium_api_npds_proto_rawDescGZIP(), []int{7, 1} +} + +// A delta NPDS resource that carries either an endpoint policy or a shared selector. +type NetworkPolicyResource struct { + state protoimpl.MessageState `protogen:"open.v1"` + // Types that are valid to be assigned to Resource: + // + // *NetworkPolicyResource_Policy + // *NetworkPolicyResource_Selector + Resource isNetworkPolicyResource_Resource `protobuf_oneof:"resource"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *NetworkPolicyResource) Reset() { + *x = NetworkPolicyResource{} + mi := &file_cilium_api_npds_proto_msgTypes[0] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *NetworkPolicyResource) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*NetworkPolicyResource) ProtoMessage() {} + +func (x *NetworkPolicyResource) ProtoReflect() protoreflect.Message { + mi := &file_cilium_api_npds_proto_msgTypes[0] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use NetworkPolicyResource.ProtoReflect.Descriptor instead. +func (*NetworkPolicyResource) Descriptor() ([]byte, []int) { + return file_cilium_api_npds_proto_rawDescGZIP(), []int{0} +} + +func (x *NetworkPolicyResource) GetResource() isNetworkPolicyResource_Resource { + if x != nil { + return x.Resource + } + return nil +} + +func (x *NetworkPolicyResource) GetPolicy() *NetworkPolicy { + if x != nil { + if x, ok := x.Resource.(*NetworkPolicyResource_Policy); ok { + return x.Policy + } + } + return nil +} + +func (x *NetworkPolicyResource) GetSelector() *Selector { + if x != nil { + if x, ok := x.Resource.(*NetworkPolicyResource_Selector); ok { + return x.Selector + } + } + return nil +} + +type isNetworkPolicyResource_Resource interface { + isNetworkPolicyResource_Resource() +} + +type NetworkPolicyResource_Policy struct { + Policy *NetworkPolicy `protobuf:"bytes,1,opt,name=policy,proto3,oneof"` +} + +type NetworkPolicyResource_Selector struct { + Selector *Selector `protobuf:"bytes,2,opt,name=selector,proto3,oneof"` +} + +func (*NetworkPolicyResource_Policy) isNetworkPolicyResource_Resource() {} + +func (*NetworkPolicyResource_Selector) isNetworkPolicyResource_Resource() {} + +// A shared set of remote identities referenced by selector resource name. +// Unlike the old state-of-the-world remote identity lists, an empty selector +// matches nothing. +type Selector struct { + state protoimpl.MessageState `protogen:"open.v1"` + // The set of numeric remote security IDs selected by this selector. + // If empty, this selector selects no remote identities. + RemoteIdentities []uint32 `protobuf:"varint,1,rep,packed,name=remote_identities,json=remoteIdentities,proto3" json:"remote_identities,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *Selector) Reset() { + *x = Selector{} + mi := &file_cilium_api_npds_proto_msgTypes[1] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *Selector) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*Selector) ProtoMessage() {} + +func (x *Selector) ProtoReflect() protoreflect.Message { + mi := &file_cilium_api_npds_proto_msgTypes[1] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use Selector.ProtoReflect.Descriptor instead. +func (*Selector) Descriptor() ([]byte, []int) { + return file_cilium_api_npds_proto_rawDescGZIP(), []int{1} +} + +func (x *Selector) GetRemoteIdentities() []uint32 { + if x != nil { + return x.RemoteIdentities + } + return nil } // A network policy that is enforced by a filter on the network flows to/from @@ -161,7 +293,7 @@ type NetworkPolicy struct { func (x *NetworkPolicy) Reset() { *x = NetworkPolicy{} - mi := &file_cilium_api_npds_proto_msgTypes[0] + mi := &file_cilium_api_npds_proto_msgTypes[2] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -173,7 +305,7 @@ func (x *NetworkPolicy) String() string { func (*NetworkPolicy) ProtoMessage() {} func (x *NetworkPolicy) ProtoReflect() protoreflect.Message { - mi := &file_cilium_api_npds_proto_msgTypes[0] + mi := &file_cilium_api_npds_proto_msgTypes[2] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -186,7 +318,7 @@ func (x *NetworkPolicy) ProtoReflect() protoreflect.Message { // Deprecated: Use NetworkPolicy.ProtoReflect.Descriptor instead. func (*NetworkPolicy) Descriptor() ([]byte, []int) { - return file_cilium_api_npds_proto_rawDescGZIP(), []int{0} + return file_cilium_api_npds_proto_rawDescGZIP(), []int{2} } func (x *NetworkPolicy) GetEndpointIps() []string { @@ -240,7 +372,7 @@ type PortNetworkPolicy struct { func (x *PortNetworkPolicy) Reset() { *x = PortNetworkPolicy{} - mi := &file_cilium_api_npds_proto_msgTypes[1] + mi := &file_cilium_api_npds_proto_msgTypes[3] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -252,7 +384,7 @@ func (x *PortNetworkPolicy) String() string { func (*PortNetworkPolicy) ProtoMessage() {} func (x *PortNetworkPolicy) ProtoReflect() protoreflect.Message { - mi := &file_cilium_api_npds_proto_msgTypes[1] + mi := &file_cilium_api_npds_proto_msgTypes[3] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -265,7 +397,7 @@ func (x *PortNetworkPolicy) ProtoReflect() protoreflect.Message { // Deprecated: Use PortNetworkPolicy.ProtoReflect.Descriptor instead. func (*PortNetworkPolicy) Descriptor() ([]byte, []int) { - return file_cilium_api_npds_proto_rawDescGZIP(), []int{1} + return file_cilium_api_npds_proto_rawDescGZIP(), []int{3} } func (x *PortNetworkPolicy) GetPort() uint32 { @@ -328,7 +460,7 @@ type TLSContext struct { func (x *TLSContext) Reset() { *x = TLSContext{} - mi := &file_cilium_api_npds_proto_msgTypes[2] + mi := &file_cilium_api_npds_proto_msgTypes[4] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -340,7 +472,7 @@ func (x *TLSContext) String() string { func (*TLSContext) ProtoMessage() {} func (x *TLSContext) ProtoReflect() protoreflect.Message { - mi := &file_cilium_api_npds_proto_msgTypes[2] + mi := &file_cilium_api_npds_proto_msgTypes[4] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -353,7 +485,7 @@ func (x *TLSContext) ProtoReflect() protoreflect.Message { // Deprecated: Use TLSContext.ProtoReflect.Descriptor instead. func (*TLSContext) Descriptor() ([]byte, []int) { - return file_cilium_api_npds_proto_rawDescGZIP(), []int{2} + return file_cilium_api_npds_proto_rawDescGZIP(), []int{4} } func (x *TLSContext) GetTrustedCa() string { @@ -433,6 +565,11 @@ type PortNetworkPolicyRule struct { // applied on the flow's remote host is contained in this set. // Optional. If not specified, any remote host is matched by this predicate. RemotePolicies []uint32 `protobuf:"varint,7,rep,packed,name=remote_policies,json=remotePolicies,proto3" json:"remote_policies,omitempty"` + // Optional selector resource names that can be resolved to shared remote + // policy sets in delta NPDS. + // Selector references are matched by exact selector resource name. + // Optional. If not specified, any remote host is matched by this predicate. + Selectors []string `protobuf:"bytes,11,rep,name=selectors,proto3" json:"selectors,omitempty"` // Optional downstream TLS context. If present, the incoming connection must // be a TLS connection. DownstreamTlsContext *TLSContext `protobuf:"bytes,3,opt,name=downstream_tls_context,json=downstreamTlsContext,proto3" json:"downstream_tls_context,omitempty"` @@ -473,7 +610,7 @@ type PortNetworkPolicyRule struct { func (x *PortNetworkPolicyRule) Reset() { *x = PortNetworkPolicyRule{} - mi := &file_cilium_api_npds_proto_msgTypes[3] + mi := &file_cilium_api_npds_proto_msgTypes[5] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -485,7 +622,7 @@ func (x *PortNetworkPolicyRule) String() string { func (*PortNetworkPolicyRule) ProtoMessage() {} func (x *PortNetworkPolicyRule) ProtoReflect() protoreflect.Message { - mi := &file_cilium_api_npds_proto_msgTypes[3] + mi := &file_cilium_api_npds_proto_msgTypes[5] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -498,7 +635,7 @@ func (x *PortNetworkPolicyRule) ProtoReflect() protoreflect.Message { // Deprecated: Use PortNetworkPolicyRule.ProtoReflect.Descriptor instead. func (*PortNetworkPolicyRule) Descriptor() ([]byte, []int) { - return file_cilium_api_npds_proto_rawDescGZIP(), []int{3} + return file_cilium_api_npds_proto_rawDescGZIP(), []int{5} } func (x *PortNetworkPolicyRule) GetPrecedence() uint32 { @@ -554,6 +691,13 @@ func (x *PortNetworkPolicyRule) GetRemotePolicies() []uint32 { return nil } +func (x *PortNetworkPolicyRule) GetSelectors() []string { + if x != nil { + return x.Selectors + } + return nil +} + func (x *PortNetworkPolicyRule) GetDownstreamTlsContext() *TLSContext { if x != nil { return x.DownstreamTlsContext @@ -679,7 +823,7 @@ type HttpNetworkPolicyRules struct { func (x *HttpNetworkPolicyRules) Reset() { *x = HttpNetworkPolicyRules{} - mi := &file_cilium_api_npds_proto_msgTypes[4] + mi := &file_cilium_api_npds_proto_msgTypes[6] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -691,7 +835,7 @@ func (x *HttpNetworkPolicyRules) String() string { func (*HttpNetworkPolicyRules) ProtoMessage() {} func (x *HttpNetworkPolicyRules) ProtoReflect() protoreflect.Message { - mi := &file_cilium_api_npds_proto_msgTypes[4] + mi := &file_cilium_api_npds_proto_msgTypes[6] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -704,7 +848,7 @@ func (x *HttpNetworkPolicyRules) ProtoReflect() protoreflect.Message { // Deprecated: Use HttpNetworkPolicyRules.ProtoReflect.Descriptor instead. func (*HttpNetworkPolicyRules) Descriptor() ([]byte, []int) { - return file_cilium_api_npds_proto_rawDescGZIP(), []int{4} + return file_cilium_api_npds_proto_rawDescGZIP(), []int{6} } func (x *HttpNetworkPolicyRules) GetHttpRules() []*HttpNetworkPolicyRule { @@ -729,7 +873,7 @@ type HeaderMatch struct { func (x *HeaderMatch) Reset() { *x = HeaderMatch{} - mi := &file_cilium_api_npds_proto_msgTypes[5] + mi := &file_cilium_api_npds_proto_msgTypes[7] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -741,7 +885,7 @@ func (x *HeaderMatch) String() string { func (*HeaderMatch) ProtoMessage() {} func (x *HeaderMatch) ProtoReflect() protoreflect.Message { - mi := &file_cilium_api_npds_proto_msgTypes[5] + mi := &file_cilium_api_npds_proto_msgTypes[7] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -754,7 +898,7 @@ func (x *HeaderMatch) ProtoReflect() protoreflect.Message { // Deprecated: Use HeaderMatch.ProtoReflect.Descriptor instead. func (*HeaderMatch) Descriptor() ([]byte, []int) { - return file_cilium_api_npds_proto_rawDescGZIP(), []int{5} + return file_cilium_api_npds_proto_rawDescGZIP(), []int{7} } func (x *HeaderMatch) GetName() string { @@ -822,7 +966,7 @@ type HttpNetworkPolicyRule struct { func (x *HttpNetworkPolicyRule) Reset() { *x = HttpNetworkPolicyRule{} - mi := &file_cilium_api_npds_proto_msgTypes[6] + mi := &file_cilium_api_npds_proto_msgTypes[8] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -834,7 +978,7 @@ func (x *HttpNetworkPolicyRule) String() string { func (*HttpNetworkPolicyRule) ProtoMessage() {} func (x *HttpNetworkPolicyRule) ProtoReflect() protoreflect.Message { - mi := &file_cilium_api_npds_proto_msgTypes[6] + mi := &file_cilium_api_npds_proto_msgTypes[8] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -847,7 +991,7 @@ func (x *HttpNetworkPolicyRule) ProtoReflect() protoreflect.Message { // Deprecated: Use HttpNetworkPolicyRule.ProtoReflect.Descriptor instead. func (*HttpNetworkPolicyRule) Descriptor() ([]byte, []int) { - return file_cilium_api_npds_proto_rawDescGZIP(), []int{6} + return file_cilium_api_npds_proto_rawDescGZIP(), []int{8} } func (x *HttpNetworkPolicyRule) GetHeaders() []*v31.HeaderMatcher { @@ -877,7 +1021,7 @@ type KafkaNetworkPolicyRules struct { func (x *KafkaNetworkPolicyRules) Reset() { *x = KafkaNetworkPolicyRules{} - mi := &file_cilium_api_npds_proto_msgTypes[7] + mi := &file_cilium_api_npds_proto_msgTypes[9] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -889,7 +1033,7 @@ func (x *KafkaNetworkPolicyRules) String() string { func (*KafkaNetworkPolicyRules) ProtoMessage() {} func (x *KafkaNetworkPolicyRules) ProtoReflect() protoreflect.Message { - mi := &file_cilium_api_npds_proto_msgTypes[7] + mi := &file_cilium_api_npds_proto_msgTypes[9] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -902,7 +1046,7 @@ func (x *KafkaNetworkPolicyRules) ProtoReflect() protoreflect.Message { // Deprecated: Use KafkaNetworkPolicyRules.ProtoReflect.Descriptor instead. func (*KafkaNetworkPolicyRules) Descriptor() ([]byte, []int) { - return file_cilium_api_npds_proto_rawDescGZIP(), []int{7} + return file_cilium_api_npds_proto_rawDescGZIP(), []int{9} } func (x *KafkaNetworkPolicyRules) GetKafkaRules() []*KafkaNetworkPolicyRule { @@ -941,7 +1085,7 @@ type KafkaNetworkPolicyRule struct { func (x *KafkaNetworkPolicyRule) Reset() { *x = KafkaNetworkPolicyRule{} - mi := &file_cilium_api_npds_proto_msgTypes[8] + mi := &file_cilium_api_npds_proto_msgTypes[10] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -953,7 +1097,7 @@ func (x *KafkaNetworkPolicyRule) String() string { func (*KafkaNetworkPolicyRule) ProtoMessage() {} func (x *KafkaNetworkPolicyRule) ProtoReflect() protoreflect.Message { - mi := &file_cilium_api_npds_proto_msgTypes[8] + mi := &file_cilium_api_npds_proto_msgTypes[10] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -966,7 +1110,7 @@ func (x *KafkaNetworkPolicyRule) ProtoReflect() protoreflect.Message { // Deprecated: Use KafkaNetworkPolicyRule.ProtoReflect.Descriptor instead. func (*KafkaNetworkPolicyRule) Descriptor() ([]byte, []int) { - return file_cilium_api_npds_proto_rawDescGZIP(), []int{8} + return file_cilium_api_npds_proto_rawDescGZIP(), []int{10} } func (x *KafkaNetworkPolicyRule) GetApiVersion() int32 { @@ -1017,7 +1161,7 @@ type L7NetworkPolicyRules struct { func (x *L7NetworkPolicyRules) Reset() { *x = L7NetworkPolicyRules{} - mi := &file_cilium_api_npds_proto_msgTypes[9] + mi := &file_cilium_api_npds_proto_msgTypes[11] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -1029,7 +1173,7 @@ func (x *L7NetworkPolicyRules) String() string { func (*L7NetworkPolicyRules) ProtoMessage() {} func (x *L7NetworkPolicyRules) ProtoReflect() protoreflect.Message { - mi := &file_cilium_api_npds_proto_msgTypes[9] + mi := &file_cilium_api_npds_proto_msgTypes[11] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -1042,7 +1186,7 @@ func (x *L7NetworkPolicyRules) ProtoReflect() protoreflect.Message { // Deprecated: Use L7NetworkPolicyRules.ProtoReflect.Descriptor instead. func (*L7NetworkPolicyRules) Descriptor() ([]byte, []int) { - return file_cilium_api_npds_proto_rawDescGZIP(), []int{9} + return file_cilium_api_npds_proto_rawDescGZIP(), []int{11} } func (x *L7NetworkPolicyRules) GetL7AllowRules() []*L7NetworkPolicyRule { @@ -1080,7 +1224,7 @@ type L7NetworkPolicyRule struct { func (x *L7NetworkPolicyRule) Reset() { *x = L7NetworkPolicyRule{} - mi := &file_cilium_api_npds_proto_msgTypes[10] + mi := &file_cilium_api_npds_proto_msgTypes[12] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -1092,7 +1236,7 @@ func (x *L7NetworkPolicyRule) String() string { func (*L7NetworkPolicyRule) ProtoMessage() {} func (x *L7NetworkPolicyRule) ProtoReflect() protoreflect.Message { - mi := &file_cilium_api_npds_proto_msgTypes[10] + mi := &file_cilium_api_npds_proto_msgTypes[12] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -1105,7 +1249,7 @@ func (x *L7NetworkPolicyRule) ProtoReflect() protoreflect.Message { // Deprecated: Use L7NetworkPolicyRule.ProtoReflect.Descriptor instead. func (*L7NetworkPolicyRule) Descriptor() ([]byte, []int) { - return file_cilium_api_npds_proto_rawDescGZIP(), []int{10} + return file_cilium_api_npds_proto_rawDescGZIP(), []int{12} } func (x *L7NetworkPolicyRule) GetName() string { @@ -1140,7 +1284,7 @@ type NetworkPoliciesConfigDump struct { func (x *NetworkPoliciesConfigDump) Reset() { *x = NetworkPoliciesConfigDump{} - mi := &file_cilium_api_npds_proto_msgTypes[11] + mi := &file_cilium_api_npds_proto_msgTypes[13] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -1152,7 +1296,7 @@ func (x *NetworkPoliciesConfigDump) String() string { func (*NetworkPoliciesConfigDump) ProtoMessage() {} func (x *NetworkPoliciesConfigDump) ProtoReflect() protoreflect.Message { - mi := &file_cilium_api_npds_proto_msgTypes[11] + mi := &file_cilium_api_npds_proto_msgTypes[13] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -1165,7 +1309,7 @@ func (x *NetworkPoliciesConfigDump) ProtoReflect() protoreflect.Message { // Deprecated: Use NetworkPoliciesConfigDump.ProtoReflect.Descriptor instead. func (*NetworkPoliciesConfigDump) Descriptor() ([]byte, []int) { - return file_cilium_api_npds_proto_rawDescGZIP(), []int{11} + return file_cilium_api_npds_proto_rawDescGZIP(), []int{13} } func (x *NetworkPoliciesConfigDump) GetNetworkpolicies() []*NetworkPolicy { @@ -1179,7 +1323,14 @@ var File_cilium_api_npds_proto protoreflect.FileDescriptor const file_cilium_api_npds_proto_rawDesc = "" + "\n" + - "\x15cilium/api/npds.proto\x12\x06cilium\x1a\"envoy/config/core/v3/address.proto\x1a,envoy/config/route/v3/route_components.proto\x1a*envoy/service/discovery/v3/discovery.proto\x1a$envoy/type/matcher/v3/metadata.proto\x1a\x1cgoogle/api/annotations.proto\x1a envoy/annotations/resource.proto\x1a\x17validate/validate.proto\"\x95\x02\n" + + "\x15cilium/api/npds.proto\x12\x06cilium\x1a\"envoy/config/core/v3/address.proto\x1a,envoy/config/route/v3/route_components.proto\x1a*envoy/service/discovery/v3/discovery.proto\x1a$envoy/type/matcher/v3/metadata.proto\x1a\x1cgoogle/api/annotations.proto\x1a envoy/annotations/resource.proto\x1a\x17validate/validate.proto\"\x84\x01\n" + + "\x15NetworkPolicyResource\x12/\n" + + "\x06policy\x18\x01 \x01(\v2\x15.cilium.NetworkPolicyH\x00R\x06policy\x12.\n" + + "\bselector\x18\x02 \x01(\v2\x10.cilium.SelectorH\x00R\bselectorB\n" + + "\n" + + "\bresource\"7\n" + + "\bSelector\x12+\n" + + "\x11remote_identities\x18\x01 \x03(\rR\x10remoteIdentities\"\x95\x02\n" + "\rNetworkPolicy\x123\n" + "\fendpoint_ips\x18\x01 \x03(\tB\x10\xfaB\r\x92\x01\n" + "\b\x01\x10\x02\"\x04r\x02\x10\x01R\vendpointIps\x12\x1f\n" + @@ -1202,7 +1353,7 @@ const file_cilium_api_npds_proto_rawDesc = "" + "\fserver_names\x18\x04 \x03(\tR\vserverNames\x12A\n" + "\x1dvalidation_context_sds_secret\x18\x05 \x01(\tR\x1avalidationContextSdsSecret\x12$\n" + "\x0etls_sds_secret\x18\x06 \x01(\tR\ftlsSdsSecret\x12%\n" + - "\x0ealpn_protocols\x18\a \x03(\tR\ralpnProtocols\"\xfb\x05\n" + + "\x0ealpn_protocols\x18\a \x03(\tR\ralpnProtocols\"\x99\x06\n" + "\x15PortNetworkPolicyRule\x12\x1e\n" + "\n" + "precedence\x18\n" + @@ -1212,7 +1363,8 @@ const file_cilium_api_npds_proto_rawDesc = "" + "\x04deny\x18\b \x01(\bH\x00R\x04deny\x12$\n" + "\bproxy_id\x18\t \x01(\rB\t\xfaB\x06*\x04\x18\xff\xff\x03R\aproxyId\x12\x12\n" + "\x04name\x18\x05 \x01(\tR\x04name\x12'\n" + - "\x0fremote_policies\x18\a \x03(\rR\x0eremotePolicies\x12H\n" + + "\x0fremote_policies\x18\a \x03(\rR\x0eremotePolicies\x12\x1c\n" + + "\tselectors\x18\v \x03(\tR\tselectors\x12H\n" + "\x16downstream_tls_context\x18\x03 \x01(\v2\x12.cilium.TLSContextR\x14downstreamTlsContext\x12D\n" + "\x14upstream_tls_context\x18\x04 \x01(\v2\x12.cilium.TLSContextR\x12upstreamTlsContext\x12\xa1\x01\n" + "\fserver_names\x18\x06 \x03(\tB~\xfaB{\x92\x01x\"vrt2r^(([*]{1,2}|[*]?[-a-zA-Z0-9_]+([*][-a-zA-Z0-9_]+)*[*]?)[.])*([*]{1,2}|[*]?[-a-zA-Z0-9_]+([*][-a-zA-Z0-9_]+)*[*]?)$R\vserverNames\x12\x19\n" + @@ -1270,7 +1422,10 @@ const file_cilium_api_npds_proto_rawDesc = "" + "\x1dNetworkPolicyDiscoveryService\x12z\n" + "\x15StreamNetworkPolicies\x12,.envoy.service.discovery.v3.DiscoveryRequest\x1a-.envoy.service.discovery.v3.DiscoveryResponse\"\x00(\x010\x01\x12\x9e\x01\n" + "\x14FetchNetworkPolicies\x12,.envoy.service.discovery.v3.DiscoveryRequest\x1a-.envoy.service.discovery.v3.DiscoveryResponse\")\x82\xd3\xe4\x93\x02#:\x01*\"\x1e/v3/discovery:network_policies\x1a\x1c\x8a\xa4\x96\xf3\a\x16\n" + - "\x14cilium.NetworkPolicyB.Z,github.com/cilium/proxy/go/cilium/api;ciliumb\x06proto3" + "\x14cilium.NetworkPolicy2\xda\x01\n" + + "%NetworkPolicyResourceDiscoveryService\x12\x8a\x01\n" + + "\x1bDeltaNetworkPolicyResources\x121.envoy.service.discovery.v3.DeltaDiscoveryRequest\x1a2.envoy.service.discovery.v3.DeltaDiscoveryResponse\"\x00(\x010\x01\x1a$\x8a\xa4\x96\xf3\a\x1e\n" + + "\x1ccilium.NetworkPolicyResourceB.Z,github.com/cilium/proxy/go/cilium/api;ciliumb\x06proto3" var ( file_cilium_api_npds_proto_rawDescOnce sync.Once @@ -1285,59 +1440,67 @@ func file_cilium_api_npds_proto_rawDescGZIP() []byte { } var file_cilium_api_npds_proto_enumTypes = make([]protoimpl.EnumInfo, 2) -var file_cilium_api_npds_proto_msgTypes = make([]protoimpl.MessageInfo, 13) +var file_cilium_api_npds_proto_msgTypes = make([]protoimpl.MessageInfo, 15) var file_cilium_api_npds_proto_goTypes = []any{ - (HeaderMatch_MatchAction)(0), // 0: cilium.HeaderMatch.MatchAction - (HeaderMatch_MismatchAction)(0), // 1: cilium.HeaderMatch.MismatchAction - (*NetworkPolicy)(nil), // 2: cilium.NetworkPolicy - (*PortNetworkPolicy)(nil), // 3: cilium.PortNetworkPolicy - (*TLSContext)(nil), // 4: cilium.TLSContext - (*PortNetworkPolicyRule)(nil), // 5: cilium.PortNetworkPolicyRule - (*HttpNetworkPolicyRules)(nil), // 6: cilium.HttpNetworkPolicyRules - (*HeaderMatch)(nil), // 7: cilium.HeaderMatch - (*HttpNetworkPolicyRule)(nil), // 8: cilium.HttpNetworkPolicyRule - (*KafkaNetworkPolicyRules)(nil), // 9: cilium.KafkaNetworkPolicyRules - (*KafkaNetworkPolicyRule)(nil), // 10: cilium.KafkaNetworkPolicyRule - (*L7NetworkPolicyRules)(nil), // 11: cilium.L7NetworkPolicyRules - (*L7NetworkPolicyRule)(nil), // 12: cilium.L7NetworkPolicyRule - (*NetworkPoliciesConfigDump)(nil), // 13: cilium.NetworkPoliciesConfigDump - nil, // 14: cilium.L7NetworkPolicyRule.RuleEntry - (v3.SocketAddress_Protocol)(0), // 15: envoy.config.core.v3.SocketAddress.Protocol - (*v31.HeaderMatcher)(nil), // 16: envoy.config.route.v3.HeaderMatcher - (*v32.MetadataMatcher)(nil), // 17: envoy.type.matcher.v3.MetadataMatcher - (*v33.DiscoveryRequest)(nil), // 18: envoy.service.discovery.v3.DiscoveryRequest - (*v33.DiscoveryResponse)(nil), // 19: envoy.service.discovery.v3.DiscoveryResponse + (HeaderMatch_MatchAction)(0), // 0: cilium.HeaderMatch.MatchAction + (HeaderMatch_MismatchAction)(0), // 1: cilium.HeaderMatch.MismatchAction + (*NetworkPolicyResource)(nil), // 2: cilium.NetworkPolicyResource + (*Selector)(nil), // 3: cilium.Selector + (*NetworkPolicy)(nil), // 4: cilium.NetworkPolicy + (*PortNetworkPolicy)(nil), // 5: cilium.PortNetworkPolicy + (*TLSContext)(nil), // 6: cilium.TLSContext + (*PortNetworkPolicyRule)(nil), // 7: cilium.PortNetworkPolicyRule + (*HttpNetworkPolicyRules)(nil), // 8: cilium.HttpNetworkPolicyRules + (*HeaderMatch)(nil), // 9: cilium.HeaderMatch + (*HttpNetworkPolicyRule)(nil), // 10: cilium.HttpNetworkPolicyRule + (*KafkaNetworkPolicyRules)(nil), // 11: cilium.KafkaNetworkPolicyRules + (*KafkaNetworkPolicyRule)(nil), // 12: cilium.KafkaNetworkPolicyRule + (*L7NetworkPolicyRules)(nil), // 13: cilium.L7NetworkPolicyRules + (*L7NetworkPolicyRule)(nil), // 14: cilium.L7NetworkPolicyRule + (*NetworkPoliciesConfigDump)(nil), // 15: cilium.NetworkPoliciesConfigDump + nil, // 16: cilium.L7NetworkPolicyRule.RuleEntry + (v3.SocketAddress_Protocol)(0), // 17: envoy.config.core.v3.SocketAddress.Protocol + (*v31.HeaderMatcher)(nil), // 18: envoy.config.route.v3.HeaderMatcher + (*v32.MetadataMatcher)(nil), // 19: envoy.type.matcher.v3.MetadataMatcher + (*v33.DiscoveryRequest)(nil), // 20: envoy.service.discovery.v3.DiscoveryRequest + (*v33.DeltaDiscoveryRequest)(nil), // 21: envoy.service.discovery.v3.DeltaDiscoveryRequest + (*v33.DiscoveryResponse)(nil), // 22: envoy.service.discovery.v3.DiscoveryResponse + (*v33.DeltaDiscoveryResponse)(nil), // 23: envoy.service.discovery.v3.DeltaDiscoveryResponse } var file_cilium_api_npds_proto_depIdxs = []int32{ - 3, // 0: cilium.NetworkPolicy.ingress_per_port_policies:type_name -> cilium.PortNetworkPolicy - 3, // 1: cilium.NetworkPolicy.egress_per_port_policies:type_name -> cilium.PortNetworkPolicy - 15, // 2: cilium.PortNetworkPolicy.protocol:type_name -> envoy.config.core.v3.SocketAddress.Protocol - 5, // 3: cilium.PortNetworkPolicy.rules:type_name -> cilium.PortNetworkPolicyRule - 4, // 4: cilium.PortNetworkPolicyRule.downstream_tls_context:type_name -> cilium.TLSContext - 4, // 5: cilium.PortNetworkPolicyRule.upstream_tls_context:type_name -> cilium.TLSContext - 6, // 6: cilium.PortNetworkPolicyRule.http_rules:type_name -> cilium.HttpNetworkPolicyRules - 9, // 7: cilium.PortNetworkPolicyRule.kafka_rules:type_name -> cilium.KafkaNetworkPolicyRules - 11, // 8: cilium.PortNetworkPolicyRule.l7_rules:type_name -> cilium.L7NetworkPolicyRules - 8, // 9: cilium.HttpNetworkPolicyRules.http_rules:type_name -> cilium.HttpNetworkPolicyRule - 0, // 10: cilium.HeaderMatch.match_action:type_name -> cilium.HeaderMatch.MatchAction - 1, // 11: cilium.HeaderMatch.mismatch_action:type_name -> cilium.HeaderMatch.MismatchAction - 16, // 12: cilium.HttpNetworkPolicyRule.headers:type_name -> envoy.config.route.v3.HeaderMatcher - 7, // 13: cilium.HttpNetworkPolicyRule.header_matches:type_name -> cilium.HeaderMatch - 10, // 14: cilium.KafkaNetworkPolicyRules.kafka_rules:type_name -> cilium.KafkaNetworkPolicyRule - 12, // 15: cilium.L7NetworkPolicyRules.l7_allow_rules:type_name -> cilium.L7NetworkPolicyRule - 12, // 16: cilium.L7NetworkPolicyRules.l7_deny_rules:type_name -> cilium.L7NetworkPolicyRule - 14, // 17: cilium.L7NetworkPolicyRule.rule:type_name -> cilium.L7NetworkPolicyRule.RuleEntry - 17, // 18: cilium.L7NetworkPolicyRule.metadata_rule:type_name -> envoy.type.matcher.v3.MetadataMatcher - 2, // 19: cilium.NetworkPoliciesConfigDump.networkpolicies:type_name -> cilium.NetworkPolicy - 18, // 20: cilium.NetworkPolicyDiscoveryService.StreamNetworkPolicies:input_type -> envoy.service.discovery.v3.DiscoveryRequest - 18, // 21: cilium.NetworkPolicyDiscoveryService.FetchNetworkPolicies:input_type -> envoy.service.discovery.v3.DiscoveryRequest - 19, // 22: cilium.NetworkPolicyDiscoveryService.StreamNetworkPolicies:output_type -> envoy.service.discovery.v3.DiscoveryResponse - 19, // 23: cilium.NetworkPolicyDiscoveryService.FetchNetworkPolicies:output_type -> envoy.service.discovery.v3.DiscoveryResponse - 22, // [22:24] is the sub-list for method output_type - 20, // [20:22] is the sub-list for method input_type - 20, // [20:20] is the sub-list for extension type_name - 20, // [20:20] is the sub-list for extension extendee - 0, // [0:20] is the sub-list for field type_name + 4, // 0: cilium.NetworkPolicyResource.policy:type_name -> cilium.NetworkPolicy + 3, // 1: cilium.NetworkPolicyResource.selector:type_name -> cilium.Selector + 5, // 2: cilium.NetworkPolicy.ingress_per_port_policies:type_name -> cilium.PortNetworkPolicy + 5, // 3: cilium.NetworkPolicy.egress_per_port_policies:type_name -> cilium.PortNetworkPolicy + 17, // 4: cilium.PortNetworkPolicy.protocol:type_name -> envoy.config.core.v3.SocketAddress.Protocol + 7, // 5: cilium.PortNetworkPolicy.rules:type_name -> cilium.PortNetworkPolicyRule + 6, // 6: cilium.PortNetworkPolicyRule.downstream_tls_context:type_name -> cilium.TLSContext + 6, // 7: cilium.PortNetworkPolicyRule.upstream_tls_context:type_name -> cilium.TLSContext + 8, // 8: cilium.PortNetworkPolicyRule.http_rules:type_name -> cilium.HttpNetworkPolicyRules + 11, // 9: cilium.PortNetworkPolicyRule.kafka_rules:type_name -> cilium.KafkaNetworkPolicyRules + 13, // 10: cilium.PortNetworkPolicyRule.l7_rules:type_name -> cilium.L7NetworkPolicyRules + 10, // 11: cilium.HttpNetworkPolicyRules.http_rules:type_name -> cilium.HttpNetworkPolicyRule + 0, // 12: cilium.HeaderMatch.match_action:type_name -> cilium.HeaderMatch.MatchAction + 1, // 13: cilium.HeaderMatch.mismatch_action:type_name -> cilium.HeaderMatch.MismatchAction + 18, // 14: cilium.HttpNetworkPolicyRule.headers:type_name -> envoy.config.route.v3.HeaderMatcher + 9, // 15: cilium.HttpNetworkPolicyRule.header_matches:type_name -> cilium.HeaderMatch + 12, // 16: cilium.KafkaNetworkPolicyRules.kafka_rules:type_name -> cilium.KafkaNetworkPolicyRule + 14, // 17: cilium.L7NetworkPolicyRules.l7_allow_rules:type_name -> cilium.L7NetworkPolicyRule + 14, // 18: cilium.L7NetworkPolicyRules.l7_deny_rules:type_name -> cilium.L7NetworkPolicyRule + 16, // 19: cilium.L7NetworkPolicyRule.rule:type_name -> cilium.L7NetworkPolicyRule.RuleEntry + 19, // 20: cilium.L7NetworkPolicyRule.metadata_rule:type_name -> envoy.type.matcher.v3.MetadataMatcher + 4, // 21: cilium.NetworkPoliciesConfigDump.networkpolicies:type_name -> cilium.NetworkPolicy + 20, // 22: cilium.NetworkPolicyDiscoveryService.StreamNetworkPolicies:input_type -> envoy.service.discovery.v3.DiscoveryRequest + 20, // 23: cilium.NetworkPolicyDiscoveryService.FetchNetworkPolicies:input_type -> envoy.service.discovery.v3.DiscoveryRequest + 21, // 24: cilium.NetworkPolicyResourceDiscoveryService.DeltaNetworkPolicyResources:input_type -> envoy.service.discovery.v3.DeltaDiscoveryRequest + 22, // 25: cilium.NetworkPolicyDiscoveryService.StreamNetworkPolicies:output_type -> envoy.service.discovery.v3.DiscoveryResponse + 22, // 26: cilium.NetworkPolicyDiscoveryService.FetchNetworkPolicies:output_type -> envoy.service.discovery.v3.DiscoveryResponse + 23, // 27: cilium.NetworkPolicyResourceDiscoveryService.DeltaNetworkPolicyResources:output_type -> envoy.service.discovery.v3.DeltaDiscoveryResponse + 25, // [25:28] is the sub-list for method output_type + 22, // [22:25] is the sub-list for method input_type + 22, // [22:22] is the sub-list for extension type_name + 22, // [22:22] is the sub-list for extension extendee + 0, // [0:22] is the sub-list for field type_name } func init() { file_cilium_api_npds_proto_init() } @@ -1345,7 +1508,11 @@ func file_cilium_api_npds_proto_init() { if File_cilium_api_npds_proto != nil { return } - file_cilium_api_npds_proto_msgTypes[3].OneofWrappers = []any{ + file_cilium_api_npds_proto_msgTypes[0].OneofWrappers = []any{ + (*NetworkPolicyResource_Policy)(nil), + (*NetworkPolicyResource_Selector)(nil), + } + file_cilium_api_npds_proto_msgTypes[5].OneofWrappers = []any{ (*PortNetworkPolicyRule_PassPrecedence)(nil), (*PortNetworkPolicyRule_Deny)(nil), (*PortNetworkPolicyRule_HttpRules)(nil), @@ -1358,9 +1525,9 @@ func file_cilium_api_npds_proto_init() { GoPackagePath: reflect.TypeOf(x{}).PkgPath(), RawDescriptor: unsafe.Slice(unsafe.StringData(file_cilium_api_npds_proto_rawDesc), len(file_cilium_api_npds_proto_rawDesc)), NumEnums: 2, - NumMessages: 13, + NumMessages: 15, NumExtensions: 0, - NumServices: 1, + NumServices: 2, }, GoTypes: file_cilium_api_npds_proto_goTypes, DependencyIndexes: file_cilium_api_npds_proto_depIdxs, diff --git a/go/cilium/api/npds.pb.validate.go b/go/cilium/api/npds.pb.validate.go index 418649f39..ea344bc3f 100644 --- a/go/cilium/api/npds.pb.validate.go +++ b/go/cilium/api/npds.pb.validate.go @@ -39,6 +39,294 @@ var ( _ = corev3.SocketAddress_Protocol(0) ) +// Validate checks the field values on NetworkPolicyResource with the rules +// defined in the proto definition for this message. If any rules are +// violated, the first error encountered is returned, or nil if there are no violations. +func (m *NetworkPolicyResource) Validate() error { + return m.validate(false) +} + +// ValidateAll checks the field values on NetworkPolicyResource with the rules +// defined in the proto definition for this message. If any rules are +// violated, the result is a list of violation errors wrapped in +// NetworkPolicyResourceMultiError, or nil if none found. +func (m *NetworkPolicyResource) ValidateAll() error { + return m.validate(true) +} + +func (m *NetworkPolicyResource) validate(all bool) error { + if m == nil { + return nil + } + + var errors []error + + switch v := m.Resource.(type) { + case *NetworkPolicyResource_Policy: + if v == nil { + err := NetworkPolicyResourceValidationError{ + field: "Resource", + reason: "oneof value cannot be a typed-nil", + } + if !all { + return err + } + errors = append(errors, err) + } + + if all { + switch v := interface{}(m.GetPolicy()).(type) { + case interface{ ValidateAll() error }: + if err := v.ValidateAll(); err != nil { + errors = append(errors, NetworkPolicyResourceValidationError{ + field: "Policy", + reason: "embedded message failed validation", + cause: err, + }) + } + case interface{ Validate() error }: + if err := v.Validate(); err != nil { + errors = append(errors, NetworkPolicyResourceValidationError{ + field: "Policy", + reason: "embedded message failed validation", + cause: err, + }) + } + } + } else if v, ok := interface{}(m.GetPolicy()).(interface{ Validate() error }); ok { + if err := v.Validate(); err != nil { + return NetworkPolicyResourceValidationError{ + field: "Policy", + reason: "embedded message failed validation", + cause: err, + } + } + } + + case *NetworkPolicyResource_Selector: + if v == nil { + err := NetworkPolicyResourceValidationError{ + field: "Resource", + reason: "oneof value cannot be a typed-nil", + } + if !all { + return err + } + errors = append(errors, err) + } + + if all { + switch v := interface{}(m.GetSelector()).(type) { + case interface{ ValidateAll() error }: + if err := v.ValidateAll(); err != nil { + errors = append(errors, NetworkPolicyResourceValidationError{ + field: "Selector", + reason: "embedded message failed validation", + cause: err, + }) + } + case interface{ Validate() error }: + if err := v.Validate(); err != nil { + errors = append(errors, NetworkPolicyResourceValidationError{ + field: "Selector", + reason: "embedded message failed validation", + cause: err, + }) + } + } + } else if v, ok := interface{}(m.GetSelector()).(interface{ Validate() error }); ok { + if err := v.Validate(); err != nil { + return NetworkPolicyResourceValidationError{ + field: "Selector", + reason: "embedded message failed validation", + cause: err, + } + } + } + + default: + _ = v // ensures v is used + } + + if len(errors) > 0 { + return NetworkPolicyResourceMultiError(errors) + } + + return nil +} + +// NetworkPolicyResourceMultiError is an error wrapping multiple validation +// errors returned by NetworkPolicyResource.ValidateAll() if the designated +// constraints aren't met. +type NetworkPolicyResourceMultiError []error + +// Error returns a concatenation of all the error messages it wraps. +func (m NetworkPolicyResourceMultiError) Error() string { + msgs := make([]string, 0, len(m)) + for _, err := range m { + msgs = append(msgs, err.Error()) + } + return strings.Join(msgs, "; ") +} + +// AllErrors returns a list of validation violation errors. +func (m NetworkPolicyResourceMultiError) AllErrors() []error { return m } + +// NetworkPolicyResourceValidationError is the validation error returned by +// NetworkPolicyResource.Validate if the designated constraints aren't met. +type NetworkPolicyResourceValidationError struct { + field string + reason string + cause error + key bool +} + +// Field function returns field value. +func (e NetworkPolicyResourceValidationError) Field() string { return e.field } + +// Reason function returns reason value. +func (e NetworkPolicyResourceValidationError) Reason() string { return e.reason } + +// Cause function returns cause value. +func (e NetworkPolicyResourceValidationError) Cause() error { return e.cause } + +// Key function returns key value. +func (e NetworkPolicyResourceValidationError) Key() bool { return e.key } + +// ErrorName returns error name. +func (e NetworkPolicyResourceValidationError) ErrorName() string { + return "NetworkPolicyResourceValidationError" +} + +// Error satisfies the builtin error interface +func (e NetworkPolicyResourceValidationError) Error() string { + cause := "" + if e.cause != nil { + cause = fmt.Sprintf(" | caused by: %v", e.cause) + } + + key := "" + if e.key { + key = "key for " + } + + return fmt.Sprintf( + "invalid %sNetworkPolicyResource.%s: %s%s", + key, + e.field, + e.reason, + cause) +} + +var _ error = NetworkPolicyResourceValidationError{} + +var _ interface { + Field() string + Reason() string + Key() bool + Cause() error + ErrorName() string +} = NetworkPolicyResourceValidationError{} + +// Validate checks the field values on Selector with the rules defined in the +// proto definition for this message. If any rules are violated, the first +// error encountered is returned, or nil if there are no violations. +func (m *Selector) Validate() error { + return m.validate(false) +} + +// ValidateAll checks the field values on Selector with the rules defined in +// the proto definition for this message. If any rules are violated, the +// result is a list of violation errors wrapped in SelectorMultiError, or nil +// if none found. +func (m *Selector) ValidateAll() error { + return m.validate(true) +} + +func (m *Selector) validate(all bool) error { + if m == nil { + return nil + } + + var errors []error + + if len(errors) > 0 { + return SelectorMultiError(errors) + } + + return nil +} + +// SelectorMultiError is an error wrapping multiple validation errors returned +// by Selector.ValidateAll() if the designated constraints aren't met. +type SelectorMultiError []error + +// Error returns a concatenation of all the error messages it wraps. +func (m SelectorMultiError) Error() string { + msgs := make([]string, 0, len(m)) + for _, err := range m { + msgs = append(msgs, err.Error()) + } + return strings.Join(msgs, "; ") +} + +// AllErrors returns a list of validation violation errors. +func (m SelectorMultiError) AllErrors() []error { return m } + +// SelectorValidationError is the validation error returned by +// Selector.Validate if the designated constraints aren't met. +type SelectorValidationError struct { + field string + reason string + cause error + key bool +} + +// Field function returns field value. +func (e SelectorValidationError) Field() string { return e.field } + +// Reason function returns reason value. +func (e SelectorValidationError) Reason() string { return e.reason } + +// Cause function returns cause value. +func (e SelectorValidationError) Cause() error { return e.cause } + +// Key function returns key value. +func (e SelectorValidationError) Key() bool { return e.key } + +// ErrorName returns error name. +func (e SelectorValidationError) ErrorName() string { return "SelectorValidationError" } + +// Error satisfies the builtin error interface +func (e SelectorValidationError) Error() string { + cause := "" + if e.cause != nil { + cause = fmt.Sprintf(" | caused by: %v", e.cause) + } + + key := "" + if e.key { + key = "key for " + } + + return fmt.Sprintf( + "invalid %sSelector.%s: %s%s", + key, + e.field, + e.reason, + cause) +} + +var _ error = SelectorValidationError{} + +var _ interface { + Field() string + Reason() string + Key() bool + Cause() error + ErrorName() string +} = SelectorValidationError{} + // Validate checks the field values on NetworkPolicy with the rules defined in // the proto definition for this message. If any rules are violated, the first // error encountered is returned, or nil if there are no violations. diff --git a/go/cilium/api/npds_grpc.pb.go b/go/cilium/api/npds_grpc.pb.go index 5259d3bed..db349f173 100644 --- a/go/cilium/api/npds_grpc.pb.go +++ b/go/cilium/api/npds_grpc.pb.go @@ -29,6 +29,7 @@ const ( // For semantics around ctx use and closing/ending streaming RPCs, please refer to https://pkg.go.dev/google.golang.org/grpc/?tab=doc#ClientConn.NewStream. // // Each resource name is a network policy identifier. +// Deprecated: This service will be removed when Cilium 1.20 is the oldest supported release. type NetworkPolicyDiscoveryServiceClient interface { StreamNetworkPolicies(ctx context.Context, opts ...grpc.CallOption) (grpc.BidiStreamingClient[v3.DiscoveryRequest, v3.DiscoveryResponse], error) FetchNetworkPolicies(ctx context.Context, in *v3.DiscoveryRequest, opts ...grpc.CallOption) (*v3.DiscoveryResponse, error) @@ -70,6 +71,7 @@ func (c *networkPolicyDiscoveryServiceClient) FetchNetworkPolicies(ctx context.C // for forward compatibility. // // Each resource name is a network policy identifier. +// Deprecated: This service will be removed when Cilium 1.20 is the oldest supported release. type NetworkPolicyDiscoveryServiceServer interface { StreamNetworkPolicies(grpc.BidiStreamingServer[v3.DiscoveryRequest, v3.DiscoveryResponse]) error FetchNetworkPolicies(context.Context, *v3.DiscoveryRequest) (*v3.DiscoveryResponse, error) @@ -158,3 +160,104 @@ var NetworkPolicyDiscoveryService_ServiceDesc = grpc.ServiceDesc{ }, Metadata: "cilium/api/npds.proto", } + +const ( + NetworkPolicyResourceDiscoveryService_DeltaNetworkPolicyResources_FullMethodName = "/cilium.NetworkPolicyResourceDiscoveryService/DeltaNetworkPolicyResources" +) + +// NetworkPolicyResourceDiscoveryServiceClient is the client API for NetworkPolicyResourceDiscoveryService service. +// +// For semantics around ctx use and closing/ending streaming RPCs, please refer to https://pkg.go.dev/google.golang.org/grpc/?tab=doc#ClientConn.NewStream. +// +// Policy and selector resource names are exact-match identifiers in delta NPDS. +type NetworkPolicyResourceDiscoveryServiceClient interface { + DeltaNetworkPolicyResources(ctx context.Context, opts ...grpc.CallOption) (grpc.BidiStreamingClient[v3.DeltaDiscoveryRequest, v3.DeltaDiscoveryResponse], error) +} + +type networkPolicyResourceDiscoveryServiceClient struct { + cc grpc.ClientConnInterface +} + +func NewNetworkPolicyResourceDiscoveryServiceClient(cc grpc.ClientConnInterface) NetworkPolicyResourceDiscoveryServiceClient { + return &networkPolicyResourceDiscoveryServiceClient{cc} +} + +func (c *networkPolicyResourceDiscoveryServiceClient) DeltaNetworkPolicyResources(ctx context.Context, opts ...grpc.CallOption) (grpc.BidiStreamingClient[v3.DeltaDiscoveryRequest, v3.DeltaDiscoveryResponse], error) { + cOpts := append([]grpc.CallOption{grpc.StaticMethod()}, opts...) + stream, err := c.cc.NewStream(ctx, &NetworkPolicyResourceDiscoveryService_ServiceDesc.Streams[0], NetworkPolicyResourceDiscoveryService_DeltaNetworkPolicyResources_FullMethodName, cOpts...) + if err != nil { + return nil, err + } + x := &grpc.GenericClientStream[v3.DeltaDiscoveryRequest, v3.DeltaDiscoveryResponse]{ClientStream: stream} + return x, nil +} + +// This type alias is provided for backwards compatibility with existing code that references the prior non-generic stream type by name. +type NetworkPolicyResourceDiscoveryService_DeltaNetworkPolicyResourcesClient = grpc.BidiStreamingClient[v3.DeltaDiscoveryRequest, v3.DeltaDiscoveryResponse] + +// NetworkPolicyResourceDiscoveryServiceServer is the server API for NetworkPolicyResourceDiscoveryService service. +// All implementations must embed UnimplementedNetworkPolicyResourceDiscoveryServiceServer +// for forward compatibility. +// +// Policy and selector resource names are exact-match identifiers in delta NPDS. +type NetworkPolicyResourceDiscoveryServiceServer interface { + DeltaNetworkPolicyResources(grpc.BidiStreamingServer[v3.DeltaDiscoveryRequest, v3.DeltaDiscoveryResponse]) error + mustEmbedUnimplementedNetworkPolicyResourceDiscoveryServiceServer() +} + +// UnimplementedNetworkPolicyResourceDiscoveryServiceServer must be embedded to have +// forward compatible implementations. +// +// NOTE: this should be embedded by value instead of pointer to avoid a nil +// pointer dereference when methods are called. +type UnimplementedNetworkPolicyResourceDiscoveryServiceServer struct{} + +func (UnimplementedNetworkPolicyResourceDiscoveryServiceServer) DeltaNetworkPolicyResources(grpc.BidiStreamingServer[v3.DeltaDiscoveryRequest, v3.DeltaDiscoveryResponse]) error { + return status.Error(codes.Unimplemented, "method DeltaNetworkPolicyResources not implemented") +} +func (UnimplementedNetworkPolicyResourceDiscoveryServiceServer) mustEmbedUnimplementedNetworkPolicyResourceDiscoveryServiceServer() { +} +func (UnimplementedNetworkPolicyResourceDiscoveryServiceServer) testEmbeddedByValue() {} + +// UnsafeNetworkPolicyResourceDiscoveryServiceServer may be embedded to opt out of forward compatibility for this service. +// Use of this interface is not recommended, as added methods to NetworkPolicyResourceDiscoveryServiceServer will +// result in compilation errors. +type UnsafeNetworkPolicyResourceDiscoveryServiceServer interface { + mustEmbedUnimplementedNetworkPolicyResourceDiscoveryServiceServer() +} + +func RegisterNetworkPolicyResourceDiscoveryServiceServer(s grpc.ServiceRegistrar, srv NetworkPolicyResourceDiscoveryServiceServer) { + // If the following call panics, it indicates UnimplementedNetworkPolicyResourceDiscoveryServiceServer was + // embedded by pointer and is nil. This will cause panics if an + // unimplemented method is ever invoked, so we test this at initialization + // time to prevent it from happening at runtime later due to I/O. + if t, ok := srv.(interface{ testEmbeddedByValue() }); ok { + t.testEmbeddedByValue() + } + s.RegisterService(&NetworkPolicyResourceDiscoveryService_ServiceDesc, srv) +} + +func _NetworkPolicyResourceDiscoveryService_DeltaNetworkPolicyResources_Handler(srv interface{}, stream grpc.ServerStream) error { + return srv.(NetworkPolicyResourceDiscoveryServiceServer).DeltaNetworkPolicyResources(&grpc.GenericServerStream[v3.DeltaDiscoveryRequest, v3.DeltaDiscoveryResponse]{ServerStream: stream}) +} + +// This type alias is provided for backwards compatibility with existing code that references the prior non-generic stream type by name. +type NetworkPolicyResourceDiscoveryService_DeltaNetworkPolicyResourcesServer = grpc.BidiStreamingServer[v3.DeltaDiscoveryRequest, v3.DeltaDiscoveryResponse] + +// NetworkPolicyResourceDiscoveryService_ServiceDesc is the grpc.ServiceDesc for NetworkPolicyResourceDiscoveryService service. +// It's only intended for direct use with grpc.RegisterService, +// and not to be introspected or modified (even as a copy) +var NetworkPolicyResourceDiscoveryService_ServiceDesc = grpc.ServiceDesc{ + ServiceName: "cilium.NetworkPolicyResourceDiscoveryService", + HandlerType: (*NetworkPolicyResourceDiscoveryServiceServer)(nil), + Methods: []grpc.MethodDesc{}, + Streams: []grpc.StreamDesc{ + { + StreamName: "DeltaNetworkPolicyResources", + Handler: _NetworkPolicyResourceDiscoveryService_DeltaNetworkPolicyResources_Handler, + ServerStreams: true, + ClientStreams: true, + }, + }, + Metadata: "cilium/api/npds.proto", +} diff --git a/go/cilium/api/nphds.pb.go b/go/cilium/api/nphds.pb.go index f4a2b2dfe..c05898ac0 100644 --- a/go/cilium/api/nphds.pb.go +++ b/go/cilium/api/nphds.pb.go @@ -90,10 +90,11 @@ const file_cilium_api_nphds_proto_rawDesc = "" + "\x16cilium/api/nphds.proto\x12\x06cilium\x1a*envoy/service/discovery/v3/discovery.proto\x1a\x1cgoogle/api/annotations.proto\x1a envoy/annotations/resource.proto\x1a\x17validate/validate.proto\"c\n" + "\x12NetworkPolicyHosts\x12\x16\n" + "\x06policy\x18\x01 \x01(\x04R\x06policy\x125\n" + - "\x0ehost_addresses\x18\x02 \x03(\tB\x0e\xfaB\v\x92\x01\b\x18\x01\"\x04r\x02\x10\x01R\rhostAddresses2\xee\x02\n" + + "\x0ehost_addresses\x18\x02 \x03(\tB\x0e\xfaB\v\x92\x01\b\x18\x01\"\x04r\x02\x10\x01R\rhostAddresses2\xf7\x03\n" + "\"NetworkPolicyHostsDiscoveryService\x12}\n" + "\x18StreamNetworkPolicyHosts\x12,.envoy.service.discovery.v3.DiscoveryRequest\x1a-.envoy.service.discovery.v3.DiscoveryResponse\"\x00(\x010\x01\x12\xa5\x01\n" + - "\x17FetchNetworkPolicyHosts\x12,.envoy.service.discovery.v3.DiscoveryRequest\x1a-.envoy.service.discovery.v3.DiscoveryResponse\"-\x82\xd3\xe4\x93\x02':\x01*\"\"/v2/discovery:network_policy_hosts\x1a!\x8a\xa4\x96\xf3\a\x1b\n" + + "\x17FetchNetworkPolicyHosts\x12,.envoy.service.discovery.v3.DiscoveryRequest\x1a-.envoy.service.discovery.v3.DiscoveryResponse\"-\x82\xd3\xe4\x93\x02':\x01*\"\"/v2/discovery:network_policy_hosts\x12\x86\x01\n" + + "\x17DeltaNetworkPolicyHosts\x121.envoy.service.discovery.v3.DeltaDiscoveryRequest\x1a2.envoy.service.discovery.v3.DeltaDiscoveryResponse\"\x00(\x010\x01\x1a!\x8a\xa4\x96\xf3\a\x1b\n" + "\x19cilium.NetworkPolicyHostsB.Z,github.com/cilium/proxy/go/cilium/api;ciliumb\x06proto3" var ( @@ -110,17 +111,21 @@ func file_cilium_api_nphds_proto_rawDescGZIP() []byte { var file_cilium_api_nphds_proto_msgTypes = make([]protoimpl.MessageInfo, 1) var file_cilium_api_nphds_proto_goTypes = []any{ - (*NetworkPolicyHosts)(nil), // 0: cilium.NetworkPolicyHosts - (*v3.DiscoveryRequest)(nil), // 1: envoy.service.discovery.v3.DiscoveryRequest - (*v3.DiscoveryResponse)(nil), // 2: envoy.service.discovery.v3.DiscoveryResponse + (*NetworkPolicyHosts)(nil), // 0: cilium.NetworkPolicyHosts + (*v3.DiscoveryRequest)(nil), // 1: envoy.service.discovery.v3.DiscoveryRequest + (*v3.DeltaDiscoveryRequest)(nil), // 2: envoy.service.discovery.v3.DeltaDiscoveryRequest + (*v3.DiscoveryResponse)(nil), // 3: envoy.service.discovery.v3.DiscoveryResponse + (*v3.DeltaDiscoveryResponse)(nil), // 4: envoy.service.discovery.v3.DeltaDiscoveryResponse } var file_cilium_api_nphds_proto_depIdxs = []int32{ 1, // 0: cilium.NetworkPolicyHostsDiscoveryService.StreamNetworkPolicyHosts:input_type -> envoy.service.discovery.v3.DiscoveryRequest 1, // 1: cilium.NetworkPolicyHostsDiscoveryService.FetchNetworkPolicyHosts:input_type -> envoy.service.discovery.v3.DiscoveryRequest - 2, // 2: cilium.NetworkPolicyHostsDiscoveryService.StreamNetworkPolicyHosts:output_type -> envoy.service.discovery.v3.DiscoveryResponse - 2, // 3: cilium.NetworkPolicyHostsDiscoveryService.FetchNetworkPolicyHosts:output_type -> envoy.service.discovery.v3.DiscoveryResponse - 2, // [2:4] is the sub-list for method output_type - 0, // [0:2] is the sub-list for method input_type + 2, // 2: cilium.NetworkPolicyHostsDiscoveryService.DeltaNetworkPolicyHosts:input_type -> envoy.service.discovery.v3.DeltaDiscoveryRequest + 3, // 3: cilium.NetworkPolicyHostsDiscoveryService.StreamNetworkPolicyHosts:output_type -> envoy.service.discovery.v3.DiscoveryResponse + 3, // 4: cilium.NetworkPolicyHostsDiscoveryService.FetchNetworkPolicyHosts:output_type -> envoy.service.discovery.v3.DiscoveryResponse + 4, // 5: cilium.NetworkPolicyHostsDiscoveryService.DeltaNetworkPolicyHosts:output_type -> envoy.service.discovery.v3.DeltaDiscoveryResponse + 3, // [3:6] is the sub-list for method output_type + 0, // [0:3] is the sub-list for method input_type 0, // [0:0] is the sub-list for extension type_name 0, // [0:0] is the sub-list for extension extendee 0, // [0:0] is the sub-list for field type_name diff --git a/go/cilium/api/nphds_grpc.pb.go b/go/cilium/api/nphds_grpc.pb.go index cdd10021d..854479169 100644 --- a/go/cilium/api/nphds_grpc.pb.go +++ b/go/cilium/api/nphds_grpc.pb.go @@ -22,6 +22,7 @@ const _ = grpc.SupportPackageIsVersion9 const ( NetworkPolicyHostsDiscoveryService_StreamNetworkPolicyHosts_FullMethodName = "/cilium.NetworkPolicyHostsDiscoveryService/StreamNetworkPolicyHosts" NetworkPolicyHostsDiscoveryService_FetchNetworkPolicyHosts_FullMethodName = "/cilium.NetworkPolicyHostsDiscoveryService/FetchNetworkPolicyHosts" + NetworkPolicyHostsDiscoveryService_DeltaNetworkPolicyHosts_FullMethodName = "/cilium.NetworkPolicyHostsDiscoveryService/DeltaNetworkPolicyHosts" ) // NetworkPolicyHostsDiscoveryServiceClient is the client API for NetworkPolicyHostsDiscoveryService service. @@ -32,6 +33,7 @@ const ( type NetworkPolicyHostsDiscoveryServiceClient interface { StreamNetworkPolicyHosts(ctx context.Context, opts ...grpc.CallOption) (grpc.BidiStreamingClient[v3.DiscoveryRequest, v3.DiscoveryResponse], error) FetchNetworkPolicyHosts(ctx context.Context, in *v3.DiscoveryRequest, opts ...grpc.CallOption) (*v3.DiscoveryResponse, error) + DeltaNetworkPolicyHosts(ctx context.Context, opts ...grpc.CallOption) (grpc.BidiStreamingClient[v3.DeltaDiscoveryRequest, v3.DeltaDiscoveryResponse], error) } type networkPolicyHostsDiscoveryServiceClient struct { @@ -65,6 +67,19 @@ func (c *networkPolicyHostsDiscoveryServiceClient) FetchNetworkPolicyHosts(ctx c return out, nil } +func (c *networkPolicyHostsDiscoveryServiceClient) DeltaNetworkPolicyHosts(ctx context.Context, opts ...grpc.CallOption) (grpc.BidiStreamingClient[v3.DeltaDiscoveryRequest, v3.DeltaDiscoveryResponse], error) { + cOpts := append([]grpc.CallOption{grpc.StaticMethod()}, opts...) + stream, err := c.cc.NewStream(ctx, &NetworkPolicyHostsDiscoveryService_ServiceDesc.Streams[1], NetworkPolicyHostsDiscoveryService_DeltaNetworkPolicyHosts_FullMethodName, cOpts...) + if err != nil { + return nil, err + } + x := &grpc.GenericClientStream[v3.DeltaDiscoveryRequest, v3.DeltaDiscoveryResponse]{ClientStream: stream} + return x, nil +} + +// This type alias is provided for backwards compatibility with existing code that references the prior non-generic stream type by name. +type NetworkPolicyHostsDiscoveryService_DeltaNetworkPolicyHostsClient = grpc.BidiStreamingClient[v3.DeltaDiscoveryRequest, v3.DeltaDiscoveryResponse] + // NetworkPolicyHostsDiscoveryServiceServer is the server API for NetworkPolicyHostsDiscoveryService service. // All implementations must embed UnimplementedNetworkPolicyHostsDiscoveryServiceServer // for forward compatibility. @@ -73,6 +88,7 @@ func (c *networkPolicyHostsDiscoveryServiceClient) FetchNetworkPolicyHosts(ctx c type NetworkPolicyHostsDiscoveryServiceServer interface { StreamNetworkPolicyHosts(grpc.BidiStreamingServer[v3.DiscoveryRequest, v3.DiscoveryResponse]) error FetchNetworkPolicyHosts(context.Context, *v3.DiscoveryRequest) (*v3.DiscoveryResponse, error) + DeltaNetworkPolicyHosts(grpc.BidiStreamingServer[v3.DeltaDiscoveryRequest, v3.DeltaDiscoveryResponse]) error mustEmbedUnimplementedNetworkPolicyHostsDiscoveryServiceServer() } @@ -89,6 +105,9 @@ func (UnimplementedNetworkPolicyHostsDiscoveryServiceServer) StreamNetworkPolicy func (UnimplementedNetworkPolicyHostsDiscoveryServiceServer) FetchNetworkPolicyHosts(context.Context, *v3.DiscoveryRequest) (*v3.DiscoveryResponse, error) { return nil, status.Error(codes.Unimplemented, "method FetchNetworkPolicyHosts not implemented") } +func (UnimplementedNetworkPolicyHostsDiscoveryServiceServer) DeltaNetworkPolicyHosts(grpc.BidiStreamingServer[v3.DeltaDiscoveryRequest, v3.DeltaDiscoveryResponse]) error { + return status.Error(codes.Unimplemented, "method DeltaNetworkPolicyHosts not implemented") +} func (UnimplementedNetworkPolicyHostsDiscoveryServiceServer) mustEmbedUnimplementedNetworkPolicyHostsDiscoveryServiceServer() { } func (UnimplementedNetworkPolicyHostsDiscoveryServiceServer) testEmbeddedByValue() {} @@ -136,6 +155,13 @@ func _NetworkPolicyHostsDiscoveryService_FetchNetworkPolicyHosts_Handler(srv int return interceptor(ctx, in, info, handler) } +func _NetworkPolicyHostsDiscoveryService_DeltaNetworkPolicyHosts_Handler(srv interface{}, stream grpc.ServerStream) error { + return srv.(NetworkPolicyHostsDiscoveryServiceServer).DeltaNetworkPolicyHosts(&grpc.GenericServerStream[v3.DeltaDiscoveryRequest, v3.DeltaDiscoveryResponse]{ServerStream: stream}) +} + +// This type alias is provided for backwards compatibility with existing code that references the prior non-generic stream type by name. +type NetworkPolicyHostsDiscoveryService_DeltaNetworkPolicyHostsServer = grpc.BidiStreamingServer[v3.DeltaDiscoveryRequest, v3.DeltaDiscoveryResponse] + // NetworkPolicyHostsDiscoveryService_ServiceDesc is the grpc.ServiceDesc for NetworkPolicyHostsDiscoveryService service. // It's only intended for direct use with grpc.RegisterService, // and not to be introspected or modified (even as a copy) @@ -155,6 +181,12 @@ var NetworkPolicyHostsDiscoveryService_ServiceDesc = grpc.ServiceDesc{ ServerStreams: true, ClientStreams: true, }, + { + StreamName: "DeltaNetworkPolicyHosts", + Handler: _NetworkPolicyHostsDiscoveryService_DeltaNetworkPolicyHosts_Handler, + ServerStreams: true, + ClientStreams: true, + }, }, Metadata: "cilium/api/nphds.proto", } diff --git a/patches/0007-config-add-grpc-mux-stream-event-callback.patch b/patches/0007-config-add-grpc-mux-stream-event-callback.patch new file mode 100644 index 000000000..e92fff4dd --- /dev/null +++ b/patches/0007-config-add-grpc-mux-stream-event-callback.patch @@ -0,0 +1,448 @@ +diff --git a/envoy/config/BUILD b/envoy/config/BUILD +--- a/envoy/config/BUILD ++++ b/envoy/config/BUILD +@@ -66,6 +66,7 @@ envoy_cc_library( + name = "grpc_mux_interface", + hdrs = ["grpc_mux.h"], + deps = [ ++ "//envoy/common:callback", + ":eds_resources_cache_interface", + ":subscription_interface", + "//envoy/stats:stats_macros", +diff --git a/envoy/config/grpc_mux.h b/envoy/config/grpc_mux.h +--- a/envoy/config/grpc_mux.h ++++ b/envoy/config/grpc_mux.h +@@ -1,8 +1,10 @@ + #pragma once + ++#include + #include + + #include "envoy/common/backoff_strategy.h" ++#include "envoy/common/callback.h" + #include "envoy/common/exception.h" + #include "envoy/common/pure.h" + #include "envoy/config/custom_config_validators.h" +@@ -17,6 +19,11 @@ namespace Envoy { + namespace Config { + + using ScopedResume = std::unique_ptr; ++ ++enum class GrpcMuxStreamEvent { Established, Closed }; ++ ++using GrpcMuxStreamEventCallback = std::function; ++ + /** + * All control plane related stats. @see stats_macros.h + */ +@@ -61,6 +67,18 @@ public: + */ + virtual void start() PURE; + ++ /** ++ * Add a callback to observe gRPC mux stream lifecycle events. ++ * @return a handle that unregisters the callback when destroyed. ++ */ ++ virtual Common::CallbackHandlePtr ++ addStreamEventCallback(GrpcMuxStreamEventCallback callback) PURE; ++ ++ /** ++ * @return true if the mux currently has an established gRPC stream. ++ */ ++ virtual bool grpcStreamConnected() const PURE; ++ + /** + * Pause discovery requests for a given API type. This is useful when we're processing an update + * for LDS or CDS and don't want a flood of updates for RDS or EDS respectively. Discovery +diff --git a/source/common/config/BUILD b/source/common/config/BUILD +--- a/source/common/config/BUILD ++++ b/source/common/config/BUILD +@@ -86,10 +86,19 @@ envoy_cc_library( + ], + ) +- ++envoy_cc_library( ++ name = "grpc_mux_stream_event_tracker_lib", ++ hdrs = ["grpc_mux_stream_event_tracker.h"], ++ deps = [ ++ "//envoy/config:grpc_mux_interface", ++ "//source/common/common:callback_impl_lib", ++ ], ++) ++ + envoy_cc_library( + name = "null_grpc_mux_lib", + hdrs = ["null_grpc_mux_impl.h"], + deps = [ ++ ":grpc_mux_stream_event_tracker_lib", + "//envoy/config:grpc_mux_interface", + ], + ) +diff --git a/source/common/config/null_grpc_mux_impl.h b/source/common/config/null_grpc_mux_impl.h +--- a/source/common/config/null_grpc_mux_impl.h ++++ b/source/common/config/null_grpc_mux_impl.h +@@ -1,6 +1,7 @@ + #pragma once + + #include "envoy/config/grpc_mux.h" ++#include "source/common/config/grpc_mux_stream_event_tracker.h" + + namespace Envoy { + namespace Config { +@@ -11,5 +12,11 @@ class NullGrpcMuxImpl : public GrpcMux, + public: + void start() override {} ++ Common::CallbackHandlePtr ++ addStreamEventCallback(GrpcMuxStreamEventCallback callback) override { ++ return stream_event_tracker_.addStreamEventCallback(std::move(callback)); ++ } ++ ++ bool grpcStreamConnected() const override { return false; } + ScopedResume pause(const std::string&) override { + return std::make_unique([] {}); + } +@@ -40,6 +48,9 @@ public: + void onEstablishmentFailure(bool) override {} + void onDiscoveryResponse(std::unique_ptr&&, + ControlPlaneStats&) override {} ++ ++private: ++ GrpcMuxStreamEventTracker stream_event_tracker_; + }; + + } // namespace Config +diff --git a/source/extensions/config_subscription/grpc/BUILD b/source/extensions/config_subscription/grpc/BUILD +--- a/source/extensions/config_subscription/grpc/BUILD ++++ b/source/extensions/config_subscription/grpc/BUILD +@@ -33,6 +33,7 @@ envoy_cc_extension( + ":eds_resources_cache_lib", + ":grpc_mux_context_lib", + ":grpc_mux_failover_lib", ++ "//source/common/config:grpc_mux_stream_event_tracker_lib", + ":grpc_stream_lib", + ":xds_source_id_lib", + "//envoy/config:custom_config_validators_interface", +@@ -65,6 +66,7 @@ envoy_cc_library( + ":grpc_mux_context_lib", + ":grpc_mux_failover_lib", + ":grpc_stream_lib", ++ "//source/common/config:grpc_mux_stream_event_tracker_lib", + ":pausable_ack_queue_lib", + ":watch_map_lib", + "//envoy/config:custom_config_validators_interface", +diff --git a/source/extensions/config_subscription/grpc/grpc_mux_impl.cc b/source/extensions/config_subscription/grpc/grpc_mux_impl.cc +--- a/source/extensions/config_subscription/grpc/grpc_mux_impl.cc ++++ b/source/extensions/config_subscription/grpc/grpc_mux_impl.cc +@@ -569,6 +569,7 @@ void GrpcMuxImpl::onStreamEstablished() { + for (const auto& type_url : subscriptions_) { + queueDiscoveryRequest(type_url); + } ++ stream_event_tracker_.onStreamEstablished(); + } + + void GrpcMuxImpl::onEstablishmentFailure(bool) { +@@ -589,6 +590,7 @@ void GrpcMuxImpl::onEstablishmentFailure(bool) { + api_state.second->previously_fetched_data_ = true; + } + } ++ stream_event_tracker_.onStreamClosed(); + } + + void GrpcMuxImpl::queueDiscoveryRequest(absl::string_view queue_item) { +diff --git a/source/extensions/config_subscription/grpc/grpc_mux_impl.h b/source/extensions/config_subscription/grpc/grpc_mux_impl.h +--- a/source/extensions/config_subscription/grpc/grpc_mux_impl.h ++++ b/source/extensions/config_subscription/grpc/grpc_mux_impl.h +@@ -3,6 +3,7 @@ + #include + #include + #include ++#include + + #include "envoy/common/random_generator.h" + #include "envoy/common/time.h" +@@ -29,5 +30,6 @@ + #include "source/extensions/config_subscription/grpc/grpc_mux_context.h" + #include "source/extensions/config_subscription/grpc/grpc_mux_failover.h" ++#include "source/common/config/grpc_mux_stream_event_tracker.h" + + #include "absl/container/node_hash_map.h" + #include "xds/core/v3/resource_name.pb.h" +@@ -57,6 +59,13 @@ public: + + void start() override; + ++ Common::CallbackHandlePtr ++ addStreamEventCallback(GrpcMuxStreamEventCallback callback) override { ++ return stream_event_tracker_.addStreamEventCallback(std::move(callback)); ++ } ++ ++ bool grpcStreamConnected() const override { return stream_event_tracker_.grpcStreamConnected(); } ++ + // GrpcMux + ScopedResume pause(const std::string& type_url) override; + ScopedResume pause(const std::vector type_urls) override; +@@ -290,6 +299,7 @@ private: + const bool skip_subsequent_node_; + CustomConfigValidatorsPtr config_validators_; + XdsConfigTrackerOptRef xds_config_tracker_; ++ GrpcMuxStreamEventTracker stream_event_tracker_; + XdsResourcesDelegateOptRef xds_resources_delegate_; + EdsResourcesCachePtr eds_resources_cache_; + const std::string target_xds_authority_; +diff --git a/source/common/config/grpc_mux_stream_event_tracker.h b/source/common/config/grpc_mux_stream_event_tracker.h +new file mode 100644 +--- /dev/null ++++ b/source/common/config/grpc_mux_stream_event_tracker.h +@@ -0,0 +1,36 @@ ++#pragma once ++ ++#include ++ ++#include "envoy/config/grpc_mux.h" ++ ++#include "source/common/common/callback_impl.h" ++ ++namespace Envoy { ++namespace Config { ++ ++class GrpcMuxStreamEventTracker { ++public: ++ Common::CallbackHandlePtr addStreamEventCallback(GrpcMuxStreamEventCallback callback) { ++ return callbacks_.add(std::move(callback)); ++ } ++ ++ bool grpcStreamConnected() const { return grpc_stream_connected_; } ++ ++ void onStreamEstablished() { ++ grpc_stream_connected_ = true; ++ callbacks_.runCallbacks(GrpcMuxStreamEvent::Established); ++ } ++ ++ void onStreamClosed() { ++ grpc_stream_connected_ = false; ++ callbacks_.runCallbacks(GrpcMuxStreamEvent::Closed); ++ } ++ ++private: ++ bool grpc_stream_connected_{false}; ++ Common::CallbackManager callbacks_; ++}; ++ ++} // namespace Config ++} // namespace Envoy +diff --git a/source/extensions/config_subscription/grpc/new_grpc_mux_impl.cc b/source/extensions/config_subscription/grpc/new_grpc_mux_impl.cc +--- a/source/extensions/config_subscription/grpc/new_grpc_mux_impl.cc ++++ b/source/extensions/config_subscription/grpc/new_grpc_mux_impl.cc +@@ -194,6 +194,7 @@ void NewGrpcMuxImpl::onStreamEstablished() { + } + pausable_ack_queue_.clear(); + trySendDiscoveryRequests(); ++ stream_event_tracker_.onStreamEstablished(); + } + + void NewGrpcMuxImpl::onEstablishmentFailure(bool next_attempt_may_send_initial_resource_version) { +@@ -215,6 +216,7 @@ void NewGrpcMuxImpl::onEstablishmentFailure(bool next_attempt_may_send_initial_resource_version) { + } + } while (all_subscribed.size() != subscriptions_.size()); + should_send_initial_resource_versions_ = next_attempt_may_send_initial_resource_version; ++ stream_event_tracker_.onStreamClosed(); + } + + void NewGrpcMuxImpl::onWriteable() { trySendDiscoveryRequests(); } +diff --git a/source/extensions/config_subscription/grpc/new_grpc_mux_impl.h b/source/extensions/config_subscription/grpc/new_grpc_mux_impl.h +--- a/source/extensions/config_subscription/grpc/new_grpc_mux_impl.h ++++ b/source/extensions/config_subscription/grpc/new_grpc_mux_impl.h +@@ -1,6 +1,7 @@ + #pragma once + + #include ++#include + + #include "envoy/common/random_generator.h" + #include "envoy/common/token_bucket.h" +@@ -20,6 +21,7 @@ + #include "source/extensions/config_subscription/grpc/grpc_mux_context.h" + #include "source/extensions/config_subscription/grpc/grpc_mux_failover.h" + #include "source/extensions/config_subscription/grpc/pausable_ack_queue.h" ++#include "source/common/config/grpc_mux_stream_event_tracker.h" + #include "source/extensions/config_subscription/grpc/watch_map.h" + + namespace Envoy { +@@ -50,6 +52,13 @@ public: + static void shutdownAll(); + + void shutdown() { shutdown_ = true; } ++ Common::CallbackHandlePtr ++ addStreamEventCallback(GrpcMuxStreamEventCallback callback) override { ++ return stream_event_tracker_.addStreamEventCallback(std::move(callback)); ++ } ++ ++ bool grpcStreamConnected() const override { return stream_event_tracker_.grpcStreamConnected(); } ++ + + GrpcMuxWatchPtr addWatch(const std::string& type_url, + const absl::flat_hash_set& resources, +@@ -217,6 +226,7 @@ private: + + const LocalInfo::LocalInfo& local_info_; + CustomConfigValidatorsPtr config_validators_; ++ GrpcMuxStreamEventTracker stream_event_tracker_; + Common::CallbackHandlePtr dynamic_update_callback_handle_; + XdsConfigTrackerOptRef xds_config_tracker_; + EdsResourcesCachePtr eds_resources_cache_; +diff --git a/source/extensions/config_subscription/grpc/xds_mux/BUILD b/source/extensions/config_subscription/grpc/xds_mux/BUILD +--- a/source/extensions/config_subscription/grpc/xds_mux/BUILD ++++ b/source/extensions/config_subscription/grpc/xds_mux/BUILD +@@ -72,6 +72,7 @@ envoy_cc_extension( + "//source/extensions/config_subscription/grpc:eds_resources_cache_lib", + "//source/extensions/config_subscription/grpc:grpc_mux_context_lib", + "//source/extensions/config_subscription/grpc:grpc_mux_failover_lib", ++ "//source/common/config:grpc_mux_stream_event_tracker_lib", + "//source/extensions/config_subscription/grpc:grpc_stream_lib", + "//source/extensions/config_subscription/grpc:pausable_ack_queue_lib", + "//source/extensions/config_subscription/grpc:watch_map_lib", +diff --git a/source/extensions/config_subscription/grpc/xds_mux/grpc_mux_impl.cc b/source/extensions/config_subscription/grpc/xds_mux/grpc_mux_impl.cc +--- a/source/extensions/config_subscription/grpc/xds_mux/grpc_mux_impl.cc ++++ b/source/extensions/config_subscription/grpc/xds_mux/grpc_mux_impl.cc +@@ -322,6 +322,7 @@ void GrpcMuxImpl::handleEstablishedStream() { + maybeUpdateQueueSizeStat(0); + pausable_ack_queue_.clear(); + trySendDiscoveryRequests(); ++ stream_event_tracker_.onStreamEstablished(); + } + + template +@@ -346,6 +347,7 @@ void GrpcMuxImpl::handleStreamEstablishmentFailure( + } + } while (all_subscribed.size() != subscriptions_.size()); + should_send_initial_resource_versions_ = next_attempt_may_send_initial_resource_version; ++ stream_event_tracker_.onStreamClosed(); + } + + template +diff --git a/source/extensions/config_subscription/grpc/xds_mux/grpc_mux_impl.h b/source/extensions/config_subscription/grpc/xds_mux/grpc_mux_impl.h +--- a/source/extensions/config_subscription/grpc/xds_mux/grpc_mux_impl.h ++++ b/source/extensions/config_subscription/grpc/xds_mux/grpc_mux_impl.h +@@ -3,6 +3,7 @@ + #include + #include + #include ++#include + + #include "envoy/common/random_generator.h" + #include "envoy/common/time.h" +@@ -23,6 +24,7 @@ + #include "source/common/grpc/common.h" + #include "source/extensions/config_subscription/grpc/grpc_mux_context.h" + #include "source/extensions/config_subscription/grpc/grpc_mux_failover.h" ++#include "source/common/config/grpc_mux_stream_event_tracker.h" + #include "source/extensions/config_subscription/grpc/pausable_ack_queue.h" + #include "source/extensions/config_subscription/grpc/watch_map.h" + #include "source/extensions/config_subscription/grpc/xds_mux/delta_subscription_state.h" +@@ -82,6 +84,13 @@ public: + SubscriptionCallbacks& callbacks, + OpaqueResourceDecoderSharedPtr resource_decoder, + const SubscriptionOptions& options) override; ++ Common::CallbackHandlePtr ++ addStreamEventCallback(GrpcMuxStreamEventCallback callback) override { ++ return stream_event_tracker_.addStreamEventCallback(std::move(callback)); ++ } ++ ++ bool grpcStreamConnected() const override { return stream_event_tracker_.grpcStreamConnected(); } ++ + void updateWatch(const std::string& type_url, Watch* watch, + const absl::flat_hash_set& resources, + const SubscriptionOptions& options); +@@ -239,6 +248,7 @@ private: + // this one is up to GrpcMux. + const LocalInfo::LocalInfo& local_info_; + Common::CallbackHandlePtr dynamic_update_callback_handle_; ++ GrpcMuxStreamEventTracker stream_event_tracker_; + CustomConfigValidatorsPtr config_validators_; + XdsConfigTrackerOptRef xds_config_tracker_; + XdsResourcesDelegateOptRef xds_resources_delegate_; +@@ -291,6 +301,13 @@ private: + class NullGrpcMuxImpl : public GrpcMux { + public: + void start() override {} ++ Common::CallbackHandlePtr ++ addStreamEventCallback(GrpcMuxStreamEventCallback callback) override { ++ return stream_event_tracker_.addStreamEventCallback(std::move(callback)); ++ } ++ ++ bool grpcStreamConnected() const override { return stream_event_tracker_.grpcStreamConnected(); } ++ + + ScopedResume pause(const std::string&) override { + return std::make_unique([]() {}); +@@ -313,6 +330,9 @@ public: + void requestOnDemandUpdate(const std::string&, const absl::flat_hash_set&) override { + ENVOY_BUG(false, "unexpected request for on demand update"); + } + + EdsResourcesCacheOptRef edsResourcesCache() override { return {}; } ++ ++private: ++ GrpcMuxStreamEventTracker stream_event_tracker_; + }; +diff --git a/test/common/config/grpc_subscription_impl_test.cc b/test/common/config/grpc_subscription_impl_test.cc +--- a/test/common/config/grpc_subscription_impl_test.cc ++++ b/test/common/config/grpc_subscription_impl_test.cc +@@ -64,6 +64,42 @@ TEST_P(GrpcSubscriptionImplTest, RemoteStreamClose) { + timer_->invokeCallback(); + EXPECT_TRUE(statsAre(2, 0, 0, 1, 0, 0, 0, "")); + } + ++TEST_P(GrpcSubscriptionImplTest, StreamEventCallbacks) { ++ uint32_t established = 0; ++ uint32_t closed = 0; ++ auto callback_handle = mux_->addStreamEventCallback([&](GrpcMuxStreamEvent event) { ++ if (event == GrpcMuxStreamEvent::Established) { ++ ++established; ++ } else { ++ ++closed; ++ } ++ }); ++ ++ EXPECT_FALSE(mux_->grpcStreamConnected()); ++ startSubscription({"cluster0", "cluster1"}); ++ EXPECT_TRUE(mux_->grpcStreamConnected()); ++ EXPECT_EQ(1, established); ++ EXPECT_EQ(0, closed); ++ ++ EXPECT_CALL(callbacks_, ++ onConfigUpdateFailed(Envoy::Config::ConfigUpdateFailureReason::ConnectionFailure, _)) ++ .Times(0); ++ EXPECT_CALL(*timer_, enableTimer(_, _)); ++ EXPECT_CALL(random_, random()); ++ onRemoteClose(); ++ EXPECT_FALSE(mux_->grpcStreamConnected()); ++ EXPECT_EQ(1, established); ++ EXPECT_EQ(1, closed); ++ ++ callback_handle.reset(); ++ EXPECT_CALL(*async_client_, startRaw(_, _, _, _)).WillOnce(Return(&async_stream_)); ++ expectSendMessage({"cluster0", "cluster1"}, "", true); ++ timer_->invokeCallback(); ++ EXPECT_TRUE(mux_->grpcStreamConnected()); ++ EXPECT_EQ(1, established); ++ EXPECT_EQ(1, closed); ++} ++ + // Validate that When the management server gets multiple requests for the same version, it can + // ignore later ones. This allows the nonce to be used. + TEST_P(GrpcSubscriptionImplTest, RepeatedNonce) { +diff --git a/test/mocks/config/mocks.h b/test/mocks/config/mocks.h +--- a/test/mocks/config/mocks.h ++++ b/test/mocks/config/mocks.h +@@ -116,6 +116,10 @@ public: + MOCK_METHOD(ScopedResume, pause, (const std::string& type_url), (override)); + MOCK_METHOD(ScopedResume, pause, (const std::vector type_urls), (override)); + ++ MOCK_METHOD(Common::CallbackHandlePtr, addStreamEventCallback, ++ (GrpcMuxStreamEventCallback callback), (override)); ++ MOCK_METHOD(bool, grpcStreamConnected, (), (const, override)); ++ + MOCK_METHOD(void, addSubscription, + (const absl::flat_hash_set& resources, const std::string& type_url, + SubscriptionCallbacks& callbacks, SubscriptionStats& stats, diff --git a/tests/BUILD b/tests/BUILD index 1f6db5971..c084fdf34 100644 --- a/tests/BUILD +++ b/tests/BUILD @@ -122,6 +122,15 @@ envoy_cc_test( ], ) +envoy_cc_test( + name = "versioned_test", + srcs = ["versioned_test.cc"], + repository = "@envoy", + deps = [ + "//cilium:versioned_lib", + ], +) + envoy_cc_test( name = "bpf_metadata_config_test", srcs = ["bpf_metadata_config_test.cc"], diff --git a/tests/bpf_metadata.cc b/tests/bpf_metadata.cc index d72664bc6..84bd5e87a 100644 --- a/tests/bpf_metadata.cc +++ b/tests/bpf_metadata.cc @@ -49,10 +49,11 @@ std::string policy_path = ""; std::vector> sds_configs{}; -namespace { +namespace Cilium { std::shared_ptr -createHostMap(const std::string& config, Server::Configuration::ListenerFactoryContext& context) { +TestHelper::createHostMap(const std::string& config, + Server::Configuration::ListenerFactoryContext& context) { return context.serverFactoryContext().singletonManager().getTyped( "cilium_host_map_singleton", [&config, &context] { std::string path = TestEnvironment::writeStringToFileForTest("host_map.yaml", config); @@ -75,9 +76,9 @@ createHostMap(const std::string& config, Server::Configuration::ListenerFactoryC } std::shared_ptr -createPolicyMap(const std::string& config, - const std::vector>& secret_configs, - Server::Configuration::FactoryContext& context) { +TestHelper::createPolicyMap(const std::string& config, + const std::vector>& secret_configs, + Server::Configuration::FactoryContext& context) { return context.serverFactoryContext().singletonManager().getTyped( "cilium_network_policy_singleton", [&config, &secret_configs, &context] { if (!secret_configs.empty()) { @@ -115,17 +116,15 @@ createPolicyMap(const std::string& config, std::make_shared(context, Cilium::CILIUM_XDS_API_CONFIG); auto subscription = std::make_unique( context.serverFactoryContext().mainThreadDispatcher(), - Envoy::Config::makePathConfigSource(policy_path), map->getImpl(), + Envoy::Config::makePathConfigSource(policy_path), map->subscriptionCallbacksForTest(), std::make_shared(), stats, ProtobufMessage::getNullValidationVisitor(), context.serverFactoryContext().api()); - map->startSubscription(std::move(subscription)); + map->startSubscriptionForTest(std::move(subscription)); return map; }); } -} // namespace - -void initTestMaps(Server::Configuration::ListenerFactoryContext& context) { +void TestHelper::initTestMaps(Server::Configuration::ListenerFactoryContext& context) { // Create the file-based policy map before the filter is created, so that the // singleton is set before the gRPC subscription is attempted. hostmap = createHostMap(host_map_config, context); @@ -134,7 +133,6 @@ void initTestMaps(Server::Configuration::ListenerFactoryContext& context) { npmap = createPolicyMap(policy_config, sds_configs, context); } -namespace Cilium { namespace BpfMetadata { namespace { @@ -210,7 +208,7 @@ class TestBpfMetadataConfigFactory : public NamedListenerFilterConfigFactory { const Network::ListenerFilterMatcherSharedPtr& listener_filter_matcher, ListenerFactoryContext& context) override { - initTestMaps(context); + Cilium::TestHelper::initTestMaps(context); auto config = std::make_shared( MessageUtil::downcastAndValidate( diff --git a/tests/bpf_metadata.h b/tests/bpf_metadata.h index e57b29f09..ecdf80e22 100644 --- a/tests/bpf_metadata.h +++ b/tests/bpf_metadata.h @@ -27,9 +27,18 @@ extern std::string policy_config; extern std::string policy_path; extern std::vector> sds_configs; -extern void initTestMaps(Server::Configuration::ListenerFactoryContext& context); - namespace Cilium { + +struct TestHelper { + static std::shared_ptr + createHostMap(const std::string& config, Server::Configuration::ListenerFactoryContext&); + static std::shared_ptr + createPolicyMap(const std::string& config, + const std::vector>& secret_configs, + Server::Configuration::FactoryContext&); + static void initTestMaps(Server::Configuration::ListenerFactoryContext&); +}; + namespace BpfMetadata { class TestConfig : public Config { diff --git a/tests/bpf_metadata_config_test.cc b/tests/bpf_metadata_config_test.cc index eafd6a7b7..fd4cb9a35 100644 --- a/tests/bpf_metadata_config_test.cc +++ b/tests/bpf_metadata_config_test.cc @@ -197,7 +197,7 @@ class MetadataConfigTest : public testing::Test { host_map_config += extra_host_map_config; policy_config += extra_policy_config; - initTestMaps(context_); + Cilium::TestHelper::initTestMaps(context_); Init::WatcherImpl watcher("metadata test", []() {}); context_.initManager().initialize(watcher); @@ -226,7 +226,7 @@ class MetadataConfigTest : public testing::Test { TEST_F(MetadataConfigTest, NpdsConfigSupported) { ::cilium::BpfMetadata config{}; - config.mutable_npds_config()->mutable_api_config_source()->set_api_type( + config.mutable_config_source()->mutable_api_config_source()->set_api_type( envoy::config::core::v3::ApiConfigSource::GRPC); EXPECT_NO_THROW(initialize(config)); diff --git a/tests/bpf_metadata_integration_test.cc b/tests/bpf_metadata_integration_test.cc index 338073cd8..d7bcae333 100644 --- a/tests/bpf_metadata_integration_test.cc +++ b/tests/bpf_metadata_integration_test.cc @@ -1,6 +1,10 @@ #include +#include +#include +#include #include +#include #include #include "envoy/config/bootstrap/v3/bootstrap.pb.h" @@ -9,10 +13,11 @@ #include "envoy/config/listener/v3/listener.pb.h" #include "envoy/grpc/status.h" #include "envoy/http/codec.h" -#include "envoy/network/address.h" #include "envoy/service/discovery/v3/discovery.pb.h" #include "source/common/common/assert.h" +#include "source/common/common/logger.h" +#include "source/common/network/utility.h" #include "source/common/protobuf/utility.h" #include "test/common/grpc/grpc_client_integration.h" @@ -22,18 +27,29 @@ #include "test/test_common/resources.h" #include "test/test_common/utility.h" +#include "absl/synchronization/mutex.h" #include "cilium/api/bpf_metadata.pb.h" #include "cilium/api/npds.pb.h" #include "cilium/api/nphds.pb.h" +#include "cilium/host_map.h" +#include "cilium/network_policy.h" +#include "cilium/policy_id.h" #include "gtest/gtest.h" namespace Envoy { namespace { const std::string NetworkPolicyTypeUrl = "type.googleapis.com/cilium.NetworkPolicy"; +const std::string NetworkPolicyResourceTypeUrl = "type.googleapis.com/cilium.NetworkPolicyResource"; const std::string NetworkPolicyHostsTypeUrl = "type.googleapis.com/cilium.NetworkPolicyHosts"; +struct NetworkPolicyResourceConfig { + std::string name; + std::string version; + std::string yaml; +}; + const std::string policy1 = R"EOF( endpoint_ips: - '10.1.1.1' @@ -56,6 +72,58 @@ const std::string policy2 = R"EOF( - remote_policies: [ 111 ] )EOF"; +const std::string invalid_policy = R"EOF( + endpoint_id: 8192 +)EOF"; + +const NetworkPolicyResourceConfig selector1_resource = {"selector-1", "1", R"EOF( + selector: + remote_identities: [ 43 ] +)EOF"}; + +const NetworkPolicyResourceConfig selector2_resource = {"selector-2", "1", R"EOF( + selector: + remote_identities: [ 44 ] +)EOF"}; + +const NetworkPolicyResourceConfig selector3_resource = {"selector-3", "1", R"EOF( + selector: + remote_identities: [ 45 ] +)EOF"}; + +const NetworkPolicyResourceConfig policy42_resource = {"policy-42", "1", R"EOF( + policy: + endpoint_ips: + - "10.1.2.3" + endpoint_id: 42 + ingress_per_port_policies: + - port: 80 + rules: + - selectors: [ "selector-1" ] +)EOF"}; + +const NetworkPolicyResourceConfig policy43_resource = {"policy-43", "1", R"EOF( + policy: + endpoint_ips: + - "10.2.3.4" + endpoint_id: 43 + ingress_per_port_policies: + - port: 81 + rules: + - selectors: [ "selector-2" ] +)EOF"}; + +const NetworkPolicyResourceConfig policy42_new_stream_resource = {"policy-42", "2", R"EOF( + policy: + endpoint_ips: + - "10.1.2.3" + endpoint_id: 42 + ingress_per_port_policies: + - port: 8080 + rules: + - selectors: [ "selector-3" ] +)EOF"}; + const std::string policy_host1 = R"EOF( policy: 111 host_addresses: [ "10.1.1.1", "f00d::1:1:1" ] @@ -66,6 +134,21 @@ const std::string policy_host2 = R"EOF( host_addresses: [ "10.2.2.2", "f00d::2:2:2" ] )EOF"; +const NetworkPolicyResourceConfig policy_host1_resource = {"111", "1", R"EOF( + policy: 111 + host_addresses: [ "10.1.1.1", "f00d::1:1:1" ] +)EOF"}; + +const NetworkPolicyResourceConfig policy_host2_resource = {"222", "1", R"EOF( + policy: 222 + host_addresses: [ "10.2.2.2", "f00d::2:2:2" ] +)EOF"}; + +const NetworkPolicyResourceConfig policy_host1_new_stream_resource = {"111", "2", R"EOF( + policy: 111 + host_addresses: [ "10.1.1.1", "f00d::1:1:1" ] +)EOF"}; + class BpfMetadataIntegrationTest : public BaseIntegrationTest, public Grpc::GrpcClientIntegrationParamTest { public: @@ -80,30 +163,66 @@ class BpfMetadataIntegrationTest : public BaseIntegrationTest, cluster: cluster_0 )EOF") { skip_tag_extraction_rule_check_ = true; + use_lds_ = false; // skip built in listener setup, we do it explicitly via xDS + setUpstreamCount(1); // same as default + defer_listener_finalization_ = true; + +#if 0 + for (Logger::Logger& logger : Logger::Registry::loggers()) { + logger.setLevel(spdlog::level::trace); + } +#endif } ~BpfMetadataIntegrationTest() override { resetConnections(); } - void setGrpcServiceHelper(envoy::config::core::v3::GrpcService& grpc_service, - const std::string& cluster_name, - Network::Address::InstanceConstSharedPtr address) { - setGrpcService(grpc_service, cluster_name, address); + void setUpGrpcLds(bool use_ads = true) { + config_helper_.addConfigModifier( + [this, use_ads](envoy::config::bootstrap::v3::Bootstrap& bootstrap) { + listener_config_.Swap(bootstrap.mutable_static_resources()->mutable_listeners(0)); + listener_config_.set_name(listener_name_); + bootstrap.mutable_static_resources()->mutable_listeners()->Clear(); + + auto* lds_config_source = bootstrap.mutable_dynamic_resources()->mutable_lds_config(); + lds_config_source->Clear(); + lds_config_source->set_resource_api_version(envoy::config::core::v3::ApiVersion::V3); + if (use_ads) { + lds_config_source->mutable_ads(); + } else { + setGrpcApiConfigSource(*lds_config_source); + } + }); } - void setUpGrpcLds() { - config_helper_.addConfigModifier([this](envoy::config::bootstrap::v3::Bootstrap& bootstrap) { - listener_config_.Swap(bootstrap.mutable_static_resources()->mutable_listeners(0)); - listener_config_.set_name(listener_name_); - bootstrap.mutable_static_resources()->mutable_listeners()->Clear(); + void setGrpcApiConfigSource(envoy::config::core::v3::ConfigSource& config_source, + envoy::config::core::v3::ApiConfigSource::ApiType api_type = + envoy::config::core::v3::ApiConfigSource::GRPC) { + config_source.set_resource_api_version(envoy::config::core::v3::ApiVersion::V3); + auto* api_config_source = config_source.mutable_api_config_source(); + api_config_source->set_set_node_on_first_message_only(true); + api_config_source->set_api_type(api_type); + api_config_source->set_transport_api_version(envoy::config::core::v3::ApiVersion::V3); + api_config_source->add_grpc_services()->mutable_envoy_grpc()->set_cluster_name( + "xds-grpc-cilium"); + } - auto* lds_config_source = bootstrap.mutable_dynamic_resources()->mutable_lds_config(); - lds_config_source->set_resource_api_version(envoy::config::core::v3::ApiVersion::V3); - lds_config_source->mutable_ads(); - }); + void setBpfMetadataNpdsConfig(::cilium::BpfMetadata& bpf_config, bool use_ads, + envoy::config::core::v3::ApiConfigSource::ApiType api_type = + envoy::config::core::v3::ApiConfigSource::GRPC) { + auto* config_source = bpf_config.mutable_config_source(); + config_source->Clear(); + if (use_ads) { + config_source->set_resource_api_version(envoy::config::core::v3::ApiVersion::V3); + config_source->mutable_ads(); + } else { + setGrpcApiConfigSource(*config_source, api_type); + } } - // Inject the cilium.bpf_metadata listener filter with npds_config into the listener. - void addBpfMetadataListenerFilter(envoy::config::listener::v3::Listener& listener, bool) { + // Inject the cilium.bpf_metadata listener filter with config_source into the listener. + void addBpfMetadataListenerFilter(envoy::config::listener::v3::Listener& listener, bool use_ads, + envoy::config::core::v3::ApiConfigSource::ApiType api_type = + envoy::config::core::v3::ApiConfigSource::GRPC) { auto* listener_filter = listener.add_listener_filters(); listener_filter->set_name("cilium.bpf_metadata"); @@ -111,18 +230,29 @@ class BpfMetadataIntegrationTest : public BaseIntegrationTest, bpf_config.set_is_ingress(false); bpf_config.set_use_nphds(true); - auto* npds_config = bpf_config.mutable_npds_config(); - npds_config->set_resource_api_version(envoy::config::core::v3::ApiVersion::V3); - npds_config->mutable_ads(); + setBpfMetadataNpdsConfig(bpf_config, use_ads, api_type); listener_filter->mutable_typed_config()->PackFrom(bpf_config); } - void initialize() override { - use_lds_ = false; - setUpstreamCount(1); - defer_listener_finalization_ = true; + void updateBpfMetadataListenerFilter(envoy::config::listener::v3::Listener& listener, + envoy::config::core::v3::ApiConfigSource::ApiType api_type) { + for (auto& listener_filter : *listener.mutable_listener_filters()) { + if (listener_filter.name() != "cilium.bpf_metadata") { + continue; + } + + ::cilium::BpfMetadata bpf_config; + RELEASE_ASSERT(listener_filter.typed_config().UnpackTo(&bpf_config), + "cilium.bpf_metadata listener filter typed_config failed to unpack"); + setBpfMetadataNpdsConfig(bpf_config, /*use_ads=*/false, api_type); + listener_filter.mutable_typed_config()->PackFrom(bpf_config); + return; + } + RELEASE_ASSERT(false, "cilium.bpf_metadata listener filter not found"); + } + void initializeAds() { config_helper_.addConfigModifier([](envoy::config::bootstrap::v3::Bootstrap& bootstrap) { // Add the ADS gRPC cluster. auto* ads_cluster = bootstrap.mutable_static_resources()->add_clusters(); @@ -149,44 +279,132 @@ class BpfMetadataIntegrationTest : public BaseIntegrationTest, BaseIntegrationTest::initialize(); } + void initializeSotw() { + config_helper_.addConfigModifier([this](envoy::config::bootstrap::v3::Bootstrap& bootstrap) { + auto* xds_cluster = bootstrap.mutable_static_resources()->add_clusters(); + xds_cluster->MergeFrom(bootstrap.static_resources().clusters()[0]); + xds_cluster->set_name("xds-grpc-cilium"); + ConfigHelper::setHttp2(*xds_cluster); + + auto* cds_config = bootstrap.mutable_dynamic_resources()->mutable_cds_config(); + setGrpcApiConfigSource(*cds_config); + }); + + // Must be last modifier — it removes static listeners. + setUpGrpcLds(/*use_ads=*/false); + + BaseIntegrationTest::initialize(); + } + void createUpstreams() override { BaseIntegrationTest::createUpstreams(); - // ADS upstream (fake_upstreams_[1]). + // ADS or SotW upstream (fake_upstreams_[1]). addFakeUpstream(Envoy::Http::CodecType::HTTP2); } - FakeUpstream& getAdsFakeUpstream() const { return *fake_upstreams_[1]; } + FakeUpstream& getFakeUpstream() const { return *fake_upstreams_[1]; } - void createAdsStream() { - AssertionResult result = - getAdsFakeUpstream().waitForHttpConnection(*dispatcher_, ads_connection_); + void createXdsConnection() { + if (xds_connection_ == nullptr) { + AssertionResult result = + getFakeUpstream().waitForHttpConnection(*dispatcher_, xds_connection_); + RELEASE_ASSERT(result, result.message()); + } + } + + void createXdsStream(FakeStreamPtr& stream) { + auto result = xds_connection_->waitForNewStream(*dispatcher_, stream); RELEASE_ASSERT(result, result.message()); - auto result2 = ads_connection_->waitForNewStream(*dispatcher_, ads_stream_); - RELEASE_ASSERT(result2, result2.message()); - ads_stream_->startGrpcStream(); + result = stream->waitForHeadersComplete(); + RELEASE_ASSERT(result, result.message()); + stream->startGrpcStream(); + } + + void createAdsStream() { + createXdsConnection(); + createXdsStream(ads_stream_); + } + + void createStreamsUntil(const std::string& response_version, const std::string& type_url) { + createXdsConnection(); + + for (int i = 0; i < 8; i++) { + FakeStreamPtr stream; + createXdsStream(stream); + + std::string request_type_url; + const bool is_delta = + stream->headers().getPathValue().find("/Delta") != absl::string_view::npos; + if (is_delta) { + envoy::service::discovery::v3::DeltaDiscoveryRequest request; + auto result = stream->waitForGrpcMessage(*dispatcher_, request); + RELEASE_ASSERT(result, result.message()); + request_type_url = request.type_url(); + } else { + envoy::service::discovery::v3::DiscoveryRequest request; + auto result = stream->waitForGrpcMessage(*dispatcher_, request); + RELEASE_ASSERT(result, result.message()); + request_type_url = request.type_url(); + } + + if (request_type_url == NetworkPolicyResourceTypeUrl) { + ENVOY_LOG_MISC(info, "GOT NPRDS STREAM"); + nprds_stream_ = std::move(stream); + if (request_type_url == type_url) { + return; + } + } else if (request_type_url == NetworkPolicyTypeUrl) { + ENVOY_LOG_MISC(info, "GOT NPDS STREAM"); + npds_stream_ = std::move(stream); + if (request_type_url == type_url) { + return; + } + } else if (request_type_url == Envoy::Config::TestTypeUrl::get().Listener) { + ENVOY_LOG_MISC(info, "GOT LDS STREAM"); + lds_stream_ = std::move(stream); + sendLdsResponse(*lds_stream_, {MessageUtil::getYamlStringFromMessage(listener_config_)}, + response_version); + } else if (request_type_url == Envoy::Config::TestTypeUrl::get().Cluster) { + ENVOY_LOG_MISC(info, "GOT CDS STREAM"); + cds_stream_ = std::move(stream); + sendCdsResponse(*cds_stream_, response_version); + } else if (request_type_url == NetworkPolicyHostsTypeUrl) { + ENVOY_LOG_MISC(info, "GOT NPHDS STREAM"); + nphds_stream_ = std::move(stream); + if (!is_delta) { + sendNphdsResponse(*nphds_stream_, response_version); + } + if (request_type_url == type_url) { + return; + } + } + } + + RELEASE_ASSERT(npds_stream_ != nullptr, fmt::format("{} stream was not established", type_url)); } - void sendCdsResponse(const std::string& version) { + void sendCdsResponse(FakeStream& stream, const std::string& version) { envoy::service::discovery::v3::DiscoveryResponse response; response.set_version_info(version); + response.set_nonce(version); response.set_type_url(Envoy::Config::TestTypeUrl::get().Cluster); - ASSERT_NE(nullptr, ads_stream_); - ads_stream_->sendGrpcMessage(response); + stream.sendGrpcMessage(response); } - void sendLdsResponse(const std::vector& listener_configs, + void sendLdsResponse(FakeStream& stream, + const std::vector& listener_configs, const std::string& version) { envoy::service::discovery::v3::DiscoveryResponse response; response.set_version_info(version); + response.set_nonce(version); response.set_type_url(Envoy::Config::TestTypeUrl::get().Listener); for (const auto& listener_config : listener_configs) { response.add_resources()->PackFrom(listener_config); } - ASSERT_NE(nullptr, ads_stream_); - ads_stream_->sendGrpcMessage(response); + stream.sendGrpcMessage(response); } - void sendLdsResponse(const std::vector& listener_configs, + void sendLdsResponse(FakeStream& stream, const std::vector& listener_configs, const std::string& version) { std::vector proto_configs; proto_configs.reserve(listener_configs.size()); @@ -194,26 +412,30 @@ class BpfMetadataIntegrationTest : public BaseIntegrationTest, proto_configs.emplace_back( TestUtility::parseYaml(listener_blob)); } - sendLdsResponse(proto_configs, version); + sendLdsResponse(stream, proto_configs, version); } - void sendNpdsResponse(const std::string& version) { + void sendNpdsResponse(FakeStream& stream, const std::string& version, + const std::vector& policy_configs = {policy1, policy2}) { envoy::service::discovery::v3::DiscoveryResponse response; response.set_version_info(version); + response.set_nonce(version); response.set_type_url(NetworkPolicyTypeUrl); std::vector proto_configs; - proto_configs.emplace_back(TestUtility::parseYaml(policy1)); - proto_configs.emplace_back(TestUtility::parseYaml(policy2)); + proto_configs.reserve(policy_configs.size()); + for (const auto& policy_config : policy_configs) { + proto_configs.emplace_back(TestUtility::parseYaml(policy_config)); + } for (const auto& policy_config : proto_configs) { response.add_resources()->PackFrom(policy_config); } - ASSERT_NE(nullptr, ads_stream_); - ads_stream_->sendGrpcMessage(response); + stream.sendGrpcMessage(response); } - void sendNphdsResponse(const std::string& version) { + void sendNphdsResponse(FakeStream& stream, const std::string& version) { envoy::service::discovery::v3::DiscoveryResponse response; response.set_version_info(version); + response.set_nonce(version); response.set_type_url(NetworkPolicyHostsTypeUrl); std::vector proto_configs; proto_configs.emplace_back(TestUtility::parseYaml(policy_host1)); @@ -221,24 +443,151 @@ class BpfMetadataIntegrationTest : public BaseIntegrationTest, for (const auto& policy_host_config : proto_configs) { response.add_resources()->PackFrom(policy_host_config); } - ASSERT_NE(nullptr, ads_stream_); - ads_stream_->sendGrpcMessage(response); + stream.sendGrpcMessage(response); + } + + void sendNprdsDeltaResponse(FakeStream& stream, const std::string& version, + const std::vector& resource_configs, + const std::vector& removed_resources = {}) { + envoy::service::discovery::v3::DeltaDiscoveryResponse response; + response.set_system_version_info(version); + response.set_nonce(version); + response.set_type_url(NetworkPolicyResourceTypeUrl); + for (const auto& resource_config : resource_configs) { + envoy::service::discovery::v3::Resource* resource = response.add_resources(); + resource->set_name(resource_config.name); + resource->set_version(resource_config.version); + resource->mutable_resource()->PackFrom( + TestUtility::parseYaml(resource_config.yaml)); + } + for (const auto& removed_resource : removed_resources) { + response.add_removed_resources(removed_resource); + } + stream.sendGrpcMessage(response); + } + + void sendNphdsDeltaResponse(FakeStream& stream, const std::string& version, + const std::vector& resource_configs, + const std::vector& removed_resources = {}) { + envoy::service::discovery::v3::DeltaDiscoveryResponse response; + response.set_system_version_info(version); + response.set_nonce(version); + response.set_type_url(NetworkPolicyHostsTypeUrl); + for (const auto& resource_config : resource_configs) { + envoy::service::discovery::v3::Resource* resource = response.add_resources(); + resource->set_name(resource_config.name); + resource->set_version(resource_config.version); + resource->mutable_resource()->PackFrom( + TestUtility::parseYaml(resource_config.yaml)); + } + for (const auto& removed_resource : removed_resources) { + response.add_removed_resources(removed_resource); + } + stream.sendGrpcMessage(response); + } + + AssertionResult compareNprdsAck() { + return compareDeltaDiscoveryRequest(NetworkPolicyResourceTypeUrl, {}, {}, nprds_stream_.get(), + Grpc::Status::WellKnownGrpcStatus::Ok, "", + /*expect_node=*/false); + } + + AssertionResult compareNphdsAck() { + return compareDeltaDiscoveryRequest(NetworkPolicyHostsTypeUrl, {}, {}, nphds_stream_.get(), + Grpc::Status::WellKnownGrpcStatus::Ok, "", + /*expect_node=*/false); } + void retireStream(FakeStreamPtr& stream) { + if (stream != nullptr) { + retired_streams_.push_back(std::move(stream)); + } + } + + void resetGrpcStream(FakeStreamPtr& stream) { + stream->encodeResetStream(); + AssertionResult result = stream->waitForReset(*dispatcher_); + RELEASE_ASSERT(result, result.message()); + retireStream(stream); + } + + void resetNprdsStream() { resetGrpcStream(nprds_stream_); } + + void resetNphdsStream() { resetGrpcStream(nphds_stream_); } + void resetConnections() { - if (ads_connection_ != nullptr) { - AssertionResult result = ads_connection_->close(); + if (xds_connection_ != nullptr) { + AssertionResult result = xds_connection_->close(); RELEASE_ASSERT(result, result.message()); - result = ads_connection_->waitForDisconnect(); + result = xds_connection_->waitForDisconnect(); RELEASE_ASSERT(result, result.message()); - ads_connection_.reset(); + xds_connection_.reset(); } + ads_stream_.reset(); + lds_stream_.reset(); + cds_stream_.reset(); + npds_stream_.reset(); + nphds_stream_.reset(); + nprds_stream_.reset(); + retired_streams_.clear(); + } + + uint64_t policyStreamGeneration() const { + return test_server_->gauge("cilium.policy.policy_stream_generation")->value(); + } + + uint64_t waitForPolicyStreamGenerationAfter(uint64_t previous_generation) { + test_server_->waitForGaugeGe("cilium.policy.policy_stream_generation", previous_generation + 1); + const uint64_t generation = policyStreamGeneration(); + EXPECT_GT(generation, previous_generation); + return generation; + } + + std::shared_ptr networkPolicyMap() const { + auto map = test_server_->server().singletonManager().getTyped( + "cilium_network_policy_singleton"); + RELEASE_ASSERT(map != nullptr, "Cilium NetworkPolicyMap singleton was not created"); + return map; + } + + uint64_t resolveHostPolicyId(const std::string& address) const { + auto parsed_address = Network::Utility::parseInternetAddressNoThrow(address); + RELEASE_ASSERT(parsed_address != nullptr, + fmt::format("failed to parse host address {}", address)); + auto map = test_server_->server().singletonManager().getTyped( + "cilium_host_map_singleton"); + RELEASE_ASSERT(map != nullptr, "Cilium PolicyHostMap singleton was not created"); + + absl::Mutex lock; + bool resolved = false; + uint64_t policy_id = Cilium::ID::UNKNOWN; + + // PolicyHostMap lookups must run on an Envoy thread that has thread-local storage registered. + // The gtest/integration thread calling this helper is not such a thread, so dereferencing the + // TLS-backed host map directly here is unsafe. Posting the resolve to the server dispatcher + // keeps the lookup on a valid Envoy TLS thread while still letting the test read the result. + test_server_->server().dispatcher().post([&, map, parsed_address]() { + policy_id = map->resolve(parsed_address->ip()); + lock.Lock(); + resolved = true; + lock.Unlock(); + }); + + lock.LockWhen(absl::Condition(&resolved)); + lock.Unlock(); + return policy_id; } envoy::config::listener::v3::Listener listener_config_; std::string listener_name_{"testing-listener-0"}; - FakeHttpConnectionPtr ads_connection_; + FakeHttpConnectionPtr xds_connection_; FakeStreamPtr ads_stream_; + FakeStreamPtr lds_stream_; + FakeStreamPtr cds_stream_; + FakeStreamPtr npds_stream_; + FakeStreamPtr nphds_stream_; + FakeStreamPtr nprds_stream_; + std::vector retired_streams_; }; INSTANTIATE_TEST_SUITE_P(IpVersionsAndGrpcTypes, BpfMetadataIntegrationTest, @@ -251,20 +600,212 @@ TEST_P(BpfMetadataIntegrationTest, BpfMetadataWithNpdsAndNpdhsViaAds) { EXPECT_TRUE(compareDiscoveryRequest( Config::TestTypeUrl::get().Cluster, "", {}, {}, {}, /*expect_node=*/true, Envoy::Grpc::Status::WellKnownGrpcStatus::Ok, "", ads_stream_.get())); - sendCdsResponse("1"); + sendCdsResponse(*ads_stream_, "1"); EXPECT_TRUE(compareDiscoveryRequest( Config::TestTypeUrl::get().Listener, "", {}, {}, {}, /*expect_node=*/false, Grpc::Status::WellKnownGrpcStatus::Ok, "", ads_stream_.get())); - sendLdsResponse({MessageUtil::getYamlStringFromMessage(listener_config_)}, "1"); + sendLdsResponse(*ads_stream_, {MessageUtil::getYamlStringFromMessage(listener_config_)}, "1"); }; - initialize(); + initializeAds(); test_server_->waitForCounterGe("listener_manager.lds.update_success", 1); EXPECT_EQ(test_server_->server().listenerManager().listeners().size(), 1); - sendNpdsResponse("1"); + sendNpdsResponse(*ads_stream_, "1"); + test_server_->waitForCounterGe("cilium.policy.update_success", 1); + sendNphdsResponse(*ads_stream_, "1"); + test_server_->waitForCounterGe("cilium.hostmap.update_success", 1); +} + +TEST_P(BpfMetadataIntegrationTest, PolicyStreamGenerationTracksAcceptedAdsGrpcStreams) { + on_server_init_function_ = [&]() { + createAdsStream(); + addBpfMetadataListenerFilter(listener_config_, /*use_ads=*/true); + EXPECT_TRUE(compareDiscoveryRequest( + Config::TestTypeUrl::get().Cluster, "", {}, {}, {}, + /*expect_node=*/true, Envoy::Grpc::Status::WellKnownGrpcStatus::Ok, "", ads_stream_.get())); + sendCdsResponse(*ads_stream_, "1"); + EXPECT_TRUE(compareDiscoveryRequest( + Config::TestTypeUrl::get().Listener, "", {}, {}, {}, /*expect_node=*/false, + Grpc::Status::WellKnownGrpcStatus::Ok, "", ads_stream_.get())); + sendLdsResponse(*ads_stream_, {MessageUtil::getYamlStringFromMessage(listener_config_)}, "1"); + }; + initializeAds(); + + test_server_->waitForCounterGe("listener_manager.lds.update_success", 1); + EXPECT_EQ(policyStreamGeneration(), 0); + + sendNpdsResponse(*ads_stream_, "1"); + test_server_->waitForCounterGe("cilium.policy.update_success", 1); + const uint64_t first_generation = waitForPolicyStreamGenerationAfter(0); + + sendNpdsResponse(*ads_stream_, "2"); + test_server_->waitForCounterGe("cilium.policy.update_success", 2); + EXPECT_EQ(policyStreamGeneration(), first_generation); + + resetConnections(); + EXPECT_EQ(policyStreamGeneration(), first_generation); +} + +TEST_P(BpfMetadataIntegrationTest, PolicyStreamGenerationTracksAcceptedSotwGrpcStreams) { + on_server_init_function_ = [&]() { + addBpfMetadataListenerFilter(listener_config_, /*use_ads=*/false); + createStreamsUntil("1", NetworkPolicyTypeUrl); + }; + initializeSotw(); + test_server_->waitForCounterGe("listener_manager.lds.update_success", 1); + EXPECT_EQ(policyStreamGeneration(), 0); + + sendNpdsResponse(*npds_stream_, "1"); + test_server_->waitForCounterGe("cilium.policy.update_success", 1); + const uint64_t first_generation = waitForPolicyStreamGenerationAfter(0); + + sendNpdsResponse(*npds_stream_, "2"); + test_server_->waitForCounterGe("cilium.policy.update_success", 2); + EXPECT_EQ(policyStreamGeneration(), first_generation); + + resetConnections(); + EXPECT_EQ(policyStreamGeneration(), first_generation); + + createStreamsUntil("2", NetworkPolicyTypeUrl); + sendNpdsResponse(*npds_stream_, "3", {invalid_policy}); + // The invalid policy is rejected by the real gRPC subscription decoder/validator before + // NetworkPolicyMapImpl::onConfigUpdate() runs, so this increments NPDS subscription stats + // rather than cilium.policy.updates_rejected. + test_server_->waitForCounterGe("cilium.npds.update_rejected", 1); + EXPECT_EQ(policyStreamGeneration(), first_generation); + + sendNpdsResponse(*npds_stream_, "4"); + test_server_->waitForCounterGe("cilium.policy.update_success", 3); + waitForPolicyStreamGenerationAfter(first_generation); +} + +TEST_P(BpfMetadataIntegrationTest, PolicyStreamGenerationTracksAcceptedDeltaNprdsStreams) { + on_server_init_function_ = [&]() { + // Step 1: establish the initial SotW LDS, CDS, NPHDS, and NPDS streams. + addBpfMetadataListenerFilter(listener_config_, /*use_ads=*/false); + createStreamsUntil("1", NetworkPolicyTypeUrl); + }; + initializeSotw(); + test_server_->waitForCounterGe("listener_manager.lds.update_success", 1); + + auto policy_map = networkPolicyMap(); + EXPECT_EQ(policyStreamGeneration(), 0); + + // Step 2: accept a real SotW NPDS response so the starting mode has installed policy. + sendNpdsResponse(*npds_stream_, "1"); test_server_->waitForCounterGe("cilium.policy.update_success", 1); - sendNphdsResponse("1"); + const uint64_t sotw_generation = waitForPolicyStreamGenerationAfter(0); + EXPECT_TRUE(policy_map->exists("10.1.1.1")); + EXPECT_TRUE(policy_map->exists("10.2.2.2")); + + // Step 3: update the BpfMetadata config source; this is evidence that Delta NPRDS is available. + updateBpfMetadataListenerFilter(listener_config_, + envoy::config::core::v3::ApiConfigSource::DELTA_GRPC); + sendLdsResponse(*lds_stream_, {listener_config_}, "2"); + test_server_->waitForCounterGe("listener_manager.lds.update_success", 2); + + // Step 4: observe the immediate switch to Delta NPRDS without advancing accepted policy state. + createStreamsUntil("2", NetworkPolicyResourceTypeUrl); + EXPECT_EQ(policyStreamGeneration(), sotw_generation); + + // Step 5: accept the first Delta NPRDS update and retire the prior SotW policy resources. + sendNprdsDeltaResponse( + *nprds_stream_, "1", + {selector1_resource, selector2_resource, policy42_resource, policy43_resource}); + EXPECT_TRUE(compareNprdsAck()); + const uint64_t first_generation = waitForPolicyStreamGenerationAfter(sotw_generation); + EXPECT_FALSE(policy_map->exists("10.1.1.1")); + EXPECT_FALSE(policy_map->exists("10.2.2.2")); + EXPECT_TRUE(policy_map->exists("10.1.2.3")); + EXPECT_TRUE(policy_map->exists("10.2.3.4")); + + // Step 6: accept a same-stream Delta update; stream generation and omitted resources stay put. + sendNprdsDeltaResponse(*nprds_stream_, "2", {selector3_resource, policy42_new_stream_resource}); + EXPECT_TRUE(compareNprdsAck()); + EXPECT_EQ(policyStreamGeneration(), first_generation); + EXPECT_TRUE(policy_map->exists("10.1.2.3")); + EXPECT_TRUE(policy_map->exists("10.2.3.4")); + + // Step 7: reset the Delta NPRDS stream. + resetNprdsStream(); + + // Step 8: open the replacement Delta stream; reconnect alone must not advance policy state. + createStreamsUntil("3", NetworkPolicyResourceTypeUrl); + EXPECT_EQ(policyStreamGeneration(), first_generation); + EXPECT_TRUE(policy_map->exists("10.2.3.4")); + + // Step 9: accept the first update on the new stream and retire resources from the old stream. + sendNprdsDeltaResponse(*nprds_stream_, "3", {selector3_resource, policy42_new_stream_resource}); + waitForPolicyStreamGenerationAfter(first_generation); + EXPECT_TRUE(policy_map->exists("10.1.2.3")); + EXPECT_FALSE(policy_map->exists("10.2.3.4")); +} + +TEST_P(BpfMetadataIntegrationTest, PolicyStreamGenerationTracksAcceptedDeltaNphdsStreams) { + on_server_init_function_ = [&]() { + // Step 1: establish the initial SotW LDS, CDS, NPHDS, and NPDS streams. + addBpfMetadataListenerFilter(listener_config_, /*use_ads=*/false); + createStreamsUntil("1", NetworkPolicyTypeUrl); + }; + initializeSotw(); + test_server_->waitForCounterGe("listener_manager.lds.update_success", 1); test_server_->waitForCounterGe("cilium.hostmap.update_success", 1); + + auto policy_map = networkPolicyMap(); + EXPECT_EQ(policyStreamGeneration(), 0); + EXPECT_EQ(resolveHostPolicyId("10.1.1.1"), 111); + EXPECT_EQ(resolveHostPolicyId("10.2.2.2"), 222); + + // Step 2: accept a real SotW NPDS response so the starting mode has installed policy. + sendNpdsResponse(*npds_stream_, "1"); + test_server_->waitForCounterGe("cilium.policy.update_success", 1); + const uint64_t sotw_generation = waitForPolicyStreamGenerationAfter(0); + EXPECT_TRUE(policy_map->exists("10.1.1.1")); + EXPECT_TRUE(policy_map->exists("10.2.2.2")); + + // Step 3: update the BpfMetadata config source; this is evidence that Delta NPHDS is available. + updateBpfMetadataListenerFilter(listener_config_, + envoy::config::core::v3::ApiConfigSource::DELTA_GRPC); + sendLdsResponse(*lds_stream_, {listener_config_}, "2"); + test_server_->waitForCounterGe("listener_manager.lds.update_success", 2); + + // Step 4: observe the immediate switch to Delta NPHDS without advancing accepted policy state. + createStreamsUntil("2", NetworkPolicyHostsTypeUrl); + EXPECT_EQ(policyStreamGeneration(), sotw_generation); + EXPECT_EQ(resolveHostPolicyId("10.1.1.1"), 111); + EXPECT_EQ(resolveHostPolicyId("10.2.2.2"), 222); + + // Step 5: accept the first Delta NPHDS update. This should not change policy stream generation. + sendNphdsDeltaResponse(*nphds_stream_, "1", {policy_host1_resource, policy_host2_resource}); + EXPECT_TRUE(compareNphdsAck()); + test_server_->waitForCounterGe("cilium.hostmap.update_success", 2); + EXPECT_EQ(policyStreamGeneration(), sotw_generation); + EXPECT_EQ(resolveHostPolicyId("10.1.1.1"), 111); + EXPECT_EQ(resolveHostPolicyId("10.2.2.2"), 222); + + // Step 6: accept a same-stream Delta update; omitted resources stay present on the same stream. + sendNphdsDeltaResponse(*nphds_stream_, "2", {policy_host1_new_stream_resource}); + EXPECT_TRUE(compareNphdsAck()); + test_server_->waitForCounterGe("cilium.hostmap.update_success", 3); + EXPECT_EQ(policyStreamGeneration(), sotw_generation); + EXPECT_EQ(resolveHostPolicyId("10.1.1.1"), 111); + EXPECT_EQ(resolveHostPolicyId("10.2.2.2"), 222); + + // Step 7: reset the Delta NPHDS stream. + resetNphdsStream(); + + // Step 8: open the replacement Delta stream; reconnect alone must not advance policy state. + createStreamsUntil("3", NetworkPolicyHostsTypeUrl); + EXPECT_EQ(policyStreamGeneration(), sotw_generation); + EXPECT_EQ(resolveHostPolicyId("10.2.2.2"), 222); + + // Step 9: accept the first update on the new stream and retire resources from the old stream. + sendNphdsDeltaResponse(*nphds_stream_, "3", {policy_host1_new_stream_resource}); + EXPECT_TRUE(compareNphdsAck()); + test_server_->waitForCounterGe("cilium.hostmap.update_success", 4); + EXPECT_EQ(policyStreamGeneration(), sotw_generation); + EXPECT_EQ(resolveHostPolicyId("10.1.1.1"), 111); + EXPECT_EQ(resolveHostPolicyId("10.2.2.2"), Cilium::ID::UNKNOWN); } } // namespace diff --git a/tests/cilium_network_policy_test.cc b/tests/cilium_network_policy_test.cc index b74055409..340b7f7c9 100644 --- a/tests/cilium_network_policy_test.cc +++ b/tests/cilium_network_policy_test.cc @@ -3,14 +3,17 @@ #include #include +#include #include #include #include #include +#include #include "envoy/common/exception.h" #include "envoy/common/optref.h" #include "envoy/config/core/v3/config_source.pb.h" +#include "envoy/config/subscription.h" #include "envoy/init/manager.h" #include "envoy/server/factory_context.h" #include "envoy/service/discovery/v3/discovery.pb.h" @@ -32,6 +35,7 @@ #include "test/mocks/server/factory_context.h" #include "test/test_common/utility.h" +#include "absl/container/flat_hash_set.h" #include "absl/strings/string_view.h" #include "cilium/accesslog.h" #include "cilium/network_policy.h" @@ -70,6 +74,42 @@ extern const envoy::config::core::v3::ConfigSource CILIUM_XDS_API_CONFIG; return secret_provider; \ })) +namespace { + +envoy::config::core::v3::ConfigSource configSourceForUseDeltaXds(bool use_delta_xds) { + auto config_source = CILIUM_XDS_API_CONFIG; + config_source.mutable_api_config_source()->set_api_type( + use_delta_xds ? envoy::config::core::v3::ApiConfigSource::DELTA_GRPC + : envoy::config::core::v3::ApiConfigSource::GRPC); + return config_source; +} + +struct FakeSubscriptionState { + int start_calls_{0}; + std::vector> start_resources_; +}; + +class FakeSubscription : public Envoy::Config::Subscription { +public: + explicit FakeSubscription(std::shared_ptr state) + : state_(std::move(state)) {} + + void start(const absl::flat_hash_set& resource_names) override { + ++state_->start_calls_; + auto& started = state_->start_resources_.emplace_back(); + started.insert(started.end(), resource_names.begin(), resource_names.end()); + std::sort(started.begin(), started.end()); + } + + void updateResourceInterest(const absl::flat_hash_set&) override {} + void requestOnDemandUpdate(const absl::flat_hash_set&) override {} + +private: + std::shared_ptr state_; +}; + +} // namespace + class CiliumNetworkPolicyTest : public ::testing::Test { protected: CiliumNetworkPolicyTest() { @@ -91,8 +131,8 @@ class CiliumNetworkPolicyTest : public ::testing::Test { ON_CALL_SDS_SECRET_PROVIDER(secret_manager_, TlsSessionTicketKeysContext, TlsSessionTicketKeys); ON_CALL_SDS_SECRET_PROVIDER(secret_manager_, GenericSecret, GenericSecret); - policy_map_ = - std::make_shared(factory_context_, Cilium::CILIUM_XDS_API_CONFIG); + policy_map_ = std::make_shared(factory_context_, + configSourceForUseDeltaXds(useDeltaXds())); } void TearDown() override { @@ -100,6 +140,12 @@ class CiliumNetworkPolicyTest : public ::testing::Test { policy_map_.reset(); } + virtual bool useDeltaXds() const { return false; } + + Envoy::Config::SubscriptionCallbacks& subscriptionCallbacks() const { + return policy_map_->subscriptionCallbacksForTest(); + } + std::string updateFromYaml(const std::string& config) { envoy::service::discovery::v3::DiscoveryResponse message; MessageUtil::loadFromYaml(config, message, ProtobufMessage::getNullValidationVisitor()); @@ -109,12 +155,29 @@ class CiliumNetworkPolicyTest : public ::testing::Test { THROW_IF_NOT_OK_REF(decoded_resources_or_error.status()); const auto decoded_resources = std::move(decoded_resources_or_error.value().get()); - EXPECT_TRUE(policy_map_->getImpl() + EXPECT_TRUE(subscriptionCallbacks() .onConfigUpdate(decoded_resources->refvec_, message.version_info()) .ok()); return message.version_info(); } + std::string deltaUpdateFromYaml(const std::string& config) { + envoy::service::discovery::v3::DeltaDiscoveryResponse message; + MessageUtil::loadFromYaml(config, message, ProtobufMessage::getNullValidationVisitor()); + NetworkPolicyResourceDecoder network_policy_resource_decoder; + auto decoded_resources = std::make_unique(); + for (const auto& resource : message.resources()) { + decoded_resources->pushBack( + Config::DecodedResourceImpl::fromResource(network_policy_resource_decoder, resource)); + } + + EXPECT_TRUE(subscriptionCallbacks() + .onConfigUpdate(decoded_resources->refvec_, message.removed_resources(), + message.system_version_info()) + .ok()); + return message.system_version_info(); + } + testing::AssertionResult validate(const std::string& pod_ip, const std::string& expected) { const auto& policy = policy_map_->getPolicyInstance(pod_ip, false); auto str = policy.string(); @@ -231,8 +294,36 @@ class CiliumNetworkPolicyTest : public ::testing::Test { } std::string updatesRejectedStatName() { - return policy_map_->getImpl().stats_.updates_rejected_.name(); + return policy_map_->statsForTest().updates_rejected_.name(); + } + + PolicyInstanceConstSharedPtr policyInstanceShared(const std::string& pod_ip) const { + return policy_map_->getPolicyInstanceSharedForTest(pod_ip); + } + + uint64_t selectorStreamGenerationForTest(const PolicyInstance& policy) const { + return policy_map_->policySelectorStreamGenerationForTest(policy); + } + + SelectorVersion selectorVersionForTest(const PolicyInstance& policy) const { + return policy_map_->policySelectorVersionForTest(policy); + } + + void resetStreamForTest() { policy_map_->resetStreamForTest(); } + bool configuredUseDeltaXds() const { return policy_map_->useDeltaXds(); } + void setUseDeltaXds(bool use_delta_xds) const { + policy_map_->setConfigSource(configSourceForUseDeltaXds(use_delta_xds)); + } + void startManagedSubscriptionForTest() { policy_map_->startManagedSubscriptionForTest(); } + void setSubscriptionFactoryForTest(NetworkPolicyMap::SubscriptionFactoryForTest factory) { + policy_map_->setSubscriptionFactoryForTest(std::move(factory)); } + void onSubscriptionConnectedForTest() { policy_map_->onSubscriptionConnectedForTest(); } + void onSubscriptionTransportCloseForTest() { policy_map_->onSubscriptionTransportCloseForTest(); } + bool subscriptionUseDeltaXdsForTest() const { + return policy_map_->subscriptionUseDeltaXdsForTest(); + } + bool subscriptionConnectedForTest() const { return policy_map_->subscriptionConnectedForTest(); } NiceMock factory_context_; NiceMock secret_manager_; @@ -241,12 +332,226 @@ class CiliumNetworkPolicyTest : public ::testing::Test { uint16_t proxy_id_ = 42; }; +class CiliumNetworkPolicyDeltaTest : public CiliumNetworkPolicyTest { +protected: + bool useDeltaXds() const override { return true; } +}; + TEST_F(CiliumNetworkPolicyTest, UpdatesRejectedStatName) { EXPECT_EQ("cilium.policy.updates_rejected", updatesRejectedStatName()); } +TEST_F(CiliumNetworkPolicyTest, ManagedSubscriptionColdStartUsesConfiguredSotwMode) { + auto state = std::make_shared(); + std::vector created_modes; + setSubscriptionFactoryForTest( + [state, &created_modes](bool use_delta_xds) -> std::unique_ptr { + created_modes.push_back(use_delta_xds); + return std::make_unique(state); + }); + + startManagedSubscriptionForTest(); + + ASSERT_EQ(created_modes.size(), 1); + EXPECT_FALSE(created_modes.front()); + EXPECT_FALSE(configuredUseDeltaXds()); + EXPECT_FALSE(subscriptionUseDeltaXdsForTest()); + ASSERT_EQ(state->start_calls_, 1); + EXPECT_TRUE(state->start_resources_.front().empty()); +} + +TEST_F(CiliumNetworkPolicyDeltaTest, ManagedSubscriptionColdStartUsesConfiguredDeltaMode) { + auto state = std::make_shared(); + std::vector created_modes; + setSubscriptionFactoryForTest( + [state, &created_modes](bool use_delta_xds) -> std::unique_ptr { + created_modes.push_back(use_delta_xds); + return std::make_unique(state); + }); + + startManagedSubscriptionForTest(); + + ASSERT_EQ(created_modes.size(), 1); + EXPECT_TRUE(created_modes.front()); + EXPECT_TRUE(configuredUseDeltaXds()); + EXPECT_TRUE(subscriptionUseDeltaXdsForTest()); + ASSERT_EQ(state->start_calls_, 1); + EXPECT_TRUE(state->start_resources_.front().empty()); +} + +TEST_F(CiliumNetworkPolicyTest, FlagFlipFromSotwToDeltaOnHealthySubscriptionRecreatesImmediately) { + auto state = std::make_shared(); + std::vector created_modes; + setSubscriptionFactoryForTest( + [state, &created_modes](bool use_delta_xds) -> std::unique_ptr { + created_modes.push_back(use_delta_xds); + return std::make_unique(state); + }); + + startManagedSubscriptionForTest(); + onSubscriptionConnectedForTest(); + ASSERT_TRUE(subscriptionConnectedForTest()); + + setUseDeltaXds(true); + + EXPECT_TRUE(configuredUseDeltaXds()); + EXPECT_TRUE(subscriptionUseDeltaXdsForTest()); + EXPECT_FALSE(subscriptionConnectedForTest()); + EXPECT_THAT(created_modes, testing::ElementsAre(false, true)); + EXPECT_EQ(state->start_calls_, 2); + EXPECT_TRUE(state->start_resources_.back().empty()); +} + +TEST_F(CiliumNetworkPolicyTest, FlagFlipFromDeltaToSotwOnHealthySubscriptionWaitsForClose) { + auto state = std::make_shared(); + std::vector created_modes; + setSubscriptionFactoryForTest( + [state, &created_modes](bool use_delta_xds) -> std::unique_ptr { + created_modes.push_back(use_delta_xds); + return std::make_unique(state); + }); + + startManagedSubscriptionForTest(); + onSubscriptionConnectedForTest(); + ASSERT_TRUE(subscriptionConnectedForTest()); + ASSERT_FALSE(subscriptionUseDeltaXdsForTest()); + + setUseDeltaXds(true); + + EXPECT_TRUE(configuredUseDeltaXds()); + EXPECT_TRUE(subscriptionUseDeltaXdsForTest()); + EXPECT_FALSE(subscriptionConnectedForTest()); + EXPECT_THAT(created_modes, testing::ElementsAre(false, true)); + EXPECT_EQ(state->start_calls_, 2); + EXPECT_TRUE(state->start_resources_.back().empty()); + + onSubscriptionConnectedForTest(); + ASSERT_TRUE(subscriptionConnectedForTest()); + ASSERT_TRUE(subscriptionUseDeltaXdsForTest()); + + setUseDeltaXds(false); + + EXPECT_FALSE(configuredUseDeltaXds()); + EXPECT_TRUE(subscriptionUseDeltaXdsForTest()); + EXPECT_TRUE(subscriptionConnectedForTest()); + // Once we have an established delta subscription, keep it until transport close even if the + // configured desired mode flips back to SotW. + EXPECT_THAT(created_modes, testing::ElementsAre(false, true)); + EXPECT_EQ(state->start_calls_, 2); + + onSubscriptionTransportCloseForTest(); + + EXPECT_FALSE(subscriptionConnectedForTest()); + EXPECT_FALSE(subscriptionUseDeltaXdsForTest()); + EXPECT_THAT(created_modes, testing::ElementsAre(false, true, false)); + EXPECT_EQ(state->start_calls_, 3); + EXPECT_TRUE(state->start_resources_.back().empty()); +} + +TEST_F(CiliumNetworkPolicyTest, FlagFlipWhileDisconnectedRecreatesImmediately) { + auto state = std::make_shared(); + std::vector created_modes; + setSubscriptionFactoryForTest( + [state, &created_modes](bool use_delta_xds) -> std::unique_ptr { + created_modes.push_back(use_delta_xds); + return std::make_unique(state); + }); + + startManagedSubscriptionForTest(); + ASSERT_FALSE(subscriptionConnectedForTest()); + + setUseDeltaXds(true); + + EXPECT_TRUE(configuredUseDeltaXds()); + EXPECT_TRUE(subscriptionUseDeltaXdsForTest()); + EXPECT_FALSE(subscriptionConnectedForTest()); + EXPECT_THAT(created_modes, testing::ElementsAre(false, true)); + EXPECT_EQ(state->start_calls_, 2); + EXPECT_TRUE(state->start_resources_.back().empty()); +} + +TEST_F(CiliumNetworkPolicyDeltaTest, FlagFlipFromDisconnectedDeltaToSotwRecreatesImmediately) { + auto state = std::make_shared(); + std::vector created_modes; + setSubscriptionFactoryForTest( + [state, &created_modes](bool use_delta_xds) -> std::unique_ptr { + created_modes.push_back(use_delta_xds); + return std::make_unique(state); + }); + + startManagedSubscriptionForTest(); + ASSERT_FALSE(subscriptionConnectedForTest()); + ASSERT_TRUE(subscriptionUseDeltaXdsForTest()); + + setUseDeltaXds(false); + + EXPECT_FALSE(configuredUseDeltaXds()); + EXPECT_FALSE(subscriptionUseDeltaXdsForTest()); + EXPECT_FALSE(subscriptionConnectedForTest()); + EXPECT_THAT(created_modes, testing::ElementsAre(true, false)); + EXPECT_EQ(state->start_calls_, 2); + EXPECT_TRUE(state->start_resources_.back().empty()); +} + +TEST_F(CiliumNetworkPolicyDeltaTest, DowngradeFromConnectedDeltaRecreatesDisconnectedRetryToSotw) { + auto state = std::make_shared(); + std::vector created_modes; + setSubscriptionFactoryForTest( + [state, &created_modes](bool use_delta_xds) -> std::unique_ptr { + created_modes.push_back(use_delta_xds); + return std::make_unique(state); + }); + + startManagedSubscriptionForTest(); + onSubscriptionConnectedForTest(); + ASSERT_TRUE(subscriptionConnectedForTest()); + ASSERT_TRUE(subscriptionUseDeltaXdsForTest()); + + // Agent restart/downgrade drops the established delta transport. While the desired mode is still + // delta we recreate immediately and begin retrying delta. + onSubscriptionTransportCloseForTest(); + + ASSERT_FALSE(subscriptionConnectedForTest()); + ASSERT_TRUE(subscriptionUseDeltaXdsForTest()); + ASSERT_EQ(state->start_calls_, 2); + EXPECT_THAT(created_modes, testing::ElementsAre(true, true)); + EXPECT_TRUE(state->start_resources_.back().empty()); + + // When listener metadata later reveals the downgraded agent no longer supports delta, flip to + // SotW immediately rather than letting the disconnected delta retry loop forever. + setUseDeltaXds(false); + + EXPECT_FALSE(configuredUseDeltaXds()); + EXPECT_FALSE(subscriptionConnectedForTest()); + EXPECT_FALSE(subscriptionUseDeltaXdsForTest()); + EXPECT_THAT(created_modes, testing::ElementsAre(true, true, false)); + EXPECT_EQ(state->start_calls_, 3); + EXPECT_TRUE(state->start_resources_.back().empty()); +} + +TEST_F(CiliumNetworkPolicyTest, TransportCloseWithoutFlagFlipRecreatesInCurrentMode) { + auto state = std::make_shared(); + std::vector created_modes; + setSubscriptionFactoryForTest( + [state, &created_modes](bool use_delta_xds) -> std::unique_ptr { + created_modes.push_back(use_delta_xds); + return std::make_unique(state); + }); + + startManagedSubscriptionForTest(); + onSubscriptionConnectedForTest(); + + onSubscriptionTransportCloseForTest(); + + EXPECT_FALSE(subscriptionConnectedForTest()); + EXPECT_FALSE(subscriptionUseDeltaXdsForTest()); + EXPECT_THAT(created_modes, testing::ElementsAre(false, false)); + EXPECT_EQ(state->start_calls_, 2); + EXPECT_TRUE(state->start_resources_.back().empty()); +} + TEST_F(CiliumNetworkPolicyTest, EmptyPolicyUpdate) { - EXPECT_TRUE(policy_map_->getImpl().onConfigUpdate({}, "1").ok()); + EXPECT_TRUE(subscriptionCallbacks().onConfigUpdate({}, "1").ok()); EXPECT_FALSE(validate("10.1.2.3", "")); // Policy not found } @@ -258,6 +563,1488 @@ TEST_F(CiliumNetworkPolicyTest, SimplePolicyUpdate) { EXPECT_FALSE(validate("10.1.2.3", "")); // Policy not found } +TEST_F(CiliumNetworkPolicyTest, RejectsWhitespaceInSotwWrappedResourceName) { + EXPECT_THROW_WITH_MESSAGE(updateFromYaml(R"EOF(version_info: "1" +resources: +- "@type": type.googleapis.com/envoy.service.discovery.v3.Resource + name: "policy 42" + resource: + "@type": type.googleapis.com/cilium.NetworkPolicy + endpoint_ips: + - "10.1.2.3" + endpoint_id: 42 +)EOF"), + EnvoyException, + "NetworkPolicy resource name 'policy 42' must not contain whitespace"); +} + +TEST_F(CiliumNetworkPolicyDeltaTest, DeltaIncrementalPolicyUpdate) { + EXPECT_NO_THROW(deltaUpdateFromYaml(R"EOF(system_version_info: "1" +resources: +- name: "selector-1" + version: "1" + resource: + "@type": type.googleapis.com/cilium.NetworkPolicyResource + selector: + remote_identities: [ 43 ] +- name: "policy-42" + version: "1" + resource: + "@type": type.googleapis.com/cilium.NetworkPolicyResource + policy: + endpoint_ips: + - "10.1.2.3" + endpoint_id: 42 + ingress_per_port_policies: + - port: 80 + rules: + - selectors: [ "selector-1" ] +)EOF")); + + EXPECT_TRUE(ingressAllowed("10.1.2.3", 43, 80)); + EXPECT_FALSE(ingressAllowed("10.1.2.3", 43, 81)); + + EXPECT_NO_THROW(deltaUpdateFromYaml(R"EOF(system_version_info: "2" +resources: +- name: "selector-2" + version: "1" + resource: + "@type": type.googleapis.com/cilium.NetworkPolicyResource + selector: + remote_identities: [ 44 ] +- name: "policy-42" + version: "2" + resource: + "@type": type.googleapis.com/cilium.NetworkPolicyResource + policy: + endpoint_ips: + - "10.1.2.3" + endpoint_id: 42 + ingress_per_port_policies: + - port: 8080 + rules: + - selectors: [ "selector-2" ] +)EOF")); + + EXPECT_FALSE(ingressAllowed("10.1.2.3", 43, 80)); + EXPECT_TRUE(ingressAllowed("10.1.2.3", 44, 8080)); +} + +TEST_F(CiliumNetworkPolicyDeltaTest, DeltaSameStreamKeepsUntouchedResources) { + EXPECT_NO_THROW(deltaUpdateFromYaml(R"EOF(system_version_info: "1" +resources: +- name: "selector-1" + version: "1" + resource: + "@type": type.googleapis.com/cilium.NetworkPolicyResource + selector: + remote_identities: [ 43 ] +- name: "selector-2" + version: "1" + resource: + "@type": type.googleapis.com/cilium.NetworkPolicyResource + selector: + remote_identities: [ 44 ] +- name: "policy-42" + version: "1" + resource: + "@type": type.googleapis.com/cilium.NetworkPolicyResource + policy: + endpoint_ips: + - "10.1.2.3" + endpoint_id: 42 + ingress_per_port_policies: + - port: 80 + rules: + - selectors: [ "selector-1" ] +- name: "policy-43" + version: "1" + resource: + "@type": type.googleapis.com/cilium.NetworkPolicyResource + policy: + endpoint_ips: + - "10.2.3.4" + endpoint_id: 43 + ingress_per_port_policies: + - port: 81 + rules: + - selectors: [ "selector-2" ] +)EOF")); + + EXPECT_TRUE(ingressAllowed("10.1.2.3", 43, 80)); + EXPECT_TRUE(ingressAllowed("10.2.3.4", 44, 81)); + + EXPECT_NO_THROW(deltaUpdateFromYaml(R"EOF(system_version_info: "2" +resources: +- name: "selector-3" + version: "1" + resource: + "@type": type.googleapis.com/cilium.NetworkPolicyResource + selector: + remote_identities: [ 45 ] +- name: "policy-42" + version: "2" + resource: + "@type": type.googleapis.com/cilium.NetworkPolicyResource + policy: + endpoint_ips: + - "10.1.2.3" + endpoint_id: 42 + ingress_per_port_policies: + - port: 8080 + rules: + - selectors: [ "selector-3" ] +)EOF")); + + EXPECT_TRUE(ingressAllowed("10.1.2.3", 45, 8080)); + EXPECT_TRUE(ingressAllowed("10.2.3.4", 44, 81)); +} + +TEST_F(CiliumNetworkPolicyDeltaTest, DeltaRemovesPolicyByResourceName) { + EXPECT_NO_THROW(deltaUpdateFromYaml(R"EOF(system_version_info: "1" +resources: +- name: "selector-1" + version: "1" + resource: + "@type": type.googleapis.com/cilium.NetworkPolicyResource + selector: + remote_identities: [ 43 ] +- name: "selector-2" + version: "1" + resource: + "@type": type.googleapis.com/cilium.NetworkPolicyResource + selector: + remote_identities: [ 44 ] +- name: "policy-42" + version: "1" + resource: + "@type": type.googleapis.com/cilium.NetworkPolicyResource + policy: + endpoint_ips: + - "10.1.2.3" + - "f00d::1" + endpoint_id: 42 + ingress_per_port_policies: + - port: 80 + rules: + - selectors: [ "selector-1" ] +- name: "policy-43" + version: "1" + resource: + "@type": type.googleapis.com/cilium.NetworkPolicyResource + policy: + endpoint_ips: + - "10.2.3.4" + endpoint_id: 43 + ingress_per_port_policies: + - port: 81 + rules: + - selectors: [ "selector-2" ] +)EOF")); + + EXPECT_TRUE(policy_map_->exists("10.1.2.3")); + EXPECT_TRUE(policy_map_->exists("f00d::1")); + EXPECT_TRUE(ingressAllowed("10.1.2.3", 43, 80)); + EXPECT_TRUE(ingressAllowed("f00d::1", 43, 80)); + + EXPECT_TRUE(policy_map_->exists("10.2.3.4")); + EXPECT_TRUE(ingressAllowed("10.2.3.4", 44, 81)); + + EXPECT_NO_THROW(deltaUpdateFromYaml(R"EOF(system_version_info: "2" +removed_resources: +- "policy-42" +)EOF")); + + EXPECT_FALSE(policy_map_->exists("10.1.2.3")); + EXPECT_FALSE(policy_map_->exists("f00d::1")); + EXPECT_FALSE(validate("10.1.2.3", "")); + EXPECT_FALSE(validate("f00d::1", "")); + + EXPECT_TRUE(policy_map_->exists("10.2.3.4")); + EXPECT_TRUE(ingressAllowed("10.2.3.4", 44, 81)); +} + +TEST_F(CiliumNetworkPolicyDeltaTest, DeltaSelectorOnlyUpdateTakesEffectImmediately) { + EXPECT_NO_THROW(deltaUpdateFromYaml(R"EOF(system_version_info: "1" +resources: +- name: "selector-1" + version: "1" + resource: + "@type": type.googleapis.com/cilium.NetworkPolicyResource + selector: + remote_identities: [ 43 ] +- name: "policy-42" + version: "1" + resource: + "@type": type.googleapis.com/cilium.NetworkPolicyResource + policy: + endpoint_ips: + - "10.1.2.3" + endpoint_id: 42 + ingress_per_port_policies: + - port: 80 + rules: + - selectors: [ "selector-1" ] +)EOF")); + + EXPECT_TRUE(ingressAllowed("10.1.2.3", 43, 80)); + EXPECT_FALSE(ingressAllowed("10.1.2.3", 44, 80)); + + EXPECT_NO_THROW(deltaUpdateFromYaml(R"EOF(system_version_info: "2" +resources: +- name: "selector-1" + version: "1" + resource: + "@type": type.googleapis.com/cilium.NetworkPolicyResource + selector: + remote_identities: [ 44, 45 ] +)EOF")); + + EXPECT_FALSE(ingressAllowed("10.1.2.3", 43, 80)); + EXPECT_TRUE(ingressAllowed("10.1.2.3", 44, 80)); + EXPECT_TRUE(ingressAllowed("10.1.2.3", 45, 80)); +} + +TEST_F(CiliumNetworkPolicyDeltaTest, DeltaPolicyUpdateRejectsMissingSelectorResource) { + EXPECT_NO_THROW(deltaUpdateFromYaml(R"EOF(system_version_info: "1" +resources: +- name: "selector-1" + version: "1" + resource: + "@type": type.googleapis.com/cilium.NetworkPolicyResource + selector: + remote_identities: [ 43 ] +- name: "policy-42" + version: "1" + resource: + "@type": type.googleapis.com/cilium.NetworkPolicyResource + policy: + endpoint_ips: + - "10.1.2.3" + endpoint_id: 42 + ingress_per_port_policies: + - port: 80 + rules: + - selectors: [ "selector-1" ] +)EOF")); + + EXPECT_TRUE(ingressAllowed("10.1.2.3", 43, 80)); + EXPECT_FALSE(ingressAllowed("10.1.2.3", 44, 80)); + + EXPECT_THROW_WITH_MESSAGE(deltaUpdateFromYaml(R"EOF(system_version_info: "2" +resources: +- name: "policy-42" + version: "2" + resource: + "@type": type.googleapis.com/cilium.NetworkPolicyResource + policy: + endpoint_ips: + - "10.1.2.3" + endpoint_id: 42 + ingress_per_port_policies: + - port: 80 + rules: + - selectors: [ "selector-2" ] +)EOF"), + EnvoyException, + "NetworkPolicyResource rule references missing selector resource " + "'selector-2'"); + + EXPECT_TRUE(ingressAllowed("10.1.2.3", 43, 80)); + EXPECT_FALSE(ingressAllowed("10.1.2.3", 44, 80)); +} + +TEST_F(CiliumNetworkPolicyDeltaTest, DeltaRemovedAndReaddedSelectorNameDoesNotRebindOldPolicy) { + EXPECT_NO_THROW(deltaUpdateFromYaml(R"EOF(system_version_info: "1" +resources: +- name: "selector-1" + version: "1" + resource: + "@type": type.googleapis.com/cilium.NetworkPolicyResource + selector: + remote_identities: [ 43 ] +- name: "policy-42" + version: "1" + resource: + "@type": type.googleapis.com/cilium.NetworkPolicyResource + policy: + endpoint_ips: + - "10.1.2.3" + endpoint_id: 42 + ingress_per_port_policies: + - port: 80 + rules: + - selectors: [ "selector-1" ] +)EOF")); + + EXPECT_TRUE(ingressAllowed("10.1.2.3", 43, 80)); + EXPECT_FALSE(ingressAllowed("10.1.2.3", 44, 80)); + + EXPECT_NO_THROW(deltaUpdateFromYaml(R"EOF(system_version_info: "2" +removed_resources: +- "selector-1" +)EOF")); + + EXPECT_FALSE(ingressAllowed("10.1.2.3", 43, 80)); + EXPECT_FALSE(ingressAllowed("10.1.2.3", 44, 80)); + + EXPECT_NO_THROW(deltaUpdateFromYaml(R"EOF(system_version_info: "3" +resources: +- name: "selector-1" + version: "2" + resource: + "@type": type.googleapis.com/cilium.NetworkPolicyResource + selector: + remote_identities: [ 44 ] +)EOF")); + + EXPECT_FALSE(ingressAllowed("10.1.2.3", 43, 80)); + EXPECT_FALSE(ingressAllowed("10.1.2.3", 44, 80)); + + EXPECT_NO_THROW(deltaUpdateFromYaml(R"EOF(system_version_info: "4" +resources: +- name: "policy-42" + version: "2" + resource: + "@type": type.googleapis.com/cilium.NetworkPolicyResource + policy: + endpoint_ips: + - "10.1.2.3" + endpoint_id: 42 + ingress_per_port_policies: + - port: 80 + rules: + - selectors: [ "selector-1" ] +)EOF")); + + EXPECT_FALSE(ingressAllowed("10.1.2.3", 43, 80)); + EXPECT_TRUE(ingressAllowed("10.1.2.3", 44, 80)); +} + +TEST_F(CiliumNetworkPolicyDeltaTest, DeltaRemovedAndReaddedSelectorNameInSameUpdateActsAsUpdate) { + EXPECT_NO_THROW(deltaUpdateFromYaml(R"EOF(system_version_info: "1" +resources: +- name: "selector-1" + version: "1" + resource: + "@type": type.googleapis.com/cilium.NetworkPolicyResource + selector: + remote_identities: [ 43 ] +- name: "policy-42" + version: "1" + resource: + "@type": type.googleapis.com/cilium.NetworkPolicyResource + policy: + endpoint_ips: + - "10.1.2.3" + endpoint_id: 42 + ingress_per_port_policies: + - port: 80 + rules: + - selectors: [ "selector-1" ] +)EOF")); + + EXPECT_TRUE(ingressAllowed("10.1.2.3", 43, 80)); + EXPECT_FALSE(ingressAllowed("10.1.2.3", 44, 80)); + + EXPECT_NO_THROW(deltaUpdateFromYaml(R"EOF(system_version_info: "2" +removed_resources: +- "selector-1" +resources: +- name: "selector-1" + version: "2" + resource: + "@type": type.googleapis.com/cilium.NetworkPolicyResource + selector: + remote_identities: [ 43 ] +)EOF")); + + EXPECT_TRUE(ingressAllowed("10.1.2.3", 43, 80)); + EXPECT_FALSE(ingressAllowed("10.1.2.3", 44, 80)); +} + +TEST_F(CiliumNetworkPolicyDeltaTest, DeltaRejectedSelectorUpdateKeepsPublishedBehavior) { + EXPECT_NO_THROW(deltaUpdateFromYaml(R"EOF(system_version_info: "1" +resources: +- name: "selector-1" + version: "1" + resource: + "@type": type.googleapis.com/cilium.NetworkPolicyResource + selector: + remote_identities: [ 43 ] +- name: "policy-42" + version: "1" + resource: + "@type": type.googleapis.com/cilium.NetworkPolicyResource + policy: + endpoint_ips: + - "10.1.2.3" + endpoint_id: 42 + ingress_per_port_policies: + - port: 80 + rules: + - selectors: [ "selector-1" ] +)EOF")); + + EXPECT_TRUE(ingressAllowed("10.1.2.3", 43, 80)); + EXPECT_FALSE(ingressAllowed("10.1.2.3", 44, 80)); + + EXPECT_THROW_WITH_REGEX( + deltaUpdateFromYaml(R"EOF(system_version_info: "2" +resources: +- name: "10.1.2.3" + version: "1" + resource: + "@type": type.googleapis.com/cilium.NetworkPolicyResource + selector: + remote_identities: [ 44 ] +)EOF"), + EnvoyException, + R"(NetworkPolicyResource .*update for version [0-9]+ has duplicate resource key '10\.1\.2\.3'.*incoming selector resource '10\.1\.2\.3'.*existing endpoint IP alias '10\.1\.2\.3' owned by policy resource 'policy-42'.*endpoint_id 42.*10\.1\.2\.3)"); + + EXPECT_TRUE(ingressAllowed("10.1.2.3", 43, 80)); + EXPECT_FALSE(ingressAllowed("10.1.2.3", 44, 80)); +} + +TEST_F(CiliumNetworkPolicyDeltaTest, DeltaPassUsesCurrentSelectorMembership) { + EXPECT_NO_THROW(deltaUpdateFromYaml(R"EOF(system_version_info: "1" +resources: +- name: "selector-1" + version: "1" + resource: + "@type": type.googleapis.com/cilium.NetworkPolicyResource + selector: + remote_identities: [ 43 ] +- name: "selector-2" + version: "1" + resource: + "@type": type.googleapis.com/cilium.NetworkPolicyResource + selector: + remote_identities: [ 43, 44 ] +- name: "policy-42" + version: "1" + resource: + "@type": type.googleapis.com/cilium.NetworkPolicyResource + policy: + endpoint_ips: + - "10.1.2.3" + endpoint_id: 42 + ingress_per_port_policies: + - port: 80 + rules: + - precedence: 1000 + pass_precedence: 501 + selectors: [ "selector-1" ] + - precedence: 900 + deny: true + - precedence: 500 + selectors: [ "selector-2" ] + http_rules: + http_rules: + - headers: + - name: ':path' + exact_match: '/allowed' +)EOF")); + + EXPECT_TRUE(ingressAllowed("10.1.2.3", 43, 80, {{":path", "/allowed"}})); + EXPECT_FALSE(ingressAllowed("10.1.2.3", 44, 80, {{":path", "/allowed"}})); +} + +TEST_F(CiliumNetworkPolicyDeltaTest, DeltaWildcardPortPassIsMergedToExactPortRules) { + EXPECT_NO_THROW(deltaUpdateFromYaml(R"EOF(system_version_info: "1" +resources: +- name: "selector-1" + version: "1" + resource: + "@type": type.googleapis.com/cilium.NetworkPolicyResource + selector: + remote_identities: [ 43 ] +- name: "selector-2" + version: "1" + resource: + "@type": type.googleapis.com/cilium.NetworkPolicyResource + selector: + remote_identities: [ 43, 44 ] +- name: "policy-42" + version: "1" + resource: + "@type": type.googleapis.com/cilium.NetworkPolicyResource + policy: + endpoint_ips: + - "10.1.2.3" + endpoint_id: 42 + ingress_per_port_policies: + - port: 0 + rules: + - precedence: 1000 + pass_precedence: 501 + selectors: [ "selector-1" ] + - port: 80 + rules: + - precedence: 900 + deny: true + - precedence: 500 + selectors: [ "selector-2" ] + http_rules: + http_rules: + - headers: + - name: ':path' + exact_match: '/allowed' +)EOF")); + + EXPECT_TRUE(ingressAllowed("10.1.2.3", 43, 80, {{":path", "/allowed"}})); + EXPECT_FALSE(ingressAllowed("10.1.2.3", 44, 80, {{":path", "/allowed"}})); +} + +TEST_F(CiliumNetworkPolicyDeltaTest, DeltaSamePrecedenceDenyWinsOverPass) { + EXPECT_NO_THROW(deltaUpdateFromYaml(R"EOF(system_version_info: "1" +resources: +- name: "selector-1" + version: "1" + resource: + "@type": type.googleapis.com/cilium.NetworkPolicyResource + selector: + remote_identities: [ 43 ] +- name: "policy-42" + version: "1" + resource: + "@type": type.googleapis.com/cilium.NetworkPolicyResource + policy: + endpoint_ips: + - "10.1.2.3" + endpoint_id: 42 + ingress_per_port_policies: + - port: 80 + rules: + - precedence: 1000 + pass_precedence: 501 + selectors: [ "selector-1" ] + - precedence: 1000 + deny: true + selectors: [ "selector-1" ] + - precedence: 500 + selectors: [ "selector-1" ] + http_rules: + http_rules: + - headers: + - name: ':path' + exact_match: '/allowed' +)EOF")); + + EXPECT_FALSE(ingressAllowed("10.1.2.3", 43, 80, {{":path", "/allowed"}})); +} + +TEST_F(CiliumNetworkPolicyDeltaTest, DeltaSelectorOnlyUpdateChangesPassBehaviorImmediately) { + EXPECT_NO_THROW(deltaUpdateFromYaml(R"EOF(system_version_info: "1" +resources: +- name: "selector-1" + version: "1" + resource: + "@type": type.googleapis.com/cilium.NetworkPolicyResource + selector: + remote_identities: [ 43 ] +- name: "selector-2" + version: "1" + resource: + "@type": type.googleapis.com/cilium.NetworkPolicyResource + selector: + remote_identities: [ 43, 44 ] +- name: "policy-42" + version: "1" + resource: + "@type": type.googleapis.com/cilium.NetworkPolicyResource + policy: + endpoint_ips: + - "10.1.2.3" + endpoint_id: 42 + ingress_per_port_policies: + - port: 80 + rules: + - precedence: 1000 + pass_precedence: 501 + selectors: [ "selector-1" ] + - precedence: 900 + deny: true + - precedence: 500 + selectors: [ "selector-2" ] + http_rules: + http_rules: + - headers: + - name: ':path' + exact_match: '/allowed' +)EOF")); + + EXPECT_TRUE(ingressAllowed("10.1.2.3", 43, 80, {{":path", "/allowed"}})); + EXPECT_FALSE(ingressAllowed("10.1.2.3", 44, 80, {{":path", "/allowed"}})); + + EXPECT_NO_THROW(deltaUpdateFromYaml(R"EOF(system_version_info: "2" +resources: +- name: "selector-1" + version: "2" + resource: + "@type": type.googleapis.com/cilium.NetworkPolicyResource + selector: + remote_identities: [ 44 ] +)EOF")); + + EXPECT_FALSE(ingressAllowed("10.1.2.3", 43, 80, {{":path", "/allowed"}})); + EXPECT_TRUE(ingressAllowed("10.1.2.3", 44, 80, {{":path", "/allowed"}})); +} + +TEST_F(CiliumNetworkPolicyTest, SotwRejectsSelectorsInRules) { + EXPECT_THROW(updateFromYaml(R"EOF(version_info: "1" +resources: +- "@type": type.googleapis.com/cilium.NetworkPolicy + endpoint_ips: + - "10.1.2.3" + endpoint_id: 42 + ingress_per_port_policies: + - port: 80 + rules: + - selectors: [ "selector-1" ] +)EOF"), + EnvoyException); +} + +TEST_F(CiliumNetworkPolicyDeltaTest, DeltaAcceptsArbitrarySelectorResourceName) { + EXPECT_NO_THROW(deltaUpdateFromYaml(R"EOF(system_version_info: "1" +resources: +- name: "7" + version: "1" + resource: + "@type": type.googleapis.com/cilium.NetworkPolicyResource + selector: + remote_identities: [ 44, 45 ] +- name: "policy-42" + version: "1" + resource: + "@type": type.googleapis.com/cilium.NetworkPolicyResource + policy: + endpoint_ips: + - "10.1.2.3" + endpoint_id: 42 + ingress_per_port_policies: + - port: 80 + rules: + - selectors: [ "7" ] +)EOF")); + + EXPECT_TRUE(ingressAllowed("10.1.2.3", 44, 80)); +} + +TEST_F(CiliumNetworkPolicyDeltaTest, DeltaRejectsEmbeddedRemotePoliciesInRules) { + EXPECT_THROW(deltaUpdateFromYaml(R"EOF(system_version_info: "1" +resources: +- name: "policy-42" + version: "1" + resource: + "@type": type.googleapis.com/cilium.NetworkPolicyResource + policy: + endpoint_ips: + - "10.1.2.3" + endpoint_id: 42 + ingress_per_port_policies: + - port: 80 + rules: + - remote_policies: [ 43 ] +)EOF"), + EnvoyException); +} + +TEST_F(CiliumNetworkPolicyDeltaTest, DeltaAcceptsArbitraryPolicyResourceName) { + EXPECT_NO_THROW(deltaUpdateFromYaml(R"EOF(system_version_info: "1" +resources: +- name: "42" + version: "1" + resource: + "@type": type.googleapis.com/cilium.NetworkPolicyResource + policy: + endpoint_ips: + - "10.1.2.3" + endpoint_id: 42 +)EOF")); + + EXPECT_TRUE(policy_map_->exists("10.1.2.3")); +} + +TEST_F(CiliumNetworkPolicyDeltaTest, DeltaRejectsAddedResourceNamesWithWhitespace) { + EXPECT_THROW_WITH_MESSAGE( + deltaUpdateFromYaml(R"EOF(system_version_info: "1" +resources: +- name: "selector 1" + version: "1" + resource: + "@type": type.googleapis.com/cilium.NetworkPolicyResource + selector: + remote_identities: [ 43 ] +)EOF"), + EnvoyException, + "NetworkPolicyResource added resource name 'selector 1' must not contain whitespace"); +} + +TEST_F(CiliumNetworkPolicyDeltaTest, DeltaRejectsRemovedResourceNamesWithWhitespace) { + EXPECT_THROW_WITH_MESSAGE( + deltaUpdateFromYaml(R"EOF(system_version_info: "1" +removed_resources: +- "selector 1" +)EOF"), + EnvoyException, + "NetworkPolicyResource removed resource name 'selector 1' must not contain whitespace"); +} + +TEST_F(CiliumNetworkPolicyDeltaTest, DeltaRejectsDuplicatePolicyResourceNamesInSameUpdate) { + EXPECT_THROW_WITH_REGEX( + deltaUpdateFromYaml(R"EOF(system_version_info: "1" +resources: +- name: "shared-name" + version: "1" + resource: + "@type": type.googleapis.com/cilium.NetworkPolicyResource + policy: + endpoint_ips: + - "10.1.2.3" + endpoint_id: 42 +- name: "shared-name" + version: "1" + resource: + "@type": type.googleapis.com/cilium.NetworkPolicyResource + policy: + endpoint_ips: + - "10.1.2.4" + endpoint_id: 43 +)EOF"), + EnvoyException, + R"(NetworkPolicyResource update for version [0-9]+ has duplicate resource key 'shared-name'.*incoming policy resource 'shared-name'.*endpoint_id 43.*10\.1\.2\.4.*existing policy resource 'shared-name'.*endpoint_id 42.*10\.1\.2\.3)"); +} + +TEST_F(CiliumNetworkPolicyDeltaTest, DeltaAcceptsPolicyResourceNamesThatDoNotMatchEndpointId) { + EXPECT_NO_THROW(deltaUpdateFromYaml(R"EOF(system_version_info: "1" +resources: +- name: "policy-43" + version: "1" + resource: + "@type": type.googleapis.com/cilium.NetworkPolicyResource + policy: + endpoint_ips: + - "10.1.2.3" + endpoint_id: 42 +)EOF")); + + EXPECT_TRUE(policy_map_->exists("10.1.2.3")); +} + +TEST_F(CiliumNetworkPolicyDeltaTest, DeltaRejectsEndpointIpCollisionsInSameUpdate) { + EXPECT_THROW_WITH_REGEX( + deltaUpdateFromYaml(R"EOF(system_version_info: "1" +resources: +- name: "policy-a" + version: "1" + resource: + "@type": type.googleapis.com/cilium.NetworkPolicyResource + policy: + endpoint_ips: + - "10.1.2.3" + endpoint_id: 42 +- name: "policy-b" + version: "1" + resource: + "@type": type.googleapis.com/cilium.NetworkPolicyResource + policy: + endpoint_ips: + - "10.1.2.3" + endpoint_id: 43 +)EOF"), + EnvoyException, + R"(NetworkPolicyResource update for version [0-9]+ has duplicate resource key '10\.1\.2\.3'.*incoming policy resource 'policy-b'.*endpoint_id 43.*10\.1\.2\.3.*existing endpoint IP alias '10\.1\.2\.3' owned by policy resource 'policy-a'.*endpoint_id 42.*10\.1\.2\.3)"); +} + +TEST_F(CiliumNetworkPolicyDeltaTest, DeltaRejectsResourceNameEndpointIpCollisionsInSameUpdate) { + EXPECT_THROW_WITH_REGEX( + deltaUpdateFromYaml(R"EOF(system_version_info: "1" +resources: +- name: "10.1.2.4" + version: "1" + resource: + "@type": type.googleapis.com/cilium.NetworkPolicyResource + policy: + endpoint_ips: + - "10.1.2.3" + endpoint_id: 42 +- name: "policy-b" + version: "1" + resource: + "@type": type.googleapis.com/cilium.NetworkPolicyResource + policy: + endpoint_ips: + - "10.1.2.4" + endpoint_id: 43 +)EOF"), + EnvoyException, + R"(NetworkPolicyResource update for version [0-9]+ has duplicate resource key '10\.1\.2\.4'.*incoming policy resource 'policy-b'.*endpoint_id 43.*10\.1\.2\.4.*existing policy resource '10\.1\.2\.4'.*endpoint_id 42.*10\.1\.2\.3)"); +} + +TEST_F(CiliumNetworkPolicyDeltaTest, DeltaRejectsDuplicateSelectorResourceNamesInSameUpdate) { + EXPECT_THROW_WITH_REGEX( + deltaUpdateFromYaml(R"EOF(system_version_info: "1" +resources: +- name: "shared-selector" + version: "1" + resource: + "@type": type.googleapis.com/cilium.NetworkPolicyResource + selector: + remote_identities: [ 44, 45 ] +- name: "shared-selector" + version: "1" + resource: + "@type": type.googleapis.com/cilium.NetworkPolicyResource + selector: + remote_identities: [ 46, 47 ] +)EOF"), + EnvoyException, + R"(NetworkPolicyResource .*update for version [0-9]+ has duplicate resource key 'shared-selector'.*incoming selector resource 'shared-selector'.*existing selector resource 'shared-selector')"); +} + +TEST_F(CiliumNetworkPolicyDeltaTest, DeltaAcceptsArbitraryPolicyResourceNamesWithHyphens) { + EXPECT_NO_THROW(deltaUpdateFromYaml(R"EOF(system_version_info: "1" +resources: +- name: "selector-1" + version: "1" + resource: + "@type": type.googleapis.com/cilium.NetworkPolicyResource + selector: + remote_identities: [ 43 ] +- name: "policy-qualified-42" + version: "1" + resource: + "@type": type.googleapis.com/cilium.NetworkPolicyResource + policy: + endpoint_ips: + - "10.1.2.3" + endpoint_id: 42 + ingress_per_port_policies: + - port: 80 + rules: + - selectors: [ "selector-1" ] +)EOF")); + + EXPECT_TRUE(ingressAllowed("10.1.2.3", 43, 80)); +} + +TEST_F(CiliumNetworkPolicyDeltaTest, DeltaRemovesPoliciesByArbitraryResourceName) { + EXPECT_NO_THROW(deltaUpdateFromYaml(R"EOF(system_version_info: "1" +resources: +- name: "42" + version: "1" + resource: + "@type": type.googleapis.com/cilium.NetworkPolicyResource + policy: + endpoint_ips: + - "10.1.2.3" + endpoint_id: 42 +- name: "policy-qualified-43" + version: "1" + resource: + "@type": type.googleapis.com/cilium.NetworkPolicyResource + policy: + endpoint_ips: + - "10.1.2.4" + endpoint_id: 43 +)EOF")); + + EXPECT_TRUE(policy_map_->exists("10.1.2.3")); + EXPECT_TRUE(policy_map_->exists("10.1.2.4")); + + EXPECT_NO_THROW(deltaUpdateFromYaml(R"EOF(system_version_info: "2" +removed_resources: +- "42" +)EOF")); + + EXPECT_FALSE(policy_map_->exists("10.1.2.3")); + EXPECT_TRUE(policy_map_->exists("10.1.2.4")); +} + +TEST_F(CiliumNetworkPolicyDeltaTest, DeltaRejectsEndpointIpCollisionsWithExistingPolicies) { + EXPECT_NO_THROW(deltaUpdateFromYaml(R"EOF(system_version_info: "1" +resources: +- name: "policy-a" + version: "1" + resource: + "@type": type.googleapis.com/cilium.NetworkPolicyResource + policy: + endpoint_ips: + - "10.1.2.3" + endpoint_id: 42 +)EOF")); + + EXPECT_THROW_WITH_REGEX( + deltaUpdateFromYaml(R"EOF(system_version_info: "2" +resources: +- name: "policy-b" + version: "1" + resource: + "@type": type.googleapis.com/cilium.NetworkPolicyResource + policy: + endpoint_ips: + - "10.1.2.3" + endpoint_id: 43 +)EOF"), + EnvoyException, + R"(NetworkPolicyResource update for version [0-9]+ has duplicate resource key '10\.1\.2\.3'.*incoming policy resource 'policy-b'.*endpoint_id 43.*10\.1\.2\.3.*existing endpoint IP alias '10\.1\.2\.3' owned by policy resource 'policy-a'.*endpoint_id 42.*10\.1\.2\.3)"); +} + +TEST_F(CiliumNetworkPolicyDeltaTest, + DeltaRejectsPolicyResourceNameCollidingWithExistingEndpointIp) { + EXPECT_NO_THROW(deltaUpdateFromYaml(R"EOF(system_version_info: "1" +resources: +- name: "policy-a" + version: "1" + resource: + "@type": type.googleapis.com/cilium.NetworkPolicyResource + policy: + endpoint_ips: + - "10.1.2.3" + endpoint_id: 42 +)EOF")); + + EXPECT_THROW_WITH_REGEX( + deltaUpdateFromYaml(R"EOF(system_version_info: "2" +resources: +- name: "10.1.2.3" + version: "1" + resource: + "@type": type.googleapis.com/cilium.NetworkPolicyResource + policy: + endpoint_ips: + - "10.1.2.4" + endpoint_id: 43 +)EOF"), + EnvoyException, + R"(NetworkPolicyResource update for version [0-9]+ has duplicate resource key '10\.1\.2\.3'.*incoming policy resource '10\.1\.2\.3'.*endpoint_id 43.*10\.1\.2\.4.*existing endpoint IP alias '10\.1\.2\.3' owned by policy resource 'policy-a'.*endpoint_id 42.*10\.1\.2\.3)"); +} + +TEST_F(CiliumNetworkPolicyDeltaTest, + DeltaRejectsEndpointIpCollidingWithExistingPolicyResourceName) { + EXPECT_NO_THROW(deltaUpdateFromYaml(R"EOF(system_version_info: "1" +resources: +- name: "10.1.2.4" + version: "1" + resource: + "@type": type.googleapis.com/cilium.NetworkPolicyResource + policy: + endpoint_ips: + - "10.1.2.3" + endpoint_id: 42 +)EOF")); + + EXPECT_THROW_WITH_REGEX( + deltaUpdateFromYaml(R"EOF(system_version_info: "2" +resources: +- name: "policy-b" + version: "1" + resource: + "@type": type.googleapis.com/cilium.NetworkPolicyResource + policy: + endpoint_ips: + - "10.1.2.4" + endpoint_id: 43 +)EOF"), + EnvoyException, + R"(NetworkPolicyResource update for version [0-9]+ has duplicate resource key '10\.1\.2\.4'.*incoming policy resource 'policy-b'.*endpoint_id 43.*10\.1\.2\.4.*existing policy resource '10\.1\.2\.4'.*endpoint_id 42.*10\.1\.2\.3)"); +} + +TEST_F(CiliumNetworkPolicyDeltaTest, + DeltaRejectsSelectorResourceNameCollidingWithExistingPolicyResourceName) { + EXPECT_NO_THROW(deltaUpdateFromYaml(R"EOF(system_version_info: "1" +resources: +- name: "shared-name" + version: "1" + resource: + "@type": type.googleapis.com/cilium.NetworkPolicyResource + policy: + endpoint_ips: + - "10.1.2.3" + endpoint_id: 42 +)EOF")); + + EXPECT_THROW_WITH_REGEX( + deltaUpdateFromYaml(R"EOF(system_version_info: "2" +resources: +- name: "shared-name" + version: "1" + resource: + "@type": type.googleapis.com/cilium.NetworkPolicyResource + selector: + remote_identities: [ 43 ] +)EOF"), + EnvoyException, + R"(NetworkPolicyResource .*update for version [0-9]+ has duplicate resource key 'shared-name'.*incoming selector resource 'shared-name'.*existing policy resource 'shared-name'.*endpoint_id 42.*10\.1\.2\.3)"); +} + +TEST_F(CiliumNetworkPolicyDeltaTest, + DeltaRejectsSelectorResourceNameCollidingWithExistingEndpointIp) { + EXPECT_NO_THROW(deltaUpdateFromYaml(R"EOF(system_version_info: "1" +resources: +- name: "policy-a" + version: "1" + resource: + "@type": type.googleapis.com/cilium.NetworkPolicyResource + policy: + endpoint_ips: + - "10.1.2.3" + endpoint_id: 42 +)EOF")); + + EXPECT_THROW_WITH_REGEX( + deltaUpdateFromYaml(R"EOF(system_version_info: "2" +resources: +- name: "10.1.2.3" + version: "1" + resource: + "@type": type.googleapis.com/cilium.NetworkPolicyResource + selector: + remote_identities: [ 43 ] +)EOF"), + EnvoyException, + R"(NetworkPolicyResource .*update for version [0-9]+ has duplicate resource key '10\.1\.2\.3'.*incoming selector resource '10\.1\.2\.3'.*existing endpoint IP alias '10\.1\.2\.3' owned by policy resource 'policy-a'.*endpoint_id 42.*10\.1\.2\.3)"); +} + +TEST_F(CiliumNetworkPolicyDeltaTest, DeltaRejectsRemovingPolicyEndpointIpAlias) { + EXPECT_NO_THROW(deltaUpdateFromYaml(R"EOF(system_version_info: "1" +resources: +- name: "policy-a" + version: "1" + resource: + "@type": type.googleapis.com/cilium.NetworkPolicyResource + policy: + endpoint_ips: + - "10.1.2.3" + endpoint_id: 42 +)EOF")); + + EXPECT_THROW_WITH_MESSAGE( + deltaUpdateFromYaml(R"EOF(system_version_info: "2" +removed_resources: +- "10.1.2.3" +)EOF"), + EnvoyException, + "NetworkPolicyResource removed resource '10.1.2.3' is a policy endpoint IP alias, not a " + "resource name"); +} + +TEST_F(CiliumNetworkPolicyDeltaTest, DeltaAllowsEndpointIpReusingRemovedPolicyResourceName) { + EXPECT_NO_THROW(deltaUpdateFromYaml(R"EOF(system_version_info: "1" +resources: +- name: "10.1.2.4" + version: "1" + resource: + "@type": type.googleapis.com/cilium.NetworkPolicyResource + policy: + endpoint_ips: + - "10.1.2.3" + endpoint_id: 42 +)EOF")); + + EXPECT_NO_THROW(deltaUpdateFromYaml(R"EOF(system_version_info: "2" +removed_resources: +- "10.1.2.4" +resources: +- name: "policy-b" + version: "1" + resource: + "@type": type.googleapis.com/cilium.NetworkPolicyResource + policy: + endpoint_ips: + - "10.1.2.4" + endpoint_id: 43 +)EOF")); + + EXPECT_FALSE(policy_map_->exists("10.1.2.3")); + EXPECT_TRUE(policy_map_->exists("10.1.2.4")); +} + +TEST_F(CiliumNetworkPolicyDeltaTest, DeltaAcceptsPolicyResourceNamesWithNumericSuffixes) { + EXPECT_NO_THROW(deltaUpdateFromYaml(R"EOF(system_version_info: "1" +resources: +- name: "policy-042" + version: "1" + resource: + "@type": type.googleapis.com/cilium.NetworkPolicyResource + policy: + endpoint_ips: + - "10.1.2.3" + endpoint_id: 42 +- name: "policy-0" + version: "1" + resource: + "@type": type.googleapis.com/cilium.NetworkPolicyResource + policy: + endpoint_ips: + - "10.1.2.4" + endpoint_id: 43 +)EOF")); + + EXPECT_TRUE(policy_map_->exists("10.1.2.3")); + EXPECT_TRUE(policy_map_->exists("10.1.2.4")); +} + +TEST_F(CiliumNetworkPolicyDeltaTest, DeltaRejectsZeroEndpointIdRegardlessOfResourceName) { + EXPECT_THROW_WITH_MESSAGE(deltaUpdateFromYaml(R"EOF(system_version_info: "1" +resources: +- name: "policy-0" + version: "1" + resource: + "@type": type.googleapis.com/cilium.NetworkPolicyResource + policy: + endpoint_ips: + - "10.1.2.3" + endpoint_id: 0 +)EOF"), + EnvoyException, "NetworkPolicyResource endpoint_id must be non-zero"); +} + +TEST_F(CiliumNetworkPolicyDeltaTest, DeltaRejectsInconsistentPassPrecedence) { + EXPECT_THROW(deltaUpdateFromYaml(R"EOF(system_version_info: "1" +resources: +- name: "selector-1" + version: "1" + resource: + "@type": type.googleapis.com/cilium.NetworkPolicyResource + selector: + remote_identities: [ 43 ] +- name: "selector-2" + version: "1" + resource: + "@type": type.googleapis.com/cilium.NetworkPolicyResource + selector: + remote_identities: [ 44 ] +- name: "policy-42" + version: "1" + resource: + "@type": type.googleapis.com/cilium.NetworkPolicyResource + policy: + endpoint_ips: + - "10.1.2.3" + endpoint_id: 42 + ingress_per_port_policies: + - port: 80 + rules: + - precedence: 1000 + pass_precedence: 100 + selectors: [ "selector-1" ] + - precedence: 900 + pass_precedence: 200 + selectors: [ "selector-2" ] +)EOF"), + EnvoyException); +} + +TEST_F(CiliumNetworkPolicyDeltaTest, DeltaNewStreamReplacesStateWithFullSnapshot) { + EXPECT_NO_THROW(deltaUpdateFromYaml(R"EOF(system_version_info: "1" +resources: +- name: "selector-1" + version: "1" + resource: + "@type": type.googleapis.com/cilium.NetworkPolicyResource + selector: + remote_identities: [ 43 ] +- name: "selector-2" + version: "1" + resource: + "@type": type.googleapis.com/cilium.NetworkPolicyResource + selector: + remote_identities: [ 44 ] +- name: "policy-42" + version: "1" + resource: + "@type": type.googleapis.com/cilium.NetworkPolicyResource + policy: + endpoint_ips: + - "10.1.2.3" + endpoint_id: 42 + ingress_per_port_policies: + - port: 80 + rules: + - selectors: [ "selector-1" ] +- name: "policy-43" + version: "1" + resource: + "@type": type.googleapis.com/cilium.NetworkPolicyResource + policy: + endpoint_ips: + - "10.2.3.4" + endpoint_id: 43 + ingress_per_port_policies: + - port: 81 + rules: + - selectors: [ "selector-2" ] +)EOF")); + + EXPECT_TRUE(ingressAllowed("10.1.2.3", 43, 80)); + EXPECT_TRUE(ingressAllowed("10.2.3.4", 44, 81)); + + resetStreamForTest(); + + EXPECT_NO_THROW(deltaUpdateFromYaml(R"EOF(system_version_info: "2" +resources: +- name: "selector-3" + version: "1" + resource: + "@type": type.googleapis.com/cilium.NetworkPolicyResource + selector: + remote_identities: [ 45 ] +- name: "policy-42" + version: "2" + resource: + "@type": type.googleapis.com/cilium.NetworkPolicyResource + policy: + endpoint_ips: + - "10.1.2.3" + endpoint_id: 42 + ingress_per_port_policies: + - port: 8080 + rules: + - selectors: [ "selector-3" ] +)EOF")); + + EXPECT_TRUE(ingressAllowed("10.1.2.3", 45, 8080)); + EXPECT_FALSE(policy_map_->exists("10.2.3.4")); +} + +TEST_F(CiliumNetworkPolicyDeltaTest, DeltaNewStreamClearsStaleResourceNamesFromResourceMap) { + EXPECT_NO_THROW(deltaUpdateFromYaml(R"EOF(system_version_info: "1" +resources: +- name: "selector-1" + version: "1" + resource: + "@type": type.googleapis.com/cilium.NetworkPolicyResource + selector: + remote_identities: [ 43 ] +- name: "selector-2" + version: "1" + resource: + "@type": type.googleapis.com/cilium.NetworkPolicyResource + selector: + remote_identities: [ 44 ] +- name: "policy-42" + version: "1" + resource: + "@type": type.googleapis.com/cilium.NetworkPolicyResource + policy: + endpoint_ips: + - "10.1.2.3" + endpoint_id: 42 + ingress_per_port_policies: + - port: 80 + rules: + - selectors: [ "selector-1" ] +- name: "policy-43" + version: "1" + resource: + "@type": type.googleapis.com/cilium.NetworkPolicyResource + policy: + endpoint_ips: + - "10.2.3.4" + endpoint_id: 43 + ingress_per_port_policies: + - port: 81 + rules: + - selectors: [ "selector-2" ] +)EOF")); + + resetStreamForTest(); + + EXPECT_NO_THROW(deltaUpdateFromYaml(R"EOF(system_version_info: "2" +resources: +- name: "selector-3" + version: "1" + resource: + "@type": type.googleapis.com/cilium.NetworkPolicyResource + selector: + remote_identities: [ 45 ] +- name: "policy-42" + version: "2" + resource: + "@type": type.googleapis.com/cilium.NetworkPolicyResource + policy: + endpoint_ips: + - "10.1.2.3" + endpoint_id: 42 + ingress_per_port_policies: + - port: 8080 + rules: + - selectors: [ "selector-3" ] +)EOF")); + + EXPECT_FALSE(policy_map_->exists("10.2.3.4")); + + EXPECT_NO_THROW(deltaUpdateFromYaml(R"EOF(system_version_info: "3" +resources: +- name: "policy-43" + version: "1" + resource: + "@type": type.googleapis.com/cilium.NetworkPolicyResource + selector: + remote_identities: [ 46 ] +)EOF")); + + EXPECT_NO_THROW(deltaUpdateFromYaml(R"EOF(system_version_info: "4" +resources: +- name: "policy-42" + version: "3" + resource: + "@type": type.googleapis.com/cilium.NetworkPolicyResource + policy: + endpoint_ips: + - "10.1.2.3" + endpoint_id: 42 + ingress_per_port_policies: + - port: 9090 + rules: + - selectors: [ "policy-43" ] +)EOF")); + + EXPECT_TRUE(ingressAllowed("10.1.2.3", 46, 9090)); +} + +TEST_F(CiliumNetworkPolicyDeltaTest, SameStreamSelectorOnlyUpdateUsesLatestSelectorSnapshot) { + EXPECT_NO_THROW(deltaUpdateFromYaml(R"EOF(system_version_info: "1" +resources: +- name: "selector-1" + version: "1" + resource: + "@type": type.googleapis.com/cilium.NetworkPolicyResource + selector: + remote_identities: [ 43 ] +- name: "policy-42" + version: "1" + resource: + "@type": type.googleapis.com/cilium.NetworkPolicyResource + policy: + endpoint_ips: + - "10.1.2.3" + endpoint_id: 42 + ingress_per_port_policies: + - port: 80 + rules: + - selectors: [ "selector-1" ] +)EOF")); + + const auto old_policy = policyInstanceShared("10.1.2.3"); + ASSERT_NE(nullptr, old_policy); + const auto initial_stream_generation = selectorStreamGenerationForTest(*old_policy); + + EXPECT_GT(initial_stream_generation, 0); + EXPECT_EQ(initial_stream_generation, selectorStreamGenerationForTest(*old_policy)); + EXPECT_EQ(1, selectorVersionForTest(*old_policy)); + + EXPECT_NO_THROW(deltaUpdateFromYaml(R"EOF(system_version_info: "2" +resources: +- name: "selector-1" + version: "2" + resource: + "@type": type.googleapis.com/cilium.NetworkPolicyResource + selector: + remote_identities: [ 44 ] +)EOF")); + + EXPECT_EQ(initial_stream_generation, selectorStreamGenerationForTest(*old_policy)); + EXPECT_EQ(2, selectorVersionForTest(*old_policy)); + EXPECT_TRUE(ingressAllowed("10.1.2.3", 44, 80)); + EXPECT_FALSE(ingressAllowed("10.1.2.3", 43, 80)); +} + +TEST_F(CiliumNetworkPolicyDeltaTest, NewStreamKeepsOldPolicyPinnedToOldSelectorSnapshot) { + EXPECT_NO_THROW(deltaUpdateFromYaml(R"EOF(system_version_info: "1" +resources: +- name: "selector-1" + version: "1" + resource: + "@type": type.googleapis.com/cilium.NetworkPolicyResource + selector: + remote_identities: [ 43 ] +- name: "selector-2" + version: "1" + resource: + "@type": type.googleapis.com/cilium.NetworkPolicyResource + selector: + remote_identities: [ 44 ] +- name: "policy-42" + version: "1" + resource: + "@type": type.googleapis.com/cilium.NetworkPolicyResource + policy: + endpoint_ips: + - "10.1.2.3" + endpoint_id: 42 + ingress_per_port_policies: + - port: 80 + rules: + - selectors: [ "selector-1" ] +)EOF")); + + const auto old_policy = policyInstanceShared("10.1.2.3"); + ASSERT_NE(nullptr, old_policy); + const auto initial_stream_generation = selectorStreamGenerationForTest(*old_policy); + + EXPECT_GT(initial_stream_generation, 0); + EXPECT_EQ(initial_stream_generation, selectorStreamGenerationForTest(*old_policy)); + EXPECT_EQ(1, selectorVersionForTest(*old_policy)); + + EXPECT_NO_THROW(deltaUpdateFromYaml(R"EOF(system_version_info: "1" +resources: +- name: "selector-1" + version: "2" + resource: + "@type": type.googleapis.com/cilium.NetworkPolicyResource + selector: + remote_identities: [ 45 ] +)EOF")); + + EXPECT_EQ(initial_stream_generation, selectorStreamGenerationForTest(*old_policy)); + EXPECT_EQ(2, selectorVersionForTest(*old_policy)); + + resetStreamForTest(); + + EXPECT_NO_THROW(deltaUpdateFromYaml(R"EOF(system_version_info: "2" +resources: +- name: "selector-1" + version: "2" + resource: + "@type": type.googleapis.com/cilium.NetworkPolicyResource + selector: + remote_identities: [ 44 ] +- name: "selector-2" + version: "2" + resource: + "@type": type.googleapis.com/cilium.NetworkPolicyResource + selector: + remote_identities: [ 43 ] +- name: "policy-42" + version: "2" + resource: + "@type": type.googleapis.com/cilium.NetworkPolicyResource + policy: + endpoint_ips: + - "10.1.2.3" + endpoint_id: 42 + ingress_per_port_policies: + - port: 80 + rules: + - selectors: [ "selector-1" ] +)EOF")); + + const auto new_policy = policyInstanceShared("10.1.2.3"); + ASSERT_NE(nullptr, new_policy); + EXPECT_NE(old_policy.get(), new_policy.get()); + + EXPECT_EQ(initial_stream_generation, selectorStreamGenerationForTest(*old_policy)); + EXPECT_EQ(2, selectorVersionForTest(*old_policy)); + EXPECT_EQ(initial_stream_generation + 1, selectorStreamGenerationForTest(*new_policy)); + EXPECT_EQ(3, selectorVersionForTest(*new_policy)); + EXPECT_TRUE(ingressAllowed("10.1.2.3", 44, 80)); + EXPECT_FALSE(ingressAllowed("10.1.2.3", 43, 80)); +} + TEST_F(CiliumNetworkPolicyTest, OverlappingPortRange) { EXPECT_NO_THROW(updateFromYaml(R"EOF(version_info: "1" resources: @@ -1205,9 +2992,11 @@ TEST_F(CiliumNetworkPolicyTest, Precedence) { rules: [80-80]: - rules: + - remotes: [] + precedence: 10 + tier_last_precedence: 1 - remotes: [] name: "default allow rule" - precedence: 9 egress: rules: [] )EOF"; @@ -1247,9 +3036,11 @@ TEST_F(CiliumNetworkPolicyTest, Precedence) { rules: [80-80]: - rules: + - remotes: [] + precedence: 10 + tier_last_precedence: 1 - remotes: [] name: "default allow rule" - precedence: 9 egress: rules: [] )EOF"; @@ -1418,14 +3209,17 @@ TEST_F(CiliumNetworkPolicyTest, Precedence) { [80-80]: - rules: - remotes: [43] - precedence: 999 + precedence: 1000 + tier_last_precedence: 501 + - remotes: [] + deny: true + precedence: 900 + - remotes: [43] + precedence: 500 http_rules: - headers: - name: ":path" value: "/allowed" - - remotes: [] - deny: true - precedence: 900 egress: rules: [] )EOF"; @@ -1477,14 +3271,17 @@ TEST_F(CiliumNetworkPolicyTest, Precedence) { [80-80]: - rules: - remotes: [43] - precedence: 999 + precedence: 1000 + tier_last_precedence: 501 + - remotes: [] + deny: true + precedence: 900 + - remotes: [43,44] + precedence: 500 http_rules: - headers: - name: ":path" value: "/allowed" - - remotes: [] - deny: true - precedence: 900 egress: rules: [] )EOF"; @@ -1536,8 +3333,14 @@ TEST_F(CiliumNetworkPolicyTest, Precedence) { rules: [80-80]: - rules: + - remotes: [] + precedence: 1000 + tier_last_precedence: 501 + - remotes: [] + deny: true + precedence: 900 - remotes: [43,44] - precedence: 999 + precedence: 500 http_rules: - headers: - name: ":path" @@ -1594,21 +3397,24 @@ TEST_F(CiliumNetworkPolicyTest, Precedence) { [80-80]: - rules: - remotes: [43] - precedence: 999 + precedence: 1000 + tier_last_precedence: 501 + - remotes: [] + deny: true + precedence: 900 + - remotes: [] + precedence: 500 http_rules: - headers: - name: ":path" value: "/allowed" - - remotes: [] - deny: true - precedence: 900 egress: rules: [] )EOF"; EXPECT_TRUE(validate("10.1.2.3", expected9)); - // Remote 43 is promoted above deny by pass. + // Remote 43 matches the pass rule and skips the intermediate deny. EXPECT_TRUE(ingressAllowed("10.1.2.3", 43, 80, {{":path", "/allowed"}})); // Other remotes are still denied by the deny rule. EXPECT_FALSE(ingressAllowed("10.1.2.3", 44, 80, {{":path", "/allowed"}})); @@ -1649,23 +3455,26 @@ TEST_F(CiliumNetworkPolicyTest, Precedence) { [80-80]: - rules: - remotes: [43] - precedence: 999 + precedence: 1000 + tier_last_precedence: 501 + - remotes: [] + deny: true + precedence: 900 + - remotes: [43,44] + precedence: 500 http_rules: - headers: - name: ":path" value: "/allowed" - - remotes: [] - deny: true - precedence: 900 egress: rules: [] )EOF"; EXPECT_TRUE(validate("10.1.2.3", expected10)); - // Pass from wildcard port should promote remote 43 above deny on port 80. + // Pass from wildcard port lets remote 43 skip the deny on port 80. EXPECT_TRUE(ingressAllowed("10.1.2.3", 43, 80, {{":path", "/allowed"}})); - // Remote 44 is denied due to only 43 being promoted. + // Remote 44 is denied because only 43 matches the pass rule. EXPECT_FALSE(ingressAllowed("10.1.2.3", 44, 80, {{":path", "/allowed"}})); // Unspecified remotes remain denied. EXPECT_FALSE(ingressAllowed("10.1.2.3", 45, 80, {{":path", "/allowed"}})); @@ -1710,15 +3519,21 @@ TEST_F(CiliumNetworkPolicyTest, Precedence) { rules: [80-80]: - rules: + - remotes: [43] + precedence: 1000 + tier_last_precedence: 501 + - remotes: [44] + precedence: 1000 + tier_last_precedence: 501 + - remotes: [] + deny: true + precedence: 900 - remotes: [43,44] - precedence: 999 + precedence: 500 http_rules: - headers: - name: ":path" value: "/allowed" - - remotes: [] - deny: true - precedence: 900 egress: rules: [] )EOF"; @@ -1731,15 +3546,11 @@ TEST_F(CiliumNetworkPolicyTest, Precedence) { EXPECT_FALSE(ingressAllowed("10.1.2.3", 45, 80, {{":path", "/allowed"}})); // - // 12th update: non-pass rule shadowing inside a pass tier + // 12th update: pass tier keeps native rules in runtime order // - // The pass rule is required to enable tier processing, but it targets only - // remote 45 so the tier is not wildcard-pass and does not pre-shadow 43/44. - // Within this tier: - // - A higher-precedence deny for remote 44 establishes a final verdict for 44. - // - A lower-precedence allow for [43,44] must have 44 removed due to shadowing. - // - A second allow at the same precedence for [43] must keep 43, confirming - // no same-precedence identity shadowing between allow rules. + // With runtime pass handling the pass rule remains present, the higher-precedence + // deny for remote 44 stays in place, and the lower allow rules are evaluated in + // their native form instead of being rewritten during preprocessing. EXPECT_NO_THROW(version = updateFromYaml(R"EOF(version_info: "12" resources: - "@type": type.googleapis.com/cilium.NetworkPolicy @@ -1784,11 +3595,8 @@ TEST_F(CiliumNetworkPolicyTest, Precedence) { [80-80]: - rules: - remotes: [45] - precedence: 999 - http_rules: - - headers: - - name: ":path" - value: "/allow-c" + precedence: 1000 + tier_last_precedence: 701 - remotes: [44] deny: true precedence: 900 @@ -1798,7 +3606,7 @@ TEST_F(CiliumNetworkPolicyTest, Precedence) { - headers: - name: ":path" value: "/allow-b" - - remotes: [43] + - remotes: [43,44] precedence: 800 http_rules: - headers: @@ -1819,12 +3627,12 @@ TEST_F(CiliumNetworkPolicyTest, Precedence) { // Remote 43 is not passed, but both same-precedence allow rules remain effective. EXPECT_TRUE(ingressAllowed("10.1.2.3", 43, 80, {{":path", "/allow-a"}})); EXPECT_TRUE(ingressAllowed("10.1.2.3", 43, 80, {{":path", "/allow-b"}})); - // Remote 44 is denied by the higher-precedence deny and removed from allow-a. + // Remote 44 is denied by the higher-precedence deny before either allow rule is reached. EXPECT_FALSE(ingressAllowed("10.1.2.3", 44, 80, {{":path", "/allow-a"}})); EXPECT_FALSE(ingressAllowed("10.1.2.3", 44, 80, {{":path", "/allow-b"}})); - // Pass remote 45 does not match /allow-a because only /allow-c is promoted for it. + // Pass remote 45 does not match /allow-a because only the lower wildcard allow matches it. EXPECT_FALSE(ingressAllowed("10.1.2.3", 45, 80, {{":path", "/allow-a"}})); - // Wildcard allow at precedence 700 is promoted to precedence 999 only for pass remote 45. + // Pass remote 45 reaches the lower wildcard allow at precedence 700. EXPECT_TRUE(ingressAllowed("10.1.2.3", 45, 80, {{":path", "/allow-c"}})); // Non-pass remotes not already denied at higher precedence still match the // original wildcard rule at precedence 700. @@ -1832,12 +3640,11 @@ TEST_F(CiliumNetworkPolicyTest, Precedence) { EXPECT_FALSE(ingressAllowed("10.1.2.3", 44, 80, {{":path", "/allow-c"}})); // - // 13th update: inherited wildcard current-tier pass fully shadowed + // 13th update: inherited wildcard and local pass rules both remain // - // Wildcard port has a current-tier pass for remote 43, and specific port has - // a higher precedence pass for the same remote on the same tier. When the - // wildcard pass is inherited, it is fully shadowed and skipped, as evidenced by the - // precedence of the passed-to rule for remote 43, which is 999 rather than 899. + // Runtime pass handling keeps both pass rules visible in precedence order. The + // specific-port pass is checked before the inherited wildcard-port pass for the + // same remote, but neither rule is rewritten away. EXPECT_NO_THROW(version = updateFromYaml(R"EOF(version_info: "13" resources: - "@type": type.googleapis.com/cilium.NetworkPolicy @@ -1873,39 +3680,38 @@ TEST_F(CiliumNetworkPolicyTest, Precedence) { [80-80]: - rules: - remotes: [43] - precedence: 999 + precedence: 1000 + tier_last_precedence: 701 + - remotes: [43] + precedence: 900 + tier_last_precedence: 701 + - remotes: [] + deny: true + precedence: 800 + - remotes: [43,44] + precedence: 700 http_rules: - headers: - name: ":path" value: "/shadowed-inherited-pass" - - remotes: [] - deny: true - precedence: 800 egress: rules: [] )EOF"; EXPECT_TRUE(validate("10.1.2.3", expected13)); - // Remote 43 is promoted above deny due to the specific-port pass. + // Remote 43 hits the specific-port pass before the intermediate deny. EXPECT_TRUE(ingressAllowed("10.1.2.3", 43, 80, {{":path", "/shadowed-inherited-pass"}})); // Remote 44 remains denied by the intermediate deny. EXPECT_FALSE(ingressAllowed("10.1.2.3", 44, 80, {{":path", "/shadowed-inherited-pass"}})); EXPECT_FALSE(ingressAllowed("10.1.2.3", 45, 80, {{":path", "/shadowed-inherited-pass"}})); // - // 14th update: multiple wildcard pass tiers inherited by a specific port + // 14th update: multiple inherited pass tiers are handled at runtime // - // Wildcard port contributes two pass tiers: - // Tier boundaries are inclusive. - // - tier 1 pass (1300/1000) for remote 41: tier boundaries [1300..1000] - // - tier 2 pass (900/700) for remote 42: tier boundaries [999..700] - // For port 80: - // - deny at 850 is within tier 2, so it is promoted by tier 1 pass for remote 41 to 1150 - // - allow [41,42,43] at 600 is split and promoted by both tiers: - // - 41 to tier 1 precedence 900 - // - 42 to tier 2 precedence 800 - // - 43 remains at tier 3 at precedence 600 + // The wildcard-port pass rules remain visible in the exact-port rule list. Remote 41 + // still reaches the intermediate deny, while remote 42 matches the lower wildcard pass + // and skips that deny to the allow at precedence 600. EXPECT_NO_THROW(version = updateFromYaml(R"EOF(version_info: "14" resources: - "@type": type.googleapis.com/cilium.NetworkPolicy @@ -1923,7 +3729,7 @@ TEST_F(CiliumNetworkPolicyTest, Precedence) { remote_policies: [ 42 ] - port: 80 rules: - - precedence: 850 + - precedence: 750 deny: true - precedence: 600 remote_policies: [ 41, 42, 43 ] @@ -1941,22 +3747,31 @@ TEST_F(CiliumNetworkPolicyTest, Precedence) { [80-80]: - rules: - remotes: [41] - deny: true - precedence: 1150 + precedence: 1300 + tier_last_precedence: 1000 + - remotes: [42] + precedence: 900 + tier_last_precedence: 700 - remotes: [] deny: true - precedence: 850 + precedence: 750 + - remotes: [41,42,43] + precedence: 600 + http_rules: + - headers: + - name: ":path" + value: "/multi-tier" egress: rules: [] )EOF"; EXPECT_TRUE(validate("10.1.2.3", expected14)); - // Remote 41 hits the promoted deny from tier 1. + // Remote 41 does not match the lower pass tier, so it still hits the deny at 850. EXPECT_FALSE(ingressAllowed("10.1.2.3", 41, 80, {{":path", "/multi-tier"}})); - // Remote 42 is promoted by the lower wildcard tier, but remains below deny. - EXPECT_FALSE(ingressAllowed("10.1.2.3", 42, 80, {{":path", "/multi-tier"}})); - // Remote 43 is not promoted and is denied. + // Remote 42 matches the lower pass tier and skips the deny to the allow at precedence 600. + EXPECT_TRUE(ingressAllowed("10.1.2.3", 42, 80, {{":path", "/multi-tier"}})); + // Remote 43 does not match either pass tier and is denied. EXPECT_FALSE(ingressAllowed("10.1.2.3", 43, 80, {{":path", "/multi-tier"}})); // @@ -1993,17 +3808,16 @@ TEST_F(CiliumNetworkPolicyTest, Precedence) { EnvoyException, "PortNetworkPolicy: Inconsistent pass precedence 600 != 700"); - // Failed update must leave policy unchanged from version 10. + // Failed update must leave policy unchanged from version 14. EXPECT_TRUE(validate("10.1.2.3", expected14)); EXPECT_FALSE(ingressAllowed("10.1.2.3", 41, 80, {{":path", "/multi-tier"}})); - EXPECT_FALSE(ingressAllowed("10.1.2.3", 42, 80, {{":path", "/multi-tier"}})); + EXPECT_TRUE(ingressAllowed("10.1.2.3", 42, 80, {{":path", "/multi-tier"}})); // - // 16th update: inherited wildcard pass skips remaining rules on that tier + // 16th update: inherited wildcard pass skips remaining rules on that tier at runtime // - // Wildcard port has a wildcard pass (2000/700), which is inherited for port 80. - // Rules in that same tier [1999..700] are skipped; a lower-tier rule at 600 is - // retained and promoted to 1900 by the inherited wildcard pass. + // The pass rule and the skipped-tier rules remain present in the rendered policy, but a + // matching remote jumps past the 1200/1100 rules and reaches the lower-tier allow at 600. EXPECT_NO_THROW(version = updateFromYaml(R"EOF(version_info: "16" resources: - "@type": type.googleapis.com/cilium.NetworkPolicy @@ -2042,8 +3856,20 @@ TEST_F(CiliumNetworkPolicyTest, Precedence) { rules: [80-80]: - rules: + - remotes: [] + precedence: 2000 + tier_last_precedence: 700 + - remotes: [43] + deny: true + precedence: 1200 + - remotes: [44] + precedence: 1100 + http_rules: + - headers: + - name: ":path" + value: "/should-skip" - remotes: [43,44] - precedence: 1900 + precedence: 600 http_rules: - headers: - name: ":path" @@ -2054,15 +3880,15 @@ TEST_F(CiliumNetworkPolicyTest, Precedence) { EXPECT_TRUE(validate("10.1.2.3", expected16)); - // Both remotes are allowed by the promoted lower-tier rule. + // Both remotes are allowed by the lower-tier rule reached after the wildcard pass. EXPECT_TRUE(ingressAllowed("10.1.2.3", 43, 80, {{":path", "/promoted-after-skip"}})); EXPECT_TRUE(ingressAllowed("10.1.2.3", 44, 80, {{":path", "/promoted-after-skip"}})); - // Tier rule at 800 is skipped by inherited wildcard pass. + // The 1100 rule remains present but is skipped by the inherited wildcard pass. EXPECT_FALSE(ingressAllowed("10.1.2.3", 44, 80, {{":path", "/should-skip"}})); EXPECT_FALSE(ingressAllowed("10.1.2.3", 45, 80, {{":path", "/promoted-after-skip"}})); // - // 17th update: Shadowed rules are eliminated + // 17th update: overlapping lower rules remain visible under runtime pass handling // EXPECT_NO_THROW(version = updateFromYaml(R"EOF(version_info: "17" resources: @@ -2102,11 +3928,20 @@ TEST_F(CiliumNetworkPolicyTest, Precedence) { rules: [80-80]: - rules: + - remotes: [] + precedence: 1000 + tier_last_precedence: 901 - remotes: [43] deny: true - precedence: 999 - - remotes: [44] - precedence: 699 + precedence: 900 + - remotes: [43] + precedence: 800 + http_rules: + - headers: + - name: ":path" + value: "/should-skip" + - remotes: [43,44] + precedence: 600 http_rules: - headers: - name: ":path" @@ -2119,7 +3954,7 @@ TEST_F(CiliumNetworkPolicyTest, Precedence) { EXPECT_FALSE(ingressAllowed("10.1.2.3", 43, 80, {{":path", "/partially-skipped"}})); EXPECT_TRUE(ingressAllowed("10.1.2.3", 44, 80, {{":path", "/partially-skipped"}})); - // Rule at 800 is shadowed by higher precedence deny + // Rule at 800 remains present, but the higher-precedence deny still wins for remote 43. EXPECT_FALSE(ingressAllowed("10.1.2.3", 43, 80, {{":path", "/should-skip"}})); // inapplicable identity EXPECT_FALSE(ingressAllowed("10.1.2.3", 45, 80, {{":path", "/partially-skipped"}})); diff --git a/tests/versioned_test.cc b/tests/versioned_test.cc new file mode 100644 index 000000000..be90c9ec5 --- /dev/null +++ b/tests/versioned_test.cc @@ -0,0 +1,1165 @@ +#include + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "source/common/common/lock_guard.h" +#include "source/common/common/thread.h" + +#include "absl/container/flat_hash_map.h" +#include "absl/container/flat_hash_set.h" +#include "cilium/versioned.h" + +// NOLINT(namespace-envoy) +namespace { + +class TrackedValue : public VersionedNode { +public: + explicit TrackedValue(int id) : TrackedValue(id, id) {} + + TrackedValue(int key_id, int generation) + : key_id_(key_id), generation_(generation), unique_id_(next_unique_id_++) { + ++constructed_count_; + ++live_count_; + } + + ~TrackedValue() { + ++destroyed_count_; + --live_count_; + ++destroyed_by_id_[generation_]; + } + + int id() const { return generation_; } + int keyId() const { return key_id_; } + int generation() const { return generation_; } + uint64_t uniqueId() const { return unique_id_; } + + static void resetCounters() { + constructed_count_ = 0; + destroyed_count_ = 0; + live_count_ = 0; + destroyed_by_id_.clear(); + next_unique_id_ = 1; + } + + static int constructedCount() { return constructed_count_; } + static int destroyedCount() { return destroyed_count_; } + static int liveCount() { return live_count_; } + + static int destroyedCountFor(int id) { + const auto it = destroyed_by_id_.find(id); + return it == destroyed_by_id_.end() ? 0 : it->second; + } + +private: + int key_id_; + int generation_; + uint64_t unique_id_; + + static inline int constructed_count_ = 0; + static inline int destroyed_count_ = 0; + static inline int live_count_ = 0; + static inline uint64_t next_unique_id_ = 1; + static inline std::map destroyed_by_id_; +}; + +using TrackedHandle = VersionedHandle; +using TrackedDeferredDeletion = DeferredDeletion; +using TrackedMap = VersionedMap; +using TrackedAccess = VersionedTestAccess; + +const TrackedValue* activeValue(const TrackedMap& map, const std::string& key, + uint64_t version = versionMax) { + auto handle = map.find(key); + return handle != nullptr ? handle->get(version) : nullptr; +} + +struct PublishedHandles { + uint64_t snapshot_id; + uint64_t version; + std::vector> handles; +}; + +struct ObservedNode { + const void* node; + const void* next; + const TrackedValue* value; + uint64_t add_version; + uint64_t remove_version; +}; + +enum class ValidationMode { + StrictConsistentView, + ConcurrentSafeInvisibleTail, +}; + +std::string describeValue(const TrackedValue* value) { + if (value == nullptr) { + return "nullptr"; + } + return testing::PrintToString( + std::make_tuple(value->keyId(), value->generation(), value->uniqueId())); +} + +std::string describeObservedChain(const std::vector& nodes) { + std::ostringstream out; + for (size_t i = 0; i < nodes.size(); ++i) { + if (i > 0) { + out << " -> "; + } + const auto& node = nodes[i]; + out << "{node=" << node.node << ", next=" << node.next + << ", value=" << describeValue(node.value) << ", add=" << node.add_version + << ", remove=" << node.remove_version << "}"; + } + if (nodes.empty()) { + out << ""; + } + return out.str(); +} + +std::string validateTraversalAtVersion(const std::string& label, + const VersionedNode* initial_head, + uint64_t version, ValidationMode mode, + const TrackedValue* expected = nullptr, + const void* handle = nullptr) { + const bool require_consistent_view = mode == ValidationMode::StrictConsistentView; + const TrackedValue* visible_value = nullptr; + size_t visible_count = 0; + uint64_t previous_add_version = versionNotRemoved; + bool in_invisible_tail = false; + absl::flat_hash_map visited_nodes; + std::vector observed_nodes; + for (const auto* node = initial_head; node != nullptr;) { + const auto* next = TrackedAccess::next(node); + observed_nodes.push_back(ObservedNode{node, next, TrackedAccess::value(node), + TrackedAccess::addVersion(node), + TrackedAccess::removeVersion(node)}); + const auto add_version = observed_nodes.back().add_version; + const auto [visited_it, inserted] = visited_nodes.emplace(node, add_version); + if (!inserted) { + if (visited_it->second == add_version) { + return "cycle detected for '" + label + "' at version " + std::to_string(version) + + " handle=" + testing::PrintToString(handle) + + " initial_head=" + testing::PrintToString(initial_head) + + " chain=" + describeObservedChain(observed_nodes); + } + visited_it->second = add_version; + } + + const auto remove_version = observed_nodes.back().remove_version; + if (add_version >= versionNotRemoved) { + return "unpublished node observed for '" + label + "' at version " + std::to_string(version) + + " handle=" + testing::PrintToString(handle) + + " initial_head=" + testing::PrintToString(initial_head) + + " chain=" + describeObservedChain(observed_nodes); + } + if ((require_consistent_view || !in_invisible_tail) && add_version > previous_add_version) { + return "add_version increased while following next links for '" + label + "' at version " + + std::to_string(version) + " handle=" + testing::PrintToString(handle) + + " initial_head=" + testing::PrintToString(initial_head) + + " previous_add=" + std::to_string(previous_add_version) + + " current_add=" + std::to_string(add_version) + + " chain=" + describeObservedChain(observed_nodes); + } + if (remove_version < versionNotRemoved && add_version > remove_version) { + return "invalid node visibility window for '" + label + "' at version " + + std::to_string(version) + " handle=" + testing::PrintToString(handle) + + " initial_head=" + testing::PrintToString(initial_head) + + " chain=" + describeObservedChain(observed_nodes); + } + + if (add_version <= version && version < remove_version) { + ++visible_count; + visible_value = TrackedAccess::value(node); + if (require_consistent_view && visible_count > 1) { + return "multiple visible nodes at version " + std::to_string(version) + " for '" + label + + "' handle=" + testing::PrintToString(handle) + + " initial_head=" + testing::PrintToString(initial_head) + + " chain=" + describeObservedChain(observed_nodes); + } + } + // Once traversal reaches the first node that is already stale for this version, production + // readers no longer depend on per-handle ordering. First-phase GC may have already rewired the + // rest of the tail into a mixed deferred-deletion list, but those nodes must remain safe to + // walk and invisible for this version. + if (!require_consistent_view && in_invisible_tail && + TrackedAccess::isVisibleInVersion(node, version)) { + return "visible node observed after entering invisible tail for '" + label + "' at version " + + std::to_string(version) + " handle=" + testing::PrintToString(handle) + + " initial_head=" + testing::PrintToString(initial_head) + + " chain=" + describeObservedChain(observed_nodes); + } + if (!TrackedAccess::isVisibleInVersion(node, version) && + !TrackedAccess::isAddedAfterVersion(node, version)) { + in_invisible_tail = true; + } + previous_add_version = add_version; + node = next; + } + + if (require_consistent_view && expected != visible_value) { + return "handle->get mismatch for '" + label + "' at version " + std::to_string(version) + + " handle=" + testing::PrintToString(handle) + + " initial_head=" + testing::PrintToString(initial_head) + + " expected=" + describeValue(expected) + ", traversed=" + describeValue(visible_value) + + ", chain=" + describeObservedChain(observed_nodes); + } + + return ""; +} + +std::string validateHandleAtVersion(const std::string& label, const TrackedHandle& handle, + uint64_t version, + ValidationMode mode = ValidationMode::StrictConsistentView) { + if (handle == nullptr) { + return "snapshot contains null handle for '" + label + "'"; + } + + return validateTraversalAtVersion( + label, TrackedAccess::head(handle), version, mode, + mode == ValidationMode::StrictConsistentView ? handle->get(version) : nullptr, handle.get()); +} + +std::string +validateNodeAtVersion(const std::string& label, const VersionedNode* node, + uint64_t version, + ValidationMode mode = ValidationMode::ConcurrentSafeInvisibleTail) { + if (node == nullptr) { + return "null node supplied for '" + label + "'"; + } + return validateTraversalAtVersion(label, node, version, mode); +} + +std::string +validatePublishedSnapshotAtVersion(const PublishedHandles& snapshot, uint64_t version, + ValidationMode mode = ValidationMode::StrictConsistentView) { + for (const auto& [label, handle] : snapshot.handles) { + auto error = validateHandleAtVersion(label, handle, version, mode); + if (!error.empty()) { + return error; + } + } + return ""; +} + +std::string validatePublishedSnapshot(const PublishedHandles& snapshot) { + return validatePublishedSnapshotAtVersion(snapshot, snapshot.version); +} + +std::shared_ptr makePublishedHandles(const TrackedMap& map, + uint64_t snapshot_id = 0) { + auto snapshot = std::make_shared(); + snapshot->snapshot_id = snapshot_id; + snapshot->version = map.getVersion(); + + absl::flat_hash_set seen_handles; + for (const auto& [key, handle] : TrackedAccess::entries(map)) { + if (handle != nullptr && seen_handles.insert(handle.get()).second) { + snapshot->handles.emplace_back(key, handle); + } + } + size_t dirty_index = 0; + for (const auto& handle : TrackedAccess::dirtyValues(map)) { + if (handle != nullptr && seen_handles.insert(handle.get()).second) { + snapshot->handles.emplace_back("dirty:" + std::to_string(dirty_index++), handle); + } + } + + return snapshot; +} + +class VersionedTest : public testing::Test { +protected: + void SetUp() override { TrackedValue::resetCounters(); } + + void TearDown() override { + EXPECT_EQ(TrackedValue::liveCount(), 0); + EXPECT_EQ(TrackedValue::constructedCount(), TrackedValue::destroyedCount()); + } +}; + +TEST_F(VersionedTest, VersionedValueFirstInsertedValueIsVisibleInPublishedVersion) { + VersionedValue value; + + value.set(1, new TrackedValue(1)); + + EXPECT_EQ(value.get(0), nullptr); + ASSERT_NE(value.get(1), nullptr); + EXPECT_EQ(value.get(1)->id(), 1); +} + +TEST_F(VersionedTest, VersionedValueNewerValueShadowsOlderValueFromNewVersionOnward) { + VersionedValue value; + + value.set(1, new TrackedValue(1)); + value.set(2, new TrackedValue(2)); + + ASSERT_NE(value.get(1), nullptr); + EXPECT_EQ(value.get(1)->id(), 1); + ASSERT_NE(value.get(2), nullptr); + EXPECT_EQ(value.get(2)->id(), 2); + EXPECT_EQ(TrackedValue::liveCount(), 2); +} + +TEST_F(VersionedTest, VersionedValueClearHidesOnlyFromThatVersionOnward) { + VersionedValue value; + + value.set(1, new TrackedValue(1)); + value.clear(2); + + ASSERT_NE(value.get(1), nullptr); + EXPECT_EQ(value.get(1)->id(), 1); + EXPECT_EQ(value.get(2), nullptr); +} + +TEST_F(VersionedTest, VersionedValueRevertRestoresPendingClear) { + VersionedValue value; + TrackedDeferredDeletion deferred; + + value.set(1, new TrackedValue(1)); + value.clear(2); + + EXPECT_EQ(value.get(2), nullptr); + + value.revert(1, deferred); + + EXPECT_TRUE(deferred.empty()); + ASSERT_NE(value.get(1), nullptr); + EXPECT_EQ(value.get(1)->id(), 1); + ASSERT_NE(value.get(2), nullptr); + EXPECT_EQ(value.get(2)->id(), 1); + EXPECT_EQ(TrackedValue::liveCount(), 1); +} + +TEST_F(VersionedTest, VersionedValueRevertDeletesUnpublishedReplacement) { + VersionedValue value; + TrackedDeferredDeletion deferred; + + value.set(1, new TrackedValue(1)); + value.set(2, new TrackedValue(2)); + + value.revert(1, deferred); + + EXPECT_FALSE(deferred.empty()); + ASSERT_NE(value.get(1), nullptr); + EXPECT_EQ(value.get(1)->id(), 1); + ASSERT_NE(value.get(2), nullptr); + EXPECT_EQ(value.get(2)->id(), 1); + EXPECT_EQ(TrackedValue::liveCount(), 2); + EXPECT_EQ(TrackedValue::destroyedCountFor(2), 0); +} + +TEST_F(VersionedTest, VersionedValueDeferredRevertDeletesReplacementOnBatchDestruction) { + { + VersionedValue value; + TrackedDeferredDeletion deferred; + + value.set(1, new TrackedValue(1)); + value.set(2, new TrackedValue(2)); + + value.revert(1, deferred); + + EXPECT_FALSE(deferred.empty()); + EXPECT_EQ(TrackedValue::liveCount(), 2); + EXPECT_EQ(TrackedValue::destroyedCountFor(2), 0); + } + + EXPECT_EQ(TrackedValue::liveCount(), 0); + EXPECT_EQ(TrackedValue::destroyedCountFor(1), 1); + EXPECT_EQ(TrackedValue::destroyedCountFor(2), 1); +} + +TEST_F(VersionedTest, VersionedValueGcDefersDeletionUntilBatchDestruction) { + VersionedValue value; + + value.set(1, new TrackedValue(1)); + value.set(2, new TrackedValue(2)); + + EXPECT_EQ(TrackedValue::liveCount(), 2); + + { + TrackedDeferredDeletion deferred; + EXPECT_FALSE(value.gcForVersion(1, deferred)); + + EXPECT_TRUE(deferred.empty()); + EXPECT_EQ(TrackedValue::liveCount(), 2); + EXPECT_EQ(TrackedValue::destroyedCountFor(1), 0); + ASSERT_NE(value.get(1), nullptr); + EXPECT_EQ(value.get(1)->id(), 1); + ASSERT_NE(value.get(2), nullptr); + EXPECT_EQ(value.get(2)->id(), 2); + } + + { + TrackedDeferredDeletion deferred; + EXPECT_TRUE(value.gcForVersion(2, deferred)); + + EXPECT_FALSE(deferred.empty()); + EXPECT_EQ(TrackedValue::liveCount(), 2); + EXPECT_EQ(TrackedValue::destroyedCountFor(1), 0); + ASSERT_NE(value.get(2), nullptr); + EXPECT_EQ(value.get(2)->id(), 2); + } + + EXPECT_EQ(TrackedValue::liveCount(), 1); + EXPECT_EQ(TrackedValue::destroyedCountFor(1), 1); + ASSERT_NE(value.get(2), nullptr); + EXPECT_EQ(value.get(2)->id(), 2); +} + +TEST_F(VersionedTest, DeferredDeletionMoveTransfersDeletionOwnership) { + VersionedValue value; + + value.set(1, new TrackedValue(1)); + value.set(2, new TrackedValue(2)); + + TrackedDeferredDeletion deferred; + EXPECT_TRUE(value.gcForVersion(2, deferred)); + EXPECT_FALSE(deferred.empty()); + EXPECT_EQ(TrackedValue::liveCount(), 2); + + { + TrackedDeferredDeletion moved = std::move(deferred); + EXPECT_FALSE(moved.empty()); + EXPECT_EQ(TrackedValue::liveCount(), 2); + } + + EXPECT_EQ(TrackedValue::liveCount(), 1); + EXPECT_EQ(TrackedValue::destroyedCountFor(1), 1); +} + +TEST_F(VersionedTest, FirstPhaseGcKeepsCurrentAndHistoricalLookupsMemorySafe) { + TrackedMap map; + + map.prepareNextVersion(); + auto handle = map.insert("a", new TrackedValue(1)); + const auto version1 = map.publishNextVersion(); + + map.prepareNextVersion(); + map.insert("a", new TrackedValue(2)); + const auto version2 = map.publishNextVersion(); + + ASSERT_NE(handle, nullptr); + + { + auto deferred = map.gc(version2); + EXPECT_FALSE(deferred.empty()); + EXPECT_EQ(TrackedValue::liveCount(), 2); + EXPECT_EQ(validateHandleAtVersion("a", handle, version1), ""); + EXPECT_EQ(validateHandleAtVersion("a", handle, version2), ""); + } + + EXPECT_EQ(TrackedValue::liveCount(), 1); + EXPECT_EQ(TrackedValue::destroyedCountFor(1), 1); +} + +TEST_F(VersionedTest, ConcurrentValidatorAcceptsMixedDeferredTailTraversal) { + TrackedMap map; + + map.prepareNextVersion(); + map.insert("a", new TrackedValue(1, 1)); + map.insert("b", new TrackedValue(2, 1)); + map.publishNextVersion(); + + auto handle_a = map.find("a"); + auto handle_b = map.find("b"); + ASSERT_NE(handle_a, nullptr); + ASSERT_NE(handle_b, nullptr); + + map.prepareNextVersion(); + map.insert("a", new TrackedValue(1, 2)); + map.insert("b", new TrackedValue(2, 2)); + const auto version2 = map.publishNextVersion(); + + const auto* stale_a = TrackedAccess::next(TrackedAccess::head(handle_a)); + const auto* stale_b = TrackedAccess::next(TrackedAccess::head(handle_b)); + ASSERT_NE(stale_a, nullptr); + ASSERT_NE(stale_b, nullptr); + + { + auto deferred = map.gc(version2); + EXPECT_FALSE(deferred.empty()); + + EXPECT_EQ(validateHandleAtVersion("a", handle_a, version2), ""); + EXPECT_EQ(validateHandleAtVersion("b", handle_b, version2), ""); + EXPECT_EQ(validateNodeAtVersion("stale-a", stale_a, version2), ""); + EXPECT_EQ(validateNodeAtVersion("stale-b", stale_b, version2), ""); + } +} + +TEST_F(VersionedTest, VersionedMapGcRetainsDirtyHandleUntilAllFutureRemovalsAreReclaimed) { + TrackedMap map; + + map.prepareNextVersion(); + map.insert("a", new TrackedValue(1)); + map.publishNextVersion(); + + map.prepareNextVersion(); + map.insert("a", new TrackedValue(2)); + const auto version2 = map.publishNextVersion(); + + map.prepareNextVersion(); + map.insert("a", new TrackedValue(3)); + const auto version3 = map.publishNextVersion(); + + EXPECT_EQ(TrackedValue::liveCount(), 3); + + { + auto deferred = map.gc(version2); + + EXPECT_FALSE(deferred.empty()); + EXPECT_EQ(TrackedValue::liveCount(), 3); + EXPECT_EQ(TrackedValue::destroyedCountFor(1), 0); + EXPECT_EQ(TrackedValue::destroyedCountFor(2), 0); + ASSERT_NE(activeValue(map, "a", version2), nullptr); + EXPECT_EQ(activeValue(map, "a", version2)->id(), 2); + ASSERT_NE(activeValue(map, "a", version3), nullptr); + EXPECT_EQ(activeValue(map, "a", version3)->id(), 3); + } + + EXPECT_EQ(TrackedValue::liveCount(), 2); + EXPECT_EQ(TrackedValue::destroyedCountFor(1), 1); + + { + auto deferred = map.gc(version3); + + EXPECT_FALSE(deferred.empty()); + EXPECT_EQ(TrackedValue::liveCount(), 2); + EXPECT_EQ(TrackedValue::destroyedCountFor(2), 0); + ASSERT_NE(activeValue(map, "a", version3), nullptr); + EXPECT_EQ(activeValue(map, "a", version3)->id(), 3); + } + + EXPECT_EQ(TrackedValue::liveCount(), 1); + EXPECT_EQ(TrackedValue::destroyedCountFor(2), 1); + ASSERT_NE(activeValue(map, "a", version3), nullptr); + EXPECT_EQ(activeValue(map, "a", version3)->id(), 3); +} + +TEST_F(VersionedTest, VersionedMapVersionStartsAtMin) { + TrackedMap map; + + EXPECT_EQ(map.getVersion(), versionMin); +} + +TEST_F(VersionedTest, VersionedMapPublishWithNoPendingChangesReturnsZero) { + TrackedMap map; + + EXPECT_EQ(map.publishNextVersion(), 0); + EXPECT_EQ(map.getVersion(), versionMin); + + map.prepareNextVersion(); + + EXPECT_EQ(map.publishNextVersion(), 0); + EXPECT_EQ(map.getVersion(), versionMin); +} + +TEST_F(VersionedTest, VersionedMapInsertNewKeyThenRevertRemovesItEntirely) { + TrackedMap map; + + map.prepareNextVersion(); + map.insert("a", new TrackedValue(1)); + + ASSERT_NE(activeValue(map, "a", 1), nullptr); + + auto deferred = map.revert(); + + EXPECT_FALSE(deferred.empty()); + EXPECT_EQ(TrackedValue::liveCount(), 1); + EXPECT_EQ(TrackedValue::destroyedCountFor(1), 0); + EXPECT_EQ(map.find("a"), nullptr); +} + +TEST_F(VersionedTest, VersionedMapDeferredRevertDeletesNewKeyAfterBatchDestruction) { + { + TrackedMap map; + + map.prepareNextVersion(); + map.insert("a", new TrackedValue(1)); + auto deferred = map.revert(); + + EXPECT_FALSE(deferred.empty()); + EXPECT_EQ(map.find("a"), nullptr); + EXPECT_EQ(TrackedValue::liveCount(), 1); + EXPECT_EQ(TrackedValue::destroyedCountFor(1), 0); + } + + EXPECT_EQ(TrackedValue::liveCount(), 0); + EXPECT_EQ(TrackedValue::destroyedCountFor(1), 1); +} + +TEST_F(VersionedTest, VersionedMapUpdateExistingKeyAndRevertRestoresPublishedValue) { + TrackedMap map; + + map.prepareNextVersion(); + map.insert("a", new TrackedValue(1)); + const auto published_version = map.publishNextVersion(); + const auto version1 = map.getVersion(); + map.gc(published_version); + + map.prepareNextVersion(); + map.insert("a", new TrackedValue(2)); + + auto deferred = map.revert(); + + EXPECT_FALSE(deferred.empty()); + EXPECT_EQ(TrackedValue::liveCount(), 2); + EXPECT_EQ(TrackedValue::destroyedCountFor(2), 0); + EXPECT_EQ(map.getVersion(), version1); + ASSERT_NE(activeValue(map, "a", version1), nullptr); + EXPECT_EQ(activeValue(map, "a", version1)->id(), 1); + ASSERT_NE(activeValue(map, "a"), nullptr); + EXPECT_EQ(activeValue(map, "a")->id(), 1); +} + +TEST_F(VersionedTest, VersionedMapDeferredRevertDeletesReplacementAfterBatchDestruction) { + { + TrackedMap map; + + map.prepareNextVersion(); + map.insert("a", new TrackedValue(1)); + const auto published_version = map.publishNextVersion(); + map.gc(published_version); + + map.prepareNextVersion(); + map.insert("a", new TrackedValue(2)); + auto deferred = map.revert(); + + EXPECT_FALSE(deferred.empty()); + ASSERT_NE(activeValue(map, "a"), nullptr); + EXPECT_EQ(activeValue(map, "a")->id(), 1); + EXPECT_EQ(TrackedValue::liveCount(), 2); + EXPECT_EQ(TrackedValue::destroyedCountFor(2), 0); + } + + EXPECT_EQ(TrackedValue::liveCount(), 0); + EXPECT_EQ(TrackedValue::destroyedCountFor(1), 1); + EXPECT_EQ(TrackedValue::destroyedCountFor(2), 1); +} + +TEST_F(VersionedTest, VersionedMapClearExistingKeyAndRevertRestoresPublishedValue) { + TrackedMap map; + + map.prepareNextVersion(); + map.insert("a", new TrackedValue(1)); + const auto published_version = map.publishNextVersion(); + const auto version1 = map.getVersion(); + map.gc(published_version); + + map.prepareNextVersion(); + map.clear("a"); + + EXPECT_EQ(activeValue(map, "a"), nullptr); + + auto deferred = map.revert(); + + EXPECT_TRUE(deferred.empty()); + ASSERT_NE(activeValue(map, "a", version1), nullptr); + EXPECT_EQ(activeValue(map, "a", version1)->id(), 1); + ASSERT_NE(activeValue(map, "a"), nullptr); + EXPECT_EQ(activeValue(map, "a")->id(), 1); +} + +TEST_F(VersionedTest, VersionedMapClearMissingKeyIsNoOp) { + TrackedMap map; + + map.prepareNextVersion(); + map.clear("missing"); + + EXPECT_EQ(map.publishNextVersion(), 0); + EXPECT_EQ(map.getVersion(), versionMin); + EXPECT_EQ(TrackedValue::liveCount(), 0); +} + +TEST_F(VersionedTest, VersionedMapSnapshotsIsolateKeysAndVersions) { + TrackedMap map; + + map.prepareNextVersion(); + map.insert("a", new TrackedValue(1)); + map.insert("b", new TrackedValue(10)); + auto published_version = map.publishNextVersion(); + const auto version1 = map.getVersion(); + map.gc(published_version); + + map.prepareNextVersion(); + map.insert("a", new TrackedValue(2)); + published_version = map.publishNextVersion(); + const auto version2 = map.getVersion(); + + ASSERT_NE(activeValue(map, "a", version1), nullptr); + EXPECT_EQ(activeValue(map, "a", version1)->id(), 1); + ASSERT_NE(activeValue(map, "a", version2), nullptr); + EXPECT_EQ(activeValue(map, "a", version2)->id(), 2); + ASSERT_NE(activeValue(map, "b", version1), nullptr); + EXPECT_EQ(activeValue(map, "b", version1)->id(), 10); + ASSERT_NE(activeValue(map, "b", version2), nullptr); + EXPECT_EQ(activeValue(map, "b", version2)->id(), 10); + + map.gc(published_version); +} + +TEST_F(VersionedTest, VersionedMapActiveUpdateReusesStableHandle) { + TrackedMap map; + + map.prepareNextVersion(); + auto handle = map.insert("a", new TrackedValue(1)); + const auto version1 = map.publishNextVersion(); + + ASSERT_NE(handle, nullptr); + ASSERT_EQ(map.find("a"), handle); + ASSERT_NE(handle->get(version1), nullptr); + EXPECT_EQ(handle->get(version1)->id(), 1); + + map.prepareNextVersion(); + map.insert("a", new TrackedValue(2)); + const auto version2 = map.publishNextVersion(); + + ASSERT_EQ(map.find("a"), handle); + ASSERT_NE(handle->get(version1), nullptr); + EXPECT_EQ(handle->get(version1)->id(), 1); + ASSERT_NE(handle->get(version2), nullptr); + EXPECT_EQ(handle->get(version2)->id(), 2); + + map.gc(version2); +} + +TEST_F(VersionedTest, VersionedMapClearAndReinsertSameKeyBeforePublishReusesStableHandle) { + TrackedMap map; + + map.prepareNextVersion(); + auto handle = map.insert("a", new TrackedValue(1)); + const auto version1 = map.publishNextVersion(); + + ASSERT_NE(handle, nullptr); + ASSERT_EQ(map.find("a"), handle); + ASSERT_NE(handle->get(version1), nullptr); + EXPECT_EQ(handle->get(version1)->id(), 1); + + map.prepareNextVersion(); + map.clear("a"); + map.insert("a", new TrackedValue(2)); + const auto version2 = map.publishNextVersion(); + + ASSERT_EQ(map.find("a"), handle); + ASSERT_NE(handle->get(version1), nullptr); + EXPECT_EQ(handle->get(version1)->id(), 1); + ASSERT_NE(handle->get(version2), nullptr); + EXPECT_EQ(handle->get(version2)->id(), 2); + + map.gc(version2); +} + +TEST_F(VersionedTest, VersionedMapRemovedHandleStaysEmptyAfterSameNameReAdd) { + TrackedMap map; + + map.prepareNextVersion(); + map.insert("a", new TrackedValue(1)); + map.publishNextVersion(); + + auto old_handle = map.find("a"); + ASSERT_NE(old_handle, nullptr); + + map.prepareNextVersion(); + map.clear("a"); + const auto version2 = map.publishNextVersion(); + map.gc(version2); + + EXPECT_EQ(map.find("a"), nullptr); + EXPECT_EQ(old_handle->get(version2), nullptr); + + map.prepareNextVersion(); + map.insert("a", new TrackedValue(2)); + const auto version3 = map.publishNextVersion(); + + auto new_handle = map.find("a"); + ASSERT_NE(new_handle, nullptr); + EXPECT_NE(new_handle, old_handle); + ASSERT_NE(new_handle->get(version3), nullptr); + EXPECT_EQ(new_handle->get(version3)->id(), 2); + EXPECT_EQ(old_handle->get(version3), nullptr); + map.gc(version3); +} + +TEST_F(VersionedTest, VersionedMapValidatorsAcceptStablePublishedSnapshots) { + TrackedMap map; + std::array generations{}; + + auto next_value = [&](int key_id) { return new TrackedValue(key_id, ++generations[key_id]); }; + auto expect_snapshot_valid = [&](const std::string& phase) { + auto snapshot = makePublishedHandles(map); + EXPECT_EQ(validatePublishedSnapshot(*snapshot), "") << phase; + }; + + map.prepareNextVersion(); + map.insert("key-10", next_value(10)); + map.insert("key-11", next_value(11)); + const auto version1 = map.publishNextVersion(); + ASSERT_EQ(version1, 1); + expect_snapshot_valid("after initial publish"); + + // Exercise same-version churn on one handle: update, clear, and re-add in one transaction. + map.prepareNextVersion(); + map.insert("key-10", next_value(10)); // generation 2 + map.clear("key-10"); + map.insert("key-10", next_value(10)); // generation 3 + const auto version2 = map.publishNextVersion(); + ASSERT_EQ(version2, 2); + expect_snapshot_valid("after same-version clear and re-add publish"); + + // Exercise both the post-unlink/pre-deletion state and the stable post-deletion state. + { + auto deferred = map.gc(version2); + EXPECT_FALSE(deferred.empty()); + expect_snapshot_valid("after first-phase gc of same-version churn publish"); + } + expect_snapshot_valid("after deferred deletion of same-version churn publish"); + + map.prepareNextVersion(); + map.clear("key-11"); + map.insert("key-12", next_value(12)); + const auto version3 = map.publishNextVersion(); + ASSERT_EQ(version3, 3); + expect_snapshot_valid("after mixed clear and insert publish"); + + { + auto deferred = map.gc(version3); + expect_snapshot_valid("after first-phase final gc"); + } + expect_snapshot_valid("after final deferred deletion"); +} + +TEST_F(VersionedTest, VersionedMapChaosConcurrentReadersAndWriter) { + TrackedMap map; + + constexpr size_t k_reader_count = 4; + constexpr int k_key_count = 32; + constexpr int k_initial_key_count = 16; + constexpr int k_max_ops_per_transaction = 8; + constexpr int k_transaction_count = 2000; + constexpr uint32_t k_seed = 1337; + + std::array key_names; + std::array generations{}; + for (int i = 0; i < k_key_count; ++i) { + key_names[i] = "key-" + std::to_string(i); + } + + auto next_value = [&](int key_id) { return new TrackedValue(key_id, ++generations[key_id]); }; + + map.prepareNextVersion(); + for (int i = 0; i < k_initial_key_count; ++i) { + map.insert(key_names[i], next_value(i)); + } + auto published_version = map.publishNextVersion(); + ASSERT_GT(published_version, 0); + + uint64_t next_snapshot_id = 1; + std::shared_ptr published_snapshot_owner = + makePublishedHandles(map, next_snapshot_id++); + std::shared_ptr previous_snapshot_owner; + std::atomic published_snapshot{published_snapshot_owner.get()}; + std::array, k_reader_count> reader_quiesced_versions; + std::array, k_reader_count> reader_snapshot_ids; + for (auto& reader_version : reader_quiesced_versions) { + reader_version.store(versionMin, std::memory_order_relaxed); + } + for (auto& reader_snapshot_id : reader_snapshot_ids) { + reader_snapshot_id.store(0, std::memory_order_relaxed); + } + + std::atomic stop{false}; + std::atomic failed{false}; + Envoy::Thread::MutexBasicLockable failure_mutex; + std::string failure_message; + Envoy::Thread::MutexBasicLockable history_mutex; + std::vector recent_history; + + auto record_failure = [&](const std::string& message) { + bool expected = false; + if (failed.compare_exchange_strong(expected, true, std::memory_order_acq_rel)) { + Envoy::Thread::LockGuard lock(failure_mutex); + failure_message = message; + } + }; + + auto record_history = [&](std::string message) { + Envoy::Thread::LockGuard lock(history_mutex); + recent_history.push_back(std::move(message)); + constexpr size_t k_history_limit = 128; + if (recent_history.size() > k_history_limit) { + recent_history.erase(recent_history.begin(), + recent_history.begin() + (recent_history.size() - k_history_limit)); + } + }; + + std::vector readers; + readers.reserve(k_reader_count); + for (size_t i = 0; i < k_reader_count; ++i) { + readers.emplace_back([&, i]() { + while (!stop.load(std::memory_order_acquire)) { + const auto* snapshot = published_snapshot.load(std::memory_order_acquire); + if (snapshot == nullptr) { + continue; + } + if (const auto error = validatePublishedSnapshotAtVersion( + *snapshot, snapshot->version, ValidationMode::ConcurrentSafeInvisibleTail); + !error.empty()) { + record_failure("reader " + std::to_string(i) + + " snapshot_id=" + std::to_string(snapshot->snapshot_id) + ": " + error); + break; + } + reader_quiesced_versions[i].store(snapshot->version, std::memory_order_release); + reader_snapshot_ids[i].store(snapshot->snapshot_id, std::memory_order_release); + } + }); + } + + auto publish_snapshot = [&]() { + previous_snapshot_owner = std::move(published_snapshot_owner); + published_snapshot_owner = makePublishedHandles(map, next_snapshot_id++); + published_snapshot.store(published_snapshot_owner.get(), std::memory_order_release); + return published_snapshot_owner; + }; + + auto release_previous_snapshot = [&]() { previous_snapshot_owner.reset(); }; + + auto wait_for_readers_to_observe_snapshot = + [&](const std::shared_ptr& snapshot) { + while (!failed.load(std::memory_order_acquire)) { + bool all_quiesced = true; + for (size_t i = 0; i < k_reader_count; ++i) { + if (reader_quiesced_versions[i].load(std::memory_order_acquire) < snapshot->version || + reader_snapshot_ids[i].load(std::memory_order_acquire) < snapshot->snapshot_id) { + all_quiesced = false; + break; + } + } + if (all_quiesced) { + return true; + } + std::this_thread::yield(); + } + return false; + }; + + auto run_deferred_gc = [&](uint64_t version) { + record_history("gc phase1 " + std::to_string(version)); + auto deferred = map.gc(version); + if (deferred.empty()) { + return true; + } + + const auto post_gc_snapshot = publish_snapshot(); + if (!wait_for_readers_to_observe_snapshot(post_gc_snapshot)) { + return false; + } + release_previous_snapshot(); + record_history("gc phase2 " + std::to_string(version)); + return true; + }; + + auto build_active_set = [&]() { + std::array active{}; + for (int i = 0; i < k_key_count; ++i) { + active[i] = map.find(key_names[i]) != nullptr; + } + return active; + }; + + auto random_matching_key = [&](std::minstd_rand& rng, const std::array& active, + bool want_active) { + std::vector candidates; + candidates.reserve(k_key_count); + for (int i = 0; i < k_key_count; ++i) { + if (active[i] == want_active) { + candidates.push_back(i); + } + } + if (candidates.empty()) { + return -1; + } + return candidates[rng() % candidates.size()]; + }; + + std::minstd_rand rng(k_seed); + uint64_t transaction_index = 0; + while (transaction_index < k_transaction_count && !failed.load(std::memory_order_acquire)) { + auto active = build_active_set(); + const auto next_version = map.prepareNextVersion(); + ++transaction_index; + record_history("tx " + std::to_string(transaction_index) + + " prepare next=" + std::to_string(next_version)); + + const int op_count = 1 + (rng() % k_max_ops_per_transaction); + for (int op_index = 0; op_index < op_count; ++op_index) { + const bool has_active = std::ranges::any_of(active, [](bool is_active) { return is_active; }); + const bool has_absent = + std::ranges::any_of(active, [](bool is_active) { return !is_active; }); + + std::vector available_ops; + if (has_active) { + available_ops.push_back(0); // update existing + available_ops.push_back(1); // clear existing + available_ops.push_back(3); // clear and re-add + } + if (has_absent) { + available_ops.push_back(2); // insert absent + } + ASSERT_FALSE(available_ops.empty()); + + switch (available_ops[rng() % available_ops.size()]) { + case 0: { + const int key_id = random_matching_key(rng, active, true); + ASSERT_GE(key_id, 0); + record_history("tx " + std::to_string(transaction_index) + " op " + + std::to_string(op_index) + ": update " + key_names[key_id] + " -> gen " + + std::to_string(generations[key_id] + 1)); + map.insert(key_names[key_id], next_value(key_id)); + break; + } + case 1: { + const int key_id = random_matching_key(rng, active, true); + ASSERT_GE(key_id, 0); + record_history("tx " + std::to_string(transaction_index) + " op " + + std::to_string(op_index) + ": clear " + key_names[key_id]); + map.clear(key_names[key_id]); + active[key_id] = false; + break; + } + case 2: { + const int key_id = random_matching_key(rng, active, false); + ASSERT_GE(key_id, 0); + record_history("tx " + std::to_string(transaction_index) + " op " + + std::to_string(op_index) + ": insert " + key_names[key_id] + " -> gen " + + std::to_string(generations[key_id] + 1)); + map.insert(key_names[key_id], next_value(key_id)); + active[key_id] = true; + break; + } + case 3: { + const int key_id = random_matching_key(rng, active, true); + ASSERT_GE(key_id, 0); + record_history("tx " + std::to_string(transaction_index) + " op " + + std::to_string(op_index) + ": clear+insert " + key_names[key_id] + + " -> gen " + std::to_string(generations[key_id] + 1)); + map.clear(key_names[key_id]); + map.insert(key_names[key_id], next_value(key_id)); + active[key_id] = true; + break; + } + default: + FAIL() << "unexpected operation"; + } + } + + if ((rng() % 5) == 0) { + record_history("tx " + std::to_string(transaction_index) + " revert"); + auto deferred = map.revert(); + if (!deferred.empty()) { + const auto reverted_snapshot = publish_snapshot(); + if (!wait_for_readers_to_observe_snapshot(reverted_snapshot)) { + break; + } + record_history("revert deferred delete"); + } + continue; + } + + published_version = map.publishNextVersion(); + record_history("tx " + std::to_string(transaction_index) + " publish -> " + + std::to_string(published_version)); + if (published_version == 0) { + continue; + } + + const auto next_snapshot = publish_snapshot(); + if (!wait_for_readers_to_observe_snapshot(next_snapshot)) { + break; + } + release_previous_snapshot(); + if (!run_deferred_gc(published_version)) { + break; + } + } + + if (!failed.load(std::memory_order_acquire)) { + const auto final_version = map.getVersion(); + const auto final_snapshot = publish_snapshot(); + if (final_version > versionMin) { + EXPECT_TRUE(wait_for_readers_to_observe_snapshot(final_snapshot)); + release_previous_snapshot(); + if (!failed.load(std::memory_order_acquire)) { + EXPECT_TRUE(run_deferred_gc(final_version)); + } + } + } + + stop.store(true, std::memory_order_release); + for (auto& reader : readers) { + reader.join(); + } + published_snapshot.store(nullptr, std::memory_order_release); + previous_snapshot_owner.reset(); + published_snapshot_owner.reset(); + + if (failed.load(std::memory_order_acquire)) { + Envoy::Thread::LockGuard lock(failure_mutex); + std::ostringstream history; + { + Envoy::Thread::LockGuard history_lock(history_mutex); + for (const auto& entry : recent_history) { + history << "\n " << entry; + } + } + FAIL() << "seed=" << k_seed << " " << failure_message << "\nrecent history:" << history.str(); + } +} + +TEST_F(VersionedTest, VersionedMapGcAndDestructorDeleteValuesExactlyOnce) { + { + TrackedMap map; + + map.prepareNextVersion(); + map.insert("a", new TrackedValue(1)); + auto published_version = map.publishNextVersion(); + map.gc(published_version); + + map.prepareNextVersion(); + map.insert("a", new TrackedValue(2)); + published_version = map.publishNextVersion(); + + EXPECT_EQ(TrackedValue::liveCount(), 2); + + { + auto deferred = map.gc(published_version); + EXPECT_FALSE(deferred.empty()); + EXPECT_EQ(TrackedValue::liveCount(), 2); + EXPECT_EQ(TrackedValue::destroyedCountFor(1), 0); + EXPECT_EQ(TrackedValue::destroyedCountFor(2), 0); + } + + EXPECT_EQ(TrackedValue::liveCount(), 1); + EXPECT_EQ(TrackedValue::destroyedCountFor(1), 1); + EXPECT_EQ(TrackedValue::destroyedCountFor(2), 0); + } + + EXPECT_EQ(TrackedValue::liveCount(), 0); + EXPECT_EQ(TrackedValue::destroyedCountFor(1), 1); + EXPECT_EQ(TrackedValue::destroyedCountFor(2), 1); +} + +} // namespace