From cbc9a2f50341012648c38914a84047cc180605c3 Mon Sep 17 00:00:00 2001 From: yanavlasov Date: Tue, 6 Dec 2022 13:01:57 -0500 Subject: [PATCH 01/23] Update libcircllhist to 39f9db7 2019-05-21 (#24374) Signed-off-by: Yan Avlasov --- bazel/repository_locations.bzl | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/bazel/repository_locations.bzl b/bazel/repository_locations.bzl index bd800d785515..a875f9e0a133 100644 --- a/bazel/repository_locations.bzl +++ b/bazel/repository_locations.bzl @@ -220,12 +220,12 @@ REPOSITORY_LOCATIONS_SPEC = dict( project_name = "libcircllhist", project_desc = "An implementation of Circonus log-linear histograms", project_url = "https://github.com/circonus-labs/libcircllhist", - version = "63a16dd6f2fc7bc841bb17ff92be8318df60e2e1", - sha256 = "8165aa25e529d7d4b9ae849d3bf30371255a99d6db0421516abcff23214cdc2c", + version = "39f9db724a81ba78f5d037f1cae79c5a07107c8e", + sha256 = "fd2492f6cc1f8734f8f57be8c2e7f2907e94ee2a4c02445ce59c4241fece144b", strip_prefix = "libcircllhist-{version}", urls = ["https://github.com/circonus-labs/libcircllhist/archive/{version}.tar.gz"], use_category = ["controlplane", "observability_core", "dataplane_core"], - release_date = "2019-02-11", + release_date = "2019-05-21", cpe = "N/A", license = "Apache-2.0", license_url = "https://github.com/circonus-labs/libcircllhist/blob/{version}/LICENSE", From 8d6c418dc95db6aff221b5447ca5d299cd2a7ec0 Mon Sep 17 00:00:00 2001 From: Chris Schoener Date: Tue, 6 Dec 2022 14:19:01 -0500 Subject: [PATCH 02/23] Fix mobile/vscode_compdb script for repo merge (#24390) This script is still necessary for now because mobile can only be built from the mobile directory due to bazelrc magic. Let's fix the script in-place for now and hopefully clean it up later once mobile can be built from the top-level directory. Verified that it runs successfully and seems to make vscode happy. Signed-off-by: caschoener --- mobile/tools/vscode_compdb.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mobile/tools/vscode_compdb.sh b/mobile/tools/vscode_compdb.sh index 485fd1ccf262..54dbb7ace83b 100755 --- a/mobile/tools/vscode_compdb.sh +++ b/mobile/tools/vscode_compdb.sh @@ -5,7 +5,7 @@ # the correct envoy-mobile Bazel targets. # Setting TEST_TMPDIR here so the compdb headers won't be overwritten by another bazel run -CC=clang TEST_TMPDIR=${BUILD_DIR:-/tmp}/envoy-mobile-compdb envoy/tools/gen_compilation_database.py --vscode --bazel ./bazelw //library/cc/... //library/common/... //test/cc/... //test/common/... +CC=clang TEST_TMPDIR=${BUILD_DIR:-/tmp}/envoy-mobile-compdb ../tools/gen_compilation_database.py --vscode --bazel ./bazelw //library/cc/... //library/common/... //test/cc/... //test/common/... # Kill clangd to reload the compilation database pkill clangd || : From 40017d7c1829972bdaa88a962c5a07f12a801b8a Mon Sep 17 00:00:00 2001 From: Adam Kotwasinski Date: Tue, 6 Dec 2022 11:49:19 -0800 Subject: [PATCH 03/23] kafka: add support for list-offsets request (#24375) Signed-off-by: Adam Kotwasinski --- .../kafka/filters/network/source/mesh/BUILD | 1 + .../source/mesh/command_handlers/BUILD | 17 ++++++ .../mesh/command_handlers/api_versions.cc | 9 +++- .../mesh/command_handlers/list_offsets.cc | 52 +++++++++++++++++++ .../mesh/command_handlers/list_offsets.h | 32 ++++++++++++ .../network/source/mesh/request_processor.cc | 9 ++++ .../network/source/mesh/request_processor.h | 1 + .../network/test/mesh/command_handlers/BUILD | 11 ++++ .../list_offsets_unit_test.cc | 46 ++++++++++++++++ .../test/mesh/request_processor_unit_test.cc | 23 ++++++-- 10 files changed, 197 insertions(+), 4 deletions(-) create mode 100644 contrib/kafka/filters/network/source/mesh/command_handlers/list_offsets.cc create mode 100644 contrib/kafka/filters/network/source/mesh/command_handlers/list_offsets.h create mode 100644 contrib/kafka/filters/network/test/mesh/command_handlers/list_offsets_unit_test.cc diff --git a/contrib/kafka/filters/network/source/mesh/BUILD b/contrib/kafka/filters/network/source/mesh/BUILD index b8ad3d39eecf..2de408fd47a8 100644 --- a/contrib/kafka/filters/network/source/mesh/BUILD +++ b/contrib/kafka/filters/network/source/mesh/BUILD @@ -69,6 +69,7 @@ envoy_cc_library( "//contrib/kafka/filters/network/source:kafka_request_codec_lib", "//contrib/kafka/filters/network/source:kafka_request_parser_lib", "//contrib/kafka/filters/network/source/mesh/command_handlers:api_versions_lib", + "//contrib/kafka/filters/network/source/mesh/command_handlers:list_offsets_lib", "//contrib/kafka/filters/network/source/mesh/command_handlers:metadata_lib", "//contrib/kafka/filters/network/source/mesh/command_handlers:produce_lib", "//source/common/common:minimal_logger_lib", diff --git a/contrib/kafka/filters/network/source/mesh/command_handlers/BUILD b/contrib/kafka/filters/network/source/mesh/command_handlers/BUILD index 3a1d58d6320a..be021a214a3a 100644 --- a/contrib/kafka/filters/network/source/mesh/command_handlers/BUILD +++ b/contrib/kafka/filters/network/source/mesh/command_handlers/BUILD @@ -45,6 +45,23 @@ envoy_cc_library( ], ) +envoy_cc_library( + name = "list_offsets_lib", + srcs = [ + "list_offsets.cc", + ], + hdrs = [ + "list_offsets.h", + ], + tags = ["skip_on_windows"], + deps = [ + "//contrib/kafka/filters/network/source:kafka_request_parser_lib", + "//contrib/kafka/filters/network/source:kafka_response_parser_lib", + "//contrib/kafka/filters/network/source/mesh:abstract_command_lib", + "//source/common/common:minimal_logger_lib", + ], +) + envoy_cc_library( name = "metadata_lib", srcs = [ diff --git a/contrib/kafka/filters/network/source/mesh/command_handlers/api_versions.cc b/contrib/kafka/filters/network/source/mesh/command_handlers/api_versions.cc index 31cb53f12a40..3f50ed73de38 100644 --- a/contrib/kafka/filters/network/source/mesh/command_handlers/api_versions.cc +++ b/contrib/kafka/filters/network/source/mesh/command_handlers/api_versions.cc @@ -15,6 +15,10 @@ namespace Mesh { // going to handle only produce requests with versions higher than 5. constexpr int16_t MIN_PRODUCE_SUPPORTED = 5; constexpr int16_t MAX_PRODUCE_SUPPORTED = PRODUCE_REQUEST_MAX_VERSION; /* Generated value. */ +// List-offsets version 0 uses old-style offsets. +constexpr int16_t MIN_LIST_OFFSETS_SUPPORTED = 1; +constexpr int16_t MAX_LIST_OFFSETS_SUPPORTED = + LIST_OFFSETS_REQUEST_MAX_VERSION; /* Generated value. */ // Right now we do not want to handle old version 0 request, as it expects us to enumerate all // topics if list of requested topics is empty. // Impl note: if filter gains knowledge of upstream cluster topics (e.g. thru admin clients), we @@ -40,9 +44,12 @@ AbstractResponseSharedPtr ApiVersionsRequestHolder::computeAnswer() const { const int16_t error_code = 0; const ApiVersion produce_entry = {PRODUCE_REQUEST_API_KEY, MIN_PRODUCE_SUPPORTED, MAX_PRODUCE_SUPPORTED}; + const ApiVersion list_offsets_entry = {LIST_OFFSETS_REQUEST_API_KEY, MIN_LIST_OFFSETS_SUPPORTED, + MAX_LIST_OFFSETS_SUPPORTED}; const ApiVersion metadata_entry = {METADATA_REQUEST_API_KEY, MIN_METADATA_SUPPORTED, MAX_METADATA_SUPPORTED}; - const ApiVersionsResponse real_response = {error_code, {produce_entry, metadata_entry}}; + const ApiVersionsResponse real_response = {error_code, + {produce_entry, list_offsets_entry, metadata_entry}}; return std::make_shared>(metadata, real_response); } diff --git a/contrib/kafka/filters/network/source/mesh/command_handlers/list_offsets.cc b/contrib/kafka/filters/network/source/mesh/command_handlers/list_offsets.cc new file mode 100644 index 000000000000..3e06f335fe50 --- /dev/null +++ b/contrib/kafka/filters/network/source/mesh/command_handlers/list_offsets.cc @@ -0,0 +1,52 @@ +#include "contrib/kafka/filters/network/source/mesh/command_handlers/list_offsets.h" + +#include "contrib/kafka/filters/network/source/external/responses.h" + +namespace Envoy { +namespace Extensions { +namespace NetworkFilters { +namespace Kafka { +namespace Mesh { + +ListOffsetsRequestHolder::ListOffsetsRequestHolder( + AbstractRequestListener& filter, const std::shared_ptr> request) + : BaseInFlightRequest{filter}, request_{request} {} + +void ListOffsetsRequestHolder::startProcessing() { notifyFilter(); } + +bool ListOffsetsRequestHolder::finished() const { return true; } + +AbstractResponseSharedPtr ListOffsetsRequestHolder::computeAnswer() const { + const auto& header = request_->request_header_; + const ResponseMetadata metadata = {header.api_key_, header.api_version_, header.correlation_id_}; + + // The response contains all the requested topics (we do not do any filtering here). + const auto& topics = request_->data_.topics_; + std::vector topic_responses; + topic_responses.reserve(topics.size()); + for (const auto& topic : topics) { + const auto& partitions = topic.partitions_; + std::vector partition_responses; + partition_responses.reserve(partitions.size()); + for (const auto& partition : partitions) { + const int16_t error_code = 0; + const int64_t timestamp = 0; + /* As we are going to ignore consumer offset requests, we can reply with dummy values. */ + const int64_t offset = 0; + const ListOffsetsPartitionResponse partition_response = {partition.partition_index_, + error_code, timestamp, offset}; + partition_responses.push_back(partition_response); + } + const ListOffsetsTopicResponse topic_response = {topic.name_, partition_responses}; + topic_responses.push_back(topic_response); + } + + const ListOffsetsResponse data = {topic_responses}; + return std::make_shared>(metadata, data); +} + +} // namespace Mesh +} // namespace Kafka +} // namespace NetworkFilters +} // namespace Extensions +} // namespace Envoy diff --git a/contrib/kafka/filters/network/source/mesh/command_handlers/list_offsets.h b/contrib/kafka/filters/network/source/mesh/command_handlers/list_offsets.h new file mode 100644 index 000000000000..26a50197813b --- /dev/null +++ b/contrib/kafka/filters/network/source/mesh/command_handlers/list_offsets.h @@ -0,0 +1,32 @@ +#pragma once + +#include "contrib/kafka/filters/network/source/external/requests.h" +#include "contrib/kafka/filters/network/source/mesh/abstract_command.h" + +namespace Envoy { +namespace Extensions { +namespace NetworkFilters { +namespace Kafka { +namespace Mesh { + +class ListOffsetsRequestHolder : public BaseInFlightRequest { +public: + ListOffsetsRequestHolder(AbstractRequestListener& filter, + const std::shared_ptr> request); + + void startProcessing() override; + + bool finished() const override; + + AbstractResponseSharedPtr computeAnswer() const override; + +private: + // Original request. + const std::shared_ptr> request_; +}; + +} // namespace Mesh +} // namespace Kafka +} // namespace NetworkFilters +} // namespace Extensions +} // namespace Envoy diff --git a/contrib/kafka/filters/network/source/mesh/request_processor.cc b/contrib/kafka/filters/network/source/mesh/request_processor.cc index 0391cdaadcf6..6e47e3ede536 100644 --- a/contrib/kafka/filters/network/source/mesh/request_processor.cc +++ b/contrib/kafka/filters/network/source/mesh/request_processor.cc @@ -3,6 +3,7 @@ #include "envoy/common/exception.h" #include "contrib/kafka/filters/network/source/mesh/command_handlers/api_versions.h" +#include "contrib/kafka/filters/network/source/mesh/command_handlers/list_offsets.h" #include "contrib/kafka/filters/network/source/mesh/command_handlers/metadata.h" #include "contrib/kafka/filters/network/source/mesh/command_handlers/produce.h" @@ -29,6 +30,9 @@ void RequestProcessor::onMessage(AbstractRequestSharedPtr arg) { case PRODUCE_REQUEST_API_KEY: process(std::dynamic_pointer_cast>(arg)); break; + case LIST_OFFSETS_REQUEST_API_KEY: + process(std::dynamic_pointer_cast>(arg)); + break; case METADATA_REQUEST_API_KEY: process(std::dynamic_pointer_cast>(arg)); break; @@ -47,6 +51,11 @@ void RequestProcessor::process(const std::shared_ptr> re origin_.onRequest(res); } +void RequestProcessor::process(const std::shared_ptr> request) const { + auto res = std::make_shared(origin_, request); + origin_.onRequest(res); +} + void RequestProcessor::process(const std::shared_ptr> request) const { auto res = std::make_shared(origin_, configuration_, request); origin_.onRequest(res); diff --git a/contrib/kafka/filters/network/source/mesh/request_processor.h b/contrib/kafka/filters/network/source/mesh/request_processor.h index b21a69ac2248..515a55ee6412 100644 --- a/contrib/kafka/filters/network/source/mesh/request_processor.h +++ b/contrib/kafka/filters/network/source/mesh/request_processor.h @@ -28,6 +28,7 @@ class RequestProcessor : public RequestCallback, private Logger::Loggable> request) const; + void process(const std::shared_ptr> request) const; void process(const std::shared_ptr> request) const; void process(const std::shared_ptr> request) const; diff --git a/contrib/kafka/filters/network/test/mesh/command_handlers/BUILD b/contrib/kafka/filters/network/test/mesh/command_handlers/BUILD index 18b75f4206f5..b65189eeff7b 100644 --- a/contrib/kafka/filters/network/test/mesh/command_handlers/BUILD +++ b/contrib/kafka/filters/network/test/mesh/command_handlers/BUILD @@ -28,6 +28,17 @@ envoy_cc_test( ], ) +envoy_cc_test( + name = "list_offsets_unit_test", + srcs = ["list_offsets_unit_test.cc"], + tags = ["skip_on_windows"], + deps = [ + "//contrib/kafka/filters/network/source/mesh/command_handlers:list_offsets_lib", + "//test/mocks/network:network_mocks", + "//test/mocks/stats:stats_mocks", + ], +) + envoy_cc_test( name = "metadata_unit_test", srcs = ["metadata_unit_test.cc"], diff --git a/contrib/kafka/filters/network/test/mesh/command_handlers/list_offsets_unit_test.cc b/contrib/kafka/filters/network/test/mesh/command_handlers/list_offsets_unit_test.cc new file mode 100644 index 000000000000..dc5bbd25ae19 --- /dev/null +++ b/contrib/kafka/filters/network/test/mesh/command_handlers/list_offsets_unit_test.cc @@ -0,0 +1,46 @@ +#include "contrib/kafka/filters/network/source/mesh/command_handlers/list_offsets.h" +#include "gmock/gmock.h" +#include "gtest/gtest.h" + +namespace Envoy { +namespace Extensions { +namespace NetworkFilters { +namespace Kafka { +namespace Mesh { +namespace { + +class MockAbstractRequestListener : public AbstractRequestListener { +public: + MOCK_METHOD(void, onRequest, (InFlightRequestSharedPtr)); + MOCK_METHOD(void, onRequestReadyForAnswer, ()); +}; + +TEST(ListOffsetsTest, shouldBeAlwaysReadyForAnswer) { + // given + MockAbstractRequestListener filter; + EXPECT_CALL(filter, onRequestReadyForAnswer()); + const RequestHeader header = {LIST_OFFSETS_REQUEST_API_KEY, 0, 0, absl::nullopt}; + const ListOffsetsRequest data = {0, {}}; + const auto message = std::make_shared>(header, data); + ListOffsetsRequestHolder testee = {filter, message}; + + // when, then - invoking should immediately notify the filter. + testee.startProcessing(); + + // when, then - should always be considered finished. + const bool finished = testee.finished(); + EXPECT_TRUE(finished); + + // when, then - the computed result is always contains correct data (confirmed by integration + // tests). + const auto answer = testee.computeAnswer(); + EXPECT_EQ(answer->metadata_.api_key_, header.api_key_); + EXPECT_EQ(answer->metadata_.correlation_id_, header.correlation_id_); +} + +} // namespace +} // namespace Mesh +} // namespace Kafka +} // namespace NetworkFilters +} // namespace Extensions +} // namespace Envoy diff --git a/contrib/kafka/filters/network/test/mesh/request_processor_unit_test.cc b/contrib/kafka/filters/network/test/mesh/request_processor_unit_test.cc index c49b449138c5..e01534a1fbe4 100644 --- a/contrib/kafka/filters/network/test/mesh/request_processor_unit_test.cc +++ b/contrib/kafka/filters/network/test/mesh/request_processor_unit_test.cc @@ -2,6 +2,7 @@ #include "contrib/kafka/filters/network/source/mesh/abstract_command.h" #include "contrib/kafka/filters/network/source/mesh/command_handlers/api_versions.h" +#include "contrib/kafka/filters/network/source/mesh/command_handlers/list_offsets.h" #include "contrib/kafka/filters/network/source/mesh/command_handlers/metadata.h" #include "contrib/kafka/filters/network/source/mesh/command_handlers/produce.h" #include "contrib/kafka/filters/network/source/mesh/request_processor.h" @@ -60,6 +61,22 @@ TEST_F(RequestProcessorTest, ShouldProcessProduceRequest) { ASSERT_NE(std::dynamic_pointer_cast(capture), nullptr); } +TEST_F(RequestProcessorTest, ShouldProcessListOffsetsRequest) { + // given + const RequestHeader header = {LIST_OFFSETS_REQUEST_API_KEY, 0, 0, absl::nullopt}; + const ListOffsetsRequest data = {0, {}}; + const auto message = std::make_shared>(header, data); + + InFlightRequestSharedPtr capture = nullptr; + EXPECT_CALL(listener_, onRequest(_)).WillOnce(testing::SaveArg<0>(&capture)); + + // when + testee_.onMessage(message); + + // then + ASSERT_NE(std::dynamic_pointer_cast(capture), nullptr); +} + TEST_F(RequestProcessorTest, ShouldProcessMetadataRequest) { // given const RequestHeader header = {METADATA_REQUEST_API_KEY, 0, 0, absl::nullopt}; @@ -94,9 +111,9 @@ TEST_F(RequestProcessorTest, ShouldProcessApiVersionsRequest) { TEST_F(RequestProcessorTest, ShouldHandleUnsupportedRequest) { // given - const RequestHeader header = {LIST_OFFSETS_REQUEST_API_KEY, 0, 0, absl::nullopt}; - const ListOffsetsRequest data = {0, {}}; - const auto message = std::make_shared>(header, data); + const RequestHeader header = {END_TXN_REQUEST_API_KEY, 0, 0, absl::nullopt}; + const EndTxnRequest data = {"", 0, 0, false}; + const auto message = std::make_shared>(header, data); // when, then - exception gets thrown. EXPECT_THROW_WITH_REGEX(testee_.onMessage(message), EnvoyException, "unsupported"); From 1211d078e0bfec5f8c5f7e1fe310beb028087783 Mon Sep 17 00:00:00 2001 From: JP Simard Date: Tue, 6 Dec 2022 15:35:57 -0500 Subject: [PATCH 04/23] ci: update `actions/setup-java` from v1 to v3.8 (#24393) https://github.com/actions/setup-java/releases/tag/v3.8.0 This should fix a number of GitHub Actions deprecations: ``` Node.js 12 actions are deprecated. For more information see: https://github.blog/changelog/2022-09-22-github-actions-all-actions-will-begin-running-on-node16-instead-of-node12/. Please update the following actions to use Node.js 16: actions/setup-java@v1 The `set-output` command is deprecated and will be disabled soon. Please upgrade to using Environment Files. For more information see: https://github.blog/changelog/2022-10-11-github-actions-deprecating-save-state-and-set-output-commands/ ``` Signed-off-by: JP Simard --- .github/workflows/android_build.yml | 15 ++++++++++----- .github/workflows/android_tests.yml | 9 ++++++--- .github/workflows/asan.yml | 3 ++- .github/workflows/format.yml | 3 ++- .github/workflows/tsan.yml | 3 ++- 5 files changed, 22 insertions(+), 11 deletions(-) diff --git a/.github/workflows/android_build.yml b/.github/workflows/android_build.yml index a0efe9cdf0fd..ace3e9860e3c 100644 --- a/.github/workflows/android_build.yml +++ b/.github/workflows/android_build.yml @@ -19,11 +19,12 @@ jobs: timeout-minutes: 90 steps: - uses: actions/checkout@v1 - - uses: actions/setup-java@v1 + - uses: actions/setup-java@c3ac5dd0ed8db40fedb61c32fbe677e6b355e94c with: java-version: '8' java-package: jdk architecture: x64 + distribution: zulu - name: 'Install dependencies' run: cd mobile && ./ci/mac_ci_setup.sh --android - name: 'Build envoy.aar distributable' @@ -42,11 +43,12 @@ jobs: timeout-minutes: 50 steps: - uses: actions/checkout@v1 - - uses: actions/setup-java@v1 + - uses: actions/setup-java@c3ac5dd0ed8db40fedb61c32fbe677e6b355e94c with: java-version: '8' java-package: jdk architecture: x64 + distribution: zulu - run: cd mobile && ./ci/mac_ci_setup.sh --android name: 'Install dependencies' - name: 'Start simulator' @@ -74,11 +76,12 @@ jobs: timeout-minutes: 50 steps: - uses: actions/checkout@v1 - - uses: actions/setup-java@v1 + - uses: actions/setup-java@c3ac5dd0ed8db40fedb61c32fbe677e6b355e94c with: java-version: '8' java-package: jdk architecture: x64 + distribution: zulu - name: 'Install dependencies' run: cd mobile && ./ci/mac_ci_setup.sh --android - name: 'Start simulator' @@ -106,11 +109,12 @@ jobs: timeout-minutes: 50 steps: - uses: actions/checkout@v1 - - uses: actions/setup-java@v1 + - uses: actions/setup-java@c3ac5dd0ed8db40fedb61c32fbe677e6b355e94c with: java-version: '8' java-package: jdk architecture: x64 + distribution: zulu - name: 'Install dependencies' run: cd mobile && ./ci/mac_ci_setup.sh --android - name: 'Start simulator' @@ -138,11 +142,12 @@ jobs: timeout-minutes: 50 steps: - uses: actions/checkout@v1 - - uses: actions/setup-java@v1 + - uses: actions/setup-java@c3ac5dd0ed8db40fedb61c32fbe677e6b355e94c with: java-version: '8' java-package: jdk architecture: x64 + distribution: zulu - name: 'Install dependencies' run: cd mobile && ./ci/mac_ci_setup.sh --android - name: 'Start simulator' diff --git a/.github/workflows/android_tests.yml b/.github/workflows/android_tests.yml index 1ecd972125e9..db85cbf3dc2e 100644 --- a/.github/workflows/android_tests.yml +++ b/.github/workflows/android_tests.yml @@ -33,11 +33,12 @@ jobs: fi - name: 'Java setup' if: steps.check_context.outputs.run_tests == 'true' - uses: actions/setup-java@v1 + uses: actions/setup-java@c3ac5dd0ed8db40fedb61c32fbe677e6b355e94c with: java-version: '8' java-package: jdk architecture: x64 + distribution: zulu - name: 'Install dependencies' if: steps.check_context.outputs.run_tests == 'true' run: cd mobile && ./ci/mac_ci_setup.sh @@ -70,11 +71,12 @@ jobs: fi - name: 'Java setup' if: steps.check_context.outputs.run_tests == 'true' - uses: actions/setup-java@v1 + uses: actions/setup-java@c3ac5dd0ed8db40fedb61c32fbe677e6b355e94c with: java-version: '8' java-package: jdk architecture: x64 + distribution: zulu - name: 'Install dependencies' if: steps.check_context.outputs.run_tests == 'true' run: cd mobile && ./ci/mac_ci_setup.sh @@ -117,11 +119,12 @@ jobs: fi - name: 'Java setup' if: steps.check_context.outputs.run_tests == 'true' - uses: actions/setup-java@v1 + uses: actions/setup-java@c3ac5dd0ed8db40fedb61c32fbe677e6b355e94c with: java-version: '8' java-package: jdk architecture: x64 + distribution: zulu - name: 'Install dependencies' if: steps.check_context.outputs.run_tests == 'true' run: cd mobile && ./ci/linux_ci_setup.sh diff --git a/.github/workflows/asan.yml b/.github/workflows/asan.yml index e2d08b359ded..d708dde3dc53 100644 --- a/.github/workflows/asan.yml +++ b/.github/workflows/asan.yml @@ -36,12 +36,13 @@ jobs: echo "Skipping tests." echo "run_tests=false" >> $GITHUB_OUTPUT fi - - uses: actions/setup-java@v1 + - uses: actions/setup-java@c3ac5dd0ed8db40fedb61c32fbe677e6b355e94c if: steps.check-cache.outputs.cache-hit != 'true' with: java-version: '8' java-package: jdk architecture: x64 + distribution: zulu - name: 'Run tests' env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} diff --git a/.github/workflows/format.yml b/.github/workflows/format.yml index 281ca9129608..f07ddd0f6c31 100644 --- a/.github/workflows/format.yml +++ b/.github/workflows/format.yml @@ -65,11 +65,12 @@ jobs: timeout-minutes: 45 steps: - uses: actions/checkout@v1 - - uses: actions/setup-java@v1 + - uses: actions/setup-java@c3ac5dd0ed8db40fedb61c32fbe677e6b355e94c with: java-version: '8' java-package: jdk architecture: x64 + distribution: zulu - run: cd mobile && ./ci/mac_ci_setup.sh name: 'Install dependencies' - name: 'Run Kotlin Lint (Detekt)' diff --git a/.github/workflows/tsan.yml b/.github/workflows/tsan.yml index 8431e4606ca9..e2f6b8e0691f 100644 --- a/.github/workflows/tsan.yml +++ b/.github/workflows/tsan.yml @@ -36,12 +36,13 @@ jobs: echo "Skipping tests." echo "run_tests=false" >> $GITHUB_OUTPUT fi - - uses: actions/setup-java@v1 + - uses: actions/setup-java@c3ac5dd0ed8db40fedb61c32fbe677e6b355e94c if: steps.check-cache.outputs.cache-hit != 'true' with: java-version: '8' java-package: jdk architecture: x64 + distribution: zulu - name: 'Run tests' env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} From 28ac8d08d71b41544969d23415247dc53ce919c1 Mon Sep 17 00:00:00 2001 From: JP Simard Date: Tue, 6 Dec 2022 16:12:22 -0500 Subject: [PATCH 05/23] mobile: build Swift with whole module optimization (#24396) Works around https://github.com/bazelbuild/rules_swift/issues/949 Signed-off-by: JP Simard --- mobile/.bazelrc | 3 +++ 1 file changed, 3 insertions(+) diff --git a/mobile/.bazelrc b/mobile/.bazelrc index 242fa4760b0b..81099a119e0f 100644 --- a/mobile/.bazelrc +++ b/mobile/.bazelrc @@ -23,6 +23,9 @@ build --workspace_status_command=../bazel/get_workspace_status build --xcode_version=14.1 build --use_top_level_targets_for_symlinks build --experimental_repository_downloader_retries=2 +# We don't have a ton of Swift in Envoy Mobile, so always build with WMO +# This also helps work around a bug in rules_swift: https://github.com/bazelbuild/rules_swift/issues/949 +build --swiftcopt=-wmo # https://github.com/bazelbuild/rules_jvm_external/issues/445 build --repo_env=JAVA_HOME=../bazel_tools/jdk build --define disable_known_issue_asserts=true From bf722e34adc36c2547899fb49d690b8210829236 Mon Sep 17 00:00:00 2001 From: alyssawilk Date: Tue, 6 Dec 2022 16:37:28 -0500 Subject: [PATCH 06/23] listeners: moving listeners to extension directory (#24248) Signed-off-by: Alyssa Wilk --- CODEOWNERS | 1 + source/common/config/well_known_names.h | 10 ++ source/exe/BUILD | 3 + .../bootstrap/internal_listener/BUILD | 2 +- .../listener_managers/listener_manager/BUILD | 83 +++++++++++++++++ .../listener_manager}/listener_impl.cc | 11 +-- .../listener_manager}/listener_impl.h | 0 .../listener_manager_impl.cc | 2 +- .../listener_manager}/listener_manager_impl.h | 7 +- source/server/BUILD | 50 ++-------- source/server/admin/BUILD | 2 +- source/server/admin/admin.cc | 2 +- source/server/config_validation/BUILD | 1 + source/server/config_validation/server.cc | 9 +- source/server/config_validation/server.h | 6 +- source/server/hot_restarting_parent.cc | 1 - source/server/listener_manager_factory.h | 2 + source/server/server.cc | 3 +- .../listener_managers/listener_manager/BUILD | 91 +++++++++++++++++++ .../listener_manager_impl_quic_only_test.cc | 2 +- .../listener_manager_impl_test.cc | 49 +++++++++- .../listener_manager_impl_test.h | 3 +- test/integration/BUILD | 2 +- test/server/BUILD | 85 +---------------- test/server/filter_chain_manager_impl_test.cc | 2 +- tools/code_format/config.yaml | 1 + tools/extensions/extensions_schema.yaml | 1 + 27 files changed, 279 insertions(+), 152 deletions(-) create mode 100644 source/extensions/listener_managers/listener_manager/BUILD rename source/{server => extensions/listener_managers/listener_manager}/listener_impl.cc (99%) rename source/{server => extensions/listener_managers/listener_manager}/listener_impl.h (100%) rename source/{server => extensions/listener_managers/listener_manager}/listener_manager_impl.cc (99%) rename source/{server => extensions/listener_managers/listener_manager}/listener_manager_impl.h (98%) create mode 100644 test/extensions/listener_managers/listener_manager/BUILD rename test/{server => extensions/listener_managers/listener_manager}/listener_manager_impl_quic_only_test.cc (99%) rename test/{server => extensions/listener_managers/listener_manager}/listener_manager_impl_test.cc (98%) rename test/{server => extensions/listener_managers/listener_manager}/listener_manager_impl_test.h (99%) diff --git a/CODEOWNERS b/CODEOWNERS index ca849987e710..dbe3a283e355 100644 --- a/CODEOWNERS +++ b/CODEOWNERS @@ -291,6 +291,7 @@ extensions/filters/http/oauth2 @derekargueta @snowp /*/extensions/clusters/logical_dns/ @UNOWNED @UNOWNED /*/extensions/clusters/common/ @UNOWNED @UNOWNED /*/extensions/clusters/eds/ @UNOWNED @UNOWNED +/*/source/extensions/listener_managers/listener_manager @UNOWNED @UNOWNED # URL Pattern Match and Rewrite Library /*/extensions/path/uri_template_lib @alyssawilk @yanjunxiang-google diff --git a/source/common/config/well_known_names.h b/source/common/config/well_known_names.h index da1085ed239a..af429199ed1e 100644 --- a/source/common/config/well_known_names.h +++ b/source/common/config/well_known_names.h @@ -173,5 +173,15 @@ class TagNameValues { using TagNames = ConstSingleton; +// This class holds extension points which will always be built into Envoy in +// server mode, but may be excluded from Envoy Mobile. +class ServerBuiltInExtensionValues { +public: + // Extension point for the default listener. + const std::string DEFAULT_LISTENER = "envoy.listener_manager_impl.default"; +}; + +using ServerExtensionValues = ConstSingleton; + } // namespace Config } // namespace Envoy diff --git a/source/exe/BUILD b/source/exe/BUILD index cc0796ed9900..390bd8a4d84d 100644 --- a/source/exe/BUILD +++ b/source/exe/BUILD @@ -89,6 +89,9 @@ envoy_cc_library( name = "envoy_main_common_lib", deps = [ ":main_common_lib", + # This is compiled as an extension so Envoy Mobile doesn't have to link it in. + # Envoy requires it. + "//source/extensions/listener_managers/listener_manager:listener_manager_lib", "//source/common/version:version_linkstamp", ], ) diff --git a/source/extensions/bootstrap/internal_listener/BUILD b/source/extensions/bootstrap/internal_listener/BUILD index 8eff9ec555eb..30f003f9e72b 100644 --- a/source/extensions/bootstrap/internal_listener/BUILD +++ b/source/extensions/bootstrap/internal_listener/BUILD @@ -30,7 +30,7 @@ envoy_cc_extension( deps = [ ":thread_local_registry", "//envoy/server:bootstrap_extension_config_interface", - "//source/server:listener_manager_lib", + "//source/extensions/listener_managers/listener_manager:listener_manager_lib", "@envoy_api//envoy/extensions/bootstrap/internal_listener/v3:pkg_cc_proto", ], alwayslink = 1, diff --git a/source/extensions/listener_managers/listener_manager/BUILD b/source/extensions/listener_managers/listener_manager/BUILD new file mode 100644 index 000000000000..0dd385ad3524 --- /dev/null +++ b/source/extensions/listener_managers/listener_manager/BUILD @@ -0,0 +1,83 @@ +load( + "//bazel:envoy_build_system.bzl", + "envoy_cc_extension", + "envoy_extension_package", + "envoy_select_enable_http3", +) + +licenses(["notice"]) # Apache 2 + +envoy_extension_package() + +envoy_cc_extension( + name = "listener_manager_lib", + srcs = [ + "listener_impl.cc", + "listener_manager_impl.cc", + ], + hdrs = [ + "listener_impl.h", + "listener_manager_impl.h", + ], + # any changes to this should be reviewed by mobile maintainers + # to ensure that listener code doesn't leak back into Envoy Mobile. + extra_visibility = [ + "//source/server/admin:__subpackages__", + "//source/server/config_validation:__subpackages__", + "//test:__subpackages__", + ], + deps = [ + "//source/server:listener_manager_factory_lib", + "//source/server:api_listener_lib", + "//source/server:active_raw_udp_listener_config", + "//source/server:configuration_lib", + "//source/server:drain_manager_lib", + "//source/server:filter_chain_manager_lib", + "//source/server:lds_api_lib", + "//source/server:transport_socket_config_lib", + "//envoy/access_log:access_log_interface", + "//envoy/config:typed_metadata_interface", + "//envoy/network:connection_interface", + "//envoy/network:listener_interface", + "//envoy/server:api_listener_interface", + "//envoy/server:filter_config_interface", + "//envoy/server:listener_manager_interface", + "//envoy/server:transport_socket_config_interface", + "//envoy/server:worker_interface", + "//source/common/access_log:access_log_lib", + "//source/common/common:basic_resource_lib", + "//source/common/common:empty_string", + "//source/common/config:utility_lib", + "//source/common/config:metadata_lib", + "//source/common/http:conn_manager_lib", + "//source/common/init:manager_lib", + "//source/common/init:target_lib", + "//source/common/network:connection_balancer_lib", + "//source/common/network:filter_matcher_lib", + "//source/common/network:listen_socket_lib", + "//source/common/network:listener_lib", + "//source/common/network:resolver_lib", + "//source/common/network:socket_option_factory_lib", + "//source/common/network:udp_packet_writer_handler_lib", + "//source/common/network:utility_lib", + "//source/common/protobuf:utility_lib", + "//source/common/stream_info:stream_info_lib", + "//source/common/quic:quic_stat_names_lib", + "//source/extensions/filters/network/http_connection_manager:config", + "//source/extensions/upstreams/http/generic:config", + "//source/extensions/udp_packet_writer/default:config", + "@envoy_api//envoy/admin/v3:pkg_cc_proto", + "@envoy_api//envoy/config/core/v3:pkg_cc_proto", + "@envoy_api//envoy/config/listener/v3:pkg_cc_proto", + "@envoy_api//envoy/extensions/filters/listener/proxy_protocol/v3:pkg_cc_proto", + "@envoy_api//envoy/extensions/transport_sockets/raw_buffer/v3:pkg_cc_proto", + "@envoy_api//envoy/extensions/udp_packet_writer/v3:pkg_cc_proto", + ] + envoy_select_enable_http3([ + "//source/common/quic:active_quic_listener_lib", + "//source/common/quic:client_connection_factory_lib", + "//source/common/quic:quic_factory_lib", + "//source/common/quic:quic_transport_socket_factory_lib", + "//source/common/quic:udp_gso_batch_writer_lib", + "//source/extensions/udp_packet_writer/gso:config", + ]), +) diff --git a/source/server/listener_impl.cc b/source/extensions/listener_managers/listener_manager/listener_impl.cc similarity index 99% rename from source/server/listener_impl.cc rename to source/extensions/listener_managers/listener_manager/listener_impl.cc index 63522b90f1e8..3e7ab8f81157 100644 --- a/source/server/listener_impl.cc +++ b/source/extensions/listener_managers/listener_manager/listener_impl.cc @@ -1,4 +1,4 @@ -#include "source/server/listener_impl.h" +#include "source/extensions/listener_managers/listener_manager/listener_impl.h" #include @@ -27,11 +27,11 @@ #include "source/common/network/utility.h" #include "source/common/protobuf/utility.h" #include "source/common/runtime/runtime_features.h" +#include "source/extensions/listener_managers/listener_manager/listener_manager_impl.h" #include "source/server/active_raw_udp_listener_config.h" #include "source/server/configuration_impl.h" #include "source/server/drain_manager_impl.h" #include "source/server/filter_chain_manager_impl.h" -#include "source/server/listener_manager_impl.h" #include "source/server/transport_socket_config_impl.h" #ifdef ENVOY_ENABLE_QUIC @@ -301,9 +301,7 @@ Stats::Scope& ListenerFactoryContextBaseImpl::listenerScope() { return *listener bool ListenerFactoryContextBaseImpl::isQuicListener() const { return is_quic_; } Network::DrainDecision& ListenerFactoryContextBaseImpl::drainDecision() { return *this; } Server::DrainManager& ListenerFactoryContextBaseImpl::drainManager() { return *drain_manager_; } - -// Must be overridden -Init::Manager& ListenerFactoryContextBaseImpl::initManager() { PANIC("not implemented"); } +Init::Manager& ListenerFactoryContextBaseImpl::initManager() { return server_.initManager(); } ListenerImpl::ListenerImpl(const envoy::config::listener::v3::Listener& config, const std::string& version_info, ListenerManagerImpl& parent, @@ -774,11 +772,10 @@ void ListenerImpl::buildConnectionBalancer(const Network::Address::Instance& add // Not in place listener update. if (config_.has_connection_balance_config()) { switch (config_.connection_balance_config().balance_type_case()) { - case envoy::config::listener::v3::Listener_ConnectionBalanceConfig::kExactBalance: { + case envoy::config::listener::v3::Listener_ConnectionBalanceConfig::kExactBalance: connection_balancers_.emplace(address.asString(), std::make_shared()); break; - } case envoy::config::listener::v3::Listener_ConnectionBalanceConfig::kExtendBalance: { const std::string connection_balance_library_type{TypeUtil::typeUrlToDescriptorFullName( config_.connection_balance_config().extend_balance().typed_config().type_url())}; diff --git a/source/server/listener_impl.h b/source/extensions/listener_managers/listener_manager/listener_impl.h similarity index 100% rename from source/server/listener_impl.h rename to source/extensions/listener_managers/listener_manager/listener_impl.h diff --git a/source/server/listener_manager_impl.cc b/source/extensions/listener_managers/listener_manager/listener_manager_impl.cc similarity index 99% rename from source/server/listener_manager_impl.cc rename to source/extensions/listener_managers/listener_manager/listener_manager_impl.cc index c08489506f4f..8b93af0ac96f 100644 --- a/source/server/listener_manager_impl.cc +++ b/source/extensions/listener_managers/listener_manager/listener_manager_impl.cc @@ -1,4 +1,4 @@ -#include "source/server/listener_manager_impl.h" +#include "source/extensions/listener_managers/listener_manager/listener_manager_impl.h" #include diff --git a/source/server/listener_manager_impl.h b/source/extensions/listener_managers/listener_manager/listener_manager_impl.h similarity index 98% rename from source/server/listener_manager_impl.h rename to source/extensions/listener_managers/listener_manager/listener_manager_impl.h index 9dba8f34d450..c8d6ac92f90f 100644 --- a/source/server/listener_manager_impl.h +++ b/source/extensions/listener_managers/listener_manager/listener_manager_impl.h @@ -18,12 +18,13 @@ #include "envoy/server/worker.h" #include "envoy/stats/scope.h" +#include "source/common/config/well_known_names.h" #include "source/common/filter/config_discovery_impl.h" #include "source/common/quic/quic_stat_names.h" +#include "source/extensions/listener_managers/listener_manager/listener_impl.h" #include "source/server/filter_chain_factory_context_callback.h" #include "source/server/filter_chain_manager_impl.h" #include "source/server/lds_api.h" -#include "source/server/listener_impl.h" #include "source/server/listener_manager_factory.h" namespace Envoy { @@ -363,7 +364,9 @@ class DefaultListenerManagerFactoryImpl : public ListenerManagerFactory { return std::make_unique(server, std::move(factory), worker_factory, enable_dispatcher_stats, quic_stat_names); } - std::string name() const override { return "envoy.listener_manager_impl.default"; } + std::string name() const override { + return Config::ServerExtensionValues::get().DEFAULT_LISTENER; + } }; } // namespace Server diff --git a/source/server/BUILD b/source/server/BUILD index a4a61bd8efa5..c40ad62578ac 100644 --- a/source/server/BUILD +++ b/source/server/BUILD @@ -3,7 +3,6 @@ load( "envoy_cc_library", "envoy_package", "envoy_proto_library", - "envoy_select_enable_http3", "envoy_select_hot_restart", ) @@ -324,7 +323,6 @@ envoy_cc_library( hdrs = envoy_select_hot_restart(["hot_restarting_parent.h"]), deps = [ ":hot_restarting_base", - ":listener_manager_lib", "//source/common/memory:stats_lib", "//source/common/stats:stat_merger_lib", "//source/common/stats:symbol_table_lib", @@ -437,74 +435,39 @@ envoy_cc_library( ], ) -# TODO(junr03): actually separate this lib from the listener and api listener lib. -# this can be done if the parent_ in the listener and the api listener becomes the ListenerManager interface. -# the issue right now is that the listener's reach into the listener manager's server_ instance variable. envoy_cc_library( - name = "listener_manager_lib", + name = "api_listener_lib", srcs = [ "api_listener_impl.cc", - "listener_impl.cc", - "listener_manager_impl.cc", ], hdrs = [ "api_listener_impl.h", - "listener_impl.h", - "listener_manager_impl.h", ], deps = [ - ":listener_manager_factory_lib", - "active_raw_udp_listener_config", ":configuration_lib", - ":drain_manager_lib", ":filter_chain_manager_lib", ":lds_api_lib", ":transport_socket_config_lib", - "//envoy/access_log:access_log_interface", - "//envoy/config:typed_metadata_interface", "//envoy/network:connection_interface", "//envoy/network:listener_interface", - "//envoy/server:api_listener_interface", "//envoy/server:filter_config_interface", + "//envoy/server:instance_interface", "//envoy/server:listener_manager_interface", "//envoy/server:transport_socket_config_interface", - "//envoy/server:worker_interface", - "//source/common/access_log:access_log_lib", "//source/common/common:basic_resource_lib", "//source/common/common:empty_string", - "//source/common/config:utility_lib", "//source/common/config:metadata_lib", - "//source/common/http:conn_manager_lib", + "//source/common/config:utility_lib", "//source/common/init:manager_lib", "//source/common/init:target_lib", - "//source/common/network:connection_balancer_lib", - "//source/common/network:filter_matcher_lib", - "//source/common/network:listen_socket_lib", - "//source/common/network:listener_lib", - "//source/common/network:resolver_lib", - "//source/common/network:socket_option_factory_lib", - "//source/common/network:udp_packet_writer_handler_lib", "//source/common/network:utility_lib", - "//source/common/protobuf:utility_lib", "//source/common/stream_info:stream_info_lib", - "//source/common/quic:quic_stat_names_lib", "//source/extensions/filters/network/http_connection_manager:config", - "//source/extensions/upstreams/http/generic:config", "//source/extensions/udp_packet_writer/default:config", - "@envoy_api//envoy/admin/v3:pkg_cc_proto", + "//source/extensions/upstreams/http/generic:config", "@envoy_api//envoy/config/core/v3:pkg_cc_proto", "@envoy_api//envoy/config/listener/v3:pkg_cc_proto", - "@envoy_api//envoy/extensions/filters/listener/proxy_protocol/v3:pkg_cc_proto", - "@envoy_api//envoy/extensions/transport_sockets/raw_buffer/v3:pkg_cc_proto", - "@envoy_api//envoy/extensions/udp_packet_writer/v3:pkg_cc_proto", - ] + envoy_select_enable_http3([ - "//source/common/quic:active_quic_listener_lib", - "//source/common/quic:client_connection_factory_lib", - "//source/common/quic:quic_factory_lib", - "//source/common/quic:quic_transport_socket_factory_lib", - "//source/common/quic:udp_gso_batch_writer_lib", - "//source/extensions/udp_packet_writer/gso:config", - ]), + ], alwayslink = True, ) @@ -586,11 +549,12 @@ envoy_cc_library( ], deps = [ ":active_raw_udp_listener_config", + ":api_listener_lib", ":configuration_lib", ":connection_handler_lib", ":guarddog_lib", ":listener_hooks_lib", - ":listener_manager_lib", + ":listener_manager_factory_lib", ":regex_engine_lib", ":ssl_context_manager_lib", ":worker_lib", diff --git a/source/server/admin/BUILD b/source/server/admin/BUILD index 0c526edc73de..7aeac01ff161 100644 --- a/source/server/admin/BUILD +++ b/source/server/admin/BUILD @@ -76,8 +76,8 @@ envoy_cc_library( "//source/common/router:scoped_config_lib", "//source/common/stats:isolated_store_lib", "//source/extensions/access_loggers/common:file_access_log_lib", + "//source/extensions/listener_managers/listener_manager:listener_manager_lib", "//source/extensions/request_id/uuid:config", - "//source/server:listener_manager_lib", ], ) diff --git a/source/server/admin/admin.cc b/source/server/admin/admin.cc index 9b58a1e0b9b5..595ff4c36734 100644 --- a/source/server/admin/admin.cc +++ b/source/server/admin/admin.cc @@ -32,9 +32,9 @@ #include "source/common/protobuf/protobuf.h" #include "source/common/protobuf/utility.h" #include "source/common/router/config_impl.h" +#include "source/extensions/listener_managers/listener_manager/listener_impl.h" #include "source/extensions/request_id/uuid/config.h" #include "source/server/admin/utils.h" -#include "source/server/listener_impl.h" #include "absl/strings/str_join.h" #include "absl/strings/str_replace.h" diff --git a/source/server/config_validation/BUILD b/source/server/config_validation/BUILD index 52326bdd8cb5..3f086dba5e46 100644 --- a/source/server/config_validation/BUILD +++ b/source/server/config_validation/BUILD @@ -99,6 +99,7 @@ envoy_cc_library( "//source/common/stats:stats_lib", "//source/common/thread_local:thread_local_lib", "//source/common/version:version_lib", + "//source/extensions/listener_managers/listener_manager:listener_manager_lib", "//source/server:configuration_lib", "//source/server:server_lib", "//source/server/admin:admin_lib", diff --git a/source/server/config_validation/server.cc b/source/server/config_validation/server.cc index 795eda570261..697cb3de93aa 100644 --- a/source/server/config_validation/server.cc +++ b/source/server/config_validation/server.cc @@ -6,6 +6,7 @@ #include "source/common/common/utility.h" #include "source/common/config/utility.h" +#include "source/common/config/well_known_names.h" #include "source/common/event/real_time_system.h" #include "source/common/local_info/local_info_impl.h" #include "source/common/protobuf/utility.h" @@ -102,9 +103,11 @@ void ValidationInstance::initialize(const Options& options, Configuration::InitialImpl initial_config(bootstrap_); initial_config.initAdminAccessLog(bootstrap_, *this); admin_ = std::make_unique(initial_config.admin().address()); - listener_manager_ = std::make_unique( - *this, std::make_unique(*this), *this, false, - quic_stat_names_); + listener_manager_ = + Config::Utility::getAndCheckFactoryByName( + Config::ServerExtensionValues::get().DEFAULT_LISTENER) + .createListenerManager(*this, std::make_unique(*this), + *this, false, quic_stat_names_); thread_local_.registerThread(*dispatcher_, true); Runtime::LoaderPtr runtime_ptr = component_factory.createRuntime(*this, initial_config); diff --git a/source/server/config_validation/server.h b/source/server/config_validation/server.h index 9db6c4a3afe0..c7757b0f9c4f 100644 --- a/source/server/config_validation/server.h +++ b/source/server/config_validation/server.h @@ -24,11 +24,13 @@ #include "source/common/runtime/runtime_impl.h" #include "source/common/secret/secret_manager_impl.h" #include "source/common/thread_local/thread_local_impl.h" + +// TODO(alyssawilk) address +#include "source/extensions/listener_managers/listener_manager/listener_manager_impl.h" #include "source/server/config_validation/admin.h" #include "source/server/config_validation/api.h" #include "source/server/config_validation/cluster_manager.h" #include "source/server/config_validation/dns.h" -#include "source/server/listener_manager_impl.h" #include "source/server/server.h" #include "absl/types/optional.h" @@ -235,7 +237,7 @@ class ValidationInstance final : Logger::Loggable, LocalInfo::LocalInfoPtr local_info_; AccessLog::AccessLogManagerImpl access_log_manager_; std::unique_ptr cluster_manager_factory_; - std::unique_ptr listener_manager_; + std::unique_ptr listener_manager_; std::unique_ptr overload_manager_; MutexTracer* mutex_tracer_; Grpc::ContextImpl grpc_context_; diff --git a/source/server/hot_restarting_parent.cc b/source/server/hot_restarting_parent.cc index 7c289fa14364..727fd2c59819 100644 --- a/source/server/hot_restarting_parent.cc +++ b/source/server/hot_restarting_parent.cc @@ -7,7 +7,6 @@ #include "source/common/stats/stat_merger.h" #include "source/common/stats/symbol_table.h" #include "source/common/stats/utility.h" -#include "source/server/listener_impl.h" namespace Envoy { namespace Server { diff --git a/source/server/listener_manager_factory.h b/source/server/listener_manager_factory.h index 705ea7737397..fb773e9936ef 100644 --- a/source/server/listener_manager_factory.h +++ b/source/server/listener_manager_factory.h @@ -9,12 +9,14 @@ namespace Envoy { namespace Server { + class ListenerManagerFactory : public Config::UntypedFactory { public: virtual std::unique_ptr createListenerManager(Instance& server, std::unique_ptr&& factory, WorkerFactory& worker_factory, bool enable_dispatcher_stats, Quic::QuicStatNames& quic_stat_names) PURE; + std::string category() const override { return "envoy.listener_manager_impl"; } }; diff --git a/source/server/server.cc b/source/server/server.cc index 77f39aca19ce..a9cc87522c17 100644 --- a/source/server/server.cc +++ b/source/server/server.cc @@ -32,6 +32,7 @@ #include "source/common/config/grpc_mux_impl.h" #include "source/common/config/new_grpc_mux_impl.h" #include "source/common/config/utility.h" +#include "source/common/config/well_known_names.h" #include "source/common/config/xds_mux/grpc_mux_impl.h" #include "source/common/config/xds_resource.h" #include "source/common/http/codes.h" @@ -625,7 +626,7 @@ void InstanceImpl::initialize(Network::Address::InstanceConstSharedPtr local_add // Workers get created first so they register for thread local updates. listener_manager_ = Config::Utility::getAndCheckFactoryByName( - "envoy.listener_manager_impl.default") + Config::ServerExtensionValues::get().DEFAULT_LISTENER) .createListenerManager(*this, nullptr, worker_factory_, bootstrap_.enable_dispatcher_stats(), quic_stat_names_); diff --git a/test/extensions/listener_managers/listener_manager/BUILD b/test/extensions/listener_managers/listener_manager/BUILD new file mode 100644 index 000000000000..4beb2f0ed5c7 --- /dev/null +++ b/test/extensions/listener_managers/listener_manager/BUILD @@ -0,0 +1,91 @@ +load( + "//bazel:envoy_build_system.bzl", + "envoy_cc_test", + "envoy_cc_test_library", + "envoy_package", +) + +licenses(["notice"]) # Apache 2 + +envoy_package() + +envoy_cc_test_library( + name = "listener_manager_impl_test_lib", + hdrs = ["listener_manager_impl_test.h"], + data = ["//test/extensions/transport_sockets/tls/test_data:certs"], + deps = [ + "//source/common/init:manager_lib", + "//source/extensions/common/matcher:trie_matcher_lib", + "//source/extensions/listener_managers/listener_manager:listener_manager_lib", + "//test/mocks/init:init_mocks", + "//test/mocks/matcher:matcher_mocks", + "//test/mocks/network:network_mocks", + "//test/mocks/server:drain_manager_mocks", + "//test/mocks/server:guard_dog_mocks", + "//test/mocks/server:instance_mocks", + "//test/mocks/server:listener_component_factory_mocks", + "//test/mocks/server:worker_factory_mocks", + "//test/mocks/server:worker_mocks", + "//test/mocks/stream_info:stream_info_mocks", + "//test/test_common:environment_lib", + "//test/test_common:simulated_time_system_lib", + "//test/test_common:test_runtime_lib", + "//test/test_common:test_time_lib", + "//test/test_common:threadsafe_singleton_injector_lib", + "@envoy_api//envoy/admin/v3:pkg_cc_proto", + "@envoy_api//envoy/config/core/v3:pkg_cc_proto", + "@envoy_api//envoy/config/listener/v3:pkg_cc_proto", + ], +) + +envoy_cc_test( + name = "listener_manager_impl_test", + srcs = ["listener_manager_impl_test.cc"], + deps = [ + ":listener_manager_impl_test_lib", + "//source/common/api:os_sys_calls_lib", + "//source/common/config:metadata_lib", + "//source/common/network:addr_family_aware_socket_option_lib", + "//source/common/network:listen_socket_lib", + "//source/common/network:socket_option_lib", + "//source/common/network:utility_lib", + "//source/common/protobuf", + "//source/extensions/filters/listener/original_dst:config", + "//source/extensions/filters/listener/proxy_protocol:config", + "//source/extensions/filters/listener/tls_inspector:config", + "//source/extensions/filters/network/http_connection_manager:config", + "//source/extensions/filters/network/tcp_proxy:config", + "//source/extensions/request_id/uuid:config", + "//source/extensions/transport_sockets/raw_buffer:config", + "//source/extensions/transport_sockets/tls:config", + "//source/extensions/transport_sockets/tls:ssl_socket_lib", + "//source/server:active_raw_udp_listener_config", + "//test/integration/filters:test_listener_filter_lib", + "//test/server:utility_lib", + "//test/test_common:network_utility_lib", + "//test/test_common:registry_lib", + "//test/test_common:test_runtime_lib", + "@envoy_api//envoy/admin/v3:pkg_cc_proto", + "@envoy_api//envoy/config/core/v3:pkg_cc_proto", + "@envoy_api//envoy/config/listener/v3:pkg_cc_proto", + ], +) + +# Stand-alone quic test because of FIPS. +envoy_cc_test( + name = "listener_manager_impl_quic_only_test", + srcs = ["listener_manager_impl_quic_only_test.cc"], + tags = ["nofips"], + deps = [ + ":listener_manager_impl_test_lib", + "//source/extensions/filters/http/router:config", + "//source/extensions/request_id/uuid:config", + "//source/extensions/transport_sockets/raw_buffer:config", + "//source/extensions/transport_sockets/tls:config", + "//test/integration/filters:test_network_filter_lib", + "//test/server:utility_lib", + "//test/test_common:threadsafe_singleton_injector_lib", + "@envoy_api//envoy/config/core/v3:pkg_cc_proto", + "@envoy_api//envoy/config/listener/v3:pkg_cc_proto", + ], +) diff --git a/test/server/listener_manager_impl_quic_only_test.cc b/test/extensions/listener_managers/listener_manager/listener_manager_impl_quic_only_test.cc similarity index 99% rename from test/server/listener_manager_impl_quic_only_test.cc rename to test/extensions/listener_managers/listener_manager/listener_manager_impl_quic_only_test.cc index a308b7631386..a21e106c1312 100644 --- a/test/server/listener_manager_impl_quic_only_test.cc +++ b/test/extensions/listener_managers/listener_manager/listener_manager_impl_quic_only_test.cc @@ -5,7 +5,7 @@ #include "source/common/quic/quic_transport_socket_factory.h" #endif -#include "test/server/listener_manager_impl_test.h" +#include "test/extensions/listener_managers/listener_manager/listener_manager_impl_test.h" #include "test/server/utility.h" #include "test/test_common/threadsafe_singleton_injector.h" diff --git a/test/server/listener_manager_impl_test.cc b/test/extensions/listener_managers/listener_manager/listener_manager_impl_test.cc similarity index 98% rename from test/server/listener_manager_impl_test.cc rename to test/extensions/listener_managers/listener_manager/listener_manager_impl_test.cc index 5f28229ce0ff..7e4dd29914cf 100644 --- a/test/server/listener_manager_impl_test.cc +++ b/test/extensions/listener_managers/listener_manager/listener_manager_impl_test.cc @@ -1,4 +1,4 @@ -#include "test/server/listener_manager_impl_test.h" +#include "test/extensions/listener_managers/listener_manager/listener_manager_impl_test.h" #include #include @@ -5660,11 +5660,56 @@ TEST_P(ListenerManagerImplWithRealFiltersTest, OriginalDstFilter) { Network::Address::IpVersion::v4); EXPECT_CALL(server_.api_.random_, uuid()); EXPECT_CALL(listener_factory_, createListenSocket(_, _, _, default_bind_type, _, 0)); + + Configuration::ListenerFactoryContext* listener_factory_context = nullptr; + // Extract listener_factory_context to unit test functions. + ON_CALL(listener_factory_, createListenerFilterFactoryList(_, _)) + .WillByDefault( + Invoke([&listener_factory_context, this]( + const Protobuf::RepeatedPtrField& + filters, + Configuration::ListenerFactoryContext& context) + -> Filter::ListenerFilterFactoriesList { + listener_factory_context = &context; + return ProdListenerComponentFactory::createListenerFilterFactoryListImpl( + filters, context, *listener_factory_.getTcpListenerConfigProviderManager()); + })); + addOrUpdateListener(parseListenerFromV3Yaml(yaml)); EXPECT_EQ(1U, manager_->listeners().size()); - Network::ListenerConfig& listener = manager_->listeners().back().get(); + // Unit test PerListenerFactoryContextImpl for coverage. + ASSERT_TRUE(listener_factory_context != nullptr); + ListenerFactoryContextBaseImpl& parent_context = + static_cast(listener_factory_context)->parentFactoryContext(); + EXPECT_EQ(&listener_factory_context->timeSource(), &listener_factory_context->api().timeSource()); + EXPECT_EQ(&listener_factory_context->initManager(), &listener.initManager()); + EXPECT_EQ(&listener_factory_context->lifecycleNotifier(), &server_.lifecycleNotifier()); + EXPECT_EQ(&listener_factory_context->messageValidationContext(), + &listener_factory_context->getServerFactoryContext().messageValidationContext()); + EXPECT_EQ(&listener_factory_context->mainThreadDispatcher(), + &parent_context.mainThreadDispatcher()); + EXPECT_EQ(&listener_factory_context->options(), &parent_context.options()); + EXPECT_EQ(&listener_factory_context->grpcContext(), &parent_context.grpcContext()); + EXPECT_EQ(listener_factory_context->healthCheckFailed(), parent_context.healthCheckFailed()); + EXPECT_EQ(&listener_factory_context->httpContext(), &parent_context.httpContext()); + EXPECT_EQ(&listener_factory_context->routerContext(), &parent_context.routerContext()); + EXPECT_EQ(&listener_factory_context->overloadManager(), &parent_context.overloadManager()); + EXPECT_EQ(listener_factory_context->admin().has_value(), parent_context.admin().has_value()); + EXPECT_EQ(&listener_factory_context->listenerTypedMetadata(), + &parent_context.listenerTypedMetadata()); + EXPECT_EQ(listener_factory_context->processContext().has_value(), + parent_context.processContext().has_value()); + EXPECT_EQ(&listener_factory_context->getTransportSocketFactoryContext(), + &parent_context.getTransportSocketFactoryContext()); + EXPECT_EQ(listener_factory_context->isQuicListener(), parent_context.isQuicListener()); + + // Unit test ListenerFactoryContextBaseImpl for coverage. + EXPECT_EQ(&parent_context.timeSource(), &listener_factory_context->api().timeSource()); + EXPECT_EQ(&parent_context.messageValidationContext(), &server_.messageValidationContext()); + EXPECT_EQ(&parent_context.lifecycleNotifier(), &server_.lifecycleNotifier()); + Network::FilterChainFactory& filterChainFactory = listener.filterChainFactory(); Network::MockListenerFilterManager manager; diff --git a/test/server/listener_manager_impl_test.h b/test/extensions/listener_managers/listener_manager/listener_manager_impl_test.h similarity index 99% rename from test/server/listener_manager_impl_test.h rename to test/extensions/listener_managers/listener_manager/listener_manager_impl_test.h index 43bb40a5f411..d426a774d167 100644 --- a/test/server/listener_manager_impl_test.h +++ b/test/extensions/listener_managers/listener_manager/listener_manager_impl_test.h @@ -9,8 +9,9 @@ #include "source/common/network/listen_socket_impl.h" #include "source/common/network/socket_option_impl.h" +#include "source/extensions/listener_managers/listener_manager/listener_impl.h" +#include "source/extensions/listener_managers/listener_manager/listener_manager_impl.h" #include "source/server/configuration_impl.h" -#include "source/server/listener_manager_impl.h" #include "test/mocks/network/mocks.h" #include "test/mocks/server/drain_manager.h" diff --git a/test/integration/BUILD b/test/integration/BUILD index c02abfb629ba..d33c2ee6f844 100644 --- a/test/integration/BUILD +++ b/test/integration/BUILD @@ -845,7 +845,7 @@ envoy_cc_test_library( deps = [ "//test/mocks/runtime:runtime_mocks", "//test/mocks/protobuf:protobuf_mocks", - "//source/server:listener_manager_lib", + "//source/extensions/listener_managers/listener_manager:listener_manager_lib", "//envoy/api:api_interface", "//envoy/grpc:status", "//envoy/http:codec_interface", diff --git a/test/server/BUILD b/test/server/BUILD index 5d4f485718c9..5ee555853a1f 100644 --- a/test/server/BUILD +++ b/test/server/BUILD @@ -21,8 +21,8 @@ envoy_cc_test( srcs = ["api_listener_test.cc"], deps = [ ":utility_lib", + "//source/extensions/listener_managers/listener_manager:listener_manager_lib", "//source/extensions/request_id/uuid:config", - "//source/server:listener_manager_lib", "//test/mocks/network:network_mocks", "//test/mocks/server:instance_mocks", "//test/mocks/server:listener_component_factory_mocks", @@ -268,87 +268,6 @@ envoy_cc_test( ], ) -envoy_cc_test_library( - name = "listener_manager_impl_test_lib", - hdrs = ["listener_manager_impl_test.h"], - data = ["//test/extensions/transport_sockets/tls/test_data:certs"], - deps = [ - "//source/common/init:manager_lib", - "//source/extensions/common/matcher:trie_matcher_lib", - "//source/server:listener_manager_lib", - "//test/mocks/init:init_mocks", - "//test/mocks/matcher:matcher_mocks", - "//test/mocks/network:network_mocks", - "//test/mocks/server:drain_manager_mocks", - "//test/mocks/server:guard_dog_mocks", - "//test/mocks/server:instance_mocks", - "//test/mocks/server:listener_component_factory_mocks", - "//test/mocks/server:worker_factory_mocks", - "//test/mocks/server:worker_mocks", - "//test/mocks/stream_info:stream_info_mocks", - "//test/test_common:environment_lib", - "//test/test_common:simulated_time_system_lib", - "//test/test_common:test_runtime_lib", - "//test/test_common:test_time_lib", - "//test/test_common:threadsafe_singleton_injector_lib", - "@envoy_api//envoy/admin/v3:pkg_cc_proto", - "@envoy_api//envoy/config/core/v3:pkg_cc_proto", - "@envoy_api//envoy/config/listener/v3:pkg_cc_proto", - ], -) - -envoy_cc_test( - name = "listener_manager_impl_test", - srcs = ["listener_manager_impl_test.cc"], - deps = [ - ":listener_manager_impl_test_lib", - ":utility_lib", - "//source/common/api:os_sys_calls_lib", - "//source/common/config:metadata_lib", - "//source/common/network:addr_family_aware_socket_option_lib", - "//source/common/network:listen_socket_lib", - "//source/common/network:socket_option_lib", - "//source/common/network:utility_lib", - "//source/common/protobuf", - "//source/extensions/filters/listener/original_dst:config", - "//source/extensions/filters/listener/proxy_protocol:config", - "//source/extensions/filters/listener/tls_inspector:config", - "//source/extensions/filters/network/http_connection_manager:config", - "//source/extensions/filters/network/tcp_proxy:config", - "//source/extensions/request_id/uuid:config", - "//source/extensions/transport_sockets/raw_buffer:config", - "//source/extensions/transport_sockets/tls:config", - "//source/extensions/transport_sockets/tls:ssl_socket_lib", - "//source/server:active_raw_udp_listener_config", - "//test/integration/filters:test_listener_filter_lib", - "//test/test_common:network_utility_lib", - "//test/test_common:registry_lib", - "//test/test_common:test_runtime_lib", - "@envoy_api//envoy/admin/v3:pkg_cc_proto", - "@envoy_api//envoy/config/core/v3:pkg_cc_proto", - "@envoy_api//envoy/config/listener/v3:pkg_cc_proto", - ], -) - -# Stand-alone quic test because of FIPS. -envoy_cc_test( - name = "listener_manager_impl_quic_only_test", - srcs = ["listener_manager_impl_quic_only_test.cc"], - tags = ["nofips"], - deps = [ - ":listener_manager_impl_test_lib", - ":utility_lib", - "//source/extensions/filters/http/router:config", - "//source/extensions/request_id/uuid:config", - "//source/extensions/transport_sockets/raw_buffer:config", - "//source/extensions/transport_sockets/tls:config", - "//test/integration/filters:test_network_filter_lib", - "//test/test_common:threadsafe_singleton_injector_lib", - "@envoy_api//envoy/config/core/v3:pkg_cc_proto", - "@envoy_api//envoy/config/listener/v3:pkg_cc_proto", - ], -) - envoy_cc_test( name = "filter_chain_manager_impl_test", srcs = ["filter_chain_manager_impl_test.cc"], @@ -363,11 +282,11 @@ envoy_cc_test( "//source/common/network:utility_lib", "//source/common/protobuf", "//source/extensions/filters/network/http_connection_manager:config", + "//source/extensions/listener_managers/listener_manager:listener_manager_lib", "//source/extensions/transport_sockets/raw_buffer:config", "//source/extensions/transport_sockets/tls:config", "//source/extensions/transport_sockets/tls:ssl_socket_lib", "//source/server:filter_chain_manager_lib", - "//source/server:listener_manager_lib", "//test/mocks/network:network_mocks", "//test/mocks/server:drain_manager_mocks", "//test/mocks/server:factory_context_mocks", diff --git a/test/server/filter_chain_manager_impl_test.cc b/test/server/filter_chain_manager_impl_test.cc index f3db9f5db033..fc844269a8ce 100644 --- a/test/server/filter_chain_manager_impl_test.cc +++ b/test/server/filter_chain_manager_impl_test.cc @@ -17,10 +17,10 @@ #include "source/common/network/socket_option_impl.h" #include "source/common/network/utility.h" #include "source/common/protobuf/protobuf.h" +#include "source/extensions/listener_managers/listener_manager/listener_impl.h" #include "source/extensions/transport_sockets/tls/ssl_socket.h" #include "source/server/configuration_impl.h" #include "source/server/filter_chain_manager_impl.h" -#include "source/server/listener_manager_impl.h" #include "test/mocks/network/mocks.h" #include "test/mocks/server/drain_manager.h" diff --git a/tools/code_format/config.yaml b/tools/code_format/config.yaml index d0f87cda7a63..8c25fb49a892 100644 --- a/tools/code_format/config.yaml +++ b/tools/code_format/config.yaml @@ -310,3 +310,4 @@ visibility_excludes: - source/extensions/path/match/uri_template/BUILD - source/extensions/path/rewrite/uri_template/BUILD - source/extensions/quic/connection_id_generator/BUILD +- source/extensions/listener_managers/listener_manager/BUILD diff --git a/tools/extensions/extensions_schema.yaml b/tools/extensions/extensions_schema.yaml index 3d787139db2a..d1fedfc3c186 100644 --- a/tools/extensions/extensions_schema.yaml +++ b/tools/extensions/extensions_schema.yaml @@ -76,6 +76,7 @@ categories: - envoy.internal_redirect_predicates - envoy.io_socket - envoy.http.original_ip_detection +- envoy.listener_manager_impl - envoy.matching.common_inputs - envoy.matching.input_matchers - envoy.tls.key_providers From a30978ec79564d4639175cf0e1fabbe2e6880b95 Mon Sep 17 00:00:00 2001 From: Keith Smiley Date: Tue, 6 Dec 2022 16:14:28 -0800 Subject: [PATCH 07/23] bazel: add another config_setting incompatible flag (#24270) Turns out the first one I added doesn't apply without this one, which I thought had already been flipped Signed-off-by: Keith Smiley --- .bazelrc | 3 ++- bazel/repositories.bzl | 7 ++++++- bazel/rules_pkg.patch | 12 ++++++++++++ 3 files changed, 20 insertions(+), 2 deletions(-) create mode 100644 bazel/rules_pkg.patch diff --git a/.bazelrc b/.bazelrc index e2d273661de6..66892483690a 100644 --- a/.bazelrc +++ b/.bazelrc @@ -31,8 +31,9 @@ build --action_env=PATH --host_action_env=PATH build --enable_platform_specific_config build --test_summary=terse -# TODO(keith): Remove once this is the default +# TODO(keith): Remove once these 2 are the default build --incompatible_config_setting_private_default_visibility +build --incompatible_enforce_config_setting_visibility # Allow tags to influence execution requirements common --experimental_allow_tags_propagation diff --git a/bazel/repositories.bzl b/bazel/repositories.bzl index 00cf975b305e..db2f8f638bf4 100644 --- a/bazel/repositories.bzl +++ b/bazel/repositories.bzl @@ -221,7 +221,12 @@ def envoy_dependencies(skip_targets = []): external_http_archive("bazel_toolchains") external_http_archive("bazel_compdb") external_http_archive("envoy_build_tools") - external_http_archive("rules_pkg") + + # TODO(keith): Remove patch when we update rules_pkg + external_http_archive( + "rules_pkg", + patches = ["@envoy//bazel:rules_pkg.patch"], + ) external_http_archive("com_github_aignas_rules_shellcheck") external_http_archive("aspect_bazel_lib") _com_github_fdio_vpp_vcl() diff --git a/bazel/rules_pkg.patch b/bazel/rules_pkg.patch new file mode 100644 index 000000000000..a7365962e298 --- /dev/null +++ b/bazel/rules_pkg.patch @@ -0,0 +1,12 @@ +--- pkg/private/BUILD ++++ pkg/private/BUILD +@@ -55,6 +55,9 @@ exports_files( + config_setting( + name = "private_stamp_detect", + values = {"stamp": "1"}, ++ # When --incompatible_config_setting_private_default_visibility is set, this fails unless this is public. ++ # TODO: refactor to clear up confusion that this is a "private" target with public access. ++ visibility = ["//visibility:public"], + ) + + py_library( From df2f00a7e2e5b841d1032ca4c2607b5e939c47bf Mon Sep 17 00:00:00 2001 From: yanjunxiang-google <78807980+yanjunxiang-google@users.noreply.github.com> Date: Tue, 6 Dec 2022 20:41:05 -0500 Subject: [PATCH 08/23] Ecds config dump recommit (#24384) * Adding back ECDS config dump support. (#23902)" which is reverted by (#24354) This reverts commit c5d61600db75c3effe25fa1a5e096d8a7936663a. * Fixing test coverage issue due to an early return and a couple of comment lines. Signed-off-by: Yanjun Xiang --- api/envoy/admin/v3/config_dump.proto | 5 + api/envoy/admin/v3/config_dump_shared.proto | 40 +++ docs/root/operations/admin.rst | 4 + source/common/filter/BUILD | 1 + source/common/filter/config_discovery_impl.cc | 5 + source/common/filter/config_discovery_impl.h | 48 ++++ test/integration/BUILD | 1 + .../extension_discovery_integration_test.cc | 248 ++++++++++++++---- ...er_extension_discovery_integration_test.cc | 192 ++++++++++++++ test/server/admin/BUILD | 1 + test/server/admin/config_dump_handler_test.cc | 68 +++++ 11 files changed, 562 insertions(+), 51 deletions(-) diff --git a/api/envoy/admin/v3/config_dump.proto b/api/envoy/admin/v3/config_dump.proto index 9ec135a4212f..8f5fa096e3f5 100644 --- a/api/envoy/admin/v3/config_dump.proto +++ b/api/envoy/admin/v3/config_dump.proto @@ -32,6 +32,8 @@ message ConfigDump { // // * ``bootstrap``: :ref:`BootstrapConfigDump ` // * ``clusters``: :ref:`ClustersConfigDump ` + // * ``ecds_filter_http``: :ref:`EcdsConfigDump ` + // * ``ecds_filter_tcp_listener``: :ref:`EcdsConfigDump ` // * ``endpoints``: :ref:`EndpointsConfigDump ` // * ``listeners``: :ref:`ListenersConfigDump ` // * ``scoped_routes``: :ref:`ScopedRoutesConfigDump ` @@ -40,6 +42,9 @@ message ConfigDump { // // EDS Configuration will only be dumped by using parameter ``?include_eds`` // + // Currently ECDS is supported in HTTP and listener filters. Note, ECDS configuration for + // either HTTP or listener filter will only be dumped if it is actually configured. + // // You can filter output with the resource and mask query parameters. // See :ref:`/config_dump?resource={} `, // :ref:`/config_dump?mask={} `, diff --git a/api/envoy/admin/v3/config_dump_shared.proto b/api/envoy/admin/v3/config_dump_shared.proto index 6677dac586d8..8de77e18e1f8 100644 --- a/api/envoy/admin/v3/config_dump_shared.proto +++ b/api/envoy/admin/v3/config_dump_shared.proto @@ -370,3 +370,43 @@ message EndpointsConfigDump { // The dynamically loaded endpoint configs. repeated DynamicEndpointConfig dynamic_endpoint_configs = 3; } + +// Envoy's ECDS service fills this message with all currently extension +// configuration. Extension configuration information can be used to recreate +// an Envoy ECDS listener and HTTP filters as static filters or by returning +// them in ECDS response. +message EcdsConfigDump { + option (udpa.annotations.versioning).previous_message_type = "envoy.admin.v2alpha.EcdsConfigDump"; + + // [#next-free-field: 6] + message EcdsFilterConfig { + option (udpa.annotations.versioning).previous_message_type = + "envoy.admin.v2alpha.EcdsConfigDump.EcdsFilterConfig"; + + // This is the per-resource version information. This version is currently + // taken from the :ref:`version_info + // ` + // field at the time that the ECDS filter was loaded. + string version_info = 1; + + // The ECDS filter config. + google.protobuf.Any ecds_filter = 2; + + // The timestamp when the ECDS filter was last updated. + google.protobuf.Timestamp last_updated = 3; + + // Set if the last update failed, cleared after the next successful update. + // The ``error_state`` field contains the rejected version of this + // particular resource along with the reason and timestamp. For successfully + // updated or acknowledged resource, this field should be empty. + // [#not-implemented-hide:] + UpdateFailureState error_state = 4; + + // The client status of this resource. + // [#not-implemented-hide:] + ClientResourceStatus client_status = 5; + } + + // The ECDS filter configs. + repeated EcdsFilterConfig ecds_filters = 1; +} diff --git a/docs/root/operations/admin.rst b/docs/root/operations/admin.rst index adf29ae92b14..205ac374741b 100644 --- a/docs/root/operations/admin.rst +++ b/docs/root/operations/admin.rst @@ -215,6 +215,10 @@ modify different aspects of the server: - :ref:`envoy.extensions.transport_sockets.tls.v3.Secret ` - :ref:`envoy.config.endpoint.v3.ClusterLoadAssignment ` + For ECDS config dump, the matched name field is the corresponding filter name, which is stored in: + + - :ref:`envoy.config.core.v3.TypedExtensionConfig.name ` + .. _operations_admin_interface_config_dump_by_resource_and_mask: .. http:get:: /config_dump?resource={}&mask={} diff --git a/source/common/filter/BUILD b/source/common/filter/BUILD index 2a2007f99aa6..e3180f7e4238 100644 --- a/source/common/filter/BUILD +++ b/source/common/filter/BUILD @@ -26,6 +26,7 @@ envoy_cc_library( "//source/common/init:target_lib", "//source/common/init:watcher_lib", "//source/common/protobuf:utility_lib", + "@envoy_api//envoy/admin/v3:pkg_cc_proto", "@envoy_api//envoy/config/core/v3:pkg_cc_proto", ], ) diff --git a/source/common/filter/config_discovery_impl.cc b/source/common/filter/config_discovery_impl.cc index c10d265b3c24..227b37949bbc 100644 --- a/source/common/filter/config_discovery_impl.cc +++ b/source/common/filter/config_discovery_impl.cc @@ -131,6 +131,7 @@ void FilterConfigSubscription::onConfigUpdate( last_type_url_ = type_url; last_version_info_ = version_info; last_factory_name_ = factory_name; + last_updated_ = factory_context_.timeSource().systemTime(); } void FilterConfigSubscription::onConfigUpdate( @@ -151,6 +152,7 @@ void FilterConfigSubscription::onConfigUpdate( last_type_url_ = ""; last_version_info_ = ""; last_factory_name_ = ""; + last_updated_ = factory_context_.timeSource().systemTime(); } else if (!added_resources.empty()) { onConfigUpdate(added_resources, added_resources[0].get().version()); } @@ -177,6 +179,9 @@ void FilterConfigSubscription::incrementConflictCounter() { stats_.config_confli std::shared_ptr FilterConfigProviderManagerImplBase::getSubscription( const envoy::config::core::v3::ConfigSource& config_source, const std::string& name, Server::Configuration::ServerFactoryContext& server_context, const std::string& stat_prefix) { + // There are ECDS filters configured. Setup ECDS config dump call backs. + setupEcdsConfigDumpCallbacks(server_context.admin()); + // FilterConfigSubscriptions are unique based on their config source and filter config name // combination. // TODO(https://github.com/envoyproxy/envoy/issues/11967) Hash collision can cause subscription diff --git a/source/common/filter/config_discovery_impl.h b/source/common/filter/config_discovery_impl.h index a6678458c18f..e5f06102cd7f 100644 --- a/source/common/filter/config_discovery_impl.h +++ b/source/common/filter/config_discovery_impl.h @@ -1,5 +1,6 @@ #pragma once +#include "envoy/admin/v3/config_dump.pb.h" #include "envoy/config/core/v3/extension.pb.h" #include "envoy/config/core/v3/extension.pb.validate.h" #include "envoy/config/extension_config_provider.h" @@ -7,6 +8,7 @@ #include "envoy/filter/config_provider_manager.h" #include "envoy/http/filter.h" #include "envoy/protobuf/message_validator.h" +#include "envoy/server/admin.h" #include "envoy/server/factory_context.h" #include "envoy/singleton/instance.h" #include "envoy/stats/scope.h" @@ -307,6 +309,8 @@ class FilterConfigSubscription const std::string& lastTypeUrl() { return last_type_url_; } const std::string& lastVersionInfo() { return last_version_info_; } const std::string& lastFactoryName() { return last_factory_name_; } + const SystemTime& lastUpdated() { return last_updated_; } + void incrementConflictCounter(); private: @@ -327,6 +331,7 @@ class FilterConfigSubscription std::string last_type_url_; std::string last_version_info_; std::string last_factory_name_; + SystemTime last_updated_; Server::Configuration::ServerFactoryContext& factory_context_; Init::SharedTargetImpl init_target_; @@ -387,9 +392,44 @@ class FilterConfigProviderManagerImplBase : Logger::Loggable absl::string_view type_url) const; void validateProtoConfigTypeUrl(const std::string& type_url, const absl::flat_hash_set& require_type_urls) const; + // Return the config dump map key string for the corresponding ECDS filter type. + virtual const std::string getConfigDumpType() const PURE; private: + void setupEcdsConfigDumpCallbacks(OptRef admin) { + if (admin.has_value()) { + if (config_tracker_entry_ == nullptr) { + config_tracker_entry_ = admin->getConfigTracker().add( + getConfigDumpType(), [this](const Matchers::StringMatcher& name_matcher) { + return dumpEcdsFilterConfigs(name_matcher); + }); + } + } + } + + ProtobufTypes::MessagePtr dumpEcdsFilterConfigs(const Matchers::StringMatcher& name_matcher) { + auto config_dump = std::make_unique(); + for (const auto& subscription : subscriptions_) { + const auto& ecds_filter = subscription.second.lock(); + if (!ecds_filter || !name_matcher.match(ecds_filter->name())) { + continue; + } + envoy::config::core::v3::TypedExtensionConfig filter_config; + filter_config.set_name(ecds_filter->name()); + if (ecds_filter->lastConfig()) { + filter_config.mutable_typed_config()->PackFrom(*ecds_filter->lastConfig()); + } + auto& filter_config_dump = *config_dump->mutable_ecds_filters()->Add(); + filter_config_dump.mutable_ecds_filter()->PackFrom(filter_config); + filter_config_dump.set_version_info(ecds_filter->lastVersionInfo()); + TimestampUtil::systemClockToTimestamp(ecds_filter->lastUpdated(), + *(filter_config_dump.mutable_last_updated())); + } + return config_dump; + } + absl::flat_hash_map> subscriptions_; + Server::ConfigTracker::EntryOwnerPtr config_tracker_entry_; friend class FilterConfigSubscription; }; @@ -528,6 +568,7 @@ class HttpFilterConfigProviderManagerImpl Config::Utility::validateTerminalFilters(filter_config_name, filter_type, filter_chain_type, is_terminal_filter, last_filter_in_filter_chain); } + const std::string getConfigDumpType() const override { return "ecds_filter_http"; } }; // HTTP filter @@ -554,6 +595,7 @@ class UpstreamHttpFilterConfigProviderManagerImpl Config::Utility::validateTerminalFilters(filter_config_name, filter_type, filter_chain_type, is_terminal_filter, last_filter_in_filter_chain); } + const std::string getConfigDumpType() const override { return "ecds_filter_upstream_http"; } }; // TCP listener filter @@ -564,6 +606,9 @@ class TcpListenerFilterConfigProviderManagerImpl TcpListenerDynamicFilterConfigProviderImpl> { public: absl::string_view statPrefix() const override { return "tcp_listener_filter."; } + +protected: + const std::string getConfigDumpType() const override { return "ecds_filter_tcp_listener"; } }; // UDP listener filter @@ -574,6 +619,9 @@ class UdpListenerFilterConfigProviderManagerImpl UdpListenerDynamicFilterConfigProviderImpl> { public: absl::string_view statPrefix() const override { return "udp_listener_filter."; } + +protected: + const std::string getConfigDumpType() const override { return "ecds_filter_udp_listener"; } }; } // namespace Filter diff --git a/test/integration/BUILD b/test/integration/BUILD index d33c2ee6f844..9132598bc380 100644 --- a/test/integration/BUILD +++ b/test/integration/BUILD @@ -1318,6 +1318,7 @@ envoy_cc_test( "//test/integration/filters:set_is_terminal_filter_lib", "//test/integration/filters:set_response_code_filter_config_proto_cc_proto", "//test/integration/filters:set_response_code_filter_lib", + "//test/integration/filters:test_listener_filter_lib", "//test/test_common:utility_lib", "@envoy_api//envoy/extensions/common/matching/v3:pkg_cc_proto", "@envoy_api//envoy/extensions/filters/network/http_connection_manager/v3:pkg_cc_proto", diff --git a/test/integration/extension_discovery_integration_test.cc b/test/integration/extension_discovery_integration_test.cc index af503ca060ff..04c6291142ca 100644 --- a/test/integration/extension_discovery_integration_test.cc +++ b/test/integration/extension_discovery_integration_test.cc @@ -7,6 +7,7 @@ #include "test/config/v2_link_hacks.h" #include "test/integration/filters/set_is_terminal_filter_config.pb.h" #include "test/integration/filters/set_response_code_filter_config.pb.h" +#include "test/integration/filters/test_listener_filter.pb.h" #include "test/integration/http_integration.h" #include "test/test_common/utility.h" @@ -131,17 +132,41 @@ class ExtensionDiscoveryIntegrationTest : public Grpc::GrpcClientIntegrationPara }); } + void addEcdsCluster(const std::string& cluster_name) { + // Add an xDS cluster for extension config discovery. + config_helper_.addConfigModifier( + [cluster_name](envoy::config::bootstrap::v3::Bootstrap& bootstrap) { + auto* ecds_cluster = bootstrap.mutable_static_resources()->add_clusters(); + ecds_cluster->MergeFrom(bootstrap.static_resources().clusters()[0]); + ecds_cluster->set_name(cluster_name); + ConfigHelper::setHttp2(*ecds_cluster); + }); + } + + void addDynamicListenerFilter(const std::string& name) { + config_helper_.addConfigModifier( + [name, this](envoy::config::bootstrap::v3::Bootstrap& bootstrap) { + auto* listener = bootstrap.mutable_static_resources()->mutable_listeners(0); + auto* listener_filter = listener->add_listener_filters(); + listener_filter->set_name(name); + auto* discovery = listener_filter->mutable_config_discovery(); + discovery->add_type_urls( + "type.googleapis.com/test.integration.filters.TestTcpListenerFilterConfig"); + discovery->mutable_config_source()->set_resource_api_version( + envoy::config::core::v3::ApiVersion::V3); + auto* api_config_source = discovery->mutable_config_source()->mutable_api_config_source(); + api_config_source->set_api_type(envoy::config::core::v3::ApiConfigSource::GRPC); + api_config_source->set_transport_api_version(envoy::config::core::v3::ApiVersion::V3); + auto* grpc_service = api_config_source->add_grpc_services(); + setGrpcService(*grpc_service, "ecds2_cluster", getEcds2FakeUpstream().localAddress()); + }); + } + void initialize() override { defer_listener_finalization_ = true; setUpstreamCount(1); - // Add an xDS cluster for extension config discovery. - config_helper_.addConfigModifier([](envoy::config::bootstrap::v3::Bootstrap& bootstrap) { - auto* ecds_cluster = bootstrap.mutable_static_resources()->add_clusters(); - ecds_cluster->MergeFrom(bootstrap.static_resources().clusters()[0]); - ecds_cluster->set_name("ecds_cluster"); - ConfigHelper::setHttp2(*ecds_cluster); - }); + addEcdsCluster("ecds_cluster"); // Make HCM do a direct response to avoid timing issues with the upstream. config_helper_.addConfigModifier( [](envoy::extensions::filters::network::http_connection_manager::v3::HttpConnectionManager& @@ -160,6 +185,12 @@ class ExtensionDiscoveryIntegrationTest : public Grpc::GrpcClientIntegrationPara lds_cluster->set_name("lds_cluster"); ConfigHelper::setHttp2(*lds_cluster); }); + + // In case to configure both HTTP and Listener ECDS filters, adding the 2nd ECDS cluster. + if (two_ecds_filters_) { + addEcdsCluster("ecds2_cluster"); + } + // Must be the last since it nukes static listeners. config_helper_.addConfigModifier([this](envoy::config::bootstrap::v3::Bootstrap& bootstrap) { listener_config_.Swap(bootstrap.mutable_static_resources()->mutable_listeners(0)); @@ -201,6 +232,19 @@ class ExtensionDiscoveryIntegrationTest : public Grpc::GrpcClientIntegrationPara addFakeUpstream(Http::CodecType::HTTP2); // Create the listener config discovery upstream (fake_upstreams_[2]). addFakeUpstream(Http::CodecType::HTTP2); + if (two_ecds_filters_) { + addFakeUpstream(Http::CodecType::HTTP2); + } + } + + // Wait for ECDS stream. + void waitForEcdsStream(FakeUpstream& upstream, FakeHttpConnectionPtr& connection, + FakeStreamPtr& stream) { + AssertionResult result = upstream.waitForHttpConnection(*dispatcher_, connection); + ASSERT_TRUE(result); + result = connection->waitForNewStream(*dispatcher_, stream); + ASSERT_TRUE(result); + stream->startGrpcStream(); } void waitXdsStream() { @@ -215,13 +259,11 @@ class ExtensionDiscoveryIntegrationTest : public Grpc::GrpcClientIntegrationPara // Response with initial LDS. sendLdsResponse("initial"); - // Wait for ECDS stream. - auto& ecds_upstream = getEcdsFakeUpstream(); - result = ecds_upstream.waitForHttpConnection(*dispatcher_, ecds_connection_); - RELEASE_ASSERT(result, result.message()); - result = ecds_connection_->waitForNewStream(*dispatcher_, ecds_stream_); - RELEASE_ASSERT(result, result.message()); - ecds_stream_->startGrpcStream(); + waitForEcdsStream(getEcdsFakeUpstream(), ecds_connection_, ecds_stream_); + if (two_ecds_filters_) { + // Wait for 2nd ECDS stream. + waitForEcdsStream(getEcds2FakeUpstream(), ecds2_connection_, ecds2_stream_); + } } void sendLdsResponse(const std::string& version) { @@ -232,19 +274,28 @@ class ExtensionDiscoveryIntegrationTest : public Grpc::GrpcClientIntegrationPara lds_stream_->sendGrpcMessage(response); } - void sendXdsResponse(const std::string& name, const std::string& version, - const std::string& yaml_config, bool ttl = false, - bool is_set_resp_code_config = true) { + void sendEcdsResponse(const envoy::config::core::v3::TypedExtensionConfig& typed_config, + const std::string& name, const std::string& version, const bool ttl, + FakeStreamPtr& ecds_stream) { + envoy::service::discovery::v3::Resource resource; + resource.set_name(name); + if (ttl) { + resource.mutable_ttl()->set_seconds(1); + } + resource.mutable_resource()->PackFrom(typed_config); + envoy::service::discovery::v3::DiscoveryResponse response; response.set_version_info(version); response.set_type_url("type.googleapis.com/envoy.config.core.v3.TypedExtensionConfig"); + response.add_resources()->PackFrom(resource); + ecds_stream->sendGrpcMessage(response); + } + void sendHttpFilterEcdsResponse(const std::string& name, const std::string& version, + const std::string& yaml_config, bool ttl = false, + bool is_set_resp_code_config = true) { envoy::config::core::v3::TypedExtensionConfig typed_config; typed_config.set_name(name); - - envoy::service::discovery::v3::Resource resource; - resource.set_name(name); - if (is_set_resp_code_config) { const auto configuration = TestUtility::parseYaml( @@ -256,29 +307,48 @@ class ExtensionDiscoveryIntegrationTest : public Grpc::GrpcClientIntegrationPara yaml_config); typed_config.mutable_typed_config()->PackFrom(configuration); } - resource.mutable_resource()->PackFrom(typed_config); - if (ttl) { - resource.mutable_ttl()->set_seconds(1); - } - response.add_resources()->PackFrom(resource); - ecds_stream_->sendGrpcMessage(response); + sendEcdsResponse(typed_config, name, version, ttl, ecds_stream_); } - void sendXdsResponseWithFullYaml(const std::string& name, const std::string& version, - const std::string& full_yaml) { - envoy::service::discovery::v3::DiscoveryResponse response; - response.set_version_info(version); - response.set_type_url("type.googleapis.com/envoy.config.core.v3.TypedExtensionConfig"); + void sendListenerFilterEcdsResponse(const std::string& name, const std::string& version, + const uint32_t drain_bytes) { + envoy::config::core::v3::TypedExtensionConfig typed_config; + typed_config.set_name(name); + auto configuration = test::integration::filters::TestTcpListenerFilterConfig(); + configuration.set_drain_bytes(drain_bytes); + typed_config.mutable_typed_config()->PackFrom(configuration); + sendEcdsResponse(typed_config, name, version, false, ecds2_stream_); + } + + void sendHttpFilterEcdsResponseWithFullYaml(const std::string& name, const std::string& version, + const std::string& full_yaml) { const auto configuration = TestUtility::parseYaml(full_yaml); envoy::config::core::v3::TypedExtensionConfig typed_config; typed_config.set_name(name); typed_config.mutable_typed_config()->MergeFrom(configuration); - response.add_resources()->PackFrom(typed_config); - ecds_stream_->sendGrpcMessage(response); + sendEcdsResponse(typed_config, name, version, false, ecds_stream_); } + absl::string_view request(const std::string port_key, const std::string method, + const std::string endpoint, BufferingStreamDecoderPtr& response) { + response = IntegrationUtil::makeSingleRequest(lookupPort(port_key), method, endpoint, "", + Http::CodecType::HTTP1, version_); + EXPECT_TRUE(response->complete()); + return response->headers().getStatusValue(); + } + + absl::string_view contentType(const BufferingStreamDecoderPtr& response) { + const Http::HeaderEntry* entry = response->headers().ContentType(); + if (entry == nullptr) { + return "(null)"; + } + return entry->value().getStringView(); + } + + bool two_ecds_filters_{false}; FakeUpstream& getEcdsFakeUpstream() const { return *fake_upstreams_[1]; } FakeUpstream& getLdsFakeUpstream() const { return *fake_upstreams_[2]; } + FakeUpstream& getEcds2FakeUpstream() const { return *fake_upstreams_[3]; } // gRPC LDS set-up envoy::config::listener::v3::Listener listener_config_; @@ -289,6 +359,8 @@ class ExtensionDiscoveryIntegrationTest : public Grpc::GrpcClientIntegrationPara // gRPC ECDS set-up FakeHttpConnectionPtr ecds_connection_{nullptr}; FakeStreamPtr ecds_stream_{nullptr}; + FakeHttpConnectionPtr ecds2_connection_{nullptr}; + FakeStreamPtr ecds2_stream_{nullptr}; }; INSTANTIATE_TEST_SUITE_P(IpVersionsClientType, ExtensionDiscoveryIntegrationTest, @@ -301,7 +373,7 @@ TEST_P(ExtensionDiscoveryIntegrationTest, BasicSuccess) { test_server_->waitForCounterGe("listener_manager.lds.update_success", 1); EXPECT_EQ(test_server_->server().initManager().state(), Init::Manager::State::Initializing); registerTestServerPorts({"http"}); - sendXdsResponse("foo", "1", denyPrivateConfig()); + sendHttpFilterEcdsResponse("foo", "1", denyPrivateConfig()); test_server_->waitForCounterGe("extension_config_discovery.http_filter.foo.config_reload", 1); test_server_->waitUntilListenersReady(); test_server_->waitForGaugeGe("listener_manager.workers_started", 1); @@ -325,7 +397,7 @@ TEST_P(ExtensionDiscoveryIntegrationTest, BasicSuccess) { } // Update again but keep the connection. { - sendXdsResponse("foo", "2", allowAllConfig()); + sendHttpFilterEcdsResponse("foo", "2", allowAllConfig()); test_server_->waitForCounterGe("extension_config_discovery.http_filter.foo.config_reload", 2); auto response = codec_client_->makeHeaderOnlyRequest(banned_request_headers); ASSERT_TRUE(response->waitForEndStream()); @@ -341,7 +413,7 @@ TEST_P(ExtensionDiscoveryIntegrationTest, BasicSuccessWithTtl) { test_server_->waitForCounterGe("listener_manager.lds.update_success", 1); EXPECT_EQ(test_server_->server().initManager().state(), Init::Manager::State::Initializing); registerTestServerPorts({"http"}); - sendXdsResponse("foo", "1", denyPrivateConfig(), true); + sendHttpFilterEcdsResponse("foo", "1", denyPrivateConfig(), true); test_server_->waitForCounterGe("extension_config_discovery.http_filter.foo.config_reload", 1); test_server_->waitUntilListenersReady(); test_server_->waitForGaugeGe("listener_manager.workers_started", 1); @@ -376,7 +448,7 @@ TEST_P(ExtensionDiscoveryIntegrationTest, BasicSuccessWithTtl) { { // Reinstate the previous configuration. - sendXdsResponse("foo", "1", denyPrivateConfig(), true); + sendHttpFilterEcdsResponse("foo", "1", denyPrivateConfig(), true); // Wait until the new configuration has been applied. test_server_->waitForCounterGe("extension_config_discovery.http_filter.foo.config_reload", 3); auto response = codec_client_->makeHeaderOnlyRequest(banned_request_headers); @@ -393,7 +465,7 @@ TEST_P(ExtensionDiscoveryIntegrationTest, BasicSuccessWithTtlWithDefault) { test_server_->waitForCounterGe("listener_manager.lds.update_success", 1); EXPECT_EQ(test_server_->server().initManager().state(), Init::Manager::State::Initializing); registerTestServerPorts({"http"}); - sendXdsResponse("foo", "1", allowAllConfig(), true); + sendHttpFilterEcdsResponse("foo", "1", allowAllConfig(), true); test_server_->waitForCounterGe("extension_config_discovery.http_filter.foo.config_reload", 1); test_server_->waitUntilListenersReady(); test_server_->waitForGaugeGe("listener_manager.workers_started", 1); @@ -427,7 +499,7 @@ TEST_P(ExtensionDiscoveryIntegrationTest, BasicSuccessWithMatcher) { test_server_->waitForCounterGe("listener_manager.lds.update_success", 1); EXPECT_EQ(test_server_->server().initManager().state(), Init::Manager::State::Initializing); registerTestServerPorts({"http"}); - sendXdsResponseWithFullYaml("foo", "1", denyPrivateConfigWithMatcher()); + sendHttpFilterEcdsResponseWithFullYaml("foo", "1", denyPrivateConfigWithMatcher()); test_server_->waitForCounterGe("extension_config_discovery.http_filter.foo.config_reload", 1); test_server_->waitUntilListenersReady(); test_server_->waitForGaugeGe("listener_manager.workers_started", 1); @@ -469,7 +541,7 @@ TEST_P(ExtensionDiscoveryIntegrationTest, BasicDefaultMatcher) { test_server_->waitForCounterGe("listener_manager.lds.update_success", 1); EXPECT_EQ(test_server_->server().initManager().state(), Init::Manager::State::Initializing); registerTestServerPorts({"http"}); - sendXdsResponse("foo", "1", invalidConfig()); + sendHttpFilterEcdsResponse("foo", "1", invalidConfig()); test_server_->waitForCounterGe("extension_config_discovery.http_filter.foo.config_fail", 1); test_server_->waitUntilListenersReady(); test_server_->waitForGaugeGe("listener_manager.workers_started", 1); @@ -506,7 +578,7 @@ TEST_P(ExtensionDiscoveryIntegrationTest, ReuseExtensionConfig) { test_server_->waitForCounterGe("listener_manager.lds.update_success", 1); EXPECT_EQ(test_server_->server().initManager().state(), Init::Manager::State::Initializing); registerTestServerPorts({"http"}); - sendXdsResponse("foo", "1", allowAllConfig()); + sendHttpFilterEcdsResponse("foo", "1", allowAllConfig()); test_server_->waitForCounterGe("extension_config_discovery.http_filter.foo.config_reload", 1); test_server_->waitUntilListenersReady(); test_server_->waitForGaugeGe("listener_manager.workers_started", 1); @@ -537,7 +609,7 @@ TEST_P(ExtensionDiscoveryIntegrationTest, ReuseExtensionConfigInvalid) { test_server_->waitForCounterGe("listener_manager.lds.update_success", 1); EXPECT_EQ(test_server_->server().initManager().state(), Init::Manager::State::Initializing); registerTestServerPorts({"http"}); - sendXdsResponseWithFullYaml("foo", "1", denyPrivateConfigWithMatcher()); + sendHttpFilterEcdsResponseWithFullYaml("foo", "1", denyPrivateConfigWithMatcher()); test_server_->waitForCounterGe("extension_config_discovery.http_filter.foo.config_reload", 1); test_server_->waitUntilListenersReady(); test_server_->waitForGaugeGe("listener_manager.workers_started", 1); @@ -574,7 +646,7 @@ TEST_P(ExtensionDiscoveryIntegrationTest, BasicFailWithDefault) { test_server_->waitForCounterGe("listener_manager.lds.update_success", 1); EXPECT_EQ(test_server_->server().initManager().state(), Init::Manager::State::Initializing); registerTestServerPorts({"http"}); - sendXdsResponse("foo", "1", invalidConfig()); + sendHttpFilterEcdsResponse("foo", "1", invalidConfig()); test_server_->waitForCounterGe("extension_config_discovery.http_filter.foo.config_fail", 1); test_server_->waitUntilListenersReady(); test_server_->waitForGaugeGe("listener_manager.workers_started", 1); @@ -595,7 +667,7 @@ TEST_P(ExtensionDiscoveryIntegrationTest, BasicFailWithoutDefault) { test_server_->waitForCounterGe("listener_manager.lds.update_success", 1); EXPECT_EQ(test_server_->server().initManager().state(), Init::Manager::State::Initializing); registerTestServerPorts({"http"}); - sendXdsResponse("foo", "1", invalidConfig()); + sendHttpFilterEcdsResponse("foo", "1", invalidConfig()); test_server_->waitForCounterGe("extension_config_discovery.http_filter.foo.config_fail", 1); test_server_->waitUntilListenersReady(); test_server_->waitForGaugeGe("listener_manager.workers_started", 1); @@ -630,7 +702,7 @@ TEST_P(ExtensionDiscoveryIntegrationTest, BasicWithoutWarming) { } // Update should cause a different response. - sendXdsResponse("bar", "1", denyPrivateConfig()); + sendHttpFilterEcdsResponse("bar", "1", denyPrivateConfig()); test_server_->waitForCounterGe("extension_config_discovery.http_filter.bar.config_reload", 1); { auto response = codec_client_->makeHeaderOnlyRequest(request_headers); @@ -651,7 +723,7 @@ TEST_P(ExtensionDiscoveryIntegrationTest, BasicWithoutWarmingFail) { test_server_->waitForGaugeGe("listener_manager.workers_started", 1); codec_client_ = makeHttpConnection(makeClientConnection((lookupPort("http")))); // Update should not cause a different response. - sendXdsResponse("bar", "1", invalidConfig()); + sendHttpFilterEcdsResponse("bar", "1", invalidConfig()); test_server_->waitForCounterGe("extension_config_discovery.http_filter.bar.config_fail", 1); Http::TestRequestHeaderMapImpl request_headers{ {":method", "GET"}, {":path", "/"}, {":scheme", "http"}, {":authority", "host"}}; @@ -669,7 +741,7 @@ TEST_P(ExtensionDiscoveryIntegrationTest, BasicTwoSubscriptionsSameName) { test_server_->waitForCounterGe("listener_manager.lds.update_success", 1); EXPECT_EQ(test_server_->server().initManager().state(), Init::Manager::State::Initializing); registerTestServerPorts({"http"}); - sendXdsResponse("baz", "1", denyPrivateConfig()); + sendHttpFilterEcdsResponse("baz", "1", denyPrivateConfig()); test_server_->waitForCounterGe("extension_config_discovery.http_filter.baz.config_reload", 1); test_server_->waitUntilListenersReady(); test_server_->waitForGaugeGe("listener_manager.workers_started", 1); @@ -708,7 +780,7 @@ TEST_P(ExtensionDiscoveryIntegrationTest, BasicFailTerminalFilterNotAtEndOfFilte test_server_->waitForCounterGe("listener_manager.lds.update_success", 1); EXPECT_EQ(test_server_->server().initManager().state(), Init::Manager::State::Initializing); registerTestServerPorts({"http"}); - sendXdsResponse("foo", "1", terminalFilterConfig(), false, false); + sendHttpFilterEcdsResponse("foo", "1", terminalFilterConfig(), false, false); test_server_->waitForCounterGe("extension_config_discovery.http_filter.foo.config_fail", 1); test_server_->waitUntilListenersReady(); test_server_->waitForGaugeGe("listener_manager.workers_started", 1); @@ -730,7 +802,7 @@ TEST_P(ExtensionDiscoveryIntegrationTest, ReloadBoth) { test_server_->waitForCounterGe("listener_manager.lds.update_success", 1); EXPECT_EQ(test_server_->server().initManager().state(), Init::Manager::State::Initializing); registerTestServerPorts({"http"}); - sendXdsResponse("foo", "1", denyPrivateConfig()); + sendHttpFilterEcdsResponse("foo", "1", denyPrivateConfig()); test_server_->waitForCounterGe("extension_config_discovery.http_filter.foo.config_reload", 1); test_server_->waitUntilListenersReady(); test_server_->waitForGaugeGe("listener_manager.workers_started", 1); @@ -765,7 +837,7 @@ TEST_P(ExtensionDiscoveryIntegrationTest, ReloadBoth) { // Update ECDS but keep the connection. { - sendXdsResponse("foo", "2", allowAllConfig()); + sendHttpFilterEcdsResponse("foo", "2", allowAllConfig()); test_server_->waitForCounterGe("extension_config_discovery.http_filter.foo.config_reload", 2); auto response = codec_client_->makeHeaderOnlyRequest(banned_request_headers); ASSERT_TRUE(response->waitForEndStream()); @@ -775,5 +847,79 @@ TEST_P(ExtensionDiscoveryIntegrationTest, ReloadBoth) { codec_client_->close(); } +// ECDS config dump test with one listener ECDS filter and one HTTP ECDS filter. +TEST_P(ExtensionDiscoveryIntegrationTest, ConfigDumpWithTwoSubscriptionTypes) { + DISABLE_IF_ADMIN_DISABLED; // Uses admin interface. + two_ecds_filters_ = true; + on_server_init_function_ = [&]() { waitXdsStream(); }; + // HTTP ECDS filter + addDynamicFilter("foo", false); + // Listener ECDS filter + addDynamicListenerFilter("bar"); + initialize(); + + test_server_->waitForCounterGe("listener_manager.lds.update_success", 1); + EXPECT_EQ(test_server_->server().initManager().state(), Init::Manager::State::Initializing); + registerTestServerPorts({"http"}); + + // Send configuration update for HTTP ECDS filter. + sendHttpFilterEcdsResponse("foo", "1", denyPrivateConfig()); + // Send configuration update for listener ECDS filter. + sendListenerFilterEcdsResponse("bar", "2", 7); + test_server_->waitForCounterGe("extension_config_discovery.http_filter.foo.config_reload", 1); + test_server_->waitUntilListenersReady(); + test_server_->waitForGaugeGe("listener_manager.workers_started", 1); + EXPECT_EQ(test_server_->server().initManager().state(), Init::Manager::State::Initialized); + + // Get config_dump and verify HTTP and Listener ECDS filters are dumped correctly. + BufferingStreamDecoderPtr response; + EXPECT_EQ("200", request("admin", "GET", "/config_dump", response)); + EXPECT_EQ("application/json", contentType(response)); + Json::ObjectSharedPtr json = Json::Factory::loadFromString(response->body()); + size_t index = 0; + const std::string expected_types[] = { + "type.googleapis.com/envoy.admin.v3.BootstrapConfigDump", + "type.googleapis.com/envoy.admin.v3.ClustersConfigDump", + "type.googleapis.com/envoy.admin.v3.EcdsConfigDump", // HTTP + "type.googleapis.com/envoy.admin.v3.EcdsConfigDump", // TCP Listener + "type.googleapis.com/envoy.admin.v3.ListenersConfigDump", + "type.googleapis.com/envoy.admin.v3.ScopedRoutesConfigDump", + "type.googleapis.com/envoy.admin.v3.RoutesConfigDump", + "type.googleapis.com/envoy.admin.v3.SecretsConfigDump"}; + + for (const Json::ObjectSharedPtr& obj_ptr : json->getObjectArray("configs")) { + EXPECT_TRUE(expected_types[index].compare(obj_ptr->getString("@type")) == 0); + index++; + } + + // Validate we can parse as proto. + envoy::admin::v3::ConfigDump config_dump; + TestUtility::loadFromJson(response->body(), config_dump); + EXPECT_EQ(8, config_dump.configs_size()); + + // Unpack the HTTP filter. + envoy::admin::v3::EcdsConfigDump ecds_config_dump_http; + config_dump.configs(2).UnpackTo(&ecds_config_dump_http); + EXPECT_EQ("1", ecds_config_dump_http.ecds_filters(0).version_info()); + envoy::config::core::v3::TypedExtensionConfig http_filter_config; + EXPECT_TRUE(ecds_config_dump_http.ecds_filters(0).ecds_filter().UnpackTo(&http_filter_config)); + EXPECT_EQ("foo", http_filter_config.name()); + test::integration::filters::SetResponseCodeFilterConfig http_config; + http_filter_config.typed_config().UnpackTo(&http_config); + EXPECT_EQ("/private", http_config.prefix()); + EXPECT_EQ(403, http_config.code()); + + // Unpack the listener filter. + envoy::admin::v3::EcdsConfigDump ecds_config_dump_listener; + config_dump.configs(3).UnpackTo(&ecds_config_dump_listener); + EXPECT_EQ("2", ecds_config_dump_listener.ecds_filters(0).version_info()); + envoy::config::core::v3::TypedExtensionConfig filter_config; + EXPECT_TRUE(ecds_config_dump_listener.ecds_filters(0).ecds_filter().UnpackTo(&filter_config)); + EXPECT_EQ("bar", filter_config.name()); + test::integration::filters::TestTcpListenerFilterConfig listener_config; + filter_config.typed_config().UnpackTo(&listener_config); + EXPECT_EQ(7, listener_config.drain_bytes()); +} + } // namespace } // namespace Envoy diff --git a/test/integration/listener_extension_discovery_integration_test.cc b/test/integration/listener_extension_discovery_integration_test.cc index 36524805d127..1c55b3a7b930 100644 --- a/test/integration/listener_extension_discovery_integration_test.cc +++ b/test/integration/listener_extension_discovery_integration_test.cc @@ -17,6 +17,12 @@ enum class ListenerMatcherType { NULLMATCHER, ANYMATCHER, NOTANYMATCHER }; constexpr absl::string_view EcdsClusterName = "ecds_cluster"; constexpr absl::string_view Ecds2ClusterName = "ecds2_cluster"; +constexpr absl::string_view expected_types[] = { + "type.googleapis.com/envoy.admin.v3.BootstrapConfigDump", + "type.googleapis.com/envoy.admin.v3.ClustersConfigDump", + "type.googleapis.com/envoy.admin.v3.EcdsConfigDump", + "type.googleapis.com/envoy.admin.v3.ListenersConfigDump", + "type.googleapis.com/envoy.admin.v3.SecretsConfigDump"}; class ListenerExtensionDiscoveryIntegrationTest : public Grpc::GrpcClientIntegrationParamTest, public BaseIntegrationTest { @@ -222,6 +228,39 @@ class ListenerExtensionDiscoveryIntegrationTest : public Grpc::GrpcClientIntegra tcp_client->close(); } + // Verify ECDS config dump data. + bool + verifyConfigDumpData(envoy::config::core::v3::TypedExtensionConfig filter_config, + test::integration::filters::TestTcpListenerFilterConfig listener_config) { + // There is no ordering. i.e, either foo or bar could be the 1st in the config dump. + if (filter_config.name() == "foo") { + EXPECT_EQ(3, listener_config.drain_bytes()); + return true; + } else if (filter_config.name() == "bar") { + EXPECT_EQ(4, listener_config.drain_bytes()); + return true; + } else { + return false; + } + } + + // Utilities used for config dump. + absl::string_view request(const std::string port_key, const std::string method, + const std::string endpoint, BufferingStreamDecoderPtr& response) { + response = IntegrationUtil::makeSingleRequest(lookupPort(port_key), method, endpoint, "", + Http::CodecType::HTTP1, version_); + EXPECT_TRUE(response->complete()); + return response->headers().getStatusValue(); + } + + absl::string_view contentType(const BufferingStreamDecoderPtr& response) { + const Http::HeaderEntry* entry = response->headers().ContentType(); + if (entry == nullptr) { + return "(null)"; + } + return entry->value().getStringView(); + } + const uint32_t default_drain_bytes_{2}; const std::string filter_name_; const std::string data_; @@ -537,5 +576,158 @@ TEST_P(ListenerExtensionDiscoveryIntegrationTest, DestroyDuringInit) { ecds_connection_.reset(); } +// Basic ECDS config dump test with one filter. +TEST_P(ListenerExtensionDiscoveryIntegrationTest, BasicSuccessWithConfigDump) { + DISABLE_IF_ADMIN_DISABLED; // Uses admin interface. + on_server_init_function_ = [&]() { waitXdsStream(); }; + addDynamicFilter(filter_name_, false); + initialize(); + + EXPECT_EQ(test_server_->server().initManager().state(), Init::Manager::State::Initializing); + + // Send 1st config update to have listener filter drain 5 bytes of data. + sendXdsResponse(filter_name_, "1", 5); + test_server_->waitForCounterGe( + "extension_config_discovery.tcp_listener_filter." + filter_name_ + ".config_reload", 1); + + // Verify ECDS config dump are working correctly. + BufferingStreamDecoderPtr response; + EXPECT_EQ("200", request("admin", "GET", "/config_dump", response)); + EXPECT_EQ("application/json", contentType(response)); + Json::ObjectSharedPtr json = Json::Factory::loadFromString(response->body()); + size_t index = 0; + for (const Json::ObjectSharedPtr& obj_ptr : json->getObjectArray("configs")) { + EXPECT_TRUE(expected_types[index].compare(obj_ptr->getString("@type")) == 0); + index++; + } + + // Validate we can parse as proto. + envoy::admin::v3::ConfigDump config_dump; + TestUtility::loadFromJson(response->body(), config_dump); + EXPECT_EQ(5, config_dump.configs_size()); + + // With /config_dump, the response has the format: EcdsConfigDump. + envoy::admin::v3::EcdsConfigDump ecds_config_dump; + config_dump.configs(2).UnpackTo(&ecds_config_dump); + EXPECT_EQ("1", ecds_config_dump.ecds_filters(0).version_info()); + envoy::config::core::v3::TypedExtensionConfig filter_config; + EXPECT_TRUE(ecds_config_dump.ecds_filters(0).ecds_filter().UnpackTo(&filter_config)); + EXPECT_EQ("foo", filter_config.name()); + test::integration::filters::TestTcpListenerFilterConfig listener_config; + filter_config.typed_config().UnpackTo(&listener_config); + EXPECT_EQ(5, listener_config.drain_bytes()); +} + +// ECDS config dump test with the filter configuration being removed by TTL expired. +TEST_P(ListenerExtensionDiscoveryIntegrationTest, ConfigDumpWithFilterConfigRemovedByTtl) { + DISABLE_IF_ADMIN_DISABLED; // Uses admin interface. + on_server_init_function_ = [&]() { waitXdsStream(); }; + addDynamicFilter(filter_name_, false, false); + initialize(); + + EXPECT_EQ(test_server_->server().initManager().state(), Init::Manager::State::Initializing); + // Send config update with TTL 1s. + sendXdsResponse(filter_name_, "1", 5, true); + test_server_->waitForCounterGe( + "extension_config_discovery.tcp_listener_filter." + filter_name_ + ".config_reload", 1); + // Wait for configuration expired. + test_server_->waitForCounterGe( + "extension_config_discovery.tcp_listener_filter." + filter_name_ + ".config_reload", 2); + + BufferingStreamDecoderPtr response; + EXPECT_EQ("200", request("admin", "GET", "/config_dump?resource=ecds_filters", response)); + envoy::admin::v3::ConfigDump config_dump; + TestUtility::loadFromJson(response->body(), config_dump); + // With /config_dump?resource=ecds_filters, the response has the format: EcdsFilterConfig. + envoy::admin::v3::EcdsConfigDump::EcdsFilterConfig ecds_msg; + config_dump.configs(0).UnpackTo(&ecds_msg); + EXPECT_EQ("", ecds_msg.version_info()); + envoy::config::core::v3::TypedExtensionConfig filter_config; + EXPECT_TRUE(ecds_msg.ecds_filter().UnpackTo(&filter_config)); + EXPECT_EQ("foo", filter_config.name()); + // Verify ECDS config dump doesn't have the filter configuration. + EXPECT_EQ(false, filter_config.has_typed_config()); +} + +// ECDS config dump test with two filters. +TEST_P(ListenerExtensionDiscoveryIntegrationTest, TwoSubscriptionsSameFilterTypeWithConfigDump) { + DISABLE_IF_ADMIN_DISABLED; // Uses admin interface. + two_connections_ = true; + on_server_init_function_ = [&]() { waitXdsStream(); }; + addDynamicFilter("foo", true); + addDynamicFilter("bar", false, true, false, ListenerMatcherType::NULLMATCHER, true); + initialize(); + EXPECT_EQ(test_server_->server().initManager().state(), Init::Manager::State::Initializing); + + sendXdsResponse("foo", "1", 3); + sendXdsResponse("bar", "1", 4, false, true); + test_server_->waitForCounterGe("extension_config_discovery.tcp_listener_filter.foo.config_reload", + 1); + test_server_->waitForCounterGe("extension_config_discovery.tcp_listener_filter.bar.config_reload", + 1); + // Verify ECDS config dump are working correctly. + BufferingStreamDecoderPtr response; + EXPECT_EQ("200", request("admin", "GET", "/config_dump", response)); + EXPECT_EQ("application/json", contentType(response)); + Json::ObjectSharedPtr json = Json::Factory::loadFromString(response->body()); + size_t index = 0; + for (const Json::ObjectSharedPtr& obj_ptr : json->getObjectArray("configs")) { + EXPECT_TRUE(expected_types[index].compare(obj_ptr->getString("@type")) == 0); + index++; + } + + envoy::admin::v3::ConfigDump config_dump; + TestUtility::loadFromJson(response->body(), config_dump); + EXPECT_EQ(5, config_dump.configs_size()); + envoy::admin::v3::EcdsConfigDump ecds_config_dump; + config_dump.configs(2).UnpackTo(&ecds_config_dump); + envoy::config::core::v3::TypedExtensionConfig filter_config; + test::integration::filters::TestTcpListenerFilterConfig listener_config; + // Verify the first filter. + EXPECT_EQ("1", ecds_config_dump.ecds_filters(0).version_info()); + EXPECT_TRUE(ecds_config_dump.ecds_filters(0).ecds_filter().UnpackTo(&filter_config)); + filter_config.typed_config().UnpackTo(&listener_config); + EXPECT_TRUE(verifyConfigDumpData(filter_config, listener_config)); + // Verify the second filter. + EXPECT_EQ("1", ecds_config_dump.ecds_filters(1).version_info()); + EXPECT_TRUE(ecds_config_dump.ecds_filters(1).ecds_filter().UnpackTo(&filter_config)); + filter_config.typed_config().UnpackTo(&listener_config); + EXPECT_TRUE(verifyConfigDumpData(filter_config, listener_config)); +} + +// ECDS config dump test with specified resource and regex name search. +TEST_P(ListenerExtensionDiscoveryIntegrationTest, TwoSubscriptionsConfigDumpWithResourceAndRegex) { + DISABLE_IF_ADMIN_DISABLED; // Uses admin interface. + two_connections_ = true; + on_server_init_function_ = [&]() { waitXdsStream(); }; + addDynamicFilter("foo", true); + addDynamicFilter("bar", false, true, false, ListenerMatcherType::NULLMATCHER, true); + initialize(); + EXPECT_EQ(test_server_->server().initManager().state(), Init::Manager::State::Initializing); + + sendXdsResponse("foo", "1", 3); + sendXdsResponse("bar", "1", 4, false, true); + test_server_->waitForCounterGe("extension_config_discovery.tcp_listener_filter.foo.config_reload", + 1); + test_server_->waitForCounterGe("extension_config_discovery.tcp_listener_filter.bar.config_reload", + 1); + BufferingStreamDecoderPtr response; + EXPECT_EQ("200", + request("admin", "GET", "/config_dump?resource=ecds_filters&name_regex=.a.", response)); + + envoy::admin::v3::ConfigDump config_dump; + TestUtility::loadFromJson(response->body(), config_dump); + EXPECT_EQ(1, config_dump.configs_size()); + envoy::admin::v3::EcdsConfigDump::EcdsFilterConfig ecds_msg; + config_dump.configs(0).UnpackTo(&ecds_msg); + EXPECT_EQ("1", ecds_msg.version_info()); + envoy::config::core::v3::TypedExtensionConfig filter_config; + EXPECT_TRUE(ecds_msg.ecds_filter().UnpackTo(&filter_config)); + EXPECT_EQ("bar", filter_config.name()); + test::integration::filters::TestTcpListenerFilterConfig listener_config; + filter_config.typed_config().UnpackTo(&listener_config); + EXPECT_EQ(4, listener_config.drain_bytes()); +} + } // namespace } // namespace Envoy diff --git a/test/server/admin/BUILD b/test/server/admin/BUILD index 7eae434ab61c..b6242f2f96cf 100644 --- a/test/server/admin/BUILD +++ b/test/server/admin/BUILD @@ -211,6 +211,7 @@ envoy_cc_test( srcs = envoy_select_admin_functionality(["config_dump_handler_test.cc"]), deps = [ ":admin_instance_lib", + "//test/integration/filters:test_listener_filter_lib", ], ) diff --git a/test/server/admin/config_dump_handler_test.cc b/test/server/admin/config_dump_handler_test.cc index 13fd66efeb98..bc3c1b0ed520 100644 --- a/test/server/admin/config_dump_handler_test.cc +++ b/test/server/admin/config_dump_handler_test.cc @@ -1,3 +1,4 @@ +#include "test/integration/filters/test_listener_filter.pb.h" #include "test/server/admin/admin_instance.h" using testing::HasSubstr; @@ -790,5 +791,72 @@ TEST_P(AdminInstanceTest, FieldMasksWorkWhenFetchingAllResources) { response.toString()); } +ProtobufTypes::MessagePtr testDumpEcdsConfig(const Matchers::StringMatcher&) { + auto msg = std::make_unique(); + auto* ecds = msg->mutable_ecds_filters()->Add(); + ecds->set_version_info("1"); + ecds->mutable_last_updated()->set_seconds(5); + + envoy::config::core::v3::TypedExtensionConfig filter_config; + filter_config.set_name("foo"); + auto listener_config = test::integration::filters::TestTcpListenerFilterConfig(); + listener_config.set_drain_bytes(5); + filter_config.mutable_typed_config()->PackFrom(listener_config); + ecds->mutable_ecds_filter()->PackFrom(filter_config); + return msg; +} + +TEST_P(AdminInstanceTest, ConfigDumpEcds) { + Buffer::OwnedImpl response; + Http::TestResponseHeaderMapImpl header_map; + auto ecds_config = admin_.getConfigTracker().add("ecds", testDumpEcdsConfig); + const std::string expected_json = R"EOF({ + "configs": [ + { + "@type": "type.googleapis.com/envoy.admin.v3.EcdsConfigDump.EcdsFilterConfig", + "version_info": "1", + "ecds_filter": { + "@type": "type.googleapis.com/envoy.config.core.v3.TypedExtensionConfig", + "name": "foo", + "typed_config": { + "@type": "type.googleapis.com/test.integration.filters.TestTcpListenerFilterConfig", + "drain_bytes": 5 + } + }, + "last_updated": "1970-01-01T00:00:05Z" + } + ] +} +)EOF"; + EXPECT_EQ(Http::Code::OK, + getCallback("/config_dump?resource=ecds_filters", header_map, response)); + std::string output = response.toString(); + EXPECT_EQ(expected_json, output); +} + +TEST_P(AdminInstanceTest, ConfigDumpEcdsByResourceAndMask) { + Buffer::OwnedImpl response; + Http::TestResponseHeaderMapImpl header_map; + auto ecds_config = admin_.getConfigTracker().add("ecds", testDumpEcdsConfig); + const std::string expected_json = R"EOF({ + "configs": [ + { + "@type": "type.googleapis.com/envoy.admin.v3.EcdsConfigDump.EcdsFilterConfig", + "version_info": "1", + "ecds_filter": { + "@type": "type.googleapis.com/envoy.config.core.v3.TypedExtensionConfig", + "name": "foo" + } + } + ] +} +)EOF"; + EXPECT_EQ(Http::Code::OK, getCallback("/config_dump?resource=ecds_filters&mask=" + "ecds_filter.name,version_info", + header_map, response)); + std::string output = response.toString(); + EXPECT_EQ(expected_json, output); +} + } // namespace Server } // namespace Envoy From 06005e7fc18131a5751db4072a9d7b9cb8909085 Mon Sep 17 00:00:00 2001 From: Keith Smiley Date: Tue, 6 Dec 2022 22:58:09 -0800 Subject: [PATCH 09/23] bazel: update rules_rust (#24409) Signed-off-by: Keith Smiley --- bazel/repository_locations.bzl | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/bazel/repository_locations.bzl b/bazel/repository_locations.bzl index a875f9e0a133..b20e4dfe26ad 100644 --- a/bazel/repository_locations.bzl +++ b/bazel/repository_locations.bzl @@ -1343,12 +1343,12 @@ REPOSITORY_LOCATIONS_SPEC = dict( project_name = "Bazel rust rules", project_desc = "Bazel rust rules (used by Wasm)", project_url = "https://github.com/bazelbuild/rules_rust", - version = "0.8.1", - sha256 = "05e15e536cc1e5fd7b395d044fc2dabf73d2b27622fbc10504b7e48219bb09bc", + version = "0.14.0", + sha256 = "dd79bd4e2e2adabae738c5e93c36d351cf18071ff2acf6590190acf4138984f6", urls = ["https://github.com/bazelbuild/rules_rust/releases/download/{version}/rules_rust-v{version}.tar.gz"], use_category = ["dataplane_ext"], extensions = ["envoy.wasm.runtime.wasmtime"], - release_date = "2022-07-26", + release_date = "2022-12-01", cpe = "N/A", license = "Apache-2.0", license_url = "https://github.com/bazelbuild/rules_rust/blob/{version}/LICENSE.txt", From 54e7d0e939387be4faaafbd57d4c49750cebcb25 Mon Sep 17 00:00:00 2001 From: Keith Smiley Date: Wed, 7 Dec 2022 03:58:47 -0800 Subject: [PATCH 10/23] bazel: update to 6.0.0rc4 (#24235) This reverts commit dede92e6e5888cdf27320bf3b6bc9987657d6e22. Originally this was reverted because of https://github.com/envoyproxy/envoy/issues/23985 --- .bazelrc | 3 ++- .bazelversion | 2 +- bazel/repository_locations.bzl | 13 +++++++++++++ 3 files changed, 16 insertions(+), 2 deletions(-) diff --git a/.bazelrc b/.bazelrc index 66892483690a..d59e60868582 100644 --- a/.bazelrc +++ b/.bazelrc @@ -180,7 +180,7 @@ build:coverage --define=dynamic_link_tests=true build:coverage --define=ENVOY_CONFIG_COVERAGE=1 build:coverage --cxxopt="-DENVOY_CONFIG_COVERAGE=1" build:coverage --coverage_support=@envoy//bazel/coverage:coverage_support -build:coverage --test_env=CC_CODE_COVERAGE_SCRIPT=external/envoy/bazel/coverage/collect_cc_coverage.sh +build:coverage --test_env=CC_CODE_COVERAGE_SCRIPT=bazel/coverage/collect_cc_coverage.sh build:coverage --test_env=HEAPCHECK= build:coverage --combined_report=lcov build:coverage --strategy=TestRunner=sandboxed,local @@ -381,6 +381,7 @@ build:windows --define tcmalloc=disabled build:windows --define wasm=disabled build:windows --define manual_stamp=manual_stamp build:windows --cxxopt="/std:c++17" +build:windows --output_groups=+pdb_file # TODO(wrowe,sunjayBhatia): Resolve bugs upstream in curl and rules_foreign_cc # See issue https://github.com/bazelbuild/rules_foreign_cc/issues/301 diff --git a/.bazelversion b/.bazelversion index 9f2e85218f69..4caf2682eddd 100644 --- a/.bazelversion +++ b/.bazelversion @@ -1 +1 @@ -6.0.0-pre.20220706.4 +6.0.0rc4 diff --git a/bazel/repository_locations.bzl b/bazel/repository_locations.bzl index b20e4dfe26ad..16fa71612e5a 100644 --- a/bazel/repository_locations.bzl +++ b/bazel/repository_locations.bzl @@ -1380,6 +1380,19 @@ REPOSITORY_LOCATIONS_SPEC = dict( release_date = "2022-03-30", cpe = "N/A", ), + rules_license = dict( + project_name = "rules_license", + project_desc = "Bazel rules for checking open source licenses", + project_url = "https://github.com/bazelbuild/rules_license", + version = "0.0.3", + sha256 = "00ccc0df21312c127ac4b12880ab0f9a26c1cff99442dc6c5a331750360de3c3", + urls = ["https://github.com/bazelbuild/rules_license/releases/download/{version}/rules_license-{version}.tar.gz"], + use_category = ["build", "dataplane_core", "controlplane"], + release_date = "2022-05-28", + cpe = "N/A", + license = "Apache-2.0", + license_url = "https://github.com/bazelbuild/rules_license/blob/{version}/LICENSE", + ), ) def _compiled_protoc_deps(locations, versions): From 75d06fb73c261c28887de6c0dc0571e9bc509f1c Mon Sep 17 00:00:00 2001 From: Raven Black Date: Wed, 7 Dec 2022 06:06:34 -0800 Subject: [PATCH 11/23] Add file size to DirectoryEntry (#24176) Commit Message: Add file size to DirectoryEntry Additional Description: Directory [find] iterator is potentially useful for file system cache and, in future, serving content directly from file system. The iterator already has access to file size from stat() or its Windows equivalent, but it is not accessible to the caller. Using the current available APIs, the caller would have to make an additional call to stat for each file they wish to know the size of. This change exposes the file size through the existing API, sparing those additional OS calls in the case that the size is useful, at the cost of a small value-copy. In passing, fixes a mistaken use of errno that should have been the error number from the os-call's returned structure, and updates a throw-test to not require a nominally private function to be public. Risk Level: Low. Simple change, existing fields should be unchanged. Testing: Tests updated to match new values, one test added to validate that file size is returned. Docs Changes: n/a Release Notes: n/a Platform Specific Features: Updated both win32 and posix versions. Signed-off-by: Raven Black --- envoy/filesystem/filesystem.h | 9 +- .../posix/directory_iterator_impl.cc | 33 ++-- .../posix/directory_iterator_impl.h | 6 +- .../win32/directory_iterator_impl.cc | 26 +-- .../win32/directory_iterator_impl.h | 2 +- test/common/filesystem/directory_test.cc | 165 ++++++++++++------ 6 files changed, 159 insertions(+), 82 deletions(-) diff --git a/envoy/filesystem/filesystem.h b/envoy/filesystem/filesystem.h index 94b30685eade..245cd936e08f 100644 --- a/envoy/filesystem/filesystem.h +++ b/envoy/filesystem/filesystem.h @@ -10,6 +10,7 @@ #include "envoy/common/pure.h" #include "absl/strings/string_view.h" +#include "absl/types/optional.h" namespace Envoy { namespace Filesystem { @@ -169,15 +170,19 @@ struct DirectoryEntry { // target. For example, if name_ is a symlink to a directory, its file type will be Directory. FileType type_; + // The file size in bytes for regular files. nullopt for FileType::Directory and FileType::Other, + // and, on Windows, also nullopt for symlinks, and on Linux nullopt for broken symlinks. + absl::optional size_bytes_; + bool operator==(const DirectoryEntry& rhs) const { - return name_ == rhs.name_ && type_ == rhs.type_; + return name_ == rhs.name_ && type_ == rhs.type_ && size_bytes_ == rhs.size_bytes_; } }; class DirectoryIteratorImpl; class DirectoryIterator { public: - DirectoryIterator() : entry_({"", FileType::Other}) {} + DirectoryIterator() : entry_({"", FileType::Other, absl::nullopt}) {} virtual ~DirectoryIterator() = default; const DirectoryEntry& operator*() const { return entry_; } diff --git a/source/common/filesystem/posix/directory_iterator_impl.cc b/source/common/filesystem/posix/directory_iterator_impl.cc index eb760c76cfdf..a50514d991dc 100644 --- a/source/common/filesystem/posix/directory_iterator_impl.cc +++ b/source/common/filesystem/posix/directory_iterator_impl.cc @@ -42,39 +42,38 @@ void DirectoryIteratorImpl::nextEntry() { } if (entry == nullptr) { - entry_ = {"", FileType::Other}; + entry_ = {"", FileType::Other, absl::nullopt}; } else { - const std::string current_path(entry->d_name); - const std::string full_path(directory_path_ + "/" + current_path); - entry_ = {current_path, fileType(full_path, os_sys_calls_)}; + entry_ = makeEntry(entry->d_name); } } -FileType DirectoryIteratorImpl::fileType(const std::string& full_path, - Api::OsSysCallsImpl& os_sys_calls) { +DirectoryEntry DirectoryIteratorImpl::makeEntry(absl::string_view filename) const { + const std::string full_path = absl::StrCat(directory_path_, "/", filename); struct stat stat_buf; - - const Api::SysCallIntResult result = os_sys_calls.stat(full_path.c_str(), &stat_buf); + const Api::SysCallIntResult result = os_sys_calls_.stat(full_path.c_str(), &stat_buf); if (result.return_value_ != 0) { - if (errno == ENOENT) { + if (result.errno_ == ENOENT) { // Special case. This directory entity is likely to be a symlink, // but the reference is broken as the target could not be stat()'ed. // If we confirm this with an lstat, treat this file entity as // a regular file, which may be unlink()'ed. if (::lstat(full_path.c_str(), &stat_buf) == 0 && S_ISLNK(stat_buf.st_mode)) { - return FileType::Regular; + return DirectoryEntry{std::string{filename}, FileType::Regular, absl::nullopt}; } } + // TODO: throwing an exception here makes this dangerous to use in worker threads, + // and in general since it's not clear to the user of Directory that an exception + // may be thrown. Perhaps make this return StatusOr and handle failures gracefully. throw EnvoyException(fmt::format("unable to stat file: '{}' ({})", full_path, errno)); - } - - if (S_ISDIR(stat_buf.st_mode)) { - return FileType::Directory; + } else if (S_ISDIR(stat_buf.st_mode)) { + return DirectoryEntry{std::string{filename}, FileType::Directory, absl::nullopt}; } else if (S_ISREG(stat_buf.st_mode)) { - return FileType::Regular; + return DirectoryEntry{std::string{filename}, FileType::Regular, + static_cast(stat_buf.st_size)}; + } else { + return DirectoryEntry{std::string{filename}, FileType::Other, absl::nullopt}; } - - return FileType::Other; } } // namespace Filesystem diff --git a/source/common/filesystem/posix/directory_iterator_impl.h b/source/common/filesystem/posix/directory_iterator_impl.h index ffb2a6b2d62a..03d45cb7512f 100644 --- a/source/common/filesystem/posix/directory_iterator_impl.h +++ b/source/common/filesystem/posix/directory_iterator_impl.h @@ -25,15 +25,17 @@ class DirectoryIteratorImpl : public DirectoryIterator { DirectoryIteratorImpl(const DirectoryIteratorImpl&) = delete; DirectoryIteratorImpl(DirectoryIteratorImpl&&) = default; - static FileType fileType(const std::string& name, Api::OsSysCallsImpl& os_sys_calls); - private: void nextEntry(); void openDirectory(); + DirectoryEntry makeEntry(absl::string_view filename) const; + std::string directory_path_; DIR* dir_{nullptr}; Api::OsSysCallsImpl& os_sys_calls_; + + friend class DirectoryTest_MakeEntryThrowsOnStatFailure_Test; }; } // namespace Filesystem diff --git a/source/common/filesystem/win32/directory_iterator_impl.cc b/source/common/filesystem/win32/directory_iterator_impl.cc index 645d5b16e521..3391bce11cb9 100644 --- a/source/common/filesystem/win32/directory_iterator_impl.cc +++ b/source/common/filesystem/win32/directory_iterator_impl.cc @@ -16,7 +16,7 @@ DirectoryIteratorImpl::DirectoryIteratorImpl(const std::string& directory_path) fmt::format("unable to open directory {}: {}", directory_path, ::GetLastError())); } - entry_ = {std::string(find_data.cFileName), fileType(find_data)}; + entry_ = makeEntry(find_data); } DirectoryIteratorImpl::~DirectoryIteratorImpl() { @@ -34,27 +34,31 @@ DirectoryIteratorImpl& DirectoryIteratorImpl::operator++() { } if (ret == 0) { - entry_ = {"", FileType::Other}; + entry_ = {"", FileType::Other, absl::nullopt}; } else { - entry_ = {std::string(find_data.cFileName), fileType(find_data)}; + entry_ = makeEntry(find_data); } return *this; } -FileType DirectoryIteratorImpl::fileType(const WIN32_FIND_DATA& find_data) const { +DirectoryEntry DirectoryIteratorImpl::makeEntry(const WIN32_FIND_DATA& find_data) { if ((find_data.dwFileAttributes & FILE_ATTRIBUTE_REPARSE_POINT) && !(find_data.dwReserved0 & IO_REPARSE_TAG_SYMLINK)) { // The file is reparse point and not a symlink, so it can't be // a regular file or a directory - return FileType::Other; - } - - if (find_data.dwFileAttributes & FILE_ATTRIBUTE_DIRECTORY) { - return FileType::Directory; + return {std::string(find_data.cFileName), FileType::Other, absl::nullopt}; + } else if (find_data.dwFileAttributes & FILE_ATTRIBUTE_DIRECTORY) { + return {std::string(find_data.cFileName), FileType::Directory, absl::nullopt}; + } else if (find_data.dwReserved0 & IO_REPARSE_TAG_SYMLINK) { + return {std::string(find_data.cFileName), FileType::Regular, absl::nullopt}; + } else { + ULARGE_INTEGER file_size; + file_size.LowPart = find_data.nFileSizeLow; + file_size.HighPart = find_data.nFileSizeHigh; + uint64_t size = static_cast(file_size.QuadPart); + return {std::string(find_data.cFileName), FileType::Regular, size}; } - - return FileType::Regular; } } // namespace Filesystem diff --git a/source/common/filesystem/win32/directory_iterator_impl.h b/source/common/filesystem/win32/directory_iterator_impl.h index bfeed6dde6cc..2a0c10aed23e 100644 --- a/source/common/filesystem/win32/directory_iterator_impl.h +++ b/source/common/filesystem/win32/directory_iterator_impl.h @@ -22,7 +22,7 @@ class DirectoryIteratorImpl : public DirectoryIterator { DirectoryIteratorImpl& operator=(DirectoryIteratorImpl&&) = default; private: - FileType fileType(const WIN32_FIND_DATA& find_data) const; + static DirectoryEntry makeEntry(const WIN32_FIND_DATA& find_data); HANDLE find_handle_; }; diff --git a/test/common/filesystem/directory_test.cc b/test/common/filesystem/directory_test.cc index ec47699d6a89..75f3715dc793 100644 --- a/test/common/filesystem/directory_test.cc +++ b/test/common/filesystem/directory_test.cc @@ -15,6 +15,17 @@ namespace Envoy { namespace Filesystem { +// NOLINTNEXTLINE(readability-identifier-naming) +void PrintTo(const DirectoryEntry& entry, std::ostream* os) { + *os << "{name=" << entry.name_ << ", type=" << static_cast(entry.type_) << ", size="; + if (entry.size_bytes_ == absl::nullopt) { + *os << "nullopt"; + } else { + *os << entry.size_bytes_.value(); + } + *os << "}"; +} + class DirectoryTest : public testing::Test { public: DirectoryTest() : dir_path_(TestEnvironment::temporaryPath("envoy_test")) { @@ -43,11 +54,26 @@ class DirectoryTest : public testing::Test { void addFiles(std::list files) { for (const std::string& file_name : files) { const std::string full_path = dir_path_ + "/" + file_name; - { const std::ofstream file(full_path); } + { + const std::ofstream file(full_path); + EXPECT_TRUE(file) << "failed to open test file"; + } files_to_remove_.push(full_path); } } + void addFileWithContents(absl::string_view file_name, absl::string_view contents) { + const std::string full_path = absl::StrCat(dir_path_, "/", file_name); + { + std::ofstream file(full_path); + EXPECT_TRUE(file) << "failed to open test file"; + file << contents; + file.close(); + EXPECT_TRUE(file) << "failed to write to test file"; + } + files_to_remove_.push(full_path); + } + void addSymlinks(std::list> symlinks) { for (const auto& link : symlinks) { const std::string target_path = dir_path_ + "/" + link.first; @@ -79,7 +105,7 @@ EntrySet getDirectoryContents(const std::string& dir_path, bool recursive) { std::string subdir_name = entry.name_; EntrySet subdir = getDirectoryContents(dir_path + "/" + subdir_name, recursive); for (const DirectoryEntry& entry : subdir) { - ret.insert({subdir_name + "/" + entry.name_, entry.type_}); + ret.insert({subdir_name + "/" + entry.name_, entry.type_, entry.size_bytes_}); } } } @@ -89,11 +115,22 @@ EntrySet getDirectoryContents(const std::string& dir_path, bool recursive) { // Test that we can list a file in a directory TEST_F(DirectoryTest, DirectoryWithOneFile) { addFiles({"file"}); + const EntrySet expected = { + {".", FileType::Directory, absl::nullopt}, + {"..", FileType::Directory, absl::nullopt}, + {"file", FileType::Regular, 0}, + }; + EXPECT_EQ(expected, getDirectoryContents(dir_path_, false)); +} + +TEST_F(DirectoryTest, DirectoryWithOneFileIncludesCorrectFileSize) { + absl::string_view contents = "hello!"; + addFileWithContents("file", contents); const EntrySet expected = { - {".", FileType::Directory}, - {"..", FileType::Directory}, - {"file", FileType::Regular}, + {".", FileType::Directory, absl::nullopt}, + {"..", FileType::Directory, absl::nullopt}, + {"file", FileType::Regular, contents.size()}, }; EXPECT_EQ(expected, getDirectoryContents(dir_path_, false)); } @@ -103,9 +140,9 @@ TEST_F(DirectoryTest, DirectoryWithOneDirectory) { addSubDirs({"sub_dir"}); const EntrySet expected = { - {".", FileType::Directory}, - {"..", FileType::Directory}, - {"sub_dir", FileType::Directory}, + {".", FileType::Directory, absl::nullopt}, + {"..", FileType::Directory, absl::nullopt}, + {"sub_dir", FileType::Directory, absl::nullopt}, }; EXPECT_EQ(expected, getDirectoryContents(dir_path_, false)); } @@ -116,9 +153,9 @@ TEST_F(DirectoryTest, DirectoryWithFileInSubDirectory) { addFiles({"sub_dir/sub_file"}); const EntrySet expected = { - {".", FileType::Directory}, - {"..", FileType::Directory}, - {"sub_dir", FileType::Directory}, + {".", FileType::Directory, absl::nullopt}, + {"..", FileType::Directory, absl::nullopt}, + {"sub_dir", FileType::Directory, absl::nullopt}, }; EXPECT_EQ(expected, getDirectoryContents(dir_path_, false)); } @@ -129,13 +166,13 @@ TEST_F(DirectoryTest, RecursionIntoSubDirectory) { addFiles({"file", "sub_dir/sub_file"}); const EntrySet expected = { - {".", FileType::Directory}, - {"..", FileType::Directory}, - {"file", FileType::Regular}, - {"sub_dir", FileType::Directory}, - {"sub_dir/sub_file", FileType::Regular}, - {"sub_dir/.", FileType::Directory}, - {"sub_dir/..", FileType::Directory}, + {".", FileType::Directory, absl::nullopt}, + {"..", FileType::Directory, absl::nullopt}, + {"file", FileType::Regular, 0}, + {"sub_dir", FileType::Directory, absl::nullopt}, + {"sub_dir/sub_file", FileType::Regular, 0}, + {"sub_dir/.", FileType::Directory, absl::nullopt}, + {"sub_dir/..", FileType::Directory, absl::nullopt}, }; EXPECT_EQ(expected, getDirectoryContents(dir_path_, true)); } @@ -146,26 +183,27 @@ TEST_F(DirectoryTest, DirectoryWithFileAndDirectory) { addFiles({"file"}); const EntrySet expected = { - {".", FileType::Directory}, - {"..", FileType::Directory}, - {"sub_dir", FileType::Directory}, - {"file", FileType::Regular}, + {".", FileType::Directory, absl::nullopt}, + {"..", FileType::Directory, absl::nullopt}, + {"sub_dir", FileType::Directory, absl::nullopt}, + {"file", FileType::Regular, 0}, }; EXPECT_EQ(expected, getDirectoryContents(dir_path_, false)); } // Test that a symlink to a file has type FileType::Regular TEST_F(DirectoryTest, DirectoryWithSymlinkToFile) { - addFiles({"file"}); + const absl::string_view contents = "hello"; + addFileWithContents("file", contents); addSymlinks({{"file", "link"}}); - const EntrySet expected = { - {".", FileType::Directory}, - {"..", FileType::Directory}, - {"file", FileType::Regular}, - {"link", FileType::Regular}, - }; - EXPECT_EQ(expected, getDirectoryContents(dir_path_, false)); + const EntrySet result = getDirectoryContents(dir_path_, false); + EXPECT_THAT(result, + testing::Contains(DirectoryEntry{"file", FileType::Regular, contents.size()})); + // Validate without size for link, as it may be nullopt or file-size depending on OS. + EXPECT_THAT(result, testing::Contains(testing::AllOf( + testing::Field(&DirectoryEntry::name_, "link"), + testing::Field(&DirectoryEntry::type_, FileType::Regular)))); } // Test that a symlink to a directory has type FileType::Directory @@ -174,10 +212,10 @@ TEST_F(DirectoryTest, DirectoryWithSymlinkToDirectory) { addSymlinks({{"sub_dir", "link_dir"}}); const EntrySet expected = { - {".", FileType::Directory}, - {"..", FileType::Directory}, - {"sub_dir", FileType::Directory}, - {"link_dir", FileType::Directory}, + {".", FileType::Directory, absl::nullopt}, + {"..", FileType::Directory, absl::nullopt}, + {"sub_dir", FileType::Directory, absl::nullopt}, + {"link_dir", FileType::Directory, absl::nullopt}, }; EXPECT_EQ(expected, getDirectoryContents(dir_path_, false)); } @@ -189,14 +227,14 @@ TEST_F(DirectoryTest, DirectoryWithBrokenSymlink) { TestEnvironment::removePath(dir_path_ + "/sub_dir"); const EntrySet expected = { - {".", FileType::Directory}, - {"..", FileType::Directory}, + {".", FileType::Directory, absl::nullopt}, + {"..", FileType::Directory, absl::nullopt}, #ifndef WIN32 // On Linux, a broken directory link is simply a symlink to be rm'ed - {"link_dir", FileType::Regular}, + {"link_dir", FileType::Regular, absl::nullopt}, #else // On Windows, a broken directory link remains a directory link to be rmdir'ed - {"link_dir", FileType::Directory}, + {"link_dir", FileType::Directory, absl::nullopt}, #endif }; EXPECT_EQ(expected, getDirectoryContents(dir_path_, false)); @@ -205,8 +243,8 @@ TEST_F(DirectoryTest, DirectoryWithBrokenSymlink) { // Test that we can list an empty directory TEST_F(DirectoryTest, DirectoryWithEmptyDirectory) { const EntrySet expected = { - {".", FileType::Directory}, - {"..", FileType::Directory}, + {".", FileType::Directory, absl::nullopt}, + {"..", FileType::Directory, absl::nullopt}, }; EXPECT_EQ(expected, getDirectoryContents(dir_path_, false)); } @@ -232,18 +270,22 @@ TEST_F(DirectoryTest, Fifo) { ASSERT_EQ(0, mkfifo(fifo_path.c_str(), 0644)); const EntrySet expected = { - {".", FileType::Directory}, - {"..", FileType::Directory}, - {"fifo", FileType::Other}, + {".", FileType::Directory, absl::nullopt}, + {"..", FileType::Directory, absl::nullopt}, + {"fifo", FileType::Other, absl::nullopt}, }; EXPECT_EQ(expected, getDirectoryContents(dir_path_, false)); remove(fifo_path.c_str()); } -TEST_F(DirectoryTest, FileTypeTest) { - auto sys_calls = Api::OsSysCallsSingleton::get(); - EXPECT_THROW_WITH_REGEX(DirectoryIteratorImpl::fileType("foo", sys_calls), EnvoyException, - "unable to stat file: 'foo' .*"); +// This test seems like it should be doable by removing a file after directory +// iteration begins, but apparently the behavior of that varies per-filesystem +// (in some cases the absent file is not seen, in others it is). So we test +// instead by directly calling the private function. +TEST_F(DirectoryTest, MakeEntryThrowsOnStatFailure) { + Directory directory(dir_path_); + EXPECT_THROW_WITH_REGEX(directory.begin().makeEntry("foo"), EnvoyException, + "unable to stat file: '.*foo' .*"); } #endif @@ -257,12 +299,37 @@ TEST(Directory, DirectoryHasTrailingPathSeparator) { TestEnvironment::createPath(dir_path); const EntrySet expected = { - {".", FileType::Directory}, - {"..", FileType::Directory}, + {".", FileType::Directory, absl::nullopt}, + {"..", FileType::Directory, absl::nullopt}, }; EXPECT_EQ(expected, getDirectoryContents(dir_path, false)); TestEnvironment::removePath(dir_path); } +TEST(DirectoryEntry, EqualityOperator) { + std::vector values{ + DirectoryEntry{"bob", FileType::Directory, absl::nullopt}, + DirectoryEntry{"bob", FileType::Regular, absl::nullopt}, + DirectoryEntry{"bob", FileType::Other, absl::nullopt}, + DirectoryEntry{"bob", FileType::Regular, 0}, + DirectoryEntry{"bob", FileType::Regular, 5}, + DirectoryEntry{"bob", FileType::Regular, 6}, + DirectoryEntry{"alice", FileType::Regular, 6}, + DirectoryEntry{"jim", FileType::Regular, absl::nullopt}, + }; + for (size_t i = 0; i < values.size(); i++) { + DirectoryEntry a = values[i]; + // Two copies of the same value should be ==, in either order. + EXPECT_THAT(a, testing::Eq(values[i])); + EXPECT_THAT(values[i], testing::Eq(a)); + for (size_t j = i + 1; j < values.size(); j++) { + DirectoryEntry b = values[j]; + // No two pairs of the above DirectoryEntries should be ==, in either order. + EXPECT_THAT(a, testing::Not(testing::Eq(b))); + EXPECT_THAT(b, testing::Not(testing::Eq(a))); + } + } +} + } // namespace Filesystem } // namespace Envoy From e521eda0a7366b284b2abd306cea603ba5f8bbfb Mon Sep 17 00:00:00 2001 From: JP Simard Date: Wed, 7 Dec 2022 09:24:25 -0500 Subject: [PATCH 12/23] mobile: remove `bump_lyft_support_rotation.sh` script (#24404) We no longer have a Lyft support rotation since the repo merge. Signed-off-by: JP Simard --- mobile/tools/bump_lyft_support_rotation.sh | 54 ---------------------- 1 file changed, 54 deletions(-) delete mode 100755 mobile/tools/bump_lyft_support_rotation.sh diff --git a/mobile/tools/bump_lyft_support_rotation.sh b/mobile/tools/bump_lyft_support_rotation.sh deleted file mode 100755 index 2b8775f1e23c..000000000000 --- a/mobile/tools/bump_lyft_support_rotation.sh +++ /dev/null @@ -1,54 +0,0 @@ -#!/usr/bin/env bash - -set -euo pipefail - -####################################################### -# bump_lyft_support_rotation.sh -# -# Assigns the next person in the Lyft maintainers list. -####################################################### - -# TODO(jpsim): Use a yaml parsing library if the format ever expands -# beyond its current very simple form. - -function next_maintainer() { - current="$1" - file="$2" - - while IFS= read -r line; do - maintainers+=("${line#" - "}") - done <<< "$(sed -n '/^ - /p' "$file")" - - for i in "${!maintainers[@]}"; do - if [[ "${maintainers[$i]}" = "${current}" ]]; then - last_index=$((${#maintainers[@]}-1)) - if [[ $i == "$last_index" ]]; then - echo "${maintainers[0]}" - break - else - echo "${maintainers[$((i + 1))]}" - break - fi - fi - done -} - -function set_maintainer() { - maintainer="$1" - file="$2" - - echo "current: $maintainer" > "$file.tmp" - tail -n +2 "$file" >> "$file.tmp" - mv "$file.tmp" "$file" -} - -maintainers_file=".github/lyft_maintainers.yml" -first_line="$(head -n 1 "$maintainers_file")" -previous=${first_line#"current: "} -next="$(next_maintainer "$previous" "$maintainers_file")" -set_maintainer "$next" "$maintainers_file" - -echo "PREVIOUS_MAINTAINER=$previous" >> "$GITHUB_OUTPUT" -echo "NEXT_MAINTAINER=$next" >> "$GITHUB_OUTPUT" - -echo "Lyft support maintainer changing from $previous to $next" From 11ca6cdb6fb8d337f1da09fed40a796ebd3cb73a Mon Sep 17 00:00:00 2001 From: JP Simard Date: Wed, 7 Dec 2022 09:30:22 -0500 Subject: [PATCH 13/23] ci: run mobile GitHub Actions on every PR (#24407) But early-exit for changes that shouldn't impact Envoy Mobile. On top of `mobile/**` changes and changes to workflow files, now when Envoy dependencies are updated in `bazel/repository_locations.bzl`, mobile CI jobs will run too. Signed-off-by: JP Simard --- .github/workflows/android_build.yml | 44 ++++++++++-- .github/workflows/android_tests.yml | 57 ++++----------- .github/workflows/asan.yml | 21 ++---- .github/workflows/core.yml | 11 ++- .github/workflows/coverage.yml | 23 ++---- .github/workflows/format.yml | 31 +++++--- .github/workflows/ios_build.yml | 90 ++++++++++++++++-------- .github/workflows/ios_tests.yml | 34 +++------ .github/workflows/python_tests.yml | 19 +---- .github/workflows/release_validation.yml | 17 ++--- .github/workflows/tsan.yml | 21 ++---- mobile/tools/should_run_ci.sh | 44 ++++++++++++ 12 files changed, 220 insertions(+), 192 deletions(-) create mode 100755 mobile/tools/should_run_ci.sh diff --git a/.github/workflows/android_build.yml b/.github/workflows/android_build.yml index ace3e9860e3c..13135ceb71cd 100644 --- a/.github/workflows/android_build.yml +++ b/.github/workflows/android_build.yml @@ -4,13 +4,7 @@ on: push: branches: - main - paths: - - 'mobile/**' - - '.github/workflows/**' pull_request: - paths: - - 'mobile/**' - - '.github/workflows/**' jobs: androidbuild: @@ -19,15 +13,21 @@ jobs: timeout-minutes: 90 steps: - uses: actions/checkout@v1 + - id: should_run + name: 'Check whether to run' + run: ./mobile/tools/should_run_ci.sh - uses: actions/setup-java@c3ac5dd0ed8db40fedb61c32fbe677e6b355e94c + if: steps.should_run.outputs.run_ci_job == 'true' with: java-version: '8' java-package: jdk architecture: x64 distribution: zulu - name: 'Install dependencies' + if: steps.should_run.outputs.run_ci_job == 'true' run: cd mobile && ./ci/mac_ci_setup.sh --android - name: 'Build envoy.aar distributable' + if: steps.should_run.outputs.run_ci_job == 'true' env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} run: | @@ -43,20 +43,27 @@ jobs: timeout-minutes: 50 steps: - uses: actions/checkout@v1 + - id: should_run + name: 'Check whether to run' + run: ./mobile/tools/should_run_ci.sh - uses: actions/setup-java@c3ac5dd0ed8db40fedb61c32fbe677e6b355e94c + if: steps.should_run.outputs.run_ci_job == 'true' with: java-version: '8' java-package: jdk architecture: x64 distribution: zulu - run: cd mobile && ./ci/mac_ci_setup.sh --android + if: steps.should_run.outputs.run_ci_job == 'true' name: 'Install dependencies' - name: 'Start simulator' + if: steps.should_run.outputs.run_ci_job == 'true' run: cd mobile && ./ci/mac_start_emulator.sh # Return to using: # cd mobile && ./bazelw mobile-install --fat_apk_cpu=x86_64 --start_app //examples/java/hello_world:hello_envoy # When https://github.com/envoyproxy/envoy-mobile/issues/853 is fixed. - name: 'Start java app' + if: steps.should_run.outputs.run_ci_job == 'true' env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} run: | @@ -68,6 +75,7 @@ jobs: adb install -r --no-incremental bazel-bin/examples/java/hello_world/hello_envoy.apk adb shell am start -n io.envoyproxy.envoymobile.helloenvoy/.MainActivity - name: 'Check connectivity' + if: steps.should_run.outputs.run_ci_job == 'true' run: adb logcat -e "received headers with status 200" -m 1 kotlinhelloworld: name: kotlin_helloworld @@ -76,20 +84,27 @@ jobs: timeout-minutes: 50 steps: - uses: actions/checkout@v1 + - id: should_run + name: 'Check whether to run' + run: ./mobile/tools/should_run_ci.sh - uses: actions/setup-java@c3ac5dd0ed8db40fedb61c32fbe677e6b355e94c + if: steps.should_run.outputs.run_ci_job == 'true' with: java-version: '8' java-package: jdk architecture: x64 distribution: zulu - name: 'Install dependencies' + if: steps.should_run.outputs.run_ci_job == 'true' run: cd mobile && ./ci/mac_ci_setup.sh --android - name: 'Start simulator' + if: steps.should_run.outputs.run_ci_job == 'true' run: cd mobile && ./ci/mac_start_emulator.sh # Return to using: # ./bazelw mobile-install --fat_apk_cpu=x86_64 --start_app //examples/kotlin/hello_world:hello_envoy_kt # When https://github.com/envoyproxy/envoy-mobile/issues/853 is fixed. - name: 'Start kotlin app' + if: steps.should_run.outputs.run_ci_job == 'true' env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} run: | @@ -101,6 +116,7 @@ jobs: adb install -r --no-incremental bazel-bin/examples/kotlin/hello_world/hello_envoy_kt.apk adb shell am start -n io.envoyproxy.envoymobile.helloenvoykotlin/.MainActivity - name: 'Check connectivity' + if: steps.should_run.outputs.run_ci_job == 'true' run: adb logcat -e "received headers with status 200" -m 1 kotlinbaselineapp: name: kotlin_baseline_app @@ -109,20 +125,27 @@ jobs: timeout-minutes: 50 steps: - uses: actions/checkout@v1 + - id: should_run + name: 'Check whether to run' + run: ./mobile/tools/should_run_ci.sh - uses: actions/setup-java@c3ac5dd0ed8db40fedb61c32fbe677e6b355e94c + if: steps.should_run.outputs.run_ci_job == 'true' with: java-version: '8' java-package: jdk architecture: x64 distribution: zulu - name: 'Install dependencies' + if: steps.should_run.outputs.run_ci_job == 'true' run: cd mobile && ./ci/mac_ci_setup.sh --android - name: 'Start simulator' + if: steps.should_run.outputs.run_ci_job == 'true' run: cd mobile && ./ci/mac_start_emulator.sh # Return to using: # ./bazelw mobile-install --fat_apk_cpu=x86_64 --start_app //examples/kotlin/hello_world:hello_envoy_kt # When https://github.com/envoyproxy/envoy-mobile/issues/853 is fixed. - name: 'Start kotlin app' + if: steps.should_run.outputs.run_ci_job == 'true' env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} run: | @@ -134,6 +157,7 @@ jobs: adb install -r --no-incremental bazel-bin/test/kotlin/apps/baseline/hello_envoy_kt.apk adb shell am start -n io.envoyproxy.envoymobile.helloenvoybaselinetest/.MainActivity - name: 'Check connectivity' + if: steps.should_run.outputs.run_ci_job == 'true' run: adb logcat -e "received headers with status 301" -m 1 kotlinexperimentalapp: name: kotlin_experimental_app @@ -142,20 +166,27 @@ jobs: timeout-minutes: 50 steps: - uses: actions/checkout@v1 + - id: should_run + name: 'Check whether to run' + run: ./mobile/tools/should_run_ci.sh - uses: actions/setup-java@c3ac5dd0ed8db40fedb61c32fbe677e6b355e94c + if: steps.should_run.outputs.run_ci_job == 'true' with: java-version: '8' java-package: jdk architecture: x64 distribution: zulu - name: 'Install dependencies' + if: steps.should_run.outputs.run_ci_job == 'true' run: cd mobile && ./ci/mac_ci_setup.sh --android - name: 'Start simulator' + if: steps.should_run.outputs.run_ci_job == 'true' run: cd mobile && ./ci/mac_start_emulator.sh # Return to using: # ./bazelw mobile-install --fat_apk_cpu=x86_64 --start_app //examples/kotlin/hello_world:hello_envoy_kt # When https://github.com/envoyproxy/envoy-mobile/issues/853 is fixed. - name: 'Start kotlin app' + if: steps.should_run.outputs.run_ci_job == 'true' env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} run: | @@ -167,4 +198,5 @@ jobs: adb install -r --no-incremental bazel-bin/test/kotlin/apps/experimental/hello_envoy_kt.apk adb shell am start -n io.envoyproxy.envoymobile.helloenvoyexperimentaltest/.MainActivity - name: 'Check connectivity' + if: steps.should_run.outputs.run_ci_job == 'true' run: adb logcat -e "received headers with status 200" -m 1 diff --git a/.github/workflows/android_tests.yml b/.github/workflows/android_tests.yml index db85cbf3dc2e..0ff44a0a79f4 100644 --- a/.github/workflows/android_tests.yml +++ b/.github/workflows/android_tests.yml @@ -4,13 +4,7 @@ on: push: branches: - main - paths: - - 'mobile/**' - - '.github/workflows/**' pull_request: - paths: - - 'mobile/**' - - '.github/workflows/**' jobs: kotlintestsmac: @@ -21,18 +15,11 @@ jobs: timeout-minutes: 90 steps: - uses: actions/checkout@v1 - - id: check_context + - id: should_run name: 'Check whether to run' - run: | - if git rev-parse --abbrev-ref HEAD | grep -q ^main$ || git diff --name-only origin/main | grep -qe common/ -e java/ -e kotlin/ -e bazel/ -e ^\.bazelrc$ -e ^envoy$ -e ^WORKSPACE$ -e ^.github/workflows/android_tests.yml$ ; then - echo "Tests will run." - echo "run_tests=true" >> $GITHUB_OUTPUT - else - echo "Skipping tests." - echo "run_tests=false" >> $GITHUB_OUTPUT - fi + run: ./mobile/tools/should_run_ci.sh - name: 'Java setup' - if: steps.check_context.outputs.run_tests == 'true' + if: steps.should_run.outputs.run_ci_job == 'true' uses: actions/setup-java@c3ac5dd0ed8db40fedb61c32fbe677e6b355e94c with: java-version: '8' @@ -40,10 +27,10 @@ jobs: architecture: x64 distribution: zulu - name: 'Install dependencies' - if: steps.check_context.outputs.run_tests == 'true' + if: steps.should_run.outputs.run_ci_job == 'true' run: cd mobile && ./ci/mac_ci_setup.sh - name: 'Run Kotlin library tests' - if: steps.check_context.outputs.run_tests == 'true' + if: steps.should_run.outputs.run_ci_job == 'true' env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} run: | @@ -59,18 +46,11 @@ jobs: timeout-minutes: 120 steps: - uses: actions/checkout@v1 - - id: check_context + - id: should_run name: 'Check whether to run' - run: | - if git rev-parse --abbrev-ref HEAD | grep -q ^main$ || git diff --name-only origin/main | grep -qe common/ -e java/ -e kotlin/ -e bazel/ -e ^\.bazelrc$ -e ^envoy$ -e ^WORKSPACE$ -e ^.github/workflows/android_tests.yml$ ; then - echo "Tests will run." - echo "run_tests=true" >> $GITHUB_OUTPUT - else - echo "Skipping tests." - echo "run_tests=false" >> $GITHUB_OUTPUT - fi + run: ./mobile/tools/should_run_ci.sh - name: 'Java setup' - if: steps.check_context.outputs.run_tests == 'true' + if: steps.should_run.outputs.run_ci_job == 'true' uses: actions/setup-java@c3ac5dd0ed8db40fedb61c32fbe677e6b355e94c with: java-version: '8' @@ -78,10 +58,10 @@ jobs: architecture: x64 distribution: zulu - name: 'Install dependencies' - if: steps.check_context.outputs.run_tests == 'true' + if: steps.should_run.outputs.run_ci_job == 'true' run: cd mobile && ./ci/mac_ci_setup.sh - name: 'Run Java library tests' - if: steps.check_context.outputs.run_tests == 'true' + if: steps.should_run.outputs.run_ci_job == 'true' env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} run: | @@ -107,18 +87,11 @@ jobs: - uses: actions/checkout@v1 - name: Add safe directory run: git config --global --add safe.directory /__w/envoy/envoy - - id: check_context + - id: should_run name: 'Check whether to run' - run: | - if git rev-parse --abbrev-ref HEAD | grep -q ^main$ || git diff --name-only origin/main | grep -qe common/ -e java/ -e kotlin/ -e bazel/ -e ^\.bazelrc$ -e ^envoy$ -e ^WORKSPACE$ -e ^.github/workflows/android_tests.yml$ ; then - echo "Tests will run." - echo "run_tests=true" >> $GITHUB_OUTPUT - else - echo "Skipping tests." - echo "run_tests=false" >> $GITHUB_OUTPUT - fi + run: ./mobile/tools/should_run_ci.sh - name: 'Java setup' - if: steps.check_context.outputs.run_tests == 'true' + if: steps.should_run.outputs.run_ci_job == 'true' uses: actions/setup-java@c3ac5dd0ed8db40fedb61c32fbe677e6b355e94c with: java-version: '8' @@ -126,10 +99,10 @@ jobs: architecture: x64 distribution: zulu - name: 'Install dependencies' - if: steps.check_context.outputs.run_tests == 'true' + if: steps.should_run.outputs.run_ci_job == 'true' run: cd mobile && ./ci/linux_ci_setup.sh - name: 'Run Kotlin library integration tests' - if: steps.check_context.outputs.run_tests == 'true' + if: steps.should_run.outputs.run_ci_job == 'true' env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} run: | diff --git a/.github/workflows/asan.yml b/.github/workflows/asan.yml index d708dde3dc53..271e331119cc 100644 --- a/.github/workflows/asan.yml +++ b/.github/workflows/asan.yml @@ -4,13 +4,7 @@ on: push: branches: - main - paths: - - 'mobile/**' - - '.github/workflows/**' pull_request: - paths: - - 'mobile/**' - - '.github/workflows/**' jobs: asan: @@ -26,18 +20,11 @@ jobs: - uses: actions/checkout@v1 - name: Add safe directory run: git config --global --add safe.directory /__w/envoy/envoy - - id: check_context + - id: should_run name: 'Check whether to run' - run: | - if git rev-parse --abbrev-ref HEAD | grep -q ^main$ || git diff --name-only origin/main | grep -qe common/ -e bazel/ -e ^\.bazelrc$ -e ^envoy$ -e ^WORKSPACE$ -e ^.github/workflows/asan.yml$ ; then - echo "Tests will run." - echo "run_tests=true" >> $GITHUB_OUTPUT - else - echo "Skipping tests." - echo "run_tests=false" >> $GITHUB_OUTPUT - fi + run: ./mobile/tools/should_run_ci.sh - uses: actions/setup-java@c3ac5dd0ed8db40fedb61c32fbe677e6b355e94c - if: steps.check-cache.outputs.cache-hit != 'true' + if: steps.should_run.outputs.run_ci_job == 'true' with: java-version: '8' java-package: jdk @@ -46,7 +33,7 @@ jobs: - name: 'Run tests' env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - if: steps.check_context.outputs.run_tests == 'true' + if: steps.should_run.outputs.run_ci_job == 'true' run: | cd mobile && ./bazelw test --test_output=all \ --test_env=ENVOY_IP_TEST_VERSIONS=v4only \ diff --git a/.github/workflows/core.yml b/.github/workflows/core.yml index 324a9e5ebff4..425b33fa475a 100644 --- a/.github/workflows/core.yml +++ b/.github/workflows/core.yml @@ -4,13 +4,7 @@ on: push: branches: - main - paths: - - 'mobile/**' - - '.github/workflows/**' pull_request: - paths: - - 'mobile/**' - - '.github/workflows/**' jobs: unittests: @@ -19,9 +13,14 @@ jobs: timeout-minutes: 120 steps: - uses: actions/checkout@v1 + - id: should_run + name: 'Check whether to run' + run: ./mobile/tools/should_run_ci.sh - name: 'Install dependencies' + if: steps.should_run.outputs.run_ci_job == 'true' run: cd mobile && ./ci/mac_ci_setup.sh - name: 'Run tests' + if: steps.should_run.outputs.run_ci_job == 'true' env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} run: | diff --git a/.github/workflows/coverage.yml b/.github/workflows/coverage.yml index b00d590746e6..b8985af4b80a 100644 --- a/.github/workflows/coverage.yml +++ b/.github/workflows/coverage.yml @@ -4,13 +4,7 @@ on: push: branches: - main - paths: - - 'mobile/**' - - '.github/workflows/**' pull_request: - paths: - - 'mobile/**' - - '.github/workflows/**' jobs: coverage: @@ -26,18 +20,11 @@ jobs: - uses: actions/checkout@v1 - name: Add safe directory run: git config --global --add safe.directory /__w/envoy/envoy - - id: check_context + - id: should_run name: 'Check whether to run' - run: | - if git rev-parse --abbrev-ref HEAD | grep -q ^main$ || git diff --name-only origin/main | grep -qe common/ -e ^.github/workflows/coverage.yml ; then - echo "Coverage will run." - echo "run_coverage=true" >> $GITHUB_OUTPUT - else - echo "Skipping coverage." - echo "run_coverage=false" >> $GITHUB_OUTPUT - fi + run: ./mobile/tools/should_run_ci.sh - name: 'Run coverage' - if: steps.check_context.outputs.run_coverage == 'true' + if: steps.should_run.outputs.run_ci_job == 'true' continue-on-error: true run: | echo "build --remote_header=\"Authorization=Bearer ${{ secrets.GITHUB_TOKEN }}\"" > ~/.bazelrc @@ -46,11 +33,11 @@ jobs: COVERAGE_THRESHOLD=95 \ ../test/run_envoy_bazel_coverage.sh //test/common/... - name: 'Package coverage' - if: steps.check_context.outputs.run_coverage == 'true' + if: steps.should_run.outputs.run_ci_job == 'true' run: | cd mobile && tar -czvf coverage.tar.gz generated/coverage - name: 'Upload report' - if: steps.check_context.outputs.run_coverage == 'true' + if: steps.should_run.outputs.run_ci_job == 'true' uses: actions/upload-artifact@v3 with: name: coverage.tar.gz diff --git a/.github/workflows/format.yml b/.github/workflows/format.yml index f07ddd0f6c31..660fef05a87f 100644 --- a/.github/workflows/format.yml +++ b/.github/workflows/format.yml @@ -4,13 +4,7 @@ on: push: branches: - main - paths: - - 'mobile/**' - - '.github/workflows/**' pull_request: - paths: - - 'mobile/**' - - '.github/workflows/**' jobs: formatall: @@ -26,7 +20,13 @@ jobs: ENVOY_BAZEL_PREFIX: "@envoy" steps: - uses: actions/checkout@v1 + - name: Add safe directory + run: git config --global --add safe.directory /__w/envoy/envoy + - id: should_run + name: 'Check whether to run' + run: ./mobile/tools/should_run_ci.sh - name: 'Run formatters' + if: steps.should_run.outputs.run_ci_job == 'true' run: cd mobile && ./tools/check_format.sh precommit: name: precommit @@ -34,9 +34,14 @@ jobs: timeout-minutes: 45 steps: - uses: actions/checkout@v1 + - id: should_run + name: 'Check whether to run' + run: ./mobile/tools/should_run_ci.sh - name: 'Install precommit' + if: steps.should_run.outputs.run_ci_job == 'true' run: brew install pre-commit - name: 'Run precommit' + if: steps.should_run.outputs.run_ci_job == 'true' run: cd mobile && find mobile/* | pre-commit run --files swiftlint: name: swift_lint @@ -55,7 +60,11 @@ jobs: timeout-minutes: 10 steps: - uses: actions/checkout@v1 + - id: should_run + name: 'Check whether to run' + run: ./mobile/tools/should_run_ci.sh - name: 'Run DrString' + if: steps.should_run.outputs.run_ci_job == 'true' env: DEVELOPER_DIR: /Applications/Xcode_14.1.app run: cd mobile && ./bazelw run @DrString//:drstring check @@ -65,15 +74,21 @@ jobs: timeout-minutes: 45 steps: - uses: actions/checkout@v1 + - id: should_run + name: 'Check whether to run' + run: ./mobile/tools/should_run_ci.sh - uses: actions/setup-java@c3ac5dd0ed8db40fedb61c32fbe677e6b355e94c + if: steps.should_run.outputs.run_ci_job == 'true' with: java-version: '8' java-package: jdk architecture: x64 distribution: zulu - run: cd mobile && ./ci/mac_ci_setup.sh + if: steps.should_run.outputs.run_ci_job == 'true' name: 'Install dependencies' - name: 'Run Kotlin Lint (Detekt)' + if: steps.should_run.outputs.run_ci_job == 'true' env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} run: | @@ -83,5 +98,5 @@ jobs: //library/kotlin/io/envoyproxy/envoymobile:envoy_lib_lint \ //examples/kotlin/hello_world:hello_envoy_kt_lint - name: 'Run Kotlin Formatter (ktlint)' - run: | - cd mobile && ./bazelw build kotlin_format + if: steps.should_run.outputs.run_ci_job == 'true' + run: cd mobile && ./bazelw build kotlin_format diff --git a/.github/workflows/ios_build.yml b/.github/workflows/ios_build.yml index 4e61e681442b..e947b602907d 100644 --- a/.github/workflows/ios_build.yml +++ b/.github/workflows/ios_build.yml @@ -4,13 +4,7 @@ on: push: branches: - main - paths: - - 'mobile/**' - - '.github/workflows/**' pull_request: - paths: - - 'mobile/**' - - '.github/workflows/**' jobs: iosbuild: @@ -19,9 +13,15 @@ jobs: timeout-minutes: 120 steps: - uses: actions/checkout@v1 + - id: should_run + name: 'Check whether to run' + run: ./mobile/tools/should_run_ci.sh - run: cd mobile && ./ci/mac_ci_setup.sh + if: steps.should_run.outputs.run_ci_job == 'true' name: 'Install dependencies' - - env: + - name: 'Build Envoy.framework distributable' + if: steps.should_run.outputs.run_ci_job == 'true' + env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} run: | cd mobile && ./bazelw shutdown @@ -30,7 +30,6 @@ jobs: $([ -z $GITHUB_TOKEN ] || echo "--config=remote-ci-macos") \ --remote_header="Authorization=Bearer $GITHUB_TOKEN" \ //library/swift:ios_framework - name: 'Build Envoy.framework distributable' swifthelloworld: name: swift_helloworld needs: iosbuild @@ -38,9 +37,15 @@ jobs: timeout-minutes: 50 steps: - uses: actions/checkout@v1 + - id: should_run + name: 'Check whether to run' + run: ./mobile/tools/should_run_ci.sh - run: cd mobile && ./ci/mac_ci_setup.sh + if: steps.should_run.outputs.run_ci_job == 'true' name: 'Install dependencies' - - env: + - name: 'Build app' + if: steps.should_run.outputs.run_ci_job == 'true' + env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} run: | cd mobile && ./bazelw build \ @@ -48,9 +53,10 @@ jobs: $([ -z $GITHUB_TOKEN ] || echo "--config=remote-ci-macos") \ --remote_header="Authorization=Bearer $GITHUB_TOKEN" \ //examples/swift/hello_world:app - name: 'Build swift app' # Run the app in the background and redirect logs. - - env: + - name: 'Run app' + if: steps.should_run.outputs.run_ci_job == 'true' + env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} run: | cd mobile && ./bazelw run \ @@ -58,8 +64,8 @@ jobs: $([ -z $GITHUB_TOKEN ] || echo "--config=remote-ci-macos") \ --remote_header="Authorization=Bearer $GITHUB_TOKEN" \ //examples/swift/hello_world:app &> /tmp/envoy.log & - name: 'Run swift app' - run: sed '/received headers with status 200/q' <(touch /tmp/envoy.log && tail -F /tmp/envoy.log) + if: steps.should_run.outputs.run_ci_job == 'true' name: 'Check connectivity' - run: cat /tmp/envoy.log if: ${{ failure() || cancelled() }} @@ -71,9 +77,15 @@ jobs: timeout-minutes: 50 steps: - uses: actions/checkout@v1 + - id: should_run + name: 'Check whether to run' + run: ./mobile/tools/should_run_ci.sh - run: cd mobile && ./ci/mac_ci_setup.sh + if: steps.should_run.outputs.run_ci_job == 'true' name: 'Install dependencies' - - env: + - name: 'Build app' + if: steps.should_run.outputs.run_ci_job == 'true' + env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} run: | cd mobile && ./bazelw build \ @@ -81,9 +93,10 @@ jobs: $([ -z $GITHUB_TOKEN ] || echo "--config=remote-ci-macos") \ --remote_header="Authorization=Bearer $GITHUB_TOKEN" \ //test/swift/apps/baseline:app - name: 'Build swift app' # Run the app in the background and redirect logs. - - env: + - name: 'Run app' + if: steps.should_run.outputs.run_ci_job == 'true' + env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} run: | cd mobile && ./bazelw run \ @@ -91,8 +104,8 @@ jobs: $([ -z $GITHUB_TOKEN ] || echo "--config=remote-ci-macos") \ --remote_header="Authorization=Bearer $GITHUB_TOKEN" \ //test/swift/apps/baseline:app &> /tmp/envoy.log & - name: 'Run swift app' - run: sed '/received headers with status 301/q' <(touch /tmp/envoy.log && tail -F /tmp/envoy.log) + if: steps.should_run.outputs.run_ci_job == 'true' name: 'Check connectivity' - run: cat /tmp/envoy.log if: ${{ failure() || cancelled() }} @@ -104,9 +117,15 @@ jobs: timeout-minutes: 50 steps: - uses: actions/checkout@v1 + - id: should_run + name: 'Check whether to run' + run: ./mobile/tools/should_run_ci.sh - run: cd mobile && ./ci/mac_ci_setup.sh + if: steps.should_run.outputs.run_ci_job == 'true' name: 'Install dependencies' - - env: + - name: 'Build app' + if: steps.should_run.outputs.run_ci_job == 'true' + env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} run: | cd mobile && ./bazelw build \ @@ -114,9 +133,10 @@ jobs: $([ -z $GITHUB_TOKEN ] || echo "--config=remote-ci-macos") \ --remote_header="Authorization=Bearer $GITHUB_TOKEN" \ //test/swift/apps/experimental:app - name: 'Build swift app' # Run the app in the background and redirect logs. - - env: + - name: 'Run app' + if: steps.should_run.outputs.run_ci_job == 'true' + env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} run: | cd mobile && ./bazelw run \ @@ -124,8 +144,8 @@ jobs: $([ -z $GITHUB_TOKEN ] || echo "--config=remote-ci-macos") \ --remote_header="Authorization=Bearer $GITHUB_TOKEN" \ //test/swift/apps/experimental:app &> /tmp/envoy.log & - name: 'Run swift app' - run: sed '/received headers with status 200/q' <(touch /tmp/envoy.log && tail -F /tmp/envoy.log) + if: steps.should_run.outputs.run_ci_job == 'true' name: 'Check connectivity' - run: cat /tmp/envoy.log if: ${{ failure() || cancelled() }} @@ -137,9 +157,15 @@ jobs: timeout-minutes: 50 steps: - uses: actions/checkout@v1 + - id: should_run + name: 'Check whether to run' + run: ./mobile/tools/should_run_ci.sh - run: cd mobile && ./ci/mac_ci_setup.sh + if: steps.should_run.outputs.run_ci_job == 'true' name: 'Install dependencies' - - env: + - name: 'Build app' + if: steps.should_run.outputs.run_ci_job == 'true' + env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} run: | cd mobile && ./bazelw build \ @@ -147,9 +173,10 @@ jobs: $([ -z $GITHUB_TOKEN ] || echo "--config=remote-ci-macos") \ --remote_header="Authorization=Bearer $GITHUB_TOKEN" \ //examples/swift/async_await:app - name: 'Build swift app' # Run the app in the background and redirect logs. - - env: + - name: 'Run app' + if: steps.should_run.outputs.run_ci_job == 'true' + env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} run: | cd mobile && ./bazelw run \ @@ -157,8 +184,8 @@ jobs: $([ -z $GITHUB_TOKEN ] || echo "--config=remote-ci-macos") \ --remote_header="Authorization=Bearer $GITHUB_TOKEN" \ //examples/swift/async_await:app &> /tmp/envoy.log & - name: 'Run swift app' - run: sed '/\[2\] Uploaded 7 MB of data/q' <(touch /tmp/envoy.log && tail -F /tmp/envoy.log) + if: steps.should_run.outputs.run_ci_job == 'true' name: 'Check upload succeeded' - run: cat /tmp/envoy.log if: ${{ failure() || cancelled() }} @@ -170,9 +197,15 @@ jobs: timeout-minutes: 50 steps: - uses: actions/checkout@v1 + - id: should_run + name: 'Check whether to run' + run: ./mobile/tools/should_run_ci.sh - run: cd mobile && ./ci/mac_ci_setup.sh + if: steps.should_run.outputs.run_ci_job == 'true' name: 'Install dependencies' - - env: + - name: 'Build app' + if: steps.should_run.outputs.run_ci_job == 'true' + env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} run: | cd mobile && ./bazelw build \ @@ -180,9 +213,10 @@ jobs: $([ -z $GITHUB_TOKEN ] || echo "--config=remote-ci-macos") \ --remote_header="Authorization=Bearer $GITHUB_TOKEN" \ //examples/objective-c/hello_world:app - name: 'Build objective-c app' # Run the app in the background and redirect logs. - - env: + - name: 'Run app' + if: steps.should_run.outputs.run_ci_job == 'true' + env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} run: | cd mobile && ./bazelw run \ @@ -190,8 +224,8 @@ jobs: $([ -z $GITHUB_TOKEN ] || echo "--config=remote-ci-macos") \ --remote_header="Authorization=Bearer $GITHUB_TOKEN" \ //examples/objective-c/hello_world:app &> /tmp/envoy.log & - name: 'Run objective-c app' - run: sed '/received headers with status 200/q' <(touch /tmp/envoy.log && tail -F /tmp/envoy.log) + if: steps.should_run.outputs.run_ci_job == 'true' name: 'Check connectivity' - run: cat /tmp/envoy.log if: ${{ failure() || cancelled() }} diff --git a/.github/workflows/ios_tests.yml b/.github/workflows/ios_tests.yml index 672b1fcc9a4a..d95e62750457 100644 --- a/.github/workflows/ios_tests.yml +++ b/.github/workflows/ios_tests.yml @@ -4,13 +4,7 @@ on: push: branches: - main - paths: - - 'mobile/**' - - '.github/workflows/**' pull_request: - paths: - - 'mobile/**' - - '.github/workflows/**' jobs: swifttests: @@ -19,20 +13,14 @@ jobs: timeout-minutes: 120 steps: - uses: actions/checkout@v1 - - id: check_context + - id: should_run name: 'Check whether to run' - run: | - if git rev-parse --abbrev-ref HEAD | grep -q ^main$ || git diff --name-only origin/main | grep -qe common/ -e objective-c/ -e swift/ -e bazel/ -e ^\.bazelrc$ -e ^envoy$ -e ^WORKSPACE$ -e ^.github/workflows/ios_tests.yml$ ; then - echo "Tests will run." - echo "run_tests=true" >> $GITHUB_OUTPUT - else - echo "Skipping tests." - echo "run_tests=false" >> $GITHUB_OUTPUT - fi + run: ./mobile/tools/should_run_ci.sh - name: 'Install dependencies' + if: steps.should_run.outputs.run_ci_job == 'true' run: cd mobile && ./ci/mac_ci_setup.sh - name: 'Run swift library tests' - if: steps.check_context.outputs.run_tests == 'true' + if: steps.should_run.outputs.run_ci_job == 'true' env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} run: | @@ -50,20 +38,14 @@ jobs: timeout-minutes: 120 steps: - uses: actions/checkout@v1 - - id: check_context + - id: should_run name: 'Check whether to run' - run: | - if git rev-parse --abbrev-ref HEAD | grep -q ^main$ || git diff --name-only origin/main | grep -qe common/ -e objective-c/ -e swift/ -e bazel/ -e ^\.bazelrc$ -e ^envoy$ -e ^WORKSPACE$ -e ^.github/workflows/ios_tests.yml$ ; then - echo "Tests will run." - echo "run_tests=true" >> $GITHUB_OUTPUT - else - echo "Skipping tests." - echo "run_tests=false" >> $GITHUB_OUTPUT - fi + run: ./mobile/tools/should_run_ci.sh - name: 'Install dependencies' + if: steps.should_run.outputs.run_ci_job == 'true' run: cd mobile && ./ci/mac_ci_setup.sh - name: 'Run Objective-C library tests' - if: steps.check_context.outputs.run_tests == 'true' + if: steps.should_run.outputs.run_ci_job == 'true' env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} run: | diff --git a/.github/workflows/python_tests.yml b/.github/workflows/python_tests.yml index a4fd5116c5f9..64e990a8b50b 100644 --- a/.github/workflows/python_tests.yml +++ b/.github/workflows/python_tests.yml @@ -4,13 +4,7 @@ on: push: branches: - main - paths: - - 'mobile/**' - - '.github/workflows/**' pull_request: - paths: - - 'mobile/**' - - '.github/workflows/**' jobs: pythontests: @@ -23,18 +17,11 @@ jobs: - uses: actions/checkout@v1 - name: Add safe directory run: git config --global --add safe.directory /__w/envoy/envoy - - id: check_context + - id: should_run name: 'Check whether to run' - run: | - if git rev-parse --abbrev-ref HEAD | grep -q ^main$ || git diff --name-only origin/main | grep -qe common/ -e cc/ -e python/ -e bazel/ -e ^\.bazelrc$ -e ^envoy$ -e ^WORKSPACE$ -e ^.github/workflows/python_tests.yml$ ; then - echo "Tests will run." - echo "run_tests=true" >> $GITHUB_OUTPUT - else - echo "Skipping tests." - echo "run_tests=false" >> $GITHUB_OUTPUT - fi + run: ./mobile/tools/should_run_ci.sh - name: 'Run tests' - if: steps.check_context.outputs.run_tests == 'true' + if: steps.should_run.outputs.run_ci_job == 'true' env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} run: | diff --git a/.github/workflows/release_validation.yml b/.github/workflows/release_validation.yml index 53786d1b0c0a..94a1c78ddad7 100644 --- a/.github/workflows/release_validation.yml +++ b/.github/workflows/release_validation.yml @@ -4,13 +4,7 @@ on: push: branches: - main - paths: - - 'mobile/**' - - '.github/workflows/**' pull_request: - paths: - - 'mobile/**' - - '.github/workflows/**' jobs: validate_swiftpm_example: @@ -19,9 +13,15 @@ jobs: timeout-minutes: 120 steps: - uses: actions/checkout@v1 + - id: should_run + name: 'Check whether to run' + run: ./mobile/tools/should_run_ci.sh - run: cd mobile && ./ci/mac_ci_setup.sh + if: steps.should_run.outputs.run_ci_job == 'true' name: 'Install dependencies' - - env: + - name: 'Build xcframework' + if: steps.should_run.outputs.run_ci_job == 'true' + env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} run: | cd mobile && ./bazelw build \ @@ -29,10 +29,11 @@ jobs: $([ -z $GITHUB_TOKEN ] || echo "--config=remote-ci-macos") \ --remote_header="Authorization=Bearer $GITHUB_TOKEN" \ //:ios_xcframework - name: 'Build xcframework' # Ignore errors: Bad CRC when unzipping large files: https://bbs.archlinux.org/viewtopic.php?id=153011 - run: unzip mobile/bazel-bin/library/swift/Envoy.xcframework.zip -d mobile/examples/swift/swiftpm/Packages || true + if: steps.should_run.outputs.run_ci_job == 'true' name: 'Unzip xcframework' - run: xcodebuild -project mobile/examples/swift/swiftpm/EnvoySwiftPMExample.xcodeproj -scheme EnvoySwiftPMExample -destination platform="iOS Simulator,name=iPhone 13 Pro Max,OS=16.1" + if: steps.should_run.outputs.run_ci_job == 'true' name: 'Build app' # TODO(jpsim): Run app and inspect logs to validate diff --git a/.github/workflows/tsan.yml b/.github/workflows/tsan.yml index e2f6b8e0691f..7509fb7324bd 100644 --- a/.github/workflows/tsan.yml +++ b/.github/workflows/tsan.yml @@ -4,13 +4,7 @@ on: push: branches: - main - paths: - - 'mobile/**' - - '.github/workflows/**' pull_request: - paths: - - 'mobile/**' - - '.github/workflows/**' jobs: tsan: @@ -26,18 +20,11 @@ jobs: - uses: actions/checkout@v1 - name: Add safe directory run: git config --global --add safe.directory /__w/envoy/envoy - - id: check_context + - id: should_run name: 'Check whether to run' - run: | - if git rev-parse --abbrev-ref HEAD | grep -q ^main$ || git diff --name-only origin/main | grep -qe common/ -e bazel/ -e ^\.bazelrc$ -e ^envoy$ -e ^WORKSPACE$ -e ^.github/workflows/tsan.yml$ ; then - echo "Tests will run." - echo "run_tests=true" >> $GITHUB_OUTPUT - else - echo "Skipping tests." - echo "run_tests=false" >> $GITHUB_OUTPUT - fi + run: ./mobile/tools/should_run_ci.sh - uses: actions/setup-java@c3ac5dd0ed8db40fedb61c32fbe677e6b355e94c - if: steps.check-cache.outputs.cache-hit != 'true' + if: steps.should_run.outputs.run_ci_job == 'true' with: java-version: '8' java-package: jdk @@ -46,7 +33,7 @@ jobs: - name: 'Run tests' env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - if: steps.check_context.outputs.run_tests == 'true' + if: steps.should_run.outputs.run_ci_job == 'true' run: | cd mobile && ./bazelw test \ --test_output=all \ diff --git a/mobile/tools/should_run_ci.sh b/mobile/tools/should_run_ci.sh new file mode 100755 index 000000000000..1c0492ec70d4 --- /dev/null +++ b/mobile/tools/should_run_ci.sh @@ -0,0 +1,44 @@ +#!/usr/bin/env bash + +set -euo pipefail + +######################################################################## +# should_run_ci.sh +# +# Checks current branch and changed PR paths to determine if mobile CI +# jobs should be run. +######################################################################## + +job="$GITHUB_JOB" +branch_name="$GITHUB_REF_NAME" + +function success() { + echo "Running $job because there are $1 changes on $branch_name" + echo "run_ci_job=true" >> "$GITHUB_OUTPUT" +} + +function failure() { + echo "Skipping $job because there are no mobile changes on $branch_name" + echo "run_ci_job=false" >> "$GITHUB_OUTPUT" +} + +# TODO(jpsim): Consider enabling this if the load on EngFlow is ok +# if [[ $branch_name == "main" ]]; then +# # Run all mobile CI jobs on `main` +# echo "Running $job because current branch is main" +# echo "run_ci_job=true" >> "$GITHUB_OUTPUT" +# exit 0 +# fi + +base_commit="$(git merge-base origin/main HEAD)" +changed_files="$(git diff "$base_commit" --name-only)" + +if grep -q "^mobile/" <<< "$changed_files"; then + success "mobile" +elif grep -q "^bazel/repository_locations\.bzl" <<< "$changed_files"; then + success "bazel/repository_locations.bzl" +elif grep -q "^\.github/workflows/" <<< "$changed_files"; then + success "GitHub Workflows" +else + failure +fi From 97f6340a2ab7f9a47c19e4edf1b70e8f1408182c Mon Sep 17 00:00:00 2001 From: phlax Date: Wed, 7 Dec 2022 14:45:15 +0000 Subject: [PATCH 14/23] ci: Skip docker/examples verification for docs or mobile only changes (#24417) Signed-off-by: Ryan Northey --- .azure-pipelines/pipelines.yml | 32 ++++++++++++++++++++++++++++++-- 1 file changed, 30 insertions(+), 2 deletions(-) diff --git a/.azure-pipelines/pipelines.yml b/.azure-pipelines/pipelines.yml index ef0a3dc9efd6..30b1bc8959e7 100644 --- a/.azure-pipelines/pipelines.yml +++ b/.azure-pipelines/pipelines.yml @@ -469,10 +469,11 @@ stages: echo "checks complete" - stage: docker - dependsOn: ["linux_x64", "linux_arm64"] + dependsOn: ["env", "linux_x64", "linux_arm64"] jobs: - job: docker displayName: "linux multiarch" + condition: and(not(canceled()), succeeded(), ne(stageDependencies.env.repo.outputs['changed.mobileOnly'], 'true'), ne(stageDependencies.env.repo.outputs['changed.docsOnly'], 'true')) timeoutInMinutes: 120 pool: vmImage: "ubuntu-20.04" @@ -540,6 +541,19 @@ stages: timeoutInMinutes: 10 condition: always() + - job: linux + dependsOn: ["docker"] + pool: + vmImage: "ubuntu-20.04" + # This condition ensures that this (required) job passes if all of + # the preceeding jobs either pass or are skipped + # adapted from: + # https://learn.microsoft.com/en-us/azure/devops/pipelines/process/expressions?view=azure-devops#job-to-job-dependencies-within-one-stage + condition: in(dependencies.docker.result, 'Succeeded', 'SucceededWithIssues', 'Skipped') + steps: + - bash: | + echo "Docker linux built" + - stage: publish dependsOn: ["docker"] condition: and(succeeded(), eq(variables['PostSubmit'], true), ne(variables['NoSync'], true)) @@ -678,9 +692,10 @@ stages: MAINTAINER_GPG_KEY_PASSPHRASE: $(MaintainerGPGKeyPassphrase) - stage: verify - dependsOn: ["docker"] + dependsOn: ["env", "docker"] jobs: - job: examples + condition: and(not(canceled()), succeeded(), ne(stageDependencies.env.repo.outputs['changed.mobileOnly'], 'true'), ne(stageDependencies.env.repo.outputs['changed.docsOnly'], 'true')) pool: vmImage: "ubuntu-20.04" steps: @@ -695,6 +710,19 @@ stages: ENVOY_DOCKER_BUILD_DIR: $(Build.StagingDirectory) NO_BUILD_SETUP: 1 + - job: verified + dependsOn: ["examples"] + pool: + vmImage: "ubuntu-20.04" + # This condition ensures that this (required) job passes if all of + # the preceeding jobs either pass or are skipped + # adapted from: + # https://learn.microsoft.com/en-us/azure/devops/pipelines/process/expressions?view=azure-devops#job-to-job-dependencies-within-one-stage + condition: in(dependencies.examples.result, 'Succeeded', 'SucceededWithIssues', 'Skipped') + steps: + - bash: | + echo "examples verifiied" + - stage: macos dependsOn: ["precheck"] jobs: From 50ac04b41af595f1df9a925146bd5314201ca425 Mon Sep 17 00:00:00 2001 From: code Date: Wed, 7 Dec 2022 22:48:36 +0800 Subject: [PATCH 15/23] lb api: moving load balancing policy specific configuration to extension configuration (#23967) Signed-off-by: wbpcode --- api/BUILD | 7 ++ .../load_balancing_policies/common/v3/BUILD | 13 ++ .../common/v3/common.proto | 115 ++++++++++++++++++ .../least_request/v3/BUILD | 2 +- .../least_request/v3/least_request.proto | 8 +- .../load_balancing_policies/maglev/v3/BUILD | 5 +- .../maglev/v3/maglev.proto | 6 +- .../load_balancing_policies/random/v3/BUILD | 5 +- .../random/v3/random.proto | 5 +- .../ring_hash/v3/BUILD | 6 +- .../ring_hash/v3/ring_hash.proto | 26 +++- .../round_robin/v3/BUILD | 2 +- .../round_robin/v3/round_robin.proto | 8 +- .../wrr_locality/v3/wrr_locality.proto | 2 +- api/versioning/BUILD | 1 + 15 files changed, 193 insertions(+), 18 deletions(-) create mode 100644 api/envoy/extensions/load_balancing_policies/common/v3/BUILD create mode 100644 api/envoy/extensions/load_balancing_policies/common/v3/common.proto diff --git a/api/BUILD b/api/BUILD index 53f57c57d87a..42744b01121e 100644 --- a/api/BUILD +++ b/api/BUILD @@ -240,6 +240,13 @@ proto_library( "//envoy/extensions/internal_redirect/previous_routes/v3:pkg", "//envoy/extensions/internal_redirect/safe_cross_scheme/v3:pkg", "//envoy/extensions/key_value/file_based/v3:pkg", + "//envoy/extensions/load_balancing_policies/common/v3:pkg", + "//envoy/extensions/load_balancing_policies/least_request/v3:pkg", + "//envoy/extensions/load_balancing_policies/maglev/v3:pkg", + "//envoy/extensions/load_balancing_policies/random/v3:pkg", + "//envoy/extensions/load_balancing_policies/ring_hash/v3:pkg", + "//envoy/extensions/load_balancing_policies/round_robin/v3:pkg", + "//envoy/extensions/load_balancing_policies/wrr_locality/v3:pkg", "//envoy/extensions/matching/common_inputs/environment_variable/v3:pkg", "//envoy/extensions/matching/common_inputs/network/v3:pkg", "//envoy/extensions/matching/common_inputs/ssl/v3:pkg", diff --git a/api/envoy/extensions/load_balancing_policies/common/v3/BUILD b/api/envoy/extensions/load_balancing_policies/common/v3/BUILD new file mode 100644 index 000000000000..ad2fc9a9a84f --- /dev/null +++ b/api/envoy/extensions/load_balancing_policies/common/v3/BUILD @@ -0,0 +1,13 @@ +# DO NOT EDIT. This file is generated by tools/proto_format/proto_sync.py. + +load("@envoy_api//bazel:api_build_system.bzl", "api_proto_package") + +licenses(["notice"]) # Apache 2 + +api_proto_package( + deps = [ + "//envoy/config/core/v3:pkg", + "//envoy/type/v3:pkg", + "@com_github_cncf_udpa//udpa/annotations:pkg", + ], +) diff --git a/api/envoy/extensions/load_balancing_policies/common/v3/common.proto b/api/envoy/extensions/load_balancing_policies/common/v3/common.proto new file mode 100644 index 000000000000..a19765f77b4b --- /dev/null +++ b/api/envoy/extensions/load_balancing_policies/common/v3/common.proto @@ -0,0 +1,115 @@ +syntax = "proto3"; + +package envoy.extensions.load_balancing_policies.common.v3; + +import "envoy/config/core/v3/base.proto"; +import "envoy/type/v3/percent.proto"; + +import "google/protobuf/duration.proto"; +import "google/protobuf/wrappers.proto"; + +import "udpa/annotations/status.proto"; +import "validate/validate.proto"; + +option java_package = "io.envoyproxy.envoy.extensions.load_balancing_policies.common.v3"; +option java_outer_classname = "CommonProto"; +option java_multiple_files = true; +option go_package = "github.com/envoyproxy/go-control-plane/envoy/extensions/load_balancing_policies/common/v3;commonv3"; +option (udpa.annotations.file_status).package_version_status = ACTIVE; + +// [#protodoc-title: Common configuration for two or more load balancing policy extensions] +// [#not-implemented-hide:] + +message LocalityLbConfig { + // Configuration for :ref:`zone aware routing + // `. + message ZoneAwareLbConfig { + // Configures percentage of requests that will be considered for zone aware routing + // if zone aware routing is configured. If not specified, the default is 100%. + // * :ref:`runtime values `. + // * :ref:`Zone aware routing support `. + type.v3.Percent routing_enabled = 1; + + // Configures minimum upstream cluster size required for zone aware routing + // If upstream cluster size is less than specified, zone aware routing is not performed + // even if zone aware routing is configured. If not specified, the default is 6. + // * :ref:`runtime values `. + // * :ref:`Zone aware routing support `. + google.protobuf.UInt64Value min_cluster_size = 2; + + // If set to true, Envoy will not consider any hosts when the cluster is in :ref:`panic + // mode`. Instead, the cluster will fail all + // requests as if all hosts are unhealthy. This can help avoid potentially overwhelming a + // failing service. + bool fail_traffic_on_panic = 3; + } + + // Configuration for :ref:`locality weighted load balancing + // ` + message LocalityWeightedLbConfig { + } + + oneof locality_config_specifier { + option (validate.required) = true; + + // Configuration for local zone aware load balancing. + ZoneAwareLbConfig zone_aware_lb_config = 1; + + // Enable locality weighted load balancing. + LocalityWeightedLbConfig locality_weighted_lb_config = 2; + } +} + +// Configuration for :ref:`slow start mode `. +message SlowStartConfig { + // Represents the size of slow start window. + // If set, the newly created host remains in slow start mode starting from its creation time + // for the duration of slow start window. + google.protobuf.Duration slow_start_window = 1; + + // This parameter controls the speed of traffic increase over the slow start window. Defaults to 1.0, + // so that endpoint would get linearly increasing amount of traffic. + // When increasing the value for this parameter, the speed of traffic ramp-up increases non-linearly. + // The value of aggression parameter should be greater than 0.0. + // By tuning the parameter, is possible to achieve polynomial or exponential shape of ramp-up curve. + // + // During slow start window, effective weight of an endpoint would be scaled with time factor and aggression: + // ``new_weight = weight * max(min_weight_percent, time_factor ^ (1 / aggression))``, + // where ``time_factor=(time_since_start_seconds / slow_start_time_seconds)``. + // + // As time progresses, more and more traffic would be sent to endpoint, which is in slow start window. + // Once host exits slow start, time_factor and aggression no longer affect its weight. + config.core.v3.RuntimeDouble aggression = 2; + + // Configures the minimum percentage of origin weight that avoids too small new weight, + // which may cause endpoints in slow start mode receive no traffic in slow start window. + // If not specified, the default is 10%. + type.v3.Percent min_weight_percent = 3; +} + +// Common Configuration for all consistent hashing load balancers (MaglevLb, RingHashLb, etc.) +message ConsistentHashingLbConfig { + // If set to ``true``, the cluster will use hostname instead of the resolved + // address as the key to consistently hash to an upstream host. Only valid for StrictDNS clusters with hostnames which resolve to a single IP address. + bool use_hostname_for_hashing = 1; + + // Configures percentage of average cluster load to bound per upstream host. For example, with a value of 150 + // no upstream host will get a load more than 1.5 times the average load of all the hosts in the cluster. + // If not specified, the load is not bounded for any upstream host. Typical value for this parameter is between 120 and 200. + // Minimum is 100. + // + // Applies to both Ring Hash and Maglev load balancers. + // + // This is implemented based on the method described in the paper https://arxiv.org/abs/1608.01350. For the specified + // ``hash_balance_factor``, requests to any upstream host are capped at ``hash_balance_factor/100`` times the average number of requests + // across the cluster. When a request arrives for an upstream host that is currently serving at its max capacity, linear probing + // is used to identify an eligible host. Further, the linear probe is implemented using a random jump in hosts ring/table to identify + // the eligible host (this technique is as described in the paper https://arxiv.org/abs/1908.08762 - the random jump avoids the + // cascading overflow effect when choosing the next host in the ring/table). + // + // If weights are specified on the hosts, they are respected. + // + // This is an O(N) algorithm, unlike other load balancers. Using a lower ``hash_balance_factor`` results in more hosts + // being probed, so use a higher value if you require better performance. + google.protobuf.UInt32Value hash_balance_factor = 2 [(validate.rules).uint32 = {gte: 100}]; +} diff --git a/api/envoy/extensions/load_balancing_policies/least_request/v3/BUILD b/api/envoy/extensions/load_balancing_policies/least_request/v3/BUILD index 2a87d27d297b..366a3c324b35 100644 --- a/api/envoy/extensions/load_balancing_policies/least_request/v3/BUILD +++ b/api/envoy/extensions/load_balancing_policies/least_request/v3/BUILD @@ -6,8 +6,8 @@ licenses(["notice"]) # Apache 2 api_proto_package( deps = [ - "//envoy/config/cluster/v3:pkg", "//envoy/config/core/v3:pkg", + "//envoy/extensions/load_balancing_policies/common/v3:pkg", "@com_github_cncf_udpa//udpa/annotations:pkg", ], ) diff --git a/api/envoy/extensions/load_balancing_policies/least_request/v3/least_request.proto b/api/envoy/extensions/load_balancing_policies/least_request/v3/least_request.proto index fd34260d05ed..d6eb834acded 100644 --- a/api/envoy/extensions/load_balancing_policies/least_request/v3/least_request.proto +++ b/api/envoy/extensions/load_balancing_policies/least_request/v3/least_request.proto @@ -2,8 +2,8 @@ syntax = "proto3"; package envoy.extensions.load_balancing_policies.least_request.v3; -import "envoy/config/cluster/v3/cluster.proto"; import "envoy/config/core/v3/base.proto"; +import "envoy/extensions/load_balancing_policies/common/v3/common.proto"; import "google/protobuf/wrappers.proto"; @@ -22,7 +22,6 @@ option (udpa.annotations.file_status).package_version_status = ACTIVE; // This configuration allows the built-in LEAST_REQUEST LB policy to be configured via the LB policy // extension point. See the :ref:`load balancing architecture overview // ` for more information. -// [#extension: envoy.load_balancing_policies] message LeastRequest { // The number of random healthy hosts from which the host with the fewest active requests will // be chosen. Defaults to 2 so that we perform two-choice selection if the field is not set. @@ -55,5 +54,8 @@ message LeastRequest { // Configuration for slow start mode. // If this configuration is not set, slow start will not be not enabled. - config.cluster.v3.Cluster.SlowStartConfig slow_start_config = 3; + common.v3.SlowStartConfig slow_start_config = 3; + + // Configuration for local zone aware load balancing or locality weighted load balancing. + common.v3.LocalityLbConfig locality_lb_config = 4; } diff --git a/api/envoy/extensions/load_balancing_policies/maglev/v3/BUILD b/api/envoy/extensions/load_balancing_policies/maglev/v3/BUILD index ee92fb652582..6a0be4c9bf97 100644 --- a/api/envoy/extensions/load_balancing_policies/maglev/v3/BUILD +++ b/api/envoy/extensions/load_balancing_policies/maglev/v3/BUILD @@ -5,5 +5,8 @@ load("@envoy_api//bazel:api_build_system.bzl", "api_proto_package") licenses(["notice"]) # Apache 2 api_proto_package( - deps = ["@com_github_cncf_udpa//udpa/annotations:pkg"], + deps = [ + "//envoy/extensions/load_balancing_policies/common/v3:pkg", + "@com_github_cncf_udpa//udpa/annotations:pkg", + ], ) diff --git a/api/envoy/extensions/load_balancing_policies/maglev/v3/maglev.proto b/api/envoy/extensions/load_balancing_policies/maglev/v3/maglev.proto index 4ed884fd7a76..d34920a7b59e 100644 --- a/api/envoy/extensions/load_balancing_policies/maglev/v3/maglev.proto +++ b/api/envoy/extensions/load_balancing_policies/maglev/v3/maglev.proto @@ -2,6 +2,8 @@ syntax = "proto3"; package envoy.extensions.load_balancing_policies.maglev.v3; +import "envoy/extensions/load_balancing_policies/common/v3/common.proto"; + import "google/protobuf/wrappers.proto"; import "udpa/annotations/status.proto"; @@ -19,11 +21,13 @@ option (udpa.annotations.file_status).package_version_status = ACTIVE; // This configuration allows the built-in Maglev LB policy to be configured via the LB policy // extension point. See the :ref:`load balancing architecture overview // ` and :ref:`Maglev` for more information. -// [#extension: envoy.load_balancing_policies] message Maglev { // The table size for Maglev hashing. Maglev aims for "minimal disruption" rather than an absolute guarantee. // Minimal disruption means that when the set of upstream hosts change, a connection will likely be sent to the same // upstream as it was before. Increasing the table size reduces the amount of disruption. // The table size must be prime number limited to 5000011. If it is not specified, the default is 65537. google.protobuf.UInt64Value table_size = 1 [(validate.rules).uint64 = {lte: 5000011}]; + + // Common configuration for hashing-based load balancing policies. + common.v3.ConsistentHashingLbConfig consistent_hashing_lb_config = 2; } diff --git a/api/envoy/extensions/load_balancing_policies/random/v3/BUILD b/api/envoy/extensions/load_balancing_policies/random/v3/BUILD index ee92fb652582..6a0be4c9bf97 100644 --- a/api/envoy/extensions/load_balancing_policies/random/v3/BUILD +++ b/api/envoy/extensions/load_balancing_policies/random/v3/BUILD @@ -5,5 +5,8 @@ load("@envoy_api//bazel:api_build_system.bzl", "api_proto_package") licenses(["notice"]) # Apache 2 api_proto_package( - deps = ["@com_github_cncf_udpa//udpa/annotations:pkg"], + deps = [ + "//envoy/extensions/load_balancing_policies/common/v3:pkg", + "@com_github_cncf_udpa//udpa/annotations:pkg", + ], ) diff --git a/api/envoy/extensions/load_balancing_policies/random/v3/random.proto b/api/envoy/extensions/load_balancing_policies/random/v3/random.proto index 1d01dfba1cd5..37efa7c13305 100644 --- a/api/envoy/extensions/load_balancing_policies/random/v3/random.proto +++ b/api/envoy/extensions/load_balancing_policies/random/v3/random.proto @@ -2,6 +2,8 @@ syntax = "proto3"; package envoy.extensions.load_balancing_policies.random.v3; +import "envoy/extensions/load_balancing_policies/common/v3/common.proto"; + import "udpa/annotations/status.proto"; option java_package = "io.envoyproxy.envoy.extensions.load_balancing_policies.random.v3"; @@ -16,6 +18,7 @@ option (udpa.annotations.file_status).package_version_status = ACTIVE; // This configuration allows the built-in Random LB policy to be configured via the LB policy // extension point. See the :ref:`load balancing architecture overview // ` for more information. -// [#extension: envoy.load_balancing_policies] message Random { + // Configuration for local zone aware load balancing or locality weighted load balancing. + common.v3.LocalityLbConfig locality_lb_config = 1; } diff --git a/api/envoy/extensions/load_balancing_policies/ring_hash/v3/BUILD b/api/envoy/extensions/load_balancing_policies/ring_hash/v3/BUILD index ee92fb652582..9ec681aa9756 100644 --- a/api/envoy/extensions/load_balancing_policies/ring_hash/v3/BUILD +++ b/api/envoy/extensions/load_balancing_policies/ring_hash/v3/BUILD @@ -5,5 +5,9 @@ load("@envoy_api//bazel:api_build_system.bzl", "api_proto_package") licenses(["notice"]) # Apache 2 api_proto_package( - deps = ["@com_github_cncf_udpa//udpa/annotations:pkg"], + deps = [ + "//envoy/annotations:pkg", + "//envoy/extensions/load_balancing_policies/common/v3:pkg", + "@com_github_cncf_udpa//udpa/annotations:pkg", + ], ) diff --git a/api/envoy/extensions/load_balancing_policies/ring_hash/v3/ring_hash.proto b/api/envoy/extensions/load_balancing_policies/ring_hash/v3/ring_hash.proto index 0001b6ee0aff..fa1da5fa60ec 100644 --- a/api/envoy/extensions/load_balancing_policies/ring_hash/v3/ring_hash.proto +++ b/api/envoy/extensions/load_balancing_policies/ring_hash/v3/ring_hash.proto @@ -2,8 +2,11 @@ syntax = "proto3"; package envoy.extensions.load_balancing_policies.ring_hash.v3; +import "envoy/extensions/load_balancing_policies/common/v3/common.proto"; + import "google/protobuf/wrappers.proto"; +import "envoy/annotations/deprecation.proto"; import "udpa/annotations/status.proto"; import "validate/validate.proto"; @@ -19,8 +22,7 @@ option (udpa.annotations.file_status).package_version_status = ACTIVE; // This configuration allows the built-in RING_HASH LB policy to be configured via the LB policy // extension point. See the :ref:`load balancing architecture overview // ` for more information. -// [#extension: envoy.load_balancing_policies] -// [#next-free-field: 6] +// [#next-free-field: 7] message RingHash { // The hash function used to hash hosts onto the ketama ring. enum HashFunction { @@ -53,7 +55,12 @@ message RingHash { // If set to `true`, the cluster will use hostname instead of the resolved // address as the key to consistently hash to an upstream host. Only valid for StrictDNS clusters with hostnames which resolve to a single IP address. - bool use_hostname_for_hashing = 4; + // + // ..note:: + // This is deprecated and please use :ref:`consistent_hashing_lb_config + // ` instead. + bool use_hostname_for_hashing = 4 + [deprecated = true, (envoy.annotations.deprecated_at_minor_version) = "3.0"]; // Configures percentage of average cluster load to bound per upstream host. For example, with a value of 150 // no upstream host will get a load more than 1.5 times the average load of all the hosts in the cluster. @@ -71,5 +78,16 @@ message RingHash { // // This is an O(N) algorithm, unlike other load balancers. Using a lower `hash_balance_factor` results in more hosts // being probed, so use a higher value if you require better performance. - google.protobuf.UInt32Value hash_balance_factor = 5 [(validate.rules).uint32 = {gte: 100}]; + // + // ..note:: + // This is deprecated and please use :ref:`consistent_hashing_lb_config + // ` instead. + google.protobuf.UInt32Value hash_balance_factor = 5 [ + deprecated = true, + (validate.rules).uint32 = {gte: 100}, + (envoy.annotations.deprecated_at_minor_version) = "3.0" + ]; + + // Common configuration for hashing-based load balancing policies. + common.v3.ConsistentHashingLbConfig consistent_hashing_lb_config = 6; } diff --git a/api/envoy/extensions/load_balancing_policies/round_robin/v3/BUILD b/api/envoy/extensions/load_balancing_policies/round_robin/v3/BUILD index b49ae9078cfc..6a0be4c9bf97 100644 --- a/api/envoy/extensions/load_balancing_policies/round_robin/v3/BUILD +++ b/api/envoy/extensions/load_balancing_policies/round_robin/v3/BUILD @@ -6,7 +6,7 @@ licenses(["notice"]) # Apache 2 api_proto_package( deps = [ - "//envoy/config/cluster/v3:pkg", + "//envoy/extensions/load_balancing_policies/common/v3:pkg", "@com_github_cncf_udpa//udpa/annotations:pkg", ], ) diff --git a/api/envoy/extensions/load_balancing_policies/round_robin/v3/round_robin.proto b/api/envoy/extensions/load_balancing_policies/round_robin/v3/round_robin.proto index 1ef8fc814500..01b8a62e002a 100644 --- a/api/envoy/extensions/load_balancing_policies/round_robin/v3/round_robin.proto +++ b/api/envoy/extensions/load_balancing_policies/round_robin/v3/round_robin.proto @@ -2,7 +2,7 @@ syntax = "proto3"; package envoy.extensions.load_balancing_policies.round_robin.v3; -import "envoy/config/cluster/v3/cluster.proto"; +import "envoy/extensions/load_balancing_policies/common/v3/common.proto"; import "udpa/annotations/status.proto"; @@ -18,9 +18,11 @@ option (udpa.annotations.file_status).package_version_status = ACTIVE; // This configuration allows the built-in ROUND_ROBIN LB policy to be configured via the LB policy // extension point. See the :ref:`load balancing architecture overview // ` for more information. -// [#extension: envoy.load_balancing_policies] message RoundRobin { // Configuration for slow start mode. // If this configuration is not set, slow start will not be not enabled. - config.cluster.v3.Cluster.SlowStartConfig slow_start_config = 1; + common.v3.SlowStartConfig slow_start_config = 1; + + // Configuration for local zone aware load balancing or locality weighted load balancing. + common.v3.LocalityLbConfig locality_lb_config = 2; } diff --git a/api/envoy/extensions/load_balancing_policies/wrr_locality/v3/wrr_locality.proto b/api/envoy/extensions/load_balancing_policies/wrr_locality/v3/wrr_locality.proto index 855cbf8fe240..ab8367a401a9 100644 --- a/api/envoy/extensions/load_balancing_policies/wrr_locality/v3/wrr_locality.proto +++ b/api/envoy/extensions/load_balancing_policies/wrr_locality/v3/wrr_locality.proto @@ -14,10 +14,10 @@ option go_package = "github.com/envoyproxy/go-control-plane/envoy/extensions/loa option (udpa.annotations.file_status).package_version_status = ACTIVE; // [#protodoc-title: Weighted Round Robin Locality-Picking Load Balancing Policy] +// [#not-implemented-hide:] // Configuration for the wrr_locality LB policy. See the :ref:`load balancing architecture overview // ` for more information. -// [#extension: envoy.load_balancing_policies] message WrrLocality { // The child LB policy to create for endpoint-picking within the chosen locality. config.cluster.v3.LoadBalancingPolicy endpoint_picking_policy = 1 diff --git a/api/versioning/BUILD b/api/versioning/BUILD index 51bbfa80b8c3..a0a178b455ad 100644 --- a/api/versioning/BUILD +++ b/api/versioning/BUILD @@ -178,6 +178,7 @@ proto_library( "//envoy/extensions/internal_redirect/previous_routes/v3:pkg", "//envoy/extensions/internal_redirect/safe_cross_scheme/v3:pkg", "//envoy/extensions/key_value/file_based/v3:pkg", + "//envoy/extensions/load_balancing_policies/common/v3:pkg", "//envoy/extensions/load_balancing_policies/least_request/v3:pkg", "//envoy/extensions/load_balancing_policies/maglev/v3:pkg", "//envoy/extensions/load_balancing_policies/random/v3:pkg", From ac201042bb7489ac2fa33946cb88b552adfdce2b Mon Sep 17 00:00:00 2001 From: alyssawilk Date: Wed, 7 Dec 2022 10:34:57 -0500 Subject: [PATCH 16/23] downstream: refactoring code to remove listener hard deps (#24394) Risk Level: low (simply moving code around) Testing: n/a Docs Changes: n/a Release Notes: n/a Signed-off-by: Alyssa Wilk --- envoy/server/factory_context.h | 7 +++ source/common/quic/BUILD | 7 +-- source/common/quic/envoy_quic_dispatcher.h | 3 +- source/common/quic/envoy_quic_proof_source.h | 3 +- ...nvoy_quic_proof_source_factory_interface.h | 2 +- .../quic/envoy_quic_server_connection.h | 1 - .../matching/actions/format_string/BUILD | 1 - .../matching/actions/format_string/config.cc | 9 ++-- .../matching/actions/format_string/config.h | 13 +++--- source/server/BUILD | 11 +++++ source/server/active_listener_base.h | 35 +-------------- source/server/filter_chain_manager_impl.cc | 5 ++- source/server/filter_chain_manager_impl.h | 6 --- source/server/listener_stats.h | 43 +++++++++++++++++++ test/common/quic/BUILD | 1 + .../common/quic/envoy_quic_dispatcher_test.cc | 1 + .../actions/format_string/config_test.cc | 4 +- 17 files changed, 87 insertions(+), 65 deletions(-) create mode 100644 source/server/listener_stats.h diff --git a/envoy/server/factory_context.h b/envoy/server/factory_context.h index cc8d3b3c9bb8..3ecb7e0bcb8a 100644 --- a/envoy/server/factory_context.h +++ b/envoy/server/factory_context.h @@ -288,6 +288,13 @@ class FilterChainFactoryContext : public virtual FactoryContext { }; using FilterChainFactoryContextPtr = std::unique_ptr; +using FilterChainsByName = absl::flat_hash_map; + +class FilterChainBaseAction : public Matcher::Action { +public: + virtual const Network::FilterChain* get(const FilterChainsByName& filter_chains_by_name, + const StreamInfo::StreamInfo& info) const PURE; +}; /** * An implementation of FactoryContext. The life time should cover the lifetime of the filter chains diff --git a/source/common/quic/BUILD b/source/common/quic/BUILD index e6db3db40fad..272477ac49ef 100644 --- a/source/common/quic/BUILD +++ b/source/common/quic/BUILD @@ -108,7 +108,7 @@ envoy_cc_library( ":quic_transport_socket_factory_lib", "//envoy/ssl:tls_certificate_config_interface", "//source/common/stream_info:stream_info_lib", - "//source/server:connection_handler_lib", + "//source/server:listener_stats", "@com_github_google_quiche//:quic_core_crypto_certificate_view_lib", ], ) @@ -324,7 +324,6 @@ envoy_cc_library( ":quic_io_handle_wrapper_lib", ":quic_network_connection_lib", "//source/common/quic:envoy_quic_utils_lib", - "//source/server:connection_handler_lib", "@com_github_google_quiche//:quic_core_connection_lib", ], ) @@ -357,7 +356,6 @@ envoy_cc_library( ":envoy_quic_server_session_lib", ":quic_stat_names_lib", "//envoy/network:listener_interface", - "//source/server:connection_handler_lib", "@com_github_google_quiche//:quic_core_server_lib", "@com_github_google_quiche//:quic_core_utils_lib", ], @@ -387,7 +385,7 @@ envoy_cc_library( "//source/common/protobuf:utility_lib", "//source/common/runtime:runtime_lib", "//source/extensions/quic/connection_id_generator:envoy_deterministic_connection_id_generator_config", - "//source/server:connection_handler_lib", + "//source/server:active_udp_listener", "@envoy_api//envoy/config/listener/v3:pkg_cc_proto", "@envoy_api//envoy/extensions/quic/connection_id_generator/v3:pkg_cc_proto", "@envoy_api//envoy/extensions/quic/crypto_stream/v3:pkg_cc_proto", @@ -516,7 +514,6 @@ envoy_cc_library( tags = ["nofips"], deps = [ "//envoy/config:typed_config_interface", - "//source/server:connection_handler_lib", "@com_github_google_quiche//:quic_core_crypto_proof_source_lib", ], ) diff --git a/source/common/quic/envoy_quic_dispatcher.h b/source/common/quic/envoy_quic_dispatcher.h index 38b13e43a21c..0807ae667389 100644 --- a/source/common/quic/envoy_quic_dispatcher.h +++ b/source/common/quic/envoy_quic_dispatcher.h @@ -7,8 +7,7 @@ #include "source/common/quic/envoy_quic_crypto_stream_factory.h" #include "source/common/quic/envoy_quic_server_session.h" #include "source/common/quic/quic_stat_names.h" -#include "source/server/active_listener_base.h" -#include "source/server/connection_handler_impl.h" +#include "source/server/listener_stats.h" #include "quiche/quic/core/quic_dispatcher.h" #include "quiche/quic/core/quic_utils.h" diff --git a/source/common/quic/envoy_quic_proof_source.h b/source/common/quic/envoy_quic_proof_source.h index acfb45707406..bc3b3c127f32 100644 --- a/source/common/quic/envoy_quic_proof_source.h +++ b/source/common/quic/envoy_quic_proof_source.h @@ -2,8 +2,7 @@ #include "source/common/quic/envoy_quic_proof_source_base.h" #include "source/common/quic/quic_transport_socket_factory.h" -#include "source/server/active_listener_base.h" -#include "source/server/connection_handler_impl.h" +#include "source/server/listener_stats.h" namespace Envoy { namespace Quic { diff --git a/source/common/quic/envoy_quic_proof_source_factory_interface.h b/source/common/quic/envoy_quic_proof_source_factory_interface.h index ca3780b7344a..9de1df10f8f8 100644 --- a/source/common/quic/envoy_quic_proof_source_factory_interface.h +++ b/source/common/quic/envoy_quic_proof_source_factory_interface.h @@ -4,7 +4,7 @@ #include "envoy/network/filter.h" #include "envoy/network/socket.h" -#include "source/server/active_listener_base.h" +#include "source/server/listener_stats.h" #include "quiche/quic/core/crypto/proof_source.h" diff --git a/source/common/quic/envoy_quic_server_connection.h b/source/common/quic/envoy_quic_server_connection.h index 38fe005d5efa..487fe81ad168 100644 --- a/source/common/quic/envoy_quic_server_connection.h +++ b/source/common/quic/envoy_quic_server_connection.h @@ -4,7 +4,6 @@ #include "source/common/quic/envoy_quic_utils.h" #include "source/common/quic/quic_network_connection.h" -#include "source/server/connection_handler_impl.h" #include "quiche/quic/core/quic_connection.h" diff --git a/source/extensions/matching/actions/format_string/BUILD b/source/extensions/matching/actions/format_string/BUILD index 086159568384..d75511b1e91b 100644 --- a/source/extensions/matching/actions/format_string/BUILD +++ b/source/extensions/matching/actions/format_string/BUILD @@ -20,7 +20,6 @@ envoy_cc_extension( "//source/common/formatter:substitution_format_string_lib", "//source/common/http:header_map_lib", "//source/common/matcher:matcher_lib", - "//source/server:filter_chain_manager_lib", "@envoy_api//envoy/config/core/v3:pkg_cc_proto", ], ) diff --git a/source/extensions/matching/actions/format_string/config.cc b/source/extensions/matching/actions/format_string/config.cc index 038e6ed6bd99..311bc2bfc6ce 100644 --- a/source/extensions/matching/actions/format_string/config.cc +++ b/source/extensions/matching/actions/format_string/config.cc @@ -12,8 +12,9 @@ namespace Matching { namespace Actions { namespace FormatString { -const Network::FilterChain* ActionImpl::get(const Server::FilterChainsByName& filter_chains_by_name, - const StreamInfo::StreamInfo& info) const { +const Network::FilterChain* +ActionImpl::get(const Server::Configuration::FilterChainsByName& filter_chains_by_name, + const StreamInfo::StreamInfo& info) const { const std::string name = formatter_->format(*Http::StaticEmptyHeaders::get().request_headers, *Http::StaticEmptyHeaders::get().response_headers, @@ -27,7 +28,7 @@ const Network::FilterChain* ActionImpl::get(const Server::FilterChainsByName& fi Matcher::ActionFactoryCb ActionFactory::createActionFactoryCb(const Protobuf::Message& proto_config, - Server::FilterChainActionFactoryContext& context, + FilterChainActionFactoryContext& context, ProtobufMessage::ValidationVisitor& validator) { const auto& config = MessageUtil::downcastAndValidate( @@ -37,7 +38,7 @@ ActionFactory::createActionFactoryCb(const Protobuf::Message& proto_config, return [formatter]() { return std::make_unique(formatter); }; } -REGISTER_FACTORY(ActionFactory, Matcher::ActionFactory); +REGISTER_FACTORY(ActionFactory, Matcher::ActionFactory); } // namespace FormatString } // namespace Actions diff --git a/source/extensions/matching/actions/format_string/config.h b/source/extensions/matching/actions/format_string/config.h index 7006a3e4aaee..359490c37523 100644 --- a/source/extensions/matching/actions/format_string/config.h +++ b/source/extensions/matching/actions/format_string/config.h @@ -7,7 +7,6 @@ #include "envoy/server/factory_context.h" #include "source/common/matcher/matcher.h" -#include "source/server/filter_chain_manager_impl.h" namespace Envoy { namespace Extensions { @@ -16,22 +15,24 @@ namespace Actions { namespace FormatString { class ActionImpl : public Matcher::ActionBase { + Server::Configuration::FilterChainBaseAction> { public: ActionImpl(const Formatter::FormatterConstSharedPtr& formatter) : formatter_(formatter) {} - const Network::FilterChain* get(const Server::FilterChainsByName& filter_chains_by_name, - const StreamInfo::StreamInfo& info) const override; + const Network::FilterChain* + get(const Server::Configuration::FilterChainsByName& filter_chains_by_name, + const StreamInfo::StreamInfo& info) const override; private: const Formatter::FormatterConstSharedPtr formatter_; }; -class ActionFactory : public Matcher::ActionFactory { +using FilterChainActionFactoryContext = Server::Configuration::ServerFactoryContext; +class ActionFactory : public Matcher::ActionFactory { public: std::string name() const override { return "envoy.matching.actions.format_string"; } Matcher::ActionFactoryCb createActionFactoryCb(const Protobuf::Message& proto_config, - Server::FilterChainActionFactoryContext& context, + FilterChainActionFactoryContext& context, ProtobufMessage::ValidationVisitor& validator) override; ProtobufTypes::MessagePtr createEmptyConfigProto() override { return std::make_unique(); diff --git a/source/server/BUILD b/source/server/BUILD index c40ad62578ac..a28b4a041377 100644 --- a/source/server/BUILD +++ b/source/server/BUILD @@ -99,12 +99,23 @@ envoy_cc_library( ], ) +envoy_cc_library( + name = "listener_stats", + hdrs = [ + "listener_stats.h", + ], + deps = [ + "//source/common/stats:timespan_lib", + ], +) + envoy_cc_library( name = "active_listener_base", hdrs = [ "active_listener_base.h", ], deps = [ + ":listener_stats", "//envoy/network:connection_handler_interface", "//envoy/network:listener_interface", "//envoy/stats:timespan_interface", diff --git a/source/server/active_listener_base.h b/source/server/active_listener_base.h index 7b71f1266280..3e3f0be9c0f4 100644 --- a/source/server/active_listener_base.h +++ b/source/server/active_listener_base.h @@ -4,42 +4,11 @@ #include "envoy/network/listener.h" #include "envoy/stats/scope.h" +#include "source/server/listener_stats.h" + namespace Envoy { namespace Server { -#define ALL_LISTENER_STATS(COUNTER, GAUGE, HISTOGRAM) \ - COUNTER(downstream_cx_destroy) \ - COUNTER(downstream_cx_overflow) \ - COUNTER(downstream_cx_total) \ - COUNTER(downstream_cx_transport_socket_connect_timeout) \ - COUNTER(downstream_cx_overload_reject) \ - COUNTER(downstream_global_cx_overflow) \ - COUNTER(downstream_pre_cx_timeout) \ - COUNTER(downstream_listener_filter_remote_close) \ - COUNTER(downstream_listener_filter_error) \ - COUNTER(no_filter_chain_match) \ - GAUGE(downstream_cx_active, Accumulate) \ - GAUGE(downstream_pre_cx_active, Accumulate) \ - HISTOGRAM(downstream_cx_length_ms, Milliseconds) - -/** - * Wrapper struct for listener stats. @see stats_macros.h - */ -struct ListenerStats { - ALL_LISTENER_STATS(GENERATE_COUNTER_STRUCT, GENERATE_GAUGE_STRUCT, GENERATE_HISTOGRAM_STRUCT) -}; - -#define ALL_PER_HANDLER_LISTENER_STATS(COUNTER, GAUGE) \ - COUNTER(downstream_cx_total) \ - GAUGE(downstream_cx_active, Accumulate) - -/** - * Wrapper struct for per-handler listener stats. @see stats_macros.h - */ -struct PerHandlerListenerStats { - ALL_PER_HANDLER_LISTENER_STATS(GENERATE_COUNTER_STRUCT, GENERATE_GAUGE_STRUCT) -}; - /** * Wrapper for an active listener owned by this handler. */ diff --git a/source/server/filter_chain_manager_impl.cc b/source/server/filter_chain_manager_impl.cc index 8624ae39110c..0bb563958d40 100644 --- a/source/server/filter_chain_manager_impl.cc +++ b/source/server/filter_chain_manager_impl.cc @@ -32,7 +32,7 @@ Network::Address::InstanceConstSharedPtr fakeAddress() { } struct FilterChainNameAction - : public Matcher::ActionBase { + : public Matcher::ActionBase { explicit FilterChainNameAction(const std::string& name) : name_(name) {} const Network::FilterChain* get(const FilterChainsByName& filter_chains_by_name, const StreamInfo::StreamInfo&) const override { @@ -591,7 +591,8 @@ FilterChainManagerImpl::findFilterChainUsingMatcher(const Network::ConnectionSoc "Matching must complete for network streams."); if (match_result.result_) { const auto result = match_result.result_(); - return result->getTyped().get(filter_chains_by_name_, info); + return result->getTyped().get(filter_chains_by_name_, + info); } return default_filter_chain_.get(); } diff --git a/source/server/filter_chain_manager_impl.h b/source/server/filter_chain_manager_impl.h index 4ae38de72ad0..56e6c53bfc2f 100644 --- a/source/server/filter_chain_manager_impl.h +++ b/source/server/filter_chain_manager_impl.h @@ -103,12 +103,6 @@ class PerFilterChainFactoryContextImpl : public Configuration::FilterChainFactor using FilterChainActionFactoryContext = Configuration::ServerFactoryContext; using FilterChainsByName = absl::flat_hash_map; -class FilterChainBaseAction : public Matcher::Action { -public: - virtual const Network::FilterChain* get(const FilterChainsByName& filter_chains_by_name, - const StreamInfo::StreamInfo& info) const PURE; -}; - class FilterChainImpl : public Network::DrainableFilterChain { public: FilterChainImpl(Network::DownstreamTransportSocketFactoryPtr&& transport_socket_factory, diff --git a/source/server/listener_stats.h b/source/server/listener_stats.h new file mode 100644 index 000000000000..c6e2b585fe04 --- /dev/null +++ b/source/server/listener_stats.h @@ -0,0 +1,43 @@ +#pragma once + +#include "envoy/stats/scope.h" +#include "envoy/stats/stats_macros.h" + +namespace Envoy { +namespace Server { + +#define ALL_LISTENER_STATS(COUNTER, GAUGE, HISTOGRAM) \ + COUNTER(downstream_cx_destroy) \ + COUNTER(downstream_cx_overflow) \ + COUNTER(downstream_cx_total) \ + COUNTER(downstream_cx_transport_socket_connect_timeout) \ + COUNTER(downstream_cx_overload_reject) \ + COUNTER(downstream_global_cx_overflow) \ + COUNTER(downstream_pre_cx_timeout) \ + COUNTER(downstream_listener_filter_remote_close) \ + COUNTER(downstream_listener_filter_error) \ + COUNTER(no_filter_chain_match) \ + GAUGE(downstream_cx_active, Accumulate) \ + GAUGE(downstream_pre_cx_active, Accumulate) \ + HISTOGRAM(downstream_cx_length_ms, Milliseconds) + +/** + * Wrapper struct for listener stats. @see stats_macros.h + */ +struct ListenerStats { + ALL_LISTENER_STATS(GENERATE_COUNTER_STRUCT, GENERATE_GAUGE_STRUCT, GENERATE_HISTOGRAM_STRUCT) +}; + +#define ALL_PER_HANDLER_LISTENER_STATS(COUNTER, GAUGE) \ + COUNTER(downstream_cx_total) \ + GAUGE(downstream_cx_active, Accumulate) + +/** + * Wrapper struct for per-handler listener stats. @see stats_macros.h + */ +struct PerHandlerListenerStats { + ALL_PER_HANDLER_LISTENER_STATS(GENERATE_COUNTER_STRUCT, GENERATE_GAUGE_STRUCT) +}; + +} // namespace Server +} // namespace Envoy diff --git a/test/common/quic/BUILD b/test/common/quic/BUILD index ed45f1bd2cca..c89319b040c0 100644 --- a/test/common/quic/BUILD +++ b/test/common/quic/BUILD @@ -242,6 +242,7 @@ envoy_cc_test( "//source/common/quic:envoy_quic_server_session_lib", "//source/extensions/quic/crypto_stream:envoy_quic_crypto_server_stream_lib", "//source/server:configuration_lib", + "//source/server:connection_handler_lib", "//test/mocks/event:event_mocks", "//test/mocks/http:http_mocks", "//test/mocks/network:network_mocks", diff --git a/test/common/quic/envoy_quic_dispatcher_test.cc b/test/common/quic/envoy_quic_dispatcher_test.cc index 5c412ec3fa47..348d488753c0 100644 --- a/test/common/quic/envoy_quic_dispatcher_test.cc +++ b/test/common/quic/envoy_quic_dispatcher_test.cc @@ -12,6 +12,7 @@ #include "source/common/quic/quic_transport_socket_factory.h" #include "source/extensions/quic/crypto_stream/envoy_quic_crypto_server_stream.h" #include "source/server/configuration_impl.h" +#include "source/server/connection_handler_impl.h" #include "test/common/quic/test_proof_source.h" #include "test/common/quic/test_utils.h" diff --git a/test/extensions/matching/actions/format_string/config_test.cc b/test/extensions/matching/actions/format_string/config_test.cc index c52cc8f2308c..8bbb3027572f 100644 --- a/test/extensions/matching/actions/format_string/config_test.cc +++ b/test/extensions/matching/actions/format_string/config_test.cc @@ -28,9 +28,9 @@ TEST(ConfigTest, TestConfig) { ASSERT_NE(nullptr, action_cb); auto action = action_cb(); ASSERT_NE(nullptr, action); - const auto& typed_action = action->getTyped(); + const auto& typed_action = action->getTyped(); - Server::FilterChainsByName chains; + Server::Configuration::FilterChainsByName chains; auto chain = std::make_shared>(); chains.emplace("foo", chain); From 39ca99f21370fd003cb95405089be4c1d393c7d3 Mon Sep 17 00:00:00 2001 From: Paul Sohn Date: Wed, 7 Dec 2022 11:25:01 -0500 Subject: [PATCH 17/23] Add setRequestDecoder to ResponseEncoder interface (#24368) Signed-off-by: Paul Sohn --- envoy/http/codec.h | 9 +++++++++ mobile/library/common/http/client.h | 1 + source/common/http/conn_manager_impl.cc | 9 ++++++--- source/common/http/http1/codec_impl.h | 5 +++++ source/common/http/http2/codec_impl.cc | 2 +- source/common/http/http2/codec_impl.h | 5 ++++- source/common/quic/envoy_quic_server_stream.h | 2 +- test/mocks/http/stream_encoder.h | 1 + 8 files changed, 28 insertions(+), 6 deletions(-) diff --git a/envoy/http/codec.h b/envoy/http/codec.h index 99c696964c58..236a4da6f250 100644 --- a/envoy/http/codec.h +++ b/envoy/http/codec.h @@ -45,6 +45,7 @@ const char MaxResponseHeadersCountOverrideKey[] = "envoy.reloadable_features.max_response_headers_count"; class Stream; +class RequestDecoder; /** * Error codes used to convey the reason for a GOAWAY. @@ -165,6 +166,14 @@ class ResponseEncoder : public virtual StreamEncoder { * error. */ virtual bool streamErrorOnInvalidHttpMessage() const PURE; + + /** + * Set a new request decoder for this ResponseEncoder. This is helpful in the case of an internal + * redirect, in which a new request decoder is created in the context of the same downstream + * request. + * @param decoder new request decoder. + */ + virtual void setRequestDecoder(RequestDecoder& decoder) PURE; }; /** diff --git a/mobile/library/common/http/client.h b/mobile/library/common/http/client.h index 68f7eab06750..4a2d459df034 100644 --- a/mobile/library/common/http/client.h +++ b/mobile/library/common/http/client.h @@ -163,6 +163,7 @@ class Client : public Logger::Loggable { PANIC("not implemented"); } bool streamErrorOnInvalidHttpMessage() const override { return false; } + void setRequestDecoder(RequestDecoder& /*decoder*/) override{}; void encodeMetadata(const MetadataMapVector&) override { PANIC("not implemented"); } diff --git a/source/common/http/conn_manager_impl.cc b/source/common/http/conn_manager_impl.cc index f9b3c21de475..2faa6fa82097 100644 --- a/source/common/http/conn_manager_impl.cc +++ b/source/common/http/conn_manager_impl.cc @@ -1747,9 +1747,6 @@ void ConnectionManagerImpl::ActiveStream::onRequestDataTooLarge() { void ConnectionManagerImpl::ActiveStream::recreateStream( StreamInfo::FilterStateSharedPtr filter_state) { - // n.b. we do not currently change the codecs to point at the new stream - // decoder because the decoder callbacks are complete. It would be good to - // null out that pointer but should not be necessary. ResponseEncoder* response_encoder = response_encoder_; response_encoder_ = nullptr; @@ -1767,6 +1764,12 @@ void ConnectionManagerImpl::ActiveStream::recreateStream( connection_manager_.doEndStream(*this, /*check_for_deferred_close*/ false); RequestDecoder& new_stream = connection_manager_.newStream(*response_encoder, true); + // Set the new RequestDecoder on the ResponseEncoder. Even though all of the decoder callbacks + // have already been called at this point, the encoder still needs the new decoder for deferred + // logging in some cases. + // This doesn't currently work for HTTP/1 as the H/1 ResponseEncoder doesn't hold the active + // stream's pointer to the RequestDecoder. + response_encoder->setRequestDecoder(new_stream); // We don't need to copy over the old parent FilterState from the old StreamInfo if it did not // store any objects with a LifeSpan at or above DownstreamRequest. This is to avoid unnecessary // heap allocation. diff --git a/source/common/http/http1/codec_impl.h b/source/common/http/http1/codec_impl.h index ec2cdc51826a..1e1bd36dc131 100644 --- a/source/common/http/http1/codec_impl.h +++ b/source/common/http/http1/codec_impl.h @@ -156,6 +156,11 @@ class ResponseEncoderImpl : public StreamEncoderImpl, public ResponseEncoder { return stream_error_on_invalid_http_message_; } + // For H/1, ResponseEncoder doesn't hold a pointer to RequestDecoder. + // TODO(paulsohn): Enable H/1 codec to get a pointer to the new + // request decoder on recreateStream, here or elsewhere. + void setRequestDecoder(Http::RequestDecoder& /*decoder*/) override {} + // Http1::StreamEncoderImpl void resetStream(StreamResetReason reason) override; diff --git a/source/common/http/http2/codec_impl.cc b/source/common/http/http2/codec_impl.cc index 812e3d88f211..e03f8e097e71 100644 --- a/source/common/http/http2/codec_impl.cc +++ b/source/common/http/http2/codec_impl.cc @@ -2076,7 +2076,7 @@ Status ServerConnectionImpl::onBeginHeaders(const nghttp2_frame* frame) { if (connection_.aboveHighWatermark()) { stream->runHighWatermarkCallbacks(); } - stream->request_decoder_ = &callbacks_.newStream(*stream); + stream->setRequestDecoder(callbacks_.newStream(*stream)); stream->stream_id_ = frame->hd.stream_id; LinkedList::moveIntoList(std::move(stream), active_streams_); adapter_->SetStreamUserData(frame->hd.stream_id, active_streams_.front().get()); diff --git a/source/common/http/http2/codec_impl.h b/source/common/http/http2/codec_impl.h index ed7808c285bf..1176c6058a78 100644 --- a/source/common/http/http2/codec_impl.h +++ b/source/common/http/http2/codec_impl.h @@ -497,16 +497,19 @@ class ConnectionImpl : public virtual Connection, void encodeTrailers(const ResponseTrailerMap& trailers) override { encodeTrailersBase(trailers); } + void setRequestDecoder(Http::RequestDecoder& decoder) override { request_decoder_ = &decoder; } // ScopeTrackedObject void dumpState(std::ostream& os, int indent_level) const override; - RequestDecoder* request_decoder_{}; absl::variant headers_or_trailers_; bool streamErrorOnInvalidHttpMessage() const override { return parent_.stream_error_on_invalid_http_messaging_; } + + private: + RequestDecoder* request_decoder_{}; }; using ServerStreamImplPtr = std::unique_ptr; diff --git a/source/common/quic/envoy_quic_server_stream.h b/source/common/quic/envoy_quic_server_stream.h index edd3aa60887c..3a2ae67c316d 100644 --- a/source/common/quic/envoy_quic_server_stream.h +++ b/source/common/quic/envoy_quic_server_stream.h @@ -18,7 +18,7 @@ class EnvoyQuicServerStream : public quic::QuicSpdyServerStreamBase, envoy::config::core::v3::HttpProtocolOptions::HeadersWithUnderscoresAction headers_with_underscores_action); - void setRequestDecoder(Http::RequestDecoder& decoder) { request_decoder_ = &decoder; } + void setRequestDecoder(Http::RequestDecoder& decoder) override { request_decoder_ = &decoder; } // Http::StreamEncoder void encode1xxHeaders(const Http::ResponseHeaderMap& headers) override; diff --git a/test/mocks/http/stream_encoder.h b/test/mocks/http/stream_encoder.h index c15da88f170d..efe3626af3ab 100644 --- a/test/mocks/http/stream_encoder.h +++ b/test/mocks/http/stream_encoder.h @@ -48,6 +48,7 @@ class MockResponseEncoder : public ResponseEncoder { MOCK_METHOD(void, encode1xxHeaders, (const ResponseHeaderMap& headers)); MOCK_METHOD(void, encodeHeaders, (const ResponseHeaderMap& headers, bool end_stream)); MOCK_METHOD(void, encodeTrailers, (const ResponseTrailerMap& trailers)); + MOCK_METHOD(void, setRequestDecoder, (RequestDecoder & decoder)); // Http::StreamEncoder MOCK_METHOD(void, encodeData, (Buffer::Instance & data, bool end_stream)); From d99cd7315dd9d8f4d73c16de16fe2071cbbac6f8 Mon Sep 17 00:00:00 2001 From: Chris Schoener Date: Wed, 7 Dec 2022 13:03:49 -0500 Subject: [PATCH 18/23] Remove uneccessary `this->` from mobile engine builder (#24389) Signed-off-by: caschoener --- mobile/library/cc/engine_builder.cc | 122 ++++++++++++++-------------- 1 file changed, 61 insertions(+), 61 deletions(-) diff --git a/mobile/library/cc/engine_builder.cc b/mobile/library/cc/engine_builder.cc index 35fcda4fc669..e713ec85c7e8 100644 --- a/mobile/library/cc/engine_builder.cc +++ b/mobile/library/cc/engine_builder.cc @@ -25,167 +25,167 @@ EngineBuilder::EngineBuilder(std::string config_template) EngineBuilder::EngineBuilder() : EngineBuilder(std::string(config_template)) {} EngineBuilder& EngineBuilder::addLogLevel(LogLevel log_level) { - this->log_level_ = log_level; + log_level_ = log_level; return *this; } EngineBuilder& EngineBuilder::setOnEngineRunning(std::function closure) { - this->callbacks_->on_engine_running = closure; + callbacks_->on_engine_running = closure; return *this; } EngineBuilder& EngineBuilder::addStatsSinks(const std::vector& stat_sinks) { - this->stat_sinks_ = stat_sinks; + stat_sinks_ = stat_sinks; return *this; } EngineBuilder& EngineBuilder::addGrpcStatsDomain(std::string stats_domain) { - this->stats_domain_ = std::move(stats_domain); + stats_domain_ = std::move(stats_domain); return *this; } EngineBuilder& EngineBuilder::addConnectTimeoutSeconds(int connect_timeout_seconds) { - this->connect_timeout_seconds_ = connect_timeout_seconds; + connect_timeout_seconds_ = connect_timeout_seconds; return *this; } EngineBuilder& EngineBuilder::addDnsRefreshSeconds(int dns_refresh_seconds) { - this->dns_refresh_seconds_ = dns_refresh_seconds; + dns_refresh_seconds_ = dns_refresh_seconds; return *this; } EngineBuilder& EngineBuilder::addDnsMinRefreshSeconds(int dns_min_refresh_seconds) { - this->dns_min_refresh_seconds_ = dns_min_refresh_seconds; + dns_min_refresh_seconds_ = dns_min_refresh_seconds; return *this; } EngineBuilder& EngineBuilder::addDnsFailureRefreshSeconds(int base, int max) { - this->dns_failure_refresh_seconds_base_ = base; - this->dns_failure_refresh_seconds_max_ = max; + dns_failure_refresh_seconds_base_ = base; + dns_failure_refresh_seconds_max_ = max; return *this; } EngineBuilder& EngineBuilder::addDnsQueryTimeoutSeconds(int dns_query_timeout_seconds) { - this->dns_query_timeout_seconds_ = dns_query_timeout_seconds; + dns_query_timeout_seconds_ = dns_query_timeout_seconds; return *this; } EngineBuilder& EngineBuilder::addDnsPreresolveHostnames(std::string dns_preresolve_hostnames) { - this->dns_preresolve_hostnames_ = std::move(dns_preresolve_hostnames); + dns_preresolve_hostnames_ = std::move(dns_preresolve_hostnames); return *this; } EngineBuilder& EngineBuilder::addMaxConnectionsPerHost(int max_connections_per_host) { - this->max_connections_per_host_ = max_connections_per_host; + max_connections_per_host_ = max_connections_per_host; return *this; } EngineBuilder& EngineBuilder::useDnsSystemResolver(bool use_system_resolver) { - this->use_system_resolver_ = use_system_resolver; + use_system_resolver_ = use_system_resolver; return *this; } EngineBuilder& EngineBuilder::addH2ConnectionKeepaliveIdleIntervalMilliseconds( int h2_connection_keepalive_idle_interval_milliseconds) { - this->h2_connection_keepalive_idle_interval_milliseconds_ = + h2_connection_keepalive_idle_interval_milliseconds_ = h2_connection_keepalive_idle_interval_milliseconds; return *this; } EngineBuilder& EngineBuilder::addH2ConnectionKeepaliveTimeoutSeconds(int h2_connection_keepalive_timeout_seconds) { - this->h2_connection_keepalive_timeout_seconds_ = h2_connection_keepalive_timeout_seconds; + h2_connection_keepalive_timeout_seconds_ = h2_connection_keepalive_timeout_seconds; return *this; } EngineBuilder& EngineBuilder::addStatsFlushSeconds(int stats_flush_seconds) { - this->stats_flush_seconds_ = stats_flush_seconds; + stats_flush_seconds_ = stats_flush_seconds; return *this; } EngineBuilder& EngineBuilder::addVirtualClusters(std::string virtual_clusters) { - this->virtual_clusters_ = std::move(virtual_clusters); + virtual_clusters_ = std::move(virtual_clusters); return *this; } EngineBuilder& EngineBuilder::addKeyValueStore(std::string name, KeyValueStoreSharedPtr key_value_store) { - this->key_value_stores_[std::move(name)] = std::move(key_value_store); + key_value_stores_[std::move(name)] = std::move(key_value_store); return *this; } EngineBuilder& EngineBuilder::setAppVersion(std::string app_version) { - this->app_version_ = std::move(app_version); + app_version_ = std::move(app_version); return *this; } EngineBuilder& EngineBuilder::setAppId(std::string app_id) { - this->app_id_ = std::move(app_id); + app_id_ = std::move(app_id); return *this; } EngineBuilder& EngineBuilder::setDeviceOs(std::string device_os) { - this->device_os_ = std::move(device_os); + device_os_ = std::move(device_os); return *this; } EngineBuilder& EngineBuilder::setStreamIdleTimeoutSeconds(int stream_idle_timeout_seconds) { - this->stream_idle_timeout_seconds_ = stream_idle_timeout_seconds; + stream_idle_timeout_seconds_ = stream_idle_timeout_seconds; return *this; } EngineBuilder& EngineBuilder::setPerTryIdleTimeoutSeconds(int per_try_idle_timeout_seconds) { - this->per_try_idle_timeout_seconds_ = per_try_idle_timeout_seconds; + per_try_idle_timeout_seconds_ = per_try_idle_timeout_seconds; return *this; } EngineBuilder& EngineBuilder::enableGzip(bool gzip_on) { - this->gzip_filter_ = gzip_on; + gzip_filter_ = gzip_on; return *this; } EngineBuilder& EngineBuilder::enableBrotli(bool brotli_on) { - this->brotli_filter_ = brotli_on; + brotli_filter_ = brotli_on; return *this; } EngineBuilder& EngineBuilder::enableSocketTagging(bool socket_tagging_on) { - this->socket_tagging_filter_ = socket_tagging_on; + socket_tagging_filter_ = socket_tagging_on; return *this; } EngineBuilder& EngineBuilder::enableAdminInterface(bool admin_interface_on) { - this->admin_interface_enabled_ = admin_interface_on; + admin_interface_enabled_ = admin_interface_on; return *this; } EngineBuilder& EngineBuilder::enableHappyEyeballs(bool happy_eyeballs_on) { - this->enable_happy_eyeballs_ = happy_eyeballs_on; + enable_happy_eyeballs_ = happy_eyeballs_on; return *this; } EngineBuilder& EngineBuilder::enableHttp3(bool http3_on) { - this->enable_http3_ = http3_on; + enable_http3_ = http3_on; return *this; } EngineBuilder& EngineBuilder::enableInterfaceBinding(bool interface_binding_on) { - this->enable_interface_binding_ = interface_binding_on; + enable_interface_binding_ = interface_binding_on; return *this; } EngineBuilder& EngineBuilder::enableDrainPostDnsRefresh(bool drain_post_dns_refresh_on) { - this->enable_drain_post_dns_refresh_ = drain_post_dns_refresh_on; + enable_drain_post_dns_refresh_ = drain_post_dns_refresh_on; return *this; } EngineBuilder& EngineBuilder::enforceTrustChainVerification(bool trust_chain_verification_on) { - this->enforce_trust_chain_verification_ = trust_chain_verification_on; + enforce_trust_chain_verification_ = trust_chain_verification_on; return *this; } EngineBuilder& EngineBuilder::enableH2ExtendKeepaliveTimeout(bool h2_extend_keepalive_timeout_on) { - this->h2_extend_keepalive_timeout_ = h2_extend_keepalive_timeout_on; + h2_extend_keepalive_timeout_ = h2_extend_keepalive_timeout_on; return *this; } @@ -196,7 +196,7 @@ EngineBuilder::enablePlatformCertificatesValidation(bool platform_certificates_v PANIC("Certificates validation using platform provided APIs is not supported in IOS."); } #endif - this->platform_certificates_validation_on_ = platform_certificates_validation_on; + platform_certificates_validation_on_ = platform_certificates_validation_on; return *this; } @@ -218,35 +218,35 @@ EngineBuilder& EngineBuilder::addPlatformFilter(std::string name) { std::string EngineBuilder::generateConfigStr() const { std::vector> replacements { - {"connect_timeout", fmt::format("{}s", this->connect_timeout_seconds_)}, - {"dns_fail_base_interval", fmt::format("{}s", this->dns_failure_refresh_seconds_base_)}, - {"dns_fail_max_interval", fmt::format("{}s", this->dns_failure_refresh_seconds_max_)}, + {"connect_timeout", fmt::format("{}s", connect_timeout_seconds_)}, + {"dns_fail_base_interval", fmt::format("{}s", dns_failure_refresh_seconds_base_)}, + {"dns_fail_max_interval", fmt::format("{}s", dns_failure_refresh_seconds_max_)}, {"dns_lookup_family", enable_happy_eyeballs_ ? "ALL" : "V4_PREFERRED"}, - {"dns_min_refresh_rate", fmt::format("{}s", this->dns_min_refresh_seconds_)}, + {"dns_min_refresh_rate", fmt::format("{}s", dns_min_refresh_seconds_)}, {"dns_multiple_addresses", enable_happy_eyeballs_ ? "true" : "false"}, - {"dns_preresolve_hostnames", this->dns_preresolve_hostnames_}, - {"dns_refresh_rate", fmt::format("{}s", this->dns_refresh_seconds_)}, - {"dns_query_timeout", fmt::format("{}s", this->dns_query_timeout_seconds_)}, + {"dns_preresolve_hostnames", dns_preresolve_hostnames_}, + {"dns_refresh_rate", fmt::format("{}s", dns_refresh_seconds_)}, + {"dns_query_timeout", fmt::format("{}s", dns_query_timeout_seconds_)}, {"enable_drain_post_dns_refresh", enable_drain_post_dns_refresh_ ? "true" : "false"}, {"enable_interface_binding", enable_interface_binding_ ? "true" : "false"}, {"h2_connection_keepalive_idle_interval", - fmt::format("{}s", this->h2_connection_keepalive_idle_interval_milliseconds_ / 1000.0)}, + fmt::format("{}s", h2_connection_keepalive_idle_interval_milliseconds_ / 1000.0)}, {"h2_connection_keepalive_timeout", - fmt::format("{}s", this->h2_connection_keepalive_timeout_seconds_)}, + fmt::format("{}s", h2_connection_keepalive_timeout_seconds_)}, {"h2_delay_keepalive_timeout", h2_extend_keepalive_timeout_ ? "true" : "false"}, { "metadata", - fmt::format("{{ device_os: {}, app_version: {}, app_id: {} }}", this->device_os_, - this->app_version_, this->app_id_), + fmt::format("{{ device_os: {}, app_version: {}, app_id: {} }}", device_os_, + app_version_, app_id_), }, - {"max_connections_per_host", fmt::format("{}", this->max_connections_per_host_)}, - {"stats_domain", this->stats_domain_}, - {"stats_flush_interval", fmt::format("{}s", this->stats_flush_seconds_)}, - {"stream_idle_timeout", fmt::format("{}s", this->stream_idle_timeout_seconds_)}, + {"max_connections_per_host", fmt::format("{}", max_connections_per_host_)}, + {"stats_domain", stats_domain_}, + {"stats_flush_interval", fmt::format("{}s", stats_flush_seconds_)}, + {"stream_idle_timeout", fmt::format("{}s", stream_idle_timeout_seconds_)}, {"trust_chain_verification", enforce_trust_chain_verification_ ? "VERIFY_TRUST_CHAIN" : "ACCEPT_UNTRUSTED"}, - {"per_try_idle_timeout", fmt::format("{}s", this->per_try_idle_timeout_seconds_)}, - {"virtual_clusters", this->virtual_clusters_}, + {"per_try_idle_timeout", fmt::format("{}s", per_try_idle_timeout_seconds_)}, + {"virtual_clusters", virtual_clusters_}, #if defined(__ANDROID_API__) {"force_ipv6", "true"}, #endif @@ -270,21 +270,21 @@ std::string EngineBuilder::generateConfigStr() const { } const std::string& cert_validation_template = - (this->platform_certificates_validation_on_ ? platform_cert_validation_context_template - : default_cert_validation_context_template); + (platform_certificates_validation_on_ ? platform_cert_validation_context_template + : default_cert_validation_context_template); config_builder << cert_validation_template << std::endl; std::string config_template = config_template_; - if (this->gzip_filter_) { + if (gzip_filter_) { insertCustomFilter(gzip_config_insert, config_template); } - if (this->brotli_filter_) { + if (brotli_filter_) { insertCustomFilter(brotli_config_insert, config_template); } - if (this->socket_tagging_filter_) { + if (socket_tagging_filter_) { insertCustomFilter(socket_tag_config_insert, config_template); } - if (this->enable_http3_) { + if (enable_http3_) { insertCustomFilter(alternate_protocols_cache_filter_insert, config_template); } @@ -324,12 +324,12 @@ EngineSharedPtr EngineBuilder::build() { std::string config_str; if (config_override_for_tests_.empty()) { - config_str = this->generateConfigStr(); + config_str = generateConfigStr(); } else { config_str = config_override_for_tests_; } envoy_engine_t envoy_engine = - init_engine(this->callbacks_->asEnvoyEngineCallbacks(), null_logger, null_tracker); + init_engine(callbacks_->asEnvoyEngineCallbacks(), null_logger, null_tracker); for (const auto& [name, store] : key_value_stores_) { // TODO(goaway): This leaks, but it's tied to the life of the engine. @@ -345,8 +345,8 @@ EngineSharedPtr EngineBuilder::build() { register_platform_api(name.c_str(), api); } - run_engine(envoy_engine, config_str.c_str(), logLevelToString(this->log_level_).c_str(), - this->admin_address_path_for_tests_.c_str()); + run_engine(envoy_engine, config_str.c_str(), logLevelToString(log_level_).c_str(), + admin_address_path_for_tests_.c_str()); // we can't construct via std::make_shared // because Engine is only constructible as a friend From d3d4a0108cf6927ddc3227334379f59c94277b24 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rafa=C5=82=20Augustyniak?= Date: Wed, 7 Dec 2022 15:33:51 -0500 Subject: [PATCH 19/23] mobile: check for pending exceptions after JNI call (#24361) Commit Message: Check for pending exceptions after some of the Java calls from within the JNI layer. At Lyft, we noticed that we have a bunch of crashes related to us not checking for exceptions after on_request_headers or/and on_response_headers methods. Additional description: I tried to use `ENVOY_LOG_EVENT_TO_LOGGER` to log when we detect a pending exception but ran into issues with deadlocks. Got rid of the extra logging to unblock my change. Risk Level: Low, mostly additive change. New code executes for when the library crashes previously. Testing: Integration test Docs Changes: Release Notes: Platform Specific Features: Signed-off-by: Rafal Augustyniak --- mobile/library/common/jni/BUILD | 4 + mobile/library/common/jni/jni_interface.cc | 69 ++++++--- mobile/library/common/jni/jni_utility.cc | 25 ++++ mobile/library/common/jni/jni_utility.h | 12 ++ mobile/library/common/types/BUILD | 12 ++ .../common/types/managed_envoy_headers.h | 32 +++++ mobile/test/kotlin/integration/BUILD | 23 ++- .../FilterThrowingExceptionTest.kt | 134 ++++++++++++++++++ 8 files changed, 294 insertions(+), 17 deletions(-) create mode 100644 mobile/library/common/types/managed_envoy_headers.h create mode 100644 mobile/test/kotlin/integration/FilterThrowingExceptionTest.kt diff --git a/mobile/library/common/jni/BUILD b/mobile/library/common/jni/BUILD index 13536a8f68d7..caaa9cc616dd 100644 --- a/mobile/library/common/jni/BUILD +++ b/mobile/library/common/jni/BUILD @@ -21,6 +21,7 @@ cc_library( deps = [ "//library/common/jni/import:jni_import_lib", "//library/common/types:c_types_lib", + "//library/common/types:managed_types_lib", "@envoy//source/common/common:assert_lib", ], ) @@ -48,6 +49,7 @@ cc_library( ":ndk_jni_support", "//library/common:envoy_main_interface_lib", "//library/common/api:c_types", + "//library/common/types:managed_types_lib", ], # We need this to ensure that we link this into the .so even though there are no code references. alwayslink = True, @@ -115,6 +117,7 @@ cc_binary( "//library/common:envoy_main_interface_lib", "//library/common/api:c_types", "//library/common/jni/import:jni_import_lib", + "//library/common/types:managed_types_lib", ], ) @@ -169,6 +172,7 @@ cc_library( "//library/common/jni/import:jni_import_lib", "//library/common:envoy_main_interface_lib", "//library/common/types:c_types_lib", + "//library/common/types:managed_types_lib", "@envoy//source/common/common:assert_lib", ] + TEST_EXTENSIONS, ) diff --git a/mobile/library/common/jni/jni_interface.cc b/mobile/library/common/jni/jni_interface.cc index cc9942f22459..8dded5a7a4f1 100644 --- a/mobile/library/common/jni/jni_interface.cc +++ b/mobile/library/common/jni/jni_interface.cc @@ -8,6 +8,7 @@ #include "library/common/jni/jni_utility.h" #include "library/common/jni/jni_version.h" #include "library/common/main_interface.h" +#include "library/common/types/managed_envoy_headers.h" // NOLINT(namespace-envoy) @@ -278,22 +279,23 @@ Java_io_envoyproxy_envoymobile_engine_JniLibrary_recordHistogramValue(JNIEnv* en // JvmCallbackContext -static void pass_headers(const char* method, envoy_headers headers, jobject j_context) { +static void passHeaders(const char* method, const Envoy::Types::ManagedEnvoyHeaders& headers, + jobject j_context) { JNIEnv* env = get_env(); jclass jcls_JvmCallbackContext = env->GetObjectClass(j_context); jmethodID jmid_passHeader = env->GetMethodID(jcls_JvmCallbackContext, method, "([B[BZ)V"); jboolean start_headers = JNI_TRUE; - for (envoy_map_size_t i = 0; i < headers.length; i++) { + for (envoy_map_size_t i = 0; i < headers.get().length; i++) { // Note: this is just an initial implementation, and we will pass a more optimized structure in // the future. // Note: the JNI function NewStringUTF would appear to be an appealing option here, except it // requires a null-terminated *modified* UTF-8 string. // Create platform byte array for header key - jbyteArray j_key = native_data_to_array(env, headers.entries[i].key); + jbyteArray j_key = native_data_to_array(env, headers.get().entries[i].key); // Create platform byte array for header value - jbyteArray j_value = native_data_to_array(env, headers.entries[i].value); + jbyteArray j_value = native_data_to_array(env, headers.get().entries[i].value); // Pass this header pair to the platform env->CallVoidMethod(j_context, jmid_passHeader, j_key, j_value, start_headers); @@ -306,19 +308,18 @@ static void pass_headers(const char* method, envoy_headers headers, jobject j_co } env->DeleteLocalRef(jcls_JvmCallbackContext); - release_envoy_headers(headers); } // Platform callback implementation // These methods call jvm methods which means the local references created will not be // released automatically. Manual bookkeeping is required for these methods. -static void* jvm_on_headers(const char* method, envoy_headers headers, bool end_stream, - envoy_stream_intel stream_intel, void* context) { +static void* jvm_on_headers(const char* method, const Envoy::Types::ManagedEnvoyHeaders& headers, + bool end_stream, envoy_stream_intel stream_intel, void* context) { jni_log("[Envoy]", "jvm_on_headers"); JNIEnv* env = get_env(); jobject j_context = static_cast(context); - pass_headers("passHeader", headers, j_context); + passHeaders("passHeader", headers, j_context); jclass jcls_JvmCallbackContext = env->GetObjectClass(j_context); jmethodID jmid_onHeaders = @@ -327,24 +328,52 @@ static void* jvm_on_headers(const char* method, envoy_headers headers, bool end_ jlongArray j_stream_intel = native_stream_intel_to_array(env, stream_intel); // Note: be careful of JVM types. Before we casted to jlong we were getting integer problems. // TODO: make this cast safer. - jobject result = env->CallObjectMethod(j_context, jmid_onHeaders, (jlong)headers.length, + jobject result = env->CallObjectMethod(j_context, jmid_onHeaders, (jlong)headers.get().length, end_stream ? JNI_TRUE : JNI_FALSE, j_stream_intel); + // TODO(Augustyniak): Pass the name of the filter in here so that we can instrument the origin of + // the JNI exception better. + bool exception_cleared = clear_pending_exceptions(env); env->DeleteLocalRef(j_stream_intel); env->DeleteLocalRef(jcls_JvmCallbackContext); - return result; + if (!exception_cleared) { + return result; + } + + // Create a "no operation" result: + // 1. Tell the filter chain to continue the iteration. + // 2. Return headers received on as method's input as part of the method's output. + jclass jcls_object_array = env->FindClass("java/lang/Object"); + jobjectArray noopResult = env->NewObjectArray(2, jcls_object_array, NULL); + + jclass jcls_int = env->FindClass("java/lang/Integer"); + jmethodID jmid_intInit = env->GetMethodID(jcls_int, "", "(I)V"); + jobject j_status = env->NewObject(jcls_int, jmid_intInit, 0); + // Set status to "0" (FilterHeadersStatus::Continue). Signal that the intent + // is to continue the iteration of the filter chain. + env->SetObjectArrayElement(noopResult, 0, j_status); + + // Since the "on headers" call threw an exception set input headers as output headers. + env->SetObjectArrayElement(noopResult, 1, ToJavaArrayOfObjectArray(env, headers)); + + env->DeleteLocalRef(jcls_object_array); + env->DeleteLocalRef(jcls_int); + + return noopResult; } static void* jvm_on_response_headers(envoy_headers headers, bool end_stream, envoy_stream_intel stream_intel, void* context) { - return jvm_on_headers("onResponseHeaders", headers, end_stream, stream_intel, context); + const auto managed_headers = Envoy::Types::ManagedEnvoyHeaders(headers); + return jvm_on_headers("onResponseHeaders", managed_headers, end_stream, stream_intel, context); } static envoy_filter_headers_status -jvm_http_filter_on_request_headers(envoy_headers headers, bool end_stream, +jvm_http_filter_on_request_headers(envoy_headers input_headers, bool end_stream, envoy_stream_intel stream_intel, const void* context) { JNIEnv* env = get_env(); + const auto headers = Envoy::Types::ManagedEnvoyHeaders(input_headers); jobjectArray result = static_cast(jvm_on_headers( "onRequestHeaders", headers, end_stream, stream_intel, const_cast(context))); @@ -363,9 +392,10 @@ jvm_http_filter_on_request_headers(envoy_headers headers, bool end_stream, } static envoy_filter_headers_status -jvm_http_filter_on_response_headers(envoy_headers headers, bool end_stream, +jvm_http_filter_on_response_headers(envoy_headers input_headers, bool end_stream, envoy_stream_intel stream_intel, const void* context) { JNIEnv* env = get_env(); + const auto headers = Envoy::Types::ManagedEnvoyHeaders(input_headers); jobjectArray result = static_cast(jvm_on_headers( "onResponseHeaders", headers, end_stream, stream_intel, const_cast(context))); @@ -484,7 +514,7 @@ static void* jvm_on_trailers(const char* method, envoy_headers trailers, JNIEnv* env = get_env(); jobject j_context = static_cast(context); - pass_headers("passHeader", trailers, j_context); + passHeaders("passHeader", trailers, j_context); jclass jcls_JvmCallbackContext = env->GetObjectClass(j_context); jmethodID jmid_onTrailers = @@ -493,6 +523,7 @@ static void* jvm_on_trailers(const char* method, envoy_headers trailers, jlongArray j_stream_intel = native_stream_intel_to_array(env, stream_intel); // Note: be careful of JVM types. Before we casted to jlong we were getting integer problems. // TODO: make this cast safer. + // TODO(Augustyniak): check for pending exceptions after returning from JNI call. jobject result = env->CallObjectMethod(j_context, jmid_onTrailers, (jlong)trailers.length, j_stream_intel); @@ -632,7 +663,7 @@ jvm_http_filter_on_resume(const char* method, envoy_headers* headers, envoy_data jlong headers_length = -1; if (headers) { headers_length = (jlong)headers->length; - pass_headers("passHeader", *headers, j_context); + passHeaders("passHeader", *headers, j_context); } jbyteArray j_in_data = nullptr; if (data) { @@ -641,7 +672,7 @@ jvm_http_filter_on_resume(const char* method, envoy_headers* headers, envoy_data jlong trailers_length = -1; if (trailers) { trailers_length = (jlong)trailers->length; - pass_headers("passTrailer", *trailers, j_context); + passHeaders("passTrailer", *trailers, j_context); } jlongArray j_stream_intel = native_stream_intel_to_array(env, stream_intel); @@ -714,6 +745,8 @@ static void* call_jvm_on_complete(envoy_stream_intel stream_intel, jobject result = env->CallObjectMethod(j_context, jmid_onComplete, j_stream_intel, j_final_stream_intel); + clear_pending_exceptions(env); + env->DeleteLocalRef(j_stream_intel); env->DeleteLocalRef(j_final_stream_intel); env->DeleteLocalRef(jcls_JvmObserverContext); @@ -737,6 +770,8 @@ static void* call_jvm_on_error(envoy_error error, envoy_stream_intel stream_inte jobject result = env->CallObjectMethod(j_context, jmid_onError, error.error_code, j_error_message, error.attempt_count, j_stream_intel, j_final_stream_intel); + clear_pending_exceptions(env); + env->DeleteLocalRef(j_stream_intel); env->DeleteLocalRef(j_final_stream_intel); env->DeleteLocalRef(j_error_message); @@ -769,6 +804,8 @@ static void* call_jvm_on_cancel(envoy_stream_intel stream_intel, jobject result = env->CallObjectMethod(j_context, jmid_onCancel, j_stream_intel, j_final_stream_intel); + clear_pending_exceptions(env); + env->DeleteLocalRef(j_stream_intel); env->DeleteLocalRef(j_final_stream_intel); env->DeleteLocalRef(jcls_JvmObserverContext); diff --git a/mobile/library/common/jni/jni_utility.cc b/mobile/library/common/jni/jni_utility.cc index 8fbc0da0b8d9..3a4debc698e6 100644 --- a/mobile/library/common/jni/jni_utility.cc +++ b/mobile/library/common/jni/jni_utility.cc @@ -66,6 +66,16 @@ void jni_delete_const_global_ref(const void* context) { jni_delete_global_ref(const_cast(context)); } +bool clear_pending_exceptions(JNIEnv* env) { + if (env->ExceptionCheck() == JNI_TRUE) { + env->ExceptionClear(); + // TODO(Augustyniak): Log exception details. + return true; + } else { + return false; + } +} + int unbox_integer(JNIEnv* env, jobject boxedInteger) { jclass jcls_Integer = env->FindClass("java/lang/Integer"); jmethodID jmid_intValue = env->GetMethodID(jcls_Integer, "intValue", "()I"); @@ -278,6 +288,21 @@ envoy_map to_native_map(JNIEnv* env, jobjectArray entries) { return native_map; } +jobjectArray ToJavaArrayOfObjectArray(JNIEnv* env, const Envoy::Types::ManagedEnvoyHeaders& map) { + jclass jcls_byte_array = env->FindClass("java/lang/Object"); + jobjectArray javaArray = env->NewObjectArray(2 * map.get().length, jcls_byte_array, nullptr); + + for (envoy_map_size_t i = 0; i < map.get().length; i++) { + jbyteArray key = native_data_to_array(env, map.get().entries[i].key); + jbyteArray value = native_data_to_array(env, map.get().entries[i].value); + + env->SetObjectArrayElement(javaArray, 2 * i, key); + env->SetObjectArrayElement(javaArray, 2 * i + 1, value); + } + + return javaArray; +} + jobjectArray ToJavaArrayOfByteArray(JNIEnv* env, const std::vector& v) { jclass jcls_byte_array = env->FindClass("[B"); jobjectArray joa = env->NewObjectArray(v.size(), jcls_byte_array, nullptr); diff --git a/mobile/library/common/jni/jni_utility.h b/mobile/library/common/jni/jni_utility.h index 2c7f2875f5ef..572fee8a0dc3 100644 --- a/mobile/library/common/jni/jni_utility.h +++ b/mobile/library/common/jni/jni_utility.h @@ -5,6 +5,7 @@ #include "library/common/jni/import/jni_import.h" #include "library/common/types/c_types.h" +#include "library/common/types/managed_envoy_headers.h" // NOLINT(namespace-envoy) @@ -43,6 +44,15 @@ void jni_delete_global_ref(void* context); void jni_delete_const_global_ref(const void* context); +/** + * Clears any pending exceptions that may have been rides in result to a call into Java code. + * + * @param env, the JNI env pointer. + * + * @return Whether any pending JNI exception was cleared. + */ +bool clear_pending_exceptions(JNIEnv* env); + int unbox_integer(JNIEnv* env, jobject boxedInteger); envoy_data array_to_native_data(JNIEnv* env, jbyteArray j_data); @@ -100,6 +110,8 @@ jbyteArray ToJavaByteArray(JNIEnv* env, const uint8_t* bytes, size_t len); jbyteArray ToJavaByteArray(JNIEnv* env, const std::string& str); +jobjectArray ToJavaArrayOfObjectArray(JNIEnv* env, const Envoy::Types::ManagedEnvoyHeaders& map); + void JavaArrayOfByteArrayToStringVector(JNIEnv* env, jobjectArray array, std::vector* out); diff --git a/mobile/library/common/types/BUILD b/mobile/library/common/types/BUILD index e4a61c2cdbb7..3d1c6b8fbf5f 100644 --- a/mobile/library/common/types/BUILD +++ b/mobile/library/common/types/BUILD @@ -18,3 +18,15 @@ envoy_cc_library( "@envoy//source/common/common:assert_lib", ], ) + +envoy_cc_library( + name = "managed_types_lib", + srcs = [ + "managed_envoy_headers.h", + ], + repository = "@envoy", + visibility = ["//visibility:public"], + deps = [ + "//library/common/types:c_types_lib", + ], +) diff --git a/mobile/library/common/types/managed_envoy_headers.h b/mobile/library/common/types/managed_envoy_headers.h new file mode 100644 index 000000000000..2e2072eb7245 --- /dev/null +++ b/mobile/library/common/types/managed_envoy_headers.h @@ -0,0 +1,32 @@ +#pragma once + +#include "library/common/types/c_types.h" + +namespace Envoy { +namespace Types { + +/** + * A wrapper around envoy_headers that's responsible for freeing + * the underlying headers when they are not needed anymore. + */ +class ManagedEnvoyHeaders { +public: + /** + * Initialize a new instance of the receiver using a given instance of envoy headers. + * + * @param headers, that should be wrapped by the receiver. The wrapper will hold onto + * the passed headers and free them once the receiver is not used anymore. + */ + ManagedEnvoyHeaders(envoy_headers headers) : headers_(headers){}; + ~ManagedEnvoyHeaders() { release_envoy_headers(headers_); } + const envoy_headers& get() const { return headers_; } + +private: + envoy_headers headers_; + // Make copy and assignment operators private to prevent copying of the receiver. + ManagedEnvoyHeaders(const ManagedEnvoyHeaders&); + ManagedEnvoyHeaders& operator=(const ManagedEnvoyHeaders&); +}; + +} // namespace Types +} // namespace Envoy diff --git a/mobile/test/kotlin/integration/BUILD b/mobile/test/kotlin/integration/BUILD index a6ce4f629f00..3325a82f6c97 100644 --- a/mobile/test/kotlin/integration/BUILD +++ b/mobile/test/kotlin/integration/BUILD @@ -1,4 +1,5 @@ -load("@envoy_mobile//bazel:kotlin_test.bzl", "envoy_mobile_jni_kt_test") +load("@envoy_mobile//bazel:kotlin_test.bzl", "envoy_mobile_android_test", "envoy_mobile_jni_kt_test") +load("@envoy_mobile//bazel:kotlin_lib.bzl", "envoy_mobile_kt_library") envoy_mobile_jni_kt_test( name = "engine_start_test", @@ -200,3 +201,23 @@ envoy_mobile_jni_kt_test( "//library/kotlin/io/envoyproxy/envoymobile:envoy_interfaces_lib", ], ) + +envoy_mobile_android_test( + name = "filter_throwing_exception_test", + srcs = [ + "FilterThrowingExceptionTest.kt", + ], + exec_properties = { + # TODO(lfpino): Remove this once the sandboxNetwork=off works for ipv4 localhost addresses. + "sandboxNetwork": "standard", + "dockerNetwork": "standard", + }, + native_deps = [ + "//library/common/jni:libndk_envoy_jni.so", + "//library/common/jni:libndk_envoy_jni.jnilib", + ], + deps = [ + "//library/kotlin/io/envoyproxy/envoymobile:envoy_interfaces_lib", + "//library/kotlin/io/envoyproxy/envoymobile:envoy_lib", + ], +) diff --git a/mobile/test/kotlin/integration/FilterThrowingExceptionTest.kt b/mobile/test/kotlin/integration/FilterThrowingExceptionTest.kt new file mode 100644 index 000000000000..aa6f993dadab --- /dev/null +++ b/mobile/test/kotlin/integration/FilterThrowingExceptionTest.kt @@ -0,0 +1,134 @@ +package test.kotlin.integration + +import android.content.Context +import androidx.test.core.app.ApplicationProvider + +import io.envoyproxy.envoymobile.AndroidEngineBuilder +import io.envoyproxy.envoymobile.EnvoyError +import io.envoyproxy.envoymobile.Engine +import io.envoyproxy.envoymobile.engine.JniLibrary +import io.envoyproxy.envoymobile.FilterDataStatus +import io.envoyproxy.envoymobile.FilterHeadersStatus +import io.envoyproxy.envoymobile.FilterTrailersStatus +import io.envoyproxy.envoymobile.FinalStreamIntel +import io.envoyproxy.envoymobile.LogLevel +import io.envoyproxy.envoymobile.RequestFilter +import io.envoyproxy.envoymobile.RequestHeaders +import io.envoyproxy.envoymobile.RequestHeadersBuilder +import io.envoyproxy.envoymobile.RequestMethod +import io.envoyproxy.envoymobile.RequestTrailers +import io.envoyproxy.envoymobile.ResponseFilter +import io.envoyproxy.envoymobile.ResponseHeaders +import io.envoyproxy.envoymobile.ResponseTrailers +import io.envoyproxy.envoymobile.StreamIntel + +import java.nio.ByteBuffer +import java.util.concurrent.CountDownLatch +import java.util.concurrent.Executors +import java.util.concurrent.TimeUnit + +import org.assertj.core.api.Assertions.assertThat +import org.junit.Assert.assertNotNull +import org.junit.Test +import org.junit.runner.RunWith +import org.robolectric.RobolectricTestRunner + +class ThrowingFilter: RequestFilter, ResponseFilter { + override fun onRequestHeaders( + headers: RequestHeaders, + endStream: Boolean, + streamIntel: StreamIntel + ): FilterHeadersStatus { + throw Exception("Simulated onRequestHeaders exception") + } + + override fun onRequestData(body: ByteBuffer, endStream: Boolean, streamIntel: StreamIntel): + FilterDataStatus { + return FilterDataStatus.Continue(body) + } + + override fun onRequestTrailers(trailers: RequestTrailers, streamIntel: StreamIntel): + FilterTrailersStatus { + return FilterTrailersStatus.Continue(trailers) + } + + override fun onResponseHeaders( + headers: ResponseHeaders, + endStream: Boolean, + streamIntel: StreamIntel + ): FilterHeadersStatus { + throw Exception("Simulated onResponseHeaders exception") + } + + override fun onResponseData( + body: ByteBuffer, + endStream: Boolean, + streamIntel: StreamIntel + ): FilterDataStatus { + return FilterDataStatus.Continue(body) + } + + override fun onResponseTrailers( + trailers: ResponseTrailers, + streamIntel: StreamIntel + ): FilterTrailersStatus { + return FilterTrailersStatus.Continue(trailers) + } + + override fun onError( + error: EnvoyError, + finalStreamIntel: FinalStreamIntel + ) {} + + override fun onCancel(finalStreamIntel: FinalStreamIntel) {} + + override fun onComplete(finalStreamIntel: FinalStreamIntel) {} +} + +@RunWith(RobolectricTestRunner::class) +class FilterThrowingExceptionTest { + init { + JniLibrary.loadTestLibrary() + } + + @Test + fun `registers a filter that throws an exception and performs an HTTP request`() { + val onEngineRunningLatch = CountDownLatch(1) + val onRespondeHeadersLatch = CountDownLatch(1) + + val context = ApplicationProvider.getApplicationContext() + val builder = AndroidEngineBuilder(context) + val engine = builder + .addLogLevel(LogLevel.DEBUG) + .addPlatformFilter(::ThrowingFilter) + .setOnEngineRunning { onEngineRunningLatch.countDown() } + .build() + + onEngineRunningLatch.await(10, TimeUnit.SECONDS) + assertThat(onEngineRunningLatch.count).isEqualTo(0) + + val requestHeaders = RequestHeadersBuilder( + method = RequestMethod.GET, + scheme = "https", + authority = "api.lyft.com", + path = "/ping" + ) + .build() + + engine + .streamClient() + .newStreamPrototype() + .setOnResponseHeaders { responseHeaders, _, _ -> + val status = responseHeaders.httpStatus ?: 0L + assertThat(status).isEqualTo(200) + onRespondeHeadersLatch.countDown() + } + .start(Executors.newSingleThreadExecutor()) + .sendHeaders(requestHeaders, true) + + onRespondeHeadersLatch.await(15, TimeUnit.SECONDS) + assertThat(onRespondeHeadersLatch.count).isEqualTo(0) + + engine.terminate() + } +} From ff9ab2bc245247529e13ee8eb732da803c2de881 Mon Sep 17 00:00:00 2001 From: JP Simard Date: Wed, 7 Dec 2022 16:33:01 -0500 Subject: [PATCH 20/23] iOS: split `EnvoyEngine.h` into multiple header files (#24397) Resolves https://github.com/envoyproxy/envoy-mobile/issues/333 Signed-off-by: JP Simard --- mobile/library/objective-c/BUILD | 12 + mobile/library/objective-c/EnvoyAliases.h | 25 + .../library/objective-c/EnvoyConfiguration.h | 101 ++++ mobile/library/objective-c/EnvoyEngine.h | 495 +----------------- .../library/objective-c/EnvoyEventTracker.h | 15 + .../library/objective-c/EnvoyEventTracker.m | 2 +- .../library/objective-c/EnvoyHTTPCallbacks.h | 87 +++ mobile/library/objective-c/EnvoyHTTPFilter.h | 124 +++++ .../objective-c/EnvoyHTTPFilterFactory.h | 15 + mobile/library/objective-c/EnvoyHTTPStream.h | 79 +++ .../library/objective-c/EnvoyKeyValueStore.h | 18 + mobile/library/objective-c/EnvoyLogger.h | 17 + .../objective-c/EnvoyNativeFilterConfig.h | 14 + .../library/objective-c/EnvoyNetworkMonitor.h | 27 + .../library/objective-c/EnvoyStringAccessor.h | 13 + 15 files changed, 561 insertions(+), 483 deletions(-) create mode 100644 mobile/library/objective-c/EnvoyAliases.h create mode 100644 mobile/library/objective-c/EnvoyConfiguration.h create mode 100644 mobile/library/objective-c/EnvoyEventTracker.h create mode 100644 mobile/library/objective-c/EnvoyHTTPCallbacks.h create mode 100644 mobile/library/objective-c/EnvoyHTTPFilter.h create mode 100644 mobile/library/objective-c/EnvoyHTTPFilterFactory.h create mode 100644 mobile/library/objective-c/EnvoyHTTPStream.h create mode 100644 mobile/library/objective-c/EnvoyKeyValueStore.h create mode 100644 mobile/library/objective-c/EnvoyLogger.h create mode 100644 mobile/library/objective-c/EnvoyNativeFilterConfig.h create mode 100644 mobile/library/objective-c/EnvoyNetworkMonitor.h create mode 100644 mobile/library/objective-c/EnvoyStringAccessor.h diff --git a/mobile/library/objective-c/BUILD b/mobile/library/objective-c/BUILD index fb4f6ad9275c..deacdbdfa9e1 100644 --- a/mobile/library/objective-c/BUILD +++ b/mobile/library/objective-c/BUILD @@ -7,18 +7,30 @@ exports_files([ objc_library( name = "envoy_engine_objc_lib", srcs = [ + "EnvoyAliases.h", + "EnvoyConfiguration.h", "EnvoyConfiguration.m", "EnvoyEngineImpl.m", + "EnvoyEventTracker.h", "EnvoyEventTracker.m", + "EnvoyHTTPCallbacks.h", "EnvoyHTTPCallbacks.m", + "EnvoyHTTPFilter.h", "EnvoyHTTPFilter.m", "EnvoyHTTPFilterCallbacksImpl.h", "EnvoyHTTPFilterCallbacksImpl.m", + "EnvoyHTTPFilterFactory.h", "EnvoyHTTPFilterFactory.m", + "EnvoyHTTPStream.h", "EnvoyHTTPStreamImpl.m", + "EnvoyKeyValueStore.h", + "EnvoyLogger.h", "EnvoyLogger.m", + "EnvoyNativeFilterConfig.h", "EnvoyNativeFilterConfig.m", + "EnvoyNetworkMonitor.h", "EnvoyNetworkMonitor.m", + "EnvoyStringAccessor.h", "EnvoyStringAccessor.m", ], hdrs = [ diff --git a/mobile/library/objective-c/EnvoyAliases.h b/mobile/library/objective-c/EnvoyAliases.h new file mode 100644 index 000000000000..22d49a138d11 --- /dev/null +++ b/mobile/library/objective-c/EnvoyAliases.h @@ -0,0 +1,25 @@ +#import + +#import "library/common/types/c_types.h" + +NS_ASSUME_NONNULL_BEGIN + +/// Return codes for Engine interface. @see /library/common/types/c_types.h +extern const int kEnvoySuccess; +extern const int kEnvoyFailure; + +/// A set of headers that may be passed to/from an Envoy stream. +typedef NSDictionary *> EnvoyHeaders; + +typedef NSDictionary EnvoyTags; + +/// A set of key-value pairs describing an event. +typedef NSDictionary EnvoyEvent; + +/// Contains internal HTTP stream metrics, context, and other details. +typedef envoy_stream_intel EnvoyStreamIntel; + +// Contains one time HTTP stream metrics, context, and other details. +typedef envoy_final_stream_intel EnvoyFinalStreamIntel; + +NS_ASSUME_NONNULL_END diff --git a/mobile/library/objective-c/EnvoyConfiguration.h b/mobile/library/objective-c/EnvoyConfiguration.h new file mode 100644 index 000000000000..46bf01f21882 --- /dev/null +++ b/mobile/library/objective-c/EnvoyConfiguration.h @@ -0,0 +1,101 @@ +#import + +@class EnvoyHTTPFilterFactory; +@class EnvoyNativeFilterConfig; +@class EnvoyStringAccessor; +@protocol EnvoyKeyValueStore; + +NS_ASSUME_NONNULL_BEGIN + +/// Typed configuration that may be used for starting Envoy. +@interface EnvoyConfiguration : NSObject + +@property (nonatomic, assign) BOOL adminInterfaceEnabled; +@property (nonatomic, strong, nullable) NSString *grpcStatsDomain; +@property (nonatomic, assign) UInt32 connectTimeoutSeconds; +@property (nonatomic, assign) UInt32 dnsFailureRefreshSecondsBase; +@property (nonatomic, assign) UInt32 dnsFailureRefreshSecondsMax; +@property (nonatomic, assign) UInt32 dnsQueryTimeoutSeconds; +@property (nonatomic, assign) UInt32 dnsMinRefreshSeconds; +@property (nonatomic, strong) NSString *dnsPreresolveHostnames; +@property (nonatomic, assign) UInt32 dnsRefreshSeconds; +@property (nonatomic, assign) BOOL enableHappyEyeballs; +@property (nonatomic, assign) BOOL enableGzip; +@property (nonatomic, assign) BOOL enableBrotli; +@property (nonatomic, assign) BOOL enableInterfaceBinding; +@property (nonatomic, assign) BOOL enableDrainPostDnsRefresh; +@property (nonatomic, assign) BOOL enforceTrustChainVerification; +@property (nonatomic, assign) BOOL enablePlatformCertificateValidation; +@property (nonatomic, assign) UInt32 h2ConnectionKeepaliveIdleIntervalMilliseconds; +@property (nonatomic, assign) UInt32 h2ConnectionKeepaliveTimeoutSeconds; +@property (nonatomic, assign) BOOL h2ExtendKeepaliveTimeout; +@property (nonatomic, assign) UInt32 maxConnectionsPerHost; +@property (nonatomic, assign) UInt32 statsFlushSeconds; +@property (nonatomic, assign) UInt32 streamIdleTimeoutSeconds; +@property (nonatomic, assign) UInt32 perTryIdleTimeoutSeconds; +@property (nonatomic, strong) NSString *appVersion; +@property (nonatomic, strong) NSString *appId; +@property (nonatomic, strong) NSString *virtualClusters; +@property (nonatomic, strong) NSString *directResponseMatchers; +@property (nonatomic, strong) NSString *directResponses; +@property (nonatomic, strong) NSArray *nativeFilterChain; +@property (nonatomic, strong) NSArray *httpPlatformFilterFactories; +@property (nonatomic, strong) NSDictionary *stringAccessors; +@property (nonatomic, strong) NSDictionary> *keyValueStores; +@property (nonatomic, strong) NSArray *statsSinks; + +/** + Create a new instance of the configuration. + */ +- (instancetype)initWithAdminInterfaceEnabled:(BOOL)adminInterfaceEnabled + GrpcStatsDomain:(nullable NSString *)grpcStatsDomain + connectTimeoutSeconds:(UInt32)connectTimeoutSeconds + dnsRefreshSeconds:(UInt32)dnsRefreshSeconds + dnsFailureRefreshSecondsBase:(UInt32)dnsFailureRefreshSecondsBase + dnsFailureRefreshSecondsMax:(UInt32)dnsFailureRefreshSecondsMax + dnsQueryTimeoutSeconds:(UInt32)dnsQueryTimeoutSeconds + dnsMinRefreshSeconds:(UInt32)dnsMinRefreshSeconds + dnsPreresolveHostnames:(NSString *)dnsPreresolveHostnames + enableHappyEyeballs:(BOOL)enableHappyEyeballs + enableGzip:(BOOL)enableGzip + enableBrotli:(BOOL)enableBrotli + enableInterfaceBinding:(BOOL)enableInterfaceBinding + enableDrainPostDnsRefresh:(BOOL)enableDrainPostDnsRefresh + enforceTrustChainVerification:(BOOL)enforceTrustChainVerification + enablePlatformCertificateValidation:(BOOL)enablePlatformCertificateValidation + h2ConnectionKeepaliveIdleIntervalMilliseconds: + (UInt32)h2ConnectionKeepaliveIdleIntervalMilliseconds + h2ConnectionKeepaliveTimeoutSeconds:(UInt32)h2ConnectionKeepaliveTimeoutSeconds + h2ExtendKeepaliveTimeout:(BOOL)h2ExtendKeepaliveTimeout + maxConnectionsPerHost:(UInt32)maxConnectionsPerHost + statsFlushSeconds:(UInt32)statsFlushSeconds + streamIdleTimeoutSeconds:(UInt32)streamIdleTimeoutSeconds + perTryIdleTimeoutSeconds:(UInt32)perTryIdleTimeoutSeconds + appVersion:(NSString *)appVersion + appId:(NSString *)appId + virtualClusters:(NSString *)virtualClusters + directResponseMatchers:(NSString *)directResponseMatchers + directResponses:(NSString *)directResponses + nativeFilterChain: + (NSArray *)nativeFilterChain + platformFilterChain: + (NSArray *)httpPlatformFilterFactories + stringAccessors: + (NSDictionary *) + stringAccessors + keyValueStores: + (NSDictionary> *) + keyValueStores + statsSinks:(NSArray *)statsSinks; + +/** + Resolves the provided configuration template using properties on this configuration. + + @param templateYAML The template configuration to resolve. + @return The resolved template. Nil if the template fails to fully resolve. + */ +- (nullable NSString *)resolveTemplate:(NSString *)templateYAML; + +@end + +NS_ASSUME_NONNULL_END diff --git a/mobile/library/objective-c/EnvoyEngine.h b/mobile/library/objective-c/EnvoyEngine.h index f063f0c57cc8..22b66ce9d87e 100644 --- a/mobile/library/objective-c/EnvoyEngine.h +++ b/mobile/library/objective-c/EnvoyEngine.h @@ -1,457 +1,24 @@ #import +#import "library/objective-c/EnvoyAliases.h" +#import "library/objective-c/EnvoyConfiguration.h" +#import "library/objective-c/EnvoyEventTracker.h" +#import "library/objective-c/EnvoyHTTPCallbacks.h" +#import "library/objective-c/EnvoyHTTPFilter.h" +#import "library/objective-c/EnvoyHTTPFilterFactory.h" +#import "library/objective-c/EnvoyHTTPStream.h" +#import "library/objective-c/EnvoyKeyValueStore.h" +#import "library/objective-c/EnvoyLogger.h" +#import "library/objective-c/EnvoyNativeFilterConfig.h" +#import "library/objective-c/EnvoyNetworkMonitor.h" +#import "library/objective-c/EnvoyStringAccessor.h" + #import "library/common/types/c_types.h" NS_ASSUME_NONNULL_BEGIN -#pragma mark - Aliases - -/// A set of headers that may be passed to/from an Envoy stream. -typedef NSDictionary *> EnvoyHeaders; - -typedef NSDictionary EnvoyTags; - -/// A set of key-value pairs describing an event. -typedef NSDictionary EnvoyEvent; - -/// Contains internal HTTP stream metrics, context, and other details. -typedef envoy_stream_intel EnvoyStreamIntel; - -// Contains one time HTTP stream metrics, context, and other details. -typedef envoy_final_stream_intel EnvoyFinalStreamIntel; - -#pragma mark - EnvoyHTTPCallbacks - -/// Interface that can handle callbacks from an HTTP stream. -@interface EnvoyHTTPCallbacks : NSObject - -/** - * Dispatch queue provided to handle callbacks. - */ -@property (nonatomic, assign) dispatch_queue_t dispatchQueue; - -// Formatting for block properties is inconsistent and not configurable. -// clang-format off - -/** - * Called when all headers get received on the async HTTP stream. - * @param headers the headers received. - * @param endStream whether the response is headers-only. - * @param streamIntel internal HTTP stream metrics, context, and other details. - */ -@property (nonatomic, copy) void (^onHeaders)( - EnvoyHeaders *headers, BOOL endStream, EnvoyStreamIntel streamIntel); - -/** - * Called when a data frame gets received on the async HTTP stream. - * This callback can be invoked multiple times if the data gets streamed. - * @param data the data received. - * @param endStream whether the data is the last data frame. - * @param streamIntel internal HTTP stream metrics, context, and other details. - */ -@property (nonatomic, copy) void (^onData)( - NSData *data, BOOL endStream, EnvoyStreamIntel streamIntel); - -/** - * Called when all trailers get received on the async HTTP stream. - * Note that end stream is implied when on_trailers is called. - * @param trailers the trailers received. - * @param streamIntel internal HTTP stream metrics, context, and other details. - */ -@property (nonatomic, copy) void (^onTrailers)( - EnvoyHeaders *trailers, EnvoyStreamIntel streamIntel); - -/** - * Called to signal there is buffer space available for continued request body upload. - * - * This is only ever called when the library is in explicit flow control mode. When enabled, - * the issuer should wait for this callback after calling sendData, before making another call - * to sendData. - * @param streamIntel internal HTTP stream metrics, context, and other details. - */ -@property (nonatomic, copy) void (^onSendWindowAvailable)(EnvoyStreamIntel streamIntel); - -/** - * Called when the async HTTP stream has an error. - * @param streamIntel internal HTTP stream metrics, context, and other details. - * @param finalStreamIntel one time HTTP stream metrics, context, and other details. - */ -@property (nonatomic, copy) void (^onError)( - uint64_t errorCode, NSString *message, int32_t attemptCount, EnvoyStreamIntel streamIntel, - EnvoyFinalStreamIntel finalStreamIntel); - -/** - * Called when the async HTTP stream is canceled. - * Note this callback will ALWAYS be fired if a stream is canceled, even if the request and/or - * response is already complete. It will fire no more than once, and no other callbacks for the - * stream will be issued afterwards. - * @param streamIntel internal HTTP stream metrics, context, and other details. - * @param finalStreamIntel one time HTTP stream metrics, context, and other details. - */ -@property (nonatomic, copy) void (^onCancel)( - EnvoyStreamIntel streamIntel, EnvoyFinalStreamIntel finalStreamIntel); - -/** - * Final call made when an HTTP stream is closed gracefully. - * Note this may already be inferred from a prior callback with endStream=TRUE, and this only needs - * to be handled if information from finalStreamIntel is desired. - * @param streamIntel internal HTTP stream metrics, context, and other details. - * @param finalStreamIntel one time HTTP stream metrics, context, and other details. - */ -@property (nonatomic, copy) void (^onComplete)( - EnvoyStreamIntel streamIntel, EnvoyFinalStreamIntel finalStreamIntel); - -// clang-format on -@end - -#pragma mark - EnvoyHTTPFilter - -/// Return codes for on-headers filter invocations. @see envoy/http/filter.h -extern const int kEnvoyFilterHeadersStatusContinue; -extern const int kEnvoyFilterHeadersStatusStopIteration; -extern const int kEnvoyFilterHeadersStatusStopAllIterationAndBuffer; - -/// Return codes for on-data filter invocations. @see envoy/http/filter.h -extern const int kEnvoyFilterDataStatusContinue; -extern const int kEnvoyFilterDataStatusStopIterationAndBuffer; -extern const int kEnvoyFilterDataStatusStopIterationNoBuffer; -extern const int kEnvoyFilterDataStatusResumeIteration; - -/// Return codes for on-trailers filter invocations. @see envoy/http/filter.h -extern const int kEnvoyFilterTrailersStatusContinue; -extern const int kEnvoyFilterTrailersStatusStopIteration; -extern const int kEnvoyFilterTrailersStatusResumeIteration; - -/// Return codes for on-resume filter invocations. These are unique to platform filters, -/// and used exclusively after an asynchronous request to resume iteration via callbacks. -extern const int kEnvoyFilterResumeStatusStopIteration; -extern const int kEnvoyFilterResumeStatusResumeIteration; - -/// Callbacks for asynchronous interaction with the filter. -@protocol EnvoyHTTPFilterCallbacks - -/// Resume filter iteration asynchronously. This will result in an on-resume invocation of the -/// filter. -- (void)resumeIteration; - -/// Reset the underlying stream idle timeout to its configured threshold. This may be useful if -/// a filter stops iteration for an extended period of time, since ordinarily timeouts will still -/// apply. This may be called periodically to continue to indicate "activity" on the stream. -- (void)resetIdleTimer; - -@end - -@interface EnvoyHTTPFilter : NSObject - -// Formatting for block properties is inconsistent and not configurable. -// clang-format off - -/// Returns tuple of: -/// 0 - NSNumber *,filter status -/// 1 - EnvoyHeaders *, forward headers -@property (nonatomic, copy) NSArray * (^onRequestHeaders)( - EnvoyHeaders *headers, BOOL endStream, EnvoyStreamIntel streamIntel); - -/// Returns tuple of: -/// 0 - NSNumber *,filter status -/// 1 - NSData *, forward data -/// 2 - EnvoyHeaders *, optional pending headers -@property (nonatomic, copy) NSArray * (^onRequestData)( - NSData *data, BOOL endStream, EnvoyStreamIntel streamIntel); - -/// Returns tuple of: -/// 0 - NSNumber *,filter status -/// 1 - EnvoyHeaders *, forward trailers -/// 2 - EnvoyHeaders *, optional pending headers -/// 3 - NSData *, optional pending data -@property (nonatomic, copy) NSArray * (^onRequestTrailers)( - EnvoyHeaders *trailers, EnvoyStreamIntel streamIntel); - -/// Returns tuple of: -/// 0 - NSNumber *,filter status -/// 1 - EnvoyHeaders *, forward headers -@property (nonatomic, copy) NSArray * (^onResponseHeaders)( - EnvoyHeaders *headers, BOOL endStream, EnvoyStreamIntel streamIntel); - -/// Returns tuple of: -/// 0 - NSNumber *,filter status -/// 1 - NSData *, forward data -/// 2 - EnvoyHeaders *, optional pending headers -@property (nonatomic, copy) NSArray * (^onResponseData)( - NSData *data, BOOL endStream, EnvoyStreamIntel streamIntel); - -/// Returns tuple of: -/// 0 - NSNumber *,filter status -/// 1 - EnvoyHeaders *, forward trailers -/// 2 - EnvoyHeaders *, optional pending headers -/// 3 - NSData *, optional pending data -@property (nonatomic, copy)NSArray * (^onResponseTrailers)( - EnvoyHeaders *trailers, EnvoyStreamIntel streamIntel); - -@property (nonatomic, copy) void (^onCancel)( - EnvoyStreamIntel streamIntel, EnvoyFinalStreamIntel finalStreamIntel); - -@property (nonatomic, copy) void (^onError)( - uint64_t errorCode, NSString *message, int32_t attemptCount, EnvoyStreamIntel streamIntel, - EnvoyFinalStreamIntel finalStreamIntel); - -@property (nonatomic, copy) void (^onComplete)( - EnvoyStreamIntel streamIntel, EnvoyFinalStreamIntel finalStreamIntel); - -@property (nonatomic, copy) void (^setRequestFilterCallbacks)( - id callbacks); - -/// Returns tuple of: -/// 0 - NSNumber *,filter status -/// 1 - EnvoyHeaders *, optional pending headers -/// 2 - NSData *, optional pending data -/// 3 - EnvoyHeaders *, optional pending trailers -@property (nonatomic, copy) NSArray * (^onResumeRequest)( - EnvoyHeaders *_Nullable headers, NSData *_Nullable data, EnvoyHeaders *_Nullable trailers, - BOOL endStream, EnvoyStreamIntel streamIntel); - -@property (nonatomic, copy) void (^setResponseFilterCallbacks)( - id callbacks); - -/// Returns tuple of: -/// 0 - NSNumber *,filter status -/// 1 - EnvoyHeaders *, optional pending headers -/// 2 - NSData *, optional pending data -/// 3 - EnvoyHeaders *, optional pending trailers -@property (nonatomic, copy) NSArray * (^onResumeResponse)( - EnvoyHeaders *_Nullable headers, NSData *_Nullable data, EnvoyHeaders *_Nullable trailers, - BOOL endStream, EnvoyStreamIntel streamIntel); - -// clang-format on -@end - -#pragma mark - EnvoyHTTPFilterFactory - -@interface EnvoyHTTPFilterFactory : NSObject - -@property (nonatomic, strong) NSString *filterName; - -@property (nonatomic, copy) EnvoyHTTPFilter * (^create)(); - -@end - -#pragma mark - EnvoyHTTPStream - -@protocol EnvoyHTTPStream - -/** - Open an underlying HTTP stream. - - @param handle Underlying handle of the HTTP stream owned by an Envoy engine. - @param engine Underlying handle of the Envoy engine. - @param callbacks The callbacks for the stream. - @param explicitFlowControl Whether explicit flow control will be enabled for this stream. - */ -- (instancetype)initWithHandle:(intptr_t)handle - engine:(intptr_t)engineHandle - callbacks:(EnvoyHTTPCallbacks *)callbacks - explicitFlowControl:(BOOL)explicitFlowControl; - -/** - Send headers over the provided stream. - - @param headers Headers to send over the stream. - @param close True if the stream should be closed after sending. - */ -- (void)sendHeaders:(EnvoyHeaders *)headers close:(BOOL)close; - -/** - Read data from the response stream. Returns immediately. - Has no effect if explicit flow control is not enabled. - - @param byteCount Maximum number of bytes that may be be passed by the next data callback. - */ -- (void)readData:(size_t)byteCount; - -/** - Send data over the provided stream. - - @param data Data to send over the stream. - @param close True if the stream should be closed after sending. - */ -- (void)sendData:(NSData *)data close:(BOOL)close; - -/** - Send trailers over the provided stream. - - @param trailers Trailers to send over the stream. - */ -- (void)sendTrailers:(EnvoyHeaders *)trailers; - -/** - Cancel the stream. This functions as an interrupt, and aborts further callbacks and handling of the - stream. - - @return Success unless the stream has already been canceled. - */ -- (int)cancel; - -/** - Clean up the stream after it's closed (by completion, cancellation, or error). - */ -- (void)cleanUp; - -@end - -#pragma mark - EnvoyHTTPStreamImpl - -// Concrete implementation of the `EnvoyHTTPStream` protocol. -@interface EnvoyHTTPStreamImpl : NSObject - -@end - -#pragma mark - EnvoyStringAccessor - -@interface EnvoyStringAccessor : NSObject - -@property (nonatomic, copy) NSString * (^getEnvoyString)(); - -- (instancetype)initWithBlock:(NSString * (^)())block; - -@end - -#pragma mark - EnvoyKeyValueStore - -@protocol EnvoyKeyValueStore - -/// Read a value from the key value store implementation. -- (NSString *_Nullable)readValueForKey:(NSString *)key; - -/// Save a value to the key value store implementation. -- (void)saveValue:(NSString *)value toKey:(NSString *)key; - -/// Remove a value from the key value store implementation. -- (void)removeKey:(NSString *)key; - -@end - -#pragma mark - EnvoyNativeFilterConfig - -@interface EnvoyNativeFilterConfig : NSObject - -@property (nonatomic, strong) NSString *name; -@property (nonatomic, strong) NSString *typedConfig; - -- (instancetype)initWithName:(NSString *)name typedConfig:(NSString *)typedConfig; - -@end - -#pragma mark - EnvoyConfiguration - -/// Typed configuration that may be used for starting Envoy. -@interface EnvoyConfiguration : NSObject - -@property (nonatomic, assign) BOOL adminInterfaceEnabled; -@property (nonatomic, strong, nullable) NSString *grpcStatsDomain; -@property (nonatomic, assign) UInt32 connectTimeoutSeconds; -@property (nonatomic, assign) UInt32 dnsFailureRefreshSecondsBase; -@property (nonatomic, assign) UInt32 dnsFailureRefreshSecondsMax; -@property (nonatomic, assign) UInt32 dnsQueryTimeoutSeconds; -@property (nonatomic, assign) UInt32 dnsMinRefreshSeconds; -@property (nonatomic, strong) NSString *dnsPreresolveHostnames; -@property (nonatomic, assign) UInt32 dnsRefreshSeconds; -@property (nonatomic, assign) BOOL enableHappyEyeballs; -@property (nonatomic, assign) BOOL enableGzip; -@property (nonatomic, assign) BOOL enableBrotli; -@property (nonatomic, assign) BOOL enableInterfaceBinding; -@property (nonatomic, assign) BOOL enableDrainPostDnsRefresh; -@property (nonatomic, assign) BOOL enforceTrustChainVerification; -@property (nonatomic, assign) BOOL enablePlatformCertificateValidation; -@property (nonatomic, assign) UInt32 h2ConnectionKeepaliveIdleIntervalMilliseconds; -@property (nonatomic, assign) UInt32 h2ConnectionKeepaliveTimeoutSeconds; -@property (nonatomic, assign) BOOL h2ExtendKeepaliveTimeout; -@property (nonatomic, assign) UInt32 maxConnectionsPerHost; -@property (nonatomic, assign) UInt32 statsFlushSeconds; -@property (nonatomic, assign) UInt32 streamIdleTimeoutSeconds; -@property (nonatomic, assign) UInt32 perTryIdleTimeoutSeconds; -@property (nonatomic, strong) NSString *appVersion; -@property (nonatomic, strong) NSString *appId; -@property (nonatomic, strong) NSString *virtualClusters; -@property (nonatomic, strong) NSString *directResponseMatchers; -@property (nonatomic, strong) NSString *directResponses; -@property (nonatomic, strong) NSArray *nativeFilterChain; -@property (nonatomic, strong) NSArray *httpPlatformFilterFactories; -@property (nonatomic, strong) NSDictionary *stringAccessors; -@property (nonatomic, strong) NSDictionary> *keyValueStores; -@property (nonatomic, strong) NSArray *statsSinks; - -/** - Create a new instance of the configuration. - */ -- (instancetype)initWithAdminInterfaceEnabled:(BOOL)adminInterfaceEnabled - GrpcStatsDomain:(nullable NSString *)grpcStatsDomain - connectTimeoutSeconds:(UInt32)connectTimeoutSeconds - dnsRefreshSeconds:(UInt32)dnsRefreshSeconds - dnsFailureRefreshSecondsBase:(UInt32)dnsFailureRefreshSecondsBase - dnsFailureRefreshSecondsMax:(UInt32)dnsFailureRefreshSecondsMax - dnsQueryTimeoutSeconds:(UInt32)dnsQueryTimeoutSeconds - dnsMinRefreshSeconds:(UInt32)dnsMinRefreshSeconds - dnsPreresolveHostnames:(NSString *)dnsPreresolveHostnames - enableHappyEyeballs:(BOOL)enableHappyEyeballs - enableGzip:(BOOL)enableGzip - enableBrotli:(BOOL)enableBrotli - enableInterfaceBinding:(BOOL)enableInterfaceBinding - enableDrainPostDnsRefresh:(BOOL)enableDrainPostDnsRefresh - enforceTrustChainVerification:(BOOL)enforceTrustChainVerification - enablePlatformCertificateValidation:(BOOL)enablePlatformCertificateValidation - h2ConnectionKeepaliveIdleIntervalMilliseconds: - (UInt32)h2ConnectionKeepaliveIdleIntervalMilliseconds - h2ConnectionKeepaliveTimeoutSeconds:(UInt32)h2ConnectionKeepaliveTimeoutSeconds - h2ExtendKeepaliveTimeout:(BOOL)h2ExtendKeepaliveTimeout - maxConnectionsPerHost:(UInt32)maxConnectionsPerHost - statsFlushSeconds:(UInt32)statsFlushSeconds - streamIdleTimeoutSeconds:(UInt32)streamIdleTimeoutSeconds - perTryIdleTimeoutSeconds:(UInt32)perTryIdleTimeoutSeconds - appVersion:(NSString *)appVersion - appId:(NSString *)appId - virtualClusters:(NSString *)virtualClusters - directResponseMatchers:(NSString *)directResponseMatchers - directResponses:(NSString *)directResponses - nativeFilterChain: - (NSArray *)nativeFilterChain - platformFilterChain: - (NSArray *)httpPlatformFilterFactories - stringAccessors: - (NSDictionary *) - stringAccessors - keyValueStores: - (NSDictionary> *) - keyValueStores - statsSinks:(NSArray *)statsSinks; - -/** - Resolves the provided configuration template using properties on this configuration. - - @param templateYAML The template configuration to resolve. - @return The resolved template. Nil if the template fails to fully resolve. - */ -- (nullable NSString *)resolveTemplate:(NSString *)templateYAML; - -@end - -#pragma mark - EnvoyEventTracker - -// Tracking events interface - -@interface EnvoyEventTracker : NSObject - -@property (nonatomic, copy, nonnull) void (^track)(EnvoyEvent *); - -- (instancetype)initWithEventTrackingClosure:(nonnull void (^)(EnvoyEvent *))track; - -@end - #pragma mark - EnvoyEngine -/// Return codes for Engine interface. @see /library/common/types/c_types.h -extern const int kEnvoySuccess; -extern const int kEnvoyFailure; - /// Wrapper layer for calling into Envoy's C/++ API. @protocol EnvoyEngine @@ -569,20 +136,6 @@ extern const int kEnvoyFailure; @end -#pragma mark - EnvoyLogger - -// Logging interface. -@interface EnvoyLogger : NSObject - -@property (nonatomic, copy) void (^log)(NSString *); - -/** - Create a new instance of the logger. - */ -- (instancetype)initWithLogClosure:(void (^)(NSString *))log; - -@end - #pragma mark - EnvoyEngineImpl // Concrete implementation of the `EnvoyEngine` interface. @@ -592,26 +145,4 @@ extern const int kEnvoyFailure; @end -#pragma mark - EnvoyNetworkMonitor - -// Monitors network changes in order to update Envoy network cluster preferences. -@interface EnvoyNetworkMonitor : NSObject - -/** - Create a new instance of the network monitor. - */ -- (instancetype)initWithEngine:(envoy_engine_t)engineHandle; - -// Start monitoring reachability using `SCNetworkReachability`, updating the -// preferred Envoy network cluster on changes. -// This is typically called by `EnvoyEngine` automatically on startup. -- (void)startReachability; - -// Start monitoring reachability using `NWPathMonitor`, updating the -// preferred Envoy network cluster on changes. -// This is typically called by `EnvoyEngine` automatically on startup. -- (void)startPathMonitor; - -@end - NS_ASSUME_NONNULL_END diff --git a/mobile/library/objective-c/EnvoyEventTracker.h b/mobile/library/objective-c/EnvoyEventTracker.h new file mode 100644 index 000000000000..1c884d34cab2 --- /dev/null +++ b/mobile/library/objective-c/EnvoyEventTracker.h @@ -0,0 +1,15 @@ +#import + +#import "library/objective-c/EnvoyAliases.h" + +NS_ASSUME_NONNULL_BEGIN + +@interface EnvoyEventTracker : NSObject + +@property (nonatomic, copy) void (^track)(EnvoyEvent *); + +- (instancetype)initWithEventTrackingClosure:(void (^)(EnvoyEvent *))track; + +@end + +NS_ASSUME_NONNULL_END diff --git a/mobile/library/objective-c/EnvoyEventTracker.m b/mobile/library/objective-c/EnvoyEventTracker.m index 5e4f8680a2fb..d9925925414a 100644 --- a/mobile/library/objective-c/EnvoyEventTracker.m +++ b/mobile/library/objective-c/EnvoyEventTracker.m @@ -2,7 +2,7 @@ @implementation EnvoyEventTracker -- (instancetype)initWithEventTrackingClosure:(nonnull void (^)(EnvoyEvent *))track { +- (instancetype)initWithEventTrackingClosure:(void (^)(EnvoyEvent *))track { self = [super init]; if (!self) { return nil; diff --git a/mobile/library/objective-c/EnvoyHTTPCallbacks.h b/mobile/library/objective-c/EnvoyHTTPCallbacks.h new file mode 100644 index 000000000000..3ade8d060649 --- /dev/null +++ b/mobile/library/objective-c/EnvoyHTTPCallbacks.h @@ -0,0 +1,87 @@ +#import + +#import "library/objective-c/EnvoyAliases.h" + +NS_ASSUME_NONNULL_BEGIN + +#pragma mark - EnvoyHTTPCallbacks + +/// Interface that can handle callbacks from an HTTP stream. +@interface EnvoyHTTPCallbacks : NSObject + +/** + * Dispatch queue provided to handle callbacks. + */ +@property (nonatomic, assign) dispatch_queue_t dispatchQueue; + +/** + * Called when all headers get received on the async HTTP stream. + * @param headers the headers received. + * @param endStream whether the response is headers-only. + * @param streamIntel internal HTTP stream metrics, context, and other details. + */ +@property (nonatomic, copy) void (^onHeaders) + (EnvoyHeaders *headers, BOOL endStream, EnvoyStreamIntel streamIntel); + +/** + * Called when a data frame gets received on the async HTTP stream. + * This callback can be invoked multiple times if the data gets streamed. + * @param data the data received. + * @param endStream whether the data is the last data frame. + * @param streamIntel internal HTTP stream metrics, context, and other details. + */ +@property (nonatomic, copy) void (^onData) + (NSData *data, BOOL endStream, EnvoyStreamIntel streamIntel); + +/** + * Called when all trailers get received on the async HTTP stream. + * Note that end stream is implied when on_trailers is called. + * @param trailers the trailers received. + * @param streamIntel internal HTTP stream metrics, context, and other details. + */ +@property (nonatomic, copy) void (^onTrailers)(EnvoyHeaders *trailers, EnvoyStreamIntel streamIntel) + ; + +/** + * Called to signal there is buffer space available for continued request body upload. + * + * This is only ever called when the library is in explicit flow control mode. When enabled, + * the issuer should wait for this callback after calling sendData, before making another call + * to sendData. + * @param streamIntel internal HTTP stream metrics, context, and other details. + */ +@property (nonatomic, copy) void (^onSendWindowAvailable)(EnvoyStreamIntel streamIntel); + +/** + * Called when the async HTTP stream has an error. + * @param streamIntel internal HTTP stream metrics, context, and other details. + * @param finalStreamIntel one time HTTP stream metrics, context, and other details. + */ +@property (nonatomic, copy) void (^onError) + (uint64_t errorCode, NSString *message, int32_t attemptCount, EnvoyStreamIntel streamIntel, + EnvoyFinalStreamIntel finalStreamIntel); + +/** + * Called when the async HTTP stream is canceled. + * Note this callback will ALWAYS be fired if a stream is canceled, even if the request and/or + * response is already complete. It will fire no more than once, and no other callbacks for the + * stream will be issued afterwards. + * @param streamIntel internal HTTP stream metrics, context, and other details. + * @param finalStreamIntel one time HTTP stream metrics, context, and other details. + */ +@property (nonatomic, copy) void (^onCancel) + (EnvoyStreamIntel streamIntel, EnvoyFinalStreamIntel finalStreamIntel); + +/** + * Final call made when an HTTP stream is closed gracefully. + * Note this may already be inferred from a prior callback with endStream=TRUE, and this only needs + * to be handled if information from finalStreamIntel is desired. + * @param streamIntel internal HTTP stream metrics, context, and other details. + * @param finalStreamIntel one time HTTP stream metrics, context, and other details. + */ +@property (nonatomic, copy) void (^onComplete) + (EnvoyStreamIntel streamIntel, EnvoyFinalStreamIntel finalStreamIntel); + +@end + +NS_ASSUME_NONNULL_END diff --git a/mobile/library/objective-c/EnvoyHTTPFilter.h b/mobile/library/objective-c/EnvoyHTTPFilter.h new file mode 100644 index 000000000000..a6bce99e8eee --- /dev/null +++ b/mobile/library/objective-c/EnvoyHTTPFilter.h @@ -0,0 +1,124 @@ +#import + +#import "library/objective-c/EnvoyAliases.h" + +#import "library/common/types/c_types.h" + +NS_ASSUME_NONNULL_BEGIN + +/// Return codes for on-headers filter invocations. @see envoy/http/filter.h +extern const int kEnvoyFilterHeadersStatusContinue; +extern const int kEnvoyFilterHeadersStatusStopIteration; +extern const int kEnvoyFilterHeadersStatusStopAllIterationAndBuffer; + +/// Return codes for on-data filter invocations. @see envoy/http/filter.h +extern const int kEnvoyFilterDataStatusContinue; +extern const int kEnvoyFilterDataStatusStopIterationAndBuffer; +extern const int kEnvoyFilterDataStatusStopIterationNoBuffer; +extern const int kEnvoyFilterDataStatusResumeIteration; + +/// Return codes for on-trailers filter invocations. @see envoy/http/filter.h +extern const int kEnvoyFilterTrailersStatusContinue; +extern const int kEnvoyFilterTrailersStatusStopIteration; +extern const int kEnvoyFilterTrailersStatusResumeIteration; + +/// Return codes for on-resume filter invocations. These are unique to platform filters, +/// and used exclusively after an asynchronous request to resume iteration via callbacks. +extern const int kEnvoyFilterResumeStatusStopIteration; +extern const int kEnvoyFilterResumeStatusResumeIteration; + +/// Callbacks for asynchronous interaction with the filter. +@protocol EnvoyHTTPFilterCallbacks + +/// Resume filter iteration asynchronously. This will result in an on-resume invocation of the +/// filter. +- (void)resumeIteration; + +/// Reset the underlying stream idle timeout to its configured threshold. This may be useful if +/// a filter stops iteration for an extended period of time, since ordinarily timeouts will still +/// apply. This may be called periodically to continue to indicate "activity" on the stream. +- (void)resetIdleTimer; + +@end + +@interface EnvoyHTTPFilter : NSObject + +/// Returns tuple of: +/// 0 - NSNumber *,filter status +/// 1 - EnvoyHeaders *, forward headers +@property (nonatomic, copy) NSArray * (^onRequestHeaders) + (EnvoyHeaders *headers, BOOL endStream, EnvoyStreamIntel streamIntel); + +/// Returns tuple of: +/// 0 - NSNumber *,filter status +/// 1 - NSData *, forward data +/// 2 - EnvoyHeaders *, optional pending headers +@property (nonatomic, copy) NSArray * (^onRequestData) + (NSData *data, BOOL endStream, EnvoyStreamIntel streamIntel); + +/// Returns tuple of: +/// 0 - NSNumber *,filter status +/// 1 - EnvoyHeaders *, forward trailers +/// 2 - EnvoyHeaders *, optional pending headers +/// 3 - NSData *, optional pending data +@property (nonatomic, copy) NSArray * (^onRequestTrailers) + (EnvoyHeaders *trailers, EnvoyStreamIntel streamIntel); + +/// Returns tuple of: +/// 0 - NSNumber *,filter status +/// 1 - EnvoyHeaders *, forward headers +@property (nonatomic, copy) NSArray * (^onResponseHeaders) + (EnvoyHeaders *headers, BOOL endStream, EnvoyStreamIntel streamIntel); + +/// Returns tuple of: +/// 0 - NSNumber *,filter status +/// 1 - NSData *, forward data +/// 2 - EnvoyHeaders *, optional pending headers +@property (nonatomic, copy) NSArray * (^onResponseData) + (NSData *data, BOOL endStream, EnvoyStreamIntel streamIntel); + +/// Returns tuple of: +/// 0 - NSNumber *,filter status +/// 1 - EnvoyHeaders *, forward trailers +/// 2 - EnvoyHeaders *, optional pending headers +/// 3 - NSData *, optional pending data +@property (nonatomic, copy) NSArray * (^onResponseTrailers) + (EnvoyHeaders *trailers, EnvoyStreamIntel streamIntel); + +@property (nonatomic, copy) void (^onCancel) + (EnvoyStreamIntel streamIntel, EnvoyFinalStreamIntel finalStreamIntel); + +@property (nonatomic, copy) void (^onError) + (uint64_t errorCode, NSString *message, int32_t attemptCount, EnvoyStreamIntel streamIntel, + EnvoyFinalStreamIntel finalStreamIntel); + +@property (nonatomic, copy) void (^onComplete) + (EnvoyStreamIntel streamIntel, EnvoyFinalStreamIntel finalStreamIntel); + +@property (nonatomic, copy) void (^setRequestFilterCallbacks) + (id callbacks); + +/// Returns tuple of: +/// 0 - NSNumber *,filter status +/// 1 - EnvoyHeaders *, optional pending headers +/// 2 - NSData *, optional pending data +/// 3 - EnvoyHeaders *, optional pending trailers +@property (nonatomic, copy) NSArray * (^onResumeRequest) + (EnvoyHeaders *_Nullable headers, NSData *_Nullable data, EnvoyHeaders *_Nullable trailers, + BOOL endStream, EnvoyStreamIntel streamIntel); + +@property (nonatomic, copy) void (^setResponseFilterCallbacks) + (id callbacks); + +/// Returns tuple of: +/// 0 - NSNumber *,filter status +/// 1 - EnvoyHeaders *, optional pending headers +/// 2 - NSData *, optional pending data +/// 3 - EnvoyHeaders *, optional pending trailers +@property (nonatomic, copy) NSArray * (^onResumeResponse) + (EnvoyHeaders *_Nullable headers, NSData *_Nullable data, EnvoyHeaders *_Nullable trailers, + BOOL endStream, EnvoyStreamIntel streamIntel); + +@end + +NS_ASSUME_NONNULL_END diff --git a/mobile/library/objective-c/EnvoyHTTPFilterFactory.h b/mobile/library/objective-c/EnvoyHTTPFilterFactory.h new file mode 100644 index 000000000000..25b7509e1737 --- /dev/null +++ b/mobile/library/objective-c/EnvoyHTTPFilterFactory.h @@ -0,0 +1,15 @@ +#import + +@class EnvoyHTTPFilter; + +NS_ASSUME_NONNULL_BEGIN + +@interface EnvoyHTTPFilterFactory : NSObject + +@property (nonatomic, strong) NSString *filterName; + +@property (nonatomic, copy) EnvoyHTTPFilter * (^create)(); + +@end + +NS_ASSUME_NONNULL_END diff --git a/mobile/library/objective-c/EnvoyHTTPStream.h b/mobile/library/objective-c/EnvoyHTTPStream.h new file mode 100644 index 000000000000..c2a425618c47 --- /dev/null +++ b/mobile/library/objective-c/EnvoyHTTPStream.h @@ -0,0 +1,79 @@ +#import + +#import "library/objective-c/EnvoyAliases.h" + +#import "library/common/types/c_types.h" + +NS_ASSUME_NONNULL_BEGIN + +#pragma mark - EnvoyHTTPStream + +@protocol EnvoyHTTPStream + +/** + Open an underlying HTTP stream. + + @param handle Underlying handle of the HTTP stream owned by an Envoy engine. + @param engine Underlying handle of the Envoy engine. + @param callbacks The callbacks for the stream. + @param explicitFlowControl Whether explicit flow control will be enabled for this stream. + */ +- (instancetype)initWithHandle:(intptr_t)handle + engine:(intptr_t)engineHandle + callbacks:(EnvoyHTTPCallbacks *)callbacks + explicitFlowControl:(BOOL)explicitFlowControl; + +/** + Send headers over the provided stream. + + @param headers Headers to send over the stream. + @param close True if the stream should be closed after sending. + */ +- (void)sendHeaders:(EnvoyHeaders *)headers close:(BOOL)close; + +/** + Read data from the response stream. Returns immediately. + Has no effect if explicit flow control is not enabled. + + @param byteCount Maximum number of bytes that may be be passed by the next data callback. + */ +- (void)readData:(size_t)byteCount; + +/** + Send data over the provided stream. + + @param data Data to send over the stream. + @param close True if the stream should be closed after sending. + */ +- (void)sendData:(NSData *)data close:(BOOL)close; + +/** + Send trailers over the provided stream. + + @param trailers Trailers to send over the stream. + */ +- (void)sendTrailers:(EnvoyHeaders *)trailers; + +/** + Cancel the stream. This functions as an interrupt, and aborts further callbacks and handling of the + stream. + + @return Success unless the stream has already been canceled. + */ +- (int)cancel; + +/** + Clean up the stream after it's closed (by completion, cancellation, or error). + */ +- (void)cleanUp; + +@end + +#pragma mark - EnvoyHTTPStreamImpl + +// Concrete implementation of the `EnvoyHTTPStream` protocol. +@interface EnvoyHTTPStreamImpl : NSObject + +@end + +NS_ASSUME_NONNULL_END diff --git a/mobile/library/objective-c/EnvoyKeyValueStore.h b/mobile/library/objective-c/EnvoyKeyValueStore.h new file mode 100644 index 000000000000..745fb353ced3 --- /dev/null +++ b/mobile/library/objective-c/EnvoyKeyValueStore.h @@ -0,0 +1,18 @@ +#import + +NS_ASSUME_NONNULL_BEGIN + +@protocol EnvoyKeyValueStore + +/// Read a value from the key value store implementation. +- (NSString *_Nullable)readValueForKey:(NSString *)key; + +/// Save a value to the key value store implementation. +- (void)saveValue:(NSString *)value toKey:(NSString *)key; + +/// Remove a value from the key value store implementation. +- (void)removeKey:(NSString *)key; + +@end + +NS_ASSUME_NONNULL_END diff --git a/mobile/library/objective-c/EnvoyLogger.h b/mobile/library/objective-c/EnvoyLogger.h new file mode 100644 index 000000000000..8c630f5efc4d --- /dev/null +++ b/mobile/library/objective-c/EnvoyLogger.h @@ -0,0 +1,17 @@ +#import + +NS_ASSUME_NONNULL_BEGIN + +// Logging interface. +@interface EnvoyLogger : NSObject + +@property (nonatomic, copy) void (^log)(NSString *); + +/** + Create a new instance of the logger. + */ +- (instancetype)initWithLogClosure:(void (^)(NSString *))log; + +@end + +NS_ASSUME_NONNULL_END diff --git a/mobile/library/objective-c/EnvoyNativeFilterConfig.h b/mobile/library/objective-c/EnvoyNativeFilterConfig.h new file mode 100644 index 000000000000..2f38ecee4619 --- /dev/null +++ b/mobile/library/objective-c/EnvoyNativeFilterConfig.h @@ -0,0 +1,14 @@ +#import + +NS_ASSUME_NONNULL_BEGIN + +@interface EnvoyNativeFilterConfig : NSObject + +@property (nonatomic, strong) NSString *name; +@property (nonatomic, strong) NSString *typedConfig; + +- (instancetype)initWithName:(NSString *)name typedConfig:(NSString *)typedConfig; + +@end + +NS_ASSUME_NONNULL_END diff --git a/mobile/library/objective-c/EnvoyNetworkMonitor.h b/mobile/library/objective-c/EnvoyNetworkMonitor.h new file mode 100644 index 000000000000..e09d3807964d --- /dev/null +++ b/mobile/library/objective-c/EnvoyNetworkMonitor.h @@ -0,0 +1,27 @@ +#import + +#import "library/common/types/c_types.h" + +NS_ASSUME_NONNULL_BEGIN + +// Monitors network changes in order to update Envoy network cluster preferences. +@interface EnvoyNetworkMonitor : NSObject + +/** + Create a new instance of the network monitor. + */ +- (instancetype)initWithEngine:(envoy_engine_t)engineHandle; + +// Start monitoring reachability using `SCNetworkReachability`, updating the +// preferred Envoy network cluster on changes. +// This is typically called by `EnvoyEngine` automatically on startup. +- (void)startReachability; + +// Start monitoring reachability using `NWPathMonitor`, updating the +// preferred Envoy network cluster on changes. +// This is typically called by `EnvoyEngine` automatically on startup. +- (void)startPathMonitor; + +@end + +NS_ASSUME_NONNULL_END diff --git a/mobile/library/objective-c/EnvoyStringAccessor.h b/mobile/library/objective-c/EnvoyStringAccessor.h new file mode 100644 index 000000000000..117c85dc8583 --- /dev/null +++ b/mobile/library/objective-c/EnvoyStringAccessor.h @@ -0,0 +1,13 @@ +#import + +NS_ASSUME_NONNULL_BEGIN + +@interface EnvoyStringAccessor : NSObject + +@property (nonatomic, copy) NSString * (^getEnvoyString)(); + +- (instancetype)initWithBlock:(NSString * (^)())block; + +@end + +NS_ASSUME_NONNULL_END From 0ce6cf5fc3b54185b068ac0b6ec2dd5e461fc3cb Mon Sep 17 00:00:00 2001 From: Christoph Pakulski Date: Wed, 7 Dec 2022 16:39:39 -0500 Subject: [PATCH 21/23] postgres: support for upstream SSL (#23990) Postgres filter can negotiate upstream SSL connection with Postgres server and enable upstream encryption. Upon receiving the initial postgres request, the filter buffers the received packet (without sending it) and sends to the upstream server a request to establish SSL connection. When the server agrees, the postgres filter enables upstream STARTTLS transport socket and sends the previously buffered initial packet. From now on, the connection to upstream is encrypted and the filter can read the postgres payloads in clear-text. If the server does not agree for SSL or converting STARTTLS transport socket to secure mode fails, depending on the configuration, the filter may continue in clear-text or may tear down the connection. Risk Level: Low Testing: unit, integration and manual tests. Docs Changes: yes. Release Notes: yes Platform Specific Features: No Fixes #19527 Signed-off-by: Christoph Pakulski --- .../v3alpha/postgres_proxy.proto | 18 + changelogs/current.yaml | 3 + .../filters/network/source/BUILD | 1 + .../filters/network/source/config.cc | 1 + .../network/source/postgres_decoder.cc | 58 ++- .../filters/network/source/postgres_decoder.h | 17 +- .../filters/network/source/postgres_filter.cc | 56 ++- .../filters/network/source/postgres_filter.h | 11 + .../postgres_proxy/filters/network/test/BUILD | 11 +- .../network/test/postgres_decoder_test.cc | 73 +++- .../network/test/postgres_filter_test.cc | 39 +- .../network/test/postgres_integration_test.cc | 369 ++++++++++++++++-- .../test/postgres_integration_test.proto | 6 + ...aml => postgres_test_config.yaml-template} | 11 +- .../network/test/postgres_test_utils.cc | 22 ++ .../network/test/postgres_test_utils.h | 1 + .../network_filters/postgres_proxy_filter.rst | 2 + 17 files changed, 631 insertions(+), 68 deletions(-) create mode 100644 contrib/postgres_proxy/filters/network/test/postgres_integration_test.proto rename contrib/postgres_proxy/filters/network/test/{postgres_test_config.yaml => postgres_test_config.yaml-template} (82%) diff --git a/api/contrib/envoy/extensions/filters/network/postgres_proxy/v3alpha/postgres_proxy.proto b/api/contrib/envoy/extensions/filters/network/postgres_proxy/v3alpha/postgres_proxy.proto index 114e97ca7e48..c7be9af020f6 100644 --- a/api/contrib/envoy/extensions/filters/network/postgres_proxy/v3alpha/postgres_proxy.proto +++ b/api/contrib/envoy/extensions/filters/network/postgres_proxy/v3alpha/postgres_proxy.proto @@ -20,6 +20,16 @@ option (udpa.annotations.file_status).package_version_status = ACTIVE; // [#extension: envoy.filters.network.postgres_proxy] message PostgresProxy { + // Upstream SSL operational modes. + enum SSLMode { + // Do not encrypt upstream connection to the server. + DISABLE = 0; + + // Establish upstream SSL connection to the server. If the server does not + // accept the request for SSL connection, the session is terminated. + REQUIRE = 1; + } + // The human readable prefix to use when emitting :ref:`statistics // `. string stat_prefix = 1 [(validate.rules).string = {min_len: 1}]; @@ -39,4 +49,12 @@ message PostgresProxy { // Refer to official documentation for details // `SSL Session Encryption Message Flow `_. bool terminate_ssl = 3; + + // Controls whether to establish upstream SSL connection to the server. + // Envoy will try to establish upstream SSL connection to the server only when + // Postgres filter is able to read Postgres payload in clear-text. It happens when + // a client established a clear-text connection to Envoy or when a client established + // SSL connection to Envoy and Postgres filter is configured to terminate SSL. + // Defaults to SSL_DISABLE. + SSLMode upstream_ssl = 4; } diff --git a/changelogs/current.yaml b/changelogs/current.yaml index e753dc623f50..0e9a43761abc 100644 --- a/changelogs/current.yaml +++ b/changelogs/current.yaml @@ -194,6 +194,9 @@ new_features: - area: matching change: | support filter chain selection based on the dynamic metadata and the filter state using :ref:`formatter actions `. +- area: postgres + change: | + added support for upstream SSL. - area: redis change: | extended :ref:`cluster support ` by adding a :ref:`dns_cache_config ` option that can be used to resolve hostnames returned by MOVED/ASK responses. diff --git a/contrib/postgres_proxy/filters/network/source/BUILD b/contrib/postgres_proxy/filters/network/source/BUILD index 4fd7bb97717a..c4b5dce9470d 100644 --- a/contrib/postgres_proxy/filters/network/source/BUILD +++ b/contrib/postgres_proxy/filters/network/source/BUILD @@ -37,6 +37,7 @@ envoy_cc_library( "//source/common/buffer:buffer_lib", "//source/common/network:filter_lib", "//source/extensions/filters/network:well_known_names", + "@envoy_api//contrib/envoy/extensions/filters/network/postgres_proxy/v3alpha:pkg_cc_proto", ], ) diff --git a/contrib/postgres_proxy/filters/network/source/config.cc b/contrib/postgres_proxy/filters/network/source/config.cc index ff19cbe37c0b..e663c02f1cce 100644 --- a/contrib/postgres_proxy/filters/network/source/config.cc +++ b/contrib/postgres_proxy/filters/network/source/config.cc @@ -19,6 +19,7 @@ NetworkFilters::PostgresProxy::PostgresConfigFactory::createFilterFactoryFromPro config_options.enable_sql_parsing_ = PROTOBUF_GET_WRAPPED_OR_DEFAULT(proto_config, enable_sql_parsing, true); config_options.terminate_ssl_ = proto_config.terminate_ssl(); + config_options.upstream_ssl_ = proto_config.upstream_ssl(); PostgresFilterConfigSharedPtr filter_config( std::make_shared(config_options, context.scope())); diff --git a/contrib/postgres_proxy/filters/network/source/postgres_decoder.cc b/contrib/postgres_proxy/filters/network/source/postgres_decoder.cc index b54935cf36f3..79951a720093 100644 --- a/contrib/postgres_proxy/filters/network/source/postgres_decoder.cc +++ b/contrib/postgres_proxy/filters/network/source/postgres_decoder.cc @@ -191,6 +191,8 @@ Decoder::Result DecoderImpl::onData(Buffer::Instance& data, bool frontend) { return onDataIgnore(data, frontend); case State::InSyncState: return onDataInSync(data, frontend); + case State::NegotiatingUpstreamSSL: + return onDataInNegotiating(data, frontend); default: PANIC("not implemented"); } @@ -240,7 +242,6 @@ Decoder::Result DecoderImpl::onDataInit(Buffer::Instance& data, bool) { Decoder::Result result = Decoder::Result::ReadyForNext; uint32_t code = data.peekBEInt(4); - data.drain(4); // Startup message with 1234 in the most significant 16 bits // indicate request to encrypt. if (code >= 0x04d20000) { @@ -268,8 +269,24 @@ Decoder::Result DecoderImpl::onDataInit(Buffer::Instance& data, bool) { } } else { ENVOY_LOG(debug, "Detected version {}.{} of Postgres", code >> 16, code & 0x0000FFFF); - state_ = State::InSyncState; + if (callbacks_->shouldEncryptUpstream()) { + // Copy the received initial request. + temp_storage_.add(data.linearize(data.length()), data.length()); + // Send SSL request to upstream. + Buffer::OwnedImpl ssl_request; + uint32_t len = 8; + ssl_request.writeBEInt(len); + uint32_t ssl_code = 0x04d2162f; + ssl_request.writeBEInt(ssl_code); + + callbacks_->sendUpstream(ssl_request); + result = Decoder::Result::Stopped; + state_ = State::NegotiatingUpstreamSSL; + } else { + state_ = State::InSyncState; + } } + data.drain(4); processMessageBody(data, FRONTEND, message_len_ - 4, first_, msgParser); data.drain(message_len_); @@ -412,6 +429,43 @@ void DecoderImpl::decodeBackendStatements() { } } +Decoder::Result DecoderImpl::onDataInNegotiating(Buffer::Instance& data, bool frontend) { + if (frontend) { + // No data from downstream is allowed when negotiating upstream SSL + // with the server. + data.drain(data.length()); + state_ = State::OutOfSyncState; + return Decoder::Result::ReadyForNext; + } + + // This should be reply from the server indicating if it accepted + // request to use SSL. It is only one character long packet, where + // 'S' means use SSL, 'E' means do not use. + + // Indicate to the filter, the response and give the initial + // packet temporarily buffered to be sent upstream. + bool upstreamSSL = false; + state_ = State::InitState; + if (data.length() == 1) { + const char c = data.peekInt(0); + if (c == 'S') { + upstreamSSL = true; + } else { + if (c != 'E') { + state_ = State::OutOfSyncState; + } + } + } else { + state_ = State::OutOfSyncState; + } + + data.drain(data.length()); + + callbacks_->encryptUpstream(upstreamSSL, temp_storage_); + + return Decoder::Result::Stopped; +} + // Method is called when X (Terminate) message // is encountered by the decoder. void DecoderImpl::decodeFrontendTerminate() { diff --git a/contrib/postgres_proxy/filters/network/source/postgres_decoder.h b/contrib/postgres_proxy/filters/network/source/postgres_decoder.h index 2d4dffcb41e4..d33c99697504 100644 --- a/contrib/postgres_proxy/filters/network/source/postgres_decoder.h +++ b/contrib/postgres_proxy/filters/network/source/postgres_decoder.h @@ -44,6 +44,9 @@ class DecoderCallbacks { virtual void processQuery(const std::string&) PURE; virtual bool onSSLRequest() PURE; + virtual bool shouldEncryptUpstream() const PURE; + virtual void sendUpstream(Buffer::Instance&) PURE; + virtual void encryptUpstream(bool, Buffer::Instance&) PURE; }; // Postgres message decoder. @@ -88,7 +91,13 @@ class DecoderImpl : public Decoder, Logger::Loggable { bool encrypted() const { return encrypted_; } - enum class State { InitState, InSyncState, OutOfSyncState, EncryptedState }; + enum class State { + InitState, + InSyncState, + OutOfSyncState, + EncryptedState, + NegotiatingUpstreamSSL + }; State state() const { return state_; } void state(State state) { state_ = state; } @@ -98,6 +107,7 @@ class DecoderImpl : public Decoder, Logger::Loggable { Result onDataInit(Buffer::Instance& data, bool frontend); Result onDataInSync(Buffer::Instance& data, bool frontend); Result onDataIgnore(Buffer::Instance& data, bool frontend); + Result onDataInNegotiating(Buffer::Instance& data, bool frontend); // MsgAction defines the Decoder's method which will be invoked // when a specific message has been decoded. @@ -188,6 +198,11 @@ class DecoderImpl : public Decoder, Logger::Loggable { MsgParserDict BE_errors_; MsgParserDict BE_notices_; + // Buffer used to temporarily store a downstream postgres packet + // while sending other packets. Currently used only when negotiating + // upstream SSL. + Buffer::OwnedImpl temp_storage_; + // MAX_STARTUP_PACKET_LENGTH is defined in Postgres source code // as maximum size of initial packet. // https://github.com/postgres/postgres/search?q=MAX_STARTUP_PACKET_LENGTH&type=code diff --git a/contrib/postgres_proxy/filters/network/source/postgres_filter.cc b/contrib/postgres_proxy/filters/network/source/postgres_filter.cc index 6255c2ba5917..2059ebaceeaa 100644 --- a/contrib/postgres_proxy/filters/network/source/postgres_filter.cc +++ b/contrib/postgres_proxy/filters/network/source/postgres_filter.cc @@ -3,6 +3,7 @@ #include "envoy/buffer/buffer.h" #include "envoy/network/connection.h" +#include "source/common/common/assert.h" #include "source/extensions/filters/network/well_known_names.h" #include "contrib/postgres_proxy/filters/network/source/postgres_decoder.h" @@ -15,8 +16,8 @@ namespace PostgresProxy { PostgresFilterConfig::PostgresFilterConfig(const PostgresFilterConfigOptions& config_options, Stats::Scope& scope) : enable_sql_parsing_(config_options.enable_sql_parsing_), - terminate_ssl_(config_options.terminate_ssl_), scope_{scope}, - stats_{generateStats(config_options.stats_prefix_, scope)} {} + terminate_ssl_(config_options.terminate_ssl_), upstream_ssl_(config_options.upstream_ssl_), + scope_{scope}, stats_{generateStats(config_options.stats_prefix_, scope)} {} PostgresFilter::PostgresFilter(PostgresFilterConfigSharedPtr config) : config_{config} { if (!decoder_) { @@ -50,7 +51,12 @@ Network::FilterStatus PostgresFilter::onWrite(Buffer::Instance& data, bool) { // Backend Buffer backend_buffer_.add(data); - return doDecode(backend_buffer_, false); + Network::FilterStatus result = doDecode(backend_buffer_, false); + if (result == Network::FilterStatus::StopIteration) { + ASSERT(backend_buffer_.length() == 0); + data.drain(data.length()); + } + return result; } DecoderPtr PostgresFilter::createDecoder(DecoderCallbacks* callbacks) { @@ -205,8 +211,9 @@ bool PostgresFilter::onSSLRequest() { // Wait until 'S' has been transmitted. if (bytes >= 1) { if (!read_callbacks_->connection().startSecureTransport()) { - ENVOY_CONN_LOG(info, "postgres_proxy: cannot enable secure transport. Check configuration.", - read_callbacks_->connection()); + ENVOY_CONN_LOG( + info, "postgres_proxy: cannot enable downstream secure transport. Check configuration.", + read_callbacks_->connection()); read_callbacks_->connection().close(Network::ConnectionCloseType::NoFlush); } else { // Unsubscribe the callback. @@ -227,6 +234,45 @@ bool PostgresFilter::onSSLRequest() { return false; } +bool PostgresFilter::shouldEncryptUpstream() const { + return (config_->upstream_ssl_ == + envoy::extensions::filters::network::postgres_proxy::v3alpha::PostgresProxy::REQUIRE); +} + +void PostgresFilter::sendUpstream(Buffer::Instance& data) { + read_callbacks_->injectReadDataToFilterChain(data, false); +} + +void PostgresFilter::encryptUpstream(bool upstream_agreed, Buffer::Instance& data) { + RELEASE_ASSERT( + config_->upstream_ssl_ != + envoy::extensions::filters::network::postgres_proxy::v3alpha::PostgresProxy::DISABLE, + "encryptUpstream should not be called when upstream SSL is disabled."); + if (!upstream_agreed) { + ENVOY_CONN_LOG(info, + "postgres_proxy: upstream server rejected request to establish SSL connection. " + "Terminating.", + read_callbacks_->connection()); + read_callbacks_->connection().close(Network::ConnectionCloseType::NoFlush); + + config_->stats_.sessions_upstream_ssl_failed_.inc(); + } else { + // Try to switch upstream connection to use a secure channel. + if (read_callbacks_->startUpstreamSecureTransport()) { + config_->stats_.sessions_upstream_ssl_success_.inc(); + read_callbacks_->injectReadDataToFilterChain(data, false); + ENVOY_CONN_LOG(trace, "postgres_proxy: upstream SSL enabled.", read_callbacks_->connection()); + } else { + ENVOY_CONN_LOG(info, + "postgres_proxy: cannot enable upstream secure transport. Check " + "configuration. Terminating.", + read_callbacks_->connection()); + read_callbacks_->connection().close(Network::ConnectionCloseType::NoFlush); + config_->stats_.sessions_upstream_ssl_failed_.inc(); + } + } +} + Network::FilterStatus PostgresFilter::doDecode(Buffer::Instance& data, bool frontend) { // Keep processing data until buffer is empty or decoder says // that it cannot process data in the buffer. diff --git a/contrib/postgres_proxy/filters/network/source/postgres_filter.h b/contrib/postgres_proxy/filters/network/source/postgres_filter.h index dbc63686e870..53f65640eabe 100644 --- a/contrib/postgres_proxy/filters/network/source/postgres_filter.h +++ b/contrib/postgres_proxy/filters/network/source/postgres_filter.h @@ -8,6 +8,7 @@ #include "source/common/buffer/buffer_impl.h" #include "source/common/common/logger.h" +#include "contrib/envoy/extensions/filters/network/postgres_proxy/v3alpha/postgres_proxy.pb.h" #include "contrib/postgres_proxy/filters/network/source/postgres_decoder.h" namespace Envoy { @@ -32,6 +33,8 @@ namespace PostgresProxy { COUNTER(sessions_encrypted) \ COUNTER(sessions_terminated_ssl) \ COUNTER(sessions_unencrypted) \ + COUNTER(sessions_upstream_ssl_success) \ + COUNTER(sessions_upstream_ssl_failed) \ COUNTER(statements) \ COUNTER(statements_insert) \ COUNTER(statements_delete) \ @@ -67,11 +70,16 @@ class PostgresFilterConfig { std::string stats_prefix_; bool enable_sql_parsing_; bool terminate_ssl_; + envoy::extensions::filters::network::postgres_proxy::v3alpha::PostgresProxy::SSLMode + upstream_ssl_; }; PostgresFilterConfig(const PostgresFilterConfigOptions& config_options, Stats::Scope& scope); bool enable_sql_parsing_{true}; bool terminate_ssl_{false}; + envoy::extensions::filters::network::postgres_proxy::v3alpha::PostgresProxy::SSLMode + upstream_ssl_{ + envoy::extensions::filters::network::postgres_proxy::v3alpha::PostgresProxy::DISABLE}; Stats::Scope& scope_; PostgresProxyStats stats_; @@ -112,6 +120,9 @@ class PostgresFilter : public Network::Filter, void incTransactionsRollback() override; void processQuery(const std::string&) override; bool onSSLRequest() override; + bool shouldEncryptUpstream() const override; + void sendUpstream(Buffer::Instance&) override; + void encryptUpstream(bool, Buffer::Instance&) override; Network::FilterStatus doDecode(Buffer::Instance& data, bool); DecoderPtr createDecoder(DecoderCallbacks* callbacks); diff --git a/contrib/postgres_proxy/filters/network/test/BUILD b/contrib/postgres_proxy/filters/network/test/BUILD index 496f8ec416e5..fe10ba4ad970 100644 --- a/contrib/postgres_proxy/filters/network/test/BUILD +++ b/contrib/postgres_proxy/filters/network/test/BUILD @@ -3,6 +3,7 @@ load( "envoy_cc_test", "envoy_cc_test_library", "envoy_contrib_package", + "envoy_proto_library", ) licenses(["notice"]) # Apache 2 @@ -54,22 +55,30 @@ envoy_cc_test( ], ) +envoy_proto_library( + name = "postgres_integration_proto", + srcs = [":postgres_integration_test.proto"], +) + envoy_cc_test( name = "postgres_integration_test", srcs = [ "postgres_integration_test.cc", ], data = [ - "postgres_test_config.yaml", + "postgres_test_config.yaml-template", "//test/config/integration/certs", ], deps = [ + ":postgres_integration_proto_cc_proto", + ":postgres_test_utils_lib", "//contrib/postgres_proxy/filters/network/source:config", "//contrib/postgres_proxy/filters/network/source:filter", "//source/common/tcp_proxy", "//source/extensions/filters/network/tcp_proxy:config", "//source/extensions/transport_sockets/starttls:config", "//test/integration:integration_lib", + "//test/test_common:registry_lib", "@envoy_api//contrib/envoy/extensions/filters/network/postgres_proxy/v3alpha:pkg_cc_proto", ], ) diff --git a/contrib/postgres_proxy/filters/network/test/postgres_decoder_test.cc b/contrib/postgres_proxy/filters/network/test/postgres_decoder_test.cc index dc273b8e55c6..5700839e915b 100644 --- a/contrib/postgres_proxy/filters/network/test/postgres_decoder_test.cc +++ b/contrib/postgres_proxy/filters/network/test/postgres_decoder_test.cc @@ -24,6 +24,9 @@ class DecoderCallbacksMock : public DecoderCallbacks { MOCK_METHOD(void, incErrors, (ErrorType), (override)); MOCK_METHOD(void, processQuery, (const std::string&), (override)); MOCK_METHOD(bool, onSSLRequest, (), (override)); + MOCK_METHOD(bool, shouldEncryptUpstream, (), (const)); + MOCK_METHOD(void, sendUpstream, (Buffer::Instance&)); + MOCK_METHOD(void, encryptUpstream, (bool, Buffer::Instance&)); }; // Define fixture class with decoder and mock callbacks. @@ -121,25 +124,8 @@ TEST_F(PostgresProxyDecoderTest, StartupMessage) { TEST_F(PostgresProxyDecoderTest, StartupMessageNoAttr) { decoder_->state(DecoderImpl::State::InitState); - buf_[0] = '\0'; - // Startup message has the following structure: - // Length (4 bytes) - payload and length field - // version (4 bytes) - // Attributes: key/value pairs separated by '\0' - data_.writeBEInt(37); - // Add version code - data_.writeBEInt(0x00030000); - // user-postgres key-pair - data_.add("user"); // 4 bytes - data_.add(buf_, 1); - data_.add("postgres"); // 8 bytes - data_.add(buf_, 1); - // database-test-db key-pair - // Some other attribute - data_.add("attribute"); // 9 bytes - data_.add(buf_, 1); - data_.add("blah"); // 4 bytes - data_.add(buf_, 1); + createInitialPostgresRequest(data_); + ASSERT_THAT(decoder_->onData(data_, true), Decoder::Result::ReadyForNext); ASSERT_THAT(decoder_->state(), DecoderImpl::State::InSyncState); ASSERT_THAT(data_.length(), 0); @@ -635,6 +621,55 @@ TEST_F(PostgresProxyDecoderTest, TerminateSSL) { ASSERT_FALSE(decoder_->encrypted()); } +class PostgresProxyUpstreamSSLTest + : public PostgresProxyDecoderTestBase, + public ::testing::TestWithParam> {}; + +TEST_F(PostgresProxyDecoderTest, UpstreamSSLDisabled) { + // Set decoder to wait for initial message. + decoder_->state(DecoderImpl::State::InitState); + + createInitialPostgresRequest(data_); + + EXPECT_CALL(callbacks_, shouldEncryptUpstream).WillOnce(testing::Return(false)); + EXPECT_CALL(callbacks_, encryptUpstream(testing::_, testing::_)).Times(0); + ASSERT_THAT(decoder_->onData(data_, true), Decoder::Result::ReadyForNext); + ASSERT_THAT(decoder_->state(), DecoderImpl::State::InSyncState); +} + +TEST_P(PostgresProxyUpstreamSSLTest, UpstreamSSLEnabled) { + // Set decoder to wait for initial message. + decoder_->state(DecoderImpl::State::InitState); + + // Create initial message + createInitialPostgresRequest(data_); + + EXPECT_CALL(callbacks_, shouldEncryptUpstream).WillOnce(testing::Return(true)); + EXPECT_CALL(callbacks_, sendUpstream); + ASSERT_THAT(decoder_->onData(data_, true), Decoder::Result::Stopped); + ASSERT_THAT(decoder_->state(), DecoderImpl::State::NegotiatingUpstreamSSL); + + // Simulate various responses from the upstream server. + // Only "S" and "E" are valid responses. + data_.add(std::get<0>(GetParam())); + + EXPECT_CALL(callbacks_, encryptUpstream(std::get<1>(GetParam()), testing::_)); + // The reply from upstream should not be delivered to the client. + ASSERT_THAT(decoder_->onData(data_, false), Decoder::Result::Stopped); + ASSERT_THAT(decoder_->state(), std::get<2>(GetParam())); + ASSERT_TRUE(data_.length() == 0); +} + +INSTANTIATE_TEST_SUITE_P(BackendEncryptedMessagesTests, PostgresProxyUpstreamSSLTest, + ::testing::Values( + // Correct response from the server (encrypt). + std::make_tuple("S", true, DecoderImpl::State::InitState), + // Correct response from the server (do not encrypt). + std::make_tuple("E", false, DecoderImpl::State::InitState), + // Incorrect response from the server. Move to out-of-sync state. + std::make_tuple("W", false, DecoderImpl::State::OutOfSyncState), + std::make_tuple("WRONG", false, DecoderImpl::State::OutOfSyncState))); + class FakeBuffer : public Buffer::Instance { public: MOCK_METHOD(void, addDrainTracker, (std::function), (override)); diff --git a/contrib/postgres_proxy/filters/network/test/postgres_filter_test.cc b/contrib/postgres_proxy/filters/network/test/postgres_filter_test.cc index 3a081ce55b68..06f478e05d92 100644 --- a/contrib/postgres_proxy/filters/network/test/postgres_filter_test.cc +++ b/contrib/postgres_proxy/filters/network/test/postgres_filter_test.cc @@ -33,7 +33,10 @@ class PostgresFilterTest public: PostgresFilterTest() { - PostgresFilterConfig::PostgresFilterConfigOptions config_options{stat_prefix_, true, false}; + PostgresFilterConfig::PostgresFilterConfigOptions config_options{ + stat_prefix_, true, false, + envoy::extensions::filters::network::postgres_proxy::v3alpha:: + PostgresProxy_SSLMode_DISABLE}; config_ = std::make_shared(config_options, scope_); filter_ = std::make_unique(config_); @@ -393,6 +396,40 @@ TEST_F(PostgresFilterTest, TerminateSSL) { ASSERT_THAT(filter_->getStats().sessions_unencrypted_.value(), 0); } +TEST_F(PostgresFilterTest, UpstreamSSL) { + EXPECT_CALL(filter_callbacks_, connection()).WillRepeatedly(ReturnRef(connection_)); + + // Configure upstream SSL to be disabled. encryptUpstream must not be called. + filter_->getConfig()->upstream_ssl_ = + envoy::extensions::filters::network::postgres_proxy::v3alpha::PostgresProxy::DISABLE; + ASSERT_FALSE(filter_->shouldEncryptUpstream()); + ASSERT_DEATH(filter_->encryptUpstream(true, data_), ".*"); + ASSERT_DEATH(filter_->encryptUpstream(false, data_), ".*"); + + // Configure upstream SSL to be required. If upstream server does not agree for SSL or + // converting upstream transport socket to secure mode fails, the filter should bump + // proper stats and close the connection to downstream client. + filter_->getConfig()->upstream_ssl_ = + envoy::extensions::filters::network::postgres_proxy::v3alpha::PostgresProxy::REQUIRE; + ASSERT_TRUE(filter_->shouldEncryptUpstream()); + // Simulate that upstream server agreed for SSL and conversion of upstream Transport socket was + // successful. + EXPECT_CALL(filter_callbacks_, startUpstreamSecureTransport()).WillOnce(testing::Return(true)); + filter_->encryptUpstream(true, data_); + ASSERT_EQ(1, filter_->getStats().sessions_upstream_ssl_success_.value()); + // Simulate that upstream server agreed for SSL but conversion of upstream Transport socket + // failed. + EXPECT_CALL(filter_callbacks_, startUpstreamSecureTransport()).WillOnce(testing::Return(false)); + filter_->encryptUpstream(true, data_); + ASSERT_EQ(1, filter_->getStats().sessions_upstream_ssl_failed_.value()); + // Simulate that upstream server does not agree for SSL. Filter should close the connection to + // downstream client. + EXPECT_CALL(filter_callbacks_, startUpstreamSecureTransport()).Times(0); + EXPECT_CALL(connection_, close(_)); + filter_->encryptUpstream(false, data_); + ASSERT_EQ(2, filter_->getStats().sessions_upstream_ssl_failed_.value()); +} + } // namespace PostgresProxy } // namespace NetworkFilters } // namespace Extensions diff --git a/contrib/postgres_proxy/filters/network/test/postgres_integration_test.cc b/contrib/postgres_proxy/filters/network/test/postgres_integration_test.cc index 58d1ddc77937..d06bb6d267a4 100644 --- a/contrib/postgres_proxy/filters/network/test/postgres_integration_test.cc +++ b/contrib/postgres_proxy/filters/network/test/postgres_integration_test.cc @@ -1,11 +1,20 @@ +#include "source/common/network/connection_impl.h" +#include "source/extensions/filters/network/common/factory_base.h" +#include "source/extensions/transport_sockets/tls/context_config_impl.h" +#include "source/extensions/transport_sockets/tls/ssl_socket.h" + #include "test/integration/fake_upstream.h" #include "test/integration/integration.h" #include "test/integration/utility.h" #include "test/mocks/network/mocks.h" #include "test/test_common/network_utility.h" +#include "test/test_common/registry.h" #include "contrib/envoy/extensions/filters/network/postgres_proxy/v3alpha/postgres_proxy.pb.h" #include "contrib/envoy/extensions/filters/network/postgres_proxy/v3alpha/postgres_proxy.pb.validate.h" +#include "contrib/postgres_proxy/filters/network/test/postgres_integration_test.pb.h" +#include "contrib/postgres_proxy/filters/network/test/postgres_integration_test.pb.validate.h" +#include "contrib/postgres_proxy/filters/network/test/postgres_test_utils.h" #include "gmock/gmock.h" #include "gtest/gtest.h" @@ -16,53 +25,49 @@ namespace PostgresProxy { class PostgresBaseIntegrationTest : public testing::TestWithParam, public BaseIntegrationTest { +public: + // Tuple to store upstream and downstream startTLS configuration. + // The first string contains string to enable/disable SSL. + // The second string contains transport socket configuration. + using SSLConfig = std::tuple; - std::string postgresConfig(bool terminate_ssl, bool add_start_tls_transport_socket) { + std::string postgresConfig(SSLConfig downstream_ssl_config, SSLConfig upstream_ssl_config, + std::string additional_filters) { std::string main_config = fmt::format( TestEnvironment::readFileToStringForTest(TestEnvironment::runfilesPath( - "contrib/postgres_proxy/filters/network/test/postgres_test_config.yaml")), + "contrib/postgres_proxy/filters/network/test/postgres_test_config.yaml-template")), Platform::null_device_path, Network::Test::getLoopbackAddressString(GetParam()), Network::Test::getLoopbackAddressString(GetParam()), - Network::Test::getAnyAddressString(GetParam()), terminate_ssl ? "true" : "false"); - - if (add_start_tls_transport_socket) { - main_config += - fmt::format(R"EOF( - transport_socket: - name: "starttls" - typed_config: - "@type": type.googleapis.com/envoy.extensions.transport_sockets.starttls.v3.StartTlsConfig - cleartext_socket_config: - tls_socket_config: - common_tls_context: - tls_certificates: - certificate_chain: - filename: {} - private_key: - filename: {} - )EOF", - TestEnvironment::runfilesPath("test/config/integration/certs/servercert.pem"), - TestEnvironment::runfilesPath("test/config/integration/certs/serverkey.pem")); - } + std::get<1>(upstream_ssl_config), // upstream SSL transport socket + Network::Test::getAnyAddressString(GetParam()), + std::get<0>(downstream_ssl_config), // downstream SSL termination + std::get<0>(upstream_ssl_config), // upstream_SSL option + additional_filters, // additional filters to insert after postgres + std::get<1>(downstream_ssl_config)); // downstream SSL transport socket return main_config; } -public: - PostgresBaseIntegrationTest(bool terminate_ssl, bool add_starttls_transport_socket) - : BaseIntegrationTest(GetParam(), - postgresConfig(terminate_ssl, add_starttls_transport_socket)) { + PostgresBaseIntegrationTest(SSLConfig downstream_ssl_config, SSLConfig upstream_ssl_config, + std::string additional_filters = "") + : BaseIntegrationTest(GetParam(), postgresConfig(downstream_ssl_config, upstream_ssl_config, + additional_filters)) { skip_tag_extraction_rule_check_ = true; }; void SetUp() override { BaseIntegrationTest::initialize(); } + + static constexpr absl::string_view empty_config_string_{""}; + static constexpr SSLConfig NoUpstreamSSL{empty_config_string_, empty_config_string_}; + static constexpr SSLConfig NoDownstreamSSL{empty_config_string_, empty_config_string_}; + FakeRawConnectionPtr fake_upstream_connection_; }; // Base class for tests with `terminate_ssl` disabled and without // `starttls` transport socket. class BasicPostgresIntegrationTest : public PostgresBaseIntegrationTest { public: - BasicPostgresIntegrationTest() : PostgresBaseIntegrationTest(false, false) {} + BasicPostgresIntegrationTest() : PostgresBaseIntegrationTest(NoDownstreamSSL, NoUpstreamSSL) {} }; // Test that the filter is properly chained and reacts to successful login @@ -107,19 +112,38 @@ TEST_P(BasicPostgresIntegrationTest, Login) { // Make sure that the successful login bumped up the number of sessions. test_server_->waitForCounterEq("postgres.postgres_stats.sessions", 1); } - INSTANTIATE_TEST_SUITE_P(IpVersions, BasicPostgresIntegrationTest, testing::ValuesIn(TestEnvironment::getIpVersionsForTest())); // Base class for tests with `terminate_ssl` enabled and `starttls` transport socket added. -class SSLPostgresIntegrationTest : public PostgresBaseIntegrationTest { +class DownstreamSSLPostgresIntegrationTest : public PostgresBaseIntegrationTest { public: - SSLPostgresIntegrationTest() : PostgresBaseIntegrationTest(true, true) {} + DownstreamSSLPostgresIntegrationTest() + : PostgresBaseIntegrationTest( + std::make_tuple( + "terminate_ssl: true", + fmt::format( + R"EOF(transport_socket: + name: "starttls" + typed_config: + "@type": type.googleapis.com/envoy.extensions.transport_sockets.starttls.v3.StartTlsConfig + cleartext_socket_config: + tls_socket_config: + common_tls_context: + tls_certificates: + certificate_chain: + filename: {} + private_key: + filename: {} + )EOF", + TestEnvironment::runfilesPath("test/config/integration/certs/servercert.pem"), + TestEnvironment::runfilesPath("test/config/integration/certs/serverkey.pem"))), + NoUpstreamSSL) {} }; // Test verifies that Postgres filter replies with correct code upon // receiving request to terminate SSL. -TEST_P(SSLPostgresIntegrationTest, TerminateSSL) { +TEST_P(DownstreamSSLPostgresIntegrationTest, TerminateSSL) { Buffer::OwnedImpl data; IntegrationTcpClientPtr tcp_client = makeTcpConnection(lookupPort("listener_0")); @@ -147,17 +171,19 @@ TEST_P(SSLPostgresIntegrationTest, TerminateSSL) { test_server_->waitForCounterEq("postgres.postgres_stats.sessions_terminated_ssl", 1); } -INSTANTIATE_TEST_SUITE_P(IpVersions, SSLPostgresIntegrationTest, +INSTANTIATE_TEST_SUITE_P(IpVersions, DownstreamSSLPostgresIntegrationTest, testing::ValuesIn(TestEnvironment::getIpVersionsForTest())); -class SSLWrongConfigPostgresIntegrationTest : public PostgresBaseIntegrationTest { +class DownstreamSSLWrongConfigPostgresIntegrationTest : public PostgresBaseIntegrationTest { public: - SSLWrongConfigPostgresIntegrationTest() : PostgresBaseIntegrationTest(true, false) {} + DownstreamSSLWrongConfigPostgresIntegrationTest() + // Enable SSL termination but do not configure downstream transport socket. + : PostgresBaseIntegrationTest(std::make_tuple("terminate_ssl: true", ""), NoUpstreamSSL) {} }; // Test verifies that Postgres filter closes connection when it is configured to // terminate SSL, but underlying transport socket does not allow for such operation. -TEST_P(SSLWrongConfigPostgresIntegrationTest, TerminateSSLNoStartTlsTransportSocket) { +TEST_P(DownstreamSSLWrongConfigPostgresIntegrationTest, TerminateSSLNoStartTlsTransportSocket) { Buffer::OwnedImpl data; IntegrationTcpClientPtr tcp_client = makeTcpConnection(lookupPort("listener_0")); @@ -183,11 +209,278 @@ TEST_P(SSLWrongConfigPostgresIntegrationTest, TerminateSSLNoStartTlsTransportSoc tcp_client->waitForDisconnect(); ASSERT_TRUE(fake_upstream_connection->waitForDisconnect()); - // Make sure that the successful login bumped up the number of sessions. test_server_->waitForCounterEq("postgres.postgres_stats.sessions_terminated_ssl", 0); } -INSTANTIATE_TEST_SUITE_P(IpVersions, SSLWrongConfigPostgresIntegrationTest, +INSTANTIATE_TEST_SUITE_P(IpVersions, DownstreamSSLWrongConfigPostgresIntegrationTest, + testing::ValuesIn(TestEnvironment::getIpVersionsForTest())); + +// Upstream SSL integration tests. +// Tests do not use the real postgres server and concentrate only on initial exchange. +// The initial packet +// sent by the downstream client, SSL request sent to fake upstream and SSL response sent back by +// fake client are valid postgres payloads, because they must be parsed by postgres filter. + +class UpstreamSSLBaseIntegrationTest : public PostgresBaseIntegrationTest { +public: + UpstreamSSLBaseIntegrationTest(SSLConfig upstream_ssl_config) + // Disable downstream SSL and attach synchronization filter. + : PostgresBaseIntegrationTest(NoDownstreamSSL, upstream_ssl_config, R"EOF( + - name: sync + typed_config: + "@type": type.googleapis.com/test.integration.postgres.SyncWriteFilterConfig +)EOF") {} + + // Helper synchronization filter which is injected between postgres filter and tcp proxy. + // Its goal is to eliminate race conditions and synchronize operations between fake upstream and + // postgres filter. + struct SyncWriteFilter : public Network::WriteFilter { + SyncWriteFilter(absl::Notification& proceed_sync, absl::Notification& recv_sync) + : proceed_sync_(proceed_sync), recv_sync_(recv_sync) {} + + Network::FilterStatus onWrite(Buffer::Instance& data, bool) override { + if (data.length() > 0) { + // Notify fake upstream that payload has been received. + recv_sync_.Notify(); + // Wait for signal to continue. This is to give fake upstream + // some time to create and attach TLS transport socket. + proceed_sync_.WaitForNotification(); + } + return Network::FilterStatus::Continue; + } + + void initializeWriteFilterCallbacks(Network::WriteFilterCallbacks& callbacks) override { + read_callbacks_ = &callbacks; + } + + Network::WriteFilterCallbacks* read_callbacks_{}; + // Synchronization object used to stop Envoy processing to allow fake upstream to + // create and attach TLS transport socket. + absl::Notification& proceed_sync_; + // Synchronization object used to notify fake upstream that a message sent + // by fake upstream was received by Envoy. + absl::Notification& recv_sync_; + }; + + // Config factory for sync helper filter. + class SyncWriteFilterConfigFactory : public Extensions::NetworkFilters::Common::FactoryBase< + test::integration::postgres::SyncWriteFilterConfig> { + public: + explicit SyncWriteFilterConfigFactory(const std::string& name, + Network::ConnectionCallbacks& /* upstream_callbacks*/) + : FactoryBase(name) {} + + Network::FilterFactoryCb + createFilterFactoryFromProtoTyped(const test::integration::postgres::SyncWriteFilterConfig&, + Server::Configuration::FactoryContext&) override { + return [&](Network::FilterManager& filter_manager) -> void { + filter_manager.addWriteFilter(std::make_shared(proceed_sync_, recv_sync_)); + }; + } + + std::string name() const override { return name_; } + + // See SyncWriteFilter for purpose and description of the following sync objects. + absl::Notification proceed_sync_, recv_sync_; + + private: + const std::string name_; + }; + + // Method prepares TLS context to be injected to fake upstream. + // Method creates and attaches TLS transport socket to fake upstream. + void enableTLSOnFakeUpstream() { + // Setup factory and context for tls transport socket. + // The tls transport socket will be inserted into fake_upstream when + // Envoy's upstream starttls transport socket is converted to secure mode. + std::unique_ptr tls_context_manager = + std::make_unique(timeSystem()); + + envoy::extensions::transport_sockets::tls::v3::DownstreamTlsContext downstream_tls_context; + + std::string yaml_plain = R"EOF( + common_tls_context: + validation_context: + trusted_ca: + filename: "{{ test_rundir }}/test/config/integration/certs/upstreamcacert.pem" + tls_certificates: + certificate_chain: + filename: "{{ test_rundir }}/test/config/integration/certs/upstreamcert.pem" + private_key: + filename: "{{ test_rundir }}/test/config/integration/certs/upstreamkey.pem" +)EOF"; + + TestUtility::loadFromYaml(TestEnvironment::substitute(yaml_plain), downstream_tls_context); + + NiceMock mock_factory_ctx; + ON_CALL(mock_factory_ctx, api()).WillByDefault(testing::ReturnRef(*api_)); + auto cfg = std::make_unique( + downstream_tls_context, mock_factory_ctx); + static auto* client_stats_store = new Stats::TestIsolatedStoreImpl(); + Network::DownstreamTransportSocketFactoryPtr tls_context = + Network::DownstreamTransportSocketFactoryPtr{ + new Extensions::TransportSockets::Tls::ServerSslSocketFactory( + std::move(cfg), *tls_context_manager, *client_stats_store, {})}; + + Network::TransportSocketPtr ts = tls_context->createDownstreamTransportSocket(); + // Synchronization object used to suspend execution + // until dispatcher completes transport socket conversion. + absl::Notification notification; + + // Execute transport socket conversion to TLS on the same thread where received data + // is dispatched. Otherwise conversion may collide with data processing. + fake_upstreams_[0]->dispatcher()->post([&]() { + auto connection = + dynamic_cast(&fake_upstream_connection_->connection()); + connection->transportSocket() = std::move(ts); + connection->transportSocket()->setTransportSocketCallbacks(*connection); + notification.Notify(); + }); + + // Wait until the transport socket conversion completes. + notification.WaitForNotification(); + } + + NiceMock upstream_callbacks_; + SyncWriteFilterConfigFactory config_factory_{"sync", upstream_callbacks_}; + Registry::InjectFactory + registered_config_factory_{config_factory_}; +}; + +// Base class for tests with disabled upstream SSL. It should behave exactly +// as without any upstream configuration specified and pass +// messages in clear-text. +class UpstreamSSLDisabledPostgresIntegrationTest : public UpstreamSSLBaseIntegrationTest { +public: + // Disable downstream SSL and upstream SSL. + UpstreamSSLDisabledPostgresIntegrationTest() + : UpstreamSSLBaseIntegrationTest(std::make_tuple("upstream_ssl: DISABLE", "")) {} +}; + +// Verify that postgres filter does not send any additional messages when +// upstream SSL is disabled. Fake upstream should receive only the initial +// postgres message. +TEST_P(UpstreamSSLDisabledPostgresIntegrationTest, BasicConnectivityTest) { + IntegrationTcpClientPtr tcp_client = makeTcpConnection(lookupPort("listener_0")); + ASSERT_TRUE(fake_upstreams_[0]->waitForRawConnection(fake_upstream_connection_)); + + // Send the startup message. + Buffer::OwnedImpl data; + std::string rcvd; + createInitialPostgresRequest(data); + ASSERT_TRUE(tcp_client->write(data.toString())); + // Make sure that upstream receives startup message in clear-text (no SSL negotiation takes + // place). + ASSERT_TRUE(fake_upstream_connection_->waitForData(data.toString().length(), &rcvd)); + data.drain(data.length()); + + tcp_client->close(); + ASSERT_TRUE(fake_upstream_connection_->waitForDisconnect()); + + test_server_->waitForCounterEq("postgres.postgres_stats.sessions_upstream_ssl_success", 0); + test_server_->waitForCounterEq("postgres.postgres_stats.sessions_upstream_ssl_failed", 0); +} + +INSTANTIATE_TEST_SUITE_P(IpVersions, UpstreamSSLDisabledPostgresIntegrationTest, + testing::ValuesIn(TestEnvironment::getIpVersionsForTest())); + +// Base class for parameterized tests with REQUIRE option for upstream SSL. +class UpstreamSSLRequirePostgresIntegrationTest : public UpstreamSSLBaseIntegrationTest { +public: + UpstreamSSLRequirePostgresIntegrationTest() + : UpstreamSSLBaseIntegrationTest(std::make_tuple("upstream_ssl: REQUIRE", + R"EOF(transport_socket: + name: "starttls" + typed_config: + "@type": type.googleapis.com/envoy.extensions.transport_sockets.starttls.v3.UpstreamStartTlsConfig + tls_socket_config: + common_tls_context: {} +)EOF")) {} +}; + +// Test verifies that postgres filter starts upstream SSL negotiation with +// fake upstream upon receiving initial postgres packet. When server agrees +// to use SSL, TLS transport socket is attached to fake upstream and +// fake upstream receives initial postgres packet over encrypted connection. +TEST_P(UpstreamSSLRequirePostgresIntegrationTest, ServerAgreesForSSLTest) { + IntegrationTcpClientPtr tcp_client = makeTcpConnection(lookupPort("listener_0")); + ASSERT_TRUE(fake_upstreams_[0]->waitForRawConnection(fake_upstream_connection_)); + + // Send the startup message. + Buffer::OwnedImpl data; + Buffer::OwnedImpl upstream_data; + std::string rcvd; + createInitialPostgresRequest(data); + ASSERT_TRUE(tcp_client->write(data.toString())); + // Postgres filter should buffer the original message and negotiate SSL upstream. + // The first 4 bytes should be length on the message (8 bytes). + // The next 4 bytes should be SSL code. + ASSERT_TRUE(fake_upstream_connection_->waitForData(8, &rcvd)); + upstream_data.add(rcvd); + ASSERT_EQ(8, upstream_data.peekBEInt(0)); + ASSERT_EQ(80877103, upstream_data.peekBEInt(4)); + upstream_data.drain(upstream_data.length()); + fake_upstream_connection_->clearData(); + + // Reply to Envoy with 'S' and attach TLS socket to upstream. + upstream_data.add("S"); + ASSERT_TRUE(fake_upstream_connection_->write(upstream_data.toString())); + + config_factory_.recv_sync_.WaitForNotification(); + enableTLSOnFakeUpstream(); + config_factory_.proceed_sync_.Notify(); + + ASSERT_TRUE(fake_upstream_connection_->waitForData(data.length(), &rcvd)); + // Make sure that upstream received initial postgres request, which + // triggered upstream SSL negotiation and TLS handshake. + ASSERT_EQ(data.toString(), rcvd); + + data.drain(data.length()); + + tcp_client->close(); + ASSERT_TRUE(fake_upstream_connection_->waitForDisconnect()); + + test_server_->waitForCounterEq("postgres.postgres_stats.sessions_upstream_ssl_success", 1); + test_server_->waitForCounterEq("postgres.postgres_stats.sessions_upstream_ssl_failed", 0); +} + +// Test verifies that postgres filter will not continue when upstream SSL +// is required and fake upstream does not agree for SSL. +TEST_P(UpstreamSSLRequirePostgresIntegrationTest, ServerDeniesSSLTest) { + IntegrationTcpClientPtr tcp_client = makeTcpConnection(lookupPort("listener_0")); + ASSERT_TRUE(fake_upstreams_[0]->waitForRawConnection(fake_upstream_connection_)); + + // Send the startup message. + Buffer::OwnedImpl data; + Buffer::OwnedImpl upstream_data; + std::string rcvd; + createInitialPostgresRequest(data); + ASSERT_TRUE(tcp_client->write(data.toString())); + // Postgres filter should buffer the original message and negotiate SSL upstream. + // The first 4 bytes should be length on the message (8 bytes). + // The next 4 bytes should be SSL code. + ASSERT_TRUE(fake_upstream_connection_->waitForData(8, &rcvd)); + upstream_data.add(rcvd); + ASSERT_EQ(8, upstream_data.peekBEInt(0)); + ASSERT_EQ(80877103, upstream_data.peekBEInt(4)); + upstream_data.drain(upstream_data.length()); + + // Reply to Envoy with 'E' (SSL not allowed). + upstream_data.add("E"); + ASSERT_TRUE(fake_upstream_connection_->write(upstream_data.toString())); + config_factory_.proceed_sync_.Notify(); + + data.drain(data.length()); + + // Connection to client should be closed. + tcp_client->waitForDisconnect(); + ASSERT_TRUE(fake_upstream_connection_->waitForDisconnect()); + + test_server_->waitForCounterEq("postgres.postgres_stats.sessions_upstream_ssl_success", 0); + test_server_->waitForCounterEq("postgres.postgres_stats.sessions_upstream_ssl_failed", 1); +} + +INSTANTIATE_TEST_SUITE_P(IpVersions, UpstreamSSLRequirePostgresIntegrationTest, testing::ValuesIn(TestEnvironment::getIpVersionsForTest())); } // namespace PostgresProxy diff --git a/contrib/postgres_proxy/filters/network/test/postgres_integration_test.proto b/contrib/postgres_proxy/filters/network/test/postgres_integration_test.proto new file mode 100644 index 000000000000..6ab902a04e6d --- /dev/null +++ b/contrib/postgres_proxy/filters/network/test/postgres_integration_test.proto @@ -0,0 +1,6 @@ +syntax = "proto3"; + +package test.integration.postgres; + +message SyncWriteFilterConfig { +} diff --git a/contrib/postgres_proxy/filters/network/test/postgres_test_config.yaml b/contrib/postgres_proxy/filters/network/test/postgres_test_config.yaml-template similarity index 82% rename from contrib/postgres_proxy/filters/network/test/postgres_test_config.yaml rename to contrib/postgres_proxy/filters/network/test/postgres_test_config.yaml-template index a60abfb26adc..e1063292769e 100644 --- a/contrib/postgres_proxy/filters/network/test/postgres_test_config.yaml +++ b/contrib/postgres_proxy/filters/network/test/postgres_test_config.yaml-template @@ -21,6 +21,8 @@ static_resources: socket_address: address: "{}" port_value: 0 + #downstream startTLS transport socket: + {} listeners: name: listener_0 address: @@ -33,9 +35,16 @@ static_resources: typed_config: "@type": type.googleapis.com/envoy.extensions.filters.network.postgres_proxy.v3alpha.PostgresProxy stat_prefix: postgres_stats - terminate_ssl: {} + # downstream SSL option: + {} + # upstream SSL option: + {} + # additional filters + {} - name: tcp typed_config: "@type": type.googleapis.com/envoy.extensions.filters.network.tcp_proxy.v3.TcpProxy stat_prefix: tcp_stats cluster: cluster_0 + # upstream startTLS transport socket + {} diff --git a/contrib/postgres_proxy/filters/network/test/postgres_test_utils.cc b/contrib/postgres_proxy/filters/network/test/postgres_test_utils.cc index 8793acf3b815..34347dd3fbf3 100644 --- a/contrib/postgres_proxy/filters/network/test/postgres_test_utils.cc +++ b/contrib/postgres_proxy/filters/network/test/postgres_test_utils.cc @@ -17,6 +17,28 @@ void createPostgresMsg(Buffer::Instance& data, std::string type, std::string pay } } +// Helper function to create an initial postgres message. +void createInitialPostgresRequest(Buffer::Instance& data) { + // Startup message has the following structure: + // Length (4 bytes) - payload and length field + // version (4 bytes) + // Attributes: key/value pairs separated by '\0' + data.writeBEInt(37); + // Add version code + data.writeBEInt(0x00030000); + // user-postgres key-pair + data.add("user"); // 4 bytes + data.writeBEInt(0); + data.add("postgres"); // 8 bytes + data.writeBEInt(0); + // database-test-db key-pair + // Some other attribute + data.add("attribute"); // 9 bytes + data.writeBEInt(0); + data.add("blah"); // 4 bytes + data.writeBEInt(0); +} + } // namespace PostgresProxy } // namespace NetworkFilters } // namespace Extensions diff --git a/contrib/postgres_proxy/filters/network/test/postgres_test_utils.h b/contrib/postgres_proxy/filters/network/test/postgres_test_utils.h index 0ee1614ec180..24f9121b1e4d 100644 --- a/contrib/postgres_proxy/filters/network/test/postgres_test_utils.h +++ b/contrib/postgres_proxy/filters/network/test/postgres_test_utils.h @@ -8,6 +8,7 @@ namespace NetworkFilters { namespace PostgresProxy { void createPostgresMsg(Buffer::Instance& data, std::string type, std::string payload = ""); +void createInitialPostgresRequest(Buffer::Instance& data); } // namespace PostgresProxy } // namespace NetworkFilters diff --git a/docs/root/configuration/listeners/network_filters/postgres_proxy_filter.rst b/docs/root/configuration/listeners/network_filters/postgres_proxy_filter.rst index 5e4834c17662..58f2e29a7616 100644 --- a/docs/root/configuration/listeners/network_filters/postgres_proxy_filter.rst +++ b/docs/root/configuration/listeners/network_filters/postgres_proxy_filter.rst @@ -75,6 +75,8 @@ Every configured Postgres proxy filter has statistics rooted at postgres. Date: Wed, 7 Dec 2022 15:44:07 -0800 Subject: [PATCH 22/23] build: fix compile error for mac (#24429) Signed-off-by: Greg Greenway --- test/common/buffer/buffer_fuzz.cc | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/test/common/buffer/buffer_fuzz.cc b/test/common/buffer/buffer_fuzz.cc index b98c62dd6d3b..899e95894bdf 100644 --- a/test/common/buffer/buffer_fuzz.cc +++ b/test/common/buffer/buffer_fuzz.cc @@ -119,7 +119,8 @@ class StringBuffer : public Buffer::Instance { uint64_t size_copied = 0; uint64_t num_slices_copied = 0; while (size_copied < length && num_slices_copied < num_slices) { - auto copy_length = std::min((length - size_copied), slices[num_slices_copied].len_); + auto copy_length = + std::min((length - size_copied), static_cast(slices[num_slices_copied].len_)); ::memcpy(slices[num_slices_copied].mem_, this->start(), copy_length); size_copied += copy_length; if (copy_length == slices[num_slices_copied].len_) { From 215a86a5b864228682f65023773f94816a2b3528 Mon Sep 17 00:00:00 2001 From: Yury Kats Date: Wed, 7 Dec 2022 19:46:32 -0500 Subject: [PATCH 23/23] Reduce Route memory utilization by avoiding RuntimeData instances when not needed (#24327) Signed-off-by: Yury Kats --- source/common/router/config_impl.cc | 22 ++++++++++------------ source/common/router/config_impl.h | 5 +++-- 2 files changed, 13 insertions(+), 14 deletions(-) diff --git a/source/common/router/config_impl.cc b/source/common/router/config_impl.cc index 552ca7275238..43621577b1b7 100644 --- a/source/common/router/config_impl.cc +++ b/source/common/router/config_impl.cc @@ -700,10 +700,11 @@ RouteEntryImplBase::RouteEntryImplBase(const VirtualHostImpl& vhost, } bool RouteEntryImplBase::evaluateRuntimeMatch(const uint64_t random_value) const { - return !runtime_ ? true - : loader_.snapshot().featureEnabled(runtime_->fractional_runtime_key_, - runtime_->fractional_runtime_default_, - random_value); + return runtime_ == nullptr + ? true + : loader_.snapshot().featureEnabled(runtime_->fractional_runtime_key_, + runtime_->fractional_runtime_default_, + random_value); } absl::string_view @@ -890,18 +891,15 @@ RouteEntryImplBase::getResponseHeaderParsers(bool specificity_ascend) const { specificity_ascend); } -absl::optional +std::unique_ptr RouteEntryImplBase::loadRuntimeData(const envoy::config::route::v3::RouteMatch& route_match) { - absl::optional runtime; - RuntimeData runtime_data; - if (route_match.has_runtime_fraction()) { - runtime_data.fractional_runtime_default_ = route_match.runtime_fraction().default_value(); - runtime_data.fractional_runtime_key_ = route_match.runtime_fraction().runtime_key(); + auto runtime_data = std::make_unique(); + runtime_data->fractional_runtime_default_ = route_match.runtime_fraction().default_value(); + runtime_data->fractional_runtime_key_ = route_match.runtime_fraction().runtime_key(); return runtime_data; } - - return runtime; + return nullptr; } const std::string& diff --git a/source/common/router/config_impl.h b/source/common/router/config_impl.h index 367f3b53e4c2..389139a889b1 100644 --- a/source/common/router/config_impl.h +++ b/source/common/router/config_impl.h @@ -967,7 +967,8 @@ class RouteEntryImplBase : public RouteEntryAndRoute, absl::InlinedVector getResponseHeaderParsers(bool specificity_ascend) const; - absl::optional loadRuntimeData(const envoy::config::route::v3::RouteMatch& route); + std::unique_ptr + loadRuntimeData(const envoy::config::route::v3::RouteMatch& route); static std::multimap parseOpaqueConfig(const envoy::config::route::v3::Route& route); @@ -1033,7 +1034,7 @@ class RouteEntryImplBase : public RouteEntryAndRoute, const absl::optional max_grpc_timeout_; const absl::optional grpc_timeout_offset_; Runtime::Loader& loader_; - const absl::optional runtime_; + std::unique_ptr runtime_; const std::string scheme_redirect_; const std::string host_redirect_; const std::string port_redirect_;