From 915ea694cead4aef6a0a38216df8ecd60f4edd4a Mon Sep 17 00:00:00 2001 From: ndr_brt Date: Tue, 12 Sep 2023 17:00:30 +0200 Subject: [PATCH] build: speed up e2e transfer tests --- .../core/base/EdcHttpClientImpl.java | 15 ++-- .../extensions/EdcClassRuntimesExtension.java | 49 +++++++++++ .../statement/SqlExecuteStatementTest.java | 0 .../TransferProcessHttpClient.java | 5 +- ...nsferProcessHttpClientIntegrationTest.java | 3 +- .../eclipse/edc/spi/http/EdcHttpClient.java | 11 +++ .../e2e/EndToEndTransferInMemoryTest.java | 88 +++++++++---------- .../e2e/EndToEndTransferPostgresqlTest.java | 87 +++++++++--------- 8 files changed, 155 insertions(+), 103 deletions(-) create mode 100644 core/common/junit/src/main/java/org/eclipse/edc/junit/extensions/EdcClassRuntimesExtension.java rename {system-tests/e2e-transfer-test/runner => extensions/common/sql/sql-core}/src/test/java/org/eclipse/edc/sql/statement/SqlExecuteStatementTest.java (100%) diff --git a/core/common/connector-core/src/main/java/org/eclipse/edc/connector/core/base/EdcHttpClientImpl.java b/core/common/connector-core/src/main/java/org/eclipse/edc/connector/core/base/EdcHttpClientImpl.java index 0e04b106867..721c97f6cf1 100644 --- a/core/common/connector-core/src/main/java/org/eclipse/edc/connector/core/base/EdcHttpClientImpl.java +++ b/core/common/connector-core/src/main/java/org/eclipse/edc/connector/core/base/EdcHttpClientImpl.java @@ -48,8 +48,15 @@ public EdcHttpClientImpl(OkHttpClient okHttpClient, RetryPolicy retryP @Override public Response execute(Request request) throws IOException { + return execute(request, emptyList()); + } + + @Override + public Response execute(Request request, List fallbacks) throws IOException { var call = okHttpClient.newCall(request); - return with(retryPolicy).compose(call).execute(); + var builder = with(retryPolicy); + fallbacks.stream().map(it -> it.create(request)).forEach(builder::compose); + return builder.compose(call).execute(); } @Override @@ -59,11 +66,7 @@ public Result execute(Request request, Function> mapp @Override public Result execute(Request request, List fallbacks, Function> mappingFunction) { - var call = okHttpClient.newCall(request); - var builder = with(retryPolicy); - fallbacks.stream().map(it -> it.create(request)).forEach(builder::compose); - - try (var response = builder.compose(call).execute()) { + try (var response = execute(request, fallbacks)) { return mappingFunction.apply(response); } catch (Throwable e) { monitor.severe("HTTP client exception caught for request " + request, e); diff --git a/core/common/junit/src/main/java/org/eclipse/edc/junit/extensions/EdcClassRuntimesExtension.java b/core/common/junit/src/main/java/org/eclipse/edc/junit/extensions/EdcClassRuntimesExtension.java new file mode 100644 index 00000000000..1012bf9490f --- /dev/null +++ b/core/common/junit/src/main/java/org/eclipse/edc/junit/extensions/EdcClassRuntimesExtension.java @@ -0,0 +1,49 @@ +/* + * Copyright (c) 2023 Bayerische Motoren Werke Aktiengesellschaft (BMW AG) + * + * This program and the accompanying materials are made available under the + * terms of the Apache License, Version 2.0 which is available at + * https://www.apache.org/licenses/LICENSE-2.0 + * + * SPDX-License-Identifier: Apache-2.0 + * + * Contributors: + * Bayerische Motoren Werke Aktiengesellschaft (BMW AG) - initial API and implementation + * + */ + +package org.eclipse.edc.junit.extensions; + +import org.junit.jupiter.api.extension.AfterAllCallback; +import org.junit.jupiter.api.extension.BeforeAllCallback; +import org.junit.jupiter.api.extension.ExtensionContext; + +/** + * Utility class that permits to run multiple EDC runtimes statically + */ +public class EdcClassRuntimesExtension implements BeforeAllCallback, AfterAllCallback { + + public final EdcRuntimeExtension[] extensions; + + public EdcClassRuntimesExtension(EdcRuntimeExtension... extensions) { + this.extensions = extensions; + } + + @Override + public void beforeAll(ExtensionContext extensionContext) { + for (var extension : this.extensions) { + try { + extension.beforeTestExecution(extensionContext); + } catch (Exception e) { + throw new RuntimeException(e); + } + } + } + + @Override + public void afterAll(ExtensionContext extensionContext) throws Exception { + for (var extension : this.extensions) { + extension.afterTestExecution(extensionContext); + } + } +} diff --git a/system-tests/e2e-transfer-test/runner/src/test/java/org/eclipse/edc/sql/statement/SqlExecuteStatementTest.java b/extensions/common/sql/sql-core/src/test/java/org/eclipse/edc/sql/statement/SqlExecuteStatementTest.java similarity index 100% rename from system-tests/e2e-transfer-test/runner/src/test/java/org/eclipse/edc/sql/statement/SqlExecuteStatementTest.java rename to extensions/common/sql/sql-core/src/test/java/org/eclipse/edc/sql/statement/SqlExecuteStatementTest.java diff --git a/extensions/control-plane/api/control-plane-api-client/src/main/java/org/eclipse/edc/connector/api/client/transferprocess/TransferProcessHttpClient.java b/extensions/control-plane/api/control-plane-api-client/src/main/java/org/eclipse/edc/connector/api/client/transferprocess/TransferProcessHttpClient.java index 086851f18b5..c17822e447a 100644 --- a/extensions/control-plane/api/control-plane-api-client/src/main/java/org/eclipse/edc/connector/api/client/transferprocess/TransferProcessHttpClient.java +++ b/extensions/control-plane/api/control-plane-api-client/src/main/java/org/eclipse/edc/connector/api/client/transferprocess/TransferProcessHttpClient.java @@ -28,6 +28,9 @@ import java.net.URI; import java.net.URISyntaxException; +import java.util.List; + +import static org.eclipse.edc.spi.http.FallbackFactories.retryWhenStatusIsNot; /** * Implementation of {@link TransferProcessApiClient} which talks to the Control Plane Transfer Process via HTTP APIs @@ -61,7 +64,7 @@ private void sendRequest(DataFlowRequest dataFlowRequest, String action, Object if (dataFlowRequest.getCallbackAddress() != null) { try { var request = createRequest(buildUrl(dataFlowRequest, action), body); - try (var response = httpClient.execute(request)) { + try (var response = httpClient.execute(request, List.of(retryWhenStatusIsNot(200)))) { if (!response.isSuccessful()) { monitor.severe(String.format("Failed to send callback request: received %s from the TransferProcess API", response.code())); } diff --git a/extensions/control-plane/api/control-plane-api-client/src/test/java/org/eclipse/edc/connector/api/client/transferprocess/TransferProcessHttpClientIntegrationTest.java b/extensions/control-plane/api/control-plane-api-client/src/test/java/org/eclipse/edc/connector/api/client/transferprocess/TransferProcessHttpClientIntegrationTest.java index 7c07f868409..42c77ee1746 100644 --- a/extensions/control-plane/api/control-plane-api-client/src/test/java/org/eclipse/edc/connector/api/client/transferprocess/TransferProcessHttpClientIntegrationTest.java +++ b/extensions/control-plane/api/control-plane-api-client/src/test/java/org/eclipse/edc/connector/api/client/transferprocess/TransferProcessHttpClientIntegrationTest.java @@ -69,7 +69,8 @@ void setUp(EdcExtension extension) { "web.http.port", String.valueOf(getFreePort()), "web.http.path", "/api", "web.http.control.port", String.valueOf(port), - "web.http.control.path", "/control" + "web.http.control.path", "/control", + "edc.core.retry.retries.max", "0" )); extension.registerSystemExtension(ServiceExtension.class, new TransferServiceMockExtension(service)); diff --git a/spi/common/http-spi/src/main/java/org/eclipse/edc/spi/http/EdcHttpClient.java b/spi/common/http-spi/src/main/java/org/eclipse/edc/spi/http/EdcHttpClient.java index 958cdc338c6..7d9803749a8 100644 --- a/spi/common/http-spi/src/main/java/org/eclipse/edc/spi/http/EdcHttpClient.java +++ b/spi/common/http-spi/src/main/java/org/eclipse/edc/spi/http/EdcHttpClient.java @@ -37,6 +37,17 @@ public interface EdcHttpClient { */ Response execute(Request request) throws IOException; + /** + * Executes the specified request synchronously. + * Accepts a list of {@link FallbackFactories} that could apply retry in particular occasions. + * + * @param request the {@link Request}. + * @param fallbacks a list of fallbacks to be applied. + * @return a {@link Response}, must be closed explicitly after consumption + * @throws IOException on connection error. + */ + Response execute(Request request, List fallbacks) throws IOException; + /** * Executes the specified request synchronously applying the mapping function to the response. * diff --git a/system-tests/e2e-transfer-test/runner/src/test/java/org/eclipse/edc/test/e2e/EndToEndTransferInMemoryTest.java b/system-tests/e2e-transfer-test/runner/src/test/java/org/eclipse/edc/test/e2e/EndToEndTransferInMemoryTest.java index a46a903ddda..038e3e675b3 100644 --- a/system-tests/e2e-transfer-test/runner/src/test/java/org/eclipse/edc/test/e2e/EndToEndTransferInMemoryTest.java +++ b/system-tests/e2e-transfer-test/runner/src/test/java/org/eclipse/edc/test/e2e/EndToEndTransferInMemoryTest.java @@ -15,6 +15,7 @@ package org.eclipse.edc.test.e2e; import org.eclipse.edc.junit.annotations.EndToEndTest; +import org.eclipse.edc.junit.extensions.EdcClassRuntimesExtension; import org.eclipse.edc.junit.extensions.EdcRuntimeExtension; import org.junit.jupiter.api.extension.RegisterExtension; @@ -24,53 +25,44 @@ class EndToEndTransferInMemoryTest extends AbstractEndToEndTransfer { @RegisterExtension - static EdcRuntimeExtension consumerControlPlane = new EdcRuntimeExtension( - ":system-tests:e2e-transfer-test:control-plane", - "consumer-control-plane", - CONSUMER.controlPlaneConfiguration() + static EdcClassRuntimesExtension runtimes = new EdcClassRuntimesExtension( + new EdcRuntimeExtension( + ":system-tests:e2e-transfer-test:control-plane", + "consumer-control-plane", + CONSUMER.controlPlaneConfiguration() + ), + new EdcRuntimeExtension( + ":system-tests:e2e-transfer-test:data-plane", + "consumer-data-plane", + CONSUMER.dataPlaneConfiguration() + ), + new EdcRuntimeExtension( + ":system-tests:e2e-transfer-test:backend-service", + "consumer-backend-service", + new HashMap<>() { + { + put("web.http.port", String.valueOf(CONSUMER.backendService().getPort())); + } + } + ), + new EdcRuntimeExtension( + ":system-tests:e2e-transfer-test:data-plane", + "provider-data-plane", + PROVIDER.dataPlaneConfiguration() + ), + new EdcRuntimeExtension( + ":system-tests:e2e-transfer-test:control-plane", + "provider-control-plane", + PROVIDER.controlPlaneConfiguration() + ), + new EdcRuntimeExtension( + ":system-tests:e2e-transfer-test:backend-service", + "provider-backend-service", + new HashMap<>() { + { + put("web.http.port", String.valueOf(PROVIDER.backendService().getPort())); + } + } + ) ); - - @RegisterExtension - static EdcRuntimeExtension consumerDataPlane = new EdcRuntimeExtension( - ":system-tests:e2e-transfer-test:data-plane", - "consumer-data-plane", - CONSUMER.dataPlaneConfiguration() - ); - - @RegisterExtension - static EdcRuntimeExtension consumerBackendService = new EdcRuntimeExtension( - ":system-tests:e2e-transfer-test:backend-service", - "consumer-backend-service", - new HashMap<>() { - { - put("web.http.port", String.valueOf(CONSUMER.backendService().getPort())); - } - } - ); - - @RegisterExtension - static EdcRuntimeExtension providerDataPlane = new EdcRuntimeExtension( - ":system-tests:e2e-transfer-test:data-plane", - "provider-data-plane", - PROVIDER.dataPlaneConfiguration() - ); - - @RegisterExtension - static EdcRuntimeExtension providerControlPlane = new EdcRuntimeExtension( - ":system-tests:e2e-transfer-test:control-plane", - "provider-control-plane", - PROVIDER.controlPlaneConfiguration() - ); - - @RegisterExtension - static EdcRuntimeExtension providerBackendService = new EdcRuntimeExtension( - ":system-tests:e2e-transfer-test:backend-service", - "provider-backend-service", - new HashMap<>() { - { - put("web.http.port", String.valueOf(PROVIDER.backendService().getPort())); - } - } - ); - } diff --git a/system-tests/e2e-transfer-test/runner/src/test/java/org/eclipse/edc/test/e2e/EndToEndTransferPostgresqlTest.java b/system-tests/e2e-transfer-test/runner/src/test/java/org/eclipse/edc/test/e2e/EndToEndTransferPostgresqlTest.java index 1ba76bb1883..1959812210c 100644 --- a/system-tests/e2e-transfer-test/runner/src/test/java/org/eclipse/edc/test/e2e/EndToEndTransferPostgresqlTest.java +++ b/system-tests/e2e-transfer-test/runner/src/test/java/org/eclipse/edc/test/e2e/EndToEndTransferPostgresqlTest.java @@ -15,6 +15,7 @@ package org.eclipse.edc.test.e2e; import org.eclipse.edc.junit.annotations.PostgresqlDbIntegrationTest; +import org.eclipse.edc.junit.extensions.EdcClassRuntimesExtension; import org.eclipse.edc.junit.extensions.EdcRuntimeExtension; import org.junit.jupiter.api.BeforeAll; import org.junit.jupiter.api.extension.RegisterExtension; @@ -29,53 +30,45 @@ class EndToEndTransferPostgresqlTest extends AbstractEndToEndTransfer { @RegisterExtension - static EdcRuntimeExtension consumerControlPlane = new EdcRuntimeExtension( - ":system-tests:e2e-transfer-test:control-plane-postgresql", - "consumer-control-plane", - CONSUMER.controlPlanePostgresConfiguration() - ); - - @RegisterExtension - static EdcRuntimeExtension consumerDataPlane = new EdcRuntimeExtension( - ":system-tests:e2e-transfer-test:data-plane", - "consumer-data-plane", - CONSUMER.dataPlaneConfiguration() - ); - - @RegisterExtension - static EdcRuntimeExtension consumerBackendService = new EdcRuntimeExtension( - ":system-tests:e2e-transfer-test:backend-service", - "consumer-backend-service", - new HashMap<>() { - { - put("web.http.port", String.valueOf(CONSUMER.backendService().getPort())); - } - } - ); - - @RegisterExtension - static EdcRuntimeExtension providerDataPlane = new EdcRuntimeExtension( - ":system-tests:e2e-transfer-test:data-plane", - "provider-data-plane", - PROVIDER.dataPlaneConfiguration() - ); - - @RegisterExtension - static EdcRuntimeExtension providerControlPlane = new EdcRuntimeExtension( - ":system-tests:e2e-transfer-test:control-plane-postgresql", - "provider-control-plane", - PROVIDER.controlPlanePostgresConfiguration() - ); - - @RegisterExtension - static EdcRuntimeExtension providerBackendService = new EdcRuntimeExtension( - ":system-tests:e2e-transfer-test:backend-service", - "provider-backend-service", - new HashMap<>() { - { - put("web.http.port", String.valueOf(PROVIDER.backendService().getPort())); - } - } + static EdcClassRuntimesExtension runtimes = new EdcClassRuntimesExtension( + new EdcRuntimeExtension( + ":system-tests:e2e-transfer-test:control-plane-postgresql", + "consumer-control-plane", + CONSUMER.controlPlanePostgresConfiguration() + ), + new EdcRuntimeExtension( + ":system-tests:e2e-transfer-test:data-plane", + "consumer-data-plane", + CONSUMER.dataPlaneConfiguration() + ), + new EdcRuntimeExtension( + ":system-tests:e2e-transfer-test:backend-service", + "consumer-backend-service", + new HashMap<>() { + { + put("web.http.port", String.valueOf(CONSUMER.backendService().getPort())); + } + } + ), + new EdcRuntimeExtension( + ":system-tests:e2e-transfer-test:data-plane", + "provider-data-plane", + PROVIDER.dataPlaneConfiguration() + ), + new EdcRuntimeExtension( + ":system-tests:e2e-transfer-test:control-plane-postgresql", + "provider-control-plane", + PROVIDER.controlPlanePostgresConfiguration() + ), + new EdcRuntimeExtension( + ":system-tests:e2e-transfer-test:backend-service", + "provider-backend-service", + new HashMap<>() { + { + put("web.http.port", String.valueOf(PROVIDER.backendService().getPort())); + } + } + ) ); @BeforeAll