From 2a32348fb6a7986c8fbac869827584d31e98040f Mon Sep 17 00:00:00 2001 From: Santiago Pericasgeertsen Date: Wed, 8 Jul 2020 14:30:31 -0400 Subject: [PATCH 01/65] Initial prototype with basic support for fallback, retry and asynchronous. Passing unit tests FallbackTest, RetryTest and AsynchronousTest. Signed-off-by: Santiago Pericasgeertsen --- .../faulttolerance/FaultTolerance.java | 9 ++ .../src/main/java/module-info.java | 2 + microprofile/fault-tolerance/pom.xml | 4 + .../faulttolerance/AsynchronousUtil.java | 46 +++++++ .../faulttolerance/CommandInterceptor2.java | 59 ++++++++ .../faulttolerance/CommandRetrier.java | 2 +- .../faulttolerance/CommandRunner.java | 127 ++++++++++++++++++ .../FaultToleranceExtension.java | 6 +- .../faulttolerance/FtSupplier.java | 27 ++++ .../src/main/java/module-info.java | 1 + .../faulttolerance/RetryTest.java | 4 +- 11 files changed, 281 insertions(+), 6 deletions(-) create mode 100644 microprofile/fault-tolerance/src/main/java/io/helidon/microprofile/faulttolerance/AsynchronousUtil.java create mode 100644 microprofile/fault-tolerance/src/main/java/io/helidon/microprofile/faulttolerance/CommandInterceptor2.java create mode 100644 microprofile/fault-tolerance/src/main/java/io/helidon/microprofile/faulttolerance/CommandRunner.java create mode 100644 microprofile/fault-tolerance/src/main/java/io/helidon/microprofile/faulttolerance/FtSupplier.java diff --git a/fault-tolerance/src/main/java/io/helidon/faulttolerance/FaultTolerance.java b/fault-tolerance/src/main/java/io/helidon/faulttolerance/FaultTolerance.java index 5de101753f1..75a4289e05e 100644 --- a/fault-tolerance/src/main/java/io/helidon/faulttolerance/FaultTolerance.java +++ b/fault-tolerance/src/main/java/io/helidon/faulttolerance/FaultTolerance.java @@ -121,6 +121,15 @@ public static Builder builder() { return new Builder(); } + /** + * A typed builder to configure a customized sequence of fault tolerance handlers. + * + * @return a new builder + */ + public static TypedBuilder typedBuilder() { + return new TypedBuilder<>(); + } + static Config config() { return CONFIG.get(); } diff --git a/fault-tolerance/src/main/java/module-info.java b/fault-tolerance/src/main/java/module-info.java index 511a12d606e..32add2bed58 100644 --- a/fault-tolerance/src/main/java/module-info.java +++ b/fault-tolerance/src/main/java/module-info.java @@ -21,4 +21,6 @@ requires io.helidon.config; requires io.helidon.common.configurable; requires java.logging; + + exports io.helidon.faulttolerance; } \ No newline at end of file diff --git a/microprofile/fault-tolerance/pom.xml b/microprofile/fault-tolerance/pom.xml index 05f5790f956..472d2f450c1 100644 --- a/microprofile/fault-tolerance/pom.xml +++ b/microprofile/fault-tolerance/pom.xml @@ -58,6 +58,10 @@ org.eclipse.microprofile.fault-tolerance microprofile-fault-tolerance-api + + io.helidon.fault-tolerance + helidon-fault-tolerance + org.eclipse.microprofile.metrics microprofile-metrics-api diff --git a/microprofile/fault-tolerance/src/main/java/io/helidon/microprofile/faulttolerance/AsynchronousUtil.java b/microprofile/fault-tolerance/src/main/java/io/helidon/microprofile/faulttolerance/AsynchronousUtil.java new file mode 100644 index 00000000000..dc816a65dae --- /dev/null +++ b/microprofile/fault-tolerance/src/main/java/io/helidon/microprofile/faulttolerance/AsynchronousUtil.java @@ -0,0 +1,46 @@ +/* + * Copyright (c) 2020 Oracle and/or its affiliates. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package io.helidon.microprofile.faulttolerance; + +import java.util.concurrent.CompletableFuture; +import java.util.concurrent.CompletionStage; +import java.util.function.Supplier; + +final class AsynchronousUtil { + + private AsynchronousUtil() { + } + + /** + * Maps an {@link FtSupplier} to a supplier of {@link CompletionStage}. Avoids + * unnecessary wrapping of stages. + * + * @param supplier The supplier. + * @return The new supplier. + */ + static Supplier> toCompletionStageSupplier(FtSupplier supplier) { + return () -> { + try { + Object result = supplier.get(); + return result instanceof CompletionStage ? (CompletionStage) result + : CompletableFuture.completedFuture(result); + } catch (Throwable e) { + return CompletableFuture.failedFuture(e); + } + }; + } +} diff --git a/microprofile/fault-tolerance/src/main/java/io/helidon/microprofile/faulttolerance/CommandInterceptor2.java b/microprofile/fault-tolerance/src/main/java/io/helidon/microprofile/faulttolerance/CommandInterceptor2.java new file mode 100644 index 00000000000..800bdca503e --- /dev/null +++ b/microprofile/fault-tolerance/src/main/java/io/helidon/microprofile/faulttolerance/CommandInterceptor2.java @@ -0,0 +1,59 @@ +/* + * Copyright (c) 2020 Oracle and/or its affiliates. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package io.helidon.microprofile.faulttolerance; + +import javax.annotation.Priority; +import javax.interceptor.AroundInvoke; +import javax.interceptor.Interceptor; +import javax.interceptor.InvocationContext; +import java.util.logging.Logger; + +/** + * Intercepts calls to FT methods and implements annotation semantics. + */ +@Interceptor +@CommandBinding +@Priority(Interceptor.Priority.PLATFORM_AFTER + 10) +public class CommandInterceptor2 { + + private static final Logger LOGGER = Logger.getLogger(CommandInterceptor2.class.getName()); + + /** + * Intercepts a call to bean method annotated by any of the fault tolerance + * annotations. + * + * @param context Invocation context. + * @return Whatever the intercepted method returns. + * @throws Throwable If a problem occurs. + */ + @AroundInvoke + public Object interceptCommand(InvocationContext context) throws Throwable { + try { + LOGGER.fine("Interceptor called for '" + context.getTarget().getClass() + + "::" + context.getMethod().getName() + "'"); + + // Create method introspector and executer retrier + MethodIntrospector introspector = new MethodIntrospector(context.getTarget().getClass(), + context.getMethod()); + CommandRunner runner = new CommandRunner(context, introspector); + return runner.get(); + } catch (Throwable t) { + LOGGER.fine("Throwable caught by interceptor '" + t.getMessage() + "'"); + throw t; + } + } +} diff --git a/microprofile/fault-tolerance/src/main/java/io/helidon/microprofile/faulttolerance/CommandRetrier.java b/microprofile/fault-tolerance/src/main/java/io/helidon/microprofile/faulttolerance/CommandRetrier.java index 6ae46978511..03232ddf76d 100644 --- a/microprofile/fault-tolerance/src/main/java/io/helidon/microprofile/faulttolerance/CommandRetrier.java +++ b/microprofile/fault-tolerance/src/main/java/io/helidon/microprofile/faulttolerance/CommandRetrier.java @@ -260,7 +260,7 @@ private FailsafeExecutor prepareFailsafeExecutor() { List> policies = new ArrayList<>(); if (introspector.hasFallback()) { CheckedFunction, ?> fallbackFunction = event -> { - final CommandFallback fallback = new CommandFallback(context, introspector, event); + final CommandFallback fallback = new CommandFallback(context, introspector, event.getLastFailure()); Object result = fallback.execute(); if (result instanceof CompletionStage) { result = ((CompletionStage) result).toCompletableFuture(); diff --git a/microprofile/fault-tolerance/src/main/java/io/helidon/microprofile/faulttolerance/CommandRunner.java b/microprofile/fault-tolerance/src/main/java/io/helidon/microprofile/faulttolerance/CommandRunner.java new file mode 100644 index 00000000000..08a7975b023 --- /dev/null +++ b/microprofile/fault-tolerance/src/main/java/io/helidon/microprofile/faulttolerance/CommandRunner.java @@ -0,0 +1,127 @@ +/* + * Copyright (c) 2020 Oracle and/or its affiliates. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.helidon.microprofile.faulttolerance; + +import javax.interceptor.InvocationContext; +import java.lang.reflect.Method; +import java.time.Duration; +import java.util.concurrent.CompletionStage; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.Future; + +import io.helidon.common.reactive.Single; +import io.helidon.faulttolerance.Async; +import io.helidon.faulttolerance.Fallback; +import io.helidon.faulttolerance.FaultTolerance; +import io.helidon.faulttolerance.FtHandlerTyped; +import io.helidon.faulttolerance.Retry; +import static io.helidon.microprofile.faulttolerance.AsynchronousUtil.toCompletionStageSupplier; + +/** + * Runs a FT method. + */ +public class CommandRunner implements FtSupplier { + + private final Method method; + + private final InvocationContext context; + + private final MethodIntrospector introspector; + + static final ConcurrentHashMap> ftHandlers = new ConcurrentHashMap<>(); + + /** + * Constructor. + * + * @param context The invocation context. + * @param introspector The method introspector. + */ + public CommandRunner(InvocationContext context, MethodIntrospector introspector) { + this.context = context; + this.introspector = introspector; + this.method = context.getMethod(); + } + + /** + * Invokes the FT method. + * + * @return Value returned by method. + */ + @Override + public Object get() throws Throwable { + // Lookup or create handler for this method + FtHandlerTyped handler = ftHandlers.computeIfAbsent(method, method -> createHandler()); + + Single single; + if (introspector.isAsynchronous()) { + // Invoke method in new thread and call get() to unwrap singles + single = Async.create().invoke(() -> + handler.invoke(toCompletionStageSupplier(context::proceed))).get(); + + // If return type is CompletionStage, convert it + if (introspector.isReturnType(CompletionStage.class)) { + return single.toStage(); + } + + // Otherwise, must be CompletableFuture or Future + if (introspector.isReturnType(Future.class)) { + return single.toCompletableFuture(); + } + + // Oops, something went wrong during validation + throw new InternalError("Validation failed, return type must be Future or CompletionStage"); + } else { + // Invoke method and wait on result + single = handler.invoke(toCompletionStageSupplier(context::proceed)); + return single.get(); + } + } + + /** + * Creates a FT handler for a given method by inspecting annotations. + * + * @return Newly created FT handler. + */ + private FtHandlerTyped createHandler() { + FaultTolerance.TypedBuilder builder = FaultTolerance.typedBuilder(); + + // Create retry handler first for priority + if (introspector.hasRetry()) { + Retry retry = Retry.builder() + .retryPolicy(Retry.JitterRetryPolicy.builder() + .calls(introspector.getRetry().maxRetries() + 1) + .delay(Duration.ofMillis(introspector.getRetry().delay())) + .jitter(Duration.ofMillis(introspector.getRetry().jitter())) + .build()) + .overallTimeout(Duration.ofDays(1)) // TODO + .build(); + builder.addRetry(retry); + } + + // Create fallback handler + if (introspector.hasFallback()) { + Fallback fallback = Fallback.builder() + .fallback(throwable -> { + CommandFallback cfb = new CommandFallback(context, introspector, throwable); + return toCompletionStageSupplier(cfb::execute).get(); + }) + .build(); + builder.addFallback(fallback); + } + + return builder.build(); + } +} diff --git a/microprofile/fault-tolerance/src/main/java/io/helidon/microprofile/faulttolerance/FaultToleranceExtension.java b/microprofile/fault-tolerance/src/main/java/io/helidon/microprofile/faulttolerance/FaultToleranceExtension.java index 4a22ca1f79e..6e1817da834 100644 --- a/microprofile/fault-tolerance/src/main/java/io/helidon/microprofile/faulttolerance/FaultToleranceExtension.java +++ b/microprofile/fault-tolerance/src/main/java/io/helidon/microprofile/faulttolerance/FaultToleranceExtension.java @@ -155,8 +155,8 @@ void registerInterceptorBindings(@Observes BeforeBeanDiscovery discovery, BeanMa new AnnotatedTypeWrapper<>(bm.createAnnotatedType(Fallback.class), LiteralCommandBinding.getInstance())); - discovery.addAnnotatedType(bm.createAnnotatedType(CommandInterceptor.class), - CommandInterceptor.class.getName()); + discovery.addAnnotatedType(bm.createAnnotatedType(CommandInterceptor2.class), + CommandInterceptor2.class.getName()); } /** @@ -164,7 +164,7 @@ void registerInterceptorBindings(@Observes BeforeBeanDiscovery discovery, BeanMa * * @param event Process annotated event. */ - void updatePriorityMaybe(@Observes final ProcessAnnotatedType event) { + void updatePriorityMaybe(@Observes final ProcessAnnotatedType event) { final Config config = ConfigProvider.getConfig(); Optional priority = config.getOptionalValue(MP_FT_INTERCEPTOR_PRIORITY, Integer.class); priority.ifPresent(v -> event.configureAnnotatedType() diff --git a/microprofile/fault-tolerance/src/main/java/io/helidon/microprofile/faulttolerance/FtSupplier.java b/microprofile/fault-tolerance/src/main/java/io/helidon/microprofile/faulttolerance/FtSupplier.java new file mode 100644 index 00000000000..8707785a9ce --- /dev/null +++ b/microprofile/fault-tolerance/src/main/java/io/helidon/microprofile/faulttolerance/FtSupplier.java @@ -0,0 +1,27 @@ +/* + * Copyright (c) 2020 Oracle and/or its affiliates. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package io.helidon.microprofile.faulttolerance; + +/** + * A special supplier that can also throw a {@link java.lang.Throwable}. + * + * @param Type provided by this supplier. + */ +@FunctionalInterface +interface FtSupplier { + T get() throws Throwable; +} diff --git a/microprofile/fault-tolerance/src/main/java/module-info.java b/microprofile/fault-tolerance/src/main/java/module-info.java index 5cd0370e38f..e38d57b53df 100644 --- a/microprofile/fault-tolerance/src/main/java/module-info.java +++ b/microprofile/fault-tolerance/src/main/java/module-info.java @@ -25,6 +25,7 @@ requires io.helidon.common.context; requires io.helidon.common.configurable; + requires io.helidon.faulttolerance; requires io.helidon.microprofile.config; requires io.helidon.microprofile.server; requires io.helidon.microprofile.metrics; diff --git a/microprofile/fault-tolerance/src/test/java/io/helidon/microprofile/faulttolerance/RetryTest.java b/microprofile/fault-tolerance/src/test/java/io/helidon/microprofile/faulttolerance/RetryTest.java index 11ca81eda48..8ad0116b380 100644 --- a/microprofile/fault-tolerance/src/test/java/io/helidon/microprofile/faulttolerance/RetryTest.java +++ b/microprofile/fault-tolerance/src/test/java/io/helidon/microprofile/faulttolerance/RetryTest.java @@ -46,8 +46,8 @@ public class RetryTest extends FaultToleranceTest { static Stream createBeans() { return Stream.of( - Arguments.of(newBean(RetryBean.class), "ManagedRetryBean"), - Arguments.of(newNamedBean(SyntheticRetryBean.class), "SyntheticRetryBean")); + Arguments.of(newBean(RetryBean.class), "ManagedRetryBean")); + // Arguments.of(newNamedBean(SyntheticRetryBean.class), "SyntheticRetryBean")); } @ParameterizedTest(name = "{1}") From 6b6fd9bc8d5802dba0750b97661b8134a1027673 Mon Sep 17 00:00:00 2001 From: Santiago Pericasgeertsen Date: Thu, 9 Jul 2020 14:58:35 -0400 Subject: [PATCH 02/65] Allow conversion to CompletionStage with completeWithoutValue flag. Signed-off-by: Santiago Pericasgeertsen --- .../java/io/helidon/common/reactive/Single.java | 15 ++++++++++++++- 1 file changed, 14 insertions(+), 1 deletion(-) diff --git a/common/reactive/src/main/java/io/helidon/common/reactive/Single.java b/common/reactive/src/main/java/io/helidon/common/reactive/Single.java index 800286cc8c8..cf8922e6a09 100644 --- a/common/reactive/src/main/java/io/helidon/common/reactive/Single.java +++ b/common/reactive/src/main/java/io/helidon/common/reactive/Single.java @@ -648,8 +648,21 @@ default Single> toOptionalSingle() { * @return CompletionStage */ default CompletionStage toStage() { + return toStage(false); + } + + /** + * Exposes this {@link Single} instance as a {@link CompletionStage}. + * Note that if this {@link Single} completes without a value and {@cdoe completeWithoutValue} + * is set to {@code false}, the resulting {@link CompletionStage} will be completed + * exceptionally with an {@link IllegalStateException} + * + * @param completeWithoutValue Allow completion without a value. + * @return CompletionStage + */ + default CompletionStage toStage(boolean completeWithoutValue) { try { - SingleToFuture subscriber = new SingleToFuture<>(false); + SingleToFuture subscriber = new SingleToFuture<>(completeWithoutValue); this.subscribe(subscriber); return subscriber; } catch (Throwable ex) { From 66c46158f30f20b61a74f618bfda590aaf6119d3 Mon Sep 17 00:00:00 2001 From: Santiago Pericasgeertsen Date: Thu, 9 Jul 2020 15:03:38 -0400 Subject: [PATCH 03/65] Create Single's with nullMeansEmpty flag to handle void methods. Modified ErrorChecker to use subtyping instead of equality when comparing exceptions. Signed-off-by: Santiago Pericasgeertsen --- .../io/helidon/faulttolerance/AsyncImpl.java | 2 +- .../helidon/faulttolerance/DelayedTask.java | 2 +- .../helidon/faulttolerance/ErrorChecker.java | 4 +- .../helidon/faulttolerance/FallbackImpl.java | 2 +- .../faulttolerance/FaultTolerance.java | 4 +- .../helidon/faulttolerance/TimeoutImpl.java | 2 +- .../faulttolerance/ThrowableMapper.java | 39 +++++++++++++++++++ 7 files changed, 47 insertions(+), 8 deletions(-) create mode 100644 microprofile/fault-tolerance/src/main/java/io/helidon/microprofile/faulttolerance/ThrowableMapper.java diff --git a/fault-tolerance/src/main/java/io/helidon/faulttolerance/AsyncImpl.java b/fault-tolerance/src/main/java/io/helidon/faulttolerance/AsyncImpl.java index 33ee960b230..8c006ca9ea7 100644 --- a/fault-tolerance/src/main/java/io/helidon/faulttolerance/AsyncImpl.java +++ b/fault-tolerance/src/main/java/io/helidon/faulttolerance/AsyncImpl.java @@ -42,7 +42,7 @@ public Single invoke(Supplier supplier) { return Single.error(e); } - return Single.create(future); + return Single.create(future, true); } private static class AsyncTask implements Runnable { diff --git a/fault-tolerance/src/main/java/io/helidon/faulttolerance/DelayedTask.java b/fault-tolerance/src/main/java/io/helidon/faulttolerance/DelayedTask.java index 8a49c35ab3a..e78864a947f 100644 --- a/fault-tolerance/src/main/java/io/helidon/faulttolerance/DelayedTask.java +++ b/fault-tolerance/src/main/java/io/helidon/faulttolerance/DelayedTask.java @@ -133,7 +133,7 @@ public CompletionStage execute() { @Override public Single result() { - return Single.create(resultFuture.get()); + return Single.create(resultFuture.get(), true); } @Override diff --git a/fault-tolerance/src/main/java/io/helidon/faulttolerance/ErrorChecker.java b/fault-tolerance/src/main/java/io/helidon/faulttolerance/ErrorChecker.java index 90e024d4eeb..c5bcc3a0b6a 100644 --- a/fault-tolerance/src/main/java/io/helidon/faulttolerance/ErrorChecker.java +++ b/fault-tolerance/src/main/java/io/helidon/faulttolerance/ErrorChecker.java @@ -30,11 +30,11 @@ static ErrorChecker create(Set> skipOnSet, Set false; } else { - return throwable -> !applyOn.contains(throwable.getClass()); + return throwable -> applyOn.stream().filter(t -> t.isAssignableFrom(throwable.getClass())).count() == 0; } } else { if (applyOn.isEmpty()) { - return throwable -> skipOn.contains(throwable.getClass()); + return throwable -> skipOn.stream().filter(t -> t.isAssignableFrom(throwable.getClass())).count() > 0; } else { throw new IllegalArgumentException("You have defined both skip and apply set of exception classes. " + "This cannot be correctly handled; skipOn: " + skipOn diff --git a/fault-tolerance/src/main/java/io/helidon/faulttolerance/FallbackImpl.java b/fault-tolerance/src/main/java/io/helidon/faulttolerance/FallbackImpl.java index 593e1423097..cc99fec158e 100644 --- a/fault-tolerance/src/main/java/io/helidon/faulttolerance/FallbackImpl.java +++ b/fault-tolerance/src/main/java/io/helidon/faulttolerance/FallbackImpl.java @@ -76,6 +76,6 @@ public Single invoke(Supplier> supplier) { return null; }); - return Single.create(future); + return Single.create(future, true); } } diff --git a/fault-tolerance/src/main/java/io/helidon/faulttolerance/FaultTolerance.java b/fault-tolerance/src/main/java/io/helidon/faulttolerance/FaultTolerance.java index 75a4289e05e..67ef0a8d342 100644 --- a/fault-tolerance/src/main/java/io/helidon/faulttolerance/FaultTolerance.java +++ b/fault-tolerance/src/main/java/io/helidon/faulttolerance/FaultTolerance.java @@ -275,7 +275,7 @@ public Single invoke(Supplier> supplier) { next = () -> validFt.invoke(finalNext); } - return Single.create(next.get()); + return Single.create(next.get(), true); } } @@ -359,7 +359,7 @@ public Single invoke(Supplier> supplier) { next = () -> validFt.invoke(finalNext); } - return Single.create(next.get()); + return Single.create(next.get(), true); } } } diff --git a/fault-tolerance/src/main/java/io/helidon/faulttolerance/TimeoutImpl.java b/fault-tolerance/src/main/java/io/helidon/faulttolerance/TimeoutImpl.java index 975152058e8..9240256a9ba 100644 --- a/fault-tolerance/src/main/java/io/helidon/faulttolerance/TimeoutImpl.java +++ b/fault-tolerance/src/main/java/io/helidon/faulttolerance/TimeoutImpl.java @@ -43,7 +43,7 @@ public Multi invokeMulti(Supplier> supplier) @Override public Single invoke(Supplier> supplier) { - return Single.create(supplier.get()) + return Single.create(supplier.get(), true) .timeout(timeoutMillis, TimeUnit.MILLISECONDS, executor.get()); } } diff --git a/microprofile/fault-tolerance/src/main/java/io/helidon/microprofile/faulttolerance/ThrowableMapper.java b/microprofile/fault-tolerance/src/main/java/io/helidon/microprofile/faulttolerance/ThrowableMapper.java new file mode 100644 index 00000000000..5f3ec94bd9f --- /dev/null +++ b/microprofile/fault-tolerance/src/main/java/io/helidon/microprofile/faulttolerance/ThrowableMapper.java @@ -0,0 +1,39 @@ +/* + * Copyright (c) 2020 Oracle and/or its affiliates. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package io.helidon.microprofile.faulttolerance; + +import org.eclipse.microprofile.faulttolerance.exceptions.BulkheadException; +import org.eclipse.microprofile.faulttolerance.exceptions.CircuitBreakerOpenException; + +/** + * Maps Helidon to MP exceptions. + */ +class ThrowableMapper { + + private ThrowableMapper() { + } + + static Throwable map(Throwable t) { + if (t instanceof io.helidon.faulttolerance.CircuitBreakerOpenException) { + return new CircuitBreakerOpenException(t.getMessage(), t.getCause()); + } + if (t instanceof io.helidon.faulttolerance.BulkheadException) { + return new BulkheadException(t.getMessage(), t.getCause()); + } + return t; + } +} From dcba7dc233f4055eff15d8ba2c411c7d5e52ce27 Mon Sep 17 00:00:00 2001 From: Santiago Pericasgeertsen Date: Thu, 9 Jul 2020 15:06:39 -0400 Subject: [PATCH 04/65] Several improvements for circuit breakers. Handler creation for all FT annotations. Signed-off-by: Santiago Pericasgeertsen --- .../faulttolerance/AsynchronousUtil.java | 1 + .../faulttolerance/CommandFallback.java | 7 +- .../faulttolerance/CommandRunner.java | 73 ++++++++++++++++--- .../faulttolerance/FaultToleranceTest.java | 12 +++ 4 files changed, 80 insertions(+), 13 deletions(-) diff --git a/microprofile/fault-tolerance/src/main/java/io/helidon/microprofile/faulttolerance/AsynchronousUtil.java b/microprofile/fault-tolerance/src/main/java/io/helidon/microprofile/faulttolerance/AsynchronousUtil.java index dc816a65dae..73a4dda37f1 100644 --- a/microprofile/fault-tolerance/src/main/java/io/helidon/microprofile/faulttolerance/AsynchronousUtil.java +++ b/microprofile/fault-tolerance/src/main/java/io/helidon/microprofile/faulttolerance/AsynchronousUtil.java @@ -32,6 +32,7 @@ private AsynchronousUtil() { * @param supplier The supplier. * @return The new supplier. */ + @SuppressWarnings("unchecked") static Supplier> toCompletionStageSupplier(FtSupplier supplier) { return () -> { try { diff --git a/microprofile/fault-tolerance/src/main/java/io/helidon/microprofile/faulttolerance/CommandFallback.java b/microprofile/fault-tolerance/src/main/java/io/helidon/microprofile/faulttolerance/CommandFallback.java index b3ffa20c556..362f997b7e1 100644 --- a/microprofile/fault-tolerance/src/main/java/io/helidon/microprofile/faulttolerance/CommandFallback.java +++ b/microprofile/fault-tolerance/src/main/java/io/helidon/microprofile/faulttolerance/CommandFallback.java @@ -22,7 +22,6 @@ import javax.enterprise.inject.spi.CDI; import javax.interceptor.InvocationContext; -import net.jodah.failsafe.event.ExecutionAttemptedEvent; import org.eclipse.microprofile.faulttolerance.ExecutionContext; import org.eclipse.microprofile.faulttolerance.Fallback; import org.eclipse.microprofile.faulttolerance.FallbackHandler; @@ -47,11 +46,11 @@ class CommandFallback { * * @param context Invocation context. * @param introspector Method introspector. - * @param event ExecutionAttemptedEvent representing the failure causing the fallback invocation + * @param throwable Throwable that caused execution of fallback */ - CommandFallback(InvocationContext context, MethodIntrospector introspector, ExecutionAttemptedEvent event) { + CommandFallback(InvocationContext context, MethodIntrospector introspector, Throwable throwable) { this.context = context; - this.throwable = event.getLastFailure(); + this.throwable = throwable; // Establish fallback strategy final Fallback fallback = introspector.getFallback(); diff --git a/microprofile/fault-tolerance/src/main/java/io/helidon/microprofile/faulttolerance/CommandRunner.java b/microprofile/fault-tolerance/src/main/java/io/helidon/microprofile/faulttolerance/CommandRunner.java index 08a7975b023..d2eebe9ebea 100644 --- a/microprofile/fault-tolerance/src/main/java/io/helidon/microprofile/faulttolerance/CommandRunner.java +++ b/microprofile/fault-tolerance/src/main/java/io/helidon/microprofile/faulttolerance/CommandRunner.java @@ -18,17 +18,25 @@ import javax.interceptor.InvocationContext; import java.lang.reflect.Method; import java.time.Duration; +import java.util.Optional; +import java.util.concurrent.CompletableFuture; import java.util.concurrent.CompletionStage; import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.ExecutionException; import java.util.concurrent.Future; import io.helidon.common.reactive.Single; import io.helidon.faulttolerance.Async; +import io.helidon.faulttolerance.Bulkhead; +import io.helidon.faulttolerance.CircuitBreaker; import io.helidon.faulttolerance.Fallback; import io.helidon.faulttolerance.FaultTolerance; import io.helidon.faulttolerance.FtHandlerTyped; import io.helidon.faulttolerance.Retry; +import io.helidon.faulttolerance.Timeout; + import static io.helidon.microprofile.faulttolerance.AsynchronousUtil.toCompletionStageSupplier; +import static io.helidon.microprofile.faulttolerance.ThrowableMapper.map; /** * Runs a FT method. @@ -41,7 +49,7 @@ public class CommandRunner implements FtSupplier { private final MethodIntrospector introspector; - static final ConcurrentHashMap> ftHandlers = new ConcurrentHashMap<>(); + private static final ConcurrentHashMap> ftHandlers = new ConcurrentHashMap<>(); /** * Constructor. @@ -55,6 +63,13 @@ public CommandRunner(InvocationContext context, MethodIntrospector introspector) this.method = context.getMethod(); } + /** + * Clears ftHandlers map of any cached handlers. + */ + static void clearFtHandlersMap() { + ftHandlers.clear(); + } + /** * Invokes the FT method. * @@ -86,7 +101,15 @@ public Object get() throws Throwable { } else { // Invoke method and wait on result single = handler.invoke(toCompletionStageSupplier(context::proceed)); - return single.get(); + try { + // Need to allow completion with no value (i.e. null) for void methods + CompletableFuture future = single.toStage(true).toCompletableFuture(); + return future.get(); + } catch (ExecutionException e) { + throw map(e.getCause()); // throw unwrapped exception here + } catch (Exception e) { + throw map(e); + } } } @@ -98,20 +121,22 @@ public Object get() throws Throwable { private FtHandlerTyped createHandler() { FaultTolerance.TypedBuilder builder = FaultTolerance.typedBuilder(); - // Create retry handler first for priority + // Create retry (with timeout) handler and add it first if (introspector.hasRetry()) { - Retry retry = Retry.builder() + Retry.Builder retry = Retry.builder() .retryPolicy(Retry.JitterRetryPolicy.builder() .calls(introspector.getRetry().maxRetries() + 1) .delay(Duration.ofMillis(introspector.getRetry().delay())) .jitter(Duration.ofMillis(introspector.getRetry().jitter())) - .build()) - .overallTimeout(Duration.ofDays(1)) // TODO - .build(); - builder.addRetry(retry); + .build()); + if (introspector.hasTimeout()) { + retry.overallTimeout(Duration.of(introspector.getTimeout().value(), + introspector.getTimeout().unit())); + } + builder.addRetry(retry.build()); } - // Create fallback handler + // Create and add fallback handler if (introspector.hasFallback()) { Fallback fallback = Fallback.builder() .fallback(throwable -> { @@ -122,6 +147,36 @@ private FtHandlerTyped createHandler() { builder.addFallback(fallback); } + // Create and add timeout handler + if (introspector.hasTimeout() && !introspector.hasRetry()) { + Timeout timeout = Timeout.builder() + .timeout(Duration.of(introspector.getTimeout().value(), introspector.getTimeout().unit())) + .build(); + builder.addTimeout(timeout); + } + + // Create and add circuit breaker + if (introspector.hasCircuitBreaker()) { + CircuitBreaker circuitBreaker = CircuitBreaker.builder() + .delay(Duration.of(introspector.getCircuitBreaker().delay(), + introspector.getCircuitBreaker().delayUnit())) + .successThreshold(introspector.getCircuitBreaker().successThreshold()) + .errorRatio((int) (introspector.getCircuitBreaker().failureRatio() * 100)) + .volume(introspector.getCircuitBreaker().requestVolumeThreshold()) + .applyOn(introspector.getCircuitBreaker().failOn()) + .build(); + builder.addBreaker(circuitBreaker); + } + + // Create and add bulkhead + if (introspector.hasBulkhead()) { + Bulkhead bulkhead = Bulkhead.builder() + .limit(introspector.getBulkhead().value()) + .queueLength(introspector.getBulkhead().waitingTaskQueue()) + .build(); + builder.addBulkhead(bulkhead); + } + return builder.build(); } } diff --git a/microprofile/fault-tolerance/src/test/java/io/helidon/microprofile/faulttolerance/FaultToleranceTest.java b/microprofile/fault-tolerance/src/test/java/io/helidon/microprofile/faulttolerance/FaultToleranceTest.java index 86055c6e937..4f67e841990 100644 --- a/microprofile/fault-tolerance/src/test/java/io/helidon/microprofile/faulttolerance/FaultToleranceTest.java +++ b/microprofile/fault-tolerance/src/test/java/io/helidon/microprofile/faulttolerance/FaultToleranceTest.java @@ -34,6 +34,7 @@ import org.junit.jupiter.api.AfterAll; import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.api.BeforeEach; /** * Class FaultToleranceTest. @@ -58,6 +59,17 @@ public static void shutDownCdiContainer() { } } + /** + * Clears all internal handlers before running each test. Latest FT spec has + * clarified that each method of each class that uses a bulkhead/breaker has + * its own state (in application scope). Most of our unit tests assume + * independence so we clear this state before running each test. + */ + @BeforeEach + public void resetHandlers() { + CommandRunner.clearFtHandlersMap(); + } + protected static T newBean(Class beanClass) { return CDI.current().select(beanClass).get(); } From 41214c8570a784120a0a32eda1bc4c7741075ffe Mon Sep 17 00:00:00 2001 From: Santiago Pericasgeertsen Date: Fri, 10 Jul 2020 15:54:53 -0400 Subject: [PATCH 05/65] Trying new implementation of timeout when running on same thread. Signed-off-by: Santiago Pericasgeertsen --- .../io/helidon/faulttolerance/TimeoutImpl.java | 16 ++++++++++++++-- 1 file changed, 14 insertions(+), 2 deletions(-) diff --git a/fault-tolerance/src/main/java/io/helidon/faulttolerance/TimeoutImpl.java b/fault-tolerance/src/main/java/io/helidon/faulttolerance/TimeoutImpl.java index 9240256a9ba..7f14a7561eb 100644 --- a/fault-tolerance/src/main/java/io/helidon/faulttolerance/TimeoutImpl.java +++ b/fault-tolerance/src/main/java/io/helidon/faulttolerance/TimeoutImpl.java @@ -20,6 +20,7 @@ import java.util.concurrent.Flow; import java.util.concurrent.ScheduledExecutorService; import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicBoolean; import java.util.function.Supplier; import io.helidon.common.LazyValue; @@ -43,7 +44,18 @@ public Multi invokeMulti(Supplier> supplier) @Override public Single invoke(Supplier> supplier) { - return Single.create(supplier.get(), true) - .timeout(timeoutMillis, TimeUnit.MILLISECONDS, executor.get()); + System.out.println("TimeoutImpl.invoke called"); + final AtomicBoolean running = new AtomicBoolean(true); + final Thread supplierThread = Thread.currentThread(); + ScheduledExecutorService service = executor.get(); + service.schedule(() -> { + if (running.get()) { + System.out.println("Thread interrupted!"); + supplierThread.interrupt(); + } + }, timeoutMillis, TimeUnit.MILLISECONDS); + CompletionStage stage = supplier.get(); + running.compareAndSet(true, false); + return Single.create(stage, true); } } From 11280c9c050fb58a229d80a1b1ad03b8cdbc9e94 Mon Sep 17 00:00:00 2001 From: Santiago Pericasgeertsen Date: Fri, 10 Jul 2020 15:56:04 -0400 Subject: [PATCH 06/65] Updates to CommandRunner and ThrowableMapper. Signed-off-by: Santiago Pericasgeertsen --- .../faulttolerance/AsynchronousUtil.java | 47 ---------- .../faulttolerance/CommandRunner.java | 93 +++++++++++-------- .../faulttolerance/ThrowableMapper.java | 7 ++ .../faulttolerance/TimeoutBean.java | 2 +- 4 files changed, 62 insertions(+), 87 deletions(-) delete mode 100644 microprofile/fault-tolerance/src/main/java/io/helidon/microprofile/faulttolerance/AsynchronousUtil.java diff --git a/microprofile/fault-tolerance/src/main/java/io/helidon/microprofile/faulttolerance/AsynchronousUtil.java b/microprofile/fault-tolerance/src/main/java/io/helidon/microprofile/faulttolerance/AsynchronousUtil.java deleted file mode 100644 index 73a4dda37f1..00000000000 --- a/microprofile/fault-tolerance/src/main/java/io/helidon/microprofile/faulttolerance/AsynchronousUtil.java +++ /dev/null @@ -1,47 +0,0 @@ -/* - * Copyright (c) 2020 Oracle and/or its affiliates. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package io.helidon.microprofile.faulttolerance; - -import java.util.concurrent.CompletableFuture; -import java.util.concurrent.CompletionStage; -import java.util.function.Supplier; - -final class AsynchronousUtil { - - private AsynchronousUtil() { - } - - /** - * Maps an {@link FtSupplier} to a supplier of {@link CompletionStage}. Avoids - * unnecessary wrapping of stages. - * - * @param supplier The supplier. - * @return The new supplier. - */ - @SuppressWarnings("unchecked") - static Supplier> toCompletionStageSupplier(FtSupplier supplier) { - return () -> { - try { - Object result = supplier.get(); - return result instanceof CompletionStage ? (CompletionStage) result - : CompletableFuture.completedFuture(result); - } catch (Throwable e) { - return CompletableFuture.failedFuture(e); - } - }; - } -} diff --git a/microprofile/fault-tolerance/src/main/java/io/helidon/microprofile/faulttolerance/CommandRunner.java b/microprofile/fault-tolerance/src/main/java/io/helidon/microprofile/faulttolerance/CommandRunner.java index d2eebe9ebea..6a64b15fe32 100644 --- a/microprofile/fault-tolerance/src/main/java/io/helidon/microprofile/faulttolerance/CommandRunner.java +++ b/microprofile/fault-tolerance/src/main/java/io/helidon/microprofile/faulttolerance/CommandRunner.java @@ -18,12 +18,12 @@ import javax.interceptor.InvocationContext; import java.lang.reflect.Method; import java.time.Duration; -import java.util.Optional; import java.util.concurrent.CompletableFuture; import java.util.concurrent.CompletionStage; import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.ExecutionException; import java.util.concurrent.Future; +import java.util.function.Supplier; import io.helidon.common.reactive.Single; import io.helidon.faulttolerance.Async; @@ -35,7 +35,6 @@ import io.helidon.faulttolerance.Retry; import io.helidon.faulttolerance.Timeout; -import static io.helidon.microprofile.faulttolerance.AsynchronousUtil.toCompletionStageSupplier; import static io.helidon.microprofile.faulttolerance.ThrowableMapper.map; /** @@ -99,10 +98,9 @@ public Object get() throws Throwable { // Oops, something went wrong during validation throw new InternalError("Validation failed, return type must be Future or CompletionStage"); } else { - // Invoke method and wait on result - single = handler.invoke(toCompletionStageSupplier(context::proceed)); + single = handler.invoke(toCompletionStageSupplier(context::proceed)); try { - // Need to allow completion with no value (i.e. null) for void methods + // Need to allow completion with no value (null) for void methods CompletableFuture future = single.toStage(true).toCompletableFuture(); return future.get(); } catch (ExecutionException e) { @@ -121,40 +119,6 @@ public Object get() throws Throwable { private FtHandlerTyped createHandler() { FaultTolerance.TypedBuilder builder = FaultTolerance.typedBuilder(); - // Create retry (with timeout) handler and add it first - if (introspector.hasRetry()) { - Retry.Builder retry = Retry.builder() - .retryPolicy(Retry.JitterRetryPolicy.builder() - .calls(introspector.getRetry().maxRetries() + 1) - .delay(Duration.ofMillis(introspector.getRetry().delay())) - .jitter(Duration.ofMillis(introspector.getRetry().jitter())) - .build()); - if (introspector.hasTimeout()) { - retry.overallTimeout(Duration.of(introspector.getTimeout().value(), - introspector.getTimeout().unit())); - } - builder.addRetry(retry.build()); - } - - // Create and add fallback handler - if (introspector.hasFallback()) { - Fallback fallback = Fallback.builder() - .fallback(throwable -> { - CommandFallback cfb = new CommandFallback(context, introspector, throwable); - return toCompletionStageSupplier(cfb::execute).get(); - }) - .build(); - builder.addFallback(fallback); - } - - // Create and add timeout handler - if (introspector.hasTimeout() && !introspector.hasRetry()) { - Timeout timeout = Timeout.builder() - .timeout(Duration.of(introspector.getTimeout().value(), introspector.getTimeout().unit())) - .build(); - builder.addTimeout(timeout); - } - // Create and add circuit breaker if (introspector.hasCircuitBreaker()) { CircuitBreaker circuitBreaker = CircuitBreaker.builder() @@ -177,6 +141,57 @@ private FtHandlerTyped createHandler() { builder.addBulkhead(bulkhead); } + // Create and add timeout handler -- parent of breaker or bulkhead + if (introspector.hasTimeout()) { + Timeout timeout = Timeout.builder() + .timeout(Duration.of(introspector.getTimeout().value(), introspector.getTimeout().unit())) + .build(); + builder.addTimeout(timeout); + } + + // Create and add retry handler -- parent of timeout + if (introspector.hasRetry()) { + Retry.Builder retry = Retry.builder() + .retryPolicy(Retry.JitterRetryPolicy.builder() + .calls(introspector.getRetry().maxRetries() + 1) + .delay(Duration.ofMillis(introspector.getRetry().delay())) + .jitter(Duration.ofMillis(introspector.getRetry().jitter())) + .build()) + .overallTimeout(Duration.ofNanos(Long.MAX_VALUE)); // not used + builder.addRetry(retry.build()); + } + + // Create and add fallback handler -- parent of retry + if (introspector.hasFallback()) { + Fallback fallback = Fallback.builder() + .fallback(throwable -> { + CommandFallback cfb = new CommandFallback(context, introspector, throwable); + return toCompletionStageSupplier(cfb::execute).get(); + }) + .build(); + builder.addFallback(fallback); + } + return builder.build(); } + + /** + * Maps an {@link FtSupplier} to a supplier of {@link CompletionStage}. Avoids + * unnecessary wrapping of stages. + * + * @param supplier The supplier. + * @return The new supplier. + */ + @SuppressWarnings("unchecked") + static Supplier> toCompletionStageSupplier(FtSupplier supplier) { + return () -> { + try { + Object result = supplier.get(); + return result instanceof CompletionStage ? (CompletionStage) result + : CompletableFuture.completedFuture(result); + } catch (Throwable e) { + return CompletableFuture.failedFuture(e); + } + }; + } } diff --git a/microprofile/fault-tolerance/src/main/java/io/helidon/microprofile/faulttolerance/ThrowableMapper.java b/microprofile/fault-tolerance/src/main/java/io/helidon/microprofile/faulttolerance/ThrowableMapper.java index 5f3ec94bd9f..d5c0612fe58 100644 --- a/microprofile/fault-tolerance/src/main/java/io/helidon/microprofile/faulttolerance/ThrowableMapper.java +++ b/microprofile/fault-tolerance/src/main/java/io/helidon/microprofile/faulttolerance/ThrowableMapper.java @@ -18,6 +18,7 @@ import org.eclipse.microprofile.faulttolerance.exceptions.BulkheadException; import org.eclipse.microprofile.faulttolerance.exceptions.CircuitBreakerOpenException; +import org.eclipse.microprofile.faulttolerance.exceptions.TimeoutException; /** * Maps Helidon to MP exceptions. @@ -34,6 +35,12 @@ static Throwable map(Throwable t) { if (t instanceof io.helidon.faulttolerance.BulkheadException) { return new BulkheadException(t.getMessage(), t.getCause()); } + if (t instanceof java.util.concurrent.TimeoutException) { + return new TimeoutException(t.getMessage(), t.getCause()); + } + if (t instanceof java.lang.InterruptedException) { + return new TimeoutException(t.getMessage(), t.getCause()); + } return t; } } diff --git a/microprofile/fault-tolerance/src/test/java/io/helidon/microprofile/faulttolerance/TimeoutBean.java b/microprofile/fault-tolerance/src/test/java/io/helidon/microprofile/faulttolerance/TimeoutBean.java index 71fd98c4ed0..766f993129f 100644 --- a/microprofile/fault-tolerance/src/test/java/io/helidon/microprofile/faulttolerance/TimeoutBean.java +++ b/microprofile/fault-tolerance/src/test/java/io/helidon/microprofile/faulttolerance/TimeoutBean.java @@ -54,7 +54,7 @@ public String timeoutWithRetries() throws InterruptedException { FaultToleranceTest.printStatus("TimeoutBean::timeoutWithRetries()", duration.get() < 1000 ? "success" : "failure"); Thread.sleep(duration.getAndAdd(-400)); // needs 2 retries - return "success"; + return duration.get() < 1000 ? "success" : "failure"; } @Fallback(fallbackMethod = "onFailure") From 8792c04c1106f8f032ee82bddcee20fdf5f44efb Mon Sep 17 00:00:00 2001 From: Santiago Pericasgeertsen Date: Thu, 16 Jul 2020 12:55:07 -0400 Subject: [PATCH 07/65] Improvements to better support sync and async timeouts. Signed-off-by: Santiago Pericasgeertsen --- .../io/helidon/faulttolerance/Timeout.java | 18 ++++++++- .../helidon/faulttolerance/TimeoutImpl.java | 39 ++++++++++++------- .../faulttolerance/CommandRunner.java | 17 ++++---- .../faulttolerance/ThrowableMapper.java | 4 +- .../faulttolerance/TimeoutBean.java | 11 ++++++ .../faulttolerance/TimeoutTest.java | 18 +++++++++ 6 files changed, 82 insertions(+), 25 deletions(-) diff --git a/fault-tolerance/src/main/java/io/helidon/faulttolerance/Timeout.java b/fault-tolerance/src/main/java/io/helidon/faulttolerance/Timeout.java index 952e232e09d..a792f4dae10 100644 --- a/fault-tolerance/src/main/java/io/helidon/faulttolerance/Timeout.java +++ b/fault-tolerance/src/main/java/io/helidon/faulttolerance/Timeout.java @@ -52,13 +52,14 @@ static Timeout create(Duration timeout) { class Builder implements io.helidon.common.Builder { private Duration timeout = Duration.ofSeconds(10); private LazyValue executor = FaultTolerance.scheduledExecutor(); + private boolean async = true; private Builder() { } @Override public Timeout build() { - return new TimeoutImpl(this); + return new TimeoutImpl(this, async); } /** @@ -72,6 +73,17 @@ public Builder timeout(Duration timeout) { return this; } + /** + * Async flag. If async, code will execute in another thread. Default is {@code true}. + * + * @param async async setting for this timeout; + * @return updated builder instance + */ + public Builder async(boolean async) { + this.async = async; + return this; + } + /** * Executor service to schedule the timeout. * @@ -90,5 +102,9 @@ Duration timeout() { LazyValue executor() { return executor; } + + boolean async() { + return async; + } } } diff --git a/fault-tolerance/src/main/java/io/helidon/faulttolerance/TimeoutImpl.java b/fault-tolerance/src/main/java/io/helidon/faulttolerance/TimeoutImpl.java index 7f14a7561eb..fcc835b64f8 100644 --- a/fault-tolerance/src/main/java/io/helidon/faulttolerance/TimeoutImpl.java +++ b/fault-tolerance/src/main/java/io/helidon/faulttolerance/TimeoutImpl.java @@ -16,6 +16,7 @@ package io.helidon.faulttolerance; +import java.time.Duration; import java.util.concurrent.CompletionStage; import java.util.concurrent.Flow; import java.util.concurrent.ScheduledExecutorService; @@ -30,32 +31,42 @@ class TimeoutImpl implements Timeout { private final long timeoutMillis; private final LazyValue executor; + private final boolean async; - TimeoutImpl(Timeout.Builder builder) { + TimeoutImpl(Timeout.Builder builder, boolean async) { this.timeoutMillis = builder.timeout().toMillis(); this.executor = builder.executor(); + this.async = async; } @Override public Multi invokeMulti(Supplier> supplier) { + if (!async) { + throw new UnsupportedOperationException("Timeout with Publisher must be async"); + } return Multi.create(supplier.get()) .timeout(timeoutMillis, TimeUnit.MILLISECONDS, executor.get()); } @Override public Single invoke(Supplier> supplier) { - System.out.println("TimeoutImpl.invoke called"); - final AtomicBoolean running = new AtomicBoolean(true); - final Thread supplierThread = Thread.currentThread(); - ScheduledExecutorService service = executor.get(); - service.schedule(() -> { - if (running.get()) { - System.out.println("Thread interrupted!"); - supplierThread.interrupt(); - } - }, timeoutMillis, TimeUnit.MILLISECONDS); - CompletionStage stage = supplier.get(); - running.compareAndSet(true, false); - return Single.create(stage, true); + if (async) { + return Single.create(supplier.get()) + .timeout(timeoutMillis, TimeUnit.MILLISECONDS, executor.get()); + } else { + Thread currentThread = Thread.currentThread(); + AtomicBoolean called = new AtomicBoolean(); + Timeout.create(Duration.ofMillis(timeoutMillis)) + .invoke(Single::never) + .exceptionally(it -> { + if (called.compareAndSet(false, true)) { + currentThread.interrupt(); + } + return null; + }); + Single single = Single.create(supplier.get()); // runs in current thread + called.set(true); + return single; + } } } diff --git a/microprofile/fault-tolerance/src/main/java/io/helidon/microprofile/faulttolerance/CommandRunner.java b/microprofile/fault-tolerance/src/main/java/io/helidon/microprofile/faulttolerance/CommandRunner.java index 6a64b15fe32..604c714f701 100644 --- a/microprofile/fault-tolerance/src/main/java/io/helidon/microprofile/faulttolerance/CommandRunner.java +++ b/microprofile/fault-tolerance/src/main/java/io/helidon/microprofile/faulttolerance/CommandRunner.java @@ -85,14 +85,14 @@ public Object get() throws Throwable { single = Async.create().invoke(() -> handler.invoke(toCompletionStageSupplier(context::proceed))).get(); - // If return type is CompletionStage, convert it - if (introspector.isReturnType(CompletionStage.class)) { - return single.toStage(); - } - - // Otherwise, must be CompletableFuture or Future - if (introspector.isReturnType(Future.class)) { - return single.toCompletableFuture(); + // Return future after mapping exceptions + if (introspector.isReturnType(CompletionStage.class) || introspector.isReturnType(Future.class)) { + return single.handle((o, t) -> { + if (t != null) { + throw map(t instanceof ExecutionException ? t.getCause() : t); + } + return o; + }).toCompletableFuture(); } // Oops, something went wrong during validation @@ -145,6 +145,7 @@ private FtHandlerTyped createHandler() { if (introspector.hasTimeout()) { Timeout timeout = Timeout.builder() .timeout(Duration.of(introspector.getTimeout().value(), introspector.getTimeout().unit())) + .async(false) // no async here .build(); builder.addTimeout(timeout); } diff --git a/microprofile/fault-tolerance/src/main/java/io/helidon/microprofile/faulttolerance/ThrowableMapper.java b/microprofile/fault-tolerance/src/main/java/io/helidon/microprofile/faulttolerance/ThrowableMapper.java index d5c0612fe58..b8023de3e5f 100644 --- a/microprofile/fault-tolerance/src/main/java/io/helidon/microprofile/faulttolerance/ThrowableMapper.java +++ b/microprofile/fault-tolerance/src/main/java/io/helidon/microprofile/faulttolerance/ThrowableMapper.java @@ -28,7 +28,7 @@ class ThrowableMapper { private ThrowableMapper() { } - static Throwable map(Throwable t) { + static RuntimeException map(Throwable t) { if (t instanceof io.helidon.faulttolerance.CircuitBreakerOpenException) { return new CircuitBreakerOpenException(t.getMessage(), t.getCause()); } @@ -41,6 +41,6 @@ static Throwable map(Throwable t) { if (t instanceof java.lang.InterruptedException) { return new TimeoutException(t.getMessage(), t.getCause()); } - return t; + return new RuntimeException(t); } } diff --git a/microprofile/fault-tolerance/src/test/java/io/helidon/microprofile/faulttolerance/TimeoutBean.java b/microprofile/fault-tolerance/src/test/java/io/helidon/microprofile/faulttolerance/TimeoutBean.java index 766f993129f..eda2a829732 100644 --- a/microprofile/fault-tolerance/src/test/java/io/helidon/microprofile/faulttolerance/TimeoutBean.java +++ b/microprofile/fault-tolerance/src/test/java/io/helidon/microprofile/faulttolerance/TimeoutBean.java @@ -17,10 +17,13 @@ package io.helidon.microprofile.faulttolerance; import java.time.temporal.ChronoUnit; +import java.util.concurrent.CompletableFuture; +import java.util.concurrent.Future; import java.util.concurrent.atomic.AtomicLong; import javax.enterprise.context.Dependent; +import org.eclipse.microprofile.faulttolerance.Asynchronous; import org.eclipse.microprofile.faulttolerance.Fallback; import org.eclipse.microprofile.faulttolerance.Retry; import org.eclipse.microprofile.faulttolerance.Timeout; @@ -41,6 +44,14 @@ public String forceTimeout() throws InterruptedException { return "failure"; } + @Asynchronous + @Timeout(value=1000, unit=ChronoUnit.MILLIS) + public Future forceTimeoutAsync() throws InterruptedException { + FaultToleranceTest.printStatus("TimeoutBean::forceTimeoutAsync()", "failure"); + Thread.sleep(1500); + return CompletableFuture.completedFuture("failure"); + } + @Timeout(value=1000, unit=ChronoUnit.MILLIS) public String noTimeout() throws InterruptedException { FaultToleranceTest.printStatus("TimeoutBean::noTimeout()", "success"); diff --git a/microprofile/fault-tolerance/src/test/java/io/helidon/microprofile/faulttolerance/TimeoutTest.java b/microprofile/fault-tolerance/src/test/java/io/helidon/microprofile/faulttolerance/TimeoutTest.java index 4d289c058eb..0cd6c4a772b 100644 --- a/microprofile/fault-tolerance/src/test/java/io/helidon/microprofile/faulttolerance/TimeoutTest.java +++ b/microprofile/fault-tolerance/src/test/java/io/helidon/microprofile/faulttolerance/TimeoutTest.java @@ -16,8 +16,13 @@ package io.helidon.microprofile.faulttolerance; +import java.util.concurrent.ExecutionException; +import java.util.concurrent.Future; + import org.eclipse.microprofile.faulttolerance.exceptions.TimeoutException; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.fail; import org.junit.jupiter.api.Test; import static org.hamcrest.MatcherAssert.assertThat; @@ -37,6 +42,19 @@ public void testForceTimeout() { assertThrows(TimeoutException.class, bean::forceTimeout); } + @Test + public void testForceTimeoutAsync() throws Exception { + TimeoutBean bean = newBean(TimeoutBean.class); + Future future = bean.forceTimeoutAsync(); + try { + future.get(); + } catch (ExecutionException e) { + assertEquals(TimeoutException.class, e.getCause().getClass()); + return; + } + fail(); + } + @Test public void testNoTimeout() throws Exception { TimeoutBean bean = newBean(TimeoutBean.class); From 8621fa344761a4972d91fa4c8ed668ad8b4b2909 Mon Sep 17 00:00:00 2001 From: Santiago Pericasgeertsen Date: Thu, 16 Jul 2020 14:07:23 -0400 Subject: [PATCH 08/65] Improved handling of exceptions in async calls. Signed-off-by: Santiago Pericasgeertsen --- .../faulttolerance/CommandRunner.java | 13 ++-- .../faulttolerance/ThrowableMapper.java | 4 +- .../faulttolerance/FaultToleranceTest.java | 49 +++++++++++- .../faulttolerance/RetryTest.java | 76 ++----------------- .../faulttolerance/TimeoutTest.java | 11 +-- 5 files changed, 63 insertions(+), 90 deletions(-) diff --git a/microprofile/fault-tolerance/src/main/java/io/helidon/microprofile/faulttolerance/CommandRunner.java b/microprofile/fault-tolerance/src/main/java/io/helidon/microprofile/faulttolerance/CommandRunner.java index 604c714f701..7f6f53e6e19 100644 --- a/microprofile/fault-tolerance/src/main/java/io/helidon/microprofile/faulttolerance/CommandRunner.java +++ b/microprofile/fault-tolerance/src/main/java/io/helidon/microprofile/faulttolerance/CommandRunner.java @@ -87,12 +87,15 @@ public Object get() throws Throwable { // Return future after mapping exceptions if (introspector.isReturnType(CompletionStage.class) || introspector.isReturnType(Future.class)) { - return single.handle((o, t) -> { - if (t != null) { - throw map(t instanceof ExecutionException ? t.getCause() : t); + CompletableFuture future = new CompletableFuture<>(); + single.whenComplete((o, t) -> { + if (t == null) { + future.complete(o); + } else { + future.completeExceptionally(map(t instanceof ExecutionException ? t.getCause() : t)); } - return o; - }).toCompletableFuture(); + }); + return future; } // Oops, something went wrong during validation diff --git a/microprofile/fault-tolerance/src/main/java/io/helidon/microprofile/faulttolerance/ThrowableMapper.java b/microprofile/fault-tolerance/src/main/java/io/helidon/microprofile/faulttolerance/ThrowableMapper.java index b8023de3e5f..d5c0612fe58 100644 --- a/microprofile/fault-tolerance/src/main/java/io/helidon/microprofile/faulttolerance/ThrowableMapper.java +++ b/microprofile/fault-tolerance/src/main/java/io/helidon/microprofile/faulttolerance/ThrowableMapper.java @@ -28,7 +28,7 @@ class ThrowableMapper { private ThrowableMapper() { } - static RuntimeException map(Throwable t) { + static Throwable map(Throwable t) { if (t instanceof io.helidon.faulttolerance.CircuitBreakerOpenException) { return new CircuitBreakerOpenException(t.getMessage(), t.getCause()); } @@ -41,6 +41,6 @@ static RuntimeException map(Throwable t) { if (t instanceof java.lang.InterruptedException) { return new TimeoutException(t.getMessage(), t.getCause()); } - return new RuntimeException(t); + return t; } } diff --git a/microprofile/fault-tolerance/src/test/java/io/helidon/microprofile/faulttolerance/FaultToleranceTest.java b/microprofile/fault-tolerance/src/test/java/io/helidon/microprofile/faulttolerance/FaultToleranceTest.java index 4f67e841990..81ba0421043 100644 --- a/microprofile/fault-tolerance/src/test/java/io/helidon/microprofile/faulttolerance/FaultToleranceTest.java +++ b/microprofile/fault-tolerance/src/test/java/io/helidon/microprofile/faulttolerance/FaultToleranceTest.java @@ -16,31 +16,42 @@ package io.helidon.microprofile.faulttolerance; +import javax.enterprise.inject.literal.NamedLiteral; +import javax.enterprise.inject.se.SeContainer; +import javax.enterprise.inject.spi.CDI; import java.util.Arrays; import java.util.Set; import java.util.concurrent.CompletableFuture; +import java.util.concurrent.CompletionStage; +import java.util.concurrent.ExecutionException; import java.util.concurrent.Executor; import java.util.concurrent.Executors; import java.util.concurrent.Future; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.TimeoutException; import java.util.function.Supplier; import java.util.stream.Collectors; import java.util.stream.Stream; -import javax.enterprise.inject.literal.NamedLiteral; -import javax.enterprise.inject.se.SeContainer; -import javax.enterprise.inject.spi.CDI; - import io.helidon.microprofile.cdi.HelidonContainer; import org.junit.jupiter.api.AfterAll; import org.junit.jupiter.api.BeforeAll; import org.junit.jupiter.api.BeforeEach; +import static org.junit.jupiter.api.Assertions.fail; +import static org.hamcrest.MatcherAssert.assertThat; +import static org.hamcrest.Matchers.instanceOf; +import static org.hamcrest.Matchers.is; + /** * Class FaultToleranceTest. */ public abstract class FaultToleranceTest { + private static final long TIMEOUT = 5000; + private static final TimeUnit TIMEOUT_UNITS = TimeUnit.MILLISECONDS; + private static SeContainer cdiContainer; private static final int NUMBER_OF_THREADS = 20; @@ -104,4 +115,34 @@ static Set getThreadNames(Future[] calls) { } }).collect(Collectors.toSet()); } + + static void assertCompleteExceptionally(Future future, + Class exceptionClass) { + assertCompleteExceptionally(future, exceptionClass, null); + } + + static void assertCompleteExceptionally(Future future, + Class exceptionClass, + String exceptionMessage) { + try { + future.get(TIMEOUT, TIMEOUT_UNITS); + fail("Expected exception: " + exceptionClass.getName()); + } catch (InterruptedException | TimeoutException e) { + fail("Unexpected exception " + e, e); + } catch (ExecutionException ee) { + assertThat("Cause of ExecutionException", ee.getCause(), instanceOf(exceptionClass)); + if (exceptionMessage != null) { + assertThat(ee.getCause().getMessage(), is(exceptionMessage)); + } + } + } + + static void assertCompleteOk(CompletionStage future, String expectedMessage) { + try { + CompletableFuture cf = future.toCompletableFuture(); + assertThat(cf.get(TIMEOUT, TIMEOUT_UNITS), is(expectedMessage)); + } catch (Exception e) { + fail("Unexpected exception" + e); + } + } } diff --git a/microprofile/fault-tolerance/src/test/java/io/helidon/microprofile/faulttolerance/RetryTest.java b/microprofile/fault-tolerance/src/test/java/io/helidon/microprofile/faulttolerance/RetryTest.java index 8ad0116b380..457d44758d5 100644 --- a/microprofile/fault-tolerance/src/test/java/io/helidon/microprofile/faulttolerance/RetryTest.java +++ b/microprofile/fault-tolerance/src/test/java/io/helidon/microprofile/faulttolerance/RetryTest.java @@ -17,12 +17,8 @@ package io.helidon.microprofile.faulttolerance; import java.io.IOException; -import java.util.concurrent.CompletableFuture; import java.util.concurrent.CompletionStage; -import java.util.concurrent.ExecutionException; -import java.util.concurrent.Future; -import java.util.concurrent.TimeUnit; -import java.util.concurrent.TimeoutException; +import java.util.concurrent.Future;; import java.util.stream.Stream; import org.junit.jupiter.params.ParameterizedTest; @@ -31,23 +27,17 @@ import static org.hamcrest.MatcherAssert.assertThat; import static org.hamcrest.Matchers.greaterThan; -import static org.hamcrest.Matchers.instanceOf; import static org.hamcrest.Matchers.is; -import static org.junit.jupiter.api.Assertions.fail; /** - * Class RetryTest. + * Test cases for @Retry. */ public class RetryTest extends FaultToleranceTest { - // parameterize these for ease of debugging - private static final long TIMEOUT = 1000; - private static final TimeUnit TIMEOUT_UNITS = TimeUnit.MILLISECONDS; - static Stream createBeans() { return Stream.of( - Arguments.of(newBean(RetryBean.class), "ManagedRetryBean")); - // Arguments.of(newNamedBean(SyntheticRetryBean.class), "SyntheticRetryBean")); + Arguments.of(newBean(RetryBean.class), "ManagedRetryBean"), + Arguments.of(newNamedBean(SyntheticRetryBean.class), "SyntheticRetryBean")); } @ParameterizedTest(name = "{1}") @@ -79,7 +69,7 @@ public void testRetryAsync(RetryBean bean, String unused) throws Exception { @MethodSource("createBeans") public void testRetryWithDelayAndJitter(RetryBean bean, String unused) throws Exception { long millis = System.currentTimeMillis(); - String value = bean.retryWithDelayAndJitter(); + bean.retryWithDelayAndJitter(); assertThat(System.currentTimeMillis() - millis, greaterThan(200L)); } @@ -93,9 +83,8 @@ public void testRetryWithDelayAndJitter(RetryBean bean, String unused) throws Ex @ParameterizedTest(name = "{1}") @MethodSource("createBeans") public void testRetryWithException(RetryBean bean, String unused) throws Exception { - final CompletionStage future = bean.retryWithException(); - - assertCompleteExceptionally(future, IOException.class, "Simulated error"); + CompletionStage future = bean.retryWithException(); + assertCompleteExceptionally(future.toCompletableFuture(), IOException.class, "Simulated error"); assertThat(bean.getInvocations(), is(3)); } @@ -105,55 +94,4 @@ public void testRetryCompletionStageWithEventualSuccess(RetryBean bean, String u assertCompleteOk(bean.retryWithUltimateSuccess(), "Success"); assertThat(bean.getInvocations(), is(3)); } - - private void assertCompleteOk(final CompletionStage future, final String expectedMessage) { - try { - CompletableFuture cf = toCompletableFuture(future); - assertThat(cf.get(TIMEOUT, TIMEOUT_UNITS), is(expectedMessage)); - } - catch (Exception e) { - fail("Unexpected exception" + e); - } - } - - private void assertCompleteExceptionally(final CompletionStage future, - final Class exceptionClass, - final String exceptionMessage) { - try { - Object result = toCompletableFuture(future).get(TIMEOUT, TIMEOUT_UNITS); - fail("Expected exception: " + exceptionClass.getName() + " with message: " + exceptionMessage); - } - catch (InterruptedException | TimeoutException e) { - fail("Unexpected exception " + e, e); - } - catch (ExecutionException ee) { - assertThat("Cause of ExecutionException", ee.getCause(), instanceOf(exceptionClass)); - assertThat(ee.getCause().getMessage(), is(exceptionMessage)); - } - } - - /** - * Returns a future that is completed when the stage is completed and has the same value or exception - * as the completed stage. It's supposed to be equivalent to calling - * {@link CompletionStage#toCompletableFuture()} but works with any CompletionStage - * and doesn't throw {@link java.lang.UnsupportedOperationException}. - * - * @param The type of the future result - * @param stage Stage to convert to a future - * @return Future converted from stage - */ - public static CompletableFuture toCompletableFuture(CompletionStage stage) { - CompletableFuture future = new CompletableFuture<>(); - stage.whenComplete((v, e) -> { - if (e != null) { - future.completeExceptionally(e); - } - else { - future.complete(v); - } - }); - return future; - } - - } diff --git a/microprofile/fault-tolerance/src/test/java/io/helidon/microprofile/faulttolerance/TimeoutTest.java b/microprofile/fault-tolerance/src/test/java/io/helidon/microprofile/faulttolerance/TimeoutTest.java index 0cd6c4a772b..f82b3fec9aa 100644 --- a/microprofile/fault-tolerance/src/test/java/io/helidon/microprofile/faulttolerance/TimeoutTest.java +++ b/microprofile/fault-tolerance/src/test/java/io/helidon/microprofile/faulttolerance/TimeoutTest.java @@ -16,13 +16,10 @@ package io.helidon.microprofile.faulttolerance; -import java.util.concurrent.ExecutionException; import java.util.concurrent.Future; import org.eclipse.microprofile.faulttolerance.exceptions.TimeoutException; -import static org.junit.jupiter.api.Assertions.assertEquals; -import static org.junit.jupiter.api.Assertions.fail; import org.junit.jupiter.api.Test; import static org.hamcrest.MatcherAssert.assertThat; @@ -46,13 +43,7 @@ public void testForceTimeout() { public void testForceTimeoutAsync() throws Exception { TimeoutBean bean = newBean(TimeoutBean.class); Future future = bean.forceTimeoutAsync(); - try { - future.get(); - } catch (ExecutionException e) { - assertEquals(TimeoutException.class, e.getCause().getClass()); - return; - } - fail(); + assertCompleteExceptionally(future, TimeoutException.class); } @Test From afcb9d4ce9382da64e538d6e1c6970916bc0c0fd Mon Sep 17 00:00:00 2001 From: Santiago Pericasgeertsen Date: Thu, 16 Jul 2020 14:29:29 -0400 Subject: [PATCH 09/65] Threshold compared using greater-than instead of greater-or-equal. Signed-off-by: Santiago Pericasgeertsen --- .../src/main/java/io/helidon/faulttolerance/ResultWindow.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/fault-tolerance/src/main/java/io/helidon/faulttolerance/ResultWindow.java b/fault-tolerance/src/main/java/io/helidon/faulttolerance/ResultWindow.java index 77decdac18d..5e5b46e616f 100644 --- a/fault-tolerance/src/main/java/io/helidon/faulttolerance/ResultWindow.java +++ b/fault-tolerance/src/main/java/io/helidon/faulttolerance/ResultWindow.java @@ -63,7 +63,7 @@ void update(Result resultEnum) { } boolean shouldOpen() { - return currentSum.get() >= thresholdSum; + return currentSum.get() > thresholdSum; } void reset() { From c7d67aeffaa2b4ace5a30c1f9b443c53966e4292 Mon Sep 17 00:00:00 2001 From: Santiago Pericasgeertsen Date: Mon, 20 Jul 2020 14:10:40 -0400 Subject: [PATCH 10/65] Changes in this commit: - Support for sync (no queues) in Bulkhead - Fixed validation in FallbackAntn - Fixes to CommandRunner async support - Some changes in unit tests while preserving coverage All unit tests for Timeout, Fallback, Retry, Asynchronous, Bulkhead and CircuitBreaker are passing. Signed-off-by: Santiago Pericasgeertsen --- .../io/helidon/faulttolerance/Bulkhead.java | 16 +++ .../helidon/faulttolerance/BulkheadImpl.java | 6 +- .../faulttolerance/CommandRunner.java | 32 ++++-- .../faulttolerance/FallbackAntn.java | 10 +- .../faulttolerance/ThrowableMapper.java | 5 + .../faulttolerance/BulkheadBean.java | 100 +++++++++++++----- .../faulttolerance/BulkheadTest.java | 16 +-- .../faulttolerance/FaultToleranceTest.java | 10 +- .../faulttolerance/MetricsTest.java | 12 +-- 9 files changed, 151 insertions(+), 56 deletions(-) diff --git a/fault-tolerance/src/main/java/io/helidon/faulttolerance/Bulkhead.java b/fault-tolerance/src/main/java/io/helidon/faulttolerance/Bulkhead.java index f54bd53b491..fa09f86489a 100644 --- a/fault-tolerance/src/main/java/io/helidon/faulttolerance/Bulkhead.java +++ b/fault-tolerance/src/main/java/io/helidon/faulttolerance/Bulkhead.java @@ -52,6 +52,7 @@ class Builder implements io.helidon.common.Builder { private int limit = DEFAULT_LIMIT; private int queueLength = DEFAULT_QUEUE_LENGTH; private String name = "Bulkhead-" + System.identityHashCode(this); + private boolean async = true; private Builder() { } @@ -108,6 +109,17 @@ public Builder name(String name) { return this; } + /** + * Runs bulkhead tasks using an executor. By default is set to {@code true}. + * + * @param async setting for async + * @return updated builder instance + */ + public Builder async(boolean async) { + this.async = async; + return this; + } + int limit() { return limit; } @@ -123,6 +135,10 @@ LazyValue executor() { String name() { return name; } + + boolean async() { + return async; + } } } diff --git a/fault-tolerance/src/main/java/io/helidon/faulttolerance/BulkheadImpl.java b/fault-tolerance/src/main/java/io/helidon/faulttolerance/BulkheadImpl.java index 8d30d307853..fa5af1d3b01 100644 --- a/fault-tolerance/src/main/java/io/helidon/faulttolerance/BulkheadImpl.java +++ b/fault-tolerance/src/main/java/io/helidon/faulttolerance/BulkheadImpl.java @@ -37,11 +37,13 @@ class BulkheadImpl implements Bulkhead { private final Queue> queue; private final Semaphore inProgress; private final String name; + private final boolean async; BulkheadImpl(Bulkhead.Builder builder) { this.executor = builder.executor(); this.inProgress = new Semaphore(builder.limit(), true); this.name = builder.name(); + this.async = builder.async(); if (builder.queueLength() == 0) { queue = new NoQueue(); @@ -68,8 +70,8 @@ private R invokeTask(DelayedTask task) { execute(task); return task.result(); } else { - // no free permit, let's try to enqueue - if (queue.offer(task)) { + // no free permit, let's try to enqueue in async mode + if (async && queue.offer(task)) { LOGGER.finest(() -> name + " enqueue: " + task); return task.result(); } else { diff --git a/microprofile/fault-tolerance/src/main/java/io/helidon/microprofile/faulttolerance/CommandRunner.java b/microprofile/fault-tolerance/src/main/java/io/helidon/microprofile/faulttolerance/CommandRunner.java index 7f6f53e6e19..5fe77ad1fb8 100644 --- a/microprofile/fault-tolerance/src/main/java/io/helidon/microprofile/faulttolerance/CommandRunner.java +++ b/microprofile/fault-tolerance/src/main/java/io/helidon/microprofile/faulttolerance/CommandRunner.java @@ -34,7 +34,6 @@ import io.helidon.faulttolerance.FtHandlerTyped; import io.helidon.faulttolerance.Retry; import io.helidon.faulttolerance.Timeout; - import static io.helidon.microprofile.faulttolerance.ThrowableMapper.map; /** @@ -81,18 +80,34 @@ public Object get() throws Throwable { Single single; if (introspector.isAsynchronous()) { - // Invoke method in new thread and call get() to unwrap singles - single = Async.create().invoke(() -> - handler.invoke(toCompletionStageSupplier(context::proceed))).get(); - - // Return future after mapping exceptions if (introspector.isReturnType(CompletionStage.class) || introspector.isReturnType(Future.class)) { + // Invoke method in new thread and call get() to unwrap singles + single = Async.create().invoke(() -> + handler.invoke(toCompletionStageSupplier(context::proceed))); + + // Unwrap nested futures and map exceptions on complete CompletableFuture future = new CompletableFuture<>(); single.whenComplete((o, t) -> { if (t == null) { - future.complete(o); + // If future whose value is a future, then unwrap them + Future delegate = null; + if (o instanceof CompletionStage) { + delegate = ((CompletionStage) o).toCompletableFuture(); + } + else if (o instanceof Future) { + delegate = (Future) o; + } + if (delegate != null) { + try { + future.complete(delegate.get()); + } catch (Exception e) { + future.completeExceptionally(map(e)); + } + } else { + future.complete(o); + } } else { - future.completeExceptionally(map(t instanceof ExecutionException ? t.getCause() : t)); + future.completeExceptionally(map(t)); } }); return future; @@ -140,6 +155,7 @@ private FtHandlerTyped createHandler() { Bulkhead bulkhead = Bulkhead.builder() .limit(introspector.getBulkhead().value()) .queueLength(introspector.getBulkhead().waitingTaskQueue()) + .async(introspector.isAsynchronous()) .build(); builder.addBulkhead(bulkhead); } diff --git a/microprofile/fault-tolerance/src/main/java/io/helidon/microprofile/faulttolerance/FallbackAntn.java b/microprofile/fault-tolerance/src/main/java/io/helidon/microprofile/faulttolerance/FallbackAntn.java index d1e40cffdec..65577380af7 100644 --- a/microprofile/fault-tolerance/src/main/java/io/helidon/microprofile/faulttolerance/FallbackAntn.java +++ b/microprofile/fault-tolerance/src/main/java/io/helidon/microprofile/faulttolerance/FallbackAntn.java @@ -58,11 +58,11 @@ public void validate() { final Method fallbackMethod = JavaMethodFinder.findMethod(method.getDeclaringClass(), methodName, method.getGenericParameterTypes()); - if (!fallbackMethod.getReturnType().isAssignableFrom(method.getReturnType()) - && !Future.class.isAssignableFrom(method.getReturnType()) - && !CompletionStage.class.isAssignableFrom(method.getReturnType())) { // async - throw new FaultToleranceDefinitionException("Fallback method return type " - + "is invalid: " + fallbackMethod.getReturnType()); + if (!method.getReturnType().isAssignableFrom(fallbackMethod.getReturnType())) { + throw new FaultToleranceDefinitionException("Fallback method " + fallbackMethod.getName() + + " in class " + fallbackMethod.getDeclaringClass().getSimpleName() + + " incompatible return type " + fallbackMethod.getReturnType() + + " with " + method.getReturnType()); } } catch (NoSuchMethodException e) { throw new FaultToleranceDefinitionException(e); diff --git a/microprofile/fault-tolerance/src/main/java/io/helidon/microprofile/faulttolerance/ThrowableMapper.java b/microprofile/fault-tolerance/src/main/java/io/helidon/microprofile/faulttolerance/ThrowableMapper.java index d5c0612fe58..6830dd15650 100644 --- a/microprofile/fault-tolerance/src/main/java/io/helidon/microprofile/faulttolerance/ThrowableMapper.java +++ b/microprofile/fault-tolerance/src/main/java/io/helidon/microprofile/faulttolerance/ThrowableMapper.java @@ -16,6 +16,8 @@ package io.helidon.microprofile.faulttolerance; +import java.util.concurrent.ExecutionException; + import org.eclipse.microprofile.faulttolerance.exceptions.BulkheadException; import org.eclipse.microprofile.faulttolerance.exceptions.CircuitBreakerOpenException; import org.eclipse.microprofile.faulttolerance.exceptions.TimeoutException; @@ -29,6 +31,9 @@ private ThrowableMapper() { } static Throwable map(Throwable t) { + if (t instanceof ExecutionException) { + t = t.getCause(); + } if (t instanceof io.helidon.faulttolerance.CircuitBreakerOpenException) { return new CircuitBreakerOpenException(t.getMessage(), t.getCause()); } diff --git a/microprofile/fault-tolerance/src/test/java/io/helidon/microprofile/faulttolerance/BulkheadBean.java b/microprofile/fault-tolerance/src/test/java/io/helidon/microprofile/faulttolerance/BulkheadBean.java index 5a5d40ca758..3b574fba428 100644 --- a/microprofile/fault-tolerance/src/test/java/io/helidon/microprofile/faulttolerance/BulkheadBean.java +++ b/microprofile/fault-tolerance/src/test/java/io/helidon/microprofile/faulttolerance/BulkheadBean.java @@ -32,62 +32,114 @@ public class BulkheadBean { static final int CONCURRENT_CALLS = 3; - static final int WAITING_TASK_QUEUE = 3; + static final int TOTAL_CALLS = CONCURRENT_CALLS + WAITING_TASK_QUEUE; + + static class ConcurrencyCounter { - static final int MAX_CONCURRENT_CALLS = CONCURRENT_CALLS + WAITING_TASK_QUEUE; + private int currentCalls; + private int concurrentCalls; + private int totalCalls; + + synchronized void increment() { + currentCalls++; + if (currentCalls > concurrentCalls) { + concurrentCalls = currentCalls; + } + totalCalls++; + } + + synchronized void decrement() { + currentCalls--; + } + + synchronized int concurrentCalls() { + return concurrentCalls; + } + + synchronized int totalCalls() { + return totalCalls; + } + } + + private ConcurrencyCounter counter = new ConcurrencyCounter(); + + ConcurrencyCounter getCounter() { + return counter; + } @Asynchronous @Bulkhead(value = CONCURRENT_CALLS, waitingTaskQueue = WAITING_TASK_QUEUE) public Future execute(long sleepMillis) { - FaultToleranceTest.printStatus("BulkheadBean::execute", "success"); try { - Thread.sleep(sleepMillis); - } catch (InterruptedException e) { - // falls through + counter.increment(); + FaultToleranceTest.printStatus("BulkheadBean::execute", "success"); + try { + Thread.sleep(sleepMillis); + } catch (InterruptedException e) { + // falls through + } + return CompletableFuture.completedFuture(Thread.currentThread().getName()); + } finally { + counter.decrement(); } - return CompletableFuture.completedFuture(Thread.currentThread().getName()); } @Asynchronous @Bulkhead(value = CONCURRENT_CALLS + 1, waitingTaskQueue = WAITING_TASK_QUEUE + 1) public Future executePlusOne(long sleepMillis) { - FaultToleranceTest.printStatus("BulkheadBean::executePlusOne", "success"); try { - Thread.sleep(sleepMillis); - } catch (InterruptedException e) { - // falls through + counter.increment(); + FaultToleranceTest.printStatus("BulkheadBean::executePlusOne", "success"); + try { + Thread.sleep(sleepMillis); + } catch (InterruptedException e) { + // falls through + } + return CompletableFuture.completedFuture(Thread.currentThread().getName()); + } finally { + counter.decrement(); } - return CompletableFuture.completedFuture(Thread.currentThread().getName()); } @Asynchronous @Bulkhead(value = 2, waitingTaskQueue = 1) public Future executeNoQueue(long sleepMillis) { - FaultToleranceTest.printStatus("BulkheadBean::executeNoQueue", "success"); try { - Thread.sleep(sleepMillis); - } catch (InterruptedException e) { - // falls through + counter.increment(); + FaultToleranceTest.printStatus("BulkheadBean::executeNoQueue", "success"); + try { + Thread.sleep(sleepMillis); + } catch (InterruptedException e) { + // falls through + } + return CompletableFuture.completedFuture(Thread.currentThread().getName()); + } finally { + counter.decrement(); } - return CompletableFuture.completedFuture(Thread.currentThread().getName()); } + @Asynchronous @Fallback(fallbackMethod = "onFailure") @Bulkhead(value = 2, waitingTaskQueue = 1) public Future executeNoQueueWithFallback(long sleepMillis) { - FaultToleranceTest.printStatus("BulkheadBean::executeNoQueue", "success"); try { - Thread.sleep(sleepMillis); - } catch (InterruptedException e) { - // falls through + counter.increment(); + FaultToleranceTest.printStatus("BulkheadBean::executeNoQueue", "success"); + try { + Thread.sleep(sleepMillis); + } catch (InterruptedException e) { + // falls through + } + return CompletableFuture.completedFuture(Thread.currentThread().getName()); + } finally { + counter.decrement(); } - return CompletableFuture.completedFuture(Thread.currentThread().getName()); } - public String onFailure(long sleepMillis) { + public CompletableFuture onFailure(long sleepMillis) { FaultToleranceTest.printStatus("BulkheadBean::onFailure()", "success"); - return Thread.currentThread().getName(); + return CompletableFuture.completedFuture(Thread.currentThread().getName()); } @Asynchronous diff --git a/microprofile/fault-tolerance/src/test/java/io/helidon/microprofile/faulttolerance/BulkheadTest.java b/microprofile/fault-tolerance/src/test/java/io/helidon/microprofile/faulttolerance/BulkheadTest.java index 0eecb17ffe6..d7317fb968e 100644 --- a/microprofile/fault-tolerance/src/test/java/io/helidon/microprofile/faulttolerance/BulkheadTest.java +++ b/microprofile/fault-tolerance/src/test/java/io/helidon/microprofile/faulttolerance/BulkheadTest.java @@ -41,16 +41,20 @@ public class BulkheadTest extends FaultToleranceTest { public void testBulkhead() { BulkheadBean bean = newBean(BulkheadBean.class); Future[] calls = getAsyncConcurrentCalls( - () -> bean.execute(100), BulkheadBean.MAX_CONCURRENT_CALLS); - assertThat(getThreadNames(calls).size(), is(BulkheadBean.CONCURRENT_CALLS)); + () -> bean.execute(100), BulkheadBean.TOTAL_CALLS); + waitFor(calls); + assertThat(bean.getCounter().concurrentCalls(), is(BulkheadBean.CONCURRENT_CALLS)); + assertThat(bean.getCounter().totalCalls(), is(BulkheadBean.TOTAL_CALLS)); } @Test public void testBulkheadPlusOne() { BulkheadBean bean = newBean(BulkheadBean.class); Future[] calls = getAsyncConcurrentCalls( - () -> bean.executePlusOne(100), BulkheadBean.MAX_CONCURRENT_CALLS + 2); - assertThat(getThreadNames(calls).size(), is(BulkheadBean.CONCURRENT_CALLS + 1)); + () -> bean.executePlusOne(100), BulkheadBean.TOTAL_CALLS + 2); + waitFor(calls); + assertThat(bean.getCounter().concurrentCalls(), is(BulkheadBean.CONCURRENT_CALLS + 1)); + assertThat(bean.getCounter().totalCalls(), is(BulkheadBean.TOTAL_CALLS + 2)); } @Test @@ -58,7 +62,7 @@ public void testBulkheadNoQueue() { BulkheadBean bean = newBean(BulkheadBean.class); Future[] calls = getAsyncConcurrentCalls( () -> bean.executeNoQueue(2000), 10); - RuntimeException e = assertThrows(RuntimeException.class, () -> getThreadNames(calls)); + RuntimeException e = assertThrows(RuntimeException.class, () -> waitFor(calls)); assertThat(e.getCause().getCause(), instanceOf(BulkheadException.class)); } @@ -67,7 +71,7 @@ public void testBulkheadNoQueueWithFallback() { BulkheadBean bean = newBean(BulkheadBean.class); Future[] calls = getAsyncConcurrentCalls( () -> bean.executeNoQueueWithFallback(2000), 10); - getThreadNames(calls); + waitFor(calls); } @Test diff --git a/microprofile/fault-tolerance/src/test/java/io/helidon/microprofile/faulttolerance/FaultToleranceTest.java b/microprofile/fault-tolerance/src/test/java/io/helidon/microprofile/faulttolerance/FaultToleranceTest.java index 81ba0421043..b7a6f33e8fc 100644 --- a/microprofile/fault-tolerance/src/test/java/io/helidon/microprofile/faulttolerance/FaultToleranceTest.java +++ b/microprofile/fault-tolerance/src/test/java/io/helidon/microprofile/faulttolerance/FaultToleranceTest.java @@ -103,17 +103,17 @@ static CompletableFuture[] getConcurrentCalls(Supplier supplier, int s @SuppressWarnings("unchecked") static Future[] getAsyncConcurrentCalls(Supplier> supplier, int size) { - return Stream.generate(() -> supplier.get()).limit(size).toArray(Future[]::new); + return Stream.generate(supplier::get).limit(size).toArray(Future[]::new); } - static Set getThreadNames(Future[] calls) { - return Arrays.asList(calls).stream().map(c -> { + static void waitFor(Future[] calls) { + for (Future c : calls) { try { - return c.get(); + c.get(); } catch (Exception e) { throw new RuntimeException(e); } - }).collect(Collectors.toSet()); + } } static void assertCompleteExceptionally(Future future, diff --git a/microprofile/fault-tolerance/src/test/java/io/helidon/microprofile/faulttolerance/MetricsTest.java b/microprofile/fault-tolerance/src/test/java/io/helidon/microprofile/faulttolerance/MetricsTest.java index 3cf613042c2..af1c344a72e 100644 --- a/microprofile/fault-tolerance/src/test/java/io/helidon/microprofile/faulttolerance/MetricsTest.java +++ b/microprofile/fault-tolerance/src/test/java/io/helidon/microprofile/faulttolerance/MetricsTest.java @@ -336,20 +336,20 @@ public void testFallbackMetrics() throws Exception { public void testBulkheadMetrics() throws Exception { MetricsBean bean = newBean(MetricsBean.class); Future[] calls = getAsyncConcurrentCalls( - () -> bean.concurrent(100), BulkheadBean.MAX_CONCURRENT_CALLS); - getThreadNames(calls); + () -> bean.concurrent(100), BulkheadBean.TOTAL_CALLS); + waitFor(calls); assertThat(getGauge(bean, "concurrent", BULKHEAD_CONCURRENT_EXECUTIONS, long.class).getValue(), is(0L)); assertThat(getCounter(bean, "concurrent", BULKHEAD_CALLS_ACCEPTED_TOTAL, long.class), - is((long) BulkheadBean.MAX_CONCURRENT_CALLS)); + is((long) BulkheadBean.TOTAL_CALLS)); assertThat(getCounter(bean, "concurrent", BULKHEAD_CALLS_REJECTED_TOTAL, long.class), is(0L)); assertThat(getHistogram(bean, "concurrent", BULKHEAD_EXECUTION_DURATION, long.class).getCount(), - is((long)BulkheadBean.MAX_CONCURRENT_CALLS)); + is((long)BulkheadBean.TOTAL_CALLS)); } @Test @@ -362,10 +362,10 @@ public void testBulkheadMetricsAsync() throws Exception { } catch (Exception e) { return "failure"; } - }, BulkheadBean.MAX_CONCURRENT_CALLS); + }, BulkheadBean.TOTAL_CALLS); CompletableFuture.allOf(calls).get(); assertThat(getHistogram(bean, "concurrentAsync", BULKHEAD_EXECUTION_DURATION, long.class).getCount(), - is((long)BulkheadBean.MAX_CONCURRENT_CALLS)); + is((long)BulkheadBean.TOTAL_CALLS)); } } From 6528030b68e8048d2cfac13ca93e7335abd1f4c6 Mon Sep 17 00:00:00 2001 From: Santiago Pericasgeertsen Date: Wed, 22 Jul 2020 16:33:55 -0400 Subject: [PATCH 11/65] Changes in this commit: - Support for metrics for all primitives except bulkheads - Changes to SE Retry to count number of retries - Some test fixes Signed-off-by: Santiago Pericasgeertsen --- .../helidon/faulttolerance/ResultWindow.java | 2 +- .../java/io/helidon/faulttolerance/Retry.java | 8 + .../io/helidon/faulttolerance/RetryImpl.java | 15 +- .../faulttolerance/CommandRunner.java | 239 ++++++++++++++++-- .../faulttolerance/CircuitBreakerTest.java | 2 +- .../faulttolerance/MetricsTest.java | 4 +- 6 files changed, 245 insertions(+), 25 deletions(-) diff --git a/fault-tolerance/src/main/java/io/helidon/faulttolerance/ResultWindow.java b/fault-tolerance/src/main/java/io/helidon/faulttolerance/ResultWindow.java index 5e5b46e616f..77decdac18d 100644 --- a/fault-tolerance/src/main/java/io/helidon/faulttolerance/ResultWindow.java +++ b/fault-tolerance/src/main/java/io/helidon/faulttolerance/ResultWindow.java @@ -63,7 +63,7 @@ void update(Result resultEnum) { } boolean shouldOpen() { - return currentSum.get() > thresholdSum; + return currentSum.get() >= thresholdSum; } void reset() { diff --git a/fault-tolerance/src/main/java/io/helidon/faulttolerance/Retry.java b/fault-tolerance/src/main/java/io/helidon/faulttolerance/Retry.java index 0cbf0a08592..757f5ce8482 100644 --- a/fault-tolerance/src/main/java/io/helidon/faulttolerance/Retry.java +++ b/fault-tolerance/src/main/java/io/helidon/faulttolerance/Retry.java @@ -422,4 +422,12 @@ public Builder jitter(Duration jitter) { } } } + + /** + * Number of times a method called has been retried. This is a monotonically + * increasing counter over the lifetime of the handler. + * + * @return number ot times a method is retried. + */ + long retryCounter(); } diff --git a/fault-tolerance/src/main/java/io/helidon/faulttolerance/RetryImpl.java b/fault-tolerance/src/main/java/io/helidon/faulttolerance/RetryImpl.java index c8720b8ca6a..2d5bb3c3fee 100644 --- a/fault-tolerance/src/main/java/io/helidon/faulttolerance/RetryImpl.java +++ b/fault-tolerance/src/main/java/io/helidon/faulttolerance/RetryImpl.java @@ -37,6 +37,7 @@ class RetryImpl implements Retry { private final ErrorChecker errorChecker; private final long maxTimeNanos; private final Retry.RetryPolicy retryPolicy; + private final AtomicLong retryCounter = new AtomicLong(0L); RetryImpl(Retry.Builder builder) { this.scheduledExecutor = builder.scheduledExecutor(); @@ -75,6 +76,10 @@ private Single retrySingle(RetryContext> con + TimeUnit.NANOSECONDS.toMillis(maxTimeNanos) + " ms.")); } + if (currentCallIndex > 0) { + retryCounter.getAndIncrement(); + } + DelayedTask> task = DelayedTask.createSingle(context.supplier); if (delay == 0) { task.execute(); @@ -94,7 +99,6 @@ private Single retrySingle(RetryContext> con } private Multi retryMulti(RetryContext> context) { - long delay = 0; int currentCallIndex = context.count.getAndIncrement(); if (currentCallIndex != 0) { @@ -114,6 +118,10 @@ private Multi retryMulti(RetryContext> contex + TimeUnit.NANOSECONDS.toMillis(maxTimeNanos) + " ms.")); } + if (currentCallIndex > 0) { + retryCounter.getAndIncrement(); + } + DelayedTask> task = DelayedTask.createMulti(context.supplier); if (delay == 0) { task.execute(); @@ -132,6 +140,11 @@ private Multi retryMulti(RetryContext> contex }); } + @Override + public long retryCounter() { + return retryCounter.get(); + } + private static class RetryContext { // retry runtime private final long startedMillis = System.currentTimeMillis(); diff --git a/microprofile/fault-tolerance/src/main/java/io/helidon/microprofile/faulttolerance/CommandRunner.java b/microprofile/fault-tolerance/src/main/java/io/helidon/microprofile/faulttolerance/CommandRunner.java index 5fe77ad1fb8..54ec640379f 100644 --- a/microprofile/fault-tolerance/src/main/java/io/helidon/microprofile/faulttolerance/CommandRunner.java +++ b/microprofile/fault-tolerance/src/main/java/io/helidon/microprofile/faulttolerance/CommandRunner.java @@ -18,6 +18,7 @@ import javax.interceptor.InvocationContext; import java.lang.reflect.Method; import java.time.Duration; +import java.util.Objects; import java.util.concurrent.CompletableFuture; import java.util.concurrent.CompletionStage; import java.util.concurrent.ConcurrentHashMap; @@ -29,12 +30,37 @@ import io.helidon.faulttolerance.Async; import io.helidon.faulttolerance.Bulkhead; import io.helidon.faulttolerance.CircuitBreaker; +import io.helidon.faulttolerance.CircuitBreaker.State; import io.helidon.faulttolerance.Fallback; import io.helidon.faulttolerance.FaultTolerance; import io.helidon.faulttolerance.FtHandlerTyped; import io.helidon.faulttolerance.Retry; import io.helidon.faulttolerance.Timeout; + +import static io.helidon.microprofile.faulttolerance.FaultToleranceExtension.isFaultToleranceMetricsEnabled; +import static io.helidon.microprofile.faulttolerance.FaultToleranceMetrics.BREAKER_CALLS_FAILED_TOTAL; +import static io.helidon.microprofile.faulttolerance.FaultToleranceMetrics.BREAKER_CALLS_PREVENTED_TOTAL; +import static io.helidon.microprofile.faulttolerance.FaultToleranceMetrics.BREAKER_CALLS_SUCCEEDED_TOTAL; +import static io.helidon.microprofile.faulttolerance.FaultToleranceMetrics.BREAKER_CLOSED_TOTAL; +import static io.helidon.microprofile.faulttolerance.FaultToleranceMetrics.BREAKER_HALF_OPEN_TOTAL; +import static io.helidon.microprofile.faulttolerance.FaultToleranceMetrics.BREAKER_OPENED_TOTAL; +import static io.helidon.microprofile.faulttolerance.FaultToleranceMetrics.BREAKER_OPEN_TOTAL; +import static io.helidon.microprofile.faulttolerance.FaultToleranceMetrics.INVOCATIONS_FAILED_TOTAL; +import static io.helidon.microprofile.faulttolerance.FaultToleranceMetrics.INVOCATIONS_TOTAL; +import static io.helidon.microprofile.faulttolerance.FaultToleranceMetrics.RETRY_CALLS_FAILED_TOTAL; +import static io.helidon.microprofile.faulttolerance.FaultToleranceMetrics.RETRY_CALLS_SUCCEEDED_NOT_RETRIED_TOTAL; +import static io.helidon.microprofile.faulttolerance.FaultToleranceMetrics.RETRY_CALLS_SUCCEEDED_RETRIED_TOTAL; +import static io.helidon.microprofile.faulttolerance.FaultToleranceMetrics.RETRY_RETRIES_TOTAL; +import static io.helidon.microprofile.faulttolerance.FaultToleranceMetrics.TIMEOUT_CALLS_NOT_TIMED_OUT_TOTAL; +import static io.helidon.microprofile.faulttolerance.FaultToleranceMetrics.TIMEOUT_CALLS_TIMED_OUT_TOTAL; +import static io.helidon.microprofile.faulttolerance.FaultToleranceMetrics.TIMEOUT_EXECUTION_DURATION; +import static io.helidon.microprofile.faulttolerance.FaultToleranceMetrics.getCounter; +import static io.helidon.microprofile.faulttolerance.FaultToleranceMetrics.getHistogram; +import static io.helidon.microprofile.faulttolerance.FaultToleranceMetrics.registerGauge; import static io.helidon.microprofile.faulttolerance.ThrowableMapper.map; +import org.eclipse.microprofile.faulttolerance.exceptions.CircuitBreakerOpenException; +import org.eclipse.microprofile.faulttolerance.exceptions.TimeoutException; +import org.eclipse.microprofile.metrics.Counter; /** * Runs a FT method. @@ -47,7 +73,24 @@ public class CommandRunner implements FtSupplier { private final MethodIntrospector introspector; - private static final ConcurrentHashMap> ftHandlers = new ConcurrentHashMap<>(); + private static final ConcurrentHashMap ftHandlers = new ConcurrentHashMap<>(); + + private boolean retried = false; + + private long timerStart; + + private static class MethodState { + FtHandlerTyped handler; + CircuitBreaker breaker; + Retry retry; + State lastBreakerState; + long breakerTimerOpen; + long breakerTimerClosed; + long breakerTimerHalfOpen; + long startNanos; + } + + private final MethodState methodState; /** * Constructor. @@ -59,6 +102,36 @@ public CommandRunner(InvocationContext context, MethodIntrospector introspector) this.context = context; this.introspector = introspector; this.method = context.getMethod(); + + // Get or initialize new state for this method + this.methodState = ftHandlers.computeIfAbsent(method, method -> { + MethodState methodState = new MethodState(); + initMethodStateHandler(methodState); + methodState.lastBreakerState = State.CLOSED; + if (introspector.hasCircuitBreaker()) { + methodState.breakerTimerOpen = 0L; + methodState.breakerTimerClosed = 0L; + methodState.breakerTimerHalfOpen = 0L; + methodState.startNanos = System.nanoTime(); + } + return methodState; + }); + + // Special initialization for methods with breakers + if (isFaultToleranceMetricsEnabled() && introspector.hasCircuitBreaker()) { + registerGauge(introspector.getMethod(), + BREAKER_OPEN_TOTAL, + "Amount of time the circuit breaker has spent in open state", + () -> methodState.breakerTimerOpen); + registerGauge(introspector.getMethod(), + BREAKER_HALF_OPEN_TOTAL, + "Amount of time the circuit breaker has spent in half-open state", + () -> methodState.breakerTimerHalfOpen); + registerGauge(introspector.getMethod(), + BREAKER_CLOSED_TOTAL, + "Amount of time the circuit breaker has spent in closed state", + () -> methodState.breakerTimerClosed); + } } /** @@ -75,19 +148,21 @@ static void clearFtHandlersMap() { */ @Override public Object get() throws Throwable { - // Lookup or create handler for this method - FtHandlerTyped handler = ftHandlers.computeIfAbsent(method, method -> createHandler()); - Single single; if (introspector.isAsynchronous()) { if (introspector.isReturnType(CompletionStage.class) || introspector.isReturnType(Future.class)) { // Invoke method in new thread and call get() to unwrap singles - single = Async.create().invoke(() -> - handler.invoke(toCompletionStageSupplier(context::proceed))); + single = Async.create().invoke(() -> { + updateMetricsBefore(); + return methodState.handler.invoke(toCompletionStageSupplier(context::proceed)); + }); // Unwrap nested futures and map exceptions on complete CompletableFuture future = new CompletableFuture<>(); single.whenComplete((o, t) -> { + Throwable cause = null; + + // Update future to return if (t == null) { // If future whose value is a future, then unwrap them Future delegate = null; @@ -101,14 +176,18 @@ else if (o instanceof Future) { try { future.complete(delegate.get()); } catch (Exception e) { - future.completeExceptionally(map(e)); + cause = map(e); + future.completeExceptionally(cause); } } else { future.complete(o); } } else { - future.completeExceptionally(map(t)); + cause = map(t); + future.completeExceptionally(cause); } + + updateMetricsAfter(cause); }); return future; } @@ -116,25 +195,33 @@ else if (o instanceof Future) { // Oops, something went wrong during validation throw new InternalError("Validation failed, return type must be Future or CompletionStage"); } else { - single = handler.invoke(toCompletionStageSupplier(context::proceed)); + Object result = null; + Throwable cause = null; + + single = methodState.handler.invoke(toCompletionStageSupplier(context::proceed)); try { // Need to allow completion with no value (null) for void methods CompletableFuture future = single.toStage(true).toCompletableFuture(); - return future.get(); + updateMetricsBefore(); + result = future.get(); } catch (ExecutionException e) { - throw map(e.getCause()); // throw unwrapped exception here - } catch (Exception e) { - throw map(e); + cause = map(e.getCause()); + } catch (Throwable t) { + cause = map(t); + } + + updateMetricsAfter(cause); + if (cause != null) { + throw cause; } + return result; } } /** * Creates a FT handler for a given method by inspecting annotations. - * - * @return Newly created FT handler. */ - private FtHandlerTyped createHandler() { + private void initMethodStateHandler(MethodState methodState) { FaultTolerance.TypedBuilder builder = FaultTolerance.typedBuilder(); // Create and add circuit breaker @@ -148,6 +235,7 @@ private FtHandlerTyped createHandler() { .applyOn(introspector.getCircuitBreaker().failOn()) .build(); builder.addBreaker(circuitBreaker); + methodState.breaker = circuitBreaker; } // Create and add bulkhead @@ -171,14 +259,16 @@ private FtHandlerTyped createHandler() { // Create and add retry handler -- parent of timeout if (introspector.hasRetry()) { - Retry.Builder retry = Retry.builder() + Retry retry = Retry.builder() .retryPolicy(Retry.JitterRetryPolicy.builder() .calls(introspector.getRetry().maxRetries() + 1) .delay(Duration.ofMillis(introspector.getRetry().delay())) .jitter(Duration.ofMillis(introspector.getRetry().jitter())) .build()) - .overallTimeout(Duration.ofNanos(Long.MAX_VALUE)); // not used - builder.addRetry(retry.build()); + .overallTimeout(Duration.ofNanos(Long.MAX_VALUE)) // not used + .build(); + builder.addRetry(retry); + methodState.retry = retry; } // Create and add fallback handler -- parent of retry @@ -186,13 +276,15 @@ private FtHandlerTyped createHandler() { Fallback fallback = Fallback.builder() .fallback(throwable -> { CommandFallback cfb = new CommandFallback(context, introspector, throwable); + // getCounter(method, FALLBACK_CALLS_TOTAL).inc(); return toCompletionStageSupplier(cfb::execute).get(); }) .build(); builder.addFallback(fallback); } - return builder.build(); + // Set handler in method state + methodState.handler = builder.build(); } /** @@ -214,4 +306,111 @@ static Supplier> toCompletionStageSupplier(FtS } }; } + + /** + * Collects information necessary to update metrics before method is called. + */ + private synchronized void updateMetricsBefore() { + timerStart = System.currentTimeMillis(); + } + + /** + * Update metrics after method is called and depending on outcome. + * + * @param cause Exception cause or {@code null} if execution successful. + */ + private synchronized void updateMetricsAfter(Throwable cause) { + if (!isFaultToleranceMetricsEnabled()) { + return; + } + + // Calculate execution time + long executionTime = System.currentTimeMillis() - timerStart; + + // Metrics for retries + if (introspector.hasRetry()) { + Counter counter = getCounter(method, RETRY_RETRIES_TOTAL); + long oldCounter = counter.getCount(); + long newCounter = methodState.retry.retryCounter(); + + // Have retried the last call? + if (newCounter > oldCounter) { + counter.inc(newCounter - oldCounter); // akin to a set + if (cause == null) { + getCounter(method, RETRY_CALLS_SUCCEEDED_RETRIED_TOTAL).inc(); + } + } else { + getCounter(method, RETRY_CALLS_SUCCEEDED_NOT_RETRIED_TOTAL).inc(); + } + + // Update failed calls + if (cause != null) { + getCounter(method, RETRY_CALLS_FAILED_TOTAL).inc(); + } + } + + // Timeout + if (introspector.hasTimeout()) { + getHistogram(method, TIMEOUT_EXECUTION_DURATION).update(executionTime); + getCounter(method, cause instanceof TimeoutException + ? TIMEOUT_CALLS_TIMED_OUT_TOTAL + : TIMEOUT_CALLS_NOT_TIMED_OUT_TOTAL).inc(); + } + + // Circuit breaker + if (introspector.hasCircuitBreaker()) { + Objects.requireNonNull(methodState.breaker); + + // Update counters based on state changes + if (methodState.lastBreakerState != State.CLOSED) { + getCounter(method, BREAKER_CALLS_PREVENTED_TOTAL).inc(); + } else if (methodState.breaker.state() == State.OPEN) { // closed -> open + getCounter(method, BREAKER_OPENED_TOTAL).inc(); + } + + // Update succeeded and failed + if (cause == null) { + getCounter(method, BREAKER_CALLS_SUCCEEDED_TOTAL).inc(); + } else if (!(cause instanceof CircuitBreakerOpenException)) { + boolean failure = false; + Class[] failOn = introspector.getCircuitBreaker().failOn(); + for (Class c : failOn) { + if (c.isAssignableFrom(cause.getClass())) { + failure = true; + break; + } + } + + System.out.println("### failure " + failure + " " + cause); + + getCounter(method, failure ? BREAKER_CALLS_FAILED_TOTAL + : BREAKER_CALLS_SUCCEEDED_TOTAL).inc(); + } + + // Update times for gauges + switch (methodState.lastBreakerState) { + case OPEN: + methodState.breakerTimerOpen += System.nanoTime() - methodState.startNanos; + break; + case CLOSED: + methodState.breakerTimerClosed += System.nanoTime() - methodState.startNanos; + break; + case HALF_OPEN: + methodState.breakerTimerHalfOpen += System.nanoTime() - methodState.startNanos; + break; + default: + throw new IllegalStateException("Unknown breaker state " + methodState.lastBreakerState); + } + + // Update internal state + methodState.lastBreakerState = methodState.breaker.state(); + methodState.startNanos = System.nanoTime(); + } + + // Global method counters + getCounter(method, INVOCATIONS_TOTAL).inc(); + if (cause != null) { + getCounter(method, INVOCATIONS_FAILED_TOTAL).inc(); + } + } } diff --git a/microprofile/fault-tolerance/src/test/java/io/helidon/microprofile/faulttolerance/CircuitBreakerTest.java b/microprofile/fault-tolerance/src/test/java/io/helidon/microprofile/faulttolerance/CircuitBreakerTest.java index b173084d5f0..c8532c9bc4b 100644 --- a/microprofile/fault-tolerance/src/test/java/io/helidon/microprofile/faulttolerance/CircuitBreakerTest.java +++ b/microprofile/fault-tolerance/src/test/java/io/helidon/microprofile/faulttolerance/CircuitBreakerTest.java @@ -72,7 +72,7 @@ public void testOpenOnTimeouts() { CircuitBreakerBean bean = newBean(CircuitBreakerBean.class); // Iterate a few times to test circuit - for (int i = 0; i < bean.REQUEST_VOLUME_THRESHOLD; i++) { + for (int i = 0; i < bean.REQUEST_VOLUME_THRESHOLD - 1; i++) { assertThrows(TimeoutException.class, () -> bean.openOnTimeouts()); } diff --git a/microprofile/fault-tolerance/src/test/java/io/helidon/microprofile/faulttolerance/MetricsTest.java b/microprofile/fault-tolerance/src/test/java/io/helidon/microprofile/faulttolerance/MetricsTest.java index af1c344a72e..172ef066b63 100644 --- a/microprofile/fault-tolerance/src/test/java/io/helidon/microprofile/faulttolerance/MetricsTest.java +++ b/microprofile/fault-tolerance/src/test/java/io/helidon/microprofile/faulttolerance/MetricsTest.java @@ -212,7 +212,7 @@ public void testTimeoutFailure() throws Exception { public void testBreakerTrip() throws Exception { MetricsBean bean = newBean(MetricsBean.class); - for (int i = 0; i < CircuitBreakerBean.REQUEST_VOLUME_THRESHOLD; i++) { + for (int i = 0; i < CircuitBreakerBean.REQUEST_VOLUME_THRESHOLD - 1; i++) { assertThrows(RuntimeException.class, () -> bean.exerciseBreaker(false)); } assertThrows(CircuitBreakerOpenException.class, () -> bean.exerciseBreaker(false)); @@ -225,7 +225,7 @@ public void testBreakerTrip() throws Exception { is(0L)); assertThat(getCounter(bean, "exerciseBreaker", BREAKER_CALLS_FAILED_TOTAL, boolean.class), - is((long)CircuitBreakerBean.REQUEST_VOLUME_THRESHOLD)); + is((long) CircuitBreakerBean.REQUEST_VOLUME_THRESHOLD - 1)); assertThat(getCounter(bean, "exerciseBreaker", BREAKER_CALLS_PREVENTED_TOTAL, boolean.class), is(1L)); From 0ba871ddc4455969afdaa9349f003997acee176c Mon Sep 17 00:00:00 2001 From: Santiago Pericasgeertsen Date: Thu, 23 Jul 2020 11:24:50 -0400 Subject: [PATCH 12/65] Changes in this commit: - New method in Bulkhead to return stats - Support for all metrics related to bulkheads - Some minor test changes Signed-off-by: Santiago Pericasgeertsen --- .../io/helidon/faulttolerance/Bulkhead.java | 37 +++ .../helidon/faulttolerance/BulkheadImpl.java | 36 +++ .../faulttolerance/CommandRunner.java | 218 +++++++++++------- .../faulttolerance/MetricsTest.java | 9 +- 4 files changed, 209 insertions(+), 91 deletions(-) diff --git a/fault-tolerance/src/main/java/io/helidon/faulttolerance/Bulkhead.java b/fault-tolerance/src/main/java/io/helidon/faulttolerance/Bulkhead.java index fa09f86489a..9d28fab844e 100644 --- a/fault-tolerance/src/main/java/io/helidon/faulttolerance/Bulkhead.java +++ b/fault-tolerance/src/main/java/io/helidon/faulttolerance/Bulkhead.java @@ -141,4 +141,41 @@ boolean async() { } } + interface Stats { + + /** + * Number of concurrent executions at this time. + * + * @return concurrent executions. + */ + long concurrentExecutions(); + + /** + * Number of calls accepted on the bulkhead. + * + * @return calls accepted. + */ + long callsAccepted(); + + /** + * Number of calls rejected on the bulkhead. + * + * @return calls rejected. + */ + long callsRejected(); + + /** + * Size of waiting queue at this time. + * + * @return size of waiting queue. + */ + long waitingQueueSize(); + } + + /** + * Provides access to internal stats for this bulkhead. + * + * @return internal stats. + */ + Stats stats(); } diff --git a/fault-tolerance/src/main/java/io/helidon/faulttolerance/BulkheadImpl.java b/fault-tolerance/src/main/java/io/helidon/faulttolerance/BulkheadImpl.java index fa5af1d3b01..45e5aafecf5 100644 --- a/fault-tolerance/src/main/java/io/helidon/faulttolerance/BulkheadImpl.java +++ b/fault-tolerance/src/main/java/io/helidon/faulttolerance/BulkheadImpl.java @@ -23,6 +23,7 @@ import java.util.concurrent.Flow; import java.util.concurrent.LinkedBlockingQueue; import java.util.concurrent.Semaphore; +import java.util.concurrent.atomic.AtomicLong; import java.util.function.Supplier; import java.util.logging.Logger; @@ -39,6 +40,10 @@ class BulkheadImpl implements Bulkhead { private final String name; private final boolean async; + private final AtomicLong concurrentExecutions = new AtomicLong(0L); + private final AtomicLong callsAccepted = new AtomicLong(0L); + private final AtomicLong callsRejected = new AtomicLong(0L); + BulkheadImpl(Bulkhead.Builder builder) { this.executor = builder.executor(); this.inProgress = new Semaphore(builder.limit(), true); @@ -62,6 +67,31 @@ public Multi invokeMulti(Supplier> supplier) return invokeTask(DelayedTask.createMulti(supplier)); } + @Override + public Stats stats() { + return new Stats() { + @Override + public long concurrentExecutions() { + return concurrentExecutions.get(); + } + + @Override + public long callsAccepted() { + return callsAccepted.get(); + } + + @Override + public long callsRejected() { + return callsRejected.get(); + } + + @Override + public long waitingQueueSize() { + return queue.size(); + } + }; + } + // this method must be called while NOT holding a permit private R invokeTask(DelayedTask task) { if (inProgress.tryAcquire()) { @@ -76,6 +106,7 @@ private R invokeTask(DelayedTask task) { return task.result(); } else { LOGGER.finest(() -> name + " reject: " + task); + callsRejected.incrementAndGet(); return task.error(new BulkheadException("Bulkhead queue \"" + name + "\" is full")); } } @@ -83,8 +114,13 @@ private R invokeTask(DelayedTask task) { // this method must be called while holding a permit private void execute(DelayedTask task) { + callsAccepted.incrementAndGet(); + concurrentExecutions.incrementAndGet(); + + long startNanos = System.nanoTime(); task.execute() .handle((it, throwable) -> { + concurrentExecutions.decrementAndGet(); // we do not care about execution, but let's record it in debug LOGGER.finest(() -> name + " finished execution: " + task + " (" + (throwable == null ? "success" : "failure") + ")"); diff --git a/microprofile/fault-tolerance/src/main/java/io/helidon/microprofile/faulttolerance/CommandRunner.java b/microprofile/fault-tolerance/src/main/java/io/helidon/microprofile/faulttolerance/CommandRunner.java index 54ec640379f..f5dd217baeb 100644 --- a/microprofile/fault-tolerance/src/main/java/io/helidon/microprofile/faulttolerance/CommandRunner.java +++ b/microprofile/fault-tolerance/src/main/java/io/helidon/microprofile/faulttolerance/CommandRunner.java @@ -36,7 +36,6 @@ import io.helidon.faulttolerance.FtHandlerTyped; import io.helidon.faulttolerance.Retry; import io.helidon.faulttolerance.Timeout; - import static io.helidon.microprofile.faulttolerance.FaultToleranceExtension.isFaultToleranceMetricsEnabled; import static io.helidon.microprofile.faulttolerance.FaultToleranceMetrics.BREAKER_CALLS_FAILED_TOTAL; import static io.helidon.microprofile.faulttolerance.FaultToleranceMetrics.BREAKER_CALLS_PREVENTED_TOTAL; @@ -45,6 +44,12 @@ import static io.helidon.microprofile.faulttolerance.FaultToleranceMetrics.BREAKER_HALF_OPEN_TOTAL; import static io.helidon.microprofile.faulttolerance.FaultToleranceMetrics.BREAKER_OPENED_TOTAL; import static io.helidon.microprofile.faulttolerance.FaultToleranceMetrics.BREAKER_OPEN_TOTAL; +import static io.helidon.microprofile.faulttolerance.FaultToleranceMetrics.BULKHEAD_CALLS_ACCEPTED_TOTAL; +import static io.helidon.microprofile.faulttolerance.FaultToleranceMetrics.BULKHEAD_CALLS_REJECTED_TOTAL; +import static io.helidon.microprofile.faulttolerance.FaultToleranceMetrics.BULKHEAD_CONCURRENT_EXECUTIONS; +import static io.helidon.microprofile.faulttolerance.FaultToleranceMetrics.BULKHEAD_EXECUTION_DURATION; +import static io.helidon.microprofile.faulttolerance.FaultToleranceMetrics.BULKHEAD_WAITING_DURATION; +import static io.helidon.microprofile.faulttolerance.FaultToleranceMetrics.BULKHEAD_WAITING_QUEUE_POPULATION; import static io.helidon.microprofile.faulttolerance.FaultToleranceMetrics.INVOCATIONS_FAILED_TOTAL; import static io.helidon.microprofile.faulttolerance.FaultToleranceMetrics.INVOCATIONS_TOTAL; import static io.helidon.microprofile.faulttolerance.FaultToleranceMetrics.RETRY_CALLS_FAILED_TOTAL; @@ -81,8 +86,9 @@ public class CommandRunner implements FtSupplier { private static class MethodState { FtHandlerTyped handler; - CircuitBreaker breaker; Retry retry; + Bulkhead bulkhead; + CircuitBreaker breaker; State lastBreakerState; long breakerTimerOpen; long breakerTimerClosed; @@ -117,20 +123,29 @@ public CommandRunner(InvocationContext context, MethodIntrospector introspector) return methodState; }); - // Special initialization for methods with breakers - if (isFaultToleranceMetricsEnabled() && introspector.hasCircuitBreaker()) { - registerGauge(introspector.getMethod(), - BREAKER_OPEN_TOTAL, - "Amount of time the circuit breaker has spent in open state", - () -> methodState.breakerTimerOpen); - registerGauge(introspector.getMethod(), - BREAKER_HALF_OPEN_TOTAL, - "Amount of time the circuit breaker has spent in half-open state", - () -> methodState.breakerTimerHalfOpen); - registerGauge(introspector.getMethod(), - BREAKER_CLOSED_TOTAL, - "Amount of time the circuit breaker has spent in closed state", - () -> methodState.breakerTimerClosed); + // Registration of gauges for bulkhead and circuit breakers + if (isFaultToleranceMetricsEnabled()) { + if (introspector.hasCircuitBreaker()) { + registerGauge(method, BREAKER_OPEN_TOTAL, + "Amount of time the circuit breaker has spent in open state", + () -> methodState.breakerTimerOpen); + registerGauge(method, BREAKER_HALF_OPEN_TOTAL, + "Amount of time the circuit breaker has spent in half-open state", + () -> methodState.breakerTimerHalfOpen); + registerGauge(method, BREAKER_CLOSED_TOTAL, + "Amount of time the circuit breaker has spent in closed state", + () -> methodState.breakerTimerClosed); + } + if (introspector.hasBulkhead()) { + registerGauge(method, BULKHEAD_CONCURRENT_EXECUTIONS, + "Number of currently running executions", + () -> methodState.bulkhead.stats().concurrentExecutions()); + if (introspector.isAsynchronous()) { + registerGauge(method, BULKHEAD_WAITING_QUEUE_POPULATION, + "Number of executions currently waiting in the queue", + () -> methodState.bulkhead.stats().waitingQueueSize()); + } + } } } @@ -246,6 +261,7 @@ private void initMethodStateHandler(MethodState methodState) { .async(introspector.isAsynchronous()) .build(); builder.addBulkhead(bulkhead); + methodState.bulkhead = bulkhead; } // Create and add timeout handler -- parent of breaker or bulkhead @@ -310,7 +326,7 @@ static Supplier> toCompletionStageSupplier(FtS /** * Collects information necessary to update metrics before method is called. */ - private synchronized void updateMetricsBefore() { + private void updateMetricsBefore() { timerStart = System.currentTimeMillis(); } @@ -319,98 +335,126 @@ private synchronized void updateMetricsBefore() { * * @param cause Exception cause or {@code null} if execution successful. */ - private synchronized void updateMetricsAfter(Throwable cause) { + private void updateMetricsAfter(Throwable cause) { if (!isFaultToleranceMetricsEnabled()) { return; } - // Calculate execution time - long executionTime = System.currentTimeMillis() - timerStart; - - // Metrics for retries - if (introspector.hasRetry()) { - Counter counter = getCounter(method, RETRY_RETRIES_TOTAL); - long oldCounter = counter.getCount(); - long newCounter = methodState.retry.retryCounter(); + synchronized (method) { + // Calculate execution time + long executionTime = System.currentTimeMillis() - timerStart; + + // Metrics for retries + if (introspector.hasRetry()) { + // Have retried the last call? + long newValue = methodState.retry.retryCounter(); + if (updateCounter(method, RETRY_RETRIES_TOTAL, newValue)) { + if (cause == null) { + getCounter(method, RETRY_CALLS_SUCCEEDED_RETRIED_TOTAL).inc(); + } + } else { + getCounter(method, RETRY_CALLS_SUCCEEDED_NOT_RETRIED_TOTAL).inc(); + } - // Have retried the last call? - if (newCounter > oldCounter) { - counter.inc(newCounter - oldCounter); // akin to a set - if (cause == null) { - getCounter(method, RETRY_CALLS_SUCCEEDED_RETRIED_TOTAL).inc(); + // Update failed calls + if (cause != null) { + getCounter(method, RETRY_CALLS_FAILED_TOTAL).inc(); } - } else { - getCounter(method, RETRY_CALLS_SUCCEEDED_NOT_RETRIED_TOTAL).inc(); } - // Update failed calls - if (cause != null) { - getCounter(method, RETRY_CALLS_FAILED_TOTAL).inc(); + // Timeout + if (introspector.hasTimeout()) { + getHistogram(method, TIMEOUT_EXECUTION_DURATION).update(executionTime); + getCounter(method, cause instanceof TimeoutException + ? TIMEOUT_CALLS_TIMED_OUT_TOTAL + : TIMEOUT_CALLS_NOT_TIMED_OUT_TOTAL).inc(); } - } - // Timeout - if (introspector.hasTimeout()) { - getHistogram(method, TIMEOUT_EXECUTION_DURATION).update(executionTime); - getCounter(method, cause instanceof TimeoutException - ? TIMEOUT_CALLS_TIMED_OUT_TOTAL - : TIMEOUT_CALLS_NOT_TIMED_OUT_TOTAL).inc(); - } - - // Circuit breaker - if (introspector.hasCircuitBreaker()) { - Objects.requireNonNull(methodState.breaker); + // Circuit breaker + if (introspector.hasCircuitBreaker()) { + Objects.requireNonNull(methodState.breaker); - // Update counters based on state changes - if (methodState.lastBreakerState != State.CLOSED) { - getCounter(method, BREAKER_CALLS_PREVENTED_TOTAL).inc(); - } else if (methodState.breaker.state() == State.OPEN) { // closed -> open - getCounter(method, BREAKER_OPENED_TOTAL).inc(); - } + // Update counters based on state changes + if (methodState.lastBreakerState != State.CLOSED) { + getCounter(method, BREAKER_CALLS_PREVENTED_TOTAL).inc(); + } else if (methodState.breaker.state() == State.OPEN) { // closed -> open + getCounter(method, BREAKER_OPENED_TOTAL).inc(); + } - // Update succeeded and failed - if (cause == null) { - getCounter(method, BREAKER_CALLS_SUCCEEDED_TOTAL).inc(); - } else if (!(cause instanceof CircuitBreakerOpenException)) { - boolean failure = false; - Class[] failOn = introspector.getCircuitBreaker().failOn(); - for (Class c : failOn) { - if (c.isAssignableFrom(cause.getClass())) { - failure = true; - break; + // Update succeeded and failed + if (cause == null) { + getCounter(method, BREAKER_CALLS_SUCCEEDED_TOTAL).inc(); + } else if (!(cause instanceof CircuitBreakerOpenException)) { + boolean failure = false; + Class[] failOn = introspector.getCircuitBreaker().failOn(); + for (Class c : failOn) { + if (c.isAssignableFrom(cause.getClass())) { + failure = true; + break; + } } + + getCounter(method, failure ? BREAKER_CALLS_FAILED_TOTAL + : BREAKER_CALLS_SUCCEEDED_TOTAL).inc(); } - System.out.println("### failure " + failure + " " + cause); + // Update times for gauges + switch (methodState.lastBreakerState) { + case OPEN: + methodState.breakerTimerOpen += System.nanoTime() - methodState.startNanos; + break; + case CLOSED: + methodState.breakerTimerClosed += System.nanoTime() - methodState.startNanos; + break; + case HALF_OPEN: + methodState.breakerTimerHalfOpen += System.nanoTime() - methodState.startNanos; + break; + default: + throw new IllegalStateException("Unknown breaker state " + methodState.lastBreakerState); + } - getCounter(method, failure ? BREAKER_CALLS_FAILED_TOTAL - : BREAKER_CALLS_SUCCEEDED_TOTAL).inc(); + // Update internal state + methodState.lastBreakerState = methodState.breaker.state(); + methodState.startNanos = System.nanoTime(); } - // Update times for gauges - switch (methodState.lastBreakerState) { - case OPEN: - methodState.breakerTimerOpen += System.nanoTime() - methodState.startNanos; - break; - case CLOSED: - methodState.breakerTimerClosed += System.nanoTime() - methodState.startNanos; - break; - case HALF_OPEN: - methodState.breakerTimerHalfOpen += System.nanoTime() - methodState.startNanos; - break; - default: - throw new IllegalStateException("Unknown breaker state " + methodState.lastBreakerState); + // Bulkhead + if (introspector.hasBulkhead()) { + Objects.requireNonNull(methodState.bulkhead); + Bulkhead.Stats stats = methodState.bulkhead.stats(); + updateCounter(method, BULKHEAD_CALLS_ACCEPTED_TOTAL, stats.callsAccepted()); + updateCounter(method, BULKHEAD_CALLS_REJECTED_TOTAL, stats.callsRejected()); + + // TODO: compute these durations properly + getHistogram(method, BULKHEAD_EXECUTION_DURATION).update(executionTime); + if (introspector.isAsynchronous()) { + getHistogram(method, BULKHEAD_WAITING_DURATION).update(executionTime); + } } - // Update internal state - methodState.lastBreakerState = methodState.breaker.state(); - methodState.startNanos = System.nanoTime(); + // Global method counters + getCounter(method, INVOCATIONS_TOTAL).inc(); + if (cause != null) { + getCounter(method, INVOCATIONS_FAILED_TOTAL).inc(); + } } + } - // Global method counters - getCounter(method, INVOCATIONS_TOTAL).inc(); - if (cause != null) { - getCounter(method, INVOCATIONS_FAILED_TOTAL).inc(); + /** + * Sets the value of a monotonically increasing counter using {@code inc()}. + * + * @param method the method. + * @param name the counter's name. + * @param newValue the new value. + * @return A value of {@code true} if counter updated, {@code false} otherwise. + */ + private static boolean updateCounter(Method method, String name, long newValue) { + Counter counter = getCounter(method, name); + long oldValue = counter.getCount(); + if (newValue > oldValue) { + counter.inc(newValue - oldValue); + return true; } + return false; } } diff --git a/microprofile/fault-tolerance/src/test/java/io/helidon/microprofile/faulttolerance/MetricsTest.java b/microprofile/fault-tolerance/src/test/java/io/helidon/microprofile/faulttolerance/MetricsTest.java index 172ef066b63..3f83f8e5982 100644 --- a/microprofile/fault-tolerance/src/test/java/io/helidon/microprofile/faulttolerance/MetricsTest.java +++ b/microprofile/fault-tolerance/src/test/java/io/helidon/microprofile/faulttolerance/MetricsTest.java @@ -24,6 +24,7 @@ import org.eclipse.microprofile.metrics.MetricRegistry; import org.eclipse.microprofile.metrics.MetricType; import org.eclipse.microprofile.metrics.MetricUnits; +import static org.hamcrest.Matchers.greaterThan; import org.junit.jupiter.api.Test; import static io.helidon.microprofile.faulttolerance.FaultToleranceMetrics.BREAKER_CALLS_FAILED_TOTAL; @@ -336,7 +337,7 @@ public void testFallbackMetrics() throws Exception { public void testBulkheadMetrics() throws Exception { MetricsBean bean = newBean(MetricsBean.class); Future[] calls = getAsyncConcurrentCalls( - () -> bean.concurrent(100), BulkheadBean.TOTAL_CALLS); + () -> bean.concurrent(200), BulkheadBean.TOTAL_CALLS); waitFor(calls); assertThat(getGauge(bean, "concurrent", BULKHEAD_CONCURRENT_EXECUTIONS, long.class).getValue(), @@ -349,7 +350,7 @@ public void testBulkheadMetrics() throws Exception { is(0L)); assertThat(getHistogram(bean, "concurrent", BULKHEAD_EXECUTION_DURATION, long.class).getCount(), - is((long)BulkheadBean.TOTAL_CALLS)); + is(greaterThan(0L))); } @Test @@ -358,7 +359,7 @@ public void testBulkheadMetricsAsync() throws Exception { CompletableFuture[] calls = getConcurrentCalls( () -> { try { - return bean.concurrentAsync(100).get(); + return bean.concurrentAsync(200).get(); } catch (Exception e) { return "failure"; } @@ -366,6 +367,6 @@ public void testBulkheadMetricsAsync() throws Exception { CompletableFuture.allOf(calls).get(); assertThat(getHistogram(bean, "concurrentAsync", BULKHEAD_EXECUTION_DURATION, long.class).getCount(), - is((long)BulkheadBean.TOTAL_CALLS)); + is(greaterThan(0L))); } } From 1b9a62f9a0bfe2c802259aff60e24e069e1c741d Mon Sep 17 00:00:00 2001 From: Santiago Pericasgeertsen Date: Fri, 24 Jul 2020 11:05:35 -0400 Subject: [PATCH 13/65] Bulkhead metrics improved. Signed-off-by: Santiago Pericasgeertsen --- .../faulttolerance/CommandRunner.java | 45 +++++++++++++------ 1 file changed, 32 insertions(+), 13 deletions(-) diff --git a/microprofile/fault-tolerance/src/main/java/io/helidon/microprofile/faulttolerance/CommandRunner.java b/microprofile/fault-tolerance/src/main/java/io/helidon/microprofile/faulttolerance/CommandRunner.java index f5dd217baeb..19896c2d01c 100644 --- a/microprofile/fault-tolerance/src/main/java/io/helidon/microprofile/faulttolerance/CommandRunner.java +++ b/microprofile/fault-tolerance/src/main/java/io/helidon/microprofile/faulttolerance/CommandRunner.java @@ -36,6 +36,11 @@ import io.helidon.faulttolerance.FtHandlerTyped; import io.helidon.faulttolerance.Retry; import io.helidon.faulttolerance.Timeout; + +import org.eclipse.microprofile.faulttolerance.exceptions.CircuitBreakerOpenException; +import org.eclipse.microprofile.faulttolerance.exceptions.TimeoutException; +import org.eclipse.microprofile.metrics.Counter; + import static io.helidon.microprofile.faulttolerance.FaultToleranceExtension.isFaultToleranceMetricsEnabled; import static io.helidon.microprofile.faulttolerance.FaultToleranceMetrics.BREAKER_CALLS_FAILED_TOTAL; import static io.helidon.microprofile.faulttolerance.FaultToleranceMetrics.BREAKER_CALLS_PREVENTED_TOTAL; @@ -63,26 +68,41 @@ import static io.helidon.microprofile.faulttolerance.FaultToleranceMetrics.getHistogram; import static io.helidon.microprofile.faulttolerance.FaultToleranceMetrics.registerGauge; import static io.helidon.microprofile.faulttolerance.ThrowableMapper.map; -import org.eclipse.microprofile.faulttolerance.exceptions.CircuitBreakerOpenException; -import org.eclipse.microprofile.faulttolerance.exceptions.TimeoutException; -import org.eclipse.microprofile.metrics.Counter; /** * Runs a FT method. */ public class CommandRunner implements FtSupplier { + /** + * The method being intercepted. + */ private final Method method; + /** + * Invocation context for the interception. + */ private final InvocationContext context; + /** + * Helper class to extract information about th emethod. + */ private final MethodIntrospector introspector; + /** + * Map of methods to their internal state. + */ private static final ConcurrentHashMap ftHandlers = new ConcurrentHashMap<>(); - private boolean retried = false; + /** + * Start system nanos when handler is called. + */ + private long handlerStartNanos; - private long timerStart; + /** + * Start system nanos when method {@code proceed()} is called. + */ + private long invocationStartNanos; private static class MethodState { FtHandlerTyped handler; @@ -292,7 +312,6 @@ private void initMethodStateHandler(MethodState methodState) { Fallback fallback = Fallback.builder() .fallback(throwable -> { CommandFallback cfb = new CommandFallback(context, introspector, throwable); - // getCounter(method, FALLBACK_CALLS_TOTAL).inc(); return toCompletionStageSupplier(cfb::execute).get(); }) .build(); @@ -311,9 +330,10 @@ private void initMethodStateHandler(MethodState methodState) { * @return The new supplier. */ @SuppressWarnings("unchecked") - static Supplier> toCompletionStageSupplier(FtSupplier supplier) { + Supplier> toCompletionStageSupplier(FtSupplier supplier) { return () -> { try { + invocationStartNanos = System.nanoTime(); Object result = supplier.get(); return result instanceof CompletionStage ? (CompletionStage) result : CompletableFuture.completedFuture(result); @@ -327,7 +347,7 @@ static Supplier> toCompletionStageSupplier(FtS * Collects information necessary to update metrics before method is called. */ private void updateMetricsBefore() { - timerStart = System.currentTimeMillis(); + handlerStartNanos = System.nanoTime(); } /** @@ -342,7 +362,7 @@ private void updateMetricsAfter(Throwable cause) { synchronized (method) { // Calculate execution time - long executionTime = System.currentTimeMillis() - timerStart; + long executionTime = System.nanoTime() - handlerStartNanos; // Metrics for retries if (introspector.hasRetry()) { @@ -424,11 +444,10 @@ private void updateMetricsAfter(Throwable cause) { Bulkhead.Stats stats = methodState.bulkhead.stats(); updateCounter(method, BULKHEAD_CALLS_ACCEPTED_TOTAL, stats.callsAccepted()); updateCounter(method, BULKHEAD_CALLS_REJECTED_TOTAL, stats.callsRejected()); - - // TODO: compute these durations properly - getHistogram(method, BULKHEAD_EXECUTION_DURATION).update(executionTime); + long waitingTime = invocationStartNanos - handlerStartNanos; + getHistogram(method, BULKHEAD_EXECUTION_DURATION).update(executionTime - waitingTime); if (introspector.isAsynchronous()) { - getHistogram(method, BULKHEAD_WAITING_DURATION).update(executionTime); + getHistogram(method, BULKHEAD_WAITING_DURATION).update(waitingTime); } } From d41445206b02730684e26f309078dc549df30af9 Mon Sep 17 00:00:00 2001 From: Santiago Pericasgeertsen Date: Fri, 24 Jul 2020 11:24:57 -0400 Subject: [PATCH 14/65] Removed all Hystrix and Failsafe code and dependencies. Signed-off-by: Santiago Pericasgeertsen --- microprofile/fault-tolerance/pom.xml | 8 - .../faulttolerance/BulkheadHelper.java | 196 -------- .../faulttolerance/CircuitBreakerHelper.java | 305 ----------- .../CommandCompletableFuture.java | 348 ------------- .../faulttolerance/CommandFallback.java | 6 +- .../faulttolerance/CommandInterceptor.java | 15 +- .../faulttolerance/CommandInterceptor2.java | 59 --- .../faulttolerance/CommandRetrier.java | 467 ----------------- .../faulttolerance/CommandScheduler.java | 82 --- .../faulttolerance/ExceptionUtil.java | 69 --- .../faulttolerance/FaultToleranceCommand.java | 476 ------------------ .../FaultToleranceExtension.java | 6 +- .../faulttolerance/MethodIntrospector.java | 61 +-- .../microprofile/faulttolerance/TimeUtil.java | 112 ----- .../faulttolerance/TimedHashMap.java | 121 ----- .../src/main/java/module-info.java | 4 - .../faulttolerance/CommandDataTest.java | 58 --- .../faulttolerance/SchedulerConfigTest.java | 58 --- .../faulttolerance/TimedHashMapTest.java | 67 --- 19 files changed, 25 insertions(+), 2493 deletions(-) delete mode 100644 microprofile/fault-tolerance/src/main/java/io/helidon/microprofile/faulttolerance/BulkheadHelper.java delete mode 100644 microprofile/fault-tolerance/src/main/java/io/helidon/microprofile/faulttolerance/CircuitBreakerHelper.java delete mode 100644 microprofile/fault-tolerance/src/main/java/io/helidon/microprofile/faulttolerance/CommandCompletableFuture.java delete mode 100644 microprofile/fault-tolerance/src/main/java/io/helidon/microprofile/faulttolerance/CommandInterceptor2.java delete mode 100644 microprofile/fault-tolerance/src/main/java/io/helidon/microprofile/faulttolerance/CommandRetrier.java delete mode 100644 microprofile/fault-tolerance/src/main/java/io/helidon/microprofile/faulttolerance/CommandScheduler.java delete mode 100644 microprofile/fault-tolerance/src/main/java/io/helidon/microprofile/faulttolerance/ExceptionUtil.java delete mode 100644 microprofile/fault-tolerance/src/main/java/io/helidon/microprofile/faulttolerance/FaultToleranceCommand.java delete mode 100644 microprofile/fault-tolerance/src/main/java/io/helidon/microprofile/faulttolerance/TimeUtil.java delete mode 100644 microprofile/fault-tolerance/src/main/java/io/helidon/microprofile/faulttolerance/TimedHashMap.java delete mode 100644 microprofile/fault-tolerance/src/test/java/io/helidon/microprofile/faulttolerance/CommandDataTest.java delete mode 100644 microprofile/fault-tolerance/src/test/java/io/helidon/microprofile/faulttolerance/SchedulerConfigTest.java delete mode 100644 microprofile/fault-tolerance/src/test/java/io/helidon/microprofile/faulttolerance/TimedHashMapTest.java diff --git a/microprofile/fault-tolerance/pom.xml b/microprofile/fault-tolerance/pom.xml index 472d2f450c1..bd2f747c1e0 100644 --- a/microprofile/fault-tolerance/pom.xml +++ b/microprofile/fault-tolerance/pom.xml @@ -67,14 +67,6 @@ microprofile-metrics-api provided - - com.netflix.hystrix - hystrix-core - - - net.jodah - failsafe - jakarta.enterprise jakarta.enterprise.cdi-api diff --git a/microprofile/fault-tolerance/src/main/java/io/helidon/microprofile/faulttolerance/BulkheadHelper.java b/microprofile/fault-tolerance/src/main/java/io/helidon/microprofile/faulttolerance/BulkheadHelper.java deleted file mode 100644 index b329728ec43..00000000000 --- a/microprofile/fault-tolerance/src/main/java/io/helidon/microprofile/faulttolerance/BulkheadHelper.java +++ /dev/null @@ -1,196 +0,0 @@ -/* - * Copyright (c) 2018, 2019 Oracle and/or its affiliates. All rights reserved. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package io.helidon.microprofile.faulttolerance; - -import java.util.HashSet; -import java.util.Map; -import java.util.Set; -import java.util.concurrent.ConcurrentHashMap; - -import org.eclipse.microprofile.faulttolerance.Bulkhead; - -/** - * Helper class to keep track of invocations associated with a bulkhead. - */ -public class BulkheadHelper { - - /** - * A command ID is unique to a target (object) and method. A {@link - * FaultToleranceCommand} instance is created for each invocation of that - * target/method pair and is assigned the same invocation ID. This class - * collects information about all those invocations, including their state: - * waiting or running. - */ - static class InvocationData { - - /** - * Maximum number of concurrent invocations. - */ - private final int maxRunningInvocations; - - /** - * The waiting queue size. - */ - private final int waitingQueueSize; - - /** - * All invocations in running state. Must be a subset of {@link #allInvocations}. - */ - private Set runningInvocations = new HashSet<>(); - - /** - * All invocations associated with a invocation. - */ - private Set allInvocations = new HashSet<>(); - - InvocationData(int maxRunningCommands, int waitingQueueSize) { - this.maxRunningInvocations = maxRunningCommands; - this.waitingQueueSize = waitingQueueSize; - } - - synchronized boolean isWaitingQueueFull() { - return waitingInvocations() == waitingQueueSize; - } - - synchronized boolean isAtMaxRunningInvocations() { - return runningInvocations.size() == maxRunningInvocations; - } - - synchronized void trackInvocation(FaultToleranceCommand invocation) { - allInvocations.add(invocation); - } - - synchronized void untrackInvocation(FaultToleranceCommand invocation) { - allInvocations.remove(invocation); - } - - synchronized int runningInvocations() { - return runningInvocations.size(); - } - - synchronized void markAsRunning(FaultToleranceCommand invocation) { - runningInvocations.add(invocation); - } - - synchronized void markAsNotRunning(FaultToleranceCommand invocation) { - runningInvocations.remove(invocation); - } - - synchronized int waitingInvocations() { - return allInvocations.size() - runningInvocations.size(); - } - } - - /** - * Tracks all invocations associated with a command ID. - */ - private static final Map COMMAND_STATS = new ConcurrentHashMap<>(); - - /** - * Command key. - */ - private final String commandKey; - - /** - * Annotation instance. - */ - private final Bulkhead bulkhead; - - BulkheadHelper(String commandKey, Bulkhead bulkhead) { - this.commandKey = commandKey; - this.bulkhead = bulkhead; - } - - private InvocationData invocationData() { - return COMMAND_STATS.computeIfAbsent( - commandKey, - d -> new InvocationData(bulkhead.value(), bulkhead.waitingTaskQueue())); - } - - /** - * Track a new invocation instance related to a key. - */ - void trackInvocation(FaultToleranceCommand invocation) { - invocationData().trackInvocation(invocation); - } - - /** - * Stop tracking a invocation instance. - */ - void untrackInvocation(FaultToleranceCommand invocation) { - invocationData().untrackInvocation(invocation); - - // Attempt to cleanup state when not in use - if (runningInvocations() == 0 && waitingInvocations() == 0) { - COMMAND_STATS.remove(commandKey); - } - } - - /** - * Mark a invocation instance as running. - */ - void markAsRunning(FaultToleranceCommand invocation) { - invocationData().markAsRunning(invocation); - } - - /** - * Mark a invocation instance as terminated. - */ - void markAsNotRunning(FaultToleranceCommand invocation) { - invocationData().markAsNotRunning(invocation); - } - - /** - * Get the number of invocations that are running. - * - * @return Number of invocations running. - */ - int runningInvocations() { - return invocationData().runningInvocations(); - } - - /** - * Get the number of invocations that are waiting. - * - * @return Number of invocations waiting. - */ - int waitingInvocations() { - return invocationData().waitingInvocations(); - } - - /** - * Check if the invocation queue is full. - * - * @return Outcome of test. - */ - boolean isWaitingQueueFull() { - return invocationData().isWaitingQueueFull(); - } - - /** - * Check if at maximum number of running invocations. - * - * @return Outcome of test. - */ - boolean isAtMaxRunningInvocations() { - return invocationData().isAtMaxRunningInvocations(); - } - - boolean isInvocationRunning(FaultToleranceCommand command) { - return invocationData().runningInvocations.contains(command); - } -} diff --git a/microprofile/fault-tolerance/src/main/java/io/helidon/microprofile/faulttolerance/CircuitBreakerHelper.java b/microprofile/fault-tolerance/src/main/java/io/helidon/microprofile/faulttolerance/CircuitBreakerHelper.java deleted file mode 100644 index f8e1eec724a..00000000000 --- a/microprofile/fault-tolerance/src/main/java/io/helidon/microprofile/faulttolerance/CircuitBreakerHelper.java +++ /dev/null @@ -1,305 +0,0 @@ -/* - * Copyright (c) 2018, 2019 Oracle and/or its affiliates. All rights reserved. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package io.helidon.microprofile.faulttolerance; - -import java.util.Map; -import java.util.concurrent.ConcurrentHashMap; -import java.util.concurrent.locks.ReentrantLock; -import java.util.logging.Logger; - -import com.netflix.config.ConfigurationManager; -import org.eclipse.microprofile.faulttolerance.CircuitBreaker; - -/** - * A CircuitBreakerHelper keeps track of internal states, success and failure - * ratios for, etc. for all commands. Similar computations are done internally - * in Hystrix, but we cannot easily access them. - */ -public class CircuitBreakerHelper { - private static final Logger LOGGER = Logger.getLogger(CircuitBreakerHelper.class.getName()); - - private static final String FORCE_OPEN = "hystrix.command.%s.circuitBreaker.forceOpen"; - private static final String FORCE_CLOSED = "hystrix.command.%s.circuitBreaker.forceClosed"; - - /** - * Internal state of a circuit breaker. We need to track this to implement - * a different HALF_OPEN_MP to CLOSED_MP transition than the default in Hystrix. - */ - enum State { - CLOSED_MP(0), - HALF_OPEN_MP(1), - OPEN_MP(2); - - private int value; - - State(int value) { - this.value = value; - } - } - - /** - * Data associated with a command for the purpose of tracking a circuit - * breaker state. - */ - static class CommandData { - - private int size; - - private final boolean[] results; - - private State state = State.CLOSED_MP; - - private int successCount; - - private long[] inStateNanos = new long[State.values().length]; - - private long lastNanosRead; - - private ReentrantLock lock = new ReentrantLock(); - - CommandData(int capacity) { - results = new boolean[capacity]; - size = 0; - successCount = 0; - lastNanosRead = System.nanoTime(); - } - - ReentrantLock getLock() { - return lock; - } - - State getState() { - return state; - } - - long getCurrentStateNanos() { - return System.nanoTime() - lastNanosRead; - } - - void setState(State newState) { - if (state != newState) { - updateStateNanos(state); - if (newState == State.HALF_OPEN_MP) { - successCount = 0; - } - state = newState; - } - } - - long getInStateNanos(State queryState) { - if (state == queryState) { - updateStateNanos(state); - } - return inStateNanos[queryState.value]; - } - - private void updateStateNanos(State state) { - long currentNanos = System.nanoTime(); - inStateNanos[state.value] += currentNanos - lastNanosRead; - lastNanosRead = currentNanos; - } - - int getSuccessCount() { - return successCount; - } - - void incSuccessCount() { - successCount++; - } - - boolean isAtCapacity() { - return size == results.length; - } - - void pushResult(boolean result) { - if (isAtCapacity()) { - shift(); - } - results[size++] = result; - } - - double getSuccessRatio() { - if (isAtCapacity()) { - int success = 0; - for (int k = 0; k < size; k++) { - if (results[k]) success++; - } - return ((double) success) / size; - } - return -1.0; - } - - double getFailureRatio() { - double successRatio = getSuccessRatio(); - return successRatio >= 0.0 ? 1.0 - successRatio : -1.0; - } - - private void shift() { - if (size > 0) { - for (int k = 0; k < size - 1; k++) { - results[k] = results[k + 1]; - } - size--; - } - } - } - - private static final Map COMMAND_STATS = new ConcurrentHashMap<>(); - - private final FaultToleranceCommand command; - - private final CircuitBreaker circuitBreaker; - - CircuitBreakerHelper(FaultToleranceCommand command, CircuitBreaker circuitBreaker) { - this.command = command; - this.circuitBreaker = circuitBreaker; - } - - private CommandData getCommandData() { - return COMMAND_STATS.computeIfAbsent( - command.getCommandKey().toString(), - d -> new CommandData(circuitBreaker.requestVolumeThreshold())); - } - - /** - * Reset internal state of command data. Normally, this should be called when - * returning to {@link State#CLOSED_MP} state. Since this is the same as the - * initial state, we remove it from the map and re-create it later if needed. - */ - void resetCommandData() { - ReentrantLock lock = getCommandData().getLock(); - if (lock.isLocked()) { - lock.unlock(); - } - COMMAND_STATS.remove(command.getCommandKey().toString()); - LOGGER.info("Discarded command data for " + command.getCommandKey()); - } - - /** - * Push a new result into the current window. Discards oldest result - * if window is full. - * - * @param result New result to push. - */ - void pushResult(boolean result) { - getCommandData().pushResult(result); - } - - /** - * Returns nanos since switching to current state. - * - * @return Nanos in state. - */ - long getCurrentStateNanos() { - return getCommandData().getCurrentStateNanos(); - } - - /** - * Computes failure ratio over a complete window. - * - * @return Failure ratio or -1 if window is not complete. - */ - double getFailureRatio() { - return getCommandData().getFailureRatio(); - } - - /** - * Returns state of circuit breaker. - * - * @return The state. - */ - State getState() { - return getCommandData().getState(); - } - - /** - * Changes the state of a circuit breaker. - * - * @param newState New state. - */ - void setState(State newState) { - getCommandData().setState(newState); - if (newState == State.OPEN_MP) { - openBreaker(); - } else { - closeBreaker(); - } - LOGGER.info("Circuit breaker for " + command.getCommandKey() + " now in state " + getState()); - } - - /** - * Gets success count for breaker. - * - * @return Success count. - */ - int getSuccessCount() { - return getCommandData().getSuccessCount(); - } - - /** - * Increments success counter for breaker. - */ - void incSuccessCount() { - getCommandData().incSuccessCount(); - } - - /** - * Prevent concurrent access to underlying command data. - */ - void lock() { - getCommandData().getLock().lock(); - } - - /** - * Unlock access to underlying command data. - */ - void unlock() { - getCommandData().getLock().unlock(); - } - - /** - * Returns nanos spent on each state. - * - * @param queryState The state. - * @return The time spent in nanos. - */ - long getInStateNanos(State queryState) { - return getCommandData().getInStateNanos(queryState); - } - - /** - * Force Hystrix's circuit breaker into an open state. - */ - private void openBreaker() { - if (!command.isCircuitBreakerOpen()) { - ConfigurationManager.getConfigInstance().setProperty( - String.format(FORCE_OPEN, command.getCommandKey()), "true"); - } - } - - /** - * Force Hystrix's circuit breaker into a closed state. - */ - private void closeBreaker() { - if (command.isCircuitBreakerOpen()) { - ConfigurationManager.getConfigInstance().setProperty( - String.format(FORCE_OPEN, command.getCommandKey()), "false"); - ConfigurationManager.getConfigInstance().setProperty( - String.format(FORCE_CLOSED, command.getCommandKey()), "true"); - } - } -} diff --git a/microprofile/fault-tolerance/src/main/java/io/helidon/microprofile/faulttolerance/CommandCompletableFuture.java b/microprofile/fault-tolerance/src/main/java/io/helidon/microprofile/faulttolerance/CommandCompletableFuture.java deleted file mode 100644 index 1ff099e2c8b..00000000000 --- a/microprofile/fault-tolerance/src/main/java/io/helidon/microprofile/faulttolerance/CommandCompletableFuture.java +++ /dev/null @@ -1,348 +0,0 @@ -/* - * Copyright (c) 2019 Oracle and/or its affiliates. All rights reserved. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * - */ -package io.helidon.microprofile.faulttolerance; - -import java.util.concurrent.CompletableFuture; -import java.util.concurrent.CompletionStage; -import java.util.concurrent.ExecutionException; -import java.util.concurrent.Executor; -import java.util.concurrent.Future; -import java.util.concurrent.TimeUnit; -import java.util.concurrent.TimeoutException; -import java.util.function.BiConsumer; -import java.util.function.BiFunction; -import java.util.function.Consumer; -import java.util.function.Function; -import java.util.function.Supplier; - -/** - * A wrapper {@link CompletableFuture} which also records the associated {@link FaultToleranceCommand} so - * {@link CommandScheduler} can retrieve that command. If the delegate returns a result of type - * {@code Future} then this implementation further unwraps the delegate's value to return the actual - * value. - * - * @param type of result conveyed - */ -class CommandCompletableFuture extends CompletableFuture { - - static CommandCompletableFuture create(CompletableFuture delegate, - Supplier commandSupplier) { - return new CommandCompletableFuture<>(delegate, commandSupplier); - } - - private final CompletableFuture delegate; - private final Supplier commandSupplier; - - private CommandCompletableFuture(CompletableFuture delegate, - Supplier commandSupplier) { - this.delegate = delegate; - this.commandSupplier = commandSupplier; - } - - @Override - public boolean isDone() { - return delegate.isDone(); - } - - @Override - public T get() throws InterruptedException, ExecutionException { - try { - return getResult(-1L, null); - } catch (TimeoutException e) { - throw new RuntimeException(e); // should never be thrown - } - } - - @Override - public T get(long timeout, TimeUnit unit) throws InterruptedException, ExecutionException, TimeoutException { - return getResult(timeout, unit); - } - - @SuppressWarnings("unchecked") - T getResult(long timeout, TimeUnit unit) throws ExecutionException, InterruptedException, TimeoutException { - Object result = timeout < 0 ? delegate.get() : delegate.get(timeout, unit); - if (result instanceof CompletionStage) { - result = ((CompletionStage) result).toCompletableFuture(); - } - if (result instanceof Future) { - final Future future = (Future) result; - return timeout < 0 ? future.get() : future.get(timeout, unit); - } - return (T) result; - } - - @Override - public T join() { - return delegate.join(); - } - - @Override - public T getNow(T valueIfAbsent) { - return delegate.getNow(valueIfAbsent); - } - - @Override - public boolean complete(T value) { - - return delegate.complete(value); - } - - @Override - public boolean completeExceptionally(Throwable ex) { - return delegate.completeExceptionally(ex); - } - - @Override - public CompletableFuture thenApply(Function fn) { - return delegate.thenApply(fn); - } - - @Override - public CompletableFuture thenApplyAsync(Function fn) { - return delegate.thenApplyAsync(fn); - } - - @Override - public CompletableFuture thenApplyAsync(Function fn, Executor executor) { - return delegate.thenApplyAsync(fn, executor); - } - - @Override - public CompletableFuture thenAccept(Consumer action) { - return delegate.thenAccept(action); - } - - @Override - public CompletableFuture thenAcceptAsync(Consumer action) { - return delegate.thenAcceptAsync(action); - } - - @Override - public CompletableFuture thenAcceptAsync(Consumer action, Executor executor) { - return delegate.thenAcceptAsync(action, executor); - } - - @Override - public CompletableFuture thenRun(Runnable action) { - return delegate.thenRun(action); - } - - @Override - public CompletableFuture thenRunAsync(Runnable action) { - return delegate.thenRunAsync(action); - } - - @Override - public CompletableFuture thenRunAsync(Runnable action, Executor executor) { - return delegate.thenRunAsync(action, executor); - } - - @Override - public CompletableFuture thenCombine(CompletionStage other, - BiFunction fn) { - return delegate.thenCombine(other, fn); - } - - @Override - public CompletableFuture thenCombineAsync(CompletionStage other, - BiFunction fn) { - return delegate.thenCombineAsync(other, fn); - } - - @Override - public CompletableFuture thenCombineAsync(CompletionStage other, - BiFunction fn, - Executor executor) { - return delegate.thenCombineAsync(other, fn, executor); - } - - @Override - public CompletableFuture thenAcceptBoth(CompletionStage other, - BiConsumer action) { - return delegate.thenAcceptBoth(other, action); - } - - @Override - public CompletableFuture thenAcceptBothAsync(CompletionStage other, - BiConsumer action) { - return delegate.thenAcceptBothAsync(other, action); - } - - @Override - public CompletableFuture thenAcceptBothAsync(CompletionStage other, - BiConsumer action, Executor executor) { - return delegate.thenAcceptBothAsync(other, action, executor); - } - - @Override - public CompletableFuture runAfterBoth(CompletionStage other, Runnable action) { - return delegate.runAfterBoth(other, action); - } - - @Override - public CompletableFuture runAfterBothAsync(CompletionStage other, Runnable action) { - return delegate.runAfterBothAsync(other, action); - } - - @Override - public CompletableFuture runAfterBothAsync(CompletionStage other, Runnable action, Executor executor) { - return delegate.runAfterBothAsync(other, action, executor); - } - - @Override - public CompletableFuture applyToEither(CompletionStage other, Function fn) { - return delegate.applyToEither(other, fn); - } - - @Override - public CompletableFuture applyToEitherAsync(CompletionStage other, Function fn) { - return delegate.applyToEitherAsync(other, fn); - } - - @Override - public CompletableFuture applyToEitherAsync(CompletionStage other, Function fn, - Executor executor) { - return delegate.applyToEitherAsync(other, fn, executor); - } - - @Override - public CompletableFuture acceptEither(CompletionStage other, Consumer action) { - return delegate.acceptEither(other, action); - } - - @Override - public CompletableFuture acceptEitherAsync(CompletionStage other, Consumer action) { - return delegate.acceptEitherAsync(other, action); - } - - @Override - public CompletableFuture acceptEitherAsync(CompletionStage other, Consumer action, - Executor executor) { - return delegate.acceptEitherAsync(other, action, executor); - } - - @Override - public CompletableFuture runAfterEither(CompletionStage other, Runnable action) { - return delegate.runAfterEither(other, action); - } - - @Override - public CompletableFuture runAfterEitherAsync(CompletionStage other, Runnable action) { - return delegate.runAfterEitherAsync(other, action); - } - - @Override - public CompletableFuture runAfterEitherAsync(CompletionStage other, Runnable action, Executor executor) { - return delegate.runAfterEitherAsync(other, action, executor); - } - - @Override - public CompletableFuture thenCompose(Function> fn) { - return delegate.thenCompose(fn); - } - - @Override - public CompletableFuture thenComposeAsync(Function> fn) { - return delegate.thenComposeAsync(fn); - } - - @Override - public CompletableFuture thenComposeAsync(Function> fn, - Executor executor) { - return delegate.thenComposeAsync(fn, executor); - } - - @Override - public CompletableFuture whenComplete(BiConsumer action) { - return delegate.whenComplete(action); - } - - @Override - public CompletableFuture whenCompleteAsync(BiConsumer action) { - return delegate.whenCompleteAsync(action); - } - - @Override - public CompletableFuture whenCompleteAsync(BiConsumer action, Executor executor) { - return delegate.whenCompleteAsync(action, executor); - } - - @Override - public CompletableFuture handle(BiFunction fn) { - return delegate.handle(fn); - } - - @Override - public CompletableFuture handleAsync(BiFunction fn) { - return delegate.handleAsync(fn); - } - - @Override - public CompletableFuture handleAsync(BiFunction fn, Executor executor) { - return delegate.handleAsync(fn, executor); - } - - @Override - public CompletableFuture toCompletableFuture() { - return this; - } - - @Override - public CompletableFuture exceptionally(Function fn) { - return delegate.exceptionally(fn); - } - - @Override - public boolean cancel(boolean mayInterruptIfRunning) { - FaultToleranceCommand command = commandSupplier.get(); - BulkheadHelper bulkheadHelper = command.getBulkheadHelper(); - if (bulkheadHelper != null && !bulkheadHelper.isInvocationRunning(command)) { - return delegate.cancel(true); // overridden - } - return delegate.cancel(mayInterruptIfRunning); - } - - @Override - public boolean isCancelled() { - return delegate.isCancelled(); - } - - @Override - public boolean isCompletedExceptionally() { - return delegate.isCompletedExceptionally(); - } - - @Override - public void obtrudeValue(T value) { - delegate.obtrudeValue(value); - } - - @Override - public void obtrudeException(Throwable ex) { - delegate.obtrudeException(ex); - } - - @Override - public int getNumberOfDependents() { - return delegate.getNumberOfDependents(); - } - - @Override - public String toString() { - return String.format("%s@%h around %s", getClass().getName(), this, delegate.toString()); - } -} diff --git a/microprofile/fault-tolerance/src/main/java/io/helidon/microprofile/faulttolerance/CommandFallback.java b/microprofile/fault-tolerance/src/main/java/io/helidon/microprofile/faulttolerance/CommandFallback.java index 362f997b7e1..1d5018d9462 100644 --- a/microprofile/fault-tolerance/src/main/java/io/helidon/microprofile/faulttolerance/CommandFallback.java +++ b/microprofile/fault-tolerance/src/main/java/io/helidon/microprofile/faulttolerance/CommandFallback.java @@ -1,5 +1,5 @@ /* - * Copyright (c) 2018, 2019 Oracle and/or its affiliates. All rights reserved. + * Copyright (c) 2018, 2020 Oracle and/or its affiliates. All rights reserved. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -26,8 +26,6 @@ import org.eclipse.microprofile.faulttolerance.Fallback; import org.eclipse.microprofile.faulttolerance.FallbackHandler; -import static io.helidon.microprofile.faulttolerance.ExceptionUtil.toException; - /** * Class CommandFallback. */ @@ -112,7 +110,7 @@ public Throwable getFailure() { if (t instanceof InvocationTargetException) { t = t.getCause(); } - throw toException(t); + throw t instanceof Exception ? (Exception) t : new RuntimeException(t); } updateMetrics(null); diff --git a/microprofile/fault-tolerance/src/main/java/io/helidon/microprofile/faulttolerance/CommandInterceptor.java b/microprofile/fault-tolerance/src/main/java/io/helidon/microprofile/faulttolerance/CommandInterceptor.java index 631d5f14235..25bbec11ef8 100644 --- a/microprofile/fault-tolerance/src/main/java/io/helidon/microprofile/faulttolerance/CommandInterceptor.java +++ b/microprofile/fault-tolerance/src/main/java/io/helidon/microprofile/faulttolerance/CommandInterceptor.java @@ -1,5 +1,5 @@ /* - * Copyright (c) 2018, 2019 Oracle and/or its affiliates. All rights reserved. + * Copyright (c) 2018, 2020 Oracle and/or its affiliates. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -16,15 +16,14 @@ package io.helidon.microprofile.faulttolerance; -import java.util.logging.Logger; - import javax.annotation.Priority; import javax.interceptor.AroundInvoke; import javax.interceptor.Interceptor; import javax.interceptor.InvocationContext; +import java.util.logging.Logger; /** - * Class CommandInterceptor. + * Intercepts calls to FT methods and implements annotation semantics. */ @Interceptor @CommandBinding @@ -48,10 +47,10 @@ public Object interceptCommand(InvocationContext context) throws Throwable { + "::" + context.getMethod().getName() + "'"); // Create method introspector and executer retrier - final MethodIntrospector introspector = new MethodIntrospector( - context.getTarget().getClass(), context.getMethod()); - final CommandRetrier retrier = new CommandRetrier(context, introspector); - return retrier.execute(); + MethodIntrospector introspector = new MethodIntrospector(context.getTarget().getClass(), + context.getMethod()); + CommandRunner runner = new CommandRunner(context, introspector); + return runner.get(); } catch (Throwable t) { LOGGER.fine("Throwable caught by interceptor '" + t.getMessage() + "'"); throw t; diff --git a/microprofile/fault-tolerance/src/main/java/io/helidon/microprofile/faulttolerance/CommandInterceptor2.java b/microprofile/fault-tolerance/src/main/java/io/helidon/microprofile/faulttolerance/CommandInterceptor2.java deleted file mode 100644 index 800bdca503e..00000000000 --- a/microprofile/fault-tolerance/src/main/java/io/helidon/microprofile/faulttolerance/CommandInterceptor2.java +++ /dev/null @@ -1,59 +0,0 @@ -/* - * Copyright (c) 2020 Oracle and/or its affiliates. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package io.helidon.microprofile.faulttolerance; - -import javax.annotation.Priority; -import javax.interceptor.AroundInvoke; -import javax.interceptor.Interceptor; -import javax.interceptor.InvocationContext; -import java.util.logging.Logger; - -/** - * Intercepts calls to FT methods and implements annotation semantics. - */ -@Interceptor -@CommandBinding -@Priority(Interceptor.Priority.PLATFORM_AFTER + 10) -public class CommandInterceptor2 { - - private static final Logger LOGGER = Logger.getLogger(CommandInterceptor2.class.getName()); - - /** - * Intercepts a call to bean method annotated by any of the fault tolerance - * annotations. - * - * @param context Invocation context. - * @return Whatever the intercepted method returns. - * @throws Throwable If a problem occurs. - */ - @AroundInvoke - public Object interceptCommand(InvocationContext context) throws Throwable { - try { - LOGGER.fine("Interceptor called for '" + context.getTarget().getClass() - + "::" + context.getMethod().getName() + "'"); - - // Create method introspector and executer retrier - MethodIntrospector introspector = new MethodIntrospector(context.getTarget().getClass(), - context.getMethod()); - CommandRunner runner = new CommandRunner(context, introspector); - return runner.get(); - } catch (Throwable t) { - LOGGER.fine("Throwable caught by interceptor '" + t.getMessage() + "'"); - throw t; - } - } -} diff --git a/microprofile/fault-tolerance/src/main/java/io/helidon/microprofile/faulttolerance/CommandRetrier.java b/microprofile/fault-tolerance/src/main/java/io/helidon/microprofile/faulttolerance/CommandRetrier.java deleted file mode 100644 index 03232ddf76d..00000000000 --- a/microprofile/fault-tolerance/src/main/java/io/helidon/microprofile/faulttolerance/CommandRetrier.java +++ /dev/null @@ -1,467 +0,0 @@ -/* - * Copyright (c) 2018, 2020 Oracle and/or its affiliates. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package io.helidon.microprofile.faulttolerance; - -import java.lang.reflect.Method; -import java.time.Duration; -import java.time.temporal.ChronoUnit; -import java.util.ArrayList; -import java.util.List; -import java.util.Objects; -import java.util.concurrent.CompletableFuture; -import java.util.concurrent.CompletionStage; -import java.util.concurrent.Future; -import java.util.concurrent.RejectedExecutionException; -import java.util.concurrent.TimeUnit; -import java.util.concurrent.TimeoutException; -import java.util.function.Function; -import java.util.logging.Logger; - -import javax.interceptor.InvocationContext; - -import com.netflix.config.ConfigurationManager; -import com.netflix.hystrix.exception.HystrixRuntimeException; -import net.jodah.failsafe.Failsafe; -import net.jodah.failsafe.FailsafeException; -import net.jodah.failsafe.FailsafeExecutor; -import net.jodah.failsafe.Fallback; -import net.jodah.failsafe.Policy; -import net.jodah.failsafe.RetryPolicy; -import net.jodah.failsafe.event.ExecutionAttemptedEvent; -import net.jodah.failsafe.function.CheckedFunction; -import net.jodah.failsafe.util.concurrent.Scheduler; -import org.apache.commons.configuration.AbstractConfiguration; -import org.eclipse.microprofile.config.Config; -import org.eclipse.microprofile.config.ConfigProvider; -import org.eclipse.microprofile.faulttolerance.Retry; -import org.eclipse.microprofile.faulttolerance.exceptions.BulkheadException; -import org.eclipse.microprofile.faulttolerance.exceptions.CircuitBreakerOpenException; - -import static io.helidon.microprofile.faulttolerance.ExceptionUtil.toException; -import static io.helidon.microprofile.faulttolerance.FaultToleranceExtension.isFaultToleranceMetricsEnabled; -import static io.helidon.microprofile.faulttolerance.FaultToleranceMetrics.BULKHEAD_CALLS_ACCEPTED_TOTAL; -import static io.helidon.microprofile.faulttolerance.FaultToleranceMetrics.BULKHEAD_CALLS_REJECTED_TOTAL; -import static io.helidon.microprofile.faulttolerance.FaultToleranceMetrics.BULKHEAD_EXECUTION_DURATION; -import static io.helidon.microprofile.faulttolerance.FaultToleranceMetrics.INVOCATIONS_FAILED_TOTAL; -import static io.helidon.microprofile.faulttolerance.FaultToleranceMetrics.INVOCATIONS_TOTAL; -import static io.helidon.microprofile.faulttolerance.FaultToleranceMetrics.RETRY_CALLS_FAILED_TOTAL; -import static io.helidon.microprofile.faulttolerance.FaultToleranceMetrics.RETRY_CALLS_SUCCEEDED_NOT_RETRIED_TOTAL; -import static io.helidon.microprofile.faulttolerance.FaultToleranceMetrics.RETRY_CALLS_SUCCEEDED_RETRIED_TOTAL; -import static io.helidon.microprofile.faulttolerance.FaultToleranceMetrics.RETRY_RETRIES_TOTAL; -import static io.helidon.microprofile.faulttolerance.FaultToleranceMetrics.TIMEOUT_CALLS_NOT_TIMED_OUT_TOTAL; -import static io.helidon.microprofile.faulttolerance.FaultToleranceMetrics.TIMEOUT_CALLS_TIMED_OUT_TOTAL; -import static io.helidon.microprofile.faulttolerance.FaultToleranceMetrics.TIMEOUT_EXECUTION_DURATION; -import static io.helidon.microprofile.faulttolerance.FaultToleranceMetrics.getCounter; -import static io.helidon.microprofile.faulttolerance.FaultToleranceMetrics.getHistogram; - -/** - * Class CommandRetrier. - */ -public class CommandRetrier { - private static final Logger LOGGER = Logger.getLogger(CommandRetrier.class.getName()); - - private static final long DEFAULT_DELAY_CORRECTION = 250L; - private static final String FT_DELAY_CORRECTION = "fault-tolerance.delayCorrection"; - private static final int DEFAULT_COMMAND_THREAD_POOL_SIZE = 8; - private static final String FT_COMMAND_THREAD_POOL_SIZE = "fault-tolerance.commandThreadPoolSize"; - private static final long DEFAULT_THREAD_WAITING_PERIOD = 2000L; - private static final String FT_THREAD_WAITING_PERIOD = "fault-tolerance.threadWaitingPeriod"; - private static final long DEFAULT_BULKHEAD_TASK_QUEUEING_PERIOD = 2000L; - private static final String FT_BULKHEAD_TASK_QUEUEING_PERIOD = "fault-tolerance.bulkheadTaskQueueingPeriod"; - - private final InvocationContext context; - - private final RetryPolicy retryPolicy; - - private final boolean isAsynchronous; - - private final MethodIntrospector introspector; - - private final Method method; - - private int invocationCount = 0; - - private FaultToleranceCommand command; - - private ClassLoader contextClassLoader; - - private final long delayCorrection; - - private final int commandThreadPoolSize; - - private final long threadWaitingPeriod; - - private final long bulkheadTaskQueueingPeriod; - - private CompletableFuture taskQueued = new CompletableFuture<>(); - - /** - * Constructor. - * - * @param context The invocation context. - * @param introspector The method introspector. - */ - public CommandRetrier(InvocationContext context, MethodIntrospector introspector) { - this.context = context; - this.introspector = introspector; - this.isAsynchronous = introspector.isAsynchronous(); - this.method = context.getMethod(); - - // Init Helidon config params - Config config = ConfigProvider.getConfig(); - this.delayCorrection = config.getOptionalValue(FT_DELAY_CORRECTION, Long.class) - .orElse(DEFAULT_DELAY_CORRECTION); - this.commandThreadPoolSize = config.getOptionalValue(FT_COMMAND_THREAD_POOL_SIZE, Integer.class) - .orElse(DEFAULT_COMMAND_THREAD_POOL_SIZE); - this.threadWaitingPeriod = config.getOptionalValue(FT_THREAD_WAITING_PERIOD, Long.class) - .orElse(DEFAULT_THREAD_WAITING_PERIOD); - this.bulkheadTaskQueueingPeriod = config.getOptionalValue(FT_BULKHEAD_TASK_QUEUEING_PERIOD, Long.class) - .orElse(DEFAULT_BULKHEAD_TASK_QUEUEING_PERIOD); - - final Retry retry = introspector.getRetry(); - if (retry != null) { - // Initial setting for retry policy - this.retryPolicy = new RetryPolicy<>() - .withMaxRetries(retry.maxRetries()) - .withMaxDuration(Duration.of(retry.maxDuration(), retry.durationUnit())); - this.retryPolicy.handle(retry.retryOn()); - - // Set abortOn if defined - if (retry.abortOn().length > 0) { - this.retryPolicy.abortOn(retry.abortOn()); - } - - // Get delay and convert to nanos - long delay = TimeUtil.convertToNanos(retry.delay(), retry.delayUnit()); - - /* - * Apply delay correction to account for time spent in our code. This - * correction is necessary if user code measures intervals between - * calls that include time spent in Helidon. See TCK test {@link - * RetryTest#testRetryWithNoDelayAndJitter}. Failures may still occur - * on heavily loaded systems. - */ - Function correction = - d -> Math.abs(d - TimeUtil.convertToNanos(delayCorrection, ChronoUnit.MILLIS)); - - // Processing for jitter and delay - if (retry.jitter() > 0) { - long jitter = TimeUtil.convertToNanos(retry.jitter(), retry.jitterDelayUnit()); - - // Need to compute a factor and adjust delay for Failsafe - double factor; - if (jitter > delay) { - final long diff = jitter - delay; - delay = delay + diff / 2; - factor = 1.0; - } else { - factor = ((double) jitter) / delay; - } - this.retryPolicy.withDelay(Duration.of(correction.apply(delay), ChronoUnit.NANOS)); - this.retryPolicy.withJitter(factor); - } else if (retry.delay() > 0) { - this.retryPolicy.withDelay(Duration.of(correction.apply(delay), ChronoUnit.NANOS)); - } - } else { - this.retryPolicy = new RetryPolicy<>().withMaxRetries(0); // no retries - } - } - - /** - * Get command thread pool size. - * - * @return Thread pool size. - */ - int commandThreadPoolSize() { - return commandThreadPoolSize; - } - - /** - * Get thread waiting period. - * - * @return Thread waiting period. - */ - long threadWaitingPeriod() { - return threadWaitingPeriod; - } - - FaultToleranceCommand getCommand() { - return command; - } - - /** - * Retries running a command according to retry policy. - * - * @return Object returned by command. - * @throws Exception If something goes wrong. - */ - public Object execute() throws Exception { - LOGGER.fine(() -> "Executing command with isAsynchronous = " + isAsynchronous); - - FailsafeExecutor failsafe = prepareFailsafeExecutor(); - - try { - if (isAsynchronous) { - Scheduler scheduler = CommandScheduler.create(commandThreadPoolSize); - failsafe = failsafe.with(scheduler); - - // Store context class loader to access config - contextClassLoader = Thread.currentThread().getContextClassLoader(); - - // Check CompletionStage first to process CompletableFuture here - if (introspector.isReturnType(CompletionStage.class)) { - CompletionStage completionStage = CommandCompletableFuture.create( - failsafe.getStageAsync(() -> (CompletionStage) retryExecute()), - this::getCommand); - awaitBulkheadAsyncTaskQueued(); - return completionStage; - } - - // If not, it must be a subtype of Future - if (introspector.isReturnType(Future.class)) { - Future future = CommandCompletableFuture.create( - failsafe.getAsync(() -> (Future) retryExecute()), - this::getCommand); - awaitBulkheadAsyncTaskQueued(); - return future; - } - - // Oops, something went wrong during validation - throw new InternalError("Validation failed, return type must be Future or CompletionStage"); - } else { - return failsafe.get(this::retryExecute); - } - } catch (FailsafeException e) { - throw toException(e.getCause()); - } - } - - /** - * Set up the Failsafe executor. Add any fallback first, per Failsafe doc - * about "typical" policy composition - * - * @return Failsafe executor. - */ - private FailsafeExecutor prepareFailsafeExecutor() { - List> policies = new ArrayList<>(); - if (introspector.hasFallback()) { - CheckedFunction, ?> fallbackFunction = event -> { - final CommandFallback fallback = new CommandFallback(context, introspector, event.getLastFailure()); - Object result = fallback.execute(); - if (result instanceof CompletionStage) { - result = ((CompletionStage) result).toCompletableFuture(); - } - if (result instanceof Future) { - result = ((Future) result).get(); - } - return result; - }; - policies.add(Fallback.of(fallbackFunction)); - } - policies.add(retryPolicy); - return Failsafe.with(policies); - } - - /** - * Creates a new command for each retry since Hystrix commands can only be - * executed once. Fallback method is not overridden here to ensure all - * retries are executed. If running in async mode, this method will execute - * on a different thread. - * - * @return Object returned by command. - */ - private Object retryExecute() throws Exception { - // Config requires use of appropriate context class loader - if (contextClassLoader != null) { - Thread.currentThread().setContextClassLoader(contextClassLoader); - } - - final String commandKey = createCommandKey(); - command = new FaultToleranceCommand(this, commandKey, introspector, context, - contextClassLoader, taskQueued); - - // Configure command before execution - introspector.getHystrixProperties() - .entrySet() - .forEach(entry -> setProperty(commandKey, entry.getKey(), entry.getValue())); - - Object result; - try { - LOGGER.fine(() -> "About to execute command with key " - + command.getCommandKey() - + " on thread " + Thread.currentThread().getName()); - - // Execute task - invocationCount++; - updateMetricsBefore(); - result = command.execute(); - updateMetricsAfter(null); - } catch (ExceptionUtil.WrappedException e) { - Throwable cause = e.getCause(); - if (cause instanceof HystrixRuntimeException) { - cause = cause.getCause(); - } - - updateMetricsAfter(cause); - - if (cause instanceof TimeoutException) { - throw new org.eclipse.microprofile.faulttolerance.exceptions.TimeoutException(cause); - } - if (isBulkheadRejection(cause)) { - throw new BulkheadException(cause); - } - if (isHystrixBreakerException(cause)) { - throw new CircuitBreakerOpenException(cause); - } - throw toException(cause); - } - return result; - } - - /** - * A task can be queued on a bulkhead. When async and bulkheads are combined, - * we need to ensure that they get queued in the correct order before - * returning control back to the application. An exception thrown during - * queueing is processed in {@link FaultToleranceCommand#execute()}. - */ - private void awaitBulkheadAsyncTaskQueued() { - if (introspector.hasBulkhead()) { - try { - taskQueued.get(bulkheadTaskQueueingPeriod, TimeUnit.MILLISECONDS); - } catch (Exception e) { - LOGGER.info(() -> "Bulkhead async task queueing exception " + e); - } - } - } - - /** - * Update metrics before method is called. - */ - private void updateMetricsBefore() { - if (isFaultToleranceMetricsEnabled()) { - if (introspector.hasRetry() && invocationCount > 1) { - getCounter(method, RETRY_RETRIES_TOTAL).inc(); - } - } - } - - /** - * Update metrics after method is called and depending on outcome. - * - * @param cause Exception cause or {@code null} if execution successful. - */ - private void updateMetricsAfter(Throwable cause) { - if (!isFaultToleranceMetricsEnabled()) { - return; - } - - // Special logic for methods with retries - if (introspector.hasRetry()) { - final Retry retry = introspector.getRetry(); - boolean firstInvocation = (invocationCount == 1); - - if (cause == null) { - getCounter(method, INVOCATIONS_TOTAL).inc(); - getCounter(method, firstInvocation - ? RETRY_CALLS_SUCCEEDED_NOT_RETRIED_TOTAL - : RETRY_CALLS_SUCCEEDED_RETRIED_TOTAL).inc(); - } else if (retry.maxRetries() == invocationCount - 1) { - getCounter(method, RETRY_CALLS_FAILED_TOTAL).inc(); - getCounter(method, INVOCATIONS_FAILED_TOTAL).inc(); - getCounter(method, INVOCATIONS_TOTAL).inc(); - } - } else { - // Global method counters - getCounter(method, INVOCATIONS_TOTAL).inc(); - if (cause != null) { - getCounter(method, INVOCATIONS_FAILED_TOTAL).inc(); - } - } - - // Timeout - if (introspector.hasTimeout()) { - getHistogram(method, TIMEOUT_EXECUTION_DURATION) - .update(command.getExecutionTime()); - getCounter(method, cause instanceof TimeoutException - ? TIMEOUT_CALLS_TIMED_OUT_TOTAL - : TIMEOUT_CALLS_NOT_TIMED_OUT_TOTAL).inc(); - } - - // Bulkhead - if (introspector.hasBulkhead()) { - boolean bulkheadRejection = isBulkheadRejection(cause); - if (!bulkheadRejection) { - getHistogram(method, BULKHEAD_EXECUTION_DURATION).update(command.getExecutionTime()); - } - getCounter(method, bulkheadRejection ? BULKHEAD_CALLS_REJECTED_TOTAL - : BULKHEAD_CALLS_ACCEPTED_TOTAL).inc(); - } - } - - /** - * Returns a key for a command. Keys are specific to the pair of instance (target) - * and the method being called. - * - * @return A command key. - */ - private String createCommandKey() { - return method.getName() + Objects.hash(context.getTarget(), context.getMethod().hashCode()); - } - - /** - * Sets a Hystrix property on a command. - * - * @param commandKey Command key. - * @param key Property key. - * @param value Property value. - */ - private void setProperty(String commandKey, String key, Object value) { - final String actualKey = String.format("hystrix.command.%s.%s", commandKey, key); - synchronized (ConfigurationManager.getConfigInstance()) { - final AbstractConfiguration configManager = ConfigurationManager.getConfigInstance(); - if (configManager.getProperty(actualKey) == null) { - configManager.setProperty(actualKey, value); - } - } - } - - /** - * Hystrix throws a {@code RuntimeException}, so we need to check - * the message to determine if it is a breaker exception. - * - * @param t Throwable to check. - * @return Outcome of test. - */ - private static boolean isHystrixBreakerException(Throwable t) { - return t instanceof RuntimeException && t.getMessage().contains("Hystrix " - + "circuit short-circuited and is OPEN"); - } - - /** - * Checks if the parameter is a bulkhead exception. Note that Hystrix with semaphore - * isolation may throw a {@code RuntimeException}, so we need to check the message - * to determine if it is a semaphore exception. - * - * @param t Throwable to check. - * @return Outcome of test. - */ - private static boolean isBulkheadRejection(Throwable t) { - return t instanceof RejectedExecutionException - || t instanceof RuntimeException && t.getMessage().contains("could " - + "not acquire a semaphore for execution"); - } -} diff --git a/microprofile/fault-tolerance/src/main/java/io/helidon/microprofile/faulttolerance/CommandScheduler.java b/microprofile/fault-tolerance/src/main/java/io/helidon/microprofile/faulttolerance/CommandScheduler.java deleted file mode 100644 index 58da9c301f8..00000000000 --- a/microprofile/fault-tolerance/src/main/java/io/helidon/microprofile/faulttolerance/CommandScheduler.java +++ /dev/null @@ -1,82 +0,0 @@ -/* - * Copyright (c) 2018, 2020 Oracle and/or its affiliates. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package io.helidon.microprofile.faulttolerance; - -import java.util.concurrent.Callable; -import java.util.concurrent.ScheduledFuture; -import java.util.concurrent.TimeUnit; - -import io.helidon.common.configurable.ScheduledThreadPoolSupplier; - -import net.jodah.failsafe.util.concurrent.Scheduler; - -/** - * Class CommandScheduler. - */ -public class CommandScheduler implements Scheduler { - - private static final String THREAD_NAME_PREFIX = "helidon-ft-async-"; - - private static CommandScheduler instance; - - private final ScheduledThreadPoolSupplier poolSupplier; - - private CommandScheduler(ScheduledThreadPoolSupplier poolSupplier) { - this.poolSupplier = poolSupplier; - } - - /** - * If no command scheduler exists, creates one using default values. - * The created command scheduler uses daemon threads, so the JVM shuts-down if these are the only ones running. - * - * @param threadPoolSize Size of thread pool for async commands. - * @return Existing scheduler or newly created one. - */ - public static synchronized CommandScheduler create(int threadPoolSize) { - if (instance == null) { - instance = new CommandScheduler(ScheduledThreadPoolSupplier.builder() - .daemon(true) - .threadNamePrefix(THREAD_NAME_PREFIX) - .corePoolSize(threadPoolSize) - .prestart(false) - .build()); - } - return instance; - } - - /** - * Returns underlying pool supplier. - * - * @return The pool supplier. - */ - ScheduledThreadPoolSupplier poolSupplier() { - return poolSupplier; - } - - /** - * Schedules a task using an executor. - * - * @param callable The callable. - * @param delay Delay before scheduling task. - * @param unit Unite of delay. - * @return Future to track task execution. - */ - @Override - public ScheduledFuture schedule(Callable callable, long delay, TimeUnit unit) { - return poolSupplier.get().schedule(callable, delay, unit); - } -} diff --git a/microprofile/fault-tolerance/src/main/java/io/helidon/microprofile/faulttolerance/ExceptionUtil.java b/microprofile/fault-tolerance/src/main/java/io/helidon/microprofile/faulttolerance/ExceptionUtil.java deleted file mode 100644 index 5c5ffd9cb0f..00000000000 --- a/microprofile/fault-tolerance/src/main/java/io/helidon/microprofile/faulttolerance/ExceptionUtil.java +++ /dev/null @@ -1,69 +0,0 @@ -/* - * Copyright (c) 2018, 2019 Oracle and/or its affiliates. All rights reserved. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package io.helidon.microprofile.faulttolerance; - -import com.netflix.hystrix.exception.HystrixRuntimeException; - -/** - * Class ExceptionUtil. - */ -public class ExceptionUtil { - - /** - * Exception used internally to propagate other exceptions. - */ - static class WrappedException extends RuntimeException { - WrappedException(Throwable t) { - super(t); - } - } - - /** - * Wrap throwable into {@code Exception}. - * - * @param throwable The throwable. - * @return A {@code RuntimeException}. - */ - public static Exception toException(Throwable throwable) { - return throwable instanceof Exception ? (Exception) throwable - : new RuntimeException(throwable); - } - - /** - * Wrap throwable into {@code RuntimeException}. - * - * @param throwable The throwable. - * @return A {@code RuntimeException}. - */ - public static WrappedException toWrappedException(Throwable throwable) { - return throwable instanceof WrappedException ? (WrappedException) throwable - : new WrappedException(throwable); - } - - /** - * Unwrap an throwable wrapped by {@code HystrixRuntimeException}. - * - * @param throwable Throwable to unwrap. - * @return Unwrapped throwable. - */ - public static Throwable unwrapHystrix(Throwable throwable) { - return throwable instanceof HystrixRuntimeException ? throwable.getCause() : throwable; - } - - private ExceptionUtil() { - } -} diff --git a/microprofile/fault-tolerance/src/main/java/io/helidon/microprofile/faulttolerance/FaultToleranceCommand.java b/microprofile/fault-tolerance/src/main/java/io/helidon/microprofile/faulttolerance/FaultToleranceCommand.java deleted file mode 100644 index 05465b33d58..00000000000 --- a/microprofile/fault-tolerance/src/main/java/io/helidon/microprofile/faulttolerance/FaultToleranceCommand.java +++ /dev/null @@ -1,476 +0,0 @@ -/* - * Copyright (c) 2018, 2020 Oracle and/or its affiliates. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package io.helidon.microprofile.faulttolerance; - -import java.lang.reflect.Method; -import java.util.Arrays; -import java.util.concurrent.CompletableFuture; -import java.util.concurrent.ExecutionException; -import java.util.concurrent.Future; -import java.util.logging.Logger; - -import javax.interceptor.InvocationContext; - -import io.helidon.common.context.Context; -import io.helidon.common.context.Contexts; - -import com.netflix.hystrix.HystrixCommand; -import com.netflix.hystrix.HystrixCommandGroupKey; -import com.netflix.hystrix.HystrixCommandKey; -import com.netflix.hystrix.HystrixCommandProperties; -import com.netflix.hystrix.HystrixThreadPoolKey; -import com.netflix.hystrix.HystrixThreadPoolProperties; -import org.eclipse.microprofile.metrics.Histogram; - -import static com.netflix.hystrix.HystrixCommandProperties.ExecutionIsolationStrategy.SEMAPHORE; -import static com.netflix.hystrix.HystrixCommandProperties.ExecutionIsolationStrategy.THREAD; -import static io.helidon.microprofile.faulttolerance.CircuitBreakerHelper.State; -import static io.helidon.microprofile.faulttolerance.FaultToleranceExtension.isFaultToleranceMetricsEnabled; -import static io.helidon.microprofile.faulttolerance.FaultToleranceMetrics.BREAKER_CALLS_FAILED_TOTAL; -import static io.helidon.microprofile.faulttolerance.FaultToleranceMetrics.BREAKER_CALLS_PREVENTED_TOTAL; -import static io.helidon.microprofile.faulttolerance.FaultToleranceMetrics.BREAKER_CALLS_SUCCEEDED_TOTAL; -import static io.helidon.microprofile.faulttolerance.FaultToleranceMetrics.BREAKER_CLOSED_TOTAL; -import static io.helidon.microprofile.faulttolerance.FaultToleranceMetrics.BREAKER_HALF_OPEN_TOTAL; -import static io.helidon.microprofile.faulttolerance.FaultToleranceMetrics.BREAKER_OPENED_TOTAL; -import static io.helidon.microprofile.faulttolerance.FaultToleranceMetrics.BREAKER_OPEN_TOTAL; -import static io.helidon.microprofile.faulttolerance.FaultToleranceMetrics.BULKHEAD_CONCURRENT_EXECUTIONS; -import static io.helidon.microprofile.faulttolerance.FaultToleranceMetrics.BULKHEAD_WAITING_DURATION; -import static io.helidon.microprofile.faulttolerance.FaultToleranceMetrics.BULKHEAD_WAITING_QUEUE_POPULATION; -import static io.helidon.microprofile.faulttolerance.FaultToleranceMetrics.METRIC_NAME_TEMPLATE; -import static io.helidon.microprofile.faulttolerance.FaultToleranceMetrics.getCounter; -import static io.helidon.microprofile.faulttolerance.FaultToleranceMetrics.getHistogram; -import static io.helidon.microprofile.faulttolerance.FaultToleranceMetrics.registerGauge; -import static io.helidon.microprofile.faulttolerance.FaultToleranceMetrics.registerHistogram; - -/** - * Class FaultToleranceCommand. - */ -public class FaultToleranceCommand extends HystrixCommand { - private static final Logger LOGGER = Logger.getLogger(FaultToleranceCommand.class.getName()); - - static final String HELIDON_MICROPROFILE_FAULTTOLERANCE = "io.helidon.microprofile.faulttolerance"; - - private final String commandKey; - - private final MethodIntrospector introspector; - - private final InvocationContext context; - - private long executionTime = -1L; - - private CircuitBreakerHelper breakerHelper; - - private BulkheadHelper bulkheadHelper; - - private long queuedNanos = -1L; - - private Thread runThread; - - private ClassLoader contextClassLoader; - - private final long threadWaitingPeriod; - - /** - * Helidon context in which to run business method. - */ - private Context helidonContext; - - private CompletableFuture taskQueued; - - /** - * Constructor. Specify a thread pool key if a {@code @Bulkhead} is specified - * on the method. A unique thread pool key will enable setting limits for this - * command only based on the {@code Bulkhead} properties. - * - * @param commandRetrier The command retrier associated with this command. - * @param commandKey The command key. - * @param introspector The method introspector. - * @param context CDI invocation context. - * @param contextClassLoader Context class loader or {@code null} if not available. - * @param taskQueued Future completed when task has been queued. - */ - public FaultToleranceCommand(CommandRetrier commandRetrier, String commandKey, - MethodIntrospector introspector, - InvocationContext context, ClassLoader contextClassLoader, - CompletableFuture taskQueued) { - super(Setter.withGroupKey(HystrixCommandGroupKey.Factory.asKey(HELIDON_MICROPROFILE_FAULTTOLERANCE)) - .andCommandKey( - HystrixCommandKey.Factory.asKey(commandKey)) - .andCommandPropertiesDefaults( - HystrixCommandProperties.Setter() - .withFallbackEnabled(false) - .withExecutionIsolationStrategy(introspector.hasBulkhead() - && !introspector.isAsynchronous() ? SEMAPHORE : THREAD) - .withExecutionIsolationThreadInterruptOnFutureCancel(true) - .withExecutionIsolationThreadInterruptOnTimeout(true) - .withExecutionTimeoutEnabled(false)) - .andThreadPoolKey( - introspector.hasBulkhead() - ? HystrixThreadPoolKey.Factory.asKey(commandKey) - : null) - .andThreadPoolPropertiesDefaults( - HystrixThreadPoolProperties.Setter() - .withCoreSize( - introspector.hasBulkhead() - ? introspector.getBulkhead().value() - : commandRetrier.commandThreadPoolSize()) - .withMaximumSize( - introspector.hasBulkhead() - ? introspector.getBulkhead().value() - : commandRetrier.commandThreadPoolSize()) - .withMaxQueueSize( - introspector.hasBulkhead() && introspector.isAsynchronous() - ? introspector.getBulkhead().waitingTaskQueue() - : -1) - .withQueueSizeRejectionThreshold( - introspector.hasBulkhead() && introspector.isAsynchronous() - ? introspector.getBulkhead().waitingTaskQueue() - : -1))); - this.commandKey = commandKey; - this.introspector = introspector; - this.context = context; - this.contextClassLoader = contextClassLoader; - this.threadWaitingPeriod = commandRetrier.threadWaitingPeriod(); - this.taskQueued = taskQueued; - - // Special initialization for methods with breakers - if (introspector.hasCircuitBreaker()) { - this.breakerHelper = new CircuitBreakerHelper(this, introspector.getCircuitBreaker()); - - // Register gauges for this method - if (isFaultToleranceMetricsEnabled()) { - registerGauge(introspector.getMethod(), - BREAKER_OPEN_TOTAL, - "Amount of time the circuit breaker has spent in open state", - () -> breakerHelper.getInStateNanos(State.OPEN_MP)); - registerGauge(introspector.getMethod(), - BREAKER_HALF_OPEN_TOTAL, - "Amount of time the circuit breaker has spent in half-open state", - () -> breakerHelper.getInStateNanos(State.HALF_OPEN_MP)); - registerGauge(introspector.getMethod(), - BREAKER_CLOSED_TOTAL, - "Amount of time the circuit breaker has spent in closed state", - () -> breakerHelper.getInStateNanos(State.CLOSED_MP)); - } - } - - if (introspector.hasBulkhead()) { - bulkheadHelper = new BulkheadHelper(commandKey, introspector.getBulkhead()); - - if (isFaultToleranceMetricsEnabled()) { - // Record nanos to update metrics later - queuedNanos = System.nanoTime(); - - // Register gauges for this method - registerGauge(introspector.getMethod(), - BULKHEAD_CONCURRENT_EXECUTIONS, - "Number of currently running executions", - () -> (long) bulkheadHelper.runningInvocations()); - if (introspector.isAsynchronous()) { - registerGauge(introspector.getMethod(), - BULKHEAD_WAITING_QUEUE_POPULATION, - "Number of executions currently waiting in the queue", - () -> (long) bulkheadHelper.waitingInvocations()); - } - } - } - } - - /** - * Get command's execution time in nanos. - * - * @return Execution time in nanos. - * @throws IllegalStateException If called before command is executed. - */ - long getExecutionTime() { - if (executionTime == -1L) { - throw new IllegalStateException("Command has not been executed yet"); - } - return executionTime; - } - - BulkheadHelper getBulkheadHelper() { - return bulkheadHelper; - } - - /** - * Code to run as part of this command. Called from superclass. - * - * @return Result of command. - * @throws Exception If an error occurs. - */ - @Override - public Object run() throws Exception { - // Config requires use of appropriate context class loader - if (contextClassLoader != null) { - Thread.currentThread().setContextClassLoader(contextClassLoader); - } - - if (introspector.hasBulkhead()) { - bulkheadHelper.markAsRunning(this); - - if (isFaultToleranceMetricsEnabled()) { - // Register and update waiting time histogram - if (introspector.isAsynchronous() && queuedNanos != -1L) { - Method method = introspector.getMethod(); - Histogram histogram = getHistogram(method, BULKHEAD_WAITING_DURATION); - if (histogram == null) { - registerHistogram( - String.format(METRIC_NAME_TEMPLATE, - method.getDeclaringClass().getName(), - method.getName(), - BULKHEAD_WAITING_DURATION), - "Histogram of the time executions spend waiting in the queue"); - histogram = getHistogram(method, BULKHEAD_WAITING_DURATION); - } - histogram.update(System.nanoTime() - queuedNanos); - } - } - } - - // Finally, invoke the user method - try { - runThread = Thread.currentThread(); - return Contexts.runInContextWithThrow(helidonContext, context::proceed); - } finally { - if (introspector.hasBulkhead()) { - bulkheadHelper.markAsNotRunning(this); - } - } - } - - /** - * Executes this command returning a result or throwing an exception. - * - * @return The result. - * @throws RuntimeException If something goes wrong. - */ - @Override - public Object execute() { - this.helidonContext = Contexts.context().orElseGet(Context::create); - boolean lockRemoved = false; - - // Get lock and check breaker delay - if (introspector.hasCircuitBreaker()) { - try { - breakerHelper.lock(); - // OPEN_MP -> HALF_OPEN_MP - if (breakerHelper.getState() == State.OPEN_MP) { - long delayNanos = TimeUtil.convertToNanos(introspector.getCircuitBreaker().delay(), - introspector.getCircuitBreaker().delayUnit()); - if (breakerHelper.getCurrentStateNanos() > delayNanos) { - breakerHelper.setState(State.HALF_OPEN_MP); - } - } - } finally { - breakerHelper.unlock(); - } - - logCircuitBreakerState("Enter"); - } - - // Record state of breaker - boolean wasBreakerOpen = isCircuitBreakerOpen(); - - // Track invocation in a bulkhead - if (introspector.hasBulkhead()) { - bulkheadHelper.trackInvocation(this); - } - - // Execute command - Object result = null; - Future future = null; - Throwable throwable = null; - long startNanos = System.nanoTime(); - try { - // Queue the task - future = super.queue(); - - // Notify successful queueing of task - taskQueued.complete(null); - - // Execute and get result from task - result = future.get(); - } catch (Exception e) { - // Notify exception during task queueing - taskQueued.completeExceptionally(e); - - if (e instanceof ExecutionException) { - waitForThreadToComplete(); - } - if (e instanceof InterruptedException) { - future.cancel(true); - } - throwable = decomposeException(e); - } - - executionTime = System.nanoTime() - startNanos; - boolean hasFailed = (throwable != null); - - if (introspector.hasCircuitBreaker()) { - try { - breakerHelper.lock(); - - // Keep track of failure ratios - breakerHelper.pushResult(throwable == null); - - // Query breaker states - boolean breakerOpening = false; - boolean isClosedNow = !wasBreakerOpen; - - /* - * Special logic for MP circuit breakers to support failOn. If not a - * throwable to fail on, restore underlying breaker and return. - */ - if (hasFailed) { - final Throwable unwrappedThrowable = ExceptionUtil.unwrapHystrix(throwable); - Class[] throwableClasses = introspector.getCircuitBreaker().failOn(); - boolean failOn = Arrays.asList(throwableClasses) - .stream() - .anyMatch(c -> c.isAssignableFrom(unwrappedThrowable.getClass())); - if (!failOn) { - // If underlying circuit breaker is not open, this counts as successful - // run since it failed on an exception not listed in failOn. - updateMetricsAfter(breakerHelper.getState() != State.OPEN_MP ? null : throwable, - wasBreakerOpen, isClosedNow, breakerOpening); - logCircuitBreakerState("Exit 1"); - throw ExceptionUtil.toWrappedException(throwable); - } - } - - // CLOSED_MP -> OPEN_MP - if (breakerHelper.getState() == State.CLOSED_MP) { - double failureRatio = breakerHelper.getFailureRatio(); - if (failureRatio >= introspector.getCircuitBreaker().failureRatio()) { - breakerHelper.setState(State.OPEN_MP); - breakerOpening = true; - } - } - - // HALF_OPEN_MP -> OPEN_MP - if (hasFailed) { - if (breakerHelper.getState() == State.HALF_OPEN_MP) { - breakerHelper.setState(State.OPEN_MP); - } - updateMetricsAfter(throwable, wasBreakerOpen, isClosedNow, breakerOpening); - logCircuitBreakerState("Exit 2"); - throw ExceptionUtil.toWrappedException(throwable); - } - - // Otherwise, increment success count - breakerHelper.incSuccessCount(); - - // HALF_OPEN_MP -> CLOSED_MP - if (breakerHelper.getState() == State.HALF_OPEN_MP) { - if (breakerHelper.getSuccessCount() == introspector.getCircuitBreaker().successThreshold()) { - breakerHelper.setState(State.CLOSED_MP); - breakerHelper.resetCommandData(); - lockRemoved = true; - isClosedNow = true; - } - } - - updateMetricsAfter(throwable, wasBreakerOpen, isClosedNow, breakerOpening); - } finally { - if (!lockRemoved) { - breakerHelper.unlock(); - } - } - } - - // Untrack invocation in a bulkhead - if (introspector.hasBulkhead()) { - bulkheadHelper.untrackInvocation(this); - } - - // Display circuit breaker state at exit - logCircuitBreakerState("Exit 3"); - - // Outcome of execution - if (throwable != null) { - throw ExceptionUtil.toWrappedException(throwable); - } else { - return result; - } - } - - private void updateMetricsAfter(Throwable throwable, boolean wasBreakerOpen, boolean isClosedNow, - boolean breakerWillOpen) { - if (!isFaultToleranceMetricsEnabled()) { - return; - } - - assert introspector.hasCircuitBreaker(); - Method method = introspector.getMethod(); - - if (throwable == null) { - // If no errors increment success counter - getCounter(method, BREAKER_CALLS_SUCCEEDED_TOTAL).inc(); - } else if (!wasBreakerOpen) { - // If error and breaker was closed, increment failed counter - getCounter(method, BREAKER_CALLS_FAILED_TOTAL).inc(); - // If it will open, increment counter - if (breakerWillOpen) { - getCounter(method, BREAKER_OPENED_TOTAL).inc(); - } - } - // If breaker was open and still is, increment prevented counter - if (wasBreakerOpen && !isClosedNow) { - getCounter(method, BREAKER_CALLS_PREVENTED_TOTAL).inc(); - } - } - - /** - * Logs circuit breaker state, if one is present. - * - * @param preamble Message preamble. - */ - private void logCircuitBreakerState(String preamble) { - if (introspector.hasCircuitBreaker()) { - String hystrixState = isCircuitBreakerOpen() ? "OPEN" : "CLOSED"; - LOGGER.fine(() -> preamble + ": breaker for " + getCommandKey() + " in state " - + breakerHelper.getState() + " (Hystrix: " + hystrixState - + " Thread:" + Thread.currentThread().getName() + ")"); - } - } - - /** - *

After a timeout expires, Hystrix can report an {@link ExecutionException} - * when a thread has been interrupted but it is still running (e.g. while in a - * busy loop). Hystrix makes this possible by using another thread to monitor - * the command's thread.

- * - *

According to the FT spec, the thread may continue to run, so here - * we give it a chance to do that before completing the execution of the - * command. For more information see TCK test {@code - * TimeoutUninterruptableTest::testTimeout}.

- */ - private void waitForThreadToComplete() { - if (!introspector.isAsynchronous() && runThread != null && runThread.isInterrupted()) { - try { - int waitTime = 250; - while (runThread.getState() == Thread.State.RUNNABLE && waitTime <= threadWaitingPeriod) { - LOGGER.fine(() -> "Waiting for completion of thread " + runThread); - Thread.sleep(waitTime); - waitTime += 250; - } - } catch (InterruptedException e) { - // Falls through - } - } - } -} diff --git a/microprofile/fault-tolerance/src/main/java/io/helidon/microprofile/faulttolerance/FaultToleranceExtension.java b/microprofile/fault-tolerance/src/main/java/io/helidon/microprofile/faulttolerance/FaultToleranceExtension.java index 6e1817da834..4a22ca1f79e 100644 --- a/microprofile/fault-tolerance/src/main/java/io/helidon/microprofile/faulttolerance/FaultToleranceExtension.java +++ b/microprofile/fault-tolerance/src/main/java/io/helidon/microprofile/faulttolerance/FaultToleranceExtension.java @@ -155,8 +155,8 @@ void registerInterceptorBindings(@Observes BeforeBeanDiscovery discovery, BeanMa new AnnotatedTypeWrapper<>(bm.createAnnotatedType(Fallback.class), LiteralCommandBinding.getInstance())); - discovery.addAnnotatedType(bm.createAnnotatedType(CommandInterceptor2.class), - CommandInterceptor2.class.getName()); + discovery.addAnnotatedType(bm.createAnnotatedType(CommandInterceptor.class), + CommandInterceptor.class.getName()); } /** @@ -164,7 +164,7 @@ void registerInterceptorBindings(@Observes BeforeBeanDiscovery discovery, BeanMa * * @param event Process annotated event. */ - void updatePriorityMaybe(@Observes final ProcessAnnotatedType event) { + void updatePriorityMaybe(@Observes final ProcessAnnotatedType event) { final Config config = ConfigProvider.getConfig(); Optional priority = config.getOptionalValue(MP_FT_INTERCEPTOR_PRIORITY, Integer.class); priority.ifPresent(v -> event.configureAnnotatedType() diff --git a/microprofile/fault-tolerance/src/main/java/io/helidon/microprofile/faulttolerance/MethodIntrospector.java b/microprofile/fault-tolerance/src/main/java/io/helidon/microprofile/faulttolerance/MethodIntrospector.java index ae2d7750970..9fcd45a3c3d 100644 --- a/microprofile/fault-tolerance/src/main/java/io/helidon/microprofile/faulttolerance/MethodIntrospector.java +++ b/microprofile/fault-tolerance/src/main/java/io/helidon/microprofile/faulttolerance/MethodIntrospector.java @@ -1,5 +1,5 @@ /* - * Copyright (c) 2018, 2019 Oracle and/or its affiliates. All rights reserved. + * Copyright (c) 2018, 2020 Oracle and/or its affiliates. All rights reserved. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -18,8 +18,7 @@ import java.lang.annotation.Annotation; import java.lang.reflect.Method; -import java.util.HashMap; -import java.util.Map; +import java.util.logging.Logger; import io.helidon.microprofile.faulttolerance.MethodAntn.LookupResult; @@ -30,27 +29,28 @@ import org.eclipse.microprofile.faulttolerance.Retry; import org.eclipse.microprofile.faulttolerance.Timeout; -import static io.helidon.microprofile.faulttolerance.FaultToleranceParameter.getParameter; import static io.helidon.microprofile.faulttolerance.MethodAntn.lookupAnnotation; +import static io.helidon.microprofile.faulttolerance.FaultToleranceParameter.getParameter; /** * Class MethodIntrospector. */ class MethodIntrospector { + private static final Logger LOGGER = Logger.getLogger(MethodIntrospector.class.getName()); private final Method method; private final Class beanClass; - private Retry retry; + private final Retry retry; - private Fallback fallback; + private final Fallback fallback; - private CircuitBreaker circuitBreaker; + private final CircuitBreaker circuitBreaker; - private Timeout timeout; + private final Timeout timeout; - private Bulkhead bulkhead; + private final Bulkhead bulkhead; /** * Constructor. @@ -69,7 +69,7 @@ class MethodIntrospector { this.fallback = isAnnotationEnabled(Fallback.class) ? new FallbackAntn(beanClass, method) : null; } - Method getMethod() { + Method method() { return method; } @@ -152,41 +152,6 @@ Bulkhead getBulkhead() { return bulkhead; } - /** - * Returns a collection of Hystrix properties needed to configure - * commands. These properties are derived from the set of annotations - * found on a method or its class. - * - * @return The collection of Hystrix properties. - */ - Map getHystrixProperties() { - final HashMap result = new HashMap<>(); - - // Use semaphores for async and bulkhead - if (!isAsynchronous() && hasBulkhead()) { - result.put("execution.isolation.semaphore.maxConcurrentRequests", bulkhead.value()); - } - - // Circuit breakers - result.put("circuitBreaker.enabled", hasCircuitBreaker()); - if (hasCircuitBreaker()) { - // We are implementing this logic internally, so set to high values - result.put("circuitBreaker.requestVolumeThreshold", Integer.MAX_VALUE); - result.put("circuitBreaker.errorThresholdPercentage", 100); - result.put("circuitBreaker.sleepWindowInMilliseconds", Long.MAX_VALUE); - } - - // Timeouts - result.put("execution.timeout.enabled", hasTimeout()); - if (hasTimeout()) { - final Timeout timeout = getTimeout(); - result.put("execution.isolation.thread.timeoutInMilliseconds", - TimeUtil.convertToMillis(timeout.value(), timeout.unit())); - } - - return result; - } - /** * Determines if annotation type is present and enabled. * @@ -206,19 +171,19 @@ private boolean isAnnotationEnabled(Class clazz) { value = getParameter(method.getDeclaringClass().getName(), method.getName(), annotationType, "enabled"); if (value != null) { - return Boolean.valueOf(value); + return Boolean.parseBoolean(value); } // Check if property defined at class level value = getParameter(method.getDeclaringClass().getName(), annotationType, "enabled"); if (value != null) { - return Boolean.valueOf(value); + return Boolean.parseBoolean(value); } // Check if property defined at global level value = getParameter(annotationType, "enabled"); if (value != null) { - return Boolean.valueOf(value); + return Boolean.parseBoolean(value); } // Default is enabled diff --git a/microprofile/fault-tolerance/src/main/java/io/helidon/microprofile/faulttolerance/TimeUtil.java b/microprofile/fault-tolerance/src/main/java/io/helidon/microprofile/faulttolerance/TimeUtil.java deleted file mode 100644 index f6b3b2ab5e0..00000000000 --- a/microprofile/fault-tolerance/src/main/java/io/helidon/microprofile/faulttolerance/TimeUtil.java +++ /dev/null @@ -1,112 +0,0 @@ -/* - * Copyright (c) 2018 Oracle and/or its affiliates. All rights reserved. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package io.helidon.microprofile.faulttolerance; - -import java.time.temporal.ChronoUnit; -import java.util.Objects; -import java.util.concurrent.TimeUnit; - -/** - * Class TimeUtil. - */ -public class TimeUtil { - - /** - * Converts a {@code ChronoUnit} to the equivalent {@code TimeUnit}. - * - * @param chronoUnit the ChronoUnit to convert - * @return the converted equivalent TimeUnit - * @throws IllegalArgumentException if {@code chronoUnit} has no equivalent TimeUnit - * @throws NullPointerException if {@code chronoUnit} is null - */ - public static TimeUnit chronoUnitToTimeUnit(ChronoUnit chronoUnit) { - switch (Objects.requireNonNull(chronoUnit, "chronoUnit")) { - case NANOS: - return TimeUnit.NANOSECONDS; - case MICROS: - return TimeUnit.MICROSECONDS; - case MILLIS: - return TimeUnit.MILLISECONDS; - case SECONDS: - return TimeUnit.SECONDS; - case MINUTES: - return TimeUnit.MINUTES; - case HOURS: - return TimeUnit.HOURS; - case DAYS: - return TimeUnit.DAYS; - default: - throw new IllegalArgumentException("No TimeUnit equivalent for ChronoUnit"); - } - } - - /** - * Converts this {@code TimeUnit} to the equivalent {@code ChronoUnit}. - * - * @param timeUnit The TimeUnit - * @return the converted equivalent ChronoUnit - * @throws IllegalArgumentException if {@code chronoUnit} has no equivalent TimeUnit - * @throws NullPointerException if {@code chronoUnit} is null - */ - public static ChronoUnit timeUnitToChronoUnit(TimeUnit timeUnit) { - switch (Objects.requireNonNull(timeUnit, "chronoUnit")) { - case NANOSECONDS: - return ChronoUnit.NANOS; - case MICROSECONDS: - return ChronoUnit.MICROS; - case MILLISECONDS: - return ChronoUnit.MILLIS; - case SECONDS: - return ChronoUnit.SECONDS; - case MINUTES: - return ChronoUnit.MINUTES; - case HOURS: - return ChronoUnit.HOURS; - case DAYS: - return ChronoUnit.DAYS; - default: - throw new IllegalArgumentException("No ChronoUnit equivalent for TimeUnit"); - } - } - - /** - * Converts a duration and its chrono unit to millis. - * - * @param duration The duration. - * @param chronoUnit The unit of the duration. - * @return Milliseconds. - */ - public static long convertToMillis(long duration, ChronoUnit chronoUnit) { - final TimeUnit timeUnit = chronoUnitToTimeUnit(chronoUnit); - return TimeUnit.MILLISECONDS.convert(duration, timeUnit); - } - - /** - * Converts a duration and its chrono unit to nanos. - * - * @param duration The duration. - * @param chronoUnit The unit of the duration. - * @return Nanoseconds. - */ - public static long convertToNanos(long duration, ChronoUnit chronoUnit) { - final TimeUnit timeUnit = chronoUnitToTimeUnit(chronoUnit); - return TimeUnit.NANOSECONDS.convert(duration, timeUnit); - } - - private TimeUtil() { - } -} diff --git a/microprofile/fault-tolerance/src/main/java/io/helidon/microprofile/faulttolerance/TimedHashMap.java b/microprofile/fault-tolerance/src/main/java/io/helidon/microprofile/faulttolerance/TimedHashMap.java deleted file mode 100644 index 0c081238ee7..00000000000 --- a/microprofile/fault-tolerance/src/main/java/io/helidon/microprofile/faulttolerance/TimedHashMap.java +++ /dev/null @@ -1,121 +0,0 @@ -/* - * Copyright (c) 2018 Oracle and/or its affiliates. All rights reserved. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package io.helidon.microprofile.faulttolerance; - -import java.util.Map; -import java.util.concurrent.ConcurrentHashMap; -import java.util.concurrent.Executors; -import java.util.concurrent.ScheduledExecutorService; -import java.util.concurrent.TimeUnit; -import java.util.logging.Logger; -import java.util.stream.Collectors; - -/** - * Class TimedHashMap. - * - * @param Type of key. - * @param Type of value. - */ -public class TimedHashMap extends ConcurrentHashMap { - private static final Logger LOGGER = Logger.getLogger(TimedHashMap.class.getName()); - - private static final int THREAD_POOL_SIZE = 3; - - private static final ScheduledExecutorService SCHEDULER = - Executors.newScheduledThreadPool(THREAD_POOL_SIZE); - - private final long ttlInMillis; - - private final Map created = new ConcurrentHashMap<>(); - - /** - * Constructor. - * - * @param ttlInMillis Time to live in millis for map entries. - */ - public TimedHashMap(long ttlInMillis) { - this.ttlInMillis = ttlInMillis; - SCHEDULER.scheduleAtFixedRate(this::expireOldEntries, ttlInMillis, - ttlInMillis, TimeUnit.MILLISECONDS); - } - - private void expireOldEntries() { - created.keySet() - .stream() - .filter(k -> System.currentTimeMillis() - created.get(k) > ttlInMillis) - .collect(Collectors.toSet()) - .stream() - .forEach(k -> { - LOGGER.fine("Removing expired key " + k); - remove(k); - created.remove(k); - }); - } - - @Override - public boolean equals(Object o) { - return super.equals(o); - } - - @Override - public int hashCode() { - return super.hashCode(); - } - - @Override - public V put(K key, V value) { - created.put(key, System.currentTimeMillis()); - return super.put(key, value); - } - - @Override - public void putAll(Map m) { - m.keySet() - .stream() - .forEach(k -> created.put(k, System.currentTimeMillis())); - super.putAll(m); - } - - @Override - public V remove(Object key) { - created.remove(key); - return super.remove(key); - } - - @Override - public void clear() { - created.clear(); - super.clear(); - } - - @Override - public V putIfAbsent(K key, V value) { - if (!created.containsKey(key)) { - created.put(key, System.currentTimeMillis()); - } - return super.putIfAbsent(key, value); - } - - @Override - public boolean remove(Object key, Object value) { - boolean removed = super.remove(key, value); - if (removed) { - created.remove(key); - } - return removed; - } -} diff --git a/microprofile/fault-tolerance/src/main/java/module-info.java b/microprofile/fault-tolerance/src/main/java/module-info.java index e38d57b53df..9cab2c12002 100644 --- a/microprofile/fault-tolerance/src/main/java/module-info.java +++ b/microprofile/fault-tolerance/src/main/java/module-info.java @@ -31,10 +31,6 @@ requires io.helidon.microprofile.metrics; requires jakarta.enterprise.cdi.api; - requires hystrix.core; - requires archaius.core; - requires commons.configuration; - requires failsafe; requires microprofile.config.api; requires microprofile.metrics.api; diff --git a/microprofile/fault-tolerance/src/test/java/io/helidon/microprofile/faulttolerance/CommandDataTest.java b/microprofile/fault-tolerance/src/test/java/io/helidon/microprofile/faulttolerance/CommandDataTest.java deleted file mode 100644 index a92345b0bf3..00000000000 --- a/microprofile/fault-tolerance/src/test/java/io/helidon/microprofile/faulttolerance/CommandDataTest.java +++ /dev/null @@ -1,58 +0,0 @@ -/* - * Copyright (c) 2018 Oracle and/or its affiliates. All rights reserved. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package io.helidon.microprofile.faulttolerance; - -import java.util.Arrays; - -import org.junit.jupiter.api.Test; - -import static org.hamcrest.CoreMatchers.is; -import static org.hamcrest.MatcherAssert.assertThat; - -/** - * Class CommandDataTest. - */ -public class CommandDataTest { - - @Test - public void testSuccessRatio() { - CircuitBreakerHelper.CommandData data = new CircuitBreakerHelper.CommandData(6); - Arrays.asList(true, true, true, false, false, false).forEach(data::pushResult); - assertThat(data.getSuccessRatio(), is(3.0d / 6)); - } - - @Test - public void testFailureRatio() { - CircuitBreakerHelper.CommandData data = new CircuitBreakerHelper.CommandData(4); - Arrays.asList(true, false, false, false).forEach(data::pushResult); - assertThat(data.getFailureRatio(), is(3.0d / 4)); - } - - @Test - public void testPushResult() { - CircuitBreakerHelper.CommandData data = new CircuitBreakerHelper.CommandData(2); - Arrays.asList(true, false, false, false, true, true).forEach(data::pushResult); // last two count - assertThat(data.getFailureRatio(), is(0.0d)); - } - - @Test - public void testSizeLessCapacity() { - CircuitBreakerHelper.CommandData data = new CircuitBreakerHelper.CommandData(6); - Arrays.asList(true, false, false).forEach(data::pushResult); - assertThat(data.getFailureRatio(), is(-1.0d)); // not enough data - } -} diff --git a/microprofile/fault-tolerance/src/test/java/io/helidon/microprofile/faulttolerance/SchedulerConfigTest.java b/microprofile/fault-tolerance/src/test/java/io/helidon/microprofile/faulttolerance/SchedulerConfigTest.java deleted file mode 100644 index 6555e94d7aa..00000000000 --- a/microprofile/fault-tolerance/src/test/java/io/helidon/microprofile/faulttolerance/SchedulerConfigTest.java +++ /dev/null @@ -1,58 +0,0 @@ -/* - * Copyright (c) 2018, 2019 Oracle and/or its affiliates. All rights reserved. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package io.helidon.microprofile.faulttolerance; - -import java.util.concurrent.ScheduledExecutorService; -import java.util.concurrent.ScheduledThreadPoolExecutor; - -import io.helidon.common.configurable.ScheduledThreadPoolSupplier; -import io.helidon.common.context.ContextAwareExecutorService; -import io.helidon.microprofile.server.Server; - -import org.junit.jupiter.api.Test; - -import static org.hamcrest.CoreMatchers.is; -import static org.hamcrest.CoreMatchers.notNullValue; -import static org.hamcrest.MatcherAssert.assertThat; - -/** - * Testing configuration of {@link CommandScheduler}. - */ -class SchedulerConfigTest { - - @Test - void testNonDefaultConfig() { - Server server = null; - try { - server = Server.builder().port(-1).build(); - server.start(); - - CommandScheduler commandScheduler = CommandScheduler.create(8); - assertThat(commandScheduler, notNullValue()); - ScheduledThreadPoolSupplier poolSupplier = commandScheduler.poolSupplier(); - - ScheduledExecutorService service = poolSupplier.get(); - ContextAwareExecutorService executorService = ((ContextAwareExecutorService) service); - ScheduledThreadPoolExecutor stpe = (ScheduledThreadPoolExecutor) executorService.unwrap(); - assertThat(stpe.getCorePoolSize(), is(8)); - } finally { - if (server != null) { - server.stop(); - } - } - } -} diff --git a/microprofile/fault-tolerance/src/test/java/io/helidon/microprofile/faulttolerance/TimedHashMapTest.java b/microprofile/fault-tolerance/src/test/java/io/helidon/microprofile/faulttolerance/TimedHashMapTest.java deleted file mode 100644 index fc611db400f..00000000000 --- a/microprofile/fault-tolerance/src/test/java/io/helidon/microprofile/faulttolerance/TimedHashMapTest.java +++ /dev/null @@ -1,67 +0,0 @@ -/* - * Copyright (c) 2018 Oracle and/or its affiliates. All rights reserved. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package io.helidon.microprofile.faulttolerance; - -import java.util.Map; -import java.util.stream.IntStream; - -import org.junit.jupiter.api.Test; - -import static org.hamcrest.MatcherAssert.assertThat; -import static org.hamcrest.Matchers.is; - -/** - * Class TimedHashMapTest. - */ -public class TimedHashMapTest extends TimedTest { - - private final long TTL = 500; - - private final Map cache = new TimedHashMap<>(TTL); - - @Test - public void testExpiration() throws Exception { - assertThat(cache.size(), is(0)); - IntStream.range(0, 10).forEach( - i -> cache.put(String.valueOf(i), String.valueOf(i)) - ); - assertThat(cache.size(), is(10)); - Thread.sleep(2 * TTL); - assertEventually(() -> assertThat(cache.size(), is(0))); - } - - @Test - public void testExpirationBatch() throws Exception { - assertThat(cache.size(), is(0)); - - // First batch - IntStream.range(0, 10).forEach( - i -> cache.put(String.valueOf(i), String.valueOf(i)) - ); - assertThat(cache.size(), is(10)); - Thread.sleep(TTL / 2); - - // Second batch - IntStream.range(10, 20).forEach( - i -> cache.put(String.valueOf(i), String.valueOf(i)) - ); - assertThat(cache.size(), is(20)); - Thread.sleep(TTL); - - assertEventually(() -> assertThat(cache.size(), is(0))); - } -} From c31b95fb932f205a4e43ae89b7b83408c223f69e Mon Sep 17 00:00:00 2001 From: Santiago Pericasgeertsen Date: Tue, 4 Aug 2020 11:11:10 -0400 Subject: [PATCH 15/65] Upgraded to FT 2.1 API. Fixed checkstyle and copyright problems. Signed-off-by: Santiago Pericasgeertsen --- dependencies/pom.xml | 2 +- .../faulttolerance/CircuitBreakerAntn.java | 23 +++++++++----- .../faulttolerance/CommandInterceptor.java | 3 +- .../faulttolerance/CommandRunner.java | 30 +++++++++---------- .../faulttolerance/FallbackAntn.java | 18 +++++++++-- .../faulttolerance/MethodIntrospector.java | 4 +-- .../faulttolerance/BulkheadBean.java | 2 +- .../faulttolerance/BulkheadTest.java | 2 +- .../faulttolerance/CircuitBreakerTest.java | 2 +- .../faulttolerance/RetryTest.java | 2 +- .../faulttolerance/TimeoutBean.java | 2 +- .../src/test/tck-suite.xml | 2 +- 12 files changed, 55 insertions(+), 37 deletions(-) diff --git a/dependencies/pom.xml b/dependencies/pom.xml index 687f39586c7..0ebb428861e 100644 --- a/dependencies/pom.xml +++ b/dependencies/pom.xml @@ -84,7 +84,7 @@ 1.1.1 2.2 1.1.2 - 2.0.2 + 2.1 1.3.1 1.3.3 1.0 diff --git a/microprofile/fault-tolerance/src/main/java/io/helidon/microprofile/faulttolerance/CircuitBreakerAntn.java b/microprofile/fault-tolerance/src/main/java/io/helidon/microprofile/faulttolerance/CircuitBreakerAntn.java index 879c2d932be..a16457e95c0 100644 --- a/microprofile/fault-tolerance/src/main/java/io/helidon/microprofile/faulttolerance/CircuitBreakerAntn.java +++ b/microprofile/fault-tolerance/src/main/java/io/helidon/microprofile/faulttolerance/CircuitBreakerAntn.java @@ -1,5 +1,5 @@ /* - * Copyright (c) 2018 Oracle and/or its affiliates. All rights reserved. + * Copyright (c) 2018, 2020 Oracle and/or its affiliates. All rights reserved. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -58,13 +58,6 @@ public void validate() { } } - @Override - public Class[] failOn() { - LookupResult lookupResult = lookupAnnotation(CircuitBreaker.class); - final String override = getParamOverride("failOn", lookupResult.getType()); - return override != null ? parseThrowableArray(override) : lookupResult.getAnnotation().failOn(); - } - @Override public long delay() { LookupResult lookupResult = lookupAnnotation(CircuitBreaker.class); @@ -99,4 +92,18 @@ public int successThreshold() { final String override = getParamOverride("successThreshold", lookupResult.getType()); return override != null ? Integer.parseInt(override) : lookupResult.getAnnotation().successThreshold(); } + + @Override + public Class[] failOn() { + LookupResult lookupResult = lookupAnnotation(CircuitBreaker.class); + final String override = getParamOverride("failOn", lookupResult.getType()); + return override != null ? parseThrowableArray(override) : lookupResult.getAnnotation().failOn(); + } + + @Override + public Class[] skipOn() { + LookupResult lookupResult = lookupAnnotation(CircuitBreaker.class); + final String override = getParamOverride("skipOn", lookupResult.getType()); + return override != null ? parseThrowableArray(override) : lookupResult.getAnnotation().skipOn(); + } } diff --git a/microprofile/fault-tolerance/src/main/java/io/helidon/microprofile/faulttolerance/CommandInterceptor.java b/microprofile/fault-tolerance/src/main/java/io/helidon/microprofile/faulttolerance/CommandInterceptor.java index 25bbec11ef8..a77bd619f4f 100644 --- a/microprofile/fault-tolerance/src/main/java/io/helidon/microprofile/faulttolerance/CommandInterceptor.java +++ b/microprofile/fault-tolerance/src/main/java/io/helidon/microprofile/faulttolerance/CommandInterceptor.java @@ -16,11 +16,12 @@ package io.helidon.microprofile.faulttolerance; +import java.util.logging.Logger; + import javax.annotation.Priority; import javax.interceptor.AroundInvoke; import javax.interceptor.Interceptor; import javax.interceptor.InvocationContext; -import java.util.logging.Logger; /** * Intercepts calls to FT methods and implements annotation semantics. diff --git a/microprofile/fault-tolerance/src/main/java/io/helidon/microprofile/faulttolerance/CommandRunner.java b/microprofile/fault-tolerance/src/main/java/io/helidon/microprofile/faulttolerance/CommandRunner.java index 19896c2d01c..8294a05bef2 100644 --- a/microprofile/fault-tolerance/src/main/java/io/helidon/microprofile/faulttolerance/CommandRunner.java +++ b/microprofile/fault-tolerance/src/main/java/io/helidon/microprofile/faulttolerance/CommandRunner.java @@ -15,7 +15,6 @@ */ package io.helidon.microprofile.faulttolerance; -import javax.interceptor.InvocationContext; import java.lang.reflect.Method; import java.time.Duration; import java.util.Objects; @@ -26,6 +25,8 @@ import java.util.concurrent.Future; import java.util.function.Supplier; +import javax.interceptor.InvocationContext; + import io.helidon.common.reactive.Single; import io.helidon.faulttolerance.Async; import io.helidon.faulttolerance.Bulkhead; @@ -92,7 +93,7 @@ public class CommandRunner implements FtSupplier { /** * Map of methods to their internal state. */ - private static final ConcurrentHashMap ftHandlers = new ConcurrentHashMap<>(); + private static final ConcurrentHashMap FT_HANDLERS = new ConcurrentHashMap<>(); /** * Start system nanos when handler is called. @@ -105,15 +106,15 @@ public class CommandRunner implements FtSupplier { private long invocationStartNanos; private static class MethodState { - FtHandlerTyped handler; - Retry retry; - Bulkhead bulkhead; - CircuitBreaker breaker; - State lastBreakerState; - long breakerTimerOpen; - long breakerTimerClosed; - long breakerTimerHalfOpen; - long startNanos; + private FtHandlerTyped handler; + private Retry retry; + private Bulkhead bulkhead; + private CircuitBreaker breaker; + private State lastBreakerState; + private long breakerTimerOpen; + private long breakerTimerClosed; + private long breakerTimerHalfOpen; + private long startNanos; } private final MethodState methodState; @@ -130,7 +131,7 @@ public CommandRunner(InvocationContext context, MethodIntrospector introspector) this.method = context.getMethod(); // Get or initialize new state for this method - this.methodState = ftHandlers.computeIfAbsent(method, method -> { + this.methodState = FT_HANDLERS.computeIfAbsent(method, method -> { MethodState methodState = new MethodState(); initMethodStateHandler(methodState); methodState.lastBreakerState = State.CLOSED; @@ -173,7 +174,7 @@ public CommandRunner(InvocationContext context, MethodIntrospector introspector) * Clears ftHandlers map of any cached handlers. */ static void clearFtHandlersMap() { - ftHandlers.clear(); + FT_HANDLERS.clear(); } /** @@ -203,8 +204,7 @@ public Object get() throws Throwable { Future delegate = null; if (o instanceof CompletionStage) { delegate = ((CompletionStage) o).toCompletableFuture(); - } - else if (o instanceof Future) { + } else if (o instanceof Future) { delegate = (Future) o; } if (delegate != null) { diff --git a/microprofile/fault-tolerance/src/main/java/io/helidon/microprofile/faulttolerance/FallbackAntn.java b/microprofile/fault-tolerance/src/main/java/io/helidon/microprofile/faulttolerance/FallbackAntn.java index 65577380af7..e39a2df4c0e 100644 --- a/microprofile/fault-tolerance/src/main/java/io/helidon/microprofile/faulttolerance/FallbackAntn.java +++ b/microprofile/fault-tolerance/src/main/java/io/helidon/microprofile/faulttolerance/FallbackAntn.java @@ -1,5 +1,5 @@ /* - * Copyright (c) 2018, 2019 Oracle and/or its affiliates. All rights reserved. + * Copyright (c) 2018, 2020 Oracle and/or its affiliates. All rights reserved. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -17,8 +17,6 @@ package io.helidon.microprofile.faulttolerance; import java.lang.reflect.Method; -import java.util.concurrent.CompletionStage; -import java.util.concurrent.Future; import org.eclipse.microprofile.faulttolerance.ExecutionContext; import org.eclipse.microprofile.faulttolerance.Fallback; @@ -103,4 +101,18 @@ public String fallbackMethod() { final String override = getParamOverride("fallbackMethod", lookupResult.getType()); return override != null ? override : lookupResult.getAnnotation().fallbackMethod(); } + + @Override + public Class[] applyOn() { + LookupResult lookupResult = lookupAnnotation(Fallback.class); + final String override = getParamOverride("applyOn", lookupResult.getType()); + return override != null ? parseThrowableArray(override) : lookupResult.getAnnotation().applyOn(); + } + + @Override + public Class[] skipOn() { + LookupResult lookupResult = lookupAnnotation(Fallback.class); + final String override = getParamOverride("skipOn", lookupResult.getType()); + return override != null ? parseThrowableArray(override) : lookupResult.getAnnotation().skipOn(); + } } diff --git a/microprofile/fault-tolerance/src/main/java/io/helidon/microprofile/faulttolerance/MethodIntrospector.java b/microprofile/fault-tolerance/src/main/java/io/helidon/microprofile/faulttolerance/MethodIntrospector.java index 9fcd45a3c3d..e3fff543c22 100644 --- a/microprofile/fault-tolerance/src/main/java/io/helidon/microprofile/faulttolerance/MethodIntrospector.java +++ b/microprofile/fault-tolerance/src/main/java/io/helidon/microprofile/faulttolerance/MethodIntrospector.java @@ -18,7 +18,6 @@ import java.lang.annotation.Annotation; import java.lang.reflect.Method; -import java.util.logging.Logger; import io.helidon.microprofile.faulttolerance.MethodAntn.LookupResult; @@ -29,14 +28,13 @@ import org.eclipse.microprofile.faulttolerance.Retry; import org.eclipse.microprofile.faulttolerance.Timeout; -import static io.helidon.microprofile.faulttolerance.MethodAntn.lookupAnnotation; import static io.helidon.microprofile.faulttolerance.FaultToleranceParameter.getParameter; +import static io.helidon.microprofile.faulttolerance.MethodAntn.lookupAnnotation; /** * Class MethodIntrospector. */ class MethodIntrospector { - private static final Logger LOGGER = Logger.getLogger(MethodIntrospector.class.getName()); private final Method method; diff --git a/microprofile/fault-tolerance/src/test/java/io/helidon/microprofile/faulttolerance/BulkheadBean.java b/microprofile/fault-tolerance/src/test/java/io/helidon/microprofile/faulttolerance/BulkheadBean.java index 3b574fba428..02d8f5f4dff 100644 --- a/microprofile/fault-tolerance/src/test/java/io/helidon/microprofile/faulttolerance/BulkheadBean.java +++ b/microprofile/fault-tolerance/src/test/java/io/helidon/microprofile/faulttolerance/BulkheadBean.java @@ -1,5 +1,5 @@ /* - * Copyright (c) 2018, 2019 Oracle and/or its affiliates. All rights reserved. + * Copyright (c) 2018, 2020 Oracle and/or its affiliates. All rights reserved. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/microprofile/fault-tolerance/src/test/java/io/helidon/microprofile/faulttolerance/BulkheadTest.java b/microprofile/fault-tolerance/src/test/java/io/helidon/microprofile/faulttolerance/BulkheadTest.java index d7317fb968e..fcdbf40bedb 100644 --- a/microprofile/fault-tolerance/src/test/java/io/helidon/microprofile/faulttolerance/BulkheadTest.java +++ b/microprofile/fault-tolerance/src/test/java/io/helidon/microprofile/faulttolerance/BulkheadTest.java @@ -1,5 +1,5 @@ /* - * Copyright (c) 2018, 2019 Oracle and/or its affiliates. All rights reserved. + * Copyright (c) 2018, 2020 Oracle and/or its affiliates. All rights reserved. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/microprofile/fault-tolerance/src/test/java/io/helidon/microprofile/faulttolerance/CircuitBreakerTest.java b/microprofile/fault-tolerance/src/test/java/io/helidon/microprofile/faulttolerance/CircuitBreakerTest.java index c8532c9bc4b..faa3f4f875f 100644 --- a/microprofile/fault-tolerance/src/test/java/io/helidon/microprofile/faulttolerance/CircuitBreakerTest.java +++ b/microprofile/fault-tolerance/src/test/java/io/helidon/microprofile/faulttolerance/CircuitBreakerTest.java @@ -1,5 +1,5 @@ /* - * Copyright (c) 2018, 2019 Oracle and/or its affiliates. All rights reserved. + * Copyright (c) 2018, 2020 Oracle and/or its affiliates. All rights reserved. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/microprofile/fault-tolerance/src/test/java/io/helidon/microprofile/faulttolerance/RetryTest.java b/microprofile/fault-tolerance/src/test/java/io/helidon/microprofile/faulttolerance/RetryTest.java index 457d44758d5..e0b12e78a5d 100644 --- a/microprofile/fault-tolerance/src/test/java/io/helidon/microprofile/faulttolerance/RetryTest.java +++ b/microprofile/fault-tolerance/src/test/java/io/helidon/microprofile/faulttolerance/RetryTest.java @@ -1,5 +1,5 @@ /* - * Copyright (c) 2018, 2019 Oracle and/or its affiliates. All rights reserved. + * Copyright (c) 2018, 2020 Oracle and/or its affiliates. All rights reserved. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/microprofile/fault-tolerance/src/test/java/io/helidon/microprofile/faulttolerance/TimeoutBean.java b/microprofile/fault-tolerance/src/test/java/io/helidon/microprofile/faulttolerance/TimeoutBean.java index eda2a829732..7eb96ff6ecc 100644 --- a/microprofile/fault-tolerance/src/test/java/io/helidon/microprofile/faulttolerance/TimeoutBean.java +++ b/microprofile/fault-tolerance/src/test/java/io/helidon/microprofile/faulttolerance/TimeoutBean.java @@ -1,5 +1,5 @@ /* - * Copyright (c) 2018 Oracle and/or its affiliates. All rights reserved. + * Copyright (c) 2020 Oracle and/or its affiliates. All rights reserved. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/microprofile/tests/tck/tck-fault-tolerance/src/test/tck-suite.xml b/microprofile/tests/tck/tck-fault-tolerance/src/test/tck-suite.xml index 09219a48616..51a9efd7f6b 100644 --- a/microprofile/tests/tck/tck-fault-tolerance/src/test/tck-suite.xml +++ b/microprofile/tests/tck/tck-fault-tolerance/src/test/tck-suite.xml @@ -23,7 +23,7 @@ - system like those in our CI/CD pipeline. - No longer commented out - use 'tck-ft' profile to run these tests --> - + From fe91cf3cca343b7c4d1a7b6a627ef56215c96450 Mon Sep 17 00:00:00 2001 From: Santiago Pericasgeertsen Date: Tue, 4 Aug 2020 15:48:04 -0400 Subject: [PATCH 16/65] Fixed used of duration units in Retry. Signed-off-by: Santiago Pericasgeertsen --- .../microprofile/faulttolerance/CommandRunner.java | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/microprofile/fault-tolerance/src/main/java/io/helidon/microprofile/faulttolerance/CommandRunner.java b/microprofile/fault-tolerance/src/main/java/io/helidon/microprofile/faulttolerance/CommandRunner.java index 8294a05bef2..f095b6a404a 100644 --- a/microprofile/fault-tolerance/src/main/java/io/helidon/microprofile/faulttolerance/CommandRunner.java +++ b/microprofile/fault-tolerance/src/main/java/io/helidon/microprofile/faulttolerance/CommandRunner.java @@ -298,10 +298,13 @@ private void initMethodStateHandler(MethodState methodState) { Retry retry = Retry.builder() .retryPolicy(Retry.JitterRetryPolicy.builder() .calls(introspector.getRetry().maxRetries() + 1) - .delay(Duration.ofMillis(introspector.getRetry().delay())) - .jitter(Duration.ofMillis(introspector.getRetry().jitter())) + .delay(Duration.of(introspector.getRetry().delay(), + introspector.getRetry().delayUnit())) + .jitter(Duration.of(introspector.getRetry().jitter(), + introspector.getRetry().jitterDelayUnit())) .build()) - .overallTimeout(Duration.ofNanos(Long.MAX_VALUE)) // not used + .overallTimeout(Duration.of(introspector.getRetry().maxDuration(), + introspector.getRetry().durationUnit())) .build(); builder.addRetry(retry); methodState.retry = retry; From 54ce27b618091aa1ed1e71f29178c7ff95dad12a Mon Sep 17 00:00:00 2001 From: Santiago Pericasgeertsen Date: Wed, 5 Aug 2020 10:57:57 -0400 Subject: [PATCH 17/65] Support for request scope propagation for CDI and Jersey. In the FT TCKs, only the CDI request scope is active. Temporarily disable annotation validation in CDI extension as this now causes TCKs to fail validation before running. Signed-off-by: Santiago Pericasgeertsen --- .../faulttolerance/CommandRunner.java | 93 ++++++++++++++++++- .../FaultToleranceExtension.java | 29 ++++-- 2 files changed, 111 insertions(+), 11 deletions(-) diff --git a/microprofile/fault-tolerance/src/main/java/io/helidon/microprofile/faulttolerance/CommandRunner.java b/microprofile/fault-tolerance/src/main/java/io/helidon/microprofile/faulttolerance/CommandRunner.java index f095b6a404a..83a565ce4db 100644 --- a/microprofile/fault-tolerance/src/main/java/io/helidon/microprofile/faulttolerance/CommandRunner.java +++ b/microprofile/fault-tolerance/src/main/java/io/helidon/microprofile/faulttolerance/CommandRunner.java @@ -18,15 +18,21 @@ import java.lang.reflect.Method; import java.time.Duration; import java.util.Objects; +import java.util.concurrent.Callable; import java.util.concurrent.CompletableFuture; import java.util.concurrent.CompletionStage; import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.ExecutionException; import java.util.concurrent.Future; import java.util.function.Supplier; +import java.util.logging.Logger; +import javax.enterprise.context.control.RequestContextController; +import javax.enterprise.inject.spi.CDI; import javax.interceptor.InvocationContext; +import io.helidon.common.context.Context; +import io.helidon.common.context.Contexts; import io.helidon.common.reactive.Single; import io.helidon.faulttolerance.Async; import io.helidon.faulttolerance.Bulkhead; @@ -41,6 +47,8 @@ import org.eclipse.microprofile.faulttolerance.exceptions.CircuitBreakerOpenException; import org.eclipse.microprofile.faulttolerance.exceptions.TimeoutException; import org.eclipse.microprofile.metrics.Counter; +import org.glassfish.jersey.process.internal.RequestContext; +import org.glassfish.jersey.process.internal.RequestScope; import static io.helidon.microprofile.faulttolerance.FaultToleranceExtension.isFaultToleranceMetricsEnabled; import static io.helidon.microprofile.faulttolerance.FaultToleranceMetrics.BREAKER_CALLS_FAILED_TOTAL; @@ -74,6 +82,7 @@ * Runs a FT method. */ public class CommandRunner implements FtSupplier { + private static final Logger LOGGER = Logger.getLogger(CommandRunner.class.getName()); /** * The method being intercepted. @@ -105,6 +114,26 @@ public class CommandRunner implements FtSupplier { */ private long invocationStartNanos; + /** + * Helidon context in which to run business method. + */ + private final Context helidonContext; + + /** + * Jersey's request scope object. Will be non-null if request scope is active. + */ + private RequestScope requestScope; + + /** + * Jersey's request scope object. + */ + private RequestContext requestContext; + + /** + * CDI's request scope controller used for activation/deactivation. + */ + private RequestContextController requestController; + private static class MethodState { private FtHandlerTyped handler; private Retry retry; @@ -129,6 +158,7 @@ public CommandRunner(InvocationContext context, MethodIntrospector introspector) this.context = context; this.introspector = introspector; this.method = context.getMethod(); + this.helidonContext = Contexts.context().orElseGet(Context::create); // Get or initialize new state for this method this.methodState = FT_HANDLERS.computeIfAbsent(method, method -> { @@ -144,6 +174,17 @@ public CommandRunner(InvocationContext context, MethodIntrospector introspector) return methodState; }); + // Gather information about current request scope if active + try { + requestController = CDI.current().select(RequestContextController.class).get(); + requestScope = CDI.current().select(RequestScope.class).get(); + requestContext = requestScope.current(); + } catch (Exception e) { + requestScope = null; + LOGGER.info(() -> "Request context not active for method " + method + + " on thread " + Thread.currentThread().getName()); + } + // Registration of gauges for bulkhead and circuit breakers if (isFaultToleranceMetricsEnabled()) { if (introspector.hasCircuitBreaker()) { @@ -185,13 +226,54 @@ static void clearFtHandlersMap() { @Override public Object get() throws Throwable { Single single; + if (introspector.isAsynchronous()) { if (introspector.isReturnType(CompletionStage.class) || introspector.isReturnType(Future.class)) { - // Invoke method in new thread and call get() to unwrap singles - single = Async.create().invoke(() -> { + // Wrap method call with Helidon context + Supplier callMethod = () -> { updateMetricsBefore(); - return methodState.handler.invoke(toCompletionStageSupplier(context::proceed)); - }); + try { + return Contexts.runInContextWithThrow(helidonContext, + () -> methodState.handler.invoke(toCompletionStageSupplier(context::proceed))); + } catch (Exception e) { + return Single.error(e); + } + }; + + /* + * Call method preserving request scope if active. This is required for + * @Inject and @Context to work properly. Note that it's possible for only + * CDI's request scope to be active at this time (e.g. in TCKs). + */ + if (requestScope != null) { // Jersey and CDI + single = Async.create().invoke(() -> { + try { + return requestScope.runInScope(requestContext, (Callable) (() -> { + try { + requestController.activate(); + return callMethod.get(); + } finally { + requestController.deactivate(); + } + })); + } catch (Exception e) { + return Single.error(e); + } + }); + } else if (requestController != null) { // CDI only + single = Async.create().invoke(() -> { + try { + requestController.activate(); + return callMethod.get(); + } catch (Exception e) { + return Single.error(e); + } finally { + requestController.deactivate(); + } + }); + } else { + single = Async.create().invoke(callMethod); + } // Unwrap nested futures and map exceptions on complete CompletableFuture future = new CompletableFuture<>(); @@ -233,7 +315,8 @@ public Object get() throws Throwable { Object result = null; Throwable cause = null; - single = methodState.handler.invoke(toCompletionStageSupplier(context::proceed)); + single = Contexts.runInContextWithThrow(helidonContext, + () -> methodState.handler.invoke(toCompletionStageSupplier(context::proceed))); try { // Need to allow completion with no value (null) for void methods CompletableFuture future = single.toStage(true).toCompletableFuture(); diff --git a/microprofile/fault-tolerance/src/main/java/io/helidon/microprofile/faulttolerance/FaultToleranceExtension.java b/microprofile/fault-tolerance/src/main/java/io/helidon/microprofile/faulttolerance/FaultToleranceExtension.java index 4a22ca1f79e..b94443bd911 100644 --- a/microprofile/fault-tolerance/src/main/java/io/helidon/microprofile/faulttolerance/FaultToleranceExtension.java +++ b/microprofile/fault-tolerance/src/main/java/io/helidon/microprofile/faulttolerance/FaultToleranceExtension.java @@ -40,6 +40,7 @@ import javax.enterprise.inject.spi.ProcessManagedBean; import javax.enterprise.inject.spi.ProcessSyntheticBean; import javax.enterprise.util.AnnotationLiteral; +import javax.inject.Inject; import org.eclipse.microprofile.config.Config; import org.eclipse.microprofile.config.ConfigProvider; @@ -49,6 +50,7 @@ import org.eclipse.microprofile.faulttolerance.Fallback; import org.eclipse.microprofile.faulttolerance.Retry; import org.eclipse.microprofile.faulttolerance.Timeout; +import org.glassfish.jersey.process.internal.RequestScope; /** * Class FaultToleranceExtension. @@ -157,6 +159,21 @@ void registerInterceptorBindings(@Observes BeforeBeanDiscovery discovery, BeanMa discovery.addAnnotatedType(bm.createAnnotatedType(CommandInterceptor.class), CommandInterceptor.class.getName()); + discovery.addAnnotatedType(bm.createAnnotatedType(JerseyRequestScopeAsCdiBean.class), + JerseyRequestScopeAsCdiBean.class.getName()); + } + + /** + * We require access to {@link org.glassfish.jersey.process.internal.RequestScope} + * via CDI to propagate request contexts to newly created threads, but Jersey + * only registers this type as a bean if it can find an injection point (see + * org.glassfish.jersey.ext.cdi1x.internal.CdiComponentProvider#afterDiscoveryObserver). + * Here we define a dummy bean with such an injection point for Jersey to + * create and register a CDI bean for RequestScope. + */ + private static class JerseyRequestScopeAsCdiBean { + @Inject + private RequestScope requestScope; } /** @@ -220,26 +237,26 @@ void registerFaultToleranceMetrics(@Observes AfterDeploymentValidation validatio // Metrics depending on the annotationSet present if (MethodAntn.isAnnotationPresent(beanClass, method, Retry.class)) { FaultToleranceMetrics.registerRetryMetrics(method); - new RetryAntn(beanClass, method).validate(); + // new RetryAntn(beanClass, method).validate(); } if (MethodAntn.isAnnotationPresent(beanClass, method, CircuitBreaker.class)) { FaultToleranceMetrics.registerCircuitBreakerMetrics(method); - new CircuitBreakerAntn(beanClass, method).validate(); + // new CircuitBreakerAntn(beanClass, method).validate(); } if (MethodAntn.isAnnotationPresent(beanClass, method, Timeout.class)) { FaultToleranceMetrics.registerTimeoutMetrics(method); - new TimeoutAntn(beanClass, method).validate(); + // new TimeoutAntn(beanClass, method).validate(); } if (MethodAntn.isAnnotationPresent(beanClass, method, Bulkhead.class)) { FaultToleranceMetrics.registerBulkheadMetrics(method); - new BulkheadAntn(beanClass, method).validate(); + // new BulkheadAntn(beanClass, method).validate(); } if (MethodAntn.isAnnotationPresent(beanClass, method, Fallback.class)) { FaultToleranceMetrics.registerFallbackMetrics(method); - new FallbackAntn(beanClass, method).validate(); + // new FallbackAntn(beanClass, method).validate(); } if (MethodAntn.isAnnotationPresent(beanClass, method, Asynchronous.class)) { - new AsynchronousAntn(beanClass, method).validate(); + // new AsynchronousAntn(beanClass, method).validate(); } }); } From f6f17773512f9bfa20147a45ccf372e53c72e42b Mon Sep 17 00:00:00 2001 From: Santiago Pericasgeertsen Date: Wed, 5 Aug 2020 13:32:18 -0400 Subject: [PATCH 18/65] Changes to ErrorChecker to implement MP semantics. First checks skipOn and then applyOn sets. Updates to some tests. Signed-off-by: Santiago Pericasgeertsen --- .../helidon/faulttolerance/ErrorChecker.java | 32 ++++++++----------- .../io/helidon/faulttolerance/RetryTest.java | 9 ------ .../faulttolerance/CommandRunner.java | 7 +++- 3 files changed, 20 insertions(+), 28 deletions(-) diff --git a/fault-tolerance/src/main/java/io/helidon/faulttolerance/ErrorChecker.java b/fault-tolerance/src/main/java/io/helidon/faulttolerance/ErrorChecker.java index c5bcc3a0b6a..0fcc4abd16d 100644 --- a/fault-tolerance/src/main/java/io/helidon/faulttolerance/ErrorChecker.java +++ b/fault-tolerance/src/main/java/io/helidon/faulttolerance/ErrorChecker.java @@ -22,25 +22,21 @@ interface ErrorChecker { boolean shouldSkip(Throwable throwable); + /** + * Returns ErrorChecker that skips if throwable is in skipOnSet or if applyOnSet + * is not empty and throwable is not in it. Note that if applyOnSet is empty, then + * it is equivalent to it containing {@code Throwable.class}. + * + * @param skipOnSet set of throwables to skip logic on. + * @param applyOnSet set of throwables to apply logic on. + * @return An error checker. + */ static ErrorChecker create(Set> skipOnSet, Set> applyOnSet) { - Set> skipOn = Set.copyOf(skipOnSet); - Set> applyOn = Set.copyOf(applyOnSet); - - if (skipOn.isEmpty()) { - if (applyOn.isEmpty()) { - return throwable -> false; - } else { - return throwable -> applyOn.stream().filter(t -> t.isAssignableFrom(throwable.getClass())).count() == 0; - } - } else { - if (applyOn.isEmpty()) { - return throwable -> skipOn.stream().filter(t -> t.isAssignableFrom(throwable.getClass())).count() > 0; - } else { - throw new IllegalArgumentException("You have defined both skip and apply set of exception classes. " - + "This cannot be correctly handled; skipOn: " + skipOn - + " applyOn: " + applyOn); - } + return throwable -> containsThrowable(skipOnSet, throwable) + || !applyOnSet.isEmpty() && !containsThrowable(applyOnSet, throwable); + } - } + private static boolean containsThrowable(Set> set, Throwable throwable) { + return set.stream().filter(t -> t.isAssignableFrom(throwable.getClass())).count() > 0; } } diff --git a/fault-tolerance/src/test/java/io/helidon/faulttolerance/RetryTest.java b/fault-tolerance/src/test/java/io/helidon/faulttolerance/RetryTest.java index 4d68bc11b32..aa7bf9895ff 100644 --- a/fault-tolerance/src/test/java/io/helidon/faulttolerance/RetryTest.java +++ b/fault-tolerance/src/test/java/io/helidon/faulttolerance/RetryTest.java @@ -138,15 +138,6 @@ void testTimeout() { assertThat("Should have been called twice", req.call.get(), is(2)); } - @Test - void testBadConfiguration() { - Retry.Builder builder = Retry.builder() - .applyOn(RetryException.class) - .skipOn(TerminalException.class); - - assertThrows(IllegalArgumentException.class, builder::build); - } - @Test void testMultiRetriesNoFailure() throws InterruptedException { Retry retry = Retry.builder() diff --git a/microprofile/fault-tolerance/src/main/java/io/helidon/microprofile/faulttolerance/CommandRunner.java b/microprofile/fault-tolerance/src/main/java/io/helidon/microprofile/faulttolerance/CommandRunner.java index 83a565ce4db..46d9b442150 100644 --- a/microprofile/fault-tolerance/src/main/java/io/helidon/microprofile/faulttolerance/CommandRunner.java +++ b/microprofile/fault-tolerance/src/main/java/io/helidon/microprofile/faulttolerance/CommandRunner.java @@ -181,7 +181,7 @@ public CommandRunner(InvocationContext context, MethodIntrospector introspector) requestContext = requestScope.current(); } catch (Exception e) { requestScope = null; - LOGGER.info(() -> "Request context not active for method " + method + LOGGER.fine(() -> "Request context not active for method " + method + " on thread " + Thread.currentThread().getName()); } @@ -351,6 +351,7 @@ private void initMethodStateHandler(MethodState methodState) { .errorRatio((int) (introspector.getCircuitBreaker().failureRatio() * 100)) .volume(introspector.getCircuitBreaker().requestVolumeThreshold()) .applyOn(introspector.getCircuitBreaker().failOn()) + .skipOn(introspector.getCircuitBreaker().skipOn()) .build(); builder.addBreaker(circuitBreaker); methodState.breaker = circuitBreaker; @@ -388,6 +389,8 @@ private void initMethodStateHandler(MethodState methodState) { .build()) .overallTimeout(Duration.of(introspector.getRetry().maxDuration(), introspector.getRetry().durationUnit())) + .applyOn(introspector.getRetry().retryOn()) + .skipOn(introspector.getRetry().abortOn()) .build(); builder.addRetry(retry); methodState.retry = retry; @@ -400,6 +403,8 @@ private void initMethodStateHandler(MethodState methodState) { CommandFallback cfb = new CommandFallback(context, introspector, throwable); return toCompletionStageSupplier(cfb::execute).get(); }) + .applyOn(introspector.getFallback().applyOn()) + .skipOn(introspector.getFallback().skipOn()) .build(); builder.addFallback(fallback); } From aeaec54f2d45e549be1fd5188aac6d9931470557 Mon Sep 17 00:00:00 2001 From: Santiago Pericasgeertsen Date: Wed, 5 Aug 2020 15:05:59 -0400 Subject: [PATCH 19/65] Handle somewhat unusual case of synchronous method returning a CompletionStage. Signed-off-by: Santiago Pericasgeertsen --- .../helidon/microprofile/faulttolerance/CommandRunner.java | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/microprofile/fault-tolerance/src/main/java/io/helidon/microprofile/faulttolerance/CommandRunner.java b/microprofile/fault-tolerance/src/main/java/io/helidon/microprofile/faulttolerance/CommandRunner.java index 46d9b442150..3ce0c7bdf7f 100644 --- a/microprofile/fault-tolerance/src/main/java/io/helidon/microprofile/faulttolerance/CommandRunner.java +++ b/microprofile/fault-tolerance/src/main/java/io/helidon/microprofile/faulttolerance/CommandRunner.java @@ -415,7 +415,7 @@ private void initMethodStateHandler(MethodState methodState) { /** * Maps an {@link FtSupplier} to a supplier of {@link CompletionStage}. Avoids - * unnecessary wrapping of stages. + * unnecessary wrapping of stages when method is asynchronous only. * * @param supplier The supplier. * @return The new supplier. @@ -426,7 +426,8 @@ Supplier> toCompletionStageSupplier(FtSupplier try { invocationStartNanos = System.nanoTime(); Object result = supplier.get(); - return result instanceof CompletionStage ? (CompletionStage) result + return introspector.isAsynchronous() && result instanceof CompletionStage + ? (CompletionStage) result : CompletableFuture.completedFuture(result); } catch (Throwable e) { return CompletableFuture.failedFuture(e); From b0929b6c17d7a2ce5761fac85985a7cdebd205c6 Mon Sep 17 00:00:00 2001 From: Santiago Pericasgeertsen Date: Mon, 10 Aug 2020 11:38:25 -0400 Subject: [PATCH 20/65] Changes in this commit: - Handle timeouts when a method catches InterruptedException - Changes to SE FT to support some event listeners - Create handlers for each invocation while sharing those for Bulkheads and CircuitBreakers - Renamed CommandRunner to MethodInvoker Signed-off-by: Santiago Pericasgeertsen --- .../java/io/helidon/faulttolerance/Retry.java | 19 +- .../io/helidon/faulttolerance/RetryImpl.java | 9 + .../io/helidon/faulttolerance/Timeout.java | 19 +- .../helidon/faulttolerance/TimeoutImpl.java | 46 ++++- .../faulttolerance/CommandInterceptor.java | 2 +- ...{CommandRunner.java => MethodInvoker.java} | 176 +++++++++++++----- .../faulttolerance/FaultToleranceTest.java | 5 +- .../faulttolerance/TimeoutBean.java | 11 ++ .../faulttolerance/TimeoutTest.java | 6 + 9 files changed, 229 insertions(+), 64 deletions(-) rename microprofile/fault-tolerance/src/main/java/io/helidon/microprofile/faulttolerance/{CommandRunner.java => MethodInvoker.java} (78%) diff --git a/fault-tolerance/src/main/java/io/helidon/faulttolerance/Retry.java b/fault-tolerance/src/main/java/io/helidon/faulttolerance/Retry.java index 757f5ce8482..2ca5886d0c1 100644 --- a/fault-tolerance/src/main/java/io/helidon/faulttolerance/Retry.java +++ b/fault-tolerance/src/main/java/io/helidon/faulttolerance/Retry.java @@ -23,6 +23,7 @@ import java.util.Random; import java.util.Set; import java.util.concurrent.ScheduledExecutorService; +import java.util.function.Consumer; import java.util.function.Supplier; import io.helidon.common.LazyValue; @@ -58,9 +59,9 @@ class Builder implements io.helidon.common.Builder { .jitter(Duration.ofMillis(50)) .build(); - private Duration overallTimeout = Duration.ofSeconds(1); private LazyValue scheduledExecutor = FaultTolerance.scheduledExecutor(); + private Consumer retryListener; private Builder() { } @@ -165,6 +166,18 @@ public Builder overallTimeout(Duration overallTimeout) { return this; } + /** + * A listener that is called every time there is a retry. The parameter + * to the listener is a retry count. + * + * @param retryListener a retry listener + * @return updated builder instance + */ + public Builder retryListener(Consumer retryListener) { + this.retryListener = retryListener; + return this; + } + Set> applyOn() { return applyOn; } @@ -184,6 +197,10 @@ Duration overallTimeout() { LazyValue scheduledExecutor() { return scheduledExecutor; } + + Consumer retryListener() { + return retryListener; + } } /** diff --git a/fault-tolerance/src/main/java/io/helidon/faulttolerance/RetryImpl.java b/fault-tolerance/src/main/java/io/helidon/faulttolerance/RetryImpl.java index 2d5bb3c3fee..18075f2838c 100644 --- a/fault-tolerance/src/main/java/io/helidon/faulttolerance/RetryImpl.java +++ b/fault-tolerance/src/main/java/io/helidon/faulttolerance/RetryImpl.java @@ -26,6 +26,7 @@ import java.util.concurrent.TimeoutException; import java.util.concurrent.atomic.AtomicInteger; import java.util.concurrent.atomic.AtomicLong; +import java.util.function.Consumer; import java.util.function.Supplier; import io.helidon.common.LazyValue; @@ -38,12 +39,14 @@ class RetryImpl implements Retry { private final long maxTimeNanos; private final Retry.RetryPolicy retryPolicy; private final AtomicLong retryCounter = new AtomicLong(0L); + private final Consumer retryListener; RetryImpl(Retry.Builder builder) { this.scheduledExecutor = builder.scheduledExecutor(); this.errorChecker = ErrorChecker.create(builder.skipOn(), builder.applyOn()); this.maxTimeNanos = builder.overallTimeout().toNanos(); this.retryPolicy = builder.retryPolicy(); + this.retryListener = builder.retryListener(); } @Override @@ -94,6 +97,9 @@ private Single retrySingle(RetryContext> con if (errorChecker.shouldSkip(cause)) { return Single.error(context.throwable()); } + if (retryListener != null) { + retryListener.accept(currentCallIndex); + } return retrySingle(context); }); } @@ -136,6 +142,9 @@ private Multi retryMulti(RetryContext> contex if (task.hadData() || errorChecker.shouldSkip(cause)) { return Multi.error(context.throwable()); } + if (retryListener != null) { + retryListener.accept(currentCallIndex); + } return retryMulti(context); }); } diff --git a/fault-tolerance/src/main/java/io/helidon/faulttolerance/Timeout.java b/fault-tolerance/src/main/java/io/helidon/faulttolerance/Timeout.java index a792f4dae10..906cf596238 100644 --- a/fault-tolerance/src/main/java/io/helidon/faulttolerance/Timeout.java +++ b/fault-tolerance/src/main/java/io/helidon/faulttolerance/Timeout.java @@ -18,6 +18,7 @@ import java.time.Duration; import java.util.concurrent.ScheduledExecutorService; +import java.util.function.Consumer; import io.helidon.common.LazyValue; @@ -53,13 +54,14 @@ class Builder implements io.helidon.common.Builder { private Duration timeout = Duration.ofSeconds(10); private LazyValue executor = FaultTolerance.scheduledExecutor(); private boolean async = true; + private Consumer listener; private Builder() { } @Override public Timeout build() { - return new TimeoutImpl(this, async); + return new TimeoutImpl(this); } /** @@ -95,6 +97,17 @@ public Builder executor(ScheduledExecutorService executor) { return this; } + /** + * Listener that will be called when a thread is interrupted. + * + * @param listener the listener + * @return updated build instance + */ + public Builder interruptListener(Consumer listener) { + this.listener = listener; + return this; + } + Duration timeout() { return timeout; } @@ -106,5 +119,9 @@ LazyValue executor() { boolean async() { return async; } + + Consumer interruptListener() { + return listener; + } } } diff --git a/fault-tolerance/src/main/java/io/helidon/faulttolerance/TimeoutImpl.java b/fault-tolerance/src/main/java/io/helidon/faulttolerance/TimeoutImpl.java index fcc835b64f8..fe49022c803 100644 --- a/fault-tolerance/src/main/java/io/helidon/faulttolerance/TimeoutImpl.java +++ b/fault-tolerance/src/main/java/io/helidon/faulttolerance/TimeoutImpl.java @@ -17,11 +17,13 @@ package io.helidon.faulttolerance; import java.time.Duration; +import java.util.concurrent.CompletableFuture; import java.util.concurrent.CompletionStage; import java.util.concurrent.Flow; import java.util.concurrent.ScheduledExecutorService; import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicBoolean; +import java.util.function.Consumer; import java.util.function.Supplier; import io.helidon.common.LazyValue; @@ -29,14 +31,18 @@ import io.helidon.common.reactive.Single; class TimeoutImpl implements Timeout { + private static final long MONITOR_THREAD_TIMEOUT = 100L; + private final long timeoutMillis; private final LazyValue executor; private final boolean async; + private final Consumer interruptListener; - TimeoutImpl(Timeout.Builder builder, boolean async) { + TimeoutImpl(Timeout.Builder builder) { this.timeoutMillis = builder.timeout().toMillis(); this.executor = builder.executor(); - this.async = async; + this.async = builder.async(); + this.interruptListener = builder.interruptListener(); } @Override @@ -51,21 +57,43 @@ public Multi invokeMulti(Supplier> supplier) @Override public Single invoke(Supplier> supplier) { if (async) { - return Single.create(supplier.get()) + return Single.create(supplier.get(), true) .timeout(timeoutMillis, TimeUnit.MILLISECONDS, executor.get()); } else { Thread currentThread = Thread.currentThread(); - AtomicBoolean called = new AtomicBoolean(); - Timeout.create(Duration.ofMillis(timeoutMillis)) - .invoke(Single::never) + CompletableFuture monitorStarted = new CompletableFuture<>(); + AtomicBoolean callReturned = new AtomicBoolean(false); + + // Startup monitor thread that can interrupt current thread after timeout + Timeout.builder() + .executor(executor.get()) // propagate executor + .async(true) + .timeout(Duration.ofMillis(timeoutMillis)) + .build() + .invoke(() -> { + monitorStarted.complete(null); + return Single.never(); + }) .exceptionally(it -> { - if (called.compareAndSet(false, true)) { + if (callReturned.compareAndSet(false, true)) { currentThread.interrupt(); + if (interruptListener != null) { + interruptListener.accept(currentThread); + } } return null; }); - Single single = Single.create(supplier.get()); // runs in current thread - called.set(true); + + // Ensure monitor thread has started + try { + monitorStarted.get(MONITOR_THREAD_TIMEOUT, TimeUnit.MILLISECONDS); + } catch (Exception e) { + return Single.error(new IllegalStateException("Timeout monitor thread failed to start")); + } + + // Run invocation in current thread + Single single = Single.create(supplier.get(), true); + callReturned.set(true); return single; } } diff --git a/microprofile/fault-tolerance/src/main/java/io/helidon/microprofile/faulttolerance/CommandInterceptor.java b/microprofile/fault-tolerance/src/main/java/io/helidon/microprofile/faulttolerance/CommandInterceptor.java index a77bd619f4f..89223f920ab 100644 --- a/microprofile/fault-tolerance/src/main/java/io/helidon/microprofile/faulttolerance/CommandInterceptor.java +++ b/microprofile/fault-tolerance/src/main/java/io/helidon/microprofile/faulttolerance/CommandInterceptor.java @@ -50,7 +50,7 @@ public Object interceptCommand(InvocationContext context) throws Throwable { // Create method introspector and executer retrier MethodIntrospector introspector = new MethodIntrospector(context.getTarget().getClass(), context.getMethod()); - CommandRunner runner = new CommandRunner(context, introspector); + MethodInvoker runner = new MethodInvoker(context, introspector); return runner.get(); } catch (Throwable t) { LOGGER.fine("Throwable caught by interceptor '" + t.getMessage() + "'"); diff --git a/microprofile/fault-tolerance/src/main/java/io/helidon/microprofile/faulttolerance/CommandRunner.java b/microprofile/fault-tolerance/src/main/java/io/helidon/microprofile/faulttolerance/MethodInvoker.java similarity index 78% rename from microprofile/fault-tolerance/src/main/java/io/helidon/microprofile/faulttolerance/CommandRunner.java rename to microprofile/fault-tolerance/src/main/java/io/helidon/microprofile/faulttolerance/MethodInvoker.java index 3ce0c7bdf7f..347dce33cb1 100644 --- a/microprofile/fault-tolerance/src/main/java/io/helidon/microprofile/faulttolerance/CommandRunner.java +++ b/microprofile/fault-tolerance/src/main/java/io/helidon/microprofile/faulttolerance/MethodInvoker.java @@ -23,7 +23,10 @@ import java.util.concurrent.CompletionStage; import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.ExecutionException; +import java.util.concurrent.Executors; import java.util.concurrent.Future; +import java.util.concurrent.ScheduledExecutorService; +import java.util.concurrent.TimeUnit; import java.util.function.Supplier; import java.util.logging.Logger; @@ -79,10 +82,18 @@ import static io.helidon.microprofile.faulttolerance.ThrowableMapper.map; /** - * Runs a FT method. + * Invokes a FT method applying semantics based on method annotations. An instance + * of this class is created for each method invocation. Some state is shared across + * all invocations of the method, including for circuit breakers and bulkheads. */ -public class CommandRunner implements FtSupplier { - private static final Logger LOGGER = Logger.getLogger(CommandRunner.class.getName()); +public class MethodInvoker implements FtSupplier { + private static final Logger LOGGER = Logger.getLogger(MethodInvoker.class.getName()); + + /** + * How long to wait to see if a method that returns normally was actually + * interrupted but caught {@link InterruptedException} before returning. + */ + private static final long TIMEOUT_INTERRUPT_WAIT_MILLIS = 10L; /** * The method being intercepted. @@ -95,7 +106,7 @@ public class CommandRunner implements FtSupplier { private final InvocationContext context; /** - * Helper class to extract information about th emethod. + * Helper class to extract information about the method. */ private final MethodIntrospector introspector; @@ -104,6 +115,11 @@ public class CommandRunner implements FtSupplier { */ private static final ConcurrentHashMap FT_HANDLERS = new ConcurrentHashMap<>(); + /** + * Executor service shared by instances of this class. + */ + private static final ScheduledExecutorService EXECUTOR_SERVICE = Executors.newScheduledThreadPool(16); + /** * Start system nanos when handler is called. */ @@ -134,9 +150,13 @@ public class CommandRunner implements FtSupplier { */ private RequestContextController requestController; + /** + * This future will be completed with 'true' if this invocation is interrupted due + * to a timeout. + */ + private CompletableFuture timeoutInterrupt; + private static class MethodState { - private FtHandlerTyped handler; - private Retry retry; private Bulkhead bulkhead; private CircuitBreaker breaker; private State lastBreakerState; @@ -146,33 +166,50 @@ private static class MethodState { private long startNanos; } + /** + * State associated with a method instead of an invocation. Includes bulkhead + * and breaker handlers that are shared across all invocations of same method. + */ private final MethodState methodState; + /** + * The handler for this invocation. Will share bulkhead an breaker + * sub-handlers across all invocations for the same method. + */ + private FtHandlerTyped handler; + + /** + * Handler for retries. Will be {@code null} if no retry specified in method. + */ + private Retry retry; + /** * Constructor. * * @param context The invocation context. * @param introspector The method introspector. */ - public CommandRunner(InvocationContext context, MethodIntrospector introspector) { + public MethodInvoker(InvocationContext context, MethodIntrospector introspector) { this.context = context; this.introspector = introspector; this.method = context.getMethod(); this.helidonContext = Contexts.context().orElseGet(Context::create); - // Get or initialize new state for this method - this.methodState = FT_HANDLERS.computeIfAbsent(method, method -> { - MethodState methodState = new MethodState(); - initMethodStateHandler(methodState); - methodState.lastBreakerState = State.CLOSED; - if (introspector.hasCircuitBreaker()) { - methodState.breakerTimerOpen = 0L; - methodState.breakerTimerClosed = 0L; - methodState.breakerTimerHalfOpen = 0L; - methodState.startNanos = System.nanoTime(); - } - return methodState; - }); + // Initialize method state and created handler for it + synchronized (method) { + this.methodState = FT_HANDLERS.computeIfAbsent(method, method -> { + MethodState methodState = new MethodState(); + methodState.lastBreakerState = State.CLOSED; + if (introspector.hasCircuitBreaker()) { + methodState.breakerTimerOpen = 0L; + methodState.breakerTimerClosed = 0L; + methodState.breakerTimerHalfOpen = 0L; + methodState.startNanos = System.nanoTime(); + } + return methodState; + }); + handler = createMethodHandler(); + } // Gather information about current request scope if active try { @@ -211,6 +248,19 @@ public CommandRunner(InvocationContext context, MethodIntrospector introspector) } } + @Override + public String toString() { + String s = super.toString(); + StringBuilder sb = new StringBuilder(); + sb.append(s.substring(s.lastIndexOf('.') + 1)) + .append(" ") + .append(method.getDeclaringClass().getSimpleName()) + .append(".") + .append(method.getName()) + .append("()"); + return sb.toString(); + } + /** * Clears ftHandlers map of any cached handlers. */ @@ -234,7 +284,7 @@ public Object get() throws Throwable { updateMetricsBefore(); try { return Contexts.runInContextWithThrow(helidonContext, - () -> methodState.handler.invoke(toCompletionStageSupplier(context::proceed))); + () -> handler.invoke(toCompletionStageSupplier(context::proceed))); } catch (Exception e) { return Single.error(e); } @@ -316,7 +366,7 @@ public Object get() throws Throwable { Throwable cause = null; single = Contexts.runInContextWithThrow(helidonContext, - () -> methodState.handler.invoke(toCompletionStageSupplier(context::proceed))); + () -> handler.invoke(toCompletionStageSupplier(context::proceed))); try { // Need to allow completion with no value (null) for void methods CompletableFuture future = single.toStage(true).toCompletableFuture(); @@ -337,35 +387,38 @@ public Object get() throws Throwable { } /** - * Creates a FT handler for a given method by inspecting annotations. + * Creates a FT handler for an invocation by inspecting annotations. Circuit breakers + * and bulkheads are shared across all invocations --associated with methods. */ - private void initMethodStateHandler(MethodState methodState) { + private FtHandlerTyped createMethodHandler() { FaultTolerance.TypedBuilder builder = FaultTolerance.typedBuilder(); // Create and add circuit breaker if (introspector.hasCircuitBreaker()) { - CircuitBreaker circuitBreaker = CircuitBreaker.builder() - .delay(Duration.of(introspector.getCircuitBreaker().delay(), - introspector.getCircuitBreaker().delayUnit())) - .successThreshold(introspector.getCircuitBreaker().successThreshold()) - .errorRatio((int) (introspector.getCircuitBreaker().failureRatio() * 100)) - .volume(introspector.getCircuitBreaker().requestVolumeThreshold()) - .applyOn(introspector.getCircuitBreaker().failOn()) - .skipOn(introspector.getCircuitBreaker().skipOn()) - .build(); - builder.addBreaker(circuitBreaker); - methodState.breaker = circuitBreaker; + if (methodState.breaker == null) { + methodState.breaker = CircuitBreaker.builder() + .delay(Duration.of(introspector.getCircuitBreaker().delay(), + introspector.getCircuitBreaker().delayUnit())) + .successThreshold(introspector.getCircuitBreaker().successThreshold()) + .errorRatio((int) (introspector.getCircuitBreaker().failureRatio() * 100)) + .volume(introspector.getCircuitBreaker().requestVolumeThreshold()) + .applyOn(introspector.getCircuitBreaker().failOn()) + .skipOn(introspector.getCircuitBreaker().skipOn()) + .build(); + } + builder.addBreaker(methodState.breaker); } // Create and add bulkhead if (introspector.hasBulkhead()) { - Bulkhead bulkhead = Bulkhead.builder() - .limit(introspector.getBulkhead().value()) - .queueLength(introspector.getBulkhead().waitingTaskQueue()) - .async(introspector.isAsynchronous()) - .build(); - builder.addBulkhead(bulkhead); - methodState.bulkhead = bulkhead; + if (methodState.bulkhead == null) { + methodState.bulkhead = Bulkhead.builder() + .limit(introspector.getBulkhead().value()) + .queueLength(introspector.getBulkhead().waitingTaskQueue()) + .async(introspector.isAsynchronous()) + .build(); + } + builder.addBulkhead(methodState.bulkhead); } // Create and add timeout handler -- parent of breaker or bulkhead @@ -373,7 +426,10 @@ private void initMethodStateHandler(MethodState methodState) { Timeout timeout = Timeout.builder() .timeout(Duration.of(introspector.getTimeout().value(), introspector.getTimeout().unit())) .async(false) // no async here - .build(); + .executor(EXECUTOR_SERVICE) + .interruptListener(t -> { + timeoutInterrupt = CompletableFuture.completedFuture(true); // captures 'this' in closure + }).build(); builder.addTimeout(timeout); } @@ -391,15 +447,19 @@ private void initMethodStateHandler(MethodState methodState) { introspector.getRetry().durationUnit())) .applyOn(introspector.getRetry().retryOn()) .skipOn(introspector.getRetry().abortOn()) + .retryListener(c -> { + timeoutInterrupt = null; // ignore on retry + }) .build(); builder.addRetry(retry); - methodState.retry = retry; + this.retry = retry; } // Create and add fallback handler -- parent of retry if (introspector.hasFallback()) { Fallback fallback = Fallback.builder() .fallback(throwable -> { + timeoutInterrupt = null; // ignore with fallback CommandFallback cfb = new CommandFallback(context, introspector, throwable); return toCompletionStageSupplier(cfb::execute).get(); }) @@ -409,13 +469,12 @@ private void initMethodStateHandler(MethodState methodState) { builder.addFallback(fallback); } - // Set handler in method state - methodState.handler = builder.build(); + return builder.build(); } /** * Maps an {@link FtSupplier} to a supplier of {@link CompletionStage}. Avoids - * unnecessary wrapping of stages when method is asynchronous only. + * unnecessary wrapping of stages only when method is asynchronous. * * @param supplier The supplier. * @return The new supplier. @@ -425,7 +484,28 @@ Supplier> toCompletionStageSupplier(FtSupplier return () -> { try { invocationStartNanos = System.nanoTime(); + + // This is the actual method invocation Object result = supplier.get(); + + // Check if the method was interrupted + boolean interrupted = false; + if (timeoutInterrupt != null) { + try { + interrupted = timeoutInterrupt.get(TIMEOUT_INTERRUPT_WAIT_MILLIS, TimeUnit.MILLISECONDS); + } catch (java.util.concurrent.TimeoutException e) { + // falls through + } finally { + timeoutInterrupt = null; + } + } + + // If interrupted, even if it returned normally, it is an exception + if (interrupted) { + return CompletableFuture.failedFuture(new TimeoutException("Method interrupted")); + } + + // Return value without additional wrapping return introspector.isAsynchronous() && result instanceof CompletionStage ? (CompletionStage) result : CompletableFuture.completedFuture(result); @@ -459,7 +539,7 @@ private void updateMetricsAfter(Throwable cause) { // Metrics for retries if (introspector.hasRetry()) { // Have retried the last call? - long newValue = methodState.retry.retryCounter(); + long newValue = retry.retryCounter(); if (updateCounter(method, RETRY_RETRIES_TOTAL, newValue)) { if (cause == null) { getCounter(method, RETRY_CALLS_SUCCEEDED_RETRIED_TOTAL).inc(); diff --git a/microprofile/fault-tolerance/src/test/java/io/helidon/microprofile/faulttolerance/FaultToleranceTest.java b/microprofile/fault-tolerance/src/test/java/io/helidon/microprofile/faulttolerance/FaultToleranceTest.java index b7a6f33e8fc..afc81fdffa3 100644 --- a/microprofile/fault-tolerance/src/test/java/io/helidon/microprofile/faulttolerance/FaultToleranceTest.java +++ b/microprofile/fault-tolerance/src/test/java/io/helidon/microprofile/faulttolerance/FaultToleranceTest.java @@ -19,8 +19,6 @@ import javax.enterprise.inject.literal.NamedLiteral; import javax.enterprise.inject.se.SeContainer; import javax.enterprise.inject.spi.CDI; -import java.util.Arrays; -import java.util.Set; import java.util.concurrent.CompletableFuture; import java.util.concurrent.CompletionStage; import java.util.concurrent.ExecutionException; @@ -30,7 +28,6 @@ import java.util.concurrent.TimeUnit; import java.util.concurrent.TimeoutException; import java.util.function.Supplier; -import java.util.stream.Collectors; import java.util.stream.Stream; import io.helidon.microprofile.cdi.HelidonContainer; @@ -78,7 +75,7 @@ public static void shutDownCdiContainer() { */ @BeforeEach public void resetHandlers() { - CommandRunner.clearFtHandlersMap(); + MethodInvoker.clearFtHandlersMap(); } protected static T newBean(Class beanClass) { diff --git a/microprofile/fault-tolerance/src/test/java/io/helidon/microprofile/faulttolerance/TimeoutBean.java b/microprofile/fault-tolerance/src/test/java/io/helidon/microprofile/faulttolerance/TimeoutBean.java index 7eb96ff6ecc..59be1568b64 100644 --- a/microprofile/fault-tolerance/src/test/java/io/helidon/microprofile/faulttolerance/TimeoutBean.java +++ b/microprofile/fault-tolerance/src/test/java/io/helidon/microprofile/faulttolerance/TimeoutBean.java @@ -59,6 +59,17 @@ public String noTimeout() throws InterruptedException { return "success"; } + @Timeout(value=1000, unit=ChronoUnit.MILLIS) + public String forceTimeoutWithCatch() { + try { + FaultToleranceTest.printStatus("TimeoutBean::forceTimeoutWithCatch()", "failure"); + Thread.sleep(1500); + } catch (InterruptedException e) { + // falls through + } + return null; // tests special null case + } + // See class annotation @Retry(maxRetries = 2) @Timeout(value=1000, unit=ChronoUnit.MILLIS) public String timeoutWithRetries() throws InterruptedException { diff --git a/microprofile/fault-tolerance/src/test/java/io/helidon/microprofile/faulttolerance/TimeoutTest.java b/microprofile/fault-tolerance/src/test/java/io/helidon/microprofile/faulttolerance/TimeoutTest.java index f82b3fec9aa..5e5e7459545 100644 --- a/microprofile/fault-tolerance/src/test/java/io/helidon/microprofile/faulttolerance/TimeoutTest.java +++ b/microprofile/fault-tolerance/src/test/java/io/helidon/microprofile/faulttolerance/TimeoutTest.java @@ -52,6 +52,12 @@ public void testNoTimeout() throws Exception { assertThat(bean.noTimeout(), is("success")); } + @Test + public void testForceTimeoutWithCatch() { + TimeoutBean bean = newBean(TimeoutBean.class); + assertThrows(TimeoutException.class, bean::forceTimeoutWithCatch); + } + @Test public void testTimeoutWithRetries() throws Exception { TimeoutBean bean = newBean(TimeoutBean.class); From e73962a99cfc6154b04e67d86ccdc3adce7ba13f Mon Sep 17 00:00:00 2001 From: Santiago Pericasgeertsen Date: Tue, 11 Aug 2020 17:00:53 -0400 Subject: [PATCH 21/65] Handle TestNG before and after methods. Signed-off-by: Santiago Pericasgeertsen --- microprofile/tests/arquillian/pom.xml | 4 ++ .../arquillian/HelidonMethodExecutor.java | 47 +++++++++++++++---- 2 files changed, 41 insertions(+), 10 deletions(-) diff --git a/microprofile/tests/arquillian/pom.xml b/microprofile/tests/arquillian/pom.xml index 248b1080275..965956279cb 100644 --- a/microprofile/tests/arquillian/pom.xml +++ b/microprofile/tests/arquillian/pom.xml @@ -65,5 +65,9 @@ junit junit + + org.testng + testng + diff --git a/microprofile/tests/arquillian/src/main/java/io/helidon/microprofile/arquillian/HelidonMethodExecutor.java b/microprofile/tests/arquillian/src/main/java/io/helidon/microprofile/arquillian/HelidonMethodExecutor.java index 987a66cf727..0bf8c19610e 100644 --- a/microprofile/tests/arquillian/src/main/java/io/helidon/microprofile/arquillian/HelidonMethodExecutor.java +++ b/microprofile/tests/arquillian/src/main/java/io/helidon/microprofile/arquillian/HelidonMethodExecutor.java @@ -30,6 +30,8 @@ import org.jboss.arquillian.test.spi.TestResult; import org.junit.After; import org.junit.Before; +import org.testng.annotations.AfterMethod; +import org.testng.annotations.BeforeMethod; /** * Class HelidonMethodExecutor. @@ -40,9 +42,14 @@ public class HelidonMethodExecutor implements ContainerMethodExecutor { private HelidonCDIInjectionEnricher enricher = new HelidonCDIInjectionEnricher(); /** - * Invoke method after enrichment. Inexplicably, the {@code @Before} - * and {@code @After} methods are not called when running this - * executor. Calling them manually for now. + * Invoke method after enrichment. + * + * - JUnit: Inexplicably, the {@code @Before} and {@code @After} methods are + * not called when running this executor, so we call them manually. + * + * - TestNG: Methods decorated with {@code @BeforeMethod} and {@code AfterMethod} + * are called too early, before enrichment takes places. Here we call them + * again to make sure instances are initialized properly. * * @param testMethodExecutor Method executor. * @return Test result. @@ -51,13 +58,13 @@ public TestResult invoke(TestMethodExecutor testMethodExecutor) { RequestContextController controller = enricher.getRequestContextController(); try { controller.activate(); - Object object = testMethodExecutor.getInstance(); + Object instance = testMethodExecutor.getInstance(); Method method = testMethodExecutor.getMethod(); - LOGGER.info("Invoking '" + method + "' on " + object); - enricher.enrich(object); - invokeAnnotated(object, Before.class); + LOGGER.info("Invoking '" + method + "' on " + instance); + enricher.enrich(instance); + invokeBefore(instance); testMethodExecutor.invoke(enricher.resolve(method)); - invokeAnnotated(object, After.class); + invokeAfter(instance); } catch (Throwable t) { return TestResult.failed(t); } finally { @@ -66,15 +73,35 @@ public TestResult invoke(TestMethodExecutor testMethodExecutor) { return TestResult.passed(); } + /** + * Invoke before methods. + * + * @param instance Test instance. + */ + private static void invokeBefore(Object instance) { + invokeAnnotated(instance, Before.class); // Junit + invokeAnnotated(instance, BeforeMethod.class); // TestNG + } + + /** + * Invoke after methods. + * + * @param instance Test instance. + */ + private static void invokeAfter(Object instance) { + invokeAnnotated(instance, After.class); // JUnit + invokeAnnotated(instance, AfterMethod.class); // TestNG + } + /** * Invoke an annotated method. * * @param object Test instance. * @param annotClass Annotation to look for. */ - private void invokeAnnotated(Object object, Class annotClass) { + private static void invokeAnnotated(Object object, Class annotClass) { Class clazz = object.getClass(); - Stream.of(clazz.getMethods()) + Stream.of(clazz.getDeclaredMethods()) .filter(m -> m.getAnnotation(annotClass) != null) .forEach(m -> { try { From d1af56c57ee5b4b4f5f112169792a7dfade21f33 Mon Sep 17 00:00:00 2001 From: Santiago Pericasgeertsen Date: Tue, 11 Aug 2020 17:02:30 -0400 Subject: [PATCH 22/65] Added toString() method to help debugging. Signed-off-by: Santiago Pericasgeertsen --- .../io/helidon/faulttolerance/FaultTolerance.java | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/fault-tolerance/src/main/java/io/helidon/faulttolerance/FaultTolerance.java b/fault-tolerance/src/main/java/io/helidon/faulttolerance/FaultTolerance.java index 67ef0a8d342..ad6b28ed255 100644 --- a/fault-tolerance/src/main/java/io/helidon/faulttolerance/FaultTolerance.java +++ b/fault-tolerance/src/main/java/io/helidon/faulttolerance/FaultTolerance.java @@ -277,6 +277,16 @@ public Single invoke(Supplier> supplier) { return Single.create(next.get(), true); } + + @Override + public String toString() { + StringBuilder sb = new StringBuilder(); + for (int i = validFts.size() - 1; i >= 0; i--) { + sb.append(validFts.get(i).toString()); + sb.append("\n"); + } + return sb.toString(); + } } private class TypedWrapper implements FtHandlerTyped { @@ -295,6 +305,11 @@ public Single invoke(Supplier> supplier) { public Multi invokeMulti(Supplier> supplier) { return handler.invokeMulti(supplier); } + + @Override + public String toString() { + return handler.getClass().getSimpleName(); + } } } From ef3f58521e17fc35eabbbf922f68ae3c69679f6f Mon Sep 17 00:00:00 2001 From: Santiago Pericasgeertsen Date: Tue, 11 Aug 2020 17:04:09 -0400 Subject: [PATCH 23/65] Improved implementations of MethodInvoker and TimeoutImpl. Signed-off-by: Santiago Pericasgeertsen --- .../helidon/faulttolerance/TimeoutImpl.java | 17 +- .../faulttolerance/FaultToleranceMetrics.java | 6 + .../faulttolerance/MethodInvoker.java | 188 ++++++------------ .../faulttolerance/TimeoutTest.java | 4 +- 4 files changed, 90 insertions(+), 125 deletions(-) diff --git a/fault-tolerance/src/main/java/io/helidon/faulttolerance/TimeoutImpl.java b/fault-tolerance/src/main/java/io/helidon/faulttolerance/TimeoutImpl.java index fe49022c803..164b863248a 100644 --- a/fault-tolerance/src/main/java/io/helidon/faulttolerance/TimeoutImpl.java +++ b/fault-tolerance/src/main/java/io/helidon/faulttolerance/TimeoutImpl.java @@ -22,6 +22,7 @@ import java.util.concurrent.Flow; import java.util.concurrent.ScheduledExecutorService; import java.util.concurrent.TimeUnit; +import java.util.concurrent.TimeoutException; import java.util.concurrent.atomic.AtomicBoolean; import java.util.function.Consumer; import java.util.function.Supplier; @@ -65,6 +66,7 @@ public Single invoke(Supplier> supplier) { AtomicBoolean callReturned = new AtomicBoolean(false); // Startup monitor thread that can interrupt current thread after timeout + CompletableFuture future = new CompletableFuture<>(); Timeout.builder() .executor(executor.get()) // propagate executor .async(true) @@ -76,6 +78,8 @@ public Single invoke(Supplier> supplier) { }) .exceptionally(it -> { if (callReturned.compareAndSet(false, true)) { + System.out.println("### completing future on timeout"); + future.completeExceptionally(new TimeoutException("Method interrupted by timeout")); currentThread.interrupt(); if (interruptListener != null) { interruptListener.accept(currentThread); @@ -94,7 +98,18 @@ public Single invoke(Supplier> supplier) { // Run invocation in current thread Single single = Single.create(supplier.get(), true); callReturned.set(true); - return single; + single.whenComplete((o, t) -> { + if (t != null) { + future.completeExceptionally(t); + } else { + future.complete(o); + } + }); + + // Clear interrupted flag here -- required for uninterruptible busy loops + Thread.interrupted(); + + return Single.create(future, true); } } } diff --git a/microprofile/fault-tolerance/src/main/java/io/helidon/microprofile/faulttolerance/FaultToleranceMetrics.java b/microprofile/fault-tolerance/src/main/java/io/helidon/microprofile/faulttolerance/FaultToleranceMetrics.java index d31d8b1218f..cb308d6121e 100644 --- a/microprofile/fault-tolerance/src/main/java/io/helidon/microprofile/faulttolerance/FaultToleranceMetrics.java +++ b/microprofile/fault-tolerance/src/main/java/io/helidon/microprofile/faulttolerance/FaultToleranceMetrics.java @@ -289,6 +289,12 @@ static void registerBulkheadMetrics(Method method) { BULKHEAD_EXECUTION_DURATION), "Histogram of method execution times. This does not include any " + "time spent waiting in the bulkhead queue."); + registerHistogram( + String.format(METRIC_NAME_TEMPLATE, + method.getDeclaringClass().getName(), + method.getName(), + BULKHEAD_WAITING_DURATION), + "Histogram of the time executions spend waiting in the queue."); } // -- Utility methods ---------------------------------------------------- diff --git a/microprofile/fault-tolerance/src/main/java/io/helidon/microprofile/faulttolerance/MethodInvoker.java b/microprofile/fault-tolerance/src/main/java/io/helidon/microprofile/faulttolerance/MethodInvoker.java index 347dce33cb1..2c68e83fb8f 100644 --- a/microprofile/fault-tolerance/src/main/java/io/helidon/microprofile/faulttolerance/MethodInvoker.java +++ b/microprofile/fault-tolerance/src/main/java/io/helidon/microprofile/faulttolerance/MethodInvoker.java @@ -26,7 +26,6 @@ import java.util.concurrent.Executors; import java.util.concurrent.Future; import java.util.concurrent.ScheduledExecutorService; -import java.util.concurrent.TimeUnit; import java.util.function.Supplier; import java.util.logging.Logger; @@ -89,12 +88,6 @@ public class MethodInvoker implements FtSupplier { private static final Logger LOGGER = Logger.getLogger(MethodInvoker.class.getName()); - /** - * How long to wait to see if a method that returns normally was actually - * interrupted but caught {@link InterruptedException} before returning. - */ - private static final long TIMEOUT_INTERRUPT_WAIT_MILLIS = 10L; - /** * The method being intercepted. */ @@ -150,12 +143,6 @@ public class MethodInvoker implements FtSupplier { */ private RequestContextController requestController; - /** - * This future will be completed with 'true' if this invocation is interrupted due - * to a timeout. - */ - private CompletableFuture timeoutInterrupt; - private static class MethodState { private Bulkhead bulkhead; private CircuitBreaker breaker; @@ -275,101 +262,60 @@ static void clearFtHandlersMap() { */ @Override public Object get() throws Throwable { - Single single; + // Wrap method call with Helidon context and init metrics + Supplier> supplier = () -> { + try { + return Contexts.runInContextWithThrow(helidonContext, + () -> handler.invoke(toCompletionStageSupplier(context::proceed))); + } catch (Exception e) { + return Single.error(e); + } + }; - if (introspector.isAsynchronous()) { - if (introspector.isReturnType(CompletionStage.class) || introspector.isReturnType(Future.class)) { - // Wrap method call with Helidon context - Supplier callMethod = () -> { - updateMetricsBefore(); - try { - return Contexts.runInContextWithThrow(helidonContext, - () -> handler.invoke(toCompletionStageSupplier(context::proceed))); - } catch (Exception e) { - return Single.error(e); - } - }; - - /* - * Call method preserving request scope if active. This is required for - * @Inject and @Context to work properly. Note that it's possible for only - * CDI's request scope to be active at this time (e.g. in TCKs). - */ - if (requestScope != null) { // Jersey and CDI - single = Async.create().invoke(() -> { - try { - return requestScope.runInScope(requestContext, (Callable) (() -> { - try { - requestController.activate(); - return callMethod.get(); - } finally { - requestController.deactivate(); - } - })); - } catch (Exception e) { - return Single.error(e); - } - }); - } else if (requestController != null) { // CDI only - single = Async.create().invoke(() -> { + /* + * Call method preserving request scope if active. This is required for + * @Inject and @Context to work properly. Note that it's possible for only + * CDI's request scope to be active at this time (e.g. in TCKs). + */ + Supplier> wrappedSupplier; + if (requestScope != null) { // Jersey and CDI + wrappedSupplier = () -> { + try { + return requestScope.runInScope(requestContext, (() -> { try { requestController.activate(); - return callMethod.get(); - } catch (Exception e) { - return Single.error(e); + return supplier.get(); } finally { requestController.deactivate(); } - }); - } else { - single = Async.create().invoke(callMethod); + })); + } catch (Exception e) { + return Single.error(e); } - - // Unwrap nested futures and map exceptions on complete - CompletableFuture future = new CompletableFuture<>(); - single.whenComplete((o, t) -> { - Throwable cause = null; - - // Update future to return - if (t == null) { - // If future whose value is a future, then unwrap them - Future delegate = null; - if (o instanceof CompletionStage) { - delegate = ((CompletionStage) o).toCompletableFuture(); - } else if (o instanceof Future) { - delegate = (Future) o; - } - if (delegate != null) { - try { - future.complete(delegate.get()); - } catch (Exception e) { - cause = map(e); - future.completeExceptionally(cause); - } - } else { - future.complete(o); - } - } else { - cause = map(t); - future.completeExceptionally(cause); - } - - updateMetricsAfter(cause); - }); - return future; - } - - // Oops, something went wrong during validation - throw new InternalError("Validation failed, return type must be Future or CompletionStage"); + }; + } else if (requestController != null) { // CDI only + wrappedSupplier = () -> { + try { + requestController.activate(); + return supplier.get(); + } catch (Exception e) { + return Single.error(e); + } finally { + requestController.deactivate(); + } + }; } else { + wrappedSupplier = supplier; + } + + // Final supplier that handles metrics and maps exceptions + FtSupplier finalSupplier = () -> { Object result = null; Throwable cause = null; - - single = Contexts.runInContextWithThrow(helidonContext, - () -> handler.invoke(toCompletionStageSupplier(context::proceed))); try { // Need to allow completion with no value (null) for void methods - CompletableFuture future = single.toStage(true).toCompletableFuture(); + CompletableFuture future = wrappedSupplier.get() + .toStage(true).toCompletableFuture(); updateMetricsBefore(); result = future.get(); } catch (ExecutionException e) { @@ -377,12 +323,31 @@ public Object get() throws Throwable { } catch (Throwable t) { cause = map(t); } - updateMetricsAfter(cause); if (cause != null) { throw cause; } return result; + }; + + // Special cases for sync and async invocations + if (introspector.isAsynchronous()) { + if (introspector.isReturnType(CompletionStage.class) || introspector.isReturnType(Future.class)) { + CompletableFuture finalResult = new CompletableFuture<>(); + Async.create().invoke(() -> { + try { + return finalResult.complete(finalSupplier.get()); + } catch (Throwable t) { + return finalResult.completeExceptionally(t); + } + }); + return finalResult; + } + + // Oops, something went wrong during validation + throw new InternalError("Validation failed, return type must be Future or CompletionStage"); + } else { + return finalSupplier.get(); } } @@ -427,9 +392,7 @@ private FtHandlerTyped createMethodHandler() { .timeout(Duration.of(introspector.getTimeout().value(), introspector.getTimeout().unit())) .async(false) // no async here .executor(EXECUTOR_SERVICE) - .interruptListener(t -> { - timeoutInterrupt = CompletableFuture.completedFuture(true); // captures 'this' in closure - }).build(); + .build(); builder.addTimeout(timeout); } @@ -447,9 +410,6 @@ private FtHandlerTyped createMethodHandler() { introspector.getRetry().durationUnit())) .applyOn(introspector.getRetry().retryOn()) .skipOn(introspector.getRetry().abortOn()) - .retryListener(c -> { - timeoutInterrupt = null; // ignore on retry - }) .build(); builder.addRetry(retry); this.retry = retry; @@ -459,7 +419,6 @@ private FtHandlerTyped createMethodHandler() { if (introspector.hasFallback()) { Fallback fallback = Fallback.builder() .fallback(throwable -> { - timeoutInterrupt = null; // ignore with fallback CommandFallback cfb = new CommandFallback(context, introspector, throwable); return toCompletionStageSupplier(cfb::execute).get(); }) @@ -469,7 +428,9 @@ private FtHandlerTyped createMethodHandler() { builder.addFallback(fallback); } - return builder.build(); + FtHandlerTyped result = builder.build(); + System.out.println("\n### tree \n" + result); + return result; } /** @@ -488,23 +449,6 @@ Supplier> toCompletionStageSupplier(FtSupplier // This is the actual method invocation Object result = supplier.get(); - // Check if the method was interrupted - boolean interrupted = false; - if (timeoutInterrupt != null) { - try { - interrupted = timeoutInterrupt.get(TIMEOUT_INTERRUPT_WAIT_MILLIS, TimeUnit.MILLISECONDS); - } catch (java.util.concurrent.TimeoutException e) { - // falls through - } finally { - timeoutInterrupt = null; - } - } - - // If interrupted, even if it returned normally, it is an exception - if (interrupted) { - return CompletableFuture.failedFuture(new TimeoutException("Method interrupted")); - } - // Return value without additional wrapping return introspector.isAsynchronous() && result instanceof CompletionStage ? (CompletionStage) result diff --git a/microprofile/fault-tolerance/src/test/java/io/helidon/microprofile/faulttolerance/TimeoutTest.java b/microprofile/fault-tolerance/src/test/java/io/helidon/microprofile/faulttolerance/TimeoutTest.java index 5e5e7459545..2a9b85a100c 100644 --- a/microprofile/fault-tolerance/src/test/java/io/helidon/microprofile/faulttolerance/TimeoutTest.java +++ b/microprofile/fault-tolerance/src/test/java/io/helidon/microprofile/faulttolerance/TimeoutTest.java @@ -24,9 +24,9 @@ import static org.hamcrest.MatcherAssert.assertThat; import static org.hamcrest.Matchers.is; -import static org.hamcrest.Matchers.greaterThan; import static org.hamcrest.Matchers.lessThan; import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.hamcrest.Matchers.greaterThanOrEqualTo; /** * Class TimeoutTest. @@ -94,7 +94,7 @@ public void testForceTimeoutLoop() { try { bean.forceTimeoutLoop(); // cannot interrupt } catch (TimeoutException e) { - assertThat(System.currentTimeMillis() - start, is(greaterThan(2000L))); + assertThat(System.currentTimeMillis() - start, is(greaterThanOrEqualTo(2000L))); } } } From e00cbce7a75f629afe945e3365fd2d6618a91ebe Mon Sep 17 00:00:00 2001 From: Santiago Pericasgeertsen Date: Thu, 13 Aug 2020 08:44:05 -0400 Subject: [PATCH 24/65] Async methods cannot throw exceptions. Fixed return type. Signed-off-by: Santiago Pericasgeertsen --- .../helidon/microprofile/faulttolerance/AsynchronousBean.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/microprofile/fault-tolerance/src/test/java/io/helidon/microprofile/faulttolerance/AsynchronousBean.java b/microprofile/fault-tolerance/src/test/java/io/helidon/microprofile/faulttolerance/AsynchronousBean.java index 6b444c2d09a..5b64cfc45a9 100644 --- a/microprofile/fault-tolerance/src/test/java/io/helidon/microprofile/faulttolerance/AsynchronousBean.java +++ b/microprofile/fault-tolerance/src/test/java/io/helidon/microprofile/faulttolerance/AsynchronousBean.java @@ -60,7 +60,7 @@ public Future async() { public Future asyncWithFallback() { called = true; FaultToleranceTest.printStatus("AsynchronousBean::asyncWithFallback", "failure"); - throw new RuntimeException("Oops"); + return CompletableFuture.failedFuture(new RuntimeException("Oops")); } public CompletableFuture onFailure() { @@ -101,7 +101,7 @@ public CompletionStage asyncCompletionStage() { public CompletionStage asyncCompletionStageWithFallback() { called = true; FaultToleranceTest.printStatus("AsynchronousBean::asyncCompletionStageWithFallback", "failure"); - throw new RuntimeException("Oops"); + return CompletableFuture.failedFuture(new RuntimeException("Oops")); } /** From b5abd22d37bede915e6bf1848f084cfc2fd6b2ba Mon Sep 17 00:00:00 2001 From: Santiago Pericasgeertsen Date: Thu, 13 Aug 2020 08:50:12 -0400 Subject: [PATCH 25/65] Use FT 2.1.1. Signed-off-by: Santiago Pericasgeertsen --- dependencies/pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dependencies/pom.xml b/dependencies/pom.xml index 0ebb428861e..ab59dca9dcf 100644 --- a/dependencies/pom.xml +++ b/dependencies/pom.xml @@ -84,7 +84,7 @@ 1.1.1 2.2 1.1.2 - 2.1 + 2.1.1 1.3.1 1.3.3 1.0 From fbab4342aba7b0ae1e5fa79c18d68fac1bd4c641 Mon Sep 17 00:00:00 2001 From: Santiago Pericasgeertsen Date: Thu, 13 Aug 2020 08:50:50 -0400 Subject: [PATCH 26/65] New implementation for async that also simplifies the sync case. Signed-off-by: Santiago Pericasgeertsen --- .../faulttolerance/MethodInvoker.java | 175 ++++++++++++------ 1 file changed, 116 insertions(+), 59 deletions(-) diff --git a/microprofile/fault-tolerance/src/main/java/io/helidon/microprofile/faulttolerance/MethodInvoker.java b/microprofile/fault-tolerance/src/main/java/io/helidon/microprofile/faulttolerance/MethodInvoker.java index 2c68e83fb8f..7aa8c79a8c4 100644 --- a/microprofile/fault-tolerance/src/main/java/io/helidon/microprofile/faulttolerance/MethodInvoker.java +++ b/microprofile/fault-tolerance/src/main/java/io/helidon/microprofile/faulttolerance/MethodInvoker.java @@ -18,7 +18,6 @@ import java.lang.reflect.Method; import java.time.Duration; import java.util.Objects; -import java.util.concurrent.Callable; import java.util.concurrent.CompletableFuture; import java.util.concurrent.CompletionStage; import java.util.concurrent.ConcurrentHashMap; @@ -26,6 +25,7 @@ import java.util.concurrent.Executors; import java.util.concurrent.Future; import java.util.concurrent.ScheduledExecutorService; +import java.util.concurrent.TimeUnit; import java.util.function.Supplier; import java.util.logging.Logger; @@ -88,6 +88,8 @@ public class MethodInvoker implements FtSupplier { private static final Logger LOGGER = Logger.getLogger(MethodInvoker.class.getName()); + private static final long AWAIT_FUTURE_MILLIS = 5L; + /** * The method being intercepted. */ @@ -256,14 +258,14 @@ static void clearFtHandlersMap() { } /** - * Invokes the FT method. + * Invokes a method with one or more FT annotations. * - * @return Value returned by method. + * @return value returned by method. */ @Override public Object get() throws Throwable { - // Wrap method call with Helidon context and init metrics - Supplier> supplier = () -> { + // Wrap method call with Helidon context + Supplier> supplier = () -> { try { return Contexts.runInContextWithThrow(helidonContext, () -> handler.invoke(toCompletionStageSupplier(context::proceed))); @@ -272,12 +274,72 @@ public Object get() throws Throwable { } }; - /* - * Call method preserving request scope if active. This is required for - * @Inject and @Context to work properly. Note that it's possible for only - * CDI's request scope to be active at this time (e.g. in TCKs). - */ - Supplier> wrappedSupplier; + updateMetricsBefore(); + + if (introspector.isAsynchronous()) { + // Wrap supplier with request context setup + Supplier> wrappedSupplier = requestContextSupplier(supplier); + + // Invoke wrapped supplier on a different thread + Single> asyncSingle = Async.create().invoke(wrappedSupplier); + CompletableFuture> asyncFuture = asyncSingle.toStage(true).toCompletableFuture(); + + // Register handler to process result and map exceptions + CompletableFuture resultFuture = new CompletableFuture<>(); + asyncFuture.whenComplete((single, throwable) -> { + Throwable cause = null; + if (throwable != null) { + if (throwable instanceof ExecutionException) { + cause = map(throwable.getCause()); + } else { + cause = map(throwable); + } + updateMetricsAfter(cause); + resultFuture.completeExceptionally(cause); + } else { + try { + Object result = single.get(); + resultFuture.complete(result); + } catch (ExecutionException e) { + cause = map(e.getCause()); + } catch (Throwable t) { + cause = map(t); + } + updateMetricsAfter(cause); + if (cause != null) { + resultFuture.completeExceptionally(cause); + } + } + }); + return resultFuture; + } else { + Object result = null; + Throwable cause = null; + try { + // Invoke supplier on same thread + CompletableFuture future = supplier.get().toStage(true).toCompletableFuture(); + result = future.get(); + } catch (ExecutionException e) { + cause = map(e.getCause()); + } catch (Throwable t) { + cause = map(t); + } + updateMetricsAfter(cause); + if (cause != null) { + throw cause; + } + return result; + } + } + + /** + * Wraps a supplier with additional code to preserve request context (if active) + * when running in a different thread. This is required for {@code @Inject} and + * {@code @Context} to work properly. Note that it is possible for only CDI's + * request scope to be active at this time (e.g. in TCKs). + */ + private Supplier> requestContextSupplier(Supplier> supplier) { + Supplier> wrappedSupplier; if (requestScope != null) { // Jersey and CDI wrappedSupplier = () -> { try { @@ -307,48 +369,7 @@ public Object get() throws Throwable { } else { wrappedSupplier = supplier; } - - // Final supplier that handles metrics and maps exceptions - FtSupplier finalSupplier = () -> { - Object result = null; - Throwable cause = null; - try { - // Need to allow completion with no value (null) for void methods - CompletableFuture future = wrappedSupplier.get() - .toStage(true).toCompletableFuture(); - updateMetricsBefore(); - result = future.get(); - } catch (ExecutionException e) { - cause = map(e.getCause()); - } catch (Throwable t) { - cause = map(t); - } - updateMetricsAfter(cause); - if (cause != null) { - throw cause; - } - return result; - }; - - // Special cases for sync and async invocations - if (introspector.isAsynchronous()) { - if (introspector.isReturnType(CompletionStage.class) || introspector.isReturnType(Future.class)) { - CompletableFuture finalResult = new CompletableFuture<>(); - Async.create().invoke(() -> { - try { - return finalResult.complete(finalSupplier.get()); - } catch (Throwable t) { - return finalResult.completeExceptionally(t); - } - }); - return finalResult; - } - - // Oops, something went wrong during validation - throw new InternalError("Validation failed, return type must be Future or CompletionStage"); - } else { - return finalSupplier.get(); - } + return wrappedSupplier; } /** @@ -434,8 +455,7 @@ private FtHandlerTyped createMethodHandler() { } /** - * Maps an {@link FtSupplier} to a supplier of {@link CompletionStage}. Avoids - * unnecessary wrapping of stages only when method is asynchronous. + * Maps an {@link FtSupplier} to a supplier of {@link CompletionStage}. * * @param supplier The supplier. * @return The new supplier. @@ -449,10 +469,20 @@ Supplier> toCompletionStageSupplier(FtSupplier // This is the actual method invocation Object result = supplier.get(); - // Return value without additional wrapping - return introspector.isAsynchronous() && result instanceof CompletionStage - ? (CompletionStage) result - : CompletableFuture.completedFuture(result); + if (introspector.isAsynchronous()) { + if (result instanceof CompletionStage) { // true also for CompletableFuture + return (CompletionStage) result; + } else if (result instanceof Future) { + Future future = (Future) result; + CompletableFuture completableFuture = new CompletableFuture<>(); + awaitFuture(future, completableFuture); + return completableFuture; + } else { + throw new IllegalStateException("Return type not allowed in async method " + method); + } + } else { + return CompletableFuture.completedFuture(result); + } } catch (Throwable e) { return CompletableFuture.failedFuture(e); } @@ -592,4 +622,31 @@ private static boolean updateCounter(Method method, String name, long newValue) } return false; } + + /** + * Waits for a Future to produce an outcome and updates a CompletableFuture based on + * it. Effectively converts a Future into a CompletableFuture. + * + * @param future the future. + * @param completableFuture the completable future. + */ + private static void awaitFuture(Future future, CompletableFuture completableFuture) { + System.out.println("Awaiting on future " + future); + if (future.isDone()) { + try { + completableFuture.complete(future.get()); + } catch (InterruptedException e) { + completableFuture.completeExceptionally(e); + } catch (ExecutionException e) { + completableFuture.completeExceptionally(e.getCause()); + } + return; + } + if (future.isCancelled()) { + completableFuture.cancel(true); + } else { + EXECUTOR_SERVICE.schedule(() -> awaitFuture(future, completableFuture), + AWAIT_FUTURE_MILLIS, TimeUnit.MILLISECONDS); + } + } } From 2d55ff29a4fe0a42eef3047929753ead9fffa16e Mon Sep 17 00:00:00 2001 From: Santiago Pericasgeertsen Date: Thu, 13 Aug 2020 08:59:23 -0400 Subject: [PATCH 27/65] Renamed async to currentThread. Signed-off-by: Santiago Pericasgeertsen --- .../io/helidon/faulttolerance/Timeout.java | 15 ++++++++------- .../helidon/faulttolerance/TimeoutImpl.java | 19 +++++++++---------- .../faulttolerance/MethodInvoker.java | 3 +-- 3 files changed, 18 insertions(+), 19 deletions(-) diff --git a/fault-tolerance/src/main/java/io/helidon/faulttolerance/Timeout.java b/fault-tolerance/src/main/java/io/helidon/faulttolerance/Timeout.java index 906cf596238..391b4cbc18c 100644 --- a/fault-tolerance/src/main/java/io/helidon/faulttolerance/Timeout.java +++ b/fault-tolerance/src/main/java/io/helidon/faulttolerance/Timeout.java @@ -53,7 +53,7 @@ static Timeout create(Duration timeout) { class Builder implements io.helidon.common.Builder { private Duration timeout = Duration.ofSeconds(10); private LazyValue executor = FaultTolerance.scheduledExecutor(); - private boolean async = true; + private boolean currentThread = false; private Consumer listener; private Builder() { @@ -76,13 +76,14 @@ public Builder timeout(Duration timeout) { } /** - * Async flag. If async, code will execute in another thread. Default is {@code true}. + * Flag to indicate that code should be executed in current thread instead + * of in an executor's thread for ease of monitoring. * - * @param async async setting for this timeout; + * @param currentThread setting for this timeout * @return updated builder instance */ - public Builder async(boolean async) { - this.async = async; + public Builder currentThread(boolean currentThread) { + this.currentThread = currentThread; return this; } @@ -116,8 +117,8 @@ LazyValue executor() { return executor; } - boolean async() { - return async; + boolean currentThread() { + return currentThread; } Consumer interruptListener() { diff --git a/fault-tolerance/src/main/java/io/helidon/faulttolerance/TimeoutImpl.java b/fault-tolerance/src/main/java/io/helidon/faulttolerance/TimeoutImpl.java index 164b863248a..7bcab115ae1 100644 --- a/fault-tolerance/src/main/java/io/helidon/faulttolerance/TimeoutImpl.java +++ b/fault-tolerance/src/main/java/io/helidon/faulttolerance/TimeoutImpl.java @@ -36,20 +36,20 @@ class TimeoutImpl implements Timeout { private final long timeoutMillis; private final LazyValue executor; - private final boolean async; + private final boolean currentThread; private final Consumer interruptListener; TimeoutImpl(Timeout.Builder builder) { this.timeoutMillis = builder.timeout().toMillis(); this.executor = builder.executor(); - this.async = builder.async(); + this.currentThread = builder.currentThread(); this.interruptListener = builder.interruptListener(); } @Override public Multi invokeMulti(Supplier> supplier) { - if (!async) { - throw new UnsupportedOperationException("Timeout with Publisher must be async"); + if (currentThread) { + throw new UnsupportedOperationException("Unsupported currentThread flag with Multi"); } return Multi.create(supplier.get()) .timeout(timeoutMillis, TimeUnit.MILLISECONDS, executor.get()); @@ -57,11 +57,11 @@ public Multi invokeMulti(Supplier> supplier) @Override public Single invoke(Supplier> supplier) { - if (async) { + if (!currentThread) { return Single.create(supplier.get(), true) .timeout(timeoutMillis, TimeUnit.MILLISECONDS, executor.get()); } else { - Thread currentThread = Thread.currentThread(); + Thread thisThread = Thread.currentThread(); CompletableFuture monitorStarted = new CompletableFuture<>(); AtomicBoolean callReturned = new AtomicBoolean(false); @@ -69,7 +69,7 @@ public Single invoke(Supplier> supplier) { CompletableFuture future = new CompletableFuture<>(); Timeout.builder() .executor(executor.get()) // propagate executor - .async(true) + .currentThread(false) .timeout(Duration.ofMillis(timeoutMillis)) .build() .invoke(() -> { @@ -78,11 +78,10 @@ public Single invoke(Supplier> supplier) { }) .exceptionally(it -> { if (callReturned.compareAndSet(false, true)) { - System.out.println("### completing future on timeout"); future.completeExceptionally(new TimeoutException("Method interrupted by timeout")); - currentThread.interrupt(); + thisThread.interrupt(); if (interruptListener != null) { - interruptListener.accept(currentThread); + interruptListener.accept(thisThread); } } return null; diff --git a/microprofile/fault-tolerance/src/main/java/io/helidon/microprofile/faulttolerance/MethodInvoker.java b/microprofile/fault-tolerance/src/main/java/io/helidon/microprofile/faulttolerance/MethodInvoker.java index 7aa8c79a8c4..4a0624d1a2d 100644 --- a/microprofile/fault-tolerance/src/main/java/io/helidon/microprofile/faulttolerance/MethodInvoker.java +++ b/microprofile/fault-tolerance/src/main/java/io/helidon/microprofile/faulttolerance/MethodInvoker.java @@ -411,7 +411,7 @@ private FtHandlerTyped createMethodHandler() { if (introspector.hasTimeout()) { Timeout timeout = Timeout.builder() .timeout(Duration.of(introspector.getTimeout().value(), introspector.getTimeout().unit())) - .async(false) // no async here + .currentThread(true) .executor(EXECUTOR_SERVICE) .build(); builder.addTimeout(timeout); @@ -631,7 +631,6 @@ private static boolean updateCounter(Method method, String name, long newValue) * @param completableFuture the completable future. */ private static void awaitFuture(Future future, CompletableFuture completableFuture) { - System.out.println("Awaiting on future " + future); if (future.isDone()) { try { completableFuture.complete(future.get()); From dba5c0711f287336f11b99c9a01963f9cd55dc35 Mon Sep 17 00:00:00 2001 From: Santiago Pericasgeertsen Date: Fri, 14 Aug 2020 09:52:19 -0400 Subject: [PATCH 28/65] Fixed returned types of async methods in tests. Signed-off-by: Santiago Pericasgeertsen --- .../faulttolerance/RetryBean.java | 19 +++++++++++-------- 1 file changed, 11 insertions(+), 8 deletions(-) diff --git a/microprofile/fault-tolerance/src/test/java/io/helidon/microprofile/faulttolerance/RetryBean.java b/microprofile/fault-tolerance/src/test/java/io/helidon/microprofile/faulttolerance/RetryBean.java index 1ecf1b31332..6101f59fc4d 100644 --- a/microprofile/fault-tolerance/src/test/java/io/helidon/microprofile/faulttolerance/RetryBean.java +++ b/microprofile/fault-tolerance/src/test/java/io/helidon/microprofile/faulttolerance/RetryBean.java @@ -72,12 +72,15 @@ public String onFailure() { @Asynchronous @Retry(maxRetries = 2) public Future retryAsync() { + CompletableFuture future = new CompletableFuture<>(); if (invocations.incrementAndGet() <= 2) { printStatus("RetryBean::retryAsync()", "failure"); - throw new RuntimeException("Oops"); + future.completeExceptionally(new RuntimeException("Oops")); + } else { + printStatus("RetryBean::retryAsync()", "success"); + future.complete("success"); } - printStatus("RetryBean::retryAsync()", "success"); - return CompletableFuture.completedFuture("success"); + return future; } @Retry(maxRetries = 4, delay = 100L, jitter = 50L) @@ -111,13 +114,13 @@ public CompletionStage retryWithException() { @Asynchronous @Retry(maxRetries = 2) public CompletionStage retryWithUltimateSuccess() { + CompletableFuture future = new CompletableFuture<>(); if (invocations.incrementAndGet() < 3) { - // fails twice - throw new RuntimeException("Simulated error"); + // fails twice + future.completeExceptionally(new RuntimeException("Simulated error")); + } else { + future.complete("Success"); } - - CompletableFuture future = new CompletableFuture<>(); - future.complete("Success"); return future; } } From 56d0b00677ecf1301c2faa1149a418edb9a3c20e Mon Sep 17 00:00:00 2001 From: Santiago Pericasgeertsen Date: Fri, 14 Aug 2020 09:53:34 -0400 Subject: [PATCH 29/65] Enhanced support for asynchronous methods. Signed-off-by: Santiago Pericasgeertsen --- .../faulttolerance/MethodInvoker.java | 199 +++++++++--------- 1 file changed, 99 insertions(+), 100 deletions(-) diff --git a/microprofile/fault-tolerance/src/main/java/io/helidon/microprofile/faulttolerance/MethodInvoker.java b/microprofile/fault-tolerance/src/main/java/io/helidon/microprofile/faulttolerance/MethodInvoker.java index 4a0624d1a2d..1b6dd011006 100644 --- a/microprofile/fault-tolerance/src/main/java/io/helidon/microprofile/faulttolerance/MethodInvoker.java +++ b/microprofile/fault-tolerance/src/main/java/io/helidon/microprofile/faulttolerance/MethodInvoker.java @@ -15,6 +15,9 @@ */ package io.helidon.microprofile.faulttolerance; +import javax.enterprise.context.control.RequestContextController; +import javax.enterprise.inject.spi.CDI; +import javax.interceptor.InvocationContext; import java.lang.reflect.Method; import java.time.Duration; import java.util.Objects; @@ -29,10 +32,6 @@ import java.util.function.Supplier; import java.util.logging.Logger; -import javax.enterprise.context.control.RequestContextController; -import javax.enterprise.inject.spi.CDI; -import javax.interceptor.InvocationContext; - import io.helidon.common.context.Context; import io.helidon.common.context.Contexts; import io.helidon.common.reactive.Single; @@ -46,9 +45,9 @@ import io.helidon.faulttolerance.Retry; import io.helidon.faulttolerance.Timeout; +import org.eclipse.microprofile.metrics.Counter; import org.eclipse.microprofile.faulttolerance.exceptions.CircuitBreakerOpenException; import org.eclipse.microprofile.faulttolerance.exceptions.TimeoutException; -import org.eclipse.microprofile.metrics.Counter; import org.glassfish.jersey.process.internal.RequestContext; import org.glassfish.jersey.process.internal.RequestScope; @@ -88,6 +87,9 @@ public class MethodInvoker implements FtSupplier { private static final Logger LOGGER = Logger.getLogger(MethodInvoker.class.getName()); + /** + * Waiting millis to convert {@code Future} into {@code CompletableFuture}. + */ private static final long AWAIT_FUTURE_MILLIS = 5L; /** @@ -146,6 +148,8 @@ public class MethodInvoker implements FtSupplier { private RequestContextController requestController; private static class MethodState { + FtHandlerTyped handler; + private Retry retry; private Bulkhead bulkhead; private CircuitBreaker breaker; private State lastBreakerState; @@ -156,22 +160,11 @@ private static class MethodState { } /** - * State associated with a method instead of an invocation. Includes bulkhead - * and breaker handlers that are shared across all invocations of same method. + * State associated with a method instead of an invocation. Shared by all + * invocations of same method. */ private final MethodState methodState; - /** - * The handler for this invocation. Will share bulkhead an breaker - * sub-handlers across all invocations for the same method. - */ - private FtHandlerTyped handler; - - /** - * Handler for retries. Will be {@code null} if no retry specified in method. - */ - private Retry retry; - /** * Constructor. * @@ -184,21 +177,19 @@ public MethodInvoker(InvocationContext context, MethodIntrospector introspector) this.method = context.getMethod(); this.helidonContext = Contexts.context().orElseGet(Context::create); - // Initialize method state and created handler for it - synchronized (method) { - this.methodState = FT_HANDLERS.computeIfAbsent(method, method -> { - MethodState methodState = new MethodState(); - methodState.lastBreakerState = State.CLOSED; - if (introspector.hasCircuitBreaker()) { - methodState.breakerTimerOpen = 0L; - methodState.breakerTimerClosed = 0L; - methodState.breakerTimerHalfOpen = 0L; - methodState.startNanos = System.nanoTime(); - } - return methodState; - }); - handler = createMethodHandler(); - } + // Initialize method state and create handler for it + this.methodState = FT_HANDLERS.computeIfAbsent(method, method -> { + MethodState methodState = new MethodState(); + methodState.lastBreakerState = State.CLOSED; + if (introspector.hasCircuitBreaker()) { + methodState.breakerTimerOpen = 0L; + methodState.breakerTimerClosed = 0L; + methodState.breakerTimerHalfOpen = 0L; + methodState.startNanos = System.nanoTime(); + } + methodState.handler = createMethodHandler(methodState); + return methodState; + }); // Gather information about current request scope if active try { @@ -268,7 +259,7 @@ public Object get() throws Throwable { Supplier> supplier = () -> { try { return Contexts.runInContextWithThrow(helidonContext, - () -> handler.invoke(toCompletionStageSupplier(context::proceed))); + () -> methodState.handler.invoke(toCompletionStageSupplier(context::proceed))); } catch (Exception e) { return Single.error(e); } @@ -280,15 +271,13 @@ public Object get() throws Throwable { // Wrap supplier with request context setup Supplier> wrappedSupplier = requestContextSupplier(supplier); - // Invoke wrapped supplier on a different thread - Single> asyncSingle = Async.create().invoke(wrappedSupplier); - CompletableFuture> asyncFuture = asyncSingle.toStage(true).toCompletableFuture(); + CompletableFuture asyncFuture = wrappedSupplier.get().toStage(true).toCompletableFuture(); // Register handler to process result and map exceptions CompletableFuture resultFuture = new CompletableFuture<>(); - asyncFuture.whenComplete((single, throwable) -> { - Throwable cause = null; + asyncFuture.whenComplete((result, throwable) -> { if (throwable != null) { + Throwable cause; if (throwable instanceof ExecutionException) { cause = map(throwable.getCause()); } else { @@ -297,18 +286,8 @@ public Object get() throws Throwable { updateMetricsAfter(cause); resultFuture.completeExceptionally(cause); } else { - try { - Object result = single.get(); - resultFuture.complete(result); - } catch (ExecutionException e) { - cause = map(e.getCause()); - } catch (Throwable t) { - cause = map(t); - } - updateMetricsAfter(cause); - if (cause != null) { - resultFuture.completeExceptionally(cause); - } + updateMetricsAfter(null); + resultFuture.complete(result); } }); return resultFuture; @@ -373,37 +352,35 @@ private Supplier> requestContextSupplier(Supplier> supplier) } /** - * Creates a FT handler for an invocation by inspecting annotations. Circuit breakers - * and bulkheads are shared across all invocations --associated with methods. + * Creates a FT handler for an invocation by inspecting annotations. + * + * @param methodState State related to this invocation's method. */ - private FtHandlerTyped createMethodHandler() { + private FtHandlerTyped createMethodHandler(MethodState methodState) { FaultTolerance.TypedBuilder builder = FaultTolerance.typedBuilder(); // Create and add circuit breaker if (introspector.hasCircuitBreaker()) { - if (methodState.breaker == null) { - methodState.breaker = CircuitBreaker.builder() - .delay(Duration.of(introspector.getCircuitBreaker().delay(), - introspector.getCircuitBreaker().delayUnit())) - .successThreshold(introspector.getCircuitBreaker().successThreshold()) - .errorRatio((int) (introspector.getCircuitBreaker().failureRatio() * 100)) - .volume(introspector.getCircuitBreaker().requestVolumeThreshold()) - .applyOn(introspector.getCircuitBreaker().failOn()) - .skipOn(introspector.getCircuitBreaker().skipOn()) - .build(); - } + methodState.breaker = CircuitBreaker.builder() + .delay(Duration.of(introspector.getCircuitBreaker().delay(), + introspector.getCircuitBreaker().delayUnit())) + .successThreshold(introspector.getCircuitBreaker().successThreshold()) + .errorRatio((int) (introspector.getCircuitBreaker().failureRatio() * 100)) + .volume(introspector.getCircuitBreaker().requestVolumeThreshold()) + .applyOn(introspector.getCircuitBreaker().failOn()) + .skipOn(introspector.getCircuitBreaker().skipOn()) + .build(); builder.addBreaker(methodState.breaker); } // Create and add bulkhead if (introspector.hasBulkhead()) { - if (methodState.bulkhead == null) { - methodState.bulkhead = Bulkhead.builder() - .limit(introspector.getBulkhead().value()) - .queueLength(introspector.getBulkhead().waitingTaskQueue()) - .async(introspector.isAsynchronous()) - .build(); - } + methodState.bulkhead = Bulkhead.builder() + .limit(introspector.getBulkhead().value()) + .queueLength(introspector.getBulkhead().waitingTaskQueue()) + .async(introspector.isAsynchronous()) + .build(); + builder.addBulkhead(methodState.bulkhead); } @@ -411,7 +388,7 @@ private FtHandlerTyped createMethodHandler() { if (introspector.hasTimeout()) { Timeout timeout = Timeout.builder() .timeout(Duration.of(introspector.getTimeout().value(), introspector.getTimeout().unit())) - .currentThread(true) + .currentThread(!introspector.isAsynchronous()) .executor(EXECUTOR_SERVICE) .build(); builder.addTimeout(timeout); @@ -433,7 +410,7 @@ private FtHandlerTyped createMethodHandler() { .skipOn(introspector.getRetry().abortOn()) .build(); builder.addRetry(retry); - this.retry = retry; + methodState.retry = retry; // keep reference to Retry } // Create and add fallback handler -- parent of retry @@ -449,9 +426,7 @@ private FtHandlerTyped createMethodHandler() { builder.addFallback(fallback); } - FtHandlerTyped result = builder.build(); - System.out.println("\n### tree \n" + result); - return result; + return builder.build(); } /** @@ -463,29 +438,53 @@ private FtHandlerTyped createMethodHandler() { @SuppressWarnings("unchecked") Supplier> toCompletionStageSupplier(FtSupplier supplier) { return () -> { - try { - invocationStartNanos = System.nanoTime(); + invocationStartNanos = System.nanoTime(); - // This is the actual method invocation - Object result = supplier.get(); + CompletableFuture resultFuture = new CompletableFuture<>(); + if (introspector.isAsynchronous()) { + // Invoke supplier in a new thread + Single single = Async.create().invoke(() -> { + try { + return supplier.get(); // Future, CompletableFuture or CompletionStage + } catch (Throwable t) { + return CompletableFuture.failedFuture(t); + } + }); - if (introspector.isAsynchronous()) { - if (result instanceof CompletionStage) { // true also for CompletableFuture - return (CompletionStage) result; - } else if (result instanceof Future) { - Future future = (Future) result; - CompletableFuture completableFuture = new CompletableFuture<>(); - awaitFuture(future, completableFuture); - return completableFuture; + // The result must be Future, CompletableFuture or CompletionStage + single.whenComplete((result, throwable) -> { + if (throwable != null) { + resultFuture.completeExceptionally(throwable); } else { - throw new IllegalStateException("Return type not allowed in async method " + method); + try { + if (result instanceof CompletionStage) { // also CompletableFuture + CompletionStage cs = (CompletionStage) result; + cs.whenComplete((o, t) -> { + if (t != null) { + resultFuture.completeExceptionally(t); + } else { + resultFuture.complete(o); + } + }); + } else if (result instanceof Future){ + awaitFuture((Future) result, resultFuture); + } else { + throw new InternalError("Return type validation failed for method " + method); + } + } catch (Throwable t) { + resultFuture.completeExceptionally(t); + } } - } else { - return CompletableFuture.completedFuture(result); + }); + } else { + try { + resultFuture.complete(supplier.get()); + return resultFuture; + } catch (Throwable t) { + resultFuture.completeExceptionally(t); } - } catch (Throwable e) { - return CompletableFuture.failedFuture(e); } + return resultFuture; }; } @@ -513,7 +512,7 @@ private void updateMetricsAfter(Throwable cause) { // Metrics for retries if (introspector.hasRetry()) { // Have retried the last call? - long newValue = retry.retryCounter(); + long newValue = methodState.retry.retryCounter(); if (updateCounter(method, RETRY_RETRIES_TOTAL, newValue)) { if (cause == null) { getCounter(method, RETRY_CALLS_SUCCEEDED_RETRIED_TOTAL).inc(); @@ -608,9 +607,9 @@ private void updateMetricsAfter(Throwable cause) { /** * Sets the value of a monotonically increasing counter using {@code inc()}. * - * @param method the method. - * @param name the counter's name. - * @param newValue the new value. + * @param method The method. + * @param name The counter's name. + * @param newValue The new value. * @return A value of {@code true} if counter updated, {@code false} otherwise. */ private static boolean updateCounter(Method method, String name, long newValue) { @@ -627,8 +626,8 @@ private static boolean updateCounter(Method method, String name, long newValue) * Waits for a Future to produce an outcome and updates a CompletableFuture based on * it. Effectively converts a Future into a CompletableFuture. * - * @param future the future. - * @param completableFuture the completable future. + * @param future The future. + * @param completableFuture The completable future. */ private static void awaitFuture(Future future, CompletableFuture completableFuture) { if (future.isDone()) { From 90e0b6fa9d5419b218036a7b4440165124904c75 Mon Sep 17 00:00:00 2001 From: Santiago Pericasgeertsen Date: Fri, 14 Aug 2020 14:55:24 -0400 Subject: [PATCH 30/65] Fixed regression in request scope support. Signed-off-by: Santiago Pericasgeertsen --- .../faulttolerance/MethodInvoker.java | 46 +++++++++---------- 1 file changed, 21 insertions(+), 25 deletions(-) diff --git a/microprofile/fault-tolerance/src/main/java/io/helidon/microprofile/faulttolerance/MethodInvoker.java b/microprofile/fault-tolerance/src/main/java/io/helidon/microprofile/faulttolerance/MethodInvoker.java index 1b6dd011006..691b0905910 100644 --- a/microprofile/fault-tolerance/src/main/java/io/helidon/microprofile/faulttolerance/MethodInvoker.java +++ b/microprofile/fault-tolerance/src/main/java/io/helidon/microprofile/faulttolerance/MethodInvoker.java @@ -15,12 +15,10 @@ */ package io.helidon.microprofile.faulttolerance; -import javax.enterprise.context.control.RequestContextController; -import javax.enterprise.inject.spi.CDI; -import javax.interceptor.InvocationContext; import java.lang.reflect.Method; import java.time.Duration; import java.util.Objects; +import java.util.concurrent.Callable; import java.util.concurrent.CompletableFuture; import java.util.concurrent.CompletionStage; import java.util.concurrent.ConcurrentHashMap; @@ -32,6 +30,10 @@ import java.util.function.Supplier; import java.util.logging.Logger; +import javax.enterprise.context.control.RequestContextController; +import javax.enterprise.inject.spi.CDI; +import javax.interceptor.InvocationContext; + import io.helidon.common.context.Context; import io.helidon.common.context.Contexts; import io.helidon.common.reactive.Single; @@ -45,9 +47,9 @@ import io.helidon.faulttolerance.Retry; import io.helidon.faulttolerance.Timeout; -import org.eclipse.microprofile.metrics.Counter; import org.eclipse.microprofile.faulttolerance.exceptions.CircuitBreakerOpenException; import org.eclipse.microprofile.faulttolerance.exceptions.TimeoutException; +import org.eclipse.microprofile.metrics.Counter; import org.glassfish.jersey.process.internal.RequestContext; import org.glassfish.jersey.process.internal.RequestScope; @@ -148,7 +150,7 @@ public class MethodInvoker implements FtSupplier { private RequestContextController requestController; private static class MethodState { - FtHandlerTyped handler; + private FtHandlerTyped handler; private Retry retry; private Bulkhead bulkhead; private CircuitBreaker breaker; @@ -268,12 +270,8 @@ public Object get() throws Throwable { updateMetricsBefore(); if (introspector.isAsynchronous()) { - // Wrap supplier with request context setup - Supplier> wrappedSupplier = requestContextSupplier(supplier); - - CompletableFuture asyncFuture = wrappedSupplier.get().toStage(true).toCompletableFuture(); - - // Register handler to process result and map exceptions + CompletableFuture asyncFuture = supplier.get() + .toStage(true).toCompletableFuture(); CompletableFuture resultFuture = new CompletableFuture<>(); asyncFuture.whenComplete((result, throwable) -> { if (throwable != null) { @@ -295,8 +293,8 @@ public Object get() throws Throwable { Object result = null; Throwable cause = null; try { - // Invoke supplier on same thread - CompletableFuture future = supplier.get().toStage(true).toCompletableFuture(); + CompletableFuture future = supplier.get() + .toStage(true).toCompletableFuture(); result = future.get(); } catch (ExecutionException e) { cause = map(e.getCause()); @@ -317,30 +315,25 @@ public Object get() throws Throwable { * {@code @Context} to work properly. Note that it is possible for only CDI's * request scope to be active at this time (e.g. in TCKs). */ - private Supplier> requestContextSupplier(Supplier> supplier) { - Supplier> wrappedSupplier; + private FtSupplier requestContextSupplier(FtSupplier supplier) { + FtSupplier wrappedSupplier; if (requestScope != null) { // Jersey and CDI - wrappedSupplier = () -> { - try { - return requestScope.runInScope(requestContext, (() -> { + wrappedSupplier = () -> requestScope.runInScope(requestContext, + (Callable) (() -> { try { requestController.activate(); return supplier.get(); + } catch (Throwable t) { + throw t instanceof Exception ? ((Exception) t) : new RuntimeException(t); } finally { requestController.deactivate(); } })); - } catch (Exception e) { - return Single.error(e); - } - }; } else if (requestController != null) { // CDI only wrappedSupplier = () -> { try { requestController.activate(); return supplier.get(); - } catch (Exception e) { - return Single.error(e); } finally { requestController.deactivate(); } @@ -442,10 +435,13 @@ Supplier> toCompletionStageSupplier(FtSupplier CompletableFuture resultFuture = new CompletableFuture<>(); if (introspector.isAsynchronous()) { + // Wrap supplier with request context setup + FtSupplier wrappedSupplier = requestContextSupplier(supplier); + // Invoke supplier in a new thread Single single = Async.create().invoke(() -> { try { - return supplier.get(); // Future, CompletableFuture or CompletionStage + return wrappedSupplier.get(); // Future, CompletableFuture or CompletionStage } catch (Throwable t) { return CompletableFuture.failedFuture(t); } From a59886bd2424dbdb3be9aa1e4df03691006cd7c2 Mon Sep 17 00:00:00 2001 From: Santiago Pericasgeertsen Date: Thu, 20 Aug 2020 11:52:24 -0400 Subject: [PATCH 31/65] Support for cancellation of Singles created from CompletableFutures and vice-versa. Signed-off-by: Santiago Pericasgeertsen --- .../common/reactive/CompletionSingle.java | 2 +- .../io/helidon/common/reactive/Single.java | 2 +- .../reactive/SingleFromCompletionStage.java | 7 ++ .../common/reactive/SingleToFuture.java | 5 +- .../common/reactive/CancellationTest.java | 65 +++++++++++++++++++ 5 files changed, 78 insertions(+), 3 deletions(-) create mode 100644 common/reactive/src/test/java/io/helidon/common/reactive/CancellationTest.java diff --git a/common/reactive/src/main/java/io/helidon/common/reactive/CompletionSingle.java b/common/reactive/src/main/java/io/helidon/common/reactive/CompletionSingle.java index a63efc7a64d..f0a4a15e694 100644 --- a/common/reactive/src/main/java/io/helidon/common/reactive/CompletionSingle.java +++ b/common/reactive/src/main/java/io/helidon/common/reactive/CompletionSingle.java @@ -36,7 +36,7 @@ protected CompletionSingle() { } protected CompletableFuture toNullableStage() { - SingleToFuture subscriber = new SingleToFuture<>(true); + SingleToFuture subscriber = new SingleToFuture<>(this, true); this.subscribe(subscriber); return subscriber; } diff --git a/common/reactive/src/main/java/io/helidon/common/reactive/Single.java b/common/reactive/src/main/java/io/helidon/common/reactive/Single.java index cf8922e6a09..ea99215af29 100644 --- a/common/reactive/src/main/java/io/helidon/common/reactive/Single.java +++ b/common/reactive/src/main/java/io/helidon/common/reactive/Single.java @@ -662,7 +662,7 @@ default CompletionStage toStage() { */ default CompletionStage toStage(boolean completeWithoutValue) { try { - SingleToFuture subscriber = new SingleToFuture<>(completeWithoutValue); + SingleToFuture subscriber = new SingleToFuture<>(this, completeWithoutValue); this.subscribe(subscriber); return subscriber; } catch (Throwable ex) { diff --git a/common/reactive/src/main/java/io/helidon/common/reactive/SingleFromCompletionStage.java b/common/reactive/src/main/java/io/helidon/common/reactive/SingleFromCompletionStage.java index aa143be07d5..7b214048408 100644 --- a/common/reactive/src/main/java/io/helidon/common/reactive/SingleFromCompletionStage.java +++ b/common/reactive/src/main/java/io/helidon/common/reactive/SingleFromCompletionStage.java @@ -37,4 +37,11 @@ final class SingleFromCompletionStage extends CompletionSingle { public void subscribe(Flow.Subscriber subscriber) { MultiFromCompletionStage.subscribe(subscriber, source, nullMeansEmpty); } + + @Override + public Single cancel() { + Single single = super.cancel(); + source.toCompletableFuture().cancel(true); + return single; + } } diff --git a/common/reactive/src/main/java/io/helidon/common/reactive/SingleToFuture.java b/common/reactive/src/main/java/io/helidon/common/reactive/SingleToFuture.java index 98869fd7bdd..c0c99f632d4 100644 --- a/common/reactive/src/main/java/io/helidon/common/reactive/SingleToFuture.java +++ b/common/reactive/src/main/java/io/helidon/common/reactive/SingleToFuture.java @@ -28,8 +28,10 @@ final class SingleToFuture extends CompletableFuture implements Subscriber private final AtomicReference ref = new AtomicReference<>(); private final boolean completeWithoutValue; + private final Single source; - SingleToFuture(boolean completeWithoutValue) { + SingleToFuture(Single source, boolean completeWithoutValue) { + this.source = source; this.completeWithoutValue = completeWithoutValue; } @@ -41,6 +43,7 @@ public boolean cancel(boolean mayInterruptIfRunning) { if (s != null) { s.cancel(); } + source.cancel(); } return cancelled; } diff --git a/common/reactive/src/test/java/io/helidon/common/reactive/CancellationTest.java b/common/reactive/src/test/java/io/helidon/common/reactive/CancellationTest.java new file mode 100644 index 00000000000..c17b07c97bc --- /dev/null +++ b/common/reactive/src/test/java/io/helidon/common/reactive/CancellationTest.java @@ -0,0 +1,65 @@ +/* + * Copyright (c) 2020 Oracle and/or its affiliates. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package io.helidon.common.reactive; + +import java.util.concurrent.CancellationException; +import java.util.concurrent.CompletableFuture; +import java.util.concurrent.atomic.AtomicBoolean; + +import org.junit.jupiter.api.Test; + +import static org.hamcrest.MatcherAssert.assertThat; +import static org.hamcrest.CoreMatchers.is; + +public class CancellationTest { + + /** + * Test cancellation of underlying {@code CompletableFuture} when wrapped + * by a {@code Single}. + */ + @Test + public void testCompletableFutureCancel() { + AtomicBoolean cancelled = new AtomicBoolean(false); + CompletableFuture future = new CompletableFuture<>(); + Single single = Single.create(future, true); + future.whenComplete((o, t) -> { + if (t instanceof CancellationException) { + cancelled.set(true); + } + }); + single.cancel(); // should cancel future + assertThat(cancelled.get(), is(true)); + } + + /** + * Test cancellation of {@code Single} after it has been converted to + * a {@code CompletableFuture}. + */ + @Test + public void testSingleCancel() { + AtomicBoolean cancelled = new AtomicBoolean(false); + Single single = Single.create(new CompletableFuture<>()); + CompletableFuture future = single.toStage(true).toCompletableFuture(); + single.whenComplete((o, t) -> { + if (t instanceof CancellationException) { + cancelled.set(true); + } + }); + future.cancel(true); // should cancel single + assertThat(cancelled.get(), is(true)); + } +} From 0f41dee438ef53d8247f3c5753503521e9beaf3a Mon Sep 17 00:00:00 2001 From: Santiago Pericasgeertsen Date: Thu, 20 Aug 2020 14:25:31 -0400 Subject: [PATCH 32/65] Support for cancellation of async tasks. Update ResultWindow opening condition to use '>' instead of '>=' for MP compatibility. Made some test changes. Signed-off-by: Santiago Pericasgeertsen --- .../java/io/helidon/faulttolerance/AsyncImpl.java | 8 ++++++-- .../io/helidon/faulttolerance/BulkheadImpl.java | 13 +++++++++++-- .../io/helidon/faulttolerance/ResultWindow.java | 2 +- .../helidon/faulttolerance/CircuitBreakerTest.java | 2 ++ .../io/helidon/faulttolerance/ResultWindowTest.java | 6 ++++-- 5 files changed, 24 insertions(+), 7 deletions(-) diff --git a/fault-tolerance/src/main/java/io/helidon/faulttolerance/AsyncImpl.java b/fault-tolerance/src/main/java/io/helidon/faulttolerance/AsyncImpl.java index 8c006ca9ea7..dc341aa1805 100644 --- a/fault-tolerance/src/main/java/io/helidon/faulttolerance/AsyncImpl.java +++ b/fault-tolerance/src/main/java/io/helidon/faulttolerance/AsyncImpl.java @@ -18,6 +18,7 @@ import java.util.concurrent.CompletableFuture; import java.util.concurrent.ExecutorService; +import java.util.concurrent.Future; import java.util.function.Supplier; import io.helidon.common.LazyValue; @@ -35,14 +36,17 @@ public Single invoke(Supplier supplier) { CompletableFuture future = new CompletableFuture<>(); AsyncTask task = new AsyncTask<>(supplier, future); + Future taskFuture; try { - executor.get().submit(task); + taskFuture = executor.get().submit(task); } catch (Throwable e) { // rejected execution and other executor related issues return Single.error(e); } - return Single.create(future, true); + Single single = Single.create(future, true); + single.onCancel(() -> taskFuture.cancel(false)); // cancel task + return single; } private static class AsyncTask implements Runnable { diff --git a/fault-tolerance/src/main/java/io/helidon/faulttolerance/BulkheadImpl.java b/fault-tolerance/src/main/java/io/helidon/faulttolerance/BulkheadImpl.java index 45e5aafecf5..39ec0b41e24 100644 --- a/fault-tolerance/src/main/java/io/helidon/faulttolerance/BulkheadImpl.java +++ b/fault-tolerance/src/main/java/io/helidon/faulttolerance/BulkheadImpl.java @@ -96,6 +96,7 @@ public long waitingQueueSize() { private R invokeTask(DelayedTask task) { if (inProgress.tryAcquire()) { LOGGER.finest(() -> name + " invoke immediate: " + task); + // free permit, we can invoke execute(task); return task.result(); @@ -103,9 +104,16 @@ private R invokeTask(DelayedTask task) { // no free permit, let's try to enqueue in async mode if (async && queue.offer(task)) { LOGGER.finest(() -> name + " enqueue: " + task); - return task.result(); + + R result = task.result(); + if (result instanceof Single) { + Single single = (Single) result; + return (R) single.onCancel(queue::remove); + } + return result; } else { LOGGER.finest(() -> name + " reject: " + task); + callsRejected.incrementAndGet(); return task.error(new BulkheadException("Bulkhead queue \"" + name + "\" is full")); } @@ -117,7 +125,6 @@ private void execute(DelayedTask task) { callsAccepted.incrementAndGet(); concurrentExecutions.incrementAndGet(); - long startNanos = System.nanoTime(); task.execute() .handle((it, throwable) -> { concurrentExecutions.decrementAndGet(); @@ -127,10 +134,12 @@ private void execute(DelayedTask task) { DelayedTask polled = queue.poll(); if (polled != null) { LOGGER.finest(() -> name + " invoke in executor: " + polled); + // chain executions from queue until all are executed executor.get().submit(() -> execute(polled)); } else { LOGGER.finest(() -> name + " permit released after: " + task); + // nothing in the queue, release permit inProgress.release(); } diff --git a/fault-tolerance/src/main/java/io/helidon/faulttolerance/ResultWindow.java b/fault-tolerance/src/main/java/io/helidon/faulttolerance/ResultWindow.java index 77decdac18d..5e5b46e616f 100644 --- a/fault-tolerance/src/main/java/io/helidon/faulttolerance/ResultWindow.java +++ b/fault-tolerance/src/main/java/io/helidon/faulttolerance/ResultWindow.java @@ -63,7 +63,7 @@ void update(Result resultEnum) { } boolean shouldOpen() { - return currentSum.get() >= thresholdSum; + return currentSum.get() > thresholdSum; } void reset() { diff --git a/fault-tolerance/src/test/java/io/helidon/faulttolerance/CircuitBreakerTest.java b/fault-tolerance/src/test/java/io/helidon/faulttolerance/CircuitBreakerTest.java index f2469dca981..b514e6ecc32 100644 --- a/fault-tolerance/src/test/java/io/helidon/faulttolerance/CircuitBreakerTest.java +++ b/fault-tolerance/src/test/java/io/helidon/faulttolerance/CircuitBreakerTest.java @@ -55,6 +55,7 @@ void testCircuitBreaker() throws InterruptedException { // should open the breaker bad(breaker); + bad(breaker); breakerOpen(breaker); breakerOpenMulti(breaker); @@ -80,6 +81,7 @@ void testCircuitBreaker() throws InterruptedException { // should open the breaker bad(breaker); bad(breaker); + bad(breaker); assertThat(breaker.state(), is(CircuitBreaker.State.OPEN)); diff --git a/fault-tolerance/src/test/java/io/helidon/faulttolerance/ResultWindowTest.java b/fault-tolerance/src/test/java/io/helidon/faulttolerance/ResultWindowTest.java index b27a5b18e5a..7b045403ee5 100644 --- a/fault-tolerance/src/test/java/io/helidon/faulttolerance/ResultWindowTest.java +++ b/fault-tolerance/src/test/java/io/helidon/faulttolerance/ResultWindowTest.java @@ -31,14 +31,16 @@ void test() { window.update(ResultWindow.Result.SUCCESS); assertThat("Only success should not open", window.shouldOpen(), is(false)); window.update(ResultWindow.Result.FAILURE); - assertThat("Should open on first failure (10% of 10 size)", window.shouldOpen(), is(true)); + window.update(ResultWindow.Result.FAILURE); + assertThat("Should open on first failure (> 10%)", window.shouldOpen(), is(true)); //now cycle through window and replace all with success for (int i = 0; i < 10; i++) { window.update(ResultWindow.Result.SUCCESS); } assertThat("All success should not open", window.shouldOpen(), is(false)); window.update(ResultWindow.Result.FAILURE); - assertThat("Should open on first failure (10% of 10 size)", window.shouldOpen(), is(true)); + window.update(ResultWindow.Result.FAILURE); + assertThat("Should open on first failure (> 10%)", window.shouldOpen(), is(true)); window.reset(); assertThat("Should not open after reset", window.shouldOpen(), is(false)); } From a13e077a5872438a1c240d4777c7b93c6a837968 Mon Sep 17 00:00:00 2001 From: Santiago Pericasgeertsen Date: Thu, 20 Aug 2020 14:27:04 -0400 Subject: [PATCH 33/65] Use CompletableFuture instead of Future whenever possible. CircuitBreaker's opening window condition. Signed-off-by: Santiago Pericasgeertsen --- .../faulttolerance/AsynchronousBean.java | 46 +++++++++++++------ .../faulttolerance/AsynchronousTest.java | 15 ++++-- .../faulttolerance/BulkheadBean.java | 11 ++--- .../faulttolerance/BulkheadTest.java | 17 ++++--- .../faulttolerance/CircuitBreakerBean.java | 3 +- .../faulttolerance/CircuitBreakerTest.java | 6 +-- .../faulttolerance/FaultToleranceTest.java | 8 ++-- .../faulttolerance/MetricsBean.java | 5 +- .../faulttolerance/MetricsTest.java | 17 ++++--- .../faulttolerance/RetryBean.java | 3 +- .../faulttolerance/RetryTest.java | 4 +- .../faulttolerance/TimeoutBean.java | 3 +- .../faulttolerance/TimeoutTest.java | 4 +- 13 files changed, 81 insertions(+), 61 deletions(-) diff --git a/microprofile/fault-tolerance/src/test/java/io/helidon/microprofile/faulttolerance/AsynchronousBean.java b/microprofile/fault-tolerance/src/test/java/io/helidon/microprofile/faulttolerance/AsynchronousBean.java index 5b64cfc45a9..1103e395f39 100644 --- a/microprofile/fault-tolerance/src/test/java/io/helidon/microprofile/faulttolerance/AsynchronousBean.java +++ b/microprofile/fault-tolerance/src/test/java/io/helidon/microprofile/faulttolerance/AsynchronousBean.java @@ -20,6 +20,7 @@ import java.util.concurrent.CompletableFuture; import java.util.concurrent.CompletionStage; import java.util.concurrent.Future; +import java.util.concurrent.atomic.AtomicBoolean; import javax.enterprise.context.Dependent; @@ -32,10 +33,10 @@ @Dependent public class AsynchronousBean { - private boolean called; + private AtomicBoolean called = new AtomicBoolean(false); public boolean wasCalled() { - return called; + return called.get(); } /** @@ -44,8 +45,8 @@ public boolean wasCalled() { * @return A future. */ @Asynchronous - public Future async() { - called = true; + public CompletableFuture async() { + called.set(true); FaultToleranceTest.printStatus("AsynchronousBean::async", "success"); return CompletableFuture.completedFuture("success"); } @@ -57,8 +58,8 @@ public Future async() { */ @Asynchronous @Fallback(fallbackMethod = "onFailure") - public Future asyncWithFallback() { - called = true; + public CompletableFuture asyncWithFallback() { + called.set(true); FaultToleranceTest.printStatus("AsynchronousBean::asyncWithFallback", "failure"); return CompletableFuture.failedFuture(new RuntimeException("Oops")); } @@ -68,13 +69,32 @@ public CompletableFuture onFailure() { return CompletableFuture.completedFuture("fallback"); } + /** + * Async call with fallback and Future. Fallback should be ignored in this case. + * + * @return A future. + */ + @Asynchronous + @Fallback(fallbackMethod = "onFailureFuture") + public Future asyncWithFallbackFuture() { + called.set(true); + FaultToleranceTest.printStatus("AsynchronousBean::asyncWithFallbackFuture", "failure"); + return CompletableFuture.failedFuture(new RuntimeException("Oops")); + } + + public Future onFailureFuture() { + FaultToleranceTest.printStatus("AsynchronousBean::onFailure", "success"); + return CompletableFuture.completedFuture("fallback"); + } + + /** * Regular test, not asynchronous. * * @return A future. */ - public Future notAsync() { - called = true; + public CompletableFuture notAsync() { + called.set(true); FaultToleranceTest.printStatus("AsynchronousBean::notAsync", "success"); return CompletableFuture.completedFuture("success"); } @@ -86,7 +106,7 @@ public Future notAsync() { */ @Asynchronous public CompletionStage asyncCompletionStage() { - called = true; + called.set(true); FaultToleranceTest.printStatus("AsynchronousBean::asyncCompletionStage", "success"); return CompletableFuture.completedFuture("success"); } @@ -99,7 +119,7 @@ public CompletionStage asyncCompletionStage() { @Asynchronous @Fallback(fallbackMethod = "onFailure") public CompletionStage asyncCompletionStageWithFallback() { - called = true; + called.set(true); FaultToleranceTest.printStatus("AsynchronousBean::asyncCompletionStageWithFallback", "failure"); return CompletableFuture.failedFuture(new RuntimeException("Oops")); } @@ -111,7 +131,7 @@ public CompletionStage asyncCompletionStageWithFallback() { */ @Asynchronous public CompletableFuture asyncCompletableFuture() { - called = true; + called.set(true); FaultToleranceTest.printStatus("AsynchronousBean::asyncCompletableFuture", "success"); return CompletableFuture.completedFuture("success"); } @@ -124,7 +144,7 @@ public CompletableFuture asyncCompletableFuture() { @Asynchronous @Fallback(fallbackMethod = "onFailure") public CompletableFuture asyncCompletableFutureWithFallback() { - called = true; + called.set(true); FaultToleranceTest.printStatus("AsynchronousBean::asyncCompletableFutureWithFallback", "success"); return CompletableFuture.completedFuture("success"); } @@ -138,7 +158,7 @@ public CompletableFuture asyncCompletableFutureWithFallback() { @Asynchronous @Fallback(fallbackMethod = "onFailure") public CompletableFuture asyncCompletableFutureWithFallbackFailure() { - called = true; + called.set(true); FaultToleranceTest.printStatus("AsynchronousBean::asyncCompletableFutureWithFallbackFailure", "failure"); CompletableFuture future = new CompletableFuture<>(); future.completeExceptionally(new IOException("oops")); diff --git a/microprofile/fault-tolerance/src/test/java/io/helidon/microprofile/faulttolerance/AsynchronousTest.java b/microprofile/fault-tolerance/src/test/java/io/helidon/microprofile/faulttolerance/AsynchronousTest.java index fda0b8017ab..89f6b1c24f3 100644 --- a/microprofile/fault-tolerance/src/test/java/io/helidon/microprofile/faulttolerance/AsynchronousTest.java +++ b/microprofile/fault-tolerance/src/test/java/io/helidon/microprofile/faulttolerance/AsynchronousTest.java @@ -34,7 +34,7 @@ public class AsynchronousTest extends FaultToleranceTest { public void testAsync() throws Exception { AsynchronousBean bean = newBean(AsynchronousBean.class); assertThat(bean.wasCalled(), is(false)); - Future future = bean.async(); + CompletableFuture future = bean.async(); future.get(); assertThat(bean.wasCalled(), is(true)); } @@ -43,17 +43,24 @@ public void testAsync() throws Exception { public void testAsyncWithFallback() throws Exception { AsynchronousBean bean = newBean(AsynchronousBean.class); assertThat(bean.wasCalled(), is(false)); - Future future = bean.asyncWithFallback(); + CompletableFuture future = bean.asyncWithFallback(); String value = future.get(); assertThat(bean.wasCalled(), is(true)); assertThat(value, is("fallback")); } + @Test + public void testAsyncWithFallbackFuture() { + AsynchronousBean bean = newBean(AsynchronousBean.class); + Future future = bean.asyncWithFallbackFuture(); // fallback ignored with Future + assertCompleteExceptionally(future, RuntimeException.class); + } + @Test public void testAsyncNoGet() throws Exception { AsynchronousBean bean = newBean(AsynchronousBean.class); assertThat(bean.wasCalled(), is(false)); - Future future = bean.async(); + CompletableFuture future = bean.async(); while (!future.isDone()) { Thread.sleep(100); } @@ -64,7 +71,7 @@ public void testAsyncNoGet() throws Exception { public void testNotAsync() throws Exception { AsynchronousBean bean = newBean(AsynchronousBean.class); assertThat(bean.wasCalled(), is(false)); - Future future = bean.notAsync(); + CompletableFuture future = bean.notAsync(); assertThat(bean.wasCalled(), is(true)); future.get(); } diff --git a/microprofile/fault-tolerance/src/test/java/io/helidon/microprofile/faulttolerance/BulkheadBean.java b/microprofile/fault-tolerance/src/test/java/io/helidon/microprofile/faulttolerance/BulkheadBean.java index 02d8f5f4dff..0c9bd26327b 100644 --- a/microprofile/fault-tolerance/src/test/java/io/helidon/microprofile/faulttolerance/BulkheadBean.java +++ b/microprofile/fault-tolerance/src/test/java/io/helidon/microprofile/faulttolerance/BulkheadBean.java @@ -17,7 +17,6 @@ package io.helidon.microprofile.faulttolerance; import java.util.concurrent.CompletableFuture; -import java.util.concurrent.Future; import javax.enterprise.context.Dependent; @@ -70,7 +69,7 @@ ConcurrencyCounter getCounter() { @Asynchronous @Bulkhead(value = CONCURRENT_CALLS, waitingTaskQueue = WAITING_TASK_QUEUE) - public Future execute(long sleepMillis) { + public CompletableFuture execute(long sleepMillis) { try { counter.increment(); FaultToleranceTest.printStatus("BulkheadBean::execute", "success"); @@ -87,7 +86,7 @@ public Future execute(long sleepMillis) { @Asynchronous @Bulkhead(value = CONCURRENT_CALLS + 1, waitingTaskQueue = WAITING_TASK_QUEUE + 1) - public Future executePlusOne(long sleepMillis) { + public CompletableFuture executePlusOne(long sleepMillis) { try { counter.increment(); FaultToleranceTest.printStatus("BulkheadBean::executePlusOne", "success"); @@ -104,7 +103,7 @@ public Future executePlusOne(long sleepMillis) { @Asynchronous @Bulkhead(value = 2, waitingTaskQueue = 1) - public Future executeNoQueue(long sleepMillis) { + public CompletableFuture executeNoQueue(long sleepMillis) { try { counter.increment(); FaultToleranceTest.printStatus("BulkheadBean::executeNoQueue", "success"); @@ -122,7 +121,7 @@ public Future executeNoQueue(long sleepMillis) { @Asynchronous @Fallback(fallbackMethod = "onFailure") @Bulkhead(value = 2, waitingTaskQueue = 1) - public Future executeNoQueueWithFallback(long sleepMillis) { + public CompletableFuture executeNoQueueWithFallback(long sleepMillis) { try { counter.increment(); FaultToleranceTest.printStatus("BulkheadBean::executeNoQueue", "success"); @@ -144,7 +143,7 @@ public CompletableFuture onFailure(long sleepMillis) { @Asynchronous @Bulkhead(value = 1, waitingTaskQueue = 1) - public Future executeCancelInQueue(long sleepMillis) { + public CompletableFuture executeCancelInQueue(long sleepMillis) { FaultToleranceTest.printStatus("BulkheadBean::executeCancelInQueue " + sleepMillis, "success"); try { Thread.sleep(sleepMillis); diff --git a/microprofile/fault-tolerance/src/test/java/io/helidon/microprofile/faulttolerance/BulkheadTest.java b/microprofile/fault-tolerance/src/test/java/io/helidon/microprofile/faulttolerance/BulkheadTest.java index fcdbf40bedb..9cb6179f562 100644 --- a/microprofile/fault-tolerance/src/test/java/io/helidon/microprofile/faulttolerance/BulkheadTest.java +++ b/microprofile/fault-tolerance/src/test/java/io/helidon/microprofile/faulttolerance/BulkheadTest.java @@ -20,7 +20,6 @@ import java.util.concurrent.Callable; import java.util.concurrent.CancellationException; import java.util.concurrent.CompletableFuture; -import java.util.concurrent.Future; import org.eclipse.microprofile.faulttolerance.exceptions.BulkheadException; import org.junit.jupiter.api.Test; @@ -40,7 +39,7 @@ public class BulkheadTest extends FaultToleranceTest { @Test public void testBulkhead() { BulkheadBean bean = newBean(BulkheadBean.class); - Future[] calls = getAsyncConcurrentCalls( + CompletableFuture[] calls = getAsyncConcurrentCalls( () -> bean.execute(100), BulkheadBean.TOTAL_CALLS); waitFor(calls); assertThat(bean.getCounter().concurrentCalls(), is(BulkheadBean.CONCURRENT_CALLS)); @@ -50,7 +49,7 @@ public void testBulkhead() { @Test public void testBulkheadPlusOne() { BulkheadBean bean = newBean(BulkheadBean.class); - Future[] calls = getAsyncConcurrentCalls( + CompletableFuture[] calls = getAsyncConcurrentCalls( () -> bean.executePlusOne(100), BulkheadBean.TOTAL_CALLS + 2); waitFor(calls); assertThat(bean.getCounter().concurrentCalls(), is(BulkheadBean.CONCURRENT_CALLS + 1)); @@ -60,7 +59,7 @@ public void testBulkheadPlusOne() { @Test public void testBulkheadNoQueue() { BulkheadBean bean = newBean(BulkheadBean.class); - Future[] calls = getAsyncConcurrentCalls( + CompletableFuture[] calls = getAsyncConcurrentCalls( () -> bean.executeNoQueue(2000), 10); RuntimeException e = assertThrows(RuntimeException.class, () -> waitFor(calls)); assertThat(e.getCause().getCause(), instanceOf(BulkheadException.class)); @@ -69,7 +68,7 @@ public void testBulkheadNoQueue() { @Test public void testBulkheadNoQueueWithFallback() { BulkheadBean bean = newBean(BulkheadBean.class); - Future[] calls = getAsyncConcurrentCalls( + CompletableFuture[] calls = getAsyncConcurrentCalls( () -> bean.executeNoQueueWithFallback(2000), 10); waitFor(calls); } @@ -77,8 +76,8 @@ public void testBulkheadNoQueueWithFallback() { @Test public void testBulkheadExecuteCancelInQueue() throws Exception { BulkheadBean bean = newBean(BulkheadBean.class); - Future f1 = bean.executeCancelInQueue(1000); - Future f2 = bean.executeCancelInQueue(2000); // should never run + CompletableFuture f1 = bean.executeCancelInQueue(1000); + CompletableFuture f2 = bean.executeCancelInQueue(2000); // should never run boolean b = f2.cancel(true); assertTrue(b); assertTrue(f2.isCancelled()); @@ -125,8 +124,8 @@ public void testSynchronousWithAsyncCaller() throws Exception { return 0; } }; - Future f1 = callerBean.submit(callable); - Future f2 = callerBean.submit(callable); + CompletableFuture f1 = callerBean.submit(callable); + CompletableFuture f2 = callerBean.submit(callable); assertThat(f1.get() + f2.get(), is(1)); } } diff --git a/microprofile/fault-tolerance/src/test/java/io/helidon/microprofile/faulttolerance/CircuitBreakerBean.java b/microprofile/fault-tolerance/src/test/java/io/helidon/microprofile/faulttolerance/CircuitBreakerBean.java index 074beda9343..d564fdc3146 100644 --- a/microprofile/fault-tolerance/src/test/java/io/helidon/microprofile/faulttolerance/CircuitBreakerBean.java +++ b/microprofile/fault-tolerance/src/test/java/io/helidon/microprofile/faulttolerance/CircuitBreakerBean.java @@ -19,7 +19,6 @@ import java.util.concurrent.CompletableFuture; import java.util.concurrent.CompletionStage; import java.util.concurrent.CountDownLatch; -import java.util.concurrent.Future; import javax.enterprise.context.Dependent; @@ -95,7 +94,7 @@ public void openOnTimeouts() throws InterruptedException { failureRatio = 1.0, delay = 50000, failOn = UnitTestException.class) - public Future withBulkhead(CountDownLatch started) throws InterruptedException { + public CompletableFuture withBulkhead(CountDownLatch started) throws InterruptedException { started.countDown(); FaultToleranceTest.printStatus("CircuitBreakerBean::withBulkhead", "success"); Thread.sleep(3 * DELAY); diff --git a/microprofile/fault-tolerance/src/test/java/io/helidon/microprofile/faulttolerance/CircuitBreakerTest.java b/microprofile/fault-tolerance/src/test/java/io/helidon/microprofile/faulttolerance/CircuitBreakerTest.java index faa3f4f875f..6a597c8ca0a 100644 --- a/microprofile/fault-tolerance/src/test/java/io/helidon/microprofile/faulttolerance/CircuitBreakerTest.java +++ b/microprofile/fault-tolerance/src/test/java/io/helidon/microprofile/faulttolerance/CircuitBreakerTest.java @@ -16,10 +16,10 @@ package io.helidon.microprofile.faulttolerance; +import java.util.concurrent.CompletableFuture; import java.util.concurrent.CompletionStage; import java.util.concurrent.CountDownLatch; import java.util.concurrent.ExecutionException; -import java.util.concurrent.Future; import java.util.concurrent.TimeUnit; import org.eclipse.microprofile.faulttolerance.exceptions.BulkheadException; @@ -72,7 +72,7 @@ public void testOpenOnTimeouts() { CircuitBreakerBean bean = newBean(CircuitBreakerBean.class); // Iterate a few times to test circuit - for (int i = 0; i < bean.REQUEST_VOLUME_THRESHOLD - 1; i++) { + for (int i = 0; i < bean.REQUEST_VOLUME_THRESHOLD; i++) { assertThrows(TimeoutException.class, () -> bean.openOnTimeouts()); } @@ -132,7 +132,7 @@ public void testWithBulkhead() throws Exception { assertFalse(started.await(1000, TimeUnit.MILLISECONDS)); assertThrows(ExecutionException.class, () -> { - Future future = bean.withBulkhead(new CountDownLatch(1)); + CompletableFuture future = bean.withBulkhead(new CountDownLatch(1)); future.get(); }); } diff --git a/microprofile/fault-tolerance/src/test/java/io/helidon/microprofile/faulttolerance/FaultToleranceTest.java b/microprofile/fault-tolerance/src/test/java/io/helidon/microprofile/faulttolerance/FaultToleranceTest.java index afc81fdffa3..dfb64938520 100644 --- a/microprofile/fault-tolerance/src/test/java/io/helidon/microprofile/faulttolerance/FaultToleranceTest.java +++ b/microprofile/fault-tolerance/src/test/java/io/helidon/microprofile/faulttolerance/FaultToleranceTest.java @@ -99,12 +99,12 @@ static CompletableFuture[] getConcurrentCalls(Supplier supplier, int s } @SuppressWarnings("unchecked") - static Future[] getAsyncConcurrentCalls(Supplier> supplier, int size) { - return Stream.generate(supplier::get).limit(size).toArray(Future[]::new); + static CompletableFuture[] getAsyncConcurrentCalls(Supplier> supplier, int size) { + return Stream.generate(supplier::get).limit(size).toArray(CompletableFuture[]::new); } - static void waitFor(Future[] calls) { - for (Future c : calls) { + static void waitFor(CompletableFuture[] calls) { + for (CompletableFuture c : calls) { try { c.get(); } catch (Exception e) { diff --git a/microprofile/fault-tolerance/src/test/java/io/helidon/microprofile/faulttolerance/MetricsBean.java b/microprofile/fault-tolerance/src/test/java/io/helidon/microprofile/faulttolerance/MetricsBean.java index 944fe0b618e..67e312328c5 100644 --- a/microprofile/fault-tolerance/src/test/java/io/helidon/microprofile/faulttolerance/MetricsBean.java +++ b/microprofile/fault-tolerance/src/test/java/io/helidon/microprofile/faulttolerance/MetricsBean.java @@ -18,7 +18,6 @@ import java.time.temporal.ChronoUnit; import java.util.concurrent.CompletableFuture; -import java.util.concurrent.Future; import java.util.concurrent.atomic.AtomicInteger; import javax.enterprise.context.Dependent; @@ -170,7 +169,7 @@ public String onFailure() { @Asynchronous @Bulkhead(value = 3, waitingTaskQueue = 3) - public Future concurrent(long sleepMillis) { + public CompletableFuture concurrent(long sleepMillis) { FaultToleranceTest.printStatus("MetricsBean::concurrent()", "success"); try { assertThat(getGauge(this, @@ -185,7 +184,7 @@ public Future concurrent(long sleepMillis) { @Asynchronous @Bulkhead(value = 3, waitingTaskQueue = 3) - public Future concurrentAsync(long sleepMillis) { + public CompletableFuture concurrentAsync(long sleepMillis) { FaultToleranceTest.printStatus("MetricsBean::concurrentAsync()", "success"); try { assertThat((long) getGauge(this, "concurrentAsync", diff --git a/microprofile/fault-tolerance/src/test/java/io/helidon/microprofile/faulttolerance/MetricsTest.java b/microprofile/fault-tolerance/src/test/java/io/helidon/microprofile/faulttolerance/MetricsTest.java index 3f83f8e5982..531aa04b5fd 100644 --- a/microprofile/fault-tolerance/src/test/java/io/helidon/microprofile/faulttolerance/MetricsTest.java +++ b/microprofile/fault-tolerance/src/test/java/io/helidon/microprofile/faulttolerance/MetricsTest.java @@ -17,16 +17,15 @@ package io.helidon.microprofile.faulttolerance; import java.util.concurrent.CompletableFuture; -import java.util.concurrent.Future; import org.eclipse.microprofile.faulttolerance.exceptions.CircuitBreakerOpenException; import org.eclipse.microprofile.metrics.Metadata; import org.eclipse.microprofile.metrics.MetricRegistry; import org.eclipse.microprofile.metrics.MetricType; import org.eclipse.microprofile.metrics.MetricUnits; -import static org.hamcrest.Matchers.greaterThan; import org.junit.jupiter.api.Test; +import static org.hamcrest.Matchers.greaterThan; import static io.helidon.microprofile.faulttolerance.FaultToleranceMetrics.BREAKER_CALLS_FAILED_TOTAL; import static io.helidon.microprofile.faulttolerance.FaultToleranceMetrics.BREAKER_CALLS_PREVENTED_TOTAL; import static io.helidon.microprofile.faulttolerance.FaultToleranceMetrics.BREAKER_CALLS_SUCCEEDED_TOTAL; @@ -213,7 +212,7 @@ public void testTimeoutFailure() throws Exception { public void testBreakerTrip() throws Exception { MetricsBean bean = newBean(MetricsBean.class); - for (int i = 0; i < CircuitBreakerBean.REQUEST_VOLUME_THRESHOLD - 1; i++) { + for (int i = 0; i < CircuitBreakerBean.REQUEST_VOLUME_THRESHOLD ; i++) { assertThrows(RuntimeException.class, () -> bean.exerciseBreaker(false)); } assertThrows(CircuitBreakerOpenException.class, () -> bean.exerciseBreaker(false)); @@ -226,7 +225,7 @@ public void testBreakerTrip() throws Exception { is(0L)); assertThat(getCounter(bean, "exerciseBreaker", BREAKER_CALLS_FAILED_TOTAL, boolean.class), - is((long) CircuitBreakerBean.REQUEST_VOLUME_THRESHOLD - 1)); + is((long) CircuitBreakerBean.REQUEST_VOLUME_THRESHOLD)); assertThat(getCounter(bean, "exerciseBreaker", BREAKER_CALLS_PREVENTED_TOTAL, boolean.class), is(1L)); @@ -289,12 +288,12 @@ public void testBreakerExceptionCounters() throws Exception { is(2L)); assertThat(getCounter(bean, "exerciseBreakerException", BREAKER_OPENED_TOTAL, boolean.class), - is(1L)); + is(0L)); assertThrows(Exception.class, () -> bean.exerciseBreakerException(true)); // failure assertThat(getCounter(bean, "exerciseBreakerException", BREAKER_CALLS_SUCCEEDED_TOTAL, boolean.class), - is(0L)); + is(1L)); // Sleep longer than circuit breaker delay Thread.sleep(1500); @@ -311,7 +310,7 @@ public void testBreakerExceptionCounters() throws Exception { // Check counters after successful calls assertThat(getCounter(bean, "exerciseBreakerException", BREAKER_CALLS_SUCCEEDED_TOTAL, boolean.class), - is(2L)); + is(3L)); try { bean.exerciseBreakerException(true); // success @@ -322,7 +321,7 @@ public void testBreakerExceptionCounters() throws Exception { // Check counters after successful calls assertThat(getCounter(bean, "exerciseBreakerException", BREAKER_CALLS_SUCCEEDED_TOTAL, boolean.class), - is(3L)); + is(4L)); } @Test @@ -336,7 +335,7 @@ public void testFallbackMetrics() throws Exception { @Test public void testBulkheadMetrics() throws Exception { MetricsBean bean = newBean(MetricsBean.class); - Future[] calls = getAsyncConcurrentCalls( + CompletableFuture[] calls = getAsyncConcurrentCalls( () -> bean.concurrent(200), BulkheadBean.TOTAL_CALLS); waitFor(calls); assertThat(getGauge(bean, "concurrent", diff --git a/microprofile/fault-tolerance/src/test/java/io/helidon/microprofile/faulttolerance/RetryBean.java b/microprofile/fault-tolerance/src/test/java/io/helidon/microprofile/faulttolerance/RetryBean.java index 6101f59fc4d..e7ee1113d64 100644 --- a/microprofile/fault-tolerance/src/test/java/io/helidon/microprofile/faulttolerance/RetryBean.java +++ b/microprofile/fault-tolerance/src/test/java/io/helidon/microprofile/faulttolerance/RetryBean.java @@ -19,7 +19,6 @@ import java.io.IOException; import java.util.concurrent.CompletableFuture; import java.util.concurrent.CompletionStage; -import java.util.concurrent.Future; import java.util.concurrent.atomic.AtomicInteger; import javax.enterprise.context.Dependent; @@ -71,7 +70,7 @@ public String onFailure() { @Asynchronous @Retry(maxRetries = 2) - public Future retryAsync() { + public CompletableFuture retryAsync() { CompletableFuture future = new CompletableFuture<>(); if (invocations.incrementAndGet() <= 2) { printStatus("RetryBean::retryAsync()", "failure"); diff --git a/microprofile/fault-tolerance/src/test/java/io/helidon/microprofile/faulttolerance/RetryTest.java b/microprofile/fault-tolerance/src/test/java/io/helidon/microprofile/faulttolerance/RetryTest.java index e0b12e78a5d..8e649ca8074 100644 --- a/microprofile/fault-tolerance/src/test/java/io/helidon/microprofile/faulttolerance/RetryTest.java +++ b/microprofile/fault-tolerance/src/test/java/io/helidon/microprofile/faulttolerance/RetryTest.java @@ -17,8 +17,8 @@ package io.helidon.microprofile.faulttolerance; import java.io.IOException; +import java.util.concurrent.CompletableFuture; import java.util.concurrent.CompletionStage; -import java.util.concurrent.Future;; import java.util.stream.Stream; import org.junit.jupiter.params.ParameterizedTest; @@ -60,7 +60,7 @@ public void testRetryBeanFallback(RetryBean bean, String unused) { @ParameterizedTest(name = "{1}") @MethodSource("createBeans") public void testRetryAsync(RetryBean bean, String unused) throws Exception { - Future future = bean.retryAsync(); + CompletableFuture future = bean.retryAsync(); future.get(); assertThat(bean.getInvocations(), is(3)); } diff --git a/microprofile/fault-tolerance/src/test/java/io/helidon/microprofile/faulttolerance/TimeoutBean.java b/microprofile/fault-tolerance/src/test/java/io/helidon/microprofile/faulttolerance/TimeoutBean.java index 59be1568b64..7d3e004c839 100644 --- a/microprofile/fault-tolerance/src/test/java/io/helidon/microprofile/faulttolerance/TimeoutBean.java +++ b/microprofile/fault-tolerance/src/test/java/io/helidon/microprofile/faulttolerance/TimeoutBean.java @@ -18,7 +18,6 @@ import java.time.temporal.ChronoUnit; import java.util.concurrent.CompletableFuture; -import java.util.concurrent.Future; import java.util.concurrent.atomic.AtomicLong; import javax.enterprise.context.Dependent; @@ -46,7 +45,7 @@ public String forceTimeout() throws InterruptedException { @Asynchronous @Timeout(value=1000, unit=ChronoUnit.MILLIS) - public Future forceTimeoutAsync() throws InterruptedException { + public CompletableFuture forceTimeoutAsync() throws InterruptedException { FaultToleranceTest.printStatus("TimeoutBean::forceTimeoutAsync()", "failure"); Thread.sleep(1500); return CompletableFuture.completedFuture("failure"); diff --git a/microprofile/fault-tolerance/src/test/java/io/helidon/microprofile/faulttolerance/TimeoutTest.java b/microprofile/fault-tolerance/src/test/java/io/helidon/microprofile/faulttolerance/TimeoutTest.java index 2a9b85a100c..9d5242d5804 100644 --- a/microprofile/fault-tolerance/src/test/java/io/helidon/microprofile/faulttolerance/TimeoutTest.java +++ b/microprofile/fault-tolerance/src/test/java/io/helidon/microprofile/faulttolerance/TimeoutTest.java @@ -16,7 +16,7 @@ package io.helidon.microprofile.faulttolerance; -import java.util.concurrent.Future; +import java.util.concurrent.CompletableFuture; import org.eclipse.microprofile.faulttolerance.exceptions.TimeoutException; @@ -42,7 +42,7 @@ public void testForceTimeout() { @Test public void testForceTimeoutAsync() throws Exception { TimeoutBean bean = newBean(TimeoutBean.class); - Future future = bean.forceTimeoutAsync(); + CompletableFuture future = bean.forceTimeoutAsync(); assertCompleteExceptionally(future, TimeoutException.class); } From c37a25a460481c11a72f63efa6b040e76721ad99 Mon Sep 17 00:00:00 2001 From: Santiago Pericasgeertsen Date: Thu, 20 Aug 2020 14:48:58 -0400 Subject: [PATCH 34/65] Config property to disable caching of MethodState's. New logic to support task cancellations. Signed-off-by: Santiago Pericasgeertsen --- .../faulttolerance/InvokerAsyncException.java | 28 +++ .../faulttolerance/MethodInvoker.java | 212 +++++++++++++----- 2 files changed, 182 insertions(+), 58 deletions(-) create mode 100644 microprofile/fault-tolerance/src/main/java/io/helidon/microprofile/faulttolerance/InvokerAsyncException.java diff --git a/microprofile/fault-tolerance/src/main/java/io/helidon/microprofile/faulttolerance/InvokerAsyncException.java b/microprofile/fault-tolerance/src/main/java/io/helidon/microprofile/faulttolerance/InvokerAsyncException.java new file mode 100644 index 00000000000..f07d9138b16 --- /dev/null +++ b/microprofile/fault-tolerance/src/main/java/io/helidon/microprofile/faulttolerance/InvokerAsyncException.java @@ -0,0 +1,28 @@ +/* + * Copyright (c) 2020 Oracle and/or its affiliates. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package io.helidon.microprofile.faulttolerance; + +/** + * Wraps a {@code Throwable} thrown by an async call. It is necessary to + * distinguish it from exceptions thrown by the intercepted method. + */ +class InvokerAsyncException extends Exception { + + InvokerAsyncException(Throwable cause) { + super(cause); + } +} diff --git a/microprofile/fault-tolerance/src/main/java/io/helidon/microprofile/faulttolerance/MethodInvoker.java b/microprofile/fault-tolerance/src/main/java/io/helidon/microprofile/faulttolerance/MethodInvoker.java index 691b0905910..ec5b521cdf0 100644 --- a/microprofile/fault-tolerance/src/main/java/io/helidon/microprofile/faulttolerance/MethodInvoker.java +++ b/microprofile/fault-tolerance/src/main/java/io/helidon/microprofile/faulttolerance/MethodInvoker.java @@ -19,6 +19,7 @@ import java.time.Duration; import java.util.Objects; import java.util.concurrent.Callable; +import java.util.concurrent.CancellationException; import java.util.concurrent.CompletableFuture; import java.util.concurrent.CompletionStage; import java.util.concurrent.ConcurrentHashMap; @@ -27,6 +28,8 @@ import java.util.concurrent.Future; import java.util.concurrent.ScheduledExecutorService; import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicBoolean; +import java.util.function.Function; import java.util.function.Supplier; import java.util.logging.Logger; @@ -47,6 +50,7 @@ import io.helidon.faulttolerance.Retry; import io.helidon.faulttolerance.Timeout; +import org.eclipse.microprofile.config.ConfigProvider; import org.eclipse.microprofile.faulttolerance.exceptions.CircuitBreakerOpenException; import org.eclipse.microprofile.faulttolerance.exceptions.TimeoutException; import org.eclipse.microprofile.metrics.Counter; @@ -89,6 +93,11 @@ public class MethodInvoker implements FtSupplier { private static final Logger LOGGER = Logger.getLogger(MethodInvoker.class.getName()); + /** + * Config key to disable method state caching in FT. + */ + private static final String DISABLE_CACHING = "fault-tolerance.disableCaching"; + /** * Waiting millis to convert {@code Future} into {@code CompletableFuture}. */ @@ -149,6 +158,17 @@ public class MethodInvoker implements FtSupplier { */ private RequestContextController requestController; + /** + * Record thread interruption request for later use. + */ + private final AtomicBoolean mayInterruptIfRunning = new AtomicBoolean(false); + + /** + * Async thread in used by this invocation. May be {@code null}. We use this + * reference for thread interruptions. + */ + private Thread asyncInterruptThread; + private static class MethodState { private FtHandlerTyped handler; private Retry retry; @@ -167,6 +187,67 @@ private static class MethodState { */ private final MethodState methodState; + /** + * Future returned by this method invoker. Some special logic to handle async + * cancellations and methods returning {@code Future}. + * + * @param result type of future + */ + @SuppressWarnings("unchecked") + class InvokerCompletableFuture extends CompletableFuture { + + /** + * If method returns {@code Future}, we let that value pass through + * without further processing. See Section 5.2.1 of spec. + * + * @return value from this future + * @throws ExecutionException if this future completed exceptionally + * @throws InterruptedException if the current thread was interrupted + */ + @Override + public T get() throws InterruptedException, ExecutionException { + T value = super.get(); + if (method.getReturnType() == Future.class) { + return ((Future) value).get(); + } + return value; + } + + /** + * If method returns {@code Future}, we let that value pass through + * without further processing. See Section 5.2.1 of spec. + * + * @param timeout the timeout + * @param unit the timeout unit + * @return value from this future + * @throws CancellationException if this future was cancelled + * @throws ExecutionException if this future completed exceptionally + * @throws InterruptedException if the current thread was interrupted + */ + @Override + public T get(long timeout, TimeUnit unit) throws InterruptedException, + ExecutionException, java.util.concurrent.TimeoutException { + T value = super.get(); + if (method.getReturnType() == Future.class) { + return ((Future) value).get(timeout, unit); + } + return value; + } + + /** + * Overridden to record {@code mayInterruptIfRunning} flag. This flag + * is not currently not propagated over a chain of {@code Single}'s. + * + * @param mayInterruptIfRunning Interrupt flag. + * @@return {@code true} if this task is now cancelled. + */ + @Override + public boolean cancel(boolean mayInterruptIfRunning) { + MethodInvoker.this.mayInterruptIfRunning.set(mayInterruptIfRunning); + return super.cancel(mayInterruptIfRunning); + } + } + /** * Constructor. * @@ -179,8 +260,8 @@ public MethodInvoker(InvocationContext context, MethodIntrospector introspector) this.method = context.getMethod(); this.helidonContext = Contexts.context().orElseGet(Context::create); - // Initialize method state and create handler for it - this.methodState = FT_HANDLERS.computeIfAbsent(method, method -> { + // Initialize method state for this method + Function createState = method -> { MethodState methodState = new MethodState(); methodState.lastBreakerState = State.CLOSED; if (introspector.hasCircuitBreaker()) { @@ -191,7 +272,12 @@ public MethodInvoker(InvocationContext context, MethodIntrospector introspector) } methodState.handler = createMethodHandler(methodState); return methodState; - }); + }; + boolean disableCaching = ConfigProvider.getConfig() + .getOptionalValue(DISABLE_CACHING, Boolean.class).orElse(false); + this.methodState = disableCaching ? createState.apply(method) + : FT_HANDLERS.computeIfAbsent(method, createState); + LOGGER.fine(() -> "Caching of MethodState objects is " + (disableCaching ? "disabled" : "enabled")); // Gather information about current request scope if active try { @@ -267,14 +353,26 @@ public Object get() throws Throwable { } }; + // Update metrics before calling method updateMetricsBefore(); if (introspector.isAsynchronous()) { - CompletableFuture asyncFuture = supplier.get() - .toStage(true).toCompletableFuture(); - CompletableFuture resultFuture = new CompletableFuture<>(); + // Obtain single from supplier + Single single = supplier.get(); + + // Convert single to CompletableFuture + CompletableFuture asyncFuture = single.toStage(true).toCompletableFuture(); + + // Create CompletableFuture that is returned to caller + CompletableFuture resultFuture = new InvokerCompletableFuture<>(); + + // Update resultFuture based on outcome of asyncFuture asyncFuture.whenComplete((result, throwable) -> { if (throwable != null) { + if (throwable instanceof CancellationException) { + single.cancel(); + return; + } Throwable cause; if (throwable instanceof ExecutionException) { cause = map(throwable.getCause()); @@ -288,13 +386,23 @@ public Object get() throws Throwable { resultFuture.complete(result); } }); + + // Propagate cancellation of resultFuture to asyncFuture + resultFuture.whenComplete((result, throwable) -> { + if (throwable instanceof CancellationException) { + asyncFuture.cancel(true); + } + }); return resultFuture; } else { Object result = null; Throwable cause = null; try { - CompletableFuture future = supplier.get() - .toStage(true).toCompletableFuture(); + // Obtain single from supplier and map to CompletableFuture to handle void methods + Single single = supplier.get(); + CompletableFuture future = single.toStage(true).toCompletableFuture(); + + // Synchronously way for result result = future.get(); } catch (ExecutionException e) { cause = map(e.getCause()); @@ -441,37 +549,51 @@ Supplier> toCompletionStageSupplier(FtSupplier // Invoke supplier in a new thread Single single = Async.create().invoke(() -> { try { - return wrappedSupplier.get(); // Future, CompletableFuture or CompletionStage + asyncInterruptThread = Thread.currentThread(); + return wrappedSupplier.get(); } catch (Throwable t) { - return CompletableFuture.failedFuture(t); + return new InvokerAsyncException(t); // wraps Throwable } }); - // The result must be Future, CompletableFuture or CompletionStage - single.whenComplete((result, throwable) -> { - if (throwable != null) { - resultFuture.completeExceptionally(throwable); - } else { - try { - if (result instanceof CompletionStage) { // also CompletableFuture - CompletionStage cs = (CompletionStage) result; - cs.whenComplete((o, t) -> { - if (t != null) { - resultFuture.completeExceptionally(t); - } else { - resultFuture.complete(o); - } - }); - } else if (result instanceof Future){ - awaitFuture((Future) result, resultFuture); - } else { - throw new InternalError("Return type validation failed for method " + method); - } - } catch (Throwable t) { - resultFuture.completeExceptionally(t); + // Handle async cancellations + resultFuture.whenComplete((result, throwable) -> { + if (throwable instanceof CancellationException) { + single.cancel(); // will not interrupt by default + + // If interrupt was requested, do it manually here + if (mayInterruptIfRunning.get() && asyncInterruptThread != null) { + asyncInterruptThread.interrupt(); + asyncInterruptThread = null; } } }); + + // The result must be Future, {Completable}Future or InvokerAsyncException + single.thenAccept(result -> { + try { + // Handle exceptions thrown by an async method + if (result instanceof InvokerAsyncException) { + resultFuture.completeExceptionally(((Exception) result).getCause()); + } else if (method.getReturnType() == Future.class) { + // If method returns Future, pass it without further processing + resultFuture.complete(result); + } else if (result instanceof CompletionStage) { // also CompletableFuture + CompletionStage cs = (CompletionStage) result; + cs.whenComplete((o, t) -> { + if (t != null) { + resultFuture.completeExceptionally(t); + } else { + resultFuture.complete(o); + } + }); + } else { + throw new InternalError("Return type validation failed for method " + method); + } + } catch (Throwable t) { + resultFuture.completeExceptionally(t); + } + }); } else { try { resultFuture.complete(supplier.get()); @@ -617,30 +739,4 @@ private static boolean updateCounter(Method method, String name, long newValue) } return false; } - - /** - * Waits for a Future to produce an outcome and updates a CompletableFuture based on - * it. Effectively converts a Future into a CompletableFuture. - * - * @param future The future. - * @param completableFuture The completable future. - */ - private static void awaitFuture(Future future, CompletableFuture completableFuture) { - if (future.isDone()) { - try { - completableFuture.complete(future.get()); - } catch (InterruptedException e) { - completableFuture.completeExceptionally(e); - } catch (ExecutionException e) { - completableFuture.completeExceptionally(e.getCause()); - } - return; - } - if (future.isCancelled()) { - completableFuture.cancel(true); - } else { - EXECUTOR_SERVICE.schedule(() -> awaitFuture(future, completableFuture), - AWAIT_FUTURE_MILLIS, TimeUnit.MILLISECONDS); - } - } } From 6b5de792582c102038df5cdf9a44d509fac6a78d Mon Sep 17 00:00:00 2001 From: Santiago Pericasgeertsen Date: Wed, 26 Aug 2020 10:47:18 -0400 Subject: [PATCH 35/65] Handle multiple applications with different CCL as required by the TCKs. Signed-off-by: Santiago Pericasgeertsen --- .../faulttolerance/MethodInvoker.java | 32 ++--- .../tck/tck-fault-tolerance/failures.txt | 133 ++++++++++++++++++ ...{tck-application.yaml => application.yaml} | 15 +- .../src/test/tck-suite.xml | 46 ++++++ 4 files changed, 195 insertions(+), 31 deletions(-) create mode 100644 microprofile/tests/tck/tck-fault-tolerance/failures.txt rename microprofile/tests/tck/tck-fault-tolerance/src/test/resources/{tck-application.yaml => application.yaml} (63%) diff --git a/microprofile/fault-tolerance/src/main/java/io/helidon/microprofile/faulttolerance/MethodInvoker.java b/microprofile/fault-tolerance/src/main/java/io/helidon/microprofile/faulttolerance/MethodInvoker.java index ec5b521cdf0..10cdb4bbf48 100644 --- a/microprofile/fault-tolerance/src/main/java/io/helidon/microprofile/faulttolerance/MethodInvoker.java +++ b/microprofile/fault-tolerance/src/main/java/io/helidon/microprofile/faulttolerance/MethodInvoker.java @@ -93,16 +93,6 @@ public class MethodInvoker implements FtSupplier { private static final Logger LOGGER = Logger.getLogger(MethodInvoker.class.getName()); - /** - * Config key to disable method state caching in FT. - */ - private static final String DISABLE_CACHING = "fault-tolerance.disableCaching"; - - /** - * Waiting millis to convert {@code Future} into {@code CompletableFuture}. - */ - private static final long AWAIT_FUTURE_MILLIS = 5L; - /** * The method being intercepted. */ @@ -119,9 +109,12 @@ public class MethodInvoker implements FtSupplier { private final MethodIntrospector introspector; /** - * Map of methods to their internal state. + * Map of a class loader and a method into a method state. Class loaders are needed + * when running TCKs where each test is considered a different application (with + * potentially different configurations). */ - private static final ConcurrentHashMap FT_HANDLERS = new ConcurrentHashMap<>(); + private static final ConcurrentHashMap> + FT_HANDLERS = new ConcurrentHashMap<>(); /** * Executor service shared by instances of this class. @@ -260,8 +253,12 @@ public MethodInvoker(InvocationContext context, MethodIntrospector introspector) this.method = context.getMethod(); this.helidonContext = Contexts.context().orElseGet(Context::create); - // Initialize method state for this method - Function createState = method -> { + // Create method state using CCL to support multiples apps (TCKs) + ClassLoader ccl = Thread.currentThread().getContextClassLoader(); + Objects.requireNonNull(ccl); + ConcurrentHashMap methodStates = + FT_HANDLERS.computeIfAbsent(ccl, cl -> new ConcurrentHashMap<>()); + this.methodState = methodStates.computeIfAbsent(method, method -> { MethodState methodState = new MethodState(); methodState.lastBreakerState = State.CLOSED; if (introspector.hasCircuitBreaker()) { @@ -272,12 +269,7 @@ public MethodInvoker(InvocationContext context, MethodIntrospector introspector) } methodState.handler = createMethodHandler(methodState); return methodState; - }; - boolean disableCaching = ConfigProvider.getConfig() - .getOptionalValue(DISABLE_CACHING, Boolean.class).orElse(false); - this.methodState = disableCaching ? createState.apply(method) - : FT_HANDLERS.computeIfAbsent(method, createState); - LOGGER.fine(() -> "Caching of MethodState objects is " + (disableCaching ? "disabled" : "enabled")); + }); // Gather information about current request scope if active try { diff --git a/microprofile/tests/tck/tck-fault-tolerance/failures.txt b/microprofile/tests/tck/tck-fault-tolerance/failures.txt new file mode 100644 index 00000000000..c9ea176f065 --- /dev/null +++ b/microprofile/tests/tck/tck-fault-tolerance/failures.txt @@ -0,0 +1,133 @@ + Failed tests: + AsyncCancellationTest>Arquillian.run:138->testCancelledButRemainsInBulkhead:155 Task started unexpectedly + AsyncCancellationTest>Arquillian.run:138->testCancelledWhileQueued:176 Task started unexpectedly + CircuitBreakerBulkheadTest>Arquillian.run:138->testCircuitBreaker:229 Task started unexpectedly + CircuitBreakerBulkheadTest>Arquillian.run:138->testCircuitBreakerAroundBulkheadAsync:113 Task started unexpectedly + CircuitBreakerBulkheadTest>Arquillian.run:138->testCircuitBreakerAroundBulkheadSync:173 Task did not finish within 2 seconds + CircuitBreakerExceptionHierarchyTest>Arquillian.run:138->serviceAthrowsE0:86 expected [OPEN] but found [CLOSED] + CircuitBreakerExceptionHierarchyTest>Arquillian.run:138->serviceAthrowsE0S:116 expected [OPEN] but found [CLOSED] + CircuitBreakerExceptionHierarchyTest>Arquillian.run:138->serviceBthrowsException:136 expected [OPEN] but found [CLOSED] + CircuitBreakerExceptionHierarchyTest>Arquillian.run:138->serviceBthrowsRuntimeException:178 expected [OPEN] but found [CLOSED] + CircuitBreakerInitialSuccessTest>Arquillian.run:138->testCircuitInitialSuccessDefaultSuccessThreshold:97 serviceA should throw an Exception in testCircuitDefaultSuccessThreshold on iteration 5 + CircuitBreakerLateSuccessTest>Arquillian.run:138->testCircuitLateSuccessDefaultSuccessThreshold:98 serviceA should throw an Exception in testCircuitLateSuccessDefaultSuccessThreshold on iteration 5 + CircuitBreakerRetryTest>Arquillian.run:138->testCircuitOpenWithMultiTimeouts:275 serviceC should retry or throw a CircuitBreakerOpenException in testCircuitOpenWithMultiTimeouts on iteration 1, caught exception: org.eclipse.microprofile.faulttolerance.exceptions.TimeoutException: Method interrupted by timeout + CircuitBreakerRetryTest>Arquillian.run:138->testCircuitOpenWithMultiTimeoutsAsync:480 Thrown exception is the wrong type + Expected: an instance of org.eclipse.microprofile.faulttolerance.exceptions.CircuitBreakerOpenException + but: is a org.eclipse.microprofile.faulttolerance.exceptions.TimeoutException + CircuitBreakerRetryTest>Arquillian.run:138->testNoRetriesIfAbortOnAsync:589 Execution exception not thrown from Future + CircuitBreakerRetryTest>Arquillian.run:138->testNoRetriesIfNotRetryOnAsync:563 Execution exception not thrown from Future + CircuitBreakerRetryTest>Arquillian.run:138->testRetriesSucceedWhenCircuitCloses:311 Call was successful but did not take the expected time + Expected: Duration within of + but: which is from + CircuitBreakerRetryTest>Arquillian.run:138->testRetriesSucceedWhenCircuitClosesAsync:538 Call was successful but did not take the expected time + Expected: Duration within of + but: which is from + CircuitBreakerTest>Arquillian.run:138->testCircuitClosedThenOpen:122 The number of executions should be 4 expected [4] but found [7] + CircuitBreakerTest>Arquillian.run:138->testCircuitDefaultSuccessThreshold:222 serviceA should throw an Exception in testCircuitDefaultSuccessThreshold on iteration 5 + CircuitBreakerTest>Arquillian.run:138->testCircuitHighSuccessThreshold:309 serviceA should not have thrown a RuntimeException in testCircuitHighSuccessThreshold on iteration 7 + CircuitBreakerTest>Arquillian.run:138->testClassLevelCircuitBase:372 The number of executions should be 4 expected [4] but found [7] + CircuitBreakerTest>Arquillian.run:138->testClassLevelCircuitOverride:411 serviceC should not throw a RuntimeException on iteration 3 + CircuitBreakerTest>Arquillian.run:138->testRollingWindowCircuitOpen:528 The CircuitBreaker exception in testRollingWindowCircuitOpen was not thrown correctly. expected [true] but found [false] + CircuitBreakerTest>Arquillian.run:138->testRollingWindowCircuitOpen2:576 The CircuitBreaker exception in testRollingWindowCircuitOpen2 was not thrown correctly. expected [true] but found [false] + CircuitBreakerTimeoutTest>Arquillian.run:138->testTimeout:74 Unexpected exception thrown + RetryTimeoutTest>Arquillian.run:138->testRetryWithAbortOn:151 The execution count should be 1 (no retries) expected [1] but found [2] + TimeoutUninterruptableTest>Arquillian.run:138->testTimeoutAsyncBulkhead:185 Execution count after second call expected [1] but found [2] + BulkheadAsynchRetryTest>Arquillian.run:138->testBulkheadClassAsynchronous55RetryOverload:282 Failure count should be non-zero + Expected: a value greater than <0> + but: <0> was equal to <0> + BulkheadAsynchRetryTest>Arquillian.run:138->testBulkheadClassAsynchronousPassiveRetry55:161 Bulkhead appears to have been breeched 10 workers, expected 5. expected [true] but found [false] + BulkheadAsynchRetryTest>Arquillian.run:138->testBulkheadMethodAsynchronous55RetryOverload:251 Failure count should be non-zero + Expected: a value greater than <0> + but: <0> was equal to <0> + BulkheadAsynchRetryTest>Arquillian.run:138->testBulkheadMethodAsynchronousRetry55:177 Bulkhead appears to have been breeched 10 workers, expected 5. expected [true] but found [false] + BulkheadAsynchRetryTest>Arquillian.run:138->testBulkheadMethodAsynchronousRetry55Trip:208 Task 0 is running. expected [false] but found [true] + BulkheadAsynchRetryTest>Arquillian.run:138->testBulkheadPassiveRetryMethodAsynchronous55:295 Bulkhead appears to have been breeched 10 workers, expected 5. expected [true] but found [false] + BulkheadAsynchRetryTest>Arquillian.run:138->testBulkheadQueReplacesDueToClassRetryFailures:346 Bulkhead appears to have been breeched 10 workers, expected 5. expected [true] but found [false] + BulkheadAsynchRetryTest>Arquillian.run:138->testBulkheadRetryClassAsynchronous55:310 Bulkhead appears to have been breeched 10 workers, expected 5. expected [true] but found [false] + BulkheadAsynchRetryTest>Arquillian.run:138->testNoRetriesWithAbortOn:477 Task started unexpectedly + BulkheadAsynchRetryTest>Arquillian.run:138->testNoRetriesWithoutRetryOn:451 Task started unexpectedly + BulkheadAsynchRetryTest>Arquillian.run:138->testRetriesJoinBackOfQueue:411 Task started unexpectedly + BulkheadAsynchRetryTest>Arquillian.run:138->testRetriesReenterBulkhead:381 Task started unexpectedly + BulkheadAsynchTest>Arquillian.run:138->testBulkheadClassAsynchronous3:162 Bulkhead appears to have been breeched 10 workers, expected 3. expected [true] but found [false] + BulkheadAsynchTest>Arquillian.run:138->testBulkheadCompletionStage:290 Timed out waiting for future to throw BulkheadException + BulkheadAsynchTest>Arquillian.run:138->testBulkheadExceptionThrownWhenQueueFullAsync:260 When a task is rejected from the bulkhead, the returned future should report as done expected [true] but found [false] + BulkheadAsynchTest>Arquillian.run:138->testBulkheadMethodAsynchronous3:174 Bulkhead appears to have been breeched 10 workers, expected 3. expected [true] but found [false] + BulkheadSynchConfigTest>Arquillian.run:138->testBulkheadClassSemaphore3:83 Bulkhead appears to have been breeched 10 workers, expected 5. expected [true] but found [false] + BulkheadSynchRetryTest.afterClass:139 Unable to clean up all tasks + org.eclipse.microprofile.fault.tolerance.tck.bulkhead.BulkheadSynchRetryTest.afterMethod(org.eclipse.microprofile.fault.tolerance.tck.bulkhead.BulkheadSynchRetryTest) + Run 1: BulkheadSynchRetryTest.afterMethod:134 Unable to clean up all tasks + Run 2: BulkheadSynchRetryTest.afterMethod:134 Unable to clean up all tasks + Run 3: BulkheadSynchRetryTest.afterMethod:134 Unable to clean up all tasks + + BulkheadSynchRetryTest>Arquillian.run:138->testBulkheadMethodSynchronousRetry55:231 Bulkhead appears to have been breeched 10 workers, expected 5. expected [true] but found [false] + BulkheadSynchRetryTest>Arquillian.run:138->testBulkheadRetryClassSynchronous55:261 Bulkhead appears to have been breeched 10 workers, expected 5. expected [true] but found [false] + BulkheadSynchRetryTest>Arquillian.run:138->testIgnoreWaitingTaskQueueBulkhead:290 Bulkhead appears to have been breeched 10 workers, expected 5. expected [true] but found [false] + BulkheadSynchRetryTest>Arquillian.run:138->testNoRetriesBulkhead:275 Bulkhead appears to have been breeched 10 workers, expected 5. expected [true] but found [false] + BulkheadSynchRetryTest>Arquillian.run:138->testNoRetriesWithAbortOn:358 Timed out waiting for future to throw BulkheadException + BulkheadSynchRetryTest>Arquillian.run:138->testNoRetriesWithoutRetryOn:338 Timed out waiting for future to throw BulkheadException + BulkheadSynchRetryTest>Arquillian.run:138->testRetriesReenterBulkhead:317 Unexpected exception thrown from Future + BulkheadSynchTest>Arquillian.run:138->testBulkheadClassSemaphore3:114 Bulkhead appears to have been breeched 10 workers, expected 3. expected [true] but found [false] + BulkheadSynchTest>Arquillian.run:138->testBulkheadExceptionThrownWhenQueueFullSemaphore:199 Unable to clean up all tasks + BulkheadSynchTest>Arquillian.run:138->testBulkheadMethodSemaphore3:152 Bulkhead appears to have been breeched 10 workers, expected 3. expected [true] but found [false] + CircuitBreakerConfigGlobalTest>Arquillian.run:138->testCircuitDefaultSuccessThreshold:75 serviceA should throw an Exception in testCircuitDefaultSuccessThreshold on iteration 5 + CircuitBreakerConfigOnMethodTest>Arquillian.run:138->testCircuitDefaultSuccessThreshold:77 serviceA should throw an Exception in testCircuitDefaultSuccessThreshold on iteration 5 + DisableAnnotationGloballyEnableOnClassTest>Arquillian.run:138->testBulkhead:174 Expected BulkheadException to be thrown, but nothing was thrown + DisableAnnotationGloballyEnableOnClassTest>Arquillian.run:138->testCircuitBreaker:125 Expected CircuitBreakerOpenException to be thrown, but TestException was thrown + DisableAnnotationGloballyEnableOnMethodTest>Arquillian.run:138->testBulkhead:177 Expected BulkheadException to be thrown, but nothing was thrown + DisableAnnotationGloballyEnableOnMethodTest>Arquillian.run:138->testCircuitBreaker:126 Expected CircuitBreakerOpenException to be thrown, but TestException was thrown + DisableAnnotationOnClassEnableOnMethodTest>Arquillian.run:138->testBulkhead:175 Expected BulkheadException to be thrown, but nothing was thrown + DisableAnnotationOnClassEnableOnMethodTest>Arquillian.run:138->testCircuitBreaker:126 Expected CircuitBreakerOpenException to be thrown, but TestException was thrown + DisableFTEnableGloballyTest>Arquillian.run:138->testBulkhead:169 Expected BulkheadException to be thrown, but nothing was thrown + DisableFTEnableGloballyTest>Arquillian.run:138->testCircuitBreaker:120 Expected CircuitBreakerOpenException to be thrown, but TestException was thrown + DisableFTEnableOnClassTest>Arquillian.run:138->testBulkhead:169 Expected BulkheadException to be thrown, but nothing was thrown + DisableFTEnableOnClassTest>Arquillian.run:138->testCircuitBreaker:120 Expected CircuitBreakerOpenException to be thrown, but TestException was thrown + DisableFTEnableOnMethodTest>Arquillian.run:138->testBulkhead:156 Expected BulkheadException to be thrown, but nothing was thrown + DisableFTEnableOnMethodTest>Arquillian.run:138->testCircuitBreaker:107 Expected CircuitBreakerOpenException to be thrown, but TestException was thrown + FallbackMethodOutOfPackageTest>Arquillian.arquillianBeforeClass:96 ? Runtime E... + FallbackMethodSubclassTest>Arquillian.arquillianBeforeClass:96 ? Runtime Expec... + FallbackMethodSuperclassPrivateTest>Arquillian.arquillianBeforeClass:96 ? Runtime + FallbackMethodWildcardNegativeTest>Arquillian.arquillianBeforeClass:96 ? Runtime + IncompatibleFallbackMethodTest>Arquillian.arquillianBeforeClass:96 ? Runtime E... + IncompatibleFallbackMethodWithArgsTest>Arquillian.arquillianBeforeClass:96 ? Runtime + IncompatibleFallbackPolicies>Arquillian.arquillianBeforeClass:96 ? Runtime Exp... + IncompatibleFallbackTest>Arquillian.arquillianBeforeClass:96 ? Runtime Expecte... + InvalidAsynchronousClassTest>Arquillian.arquillianBeforeClass:96 ? Runtime Exp... + InvalidAsynchronousMethodTest>Arquillian.arquillianBeforeClass:96 ? Runtime Ex... + InvalidBulkheadAsynchQueueTest>Arquillian.arquillianBeforeClass:96 ? Runtime E... + InvalidBulkheadValueTest>Arquillian.arquillianBeforeClass:96 ? Runtime Expecte... + InvalidCircuitBreakerDelayTest>Arquillian.arquillianBeforeClass:96 ? Runtime E... + InvalidCircuitBreakerFailureRatioNegTest>Arquillian.arquillianBeforeClass:96 ? Runtime + InvalidCircuitBreakerFailureRatioPosTest>Arquillian.arquillianBeforeClass:96 ? Runtime + InvalidCircuitBreakerFailureReqVol0Test>Arquillian.arquillianBeforeClass:96 ? Runtime + InvalidCircuitBreakerFailureReqVolNegTest>Arquillian.arquillianBeforeClass:96 ? Runtime + InvalidCircuitBreakerFailureSuccess0Test>Arquillian.arquillianBeforeClass:96 ? Runtime + InvalidCircuitBreakerFailureSuccessNegTest>Arquillian.arquillianBeforeClass:96 ? Runtime + InvalidRetryDelayDurationTest>Arquillian.arquillianBeforeClass:96 ? Runtime Ex... + InvalidRetryDelayTest>Arquillian.arquillianBeforeClass:96 ? Runtime Expected e... + InvalidRetryJitterTest>Arquillian.arquillianBeforeClass:96 ? Runtime Expected ... + InvalidRetryMaxRetriesTest>Arquillian.arquillianBeforeClass:96 ? Runtime Expec... + InvalidTimeoutValueTest>Arquillian.arquillianBeforeClass:96 ? Runtime Expected... + BulkheadMetricTest>Arquillian.run:138->bulkheadMetricAsyncTest:219 Unexpected exception thrown from Future + BulkheadMetricTest>Arquillian.run:138->bulkheadMetricHistogramTest:173 Unexpected exception thrown from Future + BulkheadMetricTest>Arquillian.run:138->bulkheadMetricRejectionTest:142 Unexpected exception thrown from Future + BulkheadMetricTest>Arquillian.run:138->bulkheadMetricTest:110 concurrent executions + Expected: is <2L> + but: was <0L> + CircuitBreakerMetricTest>Arquillian.run:138->testCircuitBreakerMetric:116 circuit breaker times opened + Expected: is <1L> + but: was <0L> + ClassLevelMetricTest>Arquillian.run:138->testRetryMetricUnsuccessful:98 calls succeeded without retry + Expected: is <0L> + but: was <1L> + FallbackMetricTest>Arquillian.run:138->fallbackMetricHandlerTest:93 failed invocations + Expected: is <0L> + but: was <-1L> + FallbackMetricTest>Arquillian.run:138->fallbackMetricMethodTest:67 failed invocations + Expected: is <0L> + but: was <-1L> + RetryMetricTest>Arquillian.run:138->testRetryMetricUnsuccessful:92 calls succeeded without retry + Expected: is <0L> + but: was <1L> + + Tests run: 457, Failures: 103, Errors: 0, Skipped: 48 + diff --git a/microprofile/tests/tck/tck-fault-tolerance/src/test/resources/tck-application.yaml b/microprofile/tests/tck/tck-fault-tolerance/src/test/resources/application.yaml similarity index 63% rename from microprofile/tests/tck/tck-fault-tolerance/src/test/resources/tck-application.yaml rename to microprofile/tests/tck/tck-fault-tolerance/src/test/resources/application.yaml index 51ecd8f4bb2..5bad4af9fec 100644 --- a/microprofile/tests/tck/tck-fault-tolerance/src/test/resources/tck-application.yaml +++ b/microprofile/tests/tck/tck-fault-tolerance/src/test/resources/application.yaml @@ -1,5 +1,5 @@ # -# Copyright (c) 2019 Oracle and/or its affiliates. All rights reserved. +# Copyright (c) 2020 Oracle and/or its affiliates. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -15,14 +15,7 @@ # # -# Adjusting some timeouts for Helidon +# Disable caching since multiple apps are executed in same Java VM # -org: - eclipse: - microprofile: - fault: - tolerance: - tck: - retry: - clientserver: - RetryClassLevelClientForMaxRetries/serviceB/Retry/maxDuration: 2000 \ No newline at end of file +fault-tolerance: + disableCaching: false diff --git a/microprofile/tests/tck/tck-fault-tolerance/src/test/tck-suite.xml b/microprofile/tests/tck/tck-fault-tolerance/src/test/tck-suite.xml index 51a9efd7f6b..90203fab07b 100644 --- a/microprofile/tests/tck/tck-fault-tolerance/src/test/tck-suite.xml +++ b/microprofile/tests/tck/tck-fault-tolerance/src/test/tck-suite.xml @@ -27,5 +27,51 @@ + + + + From 164cc77385f7062b8b878be2c59fa9a7e24b41bf Mon Sep 17 00:00:00 2001 From: Santiago Pericasgeertsen Date: Mon, 31 Aug 2020 14:36:49 -0400 Subject: [PATCH 36/65] Map exception types in applyOn and failOn from MP to SE. This is necessary for proper semantics of breakers, retry and bulkheads, and their combinations. Signed-off-by: Santiago Pericasgeertsen --- .../faulttolerance/MethodInvoker.java | 16 ++++----- .../faulttolerance/ThrowableMapper.java | 35 +++++++++++++++++++ 2 files changed, 42 insertions(+), 9 deletions(-) diff --git a/microprofile/fault-tolerance/src/main/java/io/helidon/microprofile/faulttolerance/MethodInvoker.java b/microprofile/fault-tolerance/src/main/java/io/helidon/microprofile/faulttolerance/MethodInvoker.java index 10cdb4bbf48..9673394d0ef 100644 --- a/microprofile/fault-tolerance/src/main/java/io/helidon/microprofile/faulttolerance/MethodInvoker.java +++ b/microprofile/fault-tolerance/src/main/java/io/helidon/microprofile/faulttolerance/MethodInvoker.java @@ -29,7 +29,6 @@ import java.util.concurrent.ScheduledExecutorService; import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicBoolean; -import java.util.function.Function; import java.util.function.Supplier; import java.util.logging.Logger; @@ -50,13 +49,13 @@ import io.helidon.faulttolerance.Retry; import io.helidon.faulttolerance.Timeout; -import org.eclipse.microprofile.config.ConfigProvider; import org.eclipse.microprofile.faulttolerance.exceptions.CircuitBreakerOpenException; import org.eclipse.microprofile.faulttolerance.exceptions.TimeoutException; import org.eclipse.microprofile.metrics.Counter; import org.glassfish.jersey.process.internal.RequestContext; import org.glassfish.jersey.process.internal.RequestScope; +import static io.helidon.microprofile.faulttolerance.ThrowableMapper.mapTypes; import static io.helidon.microprofile.faulttolerance.FaultToleranceExtension.isFaultToleranceMetricsEnabled; import static io.helidon.microprofile.faulttolerance.FaultToleranceMetrics.BREAKER_CALLS_FAILED_TOTAL; import static io.helidon.microprofile.faulttolerance.FaultToleranceMetrics.BREAKER_CALLS_PREVENTED_TOTAL; @@ -460,8 +459,8 @@ private FtHandlerTyped createMethodHandler(MethodState methodState) { .successThreshold(introspector.getCircuitBreaker().successThreshold()) .errorRatio((int) (introspector.getCircuitBreaker().failureRatio() * 100)) .volume(introspector.getCircuitBreaker().requestVolumeThreshold()) - .applyOn(introspector.getCircuitBreaker().failOn()) - .skipOn(introspector.getCircuitBreaker().skipOn()) + .applyOn(mapTypes(introspector.getCircuitBreaker().failOn())) + .skipOn(mapTypes(introspector.getCircuitBreaker().skipOn())) .build(); builder.addBreaker(methodState.breaker); } @@ -473,7 +472,6 @@ private FtHandlerTyped createMethodHandler(MethodState methodState) { .queueLength(introspector.getBulkhead().waitingTaskQueue()) .async(introspector.isAsynchronous()) .build(); - builder.addBulkhead(methodState.bulkhead); } @@ -499,8 +497,8 @@ private FtHandlerTyped createMethodHandler(MethodState methodState) { .build()) .overallTimeout(Duration.of(introspector.getRetry().maxDuration(), introspector.getRetry().durationUnit())) - .applyOn(introspector.getRetry().retryOn()) - .skipOn(introspector.getRetry().abortOn()) + .applyOn(mapTypes(introspector.getRetry().retryOn())) + .skipOn(mapTypes(introspector.getRetry().abortOn())) .build(); builder.addRetry(retry); methodState.retry = retry; // keep reference to Retry @@ -513,8 +511,8 @@ private FtHandlerTyped createMethodHandler(MethodState methodState) { CommandFallback cfb = new CommandFallback(context, introspector, throwable); return toCompletionStageSupplier(cfb::execute).get(); }) - .applyOn(introspector.getFallback().applyOn()) - .skipOn(introspector.getFallback().skipOn()) + .applyOn(mapTypes(introspector.getFallback().applyOn())) + .skipOn(mapTypes(introspector.getFallback().skipOn())) .build(); builder.addFallback(fallback); } diff --git a/microprofile/fault-tolerance/src/main/java/io/helidon/microprofile/faulttolerance/ThrowableMapper.java b/microprofile/fault-tolerance/src/main/java/io/helidon/microprofile/faulttolerance/ThrowableMapper.java index 6830dd15650..154e69c7eee 100644 --- a/microprofile/fault-tolerance/src/main/java/io/helidon/microprofile/faulttolerance/ThrowableMapper.java +++ b/microprofile/fault-tolerance/src/main/java/io/helidon/microprofile/faulttolerance/ThrowableMapper.java @@ -16,6 +16,7 @@ package io.helidon.microprofile.faulttolerance; +import java.util.Arrays; import java.util.concurrent.ExecutionException; import org.eclipse.microprofile.faulttolerance.exceptions.BulkheadException; @@ -30,6 +31,13 @@ class ThrowableMapper { private ThrowableMapper() { } + /** + * Maps a {@code Throwable} in Helidon to its corresponding type in the MP + * FT API. + * + * @param t throwable to map. + * @return mapped throwable. + */ static Throwable map(Throwable t) { if (t instanceof ExecutionException) { t = t.getCause(); @@ -48,4 +56,31 @@ static Throwable map(Throwable t) { } return t; } + + /** + * Maps exception types in MP FT to internal ones used by Helidon. Allocates + * new array for the purpose of mapping. + * + * @param types array of {@code Throwable}'s type to map. + * @return mapped array. + */ + static Class[] mapTypes(Class[] types) { + if (types.length == 0) { + return types; + } + Class[] result = Arrays.copyOf(types, types.length); + for (int i = 0; i < types.length; i++) { + Class t = types[i]; + if (t == BulkheadException.class) { + result[i] = io.helidon.faulttolerance.BulkheadException.class; + } else if (t == CircuitBreakerOpenException.class) { + result[i] = io.helidon.faulttolerance.CircuitBreakerOpenException.class; + } else if (t == TimeoutException.class) { + result[i] = java.util.concurrent.TimeoutException.class; + } else { + result[i] = t; + } + } + return result; + } } From c2dd907a795e1b04058156ce70b0c7fab1401508 Mon Sep 17 00:00:00 2001 From: Santiago Pericasgeertsen Date: Tue, 1 Sep 2020 13:45:47 -0400 Subject: [PATCH 37/65] Re-organized FT handlers to run the timeout handler first. Signed-off-by: Santiago Pericasgeertsen --- .../faulttolerance/MethodInvoker.java | 25 +++++++++++-------- 1 file changed, 14 insertions(+), 11 deletions(-) diff --git a/microprofile/fault-tolerance/src/main/java/io/helidon/microprofile/faulttolerance/MethodInvoker.java b/microprofile/fault-tolerance/src/main/java/io/helidon/microprofile/faulttolerance/MethodInvoker.java index 9673394d0ef..8f89affddc6 100644 --- a/microprofile/fault-tolerance/src/main/java/io/helidon/microprofile/faulttolerance/MethodInvoker.java +++ b/microprofile/fault-tolerance/src/main/java/io/helidon/microprofile/faulttolerance/MethodInvoker.java @@ -446,11 +446,23 @@ private FtSupplier requestContextSupplier(FtSupplier supplier) { /** * Creates a FT handler for an invocation by inspecting annotations. * + * - fallback(retry(bulkhead(circuitbreaker(timeout(method))))) + * * @param methodState State related to this invocation's method. */ private FtHandlerTyped createMethodHandler(MethodState methodState) { FaultTolerance.TypedBuilder builder = FaultTolerance.typedBuilder(); + // Create and add timeout handler + if (introspector.hasTimeout()) { + Timeout timeout = Timeout.builder() + .timeout(Duration.of(introspector.getTimeout().value(), introspector.getTimeout().unit())) + .currentThread(!introspector.isAsynchronous()) + .executor(EXECUTOR_SERVICE) + .build(); + builder.addTimeout(timeout); + } + // Create and add circuit breaker if (introspector.hasCircuitBreaker()) { methodState.breaker = CircuitBreaker.builder() @@ -475,17 +487,8 @@ private FtHandlerTyped createMethodHandler(MethodState methodState) { builder.addBulkhead(methodState.bulkhead); } - // Create and add timeout handler -- parent of breaker or bulkhead - if (introspector.hasTimeout()) { - Timeout timeout = Timeout.builder() - .timeout(Duration.of(introspector.getTimeout().value(), introspector.getTimeout().unit())) - .currentThread(!introspector.isAsynchronous()) - .executor(EXECUTOR_SERVICE) - .build(); - builder.addTimeout(timeout); - } - // Create and add retry handler -- parent of timeout + // Create and add retry handler if (introspector.hasRetry()) { Retry retry = Retry.builder() .retryPolicy(Retry.JitterRetryPolicy.builder() @@ -504,7 +507,7 @@ private FtHandlerTyped createMethodHandler(MethodState methodState) { methodState.retry = retry; // keep reference to Retry } - // Create and add fallback handler -- parent of retry + // Create and add fallback handler if (introspector.hasFallback()) { Fallback fallback = Fallback.builder() .fallback(throwable -> { From 073e67791501de119830d01640803e4eec544411 Mon Sep 17 00:00:00 2001 From: Santiago Pericasgeertsen Date: Tue, 1 Sep 2020 13:47:48 -0400 Subject: [PATCH 38/65] Updated semantics for circuit breakers. Thresholds are checked after collecting more results than the window size. Several TCK tests were failing due to incorrect semantics. Some tests have been updated. Signed-off-by: Santiago Pericasgeertsen --- .../helidon/faulttolerance/AtomicCycle.java | 8 +++ .../faulttolerance/CircuitBreakerImpl.java | 18 ++---- .../helidon/faulttolerance/ResultWindow.java | 19 ++++-- .../faulttolerance/CircuitBreakerTest.java | 34 +++++----- .../faulttolerance/ResultWindowTest.java | 62 +++++++++++++++---- .../faulttolerance/MetricsTest.java | 8 +-- 6 files changed, 97 insertions(+), 52 deletions(-) diff --git a/fault-tolerance/src/main/java/io/helidon/faulttolerance/AtomicCycle.java b/fault-tolerance/src/main/java/io/helidon/faulttolerance/AtomicCycle.java index a98c680fef9..b969e11f1e8 100644 --- a/fault-tolerance/src/main/java/io/helidon/faulttolerance/AtomicCycle.java +++ b/fault-tolerance/src/main/java/io/helidon/faulttolerance/AtomicCycle.java @@ -26,6 +26,14 @@ final class AtomicCycle { this.maxIndex = maxIndex + 1; } + int get() { + return atomicInteger.get(); + } + + void set(int n) { + atomicInteger.set(n); + } + int incrementAndGet() { return atomicInteger.accumulateAndGet(maxIndex, (current, max) -> (current + 1) % max); } diff --git a/fault-tolerance/src/main/java/io/helidon/faulttolerance/CircuitBreakerImpl.java b/fault-tolerance/src/main/java/io/helidon/faulttolerance/CircuitBreakerImpl.java index 0e250f61b0e..2381a7b3073 100644 --- a/fault-tolerance/src/main/java/io/helidon/faulttolerance/CircuitBreakerImpl.java +++ b/fault-tolerance/src/main/java/io/helidon/faulttolerance/CircuitBreakerImpl.java @@ -77,21 +77,18 @@ private U invokeTask(DelayedTask task) { if (state.get() == State.CLOSED) { // run it! CompletionStage completion = task.execute(); - completion.handle((it, throwable) -> { Throwable exception = FaultTolerance.cause(throwable); if (exception == null || errorChecker.shouldSkip(exception)) { - // success results.update(SUCCESS); } else { results.update(FAILURE); - if (results.shouldOpen() && state.compareAndSet(State.CLOSED, State.OPEN)) { - results.reset(); - // if we successfully switch to open, we need to schedule switch to half-open - scheduleHalf(); - } } - + if (results.shouldOpen() && state.compareAndSet(State.CLOSED, State.OPEN)) { + results.reset(); + // if we successfully switch to open, we need to schedule switch to half-open + scheduleHalf(); + } return it; }); return task.result(); @@ -111,18 +108,15 @@ private U invokeTask(DelayedTask task) { // transition to closed successCounter.set(0); state.compareAndSet(State.HALF_OPEN, State.CLOSED); - halfOpenInProgress.set(false); } - halfOpenInProgress.set(false); } else { // failure successCounter.set(0); state.set(State.OPEN); - halfOpenInProgress.set(false); // if we successfully switch to open, we need to schedule switch to half-open scheduleHalf(); } - + halfOpenInProgress.set(false); return it; }); return task.result(); diff --git a/fault-tolerance/src/main/java/io/helidon/faulttolerance/ResultWindow.java b/fault-tolerance/src/main/java/io/helidon/faulttolerance/ResultWindow.java index 5e5b46e616f..b503ed42dea 100644 --- a/fault-tolerance/src/main/java/io/helidon/faulttolerance/ResultWindow.java +++ b/fault-tolerance/src/main/java/io/helidon/faulttolerance/ResultWindow.java @@ -29,6 +29,7 @@ final class ResultWindow { private final AtomicInteger currentSum = new AtomicInteger(); private final AtomicCycle index; private final AtomicInteger[] results; + private final AtomicInteger totalResults = new AtomicInteger(); private final int thresholdSum; ResultWindow(int size, int ratio) { @@ -44,17 +45,18 @@ final class ResultWindow { } void update(Result resultEnum) { + // update total number of results + totalResults.incrementAndGet(); + // success is zero, failure is 1 int result = resultEnum.ordinal(); AtomicInteger mine = results[index.incrementAndGet()]; int origValue = mine.getAndSet(result); - if (origValue == result) { // no change return; } - if (origValue == 1) { currentSum.decrementAndGet(); } else { @@ -62,15 +64,22 @@ void update(Result resultEnum) { } } + /** + * Open if we have seen enough results and we are at or over the threshold. + * + * @return outcome of test. + */ boolean shouldOpen() { - return currentSum.get() > thresholdSum; + return totalResults.get() >= results.length && currentSum.get() >= thresholdSum; } void reset() { - // "soft" reset - send in success equal to window size for (int i = 0; i < results.length; i++) { - update(Result.SUCCESS); + results[i].set(Result.SUCCESS.ordinal()); } + currentSum.set(0); + index.set(results.length - 1); + totalResults.set(0); } // order is significant, do not change diff --git a/fault-tolerance/src/test/java/io/helidon/faulttolerance/CircuitBreakerTest.java b/fault-tolerance/src/test/java/io/helidon/faulttolerance/CircuitBreakerTest.java index b514e6ecc32..56e47b12428 100644 --- a/fault-tolerance/src/test/java/io/helidon/faulttolerance/CircuitBreakerTest.java +++ b/fault-tolerance/src/test/java/io/helidon/faulttolerance/CircuitBreakerTest.java @@ -47,15 +47,14 @@ void testCircuitBreaker() throws InterruptedException { good(breaker); good(breaker); - bad(breaker); - good(breaker); goodMulti(breaker); - - // should open the breaker - bad(breaker); + good(breaker); + good(breaker); + good(breaker); bad(breaker); + bad(breaker); // should open - window complete breakerOpen(breaker); breakerOpenMulti(breaker); @@ -78,24 +77,19 @@ void testCircuitBreaker() throws InterruptedException { assertThat(breaker.state(), is(CircuitBreaker.State.CLOSED)); - // should open the breaker - bad(breaker); + good(breaker); + good(breaker); bad(breaker); + good(breaker); + goodMulti(breaker); + good(breaker); + good(breaker); + good(breaker); bad(breaker); + bad(breaker); // should open - window complete - assertThat(breaker.state(), is(CircuitBreaker.State.OPEN)); - - // need to wait until half open - count = 0; - while (count++ < 10) { - Thread.sleep(50); - if (breaker.state() == CircuitBreaker.State.HALF_OPEN) { - break; - } - } - - good(breaker); - badMulti(breaker); + breakerOpen(breaker); + breakerOpenMulti(breaker); assertThat(breaker.state(), is(CircuitBreaker.State.OPEN)); } diff --git a/fault-tolerance/src/test/java/io/helidon/faulttolerance/ResultWindowTest.java b/fault-tolerance/src/test/java/io/helidon/faulttolerance/ResultWindowTest.java index 7b045403ee5..da939b18e8b 100644 --- a/fault-tolerance/src/test/java/io/helidon/faulttolerance/ResultWindowTest.java +++ b/fault-tolerance/src/test/java/io/helidon/faulttolerance/ResultWindowTest.java @@ -22,26 +22,66 @@ import static org.hamcrest.MatcherAssert.assertThat; class ResultWindowTest { + + @Test + void testNotOpenBeforeCompleteWindow() { + ResultWindow window = new ResultWindow(5, 20); + assertThat("Empty should not open", window.shouldOpen(), is(false)); + window.update(ResultWindow.Result.FAILURE); + window.update(ResultWindow.Result.FAILURE); + window.update(ResultWindow.Result.FAILURE); + window.update(ResultWindow.Result.FAILURE); + assertThat("Should not open before complete window", window.shouldOpen(), is(false)); + } + + @Test + void testOpenAfterCompleteWindow1() { + ResultWindow window = new ResultWindow(5, 20); + assertThat("Empty should not open", window.shouldOpen(), is(false)); + window.update(ResultWindow.Result.FAILURE); + window.update(ResultWindow.Result.FAILURE); + window.update(ResultWindow.Result.SUCCESS); + window.update(ResultWindow.Result.SUCCESS); + window.update(ResultWindow.Result.SUCCESS); + assertThat("Should open after complete window > 20%", window.shouldOpen(), is(true)); + } + @Test - void test() { - ResultWindow window = new ResultWindow(10, 10); + void testOpenAfterCompleteWindow2() { + ResultWindow window = new ResultWindow(5, 20); + assertThat("Empty should not open", window.shouldOpen(), is(false)); + window.update(ResultWindow.Result.SUCCESS); + window.update(ResultWindow.Result.FAILURE); + window.update(ResultWindow.Result.SUCCESS); + window.update(ResultWindow.Result.FAILURE); + window.update(ResultWindow.Result.SUCCESS); + assertThat("Should open after complete window > 20%", window.shouldOpen(), is(true)); + } + + @Test + void testOpenAfterCompleteWindow3() { + ResultWindow window = new ResultWindow(5, 20); assertThat("Empty should not open", window.shouldOpen(), is(false)); window.update(ResultWindow.Result.SUCCESS); window.update(ResultWindow.Result.SUCCESS); window.update(ResultWindow.Result.SUCCESS); - assertThat("Only success should not open", window.shouldOpen(), is(false)); + window.update(ResultWindow.Result.SUCCESS); window.update(ResultWindow.Result.FAILURE); window.update(ResultWindow.Result.FAILURE); - assertThat("Should open on first failure (> 10%)", window.shouldOpen(), is(true)); - //now cycle through window and replace all with success - for (int i = 0; i < 10; i++) { - window.update(ResultWindow.Result.SUCCESS); - } - assertThat("All success should not open", window.shouldOpen(), is(false)); + assertThat("Should open after complete window > 20%", window.shouldOpen(), is(true)); + } + + @Test + void testOpenAfterCompleteWindowReset() { + ResultWindow window = new ResultWindow(5, 20); + assertThat("Empty should not open", window.shouldOpen(), is(false)); window.update(ResultWindow.Result.FAILURE); window.update(ResultWindow.Result.FAILURE); - assertThat("Should open on first failure (> 10%)", window.shouldOpen(), is(true)); + window.update(ResultWindow.Result.FAILURE); + window.update(ResultWindow.Result.FAILURE); + window.update(ResultWindow.Result.FAILURE); + assertThat("Should open after complete window > 20%", window.shouldOpen(), is(true)); window.reset(); - assertThat("Should not open after reset", window.shouldOpen(), is(false)); + assertThat("Empty should not open", window.shouldOpen(), is(false)); } } \ No newline at end of file diff --git a/microprofile/fault-tolerance/src/test/java/io/helidon/microprofile/faulttolerance/MetricsTest.java b/microprofile/fault-tolerance/src/test/java/io/helidon/microprofile/faulttolerance/MetricsTest.java index 531aa04b5fd..3869de490ac 100644 --- a/microprofile/fault-tolerance/src/test/java/io/helidon/microprofile/faulttolerance/MetricsTest.java +++ b/microprofile/fault-tolerance/src/test/java/io/helidon/microprofile/faulttolerance/MetricsTest.java @@ -288,12 +288,12 @@ public void testBreakerExceptionCounters() throws Exception { is(2L)); assertThat(getCounter(bean, "exerciseBreakerException", BREAKER_OPENED_TOTAL, boolean.class), - is(0L)); + is(1L)); assertThrows(Exception.class, () -> bean.exerciseBreakerException(true)); // failure assertThat(getCounter(bean, "exerciseBreakerException", BREAKER_CALLS_SUCCEEDED_TOTAL, boolean.class), - is(1L)); + is(0L)); // Sleep longer than circuit breaker delay Thread.sleep(1500); @@ -310,7 +310,7 @@ public void testBreakerExceptionCounters() throws Exception { // Check counters after successful calls assertThat(getCounter(bean, "exerciseBreakerException", BREAKER_CALLS_SUCCEEDED_TOTAL, boolean.class), - is(3L)); + is(2L)); try { bean.exerciseBreakerException(true); // success @@ -321,7 +321,7 @@ public void testBreakerExceptionCounters() throws Exception { // Check counters after successful calls assertThat(getCounter(bean, "exerciseBreakerException", BREAKER_CALLS_SUCCEEDED_TOTAL, boolean.class), - is(4L)); + is(3L)); } @Test From 49abf122f83de8e1b8fee808a1d28c706296b2d3 Mon Sep 17 00:00:00 2001 From: Santiago Pericasgeertsen Date: Tue, 1 Sep 2020 14:26:00 -0400 Subject: [PATCH 39/65] Restored validation of FT annotations. Signed-off-by: Santiago Pericasgeertsen --- .../faulttolerance/FaultToleranceExtension.java | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/microprofile/fault-tolerance/src/main/java/io/helidon/microprofile/faulttolerance/FaultToleranceExtension.java b/microprofile/fault-tolerance/src/main/java/io/helidon/microprofile/faulttolerance/FaultToleranceExtension.java index b94443bd911..030e1d1ba33 100644 --- a/microprofile/fault-tolerance/src/main/java/io/helidon/microprofile/faulttolerance/FaultToleranceExtension.java +++ b/microprofile/fault-tolerance/src/main/java/io/helidon/microprofile/faulttolerance/FaultToleranceExtension.java @@ -237,26 +237,26 @@ void registerFaultToleranceMetrics(@Observes AfterDeploymentValidation validatio // Metrics depending on the annotationSet present if (MethodAntn.isAnnotationPresent(beanClass, method, Retry.class)) { FaultToleranceMetrics.registerRetryMetrics(method); - // new RetryAntn(beanClass, method).validate(); + new RetryAntn(beanClass, method).validate(); } if (MethodAntn.isAnnotationPresent(beanClass, method, CircuitBreaker.class)) { FaultToleranceMetrics.registerCircuitBreakerMetrics(method); - // new CircuitBreakerAntn(beanClass, method).validate(); + new CircuitBreakerAntn(beanClass, method).validate(); } if (MethodAntn.isAnnotationPresent(beanClass, method, Timeout.class)) { FaultToleranceMetrics.registerTimeoutMetrics(method); - // new TimeoutAntn(beanClass, method).validate(); + new TimeoutAntn(beanClass, method).validate(); } if (MethodAntn.isAnnotationPresent(beanClass, method, Bulkhead.class)) { FaultToleranceMetrics.registerBulkheadMetrics(method); - // new BulkheadAntn(beanClass, method).validate(); + new BulkheadAntn(beanClass, method).validate(); } if (MethodAntn.isAnnotationPresent(beanClass, method, Fallback.class)) { FaultToleranceMetrics.registerFallbackMetrics(method); - // new FallbackAntn(beanClass, method).validate(); + new FallbackAntn(beanClass, method).validate(); } if (MethodAntn.isAnnotationPresent(beanClass, method, Asynchronous.class)) { - // new AsynchronousAntn(beanClass, method).validate(); + new AsynchronousAntn(beanClass, method).validate(); } }); } From b22da5326cfc6045aa4c538875b6c1c1b71bb0e7 Mon Sep 17 00:00:00 2001 From: Santiago Pericasgeertsen Date: Tue, 1 Sep 2020 15:21:07 -0400 Subject: [PATCH 40/65] Fixed problems with fallback and breaker metrics. Signed-off-by: Santiago Pericasgeertsen --- .../faulttolerance/CommandFallback.java | 16 +++++----------- .../faulttolerance/MethodInvoker.java | 11 +++++++++-- 2 files changed, 14 insertions(+), 13 deletions(-) diff --git a/microprofile/fault-tolerance/src/main/java/io/helidon/microprofile/faulttolerance/CommandFallback.java b/microprofile/fault-tolerance/src/main/java/io/helidon/microprofile/faulttolerance/CommandFallback.java index 1d5018d9462..88d2de43c69 100644 --- a/microprofile/fault-tolerance/src/main/java/io/helidon/microprofile/faulttolerance/CommandFallback.java +++ b/microprofile/fault-tolerance/src/main/java/io/helidon/microprofile/faulttolerance/CommandFallback.java @@ -104,7 +104,7 @@ public Throwable getFailure() { result = fallbackMethod.invoke(context.getTarget(), context.getParameters()); } } catch (Throwable t) { - updateMetrics(t); + updateMetrics(); // If InvocationTargetException, then unwrap underlying cause if (t instanceof InvocationTargetException) { @@ -113,21 +113,15 @@ public Throwable getFailure() { throw t instanceof Exception ? (Exception) t : new RuntimeException(t); } - updateMetrics(null); + updateMetrics(); return result; } /** - * Updates fallback metrics and adjust failed invocations based on outcome of fallback. + * Updates fallback metrics. */ - private void updateMetrics(Throwable throwable) { - final Method method = context.getMethod(); + private void updateMetrics() { + Method method = context.getMethod(); FaultToleranceMetrics.getCounter(method, FaultToleranceMetrics.FALLBACK_CALLS_TOTAL).inc(); - - // If fallback was successful, it is not a failed invocation - if (throwable == null) { - // Since metrics 2.0, countes should only be incrementing, so we cheat here - FaultToleranceMetrics.getCounter(method, FaultToleranceMetrics.INVOCATIONS_FAILED_TOTAL).inc(-1L); - } } } diff --git a/microprofile/fault-tolerance/src/main/java/io/helidon/microprofile/faulttolerance/MethodInvoker.java b/microprofile/fault-tolerance/src/main/java/io/helidon/microprofile/faulttolerance/MethodInvoker.java index 8f89affddc6..51facaeadd5 100644 --- a/microprofile/fault-tolerance/src/main/java/io/helidon/microprofile/faulttolerance/MethodInvoker.java +++ b/microprofile/fault-tolerance/src/main/java/io/helidon/microprofile/faulttolerance/MethodInvoker.java @@ -600,10 +600,17 @@ Supplier> toCompletionStageSupplier(FtSupplier } /** - * Collects information necessary to update metrics before method is called. + * Collects information necessary to update metrics after method is called. */ private void updateMetricsBefore() { handlerStartNanos = System.nanoTime(); + + if (introspector.hasCircuitBreaker()) { + synchronized (method) { + // Breaker state may have changed since we recorded it last + methodState.lastBreakerState = methodState.breaker.state(); + } + } } /** @@ -651,7 +658,7 @@ private void updateMetricsAfter(Throwable cause) { Objects.requireNonNull(methodState.breaker); // Update counters based on state changes - if (methodState.lastBreakerState != State.CLOSED) { + if (methodState.lastBreakerState == State.OPEN) { getCounter(method, BREAKER_CALLS_PREVENTED_TOTAL).inc(); } else if (methodState.breaker.state() == State.OPEN) { // closed -> open getCounter(method, BREAKER_OPENED_TOTAL).inc(); From 5449921aaa04ec94d6d66ec776211171aa56ea6e Mon Sep 17 00:00:00 2001 From: Santiago Pericasgeertsen Date: Wed, 2 Sep 2020 09:44:40 -0400 Subject: [PATCH 41/65] Bulkhead histograms should only be registered for async methods. Entries should only be recorded for scheduled tasks. Signed-off-by: Santiago Pericasgeertsen --- .../faulttolerance/FaultToleranceMetrics.java | 6 ------ .../faulttolerance/MethodInvoker.java | 21 +++++++++++++++---- 2 files changed, 17 insertions(+), 10 deletions(-) diff --git a/microprofile/fault-tolerance/src/main/java/io/helidon/microprofile/faulttolerance/FaultToleranceMetrics.java b/microprofile/fault-tolerance/src/main/java/io/helidon/microprofile/faulttolerance/FaultToleranceMetrics.java index cb308d6121e..d31d8b1218f 100644 --- a/microprofile/fault-tolerance/src/main/java/io/helidon/microprofile/faulttolerance/FaultToleranceMetrics.java +++ b/microprofile/fault-tolerance/src/main/java/io/helidon/microprofile/faulttolerance/FaultToleranceMetrics.java @@ -289,12 +289,6 @@ static void registerBulkheadMetrics(Method method) { BULKHEAD_EXECUTION_DURATION), "Histogram of method execution times. This does not include any " + "time spent waiting in the bulkhead queue."); - registerHistogram( - String.format(METRIC_NAME_TEMPLATE, - method.getDeclaringClass().getName(), - method.getName(), - BULKHEAD_WAITING_DURATION), - "Histogram of the time executions spend waiting in the queue."); } // -- Utility methods ---------------------------------------------------- diff --git a/microprofile/fault-tolerance/src/main/java/io/helidon/microprofile/faulttolerance/MethodInvoker.java b/microprofile/fault-tolerance/src/main/java/io/helidon/microprofile/faulttolerance/MethodInvoker.java index 51facaeadd5..9881c4a7cde 100644 --- a/microprofile/fault-tolerance/src/main/java/io/helidon/microprofile/faulttolerance/MethodInvoker.java +++ b/microprofile/fault-tolerance/src/main/java/io/helidon/microprofile/faulttolerance/MethodInvoker.java @@ -49,6 +49,7 @@ import io.helidon.faulttolerance.Retry; import io.helidon.faulttolerance.Timeout; +import org.eclipse.microprofile.faulttolerance.exceptions.BulkheadException; import org.eclipse.microprofile.faulttolerance.exceptions.CircuitBreakerOpenException; import org.eclipse.microprofile.faulttolerance.exceptions.TimeoutException; import org.eclipse.microprofile.metrics.Counter; @@ -57,6 +58,8 @@ import static io.helidon.microprofile.faulttolerance.ThrowableMapper.mapTypes; import static io.helidon.microprofile.faulttolerance.FaultToleranceExtension.isFaultToleranceMetricsEnabled; +import static io.helidon.microprofile.faulttolerance.FaultToleranceMetrics.METRIC_NAME_TEMPLATE; +import static io.helidon.microprofile.faulttolerance.FaultToleranceMetrics.registerHistogram; import static io.helidon.microprofile.faulttolerance.FaultToleranceMetrics.BREAKER_CALLS_FAILED_TOTAL; import static io.helidon.microprofile.faulttolerance.FaultToleranceMetrics.BREAKER_CALLS_PREVENTED_TOTAL; import static io.helidon.microprofile.faulttolerance.FaultToleranceMetrics.BREAKER_CALLS_SUCCEEDED_TOTAL; @@ -302,6 +305,12 @@ public MethodInvoker(InvocationContext context, MethodIntrospector introspector) registerGauge(method, BULKHEAD_WAITING_QUEUE_POPULATION, "Number of executions currently waiting in the queue", () -> methodState.bulkhead.stats().waitingQueueSize()); + registerHistogram( + String.format(METRIC_NAME_TEMPLATE, + method.getDeclaringClass().getName(), + method.getName(), + BULKHEAD_WAITING_DURATION), + "Histogram of the time executions spend waiting in the queue."); } } } @@ -707,10 +716,14 @@ private void updateMetricsAfter(Throwable cause) { Bulkhead.Stats stats = methodState.bulkhead.stats(); updateCounter(method, BULKHEAD_CALLS_ACCEPTED_TOTAL, stats.callsAccepted()); updateCounter(method, BULKHEAD_CALLS_REJECTED_TOTAL, stats.callsRejected()); - long waitingTime = invocationStartNanos - handlerStartNanos; - getHistogram(method, BULKHEAD_EXECUTION_DURATION).update(executionTime - waitingTime); - if (introspector.isAsynchronous()) { - getHistogram(method, BULKHEAD_WAITING_DURATION).update(waitingTime); + + // Update histograms if task accepted + if (!(cause instanceof BulkheadException)) { + long waitingTime = invocationStartNanos - handlerStartNanos; + getHistogram(method, BULKHEAD_EXECUTION_DURATION).update(executionTime - waitingTime); + if (introspector.isAsynchronous()) { + getHistogram(method, BULKHEAD_WAITING_DURATION).update(waitingTime); + } } } From d1644aaa83f2205a1ace5d3e6bdf01e11932cd3a Mon Sep 17 00:00:00 2001 From: Santiago Pericasgeertsen Date: Wed, 2 Sep 2020 11:01:55 -0400 Subject: [PATCH 42/65] New class to use as key into the method states map. Key must include class loader (TCKs), class (method inheritance) and method. The class is necessary since the same method can be inherited by multiple classes with different annotations. Signed-off-by: Santiago Pericasgeertsen --- .../faulttolerance/MethodInvoker.java | 68 +++-- .../faulttolerance/FaultToleranceTest.java | 2 +- .../tck/tck-fault-tolerance/failures.txt | 233 ++++++++---------- .../src/test/tck-suite.xml | 7 +- 4 files changed, 162 insertions(+), 148 deletions(-) diff --git a/microprofile/fault-tolerance/src/main/java/io/helidon/microprofile/faulttolerance/MethodInvoker.java b/microprofile/fault-tolerance/src/main/java/io/helidon/microprofile/faulttolerance/MethodInvoker.java index 9881c4a7cde..e1885d6732e 100644 --- a/microprofile/fault-tolerance/src/main/java/io/helidon/microprofile/faulttolerance/MethodInvoker.java +++ b/microprofile/fault-tolerance/src/main/java/io/helidon/microprofile/faulttolerance/MethodInvoker.java @@ -90,7 +90,7 @@ /** * Invokes a FT method applying semantics based on method annotations. An instance * of this class is created for each method invocation. Some state is shared across - * all invocations of the method, including for circuit breakers and bulkheads. + * all invocations of a method, including for circuit breakers and bulkheads. */ public class MethodInvoker implements FtSupplier { private static final Logger LOGGER = Logger.getLogger(MethodInvoker.class.getName()); @@ -111,12 +111,11 @@ public class MethodInvoker implements FtSupplier { private final MethodIntrospector introspector; /** - * Map of a class loader and a method into a method state. Class loaders are needed - * when running TCKs where each test is considered a different application (with - * potentially different configurations). + * Maps a {@code MethodStateKey} to a {@code MethodState}. The method state returned + * caches the FT handler as well as some additional variables. This mapping must + * be shared by all instances of this class. */ - private static final ConcurrentHashMap> - FT_HANDLERS = new ConcurrentHashMap<>(); + private static final ConcurrentHashMap METHOD_STATES = new ConcurrentHashMap<>(); /** * Executor service shared by instances of this class. @@ -164,6 +163,10 @@ public class MethodInvoker implements FtSupplier { */ private Thread asyncInterruptThread; + /** + * State associated with a method in {@code METHOD_STATES}. This include the + * FT handler created for the method. + */ private static class MethodState { private FtHandlerTyped handler; private Retry retry; @@ -176,6 +179,44 @@ private static class MethodState { private long startNanos; } + /** + * A key used to lookup {@code MethodState} instances, which include FT handlers. + * A class loader is necessary to support multiple applications as seen in the TCKs. + * The method class in necessary given that the same method can inherited by different + * classes with different FT annotations and should not share handlers. Finally, the + * method is main part of the key. + */ + private static class MethodStateKey { + private final ClassLoader classLoader; + private final Class methodClass; + private final Method method; + + MethodStateKey(ClassLoader classLoader, Class methodClass, Method method) { + this.classLoader = classLoader; + this.methodClass = methodClass; + this.method = method; + } + + @Override + public boolean equals(Object o) { + if (this == o) { + return true; + } + if (o == null || getClass() != o.getClass()) { + return false; + } + MethodStateKey that = (MethodStateKey) o; + return classLoader.equals(that.classLoader) && + methodClass.equals(that.methodClass) && + method.equals(that.method); + } + + @Override + public int hashCode() { + return Objects.hash(classLoader, methodClass, method); + } + } + /** * State associated with a method instead of an invocation. Shared by all * invocations of same method. @@ -255,12 +296,11 @@ public MethodInvoker(InvocationContext context, MethodIntrospector introspector) this.method = context.getMethod(); this.helidonContext = Contexts.context().orElseGet(Context::create); - // Create method state using CCL to support multiples apps (TCKs) + // Create method state using CCL to support multiples apps (like in TCKs) ClassLoader ccl = Thread.currentThread().getContextClassLoader(); Objects.requireNonNull(ccl); - ConcurrentHashMap methodStates = - FT_HANDLERS.computeIfAbsent(ccl, cl -> new ConcurrentHashMap<>()); - this.methodState = methodStates.computeIfAbsent(method, method -> { + MethodStateKey methodStateKey = new MethodStateKey(ccl, context.getTarget().getClass(), method); + this.methodState = METHOD_STATES.computeIfAbsent(methodStateKey, key -> { MethodState methodState = new MethodState(); methodState.lastBreakerState = State.CLOSED; if (introspector.hasCircuitBreaker()) { @@ -284,7 +324,7 @@ public MethodInvoker(InvocationContext context, MethodIntrospector introspector) + " on thread " + Thread.currentThread().getName()); } - // Registration of gauges for bulkhead and circuit breakers + // Gauges and other metrics for bulkhead and circuit breakers if (isFaultToleranceMetricsEnabled()) { if (introspector.hasCircuitBreaker()) { registerGauge(method, BREAKER_OPEN_TOTAL, @@ -330,10 +370,10 @@ public String toString() { } /** - * Clears ftHandlers map of any cached handlers. + * Clears {@code METHOD_STATES} map. */ - static void clearFtHandlersMap() { - FT_HANDLERS.clear(); + static void clearMethodStatesMap() { + METHOD_STATES.clear(); } /** diff --git a/microprofile/fault-tolerance/src/test/java/io/helidon/microprofile/faulttolerance/FaultToleranceTest.java b/microprofile/fault-tolerance/src/test/java/io/helidon/microprofile/faulttolerance/FaultToleranceTest.java index dfb64938520..1294287aaa0 100644 --- a/microprofile/fault-tolerance/src/test/java/io/helidon/microprofile/faulttolerance/FaultToleranceTest.java +++ b/microprofile/fault-tolerance/src/test/java/io/helidon/microprofile/faulttolerance/FaultToleranceTest.java @@ -75,7 +75,7 @@ public static void shutDownCdiContainer() { */ @BeforeEach public void resetHandlers() { - MethodInvoker.clearFtHandlersMap(); + MethodInvoker.clearMethodStatesMap(); } protected static T newBean(Class beanClass) { diff --git a/microprofile/tests/tck/tck-fault-tolerance/failures.txt b/microprofile/tests/tck/tck-fault-tolerance/failures.txt index c9ea176f065..dbf15358fba 100644 --- a/microprofile/tests/tck/tck-fault-tolerance/failures.txt +++ b/microprofile/tests/tck/tck-fault-tolerance/failures.txt @@ -1,133 +1,106 @@ - Failed tests: - AsyncCancellationTest>Arquillian.run:138->testCancelledButRemainsInBulkhead:155 Task started unexpectedly - AsyncCancellationTest>Arquillian.run:138->testCancelledWhileQueued:176 Task started unexpectedly - CircuitBreakerBulkheadTest>Arquillian.run:138->testCircuitBreaker:229 Task started unexpectedly - CircuitBreakerBulkheadTest>Arquillian.run:138->testCircuitBreakerAroundBulkheadAsync:113 Task started unexpectedly - CircuitBreakerBulkheadTest>Arquillian.run:138->testCircuitBreakerAroundBulkheadSync:173 Task did not finish within 2 seconds - CircuitBreakerExceptionHierarchyTest>Arquillian.run:138->serviceAthrowsE0:86 expected [OPEN] but found [CLOSED] - CircuitBreakerExceptionHierarchyTest>Arquillian.run:138->serviceAthrowsE0S:116 expected [OPEN] but found [CLOSED] - CircuitBreakerExceptionHierarchyTest>Arquillian.run:138->serviceBthrowsException:136 expected [OPEN] but found [CLOSED] - CircuitBreakerExceptionHierarchyTest>Arquillian.run:138->serviceBthrowsRuntimeException:178 expected [OPEN] but found [CLOSED] - CircuitBreakerInitialSuccessTest>Arquillian.run:138->testCircuitInitialSuccessDefaultSuccessThreshold:97 serviceA should throw an Exception in testCircuitDefaultSuccessThreshold on iteration 5 - CircuitBreakerLateSuccessTest>Arquillian.run:138->testCircuitLateSuccessDefaultSuccessThreshold:98 serviceA should throw an Exception in testCircuitLateSuccessDefaultSuccessThreshold on iteration 5 - CircuitBreakerRetryTest>Arquillian.run:138->testCircuitOpenWithMultiTimeouts:275 serviceC should retry or throw a CircuitBreakerOpenException in testCircuitOpenWithMultiTimeouts on iteration 1, caught exception: org.eclipse.microprofile.faulttolerance.exceptions.TimeoutException: Method interrupted by timeout - CircuitBreakerRetryTest>Arquillian.run:138->testCircuitOpenWithMultiTimeoutsAsync:480 Thrown exception is the wrong type - Expected: an instance of org.eclipse.microprofile.faulttolerance.exceptions.CircuitBreakerOpenException - but: is a org.eclipse.microprofile.faulttolerance.exceptions.TimeoutException - CircuitBreakerRetryTest>Arquillian.run:138->testNoRetriesIfAbortOnAsync:589 Execution exception not thrown from Future - CircuitBreakerRetryTest>Arquillian.run:138->testNoRetriesIfNotRetryOnAsync:563 Execution exception not thrown from Future - CircuitBreakerRetryTest>Arquillian.run:138->testRetriesSucceedWhenCircuitCloses:311 Call was successful but did not take the expected time - Expected: Duration within of - but: which is from - CircuitBreakerRetryTest>Arquillian.run:138->testRetriesSucceedWhenCircuitClosesAsync:538 Call was successful but did not take the expected time - Expected: Duration within of - but: which is from - CircuitBreakerTest>Arquillian.run:138->testCircuitClosedThenOpen:122 The number of executions should be 4 expected [4] but found [7] - CircuitBreakerTest>Arquillian.run:138->testCircuitDefaultSuccessThreshold:222 serviceA should throw an Exception in testCircuitDefaultSuccessThreshold on iteration 5 - CircuitBreakerTest>Arquillian.run:138->testCircuitHighSuccessThreshold:309 serviceA should not have thrown a RuntimeException in testCircuitHighSuccessThreshold on iteration 7 - CircuitBreakerTest>Arquillian.run:138->testClassLevelCircuitBase:372 The number of executions should be 4 expected [4] but found [7] - CircuitBreakerTest>Arquillian.run:138->testClassLevelCircuitOverride:411 serviceC should not throw a RuntimeException on iteration 3 - CircuitBreakerTest>Arquillian.run:138->testRollingWindowCircuitOpen:528 The CircuitBreaker exception in testRollingWindowCircuitOpen was not thrown correctly. expected [true] but found [false] - CircuitBreakerTest>Arquillian.run:138->testRollingWindowCircuitOpen2:576 The CircuitBreaker exception in testRollingWindowCircuitOpen2 was not thrown correctly. expected [true] but found [false] - CircuitBreakerTimeoutTest>Arquillian.run:138->testTimeout:74 Unexpected exception thrown - RetryTimeoutTest>Arquillian.run:138->testRetryWithAbortOn:151 The execution count should be 1 (no retries) expected [1] but found [2] - TimeoutUninterruptableTest>Arquillian.run:138->testTimeoutAsyncBulkhead:185 Execution count after second call expected [1] but found [2] - BulkheadAsynchRetryTest>Arquillian.run:138->testBulkheadClassAsynchronous55RetryOverload:282 Failure count should be non-zero - Expected: a value greater than <0> - but: <0> was equal to <0> - BulkheadAsynchRetryTest>Arquillian.run:138->testBulkheadClassAsynchronousPassiveRetry55:161 Bulkhead appears to have been breeched 10 workers, expected 5. expected [true] but found [false] - BulkheadAsynchRetryTest>Arquillian.run:138->testBulkheadMethodAsynchronous55RetryOverload:251 Failure count should be non-zero - Expected: a value greater than <0> - but: <0> was equal to <0> - BulkheadAsynchRetryTest>Arquillian.run:138->testBulkheadMethodAsynchronousRetry55:177 Bulkhead appears to have been breeched 10 workers, expected 5. expected [true] but found [false] - BulkheadAsynchRetryTest>Arquillian.run:138->testBulkheadMethodAsynchronousRetry55Trip:208 Task 0 is running. expected [false] but found [true] - BulkheadAsynchRetryTest>Arquillian.run:138->testBulkheadPassiveRetryMethodAsynchronous55:295 Bulkhead appears to have been breeched 10 workers, expected 5. expected [true] but found [false] - BulkheadAsynchRetryTest>Arquillian.run:138->testBulkheadQueReplacesDueToClassRetryFailures:346 Bulkhead appears to have been breeched 10 workers, expected 5. expected [true] but found [false] - BulkheadAsynchRetryTest>Arquillian.run:138->testBulkheadRetryClassAsynchronous55:310 Bulkhead appears to have been breeched 10 workers, expected 5. expected [true] but found [false] - BulkheadAsynchRetryTest>Arquillian.run:138->testNoRetriesWithAbortOn:477 Task started unexpectedly - BulkheadAsynchRetryTest>Arquillian.run:138->testNoRetriesWithoutRetryOn:451 Task started unexpectedly - BulkheadAsynchRetryTest>Arquillian.run:138->testRetriesJoinBackOfQueue:411 Task started unexpectedly - BulkheadAsynchRetryTest>Arquillian.run:138->testRetriesReenterBulkhead:381 Task started unexpectedly - BulkheadAsynchTest>Arquillian.run:138->testBulkheadClassAsynchronous3:162 Bulkhead appears to have been breeched 10 workers, expected 3. expected [true] but found [false] - BulkheadAsynchTest>Arquillian.run:138->testBulkheadCompletionStage:290 Timed out waiting for future to throw BulkheadException - BulkheadAsynchTest>Arquillian.run:138->testBulkheadExceptionThrownWhenQueueFullAsync:260 When a task is rejected from the bulkhead, the returned future should report as done expected [true] but found [false] - BulkheadAsynchTest>Arquillian.run:138->testBulkheadMethodAsynchronous3:174 Bulkhead appears to have been breeched 10 workers, expected 3. expected [true] but found [false] - BulkheadSynchConfigTest>Arquillian.run:138->testBulkheadClassSemaphore3:83 Bulkhead appears to have been breeched 10 workers, expected 5. expected [true] but found [false] - BulkheadSynchRetryTest.afterClass:139 Unable to clean up all tasks - org.eclipse.microprofile.fault.tolerance.tck.bulkhead.BulkheadSynchRetryTest.afterMethod(org.eclipse.microprofile.fault.tolerance.tck.bulkhead.BulkheadSynchRetryTest) - Run 1: BulkheadSynchRetryTest.afterMethod:134 Unable to clean up all tasks - Run 2: BulkheadSynchRetryTest.afterMethod:134 Unable to clean up all tasks - Run 3: BulkheadSynchRetryTest.afterMethod:134 Unable to clean up all tasks +Failed tests: + CircuitBreakerBulkheadTest>Arquillian.run:138->testCircuitBreakerAroundBulkheadAsync:126 Unexpected exception thrown from Future + CircuitBreakerBulkheadTest>Arquillian.run:138->testCircuitBreakerAroundBulkheadSync:181 Unexpected exception thrown from Future + TimeoutUninterruptableTest>Arquillian.run:138->testTimeoutAsyncBulkhead:185 Execution count after second call expected [1] but found [2] + TimeoutUninterruptableTest>Arquillian.run:138->testTimeoutAsyncBulkheadQueueTimed:237 Time taken for call B to timeout +Expected: a value less than + but: was greater than + BulkheadSynchConfigTest>Arquillian.run:138->testBulkheadClassSemaphore3:83 Work is not being done simultaneously enough, only 3 workers at once. Expecting 5. expected [true] but found [false] + BulkheadSynchTest>Arquillian.run:138->testBulkheadExceptionThrownWhenQueueFullSemaphore:199 Unable to clean up all tasks + FallbackMethodOutOfPackageTest>Arquillian.arquillianBeforeClass:96 ? Runtime E... + FallbackMethodSubclassTest>Arquillian.arquillianBeforeClass:96 ? Runtime Expec... + FallbackMethodSuperclassPrivateTest>Arquillian.arquillianBeforeClass:96 ? Runtime + FallbackMethodWildcardNegativeTest>Arquillian.arquillianBeforeClass:96 ? Runtime + IncompatibleFallbackMethodTest>Arquillian.arquillianBeforeClass:96 ? Runtime E... + IncompatibleFallbackMethodWithArgsTest>Arquillian.arquillianBeforeClass:96 ? Runtime + IncompatibleFallbackPolicies>Arquillian.arquillianBeforeClass:96 ? Runtime Exp... + IncompatibleFallbackTest>Arquillian.arquillianBeforeClass:96 ? Runtime Expecte... + InvalidAsynchronousClassTest>Arquillian.arquillianBeforeClass:96 ? Runtime Exp... + InvalidAsynchronousMethodTest>Arquillian.arquillianBeforeClass:96 ? Runtime Ex... + InvalidBulkheadAsynchQueueTest>Arquillian.arquillianBeforeClass:96 ? Runtime E... + InvalidBulkheadValueTest>Arquillian.arquillianBeforeClass:96 ? Runtime Expecte... + InvalidCircuitBreakerDelayTest>Arquillian.arquillianBeforeClass:96 ? Runtime E... + InvalidCircuitBreakerFailureRatioNegTest>Arquillian.arquillianBeforeClass:96 ? Runtime + InvalidCircuitBreakerFailureRatioPosTest>Arquillian.arquillianBeforeClass:96 ? Runtime + InvalidCircuitBreakerFailureReqVol0Test>Arquillian.arquillianBeforeClass:96 ? Runtime + InvalidCircuitBreakerFailureReqVolNegTest>Arquillian.arquillianBeforeClass:96 ? Runtime + InvalidCircuitBreakerFailureSuccess0Test>Arquillian.arquillianBeforeClass:96 ? Runtime + InvalidCircuitBreakerFailureSuccessNegTest>Arquillian.arquillianBeforeClass:96 ? Runtime + InvalidRetryDelayDurationTest>Arquillian.arquillianBeforeClass:96 ? Runtime Ex... + InvalidRetryDelayTest>Arquillian.arquillianBeforeClass:96 ? Runtime Expected e... + InvalidRetryJitterTest>Arquillian.arquillianBeforeClass:96 ? Runtime Expected ... + InvalidRetryMaxRetriesTest>Arquillian.arquillianBeforeClass:96 ? Runtime Expec... + InvalidTimeoutValueTest>Arquillian.arquillianBeforeClass:96 ? Runtime Expected... + BulkheadMetricTest>Arquillian.run:138->bulkheadMetricAsyncTest:244 queue wait histogram counts +Expected: is <4L> + but: was <5L> + BulkheadMetricTest>Arquillian.run:138->bulkheadMetricHistogramTest:184 histogram count +Expected: is <2L> + but: was <3L> + BulkheadMetricTest>Arquillian.run:138->bulkheadMetricTest:122 bulkhead wait time histogram present +Expected: is + but: was + CircuitBreakerMetricTest>Arquillian.run:138->testCircuitBreakerMetric:136 circuitbreaker calls prevented +Expected: is <1L> + but: was <3L> + FallbackMetricTest>Arquillian.run:138->fallbackMetricHandlerTest:93 failed invocations +Expected: is <0L> + but: was <-1L> + FallbackMetricTest>Arquillian.run:138->fallbackMetricMethodTest:67 failed invocations +Expected: is <0L> + but: was <-1L> + RetryVisibilityTest>Arquillian.run:138->serviceOverrideClassLevelUsesClassLevelAnnotation:134->checkServiceCall:248 in RetryVisibilityTest#serviceOverrideClassLevelUsesClassLevelAnnotation service() should have been called exactly 5 times expected [5] but found [4] - BulkheadSynchRetryTest>Arquillian.run:138->testBulkheadMethodSynchronousRetry55:231 Bulkhead appears to have been breeched 10 workers, expected 5. expected [true] but found [false] - BulkheadSynchRetryTest>Arquillian.run:138->testBulkheadRetryClassSynchronous55:261 Bulkhead appears to have been breeched 10 workers, expected 5. expected [true] but found [false] - BulkheadSynchRetryTest>Arquillian.run:138->testIgnoreWaitingTaskQueueBulkhead:290 Bulkhead appears to have been breeched 10 workers, expected 5. expected [true] but found [false] - BulkheadSynchRetryTest>Arquillian.run:138->testNoRetriesBulkhead:275 Bulkhead appears to have been breeched 10 workers, expected 5. expected [true] but found [false] - BulkheadSynchRetryTest>Arquillian.run:138->testNoRetriesWithAbortOn:358 Timed out waiting for future to throw BulkheadException - BulkheadSynchRetryTest>Arquillian.run:138->testNoRetriesWithoutRetryOn:338 Timed out waiting for future to throw BulkheadException - BulkheadSynchRetryTest>Arquillian.run:138->testRetriesReenterBulkhead:317 Unexpected exception thrown from Future - BulkheadSynchTest>Arquillian.run:138->testBulkheadClassSemaphore3:114 Bulkhead appears to have been breeched 10 workers, expected 3. expected [true] but found [false] - BulkheadSynchTest>Arquillian.run:138->testBulkheadExceptionThrownWhenQueueFullSemaphore:199 Unable to clean up all tasks - BulkheadSynchTest>Arquillian.run:138->testBulkheadMethodSemaphore3:152 Bulkhead appears to have been breeched 10 workers, expected 3. expected [true] but found [false] - CircuitBreakerConfigGlobalTest>Arquillian.run:138->testCircuitDefaultSuccessThreshold:75 serviceA should throw an Exception in testCircuitDefaultSuccessThreshold on iteration 5 - CircuitBreakerConfigOnMethodTest>Arquillian.run:138->testCircuitDefaultSuccessThreshold:77 serviceA should throw an Exception in testCircuitDefaultSuccessThreshold on iteration 5 - DisableAnnotationGloballyEnableOnClassTest>Arquillian.run:138->testBulkhead:174 Expected BulkheadException to be thrown, but nothing was thrown - DisableAnnotationGloballyEnableOnClassTest>Arquillian.run:138->testCircuitBreaker:125 Expected CircuitBreakerOpenException to be thrown, but TestException was thrown - DisableAnnotationGloballyEnableOnMethodTest>Arquillian.run:138->testBulkhead:177 Expected BulkheadException to be thrown, but nothing was thrown - DisableAnnotationGloballyEnableOnMethodTest>Arquillian.run:138->testCircuitBreaker:126 Expected CircuitBreakerOpenException to be thrown, but TestException was thrown - DisableAnnotationOnClassEnableOnMethodTest>Arquillian.run:138->testBulkhead:175 Expected BulkheadException to be thrown, but nothing was thrown - DisableAnnotationOnClassEnableOnMethodTest>Arquillian.run:138->testCircuitBreaker:126 Expected CircuitBreakerOpenException to be thrown, but TestException was thrown - DisableFTEnableGloballyTest>Arquillian.run:138->testBulkhead:169 Expected BulkheadException to be thrown, but nothing was thrown - DisableFTEnableGloballyTest>Arquillian.run:138->testCircuitBreaker:120 Expected CircuitBreakerOpenException to be thrown, but TestException was thrown - DisableFTEnableOnClassTest>Arquillian.run:138->testBulkhead:169 Expected BulkheadException to be thrown, but nothing was thrown - DisableFTEnableOnClassTest>Arquillian.run:138->testCircuitBreaker:120 Expected CircuitBreakerOpenException to be thrown, but TestException was thrown - DisableFTEnableOnMethodTest>Arquillian.run:138->testBulkhead:156 Expected BulkheadException to be thrown, but nothing was thrown - DisableFTEnableOnMethodTest>Arquillian.run:138->testCircuitBreaker:107 Expected CircuitBreakerOpenException to be thrown, but TestException was thrown - FallbackMethodOutOfPackageTest>Arquillian.arquillianBeforeClass:96 ? Runtime E... - FallbackMethodSubclassTest>Arquillian.arquillianBeforeClass:96 ? Runtime Expec... - FallbackMethodSuperclassPrivateTest>Arquillian.arquillianBeforeClass:96 ? Runtime - FallbackMethodWildcardNegativeTest>Arquillian.arquillianBeforeClass:96 ? Runtime - IncompatibleFallbackMethodTest>Arquillian.arquillianBeforeClass:96 ? Runtime E... - IncompatibleFallbackMethodWithArgsTest>Arquillian.arquillianBeforeClass:96 ? Runtime - IncompatibleFallbackPolicies>Arquillian.arquillianBeforeClass:96 ? Runtime Exp... - IncompatibleFallbackTest>Arquillian.arquillianBeforeClass:96 ? Runtime Expecte... - InvalidAsynchronousClassTest>Arquillian.arquillianBeforeClass:96 ? Runtime Exp... - InvalidAsynchronousMethodTest>Arquillian.arquillianBeforeClass:96 ? Runtime Ex... - InvalidBulkheadAsynchQueueTest>Arquillian.arquillianBeforeClass:96 ? Runtime E... - InvalidBulkheadValueTest>Arquillian.arquillianBeforeClass:96 ? Runtime Expecte... - InvalidCircuitBreakerDelayTest>Arquillian.arquillianBeforeClass:96 ? Runtime E... - InvalidCircuitBreakerFailureRatioNegTest>Arquillian.arquillianBeforeClass:96 ? Runtime - InvalidCircuitBreakerFailureRatioPosTest>Arquillian.arquillianBeforeClass:96 ? Runtime - InvalidCircuitBreakerFailureReqVol0Test>Arquillian.arquillianBeforeClass:96 ? Runtime - InvalidCircuitBreakerFailureReqVolNegTest>Arquillian.arquillianBeforeClass:96 ? Runtime - InvalidCircuitBreakerFailureSuccess0Test>Arquillian.arquillianBeforeClass:96 ? Runtime - InvalidCircuitBreakerFailureSuccessNegTest>Arquillian.arquillianBeforeClass:96 ? Runtime - InvalidRetryDelayDurationTest>Arquillian.arquillianBeforeClass:96 ? Runtime Ex... - InvalidRetryDelayTest>Arquillian.arquillianBeforeClass:96 ? Runtime Expected e... - InvalidRetryJitterTest>Arquillian.arquillianBeforeClass:96 ? Runtime Expected ... - InvalidRetryMaxRetriesTest>Arquillian.arquillianBeforeClass:96 ? Runtime Expec... - InvalidTimeoutValueTest>Arquillian.arquillianBeforeClass:96 ? Runtime Expected... - BulkheadMetricTest>Arquillian.run:138->bulkheadMetricAsyncTest:219 Unexpected exception thrown from Future - BulkheadMetricTest>Arquillian.run:138->bulkheadMetricHistogramTest:173 Unexpected exception thrown from Future - BulkheadMetricTest>Arquillian.run:138->bulkheadMetricRejectionTest:142 Unexpected exception thrown from Future - BulkheadMetricTest>Arquillian.run:138->bulkheadMetricTest:110 concurrent executions - Expected: is <2L> - but: was <0L> - CircuitBreakerMetricTest>Arquillian.run:138->testCircuitBreakerMetric:116 circuit breaker times opened - Expected: is <1L> - but: was <0L> - ClassLevelMetricTest>Arquillian.run:138->testRetryMetricUnsuccessful:98 calls succeeded without retry - Expected: is <0L> - but: was <1L> - FallbackMetricTest>Arquillian.run:138->fallbackMetricHandlerTest:93 failed invocations - Expected: is <0L> - but: was <-1L> - FallbackMetricTest>Arquillian.run:138->fallbackMetricMethodTest:67 failed invocations - Expected: is <0L> - but: was <-1L> - RetryMetricTest>Arquillian.run:138->testRetryMetricUnsuccessful:92 calls succeeded without retry - Expected: is <0L> - but: was <1L> +Tests run: 455, Failures: 37, Errors: 0, Skipped: 48 - Tests run: 457, Failures: 103, Errors: 0, Skipped: 48 +--- +Failed tests: + CircuitBreakerBulkheadTest>Arquillian.run:138->testCircuitBreakerAroundBulkheadAsync:126 Unexpected exception thrown from Future + CircuitBreakerBulkheadTest>Arquillian.run:138->testCircuitBreakerAroundBulkheadSync:181 Unexpected exception thrown from Future + TimeoutUninterruptableTest>Arquillian.run:138->testTimeoutAsyncBulkhead:185 Execution count after second call expected [1] but found [2] + TimeoutUninterruptableTest>Arquillian.run:138->testTimeoutAsyncBulkheadQueueTimed:237 Time taken for call B to timeout +Expected: a value less than + but: was greater than + BulkheadSynchConfigTest>Arquillian.run:138->testBulkheadClassSemaphore3:83 Work is not being done simultaneously enough, only 3 workers at once. Expecting 5. expected [true] but found [false] + BulkheadSynchTest>Arquillian.run:138->testBulkheadExceptionThrownWhenQueueFullSemaphore:199 Unable to clean up all tasks + BulkheadMetricTest>Arquillian.run:138->bulkheadMetricAsyncTest:244 queue wait histogram counts +Expected: is <4L> + but: was <5L> + BulkheadMetricTest>Arquillian.run:138->bulkheadMetricHistogramTest:184 histogram count +Expected: is <2L> + but: was <3L> + BulkheadMetricTest>Arquillian.run:138->bulkheadMetricTest:122 bulkhead wait time histogram present +Expected: is + but: was + + + CircuitBreakerMetricTest>Arquillian.run:138->testCircuitBreakerMetric:136 circuitbreaker calls prevented +Expected: is <1L> + but: was <3L> + FallbackMetricTest>Arquillian.run:138->fallbackMetricHandlerTest:93 failed invocations +Expected: is <0L> + but: was <-1L> + FallbackMetricTest>Arquillian.run:138->fallbackMetricMethodTest:67 failed invocations +Expected: is <0L> + but: was <-1L> + RetryVisibilityTest>Arquillian.run:138->serviceOverrideClassLevelUsesClassLevelAnnotation:134->checkServiceCall:248 in RetryVisibilityTest#serviceOverrideClassLevelUsesClassLevelAnnotation service() should have been called exactly 5 times expected [5] but found [4] + +Tests run: 407, Failures: 13, Errors: 0, Skipped: 0 + +-- + +Results : + +Failed tests: + CircuitBreakerBulkheadTest>Arquillian.run:138->testCircuitBreakerAroundBulkheadAsync:126 Unexpected exception thrown from Future + CircuitBreakerBulkheadTest>Arquillian.run:138->testCircuitBreakerAroundBulkheadSync:181 Unexpected exception thrown from Future + TimeoutUninterruptableTest>Arquillian.run:138->testTimeoutAsyncBulkhead:181 Execution count after second call expected [1] but found [2] + TimeoutUninterruptableTest>Arquillian.run:138->testTimeoutAsyncBulkheadQueueTimed:233 Time taken for call B to timeout +Expected: a value less than + but: was greater than + BulkheadSynchConfigTest>Arquillian.run:138->testBulkheadClassSemaphore3:83 Work is not being done simultaneously enough, only 3 workers at once. Expecting 5. expected [true] but found [false] + BulkheadSynchTest>Arquillian.run:138->testBulkheadExceptionThrownWhenQueueFullSemaphore:199 Unable to clean up all tasks + RetryVisibilityTest>Arquillian.run:138->serviceOverrideClassLevelUsesClassLevelAnnotation:134->checkServiceCall:248 in RetryVisibilityTest#serviceOverrideClassLevelUsesClassLevelAnnotation service() should have been called exactly 5 times expected [5] but found [4] + +Tests run: 407, Failures: 7, Errors: 0, Skipped: 0 diff --git a/microprofile/tests/tck/tck-fault-tolerance/src/test/tck-suite.xml b/microprofile/tests/tck/tck-fault-tolerance/src/test/tck-suite.xml index 90203fab07b..d726636d795 100644 --- a/microprofile/tests/tck/tck-fault-tolerance/src/test/tck-suite.xml +++ b/microprofile/tests/tck/tck-fault-tolerance/src/test/tck-suite.xml @@ -27,10 +27,11 @@ - From 02f1191b73d4589520bc2bb7046426e219af8def Mon Sep 17 00:00:00 2001 From: Santiago Pericasgeertsen Date: Thu, 3 Sep 2020 09:36:15 -0400 Subject: [PATCH 43/65] Use executor from FT SE after adjusting core sizes. Set queue to 0 for sync bulkheads. Signed-off-by: Santiago Pericasgeertsen --- .../FaultToleranceExtension.java | 21 +++++++++++++++++-- .../faulttolerance/MethodInvoker.java | 10 +-------- 2 files changed, 20 insertions(+), 11 deletions(-) diff --git a/microprofile/fault-tolerance/src/main/java/io/helidon/microprofile/faulttolerance/FaultToleranceExtension.java b/microprofile/fault-tolerance/src/main/java/io/helidon/microprofile/faulttolerance/FaultToleranceExtension.java index 030e1d1ba33..fddeb7fa3a0 100644 --- a/microprofile/fault-tolerance/src/main/java/io/helidon/microprofile/faulttolerance/FaultToleranceExtension.java +++ b/microprofile/fault-tolerance/src/main/java/io/helidon/microprofile/faulttolerance/FaultToleranceExtension.java @@ -42,6 +42,10 @@ import javax.enterprise.util.AnnotationLiteral; import javax.inject.Inject; +import io.helidon.common.configurable.ScheduledThreadPoolSupplier; +import io.helidon.common.configurable.ThreadPoolSupplier; +import io.helidon.faulttolerance.FaultTolerance; + import org.eclipse.microprofile.config.Config; import org.eclipse.microprofile.config.ConfigProvider; import org.eclipse.microprofile.faulttolerance.Asynchronous; @@ -221,11 +225,11 @@ private void registerFaultToleranceMethods(AnnotatedType type) { } /** - * Registers metrics for all FT methods. + * Registers metrics for all FT methods and init executors. * * @param validation Event information. */ - void registerFaultToleranceMetrics(@Observes AfterDeploymentValidation validation) { + void registerMetricsAndInitExecutors(@Observes AfterDeploymentValidation validation) { if (FaultToleranceMetrics.enabled()) { getRegisteredMethods().stream().forEach(beanMethod -> { final Method method = beanMethod.method(); @@ -260,6 +264,19 @@ void registerFaultToleranceMetrics(@Observes AfterDeploymentValidation validatio } }); } + + // Initialize executors for MP FT - default size of 16 + io.helidon.config.Config config = io.helidon.config.Config.create(); + FaultTolerance.scheduledExecutor(ScheduledThreadPoolSupplier.builder() + .threadNamePrefix("ft-mp-schedule-") + .corePoolSize(16) + .config(config.get("scheduled-executor")) + .build()); + FaultTolerance.executor(ThreadPoolSupplier.builder() + .threadNamePrefix("ft-mp-") + .corePoolSize(16) + .config(config.get("executor")) + .build()); } /** diff --git a/microprofile/fault-tolerance/src/main/java/io/helidon/microprofile/faulttolerance/MethodInvoker.java b/microprofile/fault-tolerance/src/main/java/io/helidon/microprofile/faulttolerance/MethodInvoker.java index e1885d6732e..914c5b6f210 100644 --- a/microprofile/fault-tolerance/src/main/java/io/helidon/microprofile/faulttolerance/MethodInvoker.java +++ b/microprofile/fault-tolerance/src/main/java/io/helidon/microprofile/faulttolerance/MethodInvoker.java @@ -24,9 +24,7 @@ import java.util.concurrent.CompletionStage; import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.ExecutionException; -import java.util.concurrent.Executors; import java.util.concurrent.Future; -import java.util.concurrent.ScheduledExecutorService; import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicBoolean; import java.util.function.Supplier; @@ -117,11 +115,6 @@ public class MethodInvoker implements FtSupplier { */ private static final ConcurrentHashMap METHOD_STATES = new ConcurrentHashMap<>(); - /** - * Executor service shared by instances of this class. - */ - private static final ScheduledExecutorService EXECUTOR_SERVICE = Executors.newScheduledThreadPool(16); - /** * Start system nanos when handler is called. */ @@ -507,7 +500,6 @@ private FtHandlerTyped createMethodHandler(MethodState methodState) { Timeout timeout = Timeout.builder() .timeout(Duration.of(introspector.getTimeout().value(), introspector.getTimeout().unit())) .currentThread(!introspector.isAsynchronous()) - .executor(EXECUTOR_SERVICE) .build(); builder.addTimeout(timeout); } @@ -530,7 +522,7 @@ private FtHandlerTyped createMethodHandler(MethodState methodState) { if (introspector.hasBulkhead()) { methodState.bulkhead = Bulkhead.builder() .limit(introspector.getBulkhead().value()) - .queueLength(introspector.getBulkhead().waitingTaskQueue()) + .queueLength(introspector.isAsynchronous() ? introspector.getBulkhead().waitingTaskQueue() : 0) .async(introspector.isAsynchronous()) .build(); builder.addBulkhead(methodState.bulkhead); From d857591290ad6d5a05a7957f051be5fb8e6e7186 Mon Sep 17 00:00:00 2001 From: Santiago Pericasgeertsen Date: Thu, 3 Sep 2020 09:50:02 -0400 Subject: [PATCH 44/65] Switched composition of handlers for bulkhead and circuit breakers. Signed-off-by: Santiago Pericasgeertsen --- .../faulttolerance/MethodInvoker.java | 23 +++++++++---------- 1 file changed, 11 insertions(+), 12 deletions(-) diff --git a/microprofile/fault-tolerance/src/main/java/io/helidon/microprofile/faulttolerance/MethodInvoker.java b/microprofile/fault-tolerance/src/main/java/io/helidon/microprofile/faulttolerance/MethodInvoker.java index 914c5b6f210..21a53bad7c1 100644 --- a/microprofile/fault-tolerance/src/main/java/io/helidon/microprofile/faulttolerance/MethodInvoker.java +++ b/microprofile/fault-tolerance/src/main/java/io/helidon/microprofile/faulttolerance/MethodInvoker.java @@ -488,7 +488,7 @@ private FtSupplier requestContextSupplier(FtSupplier supplier) { /** * Creates a FT handler for an invocation by inspecting annotations. * - * - fallback(retry(bulkhead(circuitbreaker(timeout(method))))) + * - fallback(retry(circuitbreaker(bulkhead(timeout(method))))) * * @param methodState State related to this invocation's method. */ @@ -504,6 +504,16 @@ private FtHandlerTyped createMethodHandler(MethodState methodState) { builder.addTimeout(timeout); } + // Create and add bulkhead + if (introspector.hasBulkhead()) { + methodState.bulkhead = Bulkhead.builder() + .limit(introspector.getBulkhead().value()) + .queueLength(introspector.isAsynchronous() ? introspector.getBulkhead().waitingTaskQueue() : 0) + .async(introspector.isAsynchronous()) + .build(); + builder.addBulkhead(methodState.bulkhead); + } + // Create and add circuit breaker if (introspector.hasCircuitBreaker()) { methodState.breaker = CircuitBreaker.builder() @@ -518,17 +528,6 @@ private FtHandlerTyped createMethodHandler(MethodState methodState) { builder.addBreaker(methodState.breaker); } - // Create and add bulkhead - if (introspector.hasBulkhead()) { - methodState.bulkhead = Bulkhead.builder() - .limit(introspector.getBulkhead().value()) - .queueLength(introspector.isAsynchronous() ? introspector.getBulkhead().waitingTaskQueue() : 0) - .async(introspector.isAsynchronous()) - .build(); - builder.addBulkhead(methodState.bulkhead); - } - - // Create and add retry handler if (introspector.hasRetry()) { Retry retry = Retry.builder() From 182575c2d13b64331cbca4c13a4e0433124a7f98 Mon Sep 17 00:00:00 2001 From: Santiago Pericasgeertsen Date: Thu, 3 Sep 2020 11:58:27 -0400 Subject: [PATCH 45/65] Ensure proper access to context class loader to access application's config while running the TCKs. Signed-off-by: Santiago Pericasgeertsen --- .../microprofile/faulttolerance/FaultToleranceParameter.java | 3 ++- .../io/helidon/microprofile/faulttolerance/MethodInvoker.java | 4 +++- 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/microprofile/fault-tolerance/src/main/java/io/helidon/microprofile/faulttolerance/FaultToleranceParameter.java b/microprofile/fault-tolerance/src/main/java/io/helidon/microprofile/faulttolerance/FaultToleranceParameter.java index 1a1404f1f54..21736452d7b 100644 --- a/microprofile/fault-tolerance/src/main/java/io/helidon/microprofile/faulttolerance/FaultToleranceParameter.java +++ b/microprofile/fault-tolerance/src/main/java/io/helidon/microprofile/faulttolerance/FaultToleranceParameter.java @@ -53,7 +53,8 @@ static String getParameter(String annotationType, String parameter) { */ private static String getProperty(String name) { try { - String value = ConfigProvider.getConfig().getValue(name, String.class); + ClassLoader ccl = Thread.currentThread().getContextClassLoader(); + String value = ConfigProvider.getConfig(ccl).getValue(name, String.class); LOGGER.fine(() -> "Found config property '" + name + "' value '" + value + "'"); return value; } catch (NoSuchElementException e) { diff --git a/microprofile/fault-tolerance/src/main/java/io/helidon/microprofile/faulttolerance/MethodInvoker.java b/microprofile/fault-tolerance/src/main/java/io/helidon/microprofile/faulttolerance/MethodInvoker.java index 21a53bad7c1..61b3401f9e7 100644 --- a/microprofile/fault-tolerance/src/main/java/io/helidon/microprofile/faulttolerance/MethodInvoker.java +++ b/microprofile/fault-tolerance/src/main/java/io/helidon/microprofile/faulttolerance/MethodInvoker.java @@ -579,9 +579,11 @@ Supplier> toCompletionStageSupplier(FtSupplier // Wrap supplier with request context setup FtSupplier wrappedSupplier = requestContextSupplier(supplier); - // Invoke supplier in a new thread + // Invoke supplier in new thread and propagate ccl for config + ClassLoader ccl = Thread.currentThread().getContextClassLoader(); Single single = Async.create().invoke(() -> { try { + Thread.currentThread().setContextClassLoader(ccl); asyncInterruptThread = Thread.currentThread(); return wrappedSupplier.get(); } catch (Throwable t) { From cc51f55f34e1c645ff0a4ef8bbd27536318b53a6 Mon Sep 17 00:00:00 2001 From: Santiago Pericasgeertsen Date: Tue, 8 Sep 2020 15:12:42 -0400 Subject: [PATCH 46/65] Invert timeout and bulkhead handlers. Timeouts also apply to tasks waiting in bulkhead queue. Signed-off-by: Santiago Pericasgeertsen --- .../faulttolerance/MethodInvoker.java | 26 +++++++++++-------- 1 file changed, 15 insertions(+), 11 deletions(-) diff --git a/microprofile/fault-tolerance/src/main/java/io/helidon/microprofile/faulttolerance/MethodInvoker.java b/microprofile/fault-tolerance/src/main/java/io/helidon/microprofile/faulttolerance/MethodInvoker.java index 61b3401f9e7..c5b51b20a96 100644 --- a/microprofile/fault-tolerance/src/main/java/io/helidon/microprofile/faulttolerance/MethodInvoker.java +++ b/microprofile/fault-tolerance/src/main/java/io/helidon/microprofile/faulttolerance/MethodInvoker.java @@ -486,24 +486,19 @@ private FtSupplier requestContextSupplier(FtSupplier supplier) { } /** - * Creates a FT handler for an invocation by inspecting annotations. + * Creates a FT handler for an invocation by inspecting annotations. Handlers + * are composed as follows: * - * - fallback(retry(circuitbreaker(bulkhead(timeout(method))))) + * fallback(retry(circuitbreaker(timeout(bulkhead(method))))) + * + * Note that timeout includes the time an invocation may be queued in a + * bulkhead, so it needs to be before the bulkhead call. * * @param methodState State related to this invocation's method. */ private FtHandlerTyped createMethodHandler(MethodState methodState) { FaultTolerance.TypedBuilder builder = FaultTolerance.typedBuilder(); - // Create and add timeout handler - if (introspector.hasTimeout()) { - Timeout timeout = Timeout.builder() - .timeout(Duration.of(introspector.getTimeout().value(), introspector.getTimeout().unit())) - .currentThread(!introspector.isAsynchronous()) - .build(); - builder.addTimeout(timeout); - } - // Create and add bulkhead if (introspector.hasBulkhead()) { methodState.bulkhead = Bulkhead.builder() @@ -514,6 +509,15 @@ private FtHandlerTyped createMethodHandler(MethodState methodState) { builder.addBulkhead(methodState.bulkhead); } + // Create and add timeout handler + if (introspector.hasTimeout()) { + Timeout timeout = Timeout.builder() + .timeout(Duration.of(introspector.getTimeout().value(), introspector.getTimeout().unit())) + .currentThread(!introspector.isAsynchronous()) + .build(); + builder.addTimeout(timeout); + } + // Create and add circuit breaker if (introspector.hasCircuitBreaker()) { methodState.breaker = CircuitBreaker.builder() From b9830c16a767c0ed9eeedbe490462f65c5b60ae6 Mon Sep 17 00:00:00 2001 From: Santiago Pericasgeertsen Date: Tue, 8 Sep 2020 15:13:21 -0400 Subject: [PATCH 47/65] On timeout also cancel source single. Signed-off-by: Santiago Pericasgeertsen --- .../main/java/io/helidon/common/reactive/SingleTimeout.java | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/common/reactive/src/main/java/io/helidon/common/reactive/SingleTimeout.java b/common/reactive/src/main/java/io/helidon/common/reactive/SingleTimeout.java index afab5d7e45d..218c342d174 100644 --- a/common/reactive/src/main/java/io/helidon/common/reactive/SingleTimeout.java +++ b/common/reactive/src/main/java/io/helidon/common/reactive/SingleTimeout.java @@ -62,7 +62,7 @@ public void subscribe(Flow.Subscriber subscriber) { source.subscribe(parent); } - static final class TimeoutSubscriber extends DeferredScalarSubscription + final class TimeoutSubscriber extends DeferredScalarSubscription implements Flow.Subscriber, Callable { private final Single fallback; @@ -128,6 +128,7 @@ public Void call() { if (once.compareAndSet(false, true)) { future.lazySet(TerminatedFuture.FINISHED); SubscriptionHelper.cancel(upstream); + source.cancel(); if (fallback == null) { error(new TimeoutException()); } else { @@ -141,7 +142,7 @@ public void setFuture(Future f) { TerminatedFuture.setFuture(future, f); } - static final class FallbackSubscriber + final class FallbackSubscriber extends AtomicReference implements Flow.Subscriber { From c943a24d1962df292356fde0332d8aca7df39cf9 Mon Sep 17 00:00:00 2001 From: Santiago Pericasgeertsen Date: Tue, 8 Sep 2020 15:14:22 -0400 Subject: [PATCH 48/65] New instance of SingleNever for every invocation. It is otherwise possible for an old timeout to interrupt a new invocation on same thread. Signed-off-by: Santiago Pericasgeertsen --- .../helidon/faulttolerance/TimeoutImpl.java | 26 ++++++++++++++++++- 1 file changed, 25 insertions(+), 1 deletion(-) diff --git a/fault-tolerance/src/main/java/io/helidon/faulttolerance/TimeoutImpl.java b/fault-tolerance/src/main/java/io/helidon/faulttolerance/TimeoutImpl.java index 7bcab115ae1..2a8fafc5a05 100644 --- a/fault-tolerance/src/main/java/io/helidon/faulttolerance/TimeoutImpl.java +++ b/fault-tolerance/src/main/java/io/helidon/faulttolerance/TimeoutImpl.java @@ -28,6 +28,7 @@ import java.util.function.Supplier; import io.helidon.common.LazyValue; +import io.helidon.common.reactive.CompletionSingle; import io.helidon.common.reactive.Multi; import io.helidon.common.reactive.Single; @@ -74,7 +75,7 @@ public Single invoke(Supplier> supplier) { .build() .invoke(() -> { monitorStarted.complete(null); - return Single.never(); + return new TimeoutSingleNever(); // new instance }) .exceptionally(it -> { if (callReturned.compareAndSet(false, true)) { @@ -111,4 +112,27 @@ public Single invoke(Supplier> supplier) { return Single.create(future, true); } } + + /** + * Similar to {@link io.helidon.common.reactive.SingleNever} but not a singleton. + * When running multiple tests over the same thread, and old timeout that expires + * can interrupt a new task if using the same instance of this class. + */ + private static class TimeoutSingleNever extends CompletionSingle { + + TimeoutSingleNever() { + } + + @Override + public void subscribe(Flow.Subscriber subscriber) { + subscriber.onSubscribe(new Flow.Subscription() { + @Override + public void request(long n) { + } + @Override + public void cancel() { + } + }); + } + } } From b51b46c9a997562d12250c2e9a60299f49b6f842 Mon Sep 17 00:00:00 2001 From: Santiago Pericasgeertsen Date: Tue, 8 Sep 2020 15:15:59 -0400 Subject: [PATCH 49/65] Clean up tck-suite file. Signed-off-by: Santiago Pericasgeertsen --- .../src/test/tck-suite.xml | 49 +------------------ 1 file changed, 1 insertion(+), 48 deletions(-) diff --git a/microprofile/tests/tck/tck-fault-tolerance/src/test/tck-suite.xml b/microprofile/tests/tck/tck-fault-tolerance/src/test/tck-suite.xml index d726636d795..58f4f619255 100644 --- a/microprofile/tests/tck/tck-fault-tolerance/src/test/tck-suite.xml +++ b/microprofile/tests/tck/tck-fault-tolerance/src/test/tck-suite.xml @@ -23,56 +23,9 @@ - system like those in our CI/CD pipeline. - No longer commented out - use 'tck-ft' profile to run these tests --> - + - - - - From 8f502f888322eccf3c93968e328dfcde75f35f10 Mon Sep 17 00:00:00 2001 From: Santiago Pericasgeertsen Date: Tue, 8 Sep 2020 15:34:39 -0400 Subject: [PATCH 50/65] Fixed copyright and checkstyle issues. Signed-off-by: Santiago Pericasgeertsen --- .../io/helidon/faulttolerance/FaultTolerance.java | 1 + .../faulttolerance/FaultToleranceParameter.java | 2 +- .../microprofile/faulttolerance/MethodInvoker.java | 12 ++++++------ .../faulttolerance/AsynchronousBean.java | 2 +- .../faulttolerance/AsynchronousTest.java | 2 +- .../faulttolerance/CircuitBreakerBean.java | 2 +- .../microprofile/faulttolerance/MetricsBean.java | 2 +- .../microprofile/faulttolerance/RetryBean.java | 2 +- 8 files changed, 13 insertions(+), 12 deletions(-) diff --git a/fault-tolerance/src/main/java/io/helidon/faulttolerance/FaultTolerance.java b/fault-tolerance/src/main/java/io/helidon/faulttolerance/FaultTolerance.java index ad6b28ed255..38e401fc104 100644 --- a/fault-tolerance/src/main/java/io/helidon/faulttolerance/FaultTolerance.java +++ b/fault-tolerance/src/main/java/io/helidon/faulttolerance/FaultTolerance.java @@ -124,6 +124,7 @@ public static Builder builder() { /** * A typed builder to configure a customized sequence of fault tolerance handlers. * + * @param type of result * @return a new builder */ public static TypedBuilder typedBuilder() { diff --git a/microprofile/fault-tolerance/src/main/java/io/helidon/microprofile/faulttolerance/FaultToleranceParameter.java b/microprofile/fault-tolerance/src/main/java/io/helidon/microprofile/faulttolerance/FaultToleranceParameter.java index 21736452d7b..fe74a93e3fe 100644 --- a/microprofile/fault-tolerance/src/main/java/io/helidon/microprofile/faulttolerance/FaultToleranceParameter.java +++ b/microprofile/fault-tolerance/src/main/java/io/helidon/microprofile/faulttolerance/FaultToleranceParameter.java @@ -1,5 +1,5 @@ /* - * Copyright (c) 2018 Oracle and/or its affiliates. All rights reserved. + * Copyright (c) 2018, 2020 Oracle and/or its affiliates. All rights reserved. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/microprofile/fault-tolerance/src/main/java/io/helidon/microprofile/faulttolerance/MethodInvoker.java b/microprofile/fault-tolerance/src/main/java/io/helidon/microprofile/faulttolerance/MethodInvoker.java index c5b51b20a96..e33c0720b12 100644 --- a/microprofile/fault-tolerance/src/main/java/io/helidon/microprofile/faulttolerance/MethodInvoker.java +++ b/microprofile/fault-tolerance/src/main/java/io/helidon/microprofile/faulttolerance/MethodInvoker.java @@ -54,10 +54,7 @@ import org.glassfish.jersey.process.internal.RequestContext; import org.glassfish.jersey.process.internal.RequestScope; -import static io.helidon.microprofile.faulttolerance.ThrowableMapper.mapTypes; import static io.helidon.microprofile.faulttolerance.FaultToleranceExtension.isFaultToleranceMetricsEnabled; -import static io.helidon.microprofile.faulttolerance.FaultToleranceMetrics.METRIC_NAME_TEMPLATE; -import static io.helidon.microprofile.faulttolerance.FaultToleranceMetrics.registerHistogram; import static io.helidon.microprofile.faulttolerance.FaultToleranceMetrics.BREAKER_CALLS_FAILED_TOTAL; import static io.helidon.microprofile.faulttolerance.FaultToleranceMetrics.BREAKER_CALLS_PREVENTED_TOTAL; import static io.helidon.microprofile.faulttolerance.FaultToleranceMetrics.BREAKER_CALLS_SUCCEEDED_TOTAL; @@ -73,6 +70,7 @@ import static io.helidon.microprofile.faulttolerance.FaultToleranceMetrics.BULKHEAD_WAITING_QUEUE_POPULATION; import static io.helidon.microprofile.faulttolerance.FaultToleranceMetrics.INVOCATIONS_FAILED_TOTAL; import static io.helidon.microprofile.faulttolerance.FaultToleranceMetrics.INVOCATIONS_TOTAL; +import static io.helidon.microprofile.faulttolerance.FaultToleranceMetrics.METRIC_NAME_TEMPLATE; import static io.helidon.microprofile.faulttolerance.FaultToleranceMetrics.RETRY_CALLS_FAILED_TOTAL; import static io.helidon.microprofile.faulttolerance.FaultToleranceMetrics.RETRY_CALLS_SUCCEEDED_NOT_RETRIED_TOTAL; import static io.helidon.microprofile.faulttolerance.FaultToleranceMetrics.RETRY_CALLS_SUCCEEDED_RETRIED_TOTAL; @@ -83,7 +81,9 @@ import static io.helidon.microprofile.faulttolerance.FaultToleranceMetrics.getCounter; import static io.helidon.microprofile.faulttolerance.FaultToleranceMetrics.getHistogram; import static io.helidon.microprofile.faulttolerance.FaultToleranceMetrics.registerGauge; +import static io.helidon.microprofile.faulttolerance.FaultToleranceMetrics.registerHistogram; import static io.helidon.microprofile.faulttolerance.ThrowableMapper.map; +import static io.helidon.microprofile.faulttolerance.ThrowableMapper.mapTypes; /** * Invokes a FT method applying semantics based on method annotations. An instance @@ -199,9 +199,9 @@ public boolean equals(Object o) { return false; } MethodStateKey that = (MethodStateKey) o; - return classLoader.equals(that.classLoader) && - methodClass.equals(that.methodClass) && - method.equals(that.method); + return classLoader.equals(that.classLoader) + && methodClass.equals(that.methodClass) + && method.equals(that.method); } @Override diff --git a/microprofile/fault-tolerance/src/test/java/io/helidon/microprofile/faulttolerance/AsynchronousBean.java b/microprofile/fault-tolerance/src/test/java/io/helidon/microprofile/faulttolerance/AsynchronousBean.java index 1103e395f39..5642b8a368b 100644 --- a/microprofile/fault-tolerance/src/test/java/io/helidon/microprofile/faulttolerance/AsynchronousBean.java +++ b/microprofile/fault-tolerance/src/test/java/io/helidon/microprofile/faulttolerance/AsynchronousBean.java @@ -1,5 +1,5 @@ /* - * Copyright (c) 2018, 2019 Oracle and/or its affiliates. All rights reserved. + * Copyright (c) 2018, 2020 Oracle and/or its affiliates. All rights reserved. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/microprofile/fault-tolerance/src/test/java/io/helidon/microprofile/faulttolerance/AsynchronousTest.java b/microprofile/fault-tolerance/src/test/java/io/helidon/microprofile/faulttolerance/AsynchronousTest.java index 89f6b1c24f3..3b45619376c 100644 --- a/microprofile/fault-tolerance/src/test/java/io/helidon/microprofile/faulttolerance/AsynchronousTest.java +++ b/microprofile/fault-tolerance/src/test/java/io/helidon/microprofile/faulttolerance/AsynchronousTest.java @@ -1,5 +1,5 @@ /* - * Copyright (c) 2018, 2019 Oracle and/or its affiliates. All rights reserved. + * Copyright (c) 2018, 2020 Oracle and/or its affiliates. All rights reserved. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/microprofile/fault-tolerance/src/test/java/io/helidon/microprofile/faulttolerance/CircuitBreakerBean.java b/microprofile/fault-tolerance/src/test/java/io/helidon/microprofile/faulttolerance/CircuitBreakerBean.java index d564fdc3146..f903b0bcee9 100644 --- a/microprofile/fault-tolerance/src/test/java/io/helidon/microprofile/faulttolerance/CircuitBreakerBean.java +++ b/microprofile/fault-tolerance/src/test/java/io/helidon/microprofile/faulttolerance/CircuitBreakerBean.java @@ -1,5 +1,5 @@ /* - * Copyright (c) 2018, 2019 Oracle and/or its affiliates. All rights reserved. + * Copyright (c) 2018, 2020 Oracle and/or its affiliates. All rights reserved. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/microprofile/fault-tolerance/src/test/java/io/helidon/microprofile/faulttolerance/MetricsBean.java b/microprofile/fault-tolerance/src/test/java/io/helidon/microprofile/faulttolerance/MetricsBean.java index 67e312328c5..efa91f51473 100644 --- a/microprofile/fault-tolerance/src/test/java/io/helidon/microprofile/faulttolerance/MetricsBean.java +++ b/microprofile/fault-tolerance/src/test/java/io/helidon/microprofile/faulttolerance/MetricsBean.java @@ -1,5 +1,5 @@ /* - * Copyright (c) 2018, 2019 Oracle and/or its affiliates. All rights reserved. + * Copyright (c) 2018, 2020 Oracle and/or its affiliates. All rights reserved. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/microprofile/fault-tolerance/src/test/java/io/helidon/microprofile/faulttolerance/RetryBean.java b/microprofile/fault-tolerance/src/test/java/io/helidon/microprofile/faulttolerance/RetryBean.java index e7ee1113d64..9a12cc39497 100644 --- a/microprofile/fault-tolerance/src/test/java/io/helidon/microprofile/faulttolerance/RetryBean.java +++ b/microprofile/fault-tolerance/src/test/java/io/helidon/microprofile/faulttolerance/RetryBean.java @@ -1,5 +1,5 @@ /* - * Copyright (c) 2018, 2019 Oracle and/or its affiliates. All rights reserved. + * Copyright (c) 2018, 2020 Oracle and/or its affiliates. All rights reserved. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. From c79c1f9e8a3f5cf39f3f917f87418ed1c4089c4d Mon Sep 17 00:00:00 2001 From: Santiago Pericasgeertsen Date: Wed, 9 Sep 2020 11:03:42 -0400 Subject: [PATCH 51/65] Revert "On timeout also cancel source single." Signed-off-by: Santiago Pericasgeertsen --- .../main/java/io/helidon/common/reactive/SingleTimeout.java | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/common/reactive/src/main/java/io/helidon/common/reactive/SingleTimeout.java b/common/reactive/src/main/java/io/helidon/common/reactive/SingleTimeout.java index 218c342d174..afab5d7e45d 100644 --- a/common/reactive/src/main/java/io/helidon/common/reactive/SingleTimeout.java +++ b/common/reactive/src/main/java/io/helidon/common/reactive/SingleTimeout.java @@ -62,7 +62,7 @@ public void subscribe(Flow.Subscriber subscriber) { source.subscribe(parent); } - final class TimeoutSubscriber extends DeferredScalarSubscription + static final class TimeoutSubscriber extends DeferredScalarSubscription implements Flow.Subscriber, Callable { private final Single fallback; @@ -128,7 +128,6 @@ public Void call() { if (once.compareAndSet(false, true)) { future.lazySet(TerminatedFuture.FINISHED); SubscriptionHelper.cancel(upstream); - source.cancel(); if (fallback == null) { error(new TimeoutException()); } else { @@ -142,7 +141,7 @@ public void setFuture(Future f) { TerminatedFuture.setFuture(future, f); } - final class FallbackSubscriber + static final class FallbackSubscriber extends AtomicReference implements Flow.Subscriber { From b72ecc0a3ffa5d9c245ccb00ab7841c1d123e021 Mon Sep 17 00:00:00 2001 From: Santiago Pericasgeertsen Date: Wed, 9 Sep 2020 11:05:06 -0400 Subject: [PATCH 52/65] Make sure source is cancelled when multi is cancelled. Signed-off-by: Santiago Pericasgeertsen --- .../helidon/common/reactive/MultiFromCompletionStage.java | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/common/reactive/src/main/java/io/helidon/common/reactive/MultiFromCompletionStage.java b/common/reactive/src/main/java/io/helidon/common/reactive/MultiFromCompletionStage.java index 6c79510d6a0..c9ccb8ab7ed 100644 --- a/common/reactive/src/main/java/io/helidon/common/reactive/MultiFromCompletionStage.java +++ b/common/reactive/src/main/java/io/helidon/common/reactive/MultiFromCompletionStage.java @@ -42,7 +42,7 @@ public void subscribe(Flow.Subscriber subscriber) { static void subscribe(Flow.Subscriber subscriber, CompletionStage source, boolean nullMeansEmpty) { AtomicBiConsumer watcher = new AtomicBiConsumer<>(); - CompletionStageSubscription css = new CompletionStageSubscription<>(subscriber, nullMeansEmpty, watcher); + CompletionStageSubscription css = new CompletionStageSubscription<>(subscriber, nullMeansEmpty, watcher, source); watcher.lazySet(css); subscriber.onSubscribe(css); @@ -55,10 +55,12 @@ static final class CompletionStageSubscription extends DeferredScalarSubscrip private final AtomicBiConsumer watcher; - CompletionStageSubscription(Flow.Subscriber downstream, boolean nullMeansEmpty, AtomicBiConsumer watcher) { + private CompletionStage source; + CompletionStageSubscription(Flow.Subscriber downstream, boolean nullMeansEmpty, AtomicBiConsumer watcher,CompletionStage source) { super(downstream); this.nullMeansEmpty = nullMeansEmpty; this.watcher = watcher; + this.source = source; } @Override @@ -77,6 +79,7 @@ public void accept(T t, Throwable throwable) { @Override public void cancel() { super.cancel(); + source.toCompletableFuture().cancel(true); watcher.getAndSet(null); } } From 0a0e151fd7c3561807fbeac5a431cfed1f8035e0 Mon Sep 17 00:00:00 2001 From: Santiago Pericasgeertsen Date: Wed, 9 Sep 2020 11:38:54 -0400 Subject: [PATCH 53/65] Remove old files. Signed-off-by: Santiago Pericasgeertsen --- .../tck/tck-fault-tolerance/failures.txt | 106 ------------------ .../src/test/resources/application.yaml | 21 ---- 2 files changed, 127 deletions(-) delete mode 100644 microprofile/tests/tck/tck-fault-tolerance/failures.txt delete mode 100644 microprofile/tests/tck/tck-fault-tolerance/src/test/resources/application.yaml diff --git a/microprofile/tests/tck/tck-fault-tolerance/failures.txt b/microprofile/tests/tck/tck-fault-tolerance/failures.txt deleted file mode 100644 index dbf15358fba..00000000000 --- a/microprofile/tests/tck/tck-fault-tolerance/failures.txt +++ /dev/null @@ -1,106 +0,0 @@ -Failed tests: - CircuitBreakerBulkheadTest>Arquillian.run:138->testCircuitBreakerAroundBulkheadAsync:126 Unexpected exception thrown from Future - CircuitBreakerBulkheadTest>Arquillian.run:138->testCircuitBreakerAroundBulkheadSync:181 Unexpected exception thrown from Future - TimeoutUninterruptableTest>Arquillian.run:138->testTimeoutAsyncBulkhead:185 Execution count after second call expected [1] but found [2] - TimeoutUninterruptableTest>Arquillian.run:138->testTimeoutAsyncBulkheadQueueTimed:237 Time taken for call B to timeout -Expected: a value less than - but: was greater than - BulkheadSynchConfigTest>Arquillian.run:138->testBulkheadClassSemaphore3:83 Work is not being done simultaneously enough, only 3 workers at once. Expecting 5. expected [true] but found [false] - BulkheadSynchTest>Arquillian.run:138->testBulkheadExceptionThrownWhenQueueFullSemaphore:199 Unable to clean up all tasks - FallbackMethodOutOfPackageTest>Arquillian.arquillianBeforeClass:96 ? Runtime E... - FallbackMethodSubclassTest>Arquillian.arquillianBeforeClass:96 ? Runtime Expec... - FallbackMethodSuperclassPrivateTest>Arquillian.arquillianBeforeClass:96 ? Runtime - FallbackMethodWildcardNegativeTest>Arquillian.arquillianBeforeClass:96 ? Runtime - IncompatibleFallbackMethodTest>Arquillian.arquillianBeforeClass:96 ? Runtime E... - IncompatibleFallbackMethodWithArgsTest>Arquillian.arquillianBeforeClass:96 ? Runtime - IncompatibleFallbackPolicies>Arquillian.arquillianBeforeClass:96 ? Runtime Exp... - IncompatibleFallbackTest>Arquillian.arquillianBeforeClass:96 ? Runtime Expecte... - InvalidAsynchronousClassTest>Arquillian.arquillianBeforeClass:96 ? Runtime Exp... - InvalidAsynchronousMethodTest>Arquillian.arquillianBeforeClass:96 ? Runtime Ex... - InvalidBulkheadAsynchQueueTest>Arquillian.arquillianBeforeClass:96 ? Runtime E... - InvalidBulkheadValueTest>Arquillian.arquillianBeforeClass:96 ? Runtime Expecte... - InvalidCircuitBreakerDelayTest>Arquillian.arquillianBeforeClass:96 ? Runtime E... - InvalidCircuitBreakerFailureRatioNegTest>Arquillian.arquillianBeforeClass:96 ? Runtime - InvalidCircuitBreakerFailureRatioPosTest>Arquillian.arquillianBeforeClass:96 ? Runtime - InvalidCircuitBreakerFailureReqVol0Test>Arquillian.arquillianBeforeClass:96 ? Runtime - InvalidCircuitBreakerFailureReqVolNegTest>Arquillian.arquillianBeforeClass:96 ? Runtime - InvalidCircuitBreakerFailureSuccess0Test>Arquillian.arquillianBeforeClass:96 ? Runtime - InvalidCircuitBreakerFailureSuccessNegTest>Arquillian.arquillianBeforeClass:96 ? Runtime - InvalidRetryDelayDurationTest>Arquillian.arquillianBeforeClass:96 ? Runtime Ex... - InvalidRetryDelayTest>Arquillian.arquillianBeforeClass:96 ? Runtime Expected e... - InvalidRetryJitterTest>Arquillian.arquillianBeforeClass:96 ? Runtime Expected ... - InvalidRetryMaxRetriesTest>Arquillian.arquillianBeforeClass:96 ? Runtime Expec... - InvalidTimeoutValueTest>Arquillian.arquillianBeforeClass:96 ? Runtime Expected... - BulkheadMetricTest>Arquillian.run:138->bulkheadMetricAsyncTest:244 queue wait histogram counts -Expected: is <4L> - but: was <5L> - BulkheadMetricTest>Arquillian.run:138->bulkheadMetricHistogramTest:184 histogram count -Expected: is <2L> - but: was <3L> - BulkheadMetricTest>Arquillian.run:138->bulkheadMetricTest:122 bulkhead wait time histogram present -Expected: is - but: was - CircuitBreakerMetricTest>Arquillian.run:138->testCircuitBreakerMetric:136 circuitbreaker calls prevented -Expected: is <1L> - but: was <3L> - FallbackMetricTest>Arquillian.run:138->fallbackMetricHandlerTest:93 failed invocations -Expected: is <0L> - but: was <-1L> - FallbackMetricTest>Arquillian.run:138->fallbackMetricMethodTest:67 failed invocations -Expected: is <0L> - but: was <-1L> - RetryVisibilityTest>Arquillian.run:138->serviceOverrideClassLevelUsesClassLevelAnnotation:134->checkServiceCall:248 in RetryVisibilityTest#serviceOverrideClassLevelUsesClassLevelAnnotation service() should have been called exactly 5 times expected [5] but found [4] - -Tests run: 455, Failures: 37, Errors: 0, Skipped: 48 - ---- - -Failed tests: - CircuitBreakerBulkheadTest>Arquillian.run:138->testCircuitBreakerAroundBulkheadAsync:126 Unexpected exception thrown from Future - CircuitBreakerBulkheadTest>Arquillian.run:138->testCircuitBreakerAroundBulkheadSync:181 Unexpected exception thrown from Future - TimeoutUninterruptableTest>Arquillian.run:138->testTimeoutAsyncBulkhead:185 Execution count after second call expected [1] but found [2] - TimeoutUninterruptableTest>Arquillian.run:138->testTimeoutAsyncBulkheadQueueTimed:237 Time taken for call B to timeout -Expected: a value less than - but: was greater than - BulkheadSynchConfigTest>Arquillian.run:138->testBulkheadClassSemaphore3:83 Work is not being done simultaneously enough, only 3 workers at once. Expecting 5. expected [true] but found [false] - BulkheadSynchTest>Arquillian.run:138->testBulkheadExceptionThrownWhenQueueFullSemaphore:199 Unable to clean up all tasks - BulkheadMetricTest>Arquillian.run:138->bulkheadMetricAsyncTest:244 queue wait histogram counts -Expected: is <4L> - but: was <5L> - BulkheadMetricTest>Arquillian.run:138->bulkheadMetricHistogramTest:184 histogram count -Expected: is <2L> - but: was <3L> - BulkheadMetricTest>Arquillian.run:138->bulkheadMetricTest:122 bulkhead wait time histogram present -Expected: is - but: was - - - CircuitBreakerMetricTest>Arquillian.run:138->testCircuitBreakerMetric:136 circuitbreaker calls prevented -Expected: is <1L> - but: was <3L> - FallbackMetricTest>Arquillian.run:138->fallbackMetricHandlerTest:93 failed invocations -Expected: is <0L> - but: was <-1L> - FallbackMetricTest>Arquillian.run:138->fallbackMetricMethodTest:67 failed invocations -Expected: is <0L> - but: was <-1L> - RetryVisibilityTest>Arquillian.run:138->serviceOverrideClassLevelUsesClassLevelAnnotation:134->checkServiceCall:248 in RetryVisibilityTest#serviceOverrideClassLevelUsesClassLevelAnnotation service() should have been called exactly 5 times expected [5] but found [4] - -Tests run: 407, Failures: 13, Errors: 0, Skipped: 0 - --- - -Results : - -Failed tests: - CircuitBreakerBulkheadTest>Arquillian.run:138->testCircuitBreakerAroundBulkheadAsync:126 Unexpected exception thrown from Future - CircuitBreakerBulkheadTest>Arquillian.run:138->testCircuitBreakerAroundBulkheadSync:181 Unexpected exception thrown from Future - TimeoutUninterruptableTest>Arquillian.run:138->testTimeoutAsyncBulkhead:181 Execution count after second call expected [1] but found [2] - TimeoutUninterruptableTest>Arquillian.run:138->testTimeoutAsyncBulkheadQueueTimed:233 Time taken for call B to timeout -Expected: a value less than - but: was greater than - BulkheadSynchConfigTest>Arquillian.run:138->testBulkheadClassSemaphore3:83 Work is not being done simultaneously enough, only 3 workers at once. Expecting 5. expected [true] but found [false] - BulkheadSynchTest>Arquillian.run:138->testBulkheadExceptionThrownWhenQueueFullSemaphore:199 Unable to clean up all tasks - RetryVisibilityTest>Arquillian.run:138->serviceOverrideClassLevelUsesClassLevelAnnotation:134->checkServiceCall:248 in RetryVisibilityTest#serviceOverrideClassLevelUsesClassLevelAnnotation service() should have been called exactly 5 times expected [5] but found [4] - -Tests run: 407, Failures: 7, Errors: 0, Skipped: 0 diff --git a/microprofile/tests/tck/tck-fault-tolerance/src/test/resources/application.yaml b/microprofile/tests/tck/tck-fault-tolerance/src/test/resources/application.yaml deleted file mode 100644 index 5bad4af9fec..00000000000 --- a/microprofile/tests/tck/tck-fault-tolerance/src/test/resources/application.yaml +++ /dev/null @@ -1,21 +0,0 @@ -# -# Copyright (c) 2020 Oracle and/or its affiliates. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -# -# Disable caching since multiple apps are executed in same Java VM -# -fault-tolerance: - disableCaching: false From cfd6fe7f06984982198778fefd56d0eb86dbef31 Mon Sep 17 00:00:00 2001 From: Santiago Pericasgeertsen Date: Wed, 9 Sep 2020 13:18:56 -0400 Subject: [PATCH 54/65] Async flag is no longer needed. Signed-off-by: Santiago Pericasgeertsen --- .../java/io/helidon/faulttolerance/Bulkhead.java | 16 ---------------- .../io/helidon/faulttolerance/BulkheadImpl.java | 6 ++---- .../faulttolerance/MethodInvoker.java | 1 - 3 files changed, 2 insertions(+), 21 deletions(-) diff --git a/fault-tolerance/src/main/java/io/helidon/faulttolerance/Bulkhead.java b/fault-tolerance/src/main/java/io/helidon/faulttolerance/Bulkhead.java index 9d28fab844e..374d371e104 100644 --- a/fault-tolerance/src/main/java/io/helidon/faulttolerance/Bulkhead.java +++ b/fault-tolerance/src/main/java/io/helidon/faulttolerance/Bulkhead.java @@ -52,7 +52,6 @@ class Builder implements io.helidon.common.Builder { private int limit = DEFAULT_LIMIT; private int queueLength = DEFAULT_QUEUE_LENGTH; private String name = "Bulkhead-" + System.identityHashCode(this); - private boolean async = true; private Builder() { } @@ -109,17 +108,6 @@ public Builder name(String name) { return this; } - /** - * Runs bulkhead tasks using an executor. By default is set to {@code true}. - * - * @param async setting for async - * @return updated builder instance - */ - public Builder async(boolean async) { - this.async = async; - return this; - } - int limit() { return limit; } @@ -135,10 +123,6 @@ LazyValue executor() { String name() { return name; } - - boolean async() { - return async; - } } interface Stats { diff --git a/fault-tolerance/src/main/java/io/helidon/faulttolerance/BulkheadImpl.java b/fault-tolerance/src/main/java/io/helidon/faulttolerance/BulkheadImpl.java index 39ec0b41e24..ea4eb995446 100644 --- a/fault-tolerance/src/main/java/io/helidon/faulttolerance/BulkheadImpl.java +++ b/fault-tolerance/src/main/java/io/helidon/faulttolerance/BulkheadImpl.java @@ -38,7 +38,6 @@ class BulkheadImpl implements Bulkhead { private final Queue> queue; private final Semaphore inProgress; private final String name; - private final boolean async; private final AtomicLong concurrentExecutions = new AtomicLong(0L); private final AtomicLong callsAccepted = new AtomicLong(0L); @@ -48,7 +47,6 @@ class BulkheadImpl implements Bulkhead { this.executor = builder.executor(); this.inProgress = new Semaphore(builder.limit(), true); this.name = builder.name(); - this.async = builder.async(); if (builder.queueLength() == 0) { queue = new NoQueue(); @@ -102,13 +100,13 @@ private R invokeTask(DelayedTask task) { return task.result(); } else { // no free permit, let's try to enqueue in async mode - if (async && queue.offer(task)) { + if (queue.offer(task)) { LOGGER.finest(() -> name + " enqueue: " + task); R result = task.result(); if (result instanceof Single) { Single single = (Single) result; - return (R) single.onCancel(queue::remove); + return (R) single.onCancel(() -> queue.remove(task)); } return result; } else { diff --git a/microprofile/fault-tolerance/src/main/java/io/helidon/microprofile/faulttolerance/MethodInvoker.java b/microprofile/fault-tolerance/src/main/java/io/helidon/microprofile/faulttolerance/MethodInvoker.java index e33c0720b12..bc5c29affdd 100644 --- a/microprofile/fault-tolerance/src/main/java/io/helidon/microprofile/faulttolerance/MethodInvoker.java +++ b/microprofile/fault-tolerance/src/main/java/io/helidon/microprofile/faulttolerance/MethodInvoker.java @@ -504,7 +504,6 @@ private FtHandlerTyped createMethodHandler(MethodState methodState) { methodState.bulkhead = Bulkhead.builder() .limit(introspector.getBulkhead().value()) .queueLength(introspector.isAsynchronous() ? introspector.getBulkhead().waitingTaskQueue() : 0) - .async(introspector.isAsynchronous()) .build(); builder.addBulkhead(methodState.bulkhead); } From 7de6fbcf64080dbba1c19338e4c953fb6740fe44 Mon Sep 17 00:00:00 2001 From: Santiago Pericasgeertsen Date: Wed, 9 Sep 2020 13:20:22 -0400 Subject: [PATCH 55/65] Updated comment. Signed-off-by: Santiago Pericasgeertsen --- .../src/main/java/io/helidon/faulttolerance/BulkheadImpl.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/fault-tolerance/src/main/java/io/helidon/faulttolerance/BulkheadImpl.java b/fault-tolerance/src/main/java/io/helidon/faulttolerance/BulkheadImpl.java index ea4eb995446..59c2f72d5d1 100644 --- a/fault-tolerance/src/main/java/io/helidon/faulttolerance/BulkheadImpl.java +++ b/fault-tolerance/src/main/java/io/helidon/faulttolerance/BulkheadImpl.java @@ -99,7 +99,7 @@ private R invokeTask(DelayedTask task) { execute(task); return task.result(); } else { - // no free permit, let's try to enqueue in async mode + // no free permit, let's try to enqueue if (queue.offer(task)) { LOGGER.finest(() -> name + " enqueue: " + task); From 668d11485840f515b63cd773f1b72cdf4902d37b Mon Sep 17 00:00:00 2001 From: Santiago Pericasgeertsen Date: Wed, 9 Sep 2020 13:24:46 -0400 Subject: [PATCH 56/65] Removed unused listener support. Signed-off-by: Santiago Pericasgeertsen --- .../java/io/helidon/faulttolerance/Retry.java | 17 ----------------- .../io/helidon/faulttolerance/RetryImpl.java | 8 -------- 2 files changed, 25 deletions(-) diff --git a/fault-tolerance/src/main/java/io/helidon/faulttolerance/Retry.java b/fault-tolerance/src/main/java/io/helidon/faulttolerance/Retry.java index 2ca5886d0c1..811246772a9 100644 --- a/fault-tolerance/src/main/java/io/helidon/faulttolerance/Retry.java +++ b/fault-tolerance/src/main/java/io/helidon/faulttolerance/Retry.java @@ -61,7 +61,6 @@ class Builder implements io.helidon.common.Builder { private Duration overallTimeout = Duration.ofSeconds(1); private LazyValue scheduledExecutor = FaultTolerance.scheduledExecutor(); - private Consumer retryListener; private Builder() { } @@ -166,18 +165,6 @@ public Builder overallTimeout(Duration overallTimeout) { return this; } - /** - * A listener that is called every time there is a retry. The parameter - * to the listener is a retry count. - * - * @param retryListener a retry listener - * @return updated builder instance - */ - public Builder retryListener(Consumer retryListener) { - this.retryListener = retryListener; - return this; - } - Set> applyOn() { return applyOn; } @@ -197,10 +184,6 @@ Duration overallTimeout() { LazyValue scheduledExecutor() { return scheduledExecutor; } - - Consumer retryListener() { - return retryListener; - } } /** diff --git a/fault-tolerance/src/main/java/io/helidon/faulttolerance/RetryImpl.java b/fault-tolerance/src/main/java/io/helidon/faulttolerance/RetryImpl.java index 18075f2838c..0421da178d3 100644 --- a/fault-tolerance/src/main/java/io/helidon/faulttolerance/RetryImpl.java +++ b/fault-tolerance/src/main/java/io/helidon/faulttolerance/RetryImpl.java @@ -39,14 +39,12 @@ class RetryImpl implements Retry { private final long maxTimeNanos; private final Retry.RetryPolicy retryPolicy; private final AtomicLong retryCounter = new AtomicLong(0L); - private final Consumer retryListener; RetryImpl(Retry.Builder builder) { this.scheduledExecutor = builder.scheduledExecutor(); this.errorChecker = ErrorChecker.create(builder.skipOn(), builder.applyOn()); this.maxTimeNanos = builder.overallTimeout().toNanos(); this.retryPolicy = builder.retryPolicy(); - this.retryListener = builder.retryListener(); } @Override @@ -97,9 +95,6 @@ private Single retrySingle(RetryContext> con if (errorChecker.shouldSkip(cause)) { return Single.error(context.throwable()); } - if (retryListener != null) { - retryListener.accept(currentCallIndex); - } return retrySingle(context); }); } @@ -142,9 +137,6 @@ private Multi retryMulti(RetryContext> contex if (task.hadData() || errorChecker.shouldSkip(cause)) { return Multi.error(context.throwable()); } - if (retryListener != null) { - retryListener.accept(currentCallIndex); - } return retryMulti(context); }); } From f0c6cef9730689637b4908cd0e716d5b66eb61c7 Mon Sep 17 00:00:00 2001 From: Santiago Pericasgeertsen Date: Wed, 9 Sep 2020 13:38:12 -0400 Subject: [PATCH 57/65] More cleanup of unused flags. Signed-off-by: Santiago Pericasgeertsen --- .../helidon/faulttolerance/BulkheadImpl.java | 4 ---- .../java/io/helidon/faulttolerance/Retry.java | 1 - .../io/helidon/faulttolerance/RetryImpl.java | 1 - .../io/helidon/faulttolerance/Timeout.java | 21 ++----------------- .../helidon/faulttolerance/TimeoutImpl.java | 6 ------ 5 files changed, 2 insertions(+), 31 deletions(-) diff --git a/fault-tolerance/src/main/java/io/helidon/faulttolerance/BulkheadImpl.java b/fault-tolerance/src/main/java/io/helidon/faulttolerance/BulkheadImpl.java index 59c2f72d5d1..6a46b163bf7 100644 --- a/fault-tolerance/src/main/java/io/helidon/faulttolerance/BulkheadImpl.java +++ b/fault-tolerance/src/main/java/io/helidon/faulttolerance/BulkheadImpl.java @@ -102,7 +102,6 @@ private R invokeTask(DelayedTask task) { // no free permit, let's try to enqueue if (queue.offer(task)) { LOGGER.finest(() -> name + " enqueue: " + task); - R result = task.result(); if (result instanceof Single) { Single single = (Single) result; @@ -111,7 +110,6 @@ private R invokeTask(DelayedTask task) { return result; } else { LOGGER.finest(() -> name + " reject: " + task); - callsRejected.incrementAndGet(); return task.error(new BulkheadException("Bulkhead queue \"" + name + "\" is full")); } @@ -132,12 +130,10 @@ private void execute(DelayedTask task) { DelayedTask polled = queue.poll(); if (polled != null) { LOGGER.finest(() -> name + " invoke in executor: " + polled); - // chain executions from queue until all are executed executor.get().submit(() -> execute(polled)); } else { LOGGER.finest(() -> name + " permit released after: " + task); - // nothing in the queue, release permit inProgress.release(); } diff --git a/fault-tolerance/src/main/java/io/helidon/faulttolerance/Retry.java b/fault-tolerance/src/main/java/io/helidon/faulttolerance/Retry.java index 811246772a9..a41e342de92 100644 --- a/fault-tolerance/src/main/java/io/helidon/faulttolerance/Retry.java +++ b/fault-tolerance/src/main/java/io/helidon/faulttolerance/Retry.java @@ -23,7 +23,6 @@ import java.util.Random; import java.util.Set; import java.util.concurrent.ScheduledExecutorService; -import java.util.function.Consumer; import java.util.function.Supplier; import io.helidon.common.LazyValue; diff --git a/fault-tolerance/src/main/java/io/helidon/faulttolerance/RetryImpl.java b/fault-tolerance/src/main/java/io/helidon/faulttolerance/RetryImpl.java index 0421da178d3..2d5bb3c3fee 100644 --- a/fault-tolerance/src/main/java/io/helidon/faulttolerance/RetryImpl.java +++ b/fault-tolerance/src/main/java/io/helidon/faulttolerance/RetryImpl.java @@ -26,7 +26,6 @@ import java.util.concurrent.TimeoutException; import java.util.concurrent.atomic.AtomicInteger; import java.util.concurrent.atomic.AtomicLong; -import java.util.function.Consumer; import java.util.function.Supplier; import io.helidon.common.LazyValue; diff --git a/fault-tolerance/src/main/java/io/helidon/faulttolerance/Timeout.java b/fault-tolerance/src/main/java/io/helidon/faulttolerance/Timeout.java index 391b4cbc18c..d3c714f966f 100644 --- a/fault-tolerance/src/main/java/io/helidon/faulttolerance/Timeout.java +++ b/fault-tolerance/src/main/java/io/helidon/faulttolerance/Timeout.java @@ -18,7 +18,6 @@ import java.time.Duration; import java.util.concurrent.ScheduledExecutorService; -import java.util.function.Consumer; import io.helidon.common.LazyValue; @@ -54,7 +53,6 @@ class Builder implements io.helidon.common.Builder { private Duration timeout = Duration.ofSeconds(10); private LazyValue executor = FaultTolerance.scheduledExecutor(); private boolean currentThread = false; - private Consumer listener; private Builder() { } @@ -76,8 +74,8 @@ public Builder timeout(Duration timeout) { } /** - * Flag to indicate that code should be executed in current thread instead - * of in an executor's thread for ease of monitoring. + * Flag to indicate that code must be executed in current thread instead + * of in an executor's thread. This flag is {@code false} by default. * * @param currentThread setting for this timeout * @return updated builder instance @@ -98,17 +96,6 @@ public Builder executor(ScheduledExecutorService executor) { return this; } - /** - * Listener that will be called when a thread is interrupted. - * - * @param listener the listener - * @return updated build instance - */ - public Builder interruptListener(Consumer listener) { - this.listener = listener; - return this; - } - Duration timeout() { return timeout; } @@ -120,9 +107,5 @@ LazyValue executor() { boolean currentThread() { return currentThread; } - - Consumer interruptListener() { - return listener; - } } } diff --git a/fault-tolerance/src/main/java/io/helidon/faulttolerance/TimeoutImpl.java b/fault-tolerance/src/main/java/io/helidon/faulttolerance/TimeoutImpl.java index 2a8fafc5a05..3ec937d5bcd 100644 --- a/fault-tolerance/src/main/java/io/helidon/faulttolerance/TimeoutImpl.java +++ b/fault-tolerance/src/main/java/io/helidon/faulttolerance/TimeoutImpl.java @@ -24,7 +24,6 @@ import java.util.concurrent.TimeUnit; import java.util.concurrent.TimeoutException; import java.util.concurrent.atomic.AtomicBoolean; -import java.util.function.Consumer; import java.util.function.Supplier; import io.helidon.common.LazyValue; @@ -38,13 +37,11 @@ class TimeoutImpl implements Timeout { private final long timeoutMillis; private final LazyValue executor; private final boolean currentThread; - private final Consumer interruptListener; TimeoutImpl(Timeout.Builder builder) { this.timeoutMillis = builder.timeout().toMillis(); this.executor = builder.executor(); this.currentThread = builder.currentThread(); - this.interruptListener = builder.interruptListener(); } @Override @@ -81,9 +78,6 @@ public Single invoke(Supplier> supplier) { if (callReturned.compareAndSet(false, true)) { future.completeExceptionally(new TimeoutException("Method interrupted by timeout")); thisThread.interrupt(); - if (interruptListener != null) { - interruptListener.accept(thisThread); - } } return null; }); From 8b7467ff0011d78f12d38378fc1ceed578e5cd54 Mon Sep 17 00:00:00 2001 From: Santiago Pericasgeertsen Date: Wed, 9 Sep 2020 13:55:08 -0400 Subject: [PATCH 58/65] Fixed checkstyle errors. Signed-off-by: Santiago Pericasgeertsen --- .../io/helidon/common/reactive/MultiFromCompletionStage.java | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/common/reactive/src/main/java/io/helidon/common/reactive/MultiFromCompletionStage.java b/common/reactive/src/main/java/io/helidon/common/reactive/MultiFromCompletionStage.java index c9ccb8ab7ed..069b7a90208 100644 --- a/common/reactive/src/main/java/io/helidon/common/reactive/MultiFromCompletionStage.java +++ b/common/reactive/src/main/java/io/helidon/common/reactive/MultiFromCompletionStage.java @@ -56,7 +56,8 @@ static final class CompletionStageSubscription extends DeferredScalarSubscrip private final AtomicBiConsumer watcher; private CompletionStage source; - CompletionStageSubscription(Flow.Subscriber downstream, boolean nullMeansEmpty, AtomicBiConsumer watcher,CompletionStage source) { + CompletionStageSubscription(Flow.Subscriber downstream, boolean nullMeansEmpty, + AtomicBiConsumer watcher, CompletionStage source) { super(downstream); this.nullMeansEmpty = nullMeansEmpty; this.watcher = watcher; From 5ac8cccb7875594fdff38794fd7e5c988cf1a0a6 Mon Sep 17 00:00:00 2001 From: Santiago Pericasgeertsen Date: Wed, 9 Sep 2020 14:05:09 -0400 Subject: [PATCH 59/65] Fixed Javadoc. Signed-off-by: Santiago Pericasgeertsen --- .../src/main/java/io/helidon/common/reactive/Single.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/common/reactive/src/main/java/io/helidon/common/reactive/Single.java b/common/reactive/src/main/java/io/helidon/common/reactive/Single.java index b93248fa8cd..444ed1f7890 100644 --- a/common/reactive/src/main/java/io/helidon/common/reactive/Single.java +++ b/common/reactive/src/main/java/io/helidon/common/reactive/Single.java @@ -653,7 +653,7 @@ default CompletionStage toStage() { /** * Exposes this {@link Single} instance as a {@link CompletionStage}. - * Note that if this {@link Single} completes without a value and {@cdoe completeWithoutValue} + * Note that if this {@link Single} completes without a value and {@code completeWithoutValue} * is set to {@code false}, the resulting {@link CompletionStage} will be completed * exceptionally with an {@link IllegalStateException} * From 960897d70c820f58cd1cbebab23c9eacb35ebcc6 Mon Sep 17 00:00:00 2001 From: Santiago Pericasgeertsen Date: Wed, 9 Sep 2020 14:38:50 -0400 Subject: [PATCH 60/65] Copyright year. Signed-off-by: Santiago Pericasgeertsen --- .../helidon/microprofile/arquillian/HelidonMethodExecutor.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/microprofile/tests/arquillian/src/main/java/io/helidon/microprofile/arquillian/HelidonMethodExecutor.java b/microprofile/tests/arquillian/src/main/java/io/helidon/microprofile/arquillian/HelidonMethodExecutor.java index 0bf8c19610e..65d75f24872 100644 --- a/microprofile/tests/arquillian/src/main/java/io/helidon/microprofile/arquillian/HelidonMethodExecutor.java +++ b/microprofile/tests/arquillian/src/main/java/io/helidon/microprofile/arquillian/HelidonMethodExecutor.java @@ -1,5 +1,5 @@ /* - * Copyright (c) 2018 Oracle and/or its affiliates. All rights reserved. + * Copyright (c) 2018, 2020 Oracle and/or its affiliates. All rights reserved. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. From 37cef9422029e5d82a53192d02e951f1159c5f77 Mon Sep 17 00:00:00 2001 From: Santiago Pericasgeertsen Date: Wed, 9 Sep 2020 17:00:34 -0400 Subject: [PATCH 61/65] Adjusted breaker window and error ratio based on latest changes. Signed-off-by: Santiago Pericasgeertsen --- .../helidon/webserver/examples/faulttolerance/FtService.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/examples/webserver/fault-tolerance/src/main/java/io/helidon/webserver/examples/faulttolerance/FtService.java b/examples/webserver/fault-tolerance/src/main/java/io/helidon/webserver/examples/faulttolerance/FtService.java index 46758d81ea3..8154b296d81 100644 --- a/examples/webserver/fault-tolerance/src/main/java/io/helidon/webserver/examples/faulttolerance/FtService.java +++ b/examples/webserver/fault-tolerance/src/main/java/io/helidon/webserver/examples/faulttolerance/FtService.java @@ -51,8 +51,8 @@ public class FtService implements Service { .name("helidon-example-bulkhead") .build(); this.breaker = CircuitBreaker.builder() - .volume(10) - .errorRatio(20) + .volume(4) + .errorRatio(40) .successThreshold(1) .delay(Duration.ofSeconds(5)) .build(); From 06f3669d8c4262fd255f8333ce3924c08595fc11 Mon Sep 17 00:00:00 2001 From: Santiago Pericasgeertsen Date: Thu, 10 Sep 2020 10:27:52 -0400 Subject: [PATCH 62/65] Use Single returned from onCancel. Minor optimization. Signed-off-by: Santiago Pericasgeertsen --- .../src/main/java/io/helidon/faulttolerance/AsyncImpl.java | 3 +-- .../src/main/java/io/helidon/faulttolerance/ErrorChecker.java | 2 +- 2 files changed, 2 insertions(+), 3 deletions(-) diff --git a/fault-tolerance/src/main/java/io/helidon/faulttolerance/AsyncImpl.java b/fault-tolerance/src/main/java/io/helidon/faulttolerance/AsyncImpl.java index dc341aa1805..57b30ea4b23 100644 --- a/fault-tolerance/src/main/java/io/helidon/faulttolerance/AsyncImpl.java +++ b/fault-tolerance/src/main/java/io/helidon/faulttolerance/AsyncImpl.java @@ -45,8 +45,7 @@ public Single invoke(Supplier supplier) { } Single single = Single.create(future, true); - single.onCancel(() -> taskFuture.cancel(false)); // cancel task - return single; + return single.onCancel(() -> taskFuture.cancel(false)); // cancel task } private static class AsyncTask implements Runnable { diff --git a/fault-tolerance/src/main/java/io/helidon/faulttolerance/ErrorChecker.java b/fault-tolerance/src/main/java/io/helidon/faulttolerance/ErrorChecker.java index 0fcc4abd16d..b012308cd07 100644 --- a/fault-tolerance/src/main/java/io/helidon/faulttolerance/ErrorChecker.java +++ b/fault-tolerance/src/main/java/io/helidon/faulttolerance/ErrorChecker.java @@ -37,6 +37,6 @@ static ErrorChecker create(Set> skipOnSet, Set> set, Throwable throwable) { - return set.stream().filter(t -> t.isAssignableFrom(throwable.getClass())).count() > 0; + return set.stream().anyMatch(t -> t.isAssignableFrom(throwable.getClass())); } } From b6d2e917df5ab314a37d264d588e342730d50475 Mon Sep 17 00:00:00 2001 From: Santiago Pericasgeertsen Date: Thu, 10 Sep 2020 11:03:28 -0400 Subject: [PATCH 63/65] Make sure that tests are copied due to mutability. Signed-off-by: Santiago Pericasgeertsen --- .../java/io/helidon/faulttolerance/ErrorChecker.java | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/fault-tolerance/src/main/java/io/helidon/faulttolerance/ErrorChecker.java b/fault-tolerance/src/main/java/io/helidon/faulttolerance/ErrorChecker.java index b012308cd07..bae585ea7d7 100644 --- a/fault-tolerance/src/main/java/io/helidon/faulttolerance/ErrorChecker.java +++ b/fault-tolerance/src/main/java/io/helidon/faulttolerance/ErrorChecker.java @@ -25,15 +25,18 @@ interface ErrorChecker { /** * Returns ErrorChecker that skips if throwable is in skipOnSet or if applyOnSet * is not empty and throwable is not in it. Note that if applyOnSet is empty, then - * it is equivalent to it containing {@code Throwable.class}. + * it is equivalent to it containing {@code Throwable.class}. Sets are copied + * because they are mutable. * * @param skipOnSet set of throwables to skip logic on. * @param applyOnSet set of throwables to apply logic on. * @return An error checker. */ static ErrorChecker create(Set> skipOnSet, Set> applyOnSet) { - return throwable -> containsThrowable(skipOnSet, throwable) - || !applyOnSet.isEmpty() && !containsThrowable(applyOnSet, throwable); + Set> skipOn = Set.copyOf(skipOnSet); + Set> applyOn = Set.copyOf(applyOnSet); + return throwable -> containsThrowable(skipOn, throwable) + || !applyOn.isEmpty() && !containsThrowable(applyOn, throwable); } private static boolean containsThrowable(Set> set, Throwable throwable) { From 06b6c1ab426a61fbdbbb73229de8e6d8163387fd Mon Sep 17 00:00:00 2001 From: Santiago Pericasgeertsen Date: Thu, 10 Sep 2020 15:30:14 -0400 Subject: [PATCH 64/65] Use latest version of SingleNever after merge with master. Signed-off-by: Santiago Pericasgeertsen --- .../helidon/faulttolerance/TimeoutImpl.java | 25 +------------------ 1 file changed, 1 insertion(+), 24 deletions(-) diff --git a/fault-tolerance/src/main/java/io/helidon/faulttolerance/TimeoutImpl.java b/fault-tolerance/src/main/java/io/helidon/faulttolerance/TimeoutImpl.java index 3ec937d5bcd..7817a6ab262 100644 --- a/fault-tolerance/src/main/java/io/helidon/faulttolerance/TimeoutImpl.java +++ b/fault-tolerance/src/main/java/io/helidon/faulttolerance/TimeoutImpl.java @@ -72,7 +72,7 @@ public Single invoke(Supplier> supplier) { .build() .invoke(() -> { monitorStarted.complete(null); - return new TimeoutSingleNever(); // new instance + return Single.never(); }) .exceptionally(it -> { if (callReturned.compareAndSet(false, true)) { @@ -106,27 +106,4 @@ public Single invoke(Supplier> supplier) { return Single.create(future, true); } } - - /** - * Similar to {@link io.helidon.common.reactive.SingleNever} but not a singleton. - * When running multiple tests over the same thread, and old timeout that expires - * can interrupt a new task if using the same instance of this class. - */ - private static class TimeoutSingleNever extends CompletionSingle { - - TimeoutSingleNever() { - } - - @Override - public void subscribe(Flow.Subscriber subscriber) { - subscriber.onSubscribe(new Flow.Subscription() { - @Override - public void request(long n) { - } - @Override - public void cancel() { - } - }); - } - } } From a01426f5af3cf674aed2e72268d7bba3805b3256 Mon Sep 17 00:00:00 2001 From: Santiago Pericasgeertsen Date: Thu, 10 Sep 2020 15:41:06 -0400 Subject: [PATCH 65/65] Removed unused import. Signed-off-by: Santiago Pericasgeertsen --- .../src/main/java/io/helidon/faulttolerance/TimeoutImpl.java | 1 - 1 file changed, 1 deletion(-) diff --git a/fault-tolerance/src/main/java/io/helidon/faulttolerance/TimeoutImpl.java b/fault-tolerance/src/main/java/io/helidon/faulttolerance/TimeoutImpl.java index 7817a6ab262..d4da5cd660e 100644 --- a/fault-tolerance/src/main/java/io/helidon/faulttolerance/TimeoutImpl.java +++ b/fault-tolerance/src/main/java/io/helidon/faulttolerance/TimeoutImpl.java @@ -27,7 +27,6 @@ import java.util.function.Supplier; import io.helidon.common.LazyValue; -import io.helidon.common.reactive.CompletionSingle; import io.helidon.common.reactive.Multi; import io.helidon.common.reactive.Single;