-
Notifications
You must be signed in to change notification settings - Fork 2.9k
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
[SMALLFIX] Retry master address lookup (#6886)
* Get master address inside retry loop * Add retries around block worker registration * Fix checkstyle * Address review comments * Address review comments
- Loading branch information
Showing
19 changed files
with
724 additions
and
28 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
117 changes: 117 additions & 0 deletions
117
core/common/src/main/java/alluxio/retry/ExponentialTimeBoundedRetry.java
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,117 @@ | ||
| /* | ||
| * The Alluxio Open Foundation licenses this work under the Apache License, version 2.0 | ||
| * (the "License"). You may not use this work except in compliance with the License, which is | ||
| * available at www.apache.org/licenses/LICENSE-2.0 | ||
| * | ||
| * This software is distributed on an "AS IS" basis, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, | ||
| * either express or implied, as more fully set forth in the License. | ||
| * | ||
| * See the NOTICE file distributed with this work for information regarding copyright ownership. | ||
| */ | ||
|
|
||
| package alluxio.retry; | ||
|
|
||
| import alluxio.time.TimeContext; | ||
|
|
||
| import java.time.Duration; | ||
| import java.util.concurrent.ThreadLocalRandom; | ||
|
|
||
| /** | ||
| * A retry policy which uses exponential backoff and a maximum duration time bound. | ||
| * | ||
| * A final retry will be performed at the time bound before giving up. | ||
| * | ||
| * For example, with initial sleep 10ms, maximum sleep 100ms, and maximum duration 500ms, the sleep | ||
| * timings would be [10, 20, 40, 80, 100, 100, 100, 50], assuming the operation being retries takes | ||
| * no time. The 50 at the end is because the previous times add up to 450, so the mechanism sleeps | ||
| * for only 50ms before the final attempt. | ||
| * | ||
| * However, those are just the base sleep timings. For each sleep time, we multiply by a random | ||
| * number from 1 to 1.1 to add jitter to avoid hotspotting. | ||
| */ | ||
| public final class ExponentialTimeBoundedRetry extends TimeBoundedRetry { | ||
| private final Duration mMaxSleep; | ||
| private Duration mNextSleep; | ||
|
|
||
| /** | ||
| * See {@link Builder}. | ||
| */ | ||
| private ExponentialTimeBoundedRetry(TimeContext timeCtx, Duration maxDuration, | ||
| Duration initialSleep, Duration maxSleep) { | ||
| super(timeCtx, maxDuration); | ||
| mMaxSleep = maxSleep; | ||
| mNextSleep = initialSleep; | ||
| } | ||
|
|
||
| @Override | ||
| protected Duration computeNextWaitTime() { | ||
| Duration next = mNextSleep; | ||
| mNextSleep = mNextSleep.multipliedBy(2); | ||
| if (mNextSleep.compareTo(mMaxSleep) > 0) { | ||
| mNextSleep = mMaxSleep; | ||
| } | ||
| // Add jitter. | ||
| long jitter = Math.round(ThreadLocalRandom.current().nextDouble(0.1) * next.toMillis()); | ||
| return next.plusMillis(jitter); | ||
| } | ||
|
|
||
| /** | ||
| * @return a builder | ||
| */ | ||
| public static Builder builder() { | ||
| return new Builder(); | ||
| } | ||
|
|
||
| /** | ||
| * Builder for time bounded exponential retry mechanisms. | ||
| */ | ||
| public static class Builder { | ||
| private TimeContext mTimeCtx = TimeContext.SYSTEM; | ||
| private Duration mMaxDuration; | ||
| private Duration mInitialSleep; | ||
| private Duration mMaxSleep; | ||
|
|
||
| /** | ||
| * @param timeCtx time context | ||
| * @return the builder | ||
| */ | ||
| public Builder withTimeCtx(TimeContext timeCtx) { | ||
| mTimeCtx = timeCtx; | ||
| return this; | ||
| } | ||
|
|
||
| /** | ||
| * @param maxDuration max total duration to retry for | ||
| * @return the builder | ||
| */ | ||
| public Builder withMaxDuration(Duration maxDuration) { | ||
| mMaxDuration = maxDuration; | ||
| return this; | ||
| } | ||
|
|
||
| /** | ||
| * @param initialSleep initial sleep interval between retries | ||
| * @return the builder | ||
| */ | ||
| public Builder withInitialSleep(Duration initialSleep) { | ||
| mInitialSleep = initialSleep; | ||
| return this; | ||
| } | ||
|
|
||
| /** | ||
| * @param maxSleep maximum sleep interval between retries | ||
| * @return the builder | ||
| */ | ||
| public Builder withMaxSleep(Duration maxSleep) { | ||
| mMaxSleep = maxSleep; | ||
| return this; | ||
| } | ||
|
|
||
| /** | ||
| * @return the built retry mechanism | ||
| */ | ||
| public ExponentialTimeBoundedRetry build() { | ||
| return new ExponentialTimeBoundedRetry(mTimeCtx, mMaxDuration, mInitialSleep, mMaxSleep); | ||
| } | ||
| } | ||
| } |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,60 @@ | ||
| /* | ||
| * The Alluxio Open Foundation licenses this work under the Apache License, version 2.0 | ||
| * (the "License"). You may not use this work except in compliance with the License, which is | ||
| * available at www.apache.org/licenses/LICENSE-2.0 | ||
| * | ||
| * This software is distributed on an "AS IS" basis, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, | ||
| * either express or implied, as more fully set forth in the License. | ||
| * | ||
| * See the NOTICE file distributed with this work for information regarding copyright ownership. | ||
| */ | ||
|
|
||
| package alluxio.retry; | ||
|
|
||
| import org.slf4j.Logger; | ||
| import org.slf4j.LoggerFactory; | ||
|
|
||
| import java.io.IOException; | ||
|
|
||
| /** | ||
| * Utilities for performing retries. | ||
| */ | ||
| public final class RetryUtils { | ||
| private static final Logger LOG = LoggerFactory.getLogger(RetryUtils.class); | ||
|
|
||
| /** | ||
| * Retries the given method until it doesn't throw an IO exception or the retry policy expires. If | ||
| * the retry policy expires, the last exception generated will be rethrown. | ||
| * | ||
| * @param action a description of the action that fits the phrase "Failed to ${action}" | ||
| * @param f the function to retry | ||
| * @param policy the retry policy to use | ||
| */ | ||
| public static void retry(String action, RunnableThrowsIOException f, RetryPolicy policy) | ||
| throws IOException { | ||
| IOException e; | ||
| do { | ||
| try { | ||
| f.run(); | ||
| return; | ||
| } catch (IOException ioe) { | ||
| e = ioe; | ||
| LOG.warn("Failed to {} (attempt {}): {}", action, policy.getRetryCount() + 1, e.toString()); | ||
| } | ||
| } while (policy.attemptRetry()); | ||
| throw e; | ||
| } | ||
|
|
||
| /** | ||
| * Interface for methods which return nothing and may throw IOException. | ||
| */ | ||
| @FunctionalInterface | ||
| public interface RunnableThrowsIOException { | ||
| /** | ||
| * Runs the runnable. | ||
| */ | ||
| void run() throws IOException; | ||
| } | ||
|
|
||
| private RetryUtils() {} // prevent instantiation | ||
| } |
Oops, something went wrong.