Skip to content

Commit

Permalink
fix: improve information on CancellationException for LROs (#2236)
Browse files Browse the repository at this point in the history
* fix: point to documentation on LRO CancelledException

* mvn fmt:format

* add test for http IT in LRO

* format code

* use final variable on LRO cancellation

* Wrap guava's CancellationException in our own Exception

* reset test and basic retry future

* use logging to inform of failed LRO

* add license and comment to fake log handler

* format code

* fix license

* format souce

* decrease LRO timeout logging level to WARNING

* fix logging threshold check

* use common log handler setup for ITLongRunningOperation

* fix license header year

* format source

* reset lro it

* change to unit tests

* reset testjar exports

* simplify unit test

* use instance variables in test
  • Loading branch information
diegomarquezp committed Nov 14, 2023
1 parent 9d30ed9 commit 741e40c
Show file tree
Hide file tree
Showing 5 changed files with 193 additions and 22 deletions.
Expand Up @@ -43,6 +43,7 @@
import com.google.api.gax.rpc.StreamController;
import com.google.api.gax.rpc.UnaryCallSettings;
import com.google.api.gax.rpc.UnaryCallable;
import com.google.api.gax.util.FakeLogHandler;
import com.google.common.base.Preconditions;
import com.google.common.collect.ImmutableList;
import com.google.common.collect.Lists;
Expand All @@ -66,9 +67,6 @@
import java.util.concurrent.ScheduledFuture;
import java.util.concurrent.TimeUnit;
import java.util.concurrent.atomic.AtomicInteger;
import java.util.logging.Handler;
import java.util.logging.LogRecord;
import java.util.stream.Collectors;
import org.junit.After;
import org.junit.Assert;
import org.junit.Test;
Expand Down Expand Up @@ -717,23 +715,4 @@ public void testDoubleRelease() throws Exception {
ChannelPool.LOG.removeHandler(logHandler);
}
}

private static class FakeLogHandler extends Handler {
List<LogRecord> records = new ArrayList<>();

@Override
public void publish(LogRecord record) {
records.add(record);
}

@Override
public void flush() {}

@Override
public void close() throws SecurityException {}

List<String> getAllMessages() {
return records.stream().map(LogRecord::getMessage).collect(Collectors.toList());
}
}
}
1 change: 1 addition & 0 deletions gax-java/gax/pom.xml
Expand Up @@ -76,6 +76,7 @@
<include>com/google/api/gax/core/**</include>
<include>com/google/api/gax/rpc/testing/**</include>
<include>com/google/api/gax/rpc/mtls/**</include>
<include>com/google/api/gax/util/**</include>
</includes>
</configuration>
</execution>
Expand Down
Expand Up @@ -35,14 +35,25 @@
import com.google.api.gax.retrying.ExponentialRetryAlgorithm;
import com.google.api.gax.retrying.RetrySettings;
import com.google.api.gax.retrying.TimedAttemptSettings;
import com.google.common.annotations.VisibleForTesting;
import java.util.concurrent.CancellationException;
import java.util.logging.Level;
import java.util.logging.Logger;

/**
* Operation timed polling algorithm, which uses exponential backoff factor for determining when the
* next polling operation should be executed. If the polling exceeds the total timeout this
* algorithm cancels polling.
*/
public class OperationTimedPollAlgorithm extends ExponentialRetryAlgorithm {

@VisibleForTesting
static final Logger LOGGER = Logger.getLogger(OperationTimedPollAlgorithm.class.getName());

@VisibleForTesting
static final String LRO_TROUBLESHOOTING_LINK =
"https://github.com/googleapis/google-cloud-java#lro-timeouts";

/**
* Creates the polling algorithm, using the default {@code NanoClock} for time computations.
*
Expand Down Expand Up @@ -77,6 +88,13 @@ public boolean shouldRetry(TimedAttemptSettings nextAttemptSettings)
if (super.shouldRetry(nextAttemptSettings)) {
return true;
}
if (LOGGER.isLoggable(Level.WARNING)) {
LOGGER.log(
Level.WARNING,
"The task has been cancelled. Please refer to "
+ LRO_TROUBLESHOOTING_LINK
+ " for more information");
}
throw new CancellationException();
}

Expand Down
@@ -0,0 +1,114 @@
/*
* Copyright 2023 Google LLC
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted provided that the following conditions are
* met:
*
* * Redistributions of source code must retain the above copyright
* notice, this list of conditions and the following disclaimer.
* * Redistributions in binary form must reproduce the above
* copyright notice, this list of conditions and the following disclaimer
* in the documentation and/or other materials provided with the
* distribution.
* * Neither the name of Google LLC nor the names of its
* contributors may be used to endorse or promote products derived from
* this software without specific prior written permission.
*
* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
* "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
* LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
* A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
* OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
* SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
* LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
* DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
* THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
* (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
* OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
*/
package com.google.api.gax.longrunning;

import static org.junit.Assert.assertThrows;
import static org.junit.Assert.assertTrue;
import static org.junit.Assert.fail;

import com.google.api.gax.core.FakeApiClock;
import com.google.api.gax.retrying.RetrySettings;
import com.google.api.gax.retrying.TimedAttemptSettings;
import com.google.api.gax.util.FakeLogHandler;
import java.util.concurrent.CancellationException;
import org.junit.After;
import org.junit.Before;
import org.junit.Test;
import org.threeten.bp.Duration;

public class OperationTimedPollAlgorithmTest {

private static final RetrySettings FAST_RETRY_SETTINGS =
RetrySettings.newBuilder()
.setInitialRetryDelay(Duration.ofMillis(1L))
.setRetryDelayMultiplier(1)
.setMaxRetryDelay(Duration.ofMillis(1L))
.setInitialRpcTimeout(Duration.ofMillis(1L))
.setMaxAttempts(0)
.setJittered(false)
.setRpcTimeoutMultiplier(1)
.setMaxRpcTimeout(Duration.ofMillis(1L))
.setTotalTimeout(Duration.ofMillis(5L))
.build();
private TimedAttemptSettings timedAttemptSettings;
private FakeApiClock clock;

private FakeLogHandler logHandler;

@Before
public void setUp() {
logHandler = new FakeLogHandler();
OperationTimedPollAlgorithm.LOGGER.addHandler(logHandler);
clock = new FakeApiClock(System.nanoTime());
timedAttemptSettings =
TimedAttemptSettings.newBuilder()
.setGlobalSettings(FAST_RETRY_SETTINGS)
.setRetryDelay(Duration.ofMillis(1l))
.setRpcTimeout(Duration.ofMillis(1l))
.setRandomizedRetryDelay(Duration.ofMillis(1l))
.setAttemptCount(0)
.setFirstAttemptStartTimeNanos(clock.nanoTime())
.build();
}

@After
public void tearDown() {
OperationTimedPollAlgorithm.LOGGER.removeHandler(logHandler);
// redundant null assignment for readability - a new log handler will be used
logHandler = null;
}

@Test
public void testAlgorithmThatShouldRetry_doesNotLogTimeoutHelpMessage() {
OperationTimedPollAlgorithm algorithm =
OperationTimedPollAlgorithm.create(FAST_RETRY_SETTINGS, clock);
try {
algorithm.shouldRetry(timedAttemptSettings);
} catch (CancellationException ex) {
fail("Unexpected unsuccessful shouldRetry()");
}
assertTrue(
logHandler.getAllMessages().stream()
.noneMatch(
entry -> entry.contains(OperationTimedPollAlgorithm.LRO_TROUBLESHOOTING_LINK)));
}

@Test
public void testAlgorithmThatShouldNotRetry_logsTimeoutHelpMessage() {
OperationTimedPollAlgorithm algorithm =
OperationTimedPollAlgorithm.create(FAST_RETRY_SETTINGS, clock);
clock.incrementNanoTime(1 * 1000 * 1000 * 1000); // force rpc timeout
assertThrows(CancellationException.class, () -> algorithm.shouldRetry(timedAttemptSettings));
assertTrue(
logHandler.getAllMessages().stream()
.anyMatch(
entry -> entry.contains(OperationTimedPollAlgorithm.LRO_TROUBLESHOOTING_LINK)));
}
}
@@ -0,0 +1,59 @@
/*
* Copyright 2023 Google LLC
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted provided that the following conditions are
* met:
*
* * Redistributions of source code must retain the above copyright
* notice, this list of conditions and the following disclaimer.
* * Redistributions in binary form must reproduce the above
* copyright notice, this list of conditions and the following disclaimer
* in the documentation and/or other materials provided with the
* distribution.
* * Neither the name of Google LLC nor the names of its
* contributors may be used to endorse or promote products derived from
* this software without specific prior written permission.
*
* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
* "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
* LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
* A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
* OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
* SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
* LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
* DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
* THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
* (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
* OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
*/
package com.google.api.gax.util;

import java.util.ArrayList;
import java.util.List;
import java.util.logging.Handler;
import java.util.logging.LogRecord;
import java.util.stream.Collectors;

/*
* Convenience class that stores the log entries produced by any logger
* It can then be inspected - its entries are a list log records
*/
public class FakeLogHandler extends Handler {
List<LogRecord> records = new ArrayList<>();

@Override
public void publish(LogRecord record) {
records.add(record);
}

@Override
public void flush() {}

@Override
public void close() throws SecurityException {}

public List<String> getAllMessages() {
return records.stream().map(LogRecord::getMessage).collect(Collectors.toList());
}
}

0 comments on commit 741e40c

Please sign in to comment.