From 47178ae7fa97f7f4af7a3d9820354c8a2219a3a6 Mon Sep 17 00:00:00 2001 From: Alex Heneveld Date: Wed, 10 Sep 2014 08:30:39 +0100 Subject: [PATCH 1/2] apply back-off logic to utrace, will prevent requests for 60s after a failure --- .../java/brooklyn/location/geo/UtraceHostGeoLookup.java | 6 ++++++ utils/common/src/main/java/brooklyn/util/time/Duration.java | 4 ++++ 2 files changed, 10 insertions(+) diff --git a/core/src/main/java/brooklyn/location/geo/UtraceHostGeoLookup.java b/core/src/main/java/brooklyn/location/geo/UtraceHostGeoLookup.java index 9012032769..5a0edcb850 100644 --- a/core/src/main/java/brooklyn/location/geo/UtraceHostGeoLookup.java +++ b/core/src/main/java/brooklyn/location/geo/UtraceHostGeoLookup.java @@ -96,8 +96,13 @@ public String getLookupUrlFor(InetAddress address) { } private static boolean LOGGED_GEO_LOOKUP_UNAVAILABLE = false; + private static long LAST_FAILURE_UTC = -1; public HostGeoInfo getHostGeoInfo(InetAddress address) throws MalformedURLException, IOException { + if (Duration.sinceUtc(LAST_FAILURE_UTC).compareTo(Duration.ONE_MINUTE) < 0) { + // wait at least 60s since a failure + return null; + } String url = getLookupUrlFor(address); if (log.isDebugEnabled()) log.debug("Geo info lookup for "+address+" at "+url); @@ -105,6 +110,7 @@ public HostGeoInfo getHostGeoInfo(InetAddress address) throws MalformedURLExcept try { xml = new XmlParser().parse(getLookupUrlFor(address)); } catch (Exception e) { + LAST_FAILURE_UTC = System.currentTimeMillis(); if (log.isDebugEnabled()) log.debug("Geo info lookup for "+address+" failed: "+e); if (!LOGGED_GEO_LOOKUP_UNAVAILABLE) { diff --git a/utils/common/src/main/java/brooklyn/util/time/Duration.java b/utils/common/src/main/java/brooklyn/util/time/Duration.java index fb3d573230..3fc86584b9 100644 --- a/utils/common/src/main/java/brooklyn/util/time/Duration.java +++ b/utils/common/src/main/java/brooklyn/util/time/Duration.java @@ -221,6 +221,10 @@ public static Duration untilUtc(long millisSinceEpoch) { return millis(millisSinceEpoch - System.currentTimeMillis()); } + public static Duration sinceUtc(long millisSinceEpoch) { + return millis(System.currentTimeMillis() - millisSinceEpoch); + } + public Duration add(Duration other) { return nanos(nanos() + other.nanos()); } From 4dac3d1b8321557505c921ebed80434dc9e7deba Mon Sep 17 00:00:00 2001 From: Alex Heneveld Date: Wed, 10 Sep 2014 23:30:57 +0200 Subject: [PATCH 2/2] utrace now has timeout, defaulting to 3s, and preventing retry within 5m --- .../location/geo/UtraceHostGeoLookup.java | 48 ++++++++++++++++++- 1 file changed, 47 insertions(+), 1 deletion(-) diff --git a/core/src/main/java/brooklyn/location/geo/UtraceHostGeoLookup.java b/core/src/main/java/brooklyn/location/geo/UtraceHostGeoLookup.java index 5a0edcb850..2560b551c3 100644 --- a/core/src/main/java/brooklyn/location/geo/UtraceHostGeoLookup.java +++ b/core/src/main/java/brooklyn/location/geo/UtraceHostGeoLookup.java @@ -25,17 +25,21 @@ import java.io.IOException; import java.net.InetAddress; import java.net.MalformedURLException; +import java.util.concurrent.atomic.AtomicReference; import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import brooklyn.util.exceptions.Exceptions; import brooklyn.util.net.Networking; import brooklyn.util.time.Duration; +import brooklyn.util.time.Durations; import com.google.common.base.Throwables; public class UtraceHostGeoLookup implements HostGeoLookup { + /* * http://xml.utrace.de/?query=88.198.156.18 @@ -63,6 +67,13 @@ public class UtraceHostGeoLookup implements HostGeoLookup { (This may last for several days once blacklisting, not sure how long.) */ + /** after failures, subsequent retries within this time interval are blocked */ + private static final Duration RETRY_INTERVAL = Duration.FIVE_MINUTES; + /** requests taking longer than this period are deemed to have timed out and failed; + * set reasonably low so that if we are blacklisted for making too many requests, + * the call to get geo info does not take very long */ + private static final Duration REQUEST_TIMEOUT = Duration.seconds(3); + public static final Logger log = LoggerFactory.getLogger(UtraceHostGeoLookup.class); public String getLookupUrlForPublicIp(String ip) { @@ -98,11 +109,46 @@ public String getLookupUrlFor(InetAddress address) { private static boolean LOGGED_GEO_LOOKUP_UNAVAILABLE = false; private static long LAST_FAILURE_UTC = -1; + /** does the {@link #retrieveHostGeoInfo(InetAddress)}, but in the background with a default timeout */ public HostGeoInfo getHostGeoInfo(InetAddress address) throws MalformedURLException, IOException { - if (Duration.sinceUtc(LAST_FAILURE_UTC).compareTo(Duration.ONE_MINUTE) < 0) { + if (Duration.sinceUtc(LAST_FAILURE_UTC).compareTo(RETRY_INTERVAL) < 0) { // wait at least 60s since a failure return null; } + return getHostGeoInfo(address, REQUEST_TIMEOUT); + } + + /** does a {@link #retrieveHostGeoInfo(InetAddress)} with a timeout (returning null, interrupting, and setting failure time) */ + public HostGeoInfo getHostGeoInfo(final InetAddress address, Duration timeout) throws MalformedURLException, IOException { + final AtomicReference result = new AtomicReference(); + Thread lt = new Thread() { + public void run() { + try { + result.set(retrieveHostGeoInfo(address)); + } catch (Exception e) { + throw Exceptions.propagate(e); + } + } + }; + lt.start(); + + try { + Durations.join(lt, timeout); + } catch (InterruptedException e) { + throw Exceptions.propagate(e); + } + + if (lt.isAlive()) { + // interrupt and set the failure time so that subsequent attempts do not face this timeout + lt.interrupt(); + LAST_FAILURE_UTC = System.currentTimeMillis(); + log.debug("Geo info lookup for "+address+" timed out after "+timeout); + } + + return result.get(); + } + + public HostGeoInfo retrieveHostGeoInfo(InetAddress address) throws MalformedURLException, IOException { String url = getLookupUrlFor(address); if (log.isDebugEnabled()) log.debug("Geo info lookup for "+address+" at "+url);