Permalink
Browse files

Add LogThreshold property to degrader

LogThreshold property controls when to log degrader state updates. The idea is to skip the state logging if the degrader
is in slight fluctuation state.

Also change the rateLimitedLogger rate to 20s (4 intervals) from 10s.

RB=1226019
G=si-core-reviewers
R=dhoa
A=dhoa
  • Loading branch information...
ChaoLinkedIn committed Feb 16, 2018
1 parent a839f47 commit f697f57a31247d0947d70a000dac73b22a22bd77
@@ -1,5 +1,7 @@
19.0.1
------
(RB=1226019)
Add LogThreshold property to degrader
19.0.0
@@ -122,6 +122,12 @@
"type": "double",
"doc": "The drop rate threshold that controls whether the degrader should perform slow start(by doubling the transmission rate) or the normal stepUp/Down operation. Defaults to 0.0 which means no slow start phase will be performed.",
"optional": true
},
{
"name": "logThreshold",
"type": "double",
"doc": "The computed drop rate threshold to log degrader state changes. The log is enabled only when current drop rate higher than or equal to the threshold value.",
"optional": true
}
]
}
@@ -105,6 +105,10 @@
{
map.put(PropertyKeys.DEGRADER_SLOW_START_THRESHOLD, config.getSlowStartThreshold().toString());
}
if (config.hasLogThreshold())
{
map.put(PropertyKeys.DEGRADER_LOG_THRESHOLD, config.getLogThreshold().toString());
}
return map;
}
@@ -177,6 +181,10 @@ public static D2DegraderProperties toConfig(Map<String, String> properties)
{
config.setSlowStartThreshold(coerce(properties.get(PropertyKeys.DEGRADER_SLOW_START_THRESHOLD), Double.class));
}
if (properties.containsKey(PropertyKeys.DEGRADER_LOG_THRESHOLD))
{
config.setLogThreshold(coerce(properties.get(PropertyKeys.DEGRADER_LOG_THRESHOLD), Double.class));
}
return config;
}
}
@@ -52,6 +52,7 @@
public static final String DEGRADER_MIN_OUTSTANDING_COUNT = "degrader.minOutstandingCount";
public static final String DEGRADER_OVERRIDE_MIN_CALL_COUNT = "degrader.overrideMinCallCount";
public static final String DEGRADER_SLOW_START_THRESHOLD = "degrader.slowStartThreshold";
public static final String DEGRADER_LOG_THRESHOLD = "degrader.logThreshold";
//used by service properties
public static final String PATH = "path";
@@ -83,7 +83,7 @@
{
private static final Logger _log =
LoggerFactory.getLogger(SimpleLoadBalancerState.class);
private static final int DEGRADER_RATELIMITEDLOG_RATE_MS = 10000;
private static final int DEGRADER_RATELIMITEDLOG_RATE_MS = 20000;
private final UriLoadBalancerSubscriber _uriSubscriber;
private final ClusterLoadBalancerSubscriber _clusterSubscriber;
@@ -51,6 +51,9 @@
PropertyKeys.DEGRADER_LOG_ENABLED,
DegraderImpl.DEFAULT_LOG_ENABLED));
config.setLogThreshold(MapUtil.getWithDefault(properties, PropertyKeys.DEGRADER_LOG_THRESHOLD,
DegraderImpl.DEFAULT_LOG_THRESHOLD));
if (properties.get(PropertyKeys.DEGRADER_LATENCY_TO_USE) != null)
{
try
@@ -49,6 +49,7 @@ public void testDegraderPropertiesConverter()
final latencyType latencyToUse = latencyType.PCT50;
final Double initialDropRate = 0.1;
final Double slowStartThreshold = 0.32;
final Double logThreshold = 0.8;
Map<String, String> degraderProperties = new HashMap<>();
degraderProperties.put(PropertyKeys.DEGRADER_LOG_ENABLED, logEnabled.toString());
@@ -67,6 +68,7 @@ public void testDegraderPropertiesConverter()
degraderProperties.put(PropertyKeys.DEGRADER_LATENCY_TO_USE, latencyToUse.name());
degraderProperties.put(PropertyKeys.DEGRADER_INITIAL_DROP_RATE, initialDropRate.toString());
degraderProperties.put(PropertyKeys.DEGRADER_SLOW_START_THRESHOLD, slowStartThreshold.toString());
degraderProperties.put(PropertyKeys.DEGRADER_LOG_THRESHOLD, logThreshold.toString());
D2DegraderProperties d2DegraderProperties =
new D2DegraderProperties()
@@ -85,7 +87,8 @@ public void testDegraderPropertiesConverter()
.setMaxDropDuration(maxDropDuration)
.setLatencyToUse(latencyToUse)
.setInitialDropRate(initialDropRate)
.setSlowStartThreshold(slowStartThreshold);
.setSlowStartThreshold(slowStartThreshold)
.setLogThreshold(logThreshold);
Assert.assertEquals(DegraderPropertiesConverter.toConfig(degraderProperties), d2DegraderProperties);
Assert.assertEquals(DegraderPropertiesConverter.toProperties(d2DegraderProperties), degraderProperties);
@@ -24,6 +24,7 @@
import org.testng.annotations.Test;
import static org.testng.Assert.assertEquals;
import static org.testng.Assert.assertTrue;
/**
@@ -52,6 +53,7 @@ public void testToDegraderConfig()
Long lowOutstanding = 3000l;
Integer minOutstandingCount = 10;
Integer overrideMinCallCount = 5;
Double logThreshold = 0.8;
properties.put(PropertyKeys.DEGRADER_LOG_ENABLED, logEnabled.toString());
properties.put(PropertyKeys.DEGRADER_LATENCY_TO_USE, latencyToUse.toString());
properties.put(PropertyKeys.DEGRADER_MAX_DROP_RATE, maxDropRate.toString());
@@ -67,6 +69,7 @@ public void testToDegraderConfig()
properties.put(PropertyKeys.DEGRADER_LOW_OUTSTANDING, lowOutstanding.toString());
properties.put(PropertyKeys.DEGRADER_MIN_OUTSTANDING_COUNT, minOutstandingCount.toString());
properties.put(PropertyKeys.DEGRADER_OVERRIDE_MIN_CALL_COUNT, overrideMinCallCount.toString());
properties.put(PropertyKeys.DEGRADER_LOG_THRESHOLD, logThreshold.toString());
DegraderImpl.Config config = DegraderConfigFactory.toDegraderConfig(properties);
assertEquals(config.isLogEnabled(), logEnabled.booleanValue());
assertEquals(config.getLatencyToUse(), latencyToUse);
@@ -83,5 +86,6 @@ public void testToDegraderConfig()
assertEquals(config.getLowOutstanding(), lowOutstanding.longValue());
assertEquals(config.getMinOutstandingCount(), minOutstandingCount.longValue());
assertEquals(config.getOverrideMinCallCount(), overrideMinCallCount.intValue());
assertEquals(config.getLogThreshold(), logThreshold);
}
}
@@ -235,6 +235,12 @@ public long getLowOutstanding()
return _degrader.getConfig().getLowOutstanding();
}
@Override
public double getLogThreshold()
{
return _degrader.getConfig().getLogThreshold();
}
@Override
public int getMinOutstandingCount()
{
@@ -372,6 +378,14 @@ public void setOverrideMinCallCount(int overrideMinCallCount)
_degrader.setConfig(config);
}
@Override
public void setLogThreshold(double logThreshold)
{
DegraderImpl.Config config = new DegraderImpl.Config(_degrader.getConfig());
config.setLogThreshold(logThreshold);
_degrader.setConfig(config);
}
@Override
public void reset()
{
@@ -76,6 +76,7 @@ default boolean isLow()
long getLowOutstanding();
int getMinOutstandingCount();
default double getInitialDropRate() {return 0.0;}
default double getLogThreshold() { return 0.0; }
void setLogEnabled(boolean logEnabled);
void setLatencyToUse(String latencyToUse);
@@ -91,6 +92,7 @@ default boolean isLow()
void setHighOutstanding(long highOutstanding);
void setLowOutstanding(long lowOutstanding);
void setMinOutstandingCount(int minOutstandingCount);
default void setLogThreshold(double threshold) {};
void reset();
}
@@ -143,6 +143,7 @@
public static final Integer DEFAULT_OVERRIDE_MIN_CALL_COUNT = -1;
public static final double DEFAULT_INITIAL_DROP_RATE = 0.0d;
public static final double DEFAULT_SLOW_START_THRESHOLD = 0.0d;
public static final double DEFAULT_LOG_THRESHOLD = 0.5d;
private ImmutableConfig _config;
private String _name;
@@ -279,11 +280,12 @@ public boolean checkDrop()
}
/**
* choose logger to use: always use the default logger if isHigh() return true, otherwise go with rateLimitedLogger
* choose logger to use: always use the default logger if isHigh() return true, logEnabled flag is set, or debug
* is enabled. Otherwise go with rateLimitedLogger
*/
public Logger getLogger()
{
if (isHigh())
if (isHigh() || _config.isLogEnabled() || LOG.isDebugEnabled())
{
return LOG;
}
@@ -372,24 +374,21 @@ else if (oldDropRate > 0.0 && isLow())
}
}
if (oldDropRate != newDropRate)
if (oldDropRate != newDropRate && log.isWarnEnabled() && newDropRate >= _config.getLogThreshold())
{
if (log.isWarnEnabled())
{
log.warn(_config.getName() + " ComputedDropRate " +
(oldDropRate > newDropRate ? "decreased" : "increased") +
" from " + oldDropRate + " to " + newDropRate +
", OverrideDropRate=" + _config.getOverrideDropRate() +
", AdjustedMinCallCount=" + adjustedMinCallCount() +
", CallCount=" + _callTrackerStats.getCallCount() +
", Latency=" + _latency +
", ErrorRate=" + getErrorRateToDegrade() +
", OutstandingLatency=" + _outstandingLatency +
", OutstandingCount=" + stats.getOutstandingCount() +
", NoOverrideDropCountTotal=" + noOverrideDropCountTotal +
", DroppedCountTotal=" + droppedCountTotal +
", LastIntervalDroppedRate=" + lastIntervalDroppedRate);
}
log.warn(_config.getName() + " ComputedDropRate " +
(oldDropRate > newDropRate ? "decreased" : "increased") +
" from " + oldDropRate + " to " + newDropRate +
", OverrideDropRate=" + _config.getOverrideDropRate() +
", AdjustedMinCallCount=" + adjustedMinCallCount() +
", CallCount=" + _callTrackerStats.getCallCount() +
", Latency=" + _latency +
", ErrorRate=" + getErrorRateToDegrade() +
", OutstandingLatency=" + _outstandingLatency +
", OutstandingCount=" + stats.getOutstandingCount() +
", NoOverrideDropCountTotal=" + noOverrideDropCountTotal +
", DroppedCountTotal=" + droppedCountTotal +
", LastIntervalDroppedRate=" + lastIntervalDroppedRate);
}
else
{
@@ -680,6 +679,7 @@ public LongStats getCallTimeStats()
protected double _initialDropRate = DEFAULT_INITIAL_DROP_RATE;
protected double _slowStartThreshold = DEFAULT_SLOW_START_THRESHOLD;
protected Logger _logger = LoggerFactory.getLogger(ImmutableConfig.class);
protected double _logThreshold = DEFAULT_LOG_THRESHOLD;
public ImmutableConfig()
{
@@ -820,6 +820,11 @@ public Logger getLogger()
{
return _logger;
}
public double getLogThreshold()
{
return _logThreshold;
}
}
public static class Config extends ImmutableConfig
@@ -943,5 +948,10 @@ public void setLogger(Logger logger)
{
_logger = logger;
}
public void setLogThreshold(double threshold)
{
_logThreshold = threshold;
}
}
}

0 comments on commit f697f57

Please sign in to comment.