From 40f6b181958e4095abc8fb9b37e848e6cdbafc72 Mon Sep 17 00:00:00 2001 From: Lucas Echeverz Date: Tue, 23 Mar 2021 12:18:56 -0300 Subject: [PATCH 01/81] Cleaning up current logic --- .../split/client/LocalhostSplitFactory.java | 2 +- .../java/io/split/client/SplitClientImpl.java | 4 - .../io/split/client/SplitFactoryImpl.java | 24 +-- .../split/client/metrics/CachedMetrics.java | 142 ------------------ .../client/metrics/FireAndForgetMetrics.java | 123 --------------- .../io/split/client/SplitClientImplTest.java | 34 ----- .../client/metrics/CachedMetricsTest.java | 77 ---------- 7 files changed, 4 insertions(+), 402 deletions(-) delete mode 100644 client/src/main/java/io/split/client/metrics/CachedMetrics.java delete mode 100644 client/src/main/java/io/split/client/metrics/FireAndForgetMetrics.java delete mode 100644 client/src/test/java/io/split/client/metrics/CachedMetricsTest.java diff --git a/client/src/main/java/io/split/client/LocalhostSplitFactory.java b/client/src/main/java/io/split/client/LocalhostSplitFactory.java index 0ec01f8c9..7a7cbf888 100644 --- a/client/src/main/java/io/split/client/LocalhostSplitFactory.java +++ b/client/src/main/java/io/split/client/LocalhostSplitFactory.java @@ -58,7 +58,7 @@ public LocalhostSplitFactory(String directory, String file) throws IOException { _cacheUpdaterService = new CacheUpdaterService(splitCache); _cacheUpdaterService.updateCache(splitAndKeyToTreatment); _client = new SplitClientImpl(this, splitCache, - new ImpressionsManager.NoOpImpressionsManager(), new Metrics.NoopMetrics(), new NoopEventClient(), + new ImpressionsManager.NoOpImpressionsManager(), new NoopEventClient(), SplitClientConfig.builder().setBlockUntilReadyTimeout(1).build(), sdkReadinessGates, new EvaluatorImp(splitCache)); _manager = LocalhostSplitManager.of(splitAndKeyToTreatment); diff --git a/client/src/main/java/io/split/client/SplitClientImpl.java b/client/src/main/java/io/split/client/SplitClientImpl.java index 7792f5a1f..3129e4526 100644 --- a/client/src/main/java/io/split/client/SplitClientImpl.java +++ b/client/src/main/java/io/split/client/SplitClientImpl.java @@ -43,7 +43,6 @@ public final class SplitClientImpl implements SplitClient { private final SplitFactory _container; private final SplitCache _splitCache; private final ImpressionsManager _impressionManager; - private final Metrics _metrics; private final SplitClientConfig _config; private final EventClient _eventClient; private final SDKReadinessGates _gates; @@ -52,7 +51,6 @@ public final class SplitClientImpl implements SplitClient { public SplitClientImpl(SplitFactory container, SplitCache splitCache, ImpressionsManager impressionManager, - Metrics metrics, EventClient eventClient, SplitClientConfig config, SDKReadinessGates gates, @@ -60,7 +58,6 @@ public SplitClientImpl(SplitFactory container, _container = container; _splitCache = checkNotNull(splitCache); _impressionManager = checkNotNull(impressionManager); - _metrics = metrics; _eventClient = eventClient; _config = config; _gates = checkNotNull(gates); @@ -235,7 +232,6 @@ private void recordStats(String matchingKey, String bucketingKey, String split, String operation, String label, Long changeNumber, Map attributes) { try { _impressionManager.track(new Impression(matchingKey, bucketingKey, split, result, System.currentTimeMillis(), label, changeNumber, attributes)); - _metrics.time(operation, System.currentTimeMillis() - start); } catch (Throwable t) { _log.error("Exception", t); } diff --git a/client/src/main/java/io/split/client/SplitFactoryImpl.java b/client/src/main/java/io/split/client/SplitFactoryImpl.java index 8b576d0dd..0914e6c34 100644 --- a/client/src/main/java/io/split/client/SplitFactoryImpl.java +++ b/client/src/main/java/io/split/client/SplitFactoryImpl.java @@ -6,8 +6,6 @@ import io.split.client.interceptors.AddSplitHeadersFilter; import io.split.client.interceptors.GzipDecoderResponseInterceptor; import io.split.client.interceptors.GzipEncoderRequestInterceptor; -import io.split.client.metrics.CachedMetrics; -import io.split.client.metrics.FireAndForgetMetrics; import io.split.client.metrics.HttpMetrics; import io.split.cache.InMemoryCacheImp; import io.split.cache.SplitCache; @@ -68,12 +66,10 @@ public class SplitFactoryImpl implements SplitFactory { private final CloseableHttpClient _httpclient; private final SDKReadinessGates _gates; private final HttpMetrics _httpMetrics; - private final FireAndForgetMetrics _unCachedFireAndForget; private final SegmentSynchronizationTaskImp _segmentSynchronizationTaskImp; private final SplitFetcher _splitFetcher; private final SplitSynchronizationTask _splitSynchronizationTask; private final ImpressionsManagerImpl _impressionsManager; - private final FireAndForgetMetrics _cachedFireAndForgetMetrics; private final EventClient _eventClient; private final SyncManager _syncManager; private final Evaluator _evaluator; @@ -119,9 +115,6 @@ public SplitFactoryImpl(String apiToken, SplitClientConfig config) throws URISyn _segmentCache = new SegmentCacheInMemoryImpl(); _splitCache = new InMemoryCacheImp(); - // Metrics - _unCachedFireAndForget = FireAndForgetMetrics.instance(_httpMetrics, 2, 1000); - // Segments _segmentSynchronizationTaskImp = buildSegments(config); @@ -134,8 +127,6 @@ public SplitFactoryImpl(String apiToken, SplitClientConfig config) throws URISyn // Impressions _impressionsManager = buildImpressionsManager(config); - // CachedFireAndForgetMetrics - _cachedFireAndForgetMetrics = buildCachedFireAndForgetMetrics(config); // EventClient _eventClient = EventClientImpl.create(_httpclient, _eventsRootTarget, config.eventsQueueSize(), config.eventFlushIntervalInMillis(), config.waitBeforeShutdown()); @@ -148,7 +139,7 @@ public SplitFactoryImpl(String apiToken, SplitClientConfig config) throws URISyn _evaluator = new EvaluatorImp(_splitCache); // SplitClient - _client = new SplitClientImpl(this, _splitCache, _impressionsManager, _cachedFireAndForgetMetrics, _eventClient, config, _gates, _evaluator); + _client = new SplitClientImpl(this, _splitCache, _impressionsManager, _eventClient, config, _gates, _evaluator); // SplitManager _manager = new SplitManagerImpl(_splitCache, config, _gates); @@ -183,10 +174,6 @@ public synchronized void destroy() { _log.info("Successful shutdown of splits"); _impressionsManager.close(); _log.info("Successful shutdown of impressions manager"); - _unCachedFireAndForget.close(); - _log.info("Successful shutdown of metrics 1"); - _cachedFireAndForgetMetrics.close(); - _log.info("Successful shutdown of metrics 2"); _httpclient.close(); _log.info("Successful shutdown of httpclient"); _eventClient.close(); @@ -296,7 +283,7 @@ private static int findPollingPeriod(Random rand, int max) { } private SegmentSynchronizationTaskImp buildSegments(SplitClientConfig config) throws URISyntaxException { - SegmentChangeFetcher segmentChangeFetcher = HttpSegmentChangeFetcher.create(_httpclient, _rootTarget, _unCachedFireAndForget); + SegmentChangeFetcher segmentChangeFetcher = HttpSegmentChangeFetcher.create(_httpclient, _rootTarget); return new SegmentSynchronizationTaskImp(segmentChangeFetcher, findPollingPeriod(RANDOM, config.segmentsRefreshRate()), @@ -306,7 +293,7 @@ private SegmentSynchronizationTaskImp buildSegments(SplitClientConfig config) th } private SplitFetcher buildSplitFetcher() throws URISyntaxException { - SplitChangeFetcher splitChangeFetcher = HttpSplitChangeFetcher.create(_httpclient, _rootTarget, _unCachedFireAndForget); + SplitChangeFetcher splitChangeFetcher = HttpSplitChangeFetcher.create(_httpclient, _rootTarget); SplitParser splitParser = new SplitParser(_segmentSynchronizationTaskImp, _segmentCache); return new SplitFetcherImp(splitChangeFetcher, splitParser, _gates, _splitCache); @@ -327,9 +314,4 @@ private ImpressionsManagerImpl buildImpressionsManager(SplitClientConfig config) return ImpressionsManagerImpl.instance(_httpclient, config, impressionListeners); } - private FireAndForgetMetrics buildCachedFireAndForgetMetrics(SplitClientConfig config) { - CachedMetrics cachedMetrics = new CachedMetrics(_httpMetrics, TimeUnit.SECONDS.toMillis(config.metricsRefreshRate())); - - return FireAndForgetMetrics.instance(cachedMetrics, 2, 1000); - } } diff --git a/client/src/main/java/io/split/client/metrics/CachedMetrics.java b/client/src/main/java/io/split/client/metrics/CachedMetrics.java deleted file mode 100644 index 41f066a38..000000000 --- a/client/src/main/java/io/split/client/metrics/CachedMetrics.java +++ /dev/null @@ -1,142 +0,0 @@ - -package io.split.client.metrics; - -import com.google.common.collect.Maps; -import com.google.common.primitives.Longs; -import io.split.client.dtos.Counter; -import io.split.client.dtos.Latency; -import io.split.engine.metrics.Metrics; - -import java.util.Map; -import java.util.concurrent.TimeUnit; -import java.util.concurrent.atomic.AtomicLong; - -import static com.google.common.base.Preconditions.checkArgument; - - -/** - * Created by adilaijaz on 9/4/15. - */ -public class CachedMetrics implements Metrics { - - private final DTOMetrics _metrics; - - private final Map _latencyMap; - private final Map _countMap; - - - private final Object _latencyLock = new Object(); - private AtomicLong _latencyLastUpdateTimeMillis = new AtomicLong(System.currentTimeMillis()); - - private final Object _counterLock = new Object(); - private AtomicLong _counterLastUpdateTimeMillis = new AtomicLong(System.currentTimeMillis()); - - private long _refreshPeriodInMillis; - - private final int _queueForTheseManyCalls; - - /** - * For unit testing only. - * - * @param httpMetrics - * @param queueForTheseManyCalls - */ - /*package private*/ CachedMetrics(DTOMetrics httpMetrics, int queueForTheseManyCalls) { - this(httpMetrics, queueForTheseManyCalls, TimeUnit.MINUTES.toMillis(1)); - } - - public CachedMetrics(DTOMetrics httpMetrics, long refreshPeriodInMillis) { - this(httpMetrics, 100, refreshPeriodInMillis); - } - - private CachedMetrics(DTOMetrics metrics, int queueForTheseManyCalls, long refreshPeriodInMillis) { - _metrics = metrics; - _latencyMap = Maps.newHashMap(); - _countMap = Maps.newHashMap(); - checkArgument(queueForTheseManyCalls > 0, "queue for cache should be greater than zero"); - _queueForTheseManyCalls = queueForTheseManyCalls; - _refreshPeriodInMillis = refreshPeriodInMillis; - } - - @Override - public void count(String counter, long delta) { - if (delta <= 0) { - return; - } - - if (counter == null || counter.trim().isEmpty()) { - return; - } - - synchronized (_counterLock) { - SumAndCount sumAndCount = _countMap.get(counter); - if (sumAndCount == null) { - sumAndCount = new SumAndCount(); - _countMap.put(counter, sumAndCount); - } - - sumAndCount.addDelta(delta); - - if (sumAndCount._count >= _queueForTheseManyCalls || hasTimeElapsed(_counterLastUpdateTimeMillis)) { - Counter dto = new Counter(); - dto.name = counter; - dto.delta = sumAndCount._sum; - - sumAndCount.clear(); - _counterLastUpdateTimeMillis.set(System.currentTimeMillis()); - _metrics.count(dto); - } - } - } - - private boolean hasTimeElapsed(AtomicLong lastRefreshTime) { - return (System.currentTimeMillis() - lastRefreshTime.get()) > _refreshPeriodInMillis; - } - - @Override - public void time(String operation, long timeInMs) { - if (operation == null || operation.trim().isEmpty() || timeInMs < 0L) { - // error - return; - } - synchronized (_latencyLock) { - if (!_latencyMap.containsKey(operation)) { - ILatencyTracker latencies = new BinarySearchLatencyTracker(); - _latencyMap.put(operation, latencies); - } - - ILatencyTracker tracker = _latencyMap.get(operation); - tracker.addLatencyMillis((int) timeInMs); - - if (hasTimeElapsed(_latencyLastUpdateTimeMillis)) { - - Latency dto = new Latency(); - dto.name = operation; - dto.latencies = Longs.asList(tracker.getLatencies()); - - tracker.clear(); - _latencyLastUpdateTimeMillis.set(System.currentTimeMillis()); - _metrics.time(dto); - - } - } - } - - - private static final class SumAndCount { - private int _count = 0; - private long _sum = 0L; - - public void addDelta(long delta) { - _count++; - _sum += delta; - } - - public void clear() { - _count = 0; - _sum = 0L; - } - - } - -} diff --git a/client/src/main/java/io/split/client/metrics/FireAndForgetMetrics.java b/client/src/main/java/io/split/client/metrics/FireAndForgetMetrics.java deleted file mode 100644 index 43aa702de..000000000 --- a/client/src/main/java/io/split/client/metrics/FireAndForgetMetrics.java +++ /dev/null @@ -1,123 +0,0 @@ -package io.split.client.metrics; - -import com.google.common.util.concurrent.ThreadFactoryBuilder; -import io.split.engine.metrics.Metrics; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -import java.io.Closeable; -import java.util.List; -import java.util.concurrent.ArrayBlockingQueue; -import java.util.concurrent.ExecutorService; -import java.util.concurrent.ThreadPoolExecutor; -import java.util.concurrent.TimeUnit; - -/** - * Created by adilaijaz on 9/4/15. - */ -public class FireAndForgetMetrics implements Metrics, Closeable { - - private static final Logger _log = LoggerFactory.getLogger(FireAndForgetMetrics.class); - - private final ExecutorService _executorService; - private final Metrics _delegate; - - public static FireAndForgetMetrics instance(Metrics delegate, int numberOfThreads, int queueSize) { - ThreadFactoryBuilder threadFactoryBuilder = new ThreadFactoryBuilder(); - threadFactoryBuilder.setDaemon(true); - threadFactoryBuilder.setNameFormat("split-fireAndForgetMetrics-%d"); - threadFactoryBuilder.setUncaughtExceptionHandler(new Thread.UncaughtExceptionHandler() { - @Override - public void uncaughtException(Thread t, Throwable e) { - _log.error("Error in thread: " + t.getName(), e); - } - }); - - final ExecutorService executorService = new ThreadPoolExecutor(numberOfThreads, - numberOfThreads, - 0L, - TimeUnit.MILLISECONDS, - new ArrayBlockingQueue(queueSize), - threadFactoryBuilder.build(), - new ThreadPoolExecutor.DiscardPolicy()); - - - return new FireAndForgetMetrics(delegate, executorService); - } - - private FireAndForgetMetrics(Metrics delegate, ExecutorService executorService) { - _delegate = delegate; - _executorService = executorService; - } - - - @Override - public void count(String counter, long delta) { - try { - _executorService.submit(new CountRunnable(_delegate, counter, delta)); - } catch (Throwable t) { - _log.warn("CountRunnable failed", t); - } - } - - @Override - public void time(String operation, long timeInMs) { - try { - _executorService.submit(new TimeRunnable(_delegate, operation, timeInMs)); - } catch (Throwable t) { - _log.warn("TimeRunnable failed", t); - } - } - - public void close() { - _executorService.shutdown(); - try { - if (!_executorService.awaitTermination(10L, TimeUnit.SECONDS)) { //optional * - _log.info("Executor did not terminate in the specified time."); - List droppedTasks = _executorService.shutdownNow(); //optional ** - _log.info("Executor was abruptly shut down. These tasks will not be executed: " + droppedTasks); - } - } catch (InterruptedException e) { - // reset the interrupt. - Thread.currentThread().interrupt(); - } - } - - - private static final class CountRunnable implements Runnable { - - private final Metrics _delegate; - private final String _name; - private final long _delta; - - public CountRunnable(Metrics delegate, String name, long delta) { - _delegate = delegate; - _name = name; - _delta = delta; - } - - @Override - public void run() { - _delegate.count(_name, _delta); - } - } - - private static final class TimeRunnable implements Runnable { - - private final Metrics _delegate; - private final String _name; - private final long _timeInMs; - - public TimeRunnable(Metrics delegate, String name, long timeInMs) { - _delegate = delegate; - _name = name; - _timeInMs = timeInMs; - } - - @Override - public void run() { - _delegate.time(_name, _timeInMs); - } - } - -} diff --git a/client/src/test/java/io/split/client/SplitClientImplTest.java b/client/src/test/java/io/split/client/SplitClientImplTest.java index 4e1370ad4..dbee7934c 100644 --- a/client/src/test/java/io/split/client/SplitClientImplTest.java +++ b/client/src/test/java/io/split/client/SplitClientImplTest.java @@ -78,7 +78,6 @@ public void null_key_results_in_control() { mock(SplitFactory.class), splitCache, new ImpressionsManager.NoOpImpressionsManager(), - new Metrics.NoopMetrics(), NoopEventClient.create(), config, gates, @@ -105,7 +104,6 @@ public void null_test_results_in_control() { mock(SplitFactory.class), splitCache, new ImpressionsManager.NoOpImpressionsManager(), - new Metrics.NoopMetrics(), NoopEventClient.create(), config, gates, @@ -127,7 +125,6 @@ public void exceptions_result_in_control() { mock(SplitFactory.class), splitCache, new ImpressionsManager.NoOpImpressionsManager(), - new Metrics.NoopMetrics(), NoopEventClient.create(), config, gates, @@ -154,7 +151,6 @@ public void works() { mock(SplitFactory.class), splitCache, new ImpressionsManager.NoOpImpressionsManager(), - new Metrics.NoopMetrics(), NoopEventClient.create(), config, gates, @@ -189,7 +185,6 @@ public void works_null_config() { mock(SplitFactory.class), splitCache, new ImpressionsManager.NoOpImpressionsManager(), - new Metrics.NoopMetrics(), NoopEventClient.create(), config, gates, @@ -226,7 +221,6 @@ public void worksAndHasConfig() { mock(SplitFactory.class), splitCache, new ImpressionsManager.NoOpImpressionsManager(), - new Metrics.NoopMetrics(), NoopEventClient.create(), config, gates, @@ -261,7 +255,6 @@ public void last_condition_is_always_default() { mock(SplitFactory.class), splitCache, new ImpressionsManager.NoOpImpressionsManager(), - new Metrics.NoopMetrics(), NoopEventClient.create(), config, gates, @@ -297,7 +290,6 @@ public void last_condition_is_always_default_but_with_treatment() { mock(SplitFactory.class), splitCache, new ImpressionsManager.NoOpImpressionsManager(), - new Metrics.NoopMetrics(), NoopEventClient.create(), config, gates, @@ -330,7 +322,6 @@ public void multiple_conditions_work() { mock(SplitFactory.class), splitCache, new ImpressionsManager.NoOpImpressionsManager(), - new Metrics.NoopMetrics(), NoopEventClient.create(), config, gates, @@ -361,7 +352,6 @@ public void killed_test_always_goes_to_default() { mock(SplitFactory.class), splitCache, new ImpressionsManager.NoOpImpressionsManager(), - new Metrics.NoopMetrics(), NoopEventClient.create(), config, gates, @@ -397,7 +387,6 @@ public void killed_test_always_goes_to_default_has_config() { mock(SplitFactory.class), splitCache, new ImpressionsManager.NoOpImpressionsManager(), - new Metrics.NoopMetrics(), NoopEventClient.create(), config, gates, @@ -433,7 +422,6 @@ public void dependency_matcher_on() { mock(SplitFactory.class), splitCache, new ImpressionsManager.NoOpImpressionsManager(), - new Metrics.NoopMetrics(), NoopEventClient.create(), config, gates, @@ -466,7 +454,6 @@ public void dependency_matcher_off() { mock(SplitFactory.class), splitCache, new ImpressionsManager.NoOpImpressionsManager(), - new Metrics.NoopMetrics(), NoopEventClient.create(), config, gates, @@ -493,7 +480,6 @@ public void dependency_matcher_control() { mock(SplitFactory.class), splitCache, new ImpressionsManager.NoOpImpressionsManager(), - new Metrics.NoopMetrics(), NoopEventClient.create(), config, gates, @@ -521,7 +507,6 @@ public void attributes_work() { mock(SplitFactory.class), splitCache, new ImpressionsManager.NoOpImpressionsManager(), - new Metrics.NoopMetrics(), NoopEventClient.create(), config, gates, @@ -555,7 +540,6 @@ public void attributes_work_2() { mock(SplitFactory.class), splitCache, new ImpressionsManager.NoOpImpressionsManager(), - new Metrics.NoopMetrics(), NoopEventClient.create(), config, gates, @@ -589,7 +573,6 @@ public void attributes_greater_than_negative_number() { mock(SplitFactory.class), splitCache, new ImpressionsManager.NoOpImpressionsManager(), - new Metrics.NoopMetrics(), NoopEventClient.create(), config, gates, @@ -626,7 +609,6 @@ public void attributes_for_sets() { mock(SplitFactory.class), splitCache, new ImpressionsManager.NoOpImpressionsManager(), - new Metrics.NoopMetrics(), NoopEventClient.create(), config, gates, @@ -669,7 +651,6 @@ public void labels_are_populated() { mock(SplitFactory.class), splitCache, impressionsManager, - new Metrics.NoopMetrics(), NoopEventClient.create(), config, gates, @@ -761,7 +742,6 @@ private void traffic_allocation(String key, int trafficAllocation, int trafficAl mock(SplitFactory.class), splitCache, impressionsManager, - new Metrics.NoopMetrics(), NoopEventClient.create(), config, gates, @@ -809,7 +789,6 @@ public void notInTrafficAllocationDefaultConfig() { mock(SplitFactory.class), splitCache, impressionsManager, - new Metrics.NoopMetrics(), NoopEventClient.create(), config, gates, @@ -849,7 +828,6 @@ public void matching_bucketing_keys_work() { mock(SplitFactory.class), splitCache, new ImpressionsManager.NoOpImpressionsManager(), - new Metrics.NoopMetrics(), NoopEventClient.create(), config, gates, @@ -887,7 +865,6 @@ public void impression_metadata_is_propagated() { mock(SplitFactory.class), splitCache, impressionsManager, - new Metrics.NoopMetrics(), NoopEventClient.create(), config, gates, @@ -923,7 +900,6 @@ public void block_until_ready_does_not_time_when_sdk_is_ready() throws TimeoutEx mock(SplitFactory.class), splitCache, mock(ImpressionsManager.class), - new Metrics.NoopMetrics(), NoopEventClient.create(), config, ready, @@ -943,7 +919,6 @@ public void block_until_ready_times_when_sdk_is_not_ready() throws TimeoutExcept mock(SplitFactory.class), splitCache, mock(ImpressionsManager.class), - new Metrics.NoopMetrics(), NoopEventClient.create(), config, ready, @@ -962,7 +937,6 @@ public void track_with_valid_parameters() { mock(SplitFactory.class), splitCache, new ImpressionsManager.NoOpImpressionsManager(), - new Metrics.NoopMetrics(), NoopEventClient.create(), config, gates, @@ -988,7 +962,6 @@ public void track_with_invalid_event_type_ids() { mock(SplitFactory.class), splitCache, new ImpressionsManager.NoOpImpressionsManager(), - new Metrics.NoopMetrics(), NoopEventClient.create(), config, gates, @@ -1019,7 +992,6 @@ public void track_with_invalid_traffic_type_names() { mock(SplitFactory.class), splitCache, new ImpressionsManager.NoOpImpressionsManager(), - new Metrics.NoopMetrics(), NoopEventClient.create(), config, gates, @@ -1042,7 +1014,6 @@ public void track_with_invalid_keys() { mock(SplitFactory.class), splitCache, new ImpressionsManager.NoOpImpressionsManager(), - new Metrics.NoopMetrics(), NoopEventClient.create(), config, gates, @@ -1071,7 +1042,6 @@ public void track_with_properties() { mock(SplitFactory.class), splitCache, new ImpressionsManager.NoOpImpressionsManager(), - new Metrics.NoopMetrics(), eventClientMock, config, gates, @@ -1182,7 +1152,6 @@ public void getTreatment_with_invalid_keys() { mock(SplitFactory.class), splitCache, new ImpressionsManager.NoOpImpressionsManager(), - new Metrics.NoopMetrics(), NoopEventClient.create(), config, gates, @@ -1267,7 +1236,6 @@ public void client_cannot_perform_actions_when_destroyed() throws InterruptedExc mockFactory, splitCache, new ImpressionsManager.NoOpImpressionsManager(), - new Metrics.NoopMetrics(), NoopEventClient.create(), config, gates, @@ -1310,7 +1278,6 @@ public void worksAndHasConfigTryKetTreatmentWithKey() { mock(SplitFactory.class), splitCache, new ImpressionsManager.NoOpImpressionsManager(), - new Metrics.NoopMetrics(), NoopEventClient.create(), config, gates, @@ -1347,7 +1314,6 @@ public void blockUntilReadyException() throws TimeoutException, InterruptedExcep mock(SplitFactory.class), splitCache, new ImpressionsManager.NoOpImpressionsManager(), - new Metrics.NoopMetrics(), NoopEventClient.create(), config, gates, diff --git a/client/src/test/java/io/split/client/metrics/CachedMetricsTest.java b/client/src/test/java/io/split/client/metrics/CachedMetricsTest.java deleted file mode 100644 index 57dd8a181..000000000 --- a/client/src/test/java/io/split/client/metrics/CachedMetricsTest.java +++ /dev/null @@ -1,77 +0,0 @@ -package io.split.client.metrics; - -import com.google.common.collect.Lists; -import io.split.client.dtos.Counter; -import io.split.client.dtos.Latency; -import org.junit.Test; - -import java.util.List; - -import static org.hamcrest.Matchers.equalTo; -import static org.hamcrest.Matchers.is; -import static org.junit.Assert.assertThat; - -/** - * Created by adilaijaz on 9/23/15. - */ -public class CachedMetricsTest { - - private static final class MyDTOMetrics implements DTOMetrics { - - private List latencies = Lists.newArrayList(); - private List counters = Lists.newArrayList(); - - @Override - public void time(Latency dto) { - latencies.add(dto); - } - - @Override - public void count(Counter dto) { - counters.add(dto); - } - } - - @Test - public void count() { - MyDTOMetrics metrics = new MyDTOMetrics(); - - CachedMetrics cachedMetrics = new CachedMetrics(metrics, 2); - - cachedMetrics.count("foo", 4); - cachedMetrics.count("foo", 5); - cachedMetrics.count("foo", 6); - cachedMetrics.count("foo", 7); - cachedMetrics.count("foo", 8); - - Counter counter = new Counter(); - counter.name = "foo"; - counter.delta = 9L; - - assertThat(metrics.counters.size(), is(equalTo(2))); - assertThat(metrics.counters.get(0).name, is(equalTo("foo"))); - assertThat(metrics.counters.get(0).delta, is(equalTo(9L))); - - assertThat(metrics.counters.get(1).name, is(equalTo("foo"))); - assertThat(metrics.counters.get(1).delta, is(equalTo(13L))); - } - - @Test - public void latency() throws Exception { - MyDTOMetrics delegate = new MyDTOMetrics(); - CachedMetrics cachedMetrics = new CachedMetrics(delegate, 15L); - - cachedMetrics.time("foo", 4); - cachedMetrics.time("foo", 5); - cachedMetrics.time("foo", 6); - cachedMetrics.time("foo", 7); - Thread.sleep(30); - cachedMetrics.time("foo", 8); - - List latencies = Lists.newArrayList(0L, 0L, 0L, 0L, 2L, 2L, 1L, 0L, 0L, 0L, 0L, 0L, 0L, 0L, 0L, 0L, 0L, 0L, 0L, 0L, 0L, 0L, 0L); - - assertThat(delegate.latencies.get(0).name, is(equalTo("foo"))); - assertThat(delegate.latencies.get(0).latencies, is(equalTo(latencies))); - } - -} From bfc53017bb7e8abb9b04c1a685a12093aba53215 Mon Sep 17 00:00:00 2001 From: Lucas Echeverz Date: Mon, 29 Mar 2021 17:53:39 -0300 Subject: [PATCH 02/81] Interfaces --- .../split/telemetry/storage/TelemetryEvaluationConsumer.java | 4 ++++ .../split/telemetry/storage/TelemetryEvaluationProducer.java | 4 ++++ .../io/split/telemetry/storage/TelemetryInitConsumer.java | 4 ++++ .../io/split/telemetry/storage/TelemetryInitProducer.java | 4 ++++ .../io/split/telemetry/storage/TelemetryRuntimeConsumer.java | 4 ++++ .../io/split/telemetry/storage/TelemetryRuntimeProducer.java | 4 ++++ .../java/io/split/telemetry/storage/TelemetryStorage.java | 4 ++++ .../io/split/telemetry/storage/TelemetryStorageConsumer.java | 4 ++++ .../io/split/telemetry/storage/TelemetryStorageProducer.java | 4 ++++ 9 files changed, 36 insertions(+) create mode 100644 client/src/main/java/io/split/telemetry/storage/TelemetryEvaluationConsumer.java create mode 100644 client/src/main/java/io/split/telemetry/storage/TelemetryEvaluationProducer.java create mode 100644 client/src/main/java/io/split/telemetry/storage/TelemetryInitConsumer.java create mode 100644 client/src/main/java/io/split/telemetry/storage/TelemetryInitProducer.java create mode 100644 client/src/main/java/io/split/telemetry/storage/TelemetryRuntimeConsumer.java create mode 100644 client/src/main/java/io/split/telemetry/storage/TelemetryRuntimeProducer.java create mode 100644 client/src/main/java/io/split/telemetry/storage/TelemetryStorage.java create mode 100644 client/src/main/java/io/split/telemetry/storage/TelemetryStorageConsumer.java create mode 100644 client/src/main/java/io/split/telemetry/storage/TelemetryStorageProducer.java diff --git a/client/src/main/java/io/split/telemetry/storage/TelemetryEvaluationConsumer.java b/client/src/main/java/io/split/telemetry/storage/TelemetryEvaluationConsumer.java new file mode 100644 index 000000000..3537540ce --- /dev/null +++ b/client/src/main/java/io/split/telemetry/storage/TelemetryEvaluationConsumer.java @@ -0,0 +1,4 @@ +package io.split.telemetry.storage; + +public interface TelemetryEvaluationConsumer { +} diff --git a/client/src/main/java/io/split/telemetry/storage/TelemetryEvaluationProducer.java b/client/src/main/java/io/split/telemetry/storage/TelemetryEvaluationProducer.java new file mode 100644 index 000000000..9c4e59d4b --- /dev/null +++ b/client/src/main/java/io/split/telemetry/storage/TelemetryEvaluationProducer.java @@ -0,0 +1,4 @@ +package io.split.telemetry.storage; + +public interface TelemetryEvaluationProducer { +} diff --git a/client/src/main/java/io/split/telemetry/storage/TelemetryInitConsumer.java b/client/src/main/java/io/split/telemetry/storage/TelemetryInitConsumer.java new file mode 100644 index 000000000..2430ff765 --- /dev/null +++ b/client/src/main/java/io/split/telemetry/storage/TelemetryInitConsumer.java @@ -0,0 +1,4 @@ +package io.split.telemetry.storage; + +public interface TelemetryInitConsumer { +} diff --git a/client/src/main/java/io/split/telemetry/storage/TelemetryInitProducer.java b/client/src/main/java/io/split/telemetry/storage/TelemetryInitProducer.java new file mode 100644 index 000000000..9547967d4 --- /dev/null +++ b/client/src/main/java/io/split/telemetry/storage/TelemetryInitProducer.java @@ -0,0 +1,4 @@ +package io.split.telemetry.storage; + +public interface TelemetryInitProducer { +} diff --git a/client/src/main/java/io/split/telemetry/storage/TelemetryRuntimeConsumer.java b/client/src/main/java/io/split/telemetry/storage/TelemetryRuntimeConsumer.java new file mode 100644 index 000000000..661b75cdb --- /dev/null +++ b/client/src/main/java/io/split/telemetry/storage/TelemetryRuntimeConsumer.java @@ -0,0 +1,4 @@ +package io.split.telemetry.storage; + +public interface TelemetryRuntimeConsumer { +} diff --git a/client/src/main/java/io/split/telemetry/storage/TelemetryRuntimeProducer.java b/client/src/main/java/io/split/telemetry/storage/TelemetryRuntimeProducer.java new file mode 100644 index 000000000..8d6e3672f --- /dev/null +++ b/client/src/main/java/io/split/telemetry/storage/TelemetryRuntimeProducer.java @@ -0,0 +1,4 @@ +package io.split.telemetry.storage; + +public interface TelemetryRuntimeProducer { +} diff --git a/client/src/main/java/io/split/telemetry/storage/TelemetryStorage.java b/client/src/main/java/io/split/telemetry/storage/TelemetryStorage.java new file mode 100644 index 000000000..477965099 --- /dev/null +++ b/client/src/main/java/io/split/telemetry/storage/TelemetryStorage.java @@ -0,0 +1,4 @@ +package io.split.telemetry.storage; + +public interface TelemetryStorage extends TelemetryStorageConsumer, TelemetryStorageProducer{ +} diff --git a/client/src/main/java/io/split/telemetry/storage/TelemetryStorageConsumer.java b/client/src/main/java/io/split/telemetry/storage/TelemetryStorageConsumer.java new file mode 100644 index 000000000..217a6e27b --- /dev/null +++ b/client/src/main/java/io/split/telemetry/storage/TelemetryStorageConsumer.java @@ -0,0 +1,4 @@ +package io.split.telemetry.storage; + +public interface TelemetryStorageConsumer extends TelemetryInitConsumer, TelemetryRuntimeConsumer, TelemetryEvaluationConsumer{ +} diff --git a/client/src/main/java/io/split/telemetry/storage/TelemetryStorageProducer.java b/client/src/main/java/io/split/telemetry/storage/TelemetryStorageProducer.java new file mode 100644 index 000000000..382f5939f --- /dev/null +++ b/client/src/main/java/io/split/telemetry/storage/TelemetryStorageProducer.java @@ -0,0 +1,4 @@ +package io.split.telemetry.storage; + +public interface TelemetryStorageProducer extends TelemetryEvaluationProducer, TelemetryInitProducer, TelemetryRuntimeProducer{ +} From ae9da581d0c759ee5cc7ca842298b8d6f6ed969e Mon Sep 17 00:00:00 2001 From: Lucas Echeverz Date: Fri, 9 Apr 2021 12:45:55 -0300 Subject: [PATCH 03/81] Adding interfaces and Implementation --- .../io/split/telemetry/domain/HTTPErrors.java | 85 +++++ .../split/telemetry/domain/HTTPLatencies.java | 86 +++++ .../telemetry/domain/LastSynchronization.java | 73 +++++ .../telemetry/domain/MethodExceptions.java | 62 ++++ .../telemetry/domain/MethodLatencies.java | 73 +++++ .../telemetry/domain/StreamingEvent.java | 40 +++ .../domain/enums/EventsDataRecordsEnum.java | 6 + .../domain/enums/FactoryCountersEnum.java | 6 + .../domain/enums/HTTPLatenciesEnum.java | 10 + .../enums/LastSynchronizationRecordsEnum.java | 10 + .../telemetry/domain/enums/MethodEnum.java | 9 + .../domain/enums/PushCountersEnum.java | 6 + .../telemetry/domain/enums/ResourceEnum.java | 10 + .../domain/enums/SdkRecordsEnum.java | 5 + .../storage/InMemoryTelemetryStorage.java | 306 ++++++++++++++++++ .../storage/TelemetryConfigConsumer.java | 6 + .../storage/TelemetryConfigProducer.java | 7 + .../storage/TelemetryEvaluationConsumer.java | 5 + .../storage/TelemetryEvaluationProducer.java | 4 + .../storage/TelemetryInitConsumer.java | 4 - .../storage/TelemetryInitProducer.java | 4 - .../storage/TelemetryRuntimeConsumer.java | 20 ++ .../storage/TelemetryRuntimeProducer.java | 16 + .../storage/TelemetryStorageConsumer.java | 2 +- .../storage/TelemetryStorageProducer.java | 2 +- 25 files changed, 847 insertions(+), 10 deletions(-) create mode 100644 client/src/main/java/io/split/telemetry/domain/HTTPErrors.java create mode 100644 client/src/main/java/io/split/telemetry/domain/HTTPLatencies.java create mode 100644 client/src/main/java/io/split/telemetry/domain/LastSynchronization.java create mode 100644 client/src/main/java/io/split/telemetry/domain/MethodExceptions.java create mode 100644 client/src/main/java/io/split/telemetry/domain/MethodLatencies.java create mode 100644 client/src/main/java/io/split/telemetry/domain/StreamingEvent.java create mode 100644 client/src/main/java/io/split/telemetry/domain/enums/EventsDataRecordsEnum.java create mode 100644 client/src/main/java/io/split/telemetry/domain/enums/FactoryCountersEnum.java create mode 100644 client/src/main/java/io/split/telemetry/domain/enums/HTTPLatenciesEnum.java create mode 100644 client/src/main/java/io/split/telemetry/domain/enums/LastSynchronizationRecordsEnum.java create mode 100644 client/src/main/java/io/split/telemetry/domain/enums/MethodEnum.java create mode 100644 client/src/main/java/io/split/telemetry/domain/enums/PushCountersEnum.java create mode 100644 client/src/main/java/io/split/telemetry/domain/enums/ResourceEnum.java create mode 100644 client/src/main/java/io/split/telemetry/domain/enums/SdkRecordsEnum.java create mode 100644 client/src/main/java/io/split/telemetry/storage/InMemoryTelemetryStorage.java create mode 100644 client/src/main/java/io/split/telemetry/storage/TelemetryConfigConsumer.java create mode 100644 client/src/main/java/io/split/telemetry/storage/TelemetryConfigProducer.java delete mode 100644 client/src/main/java/io/split/telemetry/storage/TelemetryInitConsumer.java delete mode 100644 client/src/main/java/io/split/telemetry/storage/TelemetryInitProducer.java diff --git a/client/src/main/java/io/split/telemetry/domain/HTTPErrors.java b/client/src/main/java/io/split/telemetry/domain/HTTPErrors.java new file mode 100644 index 000000000..4b1051c56 --- /dev/null +++ b/client/src/main/java/io/split/telemetry/domain/HTTPErrors.java @@ -0,0 +1,85 @@ +package io.split.telemetry.domain; + +import com.google.gson.annotations.SerializedName; + +import java.util.Map; +import java.util.concurrent.ConcurrentHashMap; + +public class HTTPErrors { + /* package private */ static final String FIELD_SPLIT = "sp"; + /* package private */ static final String FIELD_SEGMENTS = "se"; + /* package private */ static final String FIELD_IMPRESSIONS = "im"; + /* package private */ static final String FIELD_EVENTS = "ev"; + /* package private */ static final String FIELD_TOKEN = "to"; + /* package private */ static final String FIELD_TELEMETRY = "te"; + + @SerializedName(FIELD_SPLIT) + private Map _splits; + @SerializedName(FIELD_SEGMENTS) + private Map _segments; + @SerializedName(FIELD_IMPRESSIONS) + private Map _impressions; + @SerializedName(FIELD_EVENTS) + private Map _events; + @SerializedName(FIELD_TOKEN) + private Map _token; + @SerializedName(FIELD_TELEMETRY) + private Map _telemetry; + + public HTTPErrors() { + _splits = new ConcurrentHashMap<>(); + _segments = new ConcurrentHashMap<>(); + _impressions = new ConcurrentHashMap<>(); + _events = new ConcurrentHashMap<>(); + _token = new ConcurrentHashMap<>(); + _telemetry = new ConcurrentHashMap<>(); + } + + public Map get_splits() { + return _splits; + } + + public void set_splits(Map _splits) { + this._splits = _splits; + } + + public Map get_segments() { + return _segments; + } + + public void set_segments(Map _segments) { + this._segments = _segments; + } + + public Map get_impressions() { + return _impressions; + } + + public void set_impressions(Map _impressions) { + this._impressions = _impressions; + } + + public Map get_events() { + return _events; + } + + public void set_events(Map _events) { + this._events = _events; + } + + public Map get_token() { + return _token; + } + + public void set_token(Map _token) { + this._token = _token; + } + + public Map get_telemetry() { + return _telemetry; + } + + public void set_telemetry(Map _telemetry) { + this._telemetry = _telemetry; + } +} diff --git a/client/src/main/java/io/split/telemetry/domain/HTTPLatencies.java b/client/src/main/java/io/split/telemetry/domain/HTTPLatencies.java new file mode 100644 index 000000000..bf32af484 --- /dev/null +++ b/client/src/main/java/io/split/telemetry/domain/HTTPLatencies.java @@ -0,0 +1,86 @@ +package io.split.telemetry.domain; + +import com.google.gson.annotations.SerializedName; + +import java.util.ArrayList; +import java.util.List; +import java.util.concurrent.ConcurrentHashMap; + +public class HTTPLatencies { + /* package private */ static final String FIELD_SPLIT = "sp"; + /* package private */ static final String FIELD_SEGMENTS = "se"; + /* package private */ static final String FIELD_IMPRESSIONS = "im"; + /* package private */ static final String FIELD_EVENTS = "ev"; + /* package private */ static final String FIELD_TOKEN = "to"; + /* package private */ static final String FIELD_TELEMETRY = "te"; + + @SerializedName(FIELD_SPLIT) + private List _splits; + @SerializedName(FIELD_SEGMENTS) + private List_segments; + @SerializedName(FIELD_IMPRESSIONS) + private List _impressions; + @SerializedName(FIELD_EVENTS) + private List _events; + @SerializedName(FIELD_TOKEN) + private List _token; + @SerializedName(FIELD_TELEMETRY) + private List _telemetry; + + public HTTPLatencies() { + _splits = new ArrayList<>(); + _segments = new ArrayList<>(); + _impressions = new ArrayList<>(); + _events = new ArrayList<>(); + _token = new ArrayList<>(); + _telemetry = new ArrayList<>(); + } + + public List get_splits() { + return _splits; + } + + public void set_splits(List _splits) { + this._splits = _splits; + } + + public List get_segments() { + return _segments; + } + + public void set_segments(List _segments) { + this._segments = _segments; + } + + public List get_impressions() { + return _impressions; + } + + public void set_impressions(List _impressions) { + this._impressions = _impressions; + } + + public List get_events() { + return _events; + } + + public void set_events(List _events) { + this._events = _events; + } + + public List get_token() { + return _token; + } + + public void set_token(List _token) { + this._token = _token; + } + + public List get_telemetry() { + return _telemetry; + } + + public void set_telemetry(List _telemetry) { + this._telemetry = _telemetry; + } +} diff --git a/client/src/main/java/io/split/telemetry/domain/LastSynchronization.java b/client/src/main/java/io/split/telemetry/domain/LastSynchronization.java new file mode 100644 index 000000000..0b77c8a06 --- /dev/null +++ b/client/src/main/java/io/split/telemetry/domain/LastSynchronization.java @@ -0,0 +1,73 @@ +package io.split.telemetry.domain; + +import com.google.gson.annotations.SerializedName; + +public class LastSynchronization { + /* package private */ static final String FIELD_SPLIT = "sp"; + /* package private */ static final String FIELD_SEGMENTS = "se"; + /* package private */ static final String FIELD_IMPRESSIONS = "im"; + /* package private */ static final String FIELD_EVENTS = "ev"; + /* package private */ static final String FIELD_TOKEN = "to"; + /* package private */ static final String FIELD_TELEMETRY = "te"; + + @SerializedName(FIELD_SPLIT) + private long _splits; + @SerializedName(FIELD_SEGMENTS) + private long _segments; + @SerializedName(FIELD_IMPRESSIONS) + private long _impressions; + @SerializedName(FIELD_EVENTS) + private long _events; + @SerializedName(FIELD_TOKEN) + private long _token; + @SerializedName(FIELD_TELEMETRY) + private long _telemetry; + + public long get_splits() { + return _splits; + } + + public void set_splits(long _splits) { + this._splits = _splits; + } + + public long get_segments() { + return _segments; + } + + public void set_segments(long _segments) { + this._segments = _segments; + } + + public long get_impressions() { + return _impressions; + } + + public void set_impressions(long _impressions) { + this._impressions = _impressions; + } + + public long get_events() { + return _events; + } + + public void set_events(long _events) { + this._events = _events; + } + + public long get_token() { + return _token; + } + + public void set_token(long _token) { + this._token = _token; + } + + public long get_telemetry() { + return _telemetry; + } + + public void set_telemetry(long _telemetry) { + this._telemetry = _telemetry; + } +} diff --git a/client/src/main/java/io/split/telemetry/domain/MethodExceptions.java b/client/src/main/java/io/split/telemetry/domain/MethodExceptions.java new file mode 100644 index 000000000..c6d6561be --- /dev/null +++ b/client/src/main/java/io/split/telemetry/domain/MethodExceptions.java @@ -0,0 +1,62 @@ +package io.split.telemetry.domain; + +import com.google.gson.annotations.SerializedName; + +public class MethodExceptions { + /* package private */ static final String FIELD_TREATMENT = "t"; + /* package private */ static final String FIELD_TREATMENTS = "ts"; + /* package private */ static final String FIELD_TREATMENT_WITH_CONFIG = "tc"; + /* package private */ static final String FIELD_TREATMENTS_WITH_CONFIG = "tcs"; + /* package private */ static final String FIELD_TRACK = "tr"; + + @SerializedName(FIELD_TREATMENT) + private long _treatment; + @SerializedName(FIELD_TREATMENTS) + private long _treatments; + @SerializedName(FIELD_TREATMENT_WITH_CONFIG) + private long _treatmentWithConfig; + @SerializedName(FIELD_TREATMENTS_WITH_CONFIG) + private long _treatmentsWithConfig; + @SerializedName(FIELD_TRACK) + private long _track; + + public long get_treatment() { + return _treatment; + } + + public void set_treatment(long _treatment) { + this._treatment = _treatment; + } + + public long get_treatments() { + return _treatments; + } + + public void set_treatments(long _treatments) { + this._treatments = _treatments; + } + + public long get_treatmentsWithConfig() { + return _treatmentsWithConfig; + } + + public void set_treatmentsWithConfig(long _treatmentsWithConfig) { + this._treatmentsWithConfig = _treatmentsWithConfig; + } + + public long get_treatmentWithConfig() { + return _treatmentWithConfig; + } + + public void set_treatmentWithConfig(long _treatmentWithConfig) { + this._treatmentWithConfig = _treatmentWithConfig; + } + + public long get_track() { + return _track; + } + + public void set_track(long _track) { + this._track = _track; + } +} diff --git a/client/src/main/java/io/split/telemetry/domain/MethodLatencies.java b/client/src/main/java/io/split/telemetry/domain/MethodLatencies.java new file mode 100644 index 000000000..21aae636c --- /dev/null +++ b/client/src/main/java/io/split/telemetry/domain/MethodLatencies.java @@ -0,0 +1,73 @@ +package io.split.telemetry.domain; + +import com.google.gson.annotations.SerializedName; + +import java.util.ArrayList; +import java.util.List; + +public class MethodLatencies { + /* package private */ static final String FIELD_TREATMENT = "t"; + /* package private */ static final String FIELD_TREATMENTS = "ts"; + /* package private */ static final String FIELD_TREATMENT_WITH_CONFIG = "tc"; + /* package private */ static final String FIELD_TREATMENTS_WITH_CONFIG = "tcs"; + /* package private */ static final String FIELD_TRACK = "tr"; + + @SerializedName(FIELD_TREATMENT) + private List _treatment; + @SerializedName(FIELD_TREATMENTS) + private List _treatments; + @SerializedName(FIELD_TREATMENT_WITH_CONFIG) + private List _treatmentWithConfig; + @SerializedName(FIELD_TREATMENTS_WITH_CONFIG) + private List _treatmentsWithConfig; + @SerializedName(FIELD_TRACK) + private List _track; + + public MethodLatencies() { + _treatment = new ArrayList<>(); + _treatments = new ArrayList<>(); + _treatmentWithConfig = new ArrayList<>(); + _treatmentsWithConfig = new ArrayList<>(); + _track = new ArrayList<>(); + } + + public List get_treatment() { + return _treatment; + } + + public void set_treatment(List _treatment) { + this._treatment = _treatment; + } + + public List get_treatments() { + return _treatments; + } + + public void set_treatments(List _treatments) { + this._treatments = _treatments; + } + + public List get_treatmentsWithConfig() { + return _treatmentsWithConfig; + } + + public void set_treatmentsWithConfig(List _treatmentsWithConfig) { + this._treatmentsWithConfig = _treatmentsWithConfig; + } + + public List get_treatmentWithConfig() { + return _treatmentWithConfig; + } + + public void set_treatmentWithConfig(List _treatmentWithConfig) { + this._treatmentWithConfig = _treatmentWithConfig; + } + + public List get_track() { + return _track; + } + + public void set_track(List _track) { + this._track = _track; + } +} diff --git a/client/src/main/java/io/split/telemetry/domain/StreamingEvent.java b/client/src/main/java/io/split/telemetry/domain/StreamingEvent.java new file mode 100644 index 000000000..eb4e49e2c --- /dev/null +++ b/client/src/main/java/io/split/telemetry/domain/StreamingEvent.java @@ -0,0 +1,40 @@ +package io.split.telemetry.domain; + +import com.google.gson.annotations.SerializedName; + +public class StreamingEvent { + /* package private */ static final String FIELD_TYPE = "sp"; + /* package private */ static final String FIELD_DATA = "se"; + /* package private */ static final String FIELD_TIMESTAMP = "im"; + + @SerializedName(FIELD_TYPE) + private int _type; + @SerializedName(FIELD_DATA) + private long _data; + @SerializedName(FIELD_TIMESTAMP) + private long timestamp; + + public int get_type() { + return _type; + } + + public void set_type(int _type) { + this._type = _type; + } + + public long get_data() { + return _data; + } + + public void set_data(long _data) { + this._data = _data; + } + + public long getTimestamp() { + return timestamp; + } + + public void setTimestamp(long timestamp) { + this.timestamp = timestamp; + } +} diff --git a/client/src/main/java/io/split/telemetry/domain/enums/EventsDataRecordsEnum.java b/client/src/main/java/io/split/telemetry/domain/enums/EventsDataRecordsEnum.java new file mode 100644 index 000000000..8beb3ac48 --- /dev/null +++ b/client/src/main/java/io/split/telemetry/domain/enums/EventsDataRecordsEnum.java @@ -0,0 +1,6 @@ +package io.split.telemetry.domain.enums; + +public enum EventsDataRecordsEnum { + EVENTS_QUEUED, + EVENTS_DROPPED +} diff --git a/client/src/main/java/io/split/telemetry/domain/enums/FactoryCountersEnum.java b/client/src/main/java/io/split/telemetry/domain/enums/FactoryCountersEnum.java new file mode 100644 index 000000000..387084612 --- /dev/null +++ b/client/src/main/java/io/split/telemetry/domain/enums/FactoryCountersEnum.java @@ -0,0 +1,6 @@ +package io.split.telemetry.domain.enums; + +public enum FactoryCountersEnum { + BUR_TIMEOUTS, + NON_READY_USAGES +} diff --git a/client/src/main/java/io/split/telemetry/domain/enums/HTTPLatenciesEnum.java b/client/src/main/java/io/split/telemetry/domain/enums/HTTPLatenciesEnum.java new file mode 100644 index 000000000..88a7a7e84 --- /dev/null +++ b/client/src/main/java/io/split/telemetry/domain/enums/HTTPLatenciesEnum.java @@ -0,0 +1,10 @@ +package io.split.telemetry.domain.enums; + +public enum HTTPLatenciesEnum { + SPLITS, + SEGMENTS, + IMPRESSIONS, + EVENTS, + TELEMETRY, + TOKEN +} diff --git a/client/src/main/java/io/split/telemetry/domain/enums/LastSynchronizationRecordsEnum.java b/client/src/main/java/io/split/telemetry/domain/enums/LastSynchronizationRecordsEnum.java new file mode 100644 index 000000000..f99b43f19 --- /dev/null +++ b/client/src/main/java/io/split/telemetry/domain/enums/LastSynchronizationRecordsEnum.java @@ -0,0 +1,10 @@ +package io.split.telemetry.domain.enums; + +public enum LastSynchronizationRecordsEnum { + SPLITS, + SEGMENTS, + IMPRESSIONS, + EVENTS, + TOKEN, + TELEMETRY +} diff --git a/client/src/main/java/io/split/telemetry/domain/enums/MethodEnum.java b/client/src/main/java/io/split/telemetry/domain/enums/MethodEnum.java new file mode 100644 index 000000000..029d59477 --- /dev/null +++ b/client/src/main/java/io/split/telemetry/domain/enums/MethodEnum.java @@ -0,0 +1,9 @@ +package io.split.telemetry.domain.enums; + +public enum MethodEnum { + TREATMENT, + TREATMENTS, + TREATMENT_WITH_CONFIG, + TREATMENTS_WITH_CONFIG, + TRACK, +} diff --git a/client/src/main/java/io/split/telemetry/domain/enums/PushCountersEnum.java b/client/src/main/java/io/split/telemetry/domain/enums/PushCountersEnum.java new file mode 100644 index 000000000..53f023f74 --- /dev/null +++ b/client/src/main/java/io/split/telemetry/domain/enums/PushCountersEnum.java @@ -0,0 +1,6 @@ +package io.split.telemetry.domain.enums; + +public enum PushCountersEnum { + AUTH_REJECTIONS, + TOKEN_REFRESHES +} diff --git a/client/src/main/java/io/split/telemetry/domain/enums/ResourceEnum.java b/client/src/main/java/io/split/telemetry/domain/enums/ResourceEnum.java new file mode 100644 index 000000000..b705aa43e --- /dev/null +++ b/client/src/main/java/io/split/telemetry/domain/enums/ResourceEnum.java @@ -0,0 +1,10 @@ +package io.split.telemetry.domain.enums; + +public enum ResourceEnum { + SPLIT_SYNC, + SEGMENT_SYNC, + IMPRESSION_SYNC, + EVENT_SYNC, + TELEMETRY_SYNC, + TOKEN_SYNC +} diff --git a/client/src/main/java/io/split/telemetry/domain/enums/SdkRecordsEnum.java b/client/src/main/java/io/split/telemetry/domain/enums/SdkRecordsEnum.java new file mode 100644 index 000000000..6f8780811 --- /dev/null +++ b/client/src/main/java/io/split/telemetry/domain/enums/SdkRecordsEnum.java @@ -0,0 +1,5 @@ +package io.split.telemetry.domain.enums; + +public enum SdkRecordsEnum { + SESSION +} diff --git a/client/src/main/java/io/split/telemetry/storage/InMemoryTelemetryStorage.java b/client/src/main/java/io/split/telemetry/storage/InMemoryTelemetryStorage.java new file mode 100644 index 000000000..91e8a0d1d --- /dev/null +++ b/client/src/main/java/io/split/telemetry/storage/InMemoryTelemetryStorage.java @@ -0,0 +1,306 @@ +package io.split.telemetry.storage; + +import com.google.common.collect.Maps; +import io.split.telemetry.domain.*; +import io.split.telemetry.domain.enums.*; + +import java.util.ArrayList; +import java.util.List; +import java.util.concurrent.ConcurrentMap; +import java.util.concurrent.atomic.AtomicLong; + +public class InMemoryTelemetryStorage implements TelemetryStorage{ + + //Latencies + private final ConcurrentMap> _methodLatencies = Maps.newConcurrentMap(); + private final ConcurrentMap> _httpLatencies = Maps.newConcurrentMap(); + + //Counters + private final ConcurrentMap _exceptionsCounters = Maps.newConcurrentMap(); + private final ConcurrentMap _pushCounters = Maps.newConcurrentMap(); + private final ConcurrentMap _factoryCounters = Maps.newConcurrentMap(); + + //Records + private final ConcurrentMap _impressionsDataRecords = Maps.newConcurrentMap(); + private final ConcurrentMap _eventsDataRecords = Maps.newConcurrentMap(); + private final ConcurrentMap _lastSynchronizationRecords = Maps.newConcurrentMap(); + private final ConcurrentMap _sdkRecords = Maps.newConcurrentMap(); + + //HTTPErrors + private final ConcurrentMap> _httpErrors = Maps.newConcurrentMap(); + + //StreamingEvents + private final Object _streamingEventsLock = new Object(); + private final List _streamingEvents = new ArrayList<>(); + + //Tags + private final Object _tagsLock = new Object(); + private final List _tags = new ArrayList<>(); + + public InMemoryTelemetryStorage() { + initMethodLatencies(); + initHttpLatencies(); + initHttpErrors(); + } + + @Override + public void recordConfigData() { + // No-Op. Config Data will be sent directly to Split Servers. No need to store. + } + + @Override + public long getBURTimeouts() { + long burTimeouts = _factoryCounters.getOrDefault(FactoryCountersEnum.BUR_TIMEOUTS, new AtomicLong()).get(); + return burTimeouts; + } + + @Override + public long getNonReadyUsages() { + long nonReadyUsages = _factoryCounters.getOrDefault(FactoryCountersEnum.NON_READY_USAGES, new AtomicLong()).get(); + return nonReadyUsages; + } + + @Override + public MethodExceptions popExceptions() { + MethodExceptions exceptions = new MethodExceptions(); + exceptions.set_treatment(_exceptionsCounters.getOrDefault(MethodEnum.TREATMENT, new AtomicLong()).get()); + exceptions.set_treatments(_exceptionsCounters.getOrDefault(MethodEnum.TREATMENTS, new AtomicLong()).get()); + exceptions.set_treatmentWithConfig(_exceptionsCounters.getOrDefault(MethodEnum.TREATMENT_WITH_CONFIG, new AtomicLong()).get()); + exceptions.set_treatmentsWithConfig(_exceptionsCounters.getOrDefault(MethodEnum.TREATMENTS_WITH_CONFIG, new AtomicLong()).get()); + exceptions.set_track(_exceptionsCounters.getOrDefault(MethodEnum.TRACK, new AtomicLong()).get()); + + _exceptionsCounters.clear(); + initMethodLatencies(); + + return exceptions; + } + + @Override + public MethodLatencies popLatencies() { + MethodLatencies latencies = new MethodLatencies(); + latencies.set_treatment(_methodLatencies.get(MethodEnum.TREATMENT)); + latencies.set_treatments(_methodLatencies.get(MethodEnum.TREATMENTS)); + latencies.set_treatmentWithConfig(_methodLatencies.get(MethodEnum.TREATMENT_WITH_CONFIG)); + latencies.set_treatmentsWithConfig(_methodLatencies.get(MethodEnum.TREATMENTS_WITH_CONFIG)); + latencies.set_track(_methodLatencies.get(MethodEnum.TRACK)); + + _methodLatencies.clear(); + initMethodLatencies(); + + return latencies; + } + + @Override + public void recordNonReadyUsage() { + _factoryCounters.putIfAbsent(FactoryCountersEnum.NON_READY_USAGES, new AtomicLong(0)); + _factoryCounters.get(FactoryCountersEnum.NON_READY_USAGES).incrementAndGet(); + + } + + @Override + public void recordBURTimeout() { + _factoryCounters.putIfAbsent(FactoryCountersEnum.BUR_TIMEOUTS, new AtomicLong(0)); + _factoryCounters.get(FactoryCountersEnum.BUR_TIMEOUTS).incrementAndGet(); + } + + @Override + public void recordLatency(String method, int latency) { + _methodLatencies.get(method).add(Long.valueOf(latency)); + } + + @Override + public void recordException(MethodEnum method) { + _exceptionsCounters.putIfAbsent(method, new AtomicLong(0)); + _exceptionsCounters.get(method).incrementAndGet(); + } + + @Override + public long getImpressionsStats(ImpressionsDataTypeEnum data) { + return _impressionsDataRecords.getOrDefault(data, new AtomicLong()).get(); + } + + @Override + public long getEventStats(EventsDataRecordsEnum type) { + return _eventsDataRecords.getOrDefault(type, new AtomicLong()).get(); + } + + @Override + public LastSynchronization getLastSynchronization() { + LastSynchronization lastSynchronization = new LastSynchronization(); + lastSynchronization.set_splits(_lastSynchronizationRecords.getOrDefault(LastSynchronizationRecordsEnum.SPLITS, new AtomicLong()).get()); + lastSynchronization.set_segments(_lastSynchronizationRecords.getOrDefault(LastSynchronizationRecordsEnum.SEGMENTS, new AtomicLong()).get()); + lastSynchronization.set_impressions(_lastSynchronizationRecords.getOrDefault(LastSynchronizationRecordsEnum.IMPRESSIONS, new AtomicLong()).get()); + lastSynchronization.set_events(_lastSynchronizationRecords.getOrDefault(LastSynchronizationRecordsEnum.EVENTS, new AtomicLong()).get()); + lastSynchronization.set_telemetry(_lastSynchronizationRecords.getOrDefault(LastSynchronizationRecordsEnum.TELEMETRY, new AtomicLong()).get()); + lastSynchronization.set_token(_lastSynchronizationRecords.getOrDefault(LastSynchronizationRecordsEnum.TOKEN, new AtomicLong()).get()); + + return lastSynchronization; + } + + @Override + public HTTPErrors popHTTPErrors() { + HTTPErrors errors = new HTTPErrors(); + errors.set_splits(_httpErrors.get(ResourceEnum.SPLIT_SYNC)); + errors.set_segments(_httpErrors.get(ResourceEnum.SEGMENT_SYNC)); + errors.set_impressions(_httpErrors.get(ResourceEnum.IMPRESSION_SYNC)); + errors.set_events(_httpErrors.get(ResourceEnum.EVENT_SYNC)); + errors.set_telemetry(_httpErrors.get(ResourceEnum.TELEMETRY_SYNC)); + errors.set_token(_httpErrors.get(ResourceEnum.TOKEN_SYNC)); + + _httpErrors.clear(); + initHttpErrors(); + + return errors; + } + + @Override + public HTTPLatencies popHTTPLatencies() { + HTTPLatencies latencies = new HTTPLatencies(); + latencies.set_splits(_httpLatencies.get(HTTPLatenciesEnum.SPLITS)); + latencies.set_segments(_httpLatencies.get(HTTPLatenciesEnum.SEGMENTS)); + latencies.set_impressions(_httpLatencies.get(HTTPLatenciesEnum.IMPRESSIONS)); + latencies.set_events(_httpLatencies.get(HTTPLatenciesEnum.EVENTS)); + latencies.set_telemetry(_httpLatencies.get(HTTPLatenciesEnum.TELEMETRY)); + latencies.set_token(_httpLatencies.get(HTTPLatenciesEnum.TOKEN)); + + _httpLatencies.clear(); + initHttpLatencies(); + + return latencies; + } + + @Override + public long popAuthRejections() { + long authRejections = _pushCounters.getOrDefault(PushCountersEnum.AUTH_REJECTIONS, new AtomicLong()).get(); + + _pushCounters.replace(PushCountersEnum.AUTH_REJECTIONS, new AtomicLong()); + + return authRejections; + } + + @Override + public long popTokenRefreshes() { + long tokenRefreshes = _pushCounters.getOrDefault(PushCountersEnum.TOKEN_REFRESHES, new AtomicLong()).get(); + + _pushCounters.replace(PushCountersEnum.TOKEN_REFRESHES, new AtomicLong()); + + return tokenRefreshes; + } + + @Override + public List popStreamingEvents() { + synchronized (_streamingEventsLock) { + List streamingEvents = _streamingEvents; + + _streamingEvents.clear(); + + return streamingEvents; + } + } + + @Override + public List popTags() { + synchronized (_tagsLock) { + List tags = _tags; + + _tags.clear(); + + return tags; + } + } + + @Override + public long getSessionLength() { + return _sdkRecords.getOrDefault(SdkRecordsEnum.SESSION, new AtomicLong()).get(); + } + + @Override + public void addTag(String tag) { + synchronized (_tagsLock) { + _tags.add(tag); + } + } + + @Override + public void recordImpressionStats(ImpressionsDataTypeEnum dataType, long count) { + _impressionsDataRecords.putIfAbsent(dataType, new AtomicLong()); + _impressionsDataRecords.get(dataType).incrementAndGet(); + } + + @Override + public void recordEventStats(EventsDataRecordsEnum dataType, long count) { + _eventsDataRecords.putIfAbsent(dataType, new AtomicLong()); + _eventsDataRecords.get(dataType).incrementAndGet(); + } + + @Override + public void recordSuccessfulSync(LastSynchronizationRecordsEnum resource, long time) { + _lastSynchronizationRecords.putIfAbsent(resource, new AtomicLong(time)); + + } + + @Override + public void recordSyncError(ResourceEnum resource, int status) { + ConcurrentMap errors = _httpErrors.get(resource); + errors.putIfAbsent(Long.valueOf(status), 0l); + errors.replace(Long.valueOf(status), errors.get(status) + 1); + } + + @Override + public void recordSyncLatency(String resource, long latency) { + _httpLatencies.get(resource).add(latency); + + } + + @Override + public void recordAuthRejections() { + _pushCounters.putIfAbsent(PushCountersEnum.AUTH_REJECTIONS, new AtomicLong(0)); + _pushCounters.get(PushCountersEnum.AUTH_REJECTIONS).incrementAndGet(); + + } + + @Override + public void recordTokenRefreshes() { + _pushCounters.putIfAbsent(PushCountersEnum.TOKEN_REFRESHES, new AtomicLong(0)); + _pushCounters.get(PushCountersEnum.TOKEN_REFRESHES).incrementAndGet(); + + } + + @Override + public void recordStreamingEvents(StreamingEvent streamingEvent) { + synchronized (_streamingEventsLock) { + _streamingEvents.add(streamingEvent); + } + } + + @Override + public void recordSessionLength(long sessionLength) { + _sdkRecords.putIfAbsent(SdkRecordsEnum.SESSION, new AtomicLong(sessionLength)); + } + + private void initMethodLatencies() { + _methodLatencies.put(MethodEnum.TREATMENT, new ArrayList<>()); + _methodLatencies.put(MethodEnum.TREATMENTS, new ArrayList<>()); + _methodLatencies.put(MethodEnum.TREATMENT_WITH_CONFIG, new ArrayList<>()); + _methodLatencies.put(MethodEnum.TREATMENTS_WITH_CONFIG, new ArrayList<>()); + _methodLatencies.put(MethodEnum.TRACK, new ArrayList<>()); + } + + private void initHttpLatencies() { + _httpLatencies.put(HTTPLatenciesEnum.SPLITS, new ArrayList<>()); + _httpLatencies.put(HTTPLatenciesEnum.SEGMENTS, new ArrayList<>()); + _httpLatencies.put(HTTPLatenciesEnum.IMPRESSIONS, new ArrayList<>()); + _httpLatencies.put(HTTPLatenciesEnum.EVENTS, new ArrayList<>()); + _httpLatencies.put(HTTPLatenciesEnum.TELEMETRY, new ArrayList<>()); + _httpLatencies.put(HTTPLatenciesEnum.TOKEN, new ArrayList<>()); + } + + private void initHttpErrors() { + _httpErrors.put(ResourceEnum.SPLIT_SYNC, Maps.newConcurrentMap()); + _httpErrors.put(ResourceEnum.SEGMENT_SYNC, Maps.newConcurrentMap()); + _httpErrors.put(ResourceEnum.IMPRESSION_SYNC, Maps.newConcurrentMap()); + _httpErrors.put(ResourceEnum.EVENT_SYNC, Maps.newConcurrentMap()); + _httpErrors.put(ResourceEnum.TELEMETRY_SYNC, Maps.newConcurrentMap()); + _httpErrors.put(ResourceEnum.TOKEN_SYNC, Maps.newConcurrentMap()); + } +} diff --git a/client/src/main/java/io/split/telemetry/storage/TelemetryConfigConsumer.java b/client/src/main/java/io/split/telemetry/storage/TelemetryConfigConsumer.java new file mode 100644 index 000000000..680d128b8 --- /dev/null +++ b/client/src/main/java/io/split/telemetry/storage/TelemetryConfigConsumer.java @@ -0,0 +1,6 @@ +package io.split.telemetry.storage; + +public interface TelemetryConfigConsumer { + long getBURTimeouts(); + long getNonReadyUsages(); +} diff --git a/client/src/main/java/io/split/telemetry/storage/TelemetryConfigProducer.java b/client/src/main/java/io/split/telemetry/storage/TelemetryConfigProducer.java new file mode 100644 index 000000000..913733d03 --- /dev/null +++ b/client/src/main/java/io/split/telemetry/storage/TelemetryConfigProducer.java @@ -0,0 +1,7 @@ +package io.split.telemetry.storage; + +public interface TelemetryConfigProducer { + void recordConfigData(); + void recordNonReadyUsage(); + void recordBURTimeout(); +} diff --git a/client/src/main/java/io/split/telemetry/storage/TelemetryEvaluationConsumer.java b/client/src/main/java/io/split/telemetry/storage/TelemetryEvaluationConsumer.java index 3537540ce..976fbb623 100644 --- a/client/src/main/java/io/split/telemetry/storage/TelemetryEvaluationConsumer.java +++ b/client/src/main/java/io/split/telemetry/storage/TelemetryEvaluationConsumer.java @@ -1,4 +1,9 @@ package io.split.telemetry.storage; +import io.split.telemetry.domain.MethodExceptions; +import io.split.telemetry.domain.MethodLatencies; + public interface TelemetryEvaluationConsumer { + MethodExceptions popExceptions(); + MethodLatencies popLatencies(); } diff --git a/client/src/main/java/io/split/telemetry/storage/TelemetryEvaluationProducer.java b/client/src/main/java/io/split/telemetry/storage/TelemetryEvaluationProducer.java index 9c4e59d4b..51795b054 100644 --- a/client/src/main/java/io/split/telemetry/storage/TelemetryEvaluationProducer.java +++ b/client/src/main/java/io/split/telemetry/storage/TelemetryEvaluationProducer.java @@ -1,4 +1,8 @@ package io.split.telemetry.storage; +import io.split.telemetry.domain.enums.MethodEnum; + public interface TelemetryEvaluationProducer { + void recordLatency(String method, int latency); + void recordException(MethodEnum method); } diff --git a/client/src/main/java/io/split/telemetry/storage/TelemetryInitConsumer.java b/client/src/main/java/io/split/telemetry/storage/TelemetryInitConsumer.java deleted file mode 100644 index 2430ff765..000000000 --- a/client/src/main/java/io/split/telemetry/storage/TelemetryInitConsumer.java +++ /dev/null @@ -1,4 +0,0 @@ -package io.split.telemetry.storage; - -public interface TelemetryInitConsumer { -} diff --git a/client/src/main/java/io/split/telemetry/storage/TelemetryInitProducer.java b/client/src/main/java/io/split/telemetry/storage/TelemetryInitProducer.java deleted file mode 100644 index 9547967d4..000000000 --- a/client/src/main/java/io/split/telemetry/storage/TelemetryInitProducer.java +++ /dev/null @@ -1,4 +0,0 @@ -package io.split.telemetry.storage; - -public interface TelemetryInitProducer { -} diff --git a/client/src/main/java/io/split/telemetry/storage/TelemetryRuntimeConsumer.java b/client/src/main/java/io/split/telemetry/storage/TelemetryRuntimeConsumer.java index 661b75cdb..9df890be3 100644 --- a/client/src/main/java/io/split/telemetry/storage/TelemetryRuntimeConsumer.java +++ b/client/src/main/java/io/split/telemetry/storage/TelemetryRuntimeConsumer.java @@ -1,4 +1,24 @@ package io.split.telemetry.storage; +import io.split.telemetry.domain.HTTPErrors; +import io.split.telemetry.domain.HTTPLatencies; +import io.split.telemetry.domain.LastSynchronization; +import io.split.telemetry.domain.StreamingEvent; +import io.split.telemetry.domain.enums.EventsDataRecordsEnum; +import io.split.telemetry.domain.enums.ImpressionsDataTypeEnum; + +import java.util.List; + public interface TelemetryRuntimeConsumer { + long getImpressionsStats(ImpressionsDataTypeEnum data); + long getEventStats(EventsDataRecordsEnum type); + LastSynchronization getLastSynchronization(); + HTTPErrors popHTTPErrors(); + HTTPLatencies popHTTPLatencies(); + long popAuthRejections(); + long popTokenRefreshes(); + List popStreamingEvents(); + List popTags(); + long getSessionLength(); + } diff --git a/client/src/main/java/io/split/telemetry/storage/TelemetryRuntimeProducer.java b/client/src/main/java/io/split/telemetry/storage/TelemetryRuntimeProducer.java index 8d6e3672f..ab6fe175f 100644 --- a/client/src/main/java/io/split/telemetry/storage/TelemetryRuntimeProducer.java +++ b/client/src/main/java/io/split/telemetry/storage/TelemetryRuntimeProducer.java @@ -1,4 +1,20 @@ package io.split.telemetry.storage; +import io.split.telemetry.domain.StreamingEvent; +import io.split.telemetry.domain.enums.EventsDataRecordsEnum; +import io.split.telemetry.domain.enums.ImpressionsDataTypeEnum; +import io.split.telemetry.domain.enums.LastSynchronizationRecordsEnum; +import io.split.telemetry.domain.enums.ResourceEnum; + public interface TelemetryRuntimeProducer { + void addTag(String tag); + void recordImpressionStats(ImpressionsDataTypeEnum dataType, long count); + void recordEventStats(EventsDataRecordsEnum dataType, long count); + void recordSuccessfulSync(LastSynchronizationRecordsEnum resource, long time); + void recordSyncError(ResourceEnum resource, int status); + void recordSyncLatency(String resource, long latency); + void recordAuthRejections(); + void recordTokenRefreshes(); + void recordStreamingEvents(StreamingEvent streamingEvent); + void recordSessionLength(long sessionLength); } diff --git a/client/src/main/java/io/split/telemetry/storage/TelemetryStorageConsumer.java b/client/src/main/java/io/split/telemetry/storage/TelemetryStorageConsumer.java index 217a6e27b..7efd537c5 100644 --- a/client/src/main/java/io/split/telemetry/storage/TelemetryStorageConsumer.java +++ b/client/src/main/java/io/split/telemetry/storage/TelemetryStorageConsumer.java @@ -1,4 +1,4 @@ package io.split.telemetry.storage; -public interface TelemetryStorageConsumer extends TelemetryInitConsumer, TelemetryRuntimeConsumer, TelemetryEvaluationConsumer{ +public interface TelemetryStorageConsumer extends TelemetryConfigConsumer, TelemetryRuntimeConsumer, TelemetryEvaluationConsumer{ } diff --git a/client/src/main/java/io/split/telemetry/storage/TelemetryStorageProducer.java b/client/src/main/java/io/split/telemetry/storage/TelemetryStorageProducer.java index 382f5939f..c3ef23031 100644 --- a/client/src/main/java/io/split/telemetry/storage/TelemetryStorageProducer.java +++ b/client/src/main/java/io/split/telemetry/storage/TelemetryStorageProducer.java @@ -1,4 +1,4 @@ package io.split.telemetry.storage; -public interface TelemetryStorageProducer extends TelemetryEvaluationProducer, TelemetryInitProducer, TelemetryRuntimeProducer{ +public interface TelemetryStorageProducer extends TelemetryEvaluationProducer, TelemetryConfigProducer, TelemetryRuntimeProducer{ } From 086150cbf611e9611a5b009d73adb4407708c1eb Mon Sep 17 00:00:00 2001 From: Lucas Echeverz Date: Fri, 9 Apr 2021 12:46:21 -0300 Subject: [PATCH 04/81] Changing name --- .../telemetry/domain/enums/ImpressionsDataTypeEnum.java | 7 +++++++ 1 file changed, 7 insertions(+) create mode 100644 client/src/main/java/io/split/telemetry/domain/enums/ImpressionsDataTypeEnum.java diff --git a/client/src/main/java/io/split/telemetry/domain/enums/ImpressionsDataTypeEnum.java b/client/src/main/java/io/split/telemetry/domain/enums/ImpressionsDataTypeEnum.java new file mode 100644 index 000000000..3ca134c93 --- /dev/null +++ b/client/src/main/java/io/split/telemetry/domain/enums/ImpressionsDataTypeEnum.java @@ -0,0 +1,7 @@ +package io.split.telemetry.domain.enums; + +public enum ImpressionsDataTypeEnum { + IMPRESSIONS_QUEUED, + IMPRESSIONS_DROPPED, + IMPRESSIONS_DEDUPED +} From 31a82081a5be3edddc43c8afa9b2fc8b2f1f9e87 Mon Sep 17 00:00:00 2001 From: Lucas Echeverz Date: Fri, 9 Apr 2021 17:41:41 -0300 Subject: [PATCH 05/81] Fixing PR comments --- .../io/split/telemetry/domain/HTTPErrors.java | 12 ++ .../split/telemetry/domain/HTTPLatencies.java | 13 +- .../telemetry/domain/LastSynchronization.java | 11 ++ .../telemetry/domain/StreamingEvent.java | 12 +- .../domain/enums/HTTPLatenciesEnum.java | 1 + .../enums/LastSynchronizationRecordsEnum.java | 1 + .../telemetry/domain/enums/ResourceEnum.java | 1 + .../storage/InMemoryTelemetryStorage.java | 176 +++++++++++------- .../storage/TelemetryConfigProducer.java | 1 - .../storage/TelemetryEvaluationConsumer.java | 2 +- .../storage/TelemetryRuntimeConsumer.java | 2 +- .../telemetry/utils/BucketCalculator.java | 64 +++++++ .../storage/InMemoryTelemetryStorageTest.java | 21 +++ 13 files changed, 243 insertions(+), 74 deletions(-) create mode 100644 client/src/main/java/io/split/telemetry/utils/BucketCalculator.java create mode 100644 client/src/test/java/io/split/telemetry/storage/InMemoryTelemetryStorageTest.java diff --git a/client/src/main/java/io/split/telemetry/domain/HTTPErrors.java b/client/src/main/java/io/split/telemetry/domain/HTTPErrors.java index 4b1051c56..dac746117 100644 --- a/client/src/main/java/io/split/telemetry/domain/HTTPErrors.java +++ b/client/src/main/java/io/split/telemetry/domain/HTTPErrors.java @@ -9,6 +9,7 @@ public class HTTPErrors { /* package private */ static final String FIELD_SPLIT = "sp"; /* package private */ static final String FIELD_SEGMENTS = "se"; /* package private */ static final String FIELD_IMPRESSIONS = "im"; + /* package private */ static final String FIELD_IMPRESSIONS_COUNT = "ic"; /* package private */ static final String FIELD_EVENTS = "ev"; /* package private */ static final String FIELD_TOKEN = "to"; /* package private */ static final String FIELD_TELEMETRY = "te"; @@ -19,6 +20,8 @@ public class HTTPErrors { private Map _segments; @SerializedName(FIELD_IMPRESSIONS) private Map _impressions; + @SerializedName(FIELD_IMPRESSIONS_COUNT) + private Map _impressionsCount; @SerializedName(FIELD_EVENTS) private Map _events; @SerializedName(FIELD_TOKEN) @@ -30,6 +33,7 @@ public HTTPErrors() { _splits = new ConcurrentHashMap<>(); _segments = new ConcurrentHashMap<>(); _impressions = new ConcurrentHashMap<>(); + _impressionsCount = new ConcurrentHashMap<>(); _events = new ConcurrentHashMap<>(); _token = new ConcurrentHashMap<>(); _telemetry = new ConcurrentHashMap<>(); @@ -82,4 +86,12 @@ public Map get_telemetry() { public void set_telemetry(Map _telemetry) { this._telemetry = _telemetry; } + + public Map get_impressionsCount() { + return _impressionsCount; + } + + public void set_impressionsCount(Map _impressionsCount) { + this._impressionsCount = _impressionsCount; + } } diff --git a/client/src/main/java/io/split/telemetry/domain/HTTPLatencies.java b/client/src/main/java/io/split/telemetry/domain/HTTPLatencies.java index bf32af484..0e0791ed9 100644 --- a/client/src/main/java/io/split/telemetry/domain/HTTPLatencies.java +++ b/client/src/main/java/io/split/telemetry/domain/HTTPLatencies.java @@ -4,12 +4,12 @@ import java.util.ArrayList; import java.util.List; -import java.util.concurrent.ConcurrentHashMap; public class HTTPLatencies { /* package private */ static final String FIELD_SPLIT = "sp"; /* package private */ static final String FIELD_SEGMENTS = "se"; /* package private */ static final String FIELD_IMPRESSIONS = "im"; + /* package private */ static final String FIELD_IMPRESSIONS_COUNT = "ic"; /* package private */ static final String FIELD_EVENTS = "ev"; /* package private */ static final String FIELD_TOKEN = "to"; /* package private */ static final String FIELD_TELEMETRY = "te"; @@ -20,6 +20,8 @@ public class HTTPLatencies { private List_segments; @SerializedName(FIELD_IMPRESSIONS) private List _impressions; + @SerializedName(FIELD_IMPRESSIONS_COUNT) + private List _impressionsCount; @SerializedName(FIELD_EVENTS) private List _events; @SerializedName(FIELD_TOKEN) @@ -31,6 +33,7 @@ public HTTPLatencies() { _splits = new ArrayList<>(); _segments = new ArrayList<>(); _impressions = new ArrayList<>(); + _impressionsCount = new ArrayList<>(); _events = new ArrayList<>(); _token = new ArrayList<>(); _telemetry = new ArrayList<>(); @@ -83,4 +86,12 @@ public List get_telemetry() { public void set_telemetry(List _telemetry) { this._telemetry = _telemetry; } + + public List get_impressionsCount() { + return _impressionsCount; + } + + public void set_impressionsCount(List _impressionsCount) { + this._impressionsCount = _impressionsCount; + } } diff --git a/client/src/main/java/io/split/telemetry/domain/LastSynchronization.java b/client/src/main/java/io/split/telemetry/domain/LastSynchronization.java index 0b77c8a06..59586562e 100644 --- a/client/src/main/java/io/split/telemetry/domain/LastSynchronization.java +++ b/client/src/main/java/io/split/telemetry/domain/LastSynchronization.java @@ -6,6 +6,7 @@ public class LastSynchronization { /* package private */ static final String FIELD_SPLIT = "sp"; /* package private */ static final String FIELD_SEGMENTS = "se"; /* package private */ static final String FIELD_IMPRESSIONS = "im"; + /* package private */ static final String FIELD_IMPRESSIONS_COUNT = "ic"; /* package private */ static final String FIELD_EVENTS = "ev"; /* package private */ static final String FIELD_TOKEN = "to"; /* package private */ static final String FIELD_TELEMETRY = "te"; @@ -16,6 +17,8 @@ public class LastSynchronization { private long _segments; @SerializedName(FIELD_IMPRESSIONS) private long _impressions; + @SerializedName(FIELD_IMPRESSIONS_COUNT) + private long _impressionsCount; @SerializedName(FIELD_EVENTS) private long _events; @SerializedName(FIELD_TOKEN) @@ -70,4 +73,12 @@ public long get_telemetry() { public void set_telemetry(long _telemetry) { this._telemetry = _telemetry; } + + public long get_impressionsCount() { + return _impressionsCount; + } + + public void set_impressionsCount(long _impressionsCount) { + this._impressionsCount = _impressionsCount; + } } diff --git a/client/src/main/java/io/split/telemetry/domain/StreamingEvent.java b/client/src/main/java/io/split/telemetry/domain/StreamingEvent.java index eb4e49e2c..e0ebca3ac 100644 --- a/client/src/main/java/io/split/telemetry/domain/StreamingEvent.java +++ b/client/src/main/java/io/split/telemetry/domain/StreamingEvent.java @@ -3,16 +3,16 @@ import com.google.gson.annotations.SerializedName; public class StreamingEvent { - /* package private */ static final String FIELD_TYPE = "sp"; - /* package private */ static final String FIELD_DATA = "se"; - /* package private */ static final String FIELD_TIMESTAMP = "im"; + /* package private */ static final String FIELD_TYPE = "e"; + /* package private */ static final String FIELD_DATA = "d"; + /* package private */ static final String FIELD_TIMESTAMP = "t"; @SerializedName(FIELD_TYPE) private int _type; @SerializedName(FIELD_DATA) private long _data; @SerializedName(FIELD_TIMESTAMP) - private long timestamp; + private long _timestamp; public int get_type() { return _type; @@ -31,10 +31,10 @@ public void set_data(long _data) { } public long getTimestamp() { - return timestamp; + return _timestamp; } public void setTimestamp(long timestamp) { - this.timestamp = timestamp; + this._timestamp = timestamp; } } diff --git a/client/src/main/java/io/split/telemetry/domain/enums/HTTPLatenciesEnum.java b/client/src/main/java/io/split/telemetry/domain/enums/HTTPLatenciesEnum.java index 88a7a7e84..6f858397b 100644 --- a/client/src/main/java/io/split/telemetry/domain/enums/HTTPLatenciesEnum.java +++ b/client/src/main/java/io/split/telemetry/domain/enums/HTTPLatenciesEnum.java @@ -4,6 +4,7 @@ public enum HTTPLatenciesEnum { SPLITS, SEGMENTS, IMPRESSIONS, + IMPRESSIONS_COUNT, EVENTS, TELEMETRY, TOKEN diff --git a/client/src/main/java/io/split/telemetry/domain/enums/LastSynchronizationRecordsEnum.java b/client/src/main/java/io/split/telemetry/domain/enums/LastSynchronizationRecordsEnum.java index f99b43f19..d2439395b 100644 --- a/client/src/main/java/io/split/telemetry/domain/enums/LastSynchronizationRecordsEnum.java +++ b/client/src/main/java/io/split/telemetry/domain/enums/LastSynchronizationRecordsEnum.java @@ -4,6 +4,7 @@ public enum LastSynchronizationRecordsEnum { SPLITS, SEGMENTS, IMPRESSIONS, + IMPRESSIONS_COUNT, EVENTS, TOKEN, TELEMETRY diff --git a/client/src/main/java/io/split/telemetry/domain/enums/ResourceEnum.java b/client/src/main/java/io/split/telemetry/domain/enums/ResourceEnum.java index b705aa43e..04af11162 100644 --- a/client/src/main/java/io/split/telemetry/domain/enums/ResourceEnum.java +++ b/client/src/main/java/io/split/telemetry/domain/enums/ResourceEnum.java @@ -4,6 +4,7 @@ public enum ResourceEnum { SPLIT_SYNC, SEGMENT_SYNC, IMPRESSION_SYNC, + IMPRESSION_COUNT_SYNC, EVENT_SYNC, TELEMETRY_SYNC, TOKEN_SYNC diff --git a/client/src/main/java/io/split/telemetry/storage/InMemoryTelemetryStorage.java b/client/src/main/java/io/split/telemetry/storage/InMemoryTelemetryStorage.java index 91e8a0d1d..967790704 100644 --- a/client/src/main/java/io/split/telemetry/storage/InMemoryTelemetryStorage.java +++ b/client/src/main/java/io/split/telemetry/storage/InMemoryTelemetryStorage.java @@ -1,6 +1,7 @@ package io.split.telemetry.storage; import com.google.common.collect.Maps; +import io.split.telemetry.utils.BucketCalculator; import io.split.telemetry.domain.*; import io.split.telemetry.domain.enums.*; @@ -10,10 +11,11 @@ import java.util.concurrent.atomic.AtomicLong; public class InMemoryTelemetryStorage implements TelemetryStorage{ + public static final int MAX_LATENCY_BUCKET_COUNT = 23; //Latencies - private final ConcurrentMap> _methodLatencies = Maps.newConcurrentMap(); - private final ConcurrentMap> _httpLatencies = Maps.newConcurrentMap(); + private final ConcurrentMap _methodLatencies = Maps.newConcurrentMap(); + private final ConcurrentMap _httpLatencies = Maps.newConcurrentMap(); //Counters private final ConcurrentMap _exceptionsCounters = Maps.newConcurrentMap(); @@ -37,37 +39,39 @@ public class InMemoryTelemetryStorage implements TelemetryStorage{ private final Object _tagsLock = new Object(); private final List _tags = new ArrayList<>(); - public InMemoryTelemetryStorage() { + public InMemoryTelemetryStorage() throws Exception { initMethodLatencies(); initHttpLatencies(); initHttpErrors(); - } - - @Override - public void recordConfigData() { - // No-Op. Config Data will be sent directly to Split Servers. No need to store. + initMethodCounters(); + initFactoryCounters(); + initImpressionDataCounters(); + initPushCounters(); + initSdkRecords(); + initLastSynchronizationRecords(); + initEventDataRecords(); } @Override public long getBURTimeouts() { - long burTimeouts = _factoryCounters.getOrDefault(FactoryCountersEnum.BUR_TIMEOUTS, new AtomicLong()).get(); + long burTimeouts = _factoryCounters.get(FactoryCountersEnum.BUR_TIMEOUTS).get(); return burTimeouts; } @Override public long getNonReadyUsages() { - long nonReadyUsages = _factoryCounters.getOrDefault(FactoryCountersEnum.NON_READY_USAGES, new AtomicLong()).get(); + long nonReadyUsages = _factoryCounters.get(FactoryCountersEnum.NON_READY_USAGES).get(); return nonReadyUsages; } @Override - public MethodExceptions popExceptions() { + public MethodExceptions popExceptions() throws Exception { MethodExceptions exceptions = new MethodExceptions(); - exceptions.set_treatment(_exceptionsCounters.getOrDefault(MethodEnum.TREATMENT, new AtomicLong()).get()); - exceptions.set_treatments(_exceptionsCounters.getOrDefault(MethodEnum.TREATMENTS, new AtomicLong()).get()); - exceptions.set_treatmentWithConfig(_exceptionsCounters.getOrDefault(MethodEnum.TREATMENT_WITH_CONFIG, new AtomicLong()).get()); - exceptions.set_treatmentsWithConfig(_exceptionsCounters.getOrDefault(MethodEnum.TREATMENTS_WITH_CONFIG, new AtomicLong()).get()); - exceptions.set_track(_exceptionsCounters.getOrDefault(MethodEnum.TRACK, new AtomicLong()).get()); + exceptions.set_treatment(_exceptionsCounters.get(MethodEnum.TREATMENT).get()); + exceptions.set_treatments(_exceptionsCounters.get(MethodEnum.TREATMENTS).get()); + exceptions.set_treatmentWithConfig(_exceptionsCounters.get(MethodEnum.TREATMENT_WITH_CONFIG).get()); + exceptions.set_treatmentsWithConfig(_exceptionsCounters.get(MethodEnum.TREATMENTS_WITH_CONFIG).get()); + exceptions.set_track(_exceptionsCounters.get(MethodEnum.TRACK).get()); _exceptionsCounters.clear(); initMethodLatencies(); @@ -78,61 +82,60 @@ public MethodExceptions popExceptions() { @Override public MethodLatencies popLatencies() { MethodLatencies latencies = new MethodLatencies(); - latencies.set_treatment(_methodLatencies.get(MethodEnum.TREATMENT)); - latencies.set_treatments(_methodLatencies.get(MethodEnum.TREATMENTS)); - latencies.set_treatmentWithConfig(_methodLatencies.get(MethodEnum.TREATMENT_WITH_CONFIG)); - latencies.set_treatmentsWithConfig(_methodLatencies.get(MethodEnum.TREATMENTS_WITH_CONFIG)); - latencies.set_track(_methodLatencies.get(MethodEnum.TRACK)); + latencies.set_treatment(_methodLatencies.get(MethodEnum.TREATMENT).fetchAndClearAll()); + latencies.set_treatments(_methodLatencies.get(MethodEnum.TREATMENTS).fetchAndClearAll()); + latencies.set_treatmentWithConfig(_methodLatencies.get(MethodEnum.TREATMENT_WITH_CONFIG).fetchAndClearAll()); + latencies.set_treatmentsWithConfig(_methodLatencies.get(MethodEnum.TREATMENTS_WITH_CONFIG).fetchAndClearAll()); + latencies.set_track(_methodLatencies.get(MethodEnum.TRACK).fetchAndClearAll()); _methodLatencies.clear(); - initMethodLatencies(); + initMethodCounters(); return latencies; } @Override public void recordNonReadyUsage() { - _factoryCounters.putIfAbsent(FactoryCountersEnum.NON_READY_USAGES, new AtomicLong(0)); _factoryCounters.get(FactoryCountersEnum.NON_READY_USAGES).incrementAndGet(); } @Override public void recordBURTimeout() { - _factoryCounters.putIfAbsent(FactoryCountersEnum.BUR_TIMEOUTS, new AtomicLong(0)); _factoryCounters.get(FactoryCountersEnum.BUR_TIMEOUTS).incrementAndGet(); } @Override public void recordLatency(String method, int latency) { - _methodLatencies.get(method).add(Long.valueOf(latency)); + int bucket = BucketCalculator.getBucketForLatencyMillis(latency); + _methodLatencies.get(method).increment(bucket); } @Override public void recordException(MethodEnum method) { - _exceptionsCounters.putIfAbsent(method, new AtomicLong(0)); _exceptionsCounters.get(method).incrementAndGet(); } @Override public long getImpressionsStats(ImpressionsDataTypeEnum data) { - return _impressionsDataRecords.getOrDefault(data, new AtomicLong()).get(); + return _impressionsDataRecords.get(data).get(); } @Override public long getEventStats(EventsDataRecordsEnum type) { - return _eventsDataRecords.getOrDefault(type, new AtomicLong()).get(); + return _eventsDataRecords.get(type).get(); } @Override public LastSynchronization getLastSynchronization() { LastSynchronization lastSynchronization = new LastSynchronization(); - lastSynchronization.set_splits(_lastSynchronizationRecords.getOrDefault(LastSynchronizationRecordsEnum.SPLITS, new AtomicLong()).get()); - lastSynchronization.set_segments(_lastSynchronizationRecords.getOrDefault(LastSynchronizationRecordsEnum.SEGMENTS, new AtomicLong()).get()); - lastSynchronization.set_impressions(_lastSynchronizationRecords.getOrDefault(LastSynchronizationRecordsEnum.IMPRESSIONS, new AtomicLong()).get()); - lastSynchronization.set_events(_lastSynchronizationRecords.getOrDefault(LastSynchronizationRecordsEnum.EVENTS, new AtomicLong()).get()); - lastSynchronization.set_telemetry(_lastSynchronizationRecords.getOrDefault(LastSynchronizationRecordsEnum.TELEMETRY, new AtomicLong()).get()); - lastSynchronization.set_token(_lastSynchronizationRecords.getOrDefault(LastSynchronizationRecordsEnum.TOKEN, new AtomicLong()).get()); + lastSynchronization.set_splits(_lastSynchronizationRecords.get(LastSynchronizationRecordsEnum.SPLITS).get()); + lastSynchronization.set_segments(_lastSynchronizationRecords.get(LastSynchronizationRecordsEnum.SEGMENTS).get()); + lastSynchronization.set_impressions(_lastSynchronizationRecords.get(LastSynchronizationRecordsEnum.IMPRESSIONS).get()); + lastSynchronization.set_impressionsCount(_lastSynchronizationRecords.get(LastSynchronizationRecordsEnum.IMPRESSIONS_COUNT).get()); + lastSynchronization.set_events(_lastSynchronizationRecords.get(LastSynchronizationRecordsEnum.EVENTS).get()); + lastSynchronization.set_telemetry(_lastSynchronizationRecords.get(LastSynchronizationRecordsEnum.TELEMETRY).get()); + lastSynchronization.set_token(_lastSynchronizationRecords.get(LastSynchronizationRecordsEnum.TOKEN).get()); return lastSynchronization; } @@ -143,6 +146,7 @@ public HTTPErrors popHTTPErrors() { errors.set_splits(_httpErrors.get(ResourceEnum.SPLIT_SYNC)); errors.set_segments(_httpErrors.get(ResourceEnum.SEGMENT_SYNC)); errors.set_impressions(_httpErrors.get(ResourceEnum.IMPRESSION_SYNC)); + errors.set_impressionsCount(_httpErrors.get(ResourceEnum.IMPRESSION_COUNT_SYNC)); errors.set_events(_httpErrors.get(ResourceEnum.EVENT_SYNC)); errors.set_telemetry(_httpErrors.get(ResourceEnum.TELEMETRY_SYNC)); errors.set_token(_httpErrors.get(ResourceEnum.TOKEN_SYNC)); @@ -154,14 +158,15 @@ public HTTPErrors popHTTPErrors() { } @Override - public HTTPLatencies popHTTPLatencies() { + public HTTPLatencies popHTTPLatencies() throws Exception { HTTPLatencies latencies = new HTTPLatencies(); - latencies.set_splits(_httpLatencies.get(HTTPLatenciesEnum.SPLITS)); - latencies.set_segments(_httpLatencies.get(HTTPLatenciesEnum.SEGMENTS)); - latencies.set_impressions(_httpLatencies.get(HTTPLatenciesEnum.IMPRESSIONS)); - latencies.set_events(_httpLatencies.get(HTTPLatenciesEnum.EVENTS)); - latencies.set_telemetry(_httpLatencies.get(HTTPLatenciesEnum.TELEMETRY)); - latencies.set_token(_httpLatencies.get(HTTPLatenciesEnum.TOKEN)); + latencies.set_splits(_httpLatencies.get(HTTPLatenciesEnum.SPLITS).fetchAndClearAll()); + latencies.set_segments(_httpLatencies.get(HTTPLatenciesEnum.SEGMENTS).fetchAndClearAll()); + latencies.set_impressions(_httpLatencies.get(HTTPLatenciesEnum.IMPRESSIONS).fetchAndClearAll()); + latencies.set_impressionsCount(_httpLatencies.get(HTTPLatenciesEnum.IMPRESSIONS_COUNT).fetchAndClearAll()); + latencies.set_events(_httpLatencies.get(HTTPLatenciesEnum.EVENTS).fetchAndClearAll()); + latencies.set_telemetry(_httpLatencies.get(HTTPLatenciesEnum.TELEMETRY).fetchAndClearAll()); + latencies.set_token(_httpLatencies.get(HTTPLatenciesEnum.TOKEN).fetchAndClearAll()); _httpLatencies.clear(); initHttpLatencies(); @@ -171,7 +176,7 @@ public HTTPLatencies popHTTPLatencies() { @Override public long popAuthRejections() { - long authRejections = _pushCounters.getOrDefault(PushCountersEnum.AUTH_REJECTIONS, new AtomicLong()).get(); + long authRejections = _pushCounters.get(PushCountersEnum.AUTH_REJECTIONS).get(); _pushCounters.replace(PushCountersEnum.AUTH_REJECTIONS, new AtomicLong()); @@ -180,7 +185,7 @@ public long popAuthRejections() { @Override public long popTokenRefreshes() { - long tokenRefreshes = _pushCounters.getOrDefault(PushCountersEnum.TOKEN_REFRESHES, new AtomicLong()).get(); + long tokenRefreshes = _pushCounters.get(PushCountersEnum.TOKEN_REFRESHES).get(); _pushCounters.replace(PushCountersEnum.TOKEN_REFRESHES, new AtomicLong()); @@ -211,7 +216,7 @@ public List popTags() { @Override public long getSessionLength() { - return _sdkRecords.getOrDefault(SdkRecordsEnum.SESSION, new AtomicLong()).get(); + return _sdkRecords.get(SdkRecordsEnum.SESSION).get(); } @Override @@ -223,19 +228,17 @@ public void addTag(String tag) { @Override public void recordImpressionStats(ImpressionsDataTypeEnum dataType, long count) { - _impressionsDataRecords.putIfAbsent(dataType, new AtomicLong()); _impressionsDataRecords.get(dataType).incrementAndGet(); } @Override public void recordEventStats(EventsDataRecordsEnum dataType, long count) { - _eventsDataRecords.putIfAbsent(dataType, new AtomicLong()); _eventsDataRecords.get(dataType).incrementAndGet(); } @Override public void recordSuccessfulSync(LastSynchronizationRecordsEnum resource, long time) { - _lastSynchronizationRecords.putIfAbsent(resource, new AtomicLong(time)); + _lastSynchronizationRecords.replace(resource, new AtomicLong(time)); } @@ -248,20 +251,19 @@ public void recordSyncError(ResourceEnum resource, int status) { @Override public void recordSyncLatency(String resource, long latency) { - _httpLatencies.get(resource).add(latency); + int bucket = BucketCalculator.getBucketForLatencyMillis(latency); + _httpLatencies.get(resource).increment(bucket); } @Override public void recordAuthRejections() { - _pushCounters.putIfAbsent(PushCountersEnum.AUTH_REJECTIONS, new AtomicLong(0)); _pushCounters.get(PushCountersEnum.AUTH_REJECTIONS).incrementAndGet(); } @Override public void recordTokenRefreshes() { - _pushCounters.putIfAbsent(PushCountersEnum.TOKEN_REFRESHES, new AtomicLong(0)); _pushCounters.get(PushCountersEnum.TOKEN_REFRESHES).incrementAndGet(); } @@ -275,32 +277,78 @@ public void recordStreamingEvents(StreamingEvent streamingEvent) { @Override public void recordSessionLength(long sessionLength) { - _sdkRecords.putIfAbsent(SdkRecordsEnum.SESSION, new AtomicLong(sessionLength)); + _sdkRecords.replace(SdkRecordsEnum.SESSION, new AtomicLong(sessionLength)); } - private void initMethodLatencies() { - _methodLatencies.put(MethodEnum.TREATMENT, new ArrayList<>()); - _methodLatencies.put(MethodEnum.TREATMENTS, new ArrayList<>()); - _methodLatencies.put(MethodEnum.TREATMENT_WITH_CONFIG, new ArrayList<>()); - _methodLatencies.put(MethodEnum.TREATMENTS_WITH_CONFIG, new ArrayList<>()); - _methodLatencies.put(MethodEnum.TRACK, new ArrayList<>()); + private void initMethodLatencies() throws Exception { + _methodLatencies.put(MethodEnum.TREATMENT, new AtomicLongArray(MAX_LATENCY_BUCKET_COUNT)); + _methodLatencies.put(MethodEnum.TREATMENTS, new AtomicLongArray(MAX_LATENCY_BUCKET_COUNT)); + _methodLatencies.put(MethodEnum.TREATMENT_WITH_CONFIG, new AtomicLongArray(MAX_LATENCY_BUCKET_COUNT)); + _methodLatencies.put(MethodEnum.TREATMENTS_WITH_CONFIG, new AtomicLongArray(MAX_LATENCY_BUCKET_COUNT)); + _methodLatencies.put(MethodEnum.TRACK, new AtomicLongArray(MAX_LATENCY_BUCKET_COUNT)); } - private void initHttpLatencies() { - _httpLatencies.put(HTTPLatenciesEnum.SPLITS, new ArrayList<>()); - _httpLatencies.put(HTTPLatenciesEnum.SEGMENTS, new ArrayList<>()); - _httpLatencies.put(HTTPLatenciesEnum.IMPRESSIONS, new ArrayList<>()); - _httpLatencies.put(HTTPLatenciesEnum.EVENTS, new ArrayList<>()); - _httpLatencies.put(HTTPLatenciesEnum.TELEMETRY, new ArrayList<>()); - _httpLatencies.put(HTTPLatenciesEnum.TOKEN, new ArrayList<>()); + private void initHttpLatencies() throws Exception { + _httpLatencies.put(HTTPLatenciesEnum.SPLITS, new AtomicLongArray(MAX_LATENCY_BUCKET_COUNT)); + _httpLatencies.put(HTTPLatenciesEnum.SEGMENTS, new AtomicLongArray(MAX_LATENCY_BUCKET_COUNT)); + _httpLatencies.put(HTTPLatenciesEnum.IMPRESSIONS, new AtomicLongArray(MAX_LATENCY_BUCKET_COUNT)); + _httpLatencies.put(HTTPLatenciesEnum.IMPRESSIONS_COUNT, new AtomicLongArray(MAX_LATENCY_BUCKET_COUNT)); + _httpLatencies.put(HTTPLatenciesEnum.EVENTS, new AtomicLongArray(MAX_LATENCY_BUCKET_COUNT)); + _httpLatencies.put(HTTPLatenciesEnum.TELEMETRY, new AtomicLongArray(MAX_LATENCY_BUCKET_COUNT)); + _httpLatencies.put(HTTPLatenciesEnum.TOKEN, new AtomicLongArray(MAX_LATENCY_BUCKET_COUNT)); } private void initHttpErrors() { _httpErrors.put(ResourceEnum.SPLIT_SYNC, Maps.newConcurrentMap()); _httpErrors.put(ResourceEnum.SEGMENT_SYNC, Maps.newConcurrentMap()); _httpErrors.put(ResourceEnum.IMPRESSION_SYNC, Maps.newConcurrentMap()); + _httpErrors.put(ResourceEnum.IMPRESSION_COUNT_SYNC, Maps.newConcurrentMap()); _httpErrors.put(ResourceEnum.EVENT_SYNC, Maps.newConcurrentMap()); _httpErrors.put(ResourceEnum.TELEMETRY_SYNC, Maps.newConcurrentMap()); _httpErrors.put(ResourceEnum.TOKEN_SYNC, Maps.newConcurrentMap()); } + + + + private void initMethodCounters() { + _exceptionsCounters.put(MethodEnum.TREATMENT, new AtomicLong()); + _exceptionsCounters.put(MethodEnum.TREATMENTS, new AtomicLong()); + _exceptionsCounters.put(MethodEnum.TREATMENT_WITH_CONFIG, new AtomicLong()); + _exceptionsCounters.put(MethodEnum.TREATMENTS_WITH_CONFIG, new AtomicLong()); + _exceptionsCounters.put(MethodEnum.TRACK, new AtomicLong()); + } + + private void initFactoryCounters() { + _factoryCounters.put(FactoryCountersEnum.BUR_TIMEOUTS, new AtomicLong()); + _factoryCounters.put(FactoryCountersEnum.NON_READY_USAGES, new AtomicLong()); + } + + private void initImpressionDataCounters() { + _impressionsDataRecords.put(ImpressionsDataTypeEnum.IMPRESSIONS_DEDUPED, new AtomicLong()); + _impressionsDataRecords.put(ImpressionsDataTypeEnum.IMPRESSIONS_DROPPED, new AtomicLong()); + _impressionsDataRecords.put(ImpressionsDataTypeEnum.IMPRESSIONS_QUEUED, new AtomicLong()); + } + + private void initPushCounters() { + _pushCounters.put(PushCountersEnum.AUTH_REJECTIONS, new AtomicLong()); + _pushCounters.put(PushCountersEnum.TOKEN_REFRESHES, new AtomicLong()); + } + + private void initSdkRecords() { + _sdkRecords.put(SdkRecordsEnum.SESSION, new AtomicLong()); + } + + private void initLastSynchronizationRecords() { + _lastSynchronizationRecords.put(LastSynchronizationRecordsEnum.SPLITS, new AtomicLong()); + _lastSynchronizationRecords.put(LastSynchronizationRecordsEnum.SEGMENTS, new AtomicLong()); + _lastSynchronizationRecords.put(LastSynchronizationRecordsEnum.EVENTS, new AtomicLong()); + _lastSynchronizationRecords.put(LastSynchronizationRecordsEnum.IMPRESSIONS, new AtomicLong()); + _lastSynchronizationRecords.put(LastSynchronizationRecordsEnum.IMPRESSIONS_COUNT, new AtomicLong()); + _lastSynchronizationRecords.put(LastSynchronizationRecordsEnum.TOKEN, new AtomicLong()); + } + + private void initEventDataRecords() { + _eventsDataRecords.put(EventsDataRecordsEnum.EVENTS_DROPPED, new AtomicLong()); + _eventsDataRecords.put(EventsDataRecordsEnum.EVENTS_QUEUED, new AtomicLong()); + } } diff --git a/client/src/main/java/io/split/telemetry/storage/TelemetryConfigProducer.java b/client/src/main/java/io/split/telemetry/storage/TelemetryConfigProducer.java index 913733d03..64348c154 100644 --- a/client/src/main/java/io/split/telemetry/storage/TelemetryConfigProducer.java +++ b/client/src/main/java/io/split/telemetry/storage/TelemetryConfigProducer.java @@ -1,7 +1,6 @@ package io.split.telemetry.storage; public interface TelemetryConfigProducer { - void recordConfigData(); void recordNonReadyUsage(); void recordBURTimeout(); } diff --git a/client/src/main/java/io/split/telemetry/storage/TelemetryEvaluationConsumer.java b/client/src/main/java/io/split/telemetry/storage/TelemetryEvaluationConsumer.java index 976fbb623..88ed7f9ca 100644 --- a/client/src/main/java/io/split/telemetry/storage/TelemetryEvaluationConsumer.java +++ b/client/src/main/java/io/split/telemetry/storage/TelemetryEvaluationConsumer.java @@ -4,6 +4,6 @@ import io.split.telemetry.domain.MethodLatencies; public interface TelemetryEvaluationConsumer { - MethodExceptions popExceptions(); + MethodExceptions popExceptions() throws Exception; MethodLatencies popLatencies(); } diff --git a/client/src/main/java/io/split/telemetry/storage/TelemetryRuntimeConsumer.java b/client/src/main/java/io/split/telemetry/storage/TelemetryRuntimeConsumer.java index 9df890be3..f1871951c 100644 --- a/client/src/main/java/io/split/telemetry/storage/TelemetryRuntimeConsumer.java +++ b/client/src/main/java/io/split/telemetry/storage/TelemetryRuntimeConsumer.java @@ -14,7 +14,7 @@ public interface TelemetryRuntimeConsumer { long getEventStats(EventsDataRecordsEnum type); LastSynchronization getLastSynchronization(); HTTPErrors popHTTPErrors(); - HTTPLatencies popHTTPLatencies(); + HTTPLatencies popHTTPLatencies() throws Exception; long popAuthRejections(); long popTokenRefreshes(); List popStreamingEvents(); diff --git a/client/src/main/java/io/split/telemetry/utils/BucketCalculator.java b/client/src/main/java/io/split/telemetry/utils/BucketCalculator.java new file mode 100644 index 000000000..b68956c97 --- /dev/null +++ b/client/src/main/java/io/split/telemetry/utils/BucketCalculator.java @@ -0,0 +1,64 @@ +package io.split.telemetry.utils; + +import java.util.Arrays; + +/** + * Calculates buckets from latency + *

+ * (1) 1.00 + * (2) 1.50 + * (3) 2.25 + * (4) 3.38 + * (5) 5.06 + * (6) 7.59 + * (7) 11.39 + * (8) 17.09 + * (9) 25.63 + * (10) 38.44 + * (11) 57.67 + * (12) 86.50 + * (13) 129.75 + * (14) 194.62 + * (15) 291.93 + * (16) 437.89 + * (17) 656.84 + * (18) 985.26 + * (19) 1,477.89 + * (20) 2,216.84 + * (21) 3,325.26 + * (22) 4,987.89 + * (23) 7,481.83 + *

+ */ +public class BucketCalculator { + + static final long[] BUCKETS = { + 1000, 1500, 2250, 3375, 5063, + 7594, 11391, 17086, 25629, 38443, + 57665, 86498, 129746, 194620, 291929, + 437894, 656841, 985261, 1477892, 2216838, + 3325257, 4987885, 7481828 + }; + + static final long MAX_LATENCY = 7481828; + + public static int getBucketForLatencyMillis(long latency) { + long micros = latency * 1000; + if (micros > MAX_LATENCY) { + return BUCKETS.length - 1; + } + + int index = Arrays.binarySearch(BUCKETS, micros); + + if (index < 0) { + + // Adjust the index based on Java Array javadocs. <0 means the value wasn't found and it's module value + // is where it should be inserted (in this case, it means the counter it applies - unless it's equals to the + // length of the array). + + index = -(index + 1); + } + return index; + } + +} diff --git a/client/src/test/java/io/split/telemetry/storage/InMemoryTelemetryStorageTest.java b/client/src/test/java/io/split/telemetry/storage/InMemoryTelemetryStorageTest.java new file mode 100644 index 000000000..98ca18132 --- /dev/null +++ b/client/src/test/java/io/split/telemetry/storage/InMemoryTelemetryStorageTest.java @@ -0,0 +1,21 @@ +package io.split.telemetry.storage; + +import io.split.telemetry.domain.MethodExceptions; +import io.split.telemetry.domain.enums.MethodEnum; +import org.junit.Assert; +import org.junit.Test; + +public class InMemoryTelemetryStorageTest{ + + @Test + public void testInMemoryTelemetryStorage() throws Exception { + InMemoryTelemetryStorage telemetryStorage = new InMemoryTelemetryStorage(); + + telemetryStorage.recordException(MethodEnum.TREATMENT); + telemetryStorage.recordException(MethodEnum.TREATMENTS); + telemetryStorage.recordException(MethodEnum.TREATMENT); + + MethodExceptions methodExceptions = telemetryStorage.popExceptions(); + Assert.assertEquals(2, methodExceptions.get_treatment()); + } +} \ No newline at end of file From b52c519056156977004804c22cb2ac957f4f06d9 Mon Sep 17 00:00:00 2001 From: Lucas Echeverz Date: Sun, 11 Apr 2021 18:32:56 -0300 Subject: [PATCH 06/81] Adding BucketCalculator, tests and several minor changes --- .../telemetry/domain/AtomicLongArray.java | 37 ++++ .../storage/InMemoryTelemetryStorage.java | 46 ++-- .../storage/TelemetryEvaluationConsumer.java | 2 +- .../storage/TelemetryEvaluationProducer.java | 2 +- .../storage/TelemetryRuntimeConsumer.java | 1 - .../storage/TelemetryRuntimeProducer.java | 7 +- .../telemetry/utils/BucketCalculator.java | 4 +- .../storage/InMemoryTelemetryStorageTest.java | 206 +++++++++++++++++- .../telemetry/utils/BucketCalculatorTest.java | 22 ++ 9 files changed, 288 insertions(+), 39 deletions(-) create mode 100644 client/src/main/java/io/split/telemetry/domain/AtomicLongArray.java create mode 100644 client/src/test/java/io/split/telemetry/utils/BucketCalculatorTest.java diff --git a/client/src/main/java/io/split/telemetry/domain/AtomicLongArray.java b/client/src/main/java/io/split/telemetry/domain/AtomicLongArray.java new file mode 100644 index 000000000..cdd51f893 --- /dev/null +++ b/client/src/main/java/io/split/telemetry/domain/AtomicLongArray.java @@ -0,0 +1,37 @@ +package io.split.telemetry.domain; + +import java.util.ArrayList; +import java.util.Arrays; +import java.util.List; +import java.util.concurrent.atomic.AtomicLong; +import java.util.stream.IntStream; + +public class AtomicLongArray { + private AtomicLong[] array; + + public AtomicLongArray(int size) throws Exception { + if(size == 0) { + throw new Exception("Invalid array size"); + } + array = new AtomicLong[size]; + IntStream.range(0, array.length).forEach(x -> array[x] = new AtomicLong()); + } + + public void increment(int index) { + if (index < 0 || index >= array.length) { + throw new ArrayIndexOutOfBoundsException(); + } + array[index].getAndIncrement(); + } + + public List fetchAndClearAll() { + List listValues = new ArrayList<>(); + for (AtomicLong a: array) { + listValues.add(a.longValue()); + } + + IntStream.range(0, array.length).forEach(x -> array[x] = new AtomicLong()); + + return listValues; + } +} diff --git a/client/src/main/java/io/split/telemetry/storage/InMemoryTelemetryStorage.java b/client/src/main/java/io/split/telemetry/storage/InMemoryTelemetryStorage.java index 967790704..927e746e4 100644 --- a/client/src/main/java/io/split/telemetry/storage/InMemoryTelemetryStorage.java +++ b/client/src/main/java/io/split/telemetry/storage/InMemoryTelemetryStorage.java @@ -9,6 +9,7 @@ import java.util.List; import java.util.concurrent.ConcurrentMap; import java.util.concurrent.atomic.AtomicLong; +import java.util.stream.Collectors; public class InMemoryTelemetryStorage implements TelemetryStorage{ public static final int MAX_LATENCY_BUCKET_COUNT = 23; @@ -54,14 +55,12 @@ public InMemoryTelemetryStorage() throws Exception { @Override public long getBURTimeouts() { - long burTimeouts = _factoryCounters.get(FactoryCountersEnum.BUR_TIMEOUTS).get(); - return burTimeouts; + return _factoryCounters.get(FactoryCountersEnum.BUR_TIMEOUTS).get(); } @Override public long getNonReadyUsages() { - long nonReadyUsages = _factoryCounters.get(FactoryCountersEnum.NON_READY_USAGES).get(); - return nonReadyUsages; + return _factoryCounters.get(FactoryCountersEnum.NON_READY_USAGES).get(); } @Override @@ -74,13 +73,13 @@ public MethodExceptions popExceptions() throws Exception { exceptions.set_track(_exceptionsCounters.get(MethodEnum.TRACK).get()); _exceptionsCounters.clear(); - initMethodLatencies(); + initMethodCounters(); return exceptions; } @Override - public MethodLatencies popLatencies() { + public MethodLatencies popLatencies() throws Exception { MethodLatencies latencies = new MethodLatencies(); latencies.set_treatment(_methodLatencies.get(MethodEnum.TREATMENT).fetchAndClearAll()); latencies.set_treatments(_methodLatencies.get(MethodEnum.TREATMENTS).fetchAndClearAll()); @@ -89,7 +88,7 @@ public MethodLatencies popLatencies() { latencies.set_track(_methodLatencies.get(MethodEnum.TRACK).fetchAndClearAll()); _methodLatencies.clear(); - initMethodCounters(); + initMethodLatencies(); return latencies; } @@ -97,7 +96,6 @@ public MethodLatencies popLatencies() { @Override public void recordNonReadyUsage() { _factoryCounters.get(FactoryCountersEnum.NON_READY_USAGES).incrementAndGet(); - } @Override @@ -106,8 +104,8 @@ public void recordBURTimeout() { } @Override - public void recordLatency(String method, int latency) { - int bucket = BucketCalculator.getBucketForLatencyMillis(latency); + public void recordLatency(MethodEnum method, long latency) { + int bucket = BucketCalculator.getBucketForLatency(latency); _methodLatencies.get(method).increment(bucket); } @@ -117,13 +115,13 @@ public void recordException(MethodEnum method) { } @Override - public long getImpressionsStats(ImpressionsDataTypeEnum data) { - return _impressionsDataRecords.get(data).get(); + public long getImpressionsStats(ImpressionsDataTypeEnum dataType) { + return _impressionsDataRecords.get(dataType).get(); } @Override - public long getEventStats(EventsDataRecordsEnum type) { - return _eventsDataRecords.get(type).get(); + public long getEventStats(EventsDataRecordsEnum dataType) { + return _eventsDataRecords.get(dataType).get(); } @Override @@ -195,7 +193,7 @@ public long popTokenRefreshes() { @Override public List popStreamingEvents() { synchronized (_streamingEventsLock) { - List streamingEvents = _streamingEvents; + List streamingEvents = _streamingEvents.stream().collect(Collectors.toList()); _streamingEvents.clear(); @@ -206,7 +204,7 @@ public List popStreamingEvents() { @Override public List popTags() { synchronized (_tagsLock) { - List tags = _tags; + List tags = _tags.stream().collect(Collectors.toList()); _tags.clear(); @@ -228,30 +226,29 @@ public void addTag(String tag) { @Override public void recordImpressionStats(ImpressionsDataTypeEnum dataType, long count) { - _impressionsDataRecords.get(dataType).incrementAndGet(); + _impressionsDataRecords.get(dataType).addAndGet(count); } @Override public void recordEventStats(EventsDataRecordsEnum dataType, long count) { - _eventsDataRecords.get(dataType).incrementAndGet(); + _eventsDataRecords.get(dataType).addAndGet(count); } @Override public void recordSuccessfulSync(LastSynchronizationRecordsEnum resource, long time) { _lastSynchronizationRecords.replace(resource, new AtomicLong(time)); - } @Override public void recordSyncError(ResourceEnum resource, int status) { ConcurrentMap errors = _httpErrors.get(resource); errors.putIfAbsent(Long.valueOf(status), 0l); - errors.replace(Long.valueOf(status), errors.get(status) + 1); + errors.replace(Long.valueOf(status), errors.get(Long.valueOf(status)) + 1); } @Override - public void recordSyncLatency(String resource, long latency) { - int bucket = BucketCalculator.getBucketForLatencyMillis(latency); + public void recordSyncLatency(HTTPLatenciesEnum resource, long latency) { + int bucket = BucketCalculator.getBucketForLatency(latency); _httpLatencies.get(resource).increment(bucket); } @@ -259,13 +256,11 @@ public void recordSyncLatency(String resource, long latency) { @Override public void recordAuthRejections() { _pushCounters.get(PushCountersEnum.AUTH_REJECTIONS).incrementAndGet(); - } @Override public void recordTokenRefreshes() { _pushCounters.get(PushCountersEnum.TOKEN_REFRESHES).incrementAndGet(); - } @Override @@ -308,8 +303,6 @@ private void initHttpErrors() { _httpErrors.put(ResourceEnum.TOKEN_SYNC, Maps.newConcurrentMap()); } - - private void initMethodCounters() { _exceptionsCounters.put(MethodEnum.TREATMENT, new AtomicLong()); _exceptionsCounters.put(MethodEnum.TREATMENTS, new AtomicLong()); @@ -345,6 +338,7 @@ private void initLastSynchronizationRecords() { _lastSynchronizationRecords.put(LastSynchronizationRecordsEnum.IMPRESSIONS, new AtomicLong()); _lastSynchronizationRecords.put(LastSynchronizationRecordsEnum.IMPRESSIONS_COUNT, new AtomicLong()); _lastSynchronizationRecords.put(LastSynchronizationRecordsEnum.TOKEN, new AtomicLong()); + _lastSynchronizationRecords.put(LastSynchronizationRecordsEnum.TELEMETRY, new AtomicLong()); } private void initEventDataRecords() { diff --git a/client/src/main/java/io/split/telemetry/storage/TelemetryEvaluationConsumer.java b/client/src/main/java/io/split/telemetry/storage/TelemetryEvaluationConsumer.java index 88ed7f9ca..ca63112ac 100644 --- a/client/src/main/java/io/split/telemetry/storage/TelemetryEvaluationConsumer.java +++ b/client/src/main/java/io/split/telemetry/storage/TelemetryEvaluationConsumer.java @@ -5,5 +5,5 @@ public interface TelemetryEvaluationConsumer { MethodExceptions popExceptions() throws Exception; - MethodLatencies popLatencies(); + MethodLatencies popLatencies() throws Exception; } diff --git a/client/src/main/java/io/split/telemetry/storage/TelemetryEvaluationProducer.java b/client/src/main/java/io/split/telemetry/storage/TelemetryEvaluationProducer.java index 51795b054..005106c30 100644 --- a/client/src/main/java/io/split/telemetry/storage/TelemetryEvaluationProducer.java +++ b/client/src/main/java/io/split/telemetry/storage/TelemetryEvaluationProducer.java @@ -3,6 +3,6 @@ import io.split.telemetry.domain.enums.MethodEnum; public interface TelemetryEvaluationProducer { - void recordLatency(String method, int latency); + void recordLatency(MethodEnum method, long latency); void recordException(MethodEnum method); } diff --git a/client/src/main/java/io/split/telemetry/storage/TelemetryRuntimeConsumer.java b/client/src/main/java/io/split/telemetry/storage/TelemetryRuntimeConsumer.java index f1871951c..7acd49afb 100644 --- a/client/src/main/java/io/split/telemetry/storage/TelemetryRuntimeConsumer.java +++ b/client/src/main/java/io/split/telemetry/storage/TelemetryRuntimeConsumer.java @@ -20,5 +20,4 @@ public interface TelemetryRuntimeConsumer { List popStreamingEvents(); List popTags(); long getSessionLength(); - } diff --git a/client/src/main/java/io/split/telemetry/storage/TelemetryRuntimeProducer.java b/client/src/main/java/io/split/telemetry/storage/TelemetryRuntimeProducer.java index ab6fe175f..2baf016f0 100644 --- a/client/src/main/java/io/split/telemetry/storage/TelemetryRuntimeProducer.java +++ b/client/src/main/java/io/split/telemetry/storage/TelemetryRuntimeProducer.java @@ -1,10 +1,7 @@ package io.split.telemetry.storage; import io.split.telemetry.domain.StreamingEvent; -import io.split.telemetry.domain.enums.EventsDataRecordsEnum; -import io.split.telemetry.domain.enums.ImpressionsDataTypeEnum; -import io.split.telemetry.domain.enums.LastSynchronizationRecordsEnum; -import io.split.telemetry.domain.enums.ResourceEnum; +import io.split.telemetry.domain.enums.*; public interface TelemetryRuntimeProducer { void addTag(String tag); @@ -12,7 +9,7 @@ public interface TelemetryRuntimeProducer { void recordEventStats(EventsDataRecordsEnum dataType, long count); void recordSuccessfulSync(LastSynchronizationRecordsEnum resource, long time); void recordSyncError(ResourceEnum resource, int status); - void recordSyncLatency(String resource, long latency); + void recordSyncLatency(HTTPLatenciesEnum resource, long latency); void recordAuthRejections(); void recordTokenRefreshes(); void recordStreamingEvents(StreamingEvent streamingEvent); diff --git a/client/src/main/java/io/split/telemetry/utils/BucketCalculator.java b/client/src/main/java/io/split/telemetry/utils/BucketCalculator.java index b68956c97..7c5ce3ced 100644 --- a/client/src/main/java/io/split/telemetry/utils/BucketCalculator.java +++ b/client/src/main/java/io/split/telemetry/utils/BucketCalculator.java @@ -42,8 +42,8 @@ public class BucketCalculator { static final long MAX_LATENCY = 7481828; - public static int getBucketForLatencyMillis(long latency) { - long micros = latency * 1000; + public static int getBucketForLatency(long latency) { + long micros = latency / 1000; //Convert to milliseconds if (micros > MAX_LATENCY) { return BUCKETS.length - 1; } diff --git a/client/src/test/java/io/split/telemetry/storage/InMemoryTelemetryStorageTest.java b/client/src/test/java/io/split/telemetry/storage/InMemoryTelemetryStorageTest.java index 98ca18132..49c1ce167 100644 --- a/client/src/test/java/io/split/telemetry/storage/InMemoryTelemetryStorageTest.java +++ b/client/src/test/java/io/split/telemetry/storage/InMemoryTelemetryStorageTest.java @@ -1,21 +1,221 @@ package io.split.telemetry.storage; -import io.split.telemetry.domain.MethodExceptions; -import io.split.telemetry.domain.enums.MethodEnum; +import io.split.telemetry.domain.*; +import io.split.telemetry.domain.enums.*; import org.junit.Assert; import org.junit.Test; +import java.util.List; + public class InMemoryTelemetryStorageTest{ @Test public void testInMemoryTelemetryStorage() throws Exception { InMemoryTelemetryStorage telemetryStorage = new InMemoryTelemetryStorage(); + //MethodLatencies + telemetryStorage.recordLatency(MethodEnum.TREATMENT, 1500l * 1000); + telemetryStorage.recordLatency(MethodEnum.TREATMENT, 2000l * 1000); + telemetryStorage.recordLatency(MethodEnum.TREATMENTS, 3000l * 1000); + telemetryStorage.recordLatency(MethodEnum.TREATMENTS, 500l * 1000); + telemetryStorage.recordLatency(MethodEnum.TREATMENT_WITH_CONFIG, 800l * 1000); + telemetryStorage.recordLatency(MethodEnum.TREATMENTS_WITH_CONFIG, 1000l * 1000); + + MethodLatencies latencies = telemetryStorage.popLatencies(); + Assert.assertEquals(2, latencies.get_treatment().stream().mapToInt(Long::intValue).sum()); + Assert.assertEquals(2, latencies.get_treatments().stream().mapToInt(Long::intValue).sum()); + Assert.assertEquals(1, latencies.get_treatmentsWithConfig().stream().mapToInt(Long::intValue).sum()); + Assert.assertEquals(1, latencies.get_treatmentWithConfig().stream().mapToInt(Long::intValue).sum()); + Assert.assertEquals(0, latencies.get_track().stream().mapToInt(Long::intValue).sum()); + + //Check empty has worked + latencies = telemetryStorage.popLatencies(); + Assert.assertEquals(0, latencies.get_treatment().stream().mapToInt(Long::intValue).sum()); + Assert.assertEquals(0, latencies.get_treatments().stream().mapToInt(Long::intValue).sum()); + Assert.assertEquals(0, latencies.get_treatmentsWithConfig().stream().mapToInt(Long::intValue).sum()); + Assert.assertEquals(0, latencies.get_treatmentWithConfig().stream().mapToInt(Long::intValue).sum()); + Assert.assertEquals(0, latencies.get_track().stream().mapToInt(Long::intValue).sum()); + + //HttpLatencies + telemetryStorage.recordSyncLatency(HTTPLatenciesEnum.TELEMETRY, 1500l * 1000); + telemetryStorage.recordSyncLatency(HTTPLatenciesEnum.TELEMETRY, 2000l * 1000); + telemetryStorage.recordSyncLatency(HTTPLatenciesEnum.EVENTS, 1500l * 1000); + telemetryStorage.recordSyncLatency(HTTPLatenciesEnum.EVENTS, 2000l * 1000); + telemetryStorage.recordSyncLatency(HTTPLatenciesEnum.SEGMENTS, 1500l * 1000); + telemetryStorage.recordSyncLatency(HTTPLatenciesEnum.SPLITS, 2000l * 1000); + telemetryStorage.recordSyncLatency(HTTPLatenciesEnum.SPLITS, 1500l * 1000); + telemetryStorage.recordSyncLatency(HTTPLatenciesEnum.SPLITS, 2000l * 1000); + telemetryStorage.recordSyncLatency(HTTPLatenciesEnum.IMPRESSIONS, 1500l * 1000); + telemetryStorage.recordSyncLatency(HTTPLatenciesEnum.IMPRESSIONS_COUNT, 2000l * 1000); + + HTTPLatencies httpLatencies = telemetryStorage.popHTTPLatencies(); + + Assert.assertEquals(3, httpLatencies.get_splits().stream().mapToInt(Long::intValue).sum()); + Assert.assertEquals(2, httpLatencies.get_telemetry().stream().mapToInt(Long::intValue).sum()); + Assert.assertEquals(2, httpLatencies.get_events().stream().mapToInt(Long::intValue).sum()); + Assert.assertEquals(1, httpLatencies.get_segments().stream().mapToInt(Long::intValue).sum()); + Assert.assertEquals(1, httpLatencies.get_impressions().stream().mapToInt(Long::intValue).sum()); + Assert.assertEquals(1, httpLatencies.get_impressionsCount().stream().mapToInt(Long::intValue).sum()); + Assert.assertEquals(0, httpLatencies.get_token().stream().mapToInt(Long::intValue).sum()); + + httpLatencies = telemetryStorage.popHTTPLatencies(); + Assert.assertEquals(0, httpLatencies.get_splits().stream().mapToInt(Long::intValue).sum()); + Assert.assertEquals(0, httpLatencies.get_telemetry().stream().mapToInt(Long::intValue).sum()); + Assert.assertEquals(0, httpLatencies.get_events().stream().mapToInt(Long::intValue).sum()); + Assert.assertEquals(0, httpLatencies.get_segments().stream().mapToInt(Long::intValue).sum()); + Assert.assertEquals(0, httpLatencies.get_impressions().stream().mapToInt(Long::intValue).sum()); + Assert.assertEquals(0, httpLatencies.get_impressionsCount().stream().mapToInt(Long::intValue).sum()); + Assert.assertEquals(0, httpLatencies.get_token().stream().mapToInt(Long::intValue).sum()); + + + //Exceptions telemetryStorage.recordException(MethodEnum.TREATMENT); telemetryStorage.recordException(MethodEnum.TREATMENTS); telemetryStorage.recordException(MethodEnum.TREATMENT); + telemetryStorage.recordException(MethodEnum.TREATMENTS); + telemetryStorage.recordException(MethodEnum.TREATMENT_WITH_CONFIG); + telemetryStorage.recordException(MethodEnum.TREATMENTS_WITH_CONFIG); MethodExceptions methodExceptions = telemetryStorage.popExceptions(); Assert.assertEquals(2, methodExceptions.get_treatment()); + Assert.assertEquals(2, methodExceptions.get_treatments()); + Assert.assertEquals(1, methodExceptions.get_treatmentsWithConfig()); + Assert.assertEquals(1, methodExceptions.get_treatmentWithConfig()); + Assert.assertEquals(0, methodExceptions.get_track()); + + //Check empty has worked + methodExceptions = telemetryStorage.popExceptions(); + Assert.assertEquals(0, methodExceptions.get_treatment()); + Assert.assertEquals(0, methodExceptions.get_treatments()); + Assert.assertEquals(0, methodExceptions.get_treatmentsWithConfig()); + Assert.assertEquals(0, methodExceptions.get_treatmentWithConfig()); + Assert.assertEquals(0, methodExceptions.get_track()); + + //AuthRejections + telemetryStorage.recordAuthRejections(); + long authRejections = telemetryStorage.popAuthRejections(); + Assert.assertEquals(1, authRejections); + + //Check amount has been reseted + authRejections = telemetryStorage.popAuthRejections(); + Assert.assertEquals(0, authRejections); + + //AuthRejections + telemetryStorage.recordTokenRefreshes(); + telemetryStorage.recordTokenRefreshes(); + long tokenRefreshes = telemetryStorage.popTokenRefreshes(); + Assert.assertEquals(2, tokenRefreshes); + + //Check amount has been reseted + tokenRefreshes = telemetryStorage.popTokenRefreshes(); + Assert.assertEquals(0, tokenRefreshes); + + //Non Ready usages + telemetryStorage.recordNonReadyUsage(); + telemetryStorage.recordNonReadyUsage(); + long nonReadyUsages = telemetryStorage.getNonReadyUsages(); + Assert.assertEquals(2, nonReadyUsages); + + //BUR Timeouts + telemetryStorage.recordBURTimeout(); + long burTimeouts = telemetryStorage.getBURTimeouts(); + Assert.assertEquals(1, burTimeouts); + + //ImpressionStats + telemetryStorage.recordImpressionStats(ImpressionsDataTypeEnum.IMPRESSIONS_DEDUPED, 3); + telemetryStorage.recordImpressionStats(ImpressionsDataTypeEnum.IMPRESSIONS_DEDUPED, 1); + telemetryStorage.recordImpressionStats(ImpressionsDataTypeEnum.IMPRESSIONS_DROPPED, 4); + telemetryStorage.recordImpressionStats(ImpressionsDataTypeEnum.IMPRESSIONS_DROPPED, 6); + telemetryStorage.recordImpressionStats(ImpressionsDataTypeEnum.IMPRESSIONS_DROPPED, 2); + + long impressionsDeduped = telemetryStorage.getImpressionsStats(ImpressionsDataTypeEnum.IMPRESSIONS_DEDUPED); + long impressionsDropped = telemetryStorage.getImpressionsStats(ImpressionsDataTypeEnum.IMPRESSIONS_DROPPED); + long impressionsQueued = telemetryStorage.getImpressionsStats(ImpressionsDataTypeEnum.IMPRESSIONS_QUEUED); + + Assert.assertEquals(4, impressionsDeduped); + Assert.assertEquals(12, impressionsDropped); + Assert.assertEquals(0, impressionsQueued); + + //Event Stats + telemetryStorage.recordEventStats(EventsDataRecordsEnum.EVENTS_DROPPED, 3); + telemetryStorage.recordEventStats(EventsDataRecordsEnum.EVENTS_DROPPED, 7); + telemetryStorage.recordEventStats(EventsDataRecordsEnum.EVENTS_QUEUED, 3); + + long eventsDropped = telemetryStorage.getEventStats(EventsDataRecordsEnum.EVENTS_DROPPED); + long eventsQueued = telemetryStorage.getEventStats(EventsDataRecordsEnum.EVENTS_QUEUED); + + Assert.assertEquals(10, eventsDropped); + Assert.assertEquals(3, eventsQueued); + + //Successfuly sync + telemetryStorage.recordSuccessfulSync(LastSynchronizationRecordsEnum.EVENTS, 1500); + telemetryStorage.recordSuccessfulSync(LastSynchronizationRecordsEnum.EVENTS, 800); + telemetryStorage.recordSuccessfulSync(LastSynchronizationRecordsEnum.IMPRESSIONS, 2500); + telemetryStorage.recordSuccessfulSync(LastSynchronizationRecordsEnum.IMPRESSIONS, 10500); + telemetryStorage.recordSuccessfulSync(LastSynchronizationRecordsEnum.IMPRESSIONS_COUNT, 1500); + telemetryStorage.recordSuccessfulSync(LastSynchronizationRecordsEnum.SEGMENTS, 1580); + telemetryStorage.recordSuccessfulSync(LastSynchronizationRecordsEnum.TELEMETRY, 265); + telemetryStorage.recordSuccessfulSync(LastSynchronizationRecordsEnum.TOKEN, 129); + + LastSynchronization lastSynchronization = telemetryStorage.getLastSynchronization(); + Assert.assertEquals(800, lastSynchronization.get_events()); + Assert.assertEquals(129, lastSynchronization.get_token()); + Assert.assertEquals(1580, lastSynchronization.get_segments()); + Assert.assertEquals(0, lastSynchronization.get_splits()); + Assert.assertEquals(10500, lastSynchronization.get_impressions()); + Assert.assertEquals(1500, lastSynchronization.get_impressionsCount()); + Assert.assertEquals(265, lastSynchronization.get_telemetry()); + + //Session length + telemetryStorage.recordSessionLength(91218); + long sessionLength = telemetryStorage.getSessionLength(); + Assert.assertEquals(91218, sessionLength); + + //Sync Error + telemetryStorage.recordSyncError(ResourceEnum.TELEMETRY_SYNC, 400); + telemetryStorage.recordSyncError(ResourceEnum.TELEMETRY_SYNC, 400); + telemetryStorage.recordSyncError(ResourceEnum.SEGMENT_SYNC, 501); + telemetryStorage.recordSyncError(ResourceEnum.IMPRESSION_SYNC, 403); + telemetryStorage.recordSyncError(ResourceEnum.IMPRESSION_SYNC, 403); + telemetryStorage.recordSyncError(ResourceEnum.EVENT_SYNC, 503); + telemetryStorage.recordSyncError(ResourceEnum.SPLIT_SYNC, 403); + telemetryStorage.recordSyncError(ResourceEnum.IMPRESSION_COUNT_SYNC, 403); + telemetryStorage.recordSyncError(ResourceEnum.TOKEN_SYNC, 403); + + HTTPErrors httpErrors = telemetryStorage.popHTTPErrors(); + Assert.assertEquals(2, httpErrors.get_telemetry().get(400l).intValue()); + Assert.assertEquals(1, httpErrors.get_segments().get(501l).intValue()); + Assert.assertEquals(2, httpErrors.get_impressions().get(403l).intValue()); + Assert.assertEquals(1, httpErrors.get_impressionsCount().get(403l).intValue()); + Assert.assertEquals(1, httpErrors.get_events().get(503l).intValue()); + Assert.assertEquals(1, httpErrors.get_splits().get(403l).intValue()); + Assert.assertEquals(1, httpErrors.get_token().get(403l).intValue()); + + //Streaming events + StreamingEvent streamingEvent = new StreamingEvent(); + streamingEvent.set_data(290); + streamingEvent.set_type(1); + streamingEvent.setTimestamp(91218); + telemetryStorage.recordStreamingEvents(streamingEvent); + + List streamingEvents = telemetryStorage.popStreamingEvents(); + Assert.assertEquals(290, streamingEvents.get(0).get_data()); + Assert.assertEquals(1, streamingEvents.get(0).get_type()); + Assert.assertEquals(91218, streamingEvents.get(0).getTimestamp()); + + //Check list has been cleared + streamingEvents = telemetryStorage.popStreamingEvents(); + Assert.assertEquals(0, streamingEvents.size()); + + //Tags + telemetryStorage.addTag("TAG_1"); + telemetryStorage.addTag("TAG_2"); + List tags = telemetryStorage.popTags(); + Assert.assertEquals(2, tags.size()); + + //Check tags have been cleared + tags = telemetryStorage.popTags(); + Assert.assertEquals(0, tags.size()); + } -} \ No newline at end of file +} diff --git a/client/src/test/java/io/split/telemetry/utils/BucketCalculatorTest.java b/client/src/test/java/io/split/telemetry/utils/BucketCalculatorTest.java new file mode 100644 index 000000000..04ba74487 --- /dev/null +++ b/client/src/test/java/io/split/telemetry/utils/BucketCalculatorTest.java @@ -0,0 +1,22 @@ +package io.split.telemetry.utils; + +import org.junit.Assert; +import org.junit.Test; + +public class BucketCalculatorTest{ + + @Test + public void testBucketCalculator() { + int bucket = BucketCalculator.getBucketForLatency(500l * 1000); + Assert.assertEquals(0, bucket); + + bucket = BucketCalculator.getBucketForLatency(1500l * 1000); + Assert.assertEquals(1, bucket); + + bucket = BucketCalculator.getBucketForLatency(8000l * 1000); + Assert.assertEquals(6, bucket); + + bucket = BucketCalculator.getBucketForLatency(7481829l * 1000); + Assert.assertEquals(22, bucket); + } +} From a963764760a6e68317f460f78c9b125a6c94d97f Mon Sep 17 00:00:00 2001 From: Lucas Echeverz Date: Mon, 12 Apr 2021 10:36:48 -0300 Subject: [PATCH 07/81] Changing name of init --- .../split/telemetry/storage/InMemoryTelemetryStorage.java | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/client/src/main/java/io/split/telemetry/storage/InMemoryTelemetryStorage.java b/client/src/main/java/io/split/telemetry/storage/InMemoryTelemetryStorage.java index 927e746e4..a115aa8d4 100644 --- a/client/src/main/java/io/split/telemetry/storage/InMemoryTelemetryStorage.java +++ b/client/src/main/java/io/split/telemetry/storage/InMemoryTelemetryStorage.java @@ -44,7 +44,7 @@ public InMemoryTelemetryStorage() throws Exception { initMethodLatencies(); initHttpLatencies(); initHttpErrors(); - initMethodCounters(); + initMethodExceptions(); initFactoryCounters(); initImpressionDataCounters(); initPushCounters(); @@ -73,7 +73,7 @@ public MethodExceptions popExceptions() throws Exception { exceptions.set_track(_exceptionsCounters.get(MethodEnum.TRACK).get()); _exceptionsCounters.clear(); - initMethodCounters(); + initMethodExceptions(); return exceptions; } @@ -103,6 +103,7 @@ public void recordBURTimeout() { _factoryCounters.get(FactoryCountersEnum.BUR_TIMEOUTS).incrementAndGet(); } + @Override public void recordLatency(MethodEnum method, long latency) { int bucket = BucketCalculator.getBucketForLatency(latency); @@ -303,7 +304,7 @@ private void initHttpErrors() { _httpErrors.put(ResourceEnum.TOKEN_SYNC, Maps.newConcurrentMap()); } - private void initMethodCounters() { + private void initMethodExceptions() { _exceptionsCounters.put(MethodEnum.TREATMENT, new AtomicLong()); _exceptionsCounters.put(MethodEnum.TREATMENTS, new AtomicLong()); _exceptionsCounters.put(MethodEnum.TREATMENT_WITH_CONFIG, new AtomicLong()); From bd086ba940d0b965ae608ddd7c54305935ea42b1 Mon Sep 17 00:00:00 2001 From: Lucas Echeverz Date: Tue, 13 Apr 2021 18:10:17 -0300 Subject: [PATCH 08/81] Commiting to be able to merge --- client/src/main/java/io/split/cache/SegmentCache.java | 6 ++++++ .../split/telemetry/storage/InMemoryTelemetryStorage.java | 1 + .../split/telemetry/{domain => utils}/AtomicLongArray.java | 5 ++--- 3 files changed, 9 insertions(+), 3 deletions(-) rename client/src/main/java/io/split/telemetry/{domain => utils}/AtomicLongArray.java (91%) diff --git a/client/src/main/java/io/split/cache/SegmentCache.java b/client/src/main/java/io/split/cache/SegmentCache.java index d75fe11d3..9a0d24f29 100644 --- a/client/src/main/java/io/split/cache/SegmentCache.java +++ b/client/src/main/java/io/split/cache/SegmentCache.java @@ -42,4 +42,10 @@ public interface SegmentCache { * clear all segments */ void clear(); + + /** + * return every segment + * @return + */ + List getAll(); } diff --git a/client/src/main/java/io/split/telemetry/storage/InMemoryTelemetryStorage.java b/client/src/main/java/io/split/telemetry/storage/InMemoryTelemetryStorage.java index a115aa8d4..1f50cb4fc 100644 --- a/client/src/main/java/io/split/telemetry/storage/InMemoryTelemetryStorage.java +++ b/client/src/main/java/io/split/telemetry/storage/InMemoryTelemetryStorage.java @@ -1,6 +1,7 @@ package io.split.telemetry.storage; import com.google.common.collect.Maps; +import io.split.telemetry.utils.AtomicLongArray; import io.split.telemetry.utils.BucketCalculator; import io.split.telemetry.domain.*; import io.split.telemetry.domain.enums.*; diff --git a/client/src/main/java/io/split/telemetry/domain/AtomicLongArray.java b/client/src/main/java/io/split/telemetry/utils/AtomicLongArray.java similarity index 91% rename from client/src/main/java/io/split/telemetry/domain/AtomicLongArray.java rename to client/src/main/java/io/split/telemetry/utils/AtomicLongArray.java index cdd51f893..232503929 100644 --- a/client/src/main/java/io/split/telemetry/domain/AtomicLongArray.java +++ b/client/src/main/java/io/split/telemetry/utils/AtomicLongArray.java @@ -1,7 +1,6 @@ -package io.split.telemetry.domain; +package io.split.telemetry.utils; import java.util.ArrayList; -import java.util.Arrays; import java.util.List; import java.util.concurrent.atomic.AtomicLong; import java.util.stream.IntStream; @@ -10,7 +9,7 @@ public class AtomicLongArray { private AtomicLong[] array; public AtomicLongArray(int size) throws Exception { - if(size == 0) { + if(size <= 0) { throw new Exception("Invalid array size"); } array = new AtomicLong[size]; From 7f2cc84519a7a515645cda96e83a13f15ce0ebd4 Mon Sep 17 00:00:00 2001 From: Lucas Echeverz Date: Tue, 13 Apr 2021 18:11:25 -0300 Subject: [PATCH 09/81] no message --- .../split/telemetry/domain/AdvanceConfig.java | 139 ++++++++++++ .../io/split/telemetry/domain/InitConfig.java | 31 +++ .../split/telemetry/domain/ManagerConfig.java | 31 +++ .../java/io/split/telemetry/domain/Stats.java | 207 ++++++++++++++++++ .../split/telemetry/domain/TaskPeriods.java | 67 ++++++ .../HttpTelemetryMemorySender.java | 47 ++++ .../synchronizer/TelemetrySynchronizer.java | 11 + .../TelemetrySynchronizerImp.java | 65 ++++++ .../telemetry/utils/AtomicLongArrayTest.java | 38 ++++ 9 files changed, 636 insertions(+) create mode 100644 client/src/main/java/io/split/telemetry/domain/AdvanceConfig.java create mode 100644 client/src/main/java/io/split/telemetry/domain/InitConfig.java create mode 100644 client/src/main/java/io/split/telemetry/domain/ManagerConfig.java create mode 100644 client/src/main/java/io/split/telemetry/domain/Stats.java create mode 100644 client/src/main/java/io/split/telemetry/domain/TaskPeriods.java create mode 100644 client/src/main/java/io/split/telemetry/synchronizer/HttpTelemetryMemorySender.java create mode 100644 client/src/main/java/io/split/telemetry/synchronizer/TelemetrySynchronizer.java create mode 100644 client/src/main/java/io/split/telemetry/synchronizer/TelemetrySynchronizerImp.java create mode 100644 client/src/test/java/io/split/telemetry/utils/AtomicLongArrayTest.java diff --git a/client/src/main/java/io/split/telemetry/domain/AdvanceConfig.java b/client/src/main/java/io/split/telemetry/domain/AdvanceConfig.java new file mode 100644 index 000000000..8e2ba548e --- /dev/null +++ b/client/src/main/java/io/split/telemetry/domain/AdvanceConfig.java @@ -0,0 +1,139 @@ +package io.split.telemetry.domain; + +public class AdvanceConfig { + private int _hTTPTimeout; + private int _segmentQueueSize; + private int _segmentWorkers; + private String _sdkURL; + private String _eventsURL; + private String _telemetryServiceURL; + private long _eventsBulkSize; + private int _eventsQueueSize; + private int _impressionsQueueSize; + private long _impressionsBulkSize; + private boolean _streamingEnabled; + private String _authServiceURL; + private String _streamingServiceURL; + private boolean _splitUpdateQueueSize; + private long _segmentUpdateQueueSize; + + public int get_hTTPTimeout() { + return _hTTPTimeout; + } + + public void set_hTTPTimeout(int _hTTPTimeout) { + this._hTTPTimeout = _hTTPTimeout; + } + + public int get_segmentQueueSize() { + return _segmentQueueSize; + } + + public void set_segmentQueueSize(int _segmentQueueSize) { + this._segmentQueueSize = _segmentQueueSize; + } + + public int get_segmentWorkers() { + return _segmentWorkers; + } + + public void set_segmentWorkers(int _segmentWorkers) { + this._segmentWorkers = _segmentWorkers; + } + + public String get_sdkURL() { + return _sdkURL; + } + + public void set_sdkURL(String _sdkURL) { + this._sdkURL = _sdkURL; + } + + public String get_eventsURL() { + return _eventsURL; + } + + public void set_eventsURL(String _eventsURL) { + this._eventsURL = _eventsURL; + } + + public String get_telemetryServiceURL() { + return _telemetryServiceURL; + } + + public void set_telemetryServiceURL(String _telemetryServiceURL) { + this._telemetryServiceURL = _telemetryServiceURL; + } + + public long get_eventsBulkSize() { + return _eventsBulkSize; + } + + public void set_eventsBulkSize(long _eventsBulkSize) { + this._eventsBulkSize = _eventsBulkSize; + } + + public int get_eventsQueueSize() { + return _eventsQueueSize; + } + + public void set_eventsQueueSize(int _eventsQueueSize) { + this._eventsQueueSize = _eventsQueueSize; + } + + public int get_impressionsQueueSize() { + return _impressionsQueueSize; + } + + public void set_impressionsQueueSize(int _impressionsQueueSize) { + this._impressionsQueueSize = _impressionsQueueSize; + } + + public long get_impressionsBulkSize() { + return _impressionsBulkSize; + } + + public void set_impressionsBulkSize(long _impressionsBulkSize) { + this._impressionsBulkSize = _impressionsBulkSize; + } + + public boolean is_streamingEnabled() { + return _streamingEnabled; + } + + public void set_streamingEnabled(boolean _streamingEnabled) { + this._streamingEnabled = _streamingEnabled; + } + + public String get_authServiceURL() { + return _authServiceURL; + } + + public void set_authServiceURL(String _authServiceURL) { + this._authServiceURL = _authServiceURL; + } + + public String get_streamingServiceURL() { + return _streamingServiceURL; + } + + public void set_streamingServiceURL(String _streamingServiceURL) { + this._streamingServiceURL = _streamingServiceURL; + } + + public boolean is_splitUpdateQueueSize() { + return _splitUpdateQueueSize; + } + + public void set_splitUpdateQueueSize(boolean _splitUpdateQueueSize) { + this._splitUpdateQueueSize = _splitUpdateQueueSize; + } + + public long get_segmentUpdateQueueSize() { + return _segmentUpdateQueueSize; + } + + public void set_segmentUpdateQueueSize(long _segmentUpdateQueueSize) { + this._segmentUpdateQueueSize = _segmentUpdateQueueSize; + } +} diff --git a/client/src/main/java/io/split/telemetry/domain/InitConfig.java b/client/src/main/java/io/split/telemetry/domain/InitConfig.java new file mode 100644 index 000000000..d2cb6569f --- /dev/null +++ b/client/src/main/java/io/split/telemetry/domain/InitConfig.java @@ -0,0 +1,31 @@ +package io.split.telemetry.domain; + +public class InitConfig { + private AdvanceConfig _advanceConfig; + private TaskPeriods _taskPeriods; + private ManagerConfig _managerConfig; + + public AdvanceConfig get_advanceConfig() { + return _advanceConfig; + } + + public void set_advanceConfig(AdvanceConfig _advanceConfig) { + this._advanceConfig = _advanceConfig; + } + + public TaskPeriods get_taskPeriods() { + return _taskPeriods; + } + + public void set_taskPeriods(TaskPeriods _taskPeriods) { + this._taskPeriods = _taskPeriods; + } + + public ManagerConfig get_managerConfig() { + return _managerConfig; + } + + public void set_managerConfig(ManagerConfig _managerConfig) { + this._managerConfig = _managerConfig; + } +} diff --git a/client/src/main/java/io/split/telemetry/domain/ManagerConfig.java b/client/src/main/java/io/split/telemetry/domain/ManagerConfig.java new file mode 100644 index 000000000..da7e00f77 --- /dev/null +++ b/client/src/main/java/io/split/telemetry/domain/ManagerConfig.java @@ -0,0 +1,31 @@ +package io.split.telemetry.domain; + +public class ManagerConfig { + private String _operationMode; + private String _impressionsMode; + private boolean _listenerEnabled; + + public String get_operationMode() { + return _operationMode; + } + + public void set_operationMode(String _operationMode) { + this._operationMode = _operationMode; + } + + public String get_impressionsMode() { + return _impressionsMode; + } + + public void set_impressionsMode(String _impressionsMode) { + this._impressionsMode = _impressionsMode; + } + + public boolean is_listenerEnabled() { + return _listenerEnabled; + } + + public void set_listenerEnabled(boolean _listenerEnabled) { + this._listenerEnabled = _listenerEnabled; + } +} diff --git a/client/src/main/java/io/split/telemetry/domain/Stats.java b/client/src/main/java/io/split/telemetry/domain/Stats.java new file mode 100644 index 000000000..8baec7261 --- /dev/null +++ b/client/src/main/java/io/split/telemetry/domain/Stats.java @@ -0,0 +1,207 @@ +package io.split.telemetry.domain; + +import com.google.gson.annotations.SerializedName; + +import java.util.List; + +public class Stats { + /* package private */ static final String FIELD_LAST_SYNCHRONIZATION = "lS"; + /* package private */ static final String FIELD_METHOD_LATENCIES = "ml"; + /* package private */ static final String FIELD_METHOD_EXCEPTIONS = "mE"; + /* package private */ static final String FIELD_HTTP_ERRORS = "hE"; + /* package private */ static final String FIELD_HTTP_LATENCIES = "hL"; + /* package private */ static final String FIELD_TOKEN_REFRESHES = "tR"; + /* package private */ static final String FIELD_AUTH_REJECTIONS = "aR"; + /* package private */ static final String FIELD_IMPRESSIONS_QUEUED = "iQ"; + /* package private */ static final String FIELD_IMPRESSIONS_DEDUPED = "iDe"; + /* package private */ static final String FIELD_IMPRESSIONS_DROPPED = "iDr"; + /* package private */ static final String FIELD_SPLITS = "spC"; + /* package private */ static final String FIELD_SEGMENTS = "seC"; + /* package private */ static final String FIELD_SEGMENTS_KEY = "skC"; + /* package private */ static final String FIELD_SESSION_LENGHT = "sL"; + /* package private */ static final String FIELD_EVENTS_QUEUED = "eQ"; + /* package private */ static final String FIELD_EVENTS_DROPPED = "eD"; + /* package private */ static final String FIELD_STREAMING_EVENT = "sE"; + /* package private */ static final String FIELD_TAGS = "t"; + + @SerializedName(FIELD_LAST_SYNCHRONIZATION) + private LastSynchronization _lastSynchronization; + @SerializedName(FIELD_METHOD_LATENCIES) + private MethodLatencies _methodLatencies; + @SerializedName(FIELD_METHOD_EXCEPTIONS) + private MethodExceptions _methodExceptions; + @SerializedName(FIELD_HTTP_ERRORS) + private HTTPErrors _httpErrors; + @SerializedName(FIELD_HTTP_LATENCIES) + private HTTPLatencies _httpLatencies; + @SerializedName(FIELD_TOKEN_REFRESHES) + private long _tokenRefreshes; + @SerializedName(FIELD_AUTH_REJECTIONS) + private long _authRejections; + @SerializedName(FIELD_IMPRESSIONS_QUEUED) + private long _impressionsQueued; + @SerializedName(FIELD_IMPRESSIONS_DEDUPED) + private long _impressionsDeduped; + @SerializedName(FIELD_IMPRESSIONS_DROPPED) + private long _impressionsDropped; + @SerializedName(FIELD_SPLITS) + private long _splitCount; + @SerializedName(FIELD_SEGMENTS) + private long _segmentCount; + @SerializedName(FIELD_SEGMENTS_KEY) + private long _segmentKeyCount; + @SerializedName(FIELD_SESSION_LENGHT) + private long _sessionLengthMs; + @SerializedName(FIELD_EVENTS_QUEUED) + private long _eventsQueued; + @SerializedName(FIELD_EVENTS_DROPPED) + private long _eventsDropped; + @SerializedName(FIELD_STREAMING_EVENT) + private List _streamingEvents; + @SerializedName(FIELD_TAGS) + private List _tags; + + public LastSynchronization get_lastSynchronization() { + return _lastSynchronization; + } + + public void set_lastSynchronization(LastSynchronization _lastSynchronization) { + this._lastSynchronization = _lastSynchronization; + } + + public MethodLatencies get_methodLatencies() { + return _methodLatencies; + } + + public void set_methodLatencies(MethodLatencies _methodLatencies) { + this._methodLatencies = _methodLatencies; + } + + public MethodExceptions get_methodExceptions() { + return _methodExceptions; + } + + public void set_methodExceptions(MethodExceptions _methodExceptions) { + this._methodExceptions = _methodExceptions; + } + + public HTTPErrors get_httpErrors() { + return _httpErrors; + } + + public void set_httpErrors(HTTPErrors _httpErrors) { + this._httpErrors = _httpErrors; + } + + public HTTPLatencies get_httpLatencies() { + return _httpLatencies; + } + + public void set_httpLatencies(HTTPLatencies _httpLatencies) { + this._httpLatencies = _httpLatencies; + } + + public long get_tokenRefreshes() { + return _tokenRefreshes; + } + + public void set_tokenRefreshes(long _tokenRefreshes) { + this._tokenRefreshes = _tokenRefreshes; + } + + public long get_authRejections() { + return _authRejections; + } + + public void set_authRejections(long _authRejections) { + this._authRejections = _authRejections; + } + + public long get_impressionsQueued() { + return _impressionsQueued; + } + + public void set_impressionsQueued(long _impressionsQueued) { + this._impressionsQueued = _impressionsQueued; + } + + public long get_impressionsDeduped() { + return _impressionsDeduped; + } + + public void set_impressionsDeduped(long _impressionsDeduped) { + this._impressionsDeduped = _impressionsDeduped; + } + + public long get_impressionsDropped() { + return _impressionsDropped; + } + + public void set_impressionsDropped(long _impressionsDropped) { + this._impressionsDropped = _impressionsDropped; + } + + public long get_splitCount() { + return _splitCount; + } + + public void set_splitCount(long _splitCount) { + this._splitCount = _splitCount; + } + + public long get_segmentCount() { + return _segmentCount; + } + + public void set_segmentCount(long _segmentCount) { + this._segmentCount = _segmentCount; + } + + public long get_segmentKeyCount() { + return _segmentKeyCount; + } + + public void set_segmentKeyCount(long _segmentKeyCount) { + this._segmentKeyCount = _segmentKeyCount; + } + + public long get_sessionLengthMs() { + return _sessionLengthMs; + } + + public void set_sessionLengthMs(long _sessionLengthMs) { + this._sessionLengthMs = _sessionLengthMs; + } + + public long get_eventsQueued() { + return _eventsQueued; + } + + public void set_eventsQueued(long _eventsQueued) { + this._eventsQueued = _eventsQueued; + } + + public long get_eventsDropped() { + return _eventsDropped; + } + + public void set_eventsDropped(long _eventsDropped) { + this._eventsDropped = _eventsDropped; + } + + public List get_streamingEvents() { + return _streamingEvents; + } + + public void set_streamingEvents(List _streamingEvents) { + this._streamingEvents = _streamingEvents; + } + + public List get_tags() { + return _tags; + } + + public void set_tags(List _tags) { + this._tags = _tags; + } +} diff --git a/client/src/main/java/io/split/telemetry/domain/TaskPeriods.java b/client/src/main/java/io/split/telemetry/domain/TaskPeriods.java new file mode 100644 index 000000000..b4f299768 --- /dev/null +++ b/client/src/main/java/io/split/telemetry/domain/TaskPeriods.java @@ -0,0 +1,67 @@ +package io.split.telemetry.domain; + +public class TaskPeriods { + private int _splitSync; + private int _segmentSync; + private int _impressionSync; + private int _gaugeSync; + private int _counterSync; + private int _latencySync; + private int _eventsSync; + + public int get_splitSync() { + return _splitSync; + } + + public void set_splitSync(int _splitSync) { + this._splitSync = _splitSync; + } + + public int get_segmentSync() { + return _segmentSync; + } + + public void set_segmentSync(int _segmentSync) { + this._segmentSync = _segmentSync; + } + + public int get_impressionSync() { + return _impressionSync; + } + + public void set_impressionSync(int _impressionSync) { + this._impressionSync = _impressionSync; + } + + public int get_gaugeSync() { + return _gaugeSync; + } + + public void set_gaugeSync(int _gaugeSync) { + this._gaugeSync = _gaugeSync; + } + + public int get_counterSync() { + return _counterSync; + } + + public void set_counterSync(int _counterSync) { + this._counterSync = _counterSync; + } + + public int get_latencySync() { + return _latencySync; + } + + public void set_latencySync(int _latencySync) { + this._latencySync = _latencySync; + } + + public int get_eventsSync() { + return _eventsSync; + } + + public void set_eventsSync(int _eventsSync) { + this._eventsSync = _eventsSync; + } +} diff --git a/client/src/main/java/io/split/telemetry/synchronizer/HttpTelemetryMemorySender.java b/client/src/main/java/io/split/telemetry/synchronizer/HttpTelemetryMemorySender.java new file mode 100644 index 000000000..f37fc0422 --- /dev/null +++ b/client/src/main/java/io/split/telemetry/synchronizer/HttpTelemetryMemorySender.java @@ -0,0 +1,47 @@ +package io.split.telemetry.synchronizer; + +import io.split.client.impressions.HttpImpressionsSender; +import io.split.client.utils.Utils; +import io.split.telemetry.domain.InitConfig; +import io.split.telemetry.domain.Stats; +import org.apache.hc.client5.http.impl.classic.CloseableHttpClient; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.net.URI; +import java.net.URISyntaxException; +import java.util.List; +import java.util.Map; + +public class HttpTelemetryMemorySender{ + + private static final String CONFIG_ENDPOINT_PATH = "api/telemetry/config"; + private static final String STATS_ENDPOINT_PATH = "api/telemetry/stats"; + + private static final Logger _logger = LoggerFactory.getLogger(HttpImpressionsSender.class); + + private final CloseableHttpClient _client; + private final URI _impressionConfigTarget; + private final URI _impressionStatsTarget; + + public static HttpTelemetryMemorySender create(CloseableHttpClient client, URI telemetryRootEndpoint) throws URISyntaxException { + return new HttpTelemetryMemorySender(client, + Utils.appendPath(telemetryRootEndpoint,CONFIG_ENDPOINT_PATH), + Utils.appendPath(telemetryRootEndpoint, STATS_ENDPOINT_PATH) + ); + } + + private HttpTelemetryMemorySender(CloseableHttpClient client, URI impressionConfigTarget, URI impressionStatsTarget) { + _client = client; + _impressionConfigTarget = impressionConfigTarget; + _impressionStatsTarget = impressionStatsTarget; + } + + public void postConfig(InitConfig config, long timedUntilReady, Map factoryInstances, List tags) { + + } + + public void postStats(Stats stats) { + + } +} diff --git a/client/src/main/java/io/split/telemetry/synchronizer/TelemetrySynchronizer.java b/client/src/main/java/io/split/telemetry/synchronizer/TelemetrySynchronizer.java new file mode 100644 index 000000000..59b454e18 --- /dev/null +++ b/client/src/main/java/io/split/telemetry/synchronizer/TelemetrySynchronizer.java @@ -0,0 +1,11 @@ +package io.split.telemetry.synchronizer; + +import io.split.telemetry.domain.InitConfig; + +import java.util.List; +import java.util.Map; + +public interface TelemetrySynchronizer { + void synchronizeConfig(InitConfig config, long timedUntilReady, Map factoryInstances, List tags); + void synchronizeStats() throws Exception; +} diff --git a/client/src/main/java/io/split/telemetry/synchronizer/TelemetrySynchronizerImp.java b/client/src/main/java/io/split/telemetry/synchronizer/TelemetrySynchronizerImp.java new file mode 100644 index 000000000..9d34d0bde --- /dev/null +++ b/client/src/main/java/io/split/telemetry/synchronizer/TelemetrySynchronizerImp.java @@ -0,0 +1,65 @@ +package io.split.telemetry.synchronizer; + +import io.split.cache.SegmentCache; +import io.split.cache.SplitCache; +import io.split.client.dtos.Split; +import io.split.telemetry.domain.InitConfig; +import io.split.telemetry.domain.Stats; +import io.split.telemetry.domain.enums.EventsDataRecordsEnum; +import io.split.telemetry.domain.enums.ImpressionsDataTypeEnum; +import io.split.telemetry.storage.TelemetryStorageConsumer; +import org.apache.hc.client5.http.impl.classic.CloseableHttpClient; + +import java.net.URI; +import java.net.URISyntaxException; +import java.util.List; +import java.util.Map; + +public class TelemetrySynchronizerImp implements TelemetrySynchronizer{ + + private HttpTelemetryMemorySender _httpHttpTelemetryMemorySender; + private TelemetryStorageConsumer _teleTelemetryStorageConsumer; + private SplitCache _splitCache; + private SegmentCache _segmentCache; + + public TelemetrySynchronizerImp(CloseableHttpClient client, URI telemetryRootEndpoint, TelemetryStorageConsumer telemetryStorageConsumer, SplitCache splitCache, + SegmentCache segmentCache) throws URISyntaxException { + _httpHttpTelemetryMemorySender = HttpTelemetryMemorySender.create(client, telemetryRootEndpoint); + _teleTelemetryStorageConsumer = telemetryStorageConsumer; + _splitCache = splitCache; + _segmentCache = segmentCache; + } + + @Override + public void synchronizeConfig(InitConfig config, long timedUntilReady, Map factoryInstances, List tags) { + _httpHttpTelemetryMemorySender.postConfig(config, timedUntilReady, factoryInstances, tags); + } + + @Override + public void synchronizeStats() throws Exception { + _httpHttpTelemetryMemorySender.postStats(generateStats()); + } + + private Stats generateStats() throws Exception { + Stats stats = new Stats(); + stats.set_lastSynchronization(_teleTelemetryStorageConsumer.getLastSynchronization()); + stats.set_methodLatencies(_teleTelemetryStorageConsumer.popLatencies()); + stats.set_methodExceptions(_teleTelemetryStorageConsumer.popExceptions()); + stats.set_httpErrors(_teleTelemetryStorageConsumer.popHTTPErrors()); + stats.set_httpLatencies(_teleTelemetryStorageConsumer.popHTTPLatencies()); + stats.set_tokenRefreshes(_teleTelemetryStorageConsumer.popTokenRefreshes()); + stats.set_authRejections(_teleTelemetryStorageConsumer.popAuthRejections()); + stats.set_impressionsQueued(_teleTelemetryStorageConsumer.getImpressionsStats(ImpressionsDataTypeEnum.IMPRESSIONS_QUEUED)); + stats.set_impressionsDeduped(_teleTelemetryStorageConsumer.getImpressionsStats(ImpressionsDataTypeEnum.IMPRESSIONS_DEDUPED)); + stats.set_impressionsDropped(_teleTelemetryStorageConsumer.getImpressionsStats(ImpressionsDataTypeEnum.IMPRESSIONS_DROPPED)); + stats.set_splitCount(_splitCache.getAll().stream().count()); //TODO + stats.set_segmentCount(1l);//TODO + stats.set_segmentKeyCount(1l);//TODO + stats.set_sessionLengthMs(_teleTelemetryStorageConsumer.getSessionLength()); + stats.set_eventsQueued(_teleTelemetryStorageConsumer.getEventStats(EventsDataRecordsEnum.EVENTS_QUEUED)); + stats.set_eventsDropped(_teleTelemetryStorageConsumer.getEventStats(EventsDataRecordsEnum.EVENTS_DROPPED)); + stats.set_streamingEvents(_teleTelemetryStorageConsumer.popStreamingEvents()); + stats.set_tags(_teleTelemetryStorageConsumer.popTags(); + return null; + } +} diff --git a/client/src/test/java/io/split/telemetry/utils/AtomicLongArrayTest.java b/client/src/test/java/io/split/telemetry/utils/AtomicLongArrayTest.java new file mode 100644 index 000000000..2dce713b0 --- /dev/null +++ b/client/src/test/java/io/split/telemetry/utils/AtomicLongArrayTest.java @@ -0,0 +1,38 @@ +package io.split.telemetry.utils; + +import org.junit.Assert; +import org.junit.Test; + +public class AtomicLongArrayTest { + + private static final int SIZE = 23; + + @Test + public void testAtomicLong() throws Exception { + AtomicLongArray atomicLongArray = new AtomicLongArray(SIZE); + Assert.assertNotNull(atomicLongArray); + } + + @Test + public void testArraySizeError() { + Exception exception = Assert.assertThrows(Exception.class, () -> { + AtomicLongArray atomicLongArray = new AtomicLongArray(0); + }); + String messageExpected = "Invalid array size"; + Assert.assertEquals(messageExpected, exception.getMessage()); + } + + @Test + public void testIncrement() throws Exception { + AtomicLongArray atomicLongArray = new AtomicLongArray(SIZE); + atomicLongArray.increment(2); + Assert.assertEquals(1, atomicLongArray.fetchAndClearAll().stream().mapToInt(Long::intValue).sum()); + } + + @Test(expected = ArrayIndexOutOfBoundsException.class) + public void testIncrementError() throws Exception { + AtomicLongArray atomicLongArray = new AtomicLongArray(SIZE); + atomicLongArray.increment(25); + } + +} \ No newline at end of file From f530d9a6d531dd7b5caec14a43f703e4829f702d Mon Sep 17 00:00:00 2001 From: Lucas Echeverz Date: Wed, 14 Apr 2021 12:20:34 -0300 Subject: [PATCH 10/81] Adding new methods to segment cache --- .../main/java/io/split/cache/SegmentCache.java | 11 ++++++++++- .../split/cache/SegmentCacheInMemoryImpl.java | 12 ++++++++++++ .../io/split/engine/segments/SegmentImp.java | 4 ++++ .../synchronizer/TelemetrySynchronizerImp.java | 8 ++++---- .../cache/SegmentCacheInMemoryImplTest.java | 17 +++++++++++++++++ 5 files changed, 47 insertions(+), 5 deletions(-) diff --git a/client/src/main/java/io/split/cache/SegmentCache.java b/client/src/main/java/io/split/cache/SegmentCache.java index 9a0d24f29..c9af2ebb7 100644 --- a/client/src/main/java/io/split/cache/SegmentCache.java +++ b/client/src/main/java/io/split/cache/SegmentCache.java @@ -1,6 +1,9 @@ package io.split.cache; +import io.split.engine.segments.SegmentImp; + import java.util.List; +import java.util.Set; /** * Memory for segments @@ -47,5 +50,11 @@ public interface SegmentCache { * return every segment * @return */ - List getAll(); + List getAll(); + + /** + * return every key + * @return + */ + Set getAllKeys(); } diff --git a/client/src/main/java/io/split/cache/SegmentCacheInMemoryImpl.java b/client/src/main/java/io/split/cache/SegmentCacheInMemoryImpl.java index 0c705c016..77a7e10d5 100644 --- a/client/src/main/java/io/split/cache/SegmentCacheInMemoryImpl.java +++ b/client/src/main/java/io/split/cache/SegmentCacheInMemoryImpl.java @@ -6,7 +6,9 @@ import org.slf4j.LoggerFactory; import java.util.List; +import java.util.Set; import java.util.concurrent.ConcurrentMap; +import java.util.stream.Collectors; /** * InMemoryCache Implementation @@ -59,4 +61,14 @@ public long getChangeNumber(String segmentName) { public void clear() { _segments.clear(); } + + @Override + public List getAll() { + return _segments.values().stream().collect(Collectors.toList()); + } + + @Override + public Set getAllKeys() { + return _segments.values().stream().flatMap(si -> si.getKeys().stream()).collect(Collectors.toSet()); + } } diff --git a/client/src/main/java/io/split/engine/segments/SegmentImp.java b/client/src/main/java/io/split/engine/segments/SegmentImp.java index 2d153d1f3..d10e3e384 100644 --- a/client/src/main/java/io/split/engine/segments/SegmentImp.java +++ b/client/src/main/java/io/split/engine/segments/SegmentImp.java @@ -41,4 +41,8 @@ public void update(List toAdd, List toRemove){ public boolean contains(String key) { return _concurrentKeySet.contains(key); } + + public Set getKeys() { + return _concurrentKeySet; + } } diff --git a/client/src/main/java/io/split/telemetry/synchronizer/TelemetrySynchronizerImp.java b/client/src/main/java/io/split/telemetry/synchronizer/TelemetrySynchronizerImp.java index 9d34d0bde..f7b003c9b 100644 --- a/client/src/main/java/io/split/telemetry/synchronizer/TelemetrySynchronizerImp.java +++ b/client/src/main/java/io/split/telemetry/synchronizer/TelemetrySynchronizerImp.java @@ -52,14 +52,14 @@ private Stats generateStats() throws Exception { stats.set_impressionsQueued(_teleTelemetryStorageConsumer.getImpressionsStats(ImpressionsDataTypeEnum.IMPRESSIONS_QUEUED)); stats.set_impressionsDeduped(_teleTelemetryStorageConsumer.getImpressionsStats(ImpressionsDataTypeEnum.IMPRESSIONS_DEDUPED)); stats.set_impressionsDropped(_teleTelemetryStorageConsumer.getImpressionsStats(ImpressionsDataTypeEnum.IMPRESSIONS_DROPPED)); - stats.set_splitCount(_splitCache.getAll().stream().count()); //TODO - stats.set_segmentCount(1l);//TODO - stats.set_segmentKeyCount(1l);//TODO + stats.set_splitCount(_splitCache.getAll().stream().count()); + stats.set_segmentCount(_segmentCache.getAll().stream().count()); + stats.set_segmentKeyCount(_segmentCache.getAllKeys().stream().count()); stats.set_sessionLengthMs(_teleTelemetryStorageConsumer.getSessionLength()); stats.set_eventsQueued(_teleTelemetryStorageConsumer.getEventStats(EventsDataRecordsEnum.EVENTS_QUEUED)); stats.set_eventsDropped(_teleTelemetryStorageConsumer.getEventStats(EventsDataRecordsEnum.EVENTS_DROPPED)); stats.set_streamingEvents(_teleTelemetryStorageConsumer.popStreamingEvents()); - stats.set_tags(_teleTelemetryStorageConsumer.popTags(); + stats.set_tags(_teleTelemetryStorageConsumer.popTags()); return null; } } diff --git a/client/src/test/java/io/split/cache/SegmentCacheInMemoryImplTest.java b/client/src/test/java/io/split/cache/SegmentCacheInMemoryImplTest.java index 95f0b54ee..76597abad 100644 --- a/client/src/test/java/io/split/cache/SegmentCacheInMemoryImplTest.java +++ b/client/src/test/java/io/split/cache/SegmentCacheInMemoryImplTest.java @@ -1,6 +1,7 @@ package io.split.cache; import junit.framework.TestCase; +import org.junit.Assert; import org.junit.Test; import java.util.ArrayList; @@ -58,4 +59,20 @@ public void testClear() { segmentCacheInMemory.clear(); assertEquals(DEFAULT_CHANGE_NUMBER, segmentCacheInMemory.getChangeNumber(SEGMENT_NAME)); } + + @Test + public void testGetAll() { + SegmentCacheInMemoryImpl segmentCacheInMemory = new SegmentCacheInMemoryImpl(); + segmentCacheInMemory.updateSegment(SEGMENT_NAME,new ArrayList<>(), new ArrayList<>()); + segmentCacheInMemory.updateSegment(FAKE_SEGMENT_NAME,new ArrayList<>(), new ArrayList<>()); + Assert.assertEquals(2, segmentCacheInMemory.getAll().stream().count()); + } + + @Test + public void testGetAllKeys() { + SegmentCacheInMemoryImpl segmentCacheInMemory = new SegmentCacheInMemoryImpl(); + segmentCacheInMemory.updateSegment(SEGMENT_NAME,Stream.of("KEY1", "KEY2").collect(Collectors.toList()), new ArrayList<>()); + segmentCacheInMemory.updateSegment(FAKE_SEGMENT_NAME,Stream.of("KEY3", "KEY2").collect(Collectors.toList()), new ArrayList<>()); + Assert.assertEquals(3, segmentCacheInMemory.getAllKeys().stream().count()); + } } \ No newline at end of file From 26ed1fa6f91d786f9d023bbdf22adfb10c89e2fb Mon Sep 17 00:00:00 2001 From: Lucas Echeverz Date: Wed, 21 Apr 2021 12:45:00 -0300 Subject: [PATCH 11/81] Almost final advances --- .../io/split/client/SplitClientConfig.java | 60 +++++- .../split/telemetry/domain/AdvanceConfig.java | 139 ------------- .../io/split/telemetry/domain/Config.java | 196 ++++++++++++++++++ .../io/split/telemetry/domain/InitConfig.java | 31 --- .../split/telemetry/domain/ManagerConfig.java | 31 --- .../java/io/split/telemetry/domain/Rates.java | 62 ++++++ .../split/telemetry/domain/TaskPeriods.java | 67 ------ .../split/telemetry/domain/URLOverrides.java | 62 ++++++ .../HttpTelemetryMemorySender.java | 24 ++- .../synchronizer/TelemetrySynchronizer.java | 4 +- .../TelemetrySynchronizerImp.java | 57 ++++- .../io/split/telemetry/utils/HttpPost.java | 42 ++++ 12 files changed, 483 insertions(+), 292 deletions(-) delete mode 100644 client/src/main/java/io/split/telemetry/domain/AdvanceConfig.java create mode 100644 client/src/main/java/io/split/telemetry/domain/Config.java delete mode 100644 client/src/main/java/io/split/telemetry/domain/InitConfig.java delete mode 100644 client/src/main/java/io/split/telemetry/domain/ManagerConfig.java create mode 100644 client/src/main/java/io/split/telemetry/domain/Rates.java delete mode 100644 client/src/main/java/io/split/telemetry/domain/TaskPeriods.java create mode 100644 client/src/main/java/io/split/telemetry/domain/URLOverrides.java create mode 100644 client/src/main/java/io/split/telemetry/utils/HttpPost.java diff --git a/client/src/main/java/io/split/client/SplitClientConfig.java b/client/src/main/java/io/split/client/SplitClientConfig.java index cafd52ebb..64e793170 100644 --- a/client/src/main/java/io/split/client/SplitClientConfig.java +++ b/client/src/main/java/io/split/client/SplitClientConfig.java @@ -17,6 +17,11 @@ public class SplitClientConfig { public static final String LOCALHOST_DEFAULT_FILE = "split.yaml"; + public static final String SDK_ENDPOINT = "https://sdk.split.io"; + public static final String EVENTS_ENDPOINT = "https://events.split.io"; + public static final String AUTH_ENDPOINT = "https://auth.split.io/api/auth"; + public static final String STREAMING_ENDPOINT = "https://streaming.split.io/sse"; + public static final String TELEMETRY_ENDPOINT = "https://telemetry.split.io/api/v1"; private final String _endpoint; private final String _eventsEndpoint; @@ -46,6 +51,8 @@ public class SplitClientConfig { private final int _streamingReconnectBackoffBase; private final String _authServiceURL; private final String _streamingServiceURL; + private final String _telemetryURL; + private final int _telemetryRefreshRate; // Proxy configs private final HttpHost _proxy; @@ -89,7 +96,9 @@ private SplitClientConfig(String endpoint, int authRetryBackoffBase, int streamingReconnectBackoffBase, String authServiceURL, - String streamingServiceURL) { + String streamingServiceURL, + String telemetryURL, + int telemetryRefreshRate) { _endpoint = endpoint; _eventsEndpoint = eventsEndpoint; _featuresRefreshRate = pollForFeatureChangesEveryNSeconds; @@ -120,6 +129,8 @@ private SplitClientConfig(String endpoint, _streamingReconnectBackoffBase = streamingReconnectBackoffBase; _authServiceURL = authServiceURL; _streamingServiceURL = streamingServiceURL; + _telemetryURL = telemetryURL; + _telemetryRefreshRate = telemetryRefreshRate; Properties props = new Properties(); try { @@ -248,11 +259,19 @@ public String streamingServiceURL() { return _streamingServiceURL; } + public String get_telemetryURL() { + return _telemetryURL; + } + + public int get_telemetryRefreshRate() { + return _telemetryRefreshRate; + } + public static final class Builder { - private String _endpoint = "https://sdk.split.io"; + private String _endpoint = SDK_ENDPOINT; private boolean _endpointSet = false; - private String _eventsEndpoint = "https://events.split.io"; + private String _eventsEndpoint = EVENTS_ENDPOINT; private boolean _eventsEndpointSet = false; private int _featuresRefreshRate = 60; private int _segmentsRefreshRate = 60; @@ -281,8 +300,10 @@ public static final class Builder { private boolean _streamingEnabled = true; private int _authRetryBackoffBase = 1; private int _streamingReconnectBackoffBase = 1; - private String _authServiceURL = "https://auth.split.io/api/auth"; - private String _streamingServiceURL = "https://streaming.split.io/sse"; + private String _authServiceURL = AUTH_ENDPOINT; + private String _streamingServiceURL = STREAMING_ENDPOINT; + private String _telemetryURl = TELEMETRY_ENDPOINT; + private int _telemetryRefreshRate = 60; public Builder() { } @@ -674,6 +695,27 @@ public Builder streamingServiceURL(String streamingServiceURL) { return this; } + /** + * Set telemetry service URL. + * @param telemetryURL + * @return + */ + public Builder telemetryURL(String telemetryURL) { + _telemetryURl = telemetryURL; + return this; + } + + /** + * How often send telemetry data + * + * @param telemetryRefreshRate + * @return this builder + */ + public Builder telemetryBuilder(int telemetryRefreshRate) { + _telemetryRefreshRate = telemetryRefreshRate; + return this; + } + public SplitClientConfig build() { if (_featuresRefreshRate < 5 ) { throw new IllegalArgumentException("featuresRefreshRate must be >= 5: " + _featuresRefreshRate); @@ -744,6 +786,10 @@ public SplitClientConfig build() { throw new IllegalArgumentException("streamingServiceURL must not be null"); } + if (_telemetryURl == null) { + throw new IllegalArgumentException("telemetryURl must not be null"); + } + return new SplitClientConfig( _endpoint, _eventsEndpoint, @@ -774,7 +820,9 @@ public SplitClientConfig build() { _authRetryBackoffBase, _streamingReconnectBackoffBase, _authServiceURL, - _streamingServiceURL); + _streamingServiceURL, + _telemetryURl, + _telemetryRefreshRate); } } } diff --git a/client/src/main/java/io/split/telemetry/domain/AdvanceConfig.java b/client/src/main/java/io/split/telemetry/domain/AdvanceConfig.java deleted file mode 100644 index 8e2ba548e..000000000 --- a/client/src/main/java/io/split/telemetry/domain/AdvanceConfig.java +++ /dev/null @@ -1,139 +0,0 @@ -package io.split.telemetry.domain; - -public class AdvanceConfig { - private int _hTTPTimeout; - private int _segmentQueueSize; - private int _segmentWorkers; - private String _sdkURL; - private String _eventsURL; - private String _telemetryServiceURL; - private long _eventsBulkSize; - private int _eventsQueueSize; - private int _impressionsQueueSize; - private long _impressionsBulkSize; - private boolean _streamingEnabled; - private String _authServiceURL; - private String _streamingServiceURL; - private boolean _splitUpdateQueueSize; - private long _segmentUpdateQueueSize; - - public int get_hTTPTimeout() { - return _hTTPTimeout; - } - - public void set_hTTPTimeout(int _hTTPTimeout) { - this._hTTPTimeout = _hTTPTimeout; - } - - public int get_segmentQueueSize() { - return _segmentQueueSize; - } - - public void set_segmentQueueSize(int _segmentQueueSize) { - this._segmentQueueSize = _segmentQueueSize; - } - - public int get_segmentWorkers() { - return _segmentWorkers; - } - - public void set_segmentWorkers(int _segmentWorkers) { - this._segmentWorkers = _segmentWorkers; - } - - public String get_sdkURL() { - return _sdkURL; - } - - public void set_sdkURL(String _sdkURL) { - this._sdkURL = _sdkURL; - } - - public String get_eventsURL() { - return _eventsURL; - } - - public void set_eventsURL(String _eventsURL) { - this._eventsURL = _eventsURL; - } - - public String get_telemetryServiceURL() { - return _telemetryServiceURL; - } - - public void set_telemetryServiceURL(String _telemetryServiceURL) { - this._telemetryServiceURL = _telemetryServiceURL; - } - - public long get_eventsBulkSize() { - return _eventsBulkSize; - } - - public void set_eventsBulkSize(long _eventsBulkSize) { - this._eventsBulkSize = _eventsBulkSize; - } - - public int get_eventsQueueSize() { - return _eventsQueueSize; - } - - public void set_eventsQueueSize(int _eventsQueueSize) { - this._eventsQueueSize = _eventsQueueSize; - } - - public int get_impressionsQueueSize() { - return _impressionsQueueSize; - } - - public void set_impressionsQueueSize(int _impressionsQueueSize) { - this._impressionsQueueSize = _impressionsQueueSize; - } - - public long get_impressionsBulkSize() { - return _impressionsBulkSize; - } - - public void set_impressionsBulkSize(long _impressionsBulkSize) { - this._impressionsBulkSize = _impressionsBulkSize; - } - - public boolean is_streamingEnabled() { - return _streamingEnabled; - } - - public void set_streamingEnabled(boolean _streamingEnabled) { - this._streamingEnabled = _streamingEnabled; - } - - public String get_authServiceURL() { - return _authServiceURL; - } - - public void set_authServiceURL(String _authServiceURL) { - this._authServiceURL = _authServiceURL; - } - - public String get_streamingServiceURL() { - return _streamingServiceURL; - } - - public void set_streamingServiceURL(String _streamingServiceURL) { - this._streamingServiceURL = _streamingServiceURL; - } - - public boolean is_splitUpdateQueueSize() { - return _splitUpdateQueueSize; - } - - public void set_splitUpdateQueueSize(boolean _splitUpdateQueueSize) { - this._splitUpdateQueueSize = _splitUpdateQueueSize; - } - - public long get_segmentUpdateQueueSize() { - return _segmentUpdateQueueSize; - } - - public void set_segmentUpdateQueueSize(long _segmentUpdateQueueSize) { - this._segmentUpdateQueueSize = _segmentUpdateQueueSize; - } -} diff --git a/client/src/main/java/io/split/telemetry/domain/Config.java b/client/src/main/java/io/split/telemetry/domain/Config.java new file mode 100644 index 000000000..1844b8e98 --- /dev/null +++ b/client/src/main/java/io/split/telemetry/domain/Config.java @@ -0,0 +1,196 @@ +package io.split.telemetry.domain; + +import com.google.gson.annotations.SerializedName; + +import java.util.List; + +public class Config { + /* package private */ static final String FIELD_OPERATION_MODE = "oM"; + /* package private */ static final String FIELD_STREAMING_ENABLED = "sE"; + /* package private */ static final String FIELD_STORAGE = "st"; + /* package private */ static final String FIELD_RATES = "rR"; + /* package private */ static final String FIELD_URL_OVERRIDES = "uO"; + /* package private */ static final String FIELD_IMPRESSIONS_QUEUE = "iQ"; + /* package private */ static final String FIELD_EVENT_QUEUE = "eQ"; + /* package private */ static final String FIELD_IMPRESSIONS_MODE = "iM"; + /* package private */ static final String FIELD_IMPRESSIONS_LISTENER = "iL"; + /* package private */ static final String FIELD_HTTP_PROXY_DETECTED = "hP"; + /* package private */ static final String FIELD_ACTIVE_FACTORIES = "aF"; + /* package private */ static final String FIELD_REDUNDANT_FACTORIES = "rF"; + /* package private */ static final String FIELD_TIME_UNTIL_READY = "tR"; + /* package private */ static final String FIELD_BUR_TIMEOUTS = "bT"; + /* package private */ static final String FIELD_NON_READY_USAGES = "nR"; + /* package private */ static final String FIELD_INTEGRATIONS = "i"; + /* package private */ static final String FIELD__TAGS = "t"; + + @SerializedName(FIELD_OPERATION_MODE) + private int _operationMode; + @SerializedName(FIELD_STREAMING_ENABLED) + private boolean _streamingEnabled; + @SerializedName(FIELD_STORAGE) + private String _storage; + @SerializedName(FIELD_RATES) + private Rates _rates; + @SerializedName(FIELD_URL_OVERRIDES) + private URLOverrides _urlOverrides; + @SerializedName(FIELD_IMPRESSIONS_QUEUE) + private long _impressionsQueueSize; + @SerializedName(FIELD_EVENT_QUEUE) + private long _eventsQueueSize; + @SerializedName(FIELD_IMPRESSIONS_MODE) + private int _impressionsMode; + @SerializedName(FIELD_IMPRESSIONS_LISTENER) + private boolean _impressionsListenerEnabled; + @SerializedName(FIELD_HTTP_PROXY_DETECTED) + private boolean _httpProxyDetected; + @SerializedName(FIELD_ACTIVE_FACTORIES) + private long _activeFactories; + @SerializedName(FIELD_REDUNDANT_FACTORIES) + private long _redundantFactories; + @SerializedName(FIELD_TIME_UNTIL_READY) + private long _timeUntilReady; + @SerializedName(FIELD_BUR_TIMEOUTS) + private long _burTimeouts; + @SerializedName(FIELD_NON_READY_USAGES) + private long _nonReadyUsages; + @SerializedName(FIELD_INTEGRATIONS) + private List _integrations; + @SerializedName(FIELD__TAGS) + private List _tags; + + public int get_operationMode() { + return _operationMode; + } + + public void set_operationMode(int _operationMode) { + this._operationMode = _operationMode; + } + + public boolean is_streamingEnabled() { + return _streamingEnabled; + } + + public void set_streamingEnabled(boolean _streamingEnabled) { + this._streamingEnabled = _streamingEnabled; + } + + public String get_storage() { + return _storage; + } + + public void set_storage(String _storage) { + this._storage = _storage; + } + + public Rates get_rates() { + return _rates; + } + + public void set_rates(Rates _rates) { + this._rates = _rates; + } + + public URLOverrides get_urlOverrides() { + return _urlOverrides; + } + + public void set_urlOverrides(URLOverrides _urlOverrides) { + this._urlOverrides = _urlOverrides; + } + + public long get_impressionsQueueSize() { + return _impressionsQueueSize; + } + + public void set_impressionsQueueSize(long _impressionsQueueSize) { + this._impressionsQueueSize = _impressionsQueueSize; + } + + public long get_eventsQueueSize() { + return _eventsQueueSize; + } + + public void set_eventsQueueSize(long _eventsQueueSize) { + this._eventsQueueSize = _eventsQueueSize; + } + + public int get_impressionsMode() { + return _impressionsMode; + } + + public void set_impressionsMode(int _impressionsMode) { + this._impressionsMode = _impressionsMode; + } + + public boolean is_impressionsListenerEnabled() { + return _impressionsListenerEnabled; + } + + public void set_impressionsListenerEnabled(boolean _impressionsListenerEnabled) { + this._impressionsListenerEnabled = _impressionsListenerEnabled; + } + + public boolean is_httpProxyDetected() { + return _httpProxyDetected; + } + + public void set_httpProxyDetected(boolean _httpProxyDetected) { + this._httpProxyDetected = _httpProxyDetected; + } + + public long get_activeFactories() { + return _activeFactories; + } + + public void set_activeFactories(long _activeFactories) { + this._activeFactories = _activeFactories; + } + + public long get_redundantFactories() { + return _redundantFactories; + } + + public void set_redundantFactories(long _redundantFactories) { + this._redundantFactories = _redundantFactories; + } + + public long get_timeUntilReady() { + return _timeUntilReady; + } + + public void set_timeUntilReady(long _timeUntilReady) { + this._timeUntilReady = _timeUntilReady; + } + + public long get_burTimeouts() { + return _burTimeouts; + } + + public void set_burTimeouts(long _burTimeouts) { + this._burTimeouts = _burTimeouts; + } + + public long get_nonReadyUsages() { + return _nonReadyUsages; + } + + public void set_nonReadyUsages(long _nonReadyUsages) { + this._nonReadyUsages = _nonReadyUsages; + } + + public List get_integrations() { + return _integrations; + } + + public void set_integrations(List _integrations) { + this._integrations = _integrations; + } + + public List get_tags() { + return _tags; + } + + public void set_tags(List _tags) { + this._tags = _tags; + } +} diff --git a/client/src/main/java/io/split/telemetry/domain/InitConfig.java b/client/src/main/java/io/split/telemetry/domain/InitConfig.java deleted file mode 100644 index d2cb6569f..000000000 --- a/client/src/main/java/io/split/telemetry/domain/InitConfig.java +++ /dev/null @@ -1,31 +0,0 @@ -package io.split.telemetry.domain; - -public class InitConfig { - private AdvanceConfig _advanceConfig; - private TaskPeriods _taskPeriods; - private ManagerConfig _managerConfig; - - public AdvanceConfig get_advanceConfig() { - return _advanceConfig; - } - - public void set_advanceConfig(AdvanceConfig _advanceConfig) { - this._advanceConfig = _advanceConfig; - } - - public TaskPeriods get_taskPeriods() { - return _taskPeriods; - } - - public void set_taskPeriods(TaskPeriods _taskPeriods) { - this._taskPeriods = _taskPeriods; - } - - public ManagerConfig get_managerConfig() { - return _managerConfig; - } - - public void set_managerConfig(ManagerConfig _managerConfig) { - this._managerConfig = _managerConfig; - } -} diff --git a/client/src/main/java/io/split/telemetry/domain/ManagerConfig.java b/client/src/main/java/io/split/telemetry/domain/ManagerConfig.java deleted file mode 100644 index da7e00f77..000000000 --- a/client/src/main/java/io/split/telemetry/domain/ManagerConfig.java +++ /dev/null @@ -1,31 +0,0 @@ -package io.split.telemetry.domain; - -public class ManagerConfig { - private String _operationMode; - private String _impressionsMode; - private boolean _listenerEnabled; - - public String get_operationMode() { - return _operationMode; - } - - public void set_operationMode(String _operationMode) { - this._operationMode = _operationMode; - } - - public String get_impressionsMode() { - return _impressionsMode; - } - - public void set_impressionsMode(String _impressionsMode) { - this._impressionsMode = _impressionsMode; - } - - public boolean is_listenerEnabled() { - return _listenerEnabled; - } - - public void set_listenerEnabled(boolean _listenerEnabled) { - this._listenerEnabled = _listenerEnabled; - } -} diff --git a/client/src/main/java/io/split/telemetry/domain/Rates.java b/client/src/main/java/io/split/telemetry/domain/Rates.java new file mode 100644 index 000000000..e80d26079 --- /dev/null +++ b/client/src/main/java/io/split/telemetry/domain/Rates.java @@ -0,0 +1,62 @@ +package io.split.telemetry.domain; + +import com.google.gson.annotations.SerializedName; + +public class Rates { + /* package private */ static final String FIELD_SPLITS = "sp"; + /* package private */ static final String FIELD_SEGMENTS = "se"; + /* package private */ static final String FIELD_IMPRESSIONS = "im"; + /* package private */ static final String FIELD_EVENTS = "ev"; + /* package private */ static final String FIELD_TELEMETRY = "te"; + + @SerializedName(FIELD_SPLITS) + private long _splits; + @SerializedName(FIELD_SEGMENTS) + private long _segments; + @SerializedName(FIELD_IMPRESSIONS) + private long _impressions; + @SerializedName(FIELD_EVENTS) + private long _events; + @SerializedName(FIELD_TELEMETRY) + private long _telemetry; + + public long get_splits() { + return _splits; + } + + public void set_splits(long _splits) { + this._splits = _splits; + } + + public long get_segments() { + return _segments; + } + + public void set_segments(long _segments) { + this._segments = _segments; + } + + public long get_impressions() { + return _impressions; + } + + public void set_impressions(long _impressions) { + this._impressions = _impressions; + } + + public long get_events() { + return _events; + } + + public void set_events(long _events) { + this._events = _events; + } + + public long get_telemetry() { + return _telemetry; + } + + public void set_telemetry(long _telemetry) { + this._telemetry = _telemetry; + } +} diff --git a/client/src/main/java/io/split/telemetry/domain/TaskPeriods.java b/client/src/main/java/io/split/telemetry/domain/TaskPeriods.java deleted file mode 100644 index b4f299768..000000000 --- a/client/src/main/java/io/split/telemetry/domain/TaskPeriods.java +++ /dev/null @@ -1,67 +0,0 @@ -package io.split.telemetry.domain; - -public class TaskPeriods { - private int _splitSync; - private int _segmentSync; - private int _impressionSync; - private int _gaugeSync; - private int _counterSync; - private int _latencySync; - private int _eventsSync; - - public int get_splitSync() { - return _splitSync; - } - - public void set_splitSync(int _splitSync) { - this._splitSync = _splitSync; - } - - public int get_segmentSync() { - return _segmentSync; - } - - public void set_segmentSync(int _segmentSync) { - this._segmentSync = _segmentSync; - } - - public int get_impressionSync() { - return _impressionSync; - } - - public void set_impressionSync(int _impressionSync) { - this._impressionSync = _impressionSync; - } - - public int get_gaugeSync() { - return _gaugeSync; - } - - public void set_gaugeSync(int _gaugeSync) { - this._gaugeSync = _gaugeSync; - } - - public int get_counterSync() { - return _counterSync; - } - - public void set_counterSync(int _counterSync) { - this._counterSync = _counterSync; - } - - public int get_latencySync() { - return _latencySync; - } - - public void set_latencySync(int _latencySync) { - this._latencySync = _latencySync; - } - - public int get_eventsSync() { - return _eventsSync; - } - - public void set_eventsSync(int _eventsSync) { - this._eventsSync = _eventsSync; - } -} diff --git a/client/src/main/java/io/split/telemetry/domain/URLOverrides.java b/client/src/main/java/io/split/telemetry/domain/URLOverrides.java new file mode 100644 index 000000000..5813f1d6c --- /dev/null +++ b/client/src/main/java/io/split/telemetry/domain/URLOverrides.java @@ -0,0 +1,62 @@ +package io.split.telemetry.domain; + +import com.google.gson.annotations.SerializedName; + +public class URLOverrides { + /* package private */ static final String FIELD_SDK = "s"; + /* package private */ static final String FIELD_EVENTS = "e"; + /* package private */ static final String FIELD_AUTH = "a"; + /* package private */ static final String FIELD_STREAM = "st"; + /* package private */ static final String FIELD_TELEMETRY = "t"; + + @SerializedName(FIELD_SDK) + private boolean _sdk; + @SerializedName(FIELD_EVENTS) + private boolean _events; + @SerializedName(FIELD_AUTH) + private boolean _auth; + @SerializedName(FIELD_STREAM) + private boolean _stream; + @SerializedName(FIELD_TELEMETRY) + private boolean _telemetry; + + public boolean is_sdk() { + return _sdk; + } + + public void set_sdk(boolean _sdk) { + this._sdk = _sdk; + } + + public boolean is_events() { + return _events; + } + + public void set_events(boolean _events) { + this._events = _events; + } + + public boolean is_auth() { + return _auth; + } + + public void set_auth(boolean _auth) { + this._auth = _auth; + } + + public boolean is_stream() { + return _stream; + } + + public void set_stream(boolean _stream) { + this._stream = _stream; + } + + public boolean is_telemetry() { + return _telemetry; + } + + public void set_telemetry(boolean _telemetry) { + this._telemetry = _telemetry; + } +} diff --git a/client/src/main/java/io/split/telemetry/synchronizer/HttpTelemetryMemorySender.java b/client/src/main/java/io/split/telemetry/synchronizer/HttpTelemetryMemorySender.java index f37fc0422..2316f8daa 100644 --- a/client/src/main/java/io/split/telemetry/synchronizer/HttpTelemetryMemorySender.java +++ b/client/src/main/java/io/split/telemetry/synchronizer/HttpTelemetryMemorySender.java @@ -1,26 +1,27 @@ package io.split.telemetry.synchronizer; +import io.split.client.SplitClientConfig; import io.split.client.impressions.HttpImpressionsSender; import io.split.client.utils.Utils; -import io.split.telemetry.domain.InitConfig; +import io.split.telemetry.domain.Config; import io.split.telemetry.domain.Stats; +import io.split.telemetry.utils.HttpPost; import org.apache.hc.client5.http.impl.classic.CloseableHttpClient; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import java.net.URI; import java.net.URISyntaxException; -import java.util.List; -import java.util.Map; -public class HttpTelemetryMemorySender{ +public class HttpTelemetryMemorySender extends HttpPost { - private static final String CONFIG_ENDPOINT_PATH = "api/telemetry/config"; - private static final String STATS_ENDPOINT_PATH = "api/telemetry/stats"; + private static final String CONFIG_ENDPOINT_PATH = "metrics/config"; + private static final String STATS_ENDPOINT_PATH = "metrics/usage"; + private static final String CONFIG_METRICS = "Config metrics "; + private static final String STATS_METRICS = "Stats metrics "; private static final Logger _logger = LoggerFactory.getLogger(HttpImpressionsSender.class); - private final CloseableHttpClient _client; private final URI _impressionConfigTarget; private final URI _impressionStatsTarget; @@ -32,16 +33,17 @@ public static HttpTelemetryMemorySender create(CloseableHttpClient client, URI t } private HttpTelemetryMemorySender(CloseableHttpClient client, URI impressionConfigTarget, URI impressionStatsTarget) { - _client = client; + super(client); _impressionConfigTarget = impressionConfigTarget; _impressionStatsTarget = impressionStatsTarget; } - public void postConfig(InitConfig config, long timedUntilReady, Map factoryInstances, List tags) { - + public void postConfig(Config config) { + post(_impressionConfigTarget, config, CONFIG_METRICS); } public void postStats(Stats stats) { - + post(_impressionStatsTarget, stats, STATS_METRICS); } + } diff --git a/client/src/main/java/io/split/telemetry/synchronizer/TelemetrySynchronizer.java b/client/src/main/java/io/split/telemetry/synchronizer/TelemetrySynchronizer.java index 59b454e18..859df44c8 100644 --- a/client/src/main/java/io/split/telemetry/synchronizer/TelemetrySynchronizer.java +++ b/client/src/main/java/io/split/telemetry/synchronizer/TelemetrySynchronizer.java @@ -1,11 +1,11 @@ package io.split.telemetry.synchronizer; -import io.split.telemetry.domain.InitConfig; +import io.split.client.SplitClientConfig; import java.util.List; import java.util.Map; public interface TelemetrySynchronizer { - void synchronizeConfig(InitConfig config, long timedUntilReady, Map factoryInstances, List tags); + void synchronizeConfig(SplitClientConfig config, long timeUntilReady, Map factoryInstances, List tags); void synchronizeStats() throws Exception; } diff --git a/client/src/main/java/io/split/telemetry/synchronizer/TelemetrySynchronizerImp.java b/client/src/main/java/io/split/telemetry/synchronizer/TelemetrySynchronizerImp.java index f7b003c9b..8de335df8 100644 --- a/client/src/main/java/io/split/telemetry/synchronizer/TelemetrySynchronizerImp.java +++ b/client/src/main/java/io/split/telemetry/synchronizer/TelemetrySynchronizerImp.java @@ -2,9 +2,11 @@ import io.split.cache.SegmentCache; import io.split.cache.SplitCache; -import io.split.client.dtos.Split; -import io.split.telemetry.domain.InitConfig; +import io.split.client.SplitClientConfig; +import io.split.telemetry.domain.Config; +import io.split.telemetry.domain.Rates; import io.split.telemetry.domain.Stats; +import io.split.telemetry.domain.URLOverrides; import io.split.telemetry.domain.enums.EventsDataRecordsEnum; import io.split.telemetry.domain.enums.ImpressionsDataTypeEnum; import io.split.telemetry.storage.TelemetryStorageConsumer; @@ -31,8 +33,8 @@ public TelemetrySynchronizerImp(CloseableHttpClient client, URI telemetryRootEnd } @Override - public void synchronizeConfig(InitConfig config, long timedUntilReady, Map factoryInstances, List tags) { - _httpHttpTelemetryMemorySender.postConfig(config, timedUntilReady, factoryInstances, tags); + public void synchronizeConfig(SplitClientConfig config, long timeUntilReady, Map factoryInstances, List tags) { + _httpHttpTelemetryMemorySender.postConfig(generateConfig(config, timeUntilReady, factoryInstances, tags)); } @Override @@ -60,6 +62,51 @@ private Stats generateStats() throws Exception { stats.set_eventsDropped(_teleTelemetryStorageConsumer.getEventStats(EventsDataRecordsEnum.EVENTS_DROPPED)); stats.set_streamingEvents(_teleTelemetryStorageConsumer.popStreamingEvents()); stats.set_tags(_teleTelemetryStorageConsumer.popTags()); - return null; + return stats; + } + + private Config generateConfig(SplitClientConfig splitClientConfig, long timeUntilReady, Map factoryInstances, List tags) { + Config config = new Config(); + Rates rates = new Rates(); + URLOverrides urlOverrides = new URLOverrides(); + + rates.set_telemetry(splitClientConfig.get_telemetryRefreshRate()); + rates.set_events(splitClientConfig.eventFlushIntervalInMillis()); + rates.set_impressions(splitClientConfig.impressionsRefreshRate()); + rates.set_segments(splitClientConfig.segmentsRefreshRate()); + rates.set_splits(splitClientConfig.featuresRefreshRate()); + + urlOverrides.set_auth(SplitClientConfig.AUTH_ENDPOINT.equals(splitClientConfig.authServiceURL())); + urlOverrides.set_stream(SplitClientConfig.STREAMING_ENDPOINT.equals(splitClientConfig.streamingServiceURL())); + urlOverrides.set_sdk(SplitClientConfig.SDK_ENDPOINT.equals(splitClientConfig.endpoint())); + urlOverrides.set_events(SplitClientConfig.EVENTS_ENDPOINT.equals(splitClientConfig.eventsEndpoint())); + urlOverrides.set_telemetry(SplitClientConfig.TELEMETRY_ENDPOINT.equals(splitClientConfig.get_telemetryURL())); + + config.set_burTimeouts(_teleTelemetryStorageConsumer.getBURTimeouts()); + config.set_nonReadyUsages(_teleTelemetryStorageConsumer.getNonReadyUsages()); + config.set_impressionsListenerEnabled(splitClientConfig.); + config.set_httpProxyDetected(splitClientConfig.proxy() != null); + config.set_impressionsMode(splitClientConfig.impressionsMode()); + config.set_integrations(splitClientConfig); + config.set_operationMode(0); //Standalone -> 0 + config.set_storage("memory"); + config.set_impressionsQueueSize(splitClientConfig.impressionsQueueSize()); + config.set_redundantFactories(getRedundantFactories(factoryInstances)); + config.set_eventsQueueSize(splitClientConfig.eventsQueueSize()); + config.set_tags(tags); + config.set_activeFactories(factoryInstances.size()); + config.set_timeUntilReady(timeUntilReady); + config.set_rates(rates); + config.set_urlOverrides(urlOverrides); + config.set_streamingEnabled(splitClientConfig.streamingEnabled()); + return config; + } + + private long getRedundantFactories(Map factoryInstances) { + long count = 0; + for(Long l :factoryInstances.values()) { + count = count + l - 1l; + } + return count; } } diff --git a/client/src/main/java/io/split/telemetry/utils/HttpPost.java b/client/src/main/java/io/split/telemetry/utils/HttpPost.java new file mode 100644 index 000000000..363cb2f1c --- /dev/null +++ b/client/src/main/java/io/split/telemetry/utils/HttpPost.java @@ -0,0 +1,42 @@ +package io.split.telemetry.utils; + +import io.split.client.utils.Utils; +import org.apache.hc.client5.http.impl.classic.CloseableHttpClient; +import org.apache.hc.client5.http.impl.classic.CloseableHttpResponse; +import org.apache.hc.core5.http.HttpEntity; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.net.URI; + +public class HttpPost { + private static final Logger _logger = LoggerFactory.getLogger(HttpPost.class); + private CloseableHttpClient _client; + + public HttpPost(CloseableHttpClient client) { + _client = client; + } + + public void post(URI uri, Object object, String posted) { + CloseableHttpResponse response = null; + + try { + HttpEntity entity = Utils.toJsonEntity(object); + org.apache.hc.client5.http.classic.methods.HttpPost request = new org.apache.hc.client5.http.classic.methods.HttpPost(uri); + request.setEntity(entity); + + response = _client.execute(request); + + int status = response.getCode(); + + if (status < 200 || status >= 300) { + _logger.warn("Response status was: " + status); + } + + } catch (Throwable t) { + _logger.warn("Exception when posting " + posted + object, t); + } finally { + Utils.forceClose(response); + } + } +} From 22cb0f0bb5c3a85215946fc261c835a571f2b01b Mon Sep 17 00:00:00 2001 From: Lucas Echeverz Date: Thu, 22 Apr 2021 17:48:57 -0300 Subject: [PATCH 12/81] Adding delay on streaming retry --- .../io/split/client/SplitClientConfig.java | 25 +++++++++++++++++-- .../io/split/client/SplitFactoryImpl.java | 2 +- .../split/engine/common/SyncManagerImp.java | 5 ++-- .../split/engine/common/SynchronizerImp.java | 19 +++++++++++++- .../split/engine/common/SynchronizerTest.java | 2 +- 5 files changed, 46 insertions(+), 7 deletions(-) diff --git a/client/src/main/java/io/split/client/SplitClientConfig.java b/client/src/main/java/io/split/client/SplitClientConfig.java index cafd52ebb..466e0f363 100644 --- a/client/src/main/java/io/split/client/SplitClientConfig.java +++ b/client/src/main/java/io/split/client/SplitClientConfig.java @@ -46,6 +46,7 @@ public class SplitClientConfig { private final int _streamingReconnectBackoffBase; private final String _authServiceURL; private final String _streamingServiceURL; + private final int _streamingRetryDelay; // Proxy configs private final HttpHost _proxy; @@ -89,7 +90,8 @@ private SplitClientConfig(String endpoint, int authRetryBackoffBase, int streamingReconnectBackoffBase, String authServiceURL, - String streamingServiceURL) { + String streamingServiceURL, + int streamingRetryDelay) { _endpoint = endpoint; _eventsEndpoint = eventsEndpoint; _featuresRefreshRate = pollForFeatureChangesEveryNSeconds; @@ -120,6 +122,7 @@ private SplitClientConfig(String endpoint, _streamingReconnectBackoffBase = streamingReconnectBackoffBase; _authServiceURL = authServiceURL; _streamingServiceURL = streamingServiceURL; + _streamingRetryDelay = streamingRetryDelay; Properties props = new Properties(); try { @@ -248,6 +251,8 @@ public String streamingServiceURL() { return _streamingServiceURL; } + public int streamingRetryDelay() {return _streamingRetryDelay;} + public static final class Builder { private String _endpoint = "https://sdk.split.io"; @@ -283,6 +288,7 @@ public static final class Builder { private int _streamingReconnectBackoffBase = 1; private String _authServiceURL = "https://auth.split.io/api/auth"; private String _streamingServiceURL = "https://streaming.split.io/sse"; + private int _streamingRetryDelay = 50; public Builder() { } @@ -674,6 +680,16 @@ public Builder streamingServiceURL(String streamingServiceURL) { return this; } + /** + * Set Streaming retry delay. + * @param streamingRetryDelay + * @return + */ + public Builder streamingRetryDelay(int streamingRetryDelay) { + _streamingRetryDelay = streamingRetryDelay; + return this; + } + public SplitClientConfig build() { if (_featuresRefreshRate < 5 ) { throw new IllegalArgumentException("featuresRefreshRate must be >= 5: " + _featuresRefreshRate); @@ -744,6 +760,10 @@ public SplitClientConfig build() { throw new IllegalArgumentException("streamingServiceURL must not be null"); } + if(_streamingRetryDelay <= 0) { + throw new IllegalStateException("streamingRetryDelay must be > 0"); + } + return new SplitClientConfig( _endpoint, _eventsEndpoint, @@ -774,7 +794,8 @@ public SplitClientConfig build() { _authRetryBackoffBase, _streamingReconnectBackoffBase, _authServiceURL, - _streamingServiceURL); + _streamingServiceURL, + _streamingRetryDelay); } } } diff --git a/client/src/main/java/io/split/client/SplitFactoryImpl.java b/client/src/main/java/io/split/client/SplitFactoryImpl.java index 8b576d0dd..a2d68ba64 100644 --- a/client/src/main/java/io/split/client/SplitFactoryImpl.java +++ b/client/src/main/java/io/split/client/SplitFactoryImpl.java @@ -141,7 +141,7 @@ public SplitFactoryImpl(String apiToken, SplitClientConfig config) throws URISyn _eventClient = EventClientImpl.create(_httpclient, _eventsRootTarget, config.eventsQueueSize(), config.eventFlushIntervalInMillis(), config.waitBeforeShutdown()); // SyncManager - _syncManager = SyncManagerImp.build(config.streamingEnabled(), _splitSynchronizationTask, _splitFetcher, _segmentSynchronizationTaskImp, _splitCache, config.authServiceURL(), _httpclient, config.streamingServiceURL(), config.authRetryBackoffBase(), buildSSEdHttpClient(config), _segmentCache); + _syncManager = SyncManagerImp.build(config.streamingEnabled(), _splitSynchronizationTask, _splitFetcher, _segmentSynchronizationTaskImp, _splitCache, config.authServiceURL(), _httpclient, config.streamingServiceURL(), config.authRetryBackoffBase(), buildSSEdHttpClient(config), _segmentCache, config.streamingRetryDelay()); _syncManager.start(); // Evaluator diff --git a/client/src/main/java/io/split/engine/common/SyncManagerImp.java b/client/src/main/java/io/split/engine/common/SyncManagerImp.java index 2d0fe2fa9..69e6c75ae 100644 --- a/client/src/main/java/io/split/engine/common/SyncManagerImp.java +++ b/client/src/main/java/io/split/engine/common/SyncManagerImp.java @@ -59,9 +59,10 @@ public static SyncManagerImp build(boolean streamingEnabledConfig, String streamingServiceUrl, int authRetryBackOffBase, CloseableHttpClient sseHttpClient, - SegmentCache segmentCache) { + SegmentCache segmentCache, + int streamingRetryDelay) { LinkedBlockingQueue pushMessages = new LinkedBlockingQueue<>(); - Synchronizer synchronizer = new SynchronizerImp(splitSynchronizationTask, splitFetcher, segmentSynchronizationTaskImp, splitCache, segmentCache); + Synchronizer synchronizer = new SynchronizerImp(splitSynchronizationTask, splitFetcher, segmentSynchronizationTaskImp, splitCache, segmentCache, streamingRetryDelay); PushManager pushManager = PushManagerImp.build(synchronizer, streamingServiceUrl, authUrl, httpClient, pushMessages, sseHttpClient); return new SyncManagerImp(streamingEnabledConfig, synchronizer, pushManager, pushMessages, authRetryBackOffBase); } diff --git a/client/src/main/java/io/split/engine/common/SynchronizerImp.java b/client/src/main/java/io/split/engine/common/SynchronizerImp.java index 0d22ff9f2..8dba84128 100644 --- a/client/src/main/java/io/split/engine/common/SynchronizerImp.java +++ b/client/src/main/java/io/split/engine/common/SynchronizerImp.java @@ -27,17 +27,20 @@ public class SynchronizerImp implements Synchronizer { private final ScheduledExecutorService _syncAllScheduledExecutorService; private final SplitCache _splitCache; private final SegmentCache _segmentCache; + private final int _streamingRetryDelay; public SynchronizerImp(SplitSynchronizationTask splitSynchronizationTask, SplitFetcher splitFetcher, SegmentSynchronizationTask segmentSynchronizationTaskImp, SplitCache splitCache, - SegmentCache segmentCache) { + SegmentCache segmentCache, + int streamingRetryDelay) { _splitSynchronizationTask = checkNotNull(splitSynchronizationTask); _splitFetcher = checkNotNull(splitFetcher); _segmentSynchronizationTaskImp = checkNotNull(segmentSynchronizationTaskImp); _splitCache = checkNotNull(splitCache); _segmentCache = checkNotNull(segmentCache); + _streamingRetryDelay = checkNotNull(streamingRetryDelay); ThreadFactory splitsThreadFactory = new ThreadFactoryBuilder() .setDaemon(true) @@ -72,9 +75,13 @@ public void stopPeriodicFetching() { public void refreshSplits(long targetChangeNumber) { int retries = 1; while(targetChangeNumber > _splitCache.getChangeNumber() && retries <= RETRIES_NUMBER) { + checkNumberAttepmt(retries); _splitFetcher.forceRefresh(true); retries++; } + if(retries > 1) { + _log.debug("Refresh completed in %s attempts.", retries); + } } @Override @@ -100,4 +107,14 @@ public void refreshSegment(String segmentName, long changeNumber) { retries++; } } + + private void checkNumberAttepmt(int retry) { + if(retry > 1) { + try { + Thread.sleep(_streamingRetryDelay); + } catch (InterruptedException e) { + e.printStackTrace(); + } + } + } } diff --git a/client/src/test/java/io/split/engine/common/SynchronizerTest.java b/client/src/test/java/io/split/engine/common/SynchronizerTest.java index 5bff9d000..9a90cb5ae 100644 --- a/client/src/test/java/io/split/engine/common/SynchronizerTest.java +++ b/client/src/test/java/io/split/engine/common/SynchronizerTest.java @@ -26,7 +26,7 @@ public void beforeMethod() { _splitCache = Mockito.mock(SplitCache.class); _segmentCache = Mockito.mock(SegmentCache.class); - _synchronizer = new SynchronizerImp(_refreshableSplitFetcherTask, _splitFetcher, _segmentFetcher, _splitCache, _segmentCache); + _synchronizer = new SynchronizerImp(_refreshableSplitFetcherTask, _splitFetcher, _segmentFetcher, _splitCache, _segmentCache, 50); } @Test From 66de222b96c1d897d64eac8600290e1d294f8fe6 Mon Sep 17 00:00:00 2001 From: Lucas Echeverz Date: Thu, 22 Apr 2021 18:26:01 -0300 Subject: [PATCH 13/81] Pr comments --- .../io/split/client/SplitClientConfig.java | 20 +++++----- .../split/engine/common/SynchronizerImp.java | 39 +++++++++---------- 2 files changed, 29 insertions(+), 30 deletions(-) diff --git a/client/src/main/java/io/split/client/SplitClientConfig.java b/client/src/main/java/io/split/client/SplitClientConfig.java index 466e0f363..d86cfa4ed 100644 --- a/client/src/main/java/io/split/client/SplitClientConfig.java +++ b/client/src/main/java/io/split/client/SplitClientConfig.java @@ -46,7 +46,7 @@ public class SplitClientConfig { private final int _streamingReconnectBackoffBase; private final String _authServiceURL; private final String _streamingServiceURL; - private final int _streamingRetryDelay; + private final int _onDemandFetchRetryDelayMs; // Proxy configs private final HttpHost _proxy; @@ -91,7 +91,7 @@ private SplitClientConfig(String endpoint, int streamingReconnectBackoffBase, String authServiceURL, String streamingServiceURL, - int streamingRetryDelay) { + int onDemandFetchRetryDelayMs) { _endpoint = endpoint; _eventsEndpoint = eventsEndpoint; _featuresRefreshRate = pollForFeatureChangesEveryNSeconds; @@ -122,7 +122,7 @@ private SplitClientConfig(String endpoint, _streamingReconnectBackoffBase = streamingReconnectBackoffBase; _authServiceURL = authServiceURL; _streamingServiceURL = streamingServiceURL; - _streamingRetryDelay = streamingRetryDelay; + _onDemandFetchRetryDelayMs = onDemandFetchRetryDelayMs; Properties props = new Properties(); try { @@ -251,7 +251,7 @@ public String streamingServiceURL() { return _streamingServiceURL; } - public int streamingRetryDelay() {return _streamingRetryDelay;} + public int streamingRetryDelay() {return _onDemandFetchRetryDelayMs;} public static final class Builder { @@ -288,7 +288,7 @@ public static final class Builder { private int _streamingReconnectBackoffBase = 1; private String _authServiceURL = "https://auth.split.io/api/auth"; private String _streamingServiceURL = "https://streaming.split.io/sse"; - private int _streamingRetryDelay = 50; + private int _onDemandFetchRetryDelayMs = 50; public Builder() { } @@ -682,11 +682,11 @@ public Builder streamingServiceURL(String streamingServiceURL) { /** * Set Streaming retry delay. - * @param streamingRetryDelay + * @param onDemandFetchRetryDelayMs * @return */ - public Builder streamingRetryDelay(int streamingRetryDelay) { - _streamingRetryDelay = streamingRetryDelay; + public Builder streamingRetryDelay(int onDemandFetchRetryDelayMs) { + _onDemandFetchRetryDelayMs = onDemandFetchRetryDelayMs; return this; } @@ -760,7 +760,7 @@ public SplitClientConfig build() { throw new IllegalArgumentException("streamingServiceURL must not be null"); } - if(_streamingRetryDelay <= 0) { + if(_onDemandFetchRetryDelayMs <= 0) { throw new IllegalStateException("streamingRetryDelay must be > 0"); } @@ -795,7 +795,7 @@ public SplitClientConfig build() { _streamingReconnectBackoffBase, _authServiceURL, _streamingServiceURL, - _streamingRetryDelay); + _onDemandFetchRetryDelayMs); } } } diff --git a/client/src/main/java/io/split/engine/common/SynchronizerImp.java b/client/src/main/java/io/split/engine/common/SynchronizerImp.java index 8dba84128..3fa0b1037 100644 --- a/client/src/main/java/io/split/engine/common/SynchronizerImp.java +++ b/client/src/main/java/io/split/engine/common/SynchronizerImp.java @@ -27,20 +27,20 @@ public class SynchronizerImp implements Synchronizer { private final ScheduledExecutorService _syncAllScheduledExecutorService; private final SplitCache _splitCache; private final SegmentCache _segmentCache; - private final int _streamingRetryDelay; + private final int _onDemandFetchRetryDelayMs; public SynchronizerImp(SplitSynchronizationTask splitSynchronizationTask, SplitFetcher splitFetcher, SegmentSynchronizationTask segmentSynchronizationTaskImp, SplitCache splitCache, SegmentCache segmentCache, - int streamingRetryDelay) { + int onDemandFetchRetryDelayMs) { _splitSynchronizationTask = checkNotNull(splitSynchronizationTask); _splitFetcher = checkNotNull(splitFetcher); _segmentSynchronizationTaskImp = checkNotNull(segmentSynchronizationTaskImp); _splitCache = checkNotNull(splitCache); _segmentCache = checkNotNull(segmentCache); - _streamingRetryDelay = checkNotNull(streamingRetryDelay); + _onDemandFetchRetryDelayMs = checkNotNull(onDemandFetchRetryDelayMs); ThreadFactory splitsThreadFactory = new ThreadFactoryBuilder() .setDaemon(true) @@ -73,14 +73,23 @@ public void stopPeriodicFetching() { @Override public void refreshSplits(long targetChangeNumber) { - int retries = 1; - while(targetChangeNumber > _splitCache.getChangeNumber() && retries <= RETRIES_NUMBER) { - checkNumberAttepmt(retries); + int retries = RETRIES_NUMBER; + while(true) { + retries--; _splitFetcher.forceRefresh(true); - retries++; - } - if(retries > 1) { - _log.debug("Refresh completed in %s attempts.", retries); + if (targetChangeNumber <= _splitCache.getChangeNumber()) { + _log.debug("Refresh completed in %s attempts.", RETRIES_NUMBER - retries); + return; + } else if (retries <= 0) { + _log.warn("No changes fetched after %s attempts.", RETRIES_NUMBER); + return; + } + try { + Thread.sleep(_onDemandFetchRetryDelayMs); + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + _log.debug("Error trying to sleep current Thread."); + } } } @@ -107,14 +116,4 @@ public void refreshSegment(String segmentName, long changeNumber) { retries++; } } - - private void checkNumberAttepmt(int retry) { - if(retry > 1) { - try { - Thread.sleep(_streamingRetryDelay); - } catch (InterruptedException e) { - e.printStackTrace(); - } - } - } } From 1438f786cfaab91963a29453eb7a487a040fbeec Mon Sep 17 00:00:00 2001 From: Lucas Echeverz Date: Thu, 22 Apr 2021 18:36:17 -0300 Subject: [PATCH 14/81] PR Fix --- .../src/main/java/io/split/engine/common/SynchronizerImp.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/client/src/main/java/io/split/engine/common/SynchronizerImp.java b/client/src/main/java/io/split/engine/common/SynchronizerImp.java index 3fa0b1037..52ab2e974 100644 --- a/client/src/main/java/io/split/engine/common/SynchronizerImp.java +++ b/client/src/main/java/io/split/engine/common/SynchronizerImp.java @@ -74,7 +74,7 @@ public void stopPeriodicFetching() { @Override public void refreshSplits(long targetChangeNumber) { int retries = RETRIES_NUMBER; - while(true) { + while(targetChangeNumber > _splitCache.getChangeNumber()) { retries--; _splitFetcher.forceRefresh(true); if (targetChangeNumber <= _splitCache.getChangeNumber()) { From de70bf2a06f5b704da08764547c2ba88556ddf9f Mon Sep 17 00:00:00 2001 From: Lucas Echeverz Date: Thu, 22 Apr 2021 18:58:48 -0300 Subject: [PATCH 15/81] Release 4.1.7-rc --- client/CHANGES.txt | 3 +++ client/pom.xml | 2 +- pom.xml | 2 +- testing/pom.xml | 2 +- 4 files changed, 6 insertions(+), 3 deletions(-) diff --git a/client/CHANGES.txt b/client/CHANGES.txt index e02d816d8..50c62d45d 100644 --- a/client/CHANGES.txt +++ b/client/CHANGES.txt @@ -1,5 +1,8 @@ CHANGES +4.1.7 (Apr 22, 2021) +-Updated: added delay on fetch retry. + 4.1.6 (Apr 15, 2021) -Updated log level and message in some messages. diff --git a/client/pom.xml b/client/pom.xml index 5336cf310..8ca259a9b 100644 --- a/client/pom.xml +++ b/client/pom.xml @@ -5,7 +5,7 @@ io.split.client java-client-parent - 4.1.6 + 4.1.7-rc1 java-client jar diff --git a/pom.xml b/pom.xml index 1a40cb3c5..97eb1deb1 100644 --- a/pom.xml +++ b/pom.xml @@ -4,7 +4,7 @@ 4.0.0 io.split.client java-client-parent - 4.1.6 + 4.1.7-rc1 diff --git a/testing/pom.xml b/testing/pom.xml index 3d74dc7a8..af974473c 100644 --- a/testing/pom.xml +++ b/testing/pom.xml @@ -6,7 +6,7 @@ io.split.client java-client-parent - 4.1.6 + 4.1.7-rc1 java-client-testing From 726366b0effa7ca91e53dc2995c1c4c1d935a388 Mon Sep 17 00:00:00 2001 From: Lucas Echeverz Date: Thu, 22 Apr 2021 19:05:39 -0300 Subject: [PATCH 16/81] PR fix --- client/CHANGES.txt | 3 --- 1 file changed, 3 deletions(-) diff --git a/client/CHANGES.txt b/client/CHANGES.txt index 50c62d45d..e02d816d8 100644 --- a/client/CHANGES.txt +++ b/client/CHANGES.txt @@ -1,8 +1,5 @@ CHANGES -4.1.7 (Apr 22, 2021) --Updated: added delay on fetch retry. - 4.1.6 (Apr 15, 2021) -Updated log level and message in some messages. From 196e7892d6e673bcb5e7a8b740e66d26ce47975b Mon Sep 17 00:00:00 2001 From: Lucas Echeverz Date: Mon, 26 Apr 2021 13:03:44 -0300 Subject: [PATCH 17/81] Adding config Generator --- .../TelemetrySynchronizerImp.java | 43 ++++++++++++++++--- .../TelemetrySynchronizerImpTest.java | 13 ++++++ 2 files changed, 50 insertions(+), 6 deletions(-) create mode 100644 client/src/test/java/io/split/telemetry/synchronizer/TelemetrySynchronizerImpTest.java diff --git a/client/src/main/java/io/split/telemetry/synchronizer/TelemetrySynchronizerImp.java b/client/src/main/java/io/split/telemetry/synchronizer/TelemetrySynchronizerImp.java index 8de335df8..7f7b2d796 100644 --- a/client/src/main/java/io/split/telemetry/synchronizer/TelemetrySynchronizerImp.java +++ b/client/src/main/java/io/split/telemetry/synchronizer/TelemetrySynchronizerImp.java @@ -3,6 +3,10 @@ import io.split.cache.SegmentCache; import io.split.cache.SplitCache; import io.split.client.SplitClientConfig; +import io.split.client.impressions.ImpressionListener; +import io.split.client.impressions.ImpressionsManager; +import io.split.integrations.IntegrationsConfig; +import io.split.integrations.NewRelicListener; import io.split.telemetry.domain.Config; import io.split.telemetry.domain.Rates; import io.split.telemetry.domain.Stats; @@ -14,11 +18,16 @@ import java.net.URI; import java.net.URISyntaxException; +import java.util.ArrayList; import java.util.List; import java.util.Map; +import java.util.stream.Collectors; public class TelemetrySynchronizerImp implements TelemetrySynchronizer{ + private static final int OPERATION_MODE = 0; + private static final String STORAGE = "memory"; + private HttpTelemetryMemorySender _httpHttpTelemetryMemorySender; private TelemetryStorageConsumer _teleTelemetryStorageConsumer; private SplitCache _splitCache; @@ -69,6 +78,9 @@ private Config generateConfig(SplitClientConfig splitClientConfig, long timeUnti Config config = new Config(); Rates rates = new Rates(); URLOverrides urlOverrides = new URLOverrides(); + List impressionsListeners = splitClientConfig.integrationsConfig().getImpressionsListeners(IntegrationsConfig.Execution.ASYNC); + impressionsListeners.addAll(splitClientConfig.integrationsConfig().getImpressionsListeners(IntegrationsConfig.Execution.SYNC)); + List impressions = getImpressions(impressionsListeners); rates.set_telemetry(splitClientConfig.get_telemetryRefreshRate()); rates.set_events(splitClientConfig.eventFlushIntervalInMillis()); @@ -84,16 +96,16 @@ private Config generateConfig(SplitClientConfig splitClientConfig, long timeUnti config.set_burTimeouts(_teleTelemetryStorageConsumer.getBURTimeouts()); config.set_nonReadyUsages(_teleTelemetryStorageConsumer.getNonReadyUsages()); - config.set_impressionsListenerEnabled(splitClientConfig.); config.set_httpProxyDetected(splitClientConfig.proxy() != null); - config.set_impressionsMode(splitClientConfig.impressionsMode()); - config.set_integrations(splitClientConfig); - config.set_operationMode(0); //Standalone -> 0 - config.set_storage("memory"); + config.set_impressionsMode(getImpressionsMode(splitClientConfig)); + config.set_integrations(impressions); + config.set_impressionsListenerEnabled((impressionsListeners.size()-impressions.size()) > 0); + config.set_operationMode(OPERATION_MODE); + config.set_storage(STORAGE); config.set_impressionsQueueSize(splitClientConfig.impressionsQueueSize()); config.set_redundantFactories(getRedundantFactories(factoryInstances)); config.set_eventsQueueSize(splitClientConfig.eventsQueueSize()); - config.set_tags(tags); + config.set_tags(getListMaxSize(tags)); config.set_activeFactories(factoryInstances.size()); config.set_timeUntilReady(timeUntilReady); config.set_rates(rates); @@ -109,4 +121,23 @@ private long getRedundantFactories(Map factoryInstances) { } return count; } + + private int getImpressionsMode(SplitClientConfig config) { + return ImpressionsManager.Mode.OPTIMIZED.equals(config.impressionsMode()) ? 0 : 1; + } + + private List getListMaxSize(List list) { + return list.size()> 10 ? list.subList(0, 10) : list; + } + + private List getImpressions(List impressionsListeners) { + List impressions = new ArrayList<>(); + for(IntegrationsConfig.ImpressionListenerWithMeta il: impressionsListeners) { + ImpressionListener listener = il.listener(); + if(listener instanceof NewRelicListener) { + impressions.add(NewRelicListener.class.getName()); + } + } + return impressions; + } } diff --git a/client/src/test/java/io/split/telemetry/synchronizer/TelemetrySynchronizerImpTest.java b/client/src/test/java/io/split/telemetry/synchronizer/TelemetrySynchronizerImpTest.java new file mode 100644 index 000000000..b095df815 --- /dev/null +++ b/client/src/test/java/io/split/telemetry/synchronizer/TelemetrySynchronizerImpTest.java @@ -0,0 +1,13 @@ +package io.split.telemetry.synchronizer; + +import junit.framework.TestCase; +import org.junit.Test; + +import java.util.List; +import java.util.stream.Collectors; +import java.util.stream.Stream; + +public class TelemetrySynchronizerImpTest{ + + +} \ No newline at end of file From 8788b776de0f3e03acb59a4761bb27ec0316a6f1 Mon Sep 17 00:00:00 2001 From: Martin Redolatti Date: Mon, 26 Apr 2021 17:59:26 -0300 Subject: [PATCH 18/81] support for cdn response headers logging - wip --- .../split/client/HttpSplitChangeFetcher.java | 25 ++++-- .../io/split/client/SplitClientConfig.java | 19 ++++- .../io/split/client/SplitFactoryImpl.java | 1 - .../io/split/client/jmx/SplitJmxMonitor.java | 3 +- .../engine/common/FastlyHeadersCaptor.java | 35 ++++++++ .../io/split/engine/common/FetchOptions.java | 61 ++++++++++++++ .../split/engine/common/SynchronizerImp.java | 27 +++++- .../experiments/SplitChangeFetcher.java | 3 +- .../engine/experiments/SplitFetcher.java | 6 +- .../engine/experiments/SplitFetcherImp.java | 15 ++-- .../client/HttpSplitChangeFetcherTest.java | 3 +- .../impressions/ImpressionCounterTest.java | 84 +++++++++++++++++++ .../split/engine/common/SynchronizerTest.java | 2 +- .../AChangePerCallSplitChangeFetcher.java | 3 +- .../engine/experiments/SplitFetcherTest.java | 9 +- 15 files changed, 267 insertions(+), 29 deletions(-) create mode 100644 client/src/main/java/io/split/engine/common/FastlyHeadersCaptor.java create mode 100644 client/src/main/java/io/split/engine/common/FetchOptions.java diff --git a/client/src/main/java/io/split/client/HttpSplitChangeFetcher.java b/client/src/main/java/io/split/client/HttpSplitChangeFetcher.java index 3c5f9b8fc..729b19264 100644 --- a/client/src/main/java/io/split/client/HttpSplitChangeFetcher.java +++ b/client/src/main/java/io/split/client/HttpSplitChangeFetcher.java @@ -4,11 +4,13 @@ import io.split.client.dtos.SplitChange; import io.split.client.utils.Json; import io.split.client.utils.Utils; +import io.split.engine.common.FetchOptions; import io.split.engine.experiments.SplitChangeFetcher; import io.split.engine.metrics.Metrics; import org.apache.hc.client5.http.classic.methods.HttpGet; import org.apache.hc.client5.http.impl.classic.CloseableHttpClient; import org.apache.hc.client5.http.impl.classic.CloseableHttpResponse; +import org.apache.hc.core5.http.Header; import org.apache.hc.core5.http.io.entity.EntityUtils; import org.apache.hc.core5.net.URIBuilder; import org.slf4j.Logger; @@ -17,6 +19,8 @@ import java.net.URI; import java.net.URISyntaxException; import java.nio.charset.StandardCharsets; +import java.util.Arrays; +import java.util.stream.Collectors; import static com.google.common.base.Preconditions.checkNotNull; @@ -28,8 +32,12 @@ public final class HttpSplitChangeFetcher implements SplitChangeFetcher { private static final String SINCE = "since"; private static final String PREFIX = "splitChangeFetcher"; - private static final String NAME_CACHE = "Cache-Control"; - private static final String VALUE_CACHE = "no-cache"; + + private static final String HEADER_CACHE_CONTROL_NAME = "Cache-Control"; + private static final String HEADER_CACHE_CONTROL_VALUE = "no-cache"; + + private static final String HEADER_FASTLY_DEBUG_NAME = "Fastly-Debug"; + private static final String HEADER_FASTLY_DEBUG_VALUE = "1"; private final CloseableHttpClient _client; private final URI _target; @@ -51,7 +59,7 @@ private HttpSplitChangeFetcher(CloseableHttpClient client, URI uri, Metrics metr } @Override - public SplitChange fetch(long since, boolean addCacheHeader) { + public SplitChange fetch(long since, FetchOptions options) { long start = System.currentTimeMillis(); @@ -61,10 +69,17 @@ public SplitChange fetch(long since, boolean addCacheHeader) { URI uri = new URIBuilder(_target).addParameter(SINCE, "" + since).build(); HttpGet request = new HttpGet(uri); - if(addCacheHeader) { - request.setHeader(NAME_CACHE, VALUE_CACHE); + if(options.cacheControlHeadersEnabled()) { + request.setHeader(HEADER_CACHE_CONTROL_NAME, HEADER_CACHE_CONTROL_VALUE); } + + if (options.fastlyDebugHeaderEnabled()) { + request.addHeader(HEADER_FASTLY_DEBUG_NAME, HEADER_FASTLY_DEBUG_VALUE); + } + response = _client.execute(request); + options.handleResponseHeaders(Arrays.stream(response.getHeaders()) + .collect(Collectors.toMap(Header::getName, Header::getValue))); int statusCode = response.getCode(); diff --git a/client/src/main/java/io/split/client/SplitClientConfig.java b/client/src/main/java/io/split/client/SplitClientConfig.java index d86cfa4ed..0cfa44eec 100644 --- a/client/src/main/java/io/split/client/SplitClientConfig.java +++ b/client/src/main/java/io/split/client/SplitClientConfig.java @@ -47,6 +47,7 @@ public class SplitClientConfig { private final String _authServiceURL; private final String _streamingServiceURL; private final int _onDemandFetchRetryDelayMs; + private final boolean _cdnDebugLogging; // Proxy configs private final HttpHost _proxy; @@ -91,7 +92,8 @@ private SplitClientConfig(String endpoint, int streamingReconnectBackoffBase, String authServiceURL, String streamingServiceURL, - int onDemandFetchRetryDelayMs) { + int onDemandFetchRetryDelayMs, + boolean cdnDebugLogging) { _endpoint = endpoint; _eventsEndpoint = eventsEndpoint; _featuresRefreshRate = pollForFeatureChangesEveryNSeconds; @@ -123,6 +125,7 @@ private SplitClientConfig(String endpoint, _authServiceURL = authServiceURL; _streamingServiceURL = streamingServiceURL; _onDemandFetchRetryDelayMs = onDemandFetchRetryDelayMs; + _cdnDebugLogging = cdnDebugLogging; Properties props = new Properties(); try { @@ -289,6 +292,7 @@ public static final class Builder { private String _authServiceURL = "https://auth.split.io/api/auth"; private String _streamingServiceURL = "https://streaming.split.io/sse"; private int _onDemandFetchRetryDelayMs = 50; + private boolean _cdnDebugLogging = false; public Builder() { } @@ -690,6 +694,16 @@ public Builder streamingRetryDelay(int onDemandFetchRetryDelayMs) { return this; } + /** + * Enable logging response headers for requests made to our CDN. + * @param cdnDebugLogging + * @return + */ + public Builder cdnDebugLogging(boolean cdnDebugLogging) { + _cdnDebugLogging = cdnDebugLogging; + return this; + } + public SplitClientConfig build() { if (_featuresRefreshRate < 5 ) { throw new IllegalArgumentException("featuresRefreshRate must be >= 5: " + _featuresRefreshRate); @@ -795,7 +809,8 @@ public SplitClientConfig build() { _streamingReconnectBackoffBase, _authServiceURL, _streamingServiceURL, - _onDemandFetchRetryDelayMs); + _onDemandFetchRetryDelayMs, + _cdnDebugLogging); } } } diff --git a/client/src/main/java/io/split/client/SplitFactoryImpl.java b/client/src/main/java/io/split/client/SplitFactoryImpl.java index a2d68ba64..cf5224e8a 100644 --- a/client/src/main/java/io/split/client/SplitFactoryImpl.java +++ b/client/src/main/java/io/split/client/SplitFactoryImpl.java @@ -207,7 +207,6 @@ public boolean isDestroyed() { } private static CloseableHttpClient buildHttpClient(String apiToken, SplitClientConfig config) { - SSLConnectionSocketFactory sslSocketFactory = SSLConnectionSocketFactoryBuilder.create() .setSslContext(SSLContexts.createSystemDefault()) .setTlsVersions(TLS.V_1_1, TLS.V_1_2) diff --git a/client/src/main/java/io/split/client/jmx/SplitJmxMonitor.java b/client/src/main/java/io/split/client/jmx/SplitJmxMonitor.java index e5d49e115..559fdf25b 100644 --- a/client/src/main/java/io/split/client/jmx/SplitJmxMonitor.java +++ b/client/src/main/java/io/split/client/jmx/SplitJmxMonitor.java @@ -3,6 +3,7 @@ import io.split.cache.SegmentCache; import io.split.cache.SplitCache; import io.split.client.SplitClient; +import io.split.engine.common.FetchOptions; import io.split.engine.experiments.SplitFetcher; import io.split.engine.segments.SegmentFetcher; import io.split.engine.segments.SegmentSynchronizationTask; @@ -34,7 +35,7 @@ public SplitJmxMonitor(SplitClient splitClient, SplitFetcher featureFetcher, Spl @Override public boolean forceSyncFeatures() { - _featureFetcher.forceRefresh(true); + _featureFetcher.forceRefresh(new FetchOptions.Builder().cacheControlHeaders(true).build()); _log.info("Features successfully refreshed via JMX"); return true; } diff --git a/client/src/main/java/io/split/engine/common/FastlyHeadersCaptor.java b/client/src/main/java/io/split/engine/common/FastlyHeadersCaptor.java new file mode 100644 index 000000000..c96eb6b94 --- /dev/null +++ b/client/src/main/java/io/split/engine/common/FastlyHeadersCaptor.java @@ -0,0 +1,35 @@ +package io.split.engine.common; + +import java.util.*; +import java.util.stream.Collectors; + +public class FastlyHeadersCaptor { + + public static Set FIELDS_TO_CAPTURE = new HashSet<>(Arrays.asList( + "Fastly-Debug-Path", + "Fastly-Debug-TTL", + "Fastly-Debug-Digest", + "X-Served-By", + "X-Cache", + "X-Cache-Hits", + "X-Timer", + "Surrogate-Key", + "ETag", + "Cache-Control", + "X-Request-ID", + "Last-Modified" + )); + + private final List> _headers = new ArrayList<>(); + + public Void handle(Map responseHeaders) { + _headers.add(responseHeaders.entrySet().stream() + .filter(e -> FIELDS_TO_CAPTURE.contains(e.getKey())) + .collect(Collectors.toMap(Map.Entry::getKey, Map.Entry::getValue))); + return null; + } + + public List> get() { + return _headers; + } +} \ No newline at end of file diff --git a/client/src/main/java/io/split/engine/common/FetchOptions.java b/client/src/main/java/io/split/engine/common/FetchOptions.java new file mode 100644 index 000000000..251c15516 --- /dev/null +++ b/client/src/main/java/io/split/engine/common/FetchOptions.java @@ -0,0 +1,61 @@ +package io.split.engine.common; + +import java.util.Map; +import java.util.Objects; +import java.util.function.Function; + +public class FetchOptions { + + public static class Builder { + public Builder() {} + + public Builder cacheControlHeaders(boolean on) { + _cacheControlHeaders = on; + return this; + } + + public Builder fastlyDebugHeader(boolean on) { + _fastlyDebugHeader = on; + return this; + } + + public Builder responseHeadersCallback(Function, Void> callback) { + _responseHeadersCallback = callback; + return this; + } + + public FetchOptions build() { + return new FetchOptions(_cacheControlHeaders, _responseHeadersCallback, _fastlyDebugHeader); + } + + private boolean _cacheControlHeaders = false; + private boolean _fastlyDebugHeader = false; + private Function, Void> _responseHeadersCallback = null; + } + + public boolean cacheControlHeadersEnabled() { + return _cacheControlHeaders; + } + + public boolean fastlyDebugHeaderEnabled() { + return _fastlyDebugHeader; + } + + public void handleResponseHeaders(Map headers) { + if (Objects.isNull(_responseHeadersCallback) || Objects.isNull(headers)) { + return; + } + _responseHeadersCallback.apply(headers); + } + + private FetchOptions(boolean cacheControlHeaders, Function, Void> responseHeadersCallback, + boolean fastlyDebugHeader) { + _cacheControlHeaders = cacheControlHeaders; + _responseHeadersCallback = responseHeadersCallback; + _fastlyDebugHeader = fastlyDebugHeader; + } + + private final boolean _cacheControlHeaders; + private final boolean _fastlyDebugHeader; + private final Function, Void> _responseHeadersCallback; +} diff --git a/client/src/main/java/io/split/engine/common/SynchronizerImp.java b/client/src/main/java/io/split/engine/common/SynchronizerImp.java index 52ab2e974..dd325e2dd 100644 --- a/client/src/main/java/io/split/engine/common/SynchronizerImp.java +++ b/client/src/main/java/io/split/engine/common/SynchronizerImp.java @@ -1,6 +1,8 @@ package io.split.engine.common; import com.google.common.util.concurrent.ThreadFactoryBuilder; +import com.google.gson.Gson; +import com.google.gson.GsonBuilder; import io.split.cache.SegmentCache; import io.split.cache.SplitCache; import io.split.engine.experiments.SplitFetcher; @@ -10,6 +12,9 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import java.util.ArrayList; +import java.util.List; +import java.util.Map; import java.util.concurrent.Executors; import java.util.concurrent.ScheduledExecutorService; import java.util.concurrent.ThreadFactory; @@ -21,6 +26,7 @@ public class SynchronizerImp implements Synchronizer { private static final Logger _log = LoggerFactory.getLogger(Synchronizer.class); private static final int RETRIES_NUMBER = 10; + private final boolean _cdnResponseHeadersLogging = true; private final SplitSynchronizationTask _splitSynchronizationTask; private final SplitFetcher _splitFetcher; private final SegmentSynchronizationTask _segmentSynchronizationTaskImp; @@ -28,6 +34,7 @@ public class SynchronizerImp implements Synchronizer { private final SplitCache _splitCache; private final SegmentCache _segmentCache; private final int _onDemandFetchRetryDelayMs; + private final Gson gson = new GsonBuilder().create(); public SynchronizerImp(SplitSynchronizationTask splitSynchronizationTask, SplitFetcher splitFetcher, @@ -52,7 +59,7 @@ public SynchronizerImp(SplitSynchronizationTask splitSynchronizationTask, @Override public void syncAll() { _syncAllScheduledExecutorService.schedule(() -> { - _splitFetcher.fetchAll(true); + _splitFetcher.fetchAll(new FetchOptions.Builder().cacheControlHeaders(true).build()); _segmentSynchronizationTaskImp.fetchAll(true); }, 0, TimeUnit.SECONDS); } @@ -74,14 +81,28 @@ public void stopPeriodicFetching() { @Override public void refreshSplits(long targetChangeNumber) { int retries = RETRIES_NUMBER; - while(targetChangeNumber > _splitCache.getChangeNumber()) { + if (targetChangeNumber <= _splitCache.getChangeNumber()) { + return; + } + + FastlyHeadersCaptor captor = new FastlyHeadersCaptor(); + FetchOptions opts = new FetchOptions.Builder() + .cacheControlHeaders(true) + .fastlyDebugHeader(_cdnResponseHeadersLogging) + .responseHeadersCallback(_cdnResponseHeadersLogging ? captor::handle : null) + .build(); + + while(true) { retries--; - _splitFetcher.forceRefresh(true); + _splitFetcher.forceRefresh(opts); if (targetChangeNumber <= _splitCache.getChangeNumber()) { _log.debug("Refresh completed in %s attempts.", RETRIES_NUMBER - retries); return; } else if (retries <= 0) { _log.warn("No changes fetched after %s attempts.", RETRIES_NUMBER); + if (_cdnResponseHeadersLogging) { + _log.debug("CDN Debug headers: ", gson.toJson(captor.get())); + } return; } try { diff --git a/client/src/main/java/io/split/engine/experiments/SplitChangeFetcher.java b/client/src/main/java/io/split/engine/experiments/SplitChangeFetcher.java index 63298a5e7..7c5fbe76e 100644 --- a/client/src/main/java/io/split/engine/experiments/SplitChangeFetcher.java +++ b/client/src/main/java/io/split/engine/experiments/SplitChangeFetcher.java @@ -1,6 +1,7 @@ package io.split.engine.experiments; import io.split.client.dtos.SplitChange; +import io.split.engine.common.FetchOptions; /** * Created by adilaijaz on 5/11/15. @@ -31,5 +32,5 @@ public interface SplitChangeFetcher { * @return SegmentChange * @throws java.lang.RuntimeException if there was a problem computing split changes */ - SplitChange fetch(long since, boolean addCacheHeader); + SplitChange fetch(long since, FetchOptions options); } diff --git a/client/src/main/java/io/split/engine/experiments/SplitFetcher.java b/client/src/main/java/io/split/engine/experiments/SplitFetcher.java index 4266659b1..8733062a0 100644 --- a/client/src/main/java/io/split/engine/experiments/SplitFetcher.java +++ b/client/src/main/java/io/split/engine/experiments/SplitFetcher.java @@ -1,5 +1,7 @@ package io.split.engine.experiments; +import io.split.engine.common.FetchOptions; + /** * Created by adilaijaz on 5/8/15. */ @@ -8,11 +10,11 @@ public interface SplitFetcher extends Runnable { * Forces a sync of splits, outside of any scheduled * syncs. This method MUST NOT throw any exceptions. */ - void forceRefresh(boolean addCacheHeader); + void forceRefresh(FetchOptions options); /** * Forces a sync of ALL splits, outside of any scheduled * syncs. This method MUST NOT throw any exceptions. */ - void fetchAll(boolean addCacheHeader); + void fetchAll(FetchOptions options); } diff --git a/client/src/main/java/io/split/engine/experiments/SplitFetcherImp.java b/client/src/main/java/io/split/engine/experiments/SplitFetcherImp.java index 510001153..b9c63f811 100644 --- a/client/src/main/java/io/split/engine/experiments/SplitFetcherImp.java +++ b/client/src/main/java/io/split/engine/experiments/SplitFetcherImp.java @@ -5,6 +5,7 @@ import io.split.client.dtos.Status; import io.split.engine.SDKReadinessGates; import io.split.cache.SplitCache; +import io.split.engine.common.FetchOptions; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -43,12 +44,12 @@ public SplitFetcherImp(SplitChangeFetcher splitChangeFetcher, SplitParser parser } @Override - public void forceRefresh(boolean addCacheHeader) { + public void forceRefresh(FetchOptions options) { _log.debug("Force Refresh splits starting ..."); try { while (true) { long start = _splitCache.getChangeNumber(); - runWithoutExceptionHandling(addCacheHeader); + runWithoutExceptionHandling(options); long end = _splitCache.getChangeNumber(); if (start >= end) { @@ -65,11 +66,11 @@ public void forceRefresh(boolean addCacheHeader) { @Override public void run() { - this.fetchAll(false); + this.fetchAll(new FetchOptions.Builder().cacheControlHeaders(false).build()); } - private void runWithoutExceptionHandling(boolean addCacheHeader) throws InterruptedException { - SplitChange change = _splitChangeFetcher.fetch(_splitCache.getChangeNumber(), addCacheHeader); + private void runWithoutExceptionHandling(FetchOptions options) throws InterruptedException { + SplitChange change = _splitChangeFetcher.fetch(_splitCache.getChangeNumber(), options); if (change == null) { throw new IllegalStateException("SplitChange was null"); @@ -139,11 +140,11 @@ private void runWithoutExceptionHandling(boolean addCacheHeader) throws Interrup } } @Override - public void fetchAll(boolean addCacheHeader) { + public void fetchAll(FetchOptions options) { _log.debug("Fetch splits starting ..."); long start = _splitCache.getChangeNumber(); try { - runWithoutExceptionHandling(addCacheHeader); + runWithoutExceptionHandling(options); _gates.splitsAreReady(); } catch (InterruptedException e) { _log.warn("Interrupting split fetcher task"); diff --git a/client/src/test/java/io/split/client/HttpSplitChangeFetcherTest.java b/client/src/test/java/io/split/client/HttpSplitChangeFetcherTest.java index 564339db7..49abe8059 100644 --- a/client/src/test/java/io/split/client/HttpSplitChangeFetcherTest.java +++ b/client/src/test/java/io/split/client/HttpSplitChangeFetcherTest.java @@ -3,6 +3,7 @@ import io.split.TestHelper; import io.split.client.dtos.Split; import io.split.client.dtos.SplitChange; +import io.split.engine.common.FetchOptions; import io.split.engine.metrics.Metrics; import org.apache.hc.client5.http.impl.classic.CloseableHttpClient; import org.apache.hc.client5.http.impl.classic.HttpClients; @@ -63,7 +64,7 @@ public void testFetcherWithSpecialCharacters() throws URISyntaxException, Invoca Metrics.NoopMetrics metrics = new Metrics.NoopMetrics(); HttpSplitChangeFetcher fetcher = HttpSplitChangeFetcher.create(httpClientMock, rootTarget, metrics); - SplitChange change = fetcher.fetch(1234567, true); + SplitChange change = fetcher.fetch(1234567, new FetchOptions.Builder().cacheControlHeaders(true).build()); Assert.assertNotNull(change); Assert.assertEquals(1, change.splits.size()); diff --git a/client/src/test/java/io/split/client/impressions/ImpressionCounterTest.java b/client/src/test/java/io/split/client/impressions/ImpressionCounterTest.java index 4d8737dde..b04c93eb6 100644 --- a/client/src/test/java/io/split/client/impressions/ImpressionCounterTest.java +++ b/client/src/test/java/io/split/client/impressions/ImpressionCounterTest.java @@ -4,10 +4,15 @@ import java.time.ZoneId; import java.time.ZonedDateTime; +import java.util.ArrayList; +import java.util.List; import java.util.Map; +import static org.hamcrest.CoreMatchers.both; import static org.hamcrest.CoreMatchers.is; import static org.hamcrest.MatcherAssert.assertThat; +import static org.hamcrest.Matchers.greaterThan; +import static org.hamcrest.Matchers.lessThan; import static org.hamcrest.core.IsEqual.equalTo; public class ImpressionCounterTest { @@ -101,4 +106,83 @@ public void manyConcurrentCalls() throws InterruptedException { assertThat(counted.get(new ImpressionCounter.Key("feature1", ImpressionUtils.truncateTimeframe(nextHourTimestamp))), is(equalTo(iterations * 3))); assertThat(counted.get(new ImpressionCounter.Key("feature2", ImpressionUtils.truncateTimeframe(nextHourTimestamp))), is(equalTo(iterations * 3))); } + + @Test + public void manyConcurrentCallsWithConcurrentPops() throws InterruptedException { + final int iterations = 10000000; + final long timestamp = makeTimestamp(2020, 9, 2, 10, 10, 12); + final long nextHourTimestamp = makeTimestamp(2020, 9, 2, 11, 10, 12); + ImpressionCounter counter = new ImpressionCounter(); + Thread t1 = new Thread(() -> { + int times = iterations; + while (times-- > 0) { + counter.inc("feature1", timestamp, 1); + counter.inc("feature2", timestamp, 1); + counter.inc("feature1", nextHourTimestamp, 2); + counter.inc("feature2", nextHourTimestamp, 2); + } + }); + Thread t2 = new Thread(() -> { + int times = iterations; + while (times-- > 0) { + counter.inc("feature1", timestamp, 2); + counter.inc("feature2", timestamp, 2); + counter.inc("feature1", nextHourTimestamp, 1); + counter.inc("feature2", nextHourTimestamp, 1); + } + }); + + // Pushing to this list will be done from a single thread. And querying will be done from the main one + // after all other threads have ended. No need for extra sync logic. + List> pops = new ArrayList<>(); + Thread t3 = new Thread(() -> { + try { + for (int i=10; i > 0; --i){ + Thread.sleep(1); + pops.add(counter.popAll()); + } + } catch (InterruptedException e) { + e.printStackTrace(); + } + }); + + t1.setDaemon(true); t2.setDaemon(true); t2.setDaemon(true); + t1.start(); t2.start(); t3.start(); + t1.join(); t2.join(); t3.join(); + + // --- No other thread is running at this point. + + // Do an extra pop in case there's still some data in the counter + pops.add(counter.popAll()); + + Long feature1TSCount = pops.stream() + .reduce(0L, + (accum, next) -> accum + next.getOrDefault(new ImpressionCounter.Key("feature1", ImpressionUtils.truncateTimeframe(timestamp)), 0), + (x, y) -> x + y); + + Long feature1NextTSCount = pops.stream() + .reduce(0L, + (accum, next) -> accum + next.getOrDefault(new ImpressionCounter.Key("feature1", ImpressionUtils.truncateTimeframe(nextHourTimestamp)), 0), + (x, y) -> x + y); + + Long feature2TSCount = pops.stream() + .reduce(0L, + (accum, next) -> accum + next.getOrDefault(new ImpressionCounter.Key("feature2", ImpressionUtils.truncateTimeframe(timestamp)), 0), + (x, y) -> x + y); + + Long feature2NextTSCount = pops.stream() + .reduce(0L, + (accum, next) -> accum + next.getOrDefault(new ImpressionCounter.Key("feature2", ImpressionUtils.truncateTimeframe(nextHourTimestamp)), 0), + (x, y) -> x + y); + + + // Using lockless/atomic structures for higher performance at the cost of 0.001% margin error accepted in very high concurrency + Long lowerBound = (long) ((iterations * 3) * 0.99999); + Long upperBound = (long) ((iterations * 3) * 1.00001); + + assertThat(feature1TSCount, is(both(greaterThan(lowerBound)).and(lessThan(upperBound)))); + assertThat(feature1NextTSCount, is(both(greaterThan(lowerBound)).and(lessThan(upperBound)))); + assertThat(feature2TSCount, is(both(greaterThan(lowerBound)).and(lessThan(upperBound)))); + assertThat(feature2NextTSCount, is(both(greaterThan(lowerBound)).and(lessThan(upperBound)))); + } } diff --git a/client/src/test/java/io/split/engine/common/SynchronizerTest.java b/client/src/test/java/io/split/engine/common/SynchronizerTest.java index 9a90cb5ae..742b1d3f8 100644 --- a/client/src/test/java/io/split/engine/common/SynchronizerTest.java +++ b/client/src/test/java/io/split/engine/common/SynchronizerTest.java @@ -34,7 +34,7 @@ public void syncAll() throws InterruptedException { _synchronizer.syncAll(); Thread.sleep(100); - Mockito.verify(_splitFetcher, Mockito.times(1)).fetchAll(true); + Mockito.verify(_splitFetcher, Mockito.times(1)).fetchAll(new FetchOptions.Builder().cacheControlHeaders(true).build()); Mockito.verify(_segmentFetcher, Mockito.times(1)).fetchAll(true); } diff --git a/client/src/test/java/io/split/engine/experiments/AChangePerCallSplitChangeFetcher.java b/client/src/test/java/io/split/engine/experiments/AChangePerCallSplitChangeFetcher.java index 6b0114566..64495e112 100644 --- a/client/src/test/java/io/split/engine/experiments/AChangePerCallSplitChangeFetcher.java +++ b/client/src/test/java/io/split/engine/experiments/AChangePerCallSplitChangeFetcher.java @@ -7,6 +7,7 @@ import io.split.client.dtos.SplitChange; import io.split.client.dtos.Status; import io.split.engine.ConditionsTestUtil; +import io.split.engine.common.FetchOptions; import io.split.grammar.Treatments; import java.util.concurrent.atomic.AtomicLong; @@ -31,7 +32,7 @@ public AChangePerCallSplitChangeFetcher(String segmentName) { @Override - public SplitChange fetch(long since, boolean addCacheHeader) { + public SplitChange fetch(long since, FetchOptions options) { long latestChangeNumber = since + 1; Condition condition = null; diff --git a/client/src/test/java/io/split/engine/experiments/SplitFetcherTest.java b/client/src/test/java/io/split/engine/experiments/SplitFetcherTest.java index cb9352994..a68fc8435 100644 --- a/client/src/test/java/io/split/engine/experiments/SplitFetcherTest.java +++ b/client/src/test/java/io/split/engine/experiments/SplitFetcherTest.java @@ -8,6 +8,7 @@ import io.split.client.dtos.*; import io.split.engine.ConditionsTestUtil; import io.split.engine.SDKReadinessGates; +import io.split.engine.common.FetchOptions; import io.split.engine.matchers.AllKeysMatcher; import io.split.engine.matchers.CombiningMatcher; import io.split.engine.segments.SegmentChangeFetcher; @@ -122,9 +123,9 @@ public void when_parser_fails_we_remove_the_experiment() throws InterruptedExcep noReturn.till = 1L; SplitChangeFetcher splitChangeFetcher = mock(SplitChangeFetcher.class); - when(splitChangeFetcher.fetch(-1L, false)).thenReturn(validReturn); - when(splitChangeFetcher.fetch(0L, false)).thenReturn(invalidReturn); - when(splitChangeFetcher.fetch(1L, false)).thenReturn(noReturn); + when(splitChangeFetcher.fetch(-1L, new FetchOptions.Builder().build())).thenReturn(validReturn); + when(splitChangeFetcher.fetch(0L, new FetchOptions.Builder().build())).thenReturn(invalidReturn); + when(splitChangeFetcher.fetch(1L, new FetchOptions.Builder().build())).thenReturn(noReturn); SegmentCache segmentCache = new SegmentCacheInMemoryImpl(); @@ -148,7 +149,7 @@ public void if_there_is_a_problem_talking_to_split_change_count_down_latch_is_no SplitCache cache = new InMemoryCacheImp(-1); SplitChangeFetcher splitChangeFetcher = mock(SplitChangeFetcher.class); - when(splitChangeFetcher.fetch(-1L, false)).thenThrow(new RuntimeException()); + when(splitChangeFetcher.fetch(-1L, new FetchOptions.Builder().build())).thenThrow(new RuntimeException()); SegmentCache segmentCache = new SegmentCacheInMemoryImpl(); SegmentChangeFetcher segmentChangeFetcher = mock(SegmentChangeFetcher.class); From 4df2dafbf6ae8d46bf6d22bbdcc2f2ef69c5f117 Mon Sep 17 00:00:00 2001 From: Mauro Sanz <51236193+sanzmauro@users.noreply.github.com> Date: Mon, 26 Apr 2021 18:43:56 -0300 Subject: [PATCH 19/81] Update README.md --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 2771f2a1c..c361a6894 100644 --- a/README.md +++ b/README.md @@ -64,7 +64,7 @@ Split has built and maintains SDKs for: * Java [Github](https://github.com/splitio/java-client) [Docs](https://help.split.io/hc/en-us/articles/360020405151-Java-SDK) * Javascript [Github](https://github.com/splitio/javascript-client) [Docs](https://help.split.io/hc/en-us/articles/360020448791-JavaScript-SDK) * Node [Github](https://github.com/splitio/javascript-client) [Docs](https://help.split.io/hc/en-us/articles/360020564931-Node-js-SDK) -* .NET [Github](https://github.com/splitio/.net-core-client) [Docs](https://help.split.io/hc/en-us/articles/360020240172--NET-SDK) +* .NET [Github](https://github.com/splitio/dotnet-client) [Docs](https://help.split.io/hc/en-us/articles/360020240172--NET-SDK) * Ruby [Github](https://github.com/splitio/ruby-client) [Docs](https://help.split.io/hc/en-us/articles/360020673251-Ruby-SDK) * PHP [Github](https://github.com/splitio/php-client) [Docs](https://help.split.io/hc/en-us/articles/360020350372-PHP-SDK) * Python [Github](https://github.com/splitio/python-client) [Docs](https://help.split.io/hc/en-us/articles/360020359652-Python-SDK) From 6228c6fa0df597d948ed20bcf39e47e058f608c0 Mon Sep 17 00:00:00 2001 From: Lucas Echeverz Date: Tue, 27 Apr 2021 15:52:11 -0300 Subject: [PATCH 20/81] Adding tests --- client/pom.xml | 2 +- .../HttpPostImp.java} | 15 +++-- .../HttpTelemetryMemorySender.java | 9 +-- .../TelemetrySynchronizerImp.java | 8 ++- .../io/split/service/HttpPostImpTest.java | 34 +++++++++++ .../TelemetrySynchronizerImpTest.java | 56 +++++++++++++++++-- pom.xml | 2 +- testing/pom.xml | 2 +- 8 files changed, 108 insertions(+), 20 deletions(-) rename client/src/main/java/io/split/{telemetry/utils/HttpPost.java => service/HttpPostImp.java} (72%) create mode 100644 client/src/test/java/io/split/service/HttpPostImpTest.java diff --git a/client/pom.xml b/client/pom.xml index ce5a16e1e..1ece412a1 100644 --- a/client/pom.xml +++ b/client/pom.xml @@ -5,7 +5,7 @@ io.split.client java-client-parent - 4.1.5 + 4.2.0-rc1 java-client jar diff --git a/client/src/main/java/io/split/telemetry/utils/HttpPost.java b/client/src/main/java/io/split/service/HttpPostImp.java similarity index 72% rename from client/src/main/java/io/split/telemetry/utils/HttpPost.java rename to client/src/main/java/io/split/service/HttpPostImp.java index 363cb2f1c..afd72da2d 100644 --- a/client/src/main/java/io/split/telemetry/utils/HttpPost.java +++ b/client/src/main/java/io/split/service/HttpPostImp.java @@ -1,4 +1,4 @@ -package io.split.telemetry.utils; +package io.split.service; import io.split.client.utils.Utils; import org.apache.hc.client5.http.impl.classic.CloseableHttpClient; @@ -6,14 +6,17 @@ import org.apache.hc.core5.http.HttpEntity; import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import org.apache.hc.client5.http.classic.methods.HttpPost; import java.net.URI; -public class HttpPost { - private static final Logger _logger = LoggerFactory.getLogger(HttpPost.class); +public class HttpPostImp { + private static final int STATUS_OKEY = 200; + private static final int STATUS_MIN = 300; + private static final Logger _logger = LoggerFactory.getLogger(HttpPostImp.class); private CloseableHttpClient _client; - public HttpPost(CloseableHttpClient client) { + public HttpPostImp(CloseableHttpClient client) { _client = client; } @@ -22,14 +25,14 @@ public void post(URI uri, Object object, String posted) { try { HttpEntity entity = Utils.toJsonEntity(object); - org.apache.hc.client5.http.classic.methods.HttpPost request = new org.apache.hc.client5.http.classic.methods.HttpPost(uri); + HttpPost request = new HttpPost(uri); request.setEntity(entity); response = _client.execute(request); int status = response.getCode(); - if (status < 200 || status >= 300) { + if (status < STATUS_OKEY || status >= STATUS_MIN) { _logger.warn("Response status was: " + status); } diff --git a/client/src/main/java/io/split/telemetry/synchronizer/HttpTelemetryMemorySender.java b/client/src/main/java/io/split/telemetry/synchronizer/HttpTelemetryMemorySender.java index 2316f8daa..2f5c932cd 100644 --- a/client/src/main/java/io/split/telemetry/synchronizer/HttpTelemetryMemorySender.java +++ b/client/src/main/java/io/split/telemetry/synchronizer/HttpTelemetryMemorySender.java @@ -1,11 +1,11 @@ package io.split.telemetry.synchronizer; -import io.split.client.SplitClientConfig; +import com.google.common.annotations.VisibleForTesting; import io.split.client.impressions.HttpImpressionsSender; import io.split.client.utils.Utils; import io.split.telemetry.domain.Config; import io.split.telemetry.domain.Stats; -import io.split.telemetry.utils.HttpPost; +import io.split.service.HttpPostImp; import org.apache.hc.client5.http.impl.classic.CloseableHttpClient; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -13,7 +13,7 @@ import java.net.URI; import java.net.URISyntaxException; -public class HttpTelemetryMemorySender extends HttpPost { +public class HttpTelemetryMemorySender extends HttpPostImp { private static final String CONFIG_ENDPOINT_PATH = "metrics/config"; private static final String STATS_ENDPOINT_PATH = "metrics/usage"; @@ -32,7 +32,8 @@ public static HttpTelemetryMemorySender create(CloseableHttpClient client, URI t ); } - private HttpTelemetryMemorySender(CloseableHttpClient client, URI impressionConfigTarget, URI impressionStatsTarget) { + @VisibleForTesting + HttpTelemetryMemorySender(CloseableHttpClient client, URI impressionConfigTarget, URI impressionStatsTarget) { super(client); _impressionConfigTarget = impressionConfigTarget; _impressionStatsTarget = impressionStatsTarget; diff --git a/client/src/main/java/io/split/telemetry/synchronizer/TelemetrySynchronizerImp.java b/client/src/main/java/io/split/telemetry/synchronizer/TelemetrySynchronizerImp.java index 7f7b2d796..a6a35d04d 100644 --- a/client/src/main/java/io/split/telemetry/synchronizer/TelemetrySynchronizerImp.java +++ b/client/src/main/java/io/split/telemetry/synchronizer/TelemetrySynchronizerImp.java @@ -19,6 +19,7 @@ import java.net.URI; import java.net.URISyntaxException; import java.util.ArrayList; +import java.util.Arrays; import java.util.List; import java.util.Map; import java.util.stream.Collectors; @@ -78,8 +79,11 @@ private Config generateConfig(SplitClientConfig splitClientConfig, long timeUnti Config config = new Config(); Rates rates = new Rates(); URLOverrides urlOverrides = new URLOverrides(); - List impressionsListeners = splitClientConfig.integrationsConfig().getImpressionsListeners(IntegrationsConfig.Execution.ASYNC); - impressionsListeners.addAll(splitClientConfig.integrationsConfig().getImpressionsListeners(IntegrationsConfig.Execution.SYNC)); + List impressionsListeners = new ArrayList<>(); + if(splitClientConfig.integrationsConfig() != null) { + impressionsListeners.addAll(splitClientConfig.integrationsConfig().getImpressionsListeners(IntegrationsConfig.Execution.ASYNC)); + impressionsListeners.addAll(splitClientConfig.integrationsConfig().getImpressionsListeners(IntegrationsConfig.Execution.SYNC)); + } List impressions = getImpressions(impressionsListeners); rates.set_telemetry(splitClientConfig.get_telemetryRefreshRate()); diff --git a/client/src/test/java/io/split/service/HttpPostImpTest.java b/client/src/test/java/io/split/service/HttpPostImpTest.java new file mode 100644 index 000000000..69c28b78e --- /dev/null +++ b/client/src/test/java/io/split/service/HttpPostImpTest.java @@ -0,0 +1,34 @@ +package io.split.service; + +import io.split.TestHelper; +import junit.framework.TestCase; +import org.apache.hc.client5.http.impl.classic.CloseableHttpClient; +import org.junit.Test; +import org.mockito.Mockito; + +import java.io.IOException; +import java.lang.reflect.InvocationTargetException; +import java.net.URI; + +public class HttpPostImpTest{ + + private static final String URL = "www.split.io"; + private static int SUCCESSS_CODE = 200; + private static int ERROR_CODE = 400; + + @Test + public void testPostWith200() throws InvocationTargetException, NoSuchMethodException, IllegalAccessException, IOException { + CloseableHttpClient client =TestHelper.mockHttpClient(URL, SUCCESSS_CODE); + HttpPostImp httpPostImp = new HttpPostImp(client); + httpPostImp.post(URI.create(URL), new Object(), "Metrics"); + Mockito.verify(client, Mockito.times(1)).execute(Mockito.any()); + } + + @Test + public void testPostWith400() throws InvocationTargetException, NoSuchMethodException, IllegalAccessException, IOException { + CloseableHttpClient client =TestHelper.mockHttpClient(URL, ERROR_CODE); + HttpPostImp httpPostImp = new HttpPostImp(client); + httpPostImp.post(URI.create(URL), new Object(), "Metrics"); + Mockito.verify(client, Mockito.times(1)).execute(Mockito.any()); + } +} \ No newline at end of file diff --git a/client/src/test/java/io/split/telemetry/synchronizer/TelemetrySynchronizerImpTest.java b/client/src/test/java/io/split/telemetry/synchronizer/TelemetrySynchronizerImpTest.java index b095df815..9ab844377 100644 --- a/client/src/test/java/io/split/telemetry/synchronizer/TelemetrySynchronizerImpTest.java +++ b/client/src/test/java/io/split/telemetry/synchronizer/TelemetrySynchronizerImpTest.java @@ -1,13 +1,59 @@ package io.split.telemetry.synchronizer; -import junit.framework.TestCase; +import io.split.TestHelper; +import io.split.cache.InMemoryCacheImp; +import io.split.cache.SegmentCache; +import io.split.cache.SegmentCacheInMemoryImpl; +import io.split.cache.SplitCache; +import io.split.client.SplitClientConfig; +import io.split.service.HttpPostImp; +import io.split.telemetry.storage.InMemoryTelemetryStorage; +import io.split.telemetry.storage.TelemetryStorageConsumer; +import org.apache.hc.client5.http.classic.methods.HttpOptions; +import org.apache.hc.client5.http.impl.classic.CloseableHttpClient; +import org.apache.hc.client5.http.impl.classic.CloseableHttpResponse; +import org.apache.hc.core5.http.ClassicHttpRequest; import org.junit.Test; +import org.mockito.Mock; +import org.mockito.Mockito; -import java.util.List; -import java.util.stream.Collectors; -import java.util.stream.Stream; +import java.io.IOException; +import java.lang.reflect.InvocationTargetException; +import java.net.URI; +import java.net.URISyntaxException; +import java.util.ArrayList; +import java.util.HashMap; public class TelemetrySynchronizerImpTest{ - + + public static final String TELEMETRY_ENDPOINT = "https://telemetry.split.io/api/v1"; + + @Test + public void testSynchronizeConfig() throws URISyntaxException, NoSuchMethodException, IOException, IllegalAccessException, InvocationTargetException { + CloseableHttpClient httpClient = TestHelper.mockHttpClient("", 200); + TelemetrySynchronizer telemetrySynchronizer = getTelemetrySynchronizer(httpClient); + SplitClientConfig splitClientConfig = SplitClientConfig.builder().build(); + + telemetrySynchronizer.synchronizeConfig(splitClientConfig, 100l, new HashMap(), new ArrayList()); + Mockito.verify(httpClient, Mockito.times(1)).execute(Mockito.any()); + } + + + @Test + public void testSynchronizeStats() throws Exception { + CloseableHttpClient httpClient = TestHelper.mockHttpClient("", 200); + TelemetrySynchronizer telemetrySynchronizer = getTelemetrySynchronizer(httpClient); + + telemetrySynchronizer.synchronizeStats(); + Mockito.verify(httpClient, Mockito.times(1)).execute(Mockito.any()); + } + + private TelemetrySynchronizer getTelemetrySynchronizer(CloseableHttpClient httpClient) throws URISyntaxException, InvocationTargetException, NoSuchMethodException, IllegalAccessException, IOException { + TelemetryStorageConsumer consumer = Mockito.mock(InMemoryTelemetryStorage.class); + SplitCache splitCache = Mockito.mock(SplitCache.class); + SegmentCache segmentCache = Mockito.mock(SegmentCacheInMemoryImpl.class); + TelemetrySynchronizer telemetrySynchronizer = new TelemetrySynchronizerImp(httpClient, URI.create(TELEMETRY_ENDPOINT), consumer, splitCache, segmentCache); + return telemetrySynchronizer; + } } \ No newline at end of file diff --git a/pom.xml b/pom.xml index a0384c7cf..147baff25 100644 --- a/pom.xml +++ b/pom.xml @@ -4,7 +4,7 @@ 4.0.0 io.split.client java-client-parent - 4.1.5 + 4.2.0-rc1 diff --git a/testing/pom.xml b/testing/pom.xml index 97f24a133..a93c32b14 100644 --- a/testing/pom.xml +++ b/testing/pom.xml @@ -6,7 +6,7 @@ io.split.client java-client-parent - 4.1.5 + 4.2.0-rc1 java-client-testing From 02f48e29c04e4cd9451a21ce0d71ac52f746e260 Mon Sep 17 00:00:00 2001 From: Lucas Echeverz Date: Tue, 27 Apr 2021 16:19:19 -0300 Subject: [PATCH 21/81] Fixing consts --- client/src/main/java/io/split/service/HttpPostImp.java | 5 ++--- client/src/test/java/io/split/service/HttpPostImpTest.java | 7 +++---- .../synchronizer/TelemetrySynchronizerImpTest.java | 5 +++-- 3 files changed, 8 insertions(+), 9 deletions(-) diff --git a/client/src/main/java/io/split/service/HttpPostImp.java b/client/src/main/java/io/split/service/HttpPostImp.java index afd72da2d..3555d01e0 100644 --- a/client/src/main/java/io/split/service/HttpPostImp.java +++ b/client/src/main/java/io/split/service/HttpPostImp.java @@ -4,6 +4,7 @@ import org.apache.hc.client5.http.impl.classic.CloseableHttpClient; import org.apache.hc.client5.http.impl.classic.CloseableHttpResponse; import org.apache.hc.core5.http.HttpEntity; +import org.apache.hc.core5.http.HttpStatus; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.apache.hc.client5.http.classic.methods.HttpPost; @@ -11,8 +12,6 @@ import java.net.URI; public class HttpPostImp { - private static final int STATUS_OKEY = 200; - private static final int STATUS_MIN = 300; private static final Logger _logger = LoggerFactory.getLogger(HttpPostImp.class); private CloseableHttpClient _client; @@ -32,7 +31,7 @@ public void post(URI uri, Object object, String posted) { int status = response.getCode(); - if (status < STATUS_OKEY || status >= STATUS_MIN) { + if (status < HttpStatus.SC_OK || status >= HttpStatus.SC_MULTIPLE_CHOICES) { _logger.warn("Response status was: " + status); } diff --git a/client/src/test/java/io/split/service/HttpPostImpTest.java b/client/src/test/java/io/split/service/HttpPostImpTest.java index 69c28b78e..1041d4593 100644 --- a/client/src/test/java/io/split/service/HttpPostImpTest.java +++ b/client/src/test/java/io/split/service/HttpPostImpTest.java @@ -3,6 +3,7 @@ import io.split.TestHelper; import junit.framework.TestCase; import org.apache.hc.client5.http.impl.classic.CloseableHttpClient; +import org.apache.hc.core5.http.HttpStatus; import org.junit.Test; import org.mockito.Mockito; @@ -13,12 +14,10 @@ public class HttpPostImpTest{ private static final String URL = "www.split.io"; - private static int SUCCESSS_CODE = 200; - private static int ERROR_CODE = 400; @Test public void testPostWith200() throws InvocationTargetException, NoSuchMethodException, IllegalAccessException, IOException { - CloseableHttpClient client =TestHelper.mockHttpClient(URL, SUCCESSS_CODE); + CloseableHttpClient client =TestHelper.mockHttpClient(URL, HttpStatus.SC_OK); HttpPostImp httpPostImp = new HttpPostImp(client); httpPostImp.post(URI.create(URL), new Object(), "Metrics"); Mockito.verify(client, Mockito.times(1)).execute(Mockito.any()); @@ -26,7 +25,7 @@ public void testPostWith200() throws InvocationTargetException, NoSuchMethodExce @Test public void testPostWith400() throws InvocationTargetException, NoSuchMethodException, IllegalAccessException, IOException { - CloseableHttpClient client =TestHelper.mockHttpClient(URL, ERROR_CODE); + CloseableHttpClient client =TestHelper.mockHttpClient(URL, HttpStatus.SC_CLIENT_ERROR); HttpPostImp httpPostImp = new HttpPostImp(client); httpPostImp.post(URI.create(URL), new Object(), "Metrics"); Mockito.verify(client, Mockito.times(1)).execute(Mockito.any()); diff --git a/client/src/test/java/io/split/telemetry/synchronizer/TelemetrySynchronizerImpTest.java b/client/src/test/java/io/split/telemetry/synchronizer/TelemetrySynchronizerImpTest.java index 9ab844377..182f68d61 100644 --- a/client/src/test/java/io/split/telemetry/synchronizer/TelemetrySynchronizerImpTest.java +++ b/client/src/test/java/io/split/telemetry/synchronizer/TelemetrySynchronizerImpTest.java @@ -13,6 +13,7 @@ import org.apache.hc.client5.http.impl.classic.CloseableHttpClient; import org.apache.hc.client5.http.impl.classic.CloseableHttpResponse; import org.apache.hc.core5.http.ClassicHttpRequest; +import org.apache.hc.core5.http.HttpStatus; import org.junit.Test; import org.mockito.Mock; import org.mockito.Mockito; @@ -30,7 +31,7 @@ public class TelemetrySynchronizerImpTest{ @Test public void testSynchronizeConfig() throws URISyntaxException, NoSuchMethodException, IOException, IllegalAccessException, InvocationTargetException { - CloseableHttpClient httpClient = TestHelper.mockHttpClient("", 200); + CloseableHttpClient httpClient = TestHelper.mockHttpClient(TELEMETRY_ENDPOINT, HttpStatus.SC_OK); TelemetrySynchronizer telemetrySynchronizer = getTelemetrySynchronizer(httpClient); SplitClientConfig splitClientConfig = SplitClientConfig.builder().build(); @@ -41,7 +42,7 @@ public void testSynchronizeConfig() throws URISyntaxException, NoSuchMethodExcep @Test public void testSynchronizeStats() throws Exception { - CloseableHttpClient httpClient = TestHelper.mockHttpClient("", 200); + CloseableHttpClient httpClient = TestHelper.mockHttpClient(TELEMETRY_ENDPOINT, HttpStatus.SC_OK); TelemetrySynchronizer telemetrySynchronizer = getTelemetrySynchronizer(httpClient); telemetrySynchronizer.synchronizeStats(); From 86be0c08be2e7f895b8ef6528b13cdd3e3fd2074 Mon Sep 17 00:00:00 2001 From: Lucas Echeverz Date: Tue, 27 Apr 2021 18:41:14 -0300 Subject: [PATCH 22/81] FIxing travis --- .../io/split/client/SplitClientConfig.java | 86 +++++++++---------- 1 file changed, 40 insertions(+), 46 deletions(-) diff --git a/client/src/main/java/io/split/client/SplitClientConfig.java b/client/src/main/java/io/split/client/SplitClientConfig.java index 9e7e6fead..a98efdc1f 100644 --- a/client/src/main/java/io/split/client/SplitClientConfig.java +++ b/client/src/main/java/io/split/client/SplitClientConfig.java @@ -99,7 +99,7 @@ private SplitClientConfig(String endpoint, String authServiceURL, String streamingServiceURL, String telemetryURL, - int telemetryRefreshRate) { + int telemetryRefreshRate, int onDemandFetchRetryDelayMs) { _endpoint = endpoint; _eventsEndpoint = eventsEndpoint; @@ -308,8 +308,6 @@ public static final class Builder { private String _streamingServiceURL = STREAMING_ENDPOINT; private String _telemetryURl = TELEMETRY_ENDPOINT; private int _telemetryRefreshRate = 60; - private String _authServiceURL = "https://auth.split.io/api/auth"; - private String _streamingServiceURL = "https://streaming.split.io/sse"; private int _onDemandFetchRetryDelayMs = 50; public Builder() { @@ -718,14 +716,8 @@ public Builder telemetryURL(String telemetryURL) { * @param telemetryRefreshRate * @return this builder */ - public Builder telemetryBuilder(int telemetryRefreshRate) { + public Builder telemetryRefreshRate(int telemetryRefreshRate) { _telemetryRefreshRate = telemetryRefreshRate; - * Set Streaming retry delay. - * @param onDemandFetchRetryDelayMs - * @return - */ - public Builder streamingRetryDelay(int onDemandFetchRetryDelayMs) { - _onDemandFetchRetryDelayMs = onDemandFetchRetryDelayMs; return this; } @@ -801,44 +793,46 @@ public SplitClientConfig build() { if (_telemetryURl == null) { throw new IllegalArgumentException("telemetryURl must not be null"); - if(_onDemandFetchRetryDelayMs <= 0) { + } + + if (_onDemandFetchRetryDelayMs <= 0) { throw new IllegalStateException("streamingRetryDelay must be > 0"); } - return new SplitClientConfig( - _endpoint, - _eventsEndpoint, - _featuresRefreshRate, - _segmentsRefreshRate, - _impressionsRefreshRate, - _impressionsQueueSize, - _impressionsMode, - _metricsRefreshRate, - _connectionTimeout, - _readTimeout, - _numThreadsForSegmentFetch, - _ready, - _debugEnabled, - _labelsEnabled, - _ipAddressEnabled, - _waitBeforeShutdown, - proxy(), - _proxyUsername, - _proxyPassword, - _eventsQueueSize, - _eventFlushIntervalInMillis, - _maxStringLength, - _destroyOnShutDown, - _splitFile, - _integrationsConfig, - _streamingEnabled, - _authRetryBackoffBase, - _streamingReconnectBackoffBase, - _authServiceURL, - _streamingServiceURL, - _telemetryURl, - _telemetryRefreshRate); - _onDemandFetchRetryDelayMs); - } + return new SplitClientConfig( + _endpoint, + _eventsEndpoint, + _featuresRefreshRate, + _segmentsRefreshRate, + _impressionsRefreshRate, + _impressionsQueueSize, + _impressionsMode, + _metricsRefreshRate, + _connectionTimeout, + _readTimeout, + _numThreadsForSegmentFetch, + _ready, + _debugEnabled, + _labelsEnabled, + _ipAddressEnabled, + _waitBeforeShutdown, + proxy(), + _proxyUsername, + _proxyPassword, + _eventsQueueSize, + _eventFlushIntervalInMillis, + _maxStringLength, + _destroyOnShutDown, + _splitFile, + _integrationsConfig, + _streamingEnabled, + _authRetryBackoffBase, + _streamingReconnectBackoffBase, + _authServiceURL, + _streamingServiceURL, + _telemetryURl, + _telemetryRefreshRate, + _onDemandFetchRetryDelayMs); + } } } From 71c0ca8cefb41a5c302deb51e4eb58f87f3f75c4 Mon Sep 17 00:00:00 2001 From: Martin Redolatti Date: Tue, 27 Apr 2021 18:47:56 -0300 Subject: [PATCH 23/81] add logging for cdn headers --- client/pom.xml | 2 +- .../io/split/client/SplitClientConfig.java | 3 + .../io/split/client/SplitFactoryImpl.java | 39 ++++++++++-- .../engine/common/FastlyHeadersCaptor.java | 4 +- .../io/split/engine/common/FetchOptions.java | 15 +++++ .../split/engine/common/SyncManagerImp.java | 10 ++- .../split/engine/common/SynchronizerImp.java | 38 ++++++------ .../io/split/engine/sse/workers/Worker.java | 1 + client/src/test/java/io/split/TestHelper.java | 3 +- .../common/FastlyHeadersCaptorTest.java | 62 +++++++++++++++++++ .../engine/common/FetcherOptionsTest.java | 47 ++++++++++++++ .../split/engine/common/SynchronizerTest.java | 2 +- .../engine/experiments/SplitFetcherTest.java | 7 ++- pom.xml | 2 +- testing/pom.xml | 2 +- 15 files changed, 200 insertions(+), 37 deletions(-) create mode 100644 client/src/test/java/io/split/engine/common/FastlyHeadersCaptorTest.java create mode 100644 client/src/test/java/io/split/engine/common/FetcherOptionsTest.java diff --git a/client/pom.xml b/client/pom.xml index 8ca259a9b..c797762fa 100644 --- a/client/pom.xml +++ b/client/pom.xml @@ -5,7 +5,7 @@ io.split.client java-client-parent - 4.1.7-rc1 + 4.1.7-rc2 java-client jar diff --git a/client/src/main/java/io/split/client/SplitClientConfig.java b/client/src/main/java/io/split/client/SplitClientConfig.java index 0cfa44eec..1ce3138b6 100644 --- a/client/src/main/java/io/split/client/SplitClientConfig.java +++ b/client/src/main/java/io/split/client/SplitClientConfig.java @@ -256,6 +256,9 @@ public String streamingServiceURL() { public int streamingRetryDelay() {return _onDemandFetchRetryDelayMs;} + public boolean cdnDebugLogging() { return _cdnDebugLogging; } + + public static final class Builder { private String _endpoint = "https://sdk.split.io"; diff --git a/client/src/main/java/io/split/client/SplitFactoryImpl.java b/client/src/main/java/io/split/client/SplitFactoryImpl.java index cf5224e8a..e927460b2 100644 --- a/client/src/main/java/io/split/client/SplitFactoryImpl.java +++ b/client/src/main/java/io/split/client/SplitFactoryImpl.java @@ -129,7 +129,9 @@ public SplitFactoryImpl(String apiToken, SplitClientConfig config) throws URISyn _splitFetcher = buildSplitFetcher(); // SplitSynchronizationTask - _splitSynchronizationTask = new SplitSynchronizationTask(_splitFetcher, _splitCache, findPollingPeriod(RANDOM, config.featuresRefreshRate())); + _splitSynchronizationTask = new SplitSynchronizationTask(_splitFetcher, + _splitCache, + findPollingPeriod(RANDOM, config.featuresRefreshRate())); // Impressions _impressionsManager = buildImpressionsManager(config); @@ -138,27 +140,52 @@ public SplitFactoryImpl(String apiToken, SplitClientConfig config) throws URISyn _cachedFireAndForgetMetrics = buildCachedFireAndForgetMetrics(config); // EventClient - _eventClient = EventClientImpl.create(_httpclient, _eventsRootTarget, config.eventsQueueSize(), config.eventFlushIntervalInMillis(), config.waitBeforeShutdown()); + _eventClient = EventClientImpl.create(_httpclient, + _eventsRootTarget, + config.eventsQueueSize(), + config.eventFlushIntervalInMillis(), + config.waitBeforeShutdown()); // SyncManager - _syncManager = SyncManagerImp.build(config.streamingEnabled(), _splitSynchronizationTask, _splitFetcher, _segmentSynchronizationTaskImp, _splitCache, config.authServiceURL(), _httpclient, config.streamingServiceURL(), config.authRetryBackoffBase(), buildSSEdHttpClient(config), _segmentCache, config.streamingRetryDelay()); + _syncManager = SyncManagerImp.build(config.streamingEnabled(), + _splitSynchronizationTask, + _splitFetcher, + _segmentSynchronizationTaskImp, + _splitCache, + config.authServiceURL(), + _httpclient, + config.streamingServiceURL(), + config.authRetryBackoffBase(), + buildSSEdHttpClient(config), + _segmentCache, + config.streamingRetryDelay(), + config.cdnDebugLogging()); _syncManager.start(); // Evaluator _evaluator = new EvaluatorImp(_splitCache); // SplitClient - _client = new SplitClientImpl(this, _splitCache, _impressionsManager, _cachedFireAndForgetMetrics, _eventClient, config, _gates, _evaluator); + _client = new SplitClientImpl(this, + _splitCache, + _impressionsManager, + _cachedFireAndForgetMetrics, + _eventClient, + config, + _gates, + _evaluator); // SplitManager _manager = new SplitManagerImpl(_splitCache, config, _gates); // DestroyOnShutDown if (config.destroyOnShutDown()) { - Runtime.getRuntime().addShutdownHook(new Thread(() -> { + Thread shutdown = new Thread(() -> { // Using the full path to avoid conflicting with Thread.destroy() SplitFactoryImpl.this.destroy(); - })); + }); + shutdown.setName("split-destroy-worker"); + Runtime.getRuntime().addShutdownHook(shutdown); } } diff --git a/client/src/main/java/io/split/engine/common/FastlyHeadersCaptor.java b/client/src/main/java/io/split/engine/common/FastlyHeadersCaptor.java index c96eb6b94..143f9523b 100644 --- a/client/src/main/java/io/split/engine/common/FastlyHeadersCaptor.java +++ b/client/src/main/java/io/split/engine/common/FastlyHeadersCaptor.java @@ -5,7 +5,7 @@ public class FastlyHeadersCaptor { - public static Set FIELDS_TO_CAPTURE = new HashSet<>(Arrays.asList( + private static final Set HEADERS_TO_CAPTURE = new HashSet<>(Arrays.asList( "Fastly-Debug-Path", "Fastly-Debug-TTL", "Fastly-Debug-Digest", @@ -24,7 +24,7 @@ public class FastlyHeadersCaptor { public Void handle(Map responseHeaders) { _headers.add(responseHeaders.entrySet().stream() - .filter(e -> FIELDS_TO_CAPTURE.contains(e.getKey())) + .filter(e -> HEADERS_TO_CAPTURE.contains(e.getKey())) .collect(Collectors.toMap(Map.Entry::getKey, Map.Entry::getValue))); return null; } diff --git a/client/src/main/java/io/split/engine/common/FetchOptions.java b/client/src/main/java/io/split/engine/common/FetchOptions.java index 251c15516..ca51b8206 100644 --- a/client/src/main/java/io/split/engine/common/FetchOptions.java +++ b/client/src/main/java/io/split/engine/common/FetchOptions.java @@ -1,5 +1,8 @@ package io.split.engine.common; +import io.split.engine.matchers.AttributeMatcher; +import org.checkerframework.checker.units.qual.A; + import java.util.Map; import java.util.Objects; import java.util.function.Function; @@ -55,6 +58,18 @@ private FetchOptions(boolean cacheControlHeaders, Function, _fastlyDebugHeader = fastlyDebugHeader; } + public boolean equals(Object obj) { + if (null == obj) return false; + if (this == obj) return true; + if (!(obj instanceof FetchOptions)) return false; + + FetchOptions other = (FetchOptions) obj; + + return Objects.equals(_cacheControlHeaders, other._cacheControlHeaders) + && Objects.equals(_fastlyDebugHeader, other._fastlyDebugHeader) + && Objects.equals(_responseHeadersCallback, other._responseHeadersCallback); + } + private final boolean _cacheControlHeaders; private final boolean _fastlyDebugHeader; private final Function, Void> _responseHeadersCallback; diff --git a/client/src/main/java/io/split/engine/common/SyncManagerImp.java b/client/src/main/java/io/split/engine/common/SyncManagerImp.java index 69e6c75ae..4095275d0 100644 --- a/client/src/main/java/io/split/engine/common/SyncManagerImp.java +++ b/client/src/main/java/io/split/engine/common/SyncManagerImp.java @@ -60,9 +60,10 @@ public static SyncManagerImp build(boolean streamingEnabledConfig, int authRetryBackOffBase, CloseableHttpClient sseHttpClient, SegmentCache segmentCache, - int streamingRetryDelay) { + int streamingRetryDelay, + boolean cdnDebugLogging) { LinkedBlockingQueue pushMessages = new LinkedBlockingQueue<>(); - Synchronizer synchronizer = new SynchronizerImp(splitSynchronizationTask, splitFetcher, segmentSynchronizationTaskImp, splitCache, segmentCache, streamingRetryDelay); + Synchronizer synchronizer = new SynchronizerImp(splitSynchronizationTask, splitFetcher, segmentSynchronizationTaskImp, splitCache, segmentCache, streamingRetryDelay, cdnDebugLogging); PushManager pushManager = PushManagerImp.build(synchronizer, streamingServiceUrl, authUrl, httpClient, pushMessages, sseHttpClient); return new SyncManagerImp(streamingEnabledConfig, synchronizer, pushManager, pushMessages, authRetryBackOffBase); } @@ -111,14 +112,16 @@ private void startPollingMode() { _pushManager.startWorkers(); _pushManager.scheduleConnectionReset(); _backoff.reset(); + _log.info("Streaming up and running."); break; case STREAMING_DOWN: + _log.info("Streaming service temporarily unavailable, working in polling mode."); _pushManager.stopWorkers(); _synchronizer.startPeriodicFetching(); break; case STREAMING_BACKOFF: long howLong = _backoff.interval() * 1000; - _log.error(String.format("Retryable error in streaming subsystem. Switching to polling and retrying in %d seconds", howLong/1000)); + _log.info(String.format("Retryable error in streaming subsystem. Switching to polling and retrying in %d seconds", howLong/1000)); _synchronizer.startPeriodicFetching(); _pushManager.stopWorkers(); _pushManager.stop(); @@ -127,6 +130,7 @@ private void startPollingMode() { _pushManager.start(); break; case STREAMING_OFF: + _log.info("Unrecoverable error in streaming subsystem. SDK will work in polling-mode and will not retry an SSE connection."); _pushManager.stop(); _synchronizer.startPeriodicFetching(); if (null != _pushStatusMonitorTask) { diff --git a/client/src/main/java/io/split/engine/common/SynchronizerImp.java b/client/src/main/java/io/split/engine/common/SynchronizerImp.java index dd325e2dd..155af6767 100644 --- a/client/src/main/java/io/split/engine/common/SynchronizerImp.java +++ b/client/src/main/java/io/split/engine/common/SynchronizerImp.java @@ -12,9 +12,6 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import java.util.ArrayList; -import java.util.List; -import java.util.Map; import java.util.concurrent.Executors; import java.util.concurrent.ScheduledExecutorService; import java.util.concurrent.ThreadFactory; @@ -23,10 +20,10 @@ import static com.google.common.base.Preconditions.checkNotNull; public class SynchronizerImp implements Synchronizer { - private static final Logger _log = LoggerFactory.getLogger(Synchronizer.class); - private static final int RETRIES_NUMBER = 10; - private final boolean _cdnResponseHeadersLogging = true; + private static final int MAX_ATTEMPTS = 10; + + private static final Logger _log = LoggerFactory.getLogger(Synchronizer.class); private final SplitSynchronizationTask _splitSynchronizationTask; private final SplitFetcher _splitFetcher; private final SegmentSynchronizationTask _segmentSynchronizationTaskImp; @@ -34,6 +31,7 @@ public class SynchronizerImp implements Synchronizer { private final SplitCache _splitCache; private final SegmentCache _segmentCache; private final int _onDemandFetchRetryDelayMs; + private final boolean _cdnResponseHeadersLogging; private final Gson gson = new GsonBuilder().create(); public SynchronizerImp(SplitSynchronizationTask splitSynchronizationTask, @@ -41,13 +39,15 @@ public SynchronizerImp(SplitSynchronizationTask splitSynchronizationTask, SegmentSynchronizationTask segmentSynchronizationTaskImp, SplitCache splitCache, SegmentCache segmentCache, - int onDemandFetchRetryDelayMs) { + int onDemandFetchRetryDelayMs, + boolean cdnResponseHeadersLogging) { _splitSynchronizationTask = checkNotNull(splitSynchronizationTask); _splitFetcher = checkNotNull(splitFetcher); _segmentSynchronizationTaskImp = checkNotNull(segmentSynchronizationTaskImp); _splitCache = checkNotNull(splitCache); _segmentCache = checkNotNull(segmentCache); _onDemandFetchRetryDelayMs = checkNotNull(onDemandFetchRetryDelayMs); + _cdnResponseHeadersLogging = cdnResponseHeadersLogging; ThreadFactory splitsThreadFactory = new ThreadFactoryBuilder() .setDaemon(true) @@ -80,7 +80,7 @@ public void stopPeriodicFetching() { @Override public void refreshSplits(long targetChangeNumber) { - int retries = RETRIES_NUMBER; + if (targetChangeNumber <= _splitCache.getChangeNumber()) { return; } @@ -92,18 +92,16 @@ public void refreshSplits(long targetChangeNumber) { .responseHeadersCallback(_cdnResponseHeadersLogging ? captor::handle : null) .build(); + int remainingAttempts = MAX_ATTEMPTS; while(true) { - retries--; + remainingAttempts--; _splitFetcher.forceRefresh(opts); if (targetChangeNumber <= _splitCache.getChangeNumber()) { - _log.debug("Refresh completed in %s attempts.", RETRIES_NUMBER - retries); - return; - } else if (retries <= 0) { - _log.warn("No changes fetched after %s attempts.", RETRIES_NUMBER); - if (_cdnResponseHeadersLogging) { - _log.debug("CDN Debug headers: ", gson.toJson(captor.get())); - } - return; + _log.debug(String.format("Refresh completed in %s attempts.", MAX_ATTEMPTS - remainingAttempts)); + break; + } else if (remainingAttempts <= 0) { + _log.info(String.format("No changes fetched after %s attempts.", MAX_ATTEMPTS)); + break; } try { Thread.sleep(_onDemandFetchRetryDelayMs); @@ -112,6 +110,10 @@ public void refreshSplits(long targetChangeNumber) { _log.debug("Error trying to sleep current Thread."); } } + + if (_cdnResponseHeadersLogging && remainingAttempts <= (MAX_ATTEMPTS / 2)) { + _log.info(String.format("CDN Debug headers: %s", gson.toJson(captor.get()))); + } } @Override @@ -125,7 +127,7 @@ public void localKillSplit(String splitName, String defaultTreatment, long newCh @Override public void refreshSegment(String segmentName, long changeNumber) { int retries = 1; - while(changeNumber > _segmentCache.getChangeNumber(segmentName) && retries <= RETRIES_NUMBER) { + while(changeNumber > _segmentCache.getChangeNumber(segmentName) && retries <= MAX_ATTEMPTS) { SegmentFetcher fetcher = _segmentSynchronizationTaskImp.getFetcher(segmentName); try{ fetcher.fetch(true); diff --git a/client/src/main/java/io/split/engine/sse/workers/Worker.java b/client/src/main/java/io/split/engine/sse/workers/Worker.java index d6b8db5d4..7d2dd21ab 100644 --- a/client/src/main/java/io/split/engine/sse/workers/Worker.java +++ b/client/src/main/java/io/split/engine/sse/workers/Worker.java @@ -25,6 +25,7 @@ public void start() { _log.debug(String.format("%s Worker starting ...", _workerName)); _queue.clear(); _thread = new Thread( this); + _thread.setName(String.format("%s-worker", _workerName)); _thread.start(); } else { _log.debug(String.format("%s Worker already running.", _workerName)); diff --git a/client/src/test/java/io/split/TestHelper.java b/client/src/test/java/io/split/TestHelper.java index b0e424a56..1cb95eaa8 100644 --- a/client/src/test/java/io/split/TestHelper.java +++ b/client/src/test/java/io/split/TestHelper.java @@ -3,6 +3,7 @@ import org.apache.hc.client5.http.impl.classic.CloseableHttpClient; import org.apache.hc.client5.http.impl.classic.CloseableHttpResponse; import org.apache.hc.core5.http.ClassicHttpResponse; +import org.apache.hc.core5.http.Header; import org.apache.hc.core5.http.HttpEntity; import org.mockito.Mockito; @@ -18,7 +19,7 @@ public static CloseableHttpClient mockHttpClient(String jsonName, int httpStatus ClassicHttpResponse httpResponseMock = Mockito.mock(ClassicHttpResponse.class); Mockito.when(httpResponseMock.getEntity()).thenReturn(entityMock); Mockito.when(httpResponseMock.getCode()).thenReturn(httpStatus); - + Mockito.when(httpResponseMock.getHeaders()).thenReturn(new Header[0]); CloseableHttpClient httpClientMock = Mockito.mock(CloseableHttpClient.class); Mockito.when(httpClientMock.execute(Mockito.anyObject())).thenReturn(classicResponseToCloseableMock(httpResponseMock)); diff --git a/client/src/test/java/io/split/engine/common/FastlyHeadersCaptorTest.java b/client/src/test/java/io/split/engine/common/FastlyHeadersCaptorTest.java new file mode 100644 index 000000000..7fd4a4106 --- /dev/null +++ b/client/src/test/java/io/split/engine/common/FastlyHeadersCaptorTest.java @@ -0,0 +1,62 @@ +package io.split.engine.common; + +import org.junit.Test; + +import java.util.stream.Collectors; +import java.util.stream.Stream; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; + +public class FastlyHeadersCaptorTest { + + @Test + public void filterWorks() { + FastlyHeadersCaptor captor = new FastlyHeadersCaptor(); + captor.handle(Stream.of(new String[][] { + {"Fastly-Debug-Path", "something"}, + {"Fastly-Debug-TTL", "something"}, + {"Fastly-Debug-Digest", "something"}, + {"X-Served-By", "something"}, + {"X-Cache", "something"}, + {"X-Cache-Hits", "something"}, + {"X-Timer", "something"}, + {"Surrogate-Key", "something"}, + {"ETag", "something"}, + {"Cache-Control", "something"}, + {"X-Request-ID", "something"}, + {"Last-Modified", "something"}, + {"NON_IMPORTANT_1", "something"}, + {"ANOTHER_NON_IMPORTANT", "something"} + }).collect(Collectors.toMap(d -> d[0], d -> d[1]))); + + assertEquals(captor.get().size(), 1); + assertEquals(captor.get().get(0).size(), 12); + assertFalse(captor.get().get(0).containsKey("NON_IMPORTANT_1")); + assertFalse(captor.get().get(0).containsKey("ANOTHER_NON_IMPORTANT")); + } + + @Test + public void orderIsPreserved() { + FastlyHeadersCaptor captor = new FastlyHeadersCaptor(); + captor.handle(Stream.of(new String[][]{ + {"Fastly-Debug-Path", "first"}, + }).collect(Collectors.toMap(d -> d[0], d -> d[1]))); + + captor.handle(Stream.of(new String[][]{ + {"Fastly-Debug-Path", "second"}, + }).collect(Collectors.toMap(d -> d[0], d -> d[1]))); + + captor.handle(Stream.of(new String[][]{ + {"Fastly-Debug-Path", "third"}, + }).collect(Collectors.toMap(d -> d[0], d -> d[1]))); + + assertEquals(captor.get().size(), 3); + assertEquals(captor.get().get(0).size(), 1); + assertEquals(captor.get().get(1).size(), 1); + assertEquals(captor.get().get(2).size(), 1); + assertEquals(captor.get().get(0).get("Fastly-Debug-Path"), "first"); + assertEquals(captor.get().get(1).get("Fastly-Debug-Path"), "second"); + assertEquals(captor.get().get(2).get("Fastly-Debug-Path"), "third"); + } +} diff --git a/client/src/test/java/io/split/engine/common/FetcherOptionsTest.java b/client/src/test/java/io/split/engine/common/FetcherOptionsTest.java new file mode 100644 index 000000000..56b5b6da9 --- /dev/null +++ b/client/src/test/java/io/split/engine/common/FetcherOptionsTest.java @@ -0,0 +1,47 @@ +package io.split.engine.common; + +import org.junit.Test; + +import java.util.HashMap; +import java.util.Map; +import java.util.function.Function; + +import static org.junit.Assert.assertEquals; + +public class FetcherOptionsTest { + + @Test + public void optionsPropagatedOk() { + final boolean[] called = {false}; + Function, Void> func = new Function, Void>() { + @Override + public Void apply(Map unused) { + called[0] = true; + return null; + } + }; + + FetchOptions options = new FetchOptions.Builder() + .cacheControlHeaders(true) + .fastlyDebugHeader(true) + .responseHeadersCallback(func) + .build(); + + assertEquals(options.cacheControlHeadersEnabled(), true); + assertEquals(options.fastlyDebugHeaderEnabled(), true); + options.handleResponseHeaders(new HashMap<>()); + assertEquals(called[0], true); + } + + @Test + public void nullHandlerDoesNotExplode() { + + FetchOptions options = new FetchOptions.Builder() + .cacheControlHeaders(true) + .fastlyDebugHeader(true) + .responseHeadersCallback(null) + .build(); + + options.handleResponseHeaders(new HashMap<>()); + } +} diff --git a/client/src/test/java/io/split/engine/common/SynchronizerTest.java b/client/src/test/java/io/split/engine/common/SynchronizerTest.java index 742b1d3f8..2438bbdac 100644 --- a/client/src/test/java/io/split/engine/common/SynchronizerTest.java +++ b/client/src/test/java/io/split/engine/common/SynchronizerTest.java @@ -26,7 +26,7 @@ public void beforeMethod() { _splitCache = Mockito.mock(SplitCache.class); _segmentCache = Mockito.mock(SegmentCache.class); - _synchronizer = new SynchronizerImp(_refreshableSplitFetcherTask, _splitFetcher, _segmentFetcher, _splitCache, _segmentCache, 50); + _synchronizer = new SynchronizerImp(_refreshableSplitFetcherTask, _splitFetcher, _segmentFetcher, _splitCache, _segmentCache, 50, false); } @Test diff --git a/client/src/test/java/io/split/engine/experiments/SplitFetcherTest.java b/client/src/test/java/io/split/engine/experiments/SplitFetcherTest.java index a68fc8435..ec936a64b 100644 --- a/client/src/test/java/io/split/engine/experiments/SplitFetcherTest.java +++ b/client/src/test/java/io/split/engine/experiments/SplitFetcherTest.java @@ -17,6 +17,7 @@ import io.split.grammar.Treatments; import org.junit.Test; import org.mockito.Mockito; +import org.mockito.internal.matchers.Any; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -123,9 +124,9 @@ public void when_parser_fails_we_remove_the_experiment() throws InterruptedExcep noReturn.till = 1L; SplitChangeFetcher splitChangeFetcher = mock(SplitChangeFetcher.class); - when(splitChangeFetcher.fetch(-1L, new FetchOptions.Builder().build())).thenReturn(validReturn); - when(splitChangeFetcher.fetch(0L, new FetchOptions.Builder().build())).thenReturn(invalidReturn); - when(splitChangeFetcher.fetch(1L, new FetchOptions.Builder().build())).thenReturn(noReturn); + when(splitChangeFetcher.fetch(Mockito.eq(-1L), Mockito.any())).thenReturn(validReturn); + when(splitChangeFetcher.fetch(Mockito.eq(0L), Mockito.any())).thenReturn(invalidReturn); + when(splitChangeFetcher.fetch(Mockito.eq(1L), Mockito.any())).thenReturn(noReturn); SegmentCache segmentCache = new SegmentCacheInMemoryImpl(); diff --git a/pom.xml b/pom.xml index 97eb1deb1..b6fe51919 100644 --- a/pom.xml +++ b/pom.xml @@ -4,7 +4,7 @@ 4.0.0 io.split.client java-client-parent - 4.1.7-rc1 + 4.1.7-rc2 diff --git a/testing/pom.xml b/testing/pom.xml index af974473c..d52d88807 100644 --- a/testing/pom.xml +++ b/testing/pom.xml @@ -6,7 +6,7 @@ io.split.client java-client-parent - 4.1.7-rc1 + 4.1.7-rc2 java-client-testing From d7f0531c32fbdfa4aed1e340ab710ce7b31cb5af Mon Sep 17 00:00:00 2001 From: Martin Redolatti Date: Tue, 27 Apr 2021 19:05:51 -0300 Subject: [PATCH 24/81] true by default --- client/src/main/java/io/split/client/SplitClientConfig.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/client/src/main/java/io/split/client/SplitClientConfig.java b/client/src/main/java/io/split/client/SplitClientConfig.java index 1ce3138b6..e245125c0 100644 --- a/client/src/main/java/io/split/client/SplitClientConfig.java +++ b/client/src/main/java/io/split/client/SplitClientConfig.java @@ -295,7 +295,7 @@ public static final class Builder { private String _authServiceURL = "https://auth.split.io/api/auth"; private String _streamingServiceURL = "https://streaming.split.io/sse"; private int _onDemandFetchRetryDelayMs = 50; - private boolean _cdnDebugLogging = false; + private boolean _cdnDebugLogging = true; public Builder() { } From f49ba00dee3c241d6021712483e79ecad77c5192 Mon Sep 17 00:00:00 2001 From: Martin Redolatti Date: Tue, 27 Apr 2021 19:11:09 -0300 Subject: [PATCH 25/81] sonarqube comment --- .../src/main/java/io/split/engine/common/FetchOptions.java | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/client/src/main/java/io/split/engine/common/FetchOptions.java b/client/src/main/java/io/split/engine/common/FetchOptions.java index ca51b8206..ce064c349 100644 --- a/client/src/main/java/io/split/engine/common/FetchOptions.java +++ b/client/src/main/java/io/split/engine/common/FetchOptions.java @@ -58,6 +58,7 @@ private FetchOptions(boolean cacheControlHeaders, Function, _fastlyDebugHeader = fastlyDebugHeader; } + @Override public boolean equals(Object obj) { if (null == obj) return false; if (this == obj) return true; @@ -70,6 +71,11 @@ public boolean equals(Object obj) { && Objects.equals(_responseHeadersCallback, other._responseHeadersCallback); } + @Override + public int hashCode() { + return com.google.common.base.Objects.hashCode(_cacheControlHeaders, _fastlyDebugHeader, _responseHeadersCallback); + } + private final boolean _cacheControlHeaders; private final boolean _fastlyDebugHeader; private final Function, Void> _responseHeadersCallback; From d3c29f08dd0389b02fbf084ebd5acd4ba151ede5 Mon Sep 17 00:00:00 2001 From: Lucas Echeverz Date: Wed, 28 Apr 2021 16:48:38 -0300 Subject: [PATCH 26/81] Adding TelemtrySyncTask --- .../HttpTelemetryMemorySender.java | 7 +--- ...onizerImp.java => SynchronizerMemory.java} | 8 ++-- .../synchronizer/TelemetrySyncTask.java | 37 +++++++++++++++++++ ...pTest.java => SynchronizerMemoryTest.java} | 10 +---- .../synchronizer/TelemetrySyncTaskTest.java | 17 +++++++++ 5 files changed, 60 insertions(+), 19 deletions(-) rename client/src/main/java/io/split/telemetry/synchronizer/{TelemetrySynchronizerImp.java => SynchronizerMemory.java} (95%) create mode 100644 client/src/main/java/io/split/telemetry/synchronizer/TelemetrySyncTask.java rename client/src/test/java/io/split/telemetry/synchronizer/{TelemetrySynchronizerImpTest.java => SynchronizerMemoryTest.java} (82%) create mode 100644 client/src/test/java/io/split/telemetry/synchronizer/TelemetrySyncTaskTest.java diff --git a/client/src/main/java/io/split/telemetry/synchronizer/HttpTelemetryMemorySender.java b/client/src/main/java/io/split/telemetry/synchronizer/HttpTelemetryMemorySender.java index 2f5c932cd..e6c1a6ceb 100644 --- a/client/src/main/java/io/split/telemetry/synchronizer/HttpTelemetryMemorySender.java +++ b/client/src/main/java/io/split/telemetry/synchronizer/HttpTelemetryMemorySender.java @@ -1,14 +1,11 @@ package io.split.telemetry.synchronizer; import com.google.common.annotations.VisibleForTesting; -import io.split.client.impressions.HttpImpressionsSender; import io.split.client.utils.Utils; +import io.split.service.HttpPostImp; import io.split.telemetry.domain.Config; import io.split.telemetry.domain.Stats; -import io.split.service.HttpPostImp; import org.apache.hc.client5.http.impl.classic.CloseableHttpClient; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; import java.net.URI; import java.net.URISyntaxException; @@ -20,8 +17,6 @@ public class HttpTelemetryMemorySender extends HttpPostImp { private static final String CONFIG_METRICS = "Config metrics "; private static final String STATS_METRICS = "Stats metrics "; - private static final Logger _logger = LoggerFactory.getLogger(HttpImpressionsSender.class); - private final URI _impressionConfigTarget; private final URI _impressionStatsTarget; diff --git a/client/src/main/java/io/split/telemetry/synchronizer/TelemetrySynchronizerImp.java b/client/src/main/java/io/split/telemetry/synchronizer/SynchronizerMemory.java similarity index 95% rename from client/src/main/java/io/split/telemetry/synchronizer/TelemetrySynchronizerImp.java rename to client/src/main/java/io/split/telemetry/synchronizer/SynchronizerMemory.java index a6a35d04d..1bd4f0356 100644 --- a/client/src/main/java/io/split/telemetry/synchronizer/TelemetrySynchronizerImp.java +++ b/client/src/main/java/io/split/telemetry/synchronizer/SynchronizerMemory.java @@ -19,12 +19,10 @@ import java.net.URI; import java.net.URISyntaxException; import java.util.ArrayList; -import java.util.Arrays; import java.util.List; import java.util.Map; -import java.util.stream.Collectors; -public class TelemetrySynchronizerImp implements TelemetrySynchronizer{ +public class SynchronizerMemory implements TelemetrySynchronizer{ private static final int OPERATION_MODE = 0; private static final String STORAGE = "memory"; @@ -34,8 +32,8 @@ public class TelemetrySynchronizerImp implements TelemetrySynchronizer{ private SplitCache _splitCache; private SegmentCache _segmentCache; - public TelemetrySynchronizerImp(CloseableHttpClient client, URI telemetryRootEndpoint, TelemetryStorageConsumer telemetryStorageConsumer, SplitCache splitCache, - SegmentCache segmentCache) throws URISyntaxException { + public SynchronizerMemory(CloseableHttpClient client, URI telemetryRootEndpoint, TelemetryStorageConsumer telemetryStorageConsumer, SplitCache splitCache, + SegmentCache segmentCache) throws URISyntaxException { _httpHttpTelemetryMemorySender = HttpTelemetryMemorySender.create(client, telemetryRootEndpoint); _teleTelemetryStorageConsumer = telemetryStorageConsumer; _splitCache = splitCache; diff --git a/client/src/main/java/io/split/telemetry/synchronizer/TelemetrySyncTask.java b/client/src/main/java/io/split/telemetry/synchronizer/TelemetrySyncTask.java new file mode 100644 index 000000000..465de6e5e --- /dev/null +++ b/client/src/main/java/io/split/telemetry/synchronizer/TelemetrySyncTask.java @@ -0,0 +1,37 @@ +package io.split.telemetry.synchronizer; + +import com.google.common.annotations.VisibleForTesting; +import com.google.common.util.concurrent.ThreadFactoryBuilder; + +import java.util.concurrent.Executors; +import java.util.concurrent.ScheduledExecutorService; +import java.util.concurrent.ThreadFactory; +import java.util.concurrent.TimeUnit; + +public class TelemetrySyncTask { + + private final ScheduledExecutorService _telemetrySyncScheduledExecutorService; + private final TelemetrySynchronizer _telemetrySynchronizer; + private final int _telemetryRefreshRate; + + public TelemetrySyncTask(int telemetryRefreshRate, TelemetrySynchronizer telemetrySynchronizer) { + ThreadFactory telemetrySyncThreadFactory = new ThreadFactoryBuilder() + .setDaemon(true) + .setNameFormat("Telemetry-synk-%d") + .build(); + _telemetrySynchronizer = telemetrySynchronizer; //TODO + _telemetryRefreshRate = telemetryRefreshRate; + _telemetrySyncScheduledExecutorService = Executors.newSingleThreadScheduledExecutor(telemetrySyncThreadFactory); + } + + @VisibleForTesting + protected void startScheduledTask() throws Exception { + _telemetrySyncScheduledExecutorService.scheduleWithFixedDelay(() -> { + try { + _telemetrySynchronizer.synchronizeStats(); + } catch (Exception e) { + e.printStackTrace(); + } + },0l, _telemetryRefreshRate, TimeUnit.SECONDS); + } +} diff --git a/client/src/test/java/io/split/telemetry/synchronizer/TelemetrySynchronizerImpTest.java b/client/src/test/java/io/split/telemetry/synchronizer/SynchronizerMemoryTest.java similarity index 82% rename from client/src/test/java/io/split/telemetry/synchronizer/TelemetrySynchronizerImpTest.java rename to client/src/test/java/io/split/telemetry/synchronizer/SynchronizerMemoryTest.java index 182f68d61..d7d9bee8c 100644 --- a/client/src/test/java/io/split/telemetry/synchronizer/TelemetrySynchronizerImpTest.java +++ b/client/src/test/java/io/split/telemetry/synchronizer/SynchronizerMemoryTest.java @@ -1,21 +1,15 @@ package io.split.telemetry.synchronizer; import io.split.TestHelper; -import io.split.cache.InMemoryCacheImp; import io.split.cache.SegmentCache; import io.split.cache.SegmentCacheInMemoryImpl; import io.split.cache.SplitCache; import io.split.client.SplitClientConfig; -import io.split.service.HttpPostImp; import io.split.telemetry.storage.InMemoryTelemetryStorage; import io.split.telemetry.storage.TelemetryStorageConsumer; -import org.apache.hc.client5.http.classic.methods.HttpOptions; import org.apache.hc.client5.http.impl.classic.CloseableHttpClient; -import org.apache.hc.client5.http.impl.classic.CloseableHttpResponse; -import org.apache.hc.core5.http.ClassicHttpRequest; import org.apache.hc.core5.http.HttpStatus; import org.junit.Test; -import org.mockito.Mock; import org.mockito.Mockito; import java.io.IOException; @@ -25,7 +19,7 @@ import java.util.ArrayList; import java.util.HashMap; -public class TelemetrySynchronizerImpTest{ +public class SynchronizerMemoryTest { public static final String TELEMETRY_ENDPOINT = "https://telemetry.split.io/api/v1"; @@ -53,7 +47,7 @@ private TelemetrySynchronizer getTelemetrySynchronizer(CloseableHttpClient httpC TelemetryStorageConsumer consumer = Mockito.mock(InMemoryTelemetryStorage.class); SplitCache splitCache = Mockito.mock(SplitCache.class); SegmentCache segmentCache = Mockito.mock(SegmentCacheInMemoryImpl.class); - TelemetrySynchronizer telemetrySynchronizer = new TelemetrySynchronizerImp(httpClient, URI.create(TELEMETRY_ENDPOINT), consumer, splitCache, segmentCache); + TelemetrySynchronizer telemetrySynchronizer = new SynchronizerMemory(httpClient, URI.create(TELEMETRY_ENDPOINT), consumer, splitCache, segmentCache); return telemetrySynchronizer; } diff --git a/client/src/test/java/io/split/telemetry/synchronizer/TelemetrySyncTaskTest.java b/client/src/test/java/io/split/telemetry/synchronizer/TelemetrySyncTaskTest.java new file mode 100644 index 000000000..9d3e3d460 --- /dev/null +++ b/client/src/test/java/io/split/telemetry/synchronizer/TelemetrySyncTaskTest.java @@ -0,0 +1,17 @@ +package io.split.telemetry.synchronizer; + +import org.junit.Test; +import org.mockito.Mockito; + +public class TelemetrySyncTaskTest { + + @Test + public void testSynchronizationTask() throws Exception { + TelemetrySynchronizer telemetrySynchronizer = Mockito.mock(SynchronizerMemory.class); + TelemetrySyncTask telemetrySyncTask = new TelemetrySyncTask(1, telemetrySynchronizer); + telemetrySyncTask.startScheduledTask(); + Thread.sleep(3000); + Mockito.verify(telemetrySynchronizer, Mockito.times(3)).synchronizeStats(); + } + +} \ No newline at end of file From 35e2f1b84f381c7fed8c1f667afa17139b9593bd Mon Sep 17 00:00:00 2001 From: Lucas Echeverz Date: Wed, 28 Apr 2021 18:24:46 -0300 Subject: [PATCH 27/81] Adding Stop --- .../telemetry/synchronizer/TelemetrySyncTask.java | 9 +++++++++ .../synchronizer/TelemetrySyncTaskTest.java | 13 +++++++++++++ 2 files changed, 22 insertions(+) diff --git a/client/src/main/java/io/split/telemetry/synchronizer/TelemetrySyncTask.java b/client/src/main/java/io/split/telemetry/synchronizer/TelemetrySyncTask.java index 465de6e5e..992e73387 100644 --- a/client/src/main/java/io/split/telemetry/synchronizer/TelemetrySyncTask.java +++ b/client/src/main/java/io/split/telemetry/synchronizer/TelemetrySyncTask.java @@ -34,4 +34,13 @@ protected void startScheduledTask() throws Exception { } },0l, _telemetryRefreshRate, TimeUnit.SECONDS); } + + protected void stopScheduledTask() { + try { + _telemetrySynchronizer.synchronizeStats(); + } catch (Exception e) { + e.printStackTrace(); + } + _telemetrySyncScheduledExecutorService.shutdown(); + } } diff --git a/client/src/test/java/io/split/telemetry/synchronizer/TelemetrySyncTaskTest.java b/client/src/test/java/io/split/telemetry/synchronizer/TelemetrySyncTaskTest.java index 9d3e3d460..9127b3a3d 100644 --- a/client/src/test/java/io/split/telemetry/synchronizer/TelemetrySyncTaskTest.java +++ b/client/src/test/java/io/split/telemetry/synchronizer/TelemetrySyncTaskTest.java @@ -14,4 +14,17 @@ public void testSynchronizationTask() throws Exception { Mockito.verify(telemetrySynchronizer, Mockito.times(3)).synchronizeStats(); } + @Test + public void testStopSynchronizationTask() throws Exception { + TelemetrySynchronizer telemetrySynchronizer = Mockito.mock(SynchronizerMemory.class); + TelemetrySyncTask telemetrySyncTask = new TelemetrySyncTask(1, telemetrySynchronizer); + telemetrySyncTask.startScheduledTask(); + Thread.sleep(3000); + Mockito.verify(telemetrySynchronizer, Mockito.times(3)).synchronizeStats(); + telemetrySyncTask.stopScheduledTask(); + Thread.sleep(2000); + Mockito.verify(telemetrySynchronizer, Mockito.times(4)).synchronizeStats(); + + } + } \ No newline at end of file From 422c156a621fe42945af9b2b02dd588c64f23e01 Mon Sep 17 00:00:00 2001 From: Mauro Sanz Date: Thu, 29 Apr 2021 15:13:46 -0300 Subject: [PATCH 28/81] sdk internal ready implementation --- .../io/split/client/SplitFactoryImpl.java | 4 +-- .../io/split/engine/SDKReadinessGates.java | 24 +++++++++++-- .../split/engine/common/SyncManagerImp.java | 10 +++--- .../split/engine/common/SynchronizerImp.java | 10 ++++-- .../engine/experiments/SplitFetcherImp.java | 6 ++++ .../engine/segments/SegmentFetcherImp.java | 2 -- .../segments/SegmentSynchronizationTask.java | 5 +++ .../SegmentSynchronizationTaskImp.java | 36 ++++++++++++++----- .../split/engine/common/SyncManagerTest.java | 2 +- .../split/engine/common/SynchronizerTest.java | 8 +++-- .../engine/experiments/SplitFetcherTest.java | 6 +++- 11 files changed, 88 insertions(+), 25 deletions(-) diff --git a/client/src/main/java/io/split/client/SplitFactoryImpl.java b/client/src/main/java/io/split/client/SplitFactoryImpl.java index 84b7478a9..c65c8dd89 100644 --- a/client/src/main/java/io/split/client/SplitFactoryImpl.java +++ b/client/src/main/java/io/split/client/SplitFactoryImpl.java @@ -51,7 +51,6 @@ import java.util.ArrayList; import java.util.List; import java.util.Random; -import java.util.concurrent.TimeUnit; import java.util.stream.Collectors; public class SplitFactoryImpl implements SplitFactory { @@ -127,12 +126,11 @@ public SplitFactoryImpl(String apiToken, SplitClientConfig config) throws URISyn // Impressions _impressionsManager = buildImpressionsManager(config); - // EventClient _eventClient = EventClientImpl.create(_httpclient, _eventsRootTarget, config.eventsQueueSize(), config.eventFlushIntervalInMillis(), config.waitBeforeShutdown()); // SyncManager - _syncManager = SyncManagerImp.build(config.streamingEnabled(), _splitSynchronizationTask, _splitFetcher, _segmentSynchronizationTaskImp, _splitCache, config.authServiceURL(), _httpclient, config.streamingServiceURL(), config.authRetryBackoffBase(), buildSSEdHttpClient(config), _segmentCache, config.streamingRetryDelay()); + _syncManager = SyncManagerImp.build(config.streamingEnabled(), _splitSynchronizationTask, _splitFetcher, _segmentSynchronizationTaskImp, _splitCache, config.authServiceURL(), _httpclient, config.streamingServiceURL(), config.authRetryBackoffBase(), buildSSEdHttpClient(config), _segmentCache, config.streamingRetryDelay(), _gates); _syncManager.start(); // Evaluator diff --git a/client/src/main/java/io/split/engine/SDKReadinessGates.java b/client/src/main/java/io/split/engine/SDKReadinessGates.java index ae0fd8ad4..fe6ec0749 100644 --- a/client/src/main/java/io/split/engine/SDKReadinessGates.java +++ b/client/src/main/java/io/split/engine/SDKReadinessGates.java @@ -16,9 +16,9 @@ public class SDKReadinessGates { private static final Logger _log = LoggerFactory.getLogger(SDKReadinessGates.class); private final CountDownLatch _splitsAreReady = new CountDownLatch(1); + private final CountDownLatch _internalReady = new CountDownLatch(1); private final ConcurrentMap _segmentsAreReady = new ConcurrentHashMap<>(); - /** * Returns true if the SDK is ready. The SDK is ready when: *

    @@ -45,7 +45,11 @@ public boolean isSDKReady(long milliseconds) throws InterruptedException { timeLeft = end - System.currentTimeMillis(); - return areSegmentsReady(timeLeft); + boolean ready = areSegmentsReady(timeLeft); + + if (ready) sdkInternalReady(); + + return ready; } public boolean isSDKReadyNow() { @@ -166,4 +170,20 @@ public boolean areSegmentsReady(long milliseconds) throws InterruptedException { public boolean areSplitsReady(long milliseconds) throws InterruptedException { return _splitsAreReady.await(milliseconds, TimeUnit.MILLISECONDS); } + + public void sdkInternalReady() { + if (_internalReady.getCount() == 0) { + return; + } + + _internalReady.countDown(); + } + + public void waitUntilInternalReady() throws InterruptedException { + if (_internalReady.getCount() == 0) { + return; + } + + _internalReady.await(); + } } diff --git a/client/src/main/java/io/split/engine/common/SyncManagerImp.java b/client/src/main/java/io/split/engine/common/SyncManagerImp.java index 69e6c75ae..f26aac4cc 100644 --- a/client/src/main/java/io/split/engine/common/SyncManagerImp.java +++ b/client/src/main/java/io/split/engine/common/SyncManagerImp.java @@ -4,6 +4,7 @@ import com.google.common.util.concurrent.ThreadFactoryBuilder; import io.split.cache.SegmentCache; import io.split.cache.SplitCache; +import io.split.engine.SDKReadinessGates; import io.split.engine.experiments.SplitFetcher; import io.split.engine.experiments.SplitSynchronizationTask; import io.split.engine.segments.SegmentSynchronizationTaskImp; @@ -60,15 +61,18 @@ public static SyncManagerImp build(boolean streamingEnabledConfig, int authRetryBackOffBase, CloseableHttpClient sseHttpClient, SegmentCache segmentCache, - int streamingRetryDelay) { + int streamingRetryDelay, + SDKReadinessGates gates) { LinkedBlockingQueue pushMessages = new LinkedBlockingQueue<>(); - Synchronizer synchronizer = new SynchronizerImp(splitSynchronizationTask, splitFetcher, segmentSynchronizationTaskImp, splitCache, segmentCache, streamingRetryDelay); + Synchronizer synchronizer = new SynchronizerImp(splitSynchronizationTask, splitFetcher, segmentSynchronizationTaskImp, splitCache, segmentCache, streamingRetryDelay, gates); PushManager pushManager = PushManagerImp.build(synchronizer, streamingServiceUrl, authUrl, httpClient, pushMessages, sseHttpClient); return new SyncManagerImp(streamingEnabledConfig, synchronizer, pushManager, pushMessages, authRetryBackOffBase); } @Override public void start() { + _synchronizer.syncAll(); + if (_streamingEnabledConfig.get()) { startStreamingMode(); } else { @@ -85,12 +89,10 @@ public void shutdown() { private void startStreamingMode() { _log.debug("Starting in streaming mode ..."); - _synchronizer.syncAll(); if (null == _pushStatusMonitorTask) { _pushStatusMonitorTask = _executorService.submit(this::incomingPushStatusHandler); } _pushManager.start(); - } private void startPollingMode() { diff --git a/client/src/main/java/io/split/engine/common/SynchronizerImp.java b/client/src/main/java/io/split/engine/common/SynchronizerImp.java index 52ab2e974..bab7170b0 100644 --- a/client/src/main/java/io/split/engine/common/SynchronizerImp.java +++ b/client/src/main/java/io/split/engine/common/SynchronizerImp.java @@ -3,6 +3,7 @@ import com.google.common.util.concurrent.ThreadFactoryBuilder; import io.split.cache.SegmentCache; import io.split.cache.SplitCache; +import io.split.engine.SDKReadinessGates; import io.split.engine.experiments.SplitFetcher; import io.split.engine.experiments.SplitSynchronizationTask; import io.split.engine.segments.SegmentFetcher; @@ -10,6 +11,7 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import java.util.ArrayList; import java.util.concurrent.Executors; import java.util.concurrent.ScheduledExecutorService; import java.util.concurrent.ThreadFactory; @@ -28,19 +30,22 @@ public class SynchronizerImp implements Synchronizer { private final SplitCache _splitCache; private final SegmentCache _segmentCache; private final int _onDemandFetchRetryDelayMs; + private final SDKReadinessGates _gates; public SynchronizerImp(SplitSynchronizationTask splitSynchronizationTask, SplitFetcher splitFetcher, SegmentSynchronizationTask segmentSynchronizationTaskImp, SplitCache splitCache, SegmentCache segmentCache, - int onDemandFetchRetryDelayMs) { + int onDemandFetchRetryDelayMs, + SDKReadinessGates gates) { _splitSynchronizationTask = checkNotNull(splitSynchronizationTask); _splitFetcher = checkNotNull(splitFetcher); _segmentSynchronizationTaskImp = checkNotNull(segmentSynchronizationTaskImp); _splitCache = checkNotNull(splitCache); _segmentCache = checkNotNull(segmentCache); _onDemandFetchRetryDelayMs = checkNotNull(onDemandFetchRetryDelayMs); + _gates = checkNotNull(gates); ThreadFactory splitsThreadFactory = new ThreadFactoryBuilder() .setDaemon(true) @@ -53,7 +58,8 @@ public SynchronizerImp(SplitSynchronizationTask splitSynchronizationTask, public void syncAll() { _syncAllScheduledExecutorService.schedule(() -> { _splitFetcher.fetchAll(true); - _segmentSynchronizationTaskImp.fetchAll(true); + _segmentSynchronizationTaskImp.fetchAllSynchronous(); + _gates.sdkInternalReady(); }, 0, TimeUnit.SECONDS); } diff --git a/client/src/main/java/io/split/engine/experiments/SplitFetcherImp.java b/client/src/main/java/io/split/engine/experiments/SplitFetcherImp.java index 510001153..17f12c58b 100644 --- a/client/src/main/java/io/split/engine/experiments/SplitFetcherImp.java +++ b/client/src/main/java/io/split/engine/experiments/SplitFetcherImp.java @@ -65,6 +65,12 @@ public void forceRefresh(boolean addCacheHeader) { @Override public void run() { + try { + _gates.waitUntilInternalReady(); + } catch (InterruptedException ex) { + _log.debug(ex.getMessage()); + } + this.fetchAll(false); } diff --git a/client/src/main/java/io/split/engine/segments/SegmentFetcherImp.java b/client/src/main/java/io/split/engine/segments/SegmentFetcherImp.java index ac21e8461..f8d613bf7 100644 --- a/client/src/main/java/io/split/engine/segments/SegmentFetcherImp.java +++ b/client/src/main/java/io/split/engine/segments/SegmentFetcherImp.java @@ -152,6 +152,4 @@ private void fetchAndUpdate(boolean addCacheHeader) { public void fetchAll() { this.fetchAndUpdate(false); } - - } diff --git a/client/src/main/java/io/split/engine/segments/SegmentSynchronizationTask.java b/client/src/main/java/io/split/engine/segments/SegmentSynchronizationTask.java index 0bed99225..5ad181cf4 100644 --- a/client/src/main/java/io/split/engine/segments/SegmentSynchronizationTask.java +++ b/client/src/main/java/io/split/engine/segments/SegmentSynchronizationTask.java @@ -29,4 +29,9 @@ public interface SegmentSynchronizationTask extends Runnable { * @param addCacheHeader */ void fetchAll(boolean addCacheHeader); + + /** + * fetch every Segment Synchronous + */ + void fetchAllSynchronous(); } diff --git a/client/src/main/java/io/split/engine/segments/SegmentSynchronizationTaskImp.java b/client/src/main/java/io/split/engine/segments/SegmentSynchronizationTaskImp.java index 7f3931f98..15d4bd412 100644 --- a/client/src/main/java/io/split/engine/segments/SegmentSynchronizationTaskImp.java +++ b/client/src/main/java/io/split/engine/segments/SegmentSynchronizationTaskImp.java @@ -8,16 +8,13 @@ import org.slf4j.LoggerFactory; import java.io.Closeable; +import java.util.ArrayList; import java.util.List; import java.util.Map; -import java.util.concurrent.ConcurrentMap; -import java.util.concurrent.ScheduledExecutorService; -import java.util.concurrent.ScheduledFuture; -import java.util.concurrent.ThreadFactory; +import java.util.concurrent.*; import java.util.concurrent.atomic.AtomicBoolean; import java.util.concurrent.atomic.AtomicLong; -import java.util.concurrent.Executors; -import java.util.concurrent.TimeUnit; +import java.util.stream.Collectors; import static com.google.common.base.Preconditions.checkArgument; import static com.google.common.base.Preconditions.checkNotNull; @@ -58,6 +55,12 @@ public SegmentSynchronizationTaskImp(SegmentChangeFetcher segmentChangeFetcher, @Override public void run() { + try { + _gates.waitUntilInternalReady(); + } catch (InterruptedException ex) { + _log.debug(ex.getMessage()); + } + this.fetchAll(false); } @@ -143,6 +146,8 @@ public void close() { @Override public void fetchAll(boolean addCacheHeader) { + ArrayList> futures = new ArrayList<>(); + for (Map.Entry entry : _segmentFetchers.entrySet()) { SegmentFetcher fetcher = entry.getValue(); @@ -151,10 +156,25 @@ public void fetchAll(boolean addCacheHeader) { } if(addCacheHeader) { - _scheduledExecutorService.submit(fetcher::runWhitCacheHeader); + futures.add(_scheduledExecutorService.submit(fetcher::runWhitCacheHeader)); continue; } - _scheduledExecutorService.submit(fetcher::fetchAll); + + futures.add(_scheduledExecutorService.submit(fetcher::fetchAll)); } } + + @Override + public void fetchAllSynchronous() { + _segmentFetchers + .entrySet() + .stream().map(e -> _scheduledExecutorService.submit(e.getValue()::runWhitCacheHeader)) + .collect(Collectors.toList()) + .stream().forEach(future -> { + try { + future.get(); + } catch (Exception ex) { + _log.error(ex.getMessage()); + }}); + } } diff --git a/client/src/test/java/io/split/engine/common/SyncManagerTest.java b/client/src/test/java/io/split/engine/common/SyncManagerTest.java index 551ebd7f3..d42b19c45 100644 --- a/client/src/test/java/io/split/engine/common/SyncManagerTest.java +++ b/client/src/test/java/io/split/engine/common/SyncManagerTest.java @@ -22,7 +22,7 @@ public void startWithStreamingFalseShouldStartPolling() { SyncManagerImp syncManager = new SyncManagerImp(false, _synchronizer, _pushManager, new LinkedBlockingQueue<>(), BACKOFF_BASE); syncManager.start(); Mockito.verify(_synchronizer, Mockito.times(1)).startPeriodicFetching(); - Mockito.verify(_synchronizer, Mockito.times(0)).syncAll(); + Mockito.verify(_synchronizer, Mockito.times(1)).syncAll(); Mockito.verify(_pushManager, Mockito.times(0)).start(); } diff --git a/client/src/test/java/io/split/engine/common/SynchronizerTest.java b/client/src/test/java/io/split/engine/common/SynchronizerTest.java index 9a90cb5ae..53e4175d1 100644 --- a/client/src/test/java/io/split/engine/common/SynchronizerTest.java +++ b/client/src/test/java/io/split/engine/common/SynchronizerTest.java @@ -2,6 +2,7 @@ import io.split.cache.SegmentCache; import io.split.cache.SplitCache; +import io.split.engine.SDKReadinessGates; import io.split.engine.experiments.SplitFetcherImp; import io.split.engine.experiments.SplitSynchronizationTask; import io.split.engine.segments.SegmentFetcher; @@ -17,6 +18,7 @@ public class SynchronizerTest { private SplitCache _splitCache; private Synchronizer _synchronizer; private SegmentCache _segmentCache; + private SDKReadinessGates _gates; @Before public void beforeMethod() { @@ -25,8 +27,9 @@ public void beforeMethod() { _splitFetcher = Mockito.mock(SplitFetcherImp.class); _splitCache = Mockito.mock(SplitCache.class); _segmentCache = Mockito.mock(SegmentCache.class); + _gates = Mockito.mock(SDKReadinessGates.class); - _synchronizer = new SynchronizerImp(_refreshableSplitFetcherTask, _splitFetcher, _segmentFetcher, _splitCache, _segmentCache, 50); + _synchronizer = new SynchronizerImp(_refreshableSplitFetcherTask, _splitFetcher, _segmentFetcher, _splitCache, _segmentCache, 50, _gates); } @Test @@ -35,7 +38,8 @@ public void syncAll() throws InterruptedException { Thread.sleep(100); Mockito.verify(_splitFetcher, Mockito.times(1)).fetchAll(true); - Mockito.verify(_segmentFetcher, Mockito.times(1)).fetchAll(true); + Mockito.verify(_segmentFetcher, Mockito.times(1)).fetchAllSynchronous(); + Mockito.verify(_gates, Mockito.times(1)).sdkInternalReady(); } @Test diff --git a/client/src/test/java/io/split/engine/experiments/SplitFetcherTest.java b/client/src/test/java/io/split/engine/experiments/SplitFetcherTest.java index cb9352994..2016f6e82 100644 --- a/client/src/test/java/io/split/engine/experiments/SplitFetcherTest.java +++ b/client/src/test/java/io/split/engine/experiments/SplitFetcherTest.java @@ -54,6 +54,7 @@ public void works_when_we_start_with_any_state() throws InterruptedException { private void works(long startingChangeNumber) throws InterruptedException { AChangePerCallSplitChangeFetcher splitChangeFetcher = new AChangePerCallSplitChangeFetcher(); SDKReadinessGates gates = new SDKReadinessGates(); + gates.sdkInternalReady(); SegmentCache segmentCache = new SegmentCacheInMemoryImpl(); SegmentChangeFetcher segmentChangeFetcher = Mockito.mock(SegmentChangeFetcher.class); SegmentSynchronizationTask segmentSynchronizationTask = new SegmentSynchronizationTaskImp(segmentChangeFetcher,1,10, gates, segmentCache); @@ -85,6 +86,7 @@ private void works(long startingChangeNumber) throws InterruptedException { @Test public void when_parser_fails_we_remove_the_experiment() throws InterruptedException { SDKReadinessGates gates = new SDKReadinessGates(); + gates.sdkInternalReady(); Split validSplit = new Split(); validSplit.status = Status.ACTIVE; validSplit.seed = (int) -1; @@ -132,7 +134,7 @@ public void when_parser_fails_we_remove_the_experiment() throws InterruptedExcep SegmentSynchronizationTask segmentSynchronizationTask = new SegmentSynchronizationTaskImp(segmentChangeFetcher, 1,10, gates, segmentCache); segmentSynchronizationTask.startPeriodicFetching(); SplitCache cache = new InMemoryCacheImp(-1); - SplitFetcherImp fetcher = new SplitFetcherImp(splitChangeFetcher, new SplitParser(segmentSynchronizationTask, segmentCache), new SDKReadinessGates(), cache); + SplitFetcherImp fetcher = new SplitFetcherImp(splitChangeFetcher, new SplitParser(segmentSynchronizationTask, segmentCache), gates, cache); // execute the fetcher for a little bit. executeWaitAndTerminate(fetcher, 1, 5, TimeUnit.SECONDS); @@ -145,6 +147,7 @@ public void when_parser_fails_we_remove_the_experiment() throws InterruptedExcep @Test public void if_there_is_a_problem_talking_to_split_change_count_down_latch_is_not_decremented() throws Exception { SDKReadinessGates gates = new SDKReadinessGates(); + gates.sdkInternalReady(); SplitCache cache = new InMemoryCacheImp(-1); SplitChangeFetcher splitChangeFetcher = mock(SplitChangeFetcher.class); @@ -188,6 +191,7 @@ public void works_with_user_defined_segments() throws Exception { String segmentName = "foosegment"; AChangePerCallSplitChangeFetcher experimentChangeFetcher = new AChangePerCallSplitChangeFetcher(segmentName); SDKReadinessGates gates = new SDKReadinessGates(); + gates.sdkInternalReady(); SplitCache cache = new InMemoryCacheImp(startingChangeNumber); SegmentCache segmentCache = new SegmentCacheInMemoryImpl(); From 624851b3087b81004be0b3b0e7c03e1eeae5f9aa Mon Sep 17 00:00:00 2001 From: Mauro Sanz Date: Thu, 29 Apr 2021 16:18:45 -0300 Subject: [PATCH 29/81] pr feedback --- .../engine/segments/SegmentSynchronizationTaskImp.java | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/client/src/main/java/io/split/engine/segments/SegmentSynchronizationTaskImp.java b/client/src/main/java/io/split/engine/segments/SegmentSynchronizationTaskImp.java index 15d4bd412..f4a61971f 100644 --- a/client/src/main/java/io/split/engine/segments/SegmentSynchronizationTaskImp.java +++ b/client/src/main/java/io/split/engine/segments/SegmentSynchronizationTaskImp.java @@ -146,8 +146,6 @@ public void close() { @Override public void fetchAll(boolean addCacheHeader) { - ArrayList> futures = new ArrayList<>(); - for (Map.Entry entry : _segmentFetchers.entrySet()) { SegmentFetcher fetcher = entry.getValue(); @@ -156,11 +154,11 @@ public void fetchAll(boolean addCacheHeader) { } if(addCacheHeader) { - futures.add(_scheduledExecutorService.submit(fetcher::runWhitCacheHeader)); + _scheduledExecutorService.submit(fetcher::runWhitCacheHeader); continue; } - futures.add(_scheduledExecutorService.submit(fetcher::fetchAll)); + _scheduledExecutorService.submit(fetcher::fetchAll); } } From bed70a354e6e0fa03c86c9e9c7c717b2dd752d89 Mon Sep 17 00:00:00 2001 From: Mauro Sanz Date: Fri, 30 Apr 2021 10:04:08 -0300 Subject: [PATCH 30/81] feedback --- .../split/engine/segments/SegmentSynchronizationTaskImp.java | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/client/src/main/java/io/split/engine/segments/SegmentSynchronizationTaskImp.java b/client/src/main/java/io/split/engine/segments/SegmentSynchronizationTaskImp.java index f4a61971f..6ce3fcd53 100644 --- a/client/src/main/java/io/split/engine/segments/SegmentSynchronizationTaskImp.java +++ b/client/src/main/java/io/split/engine/segments/SegmentSynchronizationTaskImp.java @@ -57,6 +57,7 @@ public SegmentSynchronizationTaskImp(SegmentChangeFetcher segmentChangeFetcher, public void run() { try { _gates.waitUntilInternalReady(); + _running.set(true); } catch (InterruptedException ex) { _log.debug(ex.getMessage()); } @@ -105,7 +106,7 @@ public SegmentFetcher getFetcher(String segmentName) { @Override public void startPeriodicFetching() { - if (_running.getAndSet(true)) { + if (_running.get()) { _log.debug("Segments PeriodicFetching is running..."); return; } From de2b36fdbd9fd77fc6dfe26f38a27791551f7d72 Mon Sep 17 00:00:00 2001 From: Mauro Sanz Date: Fri, 30 Apr 2021 10:43:09 -0300 Subject: [PATCH 31/81] improvements --- client/src/main/java/io/split/engine/common/SyncManager.java | 3 --- .../split/engine/segments/SegmentSynchronizationTaskImp.java | 5 ++--- 2 files changed, 2 insertions(+), 6 deletions(-) diff --git a/client/src/main/java/io/split/engine/common/SyncManager.java b/client/src/main/java/io/split/engine/common/SyncManager.java index b6a3c9b3e..147641748 100644 --- a/client/src/main/java/io/split/engine/common/SyncManager.java +++ b/client/src/main/java/io/split/engine/common/SyncManager.java @@ -1,8 +1,5 @@ package io.split.engine.common; -import io.split.engine.sse.listeners.FeedbackLoopListener; -import io.split.engine.sse.listeners.NotificationKeeperListener; - public interface SyncManager { void start(); void shutdown(); diff --git a/client/src/main/java/io/split/engine/segments/SegmentSynchronizationTaskImp.java b/client/src/main/java/io/split/engine/segments/SegmentSynchronizationTaskImp.java index 6ce3fcd53..a2e943783 100644 --- a/client/src/main/java/io/split/engine/segments/SegmentSynchronizationTaskImp.java +++ b/client/src/main/java/io/split/engine/segments/SegmentSynchronizationTaskImp.java @@ -8,7 +8,6 @@ import org.slf4j.LoggerFactory; import java.io.Closeable; -import java.util.ArrayList; import java.util.List; import java.util.Map; import java.util.concurrent.*; @@ -57,11 +56,12 @@ public SegmentSynchronizationTaskImp(SegmentChangeFetcher segmentChangeFetcher, public void run() { try { _gates.waitUntilInternalReady(); - _running.set(true); } catch (InterruptedException ex) { _log.debug(ex.getMessage()); } + _running.set(true); + _log.debug("Starting PeriodicFetching Segments ..."); this.fetchAll(false); } @@ -111,7 +111,6 @@ public void startPeriodicFetching() { return; } - _log.debug("Starting PeriodicFetching Segments ..."); _scheduledFuture = _scheduledExecutorService.scheduleWithFixedDelay(this, 0L, _refreshEveryNSeconds.get(), TimeUnit.SECONDS); } From ab64d08c62780b9c2e16772dbf8fe94fbeca70ef Mon Sep 17 00:00:00 2001 From: Mauro Sanz Date: Fri, 30 Apr 2021 13:43:21 -0300 Subject: [PATCH 32/81] pr feedback --- .../io/split/engine/SDKReadinessGates.java | 14 +---------- .../split/engine/common/SyncManagerImp.java | 25 +++++++++++++------ .../split/engine/common/SynchronizerImp.java | 1 - .../engine/experiments/SplitFetcherImp.java | 6 ----- .../SegmentSynchronizationTaskImp.java | 9 +------ .../split/engine/common/SyncManagerTest.java | 23 ++++++++++------- 6 files changed, 34 insertions(+), 44 deletions(-) diff --git a/client/src/main/java/io/split/engine/SDKReadinessGates.java b/client/src/main/java/io/split/engine/SDKReadinessGates.java index fe6ec0749..e8a4e6453 100644 --- a/client/src/main/java/io/split/engine/SDKReadinessGates.java +++ b/client/src/main/java/io/split/engine/SDKReadinessGates.java @@ -45,11 +45,7 @@ public boolean isSDKReady(long milliseconds) throws InterruptedException { timeLeft = end - System.currentTimeMillis(); - boolean ready = areSegmentsReady(timeLeft); - - if (ready) sdkInternalReady(); - - return ready; + return areSegmentsReady(timeLeft); } public boolean isSDKReadyNow() { @@ -172,18 +168,10 @@ public boolean areSplitsReady(long milliseconds) throws InterruptedException { } public void sdkInternalReady() { - if (_internalReady.getCount() == 0) { - return; - } - _internalReady.countDown(); } public void waitUntilInternalReady() throws InterruptedException { - if (_internalReady.getCount() == 0) { - return; - } - _internalReady.await(); } } diff --git a/client/src/main/java/io/split/engine/common/SyncManagerImp.java b/client/src/main/java/io/split/engine/common/SyncManagerImp.java index f26aac4cc..418964db1 100644 --- a/client/src/main/java/io/split/engine/common/SyncManagerImp.java +++ b/client/src/main/java/io/split/engine/common/SyncManagerImp.java @@ -12,10 +12,7 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import java.util.concurrent.ExecutorService; -import java.util.concurrent.Executors; -import java.util.concurrent.Future; -import java.util.concurrent.LinkedBlockingQueue; +import java.util.concurrent.*; import java.util.concurrent.atomic.AtomicBoolean; import static com.google.common.base.Preconditions.checkNotNull; @@ -29,6 +26,8 @@ public class SyncManagerImp implements SyncManager { private final AtomicBoolean _shutdown; private final LinkedBlockingQueue _incomingPushStatus; private final ExecutorService _executorService; + private final ExecutorService _pollingExecutorService; + private final SDKReadinessGates _gates; private Future _pushStatusMonitorTask; private Backoff _backoff; @@ -37,7 +36,8 @@ public class SyncManagerImp implements SyncManager { Synchronizer synchronizer, PushManager pushManager, LinkedBlockingQueue pushMessages, - int authRetryBackOffBase) { + int authRetryBackOffBase, + SDKReadinessGates gates) { _streamingEnabledConfig = new AtomicBoolean(streamingEnabledConfig); _synchronizer = checkNotNull(synchronizer); _pushManager = checkNotNull(pushManager); @@ -47,7 +47,12 @@ public class SyncManagerImp implements SyncManager { .setNameFormat("SPLIT-PushStatusMonitor-%d") .setDaemon(true) .build()); + _pollingExecutorService = Executors.newSingleThreadExecutor(new ThreadFactoryBuilder() + .setNameFormat("SPLIT-PollingMode-%d") + .setDaemon(true) + .build()); _backoff = new Backoff(authRetryBackOffBase); + _gates = checkNotNull(gates); } public static SyncManagerImp build(boolean streamingEnabledConfig, @@ -66,7 +71,7 @@ public static SyncManagerImp build(boolean streamingEnabledConfig, LinkedBlockingQueue pushMessages = new LinkedBlockingQueue<>(); Synchronizer synchronizer = new SynchronizerImp(splitSynchronizationTask, splitFetcher, segmentSynchronizationTaskImp, splitCache, segmentCache, streamingRetryDelay, gates); PushManager pushManager = PushManagerImp.build(synchronizer, streamingServiceUrl, authUrl, httpClient, pushMessages, sseHttpClient); - return new SyncManagerImp(streamingEnabledConfig, synchronizer, pushManager, pushMessages, authRetryBackOffBase); + return new SyncManagerImp(streamingEnabledConfig, synchronizer, pushManager, pushMessages, authRetryBackOffBase, gates); } @Override @@ -76,7 +81,7 @@ public void start() { if (_streamingEnabledConfig.get()) { startStreamingMode(); } else { - startPollingMode(); + _pollingExecutorService.submit(this::startPollingMode); } } @@ -96,6 +101,12 @@ private void startStreamingMode() { } private void startPollingMode() { + try { + _gates.waitUntilInternalReady(); + } catch (InterruptedException ex) { + _log.debug(ex.getMessage()); + } + _log.debug("Starting in polling mode ..."); _synchronizer.startPeriodicFetching(); } diff --git a/client/src/main/java/io/split/engine/common/SynchronizerImp.java b/client/src/main/java/io/split/engine/common/SynchronizerImp.java index bab7170b0..d63f5417f 100644 --- a/client/src/main/java/io/split/engine/common/SynchronizerImp.java +++ b/client/src/main/java/io/split/engine/common/SynchronizerImp.java @@ -11,7 +11,6 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import java.util.ArrayList; import java.util.concurrent.Executors; import java.util.concurrent.ScheduledExecutorService; import java.util.concurrent.ThreadFactory; diff --git a/client/src/main/java/io/split/engine/experiments/SplitFetcherImp.java b/client/src/main/java/io/split/engine/experiments/SplitFetcherImp.java index 17f12c58b..510001153 100644 --- a/client/src/main/java/io/split/engine/experiments/SplitFetcherImp.java +++ b/client/src/main/java/io/split/engine/experiments/SplitFetcherImp.java @@ -65,12 +65,6 @@ public void forceRefresh(boolean addCacheHeader) { @Override public void run() { - try { - _gates.waitUntilInternalReady(); - } catch (InterruptedException ex) { - _log.debug(ex.getMessage()); - } - this.fetchAll(false); } diff --git a/client/src/main/java/io/split/engine/segments/SegmentSynchronizationTaskImp.java b/client/src/main/java/io/split/engine/segments/SegmentSynchronizationTaskImp.java index a2e943783..b0cbb43b0 100644 --- a/client/src/main/java/io/split/engine/segments/SegmentSynchronizationTaskImp.java +++ b/client/src/main/java/io/split/engine/segments/SegmentSynchronizationTaskImp.java @@ -54,13 +54,6 @@ public SegmentSynchronizationTaskImp(SegmentChangeFetcher segmentChangeFetcher, @Override public void run() { - try { - _gates.waitUntilInternalReady(); - } catch (InterruptedException ex) { - _log.debug(ex.getMessage()); - } - - _running.set(true); _log.debug("Starting PeriodicFetching Segments ..."); this.fetchAll(false); } @@ -106,7 +99,7 @@ public SegmentFetcher getFetcher(String segmentName) { @Override public void startPeriodicFetching() { - if (_running.get()) { + if (_running.getAndSet(true) ) { _log.debug("Segments PeriodicFetching is running..."); return; } diff --git a/client/src/test/java/io/split/engine/common/SyncManagerTest.java b/client/src/test/java/io/split/engine/common/SyncManagerTest.java index d42b19c45..76197ff33 100644 --- a/client/src/test/java/io/split/engine/common/SyncManagerTest.java +++ b/client/src/test/java/io/split/engine/common/SyncManagerTest.java @@ -1,5 +1,6 @@ package io.split.engine.common; +import io.split.engine.SDKReadinessGates; import org.junit.Before; import org.junit.Test; import org.mockito.Mockito; @@ -10,17 +11,21 @@ public class SyncManagerTest { private static final int BACKOFF_BASE = 1; private Synchronizer _synchronizer; private PushManager _pushManager; + private SDKReadinessGates _gates; @Before public void setUp() { _synchronizer = Mockito.mock(Synchronizer.class); _pushManager = Mockito.mock(PushManager.class); + _gates = Mockito.mock(SDKReadinessGates.class); } @Test - public void startWithStreamingFalseShouldStartPolling() { - SyncManagerImp syncManager = new SyncManagerImp(false, _synchronizer, _pushManager, new LinkedBlockingQueue<>(), BACKOFF_BASE); + public void startWithStreamingFalseShouldStartPolling() throws InterruptedException { + _gates.sdkInternalReady(); + SyncManagerImp syncManager = new SyncManagerImp(false, _synchronizer, _pushManager, new LinkedBlockingQueue<>(), BACKOFF_BASE, _gates); syncManager.start(); + Thread.sleep(1000); Mockito.verify(_synchronizer, Mockito.times(1)).startPeriodicFetching(); Mockito.verify(_synchronizer, Mockito.times(1)).syncAll(); Mockito.verify(_pushManager, Mockito.times(0)).start(); @@ -28,7 +33,7 @@ public void startWithStreamingFalseShouldStartPolling() { @Test public void startWithStreamingTrueShouldStartSyncAll() { - SyncManager sm = new SyncManagerImp(true, _synchronizer, _pushManager, new LinkedBlockingQueue<>(), BACKOFF_BASE); + SyncManager sm = new SyncManagerImp(true, _synchronizer, _pushManager, new LinkedBlockingQueue<>(), BACKOFF_BASE, _gates); sm.start(); Mockito.verify(_synchronizer, Mockito.times(0)).startPeriodicFetching(); Mockito.verify(_synchronizer, Mockito.times(1)).syncAll(); @@ -38,7 +43,7 @@ public void startWithStreamingTrueShouldStartSyncAll() { @Test public void onStreamingAvailable() throws InterruptedException { LinkedBlockingQueue messsages = new LinkedBlockingQueue<>(); - SyncManagerImp syncManager = new SyncManagerImp(true, _synchronizer, _pushManager, messsages, BACKOFF_BASE); + SyncManagerImp syncManager = new SyncManagerImp(true, _synchronizer, _pushManager, messsages, BACKOFF_BASE, _gates); Thread t = new Thread(syncManager::incomingPushStatusHandler); t.start(); messsages.offer(PushManager.Status.STREAMING_READY); @@ -52,7 +57,7 @@ public void onStreamingAvailable() throws InterruptedException { @Test public void onStreamingDisabled() throws InterruptedException { LinkedBlockingQueue messsages = new LinkedBlockingQueue<>(); - SyncManagerImp syncManager = new SyncManagerImp(true, _synchronizer, _pushManager, messsages, BACKOFF_BASE); + SyncManagerImp syncManager = new SyncManagerImp(true, _synchronizer, _pushManager, messsages, BACKOFF_BASE, _gates); Thread t = new Thread(syncManager::incomingPushStatusHandler); t.start(); messsages.offer(PushManager.Status.STREAMING_DOWN); @@ -66,7 +71,7 @@ public void onStreamingDisabled() throws InterruptedException { @Test public void onStreamingShutdown() throws InterruptedException { LinkedBlockingQueue messsages = new LinkedBlockingQueue<>(); - SyncManagerImp syncManager = new SyncManagerImp(true, _synchronizer, _pushManager, messsages, BACKOFF_BASE); + SyncManagerImp syncManager = new SyncManagerImp(true, _synchronizer, _pushManager, messsages, BACKOFF_BASE, _gates); Thread t = new Thread(syncManager::incomingPushStatusHandler); t.start(); messsages.offer(PushManager.Status.STREAMING_OFF); @@ -78,7 +83,7 @@ public void onStreamingShutdown() throws InterruptedException { @Test public void onConnected() throws InterruptedException { LinkedBlockingQueue messsages = new LinkedBlockingQueue<>(); - SyncManagerImp syncManager = new SyncManagerImp(true, _synchronizer, _pushManager, messsages, BACKOFF_BASE); + SyncManagerImp syncManager = new SyncManagerImp(true, _synchronizer, _pushManager, messsages, BACKOFF_BASE, _gates); Thread t = new Thread(syncManager::incomingPushStatusHandler); t.start(); messsages.offer(PushManager.Status.STREAMING_READY); @@ -91,7 +96,7 @@ public void onConnected() throws InterruptedException { @Test public void onDisconnect() throws InterruptedException { LinkedBlockingQueue messsages = new LinkedBlockingQueue<>(); - SyncManagerImp syncManager = new SyncManagerImp(true, _synchronizer, _pushManager, messsages, BACKOFF_BASE); + SyncManagerImp syncManager = new SyncManagerImp(true, _synchronizer, _pushManager, messsages, BACKOFF_BASE, _gates); Thread t = new Thread(syncManager::incomingPushStatusHandler); t.start(); messsages.offer(PushManager.Status.STREAMING_OFF); @@ -103,7 +108,7 @@ public void onDisconnect() throws InterruptedException { @Test public void onDisconnectAndReconnect() throws InterruptedException { // Check with mauro. reconnect should call pushManager.start again, right? LinkedBlockingQueue messsages = new LinkedBlockingQueue<>(); - SyncManagerImp syncManager = new SyncManagerImp(true, _synchronizer, _pushManager, messsages, BACKOFF_BASE); + SyncManagerImp syncManager = new SyncManagerImp(true, _synchronizer, _pushManager, messsages, BACKOFF_BASE, _gates); syncManager.start(); messsages.offer(PushManager.Status.STREAMING_BACKOFF); Thread.sleep(1200); From 084eee0d9fbe5ed39ba587a015ede1453c07fb2f Mon Sep 17 00:00:00 2001 From: Mauro Sanz Date: Fri, 30 Apr 2021 13:51:36 -0300 Subject: [PATCH 33/81] feedback --- .../engine/segments/SegmentSynchronizationTaskImp.java | 9 +++++++-- .../io/split/engine/experiments/SplitFetcherTest.java | 4 ---- 2 files changed, 7 insertions(+), 6 deletions(-) diff --git a/client/src/main/java/io/split/engine/segments/SegmentSynchronizationTaskImp.java b/client/src/main/java/io/split/engine/segments/SegmentSynchronizationTaskImp.java index b0cbb43b0..da0ca42aa 100644 --- a/client/src/main/java/io/split/engine/segments/SegmentSynchronizationTaskImp.java +++ b/client/src/main/java/io/split/engine/segments/SegmentSynchronizationTaskImp.java @@ -10,7 +10,12 @@ import java.io.Closeable; import java.util.List; import java.util.Map; -import java.util.concurrent.*; +import java.util.concurrent.ConcurrentMap; +import java.util.concurrent.Executors; +import java.util.concurrent.ScheduledExecutorService; +import java.util.concurrent.ScheduledFuture; +import java.util.concurrent.ThreadFactory; +import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicBoolean; import java.util.concurrent.atomic.AtomicLong; import java.util.stream.Collectors; @@ -54,7 +59,6 @@ public SegmentSynchronizationTaskImp(SegmentChangeFetcher segmentChangeFetcher, @Override public void run() { - _log.debug("Starting PeriodicFetching Segments ..."); this.fetchAll(false); } @@ -104,6 +108,7 @@ public void startPeriodicFetching() { return; } + _log.debug("Starting PeriodicFetching Segments ..."); _scheduledFuture = _scheduledExecutorService.scheduleWithFixedDelay(this, 0L, _refreshEveryNSeconds.get(), TimeUnit.SECONDS); } diff --git a/client/src/test/java/io/split/engine/experiments/SplitFetcherTest.java b/client/src/test/java/io/split/engine/experiments/SplitFetcherTest.java index 2016f6e82..62ed695dd 100644 --- a/client/src/test/java/io/split/engine/experiments/SplitFetcherTest.java +++ b/client/src/test/java/io/split/engine/experiments/SplitFetcherTest.java @@ -54,7 +54,6 @@ public void works_when_we_start_with_any_state() throws InterruptedException { private void works(long startingChangeNumber) throws InterruptedException { AChangePerCallSplitChangeFetcher splitChangeFetcher = new AChangePerCallSplitChangeFetcher(); SDKReadinessGates gates = new SDKReadinessGates(); - gates.sdkInternalReady(); SegmentCache segmentCache = new SegmentCacheInMemoryImpl(); SegmentChangeFetcher segmentChangeFetcher = Mockito.mock(SegmentChangeFetcher.class); SegmentSynchronizationTask segmentSynchronizationTask = new SegmentSynchronizationTaskImp(segmentChangeFetcher,1,10, gates, segmentCache); @@ -86,7 +85,6 @@ private void works(long startingChangeNumber) throws InterruptedException { @Test public void when_parser_fails_we_remove_the_experiment() throws InterruptedException { SDKReadinessGates gates = new SDKReadinessGates(); - gates.sdkInternalReady(); Split validSplit = new Split(); validSplit.status = Status.ACTIVE; validSplit.seed = (int) -1; @@ -147,7 +145,6 @@ public void when_parser_fails_we_remove_the_experiment() throws InterruptedExcep @Test public void if_there_is_a_problem_talking_to_split_change_count_down_latch_is_not_decremented() throws Exception { SDKReadinessGates gates = new SDKReadinessGates(); - gates.sdkInternalReady(); SplitCache cache = new InMemoryCacheImp(-1); SplitChangeFetcher splitChangeFetcher = mock(SplitChangeFetcher.class); @@ -191,7 +188,6 @@ public void works_with_user_defined_segments() throws Exception { String segmentName = "foosegment"; AChangePerCallSplitChangeFetcher experimentChangeFetcher = new AChangePerCallSplitChangeFetcher(segmentName); SDKReadinessGates gates = new SDKReadinessGates(); - gates.sdkInternalReady(); SplitCache cache = new InMemoryCacheImp(startingChangeNumber); SegmentCache segmentCache = new SegmentCacheInMemoryImpl(); From c21fcd11cfcd945ed0e0f9f7e5cb733ef702d5a8 Mon Sep 17 00:00:00 2001 From: Mauro Sanz Date: Mon, 3 May 2021 15:13:51 -0300 Subject: [PATCH 34/81] feedback --- .../src/main/java/io/split/engine/common/SyncManagerImp.java | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/client/src/main/java/io/split/engine/common/SyncManagerImp.java b/client/src/main/java/io/split/engine/common/SyncManagerImp.java index 418964db1..5b830ca65 100644 --- a/client/src/main/java/io/split/engine/common/SyncManagerImp.java +++ b/client/src/main/java/io/split/engine/common/SyncManagerImp.java @@ -12,7 +12,10 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import java.util.concurrent.*; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; +import java.util.concurrent.Future; +import java.util.concurrent.LinkedBlockingQueue; import java.util.concurrent.atomic.AtomicBoolean; import static com.google.common.base.Preconditions.checkNotNull; From 4b4bb9979ddad16757ee3ddf9756b7c22c8d6a3e Mon Sep 17 00:00:00 2001 From: Mauro Sanz Date: Fri, 30 Apr 2021 19:07:32 -0300 Subject: [PATCH 35/81] added headers --- .../io/split/client/SplitFactoryImpl.java | 14 +++-- .../interceptors/AddSplitHeadersFilter.java | 30 ++++++++--- .../split/engine/common/PushManagerImp.java | 1 + .../engine/sse/EventSourceClientImp.java | 1 + .../io/split/engine/sse/client/SSEClient.java | 3 +- .../AddSplitHeadersFilterTest.java | 53 ++++++++++++++++--- 6 files changed, 80 insertions(+), 22 deletions(-) diff --git a/client/src/main/java/io/split/client/SplitFactoryImpl.java b/client/src/main/java/io/split/client/SplitFactoryImpl.java index c65c8dd89..a9c0df6e2 100644 --- a/client/src/main/java/io/split/client/SplitFactoryImpl.java +++ b/client/src/main/java/io/split/client/SplitFactoryImpl.java @@ -48,9 +48,7 @@ import java.io.IOException; import java.net.URI; import java.net.URISyntaxException; -import java.util.ArrayList; -import java.util.List; -import java.util.Random; +import java.util.*; import java.util.stream.Collectors; public class SplitFactoryImpl implements SplitFactory { @@ -130,7 +128,7 @@ public SplitFactoryImpl(String apiToken, SplitClientConfig config) throws URISyn _eventClient = EventClientImpl.create(_httpclient, _eventsRootTarget, config.eventsQueueSize(), config.eventFlushIntervalInMillis(), config.waitBeforeShutdown()); // SyncManager - _syncManager = SyncManagerImp.build(config.streamingEnabled(), _splitSynchronizationTask, _splitFetcher, _segmentSynchronizationTaskImp, _splitCache, config.authServiceURL(), _httpclient, config.streamingServiceURL(), config.authRetryBackoffBase(), buildSSEdHttpClient(config), _segmentCache, config.streamingRetryDelay(), _gates); + _syncManager = SyncManagerImp.build(config.streamingEnabled(), _splitSynchronizationTask, _splitFetcher, _segmentSynchronizationTaskImp, _splitCache, config.authServiceURL(), _httpclient, config.streamingServiceURL(), config.authRetryBackoffBase(), buildSSEdHttpClient(apiToken, config), _segmentCache, config.streamingRetryDelay(), _gates); _syncManager.start(); // Evaluator @@ -215,7 +213,7 @@ private static CloseableHttpClient buildHttpClient(String apiToken, SplitClientC HttpClientBuilder httpClientbuilder = HttpClients.custom() .setConnectionManager(cm) .setDefaultRequestConfig(requestConfig) - .addRequestInterceptorLast(AddSplitHeadersFilter.instance(apiToken, config.ipAddressEnabled())) + .addRequestInterceptorLast(AddSplitHeadersFilter.instance(apiToken, config.ipAddressEnabled(), false)) .addRequestInterceptorLast(new GzipEncoderRequestInterceptor()) .addResponseInterceptorLast((new GzipDecoderResponseInterceptor())); @@ -227,7 +225,7 @@ private static CloseableHttpClient buildHttpClient(String apiToken, SplitClientC return httpClientbuilder.build(); } - private static CloseableHttpClient buildSSEdHttpClient(SplitClientConfig config) { + private static CloseableHttpClient buildSSEdHttpClient(String apiToken, SplitClientConfig config) { RequestConfig requestConfig = RequestConfig.custom() .setConnectTimeout(Timeout.ofMilliseconds(SSE_CONNECT_TIMEOUT)) .build(); @@ -248,7 +246,8 @@ private static CloseableHttpClient buildSSEdHttpClient(SplitClientConfig config) HttpClientBuilder httpClientbuilder = HttpClients.custom() .setConnectionManager(cm) - .setDefaultRequestConfig(requestConfig); + .setDefaultRequestConfig(requestConfig) + .addRequestInterceptorLast(AddSplitHeadersFilter.instance(apiToken, config.ipAddressEnabled(), true)); // Set up proxy is it exists if (config.proxy() != null) { @@ -311,5 +310,4 @@ private ImpressionsManagerImpl buildImpressionsManager(SplitClientConfig config) return ImpressionsManagerImpl.instance(_httpclient, config, impressionListeners); } - } diff --git a/client/src/main/java/io/split/client/interceptors/AddSplitHeadersFilter.java b/client/src/main/java/io/split/client/interceptors/AddSplitHeadersFilter.java index 3367ac53d..0d39016e3 100644 --- a/client/src/main/java/io/split/client/interceptors/AddSplitHeadersFilter.java +++ b/client/src/main/java/io/split/client/interceptors/AddSplitHeadersFilter.java @@ -25,14 +25,16 @@ public class AddSplitHeadersFilter implements HttpRequestInterceptor { static final String CLIENT_MACHINE_NAME_HEADER = "SplitSDKMachineName"; static final String CLIENT_MACHINE_IP_HEADER = "SplitSDKMachineIP"; static final String CLIENT_VERSION = "SplitSDKVersion"; + static final String CLIENT_KEY = "SplitSDKClientKey"; private final String _apiTokenBearer; private final String _hostname; private final String _ip; + private final String _clientKey; - public static AddSplitHeadersFilter instance(String apiToken, boolean ipAddressEnabled) { + public static AddSplitHeadersFilter instance(String apiToken, boolean ipAddressEnabled, boolean externalClient) { if (!ipAddressEnabled) { - return new AddSplitHeadersFilter(apiToken, null, null); + return new AddSplitHeadersFilter(apiToken, null, null, externalClient); } String hostname = null; @@ -46,20 +48,30 @@ public static AddSplitHeadersFilter instance(String apiToken, boolean ipAddressE _log.error("Could not resolve InetAddress", e); } - return new AddSplitHeadersFilter(apiToken, hostname, ip); + return new AddSplitHeadersFilter(apiToken, hostname, ip, externalClient); } - private AddSplitHeadersFilter(String apiToken, String hostname, String ip) { + private AddSplitHeadersFilter(String apiToken, String hostname, String ip, boolean externalClient) { checkNotNull(apiToken); - _apiTokenBearer = "Bearer " + apiToken; + if (externalClient) { + _apiTokenBearer = null; + _clientKey = apiToken.substring(apiToken.length() - 4); + } else { + _apiTokenBearer = "Bearer " + apiToken; + _clientKey = null; + } + _hostname = hostname; _ip = ip; } @Override - public void process(HttpRequest request, EntityDetails entity, HttpContext context) throws HttpException, IOException { - request.addHeader(AUTHORIZATION_HEADER, _apiTokenBearer); + public void process(HttpRequest request, EntityDetails entity, HttpContext context) { + if (_apiTokenBearer != null) { + request.addHeader(AUTHORIZATION_HEADER, _apiTokenBearer); + } + request.addHeader(CLIENT_VERSION, SplitClientConfig.splitSdkVersion); if (_hostname != null) { @@ -69,5 +81,9 @@ public void process(HttpRequest request, EntityDetails entity, HttpContext conte if (_ip != null) { request.addHeader(CLIENT_MACHINE_IP_HEADER, _ip); } + + if (_clientKey != null) { + request.addHeader(CLIENT_KEY, _clientKey); + } } } diff --git a/client/src/main/java/io/split/engine/common/PushManagerImp.java b/client/src/main/java/io/split/engine/common/PushManagerImp.java index 1d770b6b1..26a34d9a2 100644 --- a/client/src/main/java/io/split/engine/common/PushManagerImp.java +++ b/client/src/main/java/io/split/engine/common/PushManagerImp.java @@ -20,6 +20,7 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import java.util.Map; import java.util.concurrent.Future; import java.util.concurrent.LinkedBlockingQueue; import java.util.concurrent.ScheduledExecutorService; diff --git a/client/src/main/java/io/split/engine/sse/EventSourceClientImp.java b/client/src/main/java/io/split/engine/sse/EventSourceClientImp.java index 7d8bf990d..142d7c80c 100644 --- a/client/src/main/java/io/split/engine/sse/EventSourceClientImp.java +++ b/client/src/main/java/io/split/engine/sse/EventSourceClientImp.java @@ -14,6 +14,7 @@ import java.net.URI; import java.net.URISyntaxException; +import java.util.Map; import java.util.concurrent.atomic.AtomicBoolean; import static com.google.common.base.Preconditions.checkNotNull; diff --git a/client/src/main/java/io/split/engine/sse/client/SSEClient.java b/client/src/main/java/io/split/engine/sse/client/SSEClient.java index 6f072400d..f6f6e13bc 100644 --- a/client/src/main/java/io/split/engine/sse/client/SSEClient.java +++ b/client/src/main/java/io/split/engine/sse/client/SSEClient.java @@ -15,6 +15,8 @@ import java.io.InputStreamReader; import java.net.SocketException; import java.net.URI; +import java.util.HashMap; +import java.util.Map; import java.util.concurrent.CountDownLatch; import java.util.concurrent.ExecutorService; import java.util.concurrent.Executors; @@ -156,7 +158,6 @@ private void connectAndLoop(URI uri, CountDownLatch signal) { } private boolean establishConnection(URI uri, CountDownLatch signal) { - _ongoingRequest.set(new HttpGet(uri)); try { diff --git a/client/src/test/java/io/split/client/interceptors/AddSplitHeadersFilterTest.java b/client/src/test/java/io/split/client/interceptors/AddSplitHeadersFilterTest.java index 2c68b0335..b35abb8ba 100644 --- a/client/src/test/java/io/split/client/interceptors/AddSplitHeadersFilterTest.java +++ b/client/src/test/java/io/split/client/interceptors/AddSplitHeadersFilterTest.java @@ -1,6 +1,5 @@ package io.split.client.interceptors; -import org.apache.hc.core5.http.HttpException; import org.apache.hc.core5.http.HttpRequest; import org.apache.hc.core5.http.protocol.HttpContext; import org.junit.Test; @@ -11,11 +10,11 @@ import org.mockito.runners.MockitoJUnitRunner; -import java.io.IOException; import java.util.List; import static org.hamcrest.Matchers.*; import static org.hamcrest.core.IsEqual.equalTo; +import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertThat; @@ -29,8 +28,8 @@ public class AddSplitHeadersFilterTest { private ArgumentCaptor headerValueCaptor; @Test - public void testHeadersWithIpAndHostname() throws IOException, HttpException { - AddSplitHeadersFilter filter = AddSplitHeadersFilter.instance("abc", true); + public void testHeadersWithIpAndHostname() { + AddSplitHeadersFilter filter = AddSplitHeadersFilter.instance("abc", true, false); HttpRequest req = Mockito.mock(HttpRequest.class); HttpContext ctx = Mockito.mock(HttpContext.class); @@ -50,8 +49,8 @@ public void testHeadersWithIpAndHostname() throws IOException, HttpException { } @Test - public void testHeadersWithoutIpAndHostname() throws IOException, HttpException { - AddSplitHeadersFilter filter = AddSplitHeadersFilter.instance("abc", false); + public void testHeadersWithoutIpAndHostname() { + AddSplitHeadersFilter filter = AddSplitHeadersFilter.instance("abc", false, false); HttpRequest req = Mockito.mock(HttpRequest.class); HttpContext ctx = Mockito.mock(HttpContext.class); @@ -70,4 +69,46 @@ public void testHeadersWithoutIpAndHostname() throws IOException, HttpException assertThat(headerNames, not(contains(AddSplitHeadersFilter.CLIENT_MACHINE_NAME_HEADER, AddSplitHeadersFilter.CLIENT_MACHINE_IP_HEADER))); } + + @Test + public void testHeadersWithoutIpAndHostnameAndWithClientKey() { + AddSplitHeadersFilter filter = AddSplitHeadersFilter.instance("ljsdfkjkldfjksldjflksdjflsdjflksd", false, true); + HttpRequest req = Mockito.mock(HttpRequest.class); + HttpContext ctx = Mockito.mock(HttpContext.class); + + filter.process(req, null, ctx); + Mockito.verify(req, Mockito.times(2)).addHeader(headerNameCaptor.capture(), headerValueCaptor.capture()); + + List headerNames = headerNameCaptor.getAllValues(); + List headerValues = headerValueCaptor.getAllValues(); + + assertEquals(2 ,headerNames.size()); + assertEquals(2, headerValues.size()); + assertThat(headerNames, contains(AddSplitHeadersFilter.CLIENT_VERSION, + AddSplitHeadersFilter.CLIENT_KEY)); + assertThat(headerNames, not(contains(AddSplitHeadersFilter.CLIENT_MACHINE_NAME_HEADER, + AddSplitHeadersFilter.CLIENT_MACHINE_IP_HEADER, + AddSplitHeadersFilter.AUTHORIZATION_HEADER))); + } + + @Test + public void testHeadersWithIpAndHostnameAndWithClientKey() { + AddSplitHeadersFilter filter = AddSplitHeadersFilter.instance("ljsdfkjkldfjksldjflksdjflsdjflksd", true, true); + HttpRequest req = Mockito.mock(HttpRequest.class); + HttpContext ctx = Mockito.mock(HttpContext.class); + + filter.process(req, null, ctx); + Mockito.verify(req, Mockito.times(4)).addHeader(headerNameCaptor.capture(), headerValueCaptor.capture()); + + List headerNames = headerNameCaptor.getAllValues(); + List headerValues = headerValueCaptor.getAllValues(); + + assertEquals(4 ,headerNames.size()); + assertEquals(4, headerValues.size()); + assertThat(headerNames, contains(AddSplitHeadersFilter.CLIENT_VERSION, + AddSplitHeadersFilter.CLIENT_MACHINE_NAME_HEADER, + AddSplitHeadersFilter.CLIENT_MACHINE_IP_HEADER, + AddSplitHeadersFilter.CLIENT_KEY)); + assertThat(headerNames, not(contains(AddSplitHeadersFilter.AUTHORIZATION_HEADER))); + } } From 66bd0ac8297ebd6441ff55194574d636e5d5adc1 Mon Sep 17 00:00:00 2001 From: Mauro Sanz Date: Mon, 3 May 2021 15:11:57 -0300 Subject: [PATCH 36/81] pr feedback --- client/src/main/java/io/split/client/SplitFactoryImpl.java | 5 ++++- .../io/split/client/interceptors/AddSplitHeadersFilter.java | 2 -- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/client/src/main/java/io/split/client/SplitFactoryImpl.java b/client/src/main/java/io/split/client/SplitFactoryImpl.java index a9c0df6e2..4015c9681 100644 --- a/client/src/main/java/io/split/client/SplitFactoryImpl.java +++ b/client/src/main/java/io/split/client/SplitFactoryImpl.java @@ -48,7 +48,10 @@ import java.io.IOException; import java.net.URI; import java.net.URISyntaxException; -import java.util.*; + +import java.util.ArrayList; +import java.util.List; +import java.util.Random; import java.util.stream.Collectors; public class SplitFactoryImpl implements SplitFactory { diff --git a/client/src/main/java/io/split/client/interceptors/AddSplitHeadersFilter.java b/client/src/main/java/io/split/client/interceptors/AddSplitHeadersFilter.java index 0d39016e3..b7a80fc89 100644 --- a/client/src/main/java/io/split/client/interceptors/AddSplitHeadersFilter.java +++ b/client/src/main/java/io/split/client/interceptors/AddSplitHeadersFilter.java @@ -2,14 +2,12 @@ import io.split.client.SplitClientConfig; import org.apache.hc.core5.http.EntityDetails; -import org.apache.hc.core5.http.HttpException; import org.apache.hc.core5.http.HttpRequest; import org.apache.hc.core5.http.HttpRequestInterceptor; import org.apache.hc.core5.http.protocol.HttpContext; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import java.io.IOException; import java.net.InetAddress; import static com.google.common.base.Preconditions.checkNotNull; From b0ffa880db52da7afc347aac9b2d532b961a23d0 Mon Sep 17 00:00:00 2001 From: Mauro Sanz Date: Mon, 3 May 2021 17:16:46 -0300 Subject: [PATCH 37/81] pr feedback --- .../io/split/client/SplitFactoryImpl.java | 10 +- .../interceptors/AddSplitHeadersFilter.java | 87 ------------- .../AuthorizationInterceptorFilter.java | 30 +++++ .../ClientKeyInterceptorFilter.java | 28 +++++ .../SdkMetadataInterceptorFilter.java | 61 ++++++++++ .../AddSplitHeadersFilterTest.java | 114 ------------------ .../AuthorizationInterceptorFilterTest.java | 51 ++++++++ .../ClientKeyInterceptorFilterTest.java | 46 +++++++ .../SdkMetadataInterceptorFilterTest.java | 65 ++++++++++ 9 files changed, 288 insertions(+), 204 deletions(-) delete mode 100644 client/src/main/java/io/split/client/interceptors/AddSplitHeadersFilter.java create mode 100644 client/src/main/java/io/split/client/interceptors/AuthorizationInterceptorFilter.java create mode 100644 client/src/main/java/io/split/client/interceptors/ClientKeyInterceptorFilter.java create mode 100644 client/src/main/java/io/split/client/interceptors/SdkMetadataInterceptorFilter.java delete mode 100644 client/src/test/java/io/split/client/interceptors/AddSplitHeadersFilterTest.java create mode 100644 client/src/test/java/io/split/client/interceptors/AuthorizationInterceptorFilterTest.java create mode 100644 client/src/test/java/io/split/client/interceptors/ClientKeyInterceptorFilterTest.java create mode 100644 client/src/test/java/io/split/client/interceptors/SdkMetadataInterceptorFilterTest.java diff --git a/client/src/main/java/io/split/client/SplitFactoryImpl.java b/client/src/main/java/io/split/client/SplitFactoryImpl.java index 4015c9681..7b2b5f930 100644 --- a/client/src/main/java/io/split/client/SplitFactoryImpl.java +++ b/client/src/main/java/io/split/client/SplitFactoryImpl.java @@ -3,9 +3,11 @@ import io.split.client.impressions.AsynchronousImpressionListener; import io.split.client.impressions.ImpressionListener; import io.split.client.impressions.ImpressionsManagerImpl; -import io.split.client.interceptors.AddSplitHeadersFilter; +import io.split.client.interceptors.AuthorizationInterceptorFilter; +import io.split.client.interceptors.ClientKeyInterceptorFilter; import io.split.client.interceptors.GzipDecoderResponseInterceptor; import io.split.client.interceptors.GzipEncoderRequestInterceptor; +import io.split.client.interceptors.SdkMetadataInterceptorFilter; import io.split.client.metrics.HttpMetrics; import io.split.cache.InMemoryCacheImp; import io.split.cache.SplitCache; @@ -216,7 +218,8 @@ private static CloseableHttpClient buildHttpClient(String apiToken, SplitClientC HttpClientBuilder httpClientbuilder = HttpClients.custom() .setConnectionManager(cm) .setDefaultRequestConfig(requestConfig) - .addRequestInterceptorLast(AddSplitHeadersFilter.instance(apiToken, config.ipAddressEnabled(), false)) + .addRequestInterceptorLast(AuthorizationInterceptorFilter.instance(apiToken)) + .addRequestInterceptorLast(SdkMetadataInterceptorFilter.instance(config.ipAddressEnabled(), SplitClientConfig.splitSdkVersion)) .addRequestInterceptorLast(new GzipEncoderRequestInterceptor()) .addResponseInterceptorLast((new GzipDecoderResponseInterceptor())); @@ -250,7 +253,8 @@ private static CloseableHttpClient buildSSEdHttpClient(String apiToken, SplitCli HttpClientBuilder httpClientbuilder = HttpClients.custom() .setConnectionManager(cm) .setDefaultRequestConfig(requestConfig) - .addRequestInterceptorLast(AddSplitHeadersFilter.instance(apiToken, config.ipAddressEnabled(), true)); + .addRequestInterceptorLast(SdkMetadataInterceptorFilter.instance(config.ipAddressEnabled(), SplitClientConfig.splitSdkVersion)) + .addRequestInterceptorLast(ClientKeyInterceptorFilter.instance(apiToken)); // Set up proxy is it exists if (config.proxy() != null) { diff --git a/client/src/main/java/io/split/client/interceptors/AddSplitHeadersFilter.java b/client/src/main/java/io/split/client/interceptors/AddSplitHeadersFilter.java deleted file mode 100644 index b7a80fc89..000000000 --- a/client/src/main/java/io/split/client/interceptors/AddSplitHeadersFilter.java +++ /dev/null @@ -1,87 +0,0 @@ -package io.split.client.interceptors; - -import io.split.client.SplitClientConfig; -import org.apache.hc.core5.http.EntityDetails; -import org.apache.hc.core5.http.HttpRequest; -import org.apache.hc.core5.http.HttpRequestInterceptor; -import org.apache.hc.core5.http.protocol.HttpContext; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -import java.net.InetAddress; - -import static com.google.common.base.Preconditions.checkNotNull; - -/** - * Created by adilaijaz on 5/22/15. - */ -public class AddSplitHeadersFilter implements HttpRequestInterceptor { - private static final Logger _log = LoggerFactory.getLogger(AddSplitHeadersFilter.class); - - /* package private for testing purposes */ - static final String AUTHORIZATION_HEADER = "Authorization"; - static final String CLIENT_MACHINE_NAME_HEADER = "SplitSDKMachineName"; - static final String CLIENT_MACHINE_IP_HEADER = "SplitSDKMachineIP"; - static final String CLIENT_VERSION = "SplitSDKVersion"; - static final String CLIENT_KEY = "SplitSDKClientKey"; - - private final String _apiTokenBearer; - private final String _hostname; - private final String _ip; - private final String _clientKey; - - public static AddSplitHeadersFilter instance(String apiToken, boolean ipAddressEnabled, boolean externalClient) { - if (!ipAddressEnabled) { - return new AddSplitHeadersFilter(apiToken, null, null, externalClient); - } - - String hostname = null; - String ip = null; - - try { - InetAddress localHost = InetAddress.getLocalHost(); - hostname = localHost.getHostName(); - ip = localHost.getHostAddress(); - } catch (Exception e) { - _log.error("Could not resolve InetAddress", e); - } - - return new AddSplitHeadersFilter(apiToken, hostname, ip, externalClient); - } - - private AddSplitHeadersFilter(String apiToken, String hostname, String ip, boolean externalClient) { - checkNotNull(apiToken); - - if (externalClient) { - _apiTokenBearer = null; - _clientKey = apiToken.substring(apiToken.length() - 4); - } else { - _apiTokenBearer = "Bearer " + apiToken; - _clientKey = null; - } - - _hostname = hostname; - _ip = ip; - } - - @Override - public void process(HttpRequest request, EntityDetails entity, HttpContext context) { - if (_apiTokenBearer != null) { - request.addHeader(AUTHORIZATION_HEADER, _apiTokenBearer); - } - - request.addHeader(CLIENT_VERSION, SplitClientConfig.splitSdkVersion); - - if (_hostname != null) { - request.addHeader(CLIENT_MACHINE_NAME_HEADER, _hostname); - } - - if (_ip != null) { - request.addHeader(CLIENT_MACHINE_IP_HEADER, _ip); - } - - if (_clientKey != null) { - request.addHeader(CLIENT_KEY, _clientKey); - } - } -} diff --git a/client/src/main/java/io/split/client/interceptors/AuthorizationInterceptorFilter.java b/client/src/main/java/io/split/client/interceptors/AuthorizationInterceptorFilter.java new file mode 100644 index 000000000..43b50949c --- /dev/null +++ b/client/src/main/java/io/split/client/interceptors/AuthorizationInterceptorFilter.java @@ -0,0 +1,30 @@ +package io.split.client.interceptors; + +import org.apache.hc.core5.http.EntityDetails; +import org.apache.hc.core5.http.HttpException; +import org.apache.hc.core5.http.HttpRequest; +import org.apache.hc.core5.http.HttpRequestInterceptor; +import org.apache.hc.core5.http.protocol.HttpContext; + +import java.io.IOException; + +import static com.google.common.base.Preconditions.checkNotNull; + +public class AuthorizationInterceptorFilter implements HttpRequestInterceptor { + static final String AUTHORIZATION_HEADER = "Authorization"; + + private final String _apiTokenBearer; + + public static AuthorizationInterceptorFilter instance(String apiToken) { + return new AuthorizationInterceptorFilter(apiToken); + } + + private AuthorizationInterceptorFilter(String apiToken) { + _apiTokenBearer = "Bearer " + checkNotNull(apiToken); + } + + @Override + public void process(HttpRequest httpRequest, EntityDetails entityDetails, HttpContext httpContext) throws HttpException, IOException { + httpRequest.addHeader(AUTHORIZATION_HEADER, _apiTokenBearer); + } +} diff --git a/client/src/main/java/io/split/client/interceptors/ClientKeyInterceptorFilter.java b/client/src/main/java/io/split/client/interceptors/ClientKeyInterceptorFilter.java new file mode 100644 index 000000000..dc096539e --- /dev/null +++ b/client/src/main/java/io/split/client/interceptors/ClientKeyInterceptorFilter.java @@ -0,0 +1,28 @@ +package io.split.client.interceptors; + +import org.apache.hc.core5.http.EntityDetails; +import org.apache.hc.core5.http.HttpException; +import org.apache.hc.core5.http.HttpRequest; +import org.apache.hc.core5.http.HttpRequestInterceptor; +import org.apache.hc.core5.http.protocol.HttpContext; + +import java.io.IOException; + +public class ClientKeyInterceptorFilter implements HttpRequestInterceptor { + static final String CLIENT_KEY = "SplitSDKClientKey"; + + private final String _clientKey; + + public static ClientKeyInterceptorFilter instance(String apiToken) { + return new ClientKeyInterceptorFilter(apiToken.substring(apiToken.length() - 4)); + } + + private ClientKeyInterceptorFilter(String clientKey) { + _clientKey = clientKey; + } + + @Override + public void process(HttpRequest httpRequest, EntityDetails entityDetails, HttpContext httpContext) throws HttpException, IOException { + httpRequest.addHeader(CLIENT_KEY, _clientKey); + } +} diff --git a/client/src/main/java/io/split/client/interceptors/SdkMetadataInterceptorFilter.java b/client/src/main/java/io/split/client/interceptors/SdkMetadataInterceptorFilter.java new file mode 100644 index 000000000..6cc059579 --- /dev/null +++ b/client/src/main/java/io/split/client/interceptors/SdkMetadataInterceptorFilter.java @@ -0,0 +1,61 @@ +package io.split.client.interceptors; + +import io.split.client.SplitClientConfig; +import org.apache.hc.core5.http.EntityDetails; +import org.apache.hc.core5.http.HttpException; +import org.apache.hc.core5.http.HttpRequest; +import org.apache.hc.core5.http.HttpRequestInterceptor; +import org.apache.hc.core5.http.protocol.HttpContext; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.io.IOException; +import java.net.InetAddress; + +public class SdkMetadataInterceptorFilter implements HttpRequestInterceptor { + private static final Logger _log = LoggerFactory.getLogger(SdkMetadataInterceptorFilter.class); + + static final String CLIENT_MACHINE_NAME_HEADER = "SplitSDKMachineName"; + static final String CLIENT_MACHINE_IP_HEADER = "SplitSDKMachineIP"; + static final String CLIENT_VERSION = "SplitSDKVersion"; + + private final String _hostname; + private final String _ip; + private final String _sdkVersion; + + public static SdkMetadataInterceptorFilter instance(boolean ipAddressEnabled, String sdkVersion) { + String hostName = null; + String ip = null; + + if (ipAddressEnabled) { + try { + InetAddress localHost = InetAddress.getLocalHost(); + hostName = localHost.getHostName(); + ip = localHost.getHostAddress(); + } catch (Exception e) { + _log.error("Could not resolve InetAddress", e); + } + } + + return new SdkMetadataInterceptorFilter(hostName, ip, sdkVersion); + } + + private SdkMetadataInterceptorFilter(String hostName, String ip, String sdkVersion) { + _sdkVersion = sdkVersion; + _hostname = hostName; + _ip = ip; + } + + @Override + public void process(HttpRequest httpRequest, EntityDetails entityDetails, HttpContext httpContext) throws HttpException, IOException { + httpRequest.addHeader(CLIENT_VERSION, SplitClientConfig.splitSdkVersion); + + if (_hostname != null) { + httpRequest.addHeader(CLIENT_MACHINE_NAME_HEADER, _hostname); + } + + if (_ip != null) { + httpRequest.addHeader(CLIENT_MACHINE_IP_HEADER, _ip); + } + } +} diff --git a/client/src/test/java/io/split/client/interceptors/AddSplitHeadersFilterTest.java b/client/src/test/java/io/split/client/interceptors/AddSplitHeadersFilterTest.java deleted file mode 100644 index b35abb8ba..000000000 --- a/client/src/test/java/io/split/client/interceptors/AddSplitHeadersFilterTest.java +++ /dev/null @@ -1,114 +0,0 @@ -package io.split.client.interceptors; - -import org.apache.hc.core5.http.HttpRequest; -import org.apache.hc.core5.http.protocol.HttpContext; -import org.junit.Test; -import org.junit.runner.RunWith; -import org.mockito.ArgumentCaptor; -import org.mockito.Captor; -import org.mockito.Mockito; - -import org.mockito.runners.MockitoJUnitRunner; - -import java.util.List; - -import static org.hamcrest.Matchers.*; -import static org.hamcrest.core.IsEqual.equalTo; -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertThat; - - -@RunWith(MockitoJUnitRunner.class) -public class AddSplitHeadersFilterTest { - - @Captor - private ArgumentCaptor headerNameCaptor; - - @Captor - private ArgumentCaptor headerValueCaptor; - - @Test - public void testHeadersWithIpAndHostname() { - AddSplitHeadersFilter filter = AddSplitHeadersFilter.instance("abc", true, false); - HttpRequest req = Mockito.mock(HttpRequest.class); - HttpContext ctx = Mockito.mock(HttpContext.class); - - filter.process(req, null, ctx); - Mockito.verify(req, Mockito.times(4)).addHeader(headerNameCaptor.capture(), headerValueCaptor.capture()); - - List headerNames = headerNameCaptor.getAllValues(); - List headerValues = headerValueCaptor.getAllValues(); - - assertThat(headerNames.size(), is(equalTo(4))); - assertThat(headerValues.size(), is(equalTo(4))); - - assertThat(headerNames, contains(AddSplitHeadersFilter.AUTHORIZATION_HEADER, - AddSplitHeadersFilter.CLIENT_VERSION, - AddSplitHeadersFilter.CLIENT_MACHINE_NAME_HEADER, - AddSplitHeadersFilter.CLIENT_MACHINE_IP_HEADER)); - } - - @Test - public void testHeadersWithoutIpAndHostname() { - AddSplitHeadersFilter filter = AddSplitHeadersFilter.instance("abc", false, false); - HttpRequest req = Mockito.mock(HttpRequest.class); - HttpContext ctx = Mockito.mock(HttpContext.class); - - filter.process(req, null, ctx); - Mockito.verify(req, Mockito.times(2)).addHeader(headerNameCaptor.capture(), headerValueCaptor.capture()); - - List headerNames = headerNameCaptor.getAllValues(); - List headerValues = headerValueCaptor.getAllValues(); - - assertThat(headerNames.size(), is(equalTo(2))); - assertThat(headerValues.size(), is(equalTo(2))); - - assertThat(headerNames, contains(AddSplitHeadersFilter.AUTHORIZATION_HEADER, - AddSplitHeadersFilter.CLIENT_VERSION)); - - assertThat(headerNames, not(contains(AddSplitHeadersFilter.CLIENT_MACHINE_NAME_HEADER, - AddSplitHeadersFilter.CLIENT_MACHINE_IP_HEADER))); - } - - @Test - public void testHeadersWithoutIpAndHostnameAndWithClientKey() { - AddSplitHeadersFilter filter = AddSplitHeadersFilter.instance("ljsdfkjkldfjksldjflksdjflsdjflksd", false, true); - HttpRequest req = Mockito.mock(HttpRequest.class); - HttpContext ctx = Mockito.mock(HttpContext.class); - - filter.process(req, null, ctx); - Mockito.verify(req, Mockito.times(2)).addHeader(headerNameCaptor.capture(), headerValueCaptor.capture()); - - List headerNames = headerNameCaptor.getAllValues(); - List headerValues = headerValueCaptor.getAllValues(); - - assertEquals(2 ,headerNames.size()); - assertEquals(2, headerValues.size()); - assertThat(headerNames, contains(AddSplitHeadersFilter.CLIENT_VERSION, - AddSplitHeadersFilter.CLIENT_KEY)); - assertThat(headerNames, not(contains(AddSplitHeadersFilter.CLIENT_MACHINE_NAME_HEADER, - AddSplitHeadersFilter.CLIENT_MACHINE_IP_HEADER, - AddSplitHeadersFilter.AUTHORIZATION_HEADER))); - } - - @Test - public void testHeadersWithIpAndHostnameAndWithClientKey() { - AddSplitHeadersFilter filter = AddSplitHeadersFilter.instance("ljsdfkjkldfjksldjflksdjflsdjflksd", true, true); - HttpRequest req = Mockito.mock(HttpRequest.class); - HttpContext ctx = Mockito.mock(HttpContext.class); - - filter.process(req, null, ctx); - Mockito.verify(req, Mockito.times(4)).addHeader(headerNameCaptor.capture(), headerValueCaptor.capture()); - - List headerNames = headerNameCaptor.getAllValues(); - List headerValues = headerValueCaptor.getAllValues(); - - assertEquals(4 ,headerNames.size()); - assertEquals(4, headerValues.size()); - assertThat(headerNames, contains(AddSplitHeadersFilter.CLIENT_VERSION, - AddSplitHeadersFilter.CLIENT_MACHINE_NAME_HEADER, - AddSplitHeadersFilter.CLIENT_MACHINE_IP_HEADER, - AddSplitHeadersFilter.CLIENT_KEY)); - assertThat(headerNames, not(contains(AddSplitHeadersFilter.AUTHORIZATION_HEADER))); - } -} diff --git a/client/src/test/java/io/split/client/interceptors/AuthorizationInterceptorFilterTest.java b/client/src/test/java/io/split/client/interceptors/AuthorizationInterceptorFilterTest.java new file mode 100644 index 000000000..1f10a90dd --- /dev/null +++ b/client/src/test/java/io/split/client/interceptors/AuthorizationInterceptorFilterTest.java @@ -0,0 +1,51 @@ +package io.split.client.interceptors; + +import org.apache.hc.core5.http.HttpException; +import org.apache.hc.core5.http.HttpRequest; +import org.apache.hc.core5.http.protocol.HttpContext; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.mockito.ArgumentCaptor; +import org.mockito.Captor; +import org.mockito.Mockito; +import org.mockito.runners.MockitoJUnitRunner; + +import java.io.IOException; +import java.util.List; + +import static org.hamcrest.Matchers.contains; +import static org.junit.Assert.*; + +@RunWith(MockitoJUnitRunner.class) +public class AuthorizationInterceptorFilterTest { + + @Captor + private ArgumentCaptor headerNameCaptor; + + @Captor + private ArgumentCaptor headerValueCaptor; + + @Test + public void authorizationInterceptorWithValue() throws IOException, HttpException { + AuthorizationInterceptorFilter filter = AuthorizationInterceptorFilter.instance("api-token-test"); + HttpRequest req = Mockito.mock(HttpRequest.class); + HttpContext ctx = Mockito.mock(HttpContext.class); + + filter.process(req, null, ctx); + Mockito.verify(req, Mockito.times(1)).addHeader(headerNameCaptor.capture(), headerValueCaptor.capture()); + + List headerNames = headerNameCaptor.getAllValues(); + List headerValues = headerValueCaptor.getAllValues(); + + assertEquals(1, headerNames.size()); + assertEquals(1, headerValues.size()); + + assertThat(headerNames, contains(AuthorizationInterceptorFilter.AUTHORIZATION_HEADER)); + assertThat(headerValues, contains("Bearer api-token-test")); + } + + @Test(expected = Exception.class) + public void authorizationInterceptorWithoutValue() { + AuthorizationInterceptorFilter filter = AuthorizationInterceptorFilter.instance(null); + } +} diff --git a/client/src/test/java/io/split/client/interceptors/ClientKeyInterceptorFilterTest.java b/client/src/test/java/io/split/client/interceptors/ClientKeyInterceptorFilterTest.java new file mode 100644 index 000000000..56121882b --- /dev/null +++ b/client/src/test/java/io/split/client/interceptors/ClientKeyInterceptorFilterTest.java @@ -0,0 +1,46 @@ +package io.split.client.interceptors; + +import org.apache.hc.core5.http.HttpException; +import org.apache.hc.core5.http.HttpRequest; +import org.apache.hc.core5.http.protocol.HttpContext; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.mockito.ArgumentCaptor; +import org.mockito.Captor; +import org.mockito.Mockito; +import org.mockito.runners.MockitoJUnitRunner; + +import java.io.IOException; +import java.util.List; + +import static org.hamcrest.Matchers.contains; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertThat; + +@RunWith(MockitoJUnitRunner.class) +public class ClientKeyInterceptorFilterTest { + @Captor + private ArgumentCaptor headerNameCaptor; + + @Captor + private ArgumentCaptor headerValueCaptor; + + @Test + public void ClientKeyInterceptorFilter() throws IOException, HttpException { + ClientKeyInterceptorFilter filter = ClientKeyInterceptorFilter.instance("api-token-test-1234"); + HttpRequest req = Mockito.mock(HttpRequest.class); + HttpContext ctx = Mockito.mock(HttpContext.class); + + filter.process(req, null, ctx); + Mockito.verify(req, Mockito.times(1)).addHeader(headerNameCaptor.capture(), headerValueCaptor.capture()); + + List headerNames = headerNameCaptor.getAllValues(); + List headerValues = headerValueCaptor.getAllValues(); + + assertEquals(1, headerNames.size()); + assertEquals(1, headerValues.size()); + + assertThat(headerNames, contains(ClientKeyInterceptorFilter.CLIENT_KEY)); + assertThat(headerValues, contains("1234")); + } +} diff --git a/client/src/test/java/io/split/client/interceptors/SdkMetadataInterceptorFilterTest.java b/client/src/test/java/io/split/client/interceptors/SdkMetadataInterceptorFilterTest.java new file mode 100644 index 000000000..6f1336516 --- /dev/null +++ b/client/src/test/java/io/split/client/interceptors/SdkMetadataInterceptorFilterTest.java @@ -0,0 +1,65 @@ +package io.split.client.interceptors; + +import org.apache.hc.core5.http.HttpException; +import org.apache.hc.core5.http.HttpRequest; +import org.apache.hc.core5.http.protocol.HttpContext; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.mockito.ArgumentCaptor; +import org.mockito.Captor; +import org.mockito.Mockito; +import org.mockito.runners.MockitoJUnitRunner; + +import java.io.IOException; +import java.util.List; + +import static org.hamcrest.Matchers.contains; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertThat; + +@RunWith(MockitoJUnitRunner.class) +public class SdkMetadataInterceptorFilterTest { + @Captor + private ArgumentCaptor headerNameCaptor; + + @Captor + private ArgumentCaptor headerValueCaptor; + + @Test + public void sdkMetadataWithIpEnabled() throws IOException, HttpException { + SdkMetadataInterceptorFilter filter = SdkMetadataInterceptorFilter.instance(true, "sdk-version-1.2.3"); + HttpRequest req = Mockito.mock(HttpRequest.class); + HttpContext ctx = Mockito.mock(HttpContext.class); + + filter.process(req, null, ctx); + Mockito.verify(req, Mockito.times(3)).addHeader(headerNameCaptor.capture(), headerValueCaptor.capture()); + + List headerNames = headerNameCaptor.getAllValues(); + List headerValues = headerValueCaptor.getAllValues(); + + assertEquals(3, headerNames.size()); + assertEquals(3, headerValues.size()); + + assertThat(headerNames, contains(SdkMetadataInterceptorFilter.CLIENT_VERSION, + SdkMetadataInterceptorFilter.CLIENT_MACHINE_NAME_HEADER, + SdkMetadataInterceptorFilter.CLIENT_MACHINE_IP_HEADER)); + } + + @Test + public void sdkMetadataWithIpDisabled() throws IOException, HttpException { + SdkMetadataInterceptorFilter filter = SdkMetadataInterceptorFilter.instance(false, "sdk-version-1.2.3"); + HttpRequest req = Mockito.mock(HttpRequest.class); + HttpContext ctx = Mockito.mock(HttpContext.class); + + filter.process(req, null, ctx); + Mockito.verify(req, Mockito.times(1)).addHeader(headerNameCaptor.capture(), headerValueCaptor.capture()); + + List headerNames = headerNameCaptor.getAllValues(); + List headerValues = headerValueCaptor.getAllValues(); + + assertEquals(1, headerNames.size()); + assertEquals(1, headerValues.size()); + + assertThat(headerNames, contains(SdkMetadataInterceptorFilter.CLIENT_VERSION)); + } +} From c83b12bb6db79805f065f678cb332ff1dfbacfc4 Mon Sep 17 00:00:00 2001 From: Lucas Echeverz Date: Mon, 3 May 2021 18:15:35 -0300 Subject: [PATCH 38/81] Recording telemetry data --- .../java/io/split/client/EventClientImpl.java | 17 +++++++++--- .../java/io/split/client/SplitClientImpl.java | 26 ++++++++++++------- .../io/split/client/SplitFactoryBuilder.java | 6 ++--- .../io/split/client/SplitFactoryImpl.java | 19 +++++++++++++- .../AsynchronousImpressionListener.java | 15 ++++++++--- .../engine/experiments/SplitFetcherImp.java | 9 ++++++- .../engine/segments/SegmentFetcherImp.java | 9 ++++++- .../synchronizer/TelemetrySyncTask.java | 7 ++++- .../io/split/client/SplitFactoryImplTest.java | 10 +++---- 9 files changed, 90 insertions(+), 28 deletions(-) diff --git a/client/src/main/java/io/split/client/EventClientImpl.java b/client/src/main/java/io/split/client/EventClientImpl.java index 5c3ae8c13..6e6e3f842 100644 --- a/client/src/main/java/io/split/client/EventClientImpl.java +++ b/client/src/main/java/io/split/client/EventClientImpl.java @@ -4,6 +4,9 @@ import io.split.client.dtos.Event; import io.split.client.utils.GenericClientUtil; import io.split.client.utils.Utils; +import io.split.telemetry.domain.enums.LastSynchronizationRecordsEnum; +import io.split.telemetry.storage.TelemetryEvaluationProducer; +import io.split.telemetry.storage.TelemetryRuntimeProducer; import org.apache.hc.client5.http.impl.classic.CloseableHttpClient; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -23,6 +26,7 @@ import java.util.concurrent.TimeUnit; import static java.lang.Thread.MIN_PRIORITY; +import static com.google.common.base.Preconditions.checkNotNull; /** * Responsible for sending events added via .track() to Split collection services @@ -45,6 +49,7 @@ public class EventClientImpl implements EventClient { private final CloseableHttpClient _httpclient; private final URI _target; private final int _waitBeforeShutdown; + private final TelemetryRuntimeProducer telemetryRuntimeProducer; ThreadFactory eventClientThreadFactory(final String name) { return new ThreadFactory() { @@ -62,17 +67,18 @@ public void run() { } - public static EventClientImpl create(CloseableHttpClient httpclient, URI eventsRootTarget, int maxQueueSize, long flushIntervalMillis, int waitBeforeShutdown) throws URISyntaxException { + public static EventClientImpl create(CloseableHttpClient httpclient, URI eventsRootTarget, int maxQueueSize, long flushIntervalMillis, int waitBeforeShutdown, TelemetryEvaluationProducer telemetryEvaluationProducer) throws URISyntaxException { return new EventClientImpl(new LinkedBlockingQueue(), httpclient, Utils.appendPath(eventsRootTarget, "api/events/bulk"), maxQueueSize, flushIntervalMillis, - waitBeforeShutdown); + waitBeforeShutdown, + telemetryEvaluationProducer); } EventClientImpl(BlockingQueue eventQueue, CloseableHttpClient httpclient, URI target, int maxQueueSize, - long flushIntervalMillis, int waitBeforeShutdown) throws URISyntaxException { + long flushIntervalMillis, int waitBeforeShutdown, TelemetryEvaluationProducer telemetryEvaluationProducer) throws URISyntaxException { _httpclient = httpclient; @@ -83,6 +89,7 @@ public static EventClientImpl create(CloseableHttpClient httpclient, URI eventsR _maxQueueSize = maxQueueSize; _flushIntervalMillis = flushIntervalMillis; + telemetryRuntimeProducer = checkNotNull(telemetryEvaluationProducer); _senderExecutor = new ThreadPoolExecutor( 1, @@ -169,7 +176,7 @@ public void run() { continue; } - + long initTime = System.currentTimeMillis(); if (events.size() >= _maxQueueSize || accumulated >= MAX_SIZE_BYTES || event == CENTINEL) { // Send over the network @@ -183,6 +190,8 @@ public void run() { // Clear the queue of events for the next batch. events = new ArrayList<>(); accumulated = 0; + long endTime = System.currentTimeMillis(); + telemetryRuntimeProducer.recordSuccessfulSync(LastSynchronizationRecordsEnum.EVENTS, endTime-initTime); } } } catch (InterruptedException e) { diff --git a/client/src/main/java/io/split/client/SplitClientImpl.java b/client/src/main/java/io/split/client/SplitClientImpl.java index 3129e4526..efeec3c3f 100644 --- a/client/src/main/java/io/split/client/SplitClientImpl.java +++ b/client/src/main/java/io/split/client/SplitClientImpl.java @@ -10,12 +10,13 @@ import io.split.engine.evaluator.Evaluator; import io.split.engine.evaluator.EvaluatorImp; import io.split.engine.evaluator.Labels; -import io.split.engine.metrics.Metrics; import io.split.grammar.Treatments; import io.split.inputValidation.EventsValidator; import io.split.inputValidation.KeyValidator; import io.split.inputValidation.SplitNameValidator; import io.split.inputValidation.TrafficTypeValidator; +import io.split.telemetry.domain.enums.MethodEnum; +import io.split.telemetry.storage.TelemetryEvaluationProducer; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -47,6 +48,7 @@ public final class SplitClientImpl implements SplitClient { private final EventClient _eventClient; private final SDKReadinessGates _gates; private final Evaluator _evaluator; + private final TelemetryEvaluationProducer _telemetryEvaluationProducer; public SplitClientImpl(SplitFactory container, SplitCache splitCache, @@ -54,7 +56,8 @@ public SplitClientImpl(SplitFactory container, EventClient eventClient, SplitClientConfig config, SDKReadinessGates gates, - Evaluator evaluator) { + Evaluator evaluator, + TelemetryEvaluationProducer telemetryEvaluationProducer) { _container = container; _splitCache = checkNotNull(splitCache); _impressionManager = checkNotNull(impressionManager); @@ -62,6 +65,7 @@ public SplitClientImpl(SplitFactory container, _config = config; _gates = checkNotNull(gates); _evaluator = checkNotNull(evaluator); + _telemetryEvaluationProducer = checkNotNull(telemetryEvaluationProducer); } @Override @@ -71,27 +75,27 @@ public String getTreatment(String key, String split) { @Override public String getTreatment(String key, String split, Map attributes) { - return getTreatmentWithConfigInternal(GET_TREATMENT, key, null, split, attributes).treatment(); + return getTreatmentWithConfigInternal(GET_TREATMENT, key, null, split, attributes, MethodEnum.TREATMENT).treatment(); } @Override public String getTreatment(Key key, String split, Map attributes) { - return getTreatmentWithConfigInternal(GET_TREATMENT, key.matchingKey(), key.bucketingKey(), split, attributes).treatment(); + return getTreatmentWithConfigInternal(GET_TREATMENT, key.matchingKey(), key.bucketingKey(), split, attributes, MethodEnum.TREATMENT).treatment(); } @Override public SplitResult getTreatmentWithConfig(String key, String split) { - return getTreatmentWithConfigInternal(GET_TREATMENT_WITH_CONFIG, key, null, split, Collections.emptyMap()); + return getTreatmentWithConfigInternal(GET_TREATMENT_WITH_CONFIG, key, null, split, Collections.emptyMap(), MethodEnum.TREATMENT_WITH_CONFIG); } @Override public SplitResult getTreatmentWithConfig(String key, String split, Map attributes) { - return getTreatmentWithConfigInternal(GET_TREATMENT_WITH_CONFIG, key, null, split, attributes); + return getTreatmentWithConfigInternal(GET_TREATMENT_WITH_CONFIG, key, null, split, attributes, MethodEnum.TREATMENT_WITH_CONFIG); } @Override public SplitResult getTreatmentWithConfig(Key key, String split, Map attributes) { - return getTreatmentWithConfigInternal(GET_TREATMENT_WITH_CONFIG, key.matchingKey(), key.bucketingKey(), split, attributes); + return getTreatmentWithConfigInternal(GET_TREATMENT_WITH_CONFIG, key.matchingKey(), key.bucketingKey(), split, attributes, MethodEnum.TREATMENT_WITH_CONFIG); } @Override @@ -174,9 +178,11 @@ private boolean track(Event event) { return _eventClient.track(event, propertiesResult.getEventSize()); } - private SplitResult getTreatmentWithConfigInternal(String method, String matchingKey, String bucketingKey, String split, Map attributes) { + private SplitResult getTreatmentWithConfigInternal(String method, String matchingKey, String bucketingKey, String split, Map attributes, MethodEnum methodEnum) { + long initTime = System.currentTimeMillis(); try { if (_container.isDestroyed()) { + _telemetryEvaluationProducer.recordException(methodEnum); _log.error("Client has already been destroyed - no calls possible"); return SPLIT_RESULT_CONTROL; } @@ -216,10 +222,12 @@ private SplitResult getTreatmentWithConfigInternal(String method, String matchin result.changeNumber, attributes ); - + long endTime = System.currentTimeMillis(); + _telemetryEvaluationProducer.recordLatency(methodEnum, endTime-initTime); return new SplitResult(result.treatment, result.configurations); } catch (Exception e) { try { + _telemetryEvaluationProducer.recordException(methodEnum); _log.error("CatchAll Exception", e); } catch (Exception e1) { // ignore diff --git a/client/src/main/java/io/split/client/SplitFactoryBuilder.java b/client/src/main/java/io/split/client/SplitFactoryBuilder.java index f18032416..7f78f4bd8 100644 --- a/client/src/main/java/io/split/client/SplitFactoryBuilder.java +++ b/client/src/main/java/io/split/client/SplitFactoryBuilder.java @@ -25,7 +25,7 @@ public class SplitFactoryBuilder { * @throws IOException if the SDK was being started in 'localhost' mode, but * there were problems reading the override file from disk. */ - public static SplitFactory build(String apiToken) throws IOException, URISyntaxException { + public static SplitFactory build(String apiToken) throws Exception { return build(apiToken, SplitClientConfig.builder().build()); } @@ -36,7 +36,7 @@ public static SplitFactory build(String apiToken) throws IOException, URISyntaxE * @throws java.io.IOException if the SDK was being started in 'localhost' mode, but * there were problems reading the override file from disk. */ - public static synchronized SplitFactory build(String apiToken, SplitClientConfig config) throws IOException, URISyntaxException { + public static synchronized SplitFactory build(String apiToken, SplitClientConfig config) throws Exception { ApiKeyValidator.validate(apiToken); if (LocalhostSplitFactory.LOCALHOST.equals(apiToken)) { @@ -66,7 +66,7 @@ public static SplitFactory local(SplitClientConfig config) throws IOException, U return LocalhostSplitFactory.createLocalhostSplitFactory(config); } - public static void main(String... args) throws IOException, InterruptedException, TimeoutException, URISyntaxException { + public static void main(String... args) throws Exception { if (args.length != 1) { System.out.println("Usage: "); System.exit(1); diff --git a/client/src/main/java/io/split/client/SplitFactoryImpl.java b/client/src/main/java/io/split/client/SplitFactoryImpl.java index 84b7478a9..bfb263dda 100644 --- a/client/src/main/java/io/split/client/SplitFactoryImpl.java +++ b/client/src/main/java/io/split/client/SplitFactoryImpl.java @@ -24,6 +24,11 @@ import io.split.cache.SegmentCacheInMemoryImpl; import io.split.engine.segments.SegmentSynchronizationTaskImp; import io.split.integrations.IntegrationsConfig; +import io.split.telemetry.storage.InMemoryTelemetryStorage; +import io.split.telemetry.storage.TelemetryStorage; +import io.split.telemetry.synchronizer.SynchronizerMemory; +import io.split.telemetry.synchronizer.TelemetrySyncTask; +import io.split.telemetry.synchronizer.TelemetrySynchronizer; import org.apache.hc.client5.http.auth.AuthScope; import org.apache.hc.client5.http.auth.Credentials; import org.apache.hc.client5.http.auth.UsernamePasswordCredentials; @@ -85,12 +90,19 @@ public class SplitFactoryImpl implements SplitFactory { private boolean isTerminated = false; private final ApiKeyCounter _apiKeyCounter; + private final TelemetryStorage _telemetryStorage; + private final TelemetrySynchronizer _telemetrySynchronizer; + private final TelemetrySyncTask _telemetrySyncTask; + private final long _startTime; - public SplitFactoryImpl(String apiToken, SplitClientConfig config) throws URISyntaxException { + public SplitFactoryImpl(String apiToken, SplitClientConfig config) throws Exception { + _startTime = System.currentTimeMillis(); _apiToken = apiToken; _apiKeyCounter = ApiKeyCounter.getApiKeyCounterInstance(); _apiKeyCounter.add(apiToken); + _telemetryStorage = new InMemoryTelemetryStorage(); + if (config.blockUntilReady() == -1) { //BlockUntilReady not been set _log.warn("no setBlockUntilReadyTimeout parameter has been set - incorrect control treatments could be logged” " + @@ -114,6 +126,8 @@ public SplitFactoryImpl(String apiToken, SplitClientConfig config) throws URISyn // Cache Initialisations _segmentCache = new SegmentCacheInMemoryImpl(); _splitCache = new InMemoryCacheImp(); + _telemetrySynchronizer = new SynchronizerMemory(_httpclient, URI.create(config.get_telemetryURL()), _telemetryStorage, _splitCache, _segmentCache); + _telemetrySyncTask = new TelemetrySyncTask(config.get_telemetryRefreshRate(), _telemetrySynchronizer); // Segments _segmentSynchronizationTaskImp = buildSegments(config); @@ -180,6 +194,9 @@ public synchronized void destroy() { _log.info("Successful shutdown of eventClient"); _syncManager.shutdown(); _log.info("Successful shutdown of syncManager"); + long endSession = System.currentTimeMillis(); + _telemetryStorage.recordSessionLength(endSession - _startTime); + _telemetrySyncTask.stopScheduledTask(); } catch (IOException e) { _log.error("We could not shutdown split", e); } diff --git a/client/src/main/java/io/split/client/impressions/AsynchronousImpressionListener.java b/client/src/main/java/io/split/client/impressions/AsynchronousImpressionListener.java index ed6a1e811..7c3a8716a 100644 --- a/client/src/main/java/io/split/client/impressions/AsynchronousImpressionListener.java +++ b/client/src/main/java/io/split/client/impressions/AsynchronousImpressionListener.java @@ -1,6 +1,8 @@ package io.split.client.impressions; import com.google.common.util.concurrent.ThreadFactoryBuilder; +import io.split.telemetry.domain.enums.LastSynchronizationRecordsEnum; +import io.split.telemetry.storage.TelemetryRuntimeProducer; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -10,6 +12,8 @@ import java.util.concurrent.ThreadPoolExecutor; import java.util.concurrent.TimeUnit; +import static com.google.common.base.Preconditions.checkNotNull; + /** * A wrapper around an ImpressionListener provided by the customer. The purpose * of the wrapper is to protect the SplitClient from any slow down happening due @@ -23,8 +27,9 @@ public class AsynchronousImpressionListener implements ImpressionListener { private final ImpressionListener _delegate; private final ExecutorService _executor; + private final TelemetryRuntimeProducer _telemetryRuntimeProducer; - public static AsynchronousImpressionListener build(ImpressionListener delegate, int capacity) { + public static AsynchronousImpressionListener build(ImpressionListener delegate, int capacity, TelemetryRuntimeProducer telemetryRuntimeProducer) { ThreadFactory threadFactory = new ThreadFactoryBuilder() .setDaemon(true) .setNameFormat("impression-listener-wrapper-%d") @@ -32,24 +37,28 @@ public static AsynchronousImpressionListener build(ImpressionListener delegate, ExecutorService executor = new ThreadPoolExecutor(2, 2, 0L, TimeUnit.MILLISECONDS, new ArrayBlockingQueue(capacity), threadFactory); - return new AsynchronousImpressionListener(delegate, executor); + return new AsynchronousImpressionListener(delegate, executor, telemetryRuntimeProducer); } - public AsynchronousImpressionListener(ImpressionListener delegate, ExecutorService executor) { + public AsynchronousImpressionListener(ImpressionListener delegate, ExecutorService executor, TelemetryRuntimeProducer telemetryRuntimeProducer) { _delegate = delegate; _executor = executor; + _telemetryRuntimeProducer = checkNotNull(telemetryRuntimeProducer); } @Override public void log(final Impression impression) { try { + long initTime = System.currentTimeMillis(); _executor.execute(new Runnable() { @Override public void run() { _delegate.log(impression); } }); + long endTime = System.currentTimeMillis(); + _telemetryRuntimeProducer.recordSuccessfulSync(LastSynchronizationRecordsEnum.IMPRESSIONS, endTime - initTime); } catch (Exception e) { _log.warn("Unable to send impression to impression listener", e); diff --git a/client/src/main/java/io/split/engine/experiments/SplitFetcherImp.java b/client/src/main/java/io/split/engine/experiments/SplitFetcherImp.java index 510001153..227224b4b 100644 --- a/client/src/main/java/io/split/engine/experiments/SplitFetcherImp.java +++ b/client/src/main/java/io/split/engine/experiments/SplitFetcherImp.java @@ -5,6 +5,8 @@ import io.split.client.dtos.Status; import io.split.engine.SDKReadinessGates; import io.split.cache.SplitCache; +import io.split.telemetry.domain.enums.LastSynchronizationRecordsEnum; +import io.split.telemetry.storage.TelemetryRuntimeProducer; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -24,6 +26,7 @@ public class SplitFetcherImp implements SplitFetcher { private final SplitCache _splitCache; private final SDKReadinessGates _gates; private final Object _lock = new Object(); + private final TelemetryRuntimeProducer _telemetryRuntimeProducer; /** * Contains all the traffic types that are currently being used by the splits and also the count @@ -35,11 +38,12 @@ public class SplitFetcherImp implements SplitFetcher { * an ARCHIVED split is received, we know if we need to remove a traffic type from the multiset. */ - public SplitFetcherImp(SplitChangeFetcher splitChangeFetcher, SplitParser parser, SDKReadinessGates gates, SplitCache splitCache) { + public SplitFetcherImp(SplitChangeFetcher splitChangeFetcher, SplitParser parser, SDKReadinessGates gates, SplitCache splitCache, TelemetryRuntimeProducer telemetryRuntimeProducer) { _splitChangeFetcher = checkNotNull(splitChangeFetcher); _parser = checkNotNull(parser); _gates = checkNotNull(gates); _splitCache = checkNotNull(splitCache); + _telemetryRuntimeProducer = checkNotNull(telemetryRuntimeProducer); } @Override @@ -69,6 +73,7 @@ public void run() { } private void runWithoutExceptionHandling(boolean addCacheHeader) throws InterruptedException { + long initTime = System.currentTimeMillis(); SplitChange change = _splitChangeFetcher.fetch(_splitCache.getChangeNumber(), addCacheHeader); if (change == null) { @@ -136,6 +141,8 @@ private void runWithoutExceptionHandling(boolean addCacheHeader) throws Interrup } _splitCache.setChangeNumber(change.till); + long endtime = System.currentTimeMillis(); + _telemetryRuntimeProducer.recordSuccessfulSync(LastSynchronizationRecordsEnum.SPLITS, endtime-initTime); } } @Override diff --git a/client/src/main/java/io/split/engine/segments/SegmentFetcherImp.java b/client/src/main/java/io/split/engine/segments/SegmentFetcherImp.java index ac21e8461..8d66ca363 100644 --- a/client/src/main/java/io/split/engine/segments/SegmentFetcherImp.java +++ b/client/src/main/java/io/split/engine/segments/SegmentFetcherImp.java @@ -3,6 +3,8 @@ import io.split.cache.SegmentCache; import io.split.client.dtos.SegmentChange; import io.split.engine.SDKReadinessGates; +import io.split.telemetry.domain.enums.LastSynchronizationRecordsEnum; +import io.split.telemetry.storage.TelemetryRuntimeProducer; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -18,14 +20,16 @@ public class SegmentFetcherImp implements SegmentFetcher { private final SegmentChangeFetcher _segmentChangeFetcher; private final SegmentCache _segmentCache; private final SDKReadinessGates _gates; + private final TelemetryRuntimeProducer _telemetryRuntimeProducer; private final Object _lock = new Object(); - public SegmentFetcherImp(String segmentName, SegmentChangeFetcher segmentChangeFetcher, SDKReadinessGates gates, SegmentCache segmentCache) { + public SegmentFetcherImp(String segmentName, SegmentChangeFetcher segmentChangeFetcher, SDKReadinessGates gates, SegmentCache segmentCache, TelemetryRuntimeProducer telemetryRuntimeProducer) { _segmentName = checkNotNull(segmentName); _segmentChangeFetcher = checkNotNull(segmentChangeFetcher); _segmentCache = checkNotNull(segmentCache); _gates = checkNotNull(gates); + _telemetryRuntimeProducer = checkNotNull(telemetryRuntimeProducer); _segmentCache.updateSegment(segmentName, new ArrayList<>(), new ArrayList<>()); } @@ -43,6 +47,7 @@ public void fetch(boolean addCacheHeader){ } private void runWithoutExceptionHandling(boolean addCacheHeader) { + long initTime = System.currentTimeMillis(); SegmentChange change = _segmentChangeFetcher.fetch(_segmentName, _segmentCache.getChangeNumber(_segmentName), addCacheHeader); if (change == null) { @@ -86,6 +91,8 @@ private void runWithoutExceptionHandling(boolean addCacheHeader) { } _segmentCache.setChangeNumber(_segmentName,change.till); + long endTime = System.currentTimeMillis(); + _telemetryRuntimeProducer.recordSuccessfulSync(LastSynchronizationRecordsEnum.SEGMENTS, endTime-initTime); } } diff --git a/client/src/main/java/io/split/telemetry/synchronizer/TelemetrySyncTask.java b/client/src/main/java/io/split/telemetry/synchronizer/TelemetrySyncTask.java index 992e73387..d8c2b3a5f 100644 --- a/client/src/main/java/io/split/telemetry/synchronizer/TelemetrySyncTask.java +++ b/client/src/main/java/io/split/telemetry/synchronizer/TelemetrySyncTask.java @@ -22,6 +22,11 @@ public TelemetrySyncTask(int telemetryRefreshRate, TelemetrySynchronizer telemet _telemetrySynchronizer = telemetrySynchronizer; //TODO _telemetryRefreshRate = telemetryRefreshRate; _telemetrySyncScheduledExecutorService = Executors.newSingleThreadScheduledExecutor(telemetrySyncThreadFactory); + try { + this.startScheduledTask(); + } catch (Exception e) { + e.printStackTrace(); + } } @VisibleForTesting @@ -35,7 +40,7 @@ protected void startScheduledTask() throws Exception { },0l, _telemetryRefreshRate, TimeUnit.SECONDS); } - protected void stopScheduledTask() { + public void stopScheduledTask() { try { _telemetrySynchronizer.synchronizeStats(); } catch (Exception e) { diff --git a/client/src/test/java/io/split/client/SplitFactoryImplTest.java b/client/src/test/java/io/split/client/SplitFactoryImplTest.java index 451bf7dcc..3e09c697e 100644 --- a/client/src/test/java/io/split/client/SplitFactoryImplTest.java +++ b/client/src/test/java/io/split/client/SplitFactoryImplTest.java @@ -15,7 +15,7 @@ public class SplitFactoryImplTest extends TestCase { @Test - public void testFactoryInstantiation() throws URISyntaxException { + public void testFactoryInstantiation() throws Exception { SplitClientConfig splitClientConfig = SplitClientConfig.builder() .enableDebug() .impressionsMode(ImpressionsManager.Mode.DEBUG) @@ -31,7 +31,7 @@ public void testFactoryInstantiation() throws URISyntaxException { } @Test - public void testFactoryInstantiationWithoutBlockUntilReady() throws URISyntaxException { + public void testFactoryInstantiationWithoutBlockUntilReady() throws Exception { SplitClientConfig splitClientConfig = SplitClientConfig.builder() .enableDebug() .impressionsMode(ImpressionsManager.Mode.DEBUG) @@ -46,7 +46,7 @@ public void testFactoryInstantiationWithoutBlockUntilReady() throws URISyntaxExc } @Test - public void testFactoryInstantiationIntegrationsConfig() throws URISyntaxException { + public void testFactoryInstantiationIntegrationsConfig() throws Exception { IntegrationsConfig integrationsConfig = new IntegrationsConfig.Builder().build(); SplitClientConfig splitClientConfig = SplitClientConfig.builder() .enableDebug() @@ -64,7 +64,7 @@ public void testFactoryInstantiationIntegrationsConfig() throws URISyntaxExcepti } @Test - public void testFactoryInstantiationWithProxy() throws URISyntaxException { + public void testFactoryInstantiationWithProxy() throws Exception { SplitClientConfig splitClientConfig = SplitClientConfig.builder() .enableDebug() .impressionsMode(ImpressionsManager.Mode.DEBUG) @@ -84,7 +84,7 @@ public void testFactoryInstantiationWithProxy() throws URISyntaxException { } @Test - public void testFactoryDestroy() throws URISyntaxException { + public void testFactoryDestroy() throws Exception { SplitClientConfig splitClientConfig = SplitClientConfig.builder() .enableDebug() .impressionsMode(ImpressionsManager.Mode.DEBUG) From 6aa82ed63e47a99d5363adb6afcb5593ef334e6c Mon Sep 17 00:00:00 2001 From: Mauro Sanz Date: Tue, 4 May 2021 10:11:15 -0300 Subject: [PATCH 39/81] pr feedback --- client/src/main/java/io/split/engine/common/PushManagerImp.java | 1 - .../src/main/java/io/split/engine/sse/EventSourceClientImp.java | 1 - client/src/main/java/io/split/engine/sse/client/SSEClient.java | 2 -- 3 files changed, 4 deletions(-) diff --git a/client/src/main/java/io/split/engine/common/PushManagerImp.java b/client/src/main/java/io/split/engine/common/PushManagerImp.java index 26a34d9a2..1d770b6b1 100644 --- a/client/src/main/java/io/split/engine/common/PushManagerImp.java +++ b/client/src/main/java/io/split/engine/common/PushManagerImp.java @@ -20,7 +20,6 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import java.util.Map; import java.util.concurrent.Future; import java.util.concurrent.LinkedBlockingQueue; import java.util.concurrent.ScheduledExecutorService; diff --git a/client/src/main/java/io/split/engine/sse/EventSourceClientImp.java b/client/src/main/java/io/split/engine/sse/EventSourceClientImp.java index 142d7c80c..7d8bf990d 100644 --- a/client/src/main/java/io/split/engine/sse/EventSourceClientImp.java +++ b/client/src/main/java/io/split/engine/sse/EventSourceClientImp.java @@ -14,7 +14,6 @@ import java.net.URI; import java.net.URISyntaxException; -import java.util.Map; import java.util.concurrent.atomic.AtomicBoolean; import static com.google.common.base.Preconditions.checkNotNull; diff --git a/client/src/main/java/io/split/engine/sse/client/SSEClient.java b/client/src/main/java/io/split/engine/sse/client/SSEClient.java index f6f6e13bc..184475671 100644 --- a/client/src/main/java/io/split/engine/sse/client/SSEClient.java +++ b/client/src/main/java/io/split/engine/sse/client/SSEClient.java @@ -15,8 +15,6 @@ import java.io.InputStreamReader; import java.net.SocketException; import java.net.URI; -import java.util.HashMap; -import java.util.Map; import java.util.concurrent.CountDownLatch; import java.util.concurrent.ExecutorService; import java.util.concurrent.Executors; From 84fd4cab14e73a7e4a9a41034160c1771821339a Mon Sep 17 00:00:00 2001 From: Lucas Echeverz Date: Thu, 6 May 2021 14:38:57 -0300 Subject: [PATCH 40/81] Attached every service --- .../java/io/split/client/EventClientImpl.java | 24 +-- .../client/HttpSegmentChangeFetcher.java | 26 +-- .../split/client/HttpSplitChangeFetcher.java | 25 ++- .../split/client/LocalhostSplitFactory.java | 4 +- .../io/split/client/SplitClientConfig.java | 2 +- .../java/io/split/client/SplitClientImpl.java | 14 +- .../io/split/client/SplitFactoryImpl.java | 21 +-- .../io/split/client/SplitManagerImpl.java | 20 ++- .../split/client/YamlLocalhostSplitFile.java | 2 +- .../AsynchronousImpressionListener.java | 9 +- .../impressions/HttpImpressionsSender.java | 26 ++- .../impressions/ImpressionsManagerImpl.java | 36 +++-- .../split/engine/common/PushManagerImp.java | 6 +- .../split/engine/common/SyncManagerImp.java | 6 +- .../engine/experiments/SplitFetcherImp.java | 4 +- .../engine/segments/SegmentFetcherImp.java | 4 +- .../SegmentSynchronizationTaskImp.java | 8 +- .../io/split/engine/sse/AuthApiClientImp.java | 16 +- .../storage/InMemoryTelemetryStorage.java | 16 +- .../storage/NoopTelemetryStorage.java | 149 ++++++++++++++++++ .../synchronizer/TelemetrySyncTask.java | 2 +- .../telemetry/utils/AtomicLongArray.java | 11 +- .../io/split/client/EventsClientImplTest.java | 16 +- .../client/HttpSegmentChangeFetcherTest.java | 15 +- .../client/HttpSplitChangeFetcherTest.java | 14 +- .../io/split/client/SplitClientImplTest.java | 71 +++++---- .../client/SplitClientIntegrationTest.java | 25 +-- .../io/split/client/SplitManagerImplTest.java | 21 +-- .../HttpImpressionsSenderTest.java | 20 ++- .../ImpressionsManagerImplTest.java | 15 +- .../engine/experiments/SplitFetcherTest.java | 19 ++- .../engine/experiments/SplitParserTest.java | 23 +-- .../segments/SegmentFetcherImpTest.java | 13 +- .../SegmentSynchronizationTaskImpTest.java | 5 +- .../split/engine/sse/AuthApiClientTest.java | 16 +- .../synchronizer/TelemetrySyncTaskTest.java | 2 - 36 files changed, 497 insertions(+), 209 deletions(-) create mode 100644 client/src/main/java/io/split/telemetry/storage/NoopTelemetryStorage.java diff --git a/client/src/main/java/io/split/client/EventClientImpl.java b/client/src/main/java/io/split/client/EventClientImpl.java index 6e6e3f842..50c95b1c6 100644 --- a/client/src/main/java/io/split/client/EventClientImpl.java +++ b/client/src/main/java/io/split/client/EventClientImpl.java @@ -4,6 +4,8 @@ import io.split.client.dtos.Event; import io.split.client.utils.GenericClientUtil; import io.split.client.utils.Utils; +import io.split.telemetry.domain.enums.EventsDataRecordsEnum; +import io.split.telemetry.domain.enums.HTTPLatenciesEnum; import io.split.telemetry.domain.enums.LastSynchronizationRecordsEnum; import io.split.telemetry.storage.TelemetryEvaluationProducer; import io.split.telemetry.storage.TelemetryRuntimeProducer; @@ -49,7 +51,7 @@ public class EventClientImpl implements EventClient { private final CloseableHttpClient _httpclient; private final URI _target; private final int _waitBeforeShutdown; - private final TelemetryRuntimeProducer telemetryRuntimeProducer; + private final TelemetryRuntimeProducer _telemetryRuntimeProducer; ThreadFactory eventClientThreadFactory(final String name) { return new ThreadFactory() { @@ -67,18 +69,18 @@ public void run() { } - public static EventClientImpl create(CloseableHttpClient httpclient, URI eventsRootTarget, int maxQueueSize, long flushIntervalMillis, int waitBeforeShutdown, TelemetryEvaluationProducer telemetryEvaluationProducer) throws URISyntaxException { + public static EventClientImpl create(CloseableHttpClient httpclient, URI eventsRootTarget, int maxQueueSize, long flushIntervalMillis, int waitBeforeShutdown, TelemetryRuntimeProducer telemetryRuntimeProducer) throws URISyntaxException { return new EventClientImpl(new LinkedBlockingQueue(), httpclient, Utils.appendPath(eventsRootTarget, "api/events/bulk"), maxQueueSize, flushIntervalMillis, waitBeforeShutdown, - telemetryEvaluationProducer); + telemetryRuntimeProducer); } EventClientImpl(BlockingQueue eventQueue, CloseableHttpClient httpclient, URI target, int maxQueueSize, - long flushIntervalMillis, int waitBeforeShutdown, TelemetryEvaluationProducer telemetryEvaluationProducer) throws URISyntaxException { + long flushIntervalMillis, int waitBeforeShutdown, TelemetryRuntimeProducer telemetryRuntimeProducer) throws URISyntaxException { _httpclient = httpclient; @@ -87,9 +89,9 @@ public static EventClientImpl create(CloseableHttpClient httpclient, URI eventsR _eventQueue = eventQueue; _waitBeforeShutdown = waitBeforeShutdown; - _maxQueueSize = maxQueueSize; + _maxQueueSize = 1; _flushIntervalMillis = flushIntervalMillis; - telemetryRuntimeProducer = checkNotNull(telemetryEvaluationProducer); + _telemetryRuntimeProducer = checkNotNull(telemetryRuntimeProducer); _senderExecutor = new ThreadPoolExecutor( 1, @@ -129,9 +131,12 @@ public boolean track(Event event, int eventSize) { if (event == null) { return false; } - _eventQueue.put(new WrappedEvent(event, eventSize)); + WrappedEvent we = new WrappedEvent(event, eventSize); + _eventQueue.put(we); + _telemetryRuntimeProducer.recordEventStats(EventsDataRecordsEnum.EVENTS_QUEUED, 1); - } catch (InterruptedException e) { + } catch (ClassCastException | NullPointerException | InterruptedException e) { + _telemetryRuntimeProducer.recordEventStats(EventsDataRecordsEnum.EVENTS_DROPPED, 1); _log.warn("Interruption when adding event withed while adding message %s.", event); return false; } @@ -191,7 +196,8 @@ public void run() { events = new ArrayList<>(); accumulated = 0; long endTime = System.currentTimeMillis(); - telemetryRuntimeProducer.recordSuccessfulSync(LastSynchronizationRecordsEnum.EVENTS, endTime-initTime); + _telemetryRuntimeProducer.recordSyncLatency(HTTPLatenciesEnum.EVENTS, endTime-initTime); + _telemetryRuntimeProducer.recordSuccessfulSync(LastSynchronizationRecordsEnum.EVENTS, endTime); } } } catch (InterruptedException e) { diff --git a/client/src/main/java/io/split/client/HttpSegmentChangeFetcher.java b/client/src/main/java/io/split/client/HttpSegmentChangeFetcher.java index 7d7d735f6..58e8e6eb0 100644 --- a/client/src/main/java/io/split/client/HttpSegmentChangeFetcher.java +++ b/client/src/main/java/io/split/client/HttpSegmentChangeFetcher.java @@ -6,6 +6,10 @@ import io.split.client.utils.Utils; import io.split.engine.metrics.Metrics; import io.split.engine.segments.SegmentChangeFetcher; +import io.split.telemetry.domain.enums.HTTPLatenciesEnum; +import io.split.telemetry.domain.enums.LastSynchronizationRecordsEnum; +import io.split.telemetry.domain.enums.ResourceEnum; +import io.split.telemetry.storage.TelemetryRuntimeProducer; import org.apache.hc.client5.http.classic.methods.HttpGet; import org.apache.hc.client5.http.impl.classic.CloseableHttpClient; import org.apache.hc.client5.http.impl.classic.CloseableHttpResponse; @@ -33,21 +37,17 @@ public final class HttpSegmentChangeFetcher implements SegmentChangeFetcher { private final CloseableHttpClient _client; private final URI _target; - private final Metrics _metrics; + private final TelemetryRuntimeProducer _telemetryRuntimeProducer; - public static HttpSegmentChangeFetcher create(CloseableHttpClient client, URI root) throws URISyntaxException { - return create(client, root, new Metrics.NoopMetrics()); + public static HttpSegmentChangeFetcher create(CloseableHttpClient client, URI root, TelemetryRuntimeProducer telemetryRuntimeProducer) throws URISyntaxException { + return new HttpSegmentChangeFetcher(client, Utils.appendPath(root, "api/segmentChanges"), telemetryRuntimeProducer); } - public static HttpSegmentChangeFetcher create(CloseableHttpClient client, URI root, Metrics metrics) throws URISyntaxException { - return new HttpSegmentChangeFetcher(client, Utils.appendPath(root, "api/segmentChanges"), metrics); - } - - private HttpSegmentChangeFetcher(CloseableHttpClient client, URI uri, Metrics metrics) { + private HttpSegmentChangeFetcher(CloseableHttpClient client, URI uri, TelemetryRuntimeProducer telemetryRuntimeProducer) { _client = client; _target = uri; - _metrics = metrics; checkNotNull(_target); + _telemetryRuntimeProducer = checkNotNull(telemetryRuntimeProducer); } @Override @@ -68,16 +68,18 @@ public SegmentChange fetch(String segmentName, long since, boolean addCacheHeade int statusCode = response.getCode(); if (statusCode < 200 || statusCode >= 300) { + _telemetryRuntimeProducer.recordSyncError(ResourceEnum.SEGMENT_SYNC, statusCode); _log.error("Response status was: " + statusCode); if (statusCode == 403) { _log.error("factory instantiation: you passed a browser type api_key, " + "please grab an api key from the Split console that is of type sdk"); } - _metrics.count(PREFIX + ".status." + statusCode, 1); throw new IllegalStateException("Could not retrieve segment changes for " + segmentName + "; http return code " + statusCode); } - + long endTime = System.currentTimeMillis(); + _telemetryRuntimeProducer.recordSyncLatency(HTTPLatenciesEnum.SEGMENTS, endTime-start); + _telemetryRuntimeProducer.recordSuccessfulSync(LastSynchronizationRecordsEnum.SEGMENTS, endTime); String json = EntityUtils.toString(response.getEntity(), StandardCharsets.UTF_8); if (_log.isDebugEnabled()) { @@ -86,11 +88,9 @@ public SegmentChange fetch(String segmentName, long since, boolean addCacheHeade return Json.fromJson(json, SegmentChange.class); } catch (Throwable t) { - _metrics.count(PREFIX + ".exception", 1); throw new IllegalStateException("Problem fetching segmentChanges: " + t.getMessage(), t); } finally { Utils.forceClose(response); - _metrics.time(PREFIX + ".time", System.currentTimeMillis() - start); } diff --git a/client/src/main/java/io/split/client/HttpSplitChangeFetcher.java b/client/src/main/java/io/split/client/HttpSplitChangeFetcher.java index 3c5f9b8fc..0f13656b4 100644 --- a/client/src/main/java/io/split/client/HttpSplitChangeFetcher.java +++ b/client/src/main/java/io/split/client/HttpSplitChangeFetcher.java @@ -6,6 +6,10 @@ import io.split.client.utils.Utils; import io.split.engine.experiments.SplitChangeFetcher; import io.split.engine.metrics.Metrics; +import io.split.telemetry.domain.enums.HTTPLatenciesEnum; +import io.split.telemetry.domain.enums.LastSynchronizationRecordsEnum; +import io.split.telemetry.domain.enums.ResourceEnum; +import io.split.telemetry.storage.TelemetryRuntimeProducer; import org.apache.hc.client5.http.classic.methods.HttpGet; import org.apache.hc.client5.http.impl.classic.CloseableHttpClient; import org.apache.hc.client5.http.impl.classic.CloseableHttpResponse; @@ -33,26 +37,21 @@ public final class HttpSplitChangeFetcher implements SplitChangeFetcher { private final CloseableHttpClient _client; private final URI _target; - private final Metrics _metrics; + private final TelemetryRuntimeProducer _telemetryRuntimeProducer; - public static HttpSplitChangeFetcher create(CloseableHttpClient client, URI root) throws URISyntaxException { - return create(client, root, new Metrics.NoopMetrics()); + public static HttpSplitChangeFetcher create(CloseableHttpClient client, URI root, TelemetryRuntimeProducer telemetryRuntimeProducer) throws URISyntaxException { + return new HttpSplitChangeFetcher(client, Utils.appendPath(root, "api/splitChanges"), telemetryRuntimeProducer); } - public static HttpSplitChangeFetcher create(CloseableHttpClient client, URI root, Metrics metrics) throws URISyntaxException { - return new HttpSplitChangeFetcher(client, Utils.appendPath(root, "api/splitChanges"), metrics); - } - - private HttpSplitChangeFetcher(CloseableHttpClient client, URI uri, Metrics metrics) { + private HttpSplitChangeFetcher(CloseableHttpClient client, URI uri, TelemetryRuntimeProducer telemetryRuntimeProducer) { _client = client; _target = uri; - _metrics = metrics; checkNotNull(_target); + _telemetryRuntimeProducer = checkNotNull(telemetryRuntimeProducer); } @Override public SplitChange fetch(long since, boolean addCacheHeader) { - long start = System.currentTimeMillis(); CloseableHttpResponse response = null; @@ -69,9 +68,11 @@ public SplitChange fetch(long since, boolean addCacheHeader) { int statusCode = response.getCode(); if (statusCode < 200 || statusCode >= 300) { - _metrics.count(PREFIX + ".status." + statusCode, 1); + _telemetryRuntimeProducer.recordSyncError(ResourceEnum.SPLIT_SYNC, statusCode); throw new IllegalStateException("Could not retrieve splitChanges; http return code " + statusCode); } + long endtime = System.currentTimeMillis(); + _telemetryRuntimeProducer.recordSyncLatency(HTTPLatenciesEnum.SPLITS, endtime-start); String json = EntityUtils.toString(response.getEntity(), StandardCharsets.UTF_8); @@ -81,11 +82,9 @@ public SplitChange fetch(long since, boolean addCacheHeader) { return Json.fromJson(json, SplitChange.class); } catch (Throwable t) { - _metrics.count(PREFIX + ".exception", 1); throw new IllegalStateException("Problem fetching splitChanges: " + t.getMessage(), t); } finally { Utils.forceClose(response); - _metrics.time(PREFIX + ".time", System.currentTimeMillis() - start); } } diff --git a/client/src/main/java/io/split/client/LocalhostSplitFactory.java b/client/src/main/java/io/split/client/LocalhostSplitFactory.java index 7a7cbf888..c716ae3ce 100644 --- a/client/src/main/java/io/split/client/LocalhostSplitFactory.java +++ b/client/src/main/java/io/split/client/LocalhostSplitFactory.java @@ -6,6 +6,8 @@ import io.split.engine.SDKReadinessGates; import io.split.engine.evaluator.EvaluatorImp; import io.split.engine.metrics.Metrics; +import io.split.telemetry.storage.InMemoryTelemetryStorage; +import io.split.telemetry.storage.NoopTelemetryStorage; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -59,7 +61,7 @@ public LocalhostSplitFactory(String directory, String file) throws IOException { _cacheUpdaterService.updateCache(splitAndKeyToTreatment); _client = new SplitClientImpl(this, splitCache, new ImpressionsManager.NoOpImpressionsManager(), new NoopEventClient(), - SplitClientConfig.builder().setBlockUntilReadyTimeout(1).build(), sdkReadinessGates, new EvaluatorImp(splitCache)); + SplitClientConfig.builder().setBlockUntilReadyTimeout(1).build(), sdkReadinessGates, new EvaluatorImp(splitCache), new NoopTelemetryStorage(), new NoopTelemetryStorage()); _manager = LocalhostSplitManager.of(splitAndKeyToTreatment); _splitFile.registerWatcher(); diff --git a/client/src/main/java/io/split/client/SplitClientConfig.java b/client/src/main/java/io/split/client/SplitClientConfig.java index a98efdc1f..7761af6fa 100644 --- a/client/src/main/java/io/split/client/SplitClientConfig.java +++ b/client/src/main/java/io/split/client/SplitClientConfig.java @@ -21,7 +21,7 @@ public class SplitClientConfig { public static final String EVENTS_ENDPOINT = "https://events.split.io"; public static final String AUTH_ENDPOINT = "https://auth.split.io/api/auth"; public static final String STREAMING_ENDPOINT = "https://streaming.split.io/sse"; - public static final String TELEMETRY_ENDPOINT = "https://telemetry.split.io/api/v1"; + public static final String TELEMETRY_ENDPOINT = "https://telemetry.split-stage.io/api/v1/"; private final String _endpoint; private final String _eventsEndpoint; diff --git a/client/src/main/java/io/split/client/SplitClientImpl.java b/client/src/main/java/io/split/client/SplitClientImpl.java index efeec3c3f..83add8aa4 100644 --- a/client/src/main/java/io/split/client/SplitClientImpl.java +++ b/client/src/main/java/io/split/client/SplitClientImpl.java @@ -15,7 +15,9 @@ import io.split.inputValidation.KeyValidator; import io.split.inputValidation.SplitNameValidator; import io.split.inputValidation.TrafficTypeValidator; +import io.split.telemetry.domain.enums.LastSynchronizationRecordsEnum; import io.split.telemetry.domain.enums.MethodEnum; +import io.split.telemetry.storage.TelemetryConfigProducer; import io.split.telemetry.storage.TelemetryEvaluationProducer; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -49,6 +51,7 @@ public final class SplitClientImpl implements SplitClient { private final SDKReadinessGates _gates; private final Evaluator _evaluator; private final TelemetryEvaluationProducer _telemetryEvaluationProducer; + private final TelemetryConfigProducer _telemetryConfigProducer; public SplitClientImpl(SplitFactory container, SplitCache splitCache, @@ -57,7 +60,8 @@ public SplitClientImpl(SplitFactory container, SplitClientConfig config, SDKReadinessGates gates, Evaluator evaluator, - TelemetryEvaluationProducer telemetryEvaluationProducer) { + TelemetryEvaluationProducer telemetryEvaluationProducer, + TelemetryConfigProducer telemetryConfigProducer) { _container = container; _splitCache = checkNotNull(splitCache); _impressionManager = checkNotNull(impressionManager); @@ -66,6 +70,7 @@ public SplitClientImpl(SplitFactory container, _gates = checkNotNull(gates); _evaluator = checkNotNull(evaluator); _telemetryEvaluationProducer = checkNotNull(telemetryEvaluationProducer); + _telemetryConfigProducer = checkNotNull(telemetryConfigProducer); } @Override @@ -145,6 +150,7 @@ public void destroy() { } private boolean track(Event event) { + long initTime = System.currentTimeMillis(); if (_container.isDestroyed()) { _log.error("Client has already been destroyed - no calls possible"); return false; @@ -174,6 +180,7 @@ private boolean track(Event event) { } event.properties = propertiesResult.getValue(); + _telemetryEvaluationProducer.recordLatency(MethodEnum.TRACK, System.currentTimeMillis() - initTime); return _eventClient.track(event, propertiesResult.getEventSize()); } @@ -181,8 +188,11 @@ private boolean track(Event event) { private SplitResult getTreatmentWithConfigInternal(String method, String matchingKey, String bucketingKey, String split, Map attributes, MethodEnum methodEnum) { long initTime = System.currentTimeMillis(); try { + if(!_gates.isSDKReadyNow()){ + _log.warn(method + ": the SDK is not ready, results may be incorrect. Make sure to wait for SDK readiness before using this method"); + _telemetryConfigProducer.recordNonReadyUsage(); + } if (_container.isDestroyed()) { - _telemetryEvaluationProducer.recordException(methodEnum); _log.error("Client has already been destroyed - no calls possible"); return SPLIT_RESULT_CONTROL; } diff --git a/client/src/main/java/io/split/client/SplitFactoryImpl.java b/client/src/main/java/io/split/client/SplitFactoryImpl.java index a9546dd18..a54ed46b8 100644 --- a/client/src/main/java/io/split/client/SplitFactoryImpl.java +++ b/client/src/main/java/io/split/client/SplitFactoryImpl.java @@ -94,7 +94,7 @@ public class SplitFactoryImpl implements SplitFactory { private final TelemetrySyncTask _telemetrySyncTask; private final long _startTime; - public SplitFactoryImpl(String apiToken, SplitClientConfig config) throws Exception { + public SplitFactoryImpl(String apiToken, SplitClientConfig config) throws URISyntaxException { _startTime = System.currentTimeMillis(); _apiToken = apiToken; _apiKeyCounter = ApiKeyCounter.getApiKeyCounterInstance(); @@ -141,20 +141,20 @@ public SplitFactoryImpl(String apiToken, SplitClientConfig config) throws Except _impressionsManager = buildImpressionsManager(config); // EventClient - _eventClient = EventClientImpl.create(_httpclient, _eventsRootTarget, config.eventsQueueSize(), config.eventFlushIntervalInMillis(), config.waitBeforeShutdown()); + _eventClient = EventClientImpl.create(_httpclient, _eventsRootTarget, config.eventsQueueSize(), config.eventFlushIntervalInMillis(), config.waitBeforeShutdown(), _telemetryStorage); // SyncManager - _syncManager = SyncManagerImp.build(config.streamingEnabled(), _splitSynchronizationTask, _splitFetcher, _segmentSynchronizationTaskImp, _splitCache, config.authServiceURL(), _httpclient, config.streamingServiceURL(), config.authRetryBackoffBase(), buildSSEdHttpClient(config), _segmentCache, config.streamingRetryDelay(), _gates); + _syncManager = SyncManagerImp.build(config.streamingEnabled(), _splitSynchronizationTask, _splitFetcher, _segmentSynchronizationTaskImp, _splitCache, config.authServiceURL(), _httpclient, config.streamingServiceURL(), config.authRetryBackoffBase(), buildSSEdHttpClient(config), _segmentCache, config.streamingRetryDelay(), _gates, _telemetryStorage); _syncManager.start(); // Evaluator _evaluator = new EvaluatorImp(_splitCache); // SplitClient - _client = new SplitClientImpl(this, _splitCache, _impressionsManager, _eventClient, config, _gates, _evaluator); + _client = new SplitClientImpl(this, _splitCache, _impressionsManager, _eventClient, config, _gates, _evaluator, _telemetryStorage, _telemetryStorage); // SplitManager - _manager = new SplitManagerImpl(_splitCache, config, _gates); + _manager = new SplitManagerImpl(_splitCache, config, _gates, _telemetryStorage); // DestroyOnShutDown if (config.destroyOnShutDown()) { @@ -298,20 +298,21 @@ private static int findPollingPeriod(Random rand, int max) { } private SegmentSynchronizationTaskImp buildSegments(SplitClientConfig config) throws URISyntaxException { - SegmentChangeFetcher segmentChangeFetcher = HttpSegmentChangeFetcher.create(_httpclient, _rootTarget); + SegmentChangeFetcher segmentChangeFetcher = HttpSegmentChangeFetcher.create(_httpclient, _rootTarget, _telemetryStorage); return new SegmentSynchronizationTaskImp(segmentChangeFetcher, findPollingPeriod(RANDOM, config.segmentsRefreshRate()), config.numThreadsForSegmentFetch(), _gates, - _segmentCache); + _segmentCache, + _telemetryStorage); } private SplitFetcher buildSplitFetcher() throws URISyntaxException { - SplitChangeFetcher splitChangeFetcher = HttpSplitChangeFetcher.create(_httpclient, _rootTarget); + SplitChangeFetcher splitChangeFetcher = HttpSplitChangeFetcher.create(_httpclient, _rootTarget, _telemetryStorage); SplitParser splitParser = new SplitParser(_segmentSynchronizationTaskImp, _segmentCache); - return new SplitFetcherImp(splitChangeFetcher, splitParser, _gates, _splitCache); + return new SplitFetcherImp(splitChangeFetcher, splitParser, _gates, _splitCache, _telemetryStorage); } private ImpressionsManagerImpl buildImpressionsManager(SplitClientConfig config) throws URISyntaxException { @@ -326,7 +327,7 @@ private ImpressionsManagerImpl buildImpressionsManager(SplitClientConfig config) .collect(Collectors.toCollection(() -> impressionListeners)); } - return ImpressionsManagerImpl.instance(_httpclient, config, impressionListeners); + return ImpressionsManagerImpl.instance(_httpclient, config, impressionListeners, _telemetryStorage); } } diff --git a/client/src/main/java/io/split/client/SplitManagerImpl.java b/client/src/main/java/io/split/client/SplitManagerImpl.java index 5304b5911..3f73d91dd 100644 --- a/client/src/main/java/io/split/client/SplitManagerImpl.java +++ b/client/src/main/java/io/split/client/SplitManagerImpl.java @@ -6,6 +6,8 @@ import io.split.cache.SplitCache; import io.split.engine.experiments.ParsedSplit; import io.split.inputValidation.SplitNameValidator; +import io.split.telemetry.domain.enums.MethodEnum; +import io.split.telemetry.storage.TelemetryConfigProducer; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -26,18 +28,25 @@ public class SplitManagerImpl implements SplitManager { private final SplitCache _splitCache; private final SplitClientConfig _config; private final SDKReadinessGates _gates; + private final TelemetryConfigProducer _telemetryConfigProducer; public SplitManagerImpl(SplitCache splitCache, SplitClientConfig config, - SDKReadinessGates gates) { + SDKReadinessGates gates, + TelemetryConfigProducer telemetryConfigProducer) { _config = Preconditions.checkNotNull(config); _splitCache = Preconditions.checkNotNull(splitCache); _gates = Preconditions.checkNotNull(gates); + _telemetryConfigProducer = telemetryConfigProducer; } @Override public List splits() { + if (_gates.isSDKReadyNow()) { { + _log.warn("splits: the SDK is not ready, results may be incorrect. Make sure to wait for SDK readiness before using this method"); + _telemetryConfigProducer.recordNonReadyUsage(); + }} List result = new ArrayList<>(); Collection parsedSplits = _splitCache.getAll(); for (ParsedSplit split : parsedSplits) { @@ -49,6 +58,10 @@ public List splits() { @Override public SplitView split(String featureName) { + if (_gates.isSDKReadyNow()) { { + _log.warn("split: the SDK is not ready, results may be incorrect. Make sure to wait for SDK readiness before using this method"); + _telemetryConfigProducer.recordNonReadyUsage(); + }} Optional result = SplitNameValidator.isValid(featureName, "split"); if (!result.isPresent()) { return null; @@ -69,6 +82,10 @@ public SplitView split(String featureName) { @Override public List splitNames() { + if (_gates.isSDKReadyNow()) { { + _log.warn("splitNames: the SDK is not ready, results may be incorrect. Make sure to wait for SDK readiness before using this method"); + _telemetryConfigProducer.recordNonReadyUsage(); + }} List result = new ArrayList<>(); Collection parsedSplits = _splitCache.getAll(); for (ParsedSplit split : parsedSplits) { @@ -84,6 +101,7 @@ public void blockUntilReady() throws TimeoutException, InterruptedException { throw new IllegalArgumentException("setBlockUntilReadyTimeout must be positive but in config was: " + _config.blockUntilReady()); } if (!_gates.isSDKReady(_config.blockUntilReady())) { + _telemetryConfigProducer.recordBURTimeout(); throw new TimeoutException("SDK was not ready in " + _config.blockUntilReady()+ " milliseconds"); } } diff --git a/client/src/main/java/io/split/client/YamlLocalhostSplitFile.java b/client/src/main/java/io/split/client/YamlLocalhostSplitFile.java index 926bab165..b9ece01c5 100644 --- a/client/src/main/java/io/split/client/YamlLocalhostSplitFile.java +++ b/client/src/main/java/io/split/client/YamlLocalhostSplitFile.java @@ -13,7 +13,7 @@ public class YamlLocalhostSplitFile extends AbstractLocalhostSplitFile { - private static final Logger _log = LoggerFactory.getLogger(LegacyLocalhostSplitFile.class); + private static final Logger _log = LoggerFactory.getLogger(YamlLocalhostSplitFile.class); public YamlLocalhostSplitFile(LocalhostSplitFactory localhostSplitFactory, String directory, String filenameYaml) throws IOException { super(localhostSplitFactory, directory, filenameYaml); diff --git a/client/src/main/java/io/split/client/impressions/AsynchronousImpressionListener.java b/client/src/main/java/io/split/client/impressions/AsynchronousImpressionListener.java index 7c3a8716a..ee9e97cb8 100644 --- a/client/src/main/java/io/split/client/impressions/AsynchronousImpressionListener.java +++ b/client/src/main/java/io/split/client/impressions/AsynchronousImpressionListener.java @@ -27,9 +27,8 @@ public class AsynchronousImpressionListener implements ImpressionListener { private final ImpressionListener _delegate; private final ExecutorService _executor; - private final TelemetryRuntimeProducer _telemetryRuntimeProducer; - public static AsynchronousImpressionListener build(ImpressionListener delegate, int capacity, TelemetryRuntimeProducer telemetryRuntimeProducer) { + public static AsynchronousImpressionListener build(ImpressionListener delegate, int capacity) { ThreadFactory threadFactory = new ThreadFactoryBuilder() .setDaemon(true) .setNameFormat("impression-listener-wrapper-%d") @@ -37,13 +36,12 @@ public static AsynchronousImpressionListener build(ImpressionListener delegate, ExecutorService executor = new ThreadPoolExecutor(2, 2, 0L, TimeUnit.MILLISECONDS, new ArrayBlockingQueue(capacity), threadFactory); - return new AsynchronousImpressionListener(delegate, executor, telemetryRuntimeProducer); + return new AsynchronousImpressionListener(delegate, executor); } - public AsynchronousImpressionListener(ImpressionListener delegate, ExecutorService executor, TelemetryRuntimeProducer telemetryRuntimeProducer) { + public AsynchronousImpressionListener(ImpressionListener delegate, ExecutorService executor) { _delegate = delegate; _executor = executor; - _telemetryRuntimeProducer = checkNotNull(telemetryRuntimeProducer); } @@ -58,7 +56,6 @@ public void run() { } }); long endTime = System.currentTimeMillis(); - _telemetryRuntimeProducer.recordSuccessfulSync(LastSynchronizationRecordsEnum.IMPRESSIONS, endTime - initTime); } catch (Exception e) { _log.warn("Unable to send impression to impression listener", e); diff --git a/client/src/main/java/io/split/client/impressions/HttpImpressionsSender.java b/client/src/main/java/io/split/client/impressions/HttpImpressionsSender.java index 336e7f2f5..5e0c5efb0 100644 --- a/client/src/main/java/io/split/client/impressions/HttpImpressionsSender.java +++ b/client/src/main/java/io/split/client/impressions/HttpImpressionsSender.java @@ -5,6 +5,10 @@ import io.split.client.dtos.TestImpressions; import io.split.client.utils.Utils; +import io.split.telemetry.domain.enums.HTTPLatenciesEnum; +import io.split.telemetry.domain.enums.LastSynchronizationRecordsEnum; +import io.split.telemetry.domain.enums.ResourceEnum; +import io.split.telemetry.storage.TelemetryRuntimeProducer; import org.apache.hc.client5.http.classic.methods.HttpPost; import org.apache.hc.client5.http.impl.classic.CloseableHttpClient; import org.apache.hc.client5.http.impl.classic.CloseableHttpResponse; @@ -18,6 +22,8 @@ import java.util.HashMap; import java.util.List; +import static com.google.common.base.Preconditions.checkNotNull; + /** * Created by patricioe on 6/20/16. */ @@ -33,26 +39,29 @@ public class HttpImpressionsSender implements ImpressionsSender { private final URI _impressionBulkTarget; private final URI _impressionCountTarget; private final ImpressionsManager.Mode _mode; + private final TelemetryRuntimeProducer _telemetryRuntimeProducer; - public static HttpImpressionsSender create(CloseableHttpClient client, URI eventsRootEndpoint, ImpressionsManager.Mode mode) throws URISyntaxException { + public static HttpImpressionsSender create(CloseableHttpClient client, URI eventsRootEndpoint, ImpressionsManager.Mode mode, TelemetryRuntimeProducer telemetryRuntimeProducer) throws URISyntaxException { return new HttpImpressionsSender(client, Utils.appendPath(eventsRootEndpoint, BULK_ENDPOINT_PATH), Utils.appendPath(eventsRootEndpoint, COUNT_ENDPOINT_PATH), - mode); + mode, + telemetryRuntimeProducer); } - private HttpImpressionsSender(CloseableHttpClient client, URI impressionBulkTarget, URI impressionCountTarget, ImpressionsManager.Mode mode) { + private HttpImpressionsSender(CloseableHttpClient client, URI impressionBulkTarget, URI impressionCountTarget, ImpressionsManager.Mode mode, TelemetryRuntimeProducer telemetryRuntimeProducer) { _client = client; _mode = mode; _impressionBulkTarget = impressionBulkTarget; _impressionCountTarget = impressionCountTarget; + _telemetryRuntimeProducer = checkNotNull(telemetryRuntimeProducer); } @Override public void postImpressionsBulk(List impressions) { CloseableHttpResponse response = null; - + long initTime = System.currentTimeMillis(); try { HttpEntity entity = Utils.toJsonEntity(impressions); @@ -65,8 +74,12 @@ public void postImpressionsBulk(List impressions) { int status = response.getCode(); if (status < 200 || status >= 300) { + _telemetryRuntimeProducer.recordSyncError(ResourceEnum.IMPRESSION_SYNC, status); _logger.warn("Response status was: " + status); } + long endTime = System.currentTimeMillis(); + _telemetryRuntimeProducer.recordSyncLatency(HTTPLatenciesEnum.IMPRESSIONS, endTime - initTime); + _telemetryRuntimeProducer.recordSuccessfulSync(LastSynchronizationRecordsEnum.IMPRESSIONS, endTime); } catch (Throwable t) { _logger.warn("Exception when posting impressions" + impressions, t); @@ -78,6 +91,7 @@ public void postImpressionsBulk(List impressions) { @Override public void postCounters(HashMap raw) { + long initTime = System.currentTimeMillis(); if (_mode.equals(ImpressionsManager.Mode.DEBUG)) { _logger.warn("Attempted to submit counters in impressions debugging mode. Ignoring"); return; @@ -88,8 +102,12 @@ public void postCounters(HashMap raw) { try (CloseableHttpResponse response = _client.execute(request)) { int status = response.getCode(); if (status < 200 || status >= 300) { + _telemetryRuntimeProducer.recordSyncError(ResourceEnum.IMPRESSION_COUNT_SYNC, status); _logger.warn("Response status was: " + status); } + long endTime = System.currentTimeMillis(); + _telemetryRuntimeProducer.recordSyncLatency(HTTPLatenciesEnum.IMPRESSIONS_COUNT, endTime - initTime); + _telemetryRuntimeProducer.recordSuccessfulSync(LastSynchronizationRecordsEnum.IMPRESSIONS_COUNT, endTime); } catch (IOException exc) { _logger.warn("Exception when posting impression counters: ", exc); } diff --git a/client/src/main/java/io/split/client/impressions/ImpressionsManagerImpl.java b/client/src/main/java/io/split/client/impressions/ImpressionsManagerImpl.java index ef20caff6..f5a5343a7 100644 --- a/client/src/main/java/io/split/client/impressions/ImpressionsManagerImpl.java +++ b/client/src/main/java/io/split/client/impressions/ImpressionsManagerImpl.java @@ -5,6 +5,8 @@ import io.split.client.SplitClientConfig; import io.split.client.dtos.KeyImpression; import io.split.client.dtos.TestImpressions; +import io.split.telemetry.domain.enums.ImpressionsDataTypeEnum; +import io.split.telemetry.storage.TelemetryRuntimeProducer; import org.apache.hc.client5.http.impl.classic.CloseableHttpClient; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -41,36 +43,41 @@ public class ImpressionsManagerImpl implements ImpressionsManager, Closeable { private final ImpressionCounter _counter; private final ImpressionListener _listener; private final ImpressionsManager.Mode _mode; + private final TelemetryRuntimeProducer _telemetryRuntimeProducer; public static ImpressionsManagerImpl instance(CloseableHttpClient client, SplitClientConfig config, - List listeners) throws URISyntaxException { - return new ImpressionsManagerImpl(client, config, null, listeners); + List listeners, + TelemetryRuntimeProducer telemetryRuntimeProducer) throws URISyntaxException { + return new ImpressionsManagerImpl(client, config, null, listeners, telemetryRuntimeProducer); } public static ImpressionsManagerImpl instanceForTest(CloseableHttpClient client, SplitClientConfig config, ImpressionsSender impressionsSender, - List listeners) throws URISyntaxException { - return new ImpressionsManagerImpl(client, config, impressionsSender, listeners); + List listeners, + TelemetryRuntimeProducer telemetryRuntimeProducer) throws URISyntaxException { + return new ImpressionsManagerImpl(client, config, impressionsSender, listeners, telemetryRuntimeProducer); } private ImpressionsManagerImpl(CloseableHttpClient client, SplitClientConfig config, ImpressionsSender impressionsSender, - List listeners) throws URISyntaxException { + List listeners, + TelemetryRuntimeProducer telemetryRuntimeProducer) throws URISyntaxException { _config = checkNotNull(config); _mode = checkNotNull(config.impressionsMode()); + _telemetryRuntimeProducer = checkNotNull(telemetryRuntimeProducer); _storage = new InMemoryImpressionsStorage(config.impressionsQueueSize()); _impressionObserver = new ImpressionObserver(LAST_SEEN_CACHE_SIZE); _counter = new ImpressionCounter(); _impressionsSender = (null != impressionsSender) ? impressionsSender - : HttpImpressionsSender.create(client, URI.create(config.eventsEndpoint()), _mode); + : HttpImpressionsSender.create(client, URI.create(config.eventsEndpoint()), _mode, telemetryRuntimeProducer); _scheduler = buildExecutor(); - _scheduler.scheduleAtFixedRate(this::sendImpressions, BULK_INITIAL_DELAY_SECONDS,config.impressionsRefreshRate(), TimeUnit.SECONDS); + _scheduler.scheduleAtFixedRate(this::sendImpressions, BULK_INITIAL_DELAY_SECONDS, config.impressionsRefreshRate(), TimeUnit.SECONDS); if (Mode.OPTIMIZED.equals(_mode)) { _scheduler.scheduleAtFixedRate(this::sendImpressionCounters, COUNT_INITIAL_DELAY_SECONDS, COUNT_REFRESH_RATE_SECONDS, TimeUnit.SECONDS); } @@ -98,7 +105,14 @@ public void track(Impression impression) { } if (Mode.DEBUG.equals(_mode) || shouldQueueImpression(impression)) { - _storage.put(KeyImpression.fromImpression(impression)); + if (shouldQueueImpression(impression)) { + _telemetryRuntimeProducer.recordImpressionStats(ImpressionsDataTypeEnum.IMPRESSIONS_DEDUPED, 1); + } + if (_storage.put(KeyImpression.fromImpression(impression))) { + _telemetryRuntimeProducer.recordImpressionStats(ImpressionsDataTypeEnum.IMPRESSIONS_QUEUED, 1); + } + else { + _telemetryRuntimeProducer.recordImpressionStats(ImpressionsDataTypeEnum.IMPRESSIONS_DROPPED, 1); } } } @@ -117,7 +131,7 @@ public void close() { } @VisibleForTesting - /* package private */ void sendImpressions() { + /* package private */ void sendImpressions() { if (_storage.isFull()) { _log.warn("Split SDK impressions queue is full. Impressions may have been dropped. Consider increasing capacity."); } @@ -129,14 +143,14 @@ public void close() { } _impressionsSender.postImpressionsBulk(TestImpressions.fromKeyImpressions(impressions)); - if(_config.debugEnabled()) { + if (_config.debugEnabled()) { _log.info(String.format("Posting %d Split impressions took %d millis", impressions.size(), (System.currentTimeMillis() - start))); } } @VisibleForTesting - /* package private */ void sendImpressionCounters() { + /* package private */ void sendImpressionCounters() { if (!_counter.isEmpty()) { _impressionsSender.postCounters(_counter.popAll()); } diff --git a/client/src/main/java/io/split/engine/common/PushManagerImp.java b/client/src/main/java/io/split/engine/common/PushManagerImp.java index 1d770b6b1..601bb2ac5 100644 --- a/client/src/main/java/io/split/engine/common/PushManagerImp.java +++ b/client/src/main/java/io/split/engine/common/PushManagerImp.java @@ -16,6 +16,7 @@ import io.split.engine.sse.workers.SplitsWorkerImp; import io.split.engine.sse.workers.Worker; +import io.split.telemetry.storage.TelemetryRuntimeProducer; import org.apache.hc.client5.http.impl.classic.CloseableHttpClient; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -66,11 +67,12 @@ public static PushManagerImp build(Synchronizer synchronizer, String authUrl, CloseableHttpClient httpClient, LinkedBlockingQueue statusMessages, - CloseableHttpClient sseHttpClient) { + CloseableHttpClient sseHttpClient, + TelemetryRuntimeProducer telemetryRuntimeProducer) { SplitsWorker splitsWorker = new SplitsWorkerImp(synchronizer); Worker segmentWorker = new SegmentsWorkerImp(synchronizer); PushStatusTracker pushStatusTracker = new PushStatusTrackerImp(statusMessages); - return new PushManagerImp(new AuthApiClientImp(authUrl, httpClient), + return new PushManagerImp(new AuthApiClientImp(authUrl, httpClient, telemetryRuntimeProducer), EventSourceClientImp.build(streamingUrl, splitsWorker, segmentWorker, pushStatusTracker, sseHttpClient), splitsWorker, segmentWorker, diff --git a/client/src/main/java/io/split/engine/common/SyncManagerImp.java b/client/src/main/java/io/split/engine/common/SyncManagerImp.java index 5b830ca65..31e245816 100644 --- a/client/src/main/java/io/split/engine/common/SyncManagerImp.java +++ b/client/src/main/java/io/split/engine/common/SyncManagerImp.java @@ -8,6 +8,7 @@ import io.split.engine.experiments.SplitFetcher; import io.split.engine.experiments.SplitSynchronizationTask; import io.split.engine.segments.SegmentSynchronizationTaskImp; +import io.split.telemetry.storage.TelemetryRuntimeProducer; import org.apache.hc.client5.http.impl.classic.CloseableHttpClient; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -70,10 +71,11 @@ public static SyncManagerImp build(boolean streamingEnabledConfig, CloseableHttpClient sseHttpClient, SegmentCache segmentCache, int streamingRetryDelay, - SDKReadinessGates gates) { + SDKReadinessGates gates, + TelemetryRuntimeProducer telemetryRuntimeProducer) { LinkedBlockingQueue pushMessages = new LinkedBlockingQueue<>(); Synchronizer synchronizer = new SynchronizerImp(splitSynchronizationTask, splitFetcher, segmentSynchronizationTaskImp, splitCache, segmentCache, streamingRetryDelay, gates); - PushManager pushManager = PushManagerImp.build(synchronizer, streamingServiceUrl, authUrl, httpClient, pushMessages, sseHttpClient); + PushManager pushManager = PushManagerImp.build(synchronizer, streamingServiceUrl, authUrl, httpClient, pushMessages, sseHttpClient, telemetryRuntimeProducer); return new SyncManagerImp(streamingEnabledConfig, synchronizer, pushManager, pushMessages, authRetryBackOffBase, gates); } diff --git a/client/src/main/java/io/split/engine/experiments/SplitFetcherImp.java b/client/src/main/java/io/split/engine/experiments/SplitFetcherImp.java index 227224b4b..0f2e91e54 100644 --- a/client/src/main/java/io/split/engine/experiments/SplitFetcherImp.java +++ b/client/src/main/java/io/split/engine/experiments/SplitFetcherImp.java @@ -5,6 +5,7 @@ import io.split.client.dtos.Status; import io.split.engine.SDKReadinessGates; import io.split.cache.SplitCache; +import io.split.telemetry.domain.enums.HTTPLatenciesEnum; import io.split.telemetry.domain.enums.LastSynchronizationRecordsEnum; import io.split.telemetry.storage.TelemetryRuntimeProducer; import org.slf4j.Logger; @@ -141,8 +142,7 @@ private void runWithoutExceptionHandling(boolean addCacheHeader) throws Interrup } _splitCache.setChangeNumber(change.till); - long endtime = System.currentTimeMillis(); - _telemetryRuntimeProducer.recordSuccessfulSync(LastSynchronizationRecordsEnum.SPLITS, endtime-initTime); + _telemetryRuntimeProducer.recordSuccessfulSync(LastSynchronizationRecordsEnum.SPLITS, System.currentTimeMillis()); } } @Override diff --git a/client/src/main/java/io/split/engine/segments/SegmentFetcherImp.java b/client/src/main/java/io/split/engine/segments/SegmentFetcherImp.java index d5673a7d7..6492c5740 100644 --- a/client/src/main/java/io/split/engine/segments/SegmentFetcherImp.java +++ b/client/src/main/java/io/split/engine/segments/SegmentFetcherImp.java @@ -3,6 +3,7 @@ import io.split.cache.SegmentCache; import io.split.client.dtos.SegmentChange; import io.split.engine.SDKReadinessGates; +import io.split.telemetry.domain.enums.HTTPLatenciesEnum; import io.split.telemetry.domain.enums.LastSynchronizationRecordsEnum; import io.split.telemetry.storage.TelemetryRuntimeProducer; import org.slf4j.Logger; @@ -91,8 +92,7 @@ private void runWithoutExceptionHandling(boolean addCacheHeader) { } _segmentCache.setChangeNumber(_segmentName,change.till); - long endTime = System.currentTimeMillis(); - _telemetryRuntimeProducer.recordSuccessfulSync(LastSynchronizationRecordsEnum.SEGMENTS, endTime-initTime); + _telemetryRuntimeProducer.recordSuccessfulSync(LastSynchronizationRecordsEnum.SEGMENTS, System.currentTimeMillis()); } } diff --git a/client/src/main/java/io/split/engine/segments/SegmentSynchronizationTaskImp.java b/client/src/main/java/io/split/engine/segments/SegmentSynchronizationTaskImp.java index da0ca42aa..d3ed0dbfe 100644 --- a/client/src/main/java/io/split/engine/segments/SegmentSynchronizationTaskImp.java +++ b/client/src/main/java/io/split/engine/segments/SegmentSynchronizationTaskImp.java @@ -4,6 +4,7 @@ import com.google.common.util.concurrent.ThreadFactoryBuilder; import io.split.cache.SegmentCache; import io.split.engine.SDKReadinessGates; +import io.split.telemetry.storage.TelemetryRuntimeProducer; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -34,10 +35,12 @@ public class SegmentSynchronizationTaskImp implements SegmentSynchronizationTask private final SegmentCache _segmentCache; private final SDKReadinessGates _gates; private final ScheduledExecutorService _scheduledExecutorService; + private final TelemetryRuntimeProducer _telemetryRuntimeProducer; private ScheduledFuture _scheduledFuture; - public SegmentSynchronizationTaskImp(SegmentChangeFetcher segmentChangeFetcher, long refreshEveryNSeconds, int numThreads, SDKReadinessGates gates, SegmentCache segmentCache) { + public SegmentSynchronizationTaskImp(SegmentChangeFetcher segmentChangeFetcher, long refreshEveryNSeconds, int numThreads, SDKReadinessGates gates, SegmentCache segmentCache, + TelemetryRuntimeProducer telemetryRuntimeProducer) { _segmentChangeFetcher = checkNotNull(segmentChangeFetcher); checkArgument(refreshEveryNSeconds >= 0L); @@ -55,6 +58,7 @@ public SegmentSynchronizationTaskImp(SegmentChangeFetcher segmentChangeFetcher, _running = new AtomicBoolean(false); _segmentCache = checkNotNull(segmentCache); + _telemetryRuntimeProducer = checkNotNull(telemetryRuntimeProducer); } @Override @@ -84,7 +88,7 @@ public void initializeSegment(String segmentName) { _log.error("Unable to register segment " + segmentName); } - segment = new SegmentFetcherImp(segmentName, _segmentChangeFetcher, _gates, _segmentCache); + segment = new SegmentFetcherImp(segmentName, _segmentChangeFetcher, _gates, _segmentCache, _telemetryRuntimeProducer); if (_running.get()) { _scheduledExecutorService.submit(segment::fetchAll); diff --git a/client/src/main/java/io/split/engine/sse/AuthApiClientImp.java b/client/src/main/java/io/split/engine/sse/AuthApiClientImp.java index 2912c9ad1..da1ff89dc 100644 --- a/client/src/main/java/io/split/engine/sse/AuthApiClientImp.java +++ b/client/src/main/java/io/split/engine/sse/AuthApiClientImp.java @@ -4,6 +4,9 @@ import io.split.client.utils.Json; import io.split.engine.sse.dtos.AuthenticationResponse; import io.split.engine.sse.dtos.RawAuthResponse; +import io.split.telemetry.domain.enums.HTTPLatenciesEnum; +import io.split.telemetry.domain.enums.LastSynchronizationRecordsEnum; +import io.split.telemetry.storage.TelemetryRuntimeProducer; import org.apache.hc.client5.http.classic.methods.HttpGet; import org.apache.hc.client5.http.impl.classic.CloseableHttpClient; import org.apache.hc.client5.http.impl.classic.CloseableHttpResponse; @@ -23,16 +26,18 @@ public class AuthApiClientImp implements AuthApiClient { private final CloseableHttpClient _httpClient; private final String _target; + private final TelemetryRuntimeProducer _telemetryRuntimeProducer; - public AuthApiClientImp(String url, - CloseableHttpClient httpClient) { + public AuthApiClientImp(String url, CloseableHttpClient httpClient, TelemetryRuntimeProducer telemetryRuntimeProducer) { _httpClient = checkNotNull(httpClient); _target = checkNotNull(url); + _telemetryRuntimeProducer = checkNotNull(telemetryRuntimeProducer); } @Override public AuthenticationResponse Authenticate() { try { + long initTime = System.currentTimeMillis(); URI uri = new URIBuilder(_target).build(); HttpGet request = new HttpGet(uri); @@ -43,11 +48,18 @@ public AuthenticationResponse Authenticate() { _log.debug(String.format("Success connection to: %s", _target)); String jsonContent = EntityUtils.toString(response.getEntity(), StandardCharsets.UTF_8); + long endTime = System.currentTimeMillis(); + _telemetryRuntimeProducer.recordTokenRefreshes(); + _telemetryRuntimeProducer.recordSuccessfulSync(LastSynchronizationRecordsEnum.TOKEN, endTime); + _telemetryRuntimeProducer.recordSyncLatency(HTTPLatenciesEnum.TOKEN, endTime-initTime); return getSuccessResponse(jsonContent); } _log.error(String.format("Problem to connect to : %s. Response status: %s", _target, statusCode)); if (statusCode >= HttpStatus.SC_BAD_REQUEST && statusCode < HttpStatus.SC_INTERNAL_SERVER_ERROR) { + if (statusCode == HttpStatus.SC_UNAUTHORIZED) { + _telemetryRuntimeProducer.recordAuthRejections(); + } return new AuthenticationResponse(false,false); } diff --git a/client/src/main/java/io/split/telemetry/storage/InMemoryTelemetryStorage.java b/client/src/main/java/io/split/telemetry/storage/InMemoryTelemetryStorage.java index 1f50cb4fc..644f43eb5 100644 --- a/client/src/main/java/io/split/telemetry/storage/InMemoryTelemetryStorage.java +++ b/client/src/main/java/io/split/telemetry/storage/InMemoryTelemetryStorage.java @@ -1,10 +1,10 @@ package io.split.telemetry.storage; import com.google.common.collect.Maps; -import io.split.telemetry.utils.AtomicLongArray; -import io.split.telemetry.utils.BucketCalculator; import io.split.telemetry.domain.*; import io.split.telemetry.domain.enums.*; +import io.split.telemetry.utils.AtomicLongArray; +import io.split.telemetry.utils.BucketCalculator; import java.util.ArrayList; import java.util.List; @@ -41,7 +41,7 @@ public class InMemoryTelemetryStorage implements TelemetryStorage{ private final Object _tagsLock = new Object(); private final List _tags = new ArrayList<>(); - public InMemoryTelemetryStorage() throws Exception { + public InMemoryTelemetryStorage() { initMethodLatencies(); initHttpLatencies(); initHttpErrors(); @@ -65,7 +65,7 @@ public long getNonReadyUsages() { } @Override - public MethodExceptions popExceptions() throws Exception { + public MethodExceptions popExceptions() { MethodExceptions exceptions = new MethodExceptions(); exceptions.set_treatment(_exceptionsCounters.get(MethodEnum.TREATMENT).get()); exceptions.set_treatments(_exceptionsCounters.get(MethodEnum.TREATMENTS).get()); @@ -80,7 +80,7 @@ public MethodExceptions popExceptions() throws Exception { } @Override - public MethodLatencies popLatencies() throws Exception { + public MethodLatencies popLatencies() { MethodLatencies latencies = new MethodLatencies(); latencies.set_treatment(_methodLatencies.get(MethodEnum.TREATMENT).fetchAndClearAll()); latencies.set_treatments(_methodLatencies.get(MethodEnum.TREATMENTS).fetchAndClearAll()); @@ -158,7 +158,7 @@ public HTTPErrors popHTTPErrors() { } @Override - public HTTPLatencies popHTTPLatencies() throws Exception { + public HTTPLatencies popHTTPLatencies(){ HTTPLatencies latencies = new HTTPLatencies(); latencies.set_splits(_httpLatencies.get(HTTPLatenciesEnum.SPLITS).fetchAndClearAll()); latencies.set_segments(_httpLatencies.get(HTTPLatenciesEnum.SEGMENTS).fetchAndClearAll()); @@ -277,7 +277,7 @@ public void recordSessionLength(long sessionLength) { _sdkRecords.replace(SdkRecordsEnum.SESSION, new AtomicLong(sessionLength)); } - private void initMethodLatencies() throws Exception { + private void initMethodLatencies() { _methodLatencies.put(MethodEnum.TREATMENT, new AtomicLongArray(MAX_LATENCY_BUCKET_COUNT)); _methodLatencies.put(MethodEnum.TREATMENTS, new AtomicLongArray(MAX_LATENCY_BUCKET_COUNT)); _methodLatencies.put(MethodEnum.TREATMENT_WITH_CONFIG, new AtomicLongArray(MAX_LATENCY_BUCKET_COUNT)); @@ -285,7 +285,7 @@ private void initMethodLatencies() throws Exception { _methodLatencies.put(MethodEnum.TRACK, new AtomicLongArray(MAX_LATENCY_BUCKET_COUNT)); } - private void initHttpLatencies() throws Exception { + private void initHttpLatencies() { _httpLatencies.put(HTTPLatenciesEnum.SPLITS, new AtomicLongArray(MAX_LATENCY_BUCKET_COUNT)); _httpLatencies.put(HTTPLatenciesEnum.SEGMENTS, new AtomicLongArray(MAX_LATENCY_BUCKET_COUNT)); _httpLatencies.put(HTTPLatenciesEnum.IMPRESSIONS, new AtomicLongArray(MAX_LATENCY_BUCKET_COUNT)); diff --git a/client/src/main/java/io/split/telemetry/storage/NoopTelemetryStorage.java b/client/src/main/java/io/split/telemetry/storage/NoopTelemetryStorage.java new file mode 100644 index 000000000..aa37fae43 --- /dev/null +++ b/client/src/main/java/io/split/telemetry/storage/NoopTelemetryStorage.java @@ -0,0 +1,149 @@ +package io.split.telemetry.storage; + +import io.split.telemetry.domain.*; +import io.split.telemetry.domain.enums.*; + +import java.util.List; + +public class NoopTelemetryStorage implements TelemetryStorage{ + + @Override + public void recordNonReadyUsage() { + + } + + @Override + public void recordBURTimeout() { + + } + + @Override + public void recordLatency(MethodEnum method, long latency) { + + } + + @Override + public void recordException(MethodEnum method) { + + } + + @Override + public void addTag(String tag) { + + } + + @Override + public void recordImpressionStats(ImpressionsDataTypeEnum dataType, long count) { + + } + + @Override + public void recordEventStats(EventsDataRecordsEnum dataType, long count) { + + } + + @Override + public void recordSuccessfulSync(LastSynchronizationRecordsEnum resource, long time) { + + } + + @Override + public void recordSyncError(ResourceEnum resource, int status) { + + } + + @Override + public void recordSyncLatency(HTTPLatenciesEnum resource, long latency) { + + } + + @Override + public void recordAuthRejections() { + + } + + @Override + public void recordTokenRefreshes() { + + } + + @Override + public void recordStreamingEvents(StreamingEvent streamingEvent) { + + } + + @Override + public void recordSessionLength(long sessionLength) { + + } + + @Override + public long getBURTimeouts() { + return 0; + } + + @Override + public long getNonReadyUsages() { + return 0; + } + + @Override + public MethodExceptions popExceptions() throws Exception { + return null; + } + + @Override + public MethodLatencies popLatencies() throws Exception { + return null; + } + + @Override + public long getImpressionsStats(ImpressionsDataTypeEnum data) { + return 0; + } + + @Override + public long getEventStats(EventsDataRecordsEnum type) { + return 0; + } + + @Override + public LastSynchronization getLastSynchronization() { + return null; + } + + @Override + public HTTPErrors popHTTPErrors() { + return null; + } + + @Override + public HTTPLatencies popHTTPLatencies() throws Exception { + return null; + } + + @Override + public long popAuthRejections() { + return 0; + } + + @Override + public long popTokenRefreshes() { + return 0; + } + + @Override + public List popStreamingEvents() { + return null; + } + + @Override + public List popTags() { + return null; + } + + @Override + public long getSessionLength() { + return 0; + } +} diff --git a/client/src/main/java/io/split/telemetry/synchronizer/TelemetrySyncTask.java b/client/src/main/java/io/split/telemetry/synchronizer/TelemetrySyncTask.java index d8c2b3a5f..725e22148 100644 --- a/client/src/main/java/io/split/telemetry/synchronizer/TelemetrySyncTask.java +++ b/client/src/main/java/io/split/telemetry/synchronizer/TelemetrySyncTask.java @@ -17,7 +17,7 @@ public class TelemetrySyncTask { public TelemetrySyncTask(int telemetryRefreshRate, TelemetrySynchronizer telemetrySynchronizer) { ThreadFactory telemetrySyncThreadFactory = new ThreadFactoryBuilder() .setDaemon(true) - .setNameFormat("Telemetry-synk-%d") + .setNameFormat("Telemetry-sync-%d") .build(); _telemetrySynchronizer = telemetrySynchronizer; //TODO _telemetryRefreshRate = telemetryRefreshRate; diff --git a/client/src/main/java/io/split/telemetry/utils/AtomicLongArray.java b/client/src/main/java/io/split/telemetry/utils/AtomicLongArray.java index 232503929..8413c9b66 100644 --- a/client/src/main/java/io/split/telemetry/utils/AtomicLongArray.java +++ b/client/src/main/java/io/split/telemetry/utils/AtomicLongArray.java @@ -1,5 +1,8 @@ package io.split.telemetry.utils; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + import java.util.ArrayList; import java.util.List; import java.util.concurrent.atomic.AtomicLong; @@ -7,10 +10,14 @@ public class AtomicLongArray { private AtomicLong[] array; + private static final int MAX_LENGTH = 23; + + private static final Logger _log = LoggerFactory.getLogger(AtomicLongArray.class); - public AtomicLongArray(int size) throws Exception { + public AtomicLongArray(int size) { if(size <= 0) { - throw new Exception("Invalid array size"); + _log.error("Invalid array size. Using default size: " + MAX_LENGTH); + size = MAX_LENGTH; } array = new AtomicLong[size]; IntStream.range(0, array.length).forEach(x -> array[x] = new AtomicLong()); diff --git a/client/src/test/java/io/split/client/EventsClientImplTest.java b/client/src/test/java/io/split/client/EventsClientImplTest.java index 1b2706ac1..85307a213 100644 --- a/client/src/test/java/io/split/client/EventsClientImplTest.java +++ b/client/src/test/java/io/split/client/EventsClientImplTest.java @@ -1,11 +1,14 @@ package io.split.client; import io.split.client.dtos.Event; +import io.split.telemetry.storage.InMemoryTelemetryStorage; +import io.split.telemetry.storage.TelemetryStorage; import org.apache.hc.client5.http.classic.methods.HttpUriRequest; import org.apache.hc.client5.http.impl.classic.CloseableHttpClient; import org.apache.hc.client5.http.impl.classic.HttpClients; import org.hamcrest.Matchers; import org.junit.Assert; +import org.junit.Ignore; import org.junit.Test; import org.mockito.Mockito; @@ -16,11 +19,13 @@ import java.util.concurrent.LinkedBlockingQueue; public class EventsClientImplTest { + private static final TelemetryStorage TELEMETRY_STORAGE = Mockito.mock(InMemoryTelemetryStorage.class); + @Test public void testDefaultURL() throws URISyntaxException { URI rootTarget = URI.create("https://api.split.io"); CloseableHttpClient httpClient = HttpClients.custom().build(); - EventClientImpl fetcher = EventClientImpl.create(httpClient, rootTarget, 5, 5, 5); + EventClientImpl fetcher = EventClientImpl.create(httpClient, rootTarget, 5, 5, 5, TELEMETRY_STORAGE); Assert.assertThat(fetcher.getTarget().toString(), Matchers.is(Matchers.equalTo("https://api.split.io/api/events/bulk"))); } @@ -28,7 +33,7 @@ public void testDefaultURL() throws URISyntaxException { public void testCustomURLNoPathNoBackslash() throws URISyntaxException { URI rootTarget = URI.create("https://kubernetesturl.com"); CloseableHttpClient httpClient = HttpClients.custom().build(); - EventClientImpl fetcher = EventClientImpl.create(httpClient, rootTarget, 5, 5, 5); + EventClientImpl fetcher = EventClientImpl.create(httpClient, rootTarget, 5, 5, 5, TELEMETRY_STORAGE); Assert.assertThat(fetcher.getTarget().toString(), Matchers.is(Matchers.equalTo("https://kubernetesturl.com/api/events/bulk"))); } @@ -36,7 +41,7 @@ public void testCustomURLNoPathNoBackslash() throws URISyntaxException { public void testCustomURLAppendingPath() throws URISyntaxException { URI rootTarget = URI.create("https://kubernetesturl.com/split/"); CloseableHttpClient httpClient = HttpClients.custom().build(); - EventClientImpl fetcher = EventClientImpl.create(httpClient, rootTarget, 5, 5, 5); + EventClientImpl fetcher = EventClientImpl.create(httpClient, rootTarget, 5, 5, 5, TELEMETRY_STORAGE); Assert.assertThat(fetcher.getTarget().toString(), Matchers.is(Matchers.equalTo("https://kubernetesturl.com/split/api/events/bulk"))); } @@ -44,11 +49,12 @@ public void testCustomURLAppendingPath() throws URISyntaxException { public void testCustomURLAppendingPathNoBackslash() throws URISyntaxException { URI rootTarget = URI.create("https://kubernetesturl.com/split"); CloseableHttpClient httpClient = HttpClients.custom().build(); - EventClientImpl fetcher = EventClientImpl.create(httpClient, rootTarget, 5, 5, 5); + EventClientImpl fetcher = EventClientImpl.create(httpClient, rootTarget, 5, 5, 5, TELEMETRY_STORAGE); Assert.assertThat(fetcher.getTarget().toString(), Matchers.is(Matchers.equalTo("https://kubernetesturl.com/split/api/events/bulk"))); } @Test + @Ignore public void testEventsFlushedWhenSizeLimitReached() throws URISyntaxException, InterruptedException, IOException { CloseableHttpClient client = Mockito.mock(CloseableHttpClient.class); EventClientImpl eventClient = new EventClientImpl(new LinkedBlockingQueue(), @@ -56,7 +62,7 @@ public void testEventsFlushedWhenSizeLimitReached() throws URISyntaxException, I URI.create("https://kubernetesturl.com/split"), 10000, // Long queue so it doesn't flush by # of events 100000, // Long period so it doesn't flush by timeout expiration. - 0); + 0, TELEMETRY_STORAGE); for (int i = 0; i < 159; ++i) { Event event = new Event(); diff --git a/client/src/test/java/io/split/client/HttpSegmentChangeFetcherTest.java b/client/src/test/java/io/split/client/HttpSegmentChangeFetcherTest.java index afb238552..ea7e57afd 100644 --- a/client/src/test/java/io/split/client/HttpSegmentChangeFetcherTest.java +++ b/client/src/test/java/io/split/client/HttpSegmentChangeFetcherTest.java @@ -3,12 +3,15 @@ import io.split.TestHelper; import io.split.client.dtos.SegmentChange; import io.split.engine.metrics.Metrics; +import io.split.telemetry.storage.InMemoryTelemetryStorage; +import io.split.telemetry.storage.TelemetryStorage; import org.apache.hc.client5.http.impl.classic.CloseableHttpClient; import org.apache.hc.client5.http.impl.classic.HttpClients; import org.apache.hc.core5.http.HttpStatus; import org.hamcrest.Matchers; import org.junit.Assert; import org.junit.Test; +import org.mockito.Mockito; import java.io.IOException; import java.lang.reflect.InvocationTargetException; @@ -16,12 +19,14 @@ import java.net.URISyntaxException; public class HttpSegmentChangeFetcherTest { + private static final TelemetryStorage TELEMETRY_STORAGE = Mockito.mock(InMemoryTelemetryStorage.class); + @Test public void testDefaultURL() throws URISyntaxException { URI rootTarget = URI.create("https://api.split.io"); CloseableHttpClient httpClient = HttpClients.custom().build(); Metrics.NoopMetrics metrics = new Metrics.NoopMetrics(); - HttpSegmentChangeFetcher fetcher = HttpSegmentChangeFetcher.create(httpClient, rootTarget, metrics); + HttpSegmentChangeFetcher fetcher = HttpSegmentChangeFetcher.create(httpClient, rootTarget, TELEMETRY_STORAGE); Assert.assertThat(fetcher.getTarget().toString(), Matchers.is(Matchers.equalTo("https://api.split.io/api/segmentChanges"))); } @@ -30,7 +35,7 @@ public void testCustomURLNoPathNoBackslash() throws URISyntaxException { URI rootTarget = URI.create("https://kubernetesturl.com/split"); CloseableHttpClient httpClient = HttpClients.custom().build(); Metrics.NoopMetrics metrics = new Metrics.NoopMetrics(); - HttpSegmentChangeFetcher fetcher = HttpSegmentChangeFetcher.create(httpClient, rootTarget, metrics); + HttpSegmentChangeFetcher fetcher = HttpSegmentChangeFetcher.create(httpClient, rootTarget, TELEMETRY_STORAGE); Assert.assertThat(fetcher.getTarget().toString(), Matchers.is(Matchers.equalTo("https://kubernetesturl.com/split/api/segmentChanges"))); } @@ -39,7 +44,7 @@ public void testCustomURLAppendingPath() throws URISyntaxException { URI rootTarget = URI.create("https://kubernetesturl.com/split/"); CloseableHttpClient httpClient = HttpClients.custom().build(); Metrics.NoopMetrics metrics = new Metrics.NoopMetrics(); - HttpSegmentChangeFetcher fetcher = HttpSegmentChangeFetcher.create(httpClient, rootTarget, metrics); + HttpSegmentChangeFetcher fetcher = HttpSegmentChangeFetcher.create(httpClient, rootTarget, TELEMETRY_STORAGE); Assert.assertThat(fetcher.getTarget().toString(), Matchers.is(Matchers.equalTo("https://kubernetesturl.com/split/api/segmentChanges"))); } @@ -48,7 +53,7 @@ public void testCustomURLAppendingPathNoBackslash() throws URISyntaxException { URI rootTarget = URI.create("https://kubernetesturl.com/split"); CloseableHttpClient httpClient = HttpClients.custom().build(); Metrics.NoopMetrics metrics = new Metrics.NoopMetrics(); - HttpSegmentChangeFetcher fetcher = HttpSegmentChangeFetcher.create(httpClient, rootTarget, metrics); + HttpSegmentChangeFetcher fetcher = HttpSegmentChangeFetcher.create(httpClient, rootTarget, TELEMETRY_STORAGE); Assert.assertThat(fetcher.getTarget().toString(), Matchers.is(Matchers.equalTo("https://kubernetesturl.com/split/api/segmentChanges"))); } @@ -59,7 +64,7 @@ public void testFetcherWithSpecialCharacters() throws URISyntaxException, IOExce CloseableHttpClient httpClientMock = TestHelper.mockHttpClient("segment-change-special-chatacters.json", HttpStatus.SC_OK); Metrics.NoopMetrics metrics = new Metrics.NoopMetrics(); - HttpSegmentChangeFetcher fetcher = HttpSegmentChangeFetcher.create(httpClientMock, rootTarget, metrics); + HttpSegmentChangeFetcher fetcher = HttpSegmentChangeFetcher.create(httpClientMock, rootTarget, TELEMETRY_STORAGE); SegmentChange change = fetcher.fetch("some_segment", 1234567, true); diff --git a/client/src/test/java/io/split/client/HttpSplitChangeFetcherTest.java b/client/src/test/java/io/split/client/HttpSplitChangeFetcherTest.java index 564339db7..a27764ce4 100644 --- a/client/src/test/java/io/split/client/HttpSplitChangeFetcherTest.java +++ b/client/src/test/java/io/split/client/HttpSplitChangeFetcherTest.java @@ -4,12 +4,15 @@ import io.split.client.dtos.Split; import io.split.client.dtos.SplitChange; import io.split.engine.metrics.Metrics; +import io.split.telemetry.storage.InMemoryTelemetryStorage; +import io.split.telemetry.storage.TelemetryStorage; import org.apache.hc.client5.http.impl.classic.CloseableHttpClient; import org.apache.hc.client5.http.impl.classic.HttpClients; import org.apache.hc.core5.http.HttpStatus; import org.hamcrest.Matchers; import org.junit.Assert; import org.junit.Test; +import org.mockito.Mockito; import java.io.IOException; import java.lang.reflect.InvocationTargetException; @@ -18,12 +21,13 @@ import java.util.Map; public class HttpSplitChangeFetcherTest { + private static final TelemetryStorage TELEMETRY_STORAGE = Mockito.mock(InMemoryTelemetryStorage.class); @Test public void testDefaultURL() throws URISyntaxException { URI rootTarget = URI.create("https://api.split.io"); CloseableHttpClient httpClient = HttpClients.custom().build(); Metrics.NoopMetrics metrics = new Metrics.NoopMetrics(); - HttpSplitChangeFetcher fetcher = HttpSplitChangeFetcher.create(httpClient, rootTarget, metrics); + HttpSplitChangeFetcher fetcher = HttpSplitChangeFetcher.create(httpClient, rootTarget, TELEMETRY_STORAGE); Assert.assertThat(fetcher.getTarget().toString(), Matchers.is(Matchers.equalTo("https://api.split.io/api/splitChanges"))); } @@ -32,7 +36,7 @@ public void testCustomURLNoPathNoBackslash() throws URISyntaxException { URI rootTarget = URI.create("https://kubernetesturl.com/split"); CloseableHttpClient httpClient = HttpClients.custom().build(); Metrics.NoopMetrics metrics = new Metrics.NoopMetrics(); - HttpSplitChangeFetcher fetcher = HttpSplitChangeFetcher.create(httpClient, rootTarget, metrics); + HttpSplitChangeFetcher fetcher = HttpSplitChangeFetcher.create(httpClient, rootTarget, TELEMETRY_STORAGE); Assert.assertThat(fetcher.getTarget().toString(), Matchers.is(Matchers.equalTo("https://kubernetesturl.com/split/api/splitChanges"))); } @@ -41,7 +45,7 @@ public void testCustomURLAppendingPath() throws URISyntaxException { URI rootTarget = URI.create("https://kubernetesturl.com/split/"); CloseableHttpClient httpClient = HttpClients.custom().build(); Metrics.NoopMetrics metrics = new Metrics.NoopMetrics(); - HttpSplitChangeFetcher fetcher = HttpSplitChangeFetcher.create(httpClient, rootTarget, metrics); + HttpSplitChangeFetcher fetcher = HttpSplitChangeFetcher.create(httpClient, rootTarget, TELEMETRY_STORAGE); Assert.assertThat(fetcher.getTarget().toString(), Matchers.is(Matchers.equalTo("https://kubernetesturl.com/split/api/splitChanges"))); } @@ -50,7 +54,7 @@ public void testCustomURLAppendingPathNoBackslash() throws URISyntaxException { URI rootTarget = URI.create("https://kubernetesturl.com/split"); CloseableHttpClient httpClient = HttpClients.custom().build(); Metrics.NoopMetrics metrics = new Metrics.NoopMetrics(); - HttpSplitChangeFetcher fetcher = HttpSplitChangeFetcher.create(httpClient, rootTarget, metrics); + HttpSplitChangeFetcher fetcher = HttpSplitChangeFetcher.create(httpClient, rootTarget, TELEMETRY_STORAGE); Assert.assertThat(fetcher.getTarget().toString(), Matchers.is(Matchers.equalTo("https://kubernetesturl.com/split/api/splitChanges"))); } @@ -61,7 +65,7 @@ public void testFetcherWithSpecialCharacters() throws URISyntaxException, Invoca CloseableHttpClient httpClientMock = TestHelper.mockHttpClient("split-change-special-characters.json", HttpStatus.SC_OK); Metrics.NoopMetrics metrics = new Metrics.NoopMetrics(); - HttpSplitChangeFetcher fetcher = HttpSplitChangeFetcher.create(httpClientMock, rootTarget, metrics); + HttpSplitChangeFetcher fetcher = HttpSplitChangeFetcher.create(httpClientMock, rootTarget, TELEMETRY_STORAGE); SplitChange change = fetcher.fetch(1234567, true); diff --git a/client/src/test/java/io/split/client/SplitClientImplTest.java b/client/src/test/java/io/split/client/SplitClientImplTest.java index dbee7934c..d4d7a3353 100644 --- a/client/src/test/java/io/split/client/SplitClientImplTest.java +++ b/client/src/test/java/io/split/client/SplitClientImplTest.java @@ -25,6 +25,8 @@ import io.split.engine.matchers.strings.WhitelistMatcher; import io.split.engine.metrics.Metrics; import io.split.grammar.Treatments; +import io.split.telemetry.storage.InMemoryTelemetryStorage; +import io.split.telemetry.storage.TelemetryStorage; import org.apache.commons.lang3.RandomStringUtils; import org.junit.Assert; import org.junit.Test; @@ -61,6 +63,7 @@ */ public class SplitClientImplTest { + private static final TelemetryStorage TELEMETRY_STORAGE = Mockito.mock(InMemoryTelemetryStorage.class); private SplitClientConfig config = SplitClientConfig.builder().setBlockUntilReadyTimeout(100).build(); @Test @@ -81,7 +84,7 @@ public void null_key_results_in_control() { NoopEventClient.create(), config, gates, - new EvaluatorImp(splitCache) + new EvaluatorImp(splitCache), TELEMETRY_STORAGE, TELEMETRY_STORAGE ); assertThat(client.getTreatment(null, "test1"), is(equalTo(Treatments.CONTROL))); @@ -107,7 +110,7 @@ public void null_test_results_in_control() { NoopEventClient.create(), config, gates, - new EvaluatorImp(splitCache) + new EvaluatorImp(splitCache), TELEMETRY_STORAGE, TELEMETRY_STORAGE ); assertThat(client.getTreatment("adil@relateiq.com", null), is(equalTo(Treatments.CONTROL))); @@ -128,7 +131,7 @@ public void exceptions_result_in_control() { NoopEventClient.create(), config, gates, - new EvaluatorImp(splitCache) + new EvaluatorImp(splitCache), TELEMETRY_STORAGE, TELEMETRY_STORAGE ); assertThat(client.getTreatment("adil@relateiq.com", "test1"), is(equalTo(Treatments.CONTROL))); @@ -154,7 +157,7 @@ public void works() { NoopEventClient.create(), config, gates, - new EvaluatorImp(splitCache) + new EvaluatorImp(splitCache), TELEMETRY_STORAGE, TELEMETRY_STORAGE ); int numKeys = 5; @@ -188,7 +191,7 @@ public void works_null_config() { NoopEventClient.create(), config, gates, - new EvaluatorImp(splitCache) + new EvaluatorImp(splitCache), TELEMETRY_STORAGE, TELEMETRY_STORAGE ); @@ -224,7 +227,7 @@ public void worksAndHasConfig() { NoopEventClient.create(), config, gates, - new EvaluatorImp(splitCache) + new EvaluatorImp(splitCache), TELEMETRY_STORAGE, TELEMETRY_STORAGE ); int numKeys = 5; @@ -258,7 +261,7 @@ public void last_condition_is_always_default() { NoopEventClient.create(), config, gates, - new EvaluatorImp(splitCache) + new EvaluatorImp(splitCache), TELEMETRY_STORAGE, TELEMETRY_STORAGE ); assertThat(client.getTreatment("pato@codigo.com", test), is(equalTo(Treatments.OFF))); @@ -293,7 +296,7 @@ public void last_condition_is_always_default_but_with_treatment() { NoopEventClient.create(), config, gates, - new EvaluatorImp(splitCache) + new EvaluatorImp(splitCache), TELEMETRY_STORAGE, TELEMETRY_STORAGE ); SplitResult result = client.getTreatmentWithConfig("pato@codigo.com", test); @@ -325,7 +328,7 @@ public void multiple_conditions_work() { NoopEventClient.create(), config, gates, - new EvaluatorImp(splitCache) + new EvaluatorImp(splitCache), TELEMETRY_STORAGE, TELEMETRY_STORAGE ); assertThat(client.getTreatment("adil@codigo.com", test), is(equalTo("on"))); @@ -355,7 +358,7 @@ public void killed_test_always_goes_to_default() { NoopEventClient.create(), config, gates, - new EvaluatorImp(splitCache) + new EvaluatorImp(splitCache), TELEMETRY_STORAGE, TELEMETRY_STORAGE ); assertThat(client.getTreatment("adil@codigo.com", test), is(equalTo(Treatments.OFF))); @@ -390,7 +393,7 @@ public void killed_test_always_goes_to_default_has_config() { NoopEventClient.create(), config, gates, - new EvaluatorImp(splitCache) + new EvaluatorImp(splitCache), TELEMETRY_STORAGE, TELEMETRY_STORAGE ); SplitResult result = client.getTreatmentWithConfig("adil@codigo.com", test); @@ -425,7 +428,7 @@ public void dependency_matcher_on() { NoopEventClient.create(), config, gates, - new EvaluatorImp(splitCache) + new EvaluatorImp(splitCache), TELEMETRY_STORAGE, TELEMETRY_STORAGE ); assertThat(client.getTreatment("key", parent), is(equalTo(Treatments.ON))); @@ -457,7 +460,7 @@ public void dependency_matcher_off() { NoopEventClient.create(), config, gates, - new EvaluatorImp(splitCache) + new EvaluatorImp(splitCache), TELEMETRY_STORAGE, TELEMETRY_STORAGE ); assertThat(client.getTreatment("key", parent), is(equalTo(Treatments.ON))); @@ -483,7 +486,7 @@ public void dependency_matcher_control() { NoopEventClient.create(), config, gates, - new EvaluatorImp(splitCache) + new EvaluatorImp(splitCache), TELEMETRY_STORAGE, TELEMETRY_STORAGE ); assertThat(client.getTreatment("key", dependent), is(equalTo(Treatments.ON))); @@ -510,7 +513,7 @@ public void attributes_work() { NoopEventClient.create(), config, gates, - new EvaluatorImp(splitCache) + new EvaluatorImp(splitCache), TELEMETRY_STORAGE, TELEMETRY_STORAGE ); assertThat(client.getTreatment("adil@codigo.com", test), is(equalTo("on"))); @@ -543,7 +546,7 @@ public void attributes_work_2() { NoopEventClient.create(), config, gates, - new EvaluatorImp(splitCache) + new EvaluatorImp(splitCache), TELEMETRY_STORAGE, TELEMETRY_STORAGE ); assertThat(client.getTreatment("adil@codigo.com", test), is(equalTo("off"))); @@ -576,7 +579,7 @@ public void attributes_greater_than_negative_number() { NoopEventClient.create(), config, gates, - new EvaluatorImp(splitCache) + new EvaluatorImp(splitCache), TELEMETRY_STORAGE, TELEMETRY_STORAGE ); assertThat(client.getTreatment("adil@codigo.com", test), is(equalTo("off"))); @@ -612,7 +615,7 @@ public void attributes_for_sets() { NoopEventClient.create(), config, gates, - new EvaluatorImp(splitCache) + new EvaluatorImp(splitCache), TELEMETRY_STORAGE, TELEMETRY_STORAGE ); assertThat(client.getTreatment("adil@codigo.com", test), is(equalTo("off"))); @@ -654,7 +657,7 @@ public void labels_are_populated() { NoopEventClient.create(), config, gates, - new EvaluatorImp(splitCache) + new EvaluatorImp(splitCache), TELEMETRY_STORAGE, TELEMETRY_STORAGE ); Map attributes = ImmutableMap.of("age", -20, "acv", "1000000"); @@ -745,7 +748,7 @@ private void traffic_allocation(String key, int trafficAllocation, int trafficAl NoopEventClient.create(), config, gates, - new EvaluatorImp(splitCache) + new EvaluatorImp(splitCache), TELEMETRY_STORAGE, TELEMETRY_STORAGE ); assertThat(client.getTreatment(key, test), is(equalTo(expected_treatment_on_or_off))); @@ -792,7 +795,7 @@ public void notInTrafficAllocationDefaultConfig() { NoopEventClient.create(), config, gates, - new EvaluatorImp(splitCache) + new EvaluatorImp(splitCache), TELEMETRY_STORAGE, TELEMETRY_STORAGE ); assertThat(client.getTreatment("pato@split.io", test), is(equalTo(Treatments.OFF))); @@ -831,7 +834,7 @@ public void matching_bucketing_keys_work() { NoopEventClient.create(), config, gates, - new EvaluatorImp(splitCache) + new EvaluatorImp(splitCache), TELEMETRY_STORAGE, TELEMETRY_STORAGE ); Key bad_key = new Key("adil", "aijaz"); @@ -868,7 +871,7 @@ public void impression_metadata_is_propagated() { NoopEventClient.create(), config, gates, - new EvaluatorImp(splitCache) + new EvaluatorImp(splitCache), TELEMETRY_STORAGE, TELEMETRY_STORAGE ); Map attributes = ImmutableMap.of("age", -20, "acv", "1000000"); @@ -903,7 +906,7 @@ public void block_until_ready_does_not_time_when_sdk_is_ready() throws TimeoutEx NoopEventClient.create(), config, ready, - new EvaluatorImp(splitCache) + new EvaluatorImp(splitCache), TELEMETRY_STORAGE, TELEMETRY_STORAGE ); client.blockUntilReady(); @@ -922,7 +925,7 @@ public void block_until_ready_times_when_sdk_is_not_ready() throws TimeoutExcept NoopEventClient.create(), config, ready, - new EvaluatorImp(splitCache) + new EvaluatorImp(splitCache), TELEMETRY_STORAGE, TELEMETRY_STORAGE ); client.blockUntilReady(); @@ -940,7 +943,7 @@ public void track_with_valid_parameters() { NoopEventClient.create(), config, gates, - new EvaluatorImp(splitCache) + new EvaluatorImp(splitCache), TELEMETRY_STORAGE, TELEMETRY_STORAGE ); Assert.assertThat(client.track("validKey", "valid_traffic_type", "valid_event"), @@ -965,7 +968,7 @@ public void track_with_invalid_event_type_ids() { NoopEventClient.create(), config, gates, - new EvaluatorImp(splitCache) + new EvaluatorImp(splitCache), TELEMETRY_STORAGE, TELEMETRY_STORAGE ); Assert.assertThat(client.track("validKey", "valid_traffic_type", ""), @@ -995,7 +998,7 @@ public void track_with_invalid_traffic_type_names() { NoopEventClient.create(), config, gates, - new EvaluatorImp(splitCache) + new EvaluatorImp(splitCache), TELEMETRY_STORAGE, TELEMETRY_STORAGE ); Assert.assertThat(client.track("validKey", "", "valid"), @@ -1017,7 +1020,7 @@ public void track_with_invalid_keys() { NoopEventClient.create(), config, gates, - new EvaluatorImp(splitCache) + new EvaluatorImp(splitCache), TELEMETRY_STORAGE, TELEMETRY_STORAGE ); Assert.assertThat(client.track("", "valid_traffic_type", "valid"), @@ -1045,7 +1048,7 @@ public void track_with_properties() { eventClientMock, config, gates, - new EvaluatorImp(splitCache) + new EvaluatorImp(splitCache), TELEMETRY_STORAGE, TELEMETRY_STORAGE ); HashMap properties = new HashMap<>(); @@ -1155,7 +1158,7 @@ public void getTreatment_with_invalid_keys() { NoopEventClient.create(), config, gates, - new EvaluatorImp(splitCache) + new EvaluatorImp(splitCache), TELEMETRY_STORAGE, TELEMETRY_STORAGE ); Assert.assertThat(client.getTreatment("valid", "split"), @@ -1239,7 +1242,7 @@ public void client_cannot_perform_actions_when_destroyed() throws InterruptedExc NoopEventClient.create(), config, gates, - new EvaluatorImp(splitCache) + new EvaluatorImp(splitCache), TELEMETRY_STORAGE, TELEMETRY_STORAGE ); Assert.assertThat(client.getTreatment("valid", "split"), @@ -1281,7 +1284,7 @@ public void worksAndHasConfigTryKetTreatmentWithKey() { NoopEventClient.create(), config, gates, - new EvaluatorImp(splitCache) + new EvaluatorImp(splitCache), TELEMETRY_STORAGE, TELEMETRY_STORAGE ); int numKeys = 5; @@ -1317,7 +1320,7 @@ public void blockUntilReadyException() throws TimeoutException, InterruptedExcep NoopEventClient.create(), config, gates, - new EvaluatorImp(splitCache) + new EvaluatorImp(splitCache), TELEMETRY_STORAGE, TELEMETRY_STORAGE ); client.blockUntilReady(); diff --git a/client/src/test/java/io/split/client/SplitClientIntegrationTest.java b/client/src/test/java/io/split/client/SplitClientIntegrationTest.java index d7ab9ef65..aba335b57 100644 --- a/client/src/test/java/io/split/client/SplitClientIntegrationTest.java +++ b/client/src/test/java/io/split/client/SplitClientIntegrationTest.java @@ -3,12 +3,15 @@ import io.split.SSEMockServer; import io.split.SplitMockServer; import io.split.client.api.SplitView; +import io.split.telemetry.storage.InMemoryTelemetryStorage; +import io.split.telemetry.storage.TelemetryStorage; import org.awaitility.Awaitility; import org.glassfish.grizzly.utils.Pair; import org.glassfish.jersey.media.sse.OutboundEvent; import org.junit.Assert; import org.junit.Ignore; import org.junit.Test; +import org.mockito.Mockito; import javax.ws.rs.sse.OutboundSseEvent; import java.io.IOException; @@ -20,9 +23,11 @@ public class SplitClientIntegrationTest { // TODO: review this test. + private static final TelemetryStorage TELEMETRY_STORAGE = Mockito.mock(InMemoryTelemetryStorage.class); + @Test @Ignore - public void getTreatmentWithStreamingEnabled() throws IOException, TimeoutException, InterruptedException, URISyntaxException { + public void getTreatmentWithStreamingEnabled() throws Exception { SplitMockServer splitServer = new SplitMockServer(); SSEMockServer.SseEventQueue eventQueue = new SSEMockServer.SseEventQueue(); SSEMockServer sseServer = buildSSEMockServer(eventQueue); @@ -111,7 +116,7 @@ public void getTreatmentWithStreamingEnabled() throws IOException, TimeoutExcept } @Test - public void getTreatmentWithStreamingEnabledAndAuthDisabled() throws IOException, TimeoutException, InterruptedException, URISyntaxException { + public void getTreatmentWithStreamingEnabledAndAuthDisabled() throws Exception { SplitMockServer splitServer = new SplitMockServer(); splitServer.start(); @@ -134,7 +139,7 @@ public void getTreatmentWithStreamingEnabledAndAuthDisabled() throws IOException } @Test - public void getTreatmentWithStreamingDisabled() throws IOException, TimeoutException, InterruptedException, URISyntaxException { + public void getTreatmentWithStreamingDisabled() throws Exception { SplitMockServer splitServer = new SplitMockServer(); splitServer.start(); @@ -162,7 +167,7 @@ public void getTreatmentWithStreamingDisabled() throws IOException, TimeoutExcep } @Test - public void managerSplitsWithStreamingEnabled() throws IOException, TimeoutException, InterruptedException, URISyntaxException { + public void managerSplitsWithStreamingEnabled() throws Exception { SplitMockServer splitServer = new SplitMockServer(); SSEMockServer.SseEventQueue eventQueue = new SSEMockServer.SseEventQueue(); SSEMockServer sseServer = buildSSEMockServer(eventQueue); @@ -197,7 +202,7 @@ public void managerSplitsWithStreamingEnabled() throws IOException, TimeoutExcep } @Test - public void splitClientOccupancyNotifications() throws IOException, TimeoutException, InterruptedException, URISyntaxException { + public void splitClientOccupancyNotifications() throws Exception { SplitMockServer splitServer = new SplitMockServer(); SSEMockServer.SseEventQueue eventQueue = new SSEMockServer.SseEventQueue(); SSEMockServer sseServer = buildSSEMockServer(eventQueue); @@ -259,7 +264,7 @@ public void splitClientOccupancyNotifications() throws IOException, TimeoutExcep } @Test - public void splitClientControlNotifications() throws IOException, TimeoutException, InterruptedException, URISyntaxException { + public void splitClientControlNotifications() throws Exception { SplitMockServer splitServer = new SplitMockServer(); SSEMockServer.SseEventQueue eventQueue = new SSEMockServer.SseEventQueue(); SSEMockServer sseServer = buildSSEMockServer(eventQueue); @@ -341,7 +346,7 @@ public void splitClientControlNotifications() throws IOException, TimeoutExcepti } @Test - public void splitClientMultiFactory() throws IOException, TimeoutException, InterruptedException, URISyntaxException { + public void splitClientMultiFactory() throws Exception { SplitMockServer splitServer = new SplitMockServer(); SSEMockServer.SseEventQueue eventQueue1 = new SSEMockServer.SseEventQueue(); @@ -465,7 +470,7 @@ public void splitClientMultiFactory() throws IOException, TimeoutException, Inte // TODO: review this test. @Test @Ignore - public void keepAlive() throws IOException, TimeoutException, InterruptedException, URISyntaxException { + public void keepAlive() throws Exception { SplitMockServer splitServer = new SplitMockServer(); SSEMockServer.SseEventQueue eventQueue = new SSEMockServer.SseEventQueue(); SSEMockServer sseServer = buildSSEMockServer(eventQueue); @@ -495,7 +500,7 @@ public void keepAlive() throws IOException, TimeoutException, InterruptedExcepti } @Test - public void testConnectionClosedByRemoteHostIsProperlyHandled() throws IOException, TimeoutException, InterruptedException, URISyntaxException { + public void testConnectionClosedByRemoteHostIsProperlyHandled() throws Exception { SplitMockServer splitServer = new SplitMockServer(); SSEMockServer.SseEventQueue eventQueue = new SSEMockServer.SseEventQueue(); SSEMockServer sseServer = buildSSEMockServer(eventQueue); @@ -528,7 +533,7 @@ public void testConnectionClosedByRemoteHostIsProperlyHandled() throws IOExcepti } @Test - public void testConnectionClosedIsProperlyHandled() throws IOException, TimeoutException, InterruptedException, URISyntaxException { + public void testConnectionClosedIsProperlyHandled() throws Exception { SplitMockServer splitServer = new SplitMockServer(); SSEMockServer.SseEventQueue eventQueue = new SSEMockServer.SseEventQueue(); SSEMockServer sseServer = buildSSEMockServer(eventQueue); diff --git a/client/src/test/java/io/split/client/SplitManagerImplTest.java b/client/src/test/java/io/split/client/SplitManagerImplTest.java index 7ef068ac9..c156ed973 100644 --- a/client/src/test/java/io/split/client/SplitManagerImplTest.java +++ b/client/src/test/java/io/split/client/SplitManagerImplTest.java @@ -10,6 +10,8 @@ import io.split.engine.matchers.AllKeysMatcher; import io.split.engine.matchers.CombiningMatcher; import io.split.grammar.Treatments; +import io.split.telemetry.storage.InMemoryTelemetryStorage; +import io.split.telemetry.storage.TelemetryStorage; import org.junit.Test; import org.mockito.Mockito; @@ -29,6 +31,7 @@ public class SplitManagerImplTest { private SplitClientConfig config = SplitClientConfig.builder().setBlockUntilReadyTimeout(100).build(); + private static final TelemetryStorage TELEMETRY_STORAGE = Mockito.mock(InMemoryTelemetryStorage.class); @Test public void splitCallWithNonExistentSplit() { @@ -38,7 +41,7 @@ public void splitCallWithNonExistentSplit() { SplitManagerImpl splitManager = new SplitManagerImpl(splitCache, Mockito.mock(SplitClientConfig.class), - Mockito.mock(SDKReadinessGates.class)); + Mockito.mock(SDKReadinessGates.class), TELEMETRY_STORAGE); assertThat(splitManager.split("nonExistent"), is(nullValue())); } @@ -52,7 +55,7 @@ public void splitCallWithExistentSplit() { SplitManagerImpl splitManager = new SplitManagerImpl(splitCache, Mockito.mock(SplitClientConfig.class), - Mockito.mock(SDKReadinessGates.class)); + Mockito.mock(SDKReadinessGates.class), TELEMETRY_STORAGE); SplitView theOne = splitManager.split(existent); assertThat(theOne.name, is(equalTo(response.feature()))); assertThat(theOne.changeNumber, is(equalTo(response.changeNumber()))); @@ -77,7 +80,7 @@ public void splitCallWithExistentSplitAndConfigs() { SplitManagerImpl splitManager = new SplitManagerImpl(splitCache, Mockito.mock(SplitClientConfig.class), - Mockito.mock(SDKReadinessGates.class)); + Mockito.mock(SDKReadinessGates.class), TELEMETRY_STORAGE); SplitView theOne = splitManager.split(existent); assertThat(theOne.name, is(equalTo(response.feature()))); assertThat(theOne.changeNumber, is(equalTo(response.changeNumber()))); @@ -94,7 +97,7 @@ public void splitsCallWithNoSplit() { Mockito.when(splitCache.getAll()).thenReturn(Lists.newArrayList()); SplitManagerImpl splitManager = new SplitManagerImpl(splitCache, Mockito.mock(SplitClientConfig.class), - Mockito.mock(SDKReadinessGates.class)); + Mockito.mock(SDKReadinessGates.class), TELEMETRY_STORAGE); assertThat(splitManager.splits(), is(empty())); } @@ -108,7 +111,7 @@ public void splitsCallWithSplit() { Mockito.when(splitCache.getAll()).thenReturn(parsedSplits); SplitManagerImpl splitManager = new SplitManagerImpl(splitCache, Mockito.mock(SplitClientConfig.class), - Mockito.mock(SDKReadinessGates.class)); + Mockito.mock(SDKReadinessGates.class), TELEMETRY_STORAGE); List splits = splitManager.splits(); assertThat(splits.size(), is(equalTo(1))); assertThat(splits.get(0).name, is(equalTo(response.feature()))); @@ -125,7 +128,7 @@ public void splitNamesCallWithNoSplit() { Mockito.when(splitCache.getAll()).thenReturn(Lists.newArrayList()); SplitManagerImpl splitManager = new SplitManagerImpl(splitCache, Mockito.mock(SplitClientConfig.class), - Mockito.mock(SDKReadinessGates.class)); + Mockito.mock(SDKReadinessGates.class), TELEMETRY_STORAGE); assertThat(splitManager.splitNames(), is(empty())); } @@ -139,7 +142,7 @@ public void splitNamesCallWithSplit() { Mockito.when(splitCache.getAll()).thenReturn(parsedSplits); SplitManagerImpl splitManager = new SplitManagerImpl(splitCache, Mockito.mock(SplitClientConfig.class), - Mockito.mock(SDKReadinessGates.class)); + Mockito.mock(SDKReadinessGates.class), TELEMETRY_STORAGE); List splitNames = splitManager.splitNames(); assertThat(splitNames.size(), is(equalTo(1))); assertThat(splitNames.get(0), is(equalTo(response.feature()))); @@ -151,7 +154,7 @@ public void block_until_ready_does_not_time_when_sdk_is_ready() throws TimeoutEx when(ready.isSDKReady(100)).thenReturn(true); SplitManagerImpl splitManager = new SplitManagerImpl(mock(SplitCache.class), config, - ready); + ready, TELEMETRY_STORAGE); splitManager.blockUntilReady(); } @@ -163,7 +166,7 @@ public void block_until_ready_times_when_sdk_is_not_ready() throws TimeoutExcept SplitManagerImpl splitManager = new SplitManagerImpl(mock(SplitCache.class), config, - ready); + ready, TELEMETRY_STORAGE); splitManager.blockUntilReady(); } diff --git a/client/src/test/java/io/split/client/impressions/HttpImpressionsSenderTest.java b/client/src/test/java/io/split/client/impressions/HttpImpressionsSenderTest.java index 876e2bf3c..ee586ffe4 100644 --- a/client/src/test/java/io/split/client/impressions/HttpImpressionsSenderTest.java +++ b/client/src/test/java/io/split/client/impressions/HttpImpressionsSenderTest.java @@ -6,6 +6,8 @@ import io.split.client.dtos.ImpressionCount; import io.split.client.dtos.KeyImpression; import io.split.client.dtos.TestImpressions; +import io.split.telemetry.storage.InMemoryTelemetryStorage; +import io.split.telemetry.storage.TelemetryStorage; import org.apache.hc.client5.http.classic.methods.HttpPost; import org.apache.hc.client5.http.classic.methods.HttpUriRequest; import org.apache.hc.client5.http.impl.classic.CloseableHttpClient; @@ -34,11 +36,13 @@ import static org.mockito.Mockito.verify; public class HttpImpressionsSenderTest { + private static final TelemetryStorage TELEMETRY_STORAGE = Mockito.mock(InMemoryTelemetryStorage.class); + @Test public void testDefaultURL() throws URISyntaxException { URI rootTarget = URI.create("https://api.split.io"); CloseableHttpClient httpClient = HttpClients.custom().build(); - HttpImpressionsSender fetcher = HttpImpressionsSender.create(httpClient, rootTarget, ImpressionsManager.Mode.DEBUG); + HttpImpressionsSender fetcher = HttpImpressionsSender.create(httpClient, rootTarget, ImpressionsManager.Mode.DEBUG, TELEMETRY_STORAGE); Assert.assertThat(fetcher.getTarget().toString(), Matchers.is(Matchers.equalTo("https://api.split.io/api/testImpressions/bulk"))); } @@ -46,7 +50,7 @@ public void testDefaultURL() throws URISyntaxException { public void testCustomURLNoPathNoBackslash() throws URISyntaxException { URI rootTarget = URI.create("https://kubernetesturl.com"); CloseableHttpClient httpClient = HttpClients.custom().build(); - HttpImpressionsSender fetcher = HttpImpressionsSender.create(httpClient, rootTarget, ImpressionsManager.Mode.DEBUG); + HttpImpressionsSender fetcher = HttpImpressionsSender.create(httpClient, rootTarget, ImpressionsManager.Mode.DEBUG, TELEMETRY_STORAGE); Assert.assertThat(fetcher.getTarget().toString(), Matchers.is(Matchers.equalTo("https://kubernetesturl.com/api/testImpressions/bulk"))); } @@ -54,7 +58,7 @@ public void testCustomURLNoPathNoBackslash() throws URISyntaxException { public void testCustomURLAppendingPath() throws URISyntaxException { URI rootTarget = URI.create("https://kubernetesturl.com/split/"); CloseableHttpClient httpClient = HttpClients.custom().build(); - HttpImpressionsSender fetcher = HttpImpressionsSender.create(httpClient, rootTarget, ImpressionsManager.Mode.DEBUG); + HttpImpressionsSender fetcher = HttpImpressionsSender.create(httpClient, rootTarget, ImpressionsManager.Mode.DEBUG, TELEMETRY_STORAGE); Assert.assertThat(fetcher.getTarget().toString(), Matchers.is(Matchers.equalTo("https://kubernetesturl.com/split/api/testImpressions/bulk"))); } @@ -62,7 +66,7 @@ public void testCustomURLAppendingPath() throws URISyntaxException { public void testCustomURLAppendingPathNoBackslash() throws URISyntaxException { URI rootTarget = URI.create("https://kubernetesturl.com/split"); CloseableHttpClient httpClient = HttpClients.custom().build(); - HttpImpressionsSender fetcher = HttpImpressionsSender.create(httpClient, rootTarget, ImpressionsManager.Mode.DEBUG); + HttpImpressionsSender fetcher = HttpImpressionsSender.create(httpClient, rootTarget, ImpressionsManager.Mode.DEBUG, TELEMETRY_STORAGE); Assert.assertThat(fetcher.getTarget().toString(), Matchers.is(Matchers.equalTo("https://kubernetesturl.com/split/api/testImpressions/bulk"))); } @@ -74,7 +78,7 @@ public void testImpressionCountsEndpointOptimized() throws URISyntaxException, I CloseableHttpClient httpClient = TestHelper.mockHttpClient("", HttpStatus.SC_OK); // Send counters - HttpImpressionsSender sender = HttpImpressionsSender.create(httpClient, rootTarget, ImpressionsManager.Mode.OPTIMIZED); + HttpImpressionsSender sender = HttpImpressionsSender.create(httpClient, rootTarget, ImpressionsManager.Mode.OPTIMIZED, TELEMETRY_STORAGE); HashMap toSend = new HashMap<>(); toSend.put(new ImpressionCounter.Key("test1", 0), 4); toSend.put(new ImpressionCounter.Key("test2", 0), 5); @@ -104,7 +108,7 @@ public void testImpressionCountsEndpointDebug() throws URISyntaxException, IOExc CloseableHttpClient httpClient = TestHelper.mockHttpClient("", HttpStatus.SC_OK); // Send counters - HttpImpressionsSender sender = HttpImpressionsSender.create(httpClient, rootTarget, ImpressionsManager.Mode.DEBUG); + HttpImpressionsSender sender = HttpImpressionsSender.create(httpClient, rootTarget, ImpressionsManager.Mode.DEBUG, TELEMETRY_STORAGE); HashMap toSend = new HashMap<>(); toSend.put(new ImpressionCounter.Key("test1", 0), 4); toSend.put(new ImpressionCounter.Key("test2", 0), 5); @@ -121,7 +125,7 @@ public void testImpressionBulksEndpoint() throws URISyntaxException, IOException // Setup response mock CloseableHttpClient httpClient = TestHelper.mockHttpClient("", HttpStatus.SC_OK); - HttpImpressionsSender sender = HttpImpressionsSender.create(httpClient, rootTarget, ImpressionsManager.Mode.OPTIMIZED); + HttpImpressionsSender sender = HttpImpressionsSender.create(httpClient, rootTarget, ImpressionsManager.Mode.OPTIMIZED, TELEMETRY_STORAGE); // Send impressions List toSend = Arrays.asList(new TestImpressions("t1", Arrays.asList( @@ -152,7 +156,7 @@ public void testImpressionBulksEndpoint() throws URISyntaxException, IOException // Do the same flow for imrpessionsMode = debug CloseableHttpClient httpClientDebugMode = TestHelper.mockHttpClient("", HttpStatus.SC_OK); - sender = HttpImpressionsSender.create(httpClientDebugMode, rootTarget, ImpressionsManager.Mode.DEBUG); + sender = HttpImpressionsSender.create(httpClientDebugMode, rootTarget, ImpressionsManager.Mode.DEBUG, TELEMETRY_STORAGE); sender.postImpressionsBulk(toSend); captor = ArgumentCaptor.forClass(HttpUriRequest.class); verify(httpClientDebugMode).execute(captor.capture()); diff --git a/client/src/test/java/io/split/client/impressions/ImpressionsManagerImplTest.java b/client/src/test/java/io/split/client/impressions/ImpressionsManagerImplTest.java index 4e812254e..aca79a6c5 100644 --- a/client/src/test/java/io/split/client/impressions/ImpressionsManagerImplTest.java +++ b/client/src/test/java/io/split/client/impressions/ImpressionsManagerImplTest.java @@ -4,6 +4,8 @@ import io.split.client.dtos.KeyImpression; import io.split.client.dtos.TestImpressions; +import io.split.telemetry.storage.InMemoryTelemetryStorage; +import io.split.telemetry.storage.TelemetryStorage; import org.junit.Ignore; import org.junit.Test; import org.junit.runner.RunWith; @@ -27,6 +29,7 @@ */ @RunWith(MockitoJUnitRunner.class) public class ImpressionsManagerImplTest { + private static final TelemetryStorage TELEMETRY_STORAGE = Mockito.mock(InMemoryTelemetryStorage.class); @Captor private ArgumentCaptor> impressionsCaptor; @@ -45,7 +48,7 @@ public void works() throws URISyntaxException { ImpressionsSender senderMock = Mockito.mock(ImpressionsSender.class); - ImpressionsManagerImpl treatmentLog = ImpressionsManagerImpl.instanceForTest(null, config, senderMock, null); + ImpressionsManagerImpl treatmentLog = ImpressionsManagerImpl.instanceForTest(null, config, senderMock, null, TELEMETRY_STORAGE); KeyImpression ki1 = keyImpression("test1", "adil", "on", 1L, null); KeyImpression ki2 = keyImpression("test1", "adil", "on", 2L, 1L); @@ -78,7 +81,7 @@ public void worksButDropsImpressions() throws URISyntaxException { ImpressionsSender senderMock = Mockito.mock(ImpressionsSender.class); - ImpressionsManagerImpl treatmentLog = ImpressionsManagerImpl.instanceForTest(null, config, senderMock, null); + ImpressionsManagerImpl treatmentLog = ImpressionsManagerImpl.instanceForTest(null, config, senderMock, null, TELEMETRY_STORAGE); // These 4 unique test name will cause 4 entries but we are caping at the first 3. KeyImpression ki1 = keyImpression("test1", "adil", "on", 1L, null); @@ -112,7 +115,7 @@ public void works4ImpressionsInOneTest() throws URISyntaxException { ImpressionsSender senderMock = Mockito.mock(ImpressionsSender.class); - ImpressionsManagerImpl treatmentLog = ImpressionsManagerImpl.instanceForTest(null, config, senderMock, null); + ImpressionsManagerImpl treatmentLog = ImpressionsManagerImpl.instanceForTest(null, config, senderMock, null, TELEMETRY_STORAGE); // These 4 unique test name will cause 4 entries but we are caping at the first 3. KeyImpression ki1 = keyImpression("test1", "adil", "on", 1L, 1L); @@ -147,7 +150,7 @@ public void worksNoImpressions() throws URISyntaxException { .build(); ImpressionsSender senderMock = Mockito.mock(ImpressionsSender.class); - ImpressionsManagerImpl treatmentLog = ImpressionsManagerImpl.instanceForTest(null, config, senderMock, null); + ImpressionsManagerImpl treatmentLog = ImpressionsManagerImpl.instanceForTest(null, config, senderMock, null, TELEMETRY_STORAGE); // There are no impressions to post. @@ -168,7 +171,7 @@ public void alreadySeenImpressionsAreMarked() throws URISyntaxException { ImpressionsSender senderMock = Mockito.mock(ImpressionsSender.class); - ImpressionsManagerImpl treatmentLog = ImpressionsManagerImpl.instanceForTest(null, config, senderMock, null); + ImpressionsManagerImpl treatmentLog = ImpressionsManagerImpl.instanceForTest(null, config, senderMock, null, TELEMETRY_STORAGE); // These 4 unique test name will cause 4 entries but we are caping at the first 3. KeyImpression ki1 = keyImpression("test1", "adil", "on", 1L, 1L); @@ -229,7 +232,7 @@ public void testImpressionsOptimizedMode() throws URISyntaxException { ImpressionsSender senderMock = Mockito.mock(ImpressionsSender.class); - ImpressionsManagerImpl treatmentLog = ImpressionsManagerImpl.instanceForTest(null, config, senderMock, null); + ImpressionsManagerImpl treatmentLog = ImpressionsManagerImpl.instanceForTest(null, config, senderMock, null, TELEMETRY_STORAGE); // These 4 unique test name will cause 4 entries but we are caping at the first 3. KeyImpression ki1 = keyImpression("test1", "adil", "on", 1L, 1L); diff --git a/client/src/test/java/io/split/engine/experiments/SplitFetcherTest.java b/client/src/test/java/io/split/engine/experiments/SplitFetcherTest.java index 62ed695dd..bfd720216 100644 --- a/client/src/test/java/io/split/engine/experiments/SplitFetcherTest.java +++ b/client/src/test/java/io/split/engine/experiments/SplitFetcherTest.java @@ -14,6 +14,8 @@ import io.split.engine.segments.SegmentSynchronizationTask; import io.split.engine.segments.SegmentSynchronizationTaskImp; import io.split.grammar.Treatments; +import io.split.telemetry.storage.InMemoryTelemetryStorage; +import io.split.telemetry.storage.TelemetryStorage; import org.junit.Test; import org.mockito.Mockito; import org.slf4j.Logger; @@ -40,6 +42,7 @@ */ public class SplitFetcherTest { private static final Logger _log = LoggerFactory.getLogger(SplitFetcherTest.class); + private static final TelemetryStorage TELEMETRY_STORAGE = Mockito.mock(InMemoryTelemetryStorage.class); @Test public void works_when_we_start_without_any_state() throws InterruptedException { @@ -56,9 +59,9 @@ private void works(long startingChangeNumber) throws InterruptedException { SDKReadinessGates gates = new SDKReadinessGates(); SegmentCache segmentCache = new SegmentCacheInMemoryImpl(); SegmentChangeFetcher segmentChangeFetcher = Mockito.mock(SegmentChangeFetcher.class); - SegmentSynchronizationTask segmentSynchronizationTask = new SegmentSynchronizationTaskImp(segmentChangeFetcher,1,10, gates, segmentCache); + SegmentSynchronizationTask segmentSynchronizationTask = new SegmentSynchronizationTaskImp(segmentChangeFetcher,1,10, gates, segmentCache, TELEMETRY_STORAGE); SplitCache cache = new InMemoryCacheImp(startingChangeNumber); - SplitFetcherImp fetcher = new SplitFetcherImp(splitChangeFetcher, new SplitParser(segmentSynchronizationTask, segmentCache), gates, cache); + SplitFetcherImp fetcher = new SplitFetcherImp(splitChangeFetcher, new SplitParser(segmentSynchronizationTask, segmentCache), gates, cache, TELEMETRY_STORAGE); // execute the fetcher for a little bit. executeWaitAndTerminate(fetcher, 1, 3, TimeUnit.SECONDS); @@ -129,10 +132,10 @@ public void when_parser_fails_we_remove_the_experiment() throws InterruptedExcep SegmentCache segmentCache = new SegmentCacheInMemoryImpl(); SegmentChangeFetcher segmentChangeFetcher = mock(SegmentChangeFetcher.class); - SegmentSynchronizationTask segmentSynchronizationTask = new SegmentSynchronizationTaskImp(segmentChangeFetcher, 1,10, gates, segmentCache); + SegmentSynchronizationTask segmentSynchronizationTask = new SegmentSynchronizationTaskImp(segmentChangeFetcher, 1,10, gates, segmentCache, TELEMETRY_STORAGE); segmentSynchronizationTask.startPeriodicFetching(); SplitCache cache = new InMemoryCacheImp(-1); - SplitFetcherImp fetcher = new SplitFetcherImp(splitChangeFetcher, new SplitParser(segmentSynchronizationTask, segmentCache), gates, cache); + SplitFetcherImp fetcher = new SplitFetcherImp(splitChangeFetcher, new SplitParser(segmentSynchronizationTask, segmentCache), gates, cache, TELEMETRY_STORAGE); // execute the fetcher for a little bit. executeWaitAndTerminate(fetcher, 1, 5, TimeUnit.SECONDS); @@ -152,9 +155,9 @@ public void if_there_is_a_problem_talking_to_split_change_count_down_latch_is_no SegmentCache segmentCache = new SegmentCacheInMemoryImpl(); SegmentChangeFetcher segmentChangeFetcher = mock(SegmentChangeFetcher.class); - SegmentSynchronizationTask segmentSynchronizationTask = new SegmentSynchronizationTaskImp(segmentChangeFetcher, 1,10, gates, segmentCache); + SegmentSynchronizationTask segmentSynchronizationTask = new SegmentSynchronizationTaskImp(segmentChangeFetcher, 1,10, gates, segmentCache, TELEMETRY_STORAGE); segmentSynchronizationTask.startPeriodicFetching(); - SplitFetcherImp fetcher = new SplitFetcherImp(splitChangeFetcher, new SplitParser(segmentSynchronizationTask, segmentCache), gates, cache); + SplitFetcherImp fetcher = new SplitFetcherImp(splitChangeFetcher, new SplitParser(segmentSynchronizationTask, segmentCache), gates, cache, TELEMETRY_STORAGE); // execute the fetcher for a little bit. executeWaitAndTerminate(fetcher, 1, 5, TimeUnit.SECONDS); @@ -194,9 +197,9 @@ public void works_with_user_defined_segments() throws Exception { SegmentChangeFetcher segmentChangeFetcher = mock(SegmentChangeFetcher.class); SegmentChange segmentChange = getSegmentChange(0L, 0L, segmentName); when(segmentChangeFetcher.fetch(anyString(), anyLong(), anyBoolean())).thenReturn(segmentChange); - SegmentSynchronizationTask segmentSynchronizationTask = new SegmentSynchronizationTaskImp(segmentChangeFetcher, 1,10, gates, segmentCache); + SegmentSynchronizationTask segmentSynchronizationTask = new SegmentSynchronizationTaskImp(segmentChangeFetcher, 1,10, gates, segmentCache, TELEMETRY_STORAGE); segmentSynchronizationTask.startPeriodicFetching(); - SplitFetcherImp fetcher = new SplitFetcherImp(experimentChangeFetcher, new SplitParser(segmentSynchronizationTask, segmentCache), gates, cache); + SplitFetcherImp fetcher = new SplitFetcherImp(experimentChangeFetcher, new SplitParser(segmentSynchronizationTask, segmentCache), gates, cache, TELEMETRY_STORAGE); // execute the fetcher for a little bit. executeWaitAndTerminate(fetcher, 1, 5, TimeUnit.SECONDS); diff --git a/client/src/test/java/io/split/engine/experiments/SplitParserTest.java b/client/src/test/java/io/split/engine/experiments/SplitParserTest.java index 86beed261..6dcc3d088 100644 --- a/client/src/test/java/io/split/engine/experiments/SplitParserTest.java +++ b/client/src/test/java/io/split/engine/experiments/SplitParserTest.java @@ -24,6 +24,8 @@ import io.split.engine.segments.SegmentSynchronizationTask; import io.split.engine.segments.SegmentSynchronizationTaskImp; import io.split.grammar.Treatments; +import io.split.telemetry.storage.InMemoryTelemetryStorage; +import io.split.telemetry.storage.TelemetryStorage; import org.junit.Test; import org.mockito.Mockito; @@ -48,6 +50,7 @@ public class SplitParserTest { public static final String EMPLOYEES = "employees"; public static final String SALES_PEOPLE = "salespeople"; + private static final TelemetryStorage TELEMETRY_STORAGE = Mockito.mock(InMemoryTelemetryStorage.class); @Test public void works() { @@ -60,7 +63,7 @@ public void works() { SegmentChange segmentChangeSalesPeople = getSegmentChange(-1L, -1L, SALES_PEOPLE); Mockito.when(segmentChangeFetcher.fetch(Mockito.anyString(), Mockito.anyLong(), Mockito.anyBoolean())).thenReturn(segmentChangeEmployee).thenReturn(segmentChangeSalesPeople); - SegmentSynchronizationTask segmentFetcher = new SegmentSynchronizationTaskImp(segmentChangeFetcher,1L, 1, gates, segmentCache); + SegmentSynchronizationTask segmentFetcher = new SegmentSynchronizationTaskImp(segmentChangeFetcher,1L, 1, gates, segmentCache, TELEMETRY_STORAGE); SplitParser parser = new SplitParser(segmentFetcher, segmentCache); @@ -99,7 +102,7 @@ public void worksWithConfig() { SegmentChange segmentChangeSalesPeople = getSegmentChange(-1L, -1L, SALES_PEOPLE); Mockito.when(segmentChangeFetcher.fetch(Mockito.anyString(), Mockito.anyLong(), Mockito.anyBoolean())).thenReturn(segmentChangeEmployee).thenReturn(segmentChangeSalesPeople); - SegmentSynchronizationTask segmentFetcher = new SegmentSynchronizationTaskImp(segmentChangeFetcher,1L, 1, gates, segmentCache); + SegmentSynchronizationTask segmentFetcher = new SegmentSynchronizationTaskImp(segmentChangeFetcher,1L, 1, gates, segmentCache, TELEMETRY_STORAGE); SplitParser parser = new SplitParser(segmentFetcher, segmentCache); @@ -143,7 +146,7 @@ public void works_for_two_conditions() { SegmentChange segmentChangeSalesPeople = getSegmentChange(-1L, -1L, SALES_PEOPLE); Mockito.when(segmentChangeFetcher.fetch(Mockito.anyString(), Mockito.anyLong(), Mockito.anyBoolean())).thenReturn(segmentChangeEmployee).thenReturn(segmentChangeSalesPeople); - SegmentSynchronizationTask segmentFetcher = new SegmentSynchronizationTaskImp(segmentChangeFetcher,1L, 1, gates, segmentCache); + SegmentSynchronizationTask segmentFetcher = new SegmentSynchronizationTaskImp(segmentChangeFetcher,1L, 1, gates, segmentCache, TELEMETRY_STORAGE); SplitParser parser = new SplitParser(segmentFetcher, segmentCache); @@ -182,7 +185,7 @@ public void fails_for_long_conditions() { SegmentChange segmentChangeEmployee = getSegmentChange(-1L, -1L, EMPLOYEES); Mockito.when(segmentChangeFetcher.fetch(Mockito.anyString(), Mockito.anyLong(), Mockito.anyBoolean())).thenReturn(segmentChangeEmployee); - SegmentSynchronizationTask segmentFetcher = new SegmentSynchronizationTaskImp(segmentChangeFetcher,1L, 1, gates, segmentCache); + SegmentSynchronizationTask segmentFetcher = new SegmentSynchronizationTaskImp(segmentChangeFetcher,1L, 1, gates, segmentCache, TELEMETRY_STORAGE); SplitParser parser = new SplitParser(segmentFetcher, segmentCache); @@ -212,7 +215,7 @@ public void works_with_attributes() { SegmentChange segmentChangeSalesPeople = getSegmentChange(-1L, -1L, SALES_PEOPLE); Mockito.when(segmentChangeFetcher.fetch(Mockito.anyString(), Mockito.anyLong(), Mockito.anyBoolean())).thenReturn(segmentChangeEmployee).thenReturn(segmentChangeSalesPeople); - SegmentSynchronizationTask segmentFetcher = new SegmentSynchronizationTaskImp(segmentChangeFetcher,1L, 1, gates, segmentCache); + SegmentSynchronizationTask segmentFetcher = new SegmentSynchronizationTaskImp(segmentChangeFetcher,1L, 1, gates, segmentCache, TELEMETRY_STORAGE); SplitParser parser = new SplitParser(segmentFetcher, segmentCache); @@ -258,7 +261,7 @@ public void less_than_or_equal_to() { SegmentChange segmentChangeSalesPeople = getSegmentChange(-1L, -1L, SALES_PEOPLE); Mockito.when(segmentChangeFetcher.fetch(Mockito.anyString(), Mockito.anyLong(), Mockito.anyBoolean())).thenReturn(segmentChangeEmployee).thenReturn(segmentChangeSalesPeople); - SegmentSynchronizationTask segmentFetcher = new SegmentSynchronizationTaskImp(segmentChangeFetcher,1L, 1, gates, segmentCache); + SegmentSynchronizationTask segmentFetcher = new SegmentSynchronizationTaskImp(segmentChangeFetcher,1L, 1, gates, segmentCache, TELEMETRY_STORAGE); SplitParser parser = new SplitParser(segmentFetcher, segmentCache); @@ -297,7 +300,7 @@ public void equal_to() { SegmentChange segmentChangeSalesPeople = getSegmentChange(-1L, -1L, SALES_PEOPLE); Mockito.when(segmentChangeFetcher.fetch(Mockito.anyString(), Mockito.anyLong(), Mockito.anyBoolean())).thenReturn(segmentChangeEmployee).thenReturn(segmentChangeSalesPeople); - SegmentSynchronizationTask segmentFetcher = new SegmentSynchronizationTaskImp(segmentChangeFetcher,1L, 1, gates, segmentCache); + SegmentSynchronizationTask segmentFetcher = new SegmentSynchronizationTaskImp(segmentChangeFetcher,1L, 1, gates, segmentCache, TELEMETRY_STORAGE); SplitParser parser = new SplitParser(segmentFetcher, segmentCache); @@ -336,7 +339,7 @@ public void equal_to_negative_number() { SegmentChange segmentChangeSalesPeople = getSegmentChange(-1L, -1L, SALES_PEOPLE); Mockito.when(segmentChangeFetcher.fetch(Mockito.anyString(), Mockito.anyLong(), Mockito.anyBoolean())).thenReturn(segmentChangeEmployee).thenReturn(segmentChangeSalesPeople); - SegmentSynchronizationTask segmentFetcher = new SegmentSynchronizationTaskImp(segmentChangeFetcher,1L, 1, gates, segmentCache); + SegmentSynchronizationTask segmentFetcher = new SegmentSynchronizationTaskImp(segmentChangeFetcher,1L, 1, gates, segmentCache, TELEMETRY_STORAGE); SplitParser parser = new SplitParser(segmentFetcher, segmentCache); @@ -375,7 +378,7 @@ public void between() { SegmentChange segmentChangeSalesPeople = getSegmentChange(-1L, -1L, SALES_PEOPLE); Mockito.when(segmentChangeFetcher.fetch(Mockito.anyString(), Mockito.anyLong(), Mockito.anyBoolean())).thenReturn(segmentChangeEmployee).thenReturn(segmentChangeSalesPeople); - SegmentSynchronizationTask segmentFetcher = new SegmentSynchronizationTaskImp(segmentChangeFetcher,1L, 1, gates, segmentCache); + SegmentSynchronizationTask segmentFetcher = new SegmentSynchronizationTaskImp(segmentChangeFetcher,1L, 1, gates, segmentCache, TELEMETRY_STORAGE); SplitParser parser = new SplitParser(segmentFetcher, segmentCache); @@ -552,7 +555,7 @@ public void set_matcher_test(Condition c, io.split.engine.matchers.Matcher m) { SegmentChange segmentChangeSalesPeople = getSegmentChange(-1L, -1L, SALES_PEOPLE); Mockito.when(segmentChangeFetcher.fetch(Mockito.anyString(), Mockito.anyLong(), Mockito.anyBoolean())).thenReturn(segmentChangeEmployee).thenReturn(segmentChangeSalesPeople); - SegmentSynchronizationTask segmentFetcher = new SegmentSynchronizationTaskImp(segmentChangeFetcher,1L, 1, gates, segmentCache); + SegmentSynchronizationTask segmentFetcher = new SegmentSynchronizationTaskImp(segmentChangeFetcher,1L, 1, gates, segmentCache, TELEMETRY_STORAGE); SplitParser parser = new SplitParser(segmentFetcher, segmentCache); diff --git a/client/src/test/java/io/split/engine/segments/SegmentFetcherImpTest.java b/client/src/test/java/io/split/engine/segments/SegmentFetcherImpTest.java index 79786bdd6..d41b9c829 100644 --- a/client/src/test/java/io/split/engine/segments/SegmentFetcherImpTest.java +++ b/client/src/test/java/io/split/engine/segments/SegmentFetcherImpTest.java @@ -5,6 +5,8 @@ import io.split.cache.SegmentCacheInMemoryImpl; import io.split.client.dtos.SegmentChange; import io.split.engine.SDKReadinessGates; +import io.split.telemetry.storage.InMemoryTelemetryStorage; +import io.split.telemetry.storage.TelemetryStorage; import org.junit.Test; import org.mockito.Mockito; import org.slf4j.Logger; @@ -30,6 +32,7 @@ public class SegmentFetcherImpTest { private static final Logger _log = LoggerFactory.getLogger(SegmentFetcherImpTest.class); private static final String SEGMENT_NAME = "foo"; + private static final TelemetryStorage TELEMETRY_STORAGE = Mockito.mock(InMemoryTelemetryStorage.class); @Test public void works_when_we_start_without_state() throws InterruptedException { @@ -53,7 +56,7 @@ public void works_when_there_are_no_changes() throws InterruptedException { SegmentChange segmentChange = getSegmentChange(-1L, 10L); Mockito.when(segmentChangeFetcher.fetch(Mockito.anyString(), Mockito.anyLong(), Mockito.anyBoolean())).thenReturn(segmentChange); - SegmentFetcherImp fetcher = new SegmentFetcherImp(SEGMENT_NAME, segmentChangeFetcher, gates, segmentCache); + SegmentFetcherImp fetcher = new SegmentFetcherImp(SEGMENT_NAME, segmentChangeFetcher, gates, segmentCache, TELEMETRY_STORAGE); // execute the fetcher for a little bit. ScheduledExecutorService scheduledExecutorService = Executors.newSingleThreadScheduledExecutor(); @@ -94,7 +97,7 @@ private void works(long startingChangeNumber) throws InterruptedException { Mockito.when(segmentChangeFetcher.fetch(SEGMENT_NAME, -1L, false)).thenReturn(segmentChange); Mockito.when(segmentChangeFetcher.fetch(SEGMENT_NAME, 0L, false)).thenReturn(segmentChange); - SegmentFetcher fetcher = new SegmentFetcherImp(segmentName, segmentChangeFetcher, gates, segmentCache); + SegmentFetcher fetcher = new SegmentFetcherImp(segmentName, segmentChangeFetcher, gates, segmentCache, TELEMETRY_STORAGE); // execute the fetcher for a little bit. ScheduledExecutorService scheduledExecutorService = Executors.newSingleThreadScheduledExecutor(); @@ -121,21 +124,21 @@ private void works(long startingChangeNumber) throws InterruptedException { @Test(expected = NullPointerException.class) public void does_not_work_if_segment_change_fetcher_is_null() { SegmentCache segmentCache = Mockito.mock(SegmentCache.class); - SegmentFetcher fetcher = new SegmentFetcherImp(SEGMENT_NAME, null, new SDKReadinessGates(), segmentCache); + SegmentFetcher fetcher = new SegmentFetcherImp(SEGMENT_NAME, null, new SDKReadinessGates(), segmentCache, TELEMETRY_STORAGE); } @Test(expected = NullPointerException.class) public void does_not_work_if_segment_name_is_null() { SegmentCache segmentCache = Mockito.mock(SegmentCache.class); SegmentChangeFetcher segmentChangeFetcher = Mockito.mock(SegmentChangeFetcher.class); - SegmentFetcher fetcher = new SegmentFetcherImp(null, segmentChangeFetcher, new SDKReadinessGates(), segmentCache); + SegmentFetcher fetcher = new SegmentFetcherImp(null, segmentChangeFetcher, new SDKReadinessGates(), segmentCache, TELEMETRY_STORAGE); } @Test(expected = NullPointerException.class) public void does_not_work_if_sdk_readiness_gates_are_null() { SegmentCache segmentCache = Mockito.mock(SegmentCache.class); SegmentChangeFetcher segmentChangeFetcher = Mockito.mock(SegmentChangeFetcher.class); - SegmentFetcher fetcher = new SegmentFetcherImp(SEGMENT_NAME, segmentChangeFetcher, null, segmentCache); + SegmentFetcher fetcher = new SegmentFetcherImp(SEGMENT_NAME, segmentChangeFetcher, null, segmentCache, TELEMETRY_STORAGE); } private SegmentChange getSegmentChange(long since, long till){ diff --git a/client/src/test/java/io/split/engine/segments/SegmentSynchronizationTaskImpTest.java b/client/src/test/java/io/split/engine/segments/SegmentSynchronizationTaskImpTest.java index b856c1d78..722458e3c 100644 --- a/client/src/test/java/io/split/engine/segments/SegmentSynchronizationTaskImpTest.java +++ b/client/src/test/java/io/split/engine/segments/SegmentSynchronizationTaskImpTest.java @@ -2,6 +2,8 @@ import io.split.engine.SDKReadinessGates; import io.split.cache.SegmentCache; +import io.split.telemetry.storage.InMemoryTelemetryStorage; +import io.split.telemetry.storage.TelemetryStorage; import org.junit.Before; import org.junit.Test; import org.mockito.Mockito; @@ -24,6 +26,7 @@ */ public class SegmentSynchronizationTaskImpTest { private static final Logger _log = LoggerFactory.getLogger(SegmentSynchronizationTaskImpTest.class); + private static final TelemetryStorage TELEMETRY_STORAGE = Mockito.mock(InMemoryTelemetryStorage.class); private AtomicReference fetcher1 = null; private AtomicReference fetcher2 = null; @@ -40,7 +43,7 @@ public void works() { SegmentCache segmentCache = Mockito.mock(SegmentCache.class); SegmentChangeFetcher segmentChangeFetcher = Mockito.mock(SegmentChangeFetcher.class); - final SegmentSynchronizationTaskImp fetchers = new SegmentSynchronizationTaskImp(segmentChangeFetcher, 1L, 1, gates, segmentCache); + final SegmentSynchronizationTaskImp fetchers = new SegmentSynchronizationTaskImp(segmentChangeFetcher, 1L, 1, gates, segmentCache, TELEMETRY_STORAGE); // create two tasks that will separately call segment and make sure diff --git a/client/src/test/java/io/split/engine/sse/AuthApiClientTest.java b/client/src/test/java/io/split/engine/sse/AuthApiClientTest.java index 2debfe68c..f90a45d80 100644 --- a/client/src/test/java/io/split/engine/sse/AuthApiClientTest.java +++ b/client/src/test/java/io/split/engine/sse/AuthApiClientTest.java @@ -2,21 +2,25 @@ import io.split.TestHelper; import io.split.engine.sse.dtos.AuthenticationResponse; +import io.split.telemetry.storage.InMemoryTelemetryStorage; +import io.split.telemetry.storage.TelemetryStorage; import org.apache.commons.lang3.StringUtils; import org.apache.hc.client5.http.impl.classic.CloseableHttpClient; import org.apache.hc.core5.http.HttpStatus; import org.junit.Assert; import org.junit.Test; +import org.mockito.Mockito; import java.io.IOException; import java.lang.reflect.InvocationTargetException; public class AuthApiClientTest { + private static final TelemetryStorage TELEMETRY_STORAGE = Mockito.mock(InMemoryTelemetryStorage.class); @Test public void authenticateWithPushEnabledShouldReturnSuccess() throws IOException, IllegalAccessException, NoSuchMethodException, InvocationTargetException { CloseableHttpClient httpClientMock = TestHelper.mockHttpClient("streaming-auth-push-enabled.json", HttpStatus.SC_OK); - AuthApiClient authApiClient = new AuthApiClientImp( "www.split-test.io", httpClientMock); + AuthApiClient authApiClient = new AuthApiClientImp( "www.split-test.io", httpClientMock, TELEMETRY_STORAGE); AuthenticationResponse result = authApiClient.Authenticate(); Assert.assertTrue(result.isPushEnabled()); @@ -31,7 +35,7 @@ public void authenticateWithPushEnabledShouldReturnSuccess() throws IOException, public void authenticateWithPushEnabledWithWrongTokenShouldReturnError() throws IOException, IllegalAccessException, NoSuchMethodException, InvocationTargetException { CloseableHttpClient httpClientMock = TestHelper.mockHttpClient("streaming-auth-push-enabled-wrong-token.json", HttpStatus.SC_OK); - AuthApiClient authApiClient = new AuthApiClientImp( "www.split-test.io", httpClientMock); + AuthApiClient authApiClient = new AuthApiClientImp( "www.split-test.io", httpClientMock, TELEMETRY_STORAGE); AuthenticationResponse result = authApiClient.Authenticate(); Assert.assertFalse(result.isPushEnabled()); @@ -45,7 +49,7 @@ public void authenticateWithPushEnabledWithWrongTokenShouldReturnError() throws public void authenticateWithPushDisabledShouldReturnSuccess() throws IOException, IllegalAccessException, NoSuchMethodException, InvocationTargetException { CloseableHttpClient httpClientMock = TestHelper.mockHttpClient("streaming-auth-push-disabled.json", HttpStatus.SC_OK); - AuthApiClient authApiClient = new AuthApiClientImp("www.split-test.io", httpClientMock); + AuthApiClient authApiClient = new AuthApiClientImp("www.split-test.io", httpClientMock, TELEMETRY_STORAGE); AuthenticationResponse result = authApiClient.Authenticate(); Assert.assertFalse(result.isPushEnabled()); @@ -58,7 +62,7 @@ public void authenticateWithPushDisabledShouldReturnSuccess() throws IOException public void authenticateServerErrorShouldReturnErrorWithRetry() throws IOException, IllegalAccessException, NoSuchMethodException, InvocationTargetException { CloseableHttpClient httpClientMock = TestHelper.mockHttpClient("", HttpStatus.SC_INTERNAL_SERVER_ERROR); - AuthApiClient authApiClient = new AuthApiClientImp("www.split-test.io", httpClientMock); + AuthApiClient authApiClient = new AuthApiClientImp("www.split-test.io", httpClientMock, TELEMETRY_STORAGE); AuthenticationResponse result = authApiClient.Authenticate(); Assert.assertFalse(result.isPushEnabled()); @@ -71,7 +75,7 @@ public void authenticateServerErrorShouldReturnErrorWithRetry() throws IOExcepti public void authenticateServerBadRequestShouldReturnErrorWithoutRetry() throws IOException, IllegalAccessException, NoSuchMethodException, InvocationTargetException { CloseableHttpClient httpClientMock = TestHelper.mockHttpClient("", HttpStatus.SC_BAD_REQUEST); - AuthApiClient authApiClient = new AuthApiClientImp("www.split-test.io", httpClientMock); + AuthApiClient authApiClient = new AuthApiClientImp("www.split-test.io", httpClientMock, TELEMETRY_STORAGE); AuthenticationResponse result = authApiClient.Authenticate(); Assert.assertFalse(result.isPushEnabled()); @@ -84,7 +88,7 @@ public void authenticateServerBadRequestShouldReturnErrorWithoutRetry() throws I public void authenticateServerUnauthorizedShouldReturnErrorWithoutRetry() throws IOException, IllegalAccessException, NoSuchMethodException, InvocationTargetException { CloseableHttpClient httpClientMock = TestHelper.mockHttpClient("", HttpStatus.SC_UNAUTHORIZED); - AuthApiClient authApiClient = new AuthApiClientImp("www.split-test.io", httpClientMock); + AuthApiClient authApiClient = new AuthApiClientImp("www.split-test.io", httpClientMock, TELEMETRY_STORAGE); AuthenticationResponse result = authApiClient.Authenticate(); Assert.assertFalse(result.isPushEnabled()); diff --git a/client/src/test/java/io/split/telemetry/synchronizer/TelemetrySyncTaskTest.java b/client/src/test/java/io/split/telemetry/synchronizer/TelemetrySyncTaskTest.java index 9127b3a3d..5058d3b5c 100644 --- a/client/src/test/java/io/split/telemetry/synchronizer/TelemetrySyncTaskTest.java +++ b/client/src/test/java/io/split/telemetry/synchronizer/TelemetrySyncTaskTest.java @@ -9,7 +9,6 @@ public class TelemetrySyncTaskTest { public void testSynchronizationTask() throws Exception { TelemetrySynchronizer telemetrySynchronizer = Mockito.mock(SynchronizerMemory.class); TelemetrySyncTask telemetrySyncTask = new TelemetrySyncTask(1, telemetrySynchronizer); - telemetrySyncTask.startScheduledTask(); Thread.sleep(3000); Mockito.verify(telemetrySynchronizer, Mockito.times(3)).synchronizeStats(); } @@ -18,7 +17,6 @@ public void testSynchronizationTask() throws Exception { public void testStopSynchronizationTask() throws Exception { TelemetrySynchronizer telemetrySynchronizer = Mockito.mock(SynchronizerMemory.class); TelemetrySyncTask telemetrySyncTask = new TelemetrySyncTask(1, telemetrySynchronizer); - telemetrySyncTask.startScheduledTask(); Thread.sleep(3000); Mockito.verify(telemetrySynchronizer, Mockito.times(3)).synchronizeStats(); telemetrySyncTask.stopScheduledTask(); From ca11660cbb0d7d2eefc787f13c6c312eb8ddb8de Mon Sep 17 00:00:00 2001 From: Lucas Echeverz Date: Thu, 6 May 2021 16:14:47 -0300 Subject: [PATCH 41/81] Files forgotten --- .../java/io/split/client/SplitFactoryImpl.java | 6 +----- .../telemetry/utils/AtomicLongArrayTest.java | 17 +++++++++-------- 2 files changed, 10 insertions(+), 13 deletions(-) diff --git a/client/src/main/java/io/split/client/SplitFactoryImpl.java b/client/src/main/java/io/split/client/SplitFactoryImpl.java index a8f4393ca..ca54a0bdd 100644 --- a/client/src/main/java/io/split/client/SplitFactoryImpl.java +++ b/client/src/main/java/io/split/client/SplitFactoryImpl.java @@ -72,7 +72,6 @@ public class SplitFactoryImpl implements SplitFactory { private final URI _eventsRootTarget; private final CloseableHttpClient _httpclient; private final SDKReadinessGates _gates; - private final HttpMetrics _httpMetrics; private final SegmentSynchronizationTaskImp _segmentSynchronizationTaskImp; private final SplitFetcher _splitFetcher; private final SplitSynchronizationTask _splitSynchronizationTask; @@ -122,9 +121,6 @@ public SplitFactoryImpl(String apiToken, SplitClientConfig config) throws URISyn _rootTarget = URI.create(config.endpoint()); _eventsRootTarget = URI.create(config.eventsEndpoint()); - // HttpMetrics - _httpMetrics = HttpMetrics.create(_httpclient, _eventsRootTarget); - // Cache Initialisations _segmentCache = new SegmentCacheInMemoryImpl(); _splitCache = new InMemoryCacheImp(); @@ -147,7 +143,7 @@ public SplitFactoryImpl(String apiToken, SplitClientConfig config) throws URISyn _eventClient = EventClientImpl.create(_httpclient, _eventsRootTarget, config.eventsQueueSize(), config.eventFlushIntervalInMillis(), config.waitBeforeShutdown(), _telemetryStorage); // SyncManager - _syncManager = SyncManagerImp.build(config.streamingEnabled(), _splitSynchronizationTask, _splitFetcher, _segmentSynchronizationTaskImp, _splitCache, config.authServiceURL(), _httpclient, config.streamingServiceURL(), config.authRetryBackoffBase(), buildSSEdHttpClient(apiToken, config), _segmentCache, config.streamingRetryDelay(), _gates, _telemetryStorage)); + _syncManager = SyncManagerImp.build(config.streamingEnabled(), _splitSynchronizationTask, _splitFetcher, _segmentSynchronizationTaskImp, _splitCache, config.authServiceURL(), _httpclient, config.streamingServiceURL(), config.authRetryBackoffBase(), buildSSEdHttpClient(apiToken, config), _segmentCache, config.streamingRetryDelay(), _gates, _telemetryStorage); _syncManager.start(); // Evaluator diff --git a/client/src/test/java/io/split/telemetry/utils/AtomicLongArrayTest.java b/client/src/test/java/io/split/telemetry/utils/AtomicLongArrayTest.java index 2dce713b0..bbee9b79a 100644 --- a/client/src/test/java/io/split/telemetry/utils/AtomicLongArrayTest.java +++ b/client/src/test/java/io/split/telemetry/utils/AtomicLongArrayTest.java @@ -2,35 +2,36 @@ import org.junit.Assert; import org.junit.Test; +import org.mockito.Mockito; +import org.slf4j.Logger; public class AtomicLongArrayTest { private static final int SIZE = 23; @Test - public void testAtomicLong() throws Exception { + public void testAtomicLong() { AtomicLongArray atomicLongArray = new AtomicLongArray(SIZE); Assert.assertNotNull(atomicLongArray); } @Test public void testArraySizeError() { - Exception exception = Assert.assertThrows(Exception.class, () -> { - AtomicLongArray atomicLongArray = new AtomicLongArray(0); - }); - String messageExpected = "Invalid array size"; - Assert.assertEquals(messageExpected, exception.getMessage()); + AtomicLongArray atomicLongArray = new AtomicLongArray(0); + Logger log = Mockito.mock(Logger.class); + atomicLongArray.increment(2); + Assert.assertEquals(1, atomicLongArray.fetchAndClearAll().stream().mapToInt(Long::intValue).sum()); } @Test - public void testIncrement() throws Exception { + public void testIncrement() { AtomicLongArray atomicLongArray = new AtomicLongArray(SIZE); atomicLongArray.increment(2); Assert.assertEquals(1, atomicLongArray.fetchAndClearAll().stream().mapToInt(Long::intValue).sum()); } @Test(expected = ArrayIndexOutOfBoundsException.class) - public void testIncrementError() throws Exception { + public void testIncrementError() { AtomicLongArray atomicLongArray = new AtomicLongArray(SIZE); atomicLongArray.increment(25); } From 8559abaf96ca65a5c9e33ab9666aa120bbcb40af Mon Sep 17 00:00:00 2001 From: Lucas Echeverz Date: Thu, 6 May 2021 18:14:54 -0300 Subject: [PATCH 42/81] PR comments --- .../java/io/split/client/EventClientImpl.java | 10 ++++------ .../io/split/client/HttpSegmentChangeFetcher.java | 10 +++++----- .../io/split/client/HttpSplitChangeFetcher.java | 6 +++--- .../java/io/split/client/SplitClientConfig.java | 2 +- .../java/io/split/client/SplitClientImpl.java | 3 +-- .../java/io/split/client/SplitFactoryImpl.java | 3 +-- .../client/impressions/HttpImpressionsSender.java | 15 +++++++-------- .../impressions/ImpressionsManagerImpl.java | 2 +- .../io/split/engine/sse/AuthApiClientImp.java | 5 ++--- 9 files changed, 25 insertions(+), 31 deletions(-) diff --git a/client/src/main/java/io/split/client/EventClientImpl.java b/client/src/main/java/io/split/client/EventClientImpl.java index 50c95b1c6..a95613a34 100644 --- a/client/src/main/java/io/split/client/EventClientImpl.java +++ b/client/src/main/java/io/split/client/EventClientImpl.java @@ -89,7 +89,7 @@ public static EventClientImpl create(CloseableHttpClient httpclient, URI eventsR _eventQueue = eventQueue; _waitBeforeShutdown = waitBeforeShutdown; - _maxQueueSize = 1; + _maxQueueSize = maxQueueSize; _flushIntervalMillis = flushIntervalMillis; _telemetryRuntimeProducer = checkNotNull(telemetryRuntimeProducer); @@ -131,8 +131,7 @@ public boolean track(Event event, int eventSize) { if (event == null) { return false; } - WrappedEvent we = new WrappedEvent(event, eventSize); - _eventQueue.put(we); + _eventQueue.put(new WrappedEvent(event, eventSize)); _telemetryRuntimeProducer.recordEventStats(EventsDataRecordsEnum.EVENTS_QUEUED, 1); } catch (ClassCastException | NullPointerException | InterruptedException e) { @@ -195,9 +194,8 @@ public void run() { // Clear the queue of events for the next batch. events = new ArrayList<>(); accumulated = 0; - long endTime = System.currentTimeMillis(); - _telemetryRuntimeProducer.recordSyncLatency(HTTPLatenciesEnum.EVENTS, endTime-initTime); - _telemetryRuntimeProducer.recordSuccessfulSync(LastSynchronizationRecordsEnum.EVENTS, endTime); + _telemetryRuntimeProducer.recordSyncLatency(HTTPLatenciesEnum.EVENTS, System.currentTimeMillis()-initTime); + _telemetryRuntimeProducer.recordSuccessfulSync(LastSynchronizationRecordsEnum.EVENTS, System.currentTimeMillis()); } } } catch (InterruptedException e) { diff --git a/client/src/main/java/io/split/client/HttpSegmentChangeFetcher.java b/client/src/main/java/io/split/client/HttpSegmentChangeFetcher.java index 58e8e6eb0..868e9c1ad 100644 --- a/client/src/main/java/io/split/client/HttpSegmentChangeFetcher.java +++ b/client/src/main/java/io/split/client/HttpSegmentChangeFetcher.java @@ -13,6 +13,7 @@ import org.apache.hc.client5.http.classic.methods.HttpGet; import org.apache.hc.client5.http.impl.classic.CloseableHttpClient; import org.apache.hc.client5.http.impl.classic.CloseableHttpResponse; +import org.apache.hc.core5.http.HttpStatus; import org.apache.hc.core5.http.io.entity.EntityUtils; import org.apache.hc.core5.net.URIBuilder; import org.slf4j.Logger; @@ -67,19 +68,18 @@ public SegmentChange fetch(String segmentName, long since, boolean addCacheHeade int statusCode = response.getCode(); - if (statusCode < 200 || statusCode >= 300) { + if (statusCode < HttpStatus.SC_OK || statusCode >= HttpStatus.SC_MULTIPLE_CHOICES) { _telemetryRuntimeProducer.recordSyncError(ResourceEnum.SEGMENT_SYNC, statusCode); _log.error("Response status was: " + statusCode); - if (statusCode == 403) { + if (statusCode == HttpStatus.SC_FORBIDDEN) { _log.error("factory instantiation: you passed a browser type api_key, " + "please grab an api key from the Split console that is of type sdk"); } throw new IllegalStateException("Could not retrieve segment changes for " + segmentName + "; http return code " + statusCode); } - long endTime = System.currentTimeMillis(); - _telemetryRuntimeProducer.recordSyncLatency(HTTPLatenciesEnum.SEGMENTS, endTime-start); - _telemetryRuntimeProducer.recordSuccessfulSync(LastSynchronizationRecordsEnum.SEGMENTS, endTime); + _telemetryRuntimeProducer.recordSyncLatency(HTTPLatenciesEnum.SEGMENTS, System.currentTimeMillis()-start); + _telemetryRuntimeProducer.recordSuccessfulSync(LastSynchronizationRecordsEnum.SEGMENTS, System.currentTimeMillis()); String json = EntityUtils.toString(response.getEntity(), StandardCharsets.UTF_8); if (_log.isDebugEnabled()) { diff --git a/client/src/main/java/io/split/client/HttpSplitChangeFetcher.java b/client/src/main/java/io/split/client/HttpSplitChangeFetcher.java index 0f13656b4..2b45d7f40 100644 --- a/client/src/main/java/io/split/client/HttpSplitChangeFetcher.java +++ b/client/src/main/java/io/split/client/HttpSplitChangeFetcher.java @@ -13,6 +13,7 @@ import org.apache.hc.client5.http.classic.methods.HttpGet; import org.apache.hc.client5.http.impl.classic.CloseableHttpClient; import org.apache.hc.client5.http.impl.classic.CloseableHttpResponse; +import org.apache.hc.core5.http.HttpStatus; import org.apache.hc.core5.http.io.entity.EntityUtils; import org.apache.hc.core5.net.URIBuilder; import org.slf4j.Logger; @@ -67,12 +68,11 @@ public SplitChange fetch(long since, boolean addCacheHeader) { int statusCode = response.getCode(); - if (statusCode < 200 || statusCode >= 300) { + if (statusCode < HttpStatus.SC_OK || statusCode >= HttpStatus.SC_MULTIPLE_CHOICES) { _telemetryRuntimeProducer.recordSyncError(ResourceEnum.SPLIT_SYNC, statusCode); throw new IllegalStateException("Could not retrieve splitChanges; http return code " + statusCode); } - long endtime = System.currentTimeMillis(); - _telemetryRuntimeProducer.recordSyncLatency(HTTPLatenciesEnum.SPLITS, endtime-start); + _telemetryRuntimeProducer.recordSyncLatency(HTTPLatenciesEnum.SPLITS, System.currentTimeMillis()-start); String json = EntityUtils.toString(response.getEntity(), StandardCharsets.UTF_8); diff --git a/client/src/main/java/io/split/client/SplitClientConfig.java b/client/src/main/java/io/split/client/SplitClientConfig.java index 7761af6fa..a98efdc1f 100644 --- a/client/src/main/java/io/split/client/SplitClientConfig.java +++ b/client/src/main/java/io/split/client/SplitClientConfig.java @@ -21,7 +21,7 @@ public class SplitClientConfig { public static final String EVENTS_ENDPOINT = "https://events.split.io"; public static final String AUTH_ENDPOINT = "https://auth.split.io/api/auth"; public static final String STREAMING_ENDPOINT = "https://streaming.split.io/sse"; - public static final String TELEMETRY_ENDPOINT = "https://telemetry.split-stage.io/api/v1/"; + public static final String TELEMETRY_ENDPOINT = "https://telemetry.split.io/api/v1"; private final String _endpoint; private final String _eventsEndpoint; diff --git a/client/src/main/java/io/split/client/SplitClientImpl.java b/client/src/main/java/io/split/client/SplitClientImpl.java index 83add8aa4..050acb516 100644 --- a/client/src/main/java/io/split/client/SplitClientImpl.java +++ b/client/src/main/java/io/split/client/SplitClientImpl.java @@ -232,8 +232,7 @@ private SplitResult getTreatmentWithConfigInternal(String method, String matchin result.changeNumber, attributes ); - long endTime = System.currentTimeMillis(); - _telemetryEvaluationProducer.recordLatency(methodEnum, endTime-initTime); + _telemetryEvaluationProducer.recordLatency(methodEnum, System.currentTimeMillis()-initTime); return new SplitResult(result.treatment, result.configurations); } catch (Exception e) { try { diff --git a/client/src/main/java/io/split/client/SplitFactoryImpl.java b/client/src/main/java/io/split/client/SplitFactoryImpl.java index ca54a0bdd..db27fc624 100644 --- a/client/src/main/java/io/split/client/SplitFactoryImpl.java +++ b/client/src/main/java/io/split/client/SplitFactoryImpl.java @@ -191,8 +191,7 @@ public synchronized void destroy() { _log.info("Successful shutdown of eventClient"); _syncManager.shutdown(); _log.info("Successful shutdown of syncManager"); - long endSession = System.currentTimeMillis(); - _telemetryStorage.recordSessionLength(endSession - _startTime); + _telemetryStorage.recordSessionLength(System.currentTimeMillis() - _startTime); _telemetrySyncTask.stopScheduledTask(); } catch (IOException e) { _log.error("We could not shutdown split", e); diff --git a/client/src/main/java/io/split/client/impressions/HttpImpressionsSender.java b/client/src/main/java/io/split/client/impressions/HttpImpressionsSender.java index 5e0c5efb0..8030723a5 100644 --- a/client/src/main/java/io/split/client/impressions/HttpImpressionsSender.java +++ b/client/src/main/java/io/split/client/impressions/HttpImpressionsSender.java @@ -13,6 +13,7 @@ import org.apache.hc.client5.http.impl.classic.CloseableHttpClient; import org.apache.hc.client5.http.impl.classic.CloseableHttpResponse; import org.apache.hc.core5.http.HttpEntity; +import org.apache.hc.core5.http.HttpStatus; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -73,13 +74,12 @@ public void postImpressionsBulk(List impressions) { int status = response.getCode(); - if (status < 200 || status >= 300) { + if (status < HttpStatus.SC_OK || status >= HttpStatus.SC_MULTIPLE_CHOICES) { _telemetryRuntimeProducer.recordSyncError(ResourceEnum.IMPRESSION_SYNC, status); _logger.warn("Response status was: " + status); } - long endTime = System.currentTimeMillis(); - _telemetryRuntimeProducer.recordSyncLatency(HTTPLatenciesEnum.IMPRESSIONS, endTime - initTime); - _telemetryRuntimeProducer.recordSuccessfulSync(LastSynchronizationRecordsEnum.IMPRESSIONS, endTime); + _telemetryRuntimeProducer.recordSyncLatency(HTTPLatenciesEnum.IMPRESSIONS, System.currentTimeMillis() - initTime); + _telemetryRuntimeProducer.recordSuccessfulSync(LastSynchronizationRecordsEnum.IMPRESSIONS, System.currentTimeMillis()); } catch (Throwable t) { _logger.warn("Exception when posting impressions" + impressions, t); @@ -101,13 +101,12 @@ public void postCounters(HashMap raw) { request.setEntity(Utils.toJsonEntity(ImpressionCount.fromImpressionCounterData(raw))); try (CloseableHttpResponse response = _client.execute(request)) { int status = response.getCode(); - if (status < 200 || status >= 300) { + if (status < HttpStatus.SC_OK || status >= HttpStatus.SC_MULTIPLE_CHOICES) { _telemetryRuntimeProducer.recordSyncError(ResourceEnum.IMPRESSION_COUNT_SYNC, status); _logger.warn("Response status was: " + status); } - long endTime = System.currentTimeMillis(); - _telemetryRuntimeProducer.recordSyncLatency(HTTPLatenciesEnum.IMPRESSIONS_COUNT, endTime - initTime); - _telemetryRuntimeProducer.recordSuccessfulSync(LastSynchronizationRecordsEnum.IMPRESSIONS_COUNT, endTime); + _telemetryRuntimeProducer.recordSyncLatency(HTTPLatenciesEnum.IMPRESSIONS_COUNT, System.currentTimeMillis() - initTime); + _telemetryRuntimeProducer.recordSuccessfulSync(LastSynchronizationRecordsEnum.IMPRESSIONS_COUNT, System.currentTimeMillis()); } catch (IOException exc) { _logger.warn("Exception when posting impression counters: ", exc); } diff --git a/client/src/main/java/io/split/client/impressions/ImpressionsManagerImpl.java b/client/src/main/java/io/split/client/impressions/ImpressionsManagerImpl.java index f5a5343a7..708592f94 100644 --- a/client/src/main/java/io/split/client/impressions/ImpressionsManagerImpl.java +++ b/client/src/main/java/io/split/client/impressions/ImpressionsManagerImpl.java @@ -105,7 +105,7 @@ public void track(Impression impression) { } if (Mode.DEBUG.equals(_mode) || shouldQueueImpression(impression)) { - if (shouldQueueImpression(impression)) { + if (!shouldQueueImpression(impression)) { _telemetryRuntimeProducer.recordImpressionStats(ImpressionsDataTypeEnum.IMPRESSIONS_DEDUPED, 1); } if (_storage.put(KeyImpression.fromImpression(impression))) { diff --git a/client/src/main/java/io/split/engine/sse/AuthApiClientImp.java b/client/src/main/java/io/split/engine/sse/AuthApiClientImp.java index da1ff89dc..6b4971cc4 100644 --- a/client/src/main/java/io/split/engine/sse/AuthApiClientImp.java +++ b/client/src/main/java/io/split/engine/sse/AuthApiClientImp.java @@ -48,10 +48,9 @@ public AuthenticationResponse Authenticate() { _log.debug(String.format("Success connection to: %s", _target)); String jsonContent = EntityUtils.toString(response.getEntity(), StandardCharsets.UTF_8); - long endTime = System.currentTimeMillis(); _telemetryRuntimeProducer.recordTokenRefreshes(); - _telemetryRuntimeProducer.recordSuccessfulSync(LastSynchronizationRecordsEnum.TOKEN, endTime); - _telemetryRuntimeProducer.recordSyncLatency(HTTPLatenciesEnum.TOKEN, endTime-initTime); + _telemetryRuntimeProducer.recordSuccessfulSync(LastSynchronizationRecordsEnum.TOKEN, System.currentTimeMillis()); + _telemetryRuntimeProducer.recordSyncLatency(HTTPLatenciesEnum.TOKEN, System.currentTimeMillis()-initTime); return getSuccessResponse(jsonContent); } From 23201470ccf4afaffdd83b973956a370ab16eaa1 Mon Sep 17 00:00:00 2001 From: Lucas Echeverz Date: Mon, 10 May 2021 19:30:36 -0300 Subject: [PATCH 43/81] Fixing PR Comments --- .../io/split/client/SplitFactoryImpl.java | 6 +- .../split/engine/common/PushManagerImp.java | 14 +++-- .../split/engine/common/SyncManagerImp.java | 13 ++++- .../engine/sse/EventSourceClientImp.java | 11 ++-- .../engine/sse/PushStatusTrackerImp.java | 33 ++++++++++- .../io/split/engine/sse/client/SSEClient.java | 18 +++++- .../java/io/split/service/HttpPostImp.java | 31 ++++++---- .../telemetry/domain/StreamingEvent.java | 6 ++ .../domain/enums/StreamEventsEnum.java | 23 ++++++++ .../storage/NoopTelemetryStorage.java | 2 +- .../storage/TelemetryRuntimeConsumer.java | 2 +- .../HttpTelemetryMemorySender.java | 17 ++++-- .../synchronizer/SynchronizerMemory.java | 5 +- .../synchronizer/TelemetrySyncTask.java | 2 +- .../io/split/client/EventsClientImplTest.java | 3 - .../io/split/client/SplitFactoryImplTest.java | 5 ++ .../split/engine/common/PushManagerTest.java | 8 ++- .../split/engine/common/SyncManagerTest.java | 26 ++++++--- .../engine/sse/EventSourceClientTest.java | 13 ++++- .../engine/sse/PushStatusTrackerTest.java | 58 ++++++++++++++++--- .../io/split/engine/sse/SSEClientTest.java | 6 +- .../io/split/service/HttpPostImpTest.java | 21 +++++-- .../storage/InMemoryTelemetryStorageTest.java | 5 +- .../synchronizer/SynchronizerMemoryTest.java | 5 +- .../synchronizer/TelemetrySyncTaskTest.java | 4 +- 25 files changed, 262 insertions(+), 75 deletions(-) create mode 100644 client/src/main/java/io/split/telemetry/domain/enums/StreamEventsEnum.java diff --git a/client/src/main/java/io/split/client/SplitFactoryImpl.java b/client/src/main/java/io/split/client/SplitFactoryImpl.java index db27fc624..87edaefda 100644 --- a/client/src/main/java/io/split/client/SplitFactoryImpl.java +++ b/client/src/main/java/io/split/client/SplitFactoryImpl.java @@ -124,7 +124,7 @@ public SplitFactoryImpl(String apiToken, SplitClientConfig config) throws URISyn // Cache Initialisations _segmentCache = new SegmentCacheInMemoryImpl(); _splitCache = new InMemoryCacheImp(); - _telemetrySynchronizer = new SynchronizerMemory(_httpclient, URI.create(config.get_telemetryURL()), _telemetryStorage, _splitCache, _segmentCache); + _telemetrySynchronizer = new SynchronizerMemory(_httpclient, URI.create(config.get_telemetryURL()), _telemetryStorage, _splitCache, _segmentCache, _telemetryStorage); _telemetrySyncTask = new TelemetrySyncTask(config.get_telemetryRefreshRate(), _telemetrySynchronizer); // Segments @@ -185,14 +185,14 @@ public synchronized void destroy() { _log.info("Successful shutdown of splits"); _impressionsManager.close(); _log.info("Successful shutdown of impressions manager"); - _httpclient.close(); - _log.info("Successful shutdown of httpclient"); _eventClient.close(); _log.info("Successful shutdown of eventClient"); _syncManager.shutdown(); _log.info("Successful shutdown of syncManager"); _telemetryStorage.recordSessionLength(System.currentTimeMillis() - _startTime); _telemetrySyncTask.stopScheduledTask(); + _httpclient.close(); + _log.info("Successful shutdown of httpclient"); } catch (IOException e) { _log.error("We could not shutdown split", e); } diff --git a/client/src/main/java/io/split/engine/common/PushManagerImp.java b/client/src/main/java/io/split/engine/common/PushManagerImp.java index 601bb2ac5..01d29a0dd 100644 --- a/client/src/main/java/io/split/engine/common/PushManagerImp.java +++ b/client/src/main/java/io/split/engine/common/PushManagerImp.java @@ -16,6 +16,8 @@ import io.split.engine.sse.workers.SplitsWorkerImp; import io.split.engine.sse.workers.Worker; +import io.split.telemetry.domain.StreamingEvent; +import io.split.telemetry.domain.enums.StreamEventsEnum; import io.split.telemetry.storage.TelemetryRuntimeProducer; import org.apache.hc.client5.http.impl.classic.CloseableHttpClient; import org.slf4j.Logger; @@ -42,13 +44,15 @@ public class PushManagerImp implements PushManager { private Future _nextTokenRefreshTask; private final ScheduledExecutorService _scheduledExecutorService; private AtomicLong _expirationTime; + private final TelemetryRuntimeProducer _telemetryRuntimeProducer; @VisibleForTesting /* package private */ PushManagerImp(AuthApiClient authApiClient, EventSourceClient eventSourceClient, SplitsWorker splitsWorker, Worker segmentWorker, - PushStatusTracker pushStatusTracker) { + PushStatusTracker pushStatusTracker, + TelemetryRuntimeProducer telemetryRuntimeProducer) { _authApiClient = checkNotNull(authApiClient); _eventSourceClient = checkNotNull(eventSourceClient); @@ -60,6 +64,7 @@ public class PushManagerImp implements PushManager { .setDaemon(true) .setNameFormat("Split-SSERefreshToken-%d") .build()); + _telemetryRuntimeProducer = checkNotNull(telemetryRuntimeProducer); } public static PushManagerImp build(Synchronizer synchronizer, @@ -71,12 +76,12 @@ public static PushManagerImp build(Synchronizer synchronizer, TelemetryRuntimeProducer telemetryRuntimeProducer) { SplitsWorker splitsWorker = new SplitsWorkerImp(synchronizer); Worker segmentWorker = new SegmentsWorkerImp(synchronizer); - PushStatusTracker pushStatusTracker = new PushStatusTrackerImp(statusMessages); + PushStatusTracker pushStatusTracker = new PushStatusTrackerImp(statusMessages, telemetryRuntimeProducer); return new PushManagerImp(new AuthApiClientImp(authUrl, httpClient, telemetryRuntimeProducer), - EventSourceClientImp.build(streamingUrl, splitsWorker, segmentWorker, pushStatusTracker, sseHttpClient), + EventSourceClientImp.build(streamingUrl, splitsWorker, segmentWorker, pushStatusTracker, sseHttpClient, telemetryRuntimeProducer), splitsWorker, segmentWorker, - pushStatusTracker); + pushStatusTracker, telemetryRuntimeProducer); } @Override @@ -85,6 +90,7 @@ public synchronized void start() { _log.debug(String.format("Auth service response pushEnabled: %s", response.isPushEnabled())); if (response.isPushEnabled() && startSse(response.getToken(), response.getChannels())) { _expirationTime.set(response.getExpiration()); + _telemetryRuntimeProducer.recordStreamingEvents(new StreamingEvent(StreamEventsEnum.TOKEN_REFRESH.get_type(), response.getExpiration(), System.currentTimeMillis())); return; } diff --git a/client/src/main/java/io/split/engine/common/SyncManagerImp.java b/client/src/main/java/io/split/engine/common/SyncManagerImp.java index 31e245816..c532f660c 100644 --- a/client/src/main/java/io/split/engine/common/SyncManagerImp.java +++ b/client/src/main/java/io/split/engine/common/SyncManagerImp.java @@ -8,6 +8,8 @@ import io.split.engine.experiments.SplitFetcher; import io.split.engine.experiments.SplitSynchronizationTask; import io.split.engine.segments.SegmentSynchronizationTaskImp; +import io.split.telemetry.domain.StreamingEvent; +import io.split.telemetry.domain.enums.StreamEventsEnum; import io.split.telemetry.storage.TelemetryRuntimeProducer; import org.apache.hc.client5.http.impl.classic.CloseableHttpClient; import org.slf4j.Logger; @@ -23,6 +25,8 @@ public class SyncManagerImp implements SyncManager { private static final Logger _log = LoggerFactory.getLogger(SyncManager.class); + private static final int STREAMING_STREAMING_EVENT = 0; + private static final int POLLING_STREAMING_EVENT = 1; private final AtomicBoolean _streamingEnabledConfig; private final Synchronizer _synchronizer; @@ -34,6 +38,7 @@ public class SyncManagerImp implements SyncManager { private final SDKReadinessGates _gates; private Future _pushStatusMonitorTask; private Backoff _backoff; + private final TelemetryRuntimeProducer _telemetryRuntimeProducer; @VisibleForTesting /* package private */ SyncManagerImp(boolean streamingEnabledConfig, @@ -41,7 +46,7 @@ public class SyncManagerImp implements SyncManager { PushManager pushManager, LinkedBlockingQueue pushMessages, int authRetryBackOffBase, - SDKReadinessGates gates) { + SDKReadinessGates gates, TelemetryRuntimeProducer telemetryRuntimeProducer) { _streamingEnabledConfig = new AtomicBoolean(streamingEnabledConfig); _synchronizer = checkNotNull(synchronizer); _pushManager = checkNotNull(pushManager); @@ -57,6 +62,7 @@ public class SyncManagerImp implements SyncManager { .build()); _backoff = new Backoff(authRetryBackOffBase); _gates = checkNotNull(gates); + _telemetryRuntimeProducer = checkNotNull(telemetryRuntimeProducer); } public static SyncManagerImp build(boolean streamingEnabledConfig, @@ -76,13 +82,12 @@ public static SyncManagerImp build(boolean streamingEnabledConfig, LinkedBlockingQueue pushMessages = new LinkedBlockingQueue<>(); Synchronizer synchronizer = new SynchronizerImp(splitSynchronizationTask, splitFetcher, segmentSynchronizationTaskImp, splitCache, segmentCache, streamingRetryDelay, gates); PushManager pushManager = PushManagerImp.build(synchronizer, streamingServiceUrl, authUrl, httpClient, pushMessages, sseHttpClient, telemetryRuntimeProducer); - return new SyncManagerImp(streamingEnabledConfig, synchronizer, pushManager, pushMessages, authRetryBackOffBase, gates); + return new SyncManagerImp(streamingEnabledConfig, synchronizer, pushManager, pushMessages, authRetryBackOffBase, gates, telemetryRuntimeProducer); } @Override public void start() { _synchronizer.syncAll(); - if (_streamingEnabledConfig.get()) { startStreamingMode(); } else { @@ -103,6 +108,7 @@ private void startStreamingMode() { _pushStatusMonitorTask = _executorService.submit(this::incomingPushStatusHandler); } _pushManager.start(); + _telemetryRuntimeProducer.recordStreamingEvents(new StreamingEvent(StreamEventsEnum.SYNC_MODE_UPDATE.get_type(), STREAMING_STREAMING_EVENT, System.currentTimeMillis())); } private void startPollingMode() { @@ -114,6 +120,7 @@ private void startPollingMode() { _log.debug("Starting in polling mode ..."); _synchronizer.startPeriodicFetching(); + _telemetryRuntimeProducer.recordStreamingEvents(new StreamingEvent(StreamEventsEnum.SYNC_MODE_UPDATE.get_type(), POLLING_STREAMING_EVENT, System.currentTimeMillis())); } @VisibleForTesting diff --git a/client/src/main/java/io/split/engine/sse/EventSourceClientImp.java b/client/src/main/java/io/split/engine/sse/EventSourceClientImp.java index 7d8bf990d..772ccfb48 100644 --- a/client/src/main/java/io/split/engine/sse/EventSourceClientImp.java +++ b/client/src/main/java/io/split/engine/sse/EventSourceClientImp.java @@ -7,6 +7,7 @@ import io.split.engine.sse.exceptions.EventParsingException; import io.split.engine.sse.workers.SplitsWorker; import io.split.engine.sse.workers.Worker; +import io.split.telemetry.storage.TelemetryRuntimeProducer; import org.apache.hc.client5.http.impl.classic.CloseableHttpClient; import org.apache.hc.core5.net.URIBuilder; import org.slf4j.Logger; @@ -35,7 +36,8 @@ public class EventSourceClientImp implements EventSourceClient { NotificationParser notificationParser, NotificationProcessor notificationProcessor, PushStatusTracker pushStatusTracker, - CloseableHttpClient sseHttpClient) { + CloseableHttpClient sseHttpClient, + TelemetryRuntimeProducer telemetryRuntimeProducer) { _baseStreamingUrl = checkNotNull(baseStreamingUrl); _notificationParser = checkNotNull(notificationParser); _notificationProcessor = checkNotNull(notificationProcessor); @@ -44,7 +46,7 @@ public class EventSourceClientImp implements EventSourceClient { _sseClient = new SSEClient( inboundEvent -> { onMessage(inboundEvent); return null; }, status -> { _pushStatusTracker.handleSseStatus(status); return null; }, - sseHttpClient); + sseHttpClient, telemetryRuntimeProducer); _firstEvent = new AtomicBoolean(); } @@ -52,12 +54,13 @@ public static EventSourceClientImp build(String baseStreamingUrl, SplitsWorker splitsWorker, Worker segmentWorker, PushStatusTracker pushStatusTracker, - CloseableHttpClient sseHttpClient) { + CloseableHttpClient sseHttpClient, TelemetryRuntimeProducer telemetryRuntimeProducer) { return new EventSourceClientImp(baseStreamingUrl, new NotificationParserImp(), NotificationProcessorImp.build(splitsWorker, segmentWorker, pushStatusTracker), pushStatusTracker, - sseHttpClient); + sseHttpClient, + telemetryRuntimeProducer); } @Override diff --git a/client/src/main/java/io/split/engine/sse/PushStatusTrackerImp.java b/client/src/main/java/io/split/engine/sse/PushStatusTrackerImp.java index f76cf691e..c675d4574 100644 --- a/client/src/main/java/io/split/engine/sse/PushStatusTrackerImp.java +++ b/client/src/main/java/io/split/engine/sse/PushStatusTrackerImp.java @@ -7,16 +7,27 @@ import io.split.engine.sse.dtos.ControlType; import io.split.engine.sse.dtos.ErrorNotification; import io.split.engine.sse.dtos.OccupancyNotification; +import io.split.telemetry.domain.StreamingEvent; +import io.split.telemetry.domain.enums.StreamEventsEnum; +import io.split.telemetry.storage.TelemetryRuntimeProducer; import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import java.util.TimeZone; import java.util.concurrent.ConcurrentMap; import java.util.concurrent.LinkedBlockingQueue; import java.util.concurrent.atomic.AtomicBoolean; import java.util.concurrent.atomic.AtomicReference; +import static com.google.common.base.Preconditions.checkNotNull; + public class PushStatusTrackerImp implements PushStatusTracker { private static final Logger _log = LoggerFactory.getLogger(PushStatusTracker.class); + private static final String CONTROL_PRI_CHANNEL = "control_pri"; + private static final String CONTROL_SEC_CHANNEL = "control_sec"; + private static final int STREAMING_DISABLED = 0; + private static final int STREAMING_ENABLED = 1; + private static final int STREAMING_PAUSED = 2; private final AtomicBoolean _publishersOnline = new AtomicBoolean(true); private final AtomicReference _sseStatus = new AtomicReference<>(SSEClient.StatusMessage.INITIALIZATION_IN_PROGRESS); @@ -24,8 +35,11 @@ public class PushStatusTrackerImp implements PushStatusTracker { private final LinkedBlockingQueue _statusMessages; private final ConcurrentMap regions = Maps.newConcurrentMap(); - public PushStatusTrackerImp(LinkedBlockingQueue statusMessages) { + private final TelemetryRuntimeProducer _telemetryRuntimeProducer; + + public PushStatusTrackerImp(LinkedBlockingQueue statusMessages, TelemetryRuntimeProducer telemetryRuntimeProducer) { _statusMessages = statusMessages; + _telemetryRuntimeProducer = checkNotNull(telemetryRuntimeProducer); } private synchronized void reset() { @@ -42,6 +56,8 @@ public void handleSseStatus(SSEClient.StatusMessage newStatus) { case FIRST_EVENT: if (SSEClient.StatusMessage.CONNECTED.equals(_sseStatus.get())) { _statusMessages.offer(PushManager.Status.STREAMING_READY); + _telemetryRuntimeProducer.recordStreamingEvents(new StreamingEvent(StreamEventsEnum.STREAMING_STATUS.get_type(), STREAMING_ENABLED, System.currentTimeMillis())); + _telemetryRuntimeProducer.recordStreamingEvents(new StreamingEvent(StreamEventsEnum.CONNECTION_ESTABLISHED.get_type(),0l, System.currentTimeMillis())); } case CONNECTED: _sseStatus.compareAndSet(SSEClient.StatusMessage.INITIALIZATION_IN_PROGRESS, SSEClient.StatusMessage.CONNECTED); @@ -85,12 +101,14 @@ public void handleIncomingControlEvent(ControlNotification controlNotification) } break; case STREAMING_PAUSED: + _telemetryRuntimeProducer.recordStreamingEvents(new StreamingEvent(StreamEventsEnum.STREAMING_STATUS.get_type(), STREAMING_PAUSED, System.currentTimeMillis())); if (_backendStatus.compareAndSet(ControlType.STREAMING_RESUMED, ControlType.STREAMING_PAUSED) && _publishersOnline.get()) { // If there are no publishers online, the STREAMING_DOWN message should have already been sent _statusMessages.offer(PushManager.Status.STREAMING_DOWN); } break; case STREAMING_DISABLED: + _telemetryRuntimeProducer.recordStreamingEvents(new StreamingEvent(StreamEventsEnum.STREAMING_STATUS.get_type(), STREAMING_DISABLED, System.currentTimeMillis())); _backendStatus.set(ControlType.STREAMING_DISABLED); _statusMessages.offer(PushManager.Status.STREAMING_OFF); break; @@ -102,6 +120,7 @@ public void handleIncomingOccupancyEvent(OccupancyNotification occupancyNotifica _log.debug(String.format("handleIncomingOccupancyEvent: publishers=%d", occupancyNotification.getMetrics().getPublishers())); int publishers = occupancyNotification.getMetrics().getPublishers(); + recordTelemetryOcuppancy(occupancyNotification, publishers); regions.put(occupancyNotification.getChannel(), publishers); boolean isPublishers = isPublishers(); if (!isPublishers && _publishersOnline.compareAndSet(true, false) && _backendStatus.get().equals(ControlType.STREAMING_RESUMED)) { @@ -114,7 +133,7 @@ public void handleIncomingOccupancyEvent(OccupancyNotification occupancyNotifica @Override public void handleIncomingAblyError(ErrorNotification notification) { _log.debug(String.format("handleIncomingAblyError: %s", notification.getMessage())); - + _telemetryRuntimeProducer.recordStreamingEvents(new StreamingEvent(StreamEventsEnum.ABLY_ERROR.get_type(), notification.getCode(), System.currentTimeMillis())); if (_backendStatus.get().equals(ControlType.STREAMING_DISABLED)) { return; // Ignore } @@ -145,4 +164,14 @@ private boolean isPublishers() { } return false; } + + private void recordTelemetryOcuppancy(OccupancyNotification occupancyNotification, int publishers) { + if (CONTROL_PRI_CHANNEL.equals(occupancyNotification.getChannel())) { + _telemetryRuntimeProducer.recordStreamingEvents(new StreamingEvent(StreamEventsEnum.OCCUPANCY_PRI.get_type(), publishers, System.currentTimeMillis())); + } + else if (CONTROL_SEC_CHANNEL.equals(occupancyNotification.getChannel())){ + _telemetryRuntimeProducer.recordStreamingEvents(new StreamingEvent(StreamEventsEnum.OCCUPANCY_SEC.get_type(), publishers, System.currentTimeMillis())); + } + + } } \ No newline at end of file diff --git a/client/src/main/java/io/split/engine/sse/client/SSEClient.java b/client/src/main/java/io/split/engine/sse/client/SSEClient.java index 184475671..cf5cb7649 100644 --- a/client/src/main/java/io/split/engine/sse/client/SSEClient.java +++ b/client/src/main/java/io/split/engine/sse/client/SSEClient.java @@ -2,6 +2,9 @@ import com.google.common.base.Strings; import com.google.common.util.concurrent.ThreadFactoryBuilder; +import io.split.telemetry.domain.StreamingEvent; +import io.split.telemetry.domain.enums.StreamEventsEnum; +import io.split.telemetry.storage.TelemetryRuntimeProducer; import org.apache.hc.client5.http.classic.methods.HttpGet; import org.apache.hc.client5.http.impl.classic.CloseableHttpClient; import org.apache.hc.client5.http.impl.classic.CloseableHttpResponse; @@ -45,6 +48,8 @@ private enum ConnectionState { private final static String KEEP_ALIVE_PAYLOAD = ":keepalive\n"; private final static long CONNECT_TIMEOUT = 30000; private static final Logger _log = LoggerFactory.getLogger(SSEClient.class); + private final static int REQUESTED_CONNECTION_ERROR = 0; + private final static int NON_REQUESTED_CONNECTION_ERROR = 1; private final ExecutorService _connectionExecutor = Executors.newSingleThreadExecutor(new ThreadFactoryBuilder() .setDaemon(true) @@ -58,13 +63,17 @@ private enum ConnectionState { private final AtomicReference _ongoingRequest = new AtomicReference<>(); private AtomicBoolean _forcedStop; + private final TelemetryRuntimeProducer _telemetryRuntimeProducer; + public SSEClient(Function eventCallback, Function statusCallback, - CloseableHttpClient client) { + CloseableHttpClient client, + TelemetryRuntimeProducer telemetryRuntimeProducer) { _eventCallback = eventCallback; _statusCallback = statusCallback; _client = client; _forcedStop = new AtomicBoolean(); + _telemetryRuntimeProducer = checkNotNull(telemetryRuntimeProducer); } public synchronized boolean open(URI uri) { @@ -126,20 +135,27 @@ private void connectAndLoop(URI uri, CountDownLatch signal) { _log.debug(exc.getMessage()); if (SOCKET_CLOSED_MESSAGE.equals(exc.getMessage())) { // Connection closed by us _statusCallback.apply(StatusMessage.FORCED_STOP); + _telemetryRuntimeProducer.recordStreamingEvents(new StreamingEvent(StreamEventsEnum.SSE_CONNECTION_ERROR.get_type(), REQUESTED_CONNECTION_ERROR, System.currentTimeMillis())); return; } // Connection closed by server _statusCallback.apply(StatusMessage.RETRYABLE_ERROR); + _telemetryRuntimeProducer.recordStreamingEvents(new StreamingEvent(StreamEventsEnum.SSE_CONNECTION_ERROR.get_type(), NON_REQUESTED_CONNECTION_ERROR, System.currentTimeMillis())); return; } catch (IOException exc) { // Other type of connection error if(!_forcedStop.get()) { _log.debug(String.format("SSE connection ended abruptly: %s. Retying", exc.getMessage())); + _telemetryRuntimeProducer.recordStreamingEvents(new StreamingEvent(StreamEventsEnum.SSE_CONNECTION_ERROR.get_type(), REQUESTED_CONNECTION_ERROR, System.currentTimeMillis())); _statusCallback.apply(StatusMessage.RETRYABLE_ERROR); return; } + + _telemetryRuntimeProducer.recordStreamingEvents(new StreamingEvent(StreamEventsEnum.SSE_CONNECTION_ERROR.get_type(), NON_REQUESTED_CONNECTION_ERROR, System.currentTimeMillis())); } } } catch (Exception e) { // Any other error non related to the connection disables streaming altogether + + _telemetryRuntimeProducer.recordStreamingEvents(new StreamingEvent(StreamEventsEnum.SSE_CONNECTION_ERROR.get_type(), NON_REQUESTED_CONNECTION_ERROR, System.currentTimeMillis())); _log.warn(e.getMessage(), e); _statusCallback.apply(StatusMessage.NONRETRYABLE_ERROR); } finally { diff --git a/client/src/main/java/io/split/service/HttpPostImp.java b/client/src/main/java/io/split/service/HttpPostImp.java index 3555d01e0..e0586eb47 100644 --- a/client/src/main/java/io/split/service/HttpPostImp.java +++ b/client/src/main/java/io/split/service/HttpPostImp.java @@ -1,6 +1,10 @@ package io.split.service; import io.split.client.utils.Utils; +import io.split.telemetry.domain.enums.HTTPLatenciesEnum; +import io.split.telemetry.domain.enums.LastSynchronizationRecordsEnum; +import io.split.telemetry.domain.enums.ResourceEnum; +import io.split.telemetry.storage.TelemetryRuntimeProducer; import org.apache.hc.client5.http.impl.classic.CloseableHttpClient; import org.apache.hc.client5.http.impl.classic.CloseableHttpResponse; import org.apache.hc.core5.http.HttpEntity; @@ -11,34 +15,37 @@ import java.net.URI; +import static com.google.common.base.Preconditions.checkNotNull; + public class HttpPostImp { private static final Logger _logger = LoggerFactory.getLogger(HttpPostImp.class); private CloseableHttpClient _client; + private final TelemetryRuntimeProducer _telemetryRuntimeProducer; - public HttpPostImp(CloseableHttpClient client) { + public HttpPostImp(CloseableHttpClient client, TelemetryRuntimeProducer telemetryRuntimeProducer) { _client = client; + _telemetryRuntimeProducer = checkNotNull(telemetryRuntimeProducer); } - public void post(URI uri, Object object, String posted) { - CloseableHttpResponse response = null; - - try { - HttpEntity entity = Utils.toJsonEntity(object); - HttpPost request = new HttpPost(uri); - request.setEntity(entity); + public void post(URI uri, Object object, String posted, HTTPLatenciesEnum httpLatenciesEnum, LastSynchronizationRecordsEnum lastSynchronizationRecordsEnum, ResourceEnum resourceEnum) { + long initTime = System.currentTimeMillis(); + HttpEntity entity = Utils.toJsonEntity(object); + HttpPost request = new HttpPost(uri); + request.setEntity(entity); - response = _client.execute(request); + try (CloseableHttpResponse response = _client.execute(request)) { int status = response.getCode(); if (status < HttpStatus.SC_OK || status >= HttpStatus.SC_MULTIPLE_CHOICES) { + _telemetryRuntimeProducer.recordSyncError(resourceEnum, status); _logger.warn("Response status was: " + status); + return; } - + _telemetryRuntimeProducer.recordSyncLatency(httpLatenciesEnum, System.currentTimeMillis() - initTime); + _telemetryRuntimeProducer.recordSuccessfulSync(lastSynchronizationRecordsEnum, System.currentTimeMillis()); } catch (Throwable t) { _logger.warn("Exception when posting " + posted + object, t); - } finally { - Utils.forceClose(response); } } } diff --git a/client/src/main/java/io/split/telemetry/domain/StreamingEvent.java b/client/src/main/java/io/split/telemetry/domain/StreamingEvent.java index e0ebca3ac..161b29762 100644 --- a/client/src/main/java/io/split/telemetry/domain/StreamingEvent.java +++ b/client/src/main/java/io/split/telemetry/domain/StreamingEvent.java @@ -14,6 +14,12 @@ public class StreamingEvent { @SerializedName(FIELD_TIMESTAMP) private long _timestamp; + public StreamingEvent(int _type, long _data, long _timestamp) { + this._type = _type; + this._data = _data; + this._timestamp = _timestamp; + } + public int get_type() { return _type; } diff --git a/client/src/main/java/io/split/telemetry/domain/enums/StreamEventsEnum.java b/client/src/main/java/io/split/telemetry/domain/enums/StreamEventsEnum.java new file mode 100644 index 000000000..3947f5d7e --- /dev/null +++ b/client/src/main/java/io/split/telemetry/domain/enums/StreamEventsEnum.java @@ -0,0 +1,23 @@ +package io.split.telemetry.domain.enums; + +public enum StreamEventsEnum { + CONNECTION_ESTABLISHED(0), + OCCUPANCY_PRI(10), + OCCUPANCY_SEC(20), + STREAMING_STATUS(30), + SSE_CONNECTION_ERROR(40), + TOKEN_REFRESH(50), + ABLY_ERROR(60), + SYNC_MODE_UPDATE(70); + + + private int _type; + + StreamEventsEnum(int type) { + _type = type; + } + + public int get_type() { + return _type; + } +} diff --git a/client/src/main/java/io/split/telemetry/storage/NoopTelemetryStorage.java b/client/src/main/java/io/split/telemetry/storage/NoopTelemetryStorage.java index aa37fae43..3673d0c0a 100644 --- a/client/src/main/java/io/split/telemetry/storage/NoopTelemetryStorage.java +++ b/client/src/main/java/io/split/telemetry/storage/NoopTelemetryStorage.java @@ -118,7 +118,7 @@ public HTTPErrors popHTTPErrors() { } @Override - public HTTPLatencies popHTTPLatencies() throws Exception { + public HTTPLatencies popHTTPLatencies(){ return null; } diff --git a/client/src/main/java/io/split/telemetry/storage/TelemetryRuntimeConsumer.java b/client/src/main/java/io/split/telemetry/storage/TelemetryRuntimeConsumer.java index 7acd49afb..6a746e783 100644 --- a/client/src/main/java/io/split/telemetry/storage/TelemetryRuntimeConsumer.java +++ b/client/src/main/java/io/split/telemetry/storage/TelemetryRuntimeConsumer.java @@ -14,7 +14,7 @@ public interface TelemetryRuntimeConsumer { long getEventStats(EventsDataRecordsEnum type); LastSynchronization getLastSynchronization(); HTTPErrors popHTTPErrors(); - HTTPLatencies popHTTPLatencies() throws Exception; + HTTPLatencies popHTTPLatencies(); long popAuthRejections(); long popTokenRefreshes(); List popStreamingEvents(); diff --git a/client/src/main/java/io/split/telemetry/synchronizer/HttpTelemetryMemorySender.java b/client/src/main/java/io/split/telemetry/synchronizer/HttpTelemetryMemorySender.java index e6c1a6ceb..37b05ed1a 100644 --- a/client/src/main/java/io/split/telemetry/synchronizer/HttpTelemetryMemorySender.java +++ b/client/src/main/java/io/split/telemetry/synchronizer/HttpTelemetryMemorySender.java @@ -5,6 +5,10 @@ import io.split.service.HttpPostImp; import io.split.telemetry.domain.Config; import io.split.telemetry.domain.Stats; +import io.split.telemetry.domain.enums.HTTPLatenciesEnum; +import io.split.telemetry.domain.enums.LastSynchronizationRecordsEnum; +import io.split.telemetry.domain.enums.ResourceEnum; +import io.split.telemetry.storage.TelemetryRuntimeProducer; import org.apache.hc.client5.http.impl.classic.CloseableHttpClient; import java.net.URI; @@ -20,26 +24,27 @@ public class HttpTelemetryMemorySender extends HttpPostImp { private final URI _impressionConfigTarget; private final URI _impressionStatsTarget; - public static HttpTelemetryMemorySender create(CloseableHttpClient client, URI telemetryRootEndpoint) throws URISyntaxException { + public static HttpTelemetryMemorySender create(CloseableHttpClient client, URI telemetryRootEndpoint, TelemetryRuntimeProducer telemetryRuntimeProducer) throws URISyntaxException { return new HttpTelemetryMemorySender(client, Utils.appendPath(telemetryRootEndpoint,CONFIG_ENDPOINT_PATH), - Utils.appendPath(telemetryRootEndpoint, STATS_ENDPOINT_PATH) + Utils.appendPath(telemetryRootEndpoint, STATS_ENDPOINT_PATH), + telemetryRuntimeProducer ); } @VisibleForTesting - HttpTelemetryMemorySender(CloseableHttpClient client, URI impressionConfigTarget, URI impressionStatsTarget) { - super(client); + HttpTelemetryMemorySender(CloseableHttpClient client, URI impressionConfigTarget, URI impressionStatsTarget, TelemetryRuntimeProducer telemetryRuntimeProducer) { + super(client, telemetryRuntimeProducer); _impressionConfigTarget = impressionConfigTarget; _impressionStatsTarget = impressionStatsTarget; } public void postConfig(Config config) { - post(_impressionConfigTarget, config, CONFIG_METRICS); + post(_impressionConfigTarget, config, CONFIG_METRICS, HTTPLatenciesEnum.TELEMETRY, LastSynchronizationRecordsEnum.TELEMETRY, ResourceEnum.TELEMETRY_SYNC); } public void postStats(Stats stats) { - post(_impressionStatsTarget, stats, STATS_METRICS); + post(_impressionStatsTarget, stats, STATS_METRICS, HTTPLatenciesEnum.TELEMETRY, LastSynchronizationRecordsEnum.TELEMETRY, ResourceEnum.TELEMETRY_SYNC); } } diff --git a/client/src/main/java/io/split/telemetry/synchronizer/SynchronizerMemory.java b/client/src/main/java/io/split/telemetry/synchronizer/SynchronizerMemory.java index 1bd4f0356..1ffeb1178 100644 --- a/client/src/main/java/io/split/telemetry/synchronizer/SynchronizerMemory.java +++ b/client/src/main/java/io/split/telemetry/synchronizer/SynchronizerMemory.java @@ -13,6 +13,7 @@ import io.split.telemetry.domain.URLOverrides; import io.split.telemetry.domain.enums.EventsDataRecordsEnum; import io.split.telemetry.domain.enums.ImpressionsDataTypeEnum; +import io.split.telemetry.storage.TelemetryRuntimeProducer; import io.split.telemetry.storage.TelemetryStorageConsumer; import org.apache.hc.client5.http.impl.classic.CloseableHttpClient; @@ -33,8 +34,8 @@ public class SynchronizerMemory implements TelemetrySynchronizer{ private SegmentCache _segmentCache; public SynchronizerMemory(CloseableHttpClient client, URI telemetryRootEndpoint, TelemetryStorageConsumer telemetryStorageConsumer, SplitCache splitCache, - SegmentCache segmentCache) throws URISyntaxException { - _httpHttpTelemetryMemorySender = HttpTelemetryMemorySender.create(client, telemetryRootEndpoint); + SegmentCache segmentCache, TelemetryRuntimeProducer telemetryRuntimeProducer) throws URISyntaxException { + _httpHttpTelemetryMemorySender = HttpTelemetryMemorySender.create(client, telemetryRootEndpoint, telemetryRuntimeProducer); _teleTelemetryStorageConsumer = telemetryStorageConsumer; _splitCache = splitCache; _segmentCache = segmentCache; diff --git a/client/src/main/java/io/split/telemetry/synchronizer/TelemetrySyncTask.java b/client/src/main/java/io/split/telemetry/synchronizer/TelemetrySyncTask.java index 725e22148..c5135b133 100644 --- a/client/src/main/java/io/split/telemetry/synchronizer/TelemetrySyncTask.java +++ b/client/src/main/java/io/split/telemetry/synchronizer/TelemetrySyncTask.java @@ -19,7 +19,7 @@ public TelemetrySyncTask(int telemetryRefreshRate, TelemetrySynchronizer telemet .setDaemon(true) .setNameFormat("Telemetry-sync-%d") .build(); - _telemetrySynchronizer = telemetrySynchronizer; //TODO + _telemetrySynchronizer = telemetrySynchronizer; _telemetryRefreshRate = telemetryRefreshRate; _telemetrySyncScheduledExecutorService = Executors.newSingleThreadScheduledExecutor(telemetrySyncThreadFactory); try { diff --git a/client/src/test/java/io/split/client/EventsClientImplTest.java b/client/src/test/java/io/split/client/EventsClientImplTest.java index 85307a213..4a2e3fefc 100644 --- a/client/src/test/java/io/split/client/EventsClientImplTest.java +++ b/client/src/test/java/io/split/client/EventsClientImplTest.java @@ -8,14 +8,12 @@ import org.apache.hc.client5.http.impl.classic.HttpClients; import org.hamcrest.Matchers; import org.junit.Assert; -import org.junit.Ignore; import org.junit.Test; import org.mockito.Mockito; import java.io.IOException; import java.net.URI; import java.net.URISyntaxException; - import java.util.concurrent.LinkedBlockingQueue; public class EventsClientImplTest { @@ -54,7 +52,6 @@ public void testCustomURLAppendingPathNoBackslash() throws URISyntaxException { } @Test - @Ignore public void testEventsFlushedWhenSizeLimitReached() throws URISyntaxException, InterruptedException, IOException { CloseableHttpClient client = Mockito.mock(CloseableHttpClient.class); EventClientImpl eventClient = new EventClientImpl(new LinkedBlockingQueue(), diff --git a/client/src/test/java/io/split/client/SplitFactoryImplTest.java b/client/src/test/java/io/split/client/SplitFactoryImplTest.java index 3e09c697e..1256293b4 100644 --- a/client/src/test/java/io/split/client/SplitFactoryImplTest.java +++ b/client/src/test/java/io/split/client/SplitFactoryImplTest.java @@ -23,6 +23,7 @@ public void testFactoryInstantiation() throws Exception { .endpoint(ENDPOINT,EVENTS_ENDPOINT) .authServiceURL(AUTH_SERVICE) .setBlockUntilReadyTimeout(10000) + .telemetryURL(SplitClientConfig.TELEMETRY_ENDPOINT) .build(); SplitFactoryImpl splitFactory = new SplitFactoryImpl(API_KEY, splitClientConfig); @@ -37,6 +38,7 @@ public void testFactoryInstantiationWithoutBlockUntilReady() throws Exception { .impressionsMode(ImpressionsManager.Mode.DEBUG) .impressionsRefreshRate(1) .endpoint(ENDPOINT,EVENTS_ENDPOINT) + .telemetryURL(SplitClientConfig.TELEMETRY_ENDPOINT) .authServiceURL(AUTH_SERVICE) .build(); SplitFactoryImpl splitFactory = new SplitFactoryImpl(API_KEY, splitClientConfig); @@ -53,6 +55,7 @@ public void testFactoryInstantiationIntegrationsConfig() throws Exception { .impressionsMode(ImpressionsManager.Mode.DEBUG) .impressionsRefreshRate(1) .endpoint(ENDPOINT,EVENTS_ENDPOINT) + .telemetryURL(SplitClientConfig.TELEMETRY_ENDPOINT) .authServiceURL(AUTH_SERVICE) .setBlockUntilReadyTimeout(1000) .integrations(integrationsConfig) @@ -70,6 +73,7 @@ public void testFactoryInstantiationWithProxy() throws Exception { .impressionsMode(ImpressionsManager.Mode.DEBUG) .impressionsRefreshRate(1) .endpoint(ENDPOINT,EVENTS_ENDPOINT) + .telemetryURL(SplitClientConfig.TELEMETRY_ENDPOINT) .authServiceURL(AUTH_SERVICE) .setBlockUntilReadyTimeout(1000) .proxyPort(6060) @@ -90,6 +94,7 @@ public void testFactoryDestroy() throws Exception { .impressionsMode(ImpressionsManager.Mode.DEBUG) .impressionsRefreshRate(1) .endpoint(ENDPOINT,EVENTS_ENDPOINT) + .telemetryURL(SplitClientConfig.TELEMETRY_ENDPOINT) .authServiceURL(AUTH_SERVICE) .setBlockUntilReadyTimeout(10000) .build(); diff --git a/client/src/test/java/io/split/engine/common/PushManagerTest.java b/client/src/test/java/io/split/engine/common/PushManagerTest.java index 4662ddab2..d93ccbae5 100644 --- a/client/src/test/java/io/split/engine/common/PushManagerTest.java +++ b/client/src/test/java/io/split/engine/common/PushManagerTest.java @@ -8,6 +8,9 @@ import io.split.engine.sse.dtos.AuthenticationResponse; import io.split.engine.sse.workers.SegmentsWorkerImp; import io.split.engine.sse.workers.SplitsWorker; +import io.split.telemetry.storage.InMemoryTelemetryStorage; +import io.split.telemetry.storage.TelemetryStorage; +import org.junit.Assert; import org.junit.Before; import org.junit.Test; import org.mockito.Mockito; @@ -20,6 +23,7 @@ public class PushManagerTest { private Backoff _backoff; private PushManager _pushManager; private PushStatusTracker _pushStatusTracker; + private TelemetryStorage _telemetryStorage; @Before public void setUp() { @@ -27,11 +31,12 @@ public void setUp() { _eventSourceClient = Mockito.mock(EventSourceClient.class); _backoff = Mockito.mock(Backoff.class); _pushStatusTracker = Mockito.mock(PushStatusTrackerImp.class); + _telemetryStorage = new InMemoryTelemetryStorage(); _pushManager = new PushManagerImp(_authApiClient, _eventSourceClient, Mockito.mock(SplitsWorker.class), Mockito.mock(SegmentsWorkerImp.class), - _pushStatusTracker); + _pushStatusTracker, _telemetryStorage); } @Test @@ -58,6 +63,7 @@ public void startWithPushEnabledShouldConnect() throws InterruptedException { Mockito.verify(_pushStatusTracker, Mockito.times(0)).handleSseStatus(SSEClient.StatusMessage.RETRYABLE_ERROR); Mockito.verify(_pushStatusTracker, Mockito.times(0)).forcePushDisable(); + Assert.assertEquals(1, _telemetryStorage.popStreamingEvents().size()); } @Test diff --git a/client/src/test/java/io/split/engine/common/SyncManagerTest.java b/client/src/test/java/io/split/engine/common/SyncManagerTest.java index 76197ff33..d2204ec18 100644 --- a/client/src/test/java/io/split/engine/common/SyncManagerTest.java +++ b/client/src/test/java/io/split/engine/common/SyncManagerTest.java @@ -1,6 +1,8 @@ package io.split.engine.common; import io.split.engine.SDKReadinessGates; +import io.split.telemetry.storage.InMemoryTelemetryStorage; +import io.split.telemetry.storage.TelemetryStorage; import org.junit.Before; import org.junit.Test; import org.mockito.Mockito; @@ -22,8 +24,9 @@ public void setUp() { @Test public void startWithStreamingFalseShouldStartPolling() throws InterruptedException { + TelemetryStorage telemetryStorage = new InMemoryTelemetryStorage(); _gates.sdkInternalReady(); - SyncManagerImp syncManager = new SyncManagerImp(false, _synchronizer, _pushManager, new LinkedBlockingQueue<>(), BACKOFF_BASE, _gates); + SyncManagerImp syncManager = new SyncManagerImp(false, _synchronizer, _pushManager, new LinkedBlockingQueue<>(), BACKOFF_BASE, _gates, telemetryStorage); syncManager.start(); Thread.sleep(1000); Mockito.verify(_synchronizer, Mockito.times(1)).startPeriodicFetching(); @@ -33,7 +36,8 @@ public void startWithStreamingFalseShouldStartPolling() throws InterruptedExcept @Test public void startWithStreamingTrueShouldStartSyncAll() { - SyncManager sm = new SyncManagerImp(true, _synchronizer, _pushManager, new LinkedBlockingQueue<>(), BACKOFF_BASE, _gates); + TelemetryStorage telemetryStorage = new InMemoryTelemetryStorage(); + SyncManager sm = new SyncManagerImp(true, _synchronizer, _pushManager, new LinkedBlockingQueue<>(), BACKOFF_BASE, _gates, telemetryStorage); sm.start(); Mockito.verify(_synchronizer, Mockito.times(0)).startPeriodicFetching(); Mockito.verify(_synchronizer, Mockito.times(1)).syncAll(); @@ -42,8 +46,9 @@ public void startWithStreamingTrueShouldStartSyncAll() { @Test public void onStreamingAvailable() throws InterruptedException { + TelemetryStorage telemetryStorage = new InMemoryTelemetryStorage(); LinkedBlockingQueue messsages = new LinkedBlockingQueue<>(); - SyncManagerImp syncManager = new SyncManagerImp(true, _synchronizer, _pushManager, messsages, BACKOFF_BASE, _gates); + SyncManagerImp syncManager = new SyncManagerImp(true, _synchronizer, _pushManager, messsages, BACKOFF_BASE, _gates, telemetryStorage); Thread t = new Thread(syncManager::incomingPushStatusHandler); t.start(); messsages.offer(PushManager.Status.STREAMING_READY); @@ -56,8 +61,9 @@ public void onStreamingAvailable() throws InterruptedException { @Test public void onStreamingDisabled() throws InterruptedException { + TelemetryStorage telemetryStorage = new InMemoryTelemetryStorage(); LinkedBlockingQueue messsages = new LinkedBlockingQueue<>(); - SyncManagerImp syncManager = new SyncManagerImp(true, _synchronizer, _pushManager, messsages, BACKOFF_BASE, _gates); + SyncManagerImp syncManager = new SyncManagerImp(true, _synchronizer, _pushManager, messsages, BACKOFF_BASE, _gates, telemetryStorage); Thread t = new Thread(syncManager::incomingPushStatusHandler); t.start(); messsages.offer(PushManager.Status.STREAMING_DOWN); @@ -70,8 +76,9 @@ public void onStreamingDisabled() throws InterruptedException { @Test public void onStreamingShutdown() throws InterruptedException { + TelemetryStorage telemetryStorage = new InMemoryTelemetryStorage(); LinkedBlockingQueue messsages = new LinkedBlockingQueue<>(); - SyncManagerImp syncManager = new SyncManagerImp(true, _synchronizer, _pushManager, messsages, BACKOFF_BASE, _gates); + SyncManagerImp syncManager = new SyncManagerImp(true, _synchronizer, _pushManager, messsages, BACKOFF_BASE, _gates, telemetryStorage); Thread t = new Thread(syncManager::incomingPushStatusHandler); t.start(); messsages.offer(PushManager.Status.STREAMING_OFF); @@ -82,8 +89,9 @@ public void onStreamingShutdown() throws InterruptedException { @Test public void onConnected() throws InterruptedException { + TelemetryStorage telemetryStorage = new InMemoryTelemetryStorage(); LinkedBlockingQueue messsages = new LinkedBlockingQueue<>(); - SyncManagerImp syncManager = new SyncManagerImp(true, _synchronizer, _pushManager, messsages, BACKOFF_BASE, _gates); + SyncManagerImp syncManager = new SyncManagerImp(true, _synchronizer, _pushManager, messsages, BACKOFF_BASE, _gates, telemetryStorage); Thread t = new Thread(syncManager::incomingPushStatusHandler); t.start(); messsages.offer(PushManager.Status.STREAMING_READY); @@ -95,8 +103,9 @@ public void onConnected() throws InterruptedException { @Test public void onDisconnect() throws InterruptedException { + TelemetryStorage telemetryStorage = new InMemoryTelemetryStorage(); LinkedBlockingQueue messsages = new LinkedBlockingQueue<>(); - SyncManagerImp syncManager = new SyncManagerImp(true, _synchronizer, _pushManager, messsages, BACKOFF_BASE, _gates); + SyncManagerImp syncManager = new SyncManagerImp(true, _synchronizer, _pushManager, messsages, BACKOFF_BASE, _gates, telemetryStorage); Thread t = new Thread(syncManager::incomingPushStatusHandler); t.start(); messsages.offer(PushManager.Status.STREAMING_OFF); @@ -107,8 +116,9 @@ public void onDisconnect() throws InterruptedException { @Test public void onDisconnectAndReconnect() throws InterruptedException { // Check with mauro. reconnect should call pushManager.start again, right? + TelemetryStorage telemetryStorage = new InMemoryTelemetryStorage(); LinkedBlockingQueue messsages = new LinkedBlockingQueue<>(); - SyncManagerImp syncManager = new SyncManagerImp(true, _synchronizer, _pushManager, messsages, BACKOFF_BASE, _gates); + SyncManagerImp syncManager = new SyncManagerImp(true, _synchronizer, _pushManager, messsages, BACKOFF_BASE, _gates, telemetryStorage); syncManager.start(); messsages.offer(PushManager.Status.STREAMING_BACKOFF); Thread.sleep(1200); diff --git a/client/src/test/java/io/split/engine/sse/EventSourceClientTest.java b/client/src/test/java/io/split/engine/sse/EventSourceClientTest.java index 704e8dbba..74acf65e3 100644 --- a/client/src/test/java/io/split/engine/sse/EventSourceClientTest.java +++ b/client/src/test/java/io/split/engine/sse/EventSourceClientTest.java @@ -4,6 +4,8 @@ import io.split.engine.sse.client.SSEClient; import io.split.engine.sse.dtos.ErrorNotification; import io.split.engine.sse.dtos.SplitChangeNotification; +import io.split.telemetry.storage.InMemoryTelemetryStorage; +import io.split.telemetry.storage.TelemetryRuntimeProducer; import org.apache.hc.client5.http.config.RequestConfig; import org.apache.hc.client5.http.impl.classic.CloseableHttpClient; import org.apache.hc.client5.http.impl.classic.HttpClients; @@ -37,9 +39,10 @@ public void setUp() { public void startShouldConnect() throws IOException { SSEMockServer.SseEventQueue eventQueue = new SSEMockServer.SseEventQueue(); SSEMockServer sseServer = buildSSEMockServer(eventQueue); + TelemetryRuntimeProducer telemetryRuntimeProducer = Mockito.mock(InMemoryTelemetryStorage.class); sseServer.start(); - EventSourceClient eventSourceClient = new EventSourceClientImp("http://localhost:" + sseServer.getPort(), _notificationParser, _notificationProcessor, _pushStatusTracker, buildHttpClient()); + EventSourceClient eventSourceClient = new EventSourceClientImp("http://localhost:" + sseServer.getPort(), _notificationParser, _notificationProcessor, _pushStatusTracker, buildHttpClient(), telemetryRuntimeProducer); boolean result = eventSourceClient.start("channel-test","token-test"); @@ -52,8 +55,9 @@ public void startShouldConnect() throws IOException { public void startShouldNotConnect() throws IOException { SSEMockServer.SseEventQueue eventQueue = new SSEMockServer.SseEventQueue(); SSEMockServer sseServer = buildSSEMockServer(eventQueue); + TelemetryRuntimeProducer telemetryRuntimeProducer = Mockito.mock(InMemoryTelemetryStorage.class); sseServer.start(); - EventSourceClient eventSourceClient = new EventSourceClientImp("http://fake:" + sseServer.getPort(), _notificationParser, _notificationProcessor, _pushStatusTracker, buildHttpClient()); + EventSourceClient eventSourceClient = new EventSourceClientImp("http://fake:" + sseServer.getPort(), _notificationParser, _notificationProcessor, _pushStatusTracker, buildHttpClient(), telemetryRuntimeProducer); boolean result = eventSourceClient.start("channel-test","token-test"); @@ -68,8 +72,9 @@ public void startShouldNotConnect() throws IOException { public void startAndReceiveNotification() throws IOException { SSEMockServer.SseEventQueue eventQueue = new SSEMockServer.SseEventQueue(); SSEMockServer sseServer = buildSSEMockServer(eventQueue); + TelemetryRuntimeProducer telemetryRuntimeProducer = Mockito.mock(InMemoryTelemetryStorage.class); sseServer.start(); - EventSourceClient eventSourceClient = new EventSourceClientImp("http://localhost:" + sseServer.getPort(), _notificationParser, _notificationProcessor, _pushStatusTracker, buildHttpClient()); + EventSourceClient eventSourceClient = new EventSourceClientImp("http://localhost:" + sseServer.getPort(), _notificationParser, _notificationProcessor, _pushStatusTracker, buildHttpClient(), telemetryRuntimeProducer); boolean result = eventSourceClient.start("channel-test","token-test"); @@ -106,6 +111,8 @@ public void startAndReceiveNotification() throws IOException { Awaitility.await() .atMost(50L, TimeUnit.SECONDS) .untilAsserted(() -> Mockito.verify(_pushStatusTracker, Mockito.times(1)).handleIncomingAblyError(Mockito.any(ErrorNotification.class))); + + Mockito.verify(_pushStatusTracker, Mockito.times(1)).handleSseStatus(SSEClient.StatusMessage.FIRST_EVENT); } private SSEMockServer buildSSEMockServer(SSEMockServer.SseEventQueue eventQueue) { diff --git a/client/src/test/java/io/split/engine/sse/PushStatusTrackerTest.java b/client/src/test/java/io/split/engine/sse/PushStatusTrackerTest.java index 82d8fc554..c7ccec2fb 100644 --- a/client/src/test/java/io/split/engine/sse/PushStatusTrackerTest.java +++ b/client/src/test/java/io/split/engine/sse/PushStatusTrackerTest.java @@ -1,21 +1,31 @@ package io.split.engine.sse; import io.split.engine.common.PushManager; +import io.split.engine.sse.client.SSEClient; import io.split.engine.sse.dtos.*; +import io.split.telemetry.storage.InMemoryTelemetryStorage; +import io.split.telemetry.storage.TelemetryStorage; +import org.junit.Assert; import org.junit.Test; +import org.mockito.Mockito; import java.util.concurrent.LinkedBlockingQueue; +import java.util.concurrent.atomic.AtomicLong; +import java.util.concurrent.atomic.AtomicReference; import static org.hamcrest.Matchers.equalTo; import static org.hamcrest.Matchers.is; import static org.junit.Assert.assertThat; public class PushStatusTrackerTest { + private static final String CONTROL_PRI = "control_pri"; + private static final String CONTROL_SEC = "control_sec"; @Test public void HandleControlEventStreamingPausedShouldNotifyEvent() { + TelemetryStorage telemetryStorage = new InMemoryTelemetryStorage(); LinkedBlockingQueue messages = new LinkedBlockingQueue<>(); ControlNotification controlNotification = buildControlNotification(ControlType.STREAMING_PAUSED); - PushStatusTracker pushStatusTracker = new PushStatusTrackerImp(messages); + PushStatusTracker pushStatusTracker = new PushStatusTrackerImp(messages, telemetryStorage); pushStatusTracker.handleIncomingControlEvent(controlNotification); assertThat(messages.size(), is(equalTo(1))); @@ -24,8 +34,9 @@ public void HandleControlEventStreamingPausedShouldNotifyEvent() { @Test public void HandleControlEventStreamingResumedShouldNotifyEvent() throws InterruptedException { + TelemetryStorage telemetryStorage = new InMemoryTelemetryStorage(); LinkedBlockingQueue messages = new LinkedBlockingQueue<>(); - PushStatusTracker pushStatusTracker = new PushStatusTrackerImp(messages); + PushStatusTracker pushStatusTracker = new PushStatusTrackerImp(messages, telemetryStorage); pushStatusTracker.handleIncomingControlEvent(buildControlNotification(ControlType.STREAMING_PAUSED)); pushStatusTracker.handleIncomingControlEvent(buildControlNotification(ControlType.STREAMING_RESUMED)); @@ -36,25 +47,28 @@ public void HandleControlEventStreamingResumedShouldNotifyEvent() throws Interru @Test public void HandleControlEventStreamingResumedShouldNotNotifyEvent() { + TelemetryStorage telemetryStorage = new InMemoryTelemetryStorage(); LinkedBlockingQueue messages = new LinkedBlockingQueue<>(); - OccupancyNotification occupancyNotification = buildOccupancyNotification(0, null); + OccupancyNotification occupancyNotification = buildOccupancyNotification(0, CONTROL_PRI); ControlNotification controlNotification = buildControlNotification(ControlType.STREAMING_RESUMED); - PushStatusTracker pushStatusTracker = new PushStatusTrackerImp(messages); + PushStatusTracker pushStatusTracker = new PushStatusTrackerImp(messages, telemetryStorage); pushStatusTracker.handleIncomingOccupancyEvent(occupancyNotification); pushStatusTracker.handleIncomingControlEvent(controlNotification); pushStatusTracker.handleIncomingControlEvent(controlNotification); assertThat(messages.size(), is(equalTo(1))); assertThat(messages.peek(), is(equalTo(PushManager.Status.STREAMING_DOWN))); + Assert.assertEquals(1, telemetryStorage.popStreamingEvents().size()); } @Test public void HandleControlEventStreamingDisabledShouldNotifyShutdownEvent() { + TelemetryStorage telemetryStorage = new InMemoryTelemetryStorage(); LinkedBlockingQueue messages = new LinkedBlockingQueue<>(); ControlNotification controlNotification = buildControlNotification(ControlType.STREAMING_DISABLED); - PushStatusTracker pushStatusTracker = new PushStatusTrackerImp(messages); + PushStatusTracker pushStatusTracker = new PushStatusTrackerImp(messages, telemetryStorage); pushStatusTracker.handleIncomingControlEvent(controlNotification); pushStatusTracker.handleIncomingControlEvent(controlNotification); @@ -64,18 +78,21 @@ public void HandleControlEventStreamingDisabledShouldNotifyShutdownEvent() { @Test public void HandleOccupancyEventWithPublishersFirstTimeShouldNotNotifyEvent() { + TelemetryStorage telemetryStorage = new InMemoryTelemetryStorage(); LinkedBlockingQueue messages = new LinkedBlockingQueue<>(); - OccupancyNotification occupancyNotification = buildOccupancyNotification(2, null); + OccupancyNotification occupancyNotification = buildOccupancyNotification(2, CONTROL_SEC); - PushStatusTracker pushStatusTracker = new PushStatusTrackerImp(messages); + PushStatusTracker pushStatusTracker = new PushStatusTrackerImp(messages, telemetryStorage); pushStatusTracker.handleIncomingOccupancyEvent(occupancyNotification); assertThat(messages.size(), is(equalTo(0))); + Assert.assertEquals(1, telemetryStorage.popStreamingEvents().size()); } @Test public void HandleOccupancyEventWithPublishersAndWithStreamingDisabledShouldNotifyEvent() throws InterruptedException { + TelemetryStorage telemetryStorage = new InMemoryTelemetryStorage(); LinkedBlockingQueue messages = new LinkedBlockingQueue<>(); - PushStatusTracker pushStatusTracker = new PushStatusTrackerImp(messages); + PushStatusTracker pushStatusTracker = new PushStatusTrackerImp(messages, telemetryStorage); pushStatusTracker.handleIncomingOccupancyEvent(buildOccupancyNotification(0, null)); pushStatusTracker.handleIncomingOccupancyEvent(buildOccupancyNotification(2, null)); @@ -89,8 +106,9 @@ public void HandleOccupancyEventWithPublishersAndWithStreamingDisabledShouldNoti @Test public void HandleOccupancyEventWithDifferentChannelsPublishersShouldNotifyEvent() throws InterruptedException { + TelemetryStorage telemetryStorage = new InMemoryTelemetryStorage(); LinkedBlockingQueue messages = new LinkedBlockingQueue<>(); - PushStatusTracker pushStatusTracker = new PushStatusTrackerImp(messages); + PushStatusTracker pushStatusTracker = new PushStatusTrackerImp(messages, telemetryStorage); pushStatusTracker.handleIncomingOccupancyEvent(buildOccupancyNotification(0, "control_pri")); pushStatusTracker.handleIncomingOccupancyEvent(buildOccupancyNotification(2, "control_sec")); @@ -102,6 +120,28 @@ public void HandleOccupancyEventWithDifferentChannelsPublishersShouldNotifyEvent assertThat(m2, is(equalTo(PushManager.Status.STREAMING_READY))); } + @Test + public void handleSSESTatusRecordTelemetryStreamingEvent() { + TelemetryStorage telemetryStorage = Mockito.mock(InMemoryTelemetryStorage.class); + LinkedBlockingQueue messages = new LinkedBlockingQueue<>(); + PushStatusTracker pushStatusTracker = new PushStatusTrackerImp(messages, telemetryStorage); + pushStatusTracker.handleSseStatus(SSEClient.StatusMessage.CONNECTED); + pushStatusTracker.handleSseStatus(SSEClient.StatusMessage.FIRST_EVENT); + + Mockito.verify(telemetryStorage, Mockito.times(2)).recordStreamingEvents(Mockito.any()); + } + + @Test + public void handleAblyErrorRecordTelemetryStreamingEvent() { + TelemetryStorage telemetryStorage = Mockito.mock(InMemoryTelemetryStorage.class); + LinkedBlockingQueue messages = new LinkedBlockingQueue<>(); + PushStatusTracker pushStatusTracker = new PushStatusTrackerImp(messages, telemetryStorage); + pushStatusTracker.handleIncomingAblyError(new ErrorNotification("Error", "Ably error", 401)); + + Mockito.verify(telemetryStorage, Mockito.times(1)).recordStreamingEvents(Mockito.any()); + + } + private ControlNotification buildControlNotification(ControlType controlType) { return new ControlNotification(buildGenericData(controlType, IncomingNotification.Type.CONTROL,null, null)); } diff --git a/client/src/test/java/io/split/engine/sse/SSEClientTest.java b/client/src/test/java/io/split/engine/sse/SSEClientTest.java index 37cec7224..aa9e8ba97 100644 --- a/client/src/test/java/io/split/engine/sse/SSEClientTest.java +++ b/client/src/test/java/io/split/engine/sse/SSEClientTest.java @@ -1,6 +1,8 @@ package io.split.engine.sse; import io.split.engine.sse.client.SSEClient; +import io.split.telemetry.storage.InMemoryTelemetryStorage; +import io.split.telemetry.storage.TelemetryRuntimeProducer; import org.apache.hc.client5.http.config.RequestConfig; import org.apache.hc.client5.http.impl.classic.CloseableHttpClient; import org.apache.hc.client5.http.impl.classic.HttpClientBuilder; @@ -9,6 +11,7 @@ import org.apache.hc.core5.util.Timeout; import org.junit.Ignore; import org.junit.Test; +import org.mockito.Mockito; import java.net.URI; import java.net.URISyntaxException; @@ -23,6 +26,7 @@ public void basicUsageTest() throws URISyntaxException, InterruptedException { .addParameter("v", "1,1") .addParameter("channels", "[?occupancy=metrics.publishers]control_pri") .build(); + TelemetryRuntimeProducer telemetryRuntimeProducer = Mockito.mock(InMemoryTelemetryStorage.class); RequestConfig requestConfig = RequestConfig.custom() .setConnectTimeout(Timeout.ofMilliseconds(70000)) @@ -34,7 +38,7 @@ public void basicUsageTest() throws URISyntaxException, InterruptedException { CloseableHttpClient httpClient = httpClientbuilder.build(); SSEClient sse = new SSEClient(e -> { System.out.println(e); return null; }, - s -> { System.out.println(s); return null; }, httpClient); + s -> { System.out.println(s); return null; }, httpClient, telemetryRuntimeProducer); sse.open(uri); Thread.sleep(5000); sse.close(); diff --git a/client/src/test/java/io/split/service/HttpPostImpTest.java b/client/src/test/java/io/split/service/HttpPostImpTest.java index 1041d4593..60e71299c 100644 --- a/client/src/test/java/io/split/service/HttpPostImpTest.java +++ b/client/src/test/java/io/split/service/HttpPostImpTest.java @@ -1,9 +1,16 @@ package io.split.service; import io.split.TestHelper; +import io.split.telemetry.domain.enums.HTTPLatenciesEnum; +import io.split.telemetry.domain.enums.LastSynchronizationRecordsEnum; +import io.split.telemetry.domain.enums.ResourceEnum; +import io.split.telemetry.storage.InMemoryTelemetryStorage; +import io.split.telemetry.storage.TelemetryRuntimeProducer; +import io.split.telemetry.storage.TelemetryStorage; import junit.framework.TestCase; import org.apache.hc.client5.http.impl.classic.CloseableHttpClient; import org.apache.hc.core5.http.HttpStatus; +import org.junit.Assert; import org.junit.Test; import org.mockito.Mockito; @@ -18,16 +25,22 @@ public class HttpPostImpTest{ @Test public void testPostWith200() throws InvocationTargetException, NoSuchMethodException, IllegalAccessException, IOException { CloseableHttpClient client =TestHelper.mockHttpClient(URL, HttpStatus.SC_OK); - HttpPostImp httpPostImp = new HttpPostImp(client); - httpPostImp.post(URI.create(URL), new Object(), "Metrics"); + TelemetryStorage telemetryStorage = new InMemoryTelemetryStorage(); + HttpPostImp httpPostImp = new HttpPostImp(client, telemetryStorage); + httpPostImp.post(URI.create(URL), new Object(), "Metrics", HTTPLatenciesEnum.TELEMETRY, LastSynchronizationRecordsEnum.TELEMETRY, ResourceEnum.TELEMETRY_SYNC); Mockito.verify(client, Mockito.times(1)).execute(Mockito.any()); + Assert.assertNotEquals(0, telemetryStorage.getLastSynchronization().get_telemetry()); + Assert.assertEquals(1, telemetryStorage.popHTTPLatencies().get_telemetry().stream().mapToInt(Long::intValue).sum()); } @Test public void testPostWith400() throws InvocationTargetException, NoSuchMethodException, IllegalAccessException, IOException { CloseableHttpClient client =TestHelper.mockHttpClient(URL, HttpStatus.SC_CLIENT_ERROR); - HttpPostImp httpPostImp = new HttpPostImp(client); - httpPostImp.post(URI.create(URL), new Object(), "Metrics"); + TelemetryStorage telemetryStorage = new InMemoryTelemetryStorage(); + HttpPostImp httpPostImp = new HttpPostImp(client, telemetryStorage); + httpPostImp.post(URI.create(URL), new Object(), "Metrics", HTTPLatenciesEnum.TELEMETRY, LastSynchronizationRecordsEnum.TELEMETRY, ResourceEnum.TELEMETRY_SYNC); Mockito.verify(client, Mockito.times(1)).execute(Mockito.any()); + + Assert.assertEquals(1, telemetryStorage.popHTTPErrors().get_telemetry().get(Long.valueOf(HttpStatus.SC_CLIENT_ERROR)).intValue()); } } \ No newline at end of file diff --git a/client/src/test/java/io/split/telemetry/storage/InMemoryTelemetryStorageTest.java b/client/src/test/java/io/split/telemetry/storage/InMemoryTelemetryStorageTest.java index 49c1ce167..e00544149 100644 --- a/client/src/test/java/io/split/telemetry/storage/InMemoryTelemetryStorageTest.java +++ b/client/src/test/java/io/split/telemetry/storage/InMemoryTelemetryStorageTest.java @@ -192,10 +192,7 @@ public void testInMemoryTelemetryStorage() throws Exception { Assert.assertEquals(1, httpErrors.get_token().get(403l).intValue()); //Streaming events - StreamingEvent streamingEvent = new StreamingEvent(); - streamingEvent.set_data(290); - streamingEvent.set_type(1); - streamingEvent.setTimestamp(91218); + StreamingEvent streamingEvent = new StreamingEvent(1, 290, 91218); telemetryStorage.recordStreamingEvents(streamingEvent); List streamingEvents = telemetryStorage.popStreamingEvents(); diff --git a/client/src/test/java/io/split/telemetry/synchronizer/SynchronizerMemoryTest.java b/client/src/test/java/io/split/telemetry/synchronizer/SynchronizerMemoryTest.java index d7d9bee8c..ff45f1ba7 100644 --- a/client/src/test/java/io/split/telemetry/synchronizer/SynchronizerMemoryTest.java +++ b/client/src/test/java/io/split/telemetry/synchronizer/SynchronizerMemoryTest.java @@ -6,10 +6,12 @@ import io.split.cache.SplitCache; import io.split.client.SplitClientConfig; import io.split.telemetry.storage.InMemoryTelemetryStorage; +import io.split.telemetry.storage.TelemetryRuntimeProducer; import io.split.telemetry.storage.TelemetryStorageConsumer; import org.apache.hc.client5.http.impl.classic.CloseableHttpClient; import org.apache.hc.core5.http.HttpStatus; import org.junit.Test; +import org.mockito.Mock; import org.mockito.Mockito; import java.io.IOException; @@ -45,9 +47,10 @@ public void testSynchronizeStats() throws Exception { private TelemetrySynchronizer getTelemetrySynchronizer(CloseableHttpClient httpClient) throws URISyntaxException, InvocationTargetException, NoSuchMethodException, IllegalAccessException, IOException { TelemetryStorageConsumer consumer = Mockito.mock(InMemoryTelemetryStorage.class); + TelemetryRuntimeProducer telemetryRuntimeProducer = Mockito.mock(TelemetryRuntimeProducer.class); SplitCache splitCache = Mockito.mock(SplitCache.class); SegmentCache segmentCache = Mockito.mock(SegmentCacheInMemoryImpl.class); - TelemetrySynchronizer telemetrySynchronizer = new SynchronizerMemory(httpClient, URI.create(TELEMETRY_ENDPOINT), consumer, splitCache, segmentCache); + TelemetrySynchronizer telemetrySynchronizer = new SynchronizerMemory(httpClient, URI.create(TELEMETRY_ENDPOINT), consumer, splitCache, segmentCache, telemetryRuntimeProducer); return telemetrySynchronizer; } diff --git a/client/src/test/java/io/split/telemetry/synchronizer/TelemetrySyncTaskTest.java b/client/src/test/java/io/split/telemetry/synchronizer/TelemetrySyncTaskTest.java index 5058d3b5c..4113313b6 100644 --- a/client/src/test/java/io/split/telemetry/synchronizer/TelemetrySyncTaskTest.java +++ b/client/src/test/java/io/split/telemetry/synchronizer/TelemetrySyncTaskTest.java @@ -8,14 +8,16 @@ public class TelemetrySyncTaskTest { @Test public void testSynchronizationTask() throws Exception { TelemetrySynchronizer telemetrySynchronizer = Mockito.mock(SynchronizerMemory.class); + Mockito.doNothing().when(telemetrySynchronizer).synchronizeStats(); TelemetrySyncTask telemetrySyncTask = new TelemetrySyncTask(1, telemetrySynchronizer); - Thread.sleep(3000); + Thread.sleep(2900); Mockito.verify(telemetrySynchronizer, Mockito.times(3)).synchronizeStats(); } @Test public void testStopSynchronizationTask() throws Exception { TelemetrySynchronizer telemetrySynchronizer = Mockito.mock(SynchronizerMemory.class); +// Mockito.doNothing().when(telemetrySynchronizer).synchronizeStats(); TelemetrySyncTask telemetrySyncTask = new TelemetrySyncTask(1, telemetrySynchronizer); Thread.sleep(3000); Mockito.verify(telemetrySynchronizer, Mockito.times(3)).synchronizeStats(); From b28fd2538337cdd5b2cb1974253481370103481d Mon Sep 17 00:00:00 2001 From: Lucas Echeverz Date: Tue, 11 May 2021 12:11:43 -0300 Subject: [PATCH 44/81] Fixing Travis --- .../io/split/client/SplitManagerImpl.java | 6 ++--- .../io/split/client/SplitClientImplTest.java | 17 +++++++++--- .../io/split/client/SplitManagerImplTest.java | 26 ++++++++++++++----- .../impressions/ImpressionObserverTest.java | 2 ++ 4 files changed, 38 insertions(+), 13 deletions(-) diff --git a/client/src/main/java/io/split/client/SplitManagerImpl.java b/client/src/main/java/io/split/client/SplitManagerImpl.java index 3f73d91dd..4d41ac36e 100644 --- a/client/src/main/java/io/split/client/SplitManagerImpl.java +++ b/client/src/main/java/io/split/client/SplitManagerImpl.java @@ -43,7 +43,7 @@ public SplitManagerImpl(SplitCache splitCache, @Override public List splits() { - if (_gates.isSDKReadyNow()) { { + if (!_gates.isSDKReadyNow()) { { _log.warn("splits: the SDK is not ready, results may be incorrect. Make sure to wait for SDK readiness before using this method"); _telemetryConfigProducer.recordNonReadyUsage(); }} @@ -58,7 +58,7 @@ public List splits() { @Override public SplitView split(String featureName) { - if (_gates.isSDKReadyNow()) { { + if (!_gates.isSDKReadyNow()) { { _log.warn("split: the SDK is not ready, results may be incorrect. Make sure to wait for SDK readiness before using this method"); _telemetryConfigProducer.recordNonReadyUsage(); }} @@ -82,7 +82,7 @@ public SplitView split(String featureName) { @Override public List splitNames() { - if (_gates.isSDKReadyNow()) { { + if (!_gates.isSDKReadyNow()) { { _log.warn("splitNames: the SDK is not ready, results may be incorrect. Make sure to wait for SDK readiness before using this method"); _telemetryConfigProducer.recordNonReadyUsage(); }} diff --git a/client/src/test/java/io/split/client/SplitClientImplTest.java b/client/src/test/java/io/split/client/SplitClientImplTest.java index d4d7a3353..aa508cd2f 100644 --- a/client/src/test/java/io/split/client/SplitClientImplTest.java +++ b/client/src/test/java/io/split/client/SplitClientImplTest.java @@ -23,14 +23,13 @@ import io.split.engine.matchers.GreaterThanOrEqualToMatcher; import io.split.engine.matchers.collections.ContainsAnyOfSetMatcher; import io.split.engine.matchers.strings.WhitelistMatcher; -import io.split.engine.metrics.Metrics; import io.split.grammar.Treatments; import io.split.telemetry.storage.InMemoryTelemetryStorage; import io.split.telemetry.storage.TelemetryStorage; import org.apache.commons.lang3.RandomStringUtils; import org.junit.Assert; +import org.junit.Before; import org.junit.Test; -import org.junit.rules.ExpectedException; import org.mockito.ArgumentCaptor; import org.mockito.Mockito; @@ -63,9 +62,14 @@ */ public class SplitClientImplTest { - private static final TelemetryStorage TELEMETRY_STORAGE = Mockito.mock(InMemoryTelemetryStorage.class); + private static TelemetryStorage TELEMETRY_STORAGE = Mockito.mock(InMemoryTelemetryStorage.class); private SplitClientConfig config = SplitClientConfig.builder().setBlockUntilReadyTimeout(100).build(); + @Before + public void updateTelemetryStorage() { + TELEMETRY_STORAGE = Mockito.mock(InMemoryTelemetryStorage.class); + } + @Test public void null_key_results_in_control() { String test = "test1"; @@ -149,6 +153,7 @@ public void works() { SDKReadinessGates gates = mock(SDKReadinessGates.class); SplitCache splitCache = mock(InMemoryCacheImp.class); when(splitCache.get(test)).thenReturn(parsedSplit); + when(gates.isSDKReadyNow()).thenReturn(true); SplitClientImpl client = new SplitClientImpl( mock(SplitFactory.class), @@ -167,6 +172,7 @@ public void works() { } verify(splitCache, times(numKeys)).get(test); + verify(TELEMETRY_STORAGE, times(5)).recordLatency(Mockito.anyObject(), Mockito.anyLong()); } /** @@ -320,6 +326,7 @@ public void multiple_conditions_work() { SDKReadinessGates gates = mock(SDKReadinessGates.class); SplitCache splitCache = mock(InMemoryCacheImp.class); when(splitCache.get(test)).thenReturn(parsedSplit); + when(gates.isSDKReadyNow()).thenReturn(false); SplitClientImpl client = new SplitClientImpl( mock(SplitFactory.class), @@ -336,6 +343,7 @@ public void multiple_conditions_work() { assertThat(client.getTreatment("trevor@codigo.com", test), is(equalTo("on"))); verify(splitCache, times(3)).get(test); + verify(TELEMETRY_STORAGE, times(3)).recordNonReadyUsage(); } @@ -935,7 +943,7 @@ public void block_until_ready_times_when_sdk_is_not_ready() throws TimeoutExcept public void track_with_valid_parameters() { SDKReadinessGates gates = mock(SDKReadinessGates.class); SplitCache splitCache = mock(InMemoryCacheImp.class); - + when(gates.isSDKReadyNow()).thenReturn(false); SplitClientImpl client = new SplitClientImpl( mock(SplitFactory.class), splitCache, @@ -953,6 +961,7 @@ public void track_with_valid_parameters() { String validKeySize = new String(new char[250]).replace('\0', 'a'); Assert.assertThat(client.track(validKeySize, "valid_traffic_type", validEventSize, 10), org.hamcrest.Matchers.is(org.hamcrest.Matchers.equalTo(true))); + verify(TELEMETRY_STORAGE, times(2)).recordLatency(Mockito.anyObject(), Mockito.anyLong()); } diff --git a/client/src/test/java/io/split/client/SplitManagerImplTest.java b/client/src/test/java/io/split/client/SplitManagerImplTest.java index c156ed973..a77674045 100644 --- a/client/src/test/java/io/split/client/SplitManagerImplTest.java +++ b/client/src/test/java/io/split/client/SplitManagerImplTest.java @@ -12,6 +12,7 @@ import io.split.grammar.Treatments; import io.split.telemetry.storage.InMemoryTelemetryStorage; import io.split.telemetry.storage.TelemetryStorage; +import org.junit.Before; import org.junit.Test; import org.mockito.Mockito; @@ -25,14 +26,17 @@ import static org.hamcrest.Matchers.is; import static org.hamcrest.Matchers.nullValue; import static org.junit.Assert.assertThat; -import static org.mockito.Mockito.mock; -import static org.mockito.Mockito.when; +import static org.mockito.Mockito.*; public class SplitManagerImplTest { private SplitClientConfig config = SplitClientConfig.builder().setBlockUntilReadyTimeout(100).build(); - private static final TelemetryStorage TELEMETRY_STORAGE = Mockito.mock(InMemoryTelemetryStorage.class); + private static TelemetryStorage TELEMETRY_STORAGE = Mockito.mock(InMemoryTelemetryStorage.class); + @Before + public void updateTelemetryStorage() { + TELEMETRY_STORAGE = Mockito.mock(InMemoryTelemetryStorage.class); + } @Test public void splitCallWithNonExistentSplit() { String nonExistent = "nonExistent"; @@ -95,23 +99,28 @@ public void splitCallWithExistentSplitAndConfigs() { public void splitsCallWithNoSplit() { SplitCache splitCache = Mockito.mock(SplitCache.class); Mockito.when(splitCache.getAll()).thenReturn(Lists.newArrayList()); + SDKReadinessGates gates = Mockito.mock(SDKReadinessGates.class); + Mockito.when(gates.isSDKReadyNow()).thenReturn(false); SplitManagerImpl splitManager = new SplitManagerImpl(splitCache, Mockito.mock(SplitClientConfig.class), - Mockito.mock(SDKReadinessGates.class), TELEMETRY_STORAGE); + gates, TELEMETRY_STORAGE); assertThat(splitManager.splits(), is(empty())); + verify(TELEMETRY_STORAGE, times(1)).recordNonReadyUsage(); } @Test public void splitsCallWithSplit() { SplitCache splitCache = Mockito.mock(SplitCache.class); List parsedSplits = Lists.newArrayList(); + SDKReadinessGates gates = Mockito.mock(SDKReadinessGates.class); + Mockito.when(gates.isSDKReadyNow()).thenReturn(false); ParsedSplit response = ParsedSplit.createParsedSplitForTests("FeatureName", 123, true, "off", Lists.newArrayList(getTestCondition("off")), "traffic", 456L, 1); parsedSplits.add(response); Mockito.when(splitCache.getAll()).thenReturn(parsedSplits); SplitManagerImpl splitManager = new SplitManagerImpl(splitCache, Mockito.mock(SplitClientConfig.class), - Mockito.mock(SDKReadinessGates.class), TELEMETRY_STORAGE); + gates, TELEMETRY_STORAGE); List splits = splitManager.splits(); assertThat(splits.size(), is(equalTo(1))); assertThat(splits.get(0).name, is(equalTo(response.feature()))); @@ -120,16 +129,20 @@ public void splitsCallWithSplit() { assertThat(splits.get(0).trafficType, is(equalTo(response.trafficTypeName()))); assertThat(splits.get(0).treatments.size(), is(equalTo(1))); assertThat(splits.get(0).treatments.get(0), is(equalTo("off"))); + verify(TELEMETRY_STORAGE, times(1)).recordNonReadyUsage(); } @Test public void splitNamesCallWithNoSplit() { SplitCache splitCache = Mockito.mock(SplitCache.class); Mockito.when(splitCache.getAll()).thenReturn(Lists.newArrayList()); + SDKReadinessGates gates = Mockito.mock(SDKReadinessGates.class); + Mockito.when(gates.isSDKReadyNow()).thenReturn(false); SplitManagerImpl splitManager = new SplitManagerImpl(splitCache, Mockito.mock(SplitClientConfig.class), - Mockito.mock(SDKReadinessGates.class), TELEMETRY_STORAGE); + gates, TELEMETRY_STORAGE); assertThat(splitManager.splitNames(), is(empty())); + verify(TELEMETRY_STORAGE, times(1)).recordNonReadyUsage(); } @Test @@ -169,6 +182,7 @@ public void block_until_ready_times_when_sdk_is_not_ready() throws TimeoutExcept ready, TELEMETRY_STORAGE); splitManager.blockUntilReady(); + verify(TELEMETRY_STORAGE, times(1)).recordBURTimeout(); } private ParsedCondition getTestCondition(String treatment) { diff --git a/client/src/test/java/io/split/client/impressions/ImpressionObserverTest.java b/client/src/test/java/io/split/client/impressions/ImpressionObserverTest.java index 564ce9e34..2fe3611e7 100644 --- a/client/src/test/java/io/split/client/impressions/ImpressionObserverTest.java +++ b/client/src/test/java/io/split/client/impressions/ImpressionObserverTest.java @@ -1,6 +1,7 @@ package io.split.client.impressions; import com.google.common.base.Strings; +import org.junit.Ignore; import org.junit.Test; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -118,6 +119,7 @@ private void caller(ImpressionObserver o, int count, ConcurrentLinkedQueue imps = new ConcurrentLinkedQueue<>(); From f5b39c5b1cf2fe227adafe2e6c357712fad7b111 Mon Sep 17 00:00:00 2001 From: Lucas Echeverz Date: Wed, 12 May 2021 18:35:35 -0300 Subject: [PATCH 45/81] Cleaning metrics and adding Telemetry config initializer --- .../java/io/split/client/ApiKeyCounter.java | 12 ++ .../io/split/client/SplitFactoryImpl.java | 5 +- .../metrics/BinarySearchLatencyTracker.java | 131 ----------------- .../io/split/client/metrics/DTOMetrics.java | 13 -- .../io/split/client/metrics/HttpMetrics.java | 135 ------------------ .../split/client/metrics/ILatencyTracker.java | 22 --- .../LogarithmicSearchLatencyTracker.java | 116 --------------- .../TelemetryConfigInitializer.java | 33 +++++ .../io/split/client/ApiKeyCounterTest.java | 16 +++ .../BinarySearchLatencyTrackerTest.java | 92 ------------ .../split/client/metrics/HttpMetricsTest.java | 48 ------- .../LogarithmicSearchLatencyTrackerTest.java | 112 --------------- .../TelemetryConfigInitializerTest.java | 20 +++ 13 files changed, 85 insertions(+), 670 deletions(-) delete mode 100644 client/src/main/java/io/split/client/metrics/BinarySearchLatencyTracker.java delete mode 100644 client/src/main/java/io/split/client/metrics/DTOMetrics.java delete mode 100644 client/src/main/java/io/split/client/metrics/HttpMetrics.java delete mode 100644 client/src/main/java/io/split/client/metrics/ILatencyTracker.java delete mode 100644 client/src/main/java/io/split/client/metrics/LogarithmicSearchLatencyTracker.java create mode 100644 client/src/main/java/io/split/telemetry/synchronizer/TelemetryConfigInitializer.java delete mode 100644 client/src/test/java/io/split/client/metrics/BinarySearchLatencyTrackerTest.java delete mode 100644 client/src/test/java/io/split/client/metrics/HttpMetricsTest.java delete mode 100644 client/src/test/java/io/split/client/metrics/LogarithmicSearchLatencyTrackerTest.java create mode 100644 client/src/test/java/io/split/telemetry/synchronizer/TelemetryConfigInitializerTest.java diff --git a/client/src/main/java/io/split/client/ApiKeyCounter.java b/client/src/main/java/io/split/client/ApiKeyCounter.java index 8c39394dd..546e9c78b 100644 --- a/client/src/main/java/io/split/client/ApiKeyCounter.java +++ b/client/src/main/java/io/split/client/ApiKeyCounter.java @@ -6,6 +6,9 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import java.util.HashMap; +import java.util.Map; + public class ApiKeyCounter { private static final Logger _log = LoggerFactory.getLogger(ApiKeyCounter.class); @@ -63,4 +66,13 @@ boolean isApiKeyPresent(String apiKey) { int getCount(String apiKey) { return USED_API_KEYS.count(apiKey); } + + public Map getFactoryInstances() { + Map factoryInstances = new HashMap<>(); + for (String factory :USED_API_KEYS) { + factoryInstances.putIfAbsent(factory, new Long(getCount(factory))); + } + + return factoryInstances; + } } diff --git a/client/src/main/java/io/split/client/SplitFactoryImpl.java b/client/src/main/java/io/split/client/SplitFactoryImpl.java index 87edaefda..e3455d4da 100644 --- a/client/src/main/java/io/split/client/SplitFactoryImpl.java +++ b/client/src/main/java/io/split/client/SplitFactoryImpl.java @@ -8,7 +8,6 @@ import io.split.client.interceptors.GzipDecoderResponseInterceptor; import io.split.client.interceptors.GzipEncoderRequestInterceptor; import io.split.client.interceptors.SdkMetadataInterceptorFilter; -import io.split.client.metrics.HttpMetrics; import io.split.cache.InMemoryCacheImp; import io.split.cache.SplitCache; import io.split.engine.evaluator.Evaluator; @@ -29,6 +28,7 @@ import io.split.telemetry.storage.InMemoryTelemetryStorage; import io.split.telemetry.storage.TelemetryStorage; import io.split.telemetry.synchronizer.SynchronizerMemory; +import io.split.telemetry.synchronizer.TelemetryConfigInitializer; import io.split.telemetry.synchronizer.TelemetrySyncTask; import io.split.telemetry.synchronizer.TelemetrySynchronizer; import org.apache.hc.client5.http.auth.AuthScope; @@ -95,6 +95,7 @@ public class SplitFactoryImpl implements SplitFactory { private final TelemetrySynchronizer _telemetrySynchronizer; private final TelemetrySyncTask _telemetrySyncTask; private final long _startTime; + private final TelemetryConfigInitializer _telemetryConfigInitializer; public SplitFactoryImpl(String apiToken, SplitClientConfig config) throws URISyntaxException { _startTime = System.currentTimeMillis(); @@ -126,6 +127,8 @@ public SplitFactoryImpl(String apiToken, SplitClientConfig config) throws URISyn _splitCache = new InMemoryCacheImp(); _telemetrySynchronizer = new SynchronizerMemory(_httpclient, URI.create(config.get_telemetryURL()), _telemetryStorage, _splitCache, _segmentCache, _telemetryStorage); _telemetrySyncTask = new TelemetrySyncTask(config.get_telemetryRefreshRate(), _telemetrySynchronizer); + _telemetryConfigInitializer = new TelemetryConfigInitializer(_telemetrySynchronizer,_gates,config); + // Segments _segmentSynchronizationTaskImp = buildSegments(config); diff --git a/client/src/main/java/io/split/client/metrics/BinarySearchLatencyTracker.java b/client/src/main/java/io/split/client/metrics/BinarySearchLatencyTracker.java deleted file mode 100644 index 35efff10f..000000000 --- a/client/src/main/java/io/split/client/metrics/BinarySearchLatencyTracker.java +++ /dev/null @@ -1,131 +0,0 @@ -package io.split.client.metrics; - -import java.util.Arrays; - -/** - * Tracks latencies pero bucket of time. - * Each bucket represent a latency greater than the one before - * and each number within each bucket is a number of calls in the range. - *

    - * (1) 1.00 - * (2) 1.50 - * (3) 2.25 - * (4) 3.38 - * (5) 5.06 - * (6) 7.59 - * (7) 11.39 - * (8) 17.09 - * (9) 25.63 - * (10) 38.44 - * (11) 57.67 - * (12) 86.50 - * (13) 129.75 - * (14) 194.62 - * (15) 291.93 - * (16) 437.89 - * (17) 656.84 - * (18) 985.26 - * (19) 1,477.89 - * (20) 2,216.84 - * (21) 3,325.26 - * (22) 4,987.89 - * (23) 7,481.83 - *

    - * Thread-safety: This class is not thread safe. - *

    - * Created by patricioe on 2/10/16. - */ -public class BinarySearchLatencyTracker implements ILatencyTracker { - - static final long[] BUCKETS = { - 1000, 1500, 2250, 3375, 5063, - 7594, 11391, 17086, 25629, 38443, - 57665, 86498, 129746, 194620, 291929, - 437894, 656841, 985261, 1477892, 2216838, - 3325257, 4987885, 7481828 - }; - - static final long MAX_LATENCY = 7481828; - - long[] latencies = new long[BUCKETS.length]; - - /** - * Increment the internal counter for the bucket this latency falls into. - * - * @param millis - */ - public void addLatencyMillis(long millis) { - int index = findIndex(millis * 1000); - latencies[index]++; - } - - /** - * Increment the internal counter for the bucket this latency falls into. - * - * @param micros - */ - public void addLatencyMicros(long micros) { - int index = findIndex(micros); - latencies[index]++; - } - - /** - * Returns the list of latencies buckets as an array. - * - * @return the list of latencies buckets as an array. - */ - public long[] getLatencies() { - return latencies; - } - - @Override - public long getLatency(int index) { - return latencies[index]; - } - - public void clear() { - latencies = new long[BUCKETS.length]; - } - - /** - * Returns the counts in the bucket this latency falls into. - * The latencies will no be updated. - * - * @param latency - * @return the bucket content for the latency. - */ - public long getBucketForLatencyMillis(long latency) { - return latencies[findIndex(latency * 1000)]; - } - - /** - * Returns the counts in the bucket this latency falls into. - * The latencies will no be updated. - * - * @param latency - * @return the bucket content for the latency. - */ - public long getBucketForLatencyMicros(long latency) { - return latencies[findIndex(latency)]; - } - - - private int findIndex(long micros) { - if (micros > MAX_LATENCY) { - return BUCKETS.length - 1; - } - - int index = Arrays.binarySearch(BUCKETS, micros); - - if (index < 0) { - - // Adjust the index based on Java Array javadocs. <0 means the value wasn't found and it's module value - // is where it should be inserted (in this case, it means the counter it applies - unless it's equals to the - // length of the array). - - index = -(index + 1); - } - return index; - } - -} diff --git a/client/src/main/java/io/split/client/metrics/DTOMetrics.java b/client/src/main/java/io/split/client/metrics/DTOMetrics.java deleted file mode 100644 index 86c793cb6..000000000 --- a/client/src/main/java/io/split/client/metrics/DTOMetrics.java +++ /dev/null @@ -1,13 +0,0 @@ -package io.split.client.metrics; - -import io.split.client.dtos.Counter; -import io.split.client.dtos.Latency; - -/** - * Created by adilaijaz on 6/14/16. - */ -public interface DTOMetrics { - void time(Latency dto); - - void count(Counter dto); -} diff --git a/client/src/main/java/io/split/client/metrics/HttpMetrics.java b/client/src/main/java/io/split/client/metrics/HttpMetrics.java deleted file mode 100644 index 14d63b0f4..000000000 --- a/client/src/main/java/io/split/client/metrics/HttpMetrics.java +++ /dev/null @@ -1,135 +0,0 @@ -package io.split.client.metrics; - -import com.google.common.annotations.VisibleForTesting; -import com.google.common.base.Preconditions; -import com.google.common.collect.Lists; -import io.split.client.dtos.Counter; -import io.split.client.dtos.Latency; -import io.split.client.utils.Utils; -import io.split.engine.metrics.Metrics; -import org.apache.hc.client5.http.classic.methods.HttpPost; -import org.apache.hc.client5.http.impl.classic.CloseableHttpClient; -import org.apache.hc.client5.http.impl.classic.CloseableHttpResponse; -import org.apache.hc.core5.http.HttpEntity; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -import java.net.URI; -import java.net.URISyntaxException; - -/** - * Created by adilaijaz on 9/4/15. - */ -public class HttpMetrics implements Metrics, DTOMetrics { - private static final Logger _log = LoggerFactory.getLogger(HttpMetrics.class); - - private final CloseableHttpClient _client; - private final URI _timeTarget; - private final URI _counterTarget; - - - public static HttpMetrics create(CloseableHttpClient client, URI root) throws URISyntaxException { - return new HttpMetrics(client, root); - } - - - public HttpMetrics(CloseableHttpClient client, URI root) throws URISyntaxException { - Preconditions.checkNotNull(root); - _client = Preconditions.checkNotNull(client); - _timeTarget = Utils.appendPath(root, "api/metrics/time"); - _counterTarget = Utils.appendPath(root, "api/metrics/counter"); - } - - - @Override - public void time(Latency dto) { - if (dto.latencies.isEmpty()) { - return; - } - - try { - post(_timeTarget, dto); - } catch (Throwable t) { - _log.warn("Exception when posting metric " + dto, t); - } - ; - - } - - @Override - public void count(Counter dto) { - try { - post(_counterTarget, dto); - } catch (Throwable t) { - _log.warn("Exception when posting metric " + dto, t); - } - - } - - private void post(URI uri, Object dto) { - - CloseableHttpResponse response = null; - - try { - HttpEntity entity = Utils.toJsonEntity(dto); - - HttpPost request = new HttpPost(uri); - request.setEntity(entity); - - response = _client.execute(request); - - int status = response.getCode(); - - if (status < 200 || status >= 300) { - _log.warn("Response status was: " + status); - } - - } catch (Throwable t) { - _log.warn("Exception when posting metrics: " + t.getMessage()); - if (_log.isDebugEnabled()) { - _log.debug("Reason: ", t); - } - } finally { - Utils.forceClose(response); - } - - } - - @Override - public void count(String counter, long delta) { - try { - Counter dto = new Counter(); - dto.name = counter; - dto.delta = delta; - - count(dto); - } catch (Throwable t) { - _log.info("Could not count metric " + counter, t); - } - - } - - @Override - public void time(String operation, long timeInMs) { - try { - Latency dto = new Latency(); - dto.name = operation; - dto.latencies = Lists.newArrayList(timeInMs); - - time(dto); - } catch (Throwable t) { - _log.info("Could not time metric " + operation, t); - } - } - - @VisibleForTesting - URI getTimeTarget() { - return _timeTarget; - } - - @VisibleForTesting - URI getCounterTarget() { - return _counterTarget; - } - -} diff --git a/client/src/main/java/io/split/client/metrics/ILatencyTracker.java b/client/src/main/java/io/split/client/metrics/ILatencyTracker.java deleted file mode 100644 index 282db06cb..000000000 --- a/client/src/main/java/io/split/client/metrics/ILatencyTracker.java +++ /dev/null @@ -1,22 +0,0 @@ -package io.split.client.metrics; - -/** - * Created by patricioe on 2/10/16. - */ -public interface ILatencyTracker { - - void addLatencyMillis(long millis); - - void addLatencyMicros(long micros); - - long[] getLatencies(); - - long getLatency(int index); - - void clear(); - - long getBucketForLatencyMillis(long latency); - - long getBucketForLatencyMicros(long latency); - -} diff --git a/client/src/main/java/io/split/client/metrics/LogarithmicSearchLatencyTracker.java b/client/src/main/java/io/split/client/metrics/LogarithmicSearchLatencyTracker.java deleted file mode 100644 index 4034d8de6..000000000 --- a/client/src/main/java/io/split/client/metrics/LogarithmicSearchLatencyTracker.java +++ /dev/null @@ -1,116 +0,0 @@ -package io.split.client.metrics; - -/** - * Tracks latencies pero bucket of time. - * Each bucket represent a latency greater than the one before - * and each number within each bucket is a number of calls in the range. - *

    - * (1) 1.00 - * (2) 1.50 - * (3) 2.25 - * (4) 3.38 - * (5) 5.06 - * (6) 7.59 - * (7) 11.39 - * (8) 17.09 - * (9) 25.63 - * (10) 38.44 - * (11) 57.67 - * (12) 86.50 - * (13) 129.75 - * (14) 194.62 - * (15) 291.93 - * (16) 437.89 - * (17) 656.84 - * (18) 985.26 - * (19) 1,477.89 - * (20) 2,216.84 - * (21) 3,325.26 - * (22) 4,987.89 - * (23) 7,481.83 - *

    - * Thread-safety: This class is not thread safe. - *

    - * Created by patricioe on 2/10/16. - */ -public class LogarithmicSearchLatencyTracker implements ILatencyTracker { - - static final int BUCKETS = 23; - private static final double LOG_10_1000_MICROS = Math.log10(1000); - private static final double LOG_10_1_5_MICROS = Math.log10(Double.valueOf("1.5").doubleValue()); - - - long[] latencies = new long[BUCKETS]; - - /** - * Increment the internal counter for the bucket this latency falls into. - * - * @param millis - */ - public void addLatencyMillis(long millis) { - int index = findIndex(millis * 1000); - latencies[index]++; - } - - /** - * Increment the internal counter for the bucket this latency falls into. - * - * @param micros - */ - public void addLatencyMicros(long micros) { - int index = findIndex(micros); - latencies[index]++; - } - - /** - * Returns the list of latencies buckets as an array. - * - * @return the list of latencies buckets as an array. - */ - public long[] getLatencies() { - return latencies; - } - - @Override - public long getLatency(int index) { - return latencies[index]; - } - - public void clear() { - latencies = new long[BUCKETS]; - } - - /** - * Returns the counts in the bucket this latency falls into. - * The latencies will no be updated. - * - * @param latency - * @return the bucket content for the latency. - */ - public long getBucketForLatencyMillis(long latency) { - return latencies[findIndex(latency * 1000)]; - } - - /** - * Returns the counts in the bucket this latency falls into. - * The latencies will no be updated. - * - * @param latency - * @return the bucket content for the latency. - */ - public long getBucketForLatencyMicros(long latency) { - return latencies[findIndex(latency)]; - } - - - private int findIndex(long micros) { - - if (micros <= 1000) return 0; - if (micros > 4987885) return 22; - - double raw = (Math.log10(micros) - LOG_10_1000_MICROS) / LOG_10_1_5_MICROS; - double rounded = Math.round(raw * 1000000d) / 1000000d; - return (int) Math.ceil(rounded); - } - -} diff --git a/client/src/main/java/io/split/telemetry/synchronizer/TelemetryConfigInitializer.java b/client/src/main/java/io/split/telemetry/synchronizer/TelemetryConfigInitializer.java new file mode 100644 index 000000000..b53dce7ae --- /dev/null +++ b/client/src/main/java/io/split/telemetry/synchronizer/TelemetryConfigInitializer.java @@ -0,0 +1,33 @@ +package io.split.telemetry.synchronizer; + +import io.split.client.ApiKeyCounter; +import io.split.client.SplitClientConfig; +import io.split.engine.SDKReadinessGates; + +import java.util.ArrayList; + +import static com.google.common.base.Preconditions.checkNotNull; + +public class TelemetryConfigInitializer { + + private final TelemetrySynchronizer _telemetrySynchronizer; + private final SDKReadinessGates _gates; + private final SplitClientConfig _config; + + public TelemetryConfigInitializer(TelemetrySynchronizer _telemetrySynchronizer, SDKReadinessGates _gates, SplitClientConfig config) { + this._telemetrySynchronizer = checkNotNull(_telemetrySynchronizer); + this._gates = checkNotNull(_gates); + _config = checkNotNull(config); + this.waitForSDKReady(); + } + + private void waitForSDKReady() { + long initTime = System.currentTimeMillis(); + while(true) { + if (_gates.isSDKReadyNow()) { + _telemetrySynchronizer.synchronizeConfig(_config,System.currentTimeMillis()-initTime, ApiKeyCounter.getApiKeyCounterInstance().getFactoryInstances(),new ArrayList<>()); + break; + } + } + } +} diff --git a/client/src/test/java/io/split/client/ApiKeyCounterTest.java b/client/src/test/java/io/split/client/ApiKeyCounterTest.java index c017127ce..bf1f21f11 100644 --- a/client/src/test/java/io/split/client/ApiKeyCounterTest.java +++ b/client/src/test/java/io/split/client/ApiKeyCounterTest.java @@ -1,8 +1,11 @@ package io.split.client; import junit.framework.TestCase; +import org.junit.Assert; import org.junit.Test; +import java.util.Map; + public class ApiKeyCounterTest extends TestCase { private static final String FIRST_KEY = "KEYNUMBER1"; @@ -47,4 +50,17 @@ public void testAddingNonExistingToken() { ApiKeyCounter.getApiKeyCounterInstance().remove(FIRST_KEY); ApiKeyCounter.getApiKeyCounterInstance().remove(SECOND_KEY); } + + @Test + public void testFactoryInstances() { + ApiKeyCounter.getApiKeyCounterInstance().add(FIRST_KEY); + ApiKeyCounter.getApiKeyCounterInstance().add(FIRST_KEY); + ApiKeyCounter.getApiKeyCounterInstance().add(FIRST_KEY); + ApiKeyCounter.getApiKeyCounterInstance().add(SECOND_KEY); + ApiKeyCounter.getApiKeyCounterInstance().add(SECOND_KEY); + + Map factoryInstances = ApiKeyCounter.getApiKeyCounterInstance().getFactoryInstances(); + Assert.assertEquals(2, factoryInstances.size()); + Assert.assertEquals(3, factoryInstances.get(FIRST_KEY).intValue()); + } } diff --git a/client/src/test/java/io/split/client/metrics/BinarySearchLatencyTrackerTest.java b/client/src/test/java/io/split/client/metrics/BinarySearchLatencyTrackerTest.java deleted file mode 100644 index c04fa5b49..000000000 --- a/client/src/test/java/io/split/client/metrics/BinarySearchLatencyTrackerTest.java +++ /dev/null @@ -1,92 +0,0 @@ -package io.split.client.metrics; - -import org.junit.Before; -import org.junit.Test; - -import static org.hamcrest.Matchers.equalTo; -import static org.hamcrest.Matchers.is; -import static org.junit.Assert.*; - -public class BinarySearchLatencyTrackerTest { - - BinarySearchLatencyTracker tracker; - - @Before - public void before() { - tracker = new BinarySearchLatencyTracker(); - } - - /** - * Latencies of <=1 millis or <= 1000 micros correspond to the first bucket (index 0) - */ - @Test - public void testLessThanFirstBucket() { - - tracker.addLatencyMicros(750); - tracker.addLatencyMicros(450); - assertThat(tracker.getLatency(0), is(equalTo(2L))); - - tracker.addLatencyMillis(0); - assertThat(tracker.getLatency(0), is(equalTo(3L))); - } - - /** - * Latencies of 1 millis or <= 1000 micros correspond to the first bucket (index 0) - */ - @Test - public void testFirstBucket() { - - tracker.addLatencyMicros(1000); - assertThat(tracker.getLatency(0), is(equalTo(1L))); - - tracker.addLatencyMillis(1); - assertThat(tracker.getLatency(0), is(equalTo(2L))); - } - - /** - * Latencies of 7481 millis or 7481828 micros correspond to the last bucket (index 22) - */ - @Test - public void testLastBucket() { - - tracker.addLatencyMicros(7481828); - assertThat(tracker.getLatency(22), is(equalTo(1L))); - - tracker.addLatencyMillis(7481); - assertThat(tracker.getLatency(22), is(equalTo(2L))); - } - - /** - * Latencies of more than 7481 millis or 7481828 micros correspond to the last bucket (index 22) - */ - @Test - public void testGreaterThanLastBucket() { - - tracker.addLatencyMicros(7481830); - assertThat(tracker.getLatency(22), is(equalTo(1L))); - - tracker.addLatencyMicros(7999999); - assertThat(tracker.getLatency(22), is(equalTo(2L))); - - tracker.addLatencyMillis(7482); - assertThat(tracker.getLatency(22), is(equalTo(3L))); - - tracker.addLatencyMillis(8000); - assertThat(tracker.getLatency(22), is(equalTo(4L))); - } - - /** - * Latencies between 11,392 and 17,086 are in the 8th bucket. - */ - @Test - public void test8ThBucket() { - - tracker.addLatencyMicros(11392); - assertThat(tracker.getLatency(7), is(equalTo(1L))); - - tracker.addLatencyMicros(17086); - assertThat(tracker.getLatency(7), is(equalTo(2L))); - - } - -} \ No newline at end of file diff --git a/client/src/test/java/io/split/client/metrics/HttpMetricsTest.java b/client/src/test/java/io/split/client/metrics/HttpMetricsTest.java deleted file mode 100644 index 71ade9dbe..000000000 --- a/client/src/test/java/io/split/client/metrics/HttpMetricsTest.java +++ /dev/null @@ -1,48 +0,0 @@ -package io.split.client.metrics; - -import org.apache.hc.client5.http.impl.classic.CloseableHttpClient; -import org.apache.hc.client5.http.impl.classic.HttpClients; -import org.hamcrest.Matchers; -import org.junit.Assert; -import org.junit.Test; - -import java.net.URI; -import java.net.URISyntaxException; - -public class HttpMetricsTest { - @Test - public void testDefaultURL() throws URISyntaxException { - URI rootTarget = URI.create("https://api.split.io"); - CloseableHttpClient httpClient = HttpClients.custom().build(); - HttpMetrics fetcher = HttpMetrics.create(httpClient, rootTarget); - Assert.assertThat(fetcher.getTimeTarget().toString(), Matchers.is(Matchers.equalTo("https://api.split.io/api/metrics/time"))); - Assert.assertThat(fetcher.getCounterTarget().toString(), Matchers.is(Matchers.equalTo("https://api.split.io/api/metrics/counter"))); - } - - @Test - public void testCustomURLNoPathNoBackslash() throws URISyntaxException { - URI rootTarget = URI.create("https://kubernetesturl.com"); - CloseableHttpClient httpClient = HttpClients.custom().build(); - HttpMetrics fetcher = HttpMetrics.create(httpClient, rootTarget); - Assert.assertThat(fetcher.getTimeTarget().toString(), Matchers.is(Matchers.equalTo("https://kubernetesturl.com/api/metrics/time"))); - Assert.assertThat(fetcher.getCounterTarget().toString(), Matchers.is(Matchers.equalTo("https://kubernetesturl.com/api/metrics/counter"))); - } - - @Test - public void testCustomURLAppendingPath() throws URISyntaxException { - URI rootTarget = URI.create("https://kubernetesturl.com/split/"); - CloseableHttpClient httpClient = HttpClients.custom().build(); - HttpMetrics fetcher = HttpMetrics.create(httpClient, rootTarget); - Assert.assertThat(fetcher.getTimeTarget().toString(), Matchers.is(Matchers.equalTo("https://kubernetesturl.com/split/api/metrics/time"))); - Assert.assertThat(fetcher.getCounterTarget().toString(), Matchers.is(Matchers.equalTo("https://kubernetesturl.com/split/api/metrics/counter"))); - } - - @Test - public void testCustomURLAppendingPathNoBackslash() throws URISyntaxException { - URI rootTarget = URI.create("https://kubernetesturl.com/split"); - CloseableHttpClient httpClient = HttpClients.custom().build(); - HttpMetrics fetcher = HttpMetrics.create(httpClient, rootTarget); - Assert.assertThat(fetcher.getTimeTarget().toString(), Matchers.is(Matchers.equalTo("https://kubernetesturl.com/split/api/metrics/time"))); - Assert.assertThat(fetcher.getCounterTarget().toString(), Matchers.is(Matchers.equalTo("https://kubernetesturl.com/split/api/metrics/counter"))); - } -} diff --git a/client/src/test/java/io/split/client/metrics/LogarithmicSearchLatencyTrackerTest.java b/client/src/test/java/io/split/client/metrics/LogarithmicSearchLatencyTrackerTest.java deleted file mode 100644 index fedcec0a9..000000000 --- a/client/src/test/java/io/split/client/metrics/LogarithmicSearchLatencyTrackerTest.java +++ /dev/null @@ -1,112 +0,0 @@ -package io.split.client.metrics; - -import org.junit.Before; -import org.junit.Ignore; -import org.junit.Test; - -import static org.hamcrest.Matchers.equalTo; -import static org.hamcrest.Matchers.is; -import static org.junit.Assert.assertThat; - -@Ignore -public class LogarithmicSearchLatencyTrackerTest { - - LogarithmicSearchLatencyTracker tracker; - - @Before - public void before() { - tracker = new LogarithmicSearchLatencyTracker(); - } - - /** - * Latencies of <=1 millis or <= 1000 micros correspond to the first bucket (index 0) - */ - @Test - public void testLessThanFirstBucket() { - - tracker.addLatencyMicros(750); - tracker.addLatencyMicros(450); - assertThat(tracker.getLatency(0), is(equalTo(2L))); - - tracker.addLatencyMillis(0); - assertThat(tracker.getLatency(0), is(equalTo(3L))); - } - - /** - * Latencies of 1 millis or <= 1000 micros correspond to the first bucket (index 0) - */ - @Test - public void testFirstBucket() { - - tracker.addLatencyMicros(1000); - assertThat(tracker.getLatency(0), is(equalTo(1L))); - - tracker.addLatencyMillis(1); - assertThat(tracker.getLatency(0), is(equalTo(2L))); - } - - /** - * Latencies of 7481 millis or 7481828 micros correspond to the last bucket (index 22) - */ - @Test - public void testLastBucket() { - - tracker.addLatencyMicros(7481828); - assertThat(tracker.getLatency(22), is(equalTo(1L))); - - tracker.addLatencyMillis(7481); - assertThat(tracker.getLatency(22), is(equalTo(2L))); - } - - /** - * Latencies of more than 7481 millis or 7481828 micros correspond to the last bucket (index 22) - */ - @Test - public void testGreaterThanLastBucket() { - - tracker.addLatencyMicros(7481830); - assertThat(tracker.getLatency(22), is(equalTo(1L))); - - tracker.addLatencyMicros(7999999); - assertThat(tracker.getLatency(22), is(equalTo(2L))); - - tracker.addLatencyMillis(7482); - assertThat(tracker.getLatency(22), is(equalTo(3L))); - - tracker.addLatencyMillis(8000); - assertThat(tracker.getLatency(22), is(equalTo(4L))); - } - - /** - * Latencies between 11,392 and 17,086 are in the 8th bucket. - */ - @Test - public void test8ThBucket() { - - tracker.addLatencyMicros(11392); - assertThat(tracker.getLatency(7), is(equalTo(1L))); - - tracker.addLatencyMicros(17086); - assertThat(tracker.getLatency(7), is(equalTo(2L))); - - tracker.addLatencyMillis(18); - assertThat(tracker.getLatency(7), is(equalTo(3L))); - } - - /** - * Latencies between 656,842 and 985,261 are in the 18th bucket. - */ - @Test - public void test17ThBucket() { - - tracker.addLatencyMicros(656842); - assertThat(tracker.getLatency(17), is(equalTo(1L))); - - tracker.addLatencyMicros(985261); - assertThat(tracker.getLatency(17), is(equalTo(2L))); - - tracker.addLatencyMillis(985); - assertThat(tracker.getLatency(17), is(equalTo(3L))); - } - -} \ No newline at end of file diff --git a/client/src/test/java/io/split/telemetry/synchronizer/TelemetryConfigInitializerTest.java b/client/src/test/java/io/split/telemetry/synchronizer/TelemetryConfigInitializerTest.java new file mode 100644 index 000000000..12fd8621d --- /dev/null +++ b/client/src/test/java/io/split/telemetry/synchronizer/TelemetryConfigInitializerTest.java @@ -0,0 +1,20 @@ +package io.split.telemetry.synchronizer; + +import io.split.client.SplitClientConfig; +import io.split.engine.SDKReadinessGates; +import org.junit.Test; +import org.mockito.Mockito; + +public class TelemetryConfigInitializerTest { + + @Test + public void testRun() { + SynchronizerMemory synchronizerMemory = Mockito.mock(SynchronizerMemory.class); + SDKReadinessGates gates = Mockito.mock(SDKReadinessGates.class); + SplitClientConfig config = Mockito.mock(SplitClientConfig.class); + Mockito.when(gates.isSDKReadyNow()).thenReturn(true); + TelemetryConfigInitializer telemetryConfigInitializer = new TelemetryConfigInitializer(synchronizerMemory, gates, config); + Mockito.verify(synchronizerMemory, Mockito.times(1)).synchronizeConfig(Mockito.anyObject(),Mockito.anyLong(), Mockito.anyObject(), Mockito.anyObject()); + } + +} \ No newline at end of file From a6fd325f5a277eb3b4e11f787d2202e6d1a723a9 Mon Sep 17 00:00:00 2001 From: Martin Redolatti Date: Wed, 12 May 2021 20:20:40 -0300 Subject: [PATCH 46/81] allow the user to change amount of attempts and how many attempts to wait until loging --- client/pom.xml | 2 +- .../io/split/client/SplitClientConfig.java | 31 +++++++++++++++++++ .../io/split/client/SplitFactoryImpl.java | 2 ++ .../split/engine/common/SyncManagerImp.java | 27 ++++++++++++++-- .../split/engine/common/SynchronizerImp.java | 20 +++++++----- .../split/engine/common/SynchronizerTest.java | 2 +- pom.xml | 2 +- testing/pom.xml | 2 +- 8 files changed, 74 insertions(+), 14 deletions(-) diff --git a/client/pom.xml b/client/pom.xml index c797762fa..dea2b550e 100644 --- a/client/pom.xml +++ b/client/pom.xml @@ -5,7 +5,7 @@ io.split.client java-client-parent - 4.1.7-rc2 + 4.1.7-rc3 java-client jar diff --git a/client/src/main/java/io/split/client/SplitClientConfig.java b/client/src/main/java/io/split/client/SplitClientConfig.java index e245125c0..3e21dd5f9 100644 --- a/client/src/main/java/io/split/client/SplitClientConfig.java +++ b/client/src/main/java/io/split/client/SplitClientConfig.java @@ -47,6 +47,8 @@ public class SplitClientConfig { private final String _authServiceURL; private final String _streamingServiceURL; private final int _onDemandFetchRetryDelayMs; + private final int _onDemandFetchMaxRetries; + private final int _failedAttemptsBeforeLogging; private final boolean _cdnDebugLogging; // Proxy configs @@ -93,6 +95,8 @@ private SplitClientConfig(String endpoint, String authServiceURL, String streamingServiceURL, int onDemandFetchRetryDelayMs, + int onDemandFetchMaxRetries, + int failedAttemptsBeforeLogging, boolean cdnDebugLogging) { _endpoint = endpoint; _eventsEndpoint = eventsEndpoint; @@ -125,6 +129,8 @@ private SplitClientConfig(String endpoint, _authServiceURL = authServiceURL; _streamingServiceURL = streamingServiceURL; _onDemandFetchRetryDelayMs = onDemandFetchRetryDelayMs; + _onDemandFetchMaxRetries = onDemandFetchMaxRetries; + _failedAttemptsBeforeLogging = failedAttemptsBeforeLogging; _cdnDebugLogging = cdnDebugLogging; Properties props = new Properties(); @@ -256,6 +262,10 @@ public String streamingServiceURL() { public int streamingRetryDelay() {return _onDemandFetchRetryDelayMs;} + public int streamingFetchMaxRetries() {return _onDemandFetchMaxRetries;} + + public int failedAttemptsBeforeLogging() {return _failedAttemptsBeforeLogging;} + public boolean cdnDebugLogging() { return _cdnDebugLogging; } @@ -295,6 +305,8 @@ public static final class Builder { private String _authServiceURL = "https://auth.split.io/api/auth"; private String _streamingServiceURL = "https://streaming.split.io/sse"; private int _onDemandFetchRetryDelayMs = 50; + private int _onDemandFetchMaxRetries = 10; + private int _failedAttemptsBeforeLogging = -1; private boolean _cdnDebugLogging = true; public Builder() { @@ -697,6 +709,16 @@ public Builder streamingRetryDelay(int onDemandFetchRetryDelayMs) { return this; } + public Builder streamingFetchMaxRetries(int maxRetries) { + _onDemandFetchMaxRetries = maxRetries; + return this; + } + + public Builder failedAttemptsBeforeLoggingCDNInfo(int failedAttemptsBeforeLogging) { + _failedAttemptsBeforeLogging = failedAttemptsBeforeLogging; + return this; + } + /** * Enable logging response headers for requests made to our CDN. * @param cdnDebugLogging @@ -780,6 +802,13 @@ public SplitClientConfig build() { if(_onDemandFetchRetryDelayMs <= 0) { throw new IllegalStateException("streamingRetryDelay must be > 0"); } + if(_onDemandFetchMaxRetries <= 0) { + throw new IllegalStateException("streamingRetryDelay must be > 0"); + } + + if (_failedAttemptsBeforeLogging < 0) { + _failedAttemptsBeforeLogging = _onDemandFetchMaxRetries / 2; + } return new SplitClientConfig( _endpoint, @@ -813,6 +842,8 @@ public SplitClientConfig build() { _authServiceURL, _streamingServiceURL, _onDemandFetchRetryDelayMs, + _onDemandFetchMaxRetries, + _failedAttemptsBeforeLogging, _cdnDebugLogging); } } diff --git a/client/src/main/java/io/split/client/SplitFactoryImpl.java b/client/src/main/java/io/split/client/SplitFactoryImpl.java index e927460b2..afce36af4 100644 --- a/client/src/main/java/io/split/client/SplitFactoryImpl.java +++ b/client/src/main/java/io/split/client/SplitFactoryImpl.java @@ -159,6 +159,8 @@ public SplitFactoryImpl(String apiToken, SplitClientConfig config) throws URISyn buildSSEdHttpClient(config), _segmentCache, config.streamingRetryDelay(), + config.streamingFetchMaxRetries(), + config.failedAttemptsBeforeLogging(), config.cdnDebugLogging()); _syncManager.start(); diff --git a/client/src/main/java/io/split/engine/common/SyncManagerImp.java b/client/src/main/java/io/split/engine/common/SyncManagerImp.java index 4095275d0..2534d067b 100644 --- a/client/src/main/java/io/split/engine/common/SyncManagerImp.java +++ b/client/src/main/java/io/split/engine/common/SyncManagerImp.java @@ -61,11 +61,32 @@ public static SyncManagerImp build(boolean streamingEnabledConfig, CloseableHttpClient sseHttpClient, SegmentCache segmentCache, int streamingRetryDelay, + int maxOnDemandFetchRetries, + int failedAttemptsBeforeLogging, boolean cdnDebugLogging) { LinkedBlockingQueue pushMessages = new LinkedBlockingQueue<>(); - Synchronizer synchronizer = new SynchronizerImp(splitSynchronizationTask, splitFetcher, segmentSynchronizationTaskImp, splitCache, segmentCache, streamingRetryDelay, cdnDebugLogging); - PushManager pushManager = PushManagerImp.build(synchronizer, streamingServiceUrl, authUrl, httpClient, pushMessages, sseHttpClient); - return new SyncManagerImp(streamingEnabledConfig, synchronizer, pushManager, pushMessages, authRetryBackOffBase); + Synchronizer synchronizer = new SynchronizerImp(splitSynchronizationTask, + splitFetcher, + segmentSynchronizationTaskImp, + splitCache, + segmentCache, + streamingRetryDelay, + maxOnDemandFetchRetries, + failedAttemptsBeforeLogging, + cdnDebugLogging); + + PushManager pushManager = PushManagerImp.build(synchronizer, + streamingServiceUrl, + authUrl, + httpClient, + pushMessages, + sseHttpClient); + + return new SyncManagerImp(streamingEnabledConfig, + synchronizer, + pushManager, + pushMessages, + authRetryBackOffBase); } @Override diff --git a/client/src/main/java/io/split/engine/common/SynchronizerImp.java b/client/src/main/java/io/split/engine/common/SynchronizerImp.java index 155af6767..3223827c9 100644 --- a/client/src/main/java/io/split/engine/common/SynchronizerImp.java +++ b/client/src/main/java/io/split/engine/common/SynchronizerImp.java @@ -21,8 +21,6 @@ public class SynchronizerImp implements Synchronizer { - private static final int MAX_ATTEMPTS = 10; - private static final Logger _log = LoggerFactory.getLogger(Synchronizer.class); private final SplitSynchronizationTask _splitSynchronizationTask; private final SplitFetcher _splitFetcher; @@ -31,7 +29,10 @@ public class SynchronizerImp implements Synchronizer { private final SplitCache _splitCache; private final SegmentCache _segmentCache; private final int _onDemandFetchRetryDelayMs; + private final int _onDemandFetchMaxRetries; + private final int _failedAttemptsBeforeLogging; private final boolean _cdnResponseHeadersLogging; + private final Gson gson = new GsonBuilder().create(); public SynchronizerImp(SplitSynchronizationTask splitSynchronizationTask, @@ -40,6 +41,8 @@ public SynchronizerImp(SplitSynchronizationTask splitSynchronizationTask, SplitCache splitCache, SegmentCache segmentCache, int onDemandFetchRetryDelayMs, + int onDemandFetchMaxRetries, + int failedAttemptsBeforeLogging, boolean cdnResponseHeadersLogging) { _splitSynchronizationTask = checkNotNull(splitSynchronizationTask); _splitFetcher = checkNotNull(splitFetcher); @@ -48,6 +51,8 @@ public SynchronizerImp(SplitSynchronizationTask splitSynchronizationTask, _segmentCache = checkNotNull(segmentCache); _onDemandFetchRetryDelayMs = checkNotNull(onDemandFetchRetryDelayMs); _cdnResponseHeadersLogging = cdnResponseHeadersLogging; + _onDemandFetchMaxRetries = onDemandFetchMaxRetries; + _failedAttemptsBeforeLogging = failedAttemptsBeforeLogging; ThreadFactory splitsThreadFactory = new ThreadFactoryBuilder() .setDaemon(true) @@ -92,15 +97,15 @@ public void refreshSplits(long targetChangeNumber) { .responseHeadersCallback(_cdnResponseHeadersLogging ? captor::handle : null) .build(); - int remainingAttempts = MAX_ATTEMPTS; + int remainingAttempts = _onDemandFetchMaxRetries; while(true) { remainingAttempts--; _splitFetcher.forceRefresh(opts); if (targetChangeNumber <= _splitCache.getChangeNumber()) { - _log.debug(String.format("Refresh completed in %s attempts.", MAX_ATTEMPTS - remainingAttempts)); + _log.debug(String.format("Refresh completed in %s attempts.", _onDemandFetchMaxRetries - remainingAttempts)); break; } else if (remainingAttempts <= 0) { - _log.info(String.format("No changes fetched after %s attempts.", MAX_ATTEMPTS)); + _log.info(String.format("No changes fetched after %s attempts.", _onDemandFetchMaxRetries)); break; } try { @@ -111,7 +116,8 @@ public void refreshSplits(long targetChangeNumber) { } } - if (_cdnResponseHeadersLogging && remainingAttempts <= (MAX_ATTEMPTS / 2)) { + if (_cdnResponseHeadersLogging && + (_onDemandFetchMaxRetries - remainingAttempts) > _failedAttemptsBeforeLogging) { _log.info(String.format("CDN Debug headers: %s", gson.toJson(captor.get()))); } } @@ -127,7 +133,7 @@ public void localKillSplit(String splitName, String defaultTreatment, long newCh @Override public void refreshSegment(String segmentName, long changeNumber) { int retries = 1; - while(changeNumber > _segmentCache.getChangeNumber(segmentName) && retries <= MAX_ATTEMPTS) { + while(changeNumber > _segmentCache.getChangeNumber(segmentName) && retries <= _onDemandFetchMaxRetries) { SegmentFetcher fetcher = _segmentSynchronizationTaskImp.getFetcher(segmentName); try{ fetcher.fetch(true); diff --git a/client/src/test/java/io/split/engine/common/SynchronizerTest.java b/client/src/test/java/io/split/engine/common/SynchronizerTest.java index 2438bbdac..13f692b75 100644 --- a/client/src/test/java/io/split/engine/common/SynchronizerTest.java +++ b/client/src/test/java/io/split/engine/common/SynchronizerTest.java @@ -26,7 +26,7 @@ public void beforeMethod() { _splitCache = Mockito.mock(SplitCache.class); _segmentCache = Mockito.mock(SegmentCache.class); - _synchronizer = new SynchronizerImp(_refreshableSplitFetcherTask, _splitFetcher, _segmentFetcher, _splitCache, _segmentCache, 50, false); + _synchronizer = new SynchronizerImp(_refreshableSplitFetcherTask, _splitFetcher, _segmentFetcher, _splitCache, _segmentCache, 50, 10, 5, false); } @Test diff --git a/pom.xml b/pom.xml index b6fe51919..d3d9043bf 100644 --- a/pom.xml +++ b/pom.xml @@ -4,7 +4,7 @@ 4.0.0 io.split.client java-client-parent - 4.1.7-rc2 + 4.1.7-rc3 diff --git a/testing/pom.xml b/testing/pom.xml index d52d88807..e1a6bc52a 100644 --- a/testing/pom.xml +++ b/testing/pom.xml @@ -6,7 +6,7 @@ io.split.client java-client-parent - 4.1.7-rc2 + 4.1.7-rc3 java-client-testing From d387762998ae9dfeb9c4f0fd3dc0ce66afaa557a Mon Sep 17 00:00:00 2001 From: Martin Redolatti Date: Wed, 12 May 2021 20:44:12 -0300 Subject: [PATCH 47/81] fix error message --- client/src/main/java/io/split/client/SplitClientConfig.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/client/src/main/java/io/split/client/SplitClientConfig.java b/client/src/main/java/io/split/client/SplitClientConfig.java index 3e21dd5f9..0cd01f2f7 100644 --- a/client/src/main/java/io/split/client/SplitClientConfig.java +++ b/client/src/main/java/io/split/client/SplitClientConfig.java @@ -803,7 +803,7 @@ public SplitClientConfig build() { throw new IllegalStateException("streamingRetryDelay must be > 0"); } if(_onDemandFetchMaxRetries <= 0) { - throw new IllegalStateException("streamingRetryDelay must be > 0"); + throw new IllegalStateException("_onDemandFetchMaxRetries must be > 0"); } if (_failedAttemptsBeforeLogging < 0) { From 868dc77d8c7900a5068763f11bd2b87e56a4ac90 Mon Sep 17 00:00:00 2001 From: Martin Redolatti Date: Mon, 17 May 2021 18:19:56 -0300 Subject: [PATCH 48/81] force the cdn to hit origin if stale data is server after N retries --- .../split/client/HttpSplitChangeFetcher.java | 11 ++- .../io/split/engine/common/FetchOptions.java | 27 ++++-- .../split/engine/common/SynchronizerImp.java | 89 ++++++++++++++----- .../engine/experiments/SplitFetcherImp.java | 8 ++ 4 files changed, 106 insertions(+), 29 deletions(-) diff --git a/client/src/main/java/io/split/client/HttpSplitChangeFetcher.java b/client/src/main/java/io/split/client/HttpSplitChangeFetcher.java index 729b19264..fca610581 100644 --- a/client/src/main/java/io/split/client/HttpSplitChangeFetcher.java +++ b/client/src/main/java/io/split/client/HttpSplitChangeFetcher.java @@ -31,6 +31,7 @@ public final class HttpSplitChangeFetcher implements SplitChangeFetcher { private static final Logger _log = LoggerFactory.getLogger(HttpSplitChangeFetcher.class); private static final String SINCE = "since"; + private static final String TILL = "till"; private static final String PREFIX = "splitChangeFetcher"; private static final String HEADER_CACHE_CONTROL_NAME = "Cache-Control"; @@ -58,6 +59,10 @@ private HttpSplitChangeFetcher(CloseableHttpClient client, URI uri, Metrics metr checkNotNull(_target); } + long makeRandomTill() { + return (-1)*(int)Math.floor(Math.random()*(Math.pow(2, 63))); + } + @Override public SplitChange fetch(long since, FetchOptions options) { @@ -66,7 +71,11 @@ public SplitChange fetch(long since, FetchOptions options) { CloseableHttpResponse response = null; try { - URI uri = new URIBuilder(_target).addParameter(SINCE, "" + since).build(); + URIBuilder uriBuilder = new URIBuilder(_target).addParameter(SINCE, "" + since); + if (options.cdnBypass()) { + uriBuilder.addParameter(TILL, "" + makeRandomTill()); + } + URI uri = uriBuilder.build(); HttpGet request = new HttpGet(uri); if(options.cacheControlHeadersEnabled()) { diff --git a/client/src/main/java/io/split/engine/common/FetchOptions.java b/client/src/main/java/io/split/engine/common/FetchOptions.java index ce064c349..4da472ac1 100644 --- a/client/src/main/java/io/split/engine/common/FetchOptions.java +++ b/client/src/main/java/io/split/engine/common/FetchOptions.java @@ -1,8 +1,5 @@ package io.split.engine.common; -import io.split.engine.matchers.AttributeMatcher; -import org.checkerframework.checker.units.qual.A; - import java.util.Map; import java.util.Objects; import java.util.function.Function; @@ -10,8 +7,16 @@ public class FetchOptions { public static class Builder { + public Builder() {} + public Builder(FetchOptions opts) { + _cdnBypass = opts._cdnBypass; + _cacheControlHeaders = opts._cacheControlHeaders; + _fastlyDebugHeader = opts._fastlyDebugHeader; + _responseHeadersCallback = opts._responseHeadersCallback; + } + public Builder cacheControlHeaders(boolean on) { _cacheControlHeaders = on; return this; @@ -27,10 +32,16 @@ public Builder responseHeadersCallback(Function, Void> callb return this; } + public Builder cdnBypass(boolean bypass) { + _cdnBypass = bypass; + return this; + } + public FetchOptions build() { - return new FetchOptions(_cacheControlHeaders, _responseHeadersCallback, _fastlyDebugHeader); + return new FetchOptions(_cacheControlHeaders, _cdnBypass, _responseHeadersCallback, _fastlyDebugHeader); } + private boolean _cdnBypass = false; private boolean _cacheControlHeaders = false; private boolean _fastlyDebugHeader = false; private Function, Void> _responseHeadersCallback = null; @@ -44,6 +55,8 @@ public boolean fastlyDebugHeaderEnabled() { return _fastlyDebugHeader; } + public boolean cdnBypass() { return _cdnBypass; } + public void handleResponseHeaders(Map headers) { if (Objects.isNull(_responseHeadersCallback) || Objects.isNull(headers)) { return; @@ -51,9 +64,12 @@ public void handleResponseHeaders(Map headers) { _responseHeadersCallback.apply(headers); } - private FetchOptions(boolean cacheControlHeaders, Function, Void> responseHeadersCallback, + private FetchOptions(boolean cacheControlHeaders, + boolean cdnBypass, + Function, Void> responseHeadersCallback, boolean fastlyDebugHeader) { _cacheControlHeaders = cacheControlHeaders; + _cdnBypass = cdnBypass; _responseHeadersCallback = responseHeadersCallback; _fastlyDebugHeader = fastlyDebugHeader; } @@ -78,5 +94,6 @@ public int hashCode() { private final boolean _cacheControlHeaders; private final boolean _fastlyDebugHeader; + private final boolean _cdnBypass; private final Function, Void> _responseHeadersCallback; } diff --git a/client/src/main/java/io/split/engine/common/SynchronizerImp.java b/client/src/main/java/io/split/engine/common/SynchronizerImp.java index 3223827c9..18fb49524 100644 --- a/client/src/main/java/io/split/engine/common/SynchronizerImp.java +++ b/client/src/main/java/io/split/engine/common/SynchronizerImp.java @@ -12,15 +12,18 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import java.util.concurrent.Executors; -import java.util.concurrent.ScheduledExecutorService; -import java.util.concurrent.ThreadFactory; -import java.util.concurrent.TimeUnit; +import java.util.List; +import java.util.Map; +import java.util.concurrent.*; +import java.util.function.Function; import static com.google.common.base.Preconditions.checkNotNull; public class SynchronizerImp implements Synchronizer { + private static final long ON_DEMAND_FETCH_BACKOFF_BASE_MS = 10000; //backoff base starting at 10 seconds (!) + private static final int ON_DEMAND_FETCH_BACKOFF_MAX_RETRIES = 10; + private static final Logger _log = LoggerFactory.getLogger(Synchronizer.class); private final SplitSynchronizationTask _splitSynchronizationTask; private final SplitFetcher _splitFetcher; @@ -83,42 +86,82 @@ public void stopPeriodicFetching() { _segmentSynchronizationTaskImp.stop(); } - @Override - public void refreshSplits(long targetChangeNumber) { + private static class SyncResult { - if (targetChangeNumber <= _splitCache.getChangeNumber()) { - return; + /* package private */ SyncResult(boolean success, int remainingAttempts) { + _success = success; + _remainingAttempts = remainingAttempts; } - FastlyHeadersCaptor captor = new FastlyHeadersCaptor(); - FetchOptions opts = new FetchOptions.Builder() - .cacheControlHeaders(true) - .fastlyDebugHeader(_cdnResponseHeadersLogging) - .responseHeadersCallback(_cdnResponseHeadersLogging ? captor::handle : null) - .build(); + public boolean success() { return _success; } + public int remainingAttempts() { return _remainingAttempts; } - int remainingAttempts = _onDemandFetchMaxRetries; + private final boolean _success; + private final int _remainingAttempts; + } + + private SyncResult attemptSync(long targetChangeNumber, + FetchOptions opts, + Function nextWaitMs, + int maxRetries) { + int remainingAttempts = maxRetries; while(true) { remainingAttempts--; _splitFetcher.forceRefresh(opts); if (targetChangeNumber <= _splitCache.getChangeNumber()) { - _log.debug(String.format("Refresh completed in %s attempts.", _onDemandFetchMaxRetries - remainingAttempts)); - break; + return new SyncResult(true, remainingAttempts); } else if (remainingAttempts <= 0) { - _log.info(String.format("No changes fetched after %s attempts.", _onDemandFetchMaxRetries)); - break; + _log.info(String.format("No changes fetched after %s attempts.", maxRetries)); + return new SyncResult(false, remainingAttempts); } try { - Thread.sleep(_onDemandFetchRetryDelayMs); + Thread.sleep(nextWaitMs.apply(null)); } catch (InterruptedException e) { Thread.currentThread().interrupt(); _log.debug("Error trying to sleep current Thread."); } } + } + + private void logCdnHeaders(int maxRetries, int remainingAttempts, List> headers) { + if (maxRetries - remainingAttempts > _failedAttemptsBeforeLogging) { + _log.info(String.format("CDN Debug headers: %s", gson.toJson(headers))); + } + } + + @Override + public void refreshSplits(long targetChangeNumber) { + + if (targetChangeNumber <= _splitCache.getChangeNumber()) { + return; + } + + FastlyHeadersCaptor captor = new FastlyHeadersCaptor(); + FetchOptions opts = new FetchOptions.Builder() + .cacheControlHeaders(true) + .fastlyDebugHeader(_cdnResponseHeadersLogging) + .responseHeadersCallback(_cdnResponseHeadersLogging ? captor::handle : null) + .build(); + + SyncResult regularResult = attemptSync(targetChangeNumber, opts, + (discard) -> (long) _onDemandFetchRetryDelayMs, _onDemandFetchMaxRetries); + + if (regularResult.success()) { + _log.debug(String.format("Refresh completed in %s attempts.", _onDemandFetchMaxRetries - regularResult.remainingAttempts())); + if (_cdnResponseHeadersLogging) { + logCdnHeaders(_onDemandFetchMaxRetries , regularResult.remainingAttempts(), captor.get()); + } + return; + } - if (_cdnResponseHeadersLogging && - (_onDemandFetchMaxRetries - remainingAttempts) > _failedAttemptsBeforeLogging) { - _log.info(String.format("CDN Debug headers: %s", gson.toJson(captor.get()))); + FetchOptions withCdnBypass = new FetchOptions.Builder(opts).cdnBypass(true).build(); + Backoff backoff = new Backoff(ON_DEMAND_FETCH_BACKOFF_BASE_MS); + SyncResult withCDNBypassed = attemptSync(targetChangeNumber, withCdnBypass, + (discard) -> backoff.interval(), ON_DEMAND_FETCH_BACKOFF_MAX_RETRIES); + + if (_cdnResponseHeadersLogging) { + logCdnHeaders(_onDemandFetchMaxRetries + ON_DEMAND_FETCH_BACKOFF_MAX_RETRIES, + withCDNBypassed.remainingAttempts(), captor.get()); } } diff --git a/client/src/main/java/io/split/engine/experiments/SplitFetcherImp.java b/client/src/main/java/io/split/engine/experiments/SplitFetcherImp.java index b9c63f811..9a2402d6a 100644 --- a/client/src/main/java/io/split/engine/experiments/SplitFetcherImp.java +++ b/client/src/main/java/io/split/engine/experiments/SplitFetcherImp.java @@ -46,12 +46,20 @@ public SplitFetcherImp(SplitChangeFetcher splitChangeFetcher, SplitParser parser @Override public void forceRefresh(FetchOptions options) { _log.debug("Force Refresh splits starting ..."); + final long initialCN = _splitCache.getChangeNumber(); try { while (true) { long start = _splitCache.getChangeNumber(); runWithoutExceptionHandling(options); long end = _splitCache.getChangeNumber(); + // If the previous execution was the first one, clear the `cdnBypass` flag + // for the next fetches. (This will clear a local copy of the fetch options, + // not the original object that was passed to this method). + if (initialCN == start) { + options = new FetchOptions.Builder(options).cdnBypass(false).build(); + } + if (start >= end) { break; } From ccc5c40a5d6c5d43c7728506e3c3206fc979aa77 Mon Sep 17 00:00:00 2001 From: Martin Redolatti Date: Mon, 17 May 2021 20:57:56 -0300 Subject: [PATCH 49/81] add uts --- client/pom.xml | 2 +- .../java/io/split/engine/common/Backoff.java | 10 +- .../split/engine/common/SynchronizerImp.java | 14 ++- client/src/test/java/io/split/TestHelper.java | 2 +- .../client/HttpSplitChangeFetcherTest.java | 53 ++++++++- .../engine/common/FetcherOptionsTest.java | 2 + .../split/engine/common/SynchronizerTest.java | 107 +++++++++++++++++- .../engine/experiments/SplitFetcherTest.java | 47 ++++++++ pom.xml | 2 +- testing/pom.xml | 2 +- 10 files changed, 226 insertions(+), 15 deletions(-) diff --git a/client/pom.xml b/client/pom.xml index dea2b550e..8e1c1d65c 100644 --- a/client/pom.xml +++ b/client/pom.xml @@ -5,7 +5,7 @@ io.split.client java-client-parent - 4.1.7-rc3 + 4.1.7-rc4 java-client jar diff --git a/client/src/main/java/io/split/engine/common/Backoff.java b/client/src/main/java/io/split/engine/common/Backoff.java index 13bcb5425..285e0c824 100644 --- a/client/src/main/java/io/split/engine/common/Backoff.java +++ b/client/src/main/java/io/split/engine/common/Backoff.java @@ -5,20 +5,26 @@ import static com.google.common.base.Preconditions.checkNotNull; public class Backoff { - private static final long BACKOFF_MAX_SECONDS_ALLOWED = 1800; + private static final long BACKOFF_MAX_ALLOWED = 1800; private final long _backoffBase; private AtomicInteger _attempt; + private final long _maxAllowed; public Backoff(long backoffBase) { + this(backoffBase, BACKOFF_MAX_ALLOWED); + } + + public Backoff(long backoffBase, long maxAllowed) { _backoffBase = checkNotNull(backoffBase); _attempt = new AtomicInteger(0); + _maxAllowed = maxAllowed; } public long interval() { long interval = _backoffBase * (long) Math.pow(2, _attempt.getAndIncrement()); - return interval >= BACKOFF_MAX_SECONDS_ALLOWED ? BACKOFF_MAX_SECONDS_ALLOWED : interval; + return interval >= _maxAllowed ? BACKOFF_MAX_ALLOWED : interval; } public synchronized void reset() { diff --git a/client/src/main/java/io/split/engine/common/SynchronizerImp.java b/client/src/main/java/io/split/engine/common/SynchronizerImp.java index 18fb49524..2859c5629 100644 --- a/client/src/main/java/io/split/engine/common/SynchronizerImp.java +++ b/client/src/main/java/io/split/engine/common/SynchronizerImp.java @@ -21,7 +21,10 @@ public class SynchronizerImp implements Synchronizer { - private static final long ON_DEMAND_FETCH_BACKOFF_BASE_MS = 10000; //backoff base starting at 10 seconds (!) + // The boxing here IS necessary, so that the constants are not inlined by the compiler + // and can be modified for the test (we don't want to wait that much in an UT) + private static final long ON_DEMAND_FETCH_BACKOFF_BASE_MS = new Long(10000); //backoff base starting at 10 seconds (!) + private static final long ON_DEMAND_FETCH_BACKOFF_MAX_WAIT_MS = new Long(60000); // don't sleep for more than 1 second private static final int ON_DEMAND_FETCH_BACKOFF_MAX_RETRIES = 10; private static final Logger _log = LoggerFactory.getLogger(Synchronizer.class); @@ -52,7 +55,7 @@ public SynchronizerImp(SplitSynchronizationTask splitSynchronizationTask, _segmentSynchronizationTaskImp = checkNotNull(segmentSynchronizationTaskImp); _splitCache = checkNotNull(splitCache); _segmentCache = checkNotNull(segmentCache); - _onDemandFetchRetryDelayMs = checkNotNull(onDemandFetchRetryDelayMs); + _onDemandFetchRetryDelayMs = onDemandFetchRetryDelayMs; _cdnResponseHeadersLogging = cdnResponseHeadersLogging; _onDemandFetchMaxRetries = onDemandFetchMaxRetries; _failedAttemptsBeforeLogging = failedAttemptsBeforeLogging; @@ -115,7 +118,8 @@ private SyncResult attemptSync(long targetChangeNumber, return new SyncResult(false, remainingAttempts); } try { - Thread.sleep(nextWaitMs.apply(null)); + long howLong = nextWaitMs.apply(null); + Thread.sleep(howLong); } catch (InterruptedException e) { Thread.currentThread().interrupt(); _log.debug("Error trying to sleep current Thread."); @@ -155,10 +159,10 @@ public void refreshSplits(long targetChangeNumber) { } FetchOptions withCdnBypass = new FetchOptions.Builder(opts).cdnBypass(true).build(); - Backoff backoff = new Backoff(ON_DEMAND_FETCH_BACKOFF_BASE_MS); + Backoff backoff = new Backoff(ON_DEMAND_FETCH_BACKOFF_BASE_MS, ON_DEMAND_FETCH_BACKOFF_MAX_WAIT_MS); SyncResult withCDNBypassed = attemptSync(targetChangeNumber, withCdnBypass, (discard) -> backoff.interval(), ON_DEMAND_FETCH_BACKOFF_MAX_RETRIES); - + if (_cdnResponseHeadersLogging) { logCdnHeaders(_onDemandFetchMaxRetries + ON_DEMAND_FETCH_BACKOFF_MAX_RETRIES, withCDNBypassed.remainingAttempts(), captor.get()); diff --git a/client/src/test/java/io/split/TestHelper.java b/client/src/test/java/io/split/TestHelper.java index 1cb95eaa8..39b973c78 100644 --- a/client/src/test/java/io/split/TestHelper.java +++ b/client/src/test/java/io/split/TestHelper.java @@ -26,7 +26,7 @@ public static CloseableHttpClient mockHttpClient(String jsonName, int httpStatus return httpClientMock; } - private static CloseableHttpResponse classicResponseToCloseableMock(ClassicHttpResponse mocked) throws InvocationTargetException, IllegalAccessException, NoSuchMethodException { + public static CloseableHttpResponse classicResponseToCloseableMock(ClassicHttpResponse mocked) throws InvocationTargetException, IllegalAccessException, NoSuchMethodException { Method adaptMethod = CloseableHttpResponse.class.getDeclaredMethod("adapt", ClassicHttpResponse.class); adaptMethod.setAccessible(true); return (CloseableHttpResponse) adaptMethod.invoke(null, mocked); diff --git a/client/src/test/java/io/split/client/HttpSplitChangeFetcherTest.java b/client/src/test/java/io/split/client/HttpSplitChangeFetcherTest.java index 49abe8059..9259cc896 100644 --- a/client/src/test/java/io/split/client/HttpSplitChangeFetcherTest.java +++ b/client/src/test/java/io/split/client/HttpSplitChangeFetcherTest.java @@ -5,19 +5,30 @@ import io.split.client.dtos.SplitChange; import io.split.engine.common.FetchOptions; import io.split.engine.metrics.Metrics; +import org.apache.hc.client5.http.classic.methods.HttpUriRequestBase; import org.apache.hc.client5.http.impl.classic.CloseableHttpClient; +import org.apache.hc.client5.http.impl.classic.CloseableHttpResponse; import org.apache.hc.client5.http.impl.classic.HttpClients; -import org.apache.hc.core5.http.HttpStatus; +import org.apache.hc.core5.http.*; +import org.apache.hc.core5.http.io.entity.StringEntity; +import org.apache.hc.core5.http.message.BasicClassicHttpResponse; import org.hamcrest.Matchers; import org.junit.Assert; import org.junit.Test; +import org.mockito.ArgumentCaptor; +import org.mockito.Mockito; +import java.io.Closeable; import java.io.IOException; +import java.io.StringBufferInputStream; import java.lang.reflect.InvocationTargetException; import java.net.URI; import java.net.URISyntaxException; +import java.util.List; import java.util.Map; +import static org.mockito.Mockito.when; + public class HttpSplitChangeFetcherTest { @Test public void testDefaultURL() throws URISyntaxException { @@ -76,4 +87,44 @@ public void testFetcherWithSpecialCharacters() throws URISyntaxException, Invoca Assert.assertEquals("{\"test\": \"blue\",\"grüne Straße\": 13}", configs.get("on")); Assert.assertEquals("{\"test\": \"blue\",\"size\": 15}", configs.get("off")); } + + @Test + public void testFetcherWithCDNBypassOption() throws IOException, URISyntaxException, IllegalAccessException, NoSuchMethodException, InvocationTargetException { + URI rootTarget = URI.create("https://api.split.io"); + + HttpEntity entityMock = Mockito.mock(HttpEntity.class); + when(entityMock.getContent()).thenReturn(new StringBufferInputStream("{\"till\": 1}")); + ClassicHttpResponse response = Mockito.mock(ClassicHttpResponse.class); + when(response.getCode()).thenReturn(200); + when(response.getEntity()).thenReturn(entityMock); + when(response.getHeaders()).thenReturn(new Header[0]); + + ArgumentCaptor requestCaptor = ArgumentCaptor.forClass(ClassicHttpRequest.class); + CloseableHttpClient httpClientMock = Mockito.mock(CloseableHttpClient.class); + when(httpClientMock.execute(requestCaptor.capture())).thenReturn(TestHelper.classicResponseToCloseableMock(response)); + + Metrics.NoopMetrics metrics = new Metrics.NoopMetrics(); + HttpSplitChangeFetcher fetcher = HttpSplitChangeFetcher.create(httpClientMock, rootTarget, metrics); + + fetcher.fetch(-1, new FetchOptions.Builder().cdnBypass(true).build()); + fetcher.fetch(-1, new FetchOptions.Builder().build()); + List captured = requestCaptor.getAllValues(); + Assert.assertEquals(captured.size(), 2); + Assert.assertTrue(captured.get(0).getUri().toString().contains("till=")); + Assert.assertFalse(captured.get(1).getUri().toString().contains("till=")); + } + + @Test + public void testRandomNumberGeneration() throws URISyntaxException { + URI rootTarget = URI.create("https://api.split.io"); + CloseableHttpClient httpClientMock = Mockito.mock(CloseableHttpClient.class); + Metrics.NoopMetrics metrics = new Metrics.NoopMetrics(); + HttpSplitChangeFetcher fetcher = HttpSplitChangeFetcher.create(httpClientMock, rootTarget, metrics); + + long min = (long)Math.pow(2, 63) * (-1); + for (long x = 0; x < 100000000; x++) { + long r = fetcher.makeRandomTill(); + Assert.assertTrue(r < 0 && r > min); + } + } } diff --git a/client/src/test/java/io/split/engine/common/FetcherOptionsTest.java b/client/src/test/java/io/split/engine/common/FetcherOptionsTest.java index 56b5b6da9..0c5d26b40 100644 --- a/client/src/test/java/io/split/engine/common/FetcherOptionsTest.java +++ b/client/src/test/java/io/split/engine/common/FetcherOptionsTest.java @@ -25,10 +25,12 @@ public Void apply(Map unused) { .cacheControlHeaders(true) .fastlyDebugHeader(true) .responseHeadersCallback(func) + .cdnBypass(true) .build(); assertEquals(options.cacheControlHeadersEnabled(), true); assertEquals(options.fastlyDebugHeaderEnabled(), true); + assertEquals(options.cdnBypass(), true); options.handleResponseHeaders(new HashMap<>()); assertEquals(called[0], true); } diff --git a/client/src/test/java/io/split/engine/common/SynchronizerTest.java b/client/src/test/java/io/split/engine/common/SynchronizerTest.java index 13f692b75..daf232339 100644 --- a/client/src/test/java/io/split/engine/common/SynchronizerTest.java +++ b/client/src/test/java/io/split/engine/common/SynchronizerTest.java @@ -1,15 +1,25 @@ package io.split.engine.common; +import io.split.cache.InMemoryCacheImp; import io.split.cache.SegmentCache; import io.split.cache.SplitCache; import io.split.engine.experiments.SplitFetcherImp; import io.split.engine.experiments.SplitSynchronizationTask; import io.split.engine.segments.SegmentFetcher; import io.split.engine.segments.SegmentSynchronizationTask; +import org.junit.Assert; import org.junit.Before; import org.junit.Test; +import org.mockito.ArgumentCaptor; import org.mockito.Mockito; +import java.lang.reflect.Field; +import java.lang.reflect.Modifier; +import java.util.List; +import java.util.concurrent.atomic.AtomicInteger; + +import static org.mockito.Mockito.when; + public class SynchronizerTest { private SplitSynchronizationTask _refreshableSplitFetcherTask; private SegmentSynchronizationTask _segmentFetcher; @@ -56,7 +66,7 @@ public void stopPeriodicFetching() { @Test public void streamingRetryOnSplit() { - Mockito.when(_splitCache.getChangeNumber()).thenReturn(0l).thenReturn(0l).thenReturn(1l); + when(_splitCache.getChangeNumber()).thenReturn(0l).thenReturn(0l).thenReturn(1l); _synchronizer.refreshSplits(1l); Mockito.verify(_splitCache, Mockito.times(3)).getChangeNumber(); @@ -65,11 +75,102 @@ public void streamingRetryOnSplit() { @Test public void streamingRetryOnSegment() { SegmentFetcher fetcher = Mockito.mock(SegmentFetcher.class); - Mockito.when(_segmentFetcher.getFetcher(Mockito.anyString())).thenReturn(fetcher); - Mockito.when(_segmentCache.getChangeNumber(Mockito.anyString())).thenReturn(0l).thenReturn(0l).thenReturn(1l); + when(_segmentFetcher.getFetcher(Mockito.anyString())).thenReturn(fetcher); + when(_segmentCache.getChangeNumber(Mockito.anyString())).thenReturn(0l).thenReturn(0l).thenReturn(1l); _synchronizer.refreshSegment("Segment",1l); Mockito.verify(_segmentCache, Mockito.times(3)).getChangeNumber(Mockito.anyString()); } + @Test + public void testCDNBypassIsRequestedAfterNFailures() { + + SplitCache cache = new InMemoryCacheImp(); + Synchronizer imp = new SynchronizerImp(_refreshableSplitFetcherTask, + _splitFetcher, + _segmentFetcher, + cache, + _segmentCache, + 50, + 3, + 1, + true); + + ArgumentCaptor optionsCaptor = ArgumentCaptor.forClass(FetchOptions.class); + AtomicInteger calls = new AtomicInteger(); + Mockito.doAnswer(invocationOnMock -> { + calls.getAndIncrement(); + switch (calls.get()) { + case 4: cache.setChangeNumber(1); + } + return null; + }).when(_splitFetcher).forceRefresh(optionsCaptor.capture()); + + imp.refreshSplits(1); + + List options = optionsCaptor.getAllValues(); + Assert.assertEquals(options.size(), 4); + Assert.assertFalse(options.get(0).cdnBypass()); + Assert.assertFalse(options.get(1).cdnBypass()); + Assert.assertFalse(options.get(2).cdnBypass()); + Assert.assertTrue(options.get(3).cdnBypass()); + } + + @Test + public void testCDNBypassRequestLimitAndBackoff() throws NoSuchFieldException, IllegalAccessException { + + SplitCache cache = new InMemoryCacheImp(); + Synchronizer imp = new SynchronizerImp(_refreshableSplitFetcherTask, + _splitFetcher, + _segmentFetcher, + cache, + _segmentCache, + 50, + 3, + 1, + true); + + ArgumentCaptor optionsCaptor = ArgumentCaptor.forClass(FetchOptions.class); + AtomicInteger calls = new AtomicInteger(); + Mockito.doAnswer(invocationOnMock -> { + calls.getAndIncrement(); + switch (calls.get()) { + case 14: Assert.assertTrue(false); // should never get here + } + return null; + }).when(_splitFetcher).forceRefresh(optionsCaptor.capture()); + + // Before executing, we'll update the backoff via reflection, to avoid waiting minutes for the test to run. + Field backoffBase = SynchronizerImp.class.getDeclaredField("ON_DEMAND_FETCH_BACKOFF_BASE_MS"); + backoffBase.setAccessible(true); + Field modifiersField = Field.class.getDeclaredField("modifiers"); + modifiersField.setAccessible(true); + modifiersField.setInt(backoffBase, backoffBase.getModifiers() & ~Modifier.FINAL); + backoffBase.set(imp, 1); // 1ms + + long before = System.currentTimeMillis(); + imp.refreshSplits(1); + long after = System.currentTimeMillis(); + + List options = optionsCaptor.getAllValues(); + Assert.assertEquals(options.size(), 13); + Assert.assertFalse(options.get(0).cdnBypass()); + Assert.assertFalse(options.get(1).cdnBypass()); + Assert.assertFalse(options.get(2).cdnBypass()); + Assert.assertTrue(options.get(3).cdnBypass()); + Assert.assertTrue(options.get(4).cdnBypass()); + Assert.assertTrue(options.get(5).cdnBypass()); + Assert.assertTrue(options.get(6).cdnBypass()); + Assert.assertTrue(options.get(7).cdnBypass()); + Assert.assertTrue(options.get(8).cdnBypass()); + Assert.assertTrue(options.get(9).cdnBypass()); + Assert.assertTrue(options.get(10).cdnBypass()); + Assert.assertTrue(options.get(11).cdnBypass()); + Assert.assertTrue(options.get(12).cdnBypass()); + + Assert.assertEquals(calls.get(), 13); + long minDiffExpected = 1 + 2 + 4 + 8 + 16 + 32 + 64 + 128 + 256; + Assert.assertTrue((after - before) > minDiffExpected); + } + } diff --git a/client/src/test/java/io/split/engine/experiments/SplitFetcherTest.java b/client/src/test/java/io/split/engine/experiments/SplitFetcherTest.java index ec936a64b..82776e0ac 100644 --- a/client/src/test/java/io/split/engine/experiments/SplitFetcherTest.java +++ b/client/src/test/java/io/split/engine/experiments/SplitFetcherTest.java @@ -15,7 +15,10 @@ import io.split.engine.segments.SegmentSynchronizationTask; import io.split.engine.segments.SegmentSynchronizationTaskImp; import io.split.grammar.Treatments; +import org.junit.Assert; import org.junit.Test; +import org.mockito.ArgumentCaptor; +import org.mockito.Mock; import org.mockito.Mockito; import org.mockito.internal.matchers.Any; import org.slf4j.Logger; @@ -218,6 +221,50 @@ public void works_with_user_defined_segments() throws Exception { assertThat(gates.isSDKReady(0), is(equalTo(true))); } + @Test + public void testBypassCdnClearedAfterFirstHit() { + SplitChangeFetcher mockFetcher = Mockito.mock(SplitChangeFetcher.class); + SegmentSynchronizationTask segmentSynchronizationTaskMock = Mockito.mock(SegmentSynchronizationTask.class); + SegmentCache segmentCacheMock = Mockito.mock(SegmentCache.class); + SplitParser mockParser = new SplitParser(segmentSynchronizationTaskMock, segmentCacheMock); + SDKReadinessGates mockGates = Mockito.mock(SDKReadinessGates.class); + SplitCache mockCache = new InMemoryCacheImp(); + SplitFetcherImp fetcher = new SplitFetcherImp(mockFetcher, mockParser, mockGates, mockCache); + + + SplitChange response1 = new SplitChange(); + response1.splits = new ArrayList<>(); + response1.since = -1; + response1.till = 1; + + SplitChange response2 = new SplitChange(); + response2.splits = new ArrayList<>(); + response2.since = 1; + response2.till = 1; + + + ArgumentCaptor optionsCaptor = ArgumentCaptor.forClass(FetchOptions.class); + ArgumentCaptor cnCaptor = ArgumentCaptor.forClass(Long.class); + when(mockFetcher.fetch(cnCaptor.capture(), optionsCaptor.capture())).thenReturn(response1, response2); + + FetchOptions originalOptions = new FetchOptions.Builder().cdnBypass(true).build(); + fetcher.forceRefresh(originalOptions); + List capturedCNs = cnCaptor.getAllValues(); + List capturedOptions = optionsCaptor.getAllValues(); + + Assert.assertEquals(capturedOptions.size(), 2); + Assert.assertEquals(capturedCNs.size(), 2); + + Assert.assertEquals(capturedCNs.get(0), Long.valueOf(-1)); + Assert.assertEquals(capturedCNs.get(1), Long.valueOf(1)); + + Assert.assertTrue(capturedOptions.get(0).cdnBypass()); + Assert.assertFalse(capturedOptions.get(1).cdnBypass()); + + // Ensure that the original value hasn't been modified + Assert.assertTrue(originalOptions.cdnBypass()); + } + private SegmentChange getSegmentChange(long since, long till, String segmentName){ SegmentChange segmentChange = new SegmentChange(); segmentChange.name = segmentName; diff --git a/pom.xml b/pom.xml index d3d9043bf..05290049e 100644 --- a/pom.xml +++ b/pom.xml @@ -4,7 +4,7 @@ 4.0.0 io.split.client java-client-parent - 4.1.7-rc3 + 4.1.7-rc4 diff --git a/testing/pom.xml b/testing/pom.xml index e1a6bc52a..61536b497 100644 --- a/testing/pom.xml +++ b/testing/pom.xml @@ -6,7 +6,7 @@ io.split.client java-client-parent - 4.1.7-rc3 + 4.1.7-rc4 java-client-testing From 2dd455cb04c817646aca780dc130adc3a33255d8 Mon Sep 17 00:00:00 2001 From: Lucas Echeverz Date: Tue, 18 May 2021 15:01:38 -0300 Subject: [PATCH 50/81] Making syncManager start async and telemetry init --- .../io/split/client/SplitFactoryImpl.java | 18 +++--- .../io/split/engine/SDKReadinessGates.java | 12 +--- .../split/engine/common/SyncManagerImp.java | 51 +++++++++++------ .../io/split/engine/common/Synchronizer.java | 2 +- .../split/engine/common/SynchronizerImp.java | 19 +++---- .../engine/experiments/SplitFetcher.java | 2 +- .../engine/experiments/SplitFetcherImp.java | 11 ++-- .../segments/SegmentSynchronizationTask.java | 2 +- .../SegmentSynchronizationTaskImp.java | 5 +- .../synchronizer/SynchronizerMemory.java | 12 ++-- .../TelemetryConfigInitializer.java | 33 ----------- .../io/split/client/ApiKeyCounterTest.java | 5 ++ .../split/engine/common/SyncManagerTest.java | 57 ++++++++++++++++--- .../split/engine/common/SynchronizerTest.java | 5 +- .../engine/experiments/SplitFetcherTest.java | 16 ++---- .../synchronizer/SynchronizerMemoryTest.java | 3 +- .../TelemetryConfigInitializerTest.java | 20 ------- 17 files changed, 136 insertions(+), 137 deletions(-) delete mode 100644 client/src/main/java/io/split/telemetry/synchronizer/TelemetryConfigInitializer.java delete mode 100644 client/src/test/java/io/split/telemetry/synchronizer/TelemetryConfigInitializerTest.java diff --git a/client/src/main/java/io/split/client/SplitFactoryImpl.java b/client/src/main/java/io/split/client/SplitFactoryImpl.java index e3455d4da..03aaedf3c 100644 --- a/client/src/main/java/io/split/client/SplitFactoryImpl.java +++ b/client/src/main/java/io/split/client/SplitFactoryImpl.java @@ -28,7 +28,6 @@ import io.split.telemetry.storage.InMemoryTelemetryStorage; import io.split.telemetry.storage.TelemetryStorage; import io.split.telemetry.synchronizer.SynchronizerMemory; -import io.split.telemetry.synchronizer.TelemetryConfigInitializer; import io.split.telemetry.synchronizer.TelemetrySyncTask; import io.split.telemetry.synchronizer.TelemetrySynchronizer; import org.apache.hc.client5.http.auth.AuthScope; @@ -95,7 +94,6 @@ public class SplitFactoryImpl implements SplitFactory { private final TelemetrySynchronizer _telemetrySynchronizer; private final TelemetrySyncTask _telemetrySyncTask; private final long _startTime; - private final TelemetryConfigInitializer _telemetryConfigInitializer; public SplitFactoryImpl(String apiToken, SplitClientConfig config) throws URISyntaxException { _startTime = System.currentTimeMillis(); @@ -125,9 +123,7 @@ public SplitFactoryImpl(String apiToken, SplitClientConfig config) throws URISyn // Cache Initialisations _segmentCache = new SegmentCacheInMemoryImpl(); _splitCache = new InMemoryCacheImp(); - _telemetrySynchronizer = new SynchronizerMemory(_httpclient, URI.create(config.get_telemetryURL()), _telemetryStorage, _splitCache, _segmentCache, _telemetryStorage); - _telemetrySyncTask = new TelemetrySyncTask(config.get_telemetryRefreshRate(), _telemetrySynchronizer); - _telemetryConfigInitializer = new TelemetryConfigInitializer(_telemetrySynchronizer,_gates,config); + _telemetrySynchronizer = new SynchronizerMemory(_httpclient, URI.create(config.get_telemetryURL()), _telemetryStorage, _splitCache, _segmentCache, _telemetryStorage, _startTime); // Segments @@ -145,9 +141,7 @@ public SplitFactoryImpl(String apiToken, SplitClientConfig config) throws URISyn // EventClient _eventClient = EventClientImpl.create(_httpclient, _eventsRootTarget, config.eventsQueueSize(), config.eventFlushIntervalInMillis(), config.waitBeforeShutdown(), _telemetryStorage); - // SyncManager - _syncManager = SyncManagerImp.build(config.streamingEnabled(), _splitSynchronizationTask, _splitFetcher, _segmentSynchronizationTaskImp, _splitCache, config.authServiceURL(), _httpclient, config.streamingServiceURL(), config.authRetryBackoffBase(), buildSSEdHttpClient(apiToken, config), _segmentCache, config.streamingRetryDelay(), _gates, _telemetryStorage); - _syncManager.start(); + _telemetrySyncTask = new TelemetrySyncTask(config.get_telemetryRefreshRate(), _telemetrySynchronizer); // Evaluator _evaluator = new EvaluatorImp(_splitCache); @@ -158,6 +152,12 @@ public SplitFactoryImpl(String apiToken, SplitClientConfig config) throws URISyn // SplitManager _manager = new SplitManagerImpl(_splitCache, config, _gates, _telemetryStorage); + // SyncManager + _syncManager = SyncManagerImp.build(config.streamingEnabled(), _splitSynchronizationTask, _splitFetcher, _segmentSynchronizationTaskImp, _splitCache, + config.authServiceURL(), _httpclient, config.streamingServiceURL(), config.authRetryBackoffBase(), buildSSEdHttpClient(apiToken, config), + _segmentCache, config.streamingRetryDelay(), _gates, _telemetryStorage, _telemetrySynchronizer,config); + _syncManager.start(); + // DestroyOnShutDown if (config.destroyOnShutDown()) { Runtime.getRuntime().addShutdownHook(new Thread(() -> { @@ -316,7 +316,7 @@ private SplitFetcher buildSplitFetcher() throws URISyntaxException { SplitChangeFetcher splitChangeFetcher = HttpSplitChangeFetcher.create(_httpclient, _rootTarget, _telemetryStorage); SplitParser splitParser = new SplitParser(_segmentSynchronizationTaskImp, _segmentCache); - return new SplitFetcherImp(splitChangeFetcher, splitParser, _gates, _splitCache, _telemetryStorage); + return new SplitFetcherImp(splitChangeFetcher, splitParser, _splitCache, _telemetryStorage); } private ImpressionsManagerImpl buildImpressionsManager(SplitClientConfig config) throws URISyntaxException { diff --git a/client/src/main/java/io/split/engine/SDKReadinessGates.java b/client/src/main/java/io/split/engine/SDKReadinessGates.java index e8a4e6453..fefcc0466 100644 --- a/client/src/main/java/io/split/engine/SDKReadinessGates.java +++ b/client/src/main/java/io/split/engine/SDKReadinessGates.java @@ -35,17 +35,7 @@ public class SDKReadinessGates { * @throws InterruptedException if this operation was interrupted. */ public boolean isSDKReady(long milliseconds) throws InterruptedException { - long end = System.currentTimeMillis() + milliseconds; - long timeLeft = milliseconds; - - boolean splits = areSplitsReady(timeLeft); - if (!splits) { - return false; - } - - timeLeft = end - System.currentTimeMillis(); - - return areSegmentsReady(timeLeft); + return _internalReady.await(milliseconds, TimeUnit.MILLISECONDS); } public boolean isSDKReadyNow() { diff --git a/client/src/main/java/io/split/engine/common/SyncManagerImp.java b/client/src/main/java/io/split/engine/common/SyncManagerImp.java index c532f660c..8ac52b425 100644 --- a/client/src/main/java/io/split/engine/common/SyncManagerImp.java +++ b/client/src/main/java/io/split/engine/common/SyncManagerImp.java @@ -4,6 +4,8 @@ import com.google.common.util.concurrent.ThreadFactoryBuilder; import io.split.cache.SegmentCache; import io.split.cache.SplitCache; +import io.split.client.ApiKeyCounter; +import io.split.client.SplitClientConfig; import io.split.engine.SDKReadinessGates; import io.split.engine.experiments.SplitFetcher; import io.split.engine.experiments.SplitSynchronizationTask; @@ -11,10 +13,12 @@ import io.split.telemetry.domain.StreamingEvent; import io.split.telemetry.domain.enums.StreamEventsEnum; import io.split.telemetry.storage.TelemetryRuntimeProducer; +import io.split.telemetry.synchronizer.TelemetrySynchronizer; import org.apache.hc.client5.http.impl.classic.CloseableHttpClient; import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import java.util.ArrayList; import java.util.concurrent.ExecutorService; import java.util.concurrent.Executors; import java.util.concurrent.Future; @@ -34,11 +38,13 @@ public class SyncManagerImp implements SyncManager { private final AtomicBoolean _shutdown; private final LinkedBlockingQueue _incomingPushStatus; private final ExecutorService _executorService; - private final ExecutorService _pollingExecutorService; + private final ExecutorService _startExecutorService; private final SDKReadinessGates _gates; private Future _pushStatusMonitorTask; private Backoff _backoff; private final TelemetryRuntimeProducer _telemetryRuntimeProducer; + private final TelemetrySynchronizer _telemetrySynchronizer; + private final SplitClientConfig _config; @VisibleForTesting /* package private */ SyncManagerImp(boolean streamingEnabledConfig, @@ -46,7 +52,9 @@ public class SyncManagerImp implements SyncManager { PushManager pushManager, LinkedBlockingQueue pushMessages, int authRetryBackOffBase, - SDKReadinessGates gates, TelemetryRuntimeProducer telemetryRuntimeProducer) { + SDKReadinessGates gates, TelemetryRuntimeProducer telemetryRuntimeProducer, + TelemetrySynchronizer telemetrySynchronizer, + SplitClientConfig config) { _streamingEnabledConfig = new AtomicBoolean(streamingEnabledConfig); _synchronizer = checkNotNull(synchronizer); _pushManager = checkNotNull(pushManager); @@ -56,13 +64,15 @@ public class SyncManagerImp implements SyncManager { .setNameFormat("SPLIT-PushStatusMonitor-%d") .setDaemon(true) .build()); - _pollingExecutorService = Executors.newSingleThreadExecutor(new ThreadFactoryBuilder() + _startExecutorService = Executors.newSingleThreadExecutor(new ThreadFactoryBuilder() .setNameFormat("SPLIT-PollingMode-%d") .setDaemon(true) .build()); _backoff = new Backoff(authRetryBackOffBase); _gates = checkNotNull(gates); _telemetryRuntimeProducer = checkNotNull(telemetryRuntimeProducer); + _telemetrySynchronizer = checkNotNull(telemetrySynchronizer); + _config = checkNotNull(config); } public static SyncManagerImp build(boolean streamingEnabledConfig, @@ -78,21 +88,34 @@ public static SyncManagerImp build(boolean streamingEnabledConfig, SegmentCache segmentCache, int streamingRetryDelay, SDKReadinessGates gates, - TelemetryRuntimeProducer telemetryRuntimeProducer) { + TelemetryRuntimeProducer telemetryRuntimeProducer, + TelemetrySynchronizer telemetrySynchronizer, + SplitClientConfig config) { LinkedBlockingQueue pushMessages = new LinkedBlockingQueue<>(); Synchronizer synchronizer = new SynchronizerImp(splitSynchronizationTask, splitFetcher, segmentSynchronizationTaskImp, splitCache, segmentCache, streamingRetryDelay, gates); PushManager pushManager = PushManagerImp.build(synchronizer, streamingServiceUrl, authUrl, httpClient, pushMessages, sseHttpClient, telemetryRuntimeProducer); - return new SyncManagerImp(streamingEnabledConfig, synchronizer, pushManager, pushMessages, authRetryBackOffBase, gates, telemetryRuntimeProducer); + return new SyncManagerImp(streamingEnabledConfig, synchronizer, pushManager, pushMessages, authRetryBackOffBase, gates, telemetryRuntimeProducer,telemetrySynchronizer, config); } @Override public void start() { - _synchronizer.syncAll(); - if (_streamingEnabledConfig.get()) { - startStreamingMode(); - } else { - _pollingExecutorService.submit(this::startPollingMode); - } + _startExecutorService.submit(() -> { + while(!_synchronizer.syncAll()) { + try { + Thread.currentThread().sleep(1000); + } catch (InterruptedException e) { + e.printStackTrace(); + Thread.currentThread().interrupt(); + } + } + _gates.sdkInternalReady(); + _telemetrySynchronizer.synchronizeConfig(_config, System.currentTimeMillis(), ApiKeyCounter.getApiKeyCounterInstance().getFactoryInstances(), new ArrayList<>()); + if (_streamingEnabledConfig.get()) { + startStreamingMode(); + } else { + startPollingMode(); + } + }); } @Override @@ -112,12 +135,6 @@ private void startStreamingMode() { } private void startPollingMode() { - try { - _gates.waitUntilInternalReady(); - } catch (InterruptedException ex) { - _log.debug(ex.getMessage()); - } - _log.debug("Starting in polling mode ..."); _synchronizer.startPeriodicFetching(); _telemetryRuntimeProducer.recordStreamingEvents(new StreamingEvent(StreamEventsEnum.SYNC_MODE_UPDATE.get_type(), POLLING_STREAMING_EVENT, System.currentTimeMillis())); diff --git a/client/src/main/java/io/split/engine/common/Synchronizer.java b/client/src/main/java/io/split/engine/common/Synchronizer.java index ab8467a5c..9197baacf 100644 --- a/client/src/main/java/io/split/engine/common/Synchronizer.java +++ b/client/src/main/java/io/split/engine/common/Synchronizer.java @@ -1,7 +1,7 @@ package io.split.engine.common; public interface Synchronizer { - void syncAll(); + boolean syncAll(); void startPeriodicFetching(); void stopPeriodicFetching(); void refreshSplits(long targetChangeNumber); diff --git a/client/src/main/java/io/split/engine/common/SynchronizerImp.java b/client/src/main/java/io/split/engine/common/SynchronizerImp.java index d63f5417f..5ad718073 100644 --- a/client/src/main/java/io/split/engine/common/SynchronizerImp.java +++ b/client/src/main/java/io/split/engine/common/SynchronizerImp.java @@ -11,10 +11,8 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import java.util.concurrent.Executors; -import java.util.concurrent.ScheduledExecutorService; -import java.util.concurrent.ThreadFactory; -import java.util.concurrent.TimeUnit; +import java.util.concurrent.*; +import java.util.concurrent.atomic.AtomicBoolean; import static com.google.common.base.Preconditions.checkNotNull; @@ -54,12 +52,13 @@ public SynchronizerImp(SplitSynchronizationTask splitSynchronizationTask, } @Override - public void syncAll() { - _syncAllScheduledExecutorService.schedule(() -> { - _splitFetcher.fetchAll(true); - _segmentSynchronizationTaskImp.fetchAllSynchronous(); - _gates.sdkInternalReady(); - }, 0, TimeUnit.SECONDS); + public boolean syncAll() { + AtomicBoolean syncStatus = new AtomicBoolean(false); + if(_splitFetcher.fetchAll(true) && + _segmentSynchronizationTaskImp.fetchAllSynchronous()) { + syncStatus.set(true); + } + return syncStatus.get(); } @Override diff --git a/client/src/main/java/io/split/engine/experiments/SplitFetcher.java b/client/src/main/java/io/split/engine/experiments/SplitFetcher.java index 4266659b1..8637feefd 100644 --- a/client/src/main/java/io/split/engine/experiments/SplitFetcher.java +++ b/client/src/main/java/io/split/engine/experiments/SplitFetcher.java @@ -14,5 +14,5 @@ public interface SplitFetcher extends Runnable { * Forces a sync of ALL splits, outside of any scheduled * syncs. This method MUST NOT throw any exceptions. */ - void fetchAll(boolean addCacheHeader); + boolean fetchAll(boolean addCacheHeader); } diff --git a/client/src/main/java/io/split/engine/experiments/SplitFetcherImp.java b/client/src/main/java/io/split/engine/experiments/SplitFetcherImp.java index 0f2e91e54..1d5e60e5c 100644 --- a/client/src/main/java/io/split/engine/experiments/SplitFetcherImp.java +++ b/client/src/main/java/io/split/engine/experiments/SplitFetcherImp.java @@ -25,7 +25,6 @@ public class SplitFetcherImp implements SplitFetcher { private final SplitParser _parser; private final SplitChangeFetcher _splitChangeFetcher; private final SplitCache _splitCache; - private final SDKReadinessGates _gates; private final Object _lock = new Object(); private final TelemetryRuntimeProducer _telemetryRuntimeProducer; @@ -39,10 +38,9 @@ public class SplitFetcherImp implements SplitFetcher { * an ARCHIVED split is received, we know if we need to remove a traffic type from the multiset. */ - public SplitFetcherImp(SplitChangeFetcher splitChangeFetcher, SplitParser parser, SDKReadinessGates gates, SplitCache splitCache, TelemetryRuntimeProducer telemetryRuntimeProducer) { + public SplitFetcherImp(SplitChangeFetcher splitChangeFetcher, SplitParser parser, SplitCache splitCache, TelemetryRuntimeProducer telemetryRuntimeProducer) { _splitChangeFetcher = checkNotNull(splitChangeFetcher); _parser = checkNotNull(parser); - _gates = checkNotNull(gates); _splitCache = checkNotNull(splitCache); _telemetryRuntimeProducer = checkNotNull(telemetryRuntimeProducer); } @@ -146,16 +144,18 @@ private void runWithoutExceptionHandling(boolean addCacheHeader) throws Interrup } } @Override - public void fetchAll(boolean addCacheHeader) { + public boolean fetchAll(boolean addCacheHeader) { + boolean fetchAllStatus = true; _log.debug("Fetch splits starting ..."); long start = _splitCache.getChangeNumber(); try { runWithoutExceptionHandling(addCacheHeader); - _gates.splitsAreReady(); } catch (InterruptedException e) { + fetchAllStatus = false; _log.warn("Interrupting split fetcher task"); Thread.currentThread().interrupt(); } catch (Throwable t) { + fetchAllStatus = false; _log.error("RefreshableSplitFetcher failed: " + t.getMessage()); if (_log.isDebugEnabled()) { _log.debug("Reason:", t); @@ -165,5 +165,6 @@ public void fetchAll(boolean addCacheHeader) { _log.debug("split fetch before: " + start + ", after: " + _splitCache.getChangeNumber()); } } + return fetchAllStatus; } } diff --git a/client/src/main/java/io/split/engine/segments/SegmentSynchronizationTask.java b/client/src/main/java/io/split/engine/segments/SegmentSynchronizationTask.java index 5ad181cf4..1a1764ed9 100644 --- a/client/src/main/java/io/split/engine/segments/SegmentSynchronizationTask.java +++ b/client/src/main/java/io/split/engine/segments/SegmentSynchronizationTask.java @@ -33,5 +33,5 @@ public interface SegmentSynchronizationTask extends Runnable { /** * fetch every Segment Synchronous */ - void fetchAllSynchronous(); + boolean fetchAllSynchronous(); } diff --git a/client/src/main/java/io/split/engine/segments/SegmentSynchronizationTaskImp.java b/client/src/main/java/io/split/engine/segments/SegmentSynchronizationTaskImp.java index d3ed0dbfe..c6433c7ec 100644 --- a/client/src/main/java/io/split/engine/segments/SegmentSynchronizationTaskImp.java +++ b/client/src/main/java/io/split/engine/segments/SegmentSynchronizationTaskImp.java @@ -165,7 +165,8 @@ public void fetchAll(boolean addCacheHeader) { } @Override - public void fetchAllSynchronous() { + public boolean fetchAllSynchronous() { + AtomicBoolean fetchAllStatus = new AtomicBoolean(true); _segmentFetchers .entrySet() .stream().map(e -> _scheduledExecutorService.submit(e.getValue()::runWhitCacheHeader)) @@ -174,7 +175,9 @@ public void fetchAllSynchronous() { try { future.get(); } catch (Exception ex) { + fetchAllStatus.set(false); _log.error(ex.getMessage()); }}); + return fetchAllStatus.get(); } } diff --git a/client/src/main/java/io/split/telemetry/synchronizer/SynchronizerMemory.java b/client/src/main/java/io/split/telemetry/synchronizer/SynchronizerMemory.java index 1ffeb1178..bf075c4ed 100644 --- a/client/src/main/java/io/split/telemetry/synchronizer/SynchronizerMemory.java +++ b/client/src/main/java/io/split/telemetry/synchronizer/SynchronizerMemory.java @@ -32,18 +32,20 @@ public class SynchronizerMemory implements TelemetrySynchronizer{ private TelemetryStorageConsumer _teleTelemetryStorageConsumer; private SplitCache _splitCache; private SegmentCache _segmentCache; + private final long _initStartTime; public SynchronizerMemory(CloseableHttpClient client, URI telemetryRootEndpoint, TelemetryStorageConsumer telemetryStorageConsumer, SplitCache splitCache, - SegmentCache segmentCache, TelemetryRuntimeProducer telemetryRuntimeProducer) throws URISyntaxException { + SegmentCache segmentCache, TelemetryRuntimeProducer telemetryRuntimeProducer, long initStartTime) throws URISyntaxException { _httpHttpTelemetryMemorySender = HttpTelemetryMemorySender.create(client, telemetryRootEndpoint, telemetryRuntimeProducer); _teleTelemetryStorageConsumer = telemetryStorageConsumer; _splitCache = splitCache; _segmentCache = segmentCache; + _initStartTime = initStartTime; } @Override - public void synchronizeConfig(SplitClientConfig config, long timeUntilReady, Map factoryInstances, List tags) { - _httpHttpTelemetryMemorySender.postConfig(generateConfig(config, timeUntilReady, factoryInstances, tags)); + public void synchronizeConfig(SplitClientConfig config, long readyTimeStamp, Map factoryInstances, List tags) { + _httpHttpTelemetryMemorySender.postConfig(generateConfig(config, readyTimeStamp, factoryInstances, tags)); } @Override @@ -74,7 +76,7 @@ private Stats generateStats() throws Exception { return stats; } - private Config generateConfig(SplitClientConfig splitClientConfig, long timeUntilReady, Map factoryInstances, List tags) { + private Config generateConfig(SplitClientConfig splitClientConfig, long readyTimestamp, Map factoryInstances, List tags) { Config config = new Config(); Rates rates = new Rates(); URLOverrides urlOverrides = new URLOverrides(); @@ -110,7 +112,7 @@ private Config generateConfig(SplitClientConfig splitClientConfig, long timeUnti config.set_eventsQueueSize(splitClientConfig.eventsQueueSize()); config.set_tags(getListMaxSize(tags)); config.set_activeFactories(factoryInstances.size()); - config.set_timeUntilReady(timeUntilReady); + config.set_timeUntilReady(readyTimestamp - _initStartTime); config.set_rates(rates); config.set_urlOverrides(urlOverrides); config.set_streamingEnabled(splitClientConfig.streamingEnabled()); diff --git a/client/src/main/java/io/split/telemetry/synchronizer/TelemetryConfigInitializer.java b/client/src/main/java/io/split/telemetry/synchronizer/TelemetryConfigInitializer.java deleted file mode 100644 index b53dce7ae..000000000 --- a/client/src/main/java/io/split/telemetry/synchronizer/TelemetryConfigInitializer.java +++ /dev/null @@ -1,33 +0,0 @@ -package io.split.telemetry.synchronizer; - -import io.split.client.ApiKeyCounter; -import io.split.client.SplitClientConfig; -import io.split.engine.SDKReadinessGates; - -import java.util.ArrayList; - -import static com.google.common.base.Preconditions.checkNotNull; - -public class TelemetryConfigInitializer { - - private final TelemetrySynchronizer _telemetrySynchronizer; - private final SDKReadinessGates _gates; - private final SplitClientConfig _config; - - public TelemetryConfigInitializer(TelemetrySynchronizer _telemetrySynchronizer, SDKReadinessGates _gates, SplitClientConfig config) { - this._telemetrySynchronizer = checkNotNull(_telemetrySynchronizer); - this._gates = checkNotNull(_gates); - _config = checkNotNull(config); - this.waitForSDKReady(); - } - - private void waitForSDKReady() { - long initTime = System.currentTimeMillis(); - while(true) { - if (_gates.isSDKReadyNow()) { - _telemetrySynchronizer.synchronizeConfig(_config,System.currentTimeMillis()-initTime, ApiKeyCounter.getApiKeyCounterInstance().getFactoryInstances(),new ArrayList<>()); - break; - } - } - } -} diff --git a/client/src/test/java/io/split/client/ApiKeyCounterTest.java b/client/src/test/java/io/split/client/ApiKeyCounterTest.java index bf1f21f11..64304f462 100644 --- a/client/src/test/java/io/split/client/ApiKeyCounterTest.java +++ b/client/src/test/java/io/split/client/ApiKeyCounterTest.java @@ -62,5 +62,10 @@ public void testFactoryInstances() { Map factoryInstances = ApiKeyCounter.getApiKeyCounterInstance().getFactoryInstances(); Assert.assertEquals(2, factoryInstances.size()); Assert.assertEquals(3, factoryInstances.get(FIRST_KEY).intValue()); + ApiKeyCounter.getApiKeyCounterInstance().remove(FIRST_KEY); + ApiKeyCounter.getApiKeyCounterInstance().remove(FIRST_KEY); + ApiKeyCounter.getApiKeyCounterInstance().remove(FIRST_KEY); + ApiKeyCounter.getApiKeyCounterInstance().remove(SECOND_KEY); + ApiKeyCounter.getApiKeyCounterInstance().remove(SECOND_KEY); } } diff --git a/client/src/test/java/io/split/engine/common/SyncManagerTest.java b/client/src/test/java/io/split/engine/common/SyncManagerTest.java index d2204ec18..fafe0874c 100644 --- a/client/src/test/java/io/split/engine/common/SyncManagerTest.java +++ b/client/src/test/java/io/split/engine/common/SyncManagerTest.java @@ -1,8 +1,10 @@ package io.split.engine.common; +import io.split.client.SplitClientConfig; import io.split.engine.SDKReadinessGates; import io.split.telemetry.storage.InMemoryTelemetryStorage; import io.split.telemetry.storage.TelemetryStorage; +import io.split.telemetry.synchronizer.TelemetrySynchronizer; import org.junit.Before; import org.junit.Test; import org.mockito.Mockito; @@ -26,7 +28,10 @@ public void setUp() { public void startWithStreamingFalseShouldStartPolling() throws InterruptedException { TelemetryStorage telemetryStorage = new InMemoryTelemetryStorage(); _gates.sdkInternalReady(); - SyncManagerImp syncManager = new SyncManagerImp(false, _synchronizer, _pushManager, new LinkedBlockingQueue<>(), BACKOFF_BASE, _gates, telemetryStorage); + TelemetrySynchronizer telemetrySynchronizer = Mockito.mock(TelemetrySynchronizer.class); + SplitClientConfig config = Mockito.mock(SplitClientConfig.class); + Mockito.when(_synchronizer.syncAll()).thenReturn(true); + SyncManagerImp syncManager = new SyncManagerImp(false, _synchronizer, _pushManager, new LinkedBlockingQueue<>(), BACKOFF_BASE, _gates, telemetryStorage, telemetrySynchronizer, config); syncManager.start(); Thread.sleep(1000); Mockito.verify(_synchronizer, Mockito.times(1)).startPeriodicFetching(); @@ -35,10 +40,14 @@ public void startWithStreamingFalseShouldStartPolling() throws InterruptedExcept } @Test - public void startWithStreamingTrueShouldStartSyncAll() { + public void startWithStreamingTrueShouldStartSyncAll() throws InterruptedException { TelemetryStorage telemetryStorage = new InMemoryTelemetryStorage(); - SyncManager sm = new SyncManagerImp(true, _synchronizer, _pushManager, new LinkedBlockingQueue<>(), BACKOFF_BASE, _gates, telemetryStorage); + TelemetrySynchronizer telemetrySynchronizer = Mockito.mock(TelemetrySynchronizer.class); + SplitClientConfig config = Mockito.mock(SplitClientConfig.class); + Mockito.when(_synchronizer.syncAll()).thenReturn(true); + SyncManager sm = new SyncManagerImp(true, _synchronizer, _pushManager, new LinkedBlockingQueue<>(), BACKOFF_BASE, _gates, telemetryStorage, telemetrySynchronizer, config); sm.start(); + Thread.sleep(1000); Mockito.verify(_synchronizer, Mockito.times(0)).startPeriodicFetching(); Mockito.verify(_synchronizer, Mockito.times(1)).syncAll(); Mockito.verify(_pushManager, Mockito.times(1)).start(); @@ -48,7 +57,10 @@ public void startWithStreamingTrueShouldStartSyncAll() { public void onStreamingAvailable() throws InterruptedException { TelemetryStorage telemetryStorage = new InMemoryTelemetryStorage(); LinkedBlockingQueue messsages = new LinkedBlockingQueue<>(); - SyncManagerImp syncManager = new SyncManagerImp(true, _synchronizer, _pushManager, messsages, BACKOFF_BASE, _gates, telemetryStorage); + TelemetrySynchronizer telemetrySynchronizer = Mockito.mock(TelemetrySynchronizer.class); + SplitClientConfig config = Mockito.mock(SplitClientConfig.class); + + SyncManagerImp syncManager = new SyncManagerImp(true, _synchronizer, _pushManager, messsages, BACKOFF_BASE, _gates, telemetryStorage, telemetrySynchronizer, config); Thread t = new Thread(syncManager::incomingPushStatusHandler); t.start(); messsages.offer(PushManager.Status.STREAMING_READY); @@ -63,7 +75,9 @@ public void onStreamingAvailable() throws InterruptedException { public void onStreamingDisabled() throws InterruptedException { TelemetryStorage telemetryStorage = new InMemoryTelemetryStorage(); LinkedBlockingQueue messsages = new LinkedBlockingQueue<>(); - SyncManagerImp syncManager = new SyncManagerImp(true, _synchronizer, _pushManager, messsages, BACKOFF_BASE, _gates, telemetryStorage); + TelemetrySynchronizer telemetrySynchronizer = Mockito.mock(TelemetrySynchronizer.class); + SplitClientConfig config = Mockito.mock(SplitClientConfig.class); + SyncManagerImp syncManager = new SyncManagerImp(true, _synchronizer, _pushManager, messsages, BACKOFF_BASE, _gates, telemetryStorage, telemetrySynchronizer, config); Thread t = new Thread(syncManager::incomingPushStatusHandler); t.start(); messsages.offer(PushManager.Status.STREAMING_DOWN); @@ -78,7 +92,9 @@ public void onStreamingDisabled() throws InterruptedException { public void onStreamingShutdown() throws InterruptedException { TelemetryStorage telemetryStorage = new InMemoryTelemetryStorage(); LinkedBlockingQueue messsages = new LinkedBlockingQueue<>(); - SyncManagerImp syncManager = new SyncManagerImp(true, _synchronizer, _pushManager, messsages, BACKOFF_BASE, _gates, telemetryStorage); + TelemetrySynchronizer telemetrySynchronizer = Mockito.mock(TelemetrySynchronizer.class); + SplitClientConfig config = Mockito.mock(SplitClientConfig.class); + SyncManagerImp syncManager = new SyncManagerImp(true, _synchronizer, _pushManager, messsages, BACKOFF_BASE, _gates, telemetryStorage, telemetrySynchronizer, config); Thread t = new Thread(syncManager::incomingPushStatusHandler); t.start(); messsages.offer(PushManager.Status.STREAMING_OFF); @@ -91,7 +107,9 @@ public void onStreamingShutdown() throws InterruptedException { public void onConnected() throws InterruptedException { TelemetryStorage telemetryStorage = new InMemoryTelemetryStorage(); LinkedBlockingQueue messsages = new LinkedBlockingQueue<>(); - SyncManagerImp syncManager = new SyncManagerImp(true, _synchronizer, _pushManager, messsages, BACKOFF_BASE, _gates, telemetryStorage); + TelemetrySynchronizer telemetrySynchronizer = Mockito.mock(TelemetrySynchronizer.class); + SplitClientConfig config = Mockito.mock(SplitClientConfig.class); + SyncManagerImp syncManager = new SyncManagerImp(true, _synchronizer, _pushManager, messsages, BACKOFF_BASE, _gates, telemetryStorage, telemetrySynchronizer, config); Thread t = new Thread(syncManager::incomingPushStatusHandler); t.start(); messsages.offer(PushManager.Status.STREAMING_READY); @@ -105,7 +123,9 @@ public void onConnected() throws InterruptedException { public void onDisconnect() throws InterruptedException { TelemetryStorage telemetryStorage = new InMemoryTelemetryStorage(); LinkedBlockingQueue messsages = new LinkedBlockingQueue<>(); - SyncManagerImp syncManager = new SyncManagerImp(true, _synchronizer, _pushManager, messsages, BACKOFF_BASE, _gates, telemetryStorage); + TelemetrySynchronizer telemetrySynchronizer = Mockito.mock(TelemetrySynchronizer.class); + SplitClientConfig config = Mockito.mock(SplitClientConfig.class); + SyncManagerImp syncManager = new SyncManagerImp(true, _synchronizer, _pushManager, messsages, BACKOFF_BASE, _gates, telemetryStorage, telemetrySynchronizer, config); Thread t = new Thread(syncManager::incomingPushStatusHandler); t.start(); messsages.offer(PushManager.Status.STREAMING_OFF); @@ -118,7 +138,10 @@ public void onDisconnect() throws InterruptedException { public void onDisconnectAndReconnect() throws InterruptedException { // Check with mauro. reconnect should call pushManager.start again, right? TelemetryStorage telemetryStorage = new InMemoryTelemetryStorage(); LinkedBlockingQueue messsages = new LinkedBlockingQueue<>(); - SyncManagerImp syncManager = new SyncManagerImp(true, _synchronizer, _pushManager, messsages, BACKOFF_BASE, _gates, telemetryStorage); + TelemetrySynchronizer telemetrySynchronizer = Mockito.mock(TelemetrySynchronizer.class); + SplitClientConfig config = Mockito.mock(SplitClientConfig.class); + Mockito.when(_synchronizer.syncAll()).thenReturn(true); + SyncManagerImp syncManager = new SyncManagerImp(true, _synchronizer, _pushManager, messsages, BACKOFF_BASE, _gates, telemetryStorage, telemetrySynchronizer, config); syncManager.start(); messsages.offer(PushManager.Status.STREAMING_BACKOFF); Thread.sleep(1200); @@ -126,4 +149,20 @@ public void onDisconnectAndReconnect() throws InterruptedException { // Check wi Mockito.verify(_synchronizer, Mockito.times(1)).syncAll(); Mockito.verify(_pushManager, Mockito.times(2)).start(); } + + @Test + public void syncAllRetryThenShouldStartPolling() throws InterruptedException { + TelemetryStorage telemetryStorage = new InMemoryTelemetryStorage(); + TelemetrySynchronizer telemetrySynchronizer = Mockito.mock(TelemetrySynchronizer.class); + SplitClientConfig config = Mockito.mock(SplitClientConfig.class); + Mockito.when(_synchronizer.syncAll()).thenReturn(false).thenReturn(true); + SyncManagerImp syncManager = new SyncManagerImp(false, _synchronizer, _pushManager, new LinkedBlockingQueue<>(), BACKOFF_BASE, _gates, telemetryStorage, telemetrySynchronizer, config); + syncManager.start(); + Thread.sleep(2000); + Mockito.verify(_synchronizer, Mockito.times(1)).startPeriodicFetching(); + Mockito.verify(_synchronizer, Mockito.times(2)).syncAll(); + Mockito.verify(_pushManager, Mockito.times(0)).start(); + Mockito.verify(_gates, Mockito.times(1)).sdkInternalReady(); + Mockito.verify(telemetrySynchronizer, Mockito.times(1)).synchronizeConfig(Mockito.anyObject(), Mockito.anyLong(), Mockito.anyObject(), Mockito.anyObject()); + } } diff --git a/client/src/test/java/io/split/engine/common/SynchronizerTest.java b/client/src/test/java/io/split/engine/common/SynchronizerTest.java index 53e4175d1..bb13bcccd 100644 --- a/client/src/test/java/io/split/engine/common/SynchronizerTest.java +++ b/client/src/test/java/io/split/engine/common/SynchronizerTest.java @@ -34,12 +34,13 @@ public void beforeMethod() { @Test public void syncAll() throws InterruptedException { + Mockito.when(_splitFetcher.fetchAll(true)).thenReturn(true); + Mockito.when(_segmentFetcher.fetchAllSynchronous()).thenReturn(true); _synchronizer.syncAll(); - Thread.sleep(100); + Thread.sleep(1000); Mockito.verify(_splitFetcher, Mockito.times(1)).fetchAll(true); Mockito.verify(_segmentFetcher, Mockito.times(1)).fetchAllSynchronous(); - Mockito.verify(_gates, Mockito.times(1)).sdkInternalReady(); } @Test diff --git a/client/src/test/java/io/split/engine/experiments/SplitFetcherTest.java b/client/src/test/java/io/split/engine/experiments/SplitFetcherTest.java index bfd720216..595b0669d 100644 --- a/client/src/test/java/io/split/engine/experiments/SplitFetcherTest.java +++ b/client/src/test/java/io/split/engine/experiments/SplitFetcherTest.java @@ -61,7 +61,7 @@ private void works(long startingChangeNumber) throws InterruptedException { SegmentChangeFetcher segmentChangeFetcher = Mockito.mock(SegmentChangeFetcher.class); SegmentSynchronizationTask segmentSynchronizationTask = new SegmentSynchronizationTaskImp(segmentChangeFetcher,1,10, gates, segmentCache, TELEMETRY_STORAGE); SplitCache cache = new InMemoryCacheImp(startingChangeNumber); - SplitFetcherImp fetcher = new SplitFetcherImp(splitChangeFetcher, new SplitParser(segmentSynchronizationTask, segmentCache), gates, cache, TELEMETRY_STORAGE); + SplitFetcherImp fetcher = new SplitFetcherImp(splitChangeFetcher, new SplitParser(segmentSynchronizationTask, segmentCache), cache, TELEMETRY_STORAGE); // execute the fetcher for a little bit. executeWaitAndTerminate(fetcher, 1, 3, TimeUnit.SECONDS); @@ -80,9 +80,8 @@ private void works(long startingChangeNumber) throws InterruptedException { ParsedSplit expected = ParsedSplit.createParsedSplitForTests("" + cache.getChangeNumber(), (int) cache.getChangeNumber(), false, Treatments.OFF, expectedListOfMatcherAndSplits, null, cache.getChangeNumber(), 1); ParsedSplit actual = cache.get("" + cache.getChangeNumber()); - + Thread.sleep(1000); assertThat(actual, is(equalTo(expected))); - assertThat(gates.areSplitsReady(0), is(equalTo(true))); } @Test @@ -135,7 +134,7 @@ public void when_parser_fails_we_remove_the_experiment() throws InterruptedExcep SegmentSynchronizationTask segmentSynchronizationTask = new SegmentSynchronizationTaskImp(segmentChangeFetcher, 1,10, gates, segmentCache, TELEMETRY_STORAGE); segmentSynchronizationTask.startPeriodicFetching(); SplitCache cache = new InMemoryCacheImp(-1); - SplitFetcherImp fetcher = new SplitFetcherImp(splitChangeFetcher, new SplitParser(segmentSynchronizationTask, segmentCache), gates, cache, TELEMETRY_STORAGE); + SplitFetcherImp fetcher = new SplitFetcherImp(splitChangeFetcher, new SplitParser(segmentSynchronizationTask, segmentCache), cache, TELEMETRY_STORAGE); // execute the fetcher for a little bit. executeWaitAndTerminate(fetcher, 1, 5, TimeUnit.SECONDS); @@ -157,7 +156,7 @@ public void if_there_is_a_problem_talking_to_split_change_count_down_latch_is_no SegmentChangeFetcher segmentChangeFetcher = mock(SegmentChangeFetcher.class); SegmentSynchronizationTask segmentSynchronizationTask = new SegmentSynchronizationTaskImp(segmentChangeFetcher, 1,10, gates, segmentCache, TELEMETRY_STORAGE); segmentSynchronizationTask.startPeriodicFetching(); - SplitFetcherImp fetcher = new SplitFetcherImp(splitChangeFetcher, new SplitParser(segmentSynchronizationTask, segmentCache), gates, cache, TELEMETRY_STORAGE); + SplitFetcherImp fetcher = new SplitFetcherImp(splitChangeFetcher, new SplitParser(segmentSynchronizationTask, segmentCache), cache, TELEMETRY_STORAGE); // execute the fetcher for a little bit. executeWaitAndTerminate(fetcher, 1, 5, TimeUnit.SECONDS); @@ -199,7 +198,7 @@ public void works_with_user_defined_segments() throws Exception { when(segmentChangeFetcher.fetch(anyString(), anyLong(), anyBoolean())).thenReturn(segmentChange); SegmentSynchronizationTask segmentSynchronizationTask = new SegmentSynchronizationTaskImp(segmentChangeFetcher, 1,10, gates, segmentCache, TELEMETRY_STORAGE); segmentSynchronizationTask.startPeriodicFetching(); - SplitFetcherImp fetcher = new SplitFetcherImp(experimentChangeFetcher, new SplitParser(segmentSynchronizationTask, segmentCache), gates, cache, TELEMETRY_STORAGE); + SplitFetcherImp fetcher = new SplitFetcherImp(experimentChangeFetcher, new SplitParser(segmentSynchronizationTask, segmentCache), cache, TELEMETRY_STORAGE); // execute the fetcher for a little bit. executeWaitAndTerminate(fetcher, 1, 5, TimeUnit.SECONDS); @@ -212,11 +211,6 @@ public void works_with_user_defined_segments() throws Exception { assertThat("Asking for " + i + " " + cache.getAll(), cache.get("" + i), is(not(nullValue()))); assertThat(cache.get("" + i).killed(), is(true)); } - - assertThat(gates.areSplitsReady(0), is(equalTo(true))); - assertThat(gates.isSegmentRegistered(segmentName), is(equalTo(true))); - assertThat(gates.areSegmentsReady(100), is(equalTo(true))); - assertThat(gates.isSDKReady(0), is(equalTo(true))); } private SegmentChange getSegmentChange(long since, long till, String segmentName){ diff --git a/client/src/test/java/io/split/telemetry/synchronizer/SynchronizerMemoryTest.java b/client/src/test/java/io/split/telemetry/synchronizer/SynchronizerMemoryTest.java index ff45f1ba7..0afab5ddf 100644 --- a/client/src/test/java/io/split/telemetry/synchronizer/SynchronizerMemoryTest.java +++ b/client/src/test/java/io/split/telemetry/synchronizer/SynchronizerMemoryTest.java @@ -50,7 +50,8 @@ private TelemetrySynchronizer getTelemetrySynchronizer(CloseableHttpClient httpC TelemetryRuntimeProducer telemetryRuntimeProducer = Mockito.mock(TelemetryRuntimeProducer.class); SplitCache splitCache = Mockito.mock(SplitCache.class); SegmentCache segmentCache = Mockito.mock(SegmentCacheInMemoryImpl.class); - TelemetrySynchronizer telemetrySynchronizer = new SynchronizerMemory(httpClient, URI.create(TELEMETRY_ENDPOINT), consumer, splitCache, segmentCache, telemetryRuntimeProducer); + SplitClientConfig config = Mockito.mock(SplitClientConfig.class); + TelemetrySynchronizer telemetrySynchronizer = new SynchronizerMemory(httpClient, URI.create(TELEMETRY_ENDPOINT), consumer, splitCache, segmentCache, telemetryRuntimeProducer, 0l); return telemetrySynchronizer; } diff --git a/client/src/test/java/io/split/telemetry/synchronizer/TelemetryConfigInitializerTest.java b/client/src/test/java/io/split/telemetry/synchronizer/TelemetryConfigInitializerTest.java deleted file mode 100644 index 12fd8621d..000000000 --- a/client/src/test/java/io/split/telemetry/synchronizer/TelemetryConfigInitializerTest.java +++ /dev/null @@ -1,20 +0,0 @@ -package io.split.telemetry.synchronizer; - -import io.split.client.SplitClientConfig; -import io.split.engine.SDKReadinessGates; -import org.junit.Test; -import org.mockito.Mockito; - -public class TelemetryConfigInitializerTest { - - @Test - public void testRun() { - SynchronizerMemory synchronizerMemory = Mockito.mock(SynchronizerMemory.class); - SDKReadinessGates gates = Mockito.mock(SDKReadinessGates.class); - SplitClientConfig config = Mockito.mock(SplitClientConfig.class); - Mockito.when(gates.isSDKReadyNow()).thenReturn(true); - TelemetryConfigInitializer telemetryConfigInitializer = new TelemetryConfigInitializer(synchronizerMemory, gates, config); - Mockito.verify(synchronizerMemory, Mockito.times(1)).synchronizeConfig(Mockito.anyObject(),Mockito.anyLong(), Mockito.anyObject(), Mockito.anyObject()); - } - -} \ No newline at end of file From 9043acecf8b40a737b50abe04b46c499eabcd9d6 Mon Sep 17 00:00:00 2001 From: Martin Redolatti Date: Wed, 19 May 2021 12:16:47 -0300 Subject: [PATCH 51/81] improve logs, fix crap in random till generation --- client/pom.xml | 2 +- .../java/io/split/client/HttpSplitChangeFetcher.java | 3 ++- .../java/io/split/engine/common/SynchronizerImp.java | 12 ++++++++++-- .../io/split/client/HttpSplitChangeFetcherTest.java | 9 ++++++++- pom.xml | 2 +- testing/pom.xml | 2 +- 6 files changed, 23 insertions(+), 7 deletions(-) diff --git a/client/pom.xml b/client/pom.xml index 8e1c1d65c..06b5e98af 100644 --- a/client/pom.xml +++ b/client/pom.xml @@ -5,7 +5,7 @@ io.split.client java-client-parent - 4.1.7-rc4 + 4.1.7-rc5 java-client jar diff --git a/client/src/main/java/io/split/client/HttpSplitChangeFetcher.java b/client/src/main/java/io/split/client/HttpSplitChangeFetcher.java index fca610581..2f2393e51 100644 --- a/client/src/main/java/io/split/client/HttpSplitChangeFetcher.java +++ b/client/src/main/java/io/split/client/HttpSplitChangeFetcher.java @@ -60,7 +60,8 @@ private HttpSplitChangeFetcher(CloseableHttpClient client, URI uri, Metrics metr } long makeRandomTill() { - return (-1)*(int)Math.floor(Math.random()*(Math.pow(2, 63))); + + return (-1)*(long)Math.floor(Math.random()*(Math.pow(2, 63))); } @Override diff --git a/client/src/main/java/io/split/engine/common/SynchronizerImp.java b/client/src/main/java/io/split/engine/common/SynchronizerImp.java index 2859c5629..0aeea5310 100644 --- a/client/src/main/java/io/split/engine/common/SynchronizerImp.java +++ b/client/src/main/java/io/split/engine/common/SynchronizerImp.java @@ -114,7 +114,6 @@ private SyncResult attemptSync(long targetChangeNumber, if (targetChangeNumber <= _splitCache.getChangeNumber()) { return new SyncResult(true, remainingAttempts); } else if (remainingAttempts <= 0) { - _log.info(String.format("No changes fetched after %s attempts.", maxRetries)); return new SyncResult(false, remainingAttempts); } try { @@ -150,19 +149,28 @@ public void refreshSplits(long targetChangeNumber) { SyncResult regularResult = attemptSync(targetChangeNumber, opts, (discard) -> (long) _onDemandFetchRetryDelayMs, _onDemandFetchMaxRetries); + int attempts = _onDemandFetchMaxRetries - regularResult.remainingAttempts(); if (regularResult.success()) { - _log.debug(String.format("Refresh completed in %s attempts.", _onDemandFetchMaxRetries - regularResult.remainingAttempts())); + _log.debug(String.format("Refresh completed in %s attempts.", attempts)); if (_cdnResponseHeadersLogging) { logCdnHeaders(_onDemandFetchMaxRetries , regularResult.remainingAttempts(), captor.get()); } return; } + _log.info(String.format("No changes fetched after %s attempts. Will retry bypassing CDN.", attempts)); FetchOptions withCdnBypass = new FetchOptions.Builder(opts).cdnBypass(true).build(); Backoff backoff = new Backoff(ON_DEMAND_FETCH_BACKOFF_BASE_MS, ON_DEMAND_FETCH_BACKOFF_MAX_WAIT_MS); SyncResult withCDNBypassed = attemptSync(targetChangeNumber, withCdnBypass, (discard) -> backoff.interval(), ON_DEMAND_FETCH_BACKOFF_MAX_RETRIES); + int withoutCDNAttempts = ON_DEMAND_FETCH_BACKOFF_MAX_RETRIES - withCDNBypassed._remainingAttempts; + if (withCDNBypassed.success()) { + _log.debug(String.format("Refresh completed bypassing the CDN in %s attempts.", attempts)); + } else { + _log.debug(String.format("No changes fetched after %s attempts, even with CDN bypassed.", attempts)); + } + if (_cdnResponseHeadersLogging) { logCdnHeaders(_onDemandFetchMaxRetries + ON_DEMAND_FETCH_BACKOFF_MAX_RETRIES, withCDNBypassed.remainingAttempts(), captor.get()); diff --git a/client/src/test/java/io/split/client/HttpSplitChangeFetcherTest.java b/client/src/test/java/io/split/client/HttpSplitChangeFetcherTest.java index 9259cc896..259b8179c 100644 --- a/client/src/test/java/io/split/client/HttpSplitChangeFetcherTest.java +++ b/client/src/test/java/io/split/client/HttpSplitChangeFetcherTest.java @@ -24,8 +24,10 @@ import java.lang.reflect.InvocationTargetException; import java.net.URI; import java.net.URISyntaxException; +import java.util.HashSet; import java.util.List; import java.util.Map; +import java.util.Set; import static org.mockito.Mockito.when; @@ -121,10 +123,15 @@ public void testRandomNumberGeneration() throws URISyntaxException { Metrics.NoopMetrics metrics = new Metrics.NoopMetrics(); HttpSplitChangeFetcher fetcher = HttpSplitChangeFetcher.create(httpClientMock, rootTarget, metrics); + Set seen = new HashSet<>(); long min = (long)Math.pow(2, 63) * (-1); - for (long x = 0; x < 100000000; x++) { + final long total = 10000000; + for (long x = 0; x < total; x++) { long r = fetcher.makeRandomTill(); Assert.assertTrue(r < 0 && r > min); + seen.add(r); } + + Assert.assertTrue(seen.size() >= (total * 0.9999)); } } diff --git a/pom.xml b/pom.xml index 05290049e..3f96f8b08 100644 --- a/pom.xml +++ b/pom.xml @@ -4,7 +4,7 @@ 4.0.0 io.split.client java-client-parent - 4.1.7-rc4 + 4.1.7-rc5 diff --git a/testing/pom.xml b/testing/pom.xml index 61536b497..66fe079d7 100644 --- a/testing/pom.xml +++ b/testing/pom.xml @@ -6,7 +6,7 @@ io.split.client java-client-parent - 4.1.7-rc4 + 4.1.7-rc5 java-client-testing From 31c7e486a082f8562e76d6f88690e407a30e9e85 Mon Sep 17 00:00:00 2001 From: Martin Redolatti Date: Wed, 19 May 2021 14:21:31 -0300 Subject: [PATCH 52/81] ask for target changeNumber when sending `till` --- .../split/client/HttpSplitChangeFetcher.java | 4 +- .../io/split/engine/common/FetchOptions.java | 29 +++++++------ .../split/engine/common/SynchronizerImp.java | 6 +-- .../engine/experiments/SplitFetcherImp.java | 2 +- .../client/HttpSplitChangeFetcherTest.java | 4 +- .../engine/common/FetcherOptionsTest.java | 4 +- .../split/engine/common/SynchronizerTest.java | 42 +++++++++---------- .../engine/experiments/SplitFetcherTest.java | 8 ++-- 8 files changed, 52 insertions(+), 47 deletions(-) diff --git a/client/src/main/java/io/split/client/HttpSplitChangeFetcher.java b/client/src/main/java/io/split/client/HttpSplitChangeFetcher.java index 2f2393e51..9efaad6c9 100644 --- a/client/src/main/java/io/split/client/HttpSplitChangeFetcher.java +++ b/client/src/main/java/io/split/client/HttpSplitChangeFetcher.java @@ -73,8 +73,8 @@ public SplitChange fetch(long since, FetchOptions options) { try { URIBuilder uriBuilder = new URIBuilder(_target).addParameter(SINCE, "" + since); - if (options.cdnBypass()) { - uriBuilder.addParameter(TILL, "" + makeRandomTill()); + if (options.hasCustomCN()) { + uriBuilder.addParameter(TILL, "" + options.targetCN()); } URI uri = uriBuilder.build(); diff --git a/client/src/main/java/io/split/engine/common/FetchOptions.java b/client/src/main/java/io/split/engine/common/FetchOptions.java index 4da472ac1..e1996b3e8 100644 --- a/client/src/main/java/io/split/engine/common/FetchOptions.java +++ b/client/src/main/java/io/split/engine/common/FetchOptions.java @@ -5,13 +5,15 @@ import java.util.function.Function; public class FetchOptions { - + + public static final Long DEFAULT_TARGET_CHANGENUMBER = -1L; + public static class Builder { public Builder() {} public Builder(FetchOptions opts) { - _cdnBypass = opts._cdnBypass; + _targetCN = opts._targetCN; _cacheControlHeaders = opts._cacheControlHeaders; _fastlyDebugHeader = opts._fastlyDebugHeader; _responseHeadersCallback = opts._responseHeadersCallback; @@ -32,16 +34,16 @@ public Builder responseHeadersCallback(Function, Void> callb return this; } - public Builder cdnBypass(boolean bypass) { - _cdnBypass = bypass; + public Builder targetChangeNumber(long targetCN) { + _targetCN = targetCN; return this; } public FetchOptions build() { - return new FetchOptions(_cacheControlHeaders, _cdnBypass, _responseHeadersCallback, _fastlyDebugHeader); + return new FetchOptions(_cacheControlHeaders, _targetCN, _responseHeadersCallback, _fastlyDebugHeader); } - private boolean _cdnBypass = false; + private long _targetCN = DEFAULT_TARGET_CHANGENUMBER; private boolean _cacheControlHeaders = false; private boolean _fastlyDebugHeader = false; private Function, Void> _responseHeadersCallback = null; @@ -55,7 +57,9 @@ public boolean fastlyDebugHeaderEnabled() { return _fastlyDebugHeader; } - public boolean cdnBypass() { return _cdnBypass; } + public long targetCN() { return _targetCN; } + + public boolean hasCustomCN() { return _targetCN != DEFAULT_TARGET_CHANGENUMBER; } public void handleResponseHeaders(Map headers) { if (Objects.isNull(_responseHeadersCallback) || Objects.isNull(headers)) { @@ -65,11 +69,11 @@ public void handleResponseHeaders(Map headers) { } private FetchOptions(boolean cacheControlHeaders, - boolean cdnBypass, + long targetCN, Function, Void> responseHeadersCallback, boolean fastlyDebugHeader) { _cacheControlHeaders = cacheControlHeaders; - _cdnBypass = cdnBypass; + _targetCN = targetCN; _responseHeadersCallback = responseHeadersCallback; _fastlyDebugHeader = fastlyDebugHeader; } @@ -84,16 +88,17 @@ public boolean equals(Object obj) { return Objects.equals(_cacheControlHeaders, other._cacheControlHeaders) && Objects.equals(_fastlyDebugHeader, other._fastlyDebugHeader) - && Objects.equals(_responseHeadersCallback, other._responseHeadersCallback); + && Objects.equals(_responseHeadersCallback, other._responseHeadersCallback) + && Objects.equals(_targetCN, other._targetCN); } @Override public int hashCode() { - return com.google.common.base.Objects.hashCode(_cacheControlHeaders, _fastlyDebugHeader, _responseHeadersCallback); + return com.google.common.base.Objects.hashCode(_cacheControlHeaders, _fastlyDebugHeader, _responseHeadersCallback, _targetCN); } private final boolean _cacheControlHeaders; private final boolean _fastlyDebugHeader; - private final boolean _cdnBypass; + private final long _targetCN; private final Function, Void> _responseHeadersCallback; } diff --git a/client/src/main/java/io/split/engine/common/SynchronizerImp.java b/client/src/main/java/io/split/engine/common/SynchronizerImp.java index 0aeea5310..e101e0ea9 100644 --- a/client/src/main/java/io/split/engine/common/SynchronizerImp.java +++ b/client/src/main/java/io/split/engine/common/SynchronizerImp.java @@ -159,16 +159,16 @@ public void refreshSplits(long targetChangeNumber) { } _log.info(String.format("No changes fetched after %s attempts. Will retry bypassing CDN.", attempts)); - FetchOptions withCdnBypass = new FetchOptions.Builder(opts).cdnBypass(true).build(); + FetchOptions withCdnBypass = new FetchOptions.Builder(opts).targetChangeNumber(targetChangeNumber).build(); Backoff backoff = new Backoff(ON_DEMAND_FETCH_BACKOFF_BASE_MS, ON_DEMAND_FETCH_BACKOFF_MAX_WAIT_MS); SyncResult withCDNBypassed = attemptSync(targetChangeNumber, withCdnBypass, (discard) -> backoff.interval(), ON_DEMAND_FETCH_BACKOFF_MAX_RETRIES); int withoutCDNAttempts = ON_DEMAND_FETCH_BACKOFF_MAX_RETRIES - withCDNBypassed._remainingAttempts; if (withCDNBypassed.success()) { - _log.debug(String.format("Refresh completed bypassing the CDN in %s attempts.", attempts)); + _log.debug(String.format("Refresh completed bypassing the CDN in %s attempts.", withoutCDNAttempts)); } else { - _log.debug(String.format("No changes fetched after %s attempts, even with CDN bypassed.", attempts)); + _log.debug(String.format("No changes fetched after %s attempts with CDN bypassed.", withoutCDNAttempts)); } if (_cdnResponseHeadersLogging) { diff --git a/client/src/main/java/io/split/engine/experiments/SplitFetcherImp.java b/client/src/main/java/io/split/engine/experiments/SplitFetcherImp.java index 9a2402d6a..b87f5cb2a 100644 --- a/client/src/main/java/io/split/engine/experiments/SplitFetcherImp.java +++ b/client/src/main/java/io/split/engine/experiments/SplitFetcherImp.java @@ -57,7 +57,7 @@ public void forceRefresh(FetchOptions options) { // for the next fetches. (This will clear a local copy of the fetch options, // not the original object that was passed to this method). if (initialCN == start) { - options = new FetchOptions.Builder(options).cdnBypass(false).build(); + options = new FetchOptions.Builder(options).targetChangeNumber(FetchOptions.DEFAULT_TARGET_CHANGENUMBER).build(); } if (start >= end) { diff --git a/client/src/test/java/io/split/client/HttpSplitChangeFetcherTest.java b/client/src/test/java/io/split/client/HttpSplitChangeFetcherTest.java index 259b8179c..fec74be0e 100644 --- a/client/src/test/java/io/split/client/HttpSplitChangeFetcherTest.java +++ b/client/src/test/java/io/split/client/HttpSplitChangeFetcherTest.java @@ -108,11 +108,11 @@ public void testFetcherWithCDNBypassOption() throws IOException, URISyntaxExcept Metrics.NoopMetrics metrics = new Metrics.NoopMetrics(); HttpSplitChangeFetcher fetcher = HttpSplitChangeFetcher.create(httpClientMock, rootTarget, metrics); - fetcher.fetch(-1, new FetchOptions.Builder().cdnBypass(true).build()); + fetcher.fetch(-1, new FetchOptions.Builder().targetChangeNumber(123).build()); fetcher.fetch(-1, new FetchOptions.Builder().build()); List captured = requestCaptor.getAllValues(); Assert.assertEquals(captured.size(), 2); - Assert.assertTrue(captured.get(0).getUri().toString().contains("till=")); + Assert.assertTrue(captured.get(0).getUri().toString().contains("till=123")); Assert.assertFalse(captured.get(1).getUri().toString().contains("till=")); } diff --git a/client/src/test/java/io/split/engine/common/FetcherOptionsTest.java b/client/src/test/java/io/split/engine/common/FetcherOptionsTest.java index 0c5d26b40..25b2ea0fd 100644 --- a/client/src/test/java/io/split/engine/common/FetcherOptionsTest.java +++ b/client/src/test/java/io/split/engine/common/FetcherOptionsTest.java @@ -25,12 +25,12 @@ public Void apply(Map unused) { .cacheControlHeaders(true) .fastlyDebugHeader(true) .responseHeadersCallback(func) - .cdnBypass(true) + .targetChangeNumber(123) .build(); assertEquals(options.cacheControlHeadersEnabled(), true); assertEquals(options.fastlyDebugHeaderEnabled(), true); - assertEquals(options.cdnBypass(), true); + assertEquals(options.targetCN(), 123); options.handleResponseHeaders(new HashMap<>()); assertEquals(called[0], true); } diff --git a/client/src/test/java/io/split/engine/common/SynchronizerTest.java b/client/src/test/java/io/split/engine/common/SynchronizerTest.java index daf232339..00dfd2aed 100644 --- a/client/src/test/java/io/split/engine/common/SynchronizerTest.java +++ b/client/src/test/java/io/split/engine/common/SynchronizerTest.java @@ -83,7 +83,7 @@ public void streamingRetryOnSegment() { } @Test - public void testCDNBypassIsRequestedAfterNFailures() { + public void testCDNBypassIsRequestedAfterNFailures() throws NoSuchFieldException, IllegalAccessException { SplitCache cache = new InMemoryCacheImp(); Synchronizer imp = new SynchronizerImp(_refreshableSplitFetcherTask, @@ -101,19 +101,19 @@ public void testCDNBypassIsRequestedAfterNFailures() { Mockito.doAnswer(invocationOnMock -> { calls.getAndIncrement(); switch (calls.get()) { - case 4: cache.setChangeNumber(1); + case 4: cache.setChangeNumber(123); } return null; }).when(_splitFetcher).forceRefresh(optionsCaptor.capture()); - imp.refreshSplits(1); + imp.refreshSplits(123); List options = optionsCaptor.getAllValues(); Assert.assertEquals(options.size(), 4); - Assert.assertFalse(options.get(0).cdnBypass()); - Assert.assertFalse(options.get(1).cdnBypass()); - Assert.assertFalse(options.get(2).cdnBypass()); - Assert.assertTrue(options.get(3).cdnBypass()); + Assert.assertFalse(options.get(0).hasCustomCN()); + Assert.assertFalse(options.get(1).hasCustomCN()); + Assert.assertFalse(options.get(2).hasCustomCN()); + Assert.assertTrue(options.get(3).hasCustomCN()); } @Test @@ -154,20 +154,20 @@ public void testCDNBypassRequestLimitAndBackoff() throws NoSuchFieldException, I List options = optionsCaptor.getAllValues(); Assert.assertEquals(options.size(), 13); - Assert.assertFalse(options.get(0).cdnBypass()); - Assert.assertFalse(options.get(1).cdnBypass()); - Assert.assertFalse(options.get(2).cdnBypass()); - Assert.assertTrue(options.get(3).cdnBypass()); - Assert.assertTrue(options.get(4).cdnBypass()); - Assert.assertTrue(options.get(5).cdnBypass()); - Assert.assertTrue(options.get(6).cdnBypass()); - Assert.assertTrue(options.get(7).cdnBypass()); - Assert.assertTrue(options.get(8).cdnBypass()); - Assert.assertTrue(options.get(9).cdnBypass()); - Assert.assertTrue(options.get(10).cdnBypass()); - Assert.assertTrue(options.get(11).cdnBypass()); - Assert.assertTrue(options.get(12).cdnBypass()); - + Assert.assertFalse(options.get(0).hasCustomCN()); + Assert.assertFalse(options.get(1).hasCustomCN()); + Assert.assertFalse(options.get(2).hasCustomCN()); + Assert.assertTrue(options.get(3).hasCustomCN()); + Assert.assertTrue(options.get(4).hasCustomCN()); + Assert.assertTrue(options.get(5).hasCustomCN()); + Assert.assertTrue(options.get(6).hasCustomCN()); + Assert.assertTrue(options.get(7).hasCustomCN()); + Assert.assertTrue(options.get(8).hasCustomCN()); + Assert.assertTrue(options.get(9).hasCustomCN()); + Assert.assertTrue(options.get(10).hasCustomCN()); + Assert.assertTrue(options.get(11).hasCustomCN()); + Assert.assertTrue(options.get(12).hasCustomCN()); + Assert.assertEquals(calls.get(), 13); long minDiffExpected = 1 + 2 + 4 + 8 + 16 + 32 + 64 + 128 + 256; Assert.assertTrue((after - before) > minDiffExpected); diff --git a/client/src/test/java/io/split/engine/experiments/SplitFetcherTest.java b/client/src/test/java/io/split/engine/experiments/SplitFetcherTest.java index 82776e0ac..c31e83ad8 100644 --- a/client/src/test/java/io/split/engine/experiments/SplitFetcherTest.java +++ b/client/src/test/java/io/split/engine/experiments/SplitFetcherTest.java @@ -247,7 +247,7 @@ public void testBypassCdnClearedAfterFirstHit() { ArgumentCaptor cnCaptor = ArgumentCaptor.forClass(Long.class); when(mockFetcher.fetch(cnCaptor.capture(), optionsCaptor.capture())).thenReturn(response1, response2); - FetchOptions originalOptions = new FetchOptions.Builder().cdnBypass(true).build(); + FetchOptions originalOptions = new FetchOptions.Builder().targetChangeNumber(123).build(); fetcher.forceRefresh(originalOptions); List capturedCNs = cnCaptor.getAllValues(); List capturedOptions = optionsCaptor.getAllValues(); @@ -258,11 +258,11 @@ public void testBypassCdnClearedAfterFirstHit() { Assert.assertEquals(capturedCNs.get(0), Long.valueOf(-1)); Assert.assertEquals(capturedCNs.get(1), Long.valueOf(1)); - Assert.assertTrue(capturedOptions.get(0).cdnBypass()); - Assert.assertFalse(capturedOptions.get(1).cdnBypass()); + Assert.assertEquals(capturedOptions.get(0).targetCN(), 123); + Assert.assertEquals(capturedOptions.get(1).targetCN(), -1); // Ensure that the original value hasn't been modified - Assert.assertTrue(originalOptions.cdnBypass()); + Assert.assertEquals(originalOptions.targetCN(), 123); } private SegmentChange getSegmentChange(long since, long till, String segmentName){ From bed5bfd0d6b7b464fbd482c6758526808e432137 Mon Sep 17 00:00:00 2001 From: Lucas Echeverz Date: Wed, 19 May 2021 17:34:50 -0300 Subject: [PATCH 53/81] Fix PR comments --- .../java/io/split/client/ApiKeyCounter.java | 5 + .../split/client/LocalhostSplitFactory.java | 2 +- .../java/io/split/client/SplitClientImpl.java | 7 +- .../io/split/client/SplitManagerImpl.java | 11 +- .../io/split/engine/SDKReadinessGates.java | 127 +----------------- .../engine/experiments/SplitFetcherImp.java | 7 +- .../engine/segments/SegmentFetcherImp.java | 4 - .../SegmentSynchronizationTaskImp.java | 6 - .../io/split/client/ApiKeyCounterTest.java | 112 +++++++++------ .../io/split/client/SplitClientImplTest.java | 10 +- .../io/split/client/SplitManagerImplTest.java | 10 +- .../engine/experiments/SplitFetcherTest.java | 1 - .../segments/SegmentFetcherImpTest.java | 4 - .../SegmentSynchronizationTaskImpTest.java | 2 - 14 files changed, 103 insertions(+), 205 deletions(-) diff --git a/client/src/main/java/io/split/client/ApiKeyCounter.java b/client/src/main/java/io/split/client/ApiKeyCounter.java index 546e9c78b..426ade17b 100644 --- a/client/src/main/java/io/split/client/ApiKeyCounter.java +++ b/client/src/main/java/io/split/client/ApiKeyCounter.java @@ -75,4 +75,9 @@ public Map getFactoryInstances() { return factoryInstances; } + + @VisibleForTesting + void clearApiKeys() { + USED_API_KEYS.clear(); + } } diff --git a/client/src/main/java/io/split/client/LocalhostSplitFactory.java b/client/src/main/java/io/split/client/LocalhostSplitFactory.java index c716ae3ce..3a2f0a14b 100644 --- a/client/src/main/java/io/split/client/LocalhostSplitFactory.java +++ b/client/src/main/java/io/split/client/LocalhostSplitFactory.java @@ -56,9 +56,9 @@ public LocalhostSplitFactory(String directory, String file) throws IOException { SplitCache splitCache = new InMemoryCacheImp(); SDKReadinessGates sdkReadinessGates = new SDKReadinessGates(); - sdkReadinessGates.splitsAreReady(); _cacheUpdaterService = new CacheUpdaterService(splitCache); _cacheUpdaterService.updateCache(splitAndKeyToTreatment); + sdkReadinessGates.sdkInternalReady(); _client = new SplitClientImpl(this, splitCache, new ImpressionsManager.NoOpImpressionsManager(), new NoopEventClient(), SplitClientConfig.builder().setBlockUntilReadyTimeout(1).build(), sdkReadinessGates, new EvaluatorImp(splitCache), new NoopTelemetryStorage(), new NoopTelemetryStorage()); diff --git a/client/src/main/java/io/split/client/SplitClientImpl.java b/client/src/main/java/io/split/client/SplitClientImpl.java index 050acb516..9c994030d 100644 --- a/client/src/main/java/io/split/client/SplitClientImpl.java +++ b/client/src/main/java/io/split/client/SplitClientImpl.java @@ -15,7 +15,6 @@ import io.split.inputValidation.KeyValidator; import io.split.inputValidation.SplitNameValidator; import io.split.inputValidation.TrafficTypeValidator; -import io.split.telemetry.domain.enums.LastSynchronizationRecordsEnum; import io.split.telemetry.domain.enums.MethodEnum; import io.split.telemetry.storage.TelemetryConfigProducer; import io.split.telemetry.storage.TelemetryEvaluationProducer; @@ -138,7 +137,7 @@ public void blockUntilReady() throws TimeoutException, InterruptedException { if (_config.blockUntilReady() <= 0) { throw new IllegalArgumentException("setBlockUntilReadyTimeout must be positive but in config was: " + _config.blockUntilReady()); } - if (!_gates.isSDKReady(_config.blockUntilReady())) { + if (!_gates.waitUntilInternalReady(_config.blockUntilReady())) { throw new TimeoutException("SDK was not ready in " + _config.blockUntilReady()+ " milliseconds"); } _log.debug(String.format("Split SDK ready in %d ms", (System.currentTimeMillis() - startTime))); @@ -188,7 +187,7 @@ private boolean track(Event event) { private SplitResult getTreatmentWithConfigInternal(String method, String matchingKey, String bucketingKey, String split, Map attributes, MethodEnum methodEnum) { long initTime = System.currentTimeMillis(); try { - if(!_gates.isSDKReadyNow()){ + if(!_gates.isSDKReady()){ _log.warn(method + ": the SDK is not ready, results may be incorrect. Make sure to wait for SDK readiness before using this method"); _telemetryConfigProducer.recordNonReadyUsage(); } @@ -215,7 +214,7 @@ private SplitResult getTreatmentWithConfigInternal(String method, String matchin EvaluatorImp.TreatmentLabelAndChangeNumber result = _evaluator.evaluateFeature(matchingKey, bucketingKey, split, attributes); - if (result.treatment.equals(Treatments.CONTROL) && result.label.equals(Labels.DEFINITION_NOT_FOUND) && _gates.isSDKReadyNow()) { + if (result.treatment.equals(Treatments.CONTROL) && result.label.equals(Labels.DEFINITION_NOT_FOUND) && _gates.isSDKReady()) { _log.warn( "getTreatment: you passed \"" + split + "\" that does not exist in this environment, " + "please double check what Splits exist in the web console."); diff --git a/client/src/main/java/io/split/client/SplitManagerImpl.java b/client/src/main/java/io/split/client/SplitManagerImpl.java index 4d41ac36e..0edd88aaa 100644 --- a/client/src/main/java/io/split/client/SplitManagerImpl.java +++ b/client/src/main/java/io/split/client/SplitManagerImpl.java @@ -6,7 +6,6 @@ import io.split.cache.SplitCache; import io.split.engine.experiments.ParsedSplit; import io.split.inputValidation.SplitNameValidator; -import io.split.telemetry.domain.enums.MethodEnum; import io.split.telemetry.storage.TelemetryConfigProducer; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -43,7 +42,7 @@ public SplitManagerImpl(SplitCache splitCache, @Override public List splits() { - if (!_gates.isSDKReadyNow()) { { + if (!_gates.isSDKReady()) { { _log.warn("splits: the SDK is not ready, results may be incorrect. Make sure to wait for SDK readiness before using this method"); _telemetryConfigProducer.recordNonReadyUsage(); }} @@ -58,7 +57,7 @@ public List splits() { @Override public SplitView split(String featureName) { - if (!_gates.isSDKReadyNow()) { { + if (!_gates.isSDKReady()) { { _log.warn("split: the SDK is not ready, results may be incorrect. Make sure to wait for SDK readiness before using this method"); _telemetryConfigProducer.recordNonReadyUsage(); }} @@ -70,7 +69,7 @@ public SplitView split(String featureName) { ParsedSplit parsedSplit = _splitCache.get(featureName); if (parsedSplit == null) { - if (_gates.isSDKReadyNow()) { + if (_gates.isSDKReady()) { _log.warn("split: you passed \"" + featureName + "\" that does not exist in this environment, " + "please double check what Splits exist in the web console."); } @@ -82,7 +81,7 @@ public SplitView split(String featureName) { @Override public List splitNames() { - if (!_gates.isSDKReadyNow()) { { + if (!_gates.isSDKReady()) { { _log.warn("splitNames: the SDK is not ready, results may be incorrect. Make sure to wait for SDK readiness before using this method"); _telemetryConfigProducer.recordNonReadyUsage(); }} @@ -100,7 +99,7 @@ public void blockUntilReady() throws TimeoutException, InterruptedException { if (_config.blockUntilReady() <= 0) { throw new IllegalArgumentException("setBlockUntilReadyTimeout must be positive but in config was: " + _config.blockUntilReady()); } - if (!_gates.isSDKReady(_config.blockUntilReady())) { + if (!_gates.waitUntilInternalReady(_config.blockUntilReady())) { _telemetryConfigProducer.recordBURTimeout(); throw new TimeoutException("SDK was not ready in " + _config.blockUntilReady()+ " milliseconds"); } diff --git a/client/src/main/java/io/split/engine/SDKReadinessGates.java b/client/src/main/java/io/split/engine/SDKReadinessGates.java index fefcc0466..10a18fbba 100644 --- a/client/src/main/java/io/split/engine/SDKReadinessGates.java +++ b/client/src/main/java/io/split/engine/SDKReadinessGates.java @@ -15,9 +15,7 @@ public class SDKReadinessGates { private static final Logger _log = LoggerFactory.getLogger(SDKReadinessGates.class); - private final CountDownLatch _splitsAreReady = new CountDownLatch(1); private final CountDownLatch _internalReady = new CountDownLatch(1); - private final ConcurrentMap _segmentsAreReady = new ConcurrentHashMap<>(); /** * Returns true if the SDK is ready. The SDK is ready when: @@ -34,134 +32,15 @@ public class SDKReadinessGates { * @return true if the sdk is ready, false otherwise. * @throws InterruptedException if this operation was interrupted. */ - public boolean isSDKReady(long milliseconds) throws InterruptedException { + public boolean waitUntilInternalReady(long milliseconds) throws InterruptedException { return _internalReady.await(milliseconds, TimeUnit.MILLISECONDS); } - public boolean isSDKReadyNow() { - try { - return isSDKReady(0); - } catch (InterruptedException e) { - return false; - } - } - - /** - * Records that the SDK split initialization is done. - * This operation is atomic and idempotent. Repeated invocations - * will not have any impact on the state. - */ - public void splitsAreReady() { - long originalCount = _splitsAreReady.getCount(); - _splitsAreReady.countDown(); - if (originalCount > 0L) { - _log.info("splits are ready"); - } - } - - /** - * Registers a segment that the SDK should download before it is ready. - * This method should be called right after the first successful download - * of split definitions. - *

    - * Note that if this method is called in subsequent fetches of splits, - * it will return false; meaning any segments used in new splits - * will not be able to block the SDK from being marked as complete. - * - * @param segmentName the segment to register - * @return true if the segments were registered, false otherwise. - * @throws InterruptedException - */ - public boolean registerSegment(String segmentName) throws InterruptedException { - if (segmentName == null || segmentName.isEmpty() || areSplitsReady(0L)) { - return false; - } - - _segmentsAreReady.putIfAbsent(segmentName, new CountDownLatch(1)); - _log.info("Registered segment: " + segmentName); - return true; - } - - /** - * Records that the SDK segment initialization for this segment is done. - * This operation is atomic and idempotent. Repeated invocations - * will not have any impact on the state. - */ - public void segmentIsReady(String segmentName) { - CountDownLatch cdl = _segmentsAreReady.get(segmentName); - if (cdl == null) { - return; - } - - long originalCount = cdl.getCount(); - - cdl.countDown(); - - if (originalCount > 0L) { - _log.info(segmentName + " segment is ready"); - } - } - - public boolean isSegmentRegistered(String segmentName) { - return _segmentsAreReady.get(segmentName) != null; - } - - /** - * Returns true if the SDK is ready w.r.t segments. In other words, this method returns true if: - *

      - *
    1. The SDK has fetched segment definitions the first time.
      2. - *
    - *

    - * This operation will block until the SDK is ready or 'milliseconds' have passed. If the milliseconds - * are less than or equal to zero, the operation will not block and return immediately - * - * @param milliseconds time to wait for an answer. if the value is zero or negative, we will not - * block for an answer. - * @return true if the sdk is ready w.r.t splits, false otherwise. - * @throws InterruptedException if this operation was interrupted. - */ - public boolean areSegmentsReady(long milliseconds) throws InterruptedException { - long end = System.currentTimeMillis() + milliseconds; - long timeLeft = milliseconds; - - for (Map.Entry entry : _segmentsAreReady.entrySet()) { - String segmentName = entry.getKey(); - CountDownLatch cdl = entry.getValue(); - - if (!cdl.await(timeLeft, TimeUnit.MILLISECONDS)) { - _log.error(segmentName + " is not ready yet"); - return false; - } - - timeLeft = end - System.currentTimeMillis(); - } - - return true; - } - - /** - * Returns true if the SDK is ready w.r.t splits. In other words, this method returns true if: - *

      - *
    1. The SDK has fetched Split definitions the first time.
      2. - *
    - *

    - * This operation will block until the SDK is ready or 'milliseconds' have passed. If the milliseconds - * are less than or equal to zero, the operation will not block and return immediately - * - * @param milliseconds time to wait for an answer. if the value is zero or negative, we will not - * block for an answer. - * @return true if the sdk is ready w.r.t splits, false otherwise. - * @throws InterruptedException if this operation was interrupted. - */ - public boolean areSplitsReady(long milliseconds) throws InterruptedException { - return _splitsAreReady.await(milliseconds, TimeUnit.MILLISECONDS); + public boolean isSDKReady() { + return _internalReady.getCount() == 0; } public void sdkInternalReady() { _internalReady.countDown(); } - - public void waitUntilInternalReady() throws InterruptedException { - _internalReady.await(); - } } diff --git a/client/src/main/java/io/split/engine/experiments/SplitFetcherImp.java b/client/src/main/java/io/split/engine/experiments/SplitFetcherImp.java index 1d5e60e5c..19adb994d 100644 --- a/client/src/main/java/io/split/engine/experiments/SplitFetcherImp.java +++ b/client/src/main/java/io/split/engine/experiments/SplitFetcherImp.java @@ -145,26 +145,25 @@ private void runWithoutExceptionHandling(boolean addCacheHeader) throws Interrup } @Override public boolean fetchAll(boolean addCacheHeader) { - boolean fetchAllStatus = true; _log.debug("Fetch splits starting ..."); long start = _splitCache.getChangeNumber(); try { runWithoutExceptionHandling(addCacheHeader); } catch (InterruptedException e) { - fetchAllStatus = false; _log.warn("Interrupting split fetcher task"); Thread.currentThread().interrupt(); + return false; } catch (Throwable t) { - fetchAllStatus = false; _log.error("RefreshableSplitFetcher failed: " + t.getMessage()); if (_log.isDebugEnabled()) { _log.debug("Reason:", t); } + return false; } finally { if (_log.isDebugEnabled()) { _log.debug("split fetch before: " + start + ", after: " + _splitCache.getChangeNumber()); } } - return fetchAllStatus; + return true; } } diff --git a/client/src/main/java/io/split/engine/segments/SegmentFetcherImp.java b/client/src/main/java/io/split/engine/segments/SegmentFetcherImp.java index 6492c5740..3bdab0125 100644 --- a/client/src/main/java/io/split/engine/segments/SegmentFetcherImp.java +++ b/client/src/main/java/io/split/engine/segments/SegmentFetcherImp.java @@ -142,11 +142,7 @@ public void runWhitCacheHeader(){ private void fetchAndUpdate(boolean addCacheHeader) { try { // Do this again in case the previous call errored out. - _gates.registerSegment(_segmentName); callLoopRun(true, addCacheHeader); - - _gates.segmentIsReady(_segmentName); - } catch (Throwable t) { _log.error("RefreshableSegmentFetcher failed: " + t.getMessage()); if (_log.isDebugEnabled()) { diff --git a/client/src/main/java/io/split/engine/segments/SegmentSynchronizationTaskImp.java b/client/src/main/java/io/split/engine/segments/SegmentSynchronizationTaskImp.java index c6433c7ec..c40915335 100644 --- a/client/src/main/java/io/split/engine/segments/SegmentSynchronizationTaskImp.java +++ b/client/src/main/java/io/split/engine/segments/SegmentSynchronizationTaskImp.java @@ -82,12 +82,6 @@ public void initializeSegment(String segmentName) { return; } - try { - _gates.registerSegment(segmentName); - } catch (InterruptedException e) { - _log.error("Unable to register segment " + segmentName); - } - segment = new SegmentFetcherImp(segmentName, _segmentChangeFetcher, _gates, _segmentCache, _telemetryRuntimeProducer); if (_running.get()) { diff --git a/client/src/test/java/io/split/client/ApiKeyCounterTest.java b/client/src/test/java/io/split/client/ApiKeyCounterTest.java index 64304f462..c0dde2379 100644 --- a/client/src/test/java/io/split/client/ApiKeyCounterTest.java +++ b/client/src/test/java/io/split/client/ApiKeyCounterTest.java @@ -1,6 +1,7 @@ package io.split.client; import junit.framework.TestCase; +import org.junit.After; import org.junit.Assert; import org.junit.Test; @@ -11,61 +12,94 @@ public class ApiKeyCounterTest extends TestCase { private static final String FIRST_KEY = "KEYNUMBER1"; private static final String SECOND_KEY = "KEYNUMBER2"; - @Test - public void testAddingNewToken() { - ApiKeyCounter.getApiKeyCounterInstance().add(FIRST_KEY); - assertTrue(ApiKeyCounter.getApiKeyCounterInstance().isApiKeyPresent(FIRST_KEY)); + @After + public synchronized void clearApiKeys() { + ApiKeyCounter.getApiKeyCounterInstance().clearApiKeys(); + } - ApiKeyCounter.getApiKeyCounterInstance().remove(FIRST_KEY); + @Test + public synchronized void testAddingNewToken() { + try { + ApiKeyCounter.getApiKeyCounterInstance().add(FIRST_KEY); + assertTrue(ApiKeyCounter.getApiKeyCounterInstance().isApiKeyPresent(FIRST_KEY)); + } + finally { + ApiKeyCounter.getApiKeyCounterInstance().clearApiKeys(); + } } @Test - public void testAddingExistingToken() { - ApiKeyCounter.getApiKeyCounterInstance().add(FIRST_KEY); - ApiKeyCounter.getApiKeyCounterInstance().add(FIRST_KEY); + public synchronized void testAddingExistingToken() { + try { + ApiKeyCounter.getApiKeyCounterInstance().add(FIRST_KEY); + ApiKeyCounter.getApiKeyCounterInstance().add(FIRST_KEY); - assertTrue(ApiKeyCounter.getApiKeyCounterInstance().isApiKeyPresent(FIRST_KEY)); - assertEquals(2, ApiKeyCounter.getApiKeyCounterInstance().getCount(FIRST_KEY)); - ApiKeyCounter.getApiKeyCounterInstance().remove(FIRST_KEY); - ApiKeyCounter.getApiKeyCounterInstance().remove(FIRST_KEY); + assertTrue(ApiKeyCounter.getApiKeyCounterInstance().isApiKeyPresent(FIRST_KEY)); + assertEquals(2, ApiKeyCounter.getApiKeyCounterInstance().getCount(FIRST_KEY)); + } + finally { + ApiKeyCounter.getApiKeyCounterInstance().clearApiKeys(); + } } @Test - public void testRemovingToken() { - ApiKeyCounter.getApiKeyCounterInstance().add(FIRST_KEY); - ApiKeyCounter.getApiKeyCounterInstance().remove(FIRST_KEY); + public synchronized void testRemovingToken() { + try { + ApiKeyCounter.getApiKeyCounterInstance().add(FIRST_KEY); + ApiKeyCounter.getApiKeyCounterInstance().remove(FIRST_KEY); - assertFalse(ApiKeyCounter.getApiKeyCounterInstance().isApiKeyPresent(FIRST_KEY)); - assertEquals(0, ApiKeyCounter.getApiKeyCounterInstance().getCount(FIRST_KEY)); + assertFalse(ApiKeyCounter.getApiKeyCounterInstance().isApiKeyPresent(FIRST_KEY)); + assertEquals(0, ApiKeyCounter.getApiKeyCounterInstance().getCount(FIRST_KEY)); + } + finally { + ApiKeyCounter.getApiKeyCounterInstance().clearApiKeys(); + } } @Test - public void testAddingNonExistingToken() { - ApiKeyCounter.getApiKeyCounterInstance().add(FIRST_KEY); - ApiKeyCounter.getApiKeyCounterInstance().add(SECOND_KEY); + public synchronized void testAddingNonExistingToken() { + try { + ApiKeyCounter.getApiKeyCounterInstance().add(FIRST_KEY); + ApiKeyCounter.getApiKeyCounterInstance().add(SECOND_KEY); - assertTrue(ApiKeyCounter.getApiKeyCounterInstance().isApiKeyPresent(FIRST_KEY)); - assertEquals(1, ApiKeyCounter.getApiKeyCounterInstance().getCount(FIRST_KEY)); - assertEquals(1, ApiKeyCounter.getApiKeyCounterInstance().getCount(SECOND_KEY)); - ApiKeyCounter.getApiKeyCounterInstance().remove(FIRST_KEY); - ApiKeyCounter.getApiKeyCounterInstance().remove(SECOND_KEY); + assertTrue(ApiKeyCounter.getApiKeyCounterInstance().isApiKeyPresent(FIRST_KEY)); + assertEquals(1, ApiKeyCounter.getApiKeyCounterInstance().getCount(FIRST_KEY)); + assertEquals(1, ApiKeyCounter.getApiKeyCounterInstance().getCount(SECOND_KEY)); + } + finally { + ApiKeyCounter.getApiKeyCounterInstance().clearApiKeys(); + } } @Test - public void testFactoryInstances() { - ApiKeyCounter.getApiKeyCounterInstance().add(FIRST_KEY); - ApiKeyCounter.getApiKeyCounterInstance().add(FIRST_KEY); - ApiKeyCounter.getApiKeyCounterInstance().add(FIRST_KEY); - ApiKeyCounter.getApiKeyCounterInstance().add(SECOND_KEY); - ApiKeyCounter.getApiKeyCounterInstance().add(SECOND_KEY); + public synchronized void testFactoryInstances() { + try { + ApiKeyCounter.getApiKeyCounterInstance().add(FIRST_KEY); + ApiKeyCounter.getApiKeyCounterInstance().add(FIRST_KEY); + ApiKeyCounter.getApiKeyCounterInstance().add(FIRST_KEY); + ApiKeyCounter.getApiKeyCounterInstance().add(SECOND_KEY); + ApiKeyCounter.getApiKeyCounterInstance().add(SECOND_KEY); - Map factoryInstances = ApiKeyCounter.getApiKeyCounterInstance().getFactoryInstances(); - Assert.assertEquals(2, factoryInstances.size()); - Assert.assertEquals(3, factoryInstances.get(FIRST_KEY).intValue()); - ApiKeyCounter.getApiKeyCounterInstance().remove(FIRST_KEY); - ApiKeyCounter.getApiKeyCounterInstance().remove(FIRST_KEY); - ApiKeyCounter.getApiKeyCounterInstance().remove(FIRST_KEY); - ApiKeyCounter.getApiKeyCounterInstance().remove(SECOND_KEY); - ApiKeyCounter.getApiKeyCounterInstance().remove(SECOND_KEY); + Map factoryInstances = ApiKeyCounter.getApiKeyCounterInstance().getFactoryInstances(); + Assert.assertEquals(2, factoryInstances.size()); + Assert.assertEquals(3, factoryInstances.get(FIRST_KEY).intValue()); + } + finally { + ApiKeyCounter.getApiKeyCounterInstance().clearApiKeys(); + } + } + + @Test + public synchronized void testClearApiKey() { + try { + ApiKeyCounter.getApiKeyCounterInstance().add(FIRST_KEY); + ApiKeyCounter.getApiKeyCounterInstance().add(FIRST_KEY); + ApiKeyCounter.getApiKeyCounterInstance().add(FIRST_KEY); + ApiKeyCounter.getApiKeyCounterInstance().clearApiKeys(); + Assert.assertEquals(0, ApiKeyCounter.getApiKeyCounterInstance().getCount(FIRST_KEY)); + } + finally { + ApiKeyCounter.getApiKeyCounterInstance().clearApiKeys(); + } } } diff --git a/client/src/test/java/io/split/client/SplitClientImplTest.java b/client/src/test/java/io/split/client/SplitClientImplTest.java index aa508cd2f..acb1f304b 100644 --- a/client/src/test/java/io/split/client/SplitClientImplTest.java +++ b/client/src/test/java/io/split/client/SplitClientImplTest.java @@ -153,7 +153,7 @@ public void works() { SDKReadinessGates gates = mock(SDKReadinessGates.class); SplitCache splitCache = mock(InMemoryCacheImp.class); when(splitCache.get(test)).thenReturn(parsedSplit); - when(gates.isSDKReadyNow()).thenReturn(true); + when(gates.isSDKReady()).thenReturn(true); SplitClientImpl client = new SplitClientImpl( mock(SplitFactory.class), @@ -326,7 +326,7 @@ public void multiple_conditions_work() { SDKReadinessGates gates = mock(SDKReadinessGates.class); SplitCache splitCache = mock(InMemoryCacheImp.class); when(splitCache.get(test)).thenReturn(parsedSplit); - when(gates.isSDKReadyNow()).thenReturn(false); + when(gates.isSDKReady()).thenReturn(false); SplitClientImpl client = new SplitClientImpl( mock(SplitFactory.class), @@ -905,7 +905,7 @@ private Partition partition(String treatment, int size) { public void block_until_ready_does_not_time_when_sdk_is_ready() throws TimeoutException, InterruptedException { SplitCache splitCache = mock(InMemoryCacheImp.class); SDKReadinessGates ready = mock(SDKReadinessGates.class); - when(ready.isSDKReady(100)).thenReturn(true); + when(ready.waitUntilInternalReady(100)).thenReturn(true); SplitClientImpl client = new SplitClientImpl( mock(SplitFactory.class), @@ -924,7 +924,7 @@ public void block_until_ready_does_not_time_when_sdk_is_ready() throws TimeoutEx public void block_until_ready_times_when_sdk_is_not_ready() throws TimeoutException, InterruptedException { SplitCache splitCache = mock(InMemoryCacheImp.class); SDKReadinessGates ready = mock(SDKReadinessGates.class); - when(ready.isSDKReady(100)).thenReturn(false); + when(ready.waitUntilInternalReady(100)).thenReturn(false); SplitClientImpl client = new SplitClientImpl( mock(SplitFactory.class), @@ -943,7 +943,7 @@ public void block_until_ready_times_when_sdk_is_not_ready() throws TimeoutExcept public void track_with_valid_parameters() { SDKReadinessGates gates = mock(SDKReadinessGates.class); SplitCache splitCache = mock(InMemoryCacheImp.class); - when(gates.isSDKReadyNow()).thenReturn(false); + when(gates.isSDKReady()).thenReturn(false); SplitClientImpl client = new SplitClientImpl( mock(SplitFactory.class), splitCache, diff --git a/client/src/test/java/io/split/client/SplitManagerImplTest.java b/client/src/test/java/io/split/client/SplitManagerImplTest.java index a77674045..201f1fedc 100644 --- a/client/src/test/java/io/split/client/SplitManagerImplTest.java +++ b/client/src/test/java/io/split/client/SplitManagerImplTest.java @@ -100,7 +100,7 @@ public void splitsCallWithNoSplit() { SplitCache splitCache = Mockito.mock(SplitCache.class); Mockito.when(splitCache.getAll()).thenReturn(Lists.newArrayList()); SDKReadinessGates gates = Mockito.mock(SDKReadinessGates.class); - Mockito.when(gates.isSDKReadyNow()).thenReturn(false); + Mockito.when(gates.isSDKReady()).thenReturn(false); SplitManagerImpl splitManager = new SplitManagerImpl(splitCache, Mockito.mock(SplitClientConfig.class), gates, TELEMETRY_STORAGE); @@ -113,7 +113,7 @@ public void splitsCallWithSplit() { SplitCache splitCache = Mockito.mock(SplitCache.class); List parsedSplits = Lists.newArrayList(); SDKReadinessGates gates = Mockito.mock(SDKReadinessGates.class); - Mockito.when(gates.isSDKReadyNow()).thenReturn(false); + Mockito.when(gates.isSDKReady()).thenReturn(false); ParsedSplit response = ParsedSplit.createParsedSplitForTests("FeatureName", 123, true, "off", Lists.newArrayList(getTestCondition("off")), "traffic", 456L, 1); parsedSplits.add(response); @@ -137,7 +137,7 @@ public void splitNamesCallWithNoSplit() { SplitCache splitCache = Mockito.mock(SplitCache.class); Mockito.when(splitCache.getAll()).thenReturn(Lists.newArrayList()); SDKReadinessGates gates = Mockito.mock(SDKReadinessGates.class); - Mockito.when(gates.isSDKReadyNow()).thenReturn(false); + Mockito.when(gates.isSDKReady()).thenReturn(false); SplitManagerImpl splitManager = new SplitManagerImpl(splitCache, Mockito.mock(SplitClientConfig.class), gates, TELEMETRY_STORAGE); @@ -164,7 +164,7 @@ public void splitNamesCallWithSplit() { @Test public void block_until_ready_does_not_time_when_sdk_is_ready() throws TimeoutException, InterruptedException { SDKReadinessGates ready = mock(SDKReadinessGates.class); - when(ready.isSDKReady(100)).thenReturn(true); + when(ready.waitUntilInternalReady(100)).thenReturn(true); SplitManagerImpl splitManager = new SplitManagerImpl(mock(SplitCache.class), config, ready, TELEMETRY_STORAGE); @@ -175,7 +175,7 @@ public void block_until_ready_does_not_time_when_sdk_is_ready() throws TimeoutEx @Test(expected = TimeoutException.class) public void block_until_ready_times_when_sdk_is_not_ready() throws TimeoutException, InterruptedException { SDKReadinessGates ready = mock(SDKReadinessGates.class); - when(ready.isSDKReady(100)).thenReturn(false); + when(ready.waitUntilInternalReady(100)).thenReturn(false); SplitManagerImpl splitManager = new SplitManagerImpl(mock(SplitCache.class), config, diff --git a/client/src/test/java/io/split/engine/experiments/SplitFetcherTest.java b/client/src/test/java/io/split/engine/experiments/SplitFetcherTest.java index 595b0669d..245d07381 100644 --- a/client/src/test/java/io/split/engine/experiments/SplitFetcherTest.java +++ b/client/src/test/java/io/split/engine/experiments/SplitFetcherTest.java @@ -162,7 +162,6 @@ public void if_there_is_a_problem_talking_to_split_change_count_down_latch_is_no executeWaitAndTerminate(fetcher, 1, 5, TimeUnit.SECONDS); assertThat(cache.getChangeNumber(), is(equalTo(-1L))); - assertThat(gates.areSplitsReady(0), is(equalTo(false))); } private void executeWaitAndTerminate(Runnable runnable, long frequency, long waitInBetween, TimeUnit unit) throws InterruptedException { diff --git a/client/src/test/java/io/split/engine/segments/SegmentFetcherImpTest.java b/client/src/test/java/io/split/engine/segments/SegmentFetcherImpTest.java index d41b9c829..c5f51afc8 100644 --- a/client/src/test/java/io/split/engine/segments/SegmentFetcherImpTest.java +++ b/client/src/test/java/io/split/engine/segments/SegmentFetcherImpTest.java @@ -49,7 +49,6 @@ public void works_when_we_start_with_state() throws InterruptedException { public void works_when_there_are_no_changes() throws InterruptedException { long startingChangeNumber = -1L; SDKReadinessGates gates = new SDKReadinessGates(); - gates.registerSegment(SEGMENT_NAME); SegmentCache segmentCache = new SegmentCacheInMemoryImpl(); SegmentChangeFetcher segmentChangeFetcher = Mockito.mock(SegmentChangeFetcher.class); @@ -79,14 +78,12 @@ public void works_when_there_are_no_changes() throws InterruptedException { assertNotNull(segmentCache.getChangeNumber(SEGMENT_NAME)); assertEquals(10L, segmentCache.getChangeNumber(SEGMENT_NAME)); - assertThat(gates.areSegmentsReady(10), is(true)); } private void works(long startingChangeNumber) throws InterruptedException { SDKReadinessGates gates = new SDKReadinessGates(); String segmentName = SEGMENT_NAME; - gates.registerSegment(segmentName); SegmentCache segmentCache = Mockito.mock(SegmentCache.class); Mockito.when(segmentCache.getChangeNumber(SEGMENT_NAME)).thenReturn(-1L).thenReturn(-1L) .thenReturn(-1L) @@ -116,7 +113,6 @@ private void works(long startingChangeNumber) throws InterruptedException { Thread.currentThread().interrupt(); } Mockito.verify(segmentChangeFetcher, Mockito.times(2)).fetch(Mockito.anyString(), Mockito.anyLong(), Mockito.anyBoolean()); - assertThat(gates.areSegmentsReady(10), is(true)); } diff --git a/client/src/test/java/io/split/engine/segments/SegmentSynchronizationTaskImpTest.java b/client/src/test/java/io/split/engine/segments/SegmentSynchronizationTaskImpTest.java index 722458e3c..c49d29962 100644 --- a/client/src/test/java/io/split/engine/segments/SegmentSynchronizationTaskImpTest.java +++ b/client/src/test/java/io/split/engine/segments/SegmentSynchronizationTaskImpTest.java @@ -75,8 +75,6 @@ public void run() { Thread.currentThread().interrupt(); } - gates.splitsAreReady(); - assertThat(fetcher1.get(), is(notNullValue())); assertThat(fetcher1.get(), is(sameInstance(fetcher2.get()))); } From 9d8dd475a2b4d40ea9b30a3d561c665e3e5e217f Mon Sep 17 00:00:00 2001 From: Lucas Echeverz Date: Wed, 19 May 2021 18:44:22 -0300 Subject: [PATCH 54/81] Travis fix --- client/src/test/java/io/split/client/ApiKeyCounterTest.java | 2 ++ 1 file changed, 2 insertions(+) diff --git a/client/src/test/java/io/split/client/ApiKeyCounterTest.java b/client/src/test/java/io/split/client/ApiKeyCounterTest.java index c0dde2379..1513313b8 100644 --- a/client/src/test/java/io/split/client/ApiKeyCounterTest.java +++ b/client/src/test/java/io/split/client/ApiKeyCounterTest.java @@ -74,6 +74,7 @@ public synchronized void testAddingNonExistingToken() { @Test public synchronized void testFactoryInstances() { try { + ApiKeyCounter.getApiKeyCounterInstance().clearApiKeys(); ApiKeyCounter.getApiKeyCounterInstance().add(FIRST_KEY); ApiKeyCounter.getApiKeyCounterInstance().add(FIRST_KEY); ApiKeyCounter.getApiKeyCounterInstance().add(FIRST_KEY); @@ -83,6 +84,7 @@ public synchronized void testFactoryInstances() { Map factoryInstances = ApiKeyCounter.getApiKeyCounterInstance().getFactoryInstances(); Assert.assertEquals(2, factoryInstances.size()); Assert.assertEquals(3, factoryInstances.get(FIRST_KEY).intValue()); + Assert.assertEquals(2, factoryInstances.get(SECOND_KEY).intValue()); } finally { ApiKeyCounter.getApiKeyCounterInstance().clearApiKeys(); From 1a25c13a88ac598ceb8c55f11bc6ec7280d89fea Mon Sep 17 00:00:00 2001 From: Martin Redolatti Date: Thu, 20 May 2021 17:52:37 -0300 Subject: [PATCH 55/81] add retries and cdn bypass logic to segment fetchers --- .../client/HttpSegmentChangeFetcher.java | 35 +++++-- .../io/split/client/jmx/SplitJmxMonitor.java | 2 +- .../split/engine/common/SynchronizerImp.java | 99 +++++++++++++++---- .../engine/experiments/SplitFetcherImp.java | 4 +- .../engine/segments/SegmentChangeFetcher.java | 4 +- .../split/engine/segments/SegmentFetcher.java | 4 +- .../engine/segments/SegmentFetcherImp.java | 30 +++--- .../client/HttpSegmentChangeFetcherTest.java | 3 +- .../engine/experiments/SplitFetcherTest.java | 2 +- .../engine/experiments/SplitParserTest.java | 20 ++-- .../segments/SegmentFetcherImpTest.java | 8 +- 11 files changed, 149 insertions(+), 62 deletions(-) diff --git a/client/src/main/java/io/split/client/HttpSegmentChangeFetcher.java b/client/src/main/java/io/split/client/HttpSegmentChangeFetcher.java index 7d7d735f6..5361ad11d 100644 --- a/client/src/main/java/io/split/client/HttpSegmentChangeFetcher.java +++ b/client/src/main/java/io/split/client/HttpSegmentChangeFetcher.java @@ -4,11 +4,13 @@ import io.split.client.dtos.SegmentChange; import io.split.client.utils.Json; import io.split.client.utils.Utils; +import io.split.engine.common.FetchOptions; import io.split.engine.metrics.Metrics; import io.split.engine.segments.SegmentChangeFetcher; import org.apache.hc.client5.http.classic.methods.HttpGet; import org.apache.hc.client5.http.impl.classic.CloseableHttpClient; import org.apache.hc.client5.http.impl.classic.CloseableHttpResponse; +import org.apache.hc.core5.http.Header; import org.apache.hc.core5.http.io.entity.EntityUtils; import org.apache.hc.core5.net.URIBuilder; import org.slf4j.Logger; @@ -17,6 +19,8 @@ import java.net.URI; import java.net.URISyntaxException; import java.nio.charset.StandardCharsets; +import java.util.Arrays; +import java.util.stream.Collectors; import static com.google.common.base.Preconditions.checkNotNull; @@ -27,9 +31,13 @@ public final class HttpSegmentChangeFetcher implements SegmentChangeFetcher { private static final Logger _log = LoggerFactory.getLogger(HttpSegmentChangeFetcher.class); private static final String SINCE = "since"; + private static final String TILL = "till"; private static final String PREFIX = "segmentChangeFetcher"; - private static final String NAME_CACHE = "Cache-Control"; - private static final String VALUE_CACHE = "no-cache"; + private static final String CACHE_CONTROL_HEADER_NAME = "Cache-Control"; + private static final String CACHE_CONTROL_HEADER_VALUE = "no-cache"; + + private static final String HEADER_FASTLY_DEBUG_NAME = "Fastly-Debug"; + private static final String HEADER_FASTLY_DEBUG_VALUE = "1"; private final CloseableHttpClient _client; private final URI _target; @@ -51,19 +59,34 @@ private HttpSegmentChangeFetcher(CloseableHttpClient client, URI uri, Metrics me } @Override - public SegmentChange fetch(String segmentName, long since, boolean addCacheHeader) { + public SegmentChange fetch(String segmentName, long since, FetchOptions options) { long start = System.currentTimeMillis(); CloseableHttpResponse response = null; try { String path = _target.getPath() + "/" + segmentName; - URI uri = new URIBuilder(_target).setPath(path).addParameter(SINCE, "" + since).build(); + URIBuilder uriBuilder = new URIBuilder(_target) + .setPath(path) + .addParameter(SINCE, "" + since); + if (options.hasCustomCN()) { + uriBuilder.addParameter(TILL, "" + options.targetCN()); + } + + URI uri = uriBuilder.build(); HttpGet request = new HttpGet(uri); - if(addCacheHeader) { - request.setHeader(NAME_CACHE, VALUE_CACHE); + + if(options.cacheControlHeadersEnabled()) { + request.setHeader(CACHE_CONTROL_HEADER_NAME, CACHE_CONTROL_HEADER_VALUE); } + + if (options.fastlyDebugHeaderEnabled()) { + request.addHeader(HEADER_FASTLY_DEBUG_NAME, HEADER_FASTLY_DEBUG_VALUE); + } + response = _client.execute(request); + options.handleResponseHeaders(Arrays.stream(response.getHeaders()) + .collect(Collectors.toMap(Header::getName, Header::getValue))); int statusCode = response.getCode(); diff --git a/client/src/main/java/io/split/client/jmx/SplitJmxMonitor.java b/client/src/main/java/io/split/client/jmx/SplitJmxMonitor.java index 559fdf25b..0fcbea305 100644 --- a/client/src/main/java/io/split/client/jmx/SplitJmxMonitor.java +++ b/client/src/main/java/io/split/client/jmx/SplitJmxMonitor.java @@ -44,7 +44,7 @@ public boolean forceSyncFeatures() { public boolean forceSyncSegment(String segmentName) { SegmentFetcher fetcher = _segmentSynchronizationTask.getFetcher(segmentName); try{ - fetcher.fetch(true); + fetcher.fetch(new FetchOptions.Builder().build()); } //We are sure this will never happen because getFetcher firts initiate the segment. This try/catch is for safe only. catch (NullPointerException np){ diff --git a/client/src/main/java/io/split/engine/common/SynchronizerImp.java b/client/src/main/java/io/split/engine/common/SynchronizerImp.java index e101e0ea9..5574c191e 100644 --- a/client/src/main/java/io/split/engine/common/SynchronizerImp.java +++ b/client/src/main/java/io/split/engine/common/SynchronizerImp.java @@ -103,10 +103,10 @@ private static class SyncResult { private final int _remainingAttempts; } - private SyncResult attemptSync(long targetChangeNumber, - FetchOptions opts, - Function nextWaitMs, - int maxRetries) { + private SyncResult attemptSplitsSync(long targetChangeNumber, + FetchOptions opts, + Function nextWaitMs, + int maxRetries) { int remainingAttempts = maxRetries; while(true) { remainingAttempts--; @@ -126,9 +126,9 @@ private SyncResult attemptSync(long targetChangeNumber, } } - private void logCdnHeaders(int maxRetries, int remainingAttempts, List> headers) { + private void logCdnHeaders(String prefix, int maxRetries, int remainingAttempts, List> headers) { if (maxRetries - remainingAttempts > _failedAttemptsBeforeLogging) { - _log.info(String.format("CDN Debug headers: %s", gson.toJson(headers))); + _log.info(String.format("%s: CDN Debug headers: %s", prefix, gson.toJson(headers))); } } @@ -146,14 +146,14 @@ public void refreshSplits(long targetChangeNumber) { .responseHeadersCallback(_cdnResponseHeadersLogging ? captor::handle : null) .build(); - SyncResult regularResult = attemptSync(targetChangeNumber, opts, + SyncResult regularResult = attemptSplitsSync(targetChangeNumber, opts, (discard) -> (long) _onDemandFetchRetryDelayMs, _onDemandFetchMaxRetries); int attempts = _onDemandFetchMaxRetries - regularResult.remainingAttempts(); if (regularResult.success()) { _log.debug(String.format("Refresh completed in %s attempts.", attempts)); if (_cdnResponseHeadersLogging) { - logCdnHeaders(_onDemandFetchMaxRetries , regularResult.remainingAttempts(), captor.get()); + logCdnHeaders("[splits]", _onDemandFetchMaxRetries , regularResult.remainingAttempts(), captor.get()); } return; } @@ -161,7 +161,7 @@ public void refreshSplits(long targetChangeNumber) { _log.info(String.format("No changes fetched after %s attempts. Will retry bypassing CDN.", attempts)); FetchOptions withCdnBypass = new FetchOptions.Builder(opts).targetChangeNumber(targetChangeNumber).build(); Backoff backoff = new Backoff(ON_DEMAND_FETCH_BACKOFF_BASE_MS, ON_DEMAND_FETCH_BACKOFF_MAX_WAIT_MS); - SyncResult withCDNBypassed = attemptSync(targetChangeNumber, withCdnBypass, + SyncResult withCDNBypassed = attemptSplitsSync(targetChangeNumber, withCdnBypass, (discard) -> backoff.interval(), ON_DEMAND_FETCH_BACKOFF_MAX_RETRIES); int withoutCDNAttempts = ON_DEMAND_FETCH_BACKOFF_MAX_RETRIES - withCDNBypassed._remainingAttempts; @@ -172,7 +172,7 @@ public void refreshSplits(long targetChangeNumber) { } if (_cdnResponseHeadersLogging) { - logCdnHeaders(_onDemandFetchMaxRetries + ON_DEMAND_FETCH_BACKOFF_MAX_RETRIES, + logCdnHeaders("[splits]", _onDemandFetchMaxRetries + ON_DEMAND_FETCH_BACKOFF_MAX_RETRIES, withCDNBypassed.remainingAttempts(), captor.get()); } } @@ -185,19 +185,76 @@ public void localKillSplit(String splitName, String defaultTreatment, long newCh } } - @Override - public void refreshSegment(String segmentName, long changeNumber) { - int retries = 1; - while(changeNumber > _segmentCache.getChangeNumber(segmentName) && retries <= _onDemandFetchMaxRetries) { - SegmentFetcher fetcher = _segmentSynchronizationTaskImp.getFetcher(segmentName); - try{ - fetcher.fetch(true); + public SyncResult attemptSegmentSync(String segmentName, + long targetChangeNumber, + FetchOptions opts, + Function nextWaitMs, + int maxRetries) { + + int remainingAttempts = maxRetries; + SegmentFetcher fetcher = _segmentSynchronizationTaskImp.getFetcher(segmentName); + checkNotNull(fetcher); + + while(true) { + remainingAttempts--; + fetcher.fetch(opts); + if (targetChangeNumber <= _segmentCache.getChangeNumber(segmentName)) { + return new SyncResult(true, remainingAttempts); + } else if (remainingAttempts <= 0) { + return new SyncResult(false, remainingAttempts); } - //We are sure this will never happen because getFetcher firts initiate the segment. This try/catch is for safe only. - catch (NullPointerException np){ - throw new NullPointerException(); + try { + long howLong = nextWaitMs.apply(null); + Thread.sleep(howLong); + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + _log.debug("Error trying to sleep current Thread."); + } + } + } + + @Override + public void refreshSegment(String segmentName, long targetChangeNumber) { + + if (targetChangeNumber <= _segmentCache.getChangeNumber(segmentName)) { + return; + } + + FastlyHeadersCaptor captor = new FastlyHeadersCaptor(); + FetchOptions opts = new FetchOptions.Builder() + .cacheControlHeaders(true) + .fastlyDebugHeader(_cdnResponseHeadersLogging) + .responseHeadersCallback(_cdnResponseHeadersLogging ? captor::handle : null) + .build(); + + SyncResult regularResult = attemptSegmentSync(segmentName, targetChangeNumber, opts, + (discard) -> (long) _onDemandFetchRetryDelayMs, _onDemandFetchMaxRetries); + + int attempts = _onDemandFetchMaxRetries - regularResult.remainingAttempts(); + if (regularResult.success()) { + _log.debug(String.format("Segment %s refresh completed in %s attempts.", segmentName, attempts)); + if (_cdnResponseHeadersLogging) { + logCdnHeaders(String.format("[segment/%s]", segmentName), _onDemandFetchMaxRetries , regularResult.remainingAttempts(), captor.get()); } - retries++; + return; + } + + _log.info(String.format("No changes fetched for segment %s after %s attempts. Will retry bypassing CDN.", segmentName, attempts)); + FetchOptions withCdnBypass = new FetchOptions.Builder(opts).targetChangeNumber(targetChangeNumber).build(); + Backoff backoff = new Backoff(ON_DEMAND_FETCH_BACKOFF_BASE_MS, ON_DEMAND_FETCH_BACKOFF_MAX_WAIT_MS); + SyncResult withCDNBypassed = attemptSegmentSync(segmentName, targetChangeNumber, withCdnBypass, + (discard) -> backoff.interval(), ON_DEMAND_FETCH_BACKOFF_MAX_RETRIES); + + int withoutCDNAttempts = ON_DEMAND_FETCH_BACKOFF_MAX_RETRIES - withCDNBypassed._remainingAttempts; + if (withCDNBypassed.success()) { + _log.debug(String.format("Segment %s refresh completed bypassing the CDN in %s attempts.", segmentName, withoutCDNAttempts)); + } else { + _log.debug(String.format("No changes fetched for segment %s after %s attempts with CDN bypassed.", segmentName, withoutCDNAttempts)); + } + + if (_cdnResponseHeadersLogging) { + logCdnHeaders(String.format("[segment/%s]", segmentName), _onDemandFetchMaxRetries + ON_DEMAND_FETCH_BACKOFF_MAX_RETRIES, + withCDNBypassed.remainingAttempts(), captor.get()); } } } diff --git a/client/src/main/java/io/split/engine/experiments/SplitFetcherImp.java b/client/src/main/java/io/split/engine/experiments/SplitFetcherImp.java index b87f5cb2a..dbfcd3fd8 100644 --- a/client/src/main/java/io/split/engine/experiments/SplitFetcherImp.java +++ b/client/src/main/java/io/split/engine/experiments/SplitFetcherImp.java @@ -46,7 +46,7 @@ public SplitFetcherImp(SplitChangeFetcher splitChangeFetcher, SplitParser parser @Override public void forceRefresh(FetchOptions options) { _log.debug("Force Refresh splits starting ..."); - final long initialCN = _splitCache.getChangeNumber(); + final long INITIAL_CN = _splitCache.getChangeNumber(); try { while (true) { long start = _splitCache.getChangeNumber(); @@ -56,7 +56,7 @@ public void forceRefresh(FetchOptions options) { // If the previous execution was the first one, clear the `cdnBypass` flag // for the next fetches. (This will clear a local copy of the fetch options, // not the original object that was passed to this method). - if (initialCN == start) { + if (INITIAL_CN == start) { options = new FetchOptions.Builder(options).targetChangeNumber(FetchOptions.DEFAULT_TARGET_CHANGENUMBER).build(); } diff --git a/client/src/main/java/io/split/engine/segments/SegmentChangeFetcher.java b/client/src/main/java/io/split/engine/segments/SegmentChangeFetcher.java index f4d46ed13..55fd23266 100644 --- a/client/src/main/java/io/split/engine/segments/SegmentChangeFetcher.java +++ b/client/src/main/java/io/split/engine/segments/SegmentChangeFetcher.java @@ -1,6 +1,7 @@ package io.split.engine.segments; import io.split.client.dtos.SegmentChange; +import io.split.engine.common.FetchOptions; /** * Fetches changes in the segment since a reference point. @@ -22,8 +23,9 @@ public interface SegmentChangeFetcher { * @param segmentName the name of the segment to fetch. * @param changesSinceThisChangeNumber a value less than zero implies that the client is * requesting information on this segment for the first time. + * @param options * @return SegmentChange * @throws java.lang.RuntimeException if there was a problem fetching segment changes */ - SegmentChange fetch(String segmentName, long changesSinceThisChangeNumber, boolean addCacheHeader); + SegmentChange fetch(String segmentName, long changesSinceThisChangeNumber, FetchOptions options); } diff --git a/client/src/main/java/io/split/engine/segments/SegmentFetcher.java b/client/src/main/java/io/split/engine/segments/SegmentFetcher.java index af4bbc767..81996d613 100644 --- a/client/src/main/java/io/split/engine/segments/SegmentFetcher.java +++ b/client/src/main/java/io/split/engine/segments/SegmentFetcher.java @@ -1,5 +1,7 @@ package io.split.engine.segments; +import io.split.engine.common.FetchOptions; + /** * Created by adilaijaz on 5/7/15. */ @@ -7,7 +9,7 @@ public interface SegmentFetcher { /** * fetch */ - void fetch(boolean addCacheHeader); + void fetch(FetchOptions opts); void runWhitCacheHeader(); diff --git a/client/src/main/java/io/split/engine/segments/SegmentFetcherImp.java b/client/src/main/java/io/split/engine/segments/SegmentFetcherImp.java index ac21e8461..ee219d624 100644 --- a/client/src/main/java/io/split/engine/segments/SegmentFetcherImp.java +++ b/client/src/main/java/io/split/engine/segments/SegmentFetcherImp.java @@ -3,6 +3,7 @@ import io.split.cache.SegmentCache; import io.split.client.dtos.SegmentChange; import io.split.engine.SDKReadinessGates; +import io.split.engine.common.FetchOptions; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -31,9 +32,9 @@ public SegmentFetcherImp(String segmentName, SegmentChangeFetcher segmentChangeF } @Override - public void fetch(boolean addCacheHeader){ + public void fetch(FetchOptions opts){ try { - callLoopRun(false, addCacheHeader); + callLoopRun(opts); } catch (Throwable t) { _log.error("RefreshableSegmentFetcher failed: " + t.getMessage()); if (_log.isDebugEnabled()) { @@ -42,8 +43,8 @@ public void fetch(boolean addCacheHeader){ } } - private void runWithoutExceptionHandling(boolean addCacheHeader) { - SegmentChange change = _segmentChangeFetcher.fetch(_segmentName, _segmentCache.getChangeNumber(_segmentName), addCacheHeader); + private void runWithoutExceptionHandling(FetchOptions options) { + SegmentChange change = _segmentChangeFetcher.fetch(_segmentName, _segmentCache.getChangeNumber(_segmentName), options); if (change == null) { throw new IllegalStateException("SegmentChange was null"); @@ -109,14 +110,15 @@ private String summarize(List changes) { return bldr.toString(); } - private void callLoopRun(boolean isFetch, boolean addCacheHeader){ + private void callLoopRun(FetchOptions opts){ + final long INITIAL_CN = _segmentCache.getChangeNumber(_segmentName); while (true) { long start = _segmentCache.getChangeNumber(_segmentName); - runWithoutExceptionHandling(addCacheHeader); - long end = _segmentCache.getChangeNumber(_segmentName); - if (isFetch && _log.isDebugEnabled()) { - _log.debug(_segmentName + " segment fetch before: " + start + ", after: " + _segmentCache.getChangeNumber(_segmentName) /*+ " size: " + _concurrentKeySet.size()*/); + runWithoutExceptionHandling(opts); + if (INITIAL_CN == start) { + opts = new FetchOptions.Builder(opts).targetChangeNumber(FetchOptions.DEFAULT_TARGET_CHANGENUMBER).build(); } + long end = _segmentCache.getChangeNumber(_segmentName); if (start >= end) { break; } @@ -125,18 +127,18 @@ private void callLoopRun(boolean isFetch, boolean addCacheHeader){ @Override public void runWhitCacheHeader(){ - this.fetchAndUpdate(true); + this.fetchAndUpdate(new FetchOptions.Builder().cacheControlHeaders(true).build()); } /** * Calls callLoopRun and after fetchs segment. - * @param addCacheHeader indicates if CacheHeader is required + * @param opts contains all soft of options used when issuing the fetch request */ - private void fetchAndUpdate(boolean addCacheHeader) { + private void fetchAndUpdate(FetchOptions opts) { try { // Do this again in case the previous call errored out. _gates.registerSegment(_segmentName); - callLoopRun(true, addCacheHeader); + callLoopRun(opts); _gates.segmentIsReady(_segmentName); @@ -150,7 +152,7 @@ private void fetchAndUpdate(boolean addCacheHeader) { @Override public void fetchAll() { - this.fetchAndUpdate(false); + this.fetchAndUpdate(new FetchOptions.Builder().build()); } diff --git a/client/src/test/java/io/split/client/HttpSegmentChangeFetcherTest.java b/client/src/test/java/io/split/client/HttpSegmentChangeFetcherTest.java index afb238552..8030e4110 100644 --- a/client/src/test/java/io/split/client/HttpSegmentChangeFetcherTest.java +++ b/client/src/test/java/io/split/client/HttpSegmentChangeFetcherTest.java @@ -2,6 +2,7 @@ import io.split.TestHelper; import io.split.client.dtos.SegmentChange; +import io.split.engine.common.FetchOptions; import io.split.engine.metrics.Metrics; import org.apache.hc.client5.http.impl.classic.CloseableHttpClient; import org.apache.hc.client5.http.impl.classic.HttpClients; @@ -61,7 +62,7 @@ public void testFetcherWithSpecialCharacters() throws URISyntaxException, IOExce Metrics.NoopMetrics metrics = new Metrics.NoopMetrics(); HttpSegmentChangeFetcher fetcher = HttpSegmentChangeFetcher.create(httpClientMock, rootTarget, metrics); - SegmentChange change = fetcher.fetch("some_segment", 1234567, true); + SegmentChange change = fetcher.fetch("some_segment", 1234567, new FetchOptions.Builder().build()); Assert.assertNotNull(change); Assert.assertEquals(1, change.added.size()); diff --git a/client/src/test/java/io/split/engine/experiments/SplitFetcherTest.java b/client/src/test/java/io/split/engine/experiments/SplitFetcherTest.java index c31e83ad8..8e5379176 100644 --- a/client/src/test/java/io/split/engine/experiments/SplitFetcherTest.java +++ b/client/src/test/java/io/split/engine/experiments/SplitFetcherTest.java @@ -198,7 +198,7 @@ public void works_with_user_defined_segments() throws Exception { SegmentChangeFetcher segmentChangeFetcher = mock(SegmentChangeFetcher.class); SegmentChange segmentChange = getSegmentChange(0L, 0L, segmentName); - when(segmentChangeFetcher.fetch(anyString(), anyLong(), anyBoolean())).thenReturn(segmentChange); + when(segmentChangeFetcher.fetch(anyString(), anyLong(), any())).thenReturn(segmentChange); SegmentSynchronizationTask segmentSynchronizationTask = new SegmentSynchronizationTaskImp(segmentChangeFetcher, 1,10, gates, segmentCache); segmentSynchronizationTask.startPeriodicFetching(); SplitFetcherImp fetcher = new SplitFetcherImp(experimentChangeFetcher, new SplitParser(segmentSynchronizationTask, segmentCache), gates, cache); diff --git a/client/src/test/java/io/split/engine/experiments/SplitParserTest.java b/client/src/test/java/io/split/engine/experiments/SplitParserTest.java index 86beed261..e440926bc 100644 --- a/client/src/test/java/io/split/engine/experiments/SplitParserTest.java +++ b/client/src/test/java/io/split/engine/experiments/SplitParserTest.java @@ -58,7 +58,7 @@ public void works() { SegmentChangeFetcher segmentChangeFetcher = Mockito.mock(SegmentChangeFetcher.class); SegmentChange segmentChangeEmployee = getSegmentChange(-1L, -1L, EMPLOYEES); SegmentChange segmentChangeSalesPeople = getSegmentChange(-1L, -1L, SALES_PEOPLE); - Mockito.when(segmentChangeFetcher.fetch(Mockito.anyString(), Mockito.anyLong(), Mockito.anyBoolean())).thenReturn(segmentChangeEmployee).thenReturn(segmentChangeSalesPeople); + Mockito.when(segmentChangeFetcher.fetch(Mockito.anyString(), Mockito.anyLong(), Mockito.any())).thenReturn(segmentChangeEmployee).thenReturn(segmentChangeSalesPeople); SegmentSynchronizationTask segmentFetcher = new SegmentSynchronizationTaskImp(segmentChangeFetcher,1L, 1, gates, segmentCache); SplitParser parser = new SplitParser(segmentFetcher, segmentCache); @@ -97,7 +97,7 @@ public void worksWithConfig() { SegmentChangeFetcher segmentChangeFetcher = Mockito.mock(SegmentChangeFetcher.class); SegmentChange segmentChangeEmployee = getSegmentChange(-1L, -1L, EMPLOYEES); SegmentChange segmentChangeSalesPeople = getSegmentChange(-1L, -1L, SALES_PEOPLE); - Mockito.when(segmentChangeFetcher.fetch(Mockito.anyString(), Mockito.anyLong(), Mockito.anyBoolean())).thenReturn(segmentChangeEmployee).thenReturn(segmentChangeSalesPeople); + Mockito.when(segmentChangeFetcher.fetch(Mockito.anyString(), Mockito.anyLong(), Mockito.any())).thenReturn(segmentChangeEmployee).thenReturn(segmentChangeSalesPeople); SegmentSynchronizationTask segmentFetcher = new SegmentSynchronizationTaskImp(segmentChangeFetcher,1L, 1, gates, segmentCache); @@ -141,7 +141,7 @@ public void works_for_two_conditions() { SegmentChangeFetcher segmentChangeFetcher = Mockito.mock(SegmentChangeFetcher.class); SegmentChange segmentChangeEmployee = getSegmentChange(-1L, -1L, EMPLOYEES); SegmentChange segmentChangeSalesPeople = getSegmentChange(-1L, -1L, SALES_PEOPLE); - Mockito.when(segmentChangeFetcher.fetch(Mockito.anyString(), Mockito.anyLong(), Mockito.anyBoolean())).thenReturn(segmentChangeEmployee).thenReturn(segmentChangeSalesPeople); + Mockito.when(segmentChangeFetcher.fetch(Mockito.anyString(), Mockito.anyLong(), Mockito.any())).thenReturn(segmentChangeEmployee).thenReturn(segmentChangeSalesPeople); SegmentSynchronizationTask segmentFetcher = new SegmentSynchronizationTaskImp(segmentChangeFetcher,1L, 1, gates, segmentCache); @@ -180,7 +180,7 @@ public void fails_for_long_conditions() { segmentCache.updateSegment(SALES_PEOPLE, Stream.of("kunal").collect(Collectors.toList()), new ArrayList<>()); SegmentChangeFetcher segmentChangeFetcher = Mockito.mock(SegmentChangeFetcher.class); SegmentChange segmentChangeEmployee = getSegmentChange(-1L, -1L, EMPLOYEES); - Mockito.when(segmentChangeFetcher.fetch(Mockito.anyString(), Mockito.anyLong(), Mockito.anyBoolean())).thenReturn(segmentChangeEmployee); + Mockito.when(segmentChangeFetcher.fetch(Mockito.anyString(), Mockito.anyLong(), Mockito.any())).thenReturn(segmentChangeEmployee); SegmentSynchronizationTask segmentFetcher = new SegmentSynchronizationTaskImp(segmentChangeFetcher,1L, 1, gates, segmentCache); @@ -210,7 +210,7 @@ public void works_with_attributes() { SegmentChangeFetcher segmentChangeFetcher = Mockito.mock(SegmentChangeFetcher.class); SegmentChange segmentChangeEmployee = getSegmentChange(-1L, -1L, EMPLOYEES); SegmentChange segmentChangeSalesPeople = getSegmentChange(-1L, -1L, SALES_PEOPLE); - Mockito.when(segmentChangeFetcher.fetch(Mockito.anyString(), Mockito.anyLong(), Mockito.anyBoolean())).thenReturn(segmentChangeEmployee).thenReturn(segmentChangeSalesPeople); + Mockito.when(segmentChangeFetcher.fetch(Mockito.anyString(), Mockito.anyLong(), Mockito.any())).thenReturn(segmentChangeEmployee).thenReturn(segmentChangeSalesPeople); SegmentSynchronizationTask segmentFetcher = new SegmentSynchronizationTaskImp(segmentChangeFetcher,1L, 1, gates, segmentCache); @@ -256,7 +256,7 @@ public void less_than_or_equal_to() { SegmentChangeFetcher segmentChangeFetcher = Mockito.mock(SegmentChangeFetcher.class); SegmentChange segmentChangeEmployee = getSegmentChange(-1L, -1L, EMPLOYEES); SegmentChange segmentChangeSalesPeople = getSegmentChange(-1L, -1L, SALES_PEOPLE); - Mockito.when(segmentChangeFetcher.fetch(Mockito.anyString(), Mockito.anyLong(), Mockito.anyBoolean())).thenReturn(segmentChangeEmployee).thenReturn(segmentChangeSalesPeople); + Mockito.when(segmentChangeFetcher.fetch(Mockito.anyString(), Mockito.anyLong(), Mockito.any())).thenReturn(segmentChangeEmployee).thenReturn(segmentChangeSalesPeople); SegmentSynchronizationTask segmentFetcher = new SegmentSynchronizationTaskImp(segmentChangeFetcher,1L, 1, gates, segmentCache); @@ -295,7 +295,7 @@ public void equal_to() { SegmentChangeFetcher segmentChangeFetcher = Mockito.mock(SegmentChangeFetcher.class); SegmentChange segmentChangeEmployee = getSegmentChange(-1L, -1L, EMPLOYEES); SegmentChange segmentChangeSalesPeople = getSegmentChange(-1L, -1L, SALES_PEOPLE); - Mockito.when(segmentChangeFetcher.fetch(Mockito.anyString(), Mockito.anyLong(), Mockito.anyBoolean())).thenReturn(segmentChangeEmployee).thenReturn(segmentChangeSalesPeople); + Mockito.when(segmentChangeFetcher.fetch(Mockito.anyString(), Mockito.anyLong(), Mockito.any())).thenReturn(segmentChangeEmployee).thenReturn(segmentChangeSalesPeople); SegmentSynchronizationTask segmentFetcher = new SegmentSynchronizationTaskImp(segmentChangeFetcher,1L, 1, gates, segmentCache); @@ -334,7 +334,7 @@ public void equal_to_negative_number() { SegmentChangeFetcher segmentChangeFetcher = Mockito.mock(SegmentChangeFetcher.class); SegmentChange segmentChangeEmployee = getSegmentChange(-1L, -1L, EMPLOYEES); SegmentChange segmentChangeSalesPeople = getSegmentChange(-1L, -1L, SALES_PEOPLE); - Mockito.when(segmentChangeFetcher.fetch(Mockito.anyString(), Mockito.anyLong(), Mockito.anyBoolean())).thenReturn(segmentChangeEmployee).thenReturn(segmentChangeSalesPeople); + Mockito.when(segmentChangeFetcher.fetch(Mockito.anyString(), Mockito.anyLong(), Mockito.any())).thenReturn(segmentChangeEmployee).thenReturn(segmentChangeSalesPeople); SegmentSynchronizationTask segmentFetcher = new SegmentSynchronizationTaskImp(segmentChangeFetcher,1L, 1, gates, segmentCache); @@ -373,7 +373,7 @@ public void between() { SegmentChangeFetcher segmentChangeFetcher = Mockito.mock(SegmentChangeFetcher.class); SegmentChange segmentChangeEmployee = getSegmentChange(-1L, -1L, EMPLOYEES); SegmentChange segmentChangeSalesPeople = getSegmentChange(-1L, -1L, SALES_PEOPLE); - Mockito.when(segmentChangeFetcher.fetch(Mockito.anyString(), Mockito.anyLong(), Mockito.anyBoolean())).thenReturn(segmentChangeEmployee).thenReturn(segmentChangeSalesPeople); + Mockito.when(segmentChangeFetcher.fetch(Mockito.anyString(), Mockito.anyLong(), Mockito.any())).thenReturn(segmentChangeEmployee).thenReturn(segmentChangeSalesPeople); SegmentSynchronizationTask segmentFetcher = new SegmentSynchronizationTaskImp(segmentChangeFetcher,1L, 1, gates, segmentCache); @@ -550,7 +550,7 @@ public void set_matcher_test(Condition c, io.split.engine.matchers.Matcher m) { SegmentChangeFetcher segmentChangeFetcher = Mockito.mock(SegmentChangeFetcher.class); SegmentChange segmentChangeEmployee = getSegmentChange(-1L, -1L, EMPLOYEES); SegmentChange segmentChangeSalesPeople = getSegmentChange(-1L, -1L, SALES_PEOPLE); - Mockito.when(segmentChangeFetcher.fetch(Mockito.anyString(), Mockito.anyLong(), Mockito.anyBoolean())).thenReturn(segmentChangeEmployee).thenReturn(segmentChangeSalesPeople); + Mockito.when(segmentChangeFetcher.fetch(Mockito.anyString(), Mockito.anyLong(), Mockito.any())).thenReturn(segmentChangeEmployee).thenReturn(segmentChangeSalesPeople); SegmentSynchronizationTask segmentFetcher = new SegmentSynchronizationTaskImp(segmentChangeFetcher,1L, 1, gates, segmentCache); diff --git a/client/src/test/java/io/split/engine/segments/SegmentFetcherImpTest.java b/client/src/test/java/io/split/engine/segments/SegmentFetcherImpTest.java index 79786bdd6..4c682bbf8 100644 --- a/client/src/test/java/io/split/engine/segments/SegmentFetcherImpTest.java +++ b/client/src/test/java/io/split/engine/segments/SegmentFetcherImpTest.java @@ -51,7 +51,7 @@ public void works_when_there_are_no_changes() throws InterruptedException { SegmentChangeFetcher segmentChangeFetcher = Mockito.mock(SegmentChangeFetcher.class); SegmentChange segmentChange = getSegmentChange(-1L, 10L); - Mockito.when(segmentChangeFetcher.fetch(Mockito.anyString(), Mockito.anyLong(), Mockito.anyBoolean())).thenReturn(segmentChange); + Mockito.when(segmentChangeFetcher.fetch(Mockito.anyString(), Mockito.anyLong(), Mockito.any())).thenReturn(segmentChange); SegmentFetcherImp fetcher = new SegmentFetcherImp(SEGMENT_NAME, segmentChangeFetcher, gates, segmentCache); @@ -92,8 +92,8 @@ private void works(long startingChangeNumber) throws InterruptedException { SegmentChangeFetcher segmentChangeFetcher = Mockito.mock(SegmentChangeFetcher.class); SegmentChange segmentChange = getSegmentChange(-1L, -1L); - Mockito.when(segmentChangeFetcher.fetch(SEGMENT_NAME, -1L, false)).thenReturn(segmentChange); - Mockito.when(segmentChangeFetcher.fetch(SEGMENT_NAME, 0L, false)).thenReturn(segmentChange); + Mockito.when(segmentChangeFetcher.fetch(Mockito.eq(SEGMENT_NAME),Mockito.eq( -1L), Mockito.any())).thenReturn(segmentChange); + Mockito.when(segmentChangeFetcher.fetch(Mockito.eq(SEGMENT_NAME),Mockito.eq( 0L), Mockito.any())).thenReturn(segmentChange); SegmentFetcher fetcher = new SegmentFetcherImp(segmentName, segmentChangeFetcher, gates, segmentCache); // execute the fetcher for a little bit. @@ -112,7 +112,7 @@ private void works(long startingChangeNumber) throws InterruptedException { // reset the interrupt. Thread.currentThread().interrupt(); } - Mockito.verify(segmentChangeFetcher, Mockito.times(2)).fetch(Mockito.anyString(), Mockito.anyLong(), Mockito.anyBoolean()); + Mockito.verify(segmentChangeFetcher, Mockito.times(2)).fetch(Mockito.anyString(), Mockito.anyLong(), Mockito.any()); assertThat(gates.areSegmentsReady(10), is(true)); } From d14da5c1c657c769a7d5130f7585e5295356dbab Mon Sep 17 00:00:00 2001 From: Martin Redolatti Date: Thu, 20 May 2021 18:34:54 -0300 Subject: [PATCH 56/81] add tests --- .../client/HttpSegmentChangeFetcherTest.java | 35 ++++++++++- .../split/engine/common/SynchronizerTest.java | 59 +++++++++++++++++++ .../segments/SegmentFetcherImpTest.java | 55 +++++++++++++++++ 3 files changed, 148 insertions(+), 1 deletion(-) diff --git a/client/src/test/java/io/split/client/HttpSegmentChangeFetcherTest.java b/client/src/test/java/io/split/client/HttpSegmentChangeFetcherTest.java index 8030e4110..2ed371ab1 100644 --- a/client/src/test/java/io/split/client/HttpSegmentChangeFetcherTest.java +++ b/client/src/test/java/io/split/client/HttpSegmentChangeFetcherTest.java @@ -6,15 +6,21 @@ import io.split.engine.metrics.Metrics; import org.apache.hc.client5.http.impl.classic.CloseableHttpClient; import org.apache.hc.client5.http.impl.classic.HttpClients; -import org.apache.hc.core5.http.HttpStatus; +import org.apache.hc.core5.http.*; import org.hamcrest.Matchers; import org.junit.Assert; import org.junit.Test; +import org.mockito.ArgumentCaptor; +import org.mockito.Mockito; import java.io.IOException; +import java.io.StringBufferInputStream; import java.lang.reflect.InvocationTargetException; import java.net.URI; import java.net.URISyntaxException; +import java.util.List; + +import static org.mockito.Mockito.when; public class HttpSegmentChangeFetcherTest { @Test @@ -70,4 +76,31 @@ public void testFetcherWithSpecialCharacters() throws URISyntaxException, IOExce Assert.assertEquals(1, change.removed.size()); Assert.assertEquals("other_user", change.removed.get(0)); } + + @Test + public void testFetcherWithCDNBypassOption() throws IOException, URISyntaxException, IllegalAccessException, NoSuchMethodException, InvocationTargetException { + URI rootTarget = URI.create("https://api.split.io"); + + HttpEntity entityMock = Mockito.mock(HttpEntity.class); + when(entityMock.getContent()).thenReturn(new StringBufferInputStream("{\"till\": 1}")); + ClassicHttpResponse response = Mockito.mock(ClassicHttpResponse.class); + when(response.getCode()).thenReturn(200); + when(response.getEntity()).thenReturn(entityMock); + when(response.getHeaders()).thenReturn(new Header[0]); + + ArgumentCaptor requestCaptor = ArgumentCaptor.forClass(ClassicHttpRequest.class); + CloseableHttpClient httpClientMock = Mockito.mock(CloseableHttpClient.class); + when(httpClientMock.execute(requestCaptor.capture())).thenReturn(TestHelper.classicResponseToCloseableMock(response)); + + Metrics.NoopMetrics metrics = new Metrics.NoopMetrics(); + HttpSegmentChangeFetcher fetcher = HttpSegmentChangeFetcher.create(httpClientMock, rootTarget, metrics); + + fetcher.fetch("someSegment", -1, new FetchOptions.Builder().targetChangeNumber(123).build()); + fetcher.fetch("someSegment2",-1, new FetchOptions.Builder().build()); + List captured = requestCaptor.getAllValues(); + Assert.assertEquals(captured.size(), 2); + Assert.assertTrue(captured.get(0).getUri().toString().contains("till=123")); + Assert.assertFalse(captured.get(1).getUri().toString().contains("till=")); + } + } diff --git a/client/src/test/java/io/split/engine/common/SynchronizerTest.java b/client/src/test/java/io/split/engine/common/SynchronizerTest.java index 00dfd2aed..c399230c4 100644 --- a/client/src/test/java/io/split/engine/common/SynchronizerTest.java +++ b/client/src/test/java/io/split/engine/common/SynchronizerTest.java @@ -173,4 +173,63 @@ public void testCDNBypassRequestLimitAndBackoff() throws NoSuchFieldException, I Assert.assertTrue((after - before) > minDiffExpected); } + @Test + public void testCDNBypassRequestLimitAndForSegmentsBackoff() throws NoSuchFieldException, IllegalAccessException { + + SplitCache cache = new InMemoryCacheImp(); + Synchronizer imp = new SynchronizerImp(_refreshableSplitFetcherTask, + _splitFetcher, + _segmentFetcher, + cache, + _segmentCache, + 50, + 3, + 1, + true); + + SegmentFetcher fetcher = Mockito.mock(SegmentFetcher.class); + when(_segmentFetcher.getFetcher("someSegment")).thenReturn(fetcher); + + ArgumentCaptor optionsCaptor = ArgumentCaptor.forClass(FetchOptions.class); + AtomicInteger calls = new AtomicInteger(); + Mockito.doAnswer(invocationOnMock -> { + calls.getAndIncrement(); + switch (calls.get()) { + case 14: Assert.assertTrue(false); // should never get here + } + return null; + }).when(fetcher).fetch(optionsCaptor.capture()); + + // Before executing, we'll update the backoff via reflection, to avoid waiting minutes for the test to run. + Field backoffBase = SynchronizerImp.class.getDeclaredField("ON_DEMAND_FETCH_BACKOFF_BASE_MS"); + backoffBase.setAccessible(true); + Field modifiersField = Field.class.getDeclaredField("modifiers"); + modifiersField.setAccessible(true); + modifiersField.setInt(backoffBase, backoffBase.getModifiers() & ~Modifier.FINAL); + backoffBase.set(imp, 1); // 1ms + + long before = System.currentTimeMillis(); + imp.refreshSegment("someSegment",1); + long after = System.currentTimeMillis(); + + List options = optionsCaptor.getAllValues(); + Assert.assertEquals(options.size(), 13); + Assert.assertFalse(options.get(0).hasCustomCN()); + Assert.assertFalse(options.get(1).hasCustomCN()); + Assert.assertFalse(options.get(2).hasCustomCN()); + Assert.assertTrue(options.get(3).hasCustomCN()); + Assert.assertTrue(options.get(4).hasCustomCN()); + Assert.assertTrue(options.get(5).hasCustomCN()); + Assert.assertTrue(options.get(6).hasCustomCN()); + Assert.assertTrue(options.get(7).hasCustomCN()); + Assert.assertTrue(options.get(8).hasCustomCN()); + Assert.assertTrue(options.get(9).hasCustomCN()); + Assert.assertTrue(options.get(10).hasCustomCN()); + Assert.assertTrue(options.get(11).hasCustomCN()); + Assert.assertTrue(options.get(12).hasCustomCN()); + + Assert.assertEquals(calls.get(), 13); + long minDiffExpected = 1 + 2 + 4 + 8 + 16 + 32 + 64 + 128 + 256; + Assert.assertTrue((after - before) > minDiffExpected); + } } diff --git a/client/src/test/java/io/split/engine/segments/SegmentFetcherImpTest.java b/client/src/test/java/io/split/engine/segments/SegmentFetcherImpTest.java index 4c682bbf8..92f3704dc 100644 --- a/client/src/test/java/io/split/engine/segments/SegmentFetcherImpTest.java +++ b/client/src/test/java/io/split/engine/segments/SegmentFetcherImpTest.java @@ -1,11 +1,20 @@ package io.split.engine.segments; import com.google.common.collect.Sets; +import io.split.cache.InMemoryCacheImp; import io.split.cache.SegmentCache; import io.split.cache.SegmentCacheInMemoryImpl; +import io.split.cache.SplitCache; import io.split.client.dtos.SegmentChange; +import io.split.client.dtos.SplitChange; import io.split.engine.SDKReadinessGates; +import io.split.engine.common.FetchOptions; +import io.split.engine.experiments.SplitChangeFetcher; +import io.split.engine.experiments.SplitFetcherImp; +import io.split.engine.experiments.SplitParser; +import org.junit.Assert; import org.junit.Test; +import org.mockito.ArgumentCaptor; import org.mockito.Mockito; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -21,6 +30,7 @@ import static org.junit.Assert.assertNotNull; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertThat; +import static org.mockito.Mockito.when; /** * Tests for RefreshableSegmentFetcher. @@ -138,6 +148,51 @@ public void does_not_work_if_sdk_readiness_gates_are_null() { SegmentFetcher fetcher = new SegmentFetcherImp(SEGMENT_NAME, segmentChangeFetcher, null, segmentCache); } + @Test + public void testBypassCdnClearedAfterFirstHit() { + SegmentChangeFetcher mockFetcher = Mockito.mock(SegmentChangeFetcher.class); + SegmentSynchronizationTask segmentSynchronizationTaskMock = Mockito.mock(SegmentSynchronizationTask.class); + SegmentCache segmentCacheMock = new SegmentCacheInMemoryImpl(); + SDKReadinessGates mockGates = Mockito.mock(SDKReadinessGates.class); + SegmentFetcher fetcher = new SegmentFetcherImp("someSegment", mockFetcher, mockGates, segmentCacheMock); + + + SegmentChange response1 = new SegmentChange(); + response1.name = "someSegment"; + response1.added = new ArrayList<>(); + response1.removed = new ArrayList<>(); + response1.since = -1; + response1.till = 1; + + SegmentChange response2 = new SegmentChange(); + response2.name = "someSegment"; + response2.added = new ArrayList<>(); + response2.removed = new ArrayList<>(); + response2.since = 1; + response1.till = 1; + + ArgumentCaptor optionsCaptor = ArgumentCaptor.forClass(FetchOptions.class); + ArgumentCaptor cnCaptor = ArgumentCaptor.forClass(Long.class); + when(mockFetcher.fetch(Mockito.eq("someSegment"), cnCaptor.capture(), optionsCaptor.capture())).thenReturn(response1, response2); + + FetchOptions originalOptions = new FetchOptions.Builder().targetChangeNumber(123).build(); + fetcher.fetch(originalOptions); + List capturedCNs = cnCaptor.getAllValues(); + List capturedOptions = optionsCaptor.getAllValues(); + + Assert.assertEquals(capturedOptions.size(), 2); + Assert.assertEquals(capturedCNs.size(), 2); + + Assert.assertEquals(capturedCNs.get(0), Long.valueOf(-1)); + Assert.assertEquals(capturedCNs.get(1), Long.valueOf(1)); + + Assert.assertEquals(capturedOptions.get(0).targetCN(), 123); + Assert.assertEquals(capturedOptions.get(1).targetCN(), -1); + + // Ensure that the original value hasn't been modified + Assert.assertEquals(originalOptions.targetCN(), 123); + } + private SegmentChange getSegmentChange(long since, long till){ SegmentChange segmentChange = new SegmentChange(); segmentChange.name = SEGMENT_NAME; From a0c4fd0317a42ea7d603bff9dd7eb0b0537a9deb Mon Sep 17 00:00:00 2001 From: Lucas Echeverz Date: Fri, 21 May 2021 11:39:10 -0300 Subject: [PATCH 57/81] Fixing PR comments --- .../split/engine/common/SyncManagerImp.java | 1 + .../split/engine/common/SynchronizerImp.java | 7 +-- .../engine/experiments/SplitFetcherImp.java | 2 +- .../split/engine/segments/SegmentFetcher.java | 2 +- .../engine/segments/SegmentFetcherImp.java | 13 +++-- .../SegmentSynchronizationTaskImp.java | 4 +- .../SegmentSynchronizationTaskImpTest.java | 53 +++++++++++++++++++ 7 files changed, 69 insertions(+), 13 deletions(-) diff --git a/client/src/main/java/io/split/engine/common/SyncManagerImp.java b/client/src/main/java/io/split/engine/common/SyncManagerImp.java index 8ac52b425..529d02914 100644 --- a/client/src/main/java/io/split/engine/common/SyncManagerImp.java +++ b/client/src/main/java/io/split/engine/common/SyncManagerImp.java @@ -104,6 +104,7 @@ public void start() { try { Thread.currentThread().sleep(1000); } catch (InterruptedException e) { + _log.warn("Sdk Initializer thread interrupted"); e.printStackTrace(); Thread.currentThread().interrupt(); } diff --git a/client/src/main/java/io/split/engine/common/SynchronizerImp.java b/client/src/main/java/io/split/engine/common/SynchronizerImp.java index 5ad718073..d3e800ebc 100644 --- a/client/src/main/java/io/split/engine/common/SynchronizerImp.java +++ b/client/src/main/java/io/split/engine/common/SynchronizerImp.java @@ -53,12 +53,7 @@ public SynchronizerImp(SplitSynchronizationTask splitSynchronizationTask, @Override public boolean syncAll() { - AtomicBoolean syncStatus = new AtomicBoolean(false); - if(_splitFetcher.fetchAll(true) && - _segmentSynchronizationTaskImp.fetchAllSynchronous()) { - syncStatus.set(true); - } - return syncStatus.get(); + return _splitFetcher.fetchAll(true) && _segmentSynchronizationTaskImp.fetchAllSynchronous(); } @Override diff --git a/client/src/main/java/io/split/engine/experiments/SplitFetcherImp.java b/client/src/main/java/io/split/engine/experiments/SplitFetcherImp.java index 19adb994d..0dd56ae2c 100644 --- a/client/src/main/java/io/split/engine/experiments/SplitFetcherImp.java +++ b/client/src/main/java/io/split/engine/experiments/SplitFetcherImp.java @@ -149,6 +149,7 @@ public boolean fetchAll(boolean addCacheHeader) { long start = _splitCache.getChangeNumber(); try { runWithoutExceptionHandling(addCacheHeader); + return true; } catch (InterruptedException e) { _log.warn("Interrupting split fetcher task"); Thread.currentThread().interrupt(); @@ -164,6 +165,5 @@ public boolean fetchAll(boolean addCacheHeader) { _log.debug("split fetch before: " + start + ", after: " + _splitCache.getChangeNumber()); } } - return true; } } diff --git a/client/src/main/java/io/split/engine/segments/SegmentFetcher.java b/client/src/main/java/io/split/engine/segments/SegmentFetcher.java index af4bbc767..bd9b19ddd 100644 --- a/client/src/main/java/io/split/engine/segments/SegmentFetcher.java +++ b/client/src/main/java/io/split/engine/segments/SegmentFetcher.java @@ -9,7 +9,7 @@ public interface SegmentFetcher { */ void fetch(boolean addCacheHeader); - void runWhitCacheHeader(); + boolean runWhitCacheHeader(); void fetchAll(); } diff --git a/client/src/main/java/io/split/engine/segments/SegmentFetcherImp.java b/client/src/main/java/io/split/engine/segments/SegmentFetcherImp.java index 3bdab0125..b18180264 100644 --- a/client/src/main/java/io/split/engine/segments/SegmentFetcherImp.java +++ b/client/src/main/java/io/split/engine/segments/SegmentFetcherImp.java @@ -1,5 +1,6 @@ package io.split.engine.segments; +import com.google.common.annotations.VisibleForTesting; import io.split.cache.SegmentCache; import io.split.client.dtos.SegmentChange; import io.split.engine.SDKReadinessGates; @@ -116,7 +117,8 @@ private String summarize(List changes) { return bldr.toString(); } - private void callLoopRun(boolean isFetch, boolean addCacheHeader){ + @VisibleForTesting + void callLoopRun(boolean isFetch, boolean addCacheHeader){ while (true) { long start = _segmentCache.getChangeNumber(_segmentName); runWithoutExceptionHandling(addCacheHeader); @@ -131,23 +133,26 @@ private void callLoopRun(boolean isFetch, boolean addCacheHeader){ } @Override - public void runWhitCacheHeader(){ - this.fetchAndUpdate(true); + public boolean runWhitCacheHeader(){ + return this.fetchAndUpdate(true); } /** * Calls callLoopRun and after fetchs segment. * @param addCacheHeader indicates if CacheHeader is required */ - private void fetchAndUpdate(boolean addCacheHeader) { + @VisibleForTesting + boolean fetchAndUpdate(boolean addCacheHeader) { try { // Do this again in case the previous call errored out. callLoopRun(true, addCacheHeader); + return true; } catch (Throwable t) { _log.error("RefreshableSegmentFetcher failed: " + t.getMessage()); if (_log.isDebugEnabled()) { _log.debug("Reason:", t); } + return false; } } diff --git a/client/src/main/java/io/split/engine/segments/SegmentSynchronizationTaskImp.java b/client/src/main/java/io/split/engine/segments/SegmentSynchronizationTaskImp.java index c40915335..025ca08ab 100644 --- a/client/src/main/java/io/split/engine/segments/SegmentSynchronizationTaskImp.java +++ b/client/src/main/java/io/split/engine/segments/SegmentSynchronizationTaskImp.java @@ -167,7 +167,9 @@ public boolean fetchAllSynchronous() { .collect(Collectors.toList()) .stream().forEach(future -> { try { - future.get(); + if(!future.get()) { + fetchAllStatus.set(false); + }; } catch (Exception ex) { fetchAllStatus.set(false); _log.error(ex.getMessage()); diff --git a/client/src/test/java/io/split/engine/segments/SegmentSynchronizationTaskImpTest.java b/client/src/test/java/io/split/engine/segments/SegmentSynchronizationTaskImpTest.java index c49d29962..946cba066 100644 --- a/client/src/test/java/io/split/engine/segments/SegmentSynchronizationTaskImpTest.java +++ b/client/src/test/java/io/split/engine/segments/SegmentSynchronizationTaskImpTest.java @@ -1,16 +1,21 @@ package io.split.engine.segments; +import com.google.common.collect.Maps; import io.split.engine.SDKReadinessGates; import io.split.cache.SegmentCache; import io.split.telemetry.storage.InMemoryTelemetryStorage; import io.split.telemetry.storage.TelemetryStorage; +import org.junit.Assert; import org.junit.Before; import org.junit.Test; import org.mockito.Mockito; import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import java.lang.reflect.Field; +import java.lang.reflect.Modifier; import java.util.List; +import java.util.concurrent.ConcurrentMap; import java.util.concurrent.ExecutorService; import java.util.concurrent.Executors; import java.util.concurrent.TimeUnit; @@ -79,5 +84,53 @@ public void run() { assertThat(fetcher1.get(), is(sameInstance(fetcher2.get()))); } + @Test + public void testFetchAllAsynchronousAndGetFalse() throws NoSuchFieldException, IllegalAccessException { + SDKReadinessGates gates = new SDKReadinessGates(); + SegmentCache segmentCache = Mockito.mock(SegmentCache.class); + ConcurrentMap _segmentFetchers = Maps.newConcurrentMap(); + + SegmentChangeFetcher segmentChangeFetcher = Mockito.mock(SegmentChangeFetcher.class); + SegmentFetcherImp segmentFetcher = Mockito.mock(SegmentFetcherImp.class); + _segmentFetchers.put("SF", segmentFetcher); + final SegmentSynchronizationTaskImp fetchers = new SegmentSynchronizationTaskImp(segmentChangeFetcher, 1L, 1, gates, segmentCache, TELEMETRY_STORAGE); + Mockito.doNothing().when(segmentFetcher).callLoopRun(Mockito.anyBoolean(),Mockito.anyBoolean()); + Mockito.when(segmentFetcher.runWhitCacheHeader()).thenReturn(false); + Mockito.when(segmentFetcher.fetchAndUpdate(Mockito.anyBoolean())).thenReturn(false); + Mockito.doNothing().when(segmentFetcher).callLoopRun(Mockito.anyBoolean(),Mockito.anyBoolean()); + + // Before executing, we'll update the map of segmentFecthers via reflection. + Field backoffBase = SegmentSynchronizationTaskImp.class.getDeclaredField("_segmentFetchers"); + backoffBase.setAccessible(true); + Field modifiersField = Field.class.getDeclaredField("modifiers"); + modifiersField.setAccessible(true); + modifiersField.setInt(backoffBase, backoffBase.getModifiers() & ~Modifier.FINAL); + + backoffBase.set(fetchers, _segmentFetchers); // 1ms + fetcher1.set(segmentFetcher); + boolean fetch = fetchers.fetchAllSynchronous(); + Assert.assertEquals(false, fetch); + } + @Test + public void testFetchAllAsynchronousAndGetTrue() throws NoSuchFieldException, IllegalAccessException { + SDKReadinessGates gates = new SDKReadinessGates(); + SegmentCache segmentCache = Mockito.mock(SegmentCache.class); + + SegmentChangeFetcher segmentChangeFetcher = Mockito.mock(SegmentChangeFetcher.class); + SegmentFetcherImp segmentFetcher = Mockito.mock(SegmentFetcherImp.class); + final SegmentSynchronizationTaskImp fetchers = new SegmentSynchronizationTaskImp(segmentChangeFetcher, 1L, 1, gates, segmentCache, TELEMETRY_STORAGE); + + // Before executing, we'll update the map of segmentFecthers via reflection. + Field backoffBase = SegmentSynchronizationTaskImp.class.getDeclaredField("_segmentFetchers"); + backoffBase.setAccessible(true); + Field modifiersField = Field.class.getDeclaredField("modifiers"); + modifiersField.setAccessible(true); + modifiersField.setInt(backoffBase, backoffBase.getModifiers() & ~Modifier.FINAL); + Mockito.doNothing().when(segmentFetcher).callLoopRun(Mockito.anyBoolean(),Mockito.anyBoolean()); + Mockito.when(segmentFetcher.runWhitCacheHeader()).thenReturn(true); + Mockito.when(segmentFetcher.fetchAndUpdate(Mockito.anyBoolean())).thenReturn(true); + boolean fetch = fetchers.fetchAllSynchronous(); + Assert.assertEquals(true, fetch); + } } From 78540db9fc06c864ee28e03d5b0c05f1988f7d11 Mon Sep 17 00:00:00 2001 From: Lucas Echeverz Date: Fri, 21 May 2021 12:24:08 -0300 Subject: [PATCH 58/81] Final fixes --- .../io/split/engine/common/SyncManagerImp.java | 1 - .../SegmentSynchronizationTaskImpTest.java | 14 +++++++------- 2 files changed, 7 insertions(+), 8 deletions(-) diff --git a/client/src/main/java/io/split/engine/common/SyncManagerImp.java b/client/src/main/java/io/split/engine/common/SyncManagerImp.java index 529d02914..b556beb9d 100644 --- a/client/src/main/java/io/split/engine/common/SyncManagerImp.java +++ b/client/src/main/java/io/split/engine/common/SyncManagerImp.java @@ -105,7 +105,6 @@ public void start() { Thread.currentThread().sleep(1000); } catch (InterruptedException e) { _log.warn("Sdk Initializer thread interrupted"); - e.printStackTrace(); Thread.currentThread().interrupt(); } } diff --git a/client/src/test/java/io/split/engine/segments/SegmentSynchronizationTaskImpTest.java b/client/src/test/java/io/split/engine/segments/SegmentSynchronizationTaskImpTest.java index 946cba066..db3740565 100644 --- a/client/src/test/java/io/split/engine/segments/SegmentSynchronizationTaskImpTest.java +++ b/client/src/test/java/io/split/engine/segments/SegmentSynchronizationTaskImpTest.java @@ -100,13 +100,13 @@ public void testFetchAllAsynchronousAndGetFalse() throws NoSuchFieldException, I Mockito.doNothing().when(segmentFetcher).callLoopRun(Mockito.anyBoolean(),Mockito.anyBoolean()); // Before executing, we'll update the map of segmentFecthers via reflection. - Field backoffBase = SegmentSynchronizationTaskImp.class.getDeclaredField("_segmentFetchers"); - backoffBase.setAccessible(true); + Field segmentFetchersForced = SegmentSynchronizationTaskImp.class.getDeclaredField("_segmentFetchers"); + segmentFetchersForced.setAccessible(true); Field modifiersField = Field.class.getDeclaredField("modifiers"); modifiersField.setAccessible(true); - modifiersField.setInt(backoffBase, backoffBase.getModifiers() & ~Modifier.FINAL); + modifiersField.setInt(segmentFetchersForced, segmentFetchersForced.getModifiers() & ~Modifier.FINAL); - backoffBase.set(fetchers, _segmentFetchers); // 1ms + segmentFetchersForced.set(fetchers, _segmentFetchers); // 1ms fetcher1.set(segmentFetcher); boolean fetch = fetchers.fetchAllSynchronous(); Assert.assertEquals(false, fetch); @@ -122,11 +122,11 @@ public void testFetchAllAsynchronousAndGetTrue() throws NoSuchFieldException, Il final SegmentSynchronizationTaskImp fetchers = new SegmentSynchronizationTaskImp(segmentChangeFetcher, 1L, 1, gates, segmentCache, TELEMETRY_STORAGE); // Before executing, we'll update the map of segmentFecthers via reflection. - Field backoffBase = SegmentSynchronizationTaskImp.class.getDeclaredField("_segmentFetchers"); - backoffBase.setAccessible(true); + Field segmentFetchersForced = SegmentSynchronizationTaskImp.class.getDeclaredField("_segmentFetchers"); + segmentFetchersForced.setAccessible(true); Field modifiersField = Field.class.getDeclaredField("modifiers"); modifiersField.setAccessible(true); - modifiersField.setInt(backoffBase, backoffBase.getModifiers() & ~Modifier.FINAL); + modifiersField.setInt(segmentFetchersForced, segmentFetchersForced.getModifiers() & ~Modifier.FINAL); Mockito.doNothing().when(segmentFetcher).callLoopRun(Mockito.anyBoolean(),Mockito.anyBoolean()); Mockito.when(segmentFetcher.runWhitCacheHeader()).thenReturn(true); Mockito.when(segmentFetcher.fetchAndUpdate(Mockito.anyBoolean())).thenReturn(true); From d5752440ca824d05414e6e89289d0820920d5edb Mon Sep 17 00:00:00 2001 From: Lucas Echeverz Date: Fri, 21 May 2021 16:26:14 -0300 Subject: [PATCH 59/81] Setting up size to events Queue --- .../java/io/split/client/EventClientImpl.java | 30 +++++++--------- .../io/split/client/EventsClientImplTest.java | 36 +++++++++++++++++++ .../SegmentSynchronizationTaskImpTest.java | 17 ++++----- 3 files changed, 58 insertions(+), 25 deletions(-) diff --git a/client/src/main/java/io/split/client/EventClientImpl.java b/client/src/main/java/io/split/client/EventClientImpl.java index a95613a34..3644da688 100644 --- a/client/src/main/java/io/split/client/EventClientImpl.java +++ b/client/src/main/java/io/split/client/EventClientImpl.java @@ -54,23 +54,15 @@ public class EventClientImpl implements EventClient { private final TelemetryRuntimeProducer _telemetryRuntimeProducer; ThreadFactory eventClientThreadFactory(final String name) { - return new ThreadFactory() { - @Override - public Thread newThread(final Runnable r) { - return new Thread(new Runnable() { - @Override - public void run() { - Thread.currentThread().setPriority(MIN_PRIORITY); - r.run(); - } - }, name); - } - }; + return r -> new Thread(() -> { + Thread.currentThread().setPriority(MIN_PRIORITY); + r.run(); + }, name); } public static EventClientImpl create(CloseableHttpClient httpclient, URI eventsRootTarget, int maxQueueSize, long flushIntervalMillis, int waitBeforeShutdown, TelemetryRuntimeProducer telemetryRuntimeProducer) throws URISyntaxException { - return new EventClientImpl(new LinkedBlockingQueue(), + return new EventClientImpl(new LinkedBlockingQueue<>(maxQueueSize), httpclient, Utils.appendPath(eventsRootTarget, "api/events/bulk"), maxQueueSize, @@ -131,10 +123,14 @@ public boolean track(Event event, int eventSize) { if (event == null) { return false; } - _eventQueue.put(new WrappedEvent(event, eventSize)); - _telemetryRuntimeProducer.recordEventStats(EventsDataRecordsEnum.EVENTS_QUEUED, 1); + if(_eventQueue.offer(new WrappedEvent(event, eventSize))) { + _telemetryRuntimeProducer.recordEventStats(EventsDataRecordsEnum.EVENTS_QUEUED, 1); + } + else { + _telemetryRuntimeProducer.recordEventStats(EventsDataRecordsEnum.EVENTS_DROPPED, 1); + } - } catch (ClassCastException | NullPointerException | InterruptedException e) { + } catch (ClassCastException | NullPointerException | IllegalArgumentException e) { _telemetryRuntimeProducer.recordEventStats(EventsDataRecordsEnum.EVENTS_DROPPED, 1); _log.warn("Interruption when adding event withed while adding message %s.", event); return false; @@ -164,7 +160,7 @@ public void run() { List events = new ArrayList<>(); long accumulated = 0; try { - while (true) { + while (!Thread.currentThread().isInterrupted()) { WrappedEvent data = _eventQueue.take(); Event event = data.event(); Long size = data.size(); diff --git a/client/src/test/java/io/split/client/EventsClientImplTest.java b/client/src/test/java/io/split/client/EventsClientImplTest.java index 4a2e3fefc..facd1daec 100644 --- a/client/src/test/java/io/split/client/EventsClientImplTest.java +++ b/client/src/test/java/io/split/client/EventsClientImplTest.java @@ -1,6 +1,8 @@ package io.split.client; import io.split.client.dtos.Event; +import io.split.engine.segments.SegmentSynchronizationTaskImp; +import io.split.telemetry.domain.enums.EventsDataRecordsEnum; import io.split.telemetry.storage.InMemoryTelemetryStorage; import io.split.telemetry.storage.TelemetryStorage; import org.apache.hc.client5.http.classic.methods.HttpUriRequest; @@ -12,8 +14,12 @@ import org.mockito.Mockito; import java.io.IOException; +import java.lang.reflect.Field; +import java.lang.reflect.Modifier; import java.net.URI; import java.net.URISyntaxException; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; import java.util.concurrent.LinkedBlockingQueue; public class EventsClientImplTest { @@ -74,4 +80,34 @@ public void testEventsFlushedWhenSizeLimitReached() throws URISyntaxException, I Thread.sleep(2000); Mockito.verify(client, Mockito.times(1)).execute((HttpUriRequest) Mockito.any()); } + + @Test + public void testEventDropped() throws URISyntaxException, NoSuchFieldException, IllegalAccessException { + CloseableHttpClient client = Mockito.mock(CloseableHttpClient.class); + Field eventDroppedConsumer = EventClientImpl.class.getDeclaredField("_consumerExecutor"); + ExecutorService executorService = Executors.newFixedThreadPool(2); + eventDroppedConsumer.setAccessible(true); + + Field modifiersField = Field.class.getDeclaredField("modifiers"); + modifiersField.setAccessible(true); + modifiersField.setInt(eventDroppedConsumer, eventDroppedConsumer.getModifiers() & ~Modifier.FINAL); + executorService.execute(() -> { + + }); + EventClientImpl eventClient = new EventClientImpl(new LinkedBlockingQueue<>(2), + client, + URI.create("https://kubernetesturl.com/split"), + 10000, // Long queue so it doesn't flush by # of events + 100000, // Long period so it doesn't flush by timeout expiration. + 0, TELEMETRY_STORAGE); + eventClient.close(); + eventDroppedConsumer.set(eventClient, executorService); // 1ms + for (int i = 0; i < 3; ++i) { + Event event = new Event(); + eventClient.track(event, 1); + } + + Mockito.verify(TELEMETRY_STORAGE, Mockito.times(2)).recordEventStats(EventsDataRecordsEnum.EVENTS_QUEUED, 1); + Mockito.verify(TELEMETRY_STORAGE, Mockito.times(1)).recordEventStats(EventsDataRecordsEnum.EVENTS_DROPPED, 1); + } } diff --git a/client/src/test/java/io/split/engine/segments/SegmentSynchronizationTaskImpTest.java b/client/src/test/java/io/split/engine/segments/SegmentSynchronizationTaskImpTest.java index 946cba066..a4645ee94 100644 --- a/client/src/test/java/io/split/engine/segments/SegmentSynchronizationTaskImpTest.java +++ b/client/src/test/java/io/split/engine/segments/SegmentSynchronizationTaskImpTest.java @@ -100,14 +100,13 @@ public void testFetchAllAsynchronousAndGetFalse() throws NoSuchFieldException, I Mockito.doNothing().when(segmentFetcher).callLoopRun(Mockito.anyBoolean(),Mockito.anyBoolean()); // Before executing, we'll update the map of segmentFecthers via reflection. - Field backoffBase = SegmentSynchronizationTaskImp.class.getDeclaredField("_segmentFetchers"); - backoffBase.setAccessible(true); + Field segmentFetchersForced = SegmentSynchronizationTaskImp.class.getDeclaredField("_segmentFetchers"); + segmentFetchersForced.setAccessible(true); Field modifiersField = Field.class.getDeclaredField("modifiers"); modifiersField.setAccessible(true); - modifiersField.setInt(backoffBase, backoffBase.getModifiers() & ~Modifier.FINAL); + modifiersField.setInt(segmentFetchersForced, segmentFetchersForced.getModifiers() & ~Modifier.FINAL); - backoffBase.set(fetchers, _segmentFetchers); // 1ms - fetcher1.set(segmentFetcher); + segmentFetchersForced.set(fetchers, _segmentFetchers); // 1ms boolean fetch = fetchers.fetchAllSynchronous(); Assert.assertEquals(false, fetch); } @@ -117,16 +116,18 @@ public void testFetchAllAsynchronousAndGetTrue() throws NoSuchFieldException, Il SDKReadinessGates gates = new SDKReadinessGates(); SegmentCache segmentCache = Mockito.mock(SegmentCache.class); + ConcurrentMap _segmentFetchers = Maps.newConcurrentMap(); SegmentChangeFetcher segmentChangeFetcher = Mockito.mock(SegmentChangeFetcher.class); SegmentFetcherImp segmentFetcher = Mockito.mock(SegmentFetcherImp.class); final SegmentSynchronizationTaskImp fetchers = new SegmentSynchronizationTaskImp(segmentChangeFetcher, 1L, 1, gates, segmentCache, TELEMETRY_STORAGE); // Before executing, we'll update the map of segmentFecthers via reflection. - Field backoffBase = SegmentSynchronizationTaskImp.class.getDeclaredField("_segmentFetchers"); - backoffBase.setAccessible(true); + Field segmentFetchersForced = SegmentSynchronizationTaskImp.class.getDeclaredField("_segmentFetchers"); + segmentFetchersForced.setAccessible(true); Field modifiersField = Field.class.getDeclaredField("modifiers"); modifiersField.setAccessible(true); - modifiersField.setInt(backoffBase, backoffBase.getModifiers() & ~Modifier.FINAL); + modifiersField.setInt(segmentFetchersForced, segmentFetchersForced.getModifiers() & ~Modifier.FINAL); + segmentFetchersForced.set(fetchers, _segmentFetchers); // 1ms Mockito.doNothing().when(segmentFetcher).callLoopRun(Mockito.anyBoolean(),Mockito.anyBoolean()); Mockito.when(segmentFetcher.runWhitCacheHeader()).thenReturn(true); Mockito.when(segmentFetcher.fetchAndUpdate(Mockito.anyBoolean())).thenReturn(true); From 285ae1bfc126c332a5fc2c256d2de27b3693ab68 Mon Sep 17 00:00:00 2001 From: Lucas Echeverz Date: Fri, 21 May 2021 16:48:09 -0300 Subject: [PATCH 60/81] Fixing wrong import --- client/src/test/java/io/split/client/EventsClientImplTest.java | 1 - 1 file changed, 1 deletion(-) diff --git a/client/src/test/java/io/split/client/EventsClientImplTest.java b/client/src/test/java/io/split/client/EventsClientImplTest.java index facd1daec..12a11a189 100644 --- a/client/src/test/java/io/split/client/EventsClientImplTest.java +++ b/client/src/test/java/io/split/client/EventsClientImplTest.java @@ -1,7 +1,6 @@ package io.split.client; import io.split.client.dtos.Event; -import io.split.engine.segments.SegmentSynchronizationTaskImp; import io.split.telemetry.domain.enums.EventsDataRecordsEnum; import io.split.telemetry.storage.InMemoryTelemetryStorage; import io.split.telemetry.storage.TelemetryStorage; From 563a46ebaf6f2f1d7cbf48a3889720509ba88546 Mon Sep 17 00:00:00 2001 From: Lucas Echeverz Date: Fri, 21 May 2021 16:49:26 -0300 Subject: [PATCH 61/81] Improving Test --- .../java/io/split/client/EventsClientImplTest.java | 11 ----------- 1 file changed, 11 deletions(-) diff --git a/client/src/test/java/io/split/client/EventsClientImplTest.java b/client/src/test/java/io/split/client/EventsClientImplTest.java index 12a11a189..044e6a634 100644 --- a/client/src/test/java/io/split/client/EventsClientImplTest.java +++ b/client/src/test/java/io/split/client/EventsClientImplTest.java @@ -83,16 +83,6 @@ public void testEventsFlushedWhenSizeLimitReached() throws URISyntaxException, I @Test public void testEventDropped() throws URISyntaxException, NoSuchFieldException, IllegalAccessException { CloseableHttpClient client = Mockito.mock(CloseableHttpClient.class); - Field eventDroppedConsumer = EventClientImpl.class.getDeclaredField("_consumerExecutor"); - ExecutorService executorService = Executors.newFixedThreadPool(2); - eventDroppedConsumer.setAccessible(true); - - Field modifiersField = Field.class.getDeclaredField("modifiers"); - modifiersField.setAccessible(true); - modifiersField.setInt(eventDroppedConsumer, eventDroppedConsumer.getModifiers() & ~Modifier.FINAL); - executorService.execute(() -> { - - }); EventClientImpl eventClient = new EventClientImpl(new LinkedBlockingQueue<>(2), client, URI.create("https://kubernetesturl.com/split"), @@ -100,7 +90,6 @@ public void testEventDropped() throws URISyntaxException, NoSuchFieldException, 100000, // Long period so it doesn't flush by timeout expiration. 0, TELEMETRY_STORAGE); eventClient.close(); - eventDroppedConsumer.set(eventClient, executorService); // 1ms for (int i = 0; i < 3; ++i) { Event event = new Event(); eventClient.track(event, 1); From 80e720cd456eb06621babf2f6c7cf1a951ba55ce Mon Sep 17 00:00:00 2001 From: Martin Redolatti Date: Fri, 21 May 2021 17:03:14 -0300 Subject: [PATCH 62/81] cleanup for release; --- client/pom.xml | 2 +- .../io/split/client/SplitClientConfig.java | 42 ++----------------- pom.xml | 2 +- testing/pom.xml | 2 +- 4 files changed, 7 insertions(+), 41 deletions(-) diff --git a/client/pom.xml b/client/pom.xml index 06b5e98af..b41dd223c 100644 --- a/client/pom.xml +++ b/client/pom.xml @@ -5,7 +5,7 @@ io.split.client java-client-parent - 4.1.7-rc5 + 4.1.7 java-client jar diff --git a/client/src/main/java/io/split/client/SplitClientConfig.java b/client/src/main/java/io/split/client/SplitClientConfig.java index 0cd01f2f7..b8a1e3bd6 100644 --- a/client/src/main/java/io/split/client/SplitClientConfig.java +++ b/client/src/main/java/io/split/client/SplitClientConfig.java @@ -304,10 +304,10 @@ public static final class Builder { private int _streamingReconnectBackoffBase = 1; private String _authServiceURL = "https://auth.split.io/api/auth"; private String _streamingServiceURL = "https://streaming.split.io/sse"; - private int _onDemandFetchRetryDelayMs = 50; - private int _onDemandFetchMaxRetries = 10; - private int _failedAttemptsBeforeLogging = -1; - private boolean _cdnDebugLogging = true; + private final int _onDemandFetchRetryDelayMs = 50; + private final int _onDemandFetchMaxRetries = 10; + private final int _failedAttemptsBeforeLogging = 10; + private final boolean _cdnDebugLogging = true; public Builder() { } @@ -699,36 +699,6 @@ public Builder streamingServiceURL(String streamingServiceURL) { return this; } - /** - * Set Streaming retry delay. - * @param onDemandFetchRetryDelayMs - * @return - */ - public Builder streamingRetryDelay(int onDemandFetchRetryDelayMs) { - _onDemandFetchRetryDelayMs = onDemandFetchRetryDelayMs; - return this; - } - - public Builder streamingFetchMaxRetries(int maxRetries) { - _onDemandFetchMaxRetries = maxRetries; - return this; - } - - public Builder failedAttemptsBeforeLoggingCDNInfo(int failedAttemptsBeforeLogging) { - _failedAttemptsBeforeLogging = failedAttemptsBeforeLogging; - return this; - } - - /** - * Enable logging response headers for requests made to our CDN. - * @param cdnDebugLogging - * @return - */ - public Builder cdnDebugLogging(boolean cdnDebugLogging) { - _cdnDebugLogging = cdnDebugLogging; - return this; - } - public SplitClientConfig build() { if (_featuresRefreshRate < 5 ) { throw new IllegalArgumentException("featuresRefreshRate must be >= 5: " + _featuresRefreshRate); @@ -806,10 +776,6 @@ public SplitClientConfig build() { throw new IllegalStateException("_onDemandFetchMaxRetries must be > 0"); } - if (_failedAttemptsBeforeLogging < 0) { - _failedAttemptsBeforeLogging = _onDemandFetchMaxRetries / 2; - } - return new SplitClientConfig( _endpoint, _eventsEndpoint, diff --git a/pom.xml b/pom.xml index 3f96f8b08..1574ea7dd 100644 --- a/pom.xml +++ b/pom.xml @@ -4,7 +4,7 @@ 4.0.0 io.split.client java-client-parent - 4.1.7-rc5 + 4.1.7 diff --git a/testing/pom.xml b/testing/pom.xml index 66fe079d7..e32089bf8 100644 --- a/testing/pom.xml +++ b/testing/pom.xml @@ -6,7 +6,7 @@ io.split.client java-client-parent - 4.1.7-rc5 + 4.1.7 java-client-testing From 70169ec384d4ee5af1d554119c224bfdb288b24f Mon Sep 17 00:00:00 2001 From: Lucas Echeverz Date: Fri, 21 May 2021 17:16:44 -0300 Subject: [PATCH 63/81] Fixing travis build --- client/src/test/java/io/split/client/EventsClientImplTest.java | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/client/src/test/java/io/split/client/EventsClientImplTest.java b/client/src/test/java/io/split/client/EventsClientImplTest.java index 044e6a634..a88aca630 100644 --- a/client/src/test/java/io/split/client/EventsClientImplTest.java +++ b/client/src/test/java/io/split/client/EventsClientImplTest.java @@ -81,7 +81,7 @@ public void testEventsFlushedWhenSizeLimitReached() throws URISyntaxException, I } @Test - public void testEventDropped() throws URISyntaxException, NoSuchFieldException, IllegalAccessException { + public void testEventDropped() throws URISyntaxException, NoSuchFieldException, IllegalAccessException, InterruptedException { CloseableHttpClient client = Mockito.mock(CloseableHttpClient.class); EventClientImpl eventClient = new EventClientImpl(new LinkedBlockingQueue<>(2), client, @@ -90,6 +90,7 @@ public void testEventDropped() throws URISyntaxException, NoSuchFieldException, 100000, // Long period so it doesn't flush by timeout expiration. 0, TELEMETRY_STORAGE); eventClient.close(); + Thread.sleep(1000); for (int i = 0; i < 3; ++i) { Event event = new Event(); eventClient.track(event, 1); From 154a615eebc887ad12757d3f30018dd83691febe Mon Sep 17 00:00:00 2001 From: Lucas Echeverz Date: Fri, 21 May 2021 17:29:15 -0300 Subject: [PATCH 64/81] Fixing travis --- .../java/io/split/client/EventsClientImplTest.java | 12 +++++------- 1 file changed, 5 insertions(+), 7 deletions(-) diff --git a/client/src/test/java/io/split/client/EventsClientImplTest.java b/client/src/test/java/io/split/client/EventsClientImplTest.java index a88aca630..2bc9d5553 100644 --- a/client/src/test/java/io/split/client/EventsClientImplTest.java +++ b/client/src/test/java/io/split/client/EventsClientImplTest.java @@ -9,16 +9,13 @@ import org.apache.hc.client5.http.impl.classic.HttpClients; import org.hamcrest.Matchers; import org.junit.Assert; +import org.junit.Before; import org.junit.Test; import org.mockito.Mockito; import java.io.IOException; -import java.lang.reflect.Field; -import java.lang.reflect.Modifier; import java.net.URI; import java.net.URISyntaxException; -import java.util.concurrent.ExecutorService; -import java.util.concurrent.Executors; import java.util.concurrent.LinkedBlockingQueue; public class EventsClientImplTest { @@ -82,13 +79,14 @@ public void testEventsFlushedWhenSizeLimitReached() throws URISyntaxException, I @Test public void testEventDropped() throws URISyntaxException, NoSuchFieldException, IllegalAccessException, InterruptedException { + TelemetryStorage telemetryStorage = Mockito.mock(InMemoryTelemetryStorage.class); CloseableHttpClient client = Mockito.mock(CloseableHttpClient.class); EventClientImpl eventClient = new EventClientImpl(new LinkedBlockingQueue<>(2), client, URI.create("https://kubernetesturl.com/split"), 10000, // Long queue so it doesn't flush by # of events 100000, // Long period so it doesn't flush by timeout expiration. - 0, TELEMETRY_STORAGE); + 0, telemetryStorage); eventClient.close(); Thread.sleep(1000); for (int i = 0; i < 3; ++i) { @@ -96,7 +94,7 @@ public void testEventDropped() throws URISyntaxException, NoSuchFieldException, eventClient.track(event, 1); } - Mockito.verify(TELEMETRY_STORAGE, Mockito.times(2)).recordEventStats(EventsDataRecordsEnum.EVENTS_QUEUED, 1); - Mockito.verify(TELEMETRY_STORAGE, Mockito.times(1)).recordEventStats(EventsDataRecordsEnum.EVENTS_DROPPED, 1); + Mockito.verify(telemetryStorage, Mockito.times(2)).recordEventStats(EventsDataRecordsEnum.EVENTS_QUEUED, 1); + Mockito.verify(telemetryStorage, Mockito.times(1)).recordEventStats(EventsDataRecordsEnum.EVENTS_DROPPED, 1); } } From 96c104db7c6ae214e18779287ec1d19bcbf13129 Mon Sep 17 00:00:00 2001 From: Lucas Echeverz Date: Thu, 27 May 2021 14:05:15 -0300 Subject: [PATCH 65/81] Adding tests --- .../io/split/client/SplitFactoryImpl.java | 4 +- ...zerMemory.java => TelemetrySubmitter.java} | 21 +- .../io/split/client/SplitFactoryImplTest.java | 16 ++ .../ImpressionsManagerImplTest.java | 15 +- .../split/engine/sse/AuthApiClientTest.java | 12 +- .../synchronizer/SynchronizerMemoryTest.java | 58 ----- .../synchronizer/TelemetrySubmitterTest.java | 224 ++++++++++++++++++ .../synchronizer/TelemetrySyncTaskTest.java | 4 +- 8 files changed, 279 insertions(+), 75 deletions(-) rename client/src/main/java/io/split/telemetry/synchronizer/{SynchronizerMemory.java => TelemetrySubmitter.java} (88%) delete mode 100644 client/src/test/java/io/split/telemetry/synchronizer/SynchronizerMemoryTest.java create mode 100644 client/src/test/java/io/split/telemetry/synchronizer/TelemetrySubmitterTest.java diff --git a/client/src/main/java/io/split/client/SplitFactoryImpl.java b/client/src/main/java/io/split/client/SplitFactoryImpl.java index 03aaedf3c..ac9f61555 100644 --- a/client/src/main/java/io/split/client/SplitFactoryImpl.java +++ b/client/src/main/java/io/split/client/SplitFactoryImpl.java @@ -27,7 +27,7 @@ import io.split.integrations.IntegrationsConfig; import io.split.telemetry.storage.InMemoryTelemetryStorage; import io.split.telemetry.storage.TelemetryStorage; -import io.split.telemetry.synchronizer.SynchronizerMemory; +import io.split.telemetry.synchronizer.TelemetrySubmitter; import io.split.telemetry.synchronizer.TelemetrySyncTask; import io.split.telemetry.synchronizer.TelemetrySynchronizer; import org.apache.hc.client5.http.auth.AuthScope; @@ -123,7 +123,7 @@ public SplitFactoryImpl(String apiToken, SplitClientConfig config) throws URISyn // Cache Initialisations _segmentCache = new SegmentCacheInMemoryImpl(); _splitCache = new InMemoryCacheImp(); - _telemetrySynchronizer = new SynchronizerMemory(_httpclient, URI.create(config.get_telemetryURL()), _telemetryStorage, _splitCache, _segmentCache, _telemetryStorage, _startTime); + _telemetrySynchronizer = new TelemetrySubmitter(_httpclient, URI.create(config.get_telemetryURL()), _telemetryStorage, _splitCache, _segmentCache, _telemetryStorage, _startTime); // Segments diff --git a/client/src/main/java/io/split/telemetry/synchronizer/SynchronizerMemory.java b/client/src/main/java/io/split/telemetry/synchronizer/TelemetrySubmitter.java similarity index 88% rename from client/src/main/java/io/split/telemetry/synchronizer/SynchronizerMemory.java rename to client/src/main/java/io/split/telemetry/synchronizer/TelemetrySubmitter.java index bf075c4ed..4da5339ec 100644 --- a/client/src/main/java/io/split/telemetry/synchronizer/SynchronizerMemory.java +++ b/client/src/main/java/io/split/telemetry/synchronizer/TelemetrySubmitter.java @@ -1,5 +1,6 @@ package io.split.telemetry.synchronizer; +import com.google.common.annotations.VisibleForTesting; import io.split.cache.SegmentCache; import io.split.cache.SplitCache; import io.split.client.SplitClientConfig; @@ -23,7 +24,7 @@ import java.util.List; import java.util.Map; -public class SynchronizerMemory implements TelemetrySynchronizer{ +public class TelemetrySubmitter implements TelemetrySynchronizer{ private static final int OPERATION_MODE = 0; private static final String STORAGE = "memory"; @@ -34,7 +35,7 @@ public class SynchronizerMemory implements TelemetrySynchronizer{ private SegmentCache _segmentCache; private final long _initStartTime; - public SynchronizerMemory(CloseableHttpClient client, URI telemetryRootEndpoint, TelemetryStorageConsumer telemetryStorageConsumer, SplitCache splitCache, + public TelemetrySubmitter(CloseableHttpClient client, URI telemetryRootEndpoint, TelemetryStorageConsumer telemetryStorageConsumer, SplitCache splitCache, SegmentCache segmentCache, TelemetryRuntimeProducer telemetryRuntimeProducer, long initStartTime) throws URISyntaxException { _httpHttpTelemetryMemorySender = HttpTelemetryMemorySender.create(client, telemetryRootEndpoint, telemetryRuntimeProducer); _teleTelemetryStorageConsumer = telemetryStorageConsumer; @@ -53,7 +54,8 @@ public void synchronizeStats() throws Exception { _httpHttpTelemetryMemorySender.postStats(generateStats()); } - private Stats generateStats() throws Exception { + @VisibleForTesting + Stats generateStats() throws Exception { Stats stats = new Stats(); stats.set_lastSynchronization(_teleTelemetryStorageConsumer.getLastSynchronization()); stats.set_methodLatencies(_teleTelemetryStorageConsumer.popLatencies()); @@ -76,7 +78,8 @@ private Stats generateStats() throws Exception { return stats; } - private Config generateConfig(SplitClientConfig splitClientConfig, long readyTimestamp, Map factoryInstances, List tags) { + @VisibleForTesting + Config generateConfig(SplitClientConfig splitClientConfig, long readyTimestamp, Map factoryInstances, List tags) { Config config = new Config(); Rates rates = new Rates(); URLOverrides urlOverrides = new URLOverrides(); @@ -93,11 +96,11 @@ private Config generateConfig(SplitClientConfig splitClientConfig, long readyTim rates.set_segments(splitClientConfig.segmentsRefreshRate()); rates.set_splits(splitClientConfig.featuresRefreshRate()); - urlOverrides.set_auth(SplitClientConfig.AUTH_ENDPOINT.equals(splitClientConfig.authServiceURL())); - urlOverrides.set_stream(SplitClientConfig.STREAMING_ENDPOINT.equals(splitClientConfig.streamingServiceURL())); - urlOverrides.set_sdk(SplitClientConfig.SDK_ENDPOINT.equals(splitClientConfig.endpoint())); - urlOverrides.set_events(SplitClientConfig.EVENTS_ENDPOINT.equals(splitClientConfig.eventsEndpoint())); - urlOverrides.set_telemetry(SplitClientConfig.TELEMETRY_ENDPOINT.equals(splitClientConfig.get_telemetryURL())); + urlOverrides.set_auth(!SplitClientConfig.AUTH_ENDPOINT.equals(splitClientConfig.authServiceURL())); + urlOverrides.set_stream(!SplitClientConfig.STREAMING_ENDPOINT.equals(splitClientConfig.streamingServiceURL())); + urlOverrides.set_sdk(!SplitClientConfig.SDK_ENDPOINT.equals(splitClientConfig.endpoint())); + urlOverrides.set_events(!SplitClientConfig.EVENTS_ENDPOINT.equals(splitClientConfig.eventsEndpoint())); + urlOverrides.set_telemetry(!SplitClientConfig.TELEMETRY_ENDPOINT.equals(splitClientConfig.get_telemetryURL())); config.set_burTimeouts(_teleTelemetryStorageConsumer.getBURTimeouts()); config.set_nonReadyUsages(_teleTelemetryStorageConsumer.getNonReadyUsages()); diff --git a/client/src/test/java/io/split/client/SplitFactoryImplTest.java b/client/src/test/java/io/split/client/SplitFactoryImplTest.java index 1256293b4..99123aa93 100644 --- a/client/src/test/java/io/split/client/SplitFactoryImplTest.java +++ b/client/src/test/java/io/split/client/SplitFactoryImplTest.java @@ -1,10 +1,15 @@ package io.split.client; import io.split.client.impressions.ImpressionsManager; +import io.split.engine.segments.SegmentSynchronizationTaskImp; import io.split.integrations.IntegrationsConfig; +import io.split.telemetry.storage.TelemetryStorage; import junit.framework.TestCase; import org.junit.Test; +import org.mockito.Mockito; +import java.lang.reflect.Field; +import java.lang.reflect.Modifier; import java.net.URISyntaxException; public class SplitFactoryImplTest extends TestCase { @@ -89,6 +94,7 @@ public void testFactoryInstantiationWithProxy() throws Exception { @Test public void testFactoryDestroy() throws Exception { + TelemetryStorage telemetryStorage = Mockito.mock(TelemetryStorage.class); SplitClientConfig splitClientConfig = SplitClientConfig.builder() .enableDebug() .impressionsMode(ImpressionsManager.Mode.DEBUG) @@ -98,10 +104,20 @@ public void testFactoryDestroy() throws Exception { .authServiceURL(AUTH_SERVICE) .setBlockUntilReadyTimeout(10000) .build(); + SplitFactoryImpl splitFactory = new SplitFactoryImpl(API_KEY, splitClientConfig); + //Before destroy we replace telemetryStorage via reflection. + Field factoryDestroy = SplitFactoryImpl.class.getDeclaredField("_telemetryStorage"); + factoryDestroy.setAccessible(true); + Field modifiersField = Field.class.getDeclaredField("modifiers"); + modifiersField.setAccessible(true); + modifiersField.setInt(factoryDestroy, factoryDestroy.getModifiers() & ~Modifier.FINAL); + + factoryDestroy.set(splitFactory, telemetryStorage); splitFactory.destroy(); assertTrue(splitFactory.isDestroyed()); + Mockito.verify(telemetryStorage, Mockito.times(1)).recordSessionLength(Mockito.anyLong()); } } \ No newline at end of file diff --git a/client/src/test/java/io/split/client/impressions/ImpressionsManagerImplTest.java b/client/src/test/java/io/split/client/impressions/ImpressionsManagerImplTest.java index aca79a6c5..c00caa5d4 100644 --- a/client/src/test/java/io/split/client/impressions/ImpressionsManagerImplTest.java +++ b/client/src/test/java/io/split/client/impressions/ImpressionsManagerImplTest.java @@ -4,8 +4,10 @@ import io.split.client.dtos.KeyImpression; import io.split.client.dtos.TestImpressions; +import io.split.telemetry.domain.enums.ImpressionsDataTypeEnum; import io.split.telemetry.storage.InMemoryTelemetryStorage; import io.split.telemetry.storage.TelemetryStorage; +import org.junit.Before; import org.junit.Ignore; import org.junit.Test; import org.junit.runner.RunWith; @@ -21,15 +23,19 @@ import static org.hamcrest.Matchers.*; import static org.junit.Assert.assertThat; -import static org.mockito.Mockito.never; -import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.*; /** * Created by patricioe on 6/20/16. */ @RunWith(MockitoJUnitRunner.class) public class ImpressionsManagerImplTest { - private static final TelemetryStorage TELEMETRY_STORAGE = Mockito.mock(InMemoryTelemetryStorage.class); + private static TelemetryStorage TELEMETRY_STORAGE = Mockito.mock(InMemoryTelemetryStorage.class); + + @Before + public void setUp() { + TELEMETRY_STORAGE = Mockito.mock(InMemoryTelemetryStorage.class); + } @Captor private ArgumentCaptor> impressionsCaptor; @@ -102,6 +108,7 @@ public void worksButDropsImpressions() throws URISyntaxException { List captured = impressionsCaptor.getValue(); assertThat(captured.size(), is(equalTo(3))); + Mockito.verify(TELEMETRY_STORAGE, times(1)).recordImpressionStats(ImpressionsDataTypeEnum.IMPRESSIONS_DROPPED, 1); } @Test @@ -138,6 +145,8 @@ public void works4ImpressionsInOneTest() throws URISyntaxException { assertThat(captured.size(), is(equalTo(1))); assertThat(captured.get(0).keyImpressions.size(), is(equalTo(4))); assertThat(captured.get(0).keyImpressions.get(0), is(equalTo(ki1))); + Mockito.verify(TELEMETRY_STORAGE, times(2)).recordImpressionStats(ImpressionsDataTypeEnum.IMPRESSIONS_DEDUPED, 1); + Mockito.verify(TELEMETRY_STORAGE, times(4)).recordImpressionStats(ImpressionsDataTypeEnum.IMPRESSIONS_QUEUED, 1); } @Test diff --git a/client/src/test/java/io/split/engine/sse/AuthApiClientTest.java b/client/src/test/java/io/split/engine/sse/AuthApiClientTest.java index f90a45d80..dab743602 100644 --- a/client/src/test/java/io/split/engine/sse/AuthApiClientTest.java +++ b/client/src/test/java/io/split/engine/sse/AuthApiClientTest.java @@ -8,6 +8,7 @@ import org.apache.hc.client5.http.impl.classic.CloseableHttpClient; import org.apache.hc.core5.http.HttpStatus; import org.junit.Assert; +import org.junit.Before; import org.junit.Test; import org.mockito.Mockito; @@ -15,7 +16,12 @@ import java.lang.reflect.InvocationTargetException; public class AuthApiClientTest { - private static final TelemetryStorage TELEMETRY_STORAGE = Mockito.mock(InMemoryTelemetryStorage.class); + private static TelemetryStorage TELEMETRY_STORAGE = Mockito.mock(InMemoryTelemetryStorage.class); + + @Before + public void setUp() { + TELEMETRY_STORAGE = Mockito.mock(InMemoryTelemetryStorage.class); + } @Test public void authenticateWithPushEnabledShouldReturnSuccess() throws IOException, IllegalAccessException, NoSuchMethodException, InvocationTargetException { CloseableHttpClient httpClientMock = TestHelper.mockHttpClient("streaming-auth-push-enabled.json", HttpStatus.SC_OK); @@ -28,6 +34,9 @@ public void authenticateWithPushEnabledShouldReturnSuccess() throws IOException, Assert.assertFalse(result.isRetry()); Assert.assertFalse(StringUtils.isEmpty(result.getToken())); Assert.assertTrue(result.getExpiration() > 0); + Mockito.verify(TELEMETRY_STORAGE, Mockito.times(1)).recordTokenRefreshes(); + Mockito.verify(TELEMETRY_STORAGE, Mockito.times(1)).recordSyncLatency(Mockito.anyObject(), Mockito.anyLong()); + Mockito.verify(TELEMETRY_STORAGE, Mockito.times(1)).recordSuccessfulSync(Mockito.anyObject(), Mockito.anyLong()); } @@ -95,5 +104,6 @@ public void authenticateServerUnauthorizedShouldReturnErrorWithoutRetry() throws Assert.assertTrue(StringUtils.isEmpty(result.getChannels())); Assert.assertTrue(StringUtils.isEmpty(result.getToken())); Assert.assertFalse(result.isRetry()); + Mockito.verify(TELEMETRY_STORAGE, Mockito.times(1)).recordAuthRejections(); } } diff --git a/client/src/test/java/io/split/telemetry/synchronizer/SynchronizerMemoryTest.java b/client/src/test/java/io/split/telemetry/synchronizer/SynchronizerMemoryTest.java deleted file mode 100644 index 0afab5ddf..000000000 --- a/client/src/test/java/io/split/telemetry/synchronizer/SynchronizerMemoryTest.java +++ /dev/null @@ -1,58 +0,0 @@ -package io.split.telemetry.synchronizer; - -import io.split.TestHelper; -import io.split.cache.SegmentCache; -import io.split.cache.SegmentCacheInMemoryImpl; -import io.split.cache.SplitCache; -import io.split.client.SplitClientConfig; -import io.split.telemetry.storage.InMemoryTelemetryStorage; -import io.split.telemetry.storage.TelemetryRuntimeProducer; -import io.split.telemetry.storage.TelemetryStorageConsumer; -import org.apache.hc.client5.http.impl.classic.CloseableHttpClient; -import org.apache.hc.core5.http.HttpStatus; -import org.junit.Test; -import org.mockito.Mock; -import org.mockito.Mockito; - -import java.io.IOException; -import java.lang.reflect.InvocationTargetException; -import java.net.URI; -import java.net.URISyntaxException; -import java.util.ArrayList; -import java.util.HashMap; - -public class SynchronizerMemoryTest { - - public static final String TELEMETRY_ENDPOINT = "https://telemetry.split.io/api/v1"; - - @Test - public void testSynchronizeConfig() throws URISyntaxException, NoSuchMethodException, IOException, IllegalAccessException, InvocationTargetException { - CloseableHttpClient httpClient = TestHelper.mockHttpClient(TELEMETRY_ENDPOINT, HttpStatus.SC_OK); - TelemetrySynchronizer telemetrySynchronizer = getTelemetrySynchronizer(httpClient); - SplitClientConfig splitClientConfig = SplitClientConfig.builder().build(); - - telemetrySynchronizer.synchronizeConfig(splitClientConfig, 100l, new HashMap(), new ArrayList()); - Mockito.verify(httpClient, Mockito.times(1)).execute(Mockito.any()); - } - - - @Test - public void testSynchronizeStats() throws Exception { - CloseableHttpClient httpClient = TestHelper.mockHttpClient(TELEMETRY_ENDPOINT, HttpStatus.SC_OK); - TelemetrySynchronizer telemetrySynchronizer = getTelemetrySynchronizer(httpClient); - - telemetrySynchronizer.synchronizeStats(); - Mockito.verify(httpClient, Mockito.times(1)).execute(Mockito.any()); - } - - private TelemetrySynchronizer getTelemetrySynchronizer(CloseableHttpClient httpClient) throws URISyntaxException, InvocationTargetException, NoSuchMethodException, IllegalAccessException, IOException { - TelemetryStorageConsumer consumer = Mockito.mock(InMemoryTelemetryStorage.class); - TelemetryRuntimeProducer telemetryRuntimeProducer = Mockito.mock(TelemetryRuntimeProducer.class); - SplitCache splitCache = Mockito.mock(SplitCache.class); - SegmentCache segmentCache = Mockito.mock(SegmentCacheInMemoryImpl.class); - SplitClientConfig config = Mockito.mock(SplitClientConfig.class); - TelemetrySynchronizer telemetrySynchronizer = new SynchronizerMemory(httpClient, URI.create(TELEMETRY_ENDPOINT), consumer, splitCache, segmentCache, telemetryRuntimeProducer, 0l); - return telemetrySynchronizer; - } - -} \ No newline at end of file diff --git a/client/src/test/java/io/split/telemetry/synchronizer/TelemetrySubmitterTest.java b/client/src/test/java/io/split/telemetry/synchronizer/TelemetrySubmitterTest.java new file mode 100644 index 000000000..d4bb42d06 --- /dev/null +++ b/client/src/test/java/io/split/telemetry/synchronizer/TelemetrySubmitterTest.java @@ -0,0 +1,224 @@ +package io.split.telemetry.synchronizer; + +import io.split.TestHelper; +import io.split.cache.SegmentCache; +import io.split.cache.SegmentCacheInMemoryImpl; +import io.split.cache.SplitCache; +import io.split.client.ApiKeyCounter; +import io.split.client.SplitClientConfig; +import io.split.telemetry.domain.Config; +import io.split.telemetry.domain.Stats; +import io.split.telemetry.domain.StreamingEvent; +import io.split.telemetry.domain.enums.*; +import io.split.telemetry.storage.InMemoryTelemetryStorage; +import io.split.telemetry.storage.TelemetryRuntimeProducer; +import io.split.telemetry.storage.TelemetryStorage; +import io.split.telemetry.storage.TelemetryStorageConsumer; +import org.apache.hc.client5.http.impl.classic.CloseableHttpClient; +import org.apache.hc.core5.http.HttpStatus; +import org.junit.Assert; +import org.junit.Test; +import org.mockito.Mockito; + +import java.io.IOException; +import java.lang.reflect.Field; +import java.lang.reflect.InvocationTargetException; +import java.lang.reflect.Modifier; +import java.net.URI; +import java.net.URISyntaxException; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; + +public class TelemetrySubmitterTest { + private static final String FIRST_KEY = "KEY_1"; + private static final String SECOND_KEY = "KEY_2"; + public static final String TELEMETRY_ENDPOINT = "https://telemetry.split.io/api/v1"; + + @Test + public void testSynchronizeConfig() throws URISyntaxException, NoSuchMethodException, IOException, IllegalAccessException, InvocationTargetException { + CloseableHttpClient httpClient = TestHelper.mockHttpClient(TELEMETRY_ENDPOINT, HttpStatus.SC_OK); + TelemetrySynchronizer telemetrySynchronizer = getTelemetrySynchronizer(httpClient); + SplitClientConfig splitClientConfig = SplitClientConfig.builder().build(); + + telemetrySynchronizer.synchronizeConfig(splitClientConfig, 100l, new HashMap(), new ArrayList()); + Mockito.verify(httpClient, Mockito.times(1)).execute(Mockito.any()); + } + + + @Test + public void testSynchronizeStats() throws Exception { + CloseableHttpClient httpClient = TestHelper.mockHttpClient(TELEMETRY_ENDPOINT, HttpStatus.SC_OK); + TelemetrySynchronizer telemetrySynchronizer = getTelemetrySynchronizer(httpClient); + + telemetrySynchronizer.synchronizeStats(); + Mockito.verify(httpClient, Mockito.times(1)).execute(Mockito.any()); + } + + @Test + public void testConfig() throws InvocationTargetException, NoSuchMethodException, IllegalAccessException, IOException, URISyntaxException, NoSuchFieldException, ClassNotFoundException { + ApiKeyCounter.getApiKeyCounterInstance().add(FIRST_KEY); + ApiKeyCounter.getApiKeyCounterInstance().add(FIRST_KEY); + ApiKeyCounter.getApiKeyCounterInstance().add(FIRST_KEY); + ApiKeyCounter.getApiKeyCounterInstance().add(SECOND_KEY); + ApiKeyCounter.getApiKeyCounterInstance().add(SECOND_KEY); + TelemetryStorage telemetryStorage = new InMemoryTelemetryStorage(); + CloseableHttpClient httpClient = TestHelper.mockHttpClient(TELEMETRY_ENDPOINT, HttpStatus.SC_OK); + TelemetrySubmitter telemetrySynchronizer = getTelemetrySynchronizer(httpClient); + SplitClientConfig splitClientConfig = SplitClientConfig.builder().build(); + populateConfig(telemetryStorage); + Field teleTelemetryStorageConsumer = TelemetrySubmitter.class.getDeclaredField("_teleTelemetryStorageConsumer"); + teleTelemetryStorageConsumer.setAccessible(true); + Field modifiersField = Field.class.getDeclaredField("modifiers"); + modifiersField.setAccessible(true); + modifiersField.setInt(teleTelemetryStorageConsumer, teleTelemetryStorageConsumer.getModifiers() & ~Modifier.FINAL); + teleTelemetryStorageConsumer.set(telemetrySynchronizer, telemetryStorage); + Config config = telemetrySynchronizer.generateConfig(splitClientConfig, 100l, ApiKeyCounter.getApiKeyCounterInstance().getFactoryInstances(), new ArrayList<>()); + Assert.assertEquals(3, config.get_redundantFactories()); + Assert.assertEquals(2, config.get_burTimeouts()); + Assert.assertEquals(3, config.get_nonReadyUsages()); + } + + @Test + public void testStats() throws Exception { + TelemetryStorage telemetryStorage = new InMemoryTelemetryStorage(); + CloseableHttpClient httpClient = TestHelper.mockHttpClient(TELEMETRY_ENDPOINT, HttpStatus.SC_OK); + TelemetrySubmitter telemetrySynchronizer = getTelemetrySynchronizer(httpClient); + populateStats(telemetryStorage); + Field teleTelemetryStorageConsumer = TelemetrySubmitter.class.getDeclaredField("_teleTelemetryStorageConsumer"); + teleTelemetryStorageConsumer.setAccessible(true); + Field modifiersField = Field.class.getDeclaredField("modifiers"); + modifiersField.setAccessible(true); + modifiersField.setInt(teleTelemetryStorageConsumer, teleTelemetryStorageConsumer.getModifiers() & ~Modifier.FINAL); + + teleTelemetryStorageConsumer.set(telemetrySynchronizer, telemetryStorage); + Stats stats = telemetrySynchronizer.generateStats(); + Assert.assertEquals(2, stats.get_methodLatencies().get_treatment().stream().mapToInt(Long::intValue).sum()); + Assert.assertEquals(2, stats.get_methodLatencies().get_treatments().stream().mapToInt(Long::intValue).sum()); + Assert.assertEquals(1, stats.get_methodLatencies().get_treatmentsWithConfig().stream().mapToInt(Long::intValue).sum()); + Assert.assertEquals(1, stats.get_methodLatencies().get_treatmentWithConfig().stream().mapToInt(Long::intValue).sum()); + Assert.assertEquals(0, stats.get_methodLatencies().get_track().stream().mapToInt(Long::intValue).sum()); + Assert.assertEquals(3, stats.get_httpLatencies().get_splits().stream().mapToInt(Long::intValue).sum()); + Assert.assertEquals(2, stats.get_httpLatencies().get_telemetry().stream().mapToInt(Long::intValue).sum()); + Assert.assertEquals(2, stats.get_httpLatencies().get_events().stream().mapToInt(Long::intValue).sum()); + Assert.assertEquals(1, stats.get_httpLatencies().get_segments().stream().mapToInt(Long::intValue).sum()); + Assert.assertEquals(1, stats.get_httpLatencies().get_impressions().stream().mapToInt(Long::intValue).sum()); + Assert.assertEquals(1, stats.get_httpLatencies().get_impressionsCount().stream().mapToInt(Long::intValue).sum()); + Assert.assertEquals(0, stats.get_httpLatencies().get_token().stream().mapToInt(Long::intValue).sum()); + Assert.assertEquals(2, stats.get_methodExceptions().get_treatment()); + Assert.assertEquals(2, stats.get_methodExceptions().get_treatments()); + Assert.assertEquals(1, stats.get_methodExceptions().get_treatmentsWithConfig()); + Assert.assertEquals(1, stats.get_methodExceptions().get_treatmentWithConfig()); + Assert.assertEquals(0, stats.get_methodExceptions().get_track()); + Assert.assertEquals(1, stats.get_authRejections()); + Assert.assertEquals(2, stats.get_tokenRefreshes()); + Assert.assertEquals(4, stats.get_impressionsDeduped()); + Assert.assertEquals(12, stats.get_impressionsDropped()); + Assert.assertEquals(0, stats.get_impressionsQueued()); + Assert.assertEquals(10, stats.get_eventsDropped()); + Assert.assertEquals(3, stats.get_eventsQueued()); + Assert.assertEquals(800, stats.get_lastSynchronization().get_events()); + Assert.assertEquals(129, stats.get_lastSynchronization().get_token()); + Assert.assertEquals(1580, stats.get_lastSynchronization().get_segments()); + Assert.assertEquals(0, stats.get_lastSynchronization().get_splits()); + Assert.assertEquals(10500, stats.get_lastSynchronization().get_impressions()); + Assert.assertEquals(1500, stats.get_lastSynchronization().get_impressionsCount()); + Assert.assertEquals(265, stats.get_lastSynchronization().get_telemetry()); + Assert.assertEquals(91218, stats.get_sessionLengthMs()); + Assert.assertEquals(2, stats.get_httpErrors().get_telemetry().get(400l).intValue()); + Assert.assertEquals(1, stats.get_httpErrors().get_segments().get(501l).intValue()); + Assert.assertEquals(2, stats.get_httpErrors().get_impressions().get(403l).intValue()); + Assert.assertEquals(1, stats.get_httpErrors().get_impressionsCount().get(403l).intValue()); + Assert.assertEquals(1, stats.get_httpErrors().get_events().get(503l).intValue()); + Assert.assertEquals(1, stats.get_httpErrors().get_splits().get(403l).intValue()); + Assert.assertEquals(1, stats.get_httpErrors().get_token().get(403l).intValue()); + List streamingEvents = stats.get_streamingEvents(); + Assert.assertEquals(290, streamingEvents.get(0).get_data()); + Assert.assertEquals(1, streamingEvents.get(0).get_type()); + Assert.assertEquals(91218, streamingEvents.get(0).getTimestamp()); + } + + private TelemetrySubmitter getTelemetrySynchronizer(CloseableHttpClient httpClient) throws URISyntaxException, InvocationTargetException, NoSuchMethodException, IllegalAccessException, IOException { + TelemetryStorageConsumer consumer = Mockito.mock(InMemoryTelemetryStorage.class); + TelemetryRuntimeProducer telemetryRuntimeProducer = Mockito.mock(TelemetryRuntimeProducer.class); + SplitCache splitCache = Mockito.mock(SplitCache.class); + SegmentCache segmentCache = Mockito.mock(SegmentCacheInMemoryImpl.class); + TelemetrySubmitter telemetrySynchronizer = new TelemetrySubmitter(httpClient, URI.create(TELEMETRY_ENDPOINT), consumer, splitCache, segmentCache, telemetryRuntimeProducer, 0l); + return telemetrySynchronizer; + } + + private void populateStats(TelemetryStorage telemetryStorage) { + telemetryStorage.recordLatency(MethodEnum.TREATMENT, 1500l * 1000); + telemetryStorage.recordLatency(MethodEnum.TREATMENT, 2000l * 1000); + telemetryStorage.recordLatency(MethodEnum.TREATMENTS, 3000l * 1000); + telemetryStorage.recordLatency(MethodEnum.TREATMENTS, 500l * 1000); + telemetryStorage.recordLatency(MethodEnum.TREATMENT_WITH_CONFIG, 800l * 1000); + telemetryStorage.recordLatency(MethodEnum.TREATMENTS_WITH_CONFIG, 1000l * 1000); + + telemetryStorage.recordSyncLatency(HTTPLatenciesEnum.TELEMETRY, 1500l * 1000); + telemetryStorage.recordSyncLatency(HTTPLatenciesEnum.TELEMETRY, 2000l * 1000); + telemetryStorage.recordSyncLatency(HTTPLatenciesEnum.EVENTS, 1500l * 1000); + telemetryStorage.recordSyncLatency(HTTPLatenciesEnum.EVENTS, 2000l * 1000); + telemetryStorage.recordSyncLatency(HTTPLatenciesEnum.SEGMENTS, 1500l * 1000); + telemetryStorage.recordSyncLatency(HTTPLatenciesEnum.SPLITS, 2000l * 1000); + telemetryStorage.recordSyncLatency(HTTPLatenciesEnum.SPLITS, 1500l * 1000); + telemetryStorage.recordSyncLatency(HTTPLatenciesEnum.SPLITS, 2000l * 1000); + telemetryStorage.recordSyncLatency(HTTPLatenciesEnum.IMPRESSIONS, 1500l * 1000); + telemetryStorage.recordSyncLatency(HTTPLatenciesEnum.IMPRESSIONS_COUNT, 2000l * 1000); + + telemetryStorage.recordException(MethodEnum.TREATMENT); + telemetryStorage.recordException(MethodEnum.TREATMENTS); + telemetryStorage.recordException(MethodEnum.TREATMENT); + telemetryStorage.recordException(MethodEnum.TREATMENTS); + telemetryStorage.recordException(MethodEnum.TREATMENT_WITH_CONFIG); + telemetryStorage.recordException(MethodEnum.TREATMENTS_WITH_CONFIG); + + telemetryStorage.recordAuthRejections(); + + telemetryStorage.recordTokenRefreshes(); + telemetryStorage.recordTokenRefreshes(); + + telemetryStorage.recordImpressionStats(ImpressionsDataTypeEnum.IMPRESSIONS_DEDUPED, 3); + telemetryStorage.recordImpressionStats(ImpressionsDataTypeEnum.IMPRESSIONS_DEDUPED, 1); + telemetryStorage.recordImpressionStats(ImpressionsDataTypeEnum.IMPRESSIONS_DROPPED, 4); + telemetryStorage.recordImpressionStats(ImpressionsDataTypeEnum.IMPRESSIONS_DROPPED, 6); + telemetryStorage.recordImpressionStats(ImpressionsDataTypeEnum.IMPRESSIONS_DROPPED, 2); + + telemetryStorage.recordEventStats(EventsDataRecordsEnum.EVENTS_DROPPED, 3); + telemetryStorage.recordEventStats(EventsDataRecordsEnum.EVENTS_DROPPED, 7); + telemetryStorage.recordEventStats(EventsDataRecordsEnum.EVENTS_QUEUED, 3); + + telemetryStorage.recordSuccessfulSync(LastSynchronizationRecordsEnum.EVENTS, 1500); + telemetryStorage.recordSuccessfulSync(LastSynchronizationRecordsEnum.EVENTS, 800); + telemetryStorage.recordSuccessfulSync(LastSynchronizationRecordsEnum.IMPRESSIONS, 2500); + telemetryStorage.recordSuccessfulSync(LastSynchronizationRecordsEnum.IMPRESSIONS, 10500); + telemetryStorage.recordSuccessfulSync(LastSynchronizationRecordsEnum.IMPRESSIONS_COUNT, 1500); + telemetryStorage.recordSuccessfulSync(LastSynchronizationRecordsEnum.SEGMENTS, 1580); + telemetryStorage.recordSuccessfulSync(LastSynchronizationRecordsEnum.TELEMETRY, 265); + telemetryStorage.recordSuccessfulSync(LastSynchronizationRecordsEnum.TOKEN, 129); + + telemetryStorage.recordSessionLength(91218); + + telemetryStorage.recordSyncError(ResourceEnum.TELEMETRY_SYNC, 400); + telemetryStorage.recordSyncError(ResourceEnum.TELEMETRY_SYNC, 400); + telemetryStorage.recordSyncError(ResourceEnum.SEGMENT_SYNC, 501); + telemetryStorage.recordSyncError(ResourceEnum.IMPRESSION_SYNC, 403); + telemetryStorage.recordSyncError(ResourceEnum.IMPRESSION_SYNC, 403); + telemetryStorage.recordSyncError(ResourceEnum.EVENT_SYNC, 503); + telemetryStorage.recordSyncError(ResourceEnum.SPLIT_SYNC, 403); + telemetryStorage.recordSyncError(ResourceEnum.IMPRESSION_COUNT_SYNC, 403); + telemetryStorage.recordSyncError(ResourceEnum.TOKEN_SYNC, 403); + + StreamingEvent streamingEvent = new StreamingEvent(1, 290, 91218); + telemetryStorage.recordStreamingEvents(streamingEvent); + } + + private void populateConfig(TelemetryStorage telemetryStorage) { + telemetryStorage.recordBURTimeout(); + telemetryStorage.recordBURTimeout(); + telemetryStorage.recordNonReadyUsage(); + telemetryStorage.recordNonReadyUsage(); + telemetryStorage.recordNonReadyUsage(); + } + +} \ No newline at end of file diff --git a/client/src/test/java/io/split/telemetry/synchronizer/TelemetrySyncTaskTest.java b/client/src/test/java/io/split/telemetry/synchronizer/TelemetrySyncTaskTest.java index 4113313b6..a7432c690 100644 --- a/client/src/test/java/io/split/telemetry/synchronizer/TelemetrySyncTaskTest.java +++ b/client/src/test/java/io/split/telemetry/synchronizer/TelemetrySyncTaskTest.java @@ -7,7 +7,7 @@ public class TelemetrySyncTaskTest { @Test public void testSynchronizationTask() throws Exception { - TelemetrySynchronizer telemetrySynchronizer = Mockito.mock(SynchronizerMemory.class); + TelemetrySynchronizer telemetrySynchronizer = Mockito.mock(TelemetrySubmitter.class); Mockito.doNothing().when(telemetrySynchronizer).synchronizeStats(); TelemetrySyncTask telemetrySyncTask = new TelemetrySyncTask(1, telemetrySynchronizer); Thread.sleep(2900); @@ -16,7 +16,7 @@ public void testSynchronizationTask() throws Exception { @Test public void testStopSynchronizationTask() throws Exception { - TelemetrySynchronizer telemetrySynchronizer = Mockito.mock(SynchronizerMemory.class); + TelemetrySynchronizer telemetrySynchronizer = Mockito.mock(TelemetrySubmitter.class); // Mockito.doNothing().when(telemetrySynchronizer).synchronizeStats(); TelemetrySyncTask telemetrySyncTask = new TelemetrySyncTask(1, telemetrySynchronizer); Thread.sleep(3000); From 848787da5f2448031cc65e80103c967da026b7dd Mon Sep 17 00:00:00 2001 From: Lucas Echeverz Date: Thu, 27 May 2021 22:46:53 -0300 Subject: [PATCH 66/81] Fixing old exceptions --- .../src/main/java/io/split/client/SplitFactoryBuilder.java | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/client/src/main/java/io/split/client/SplitFactoryBuilder.java b/client/src/main/java/io/split/client/SplitFactoryBuilder.java index 7f78f4bd8..f7f1fea8b 100644 --- a/client/src/main/java/io/split/client/SplitFactoryBuilder.java +++ b/client/src/main/java/io/split/client/SplitFactoryBuilder.java @@ -25,7 +25,7 @@ public class SplitFactoryBuilder { * @throws IOException if the SDK was being started in 'localhost' mode, but * there were problems reading the override file from disk. */ - public static SplitFactory build(String apiToken) throws Exception { + public static SplitFactory build(String apiToken) throws IOException, URISyntaxException { return build(apiToken, SplitClientConfig.builder().build()); } @@ -36,7 +36,7 @@ public static SplitFactory build(String apiToken) throws Exception { * @throws java.io.IOException if the SDK was being started in 'localhost' mode, but * there were problems reading the override file from disk. */ - public static synchronized SplitFactory build(String apiToken, SplitClientConfig config) throws Exception { + public static synchronized SplitFactory build(String apiToken, SplitClientConfig config) throws IOException, URISyntaxException { ApiKeyValidator.validate(apiToken); if (LocalhostSplitFactory.LOCALHOST.equals(apiToken)) { @@ -66,7 +66,7 @@ public static SplitFactory local(SplitClientConfig config) throws IOException, U return LocalhostSplitFactory.createLocalhostSplitFactory(config); } - public static void main(String... args) throws Exception { + public static void main(String... args) throws IOException, URISyntaxException { if (args.length != 1) { System.out.println("Usage: "); System.exit(1); From c637a585c445f81a9d72a622902d9515e7648f36 Mon Sep 17 00:00:00 2001 From: Lucas Echeverz Date: Fri, 28 May 2021 15:24:09 -0300 Subject: [PATCH 67/81] Fixing final stats sync --- .../java/io/split/client/SplitFactoryImpl.java | 14 +++++++++----- .../telemetry/synchronizer/TelemetrySubmitter.java | 9 +++++++++ .../telemetry/synchronizer/TelemetrySyncTask.java | 8 ++++---- .../synchronizer/TelemetrySynchronizer.java | 1 + .../synchronizer/TelemetrySyncTaskTest.java | 10 +++++----- 5 files changed, 28 insertions(+), 14 deletions(-) diff --git a/client/src/main/java/io/split/client/SplitFactoryImpl.java b/client/src/main/java/io/split/client/SplitFactoryImpl.java index ac9f61555..90a9e9c0f 100644 --- a/client/src/main/java/io/split/client/SplitFactoryImpl.java +++ b/client/src/main/java/io/split/client/SplitFactoryImpl.java @@ -182,18 +182,22 @@ public synchronized void destroy() { if (!isTerminated) { _log.info("Shutdown called for split"); try { - _segmentSynchronizationTaskImp.close(); - _log.info("Successful shutdown of segment fetchers"); - _splitSynchronizationTask.close(); - _log.info("Successful shutdown of splits"); + long splitCount = _splitCache.getAll().stream().count(); + long segmentCount = _segmentCache.getAll().stream().count(); + long segmentKeyCount = _segmentCache.getAllKeys().stream().count(); _impressionsManager.close(); _log.info("Successful shutdown of impressions manager"); _eventClient.close(); _log.info("Successful shutdown of eventClient"); + _segmentSynchronizationTaskImp.close(); + _log.info("Successful shutdown of segment fetchers"); + _splitSynchronizationTask.close(); + _log.info("Successful shutdown of splits"); _syncManager.shutdown(); _log.info("Successful shutdown of syncManager"); _telemetryStorage.recordSessionLength(System.currentTimeMillis() - _startTime); - _telemetrySyncTask.stopScheduledTask(); + _telemetrySyncTask.stopScheduledTask(splitCount, segmentCount, segmentKeyCount); + _log.info("Successful shutdown of telemetry sync task"); _httpclient.close(); _log.info("Successful shutdown of httpclient"); } catch (IOException e) { diff --git a/client/src/main/java/io/split/telemetry/synchronizer/TelemetrySubmitter.java b/client/src/main/java/io/split/telemetry/synchronizer/TelemetrySubmitter.java index 4da5339ec..b0b600a50 100644 --- a/client/src/main/java/io/split/telemetry/synchronizer/TelemetrySubmitter.java +++ b/client/src/main/java/io/split/telemetry/synchronizer/TelemetrySubmitter.java @@ -54,6 +54,15 @@ public void synchronizeStats() throws Exception { _httpHttpTelemetryMemorySender.postStats(generateStats()); } + @Override + public void finalSynchronization(long splitCount, long segmentCount, long segmentKeyCount) throws Exception { + Stats stats = generateStats(); + stats.set_splitCount(splitCount); + stats.set_segmentCount(segmentCount); + stats.set_segmentKeyCount(segmentKeyCount); + _httpHttpTelemetryMemorySender.postStats(stats); + } + @VisibleForTesting Stats generateStats() throws Exception { Stats stats = new Stats(); diff --git a/client/src/main/java/io/split/telemetry/synchronizer/TelemetrySyncTask.java b/client/src/main/java/io/split/telemetry/synchronizer/TelemetrySyncTask.java index c5135b133..6cd4aa8ef 100644 --- a/client/src/main/java/io/split/telemetry/synchronizer/TelemetrySyncTask.java +++ b/client/src/main/java/io/split/telemetry/synchronizer/TelemetrySyncTask.java @@ -30,19 +30,19 @@ public TelemetrySyncTask(int telemetryRefreshRate, TelemetrySynchronizer telemet } @VisibleForTesting - protected void startScheduledTask() throws Exception { + protected void startScheduledTask() { _telemetrySyncScheduledExecutorService.scheduleWithFixedDelay(() -> { try { _telemetrySynchronizer.synchronizeStats(); } catch (Exception e) { e.printStackTrace(); } - },0l, _telemetryRefreshRate, TimeUnit.SECONDS); + },_telemetryRefreshRate, _telemetryRefreshRate, TimeUnit.SECONDS); } - public void stopScheduledTask() { + public void stopScheduledTask(long splitCount, long segmentCount, long segmentKeyCount) { try { - _telemetrySynchronizer.synchronizeStats(); + _telemetrySynchronizer.finalSynchronization(splitCount, segmentCount, segmentKeyCount); } catch (Exception e) { e.printStackTrace(); } diff --git a/client/src/main/java/io/split/telemetry/synchronizer/TelemetrySynchronizer.java b/client/src/main/java/io/split/telemetry/synchronizer/TelemetrySynchronizer.java index 859df44c8..7600a6334 100644 --- a/client/src/main/java/io/split/telemetry/synchronizer/TelemetrySynchronizer.java +++ b/client/src/main/java/io/split/telemetry/synchronizer/TelemetrySynchronizer.java @@ -8,4 +8,5 @@ public interface TelemetrySynchronizer { void synchronizeConfig(SplitClientConfig config, long timeUntilReady, Map factoryInstances, List tags); void synchronizeStats() throws Exception; + void finalSynchronization(long splitCount, long segmentCount, long segmentKeyCount) throws Exception; } diff --git a/client/src/test/java/io/split/telemetry/synchronizer/TelemetrySyncTaskTest.java b/client/src/test/java/io/split/telemetry/synchronizer/TelemetrySyncTaskTest.java index a7432c690..46339d71f 100644 --- a/client/src/test/java/io/split/telemetry/synchronizer/TelemetrySyncTaskTest.java +++ b/client/src/test/java/io/split/telemetry/synchronizer/TelemetrySyncTaskTest.java @@ -11,7 +11,7 @@ public void testSynchronizationTask() throws Exception { Mockito.doNothing().when(telemetrySynchronizer).synchronizeStats(); TelemetrySyncTask telemetrySyncTask = new TelemetrySyncTask(1, telemetrySynchronizer); Thread.sleep(2900); - Mockito.verify(telemetrySynchronizer, Mockito.times(3)).synchronizeStats(); + Mockito.verify(telemetrySynchronizer, Mockito.times(2)).synchronizeStats(); } @Test @@ -20,11 +20,11 @@ public void testStopSynchronizationTask() throws Exception { // Mockito.doNothing().when(telemetrySynchronizer).synchronizeStats(); TelemetrySyncTask telemetrySyncTask = new TelemetrySyncTask(1, telemetrySynchronizer); Thread.sleep(3000); - Mockito.verify(telemetrySynchronizer, Mockito.times(3)).synchronizeStats(); - telemetrySyncTask.stopScheduledTask(); + Mockito.verify(telemetrySynchronizer, Mockito.times(2)).synchronizeStats(); + telemetrySyncTask.stopScheduledTask(1l, 1l, 1l); Thread.sleep(2000); - Mockito.verify(telemetrySynchronizer, Mockito.times(4)).synchronizeStats(); - + Mockito.verify(telemetrySynchronizer, Mockito.times(2)).synchronizeStats(); + Mockito.verify(telemetrySynchronizer, Mockito.times(1)).finalSynchronization(1l, 1l, 1l); } } \ No newline at end of file From ea7c9b7cfa3d067ea12bde7d120498f7b93cf410 Mon Sep 17 00:00:00 2001 From: Lucas Echeverz Date: Fri, 28 May 2021 15:43:04 -0300 Subject: [PATCH 68/81] guava fix --- client/pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/client/pom.xml b/client/pom.xml index 1ece412a1..91771c389 100644 --- a/client/pom.xml +++ b/client/pom.xml @@ -121,7 +121,7 @@ com.google.guava guava - 29.0-jre + 30.0-jre org.slf4j From a62f0ac237849f34b528bfc68ce234c190177ded Mon Sep 17 00:00:00 2001 From: Lucas Echeverz Date: Fri, 28 May 2021 16:17:18 -0300 Subject: [PATCH 69/81] Fixing PR comments --- .../io/split/telemetry/synchronizer/TelemetrySyncTask.java | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/client/src/main/java/io/split/telemetry/synchronizer/TelemetrySyncTask.java b/client/src/main/java/io/split/telemetry/synchronizer/TelemetrySyncTask.java index 6cd4aa8ef..b766f7e05 100644 --- a/client/src/main/java/io/split/telemetry/synchronizer/TelemetrySyncTask.java +++ b/client/src/main/java/io/split/telemetry/synchronizer/TelemetrySyncTask.java @@ -2,6 +2,9 @@ import com.google.common.annotations.VisibleForTesting; import com.google.common.util.concurrent.ThreadFactoryBuilder; +import io.split.client.SplitManagerImpl; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; import java.util.concurrent.Executors; import java.util.concurrent.ScheduledExecutorService; @@ -10,6 +13,7 @@ public class TelemetrySyncTask { + private static final Logger _log = LoggerFactory.getLogger(TelemetrySyncTask.class); private final ScheduledExecutorService _telemetrySyncScheduledExecutorService; private final TelemetrySynchronizer _telemetrySynchronizer; private final int _telemetryRefreshRate; @@ -44,7 +48,7 @@ public void stopScheduledTask(long splitCount, long segmentCount, long segmentKe try { _telemetrySynchronizer.finalSynchronization(splitCount, segmentCount, segmentKeyCount); } catch (Exception e) { - e.printStackTrace(); + _log.warn("Error trying to send telemetry stats."); } _telemetrySyncScheduledExecutorService.shutdown(); } From 0a525ef3f119cdbd36c9e552c04b1361e4a649fc Mon Sep 17 00:00:00 2001 From: Lucas Echeverz Date: Fri, 28 May 2021 18:38:31 -0300 Subject: [PATCH 70/81] Merge with development --- .../io/split/client/SplitClientConfig.java | 6 +-- .../io/split/client/SplitFactoryImpl.java | 40 ++++++++----------- .../split/engine/common/SyncManagerImp.java | 21 +++++----- .../split/engine/common/SynchronizerImp.java | 15 ++----- .../engine/experiments/SplitFetcherImp.java | 2 +- .../engine/segments/SegmentFetcherImp.java | 2 +- .../client/HttpSegmentChangeFetcherTest.java | 2 +- .../client/HttpSplitChangeFetcherTest.java | 7 ++-- .../split/engine/common/SynchronizerTest.java | 19 +++++---- .../engine/experiments/SplitFetcherTest.java | 5 ++- .../segments/SegmentFetcherImpTest.java | 7 ++-- .../SegmentSynchronizationTaskImpTest.java | 10 ++--- 12 files changed, 63 insertions(+), 73 deletions(-) diff --git a/client/src/main/java/io/split/client/SplitClientConfig.java b/client/src/main/java/io/split/client/SplitClientConfig.java index 7c1eab91a..90845071c 100644 --- a/client/src/main/java/io/split/client/SplitClientConfig.java +++ b/client/src/main/java/io/split/client/SplitClientConfig.java @@ -852,12 +852,12 @@ public SplitClientConfig build() { _streamingReconnectBackoffBase, _authServiceURL, _streamingServiceURL, + _telemetryURl, + _telemetryRefreshRate, _onDemandFetchRetryDelayMs, _onDemandFetchMaxRetries, _failedAttemptsBeforeLogging, - _cdnDebugLogging, - _telemetryURl, - _telemetryRefreshRate); + _cdnDebugLogging); } } } diff --git a/client/src/main/java/io/split/client/SplitFactoryImpl.java b/client/src/main/java/io/split/client/SplitFactoryImpl.java index 38b40c4dd..2eacf06d8 100644 --- a/client/src/main/java/io/split/client/SplitFactoryImpl.java +++ b/client/src/main/java/io/split/client/SplitFactoryImpl.java @@ -145,28 +145,11 @@ public SplitFactoryImpl(String apiToken, SplitClientConfig config) throws URISyn _eventsRootTarget, config.eventsQueueSize(), config.eventFlushIntervalInMillis(), - config.waitBeforeShutdown()); + config.waitBeforeShutdown(), + _telemetryStorage); _telemetrySyncTask = new TelemetrySyncTask(config.get_telemetryRefreshRate(), _telemetrySynchronizer); - // SyncManager - _syncManager = SyncManagerImp.build(config.streamingEnabled(), - _splitSynchronizationTask, - _splitFetcher, - _segmentSynchronizationTaskImp, - _splitCache, - config.authServiceURL(), - _httpclient, - config.streamingServiceURL(), - config.authRetryBackoffBase(), - buildSSEdHttpClient(config), - _segmentCache, - config.streamingRetryDelay(), - config.streamingFetchMaxRetries(), - config.failedAttemptsBeforeLogging(), - config.cdnDebugLogging()); - _syncManager.start(); - // Evaluator _evaluator = new EvaluatorImp(_splitCache); @@ -174,7 +157,6 @@ public SplitFactoryImpl(String apiToken, SplitClientConfig config) throws URISyn _client = new SplitClientImpl(this, _splitCache, _impressionsManager, - _cachedFireAndForgetMetrics, _eventClient, config, _gates, @@ -186,9 +168,21 @@ public SplitFactoryImpl(String apiToken, SplitClientConfig config) throws URISyn _manager = new SplitManagerImpl(_splitCache, config, _gates, _telemetryStorage); // SyncManager - _syncManager = SyncManagerImp.build(config.streamingEnabled(), _splitSynchronizationTask, _splitFetcher, _segmentSynchronizationTaskImp, _splitCache, - config.authServiceURL(), _httpclient, config.streamingServiceURL(), config.authRetryBackoffBase(), buildSSEdHttpClient(apiToken, config), - _segmentCache, config.streamingRetryDelay(), _gates, _telemetryStorage, _telemetrySynchronizer,config); + _syncManager = SyncManagerImp.build(config.streamingEnabled(), + _splitSynchronizationTask, + _splitFetcher, + _segmentSynchronizationTaskImp, + _splitCache, + config.authServiceURL(), + _httpclient, + config.streamingServiceURL(), + config.authRetryBackoffBase(), + buildSSEdHttpClient(apiToken, config), + _segmentCache, + config.streamingRetryDelay(), + config.streamingFetchMaxRetries(), + config.failedAttemptsBeforeLogging(), + config.cdnDebugLogging(), _gates, _telemetryStorage, _telemetrySynchronizer,config); _syncManager.start(); // DestroyOnShutDown diff --git a/client/src/main/java/io/split/engine/common/SyncManagerImp.java b/client/src/main/java/io/split/engine/common/SyncManagerImp.java index de3768fce..2431f24f9 100644 --- a/client/src/main/java/io/split/engine/common/SyncManagerImp.java +++ b/client/src/main/java/io/split/engine/common/SyncManagerImp.java @@ -96,22 +96,23 @@ public static SyncManagerImp build(boolean streamingEnabledConfig, SplitClientConfig config) { LinkedBlockingQueue pushMessages = new LinkedBlockingQueue<>(); Synchronizer synchronizer = new SynchronizerImp(splitSynchronizationTask, - splitFetcher, - segmentSynchronizationTaskImp, - splitCache, - segmentCache, - streamingRetryDelay, - maxOnDemandFetchRetries, - failedAttemptsBeforeLogging, - cdnDebugLogging, - gates); + splitFetcher, + segmentSynchronizationTaskImp, + splitCache, + segmentCache, + streamingRetryDelay, + maxOnDemandFetchRetries, + failedAttemptsBeforeLogging, + cdnDebugLogging, + gates); PushManager pushManager = PushManagerImp.build(synchronizer, streamingServiceUrl, authUrl, httpClient, pushMessages, - sseHttpClient); + sseHttpClient, + telemetryRuntimeProducer); return new SyncManagerImp(streamingEnabledConfig, synchronizer, diff --git a/client/src/main/java/io/split/engine/common/SynchronizerImp.java b/client/src/main/java/io/split/engine/common/SynchronizerImp.java index e847f46ef..9214d284b 100644 --- a/client/src/main/java/io/split/engine/common/SynchronizerImp.java +++ b/client/src/main/java/io/split/engine/common/SynchronizerImp.java @@ -32,11 +32,9 @@ public class SynchronizerImp implements Synchronizer { private final SplitSynchronizationTask _splitSynchronizationTask; private final SplitFetcher _splitFetcher; private final SegmentSynchronizationTask _segmentSynchronizationTaskImp; - private final ScheduledExecutorService _syncAllScheduledExecutorService; private final SplitCache _splitCache; private final SegmentCache _segmentCache; private final int _onDemandFetchRetryDelayMs; - private final SDKReadinessGates _gates; private final int _onDemandFetchMaxRetries; private final int _failedAttemptsBeforeLogging; private final boolean _cdnResponseHeadersLogging; @@ -49,32 +47,25 @@ public SynchronizerImp(SplitSynchronizationTask splitSynchronizationTask, SplitCache splitCache, SegmentCache segmentCache, int onDemandFetchRetryDelayMs, - SDKReadinessGates gates, int onDemandFetchMaxRetries, int failedAttemptsBeforeLogging, - boolean cdnResponseHeadersLogging) { + boolean cdnResponseHeadersLogging, + SDKReadinessGates gates) { _splitSynchronizationTask = checkNotNull(splitSynchronizationTask); _splitFetcher = checkNotNull(splitFetcher); _segmentSynchronizationTaskImp = checkNotNull(segmentSynchronizationTaskImp); _splitCache = checkNotNull(splitCache); _segmentCache = checkNotNull(segmentCache); _onDemandFetchRetryDelayMs = checkNotNull(onDemandFetchRetryDelayMs); - _gates = checkNotNull(gates); - _onDemandFetchRetryDelayMs = onDemandFetchRetryDelayMs; _cdnResponseHeadersLogging = cdnResponseHeadersLogging; _onDemandFetchMaxRetries = onDemandFetchMaxRetries; _failedAttemptsBeforeLogging = failedAttemptsBeforeLogging; - ThreadFactory splitsThreadFactory = new ThreadFactoryBuilder() - .setDaemon(true) - .setNameFormat("Split-SyncAll-%d") - .build(); - _syncAllScheduledExecutorService = Executors.newSingleThreadScheduledExecutor(splitsThreadFactory); } @Override public boolean syncAll() { - return _splitFetcher.fetchAll(true) && _segmentSynchronizationTaskImp.fetchAllSynchronous(); + return _splitFetcher.fetchAll(new FetchOptions.Builder().cacheControlHeaders(true).build()) && _segmentSynchronizationTaskImp.fetchAllSynchronous(); } @Override diff --git a/client/src/main/java/io/split/engine/experiments/SplitFetcherImp.java b/client/src/main/java/io/split/engine/experiments/SplitFetcherImp.java index 4a61311da..37ec85c6f 100644 --- a/client/src/main/java/io/split/engine/experiments/SplitFetcherImp.java +++ b/client/src/main/java/io/split/engine/experiments/SplitFetcherImp.java @@ -153,7 +153,7 @@ private void runWithoutExceptionHandling(FetchOptions options) throws Interrupte } } @Override - public void fetchAll(FetchOptions options) { + public boolean fetchAll(FetchOptions options) { _log.debug("Fetch splits starting ..."); long start = _splitCache.getChangeNumber(); try { diff --git a/client/src/main/java/io/split/engine/segments/SegmentFetcherImp.java b/client/src/main/java/io/split/engine/segments/SegmentFetcherImp.java index 4e3aa0f11..13dfac4bb 100644 --- a/client/src/main/java/io/split/engine/segments/SegmentFetcherImp.java +++ b/client/src/main/java/io/split/engine/segments/SegmentFetcherImp.java @@ -136,7 +136,7 @@ void callLoopRun(FetchOptions opts){ @Override public boolean runWhitCacheHeader(){ - this.fetchAndUpdate(new FetchOptions.Builder().cacheControlHeaders(true).build()); + return this.fetchAndUpdate(new FetchOptions.Builder().cacheControlHeaders(true).build()); } /** diff --git a/client/src/test/java/io/split/client/HttpSegmentChangeFetcherTest.java b/client/src/test/java/io/split/client/HttpSegmentChangeFetcherTest.java index d754b793f..be1fb4b3c 100644 --- a/client/src/test/java/io/split/client/HttpSegmentChangeFetcherTest.java +++ b/client/src/test/java/io/split/client/HttpSegmentChangeFetcherTest.java @@ -97,7 +97,7 @@ public void testFetcherWithCDNBypassOption() throws IOException, URISyntaxExcept when(httpClientMock.execute(requestCaptor.capture())).thenReturn(TestHelper.classicResponseToCloseableMock(response)); Metrics.NoopMetrics metrics = new Metrics.NoopMetrics(); - HttpSegmentChangeFetcher fetcher = HttpSegmentChangeFetcher.create(httpClientMock, rootTarget, metrics); + HttpSegmentChangeFetcher fetcher = HttpSegmentChangeFetcher.create(httpClientMock, rootTarget, Mockito.mock(TelemetryStorage.class)); fetcher.fetch("someSegment", -1, new FetchOptions.Builder().targetChangeNumber(123).build()); fetcher.fetch("someSegment2",-1, new FetchOptions.Builder().build()); diff --git a/client/src/test/java/io/split/client/HttpSplitChangeFetcherTest.java b/client/src/test/java/io/split/client/HttpSplitChangeFetcherTest.java index 67c12351f..55090e7b3 100644 --- a/client/src/test/java/io/split/client/HttpSplitChangeFetcherTest.java +++ b/client/src/test/java/io/split/client/HttpSplitChangeFetcherTest.java @@ -6,6 +6,7 @@ import io.split.engine.common.FetchOptions; import io.split.engine.metrics.Metrics; import io.split.telemetry.storage.InMemoryTelemetryStorage; +import io.split.telemetry.storage.TelemetryRuntimeProducer; import io.split.telemetry.storage.TelemetryStorage; import org.apache.hc.client5.http.classic.methods.HttpUriRequestBase; import org.apache.hc.client5.http.impl.classic.CloseableHttpClient; @@ -108,8 +109,7 @@ public void testFetcherWithCDNBypassOption() throws IOException, URISyntaxExcept CloseableHttpClient httpClientMock = Mockito.mock(CloseableHttpClient.class); when(httpClientMock.execute(requestCaptor.capture())).thenReturn(TestHelper.classicResponseToCloseableMock(response)); - Metrics.NoopMetrics metrics = new Metrics.NoopMetrics(); - HttpSplitChangeFetcher fetcher = HttpSplitChangeFetcher.create(httpClientMock, rootTarget, metrics); + HttpSplitChangeFetcher fetcher = HttpSplitChangeFetcher.create(httpClientMock, rootTarget, Mockito.mock(TelemetryRuntimeProducer.class)); fetcher.fetch(-1, new FetchOptions.Builder().targetChangeNumber(123).build()); fetcher.fetch(-1, new FetchOptions.Builder().build()); @@ -123,8 +123,7 @@ public void testFetcherWithCDNBypassOption() throws IOException, URISyntaxExcept public void testRandomNumberGeneration() throws URISyntaxException { URI rootTarget = URI.create("https://api.split.io"); CloseableHttpClient httpClientMock = Mockito.mock(CloseableHttpClient.class); - Metrics.NoopMetrics metrics = new Metrics.NoopMetrics(); - HttpSplitChangeFetcher fetcher = HttpSplitChangeFetcher.create(httpClientMock, rootTarget, metrics); + HttpSplitChangeFetcher fetcher = HttpSplitChangeFetcher.create(httpClientMock, rootTarget, Mockito.mock(TelemetryRuntimeProducer.class)); Set seen = new HashSet<>(); long min = (long)Math.pow(2, 63) * (-1); diff --git a/client/src/test/java/io/split/engine/common/SynchronizerTest.java b/client/src/test/java/io/split/engine/common/SynchronizerTest.java index b512af31d..88bfdb903 100644 --- a/client/src/test/java/io/split/engine/common/SynchronizerTest.java +++ b/client/src/test/java/io/split/engine/common/SynchronizerTest.java @@ -39,18 +39,18 @@ public void beforeMethod() { _segmentCache = Mockito.mock(SegmentCache.class); _gates = Mockito.mock(SDKReadinessGates.class); - _synchronizer = new SynchronizerImp(_refreshableSplitFetcherTask, _splitFetcher, _segmentFetcher, _splitCache, _segmentCache, 50, 10, 5, false); + _synchronizer = new SynchronizerImp(_refreshableSplitFetcherTask, _splitFetcher, _segmentFetcher, _splitCache, _segmentCache, 50, 10, 5, false, _gates); } @Test public void syncAll() throws InterruptedException { - Mockito.when(_splitFetcher.fetchAll(true)).thenReturn(true); + Mockito.when(_splitFetcher.fetchAll(Mockito.anyObject())).thenReturn(true); Mockito.when(_segmentFetcher.fetchAllSynchronous()).thenReturn(true); _synchronizer.syncAll(); - Thread.sleep(100); - Mockito.verify(_splitFetcher, Mockito.times(1)).fetchAll(new FetchOptions.Builder().cacheControlHeaders(true).build()); - Mockito.verify(_segmentFetcher, Mockito.times(1)).fetchAll(true); + Thread.sleep(1000); + Mockito.verify(_splitFetcher, Mockito.times(1)).fetchAll(Mockito.anyObject()); + Mockito.verify(_segmentFetcher, Mockito.times(1)).fetchAllSynchronous(); } @Test @@ -99,7 +99,8 @@ public void testCDNBypassIsRequestedAfterNFailures() throws NoSuchFieldException 50, 3, 1, - true); + true, + Mockito.mock(SDKReadinessGates.class)); ArgumentCaptor optionsCaptor = ArgumentCaptor.forClass(FetchOptions.class); AtomicInteger calls = new AtomicInteger(); @@ -133,7 +134,8 @@ public void testCDNBypassRequestLimitAndBackoff() throws NoSuchFieldException, I 50, 3, 1, - true); + true, + Mockito.mock(SDKReadinessGates.class)); ArgumentCaptor optionsCaptor = ArgumentCaptor.forClass(FetchOptions.class); AtomicInteger calls = new AtomicInteger(); @@ -190,7 +192,8 @@ public void testCDNBypassRequestLimitAndForSegmentsBackoff() throws NoSuchFieldE 50, 3, 1, - true); + true, + Mockito.mock(SDKReadinessGates.class)); SegmentFetcher fetcher = Mockito.mock(SegmentFetcher.class); when(_segmentFetcher.getFetcher("someSegment")).thenReturn(fetcher); diff --git a/client/src/test/java/io/split/engine/experiments/SplitFetcherTest.java b/client/src/test/java/io/split/engine/experiments/SplitFetcherTest.java index ad0199a07..9999da832 100644 --- a/client/src/test/java/io/split/engine/experiments/SplitFetcherTest.java +++ b/client/src/test/java/io/split/engine/experiments/SplitFetcherTest.java @@ -16,6 +16,7 @@ import io.split.engine.segments.SegmentSynchronizationTaskImp; import io.split.grammar.Treatments; import io.split.telemetry.storage.InMemoryTelemetryStorage; +import io.split.telemetry.storage.TelemetryRuntimeProducer; import io.split.telemetry.storage.TelemetryStorage; import org.junit.Assert; import org.junit.Test; @@ -200,7 +201,7 @@ public void works_with_user_defined_segments() throws Exception { SegmentChangeFetcher segmentChangeFetcher = mock(SegmentChangeFetcher.class); SegmentChange segmentChange = getSegmentChange(0L, 0L, segmentName); when(segmentChangeFetcher.fetch(anyString(), anyLong(), any())).thenReturn(segmentChange); - SegmentSynchronizationTask segmentSynchronizationTask = new SegmentSynchronizationTaskImp(segmentChangeFetcher, 1,10, gates, segmentCache); + SegmentSynchronizationTask segmentSynchronizationTask = new SegmentSynchronizationTaskImp(segmentChangeFetcher, 1,10, gates, segmentCache, Mockito.mock(TelemetryStorage.class)); segmentSynchronizationTask.startPeriodicFetching(); SplitFetcherImp fetcher = new SplitFetcherImp(experimentChangeFetcher, new SplitParser(segmentSynchronizationTask, segmentCache), cache, TELEMETRY_STORAGE); @@ -225,7 +226,7 @@ public void testBypassCdnClearedAfterFirstHit() { SplitParser mockParser = new SplitParser(segmentSynchronizationTaskMock, segmentCacheMock); SDKReadinessGates mockGates = Mockito.mock(SDKReadinessGates.class); SplitCache mockCache = new InMemoryCacheImp(); - SplitFetcherImp fetcher = new SplitFetcherImp(mockFetcher, mockParser, mockGates, mockCache); + SplitFetcherImp fetcher = new SplitFetcherImp(mockFetcher, mockParser, mockCache, Mockito.mock(TelemetryRuntimeProducer.class)); SplitChange response1 = new SplitChange(); diff --git a/client/src/test/java/io/split/engine/segments/SegmentFetcherImpTest.java b/client/src/test/java/io/split/engine/segments/SegmentFetcherImpTest.java index 7108b3deb..0f0c799e0 100644 --- a/client/src/test/java/io/split/engine/segments/SegmentFetcherImpTest.java +++ b/client/src/test/java/io/split/engine/segments/SegmentFetcherImpTest.java @@ -9,6 +9,7 @@ import io.split.client.dtos.SplitChange; import io.split.engine.SDKReadinessGates; import io.split.telemetry.storage.InMemoryTelemetryStorage; +import io.split.telemetry.storage.TelemetryRuntimeProducer; import io.split.telemetry.storage.TelemetryStorage; import io.split.engine.common.FetchOptions; import io.split.engine.experiments.SplitChangeFetcher; @@ -104,7 +105,7 @@ private void works(long startingChangeNumber) throws InterruptedException { Mockito.when(segmentChangeFetcher.fetch(Mockito.eq(SEGMENT_NAME),Mockito.eq( -1L), Mockito.any())).thenReturn(segmentChange); Mockito.when(segmentChangeFetcher.fetch(Mockito.eq(SEGMENT_NAME),Mockito.eq( 0L), Mockito.any())).thenReturn(segmentChange); - SegmentFetcher fetcher = new SegmentFetcherImp(segmentName, segmentChangeFetcher, gates, segmentCache); + SegmentFetcher fetcher = new SegmentFetcherImp(segmentName, segmentChangeFetcher, gates, segmentCache, Mockito.mock(TelemetryRuntimeProducer.class)); // execute the fetcher for a little bit. ScheduledExecutorService scheduledExecutorService = Executors.newSingleThreadScheduledExecutor(); @@ -122,7 +123,7 @@ private void works(long startingChangeNumber) throws InterruptedException { // reset the interrupt. Thread.currentThread().interrupt(); } - Mockito.verify(segmentChangeFetcher, Mockito.times(2)).fetch(Mockito.anyString(), Mockito.anyLong(), Mockito.anyBoolean()); + Mockito.verify(segmentChangeFetcher, Mockito.times(2)).fetch(Mockito.anyString(), Mockito.anyLong(), Mockito.anyObject()); } @@ -153,7 +154,7 @@ public void testBypassCdnClearedAfterFirstHit() { SegmentSynchronizationTask segmentSynchronizationTaskMock = Mockito.mock(SegmentSynchronizationTask.class); SegmentCache segmentCacheMock = new SegmentCacheInMemoryImpl(); SDKReadinessGates mockGates = Mockito.mock(SDKReadinessGates.class); - SegmentFetcher fetcher = new SegmentFetcherImp("someSegment", mockFetcher, mockGates, segmentCacheMock); + SegmentFetcher fetcher = new SegmentFetcherImp("someSegment", mockFetcher, mockGates, segmentCacheMock, Mockito.mock(TelemetryRuntimeProducer.class)); SegmentChange response1 = new SegmentChange(); diff --git a/client/src/test/java/io/split/engine/segments/SegmentSynchronizationTaskImpTest.java b/client/src/test/java/io/split/engine/segments/SegmentSynchronizationTaskImpTest.java index 30c03c044..a4e38a129 100644 --- a/client/src/test/java/io/split/engine/segments/SegmentSynchronizationTaskImpTest.java +++ b/client/src/test/java/io/split/engine/segments/SegmentSynchronizationTaskImpTest.java @@ -94,10 +94,10 @@ public void testFetchAllAsynchronousAndGetFalse() throws NoSuchFieldException, I SegmentFetcherImp segmentFetcher = Mockito.mock(SegmentFetcherImp.class); _segmentFetchers.put("SF", segmentFetcher); final SegmentSynchronizationTaskImp fetchers = new SegmentSynchronizationTaskImp(segmentChangeFetcher, 1L, 1, gates, segmentCache, TELEMETRY_STORAGE); - Mockito.doNothing().when(segmentFetcher).callLoopRun(Mockito.anyBoolean(),Mockito.anyBoolean()); + Mockito.doNothing().when(segmentFetcher).callLoopRun(Mockito.anyObject()); Mockito.when(segmentFetcher.runWhitCacheHeader()).thenReturn(false); - Mockito.when(segmentFetcher.fetchAndUpdate(Mockito.anyBoolean())).thenReturn(false); - Mockito.doNothing().when(segmentFetcher).callLoopRun(Mockito.anyBoolean(),Mockito.anyBoolean()); + Mockito.when(segmentFetcher.fetchAndUpdate(Mockito.anyObject())).thenReturn(false); + Mockito.doNothing().when(segmentFetcher).callLoopRun(Mockito.anyObject()); // Before executing, we'll update the map of segmentFecthers via reflection. Field segmentFetchersForced = SegmentSynchronizationTaskImp.class.getDeclaredField("_segmentFetchers"); @@ -128,9 +128,9 @@ public void testFetchAllAsynchronousAndGetTrue() throws NoSuchFieldException, Il modifiersField.setAccessible(true); modifiersField.setInt(segmentFetchersForced, segmentFetchersForced.getModifiers() & ~Modifier.FINAL); segmentFetchersForced.set(fetchers, _segmentFetchers); - Mockito.doNothing().when(segmentFetcher).callLoopRun(Mockito.anyBoolean(),Mockito.anyBoolean()); + Mockito.doNothing().when(segmentFetcher).callLoopRun(Mockito.anyObject()); Mockito.when(segmentFetcher.runWhitCacheHeader()).thenReturn(true); - Mockito.when(segmentFetcher.fetchAndUpdate(Mockito.anyBoolean())).thenReturn(true); + Mockito.when(segmentFetcher.fetchAndUpdate(Mockito.anyObject())).thenReturn(true); boolean fetch = fetchers.fetchAllSynchronous(); Assert.assertEquals(true, fetch); } From 88f7cd35d52c04e8ea78f23aafd981fb17df33a7 Mon Sep 17 00:00:00 2001 From: Lucas Echeverz Date: Fri, 28 May 2021 18:50:41 -0300 Subject: [PATCH 71/81] Fix Travis --- .../io/split/telemetry/synchronizer/TelemetrySyncTaskTest.java | 1 - 1 file changed, 1 deletion(-) diff --git a/client/src/test/java/io/split/telemetry/synchronizer/TelemetrySyncTaskTest.java b/client/src/test/java/io/split/telemetry/synchronizer/TelemetrySyncTaskTest.java index 46339d71f..5cd350710 100644 --- a/client/src/test/java/io/split/telemetry/synchronizer/TelemetrySyncTaskTest.java +++ b/client/src/test/java/io/split/telemetry/synchronizer/TelemetrySyncTaskTest.java @@ -22,7 +22,6 @@ public void testStopSynchronizationTask() throws Exception { Thread.sleep(3000); Mockito.verify(telemetrySynchronizer, Mockito.times(2)).synchronizeStats(); telemetrySyncTask.stopScheduledTask(1l, 1l, 1l); - Thread.sleep(2000); Mockito.verify(telemetrySynchronizer, Mockito.times(2)).synchronizeStats(); Mockito.verify(telemetrySynchronizer, Mockito.times(1)).finalSynchronization(1l, 1l, 1l); } From 91cce5d3432c17fdcd521d1690b7fefcf12a20ad Mon Sep 17 00:00:00 2001 From: Lucas Echeverz Date: Fri, 28 May 2021 19:35:59 -0300 Subject: [PATCH 72/81] FIxing PR comments --- client/pom.xml | 2 +- .../src/main/java/io/split/client/EventClientImpl.java | 1 + .../split/client/impressions/ImpressionsManagerImpl.java | 9 ++++----- pom.xml | 2 +- testing/pom.xml | 2 +- 5 files changed, 8 insertions(+), 8 deletions(-) diff --git a/client/pom.xml b/client/pom.xml index 91771c389..15019f068 100644 --- a/client/pom.xml +++ b/client/pom.xml @@ -5,7 +5,7 @@ io.split.client java-client-parent - 4.2.0-rc1 + 4.2.0 java-client jar diff --git a/client/src/main/java/io/split/client/EventClientImpl.java b/client/src/main/java/io/split/client/EventClientImpl.java index 3644da688..eb4da7703 100644 --- a/client/src/main/java/io/split/client/EventClientImpl.java +++ b/client/src/main/java/io/split/client/EventClientImpl.java @@ -127,6 +127,7 @@ public boolean track(Event event, int eventSize) { _telemetryRuntimeProducer.recordEventStats(EventsDataRecordsEnum.EVENTS_QUEUED, 1); } else { + _log.warn("Event dropped."); _telemetryRuntimeProducer.recordEventStats(EventsDataRecordsEnum.EVENTS_DROPPED, 1); } diff --git a/client/src/main/java/io/split/client/impressions/ImpressionsManagerImpl.java b/client/src/main/java/io/split/client/impressions/ImpressionsManagerImpl.java index 708592f94..84bddca33 100644 --- a/client/src/main/java/io/split/client/impressions/ImpressionsManagerImpl.java +++ b/client/src/main/java/io/split/client/impressions/ImpressionsManagerImpl.java @@ -105,14 +105,13 @@ public void track(Impression impression) { } if (Mode.DEBUG.equals(_mode) || shouldQueueImpression(impression)) { - if (!shouldQueueImpression(impression)) { - _telemetryRuntimeProducer.recordImpressionStats(ImpressionsDataTypeEnum.IMPRESSIONS_DEDUPED, 1); - } if (_storage.put(KeyImpression.fromImpression(impression))) { _telemetryRuntimeProducer.recordImpressionStats(ImpressionsDataTypeEnum.IMPRESSIONS_QUEUED, 1); + } else { + _telemetryRuntimeProducer.recordImpressionStats(ImpressionsDataTypeEnum.IMPRESSIONS_DROPPED, 1); } - else { - _telemetryRuntimeProducer.recordImpressionStats(ImpressionsDataTypeEnum.IMPRESSIONS_DROPPED, 1); } + } else { + _telemetryRuntimeProducer.recordImpressionStats(ImpressionsDataTypeEnum.IMPRESSIONS_DEDUPED, 1); } } diff --git a/pom.xml b/pom.xml index 147baff25..3e2ba35d6 100644 --- a/pom.xml +++ b/pom.xml @@ -4,7 +4,7 @@ 4.0.0 io.split.client java-client-parent - 4.2.0-rc1 + 4.2.0 diff --git a/testing/pom.xml b/testing/pom.xml index a93c32b14..ea1a844de 100644 --- a/testing/pom.xml +++ b/testing/pom.xml @@ -6,7 +6,7 @@ io.split.client java-client-parent - 4.2.0-rc1 + 4.2.0 java-client-testing From f4bd0e57a1b875af631220a5ab8d14024f02891e Mon Sep 17 00:00:00 2001 From: Lucas Echeverz Date: Tue, 1 Jun 2021 14:40:11 -0300 Subject: [PATCH 73/81] Fixing PR Comments --- .../java/io/split/cache/SegmentCache.java | 4 +-- .../split/cache/SegmentCacheInMemoryImpl.java | 4 +-- .../client/HttpSegmentChangeFetcher.java | 2 +- .../split/client/HttpSplitChangeFetcher.java | 2 +- .../io/split/client/SplitClientConfig.java | 2 +- .../java/io/split/client/SplitClientImpl.java | 25 ++++++++----------- .../io/split/client/SplitFactoryImpl.java | 4 +-- .../AsynchronousImpressionListener.java | 13 +--------- .../impressions/HttpImpressionsSender.java | 2 +- .../impressions/ImpressionsManagerImpl.java | 16 ++++++------ .../ClientKeyInterceptorFilter.java | 6 ++++- .../split/engine/common/PushManagerImp.java | 2 +- .../split/engine/common/SyncManagerImp.java | 7 +++--- .../io/split/engine/segments/SegmentImp.java | 4 +-- .../engine/sse/PushStatusTrackerImp.java | 17 +++++-------- .../io/split/engine/sse/client/SSEClient.java | 12 ++++----- .../telemetry/domain/enums/MethodEnum.java | 20 +++++++++++---- .../domain/enums/StreamEventsEnum.java | 22 +++++++++++++++- .../HttpTelemetryMemorySender.java | 9 ++++--- .../synchronizer/TelemetrySubmitter.java | 4 +-- .../synchronizer/TelemetrySyncTask.java | 9 ++++--- .../telemetry/utils/AtomicLongArray.java | 3 ++- .../cache/SegmentCacheInMemoryImplTest.java | 2 +- .../ImpressionsManagerImplTest.java | 1 - .../split/engine/common/SyncManagerTest.java | 14 ++++++----- .../engine/sse/PushStatusTrackerTest.java | 2 +- 26 files changed, 112 insertions(+), 96 deletions(-) diff --git a/client/src/main/java/io/split/cache/SegmentCache.java b/client/src/main/java/io/split/cache/SegmentCache.java index c9af2ebb7..81349a5b9 100644 --- a/client/src/main/java/io/split/cache/SegmentCache.java +++ b/client/src/main/java/io/split/cache/SegmentCache.java @@ -53,8 +53,8 @@ public interface SegmentCache { List getAll(); /** - * return every key + * return key count * @return */ - Set getAllKeys(); + long getKeyCount(); } diff --git a/client/src/main/java/io/split/cache/SegmentCacheInMemoryImpl.java b/client/src/main/java/io/split/cache/SegmentCacheInMemoryImpl.java index 77a7e10d5..774c8d20f 100644 --- a/client/src/main/java/io/split/cache/SegmentCacheInMemoryImpl.java +++ b/client/src/main/java/io/split/cache/SegmentCacheInMemoryImpl.java @@ -68,7 +68,7 @@ public List getAll() { } @Override - public Set getAllKeys() { - return _segments.values().stream().flatMap(si -> si.getKeys().stream()).collect(Collectors.toSet()); + public long getKeyCount() { + return _segments.values().stream().mapToLong(SegmentImp::getKeysSize).sum(); } } diff --git a/client/src/main/java/io/split/client/HttpSegmentChangeFetcher.java b/client/src/main/java/io/split/client/HttpSegmentChangeFetcher.java index dd0cc3d4e..a9475320f 100644 --- a/client/src/main/java/io/split/client/HttpSegmentChangeFetcher.java +++ b/client/src/main/java/io/split/client/HttpSegmentChangeFetcher.java @@ -101,7 +101,6 @@ public SegmentChange fetch(String segmentName, long since, FetchOptions options) throw new IllegalStateException("Could not retrieve segment changes for " + segmentName + "; http return code " + statusCode); } - _telemetryRuntimeProducer.recordSyncLatency(HTTPLatenciesEnum.SEGMENTS, System.currentTimeMillis()-start); _telemetryRuntimeProducer.recordSuccessfulSync(LastSynchronizationRecordsEnum.SEGMENTS, System.currentTimeMillis()); String json = EntityUtils.toString(response.getEntity(), StandardCharsets.UTF_8); @@ -113,6 +112,7 @@ public SegmentChange fetch(String segmentName, long since, FetchOptions options) } catch (Throwable t) { throw new IllegalStateException("Problem fetching segmentChanges: " + t.getMessage(), t); } finally { + _telemetryRuntimeProducer.recordSyncLatency(HTTPLatenciesEnum.SEGMENTS, System.currentTimeMillis()-start); Utils.forceClose(response); } diff --git a/client/src/main/java/io/split/client/HttpSplitChangeFetcher.java b/client/src/main/java/io/split/client/HttpSplitChangeFetcher.java index 95b20d722..9d77654e5 100644 --- a/client/src/main/java/io/split/client/HttpSplitChangeFetcher.java +++ b/client/src/main/java/io/split/client/HttpSplitChangeFetcher.java @@ -98,7 +98,6 @@ public SplitChange fetch(long since, FetchOptions options) { _telemetryRuntimeProducer.recordSyncError(ResourceEnum.SPLIT_SYNC, statusCode); throw new IllegalStateException("Could not retrieve splitChanges; http return code " + statusCode); } - _telemetryRuntimeProducer.recordSyncLatency(HTTPLatenciesEnum.SPLITS, System.currentTimeMillis()-start); String json = EntityUtils.toString(response.getEntity(), StandardCharsets.UTF_8); @@ -110,6 +109,7 @@ public SplitChange fetch(long since, FetchOptions options) { } catch (Throwable t) { throw new IllegalStateException("Problem fetching splitChanges: " + t.getMessage(), t); } finally { + _telemetryRuntimeProducer.recordSyncLatency(HTTPLatenciesEnum.SPLITS, System.currentTimeMillis()-start); Utils.forceClose(response); } } diff --git a/client/src/main/java/io/split/client/SplitClientConfig.java b/client/src/main/java/io/split/client/SplitClientConfig.java index 90845071c..7b2ae0e5b 100644 --- a/client/src/main/java/io/split/client/SplitClientConfig.java +++ b/client/src/main/java/io/split/client/SplitClientConfig.java @@ -271,7 +271,7 @@ public String streamingServiceURL() { return _streamingServiceURL; } - public String get_telemetryURL() { + public String telemetryURL() { return _telemetryURL; } diff --git a/client/src/main/java/io/split/client/SplitClientImpl.java b/client/src/main/java/io/split/client/SplitClientImpl.java index 9c994030d..7427165ab 100644 --- a/client/src/main/java/io/split/client/SplitClientImpl.java +++ b/client/src/main/java/io/split/client/SplitClientImpl.java @@ -37,9 +37,6 @@ public final class SplitClientImpl implements SplitClient { public static final SplitResult SPLIT_RESULT_CONTROL = new SplitResult(Treatments.CONTROL, null); - private static final String GET_TREATMENT = "getTreatment"; - private static final String GET_TREATMENT_WITH_CONFIG = "getTreatmentWithConfig"; - private static final Logger _log = LoggerFactory.getLogger(SplitClientImpl.class); private final SplitFactory _container; @@ -79,27 +76,27 @@ public String getTreatment(String key, String split) { @Override public String getTreatment(String key, String split, Map attributes) { - return getTreatmentWithConfigInternal(GET_TREATMENT, key, null, split, attributes, MethodEnum.TREATMENT).treatment(); + return getTreatmentWithConfigInternal(key, null, split, attributes, MethodEnum.TREATMENT).treatment(); } @Override public String getTreatment(Key key, String split, Map attributes) { - return getTreatmentWithConfigInternal(GET_TREATMENT, key.matchingKey(), key.bucketingKey(), split, attributes, MethodEnum.TREATMENT).treatment(); + return getTreatmentWithConfigInternal(key.matchingKey(), key.bucketingKey(), split, attributes, MethodEnum.TREATMENT).treatment(); } @Override public SplitResult getTreatmentWithConfig(String key, String split) { - return getTreatmentWithConfigInternal(GET_TREATMENT_WITH_CONFIG, key, null, split, Collections.emptyMap(), MethodEnum.TREATMENT_WITH_CONFIG); + return getTreatmentWithConfigInternal(key, null, split, Collections.emptyMap(), MethodEnum.TREATMENT_WITH_CONFIG); } @Override public SplitResult getTreatmentWithConfig(String key, String split, Map attributes) { - return getTreatmentWithConfigInternal(GET_TREATMENT_WITH_CONFIG, key, null, split, attributes, MethodEnum.TREATMENT_WITH_CONFIG); + return getTreatmentWithConfigInternal(key, null, split, attributes, MethodEnum.TREATMENT_WITH_CONFIG); } @Override public SplitResult getTreatmentWithConfig(Key key, String split, Map attributes) { - return getTreatmentWithConfigInternal(GET_TREATMENT_WITH_CONFIG, key.matchingKey(), key.bucketingKey(), split, attributes, MethodEnum.TREATMENT_WITH_CONFIG); + return getTreatmentWithConfigInternal(key.matchingKey(), key.bucketingKey(), split, attributes, MethodEnum.TREATMENT_WITH_CONFIG); } @Override @@ -184,11 +181,11 @@ private boolean track(Event event) { return _eventClient.track(event, propertiesResult.getEventSize()); } - private SplitResult getTreatmentWithConfigInternal(String method, String matchingKey, String bucketingKey, String split, Map attributes, MethodEnum methodEnum) { + private SplitResult getTreatmentWithConfigInternal(String matchingKey, String bucketingKey, String split, Map attributes, MethodEnum methodEnum) { long initTime = System.currentTimeMillis(); try { if(!_gates.isSDKReady()){ - _log.warn(method + ": the SDK is not ready, results may be incorrect. Make sure to wait for SDK readiness before using this method"); + _log.warn(methodEnum.getMethod() + ": the SDK is not ready, results may be incorrect. Make sure to wait for SDK readiness before using this method"); _telemetryConfigProducer.recordNonReadyUsage(); } if (_container.isDestroyed()) { @@ -196,15 +193,15 @@ private SplitResult getTreatmentWithConfigInternal(String method, String matchin return SPLIT_RESULT_CONTROL; } - if (!KeyValidator.isValid(matchingKey, "matchingKey", _config.maxStringLength(), method)) { + if (!KeyValidator.isValid(matchingKey, "matchingKey", _config.maxStringLength(), methodEnum.getMethod())) { return SPLIT_RESULT_CONTROL; } - if (!KeyValidator.bucketingKeyIsValid(bucketingKey, _config.maxStringLength(), method)) { + if (!KeyValidator.bucketingKeyIsValid(bucketingKey, _config.maxStringLength(), methodEnum.getMethod())) { return SPLIT_RESULT_CONTROL; } - Optional splitNameResult = SplitNameValidator.isValid(split, method); + Optional splitNameResult = SplitNameValidator.isValid(split, methodEnum.getMethod()); if (!splitNameResult.isPresent()) { return SPLIT_RESULT_CONTROL; } @@ -226,7 +223,7 @@ private SplitResult getTreatmentWithConfigInternal(String method, String matchin split, start, result.treatment, - String.format("sdk.%s", method), + String.format("sdk.%s", methodEnum.getMethod()), _config.labelsEnabled() ? result.label : null, result.changeNumber, attributes diff --git a/client/src/main/java/io/split/client/SplitFactoryImpl.java b/client/src/main/java/io/split/client/SplitFactoryImpl.java index 2eacf06d8..1489bafec 100644 --- a/client/src/main/java/io/split/client/SplitFactoryImpl.java +++ b/client/src/main/java/io/split/client/SplitFactoryImpl.java @@ -123,7 +123,7 @@ public SplitFactoryImpl(String apiToken, SplitClientConfig config) throws URISyn // Cache Initialisations _segmentCache = new SegmentCacheInMemoryImpl(); _splitCache = new InMemoryCacheImp(); - _telemetrySynchronizer = new TelemetrySubmitter(_httpclient, URI.create(config.get_telemetryURL()), _telemetryStorage, _splitCache, _segmentCache, _telemetryStorage, _startTime); + _telemetrySynchronizer = new TelemetrySubmitter(_httpclient, URI.create(config.telemetryURL()), _telemetryStorage, _splitCache, _segmentCache, _telemetryStorage, _startTime); // Segments @@ -213,7 +213,7 @@ public synchronized void destroy() { try { long splitCount = _splitCache.getAll().stream().count(); long segmentCount = _segmentCache.getAll().stream().count(); - long segmentKeyCount = _segmentCache.getAllKeys().stream().count(); + long segmentKeyCount = _segmentCache.getKeyCount(); _impressionsManager.close(); _log.info("Successful shutdown of impressions manager"); _eventClient.close(); diff --git a/client/src/main/java/io/split/client/impressions/AsynchronousImpressionListener.java b/client/src/main/java/io/split/client/impressions/AsynchronousImpressionListener.java index ee9e97cb8..66f1d17d4 100644 --- a/client/src/main/java/io/split/client/impressions/AsynchronousImpressionListener.java +++ b/client/src/main/java/io/split/client/impressions/AsynchronousImpressionListener.java @@ -1,8 +1,6 @@ package io.split.client.impressions; import com.google.common.util.concurrent.ThreadFactoryBuilder; -import io.split.telemetry.domain.enums.LastSynchronizationRecordsEnum; -import io.split.telemetry.storage.TelemetryRuntimeProducer; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -12,8 +10,6 @@ import java.util.concurrent.ThreadPoolExecutor; import java.util.concurrent.TimeUnit; -import static com.google.common.base.Preconditions.checkNotNull; - /** * A wrapper around an ImpressionListener provided by the customer. The purpose * of the wrapper is to protect the SplitClient from any slow down happening due @@ -48,14 +44,7 @@ public AsynchronousImpressionListener(ImpressionListener delegate, ExecutorServi @Override public void log(final Impression impression) { try { - long initTime = System.currentTimeMillis(); - _executor.execute(new Runnable() { - @Override - public void run() { - _delegate.log(impression); - } - }); - long endTime = System.currentTimeMillis(); + _executor.execute(() -> _delegate.log(impression)); } catch (Exception e) { _log.warn("Unable to send impression to impression listener", e); diff --git a/client/src/main/java/io/split/client/impressions/HttpImpressionsSender.java b/client/src/main/java/io/split/client/impressions/HttpImpressionsSender.java index 8030723a5..017ef45a3 100644 --- a/client/src/main/java/io/split/client/impressions/HttpImpressionsSender.java +++ b/client/src/main/java/io/split/client/impressions/HttpImpressionsSender.java @@ -78,12 +78,12 @@ public void postImpressionsBulk(List impressions) { _telemetryRuntimeProducer.recordSyncError(ResourceEnum.IMPRESSION_SYNC, status); _logger.warn("Response status was: " + status); } - _telemetryRuntimeProducer.recordSyncLatency(HTTPLatenciesEnum.IMPRESSIONS, System.currentTimeMillis() - initTime); _telemetryRuntimeProducer.recordSuccessfulSync(LastSynchronizationRecordsEnum.IMPRESSIONS, System.currentTimeMillis()); } catch (Throwable t) { _logger.warn("Exception when posting impressions" + impressions, t); } finally { + _telemetryRuntimeProducer.recordSyncLatency(HTTPLatenciesEnum.IMPRESSIONS, System.currentTimeMillis() - initTime); Utils.forceClose(response); } diff --git a/client/src/main/java/io/split/client/impressions/ImpressionsManagerImpl.java b/client/src/main/java/io/split/client/impressions/ImpressionsManagerImpl.java index 84bddca33..e32455fdf 100644 --- a/client/src/main/java/io/split/client/impressions/ImpressionsManagerImpl.java +++ b/client/src/main/java/io/split/client/impressions/ImpressionsManagerImpl.java @@ -104,15 +104,15 @@ public void track(Impression impression) { _counter.inc(impression.split(), impression.time(), 1); } - if (Mode.DEBUG.equals(_mode) || shouldQueueImpression(impression)) { - if (_storage.put(KeyImpression.fromImpression(impression))) { - _telemetryRuntimeProducer.recordImpressionStats(ImpressionsDataTypeEnum.IMPRESSIONS_QUEUED, 1); - } else { - _telemetryRuntimeProducer.recordImpressionStats(ImpressionsDataTypeEnum.IMPRESSIONS_DROPPED, 1); - } - } else { + if (Mode.OPTIMIZED.equals(_mode) && !shouldQueueImpression(impression)) { _telemetryRuntimeProducer.recordImpressionStats(ImpressionsDataTypeEnum.IMPRESSIONS_DEDUPED, 1); + return; + } + if (!_storage.put(KeyImpression.fromImpression(impression))) { + _telemetryRuntimeProducer.recordImpressionStats(ImpressionsDataTypeEnum.IMPRESSIONS_DROPPED, 1); + return; } + _telemetryRuntimeProducer.recordImpressionStats(ImpressionsDataTypeEnum.IMPRESSIONS_QUEUED, 1); } @Override @@ -130,7 +130,7 @@ public void close() { } @VisibleForTesting - /* package private */ void sendImpressions() { + /* package private */ void sendImpressions() { if (_storage.isFull()) { _log.warn("Split SDK impressions queue is full. Impressions may have been dropped. Consider increasing capacity."); } diff --git a/client/src/main/java/io/split/client/interceptors/ClientKeyInterceptorFilter.java b/client/src/main/java/io/split/client/interceptors/ClientKeyInterceptorFilter.java index dc096539e..ad3c82594 100644 --- a/client/src/main/java/io/split/client/interceptors/ClientKeyInterceptorFilter.java +++ b/client/src/main/java/io/split/client/interceptors/ClientKeyInterceptorFilter.java @@ -14,7 +14,7 @@ public class ClientKeyInterceptorFilter implements HttpRequestInterceptor { private final String _clientKey; public static ClientKeyInterceptorFilter instance(String apiToken) { - return new ClientKeyInterceptorFilter(apiToken.substring(apiToken.length() - 4)); + return new ClientKeyInterceptorFilter(getKey(apiToken)); } private ClientKeyInterceptorFilter(String clientKey) { @@ -25,4 +25,8 @@ private ClientKeyInterceptorFilter(String clientKey) { public void process(HttpRequest httpRequest, EntityDetails entityDetails, HttpContext httpContext) throws HttpException, IOException { httpRequest.addHeader(CLIENT_KEY, _clientKey); } + + private static String getKey(String clientKey) { + return clientKey.length() >4 ? clientKey.substring(clientKey.length() - 4) : clientKey; + } } diff --git a/client/src/main/java/io/split/engine/common/PushManagerImp.java b/client/src/main/java/io/split/engine/common/PushManagerImp.java index 01d29a0dd..9585bc1a7 100644 --- a/client/src/main/java/io/split/engine/common/PushManagerImp.java +++ b/client/src/main/java/io/split/engine/common/PushManagerImp.java @@ -90,7 +90,7 @@ public synchronized void start() { _log.debug(String.format("Auth service response pushEnabled: %s", response.isPushEnabled())); if (response.isPushEnabled() && startSse(response.getToken(), response.getChannels())) { _expirationTime.set(response.getExpiration()); - _telemetryRuntimeProducer.recordStreamingEvents(new StreamingEvent(StreamEventsEnum.TOKEN_REFRESH.get_type(), response.getExpiration(), System.currentTimeMillis())); + _telemetryRuntimeProducer.recordStreamingEvents(new StreamingEvent(StreamEventsEnum.TOKEN_REFRESH.getType(), response.getExpiration(), System.currentTimeMillis())); return; } diff --git a/client/src/main/java/io/split/engine/common/SyncManagerImp.java b/client/src/main/java/io/split/engine/common/SyncManagerImp.java index 2431f24f9..cdbec26d9 100644 --- a/client/src/main/java/io/split/engine/common/SyncManagerImp.java +++ b/client/src/main/java/io/split/engine/common/SyncManagerImp.java @@ -29,8 +29,6 @@ public class SyncManagerImp implements SyncManager { private static final Logger _log = LoggerFactory.getLogger(SyncManager.class); - private static final int STREAMING_STREAMING_EVENT = 0; - private static final int POLLING_STREAMING_EVENT = 1; private final AtomicBoolean _streamingEnabledConfig; private final Synchronizer _synchronizer; @@ -159,13 +157,13 @@ private void startStreamingMode() { _pushStatusMonitorTask = _executorService.submit(this::incomingPushStatusHandler); } _pushManager.start(); - _telemetryRuntimeProducer.recordStreamingEvents(new StreamingEvent(StreamEventsEnum.SYNC_MODE_UPDATE.get_type(), STREAMING_STREAMING_EVENT, System.currentTimeMillis())); + _telemetryRuntimeProducer.recordStreamingEvents(new StreamingEvent(StreamEventsEnum.SYNC_MODE_UPDATE.getType(), StreamEventsEnum.StreamEventsValues.STREAMING_EVENT.getValue(), System.currentTimeMillis())); } private void startPollingMode() { _log.debug("Starting in polling mode ..."); _synchronizer.startPeriodicFetching(); - _telemetryRuntimeProducer.recordStreamingEvents(new StreamingEvent(StreamEventsEnum.SYNC_MODE_UPDATE.get_type(), POLLING_STREAMING_EVENT, System.currentTimeMillis())); + _telemetryRuntimeProducer.recordStreamingEvents(new StreamingEvent(StreamEventsEnum.SYNC_MODE_UPDATE.getType(), StreamEventsEnum.StreamEventsValues.POLLING_EVENT.getValue(), System.currentTimeMillis())); } @VisibleForTesting @@ -181,6 +179,7 @@ private void startPollingMode() { _pushManager.startWorkers(); _pushManager.scheduleConnectionReset(); _backoff.reset(); + _telemetryRuntimeProducer.recordStreamingEvents(new StreamingEvent(StreamEventsEnum.STREAMING_STATUS.getType(), StreamEventsEnum.StreamEventsValues.STREAMING_ENABLED.getValue(), System.currentTimeMillis())); _log.info("Streaming up and running."); break; case STREAMING_DOWN: diff --git a/client/src/main/java/io/split/engine/segments/SegmentImp.java b/client/src/main/java/io/split/engine/segments/SegmentImp.java index d10e3e384..d62cb1100 100644 --- a/client/src/main/java/io/split/engine/segments/SegmentImp.java +++ b/client/src/main/java/io/split/engine/segments/SegmentImp.java @@ -42,7 +42,7 @@ public boolean contains(String key) { return _concurrentKeySet.contains(key); } - public Set getKeys() { - return _concurrentKeySet; + public long getKeysSize() { + return _concurrentKeySet.size(); } } diff --git a/client/src/main/java/io/split/engine/sse/PushStatusTrackerImp.java b/client/src/main/java/io/split/engine/sse/PushStatusTrackerImp.java index c675d4574..60c519b60 100644 --- a/client/src/main/java/io/split/engine/sse/PushStatusTrackerImp.java +++ b/client/src/main/java/io/split/engine/sse/PushStatusTrackerImp.java @@ -13,7 +13,6 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import java.util.TimeZone; import java.util.concurrent.ConcurrentMap; import java.util.concurrent.LinkedBlockingQueue; import java.util.concurrent.atomic.AtomicBoolean; @@ -25,9 +24,6 @@ public class PushStatusTrackerImp implements PushStatusTracker { private static final Logger _log = LoggerFactory.getLogger(PushStatusTracker.class); private static final String CONTROL_PRI_CHANNEL = "control_pri"; private static final String CONTROL_SEC_CHANNEL = "control_sec"; - private static final int STREAMING_DISABLED = 0; - private static final int STREAMING_ENABLED = 1; - private static final int STREAMING_PAUSED = 2; private final AtomicBoolean _publishersOnline = new AtomicBoolean(true); private final AtomicReference _sseStatus = new AtomicReference<>(SSEClient.StatusMessage.INITIALIZATION_IN_PROGRESS); @@ -56,8 +52,7 @@ public void handleSseStatus(SSEClient.StatusMessage newStatus) { case FIRST_EVENT: if (SSEClient.StatusMessage.CONNECTED.equals(_sseStatus.get())) { _statusMessages.offer(PushManager.Status.STREAMING_READY); - _telemetryRuntimeProducer.recordStreamingEvents(new StreamingEvent(StreamEventsEnum.STREAMING_STATUS.get_type(), STREAMING_ENABLED, System.currentTimeMillis())); - _telemetryRuntimeProducer.recordStreamingEvents(new StreamingEvent(StreamEventsEnum.CONNECTION_ESTABLISHED.get_type(),0l, System.currentTimeMillis())); + _telemetryRuntimeProducer.recordStreamingEvents(new StreamingEvent(StreamEventsEnum.CONNECTION_ESTABLISHED.getType(),0l, System.currentTimeMillis())); } case CONNECTED: _sseStatus.compareAndSet(SSEClient.StatusMessage.INITIALIZATION_IN_PROGRESS, SSEClient.StatusMessage.CONNECTED); @@ -101,14 +96,14 @@ public void handleIncomingControlEvent(ControlNotification controlNotification) } break; case STREAMING_PAUSED: - _telemetryRuntimeProducer.recordStreamingEvents(new StreamingEvent(StreamEventsEnum.STREAMING_STATUS.get_type(), STREAMING_PAUSED, System.currentTimeMillis())); + _telemetryRuntimeProducer.recordStreamingEvents(new StreamingEvent(StreamEventsEnum.STREAMING_STATUS.getType(), StreamEventsEnum.StreamEventsValues.STREAMING_PAUSED.getValue(), System.currentTimeMillis())); if (_backendStatus.compareAndSet(ControlType.STREAMING_RESUMED, ControlType.STREAMING_PAUSED) && _publishersOnline.get()) { // If there are no publishers online, the STREAMING_DOWN message should have already been sent _statusMessages.offer(PushManager.Status.STREAMING_DOWN); } break; case STREAMING_DISABLED: - _telemetryRuntimeProducer.recordStreamingEvents(new StreamingEvent(StreamEventsEnum.STREAMING_STATUS.get_type(), STREAMING_DISABLED, System.currentTimeMillis())); + _telemetryRuntimeProducer.recordStreamingEvents(new StreamingEvent(StreamEventsEnum.STREAMING_STATUS.getType(), StreamEventsEnum.StreamEventsValues.STREAMING_DISABLED.getValue(), System.currentTimeMillis())); _backendStatus.set(ControlType.STREAMING_DISABLED); _statusMessages.offer(PushManager.Status.STREAMING_OFF); break; @@ -133,7 +128,7 @@ public void handleIncomingOccupancyEvent(OccupancyNotification occupancyNotifica @Override public void handleIncomingAblyError(ErrorNotification notification) { _log.debug(String.format("handleIncomingAblyError: %s", notification.getMessage())); - _telemetryRuntimeProducer.recordStreamingEvents(new StreamingEvent(StreamEventsEnum.ABLY_ERROR.get_type(), notification.getCode(), System.currentTimeMillis())); + _telemetryRuntimeProducer.recordStreamingEvents(new StreamingEvent(StreamEventsEnum.ABLY_ERROR.getType(), notification.getCode(), System.currentTimeMillis())); if (_backendStatus.get().equals(ControlType.STREAMING_DISABLED)) { return; // Ignore } @@ -167,10 +162,10 @@ private boolean isPublishers() { private void recordTelemetryOcuppancy(OccupancyNotification occupancyNotification, int publishers) { if (CONTROL_PRI_CHANNEL.equals(occupancyNotification.getChannel())) { - _telemetryRuntimeProducer.recordStreamingEvents(new StreamingEvent(StreamEventsEnum.OCCUPANCY_PRI.get_type(), publishers, System.currentTimeMillis())); + _telemetryRuntimeProducer.recordStreamingEvents(new StreamingEvent(StreamEventsEnum.OCCUPANCY_PRI.getType(), publishers, System.currentTimeMillis())); } else if (CONTROL_SEC_CHANNEL.equals(occupancyNotification.getChannel())){ - _telemetryRuntimeProducer.recordStreamingEvents(new StreamingEvent(StreamEventsEnum.OCCUPANCY_SEC.get_type(), publishers, System.currentTimeMillis())); + _telemetryRuntimeProducer.recordStreamingEvents(new StreamingEvent(StreamEventsEnum.OCCUPANCY_SEC.getType(), publishers, System.currentTimeMillis())); } } diff --git a/client/src/main/java/io/split/engine/sse/client/SSEClient.java b/client/src/main/java/io/split/engine/sse/client/SSEClient.java index cf5cb7649..62bf3d5ce 100644 --- a/client/src/main/java/io/split/engine/sse/client/SSEClient.java +++ b/client/src/main/java/io/split/engine/sse/client/SSEClient.java @@ -48,8 +48,6 @@ private enum ConnectionState { private final static String KEEP_ALIVE_PAYLOAD = ":keepalive\n"; private final static long CONNECT_TIMEOUT = 30000; private static final Logger _log = LoggerFactory.getLogger(SSEClient.class); - private final static int REQUESTED_CONNECTION_ERROR = 0; - private final static int NON_REQUESTED_CONNECTION_ERROR = 1; private final ExecutorService _connectionExecutor = Executors.newSingleThreadExecutor(new ThreadFactoryBuilder() .setDaemon(true) @@ -135,27 +133,27 @@ private void connectAndLoop(URI uri, CountDownLatch signal) { _log.debug(exc.getMessage()); if (SOCKET_CLOSED_MESSAGE.equals(exc.getMessage())) { // Connection closed by us _statusCallback.apply(StatusMessage.FORCED_STOP); - _telemetryRuntimeProducer.recordStreamingEvents(new StreamingEvent(StreamEventsEnum.SSE_CONNECTION_ERROR.get_type(), REQUESTED_CONNECTION_ERROR, System.currentTimeMillis())); + _telemetryRuntimeProducer.recordStreamingEvents(new StreamingEvent(StreamEventsEnum.SSE_CONNECTION_ERROR.getType(), StreamEventsEnum.StreamEventsValues.REQUESTED_CONNECTION_ERROR.getValue(), System.currentTimeMillis())); return; } // Connection closed by server _statusCallback.apply(StatusMessage.RETRYABLE_ERROR); - _telemetryRuntimeProducer.recordStreamingEvents(new StreamingEvent(StreamEventsEnum.SSE_CONNECTION_ERROR.get_type(), NON_REQUESTED_CONNECTION_ERROR, System.currentTimeMillis())); + _telemetryRuntimeProducer.recordStreamingEvents(new StreamingEvent(StreamEventsEnum.SSE_CONNECTION_ERROR.getType(), StreamEventsEnum.StreamEventsValues.NON_REQUESTED_CONNECTION_ERROR.getValue(), System.currentTimeMillis())); return; } catch (IOException exc) { // Other type of connection error if(!_forcedStop.get()) { _log.debug(String.format("SSE connection ended abruptly: %s. Retying", exc.getMessage())); - _telemetryRuntimeProducer.recordStreamingEvents(new StreamingEvent(StreamEventsEnum.SSE_CONNECTION_ERROR.get_type(), REQUESTED_CONNECTION_ERROR, System.currentTimeMillis())); + _telemetryRuntimeProducer.recordStreamingEvents(new StreamingEvent(StreamEventsEnum.SSE_CONNECTION_ERROR.getType(), StreamEventsEnum.StreamEventsValues.REQUESTED_CONNECTION_ERROR.getValue(), System.currentTimeMillis())); _statusCallback.apply(StatusMessage.RETRYABLE_ERROR); return; } - _telemetryRuntimeProducer.recordStreamingEvents(new StreamingEvent(StreamEventsEnum.SSE_CONNECTION_ERROR.get_type(), NON_REQUESTED_CONNECTION_ERROR, System.currentTimeMillis())); + _telemetryRuntimeProducer.recordStreamingEvents(new StreamingEvent(StreamEventsEnum.SSE_CONNECTION_ERROR.getType(), StreamEventsEnum.StreamEventsValues.NON_REQUESTED_CONNECTION_ERROR.getValue(), System.currentTimeMillis())); } } } catch (Exception e) { // Any other error non related to the connection disables streaming altogether - _telemetryRuntimeProducer.recordStreamingEvents(new StreamingEvent(StreamEventsEnum.SSE_CONNECTION_ERROR.get_type(), NON_REQUESTED_CONNECTION_ERROR, System.currentTimeMillis())); + _telemetryRuntimeProducer.recordStreamingEvents(new StreamingEvent(StreamEventsEnum.SSE_CONNECTION_ERROR.getType(), StreamEventsEnum.StreamEventsValues.NON_REQUESTED_CONNECTION_ERROR.getValue(), System.currentTimeMillis())); _log.warn(e.getMessage(), e); _statusCallback.apply(StatusMessage.NONRETRYABLE_ERROR); } finally { diff --git a/client/src/main/java/io/split/telemetry/domain/enums/MethodEnum.java b/client/src/main/java/io/split/telemetry/domain/enums/MethodEnum.java index 029d59477..8f99527f2 100644 --- a/client/src/main/java/io/split/telemetry/domain/enums/MethodEnum.java +++ b/client/src/main/java/io/split/telemetry/domain/enums/MethodEnum.java @@ -1,9 +1,19 @@ package io.split.telemetry.domain.enums; public enum MethodEnum { - TREATMENT, - TREATMENTS, - TREATMENT_WITH_CONFIG, - TREATMENTS_WITH_CONFIG, - TRACK, + TREATMENT("getTreatment"), + TREATMENTS("getTreatments"), + TREATMENT_WITH_CONFIG("getTreatmentWithConfig"), + TREATMENTS_WITH_CONFIG("getTreatmentsWithConfig"), + TRACK("track"); + + private String _method; + + MethodEnum(String method) { + _method = method; + } + + public String getMethod() { + return _method; + } } diff --git a/client/src/main/java/io/split/telemetry/domain/enums/StreamEventsEnum.java b/client/src/main/java/io/split/telemetry/domain/enums/StreamEventsEnum.java index 3947f5d7e..53103da5e 100644 --- a/client/src/main/java/io/split/telemetry/domain/enums/StreamEventsEnum.java +++ b/client/src/main/java/io/split/telemetry/domain/enums/StreamEventsEnum.java @@ -17,7 +17,27 @@ public enum StreamEventsEnum { _type = type; } - public int get_type() { + public int getType() { return _type; } + + public enum StreamEventsValues { + STREAMING_DISABLED(0), + STREAMING_PAUSED(2), + STREAMING_EVENT(0), + POLLING_EVENT(1), + REQUESTED_CONNECTION_ERROR(0), + NON_REQUESTED_CONNECTION_ERROR (1), + STREAMING_ENABLED(1); + + private long _value; + + StreamEventsValues(long value) { + _value = value; + } + + public long getValue() { + return _value; + } + } } diff --git a/client/src/main/java/io/split/telemetry/synchronizer/HttpTelemetryMemorySender.java b/client/src/main/java/io/split/telemetry/synchronizer/HttpTelemetryMemorySender.java index 37b05ed1a..48e5cbac1 100644 --- a/client/src/main/java/io/split/telemetry/synchronizer/HttpTelemetryMemorySender.java +++ b/client/src/main/java/io/split/telemetry/synchronizer/HttpTelemetryMemorySender.java @@ -14,7 +14,7 @@ import java.net.URI; import java.net.URISyntaxException; -public class HttpTelemetryMemorySender extends HttpPostImp { +public class HttpTelemetryMemorySender{ private static final String CONFIG_ENDPOINT_PATH = "metrics/config"; private static final String STATS_ENDPOINT_PATH = "metrics/usage"; @@ -23,6 +23,7 @@ public class HttpTelemetryMemorySender extends HttpPostImp { private final URI _impressionConfigTarget; private final URI _impressionStatsTarget; + private final HttpPostImp _httpPost; public static HttpTelemetryMemorySender create(CloseableHttpClient client, URI telemetryRootEndpoint, TelemetryRuntimeProducer telemetryRuntimeProducer) throws URISyntaxException { return new HttpTelemetryMemorySender(client, @@ -34,17 +35,17 @@ public static HttpTelemetryMemorySender create(CloseableHttpClient client, URI t @VisibleForTesting HttpTelemetryMemorySender(CloseableHttpClient client, URI impressionConfigTarget, URI impressionStatsTarget, TelemetryRuntimeProducer telemetryRuntimeProducer) { - super(client, telemetryRuntimeProducer); + _httpPost = new HttpPostImp(client, telemetryRuntimeProducer); _impressionConfigTarget = impressionConfigTarget; _impressionStatsTarget = impressionStatsTarget; } public void postConfig(Config config) { - post(_impressionConfigTarget, config, CONFIG_METRICS, HTTPLatenciesEnum.TELEMETRY, LastSynchronizationRecordsEnum.TELEMETRY, ResourceEnum.TELEMETRY_SYNC); + _httpPost.post(_impressionConfigTarget, config, CONFIG_METRICS, HTTPLatenciesEnum.TELEMETRY, LastSynchronizationRecordsEnum.TELEMETRY, ResourceEnum.TELEMETRY_SYNC); } public void postStats(Stats stats) { - post(_impressionStatsTarget, stats, STATS_METRICS, HTTPLatenciesEnum.TELEMETRY, LastSynchronizationRecordsEnum.TELEMETRY, ResourceEnum.TELEMETRY_SYNC); + _httpPost.post(_impressionStatsTarget, stats, STATS_METRICS, HTTPLatenciesEnum.TELEMETRY, LastSynchronizationRecordsEnum.TELEMETRY, ResourceEnum.TELEMETRY_SYNC); } } diff --git a/client/src/main/java/io/split/telemetry/synchronizer/TelemetrySubmitter.java b/client/src/main/java/io/split/telemetry/synchronizer/TelemetrySubmitter.java index b0b600a50..2be82aa9a 100644 --- a/client/src/main/java/io/split/telemetry/synchronizer/TelemetrySubmitter.java +++ b/client/src/main/java/io/split/telemetry/synchronizer/TelemetrySubmitter.java @@ -78,7 +78,7 @@ Stats generateStats() throws Exception { stats.set_impressionsDropped(_teleTelemetryStorageConsumer.getImpressionsStats(ImpressionsDataTypeEnum.IMPRESSIONS_DROPPED)); stats.set_splitCount(_splitCache.getAll().stream().count()); stats.set_segmentCount(_segmentCache.getAll().stream().count()); - stats.set_segmentKeyCount(_segmentCache.getAllKeys().stream().count()); + stats.set_segmentKeyCount(_segmentCache.getKeyCount()); stats.set_sessionLengthMs(_teleTelemetryStorageConsumer.getSessionLength()); stats.set_eventsQueued(_teleTelemetryStorageConsumer.getEventStats(EventsDataRecordsEnum.EVENTS_QUEUED)); stats.set_eventsDropped(_teleTelemetryStorageConsumer.getEventStats(EventsDataRecordsEnum.EVENTS_DROPPED)); @@ -109,7 +109,7 @@ Config generateConfig(SplitClientConfig splitClientConfig, long readyTimestamp, urlOverrides.set_stream(!SplitClientConfig.STREAMING_ENDPOINT.equals(splitClientConfig.streamingServiceURL())); urlOverrides.set_sdk(!SplitClientConfig.SDK_ENDPOINT.equals(splitClientConfig.endpoint())); urlOverrides.set_events(!SplitClientConfig.EVENTS_ENDPOINT.equals(splitClientConfig.eventsEndpoint())); - urlOverrides.set_telemetry(!SplitClientConfig.TELEMETRY_ENDPOINT.equals(splitClientConfig.get_telemetryURL())); + urlOverrides.set_telemetry(!SplitClientConfig.TELEMETRY_ENDPOINT.equals(splitClientConfig.telemetryURL())); config.set_burTimeouts(_teleTelemetryStorageConsumer.getBURTimeouts()); config.set_nonReadyUsages(_teleTelemetryStorageConsumer.getNonReadyUsages()); diff --git a/client/src/main/java/io/split/telemetry/synchronizer/TelemetrySyncTask.java b/client/src/main/java/io/split/telemetry/synchronizer/TelemetrySyncTask.java index b766f7e05..8ad64dc5f 100644 --- a/client/src/main/java/io/split/telemetry/synchronizer/TelemetrySyncTask.java +++ b/client/src/main/java/io/split/telemetry/synchronizer/TelemetrySyncTask.java @@ -2,7 +2,6 @@ import com.google.common.annotations.VisibleForTesting; import com.google.common.util.concurrent.ThreadFactoryBuilder; -import io.split.client.SplitManagerImpl; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -11,6 +10,8 @@ import java.util.concurrent.ThreadFactory; import java.util.concurrent.TimeUnit; +import static com.google.common.base.Preconditions.checkNotNull; + public class TelemetrySyncTask { private static final Logger _log = LoggerFactory.getLogger(TelemetrySyncTask.class); @@ -23,13 +24,13 @@ public TelemetrySyncTask(int telemetryRefreshRate, TelemetrySynchronizer telemet .setDaemon(true) .setNameFormat("Telemetry-sync-%d") .build(); - _telemetrySynchronizer = telemetrySynchronizer; + _telemetrySynchronizer = checkNotNull(telemetrySynchronizer); _telemetryRefreshRate = telemetryRefreshRate; _telemetrySyncScheduledExecutorService = Executors.newSingleThreadScheduledExecutor(telemetrySyncThreadFactory); try { this.startScheduledTask(); } catch (Exception e) { - e.printStackTrace(); + _log.warn("Error trying to init telemetry stats synchronizer task."); } } @@ -39,7 +40,7 @@ protected void startScheduledTask() { try { _telemetrySynchronizer.synchronizeStats(); } catch (Exception e) { - e.printStackTrace(); + _log.warn("Error sending telemetry stats."); } },_telemetryRefreshRate, _telemetryRefreshRate, TimeUnit.SECONDS); } diff --git a/client/src/main/java/io/split/telemetry/utils/AtomicLongArray.java b/client/src/main/java/io/split/telemetry/utils/AtomicLongArray.java index 8413c9b66..eaad8b6dc 100644 --- a/client/src/main/java/io/split/telemetry/utils/AtomicLongArray.java +++ b/client/src/main/java/io/split/telemetry/utils/AtomicLongArray.java @@ -25,7 +25,8 @@ public AtomicLongArray(int size) { public void increment(int index) { if (index < 0 || index >= array.length) { - throw new ArrayIndexOutOfBoundsException(); + _log.error("Index is out of bounds. Did not incremented."); + return; } array[index].getAndIncrement(); } diff --git a/client/src/test/java/io/split/cache/SegmentCacheInMemoryImplTest.java b/client/src/test/java/io/split/cache/SegmentCacheInMemoryImplTest.java index 76597abad..1e5fcd4c3 100644 --- a/client/src/test/java/io/split/cache/SegmentCacheInMemoryImplTest.java +++ b/client/src/test/java/io/split/cache/SegmentCacheInMemoryImplTest.java @@ -73,6 +73,6 @@ public void testGetAllKeys() { SegmentCacheInMemoryImpl segmentCacheInMemory = new SegmentCacheInMemoryImpl(); segmentCacheInMemory.updateSegment(SEGMENT_NAME,Stream.of("KEY1", "KEY2").collect(Collectors.toList()), new ArrayList<>()); segmentCacheInMemory.updateSegment(FAKE_SEGMENT_NAME,Stream.of("KEY3", "KEY2").collect(Collectors.toList()), new ArrayList<>()); - Assert.assertEquals(3, segmentCacheInMemory.getAllKeys().stream().count()); + Assert.assertEquals(4, segmentCacheInMemory.getKeyCount()); } } \ No newline at end of file diff --git a/client/src/test/java/io/split/client/impressions/ImpressionsManagerImplTest.java b/client/src/test/java/io/split/client/impressions/ImpressionsManagerImplTest.java index c00caa5d4..a0fa1a7a2 100644 --- a/client/src/test/java/io/split/client/impressions/ImpressionsManagerImplTest.java +++ b/client/src/test/java/io/split/client/impressions/ImpressionsManagerImplTest.java @@ -145,7 +145,6 @@ public void works4ImpressionsInOneTest() throws URISyntaxException { assertThat(captured.size(), is(equalTo(1))); assertThat(captured.get(0).keyImpressions.size(), is(equalTo(4))); assertThat(captured.get(0).keyImpressions.get(0), is(equalTo(ki1))); - Mockito.verify(TELEMETRY_STORAGE, times(2)).recordImpressionStats(ImpressionsDataTypeEnum.IMPRESSIONS_DEDUPED, 1); Mockito.verify(TELEMETRY_STORAGE, times(4)).recordImpressionStats(ImpressionsDataTypeEnum.IMPRESSIONS_QUEUED, 1); } diff --git a/client/src/test/java/io/split/engine/common/SyncManagerTest.java b/client/src/test/java/io/split/engine/common/SyncManagerTest.java index fafe0874c..466fe38a8 100644 --- a/client/src/test/java/io/split/engine/common/SyncManagerTest.java +++ b/client/src/test/java/io/split/engine/common/SyncManagerTest.java @@ -26,7 +26,7 @@ public void setUp() { @Test public void startWithStreamingFalseShouldStartPolling() throws InterruptedException { - TelemetryStorage telemetryStorage = new InMemoryTelemetryStorage(); + TelemetryStorage telemetryStorage = Mockito.mock(TelemetryStorage.class); _gates.sdkInternalReady(); TelemetrySynchronizer telemetrySynchronizer = Mockito.mock(TelemetrySynchronizer.class); SplitClientConfig config = Mockito.mock(SplitClientConfig.class); @@ -41,7 +41,7 @@ public void startWithStreamingFalseShouldStartPolling() throws InterruptedExcept @Test public void startWithStreamingTrueShouldStartSyncAll() throws InterruptedException { - TelemetryStorage telemetryStorage = new InMemoryTelemetryStorage(); + TelemetryStorage telemetryStorage = Mockito.mock(TelemetryStorage.class); TelemetrySynchronizer telemetrySynchronizer = Mockito.mock(TelemetrySynchronizer.class); SplitClientConfig config = Mockito.mock(SplitClientConfig.class); Mockito.when(_synchronizer.syncAll()).thenReturn(true); @@ -51,23 +51,25 @@ public void startWithStreamingTrueShouldStartSyncAll() throws InterruptedExcepti Mockito.verify(_synchronizer, Mockito.times(0)).startPeriodicFetching(); Mockito.verify(_synchronizer, Mockito.times(1)).syncAll(); Mockito.verify(_pushManager, Mockito.times(1)).start(); + Mockito.verify(telemetryStorage, Mockito.times(1)).recordStreamingEvents(Mockito.any()); } @Test public void onStreamingAvailable() throws InterruptedException { - TelemetryStorage telemetryStorage = new InMemoryTelemetryStorage(); - LinkedBlockingQueue messsages = new LinkedBlockingQueue<>(); + TelemetryStorage telemetryStorage = Mockito.mock(TelemetryStorage.class); + LinkedBlockingQueue messages = new LinkedBlockingQueue<>(); TelemetrySynchronizer telemetrySynchronizer = Mockito.mock(TelemetrySynchronizer.class); SplitClientConfig config = Mockito.mock(SplitClientConfig.class); - SyncManagerImp syncManager = new SyncManagerImp(true, _synchronizer, _pushManager, messsages, BACKOFF_BASE, _gates, telemetryStorage, telemetrySynchronizer, config); + SyncManagerImp syncManager = new SyncManagerImp(true, _synchronizer, _pushManager, messages, BACKOFF_BASE, _gates, telemetryStorage, telemetrySynchronizer, config); Thread t = new Thread(syncManager::incomingPushStatusHandler); t.start(); - messsages.offer(PushManager.Status.STREAMING_READY); + messages.offer(PushManager.Status.STREAMING_READY); Thread.sleep(500); Mockito.verify(_synchronizer, Mockito.times(1)).stopPeriodicFetching(); Mockito.verify(_synchronizer, Mockito.times(1)).syncAll(); Mockito.verify(_pushManager, Mockito.times(1)).startWorkers(); + Mockito.verify(telemetryStorage, Mockito.times(1)).recordStreamingEvents(Mockito.any()); t.interrupt(); } diff --git a/client/src/test/java/io/split/engine/sse/PushStatusTrackerTest.java b/client/src/test/java/io/split/engine/sse/PushStatusTrackerTest.java index c7ccec2fb..4e3f2dbf8 100644 --- a/client/src/test/java/io/split/engine/sse/PushStatusTrackerTest.java +++ b/client/src/test/java/io/split/engine/sse/PushStatusTrackerTest.java @@ -128,7 +128,7 @@ public void handleSSESTatusRecordTelemetryStreamingEvent() { pushStatusTracker.handleSseStatus(SSEClient.StatusMessage.CONNECTED); pushStatusTracker.handleSseStatus(SSEClient.StatusMessage.FIRST_EVENT); - Mockito.verify(telemetryStorage, Mockito.times(2)).recordStreamingEvents(Mockito.any()); + Mockito.verify(telemetryStorage, Mockito.times(1)).recordStreamingEvents(Mockito.any()); } @Test From 68e338486b18be49a25a33d46d1654d4e169f0e6 Mon Sep 17 00:00:00 2001 From: Lucas Echeverz Date: Tue, 1 Jun 2021 15:16:34 -0300 Subject: [PATCH 74/81] Fixing travis --- .../telemetry/utils/AtomicLongArrayTest.java | 15 +++++++++++++-- 1 file changed, 13 insertions(+), 2 deletions(-) diff --git a/client/src/test/java/io/split/telemetry/utils/AtomicLongArrayTest.java b/client/src/test/java/io/split/telemetry/utils/AtomicLongArrayTest.java index bbee9b79a..e5a8a38a0 100644 --- a/client/src/test/java/io/split/telemetry/utils/AtomicLongArrayTest.java +++ b/client/src/test/java/io/split/telemetry/utils/AtomicLongArrayTest.java @@ -5,6 +5,9 @@ import org.mockito.Mockito; import org.slf4j.Logger; +import java.lang.reflect.Field; +import java.lang.reflect.Modifier; + public class AtomicLongArrayTest { private static final int SIZE = 23; @@ -30,10 +33,18 @@ public void testIncrement() { Assert.assertEquals(1, atomicLongArray.fetchAndClearAll().stream().mapToInt(Long::intValue).sum()); } - @Test(expected = ArrayIndexOutOfBoundsException.class) - public void testIncrementError() { + @Test + public void testIncrementError() throws NoSuchFieldException, IllegalAccessException { + Logger log = Mockito.mock(Logger.class); AtomicLongArray atomicLongArray = new AtomicLongArray(SIZE); + Field logAssert = AtomicLongArray.class.getDeclaredField("_log"); + logAssert.setAccessible(true); + Field modifiersField = Field.class.getDeclaredField("modifiers"); + modifiersField.setAccessible(true); + modifiersField.setInt(logAssert, logAssert.getModifiers() & ~Modifier.FINAL); + logAssert.set(atomicLongArray, log); atomicLongArray.increment(25); + Mockito.verify(log, Mockito.times(1)).error(Mockito.anyString()); } } \ No newline at end of file From bc4207080c248c1e27e3a6d70e6aa2ed1f55c5d5 Mon Sep 17 00:00:00 2001 From: Lucas Echeverz Date: Wed, 2 Jun 2021 10:33:49 -0300 Subject: [PATCH 75/81] Fixing travis --- .../io/split/telemetry/synchronizer/TelemetrySyncTaskTest.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/client/src/test/java/io/split/telemetry/synchronizer/TelemetrySyncTaskTest.java b/client/src/test/java/io/split/telemetry/synchronizer/TelemetrySyncTaskTest.java index 5cd350710..055b0f048 100644 --- a/client/src/test/java/io/split/telemetry/synchronizer/TelemetrySyncTaskTest.java +++ b/client/src/test/java/io/split/telemetry/synchronizer/TelemetrySyncTaskTest.java @@ -19,7 +19,7 @@ public void testStopSynchronizationTask() throws Exception { TelemetrySynchronizer telemetrySynchronizer = Mockito.mock(TelemetrySubmitter.class); // Mockito.doNothing().when(telemetrySynchronizer).synchronizeStats(); TelemetrySyncTask telemetrySyncTask = new TelemetrySyncTask(1, telemetrySynchronizer); - Thread.sleep(3000); + Thread.sleep(2100); Mockito.verify(telemetrySynchronizer, Mockito.times(2)).synchronizeStats(); telemetrySyncTask.stopScheduledTask(1l, 1l, 1l); Mockito.verify(telemetrySynchronizer, Mockito.times(2)).synchronizeStats(); From c43f1f9ba7d3c822b927fc9bfc2d114bc56813f6 Mon Sep 17 00:00:00 2001 From: Lucas Echeverz Date: Thu, 3 Jun 2021 13:02:26 -0300 Subject: [PATCH 76/81] PR Comments --- .../split/engine/common/SyncManagerImp.java | 6 +-- .../engine/sse/PushStatusTrackerImp.java | 4 +- .../io/split/engine/sse/client/SSEClient.java | 10 ++--- .../domain/enums/StreamEventsEnum.java | 38 ++++++++++++++++--- 4 files changed, 42 insertions(+), 16 deletions(-) diff --git a/client/src/main/java/io/split/engine/common/SyncManagerImp.java b/client/src/main/java/io/split/engine/common/SyncManagerImp.java index cdbec26d9..024ac02d2 100644 --- a/client/src/main/java/io/split/engine/common/SyncManagerImp.java +++ b/client/src/main/java/io/split/engine/common/SyncManagerImp.java @@ -157,13 +157,13 @@ private void startStreamingMode() { _pushStatusMonitorTask = _executorService.submit(this::incomingPushStatusHandler); } _pushManager.start(); - _telemetryRuntimeProducer.recordStreamingEvents(new StreamingEvent(StreamEventsEnum.SYNC_MODE_UPDATE.getType(), StreamEventsEnum.StreamEventsValues.STREAMING_EVENT.getValue(), System.currentTimeMillis())); + _telemetryRuntimeProducer.recordStreamingEvents(new StreamingEvent(StreamEventsEnum.SYNC_MODE_UPDATE.getType(), StreamEventsEnum.SyncModeUpdateValues.STREAMING_EVENT.getValue(), System.currentTimeMillis())); } private void startPollingMode() { _log.debug("Starting in polling mode ..."); _synchronizer.startPeriodicFetching(); - _telemetryRuntimeProducer.recordStreamingEvents(new StreamingEvent(StreamEventsEnum.SYNC_MODE_UPDATE.getType(), StreamEventsEnum.StreamEventsValues.POLLING_EVENT.getValue(), System.currentTimeMillis())); + _telemetryRuntimeProducer.recordStreamingEvents(new StreamingEvent(StreamEventsEnum.SYNC_MODE_UPDATE.getType(), StreamEventsEnum.SyncModeUpdateValues.POLLING_EVENT.getValue(), System.currentTimeMillis())); } @VisibleForTesting @@ -179,7 +179,7 @@ private void startPollingMode() { _pushManager.startWorkers(); _pushManager.scheduleConnectionReset(); _backoff.reset(); - _telemetryRuntimeProducer.recordStreamingEvents(new StreamingEvent(StreamEventsEnum.STREAMING_STATUS.getType(), StreamEventsEnum.StreamEventsValues.STREAMING_ENABLED.getValue(), System.currentTimeMillis())); + _telemetryRuntimeProducer.recordStreamingEvents(new StreamingEvent(StreamEventsEnum.STREAMING_STATUS.getType(), StreamEventsEnum.StreamingStatusValues.STREAMING_ENABLED.getValue(), System.currentTimeMillis())); _log.info("Streaming up and running."); break; case STREAMING_DOWN: diff --git a/client/src/main/java/io/split/engine/sse/PushStatusTrackerImp.java b/client/src/main/java/io/split/engine/sse/PushStatusTrackerImp.java index 60c519b60..b7e04730f 100644 --- a/client/src/main/java/io/split/engine/sse/PushStatusTrackerImp.java +++ b/client/src/main/java/io/split/engine/sse/PushStatusTrackerImp.java @@ -96,14 +96,14 @@ public void handleIncomingControlEvent(ControlNotification controlNotification) } break; case STREAMING_PAUSED: - _telemetryRuntimeProducer.recordStreamingEvents(new StreamingEvent(StreamEventsEnum.STREAMING_STATUS.getType(), StreamEventsEnum.StreamEventsValues.STREAMING_PAUSED.getValue(), System.currentTimeMillis())); + _telemetryRuntimeProducer.recordStreamingEvents(new StreamingEvent(StreamEventsEnum.STREAMING_STATUS.getType(), StreamEventsEnum.StreamingStatusValues.STREAMING_PAUSED.getValue(), System.currentTimeMillis())); if (_backendStatus.compareAndSet(ControlType.STREAMING_RESUMED, ControlType.STREAMING_PAUSED) && _publishersOnline.get()) { // If there are no publishers online, the STREAMING_DOWN message should have already been sent _statusMessages.offer(PushManager.Status.STREAMING_DOWN); } break; case STREAMING_DISABLED: - _telemetryRuntimeProducer.recordStreamingEvents(new StreamingEvent(StreamEventsEnum.STREAMING_STATUS.getType(), StreamEventsEnum.StreamEventsValues.STREAMING_DISABLED.getValue(), System.currentTimeMillis())); + _telemetryRuntimeProducer.recordStreamingEvents(new StreamingEvent(StreamEventsEnum.STREAMING_STATUS.getType(), StreamEventsEnum.StreamingStatusValues.STREAMING_DISABLED.getValue(), System.currentTimeMillis())); _backendStatus.set(ControlType.STREAMING_DISABLED); _statusMessages.offer(PushManager.Status.STREAMING_OFF); break; diff --git a/client/src/main/java/io/split/engine/sse/client/SSEClient.java b/client/src/main/java/io/split/engine/sse/client/SSEClient.java index 62bf3d5ce..abb21fee5 100644 --- a/client/src/main/java/io/split/engine/sse/client/SSEClient.java +++ b/client/src/main/java/io/split/engine/sse/client/SSEClient.java @@ -133,27 +133,27 @@ private void connectAndLoop(URI uri, CountDownLatch signal) { _log.debug(exc.getMessage()); if (SOCKET_CLOSED_MESSAGE.equals(exc.getMessage())) { // Connection closed by us _statusCallback.apply(StatusMessage.FORCED_STOP); - _telemetryRuntimeProducer.recordStreamingEvents(new StreamingEvent(StreamEventsEnum.SSE_CONNECTION_ERROR.getType(), StreamEventsEnum.StreamEventsValues.REQUESTED_CONNECTION_ERROR.getValue(), System.currentTimeMillis())); + _telemetryRuntimeProducer.recordStreamingEvents(new StreamingEvent(StreamEventsEnum.SSE_CONNECTION_ERROR.getType(), StreamEventsEnum.SseConnectionErrorValues.REQUESTED_CONNECTION_ERROR.getValue(), System.currentTimeMillis())); return; } // Connection closed by server _statusCallback.apply(StatusMessage.RETRYABLE_ERROR); - _telemetryRuntimeProducer.recordStreamingEvents(new StreamingEvent(StreamEventsEnum.SSE_CONNECTION_ERROR.getType(), StreamEventsEnum.StreamEventsValues.NON_REQUESTED_CONNECTION_ERROR.getValue(), System.currentTimeMillis())); + _telemetryRuntimeProducer.recordStreamingEvents(new StreamingEvent(StreamEventsEnum.SSE_CONNECTION_ERROR.getType(), StreamEventsEnum.SseConnectionErrorValues.NON_REQUESTED_CONNECTION_ERROR.getValue(), System.currentTimeMillis())); return; } catch (IOException exc) { // Other type of connection error if(!_forcedStop.get()) { _log.debug(String.format("SSE connection ended abruptly: %s. Retying", exc.getMessage())); - _telemetryRuntimeProducer.recordStreamingEvents(new StreamingEvent(StreamEventsEnum.SSE_CONNECTION_ERROR.getType(), StreamEventsEnum.StreamEventsValues.REQUESTED_CONNECTION_ERROR.getValue(), System.currentTimeMillis())); + _telemetryRuntimeProducer.recordStreamingEvents(new StreamingEvent(StreamEventsEnum.SSE_CONNECTION_ERROR.getType(), StreamEventsEnum.SseConnectionErrorValues.REQUESTED_CONNECTION_ERROR.getValue(), System.currentTimeMillis())); _statusCallback.apply(StatusMessage.RETRYABLE_ERROR); return; } - _telemetryRuntimeProducer.recordStreamingEvents(new StreamingEvent(StreamEventsEnum.SSE_CONNECTION_ERROR.getType(), StreamEventsEnum.StreamEventsValues.NON_REQUESTED_CONNECTION_ERROR.getValue(), System.currentTimeMillis())); + _telemetryRuntimeProducer.recordStreamingEvents(new StreamingEvent(StreamEventsEnum.SSE_CONNECTION_ERROR.getType(), StreamEventsEnum.SseConnectionErrorValues.NON_REQUESTED_CONNECTION_ERROR.getValue(), System.currentTimeMillis())); } } } catch (Exception e) { // Any other error non related to the connection disables streaming altogether - _telemetryRuntimeProducer.recordStreamingEvents(new StreamingEvent(StreamEventsEnum.SSE_CONNECTION_ERROR.getType(), StreamEventsEnum.StreamEventsValues.NON_REQUESTED_CONNECTION_ERROR.getValue(), System.currentTimeMillis())); + _telemetryRuntimeProducer.recordStreamingEvents(new StreamingEvent(StreamEventsEnum.SSE_CONNECTION_ERROR.getType(), StreamEventsEnum.SseConnectionErrorValues.NON_REQUESTED_CONNECTION_ERROR.getValue(), System.currentTimeMillis())); _log.warn(e.getMessage(), e); _statusCallback.apply(StatusMessage.NONRETRYABLE_ERROR); } finally { diff --git a/client/src/main/java/io/split/telemetry/domain/enums/StreamEventsEnum.java b/client/src/main/java/io/split/telemetry/domain/enums/StreamEventsEnum.java index 53103da5e..30f977c6f 100644 --- a/client/src/main/java/io/split/telemetry/domain/enums/StreamEventsEnum.java +++ b/client/src/main/java/io/split/telemetry/domain/enums/StreamEventsEnum.java @@ -21,18 +21,44 @@ public int getType() { return _type; } - public enum StreamEventsValues { + public enum StreamingStatusValues { STREAMING_DISABLED(0), STREAMING_PAUSED(2), - STREAMING_EVENT(0), - POLLING_EVENT(1), - REQUESTED_CONNECTION_ERROR(0), - NON_REQUESTED_CONNECTION_ERROR (1), STREAMING_ENABLED(1); private long _value; - StreamEventsValues(long value) { + StreamingStatusValues(long value) { + _value = value; + } + + public long getValue() { + return _value; + } + } + + public enum SseConnectionErrorValues { + REQUESTED_CONNECTION_ERROR(0), + NON_REQUESTED_CONNECTION_ERROR (1); + + private long _value; + + SseConnectionErrorValues(long value) { + _value = value; + } + + public long getValue() { + return _value; + } + } + + public enum SyncModeUpdateValues { + STREAMING_EVENT(0), + POLLING_EVENT(1); + + private long _value; + + SyncModeUpdateValues(long value) { _value = value; } From d398696ff40d2057a6cd33c6f5f83ee2915c2d2c Mon Sep 17 00:00:00 2001 From: Lucas Echeverz Date: Fri, 4 Jun 2021 11:09:25 -0300 Subject: [PATCH 77/81] Removing conditions limit --- .../java/io/split/engine/experiments/SplitParser.java | 7 ------- .../io/split/engine/experiments/SplitParserTest.java | 9 +++++---- 2 files changed, 5 insertions(+), 11 deletions(-) diff --git a/client/src/main/java/io/split/engine/experiments/SplitParser.java b/client/src/main/java/io/split/engine/experiments/SplitParser.java index e58292092..a5af700ff 100644 --- a/client/src/main/java/io/split/engine/experiments/SplitParser.java +++ b/client/src/main/java/io/split/engine/experiments/SplitParser.java @@ -43,7 +43,6 @@ */ public final class SplitParser { - public static final int CONDITIONS_UPPER_LIMIT = 50; private static final Logger _log = LoggerFactory.getLogger(SplitParser.class); private final SegmentSynchronizationTask _segmentSynchronizationTask; @@ -69,12 +68,6 @@ private ParsedSplit parseWithoutExceptionHandling(Split split) { return null; } - if (split.conditions.size() > CONDITIONS_UPPER_LIMIT) { - _log.warn(String.format("Dropping Split name=%s due to large number of conditions(%d)", - split.name, split.conditions.size())); - return null; - } - List parsedConditionList = Lists.newArrayList(); for (Condition condition : split.conditions) { diff --git a/client/src/test/java/io/split/engine/experiments/SplitParserTest.java b/client/src/test/java/io/split/engine/experiments/SplitParserTest.java index 8d8e4c9cb..83518da0e 100644 --- a/client/src/test/java/io/split/engine/experiments/SplitParserTest.java +++ b/client/src/test/java/io/split/engine/experiments/SplitParserTest.java @@ -26,6 +26,7 @@ import io.split.grammar.Treatments; import io.split.telemetry.storage.InMemoryTelemetryStorage; import io.split.telemetry.storage.TelemetryStorage; +import org.junit.Assert; import org.junit.Test; import org.mockito.Mockito; @@ -38,7 +39,6 @@ import static org.hamcrest.Matchers.equalTo; import static org.hamcrest.Matchers.is; -import static org.hamcrest.Matchers.nullValue; import static org.junit.Assert.assertThat; /** @@ -50,6 +50,7 @@ public class SplitParserTest { public static final String EMPLOYEES = "employees"; public static final String SALES_PEOPLE = "salespeople"; + public static final int CONDITIONS_UPPER_LIMIT = 50; private static final TelemetryStorage TELEMETRY_STORAGE = Mockito.mock(InMemoryTelemetryStorage.class); @Test @@ -176,7 +177,7 @@ public void works_for_two_conditions() { } @Test - public void fails_for_long_conditions() { + public void success_for_long_conditions() { SDKReadinessGates gates = new SDKReadinessGates(); SegmentCache segmentCache = new SegmentCacheInMemoryImpl(); segmentCache.updateSegment(EMPLOYEES, Stream.of("adil", "pato", "trevor").collect(Collectors.toList()), new ArrayList<>()); @@ -193,14 +194,14 @@ public void fails_for_long_conditions() { List conditions = Lists.newArrayList(); List p1 = Lists.newArrayList(ConditionsTestUtil.partition("on", 100)); - for (int i = 0 ; i < SplitParser.CONDITIONS_UPPER_LIMIT+1 ; i++) { + for (int i = 0 ; i < CONDITIONS_UPPER_LIMIT+1 ; i++) { Condition c = ConditionsTestUtil.and(employeesMatcher, p1); conditions.add(c); } Split split = makeSplit("first.name", 123, conditions, 1); - assertThat(parser.parse(split), is(nullValue())); + Assert.assertNotNull(parser.parse(split)); } From e7c4b3ddeb41bc786f103c340c7e6e65dd273537 Mon Sep 17 00:00:00 2001 From: Lucas Echeverz Date: Mon, 7 Jun 2021 15:57:47 -0300 Subject: [PATCH 78/81] Changes log --- client/CHANGES.txt | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/client/CHANGES.txt b/client/CHANGES.txt index e02d816d8..07ee97aa7 100644 --- a/client/CHANGES.txt +++ b/client/CHANGES.txt @@ -1,10 +1,15 @@ CHANGES +4.2.0 (Jun 7, 2021) +- Update SDK telemetry storage, metrics and updater to be more effective and send less often. +- Improved the synchronization flow to be more reliable in the event of an edge case generating delay in cache purge propagation, keeping the SDK cache properly synced. +- Fixed issue where the SDK was validating no Split had over 50 conditions (legacy code). + 4.1.6 (Apr 15, 2021) --Updated log level and message in some messages. +- Updated log level and message in some messages. 4.1.5 (Apr 6, 2021) --Updated: Streaming retry fix. +- Updated: Streaming retry fix. 4.1.4 (Mar 19, 2021) - Updated: Internal cache structure refactor. From e264811bcfce15413b10cc9ba7be86dae5a9d4bc Mon Sep 17 00:00:00 2001 From: Lucas Echeverz Date: Mon, 7 Jun 2021 16:01:09 -0300 Subject: [PATCH 79/81] Fix PR comments --- client/CHANGES.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/client/CHANGES.txt b/client/CHANGES.txt index 07ee97aa7..8a8a9e9ce 100644 --- a/client/CHANGES.txt +++ b/client/CHANGES.txt @@ -1,7 +1,7 @@ CHANGES 4.2.0 (Jun 7, 2021) -- Update SDK telemetry storage, metrics and updater to be more effective and send less often. +- Updated SDK telemetry storage, metrics and updater to be more effective and send less often. - Improved the synchronization flow to be more reliable in the event of an edge case generating delay in cache purge propagation, keeping the SDK cache properly synced. - Fixed issue where the SDK was validating no Split had over 50 conditions (legacy code). From 70f26b5a2fbb40ae36ce2255798c8e8d4dd7d03e Mon Sep 17 00:00:00 2001 From: Lucas Echeverz Date: Mon, 7 Jun 2021 17:19:58 -0300 Subject: [PATCH 80/81] PR comment --- client/CHANGES.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/client/CHANGES.txt b/client/CHANGES.txt index 8a8a9e9ce..cb1fc2794 100644 --- a/client/CHANGES.txt +++ b/client/CHANGES.txt @@ -9,7 +9,7 @@ CHANGES - Updated log level and message in some messages. 4.1.5 (Apr 6, 2021) -- Updated: Streaming retry fix. +- Updated streaming logic to use limited fetch retry attempts 4.1.4 (Mar 19, 2021) - Updated: Internal cache structure refactor. From 5047220030d194cf2fa75f0b8a750405ee56e3da Mon Sep 17 00:00:00 2001 From: Lucas Echeverz Date: Mon, 7 Jun 2021 17:20:19 -0300 Subject: [PATCH 81/81] PR comment --- client/CHANGES.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/client/CHANGES.txt b/client/CHANGES.txt index cb1fc2794..2d413bf6e 100644 --- a/client/CHANGES.txt +++ b/client/CHANGES.txt @@ -9,7 +9,7 @@ CHANGES - Updated log level and message in some messages. 4.1.5 (Apr 6, 2021) -- Updated streaming logic to use limited fetch retry attempts +- Updated streaming logic to use limited fetch retry attempts. 4.1.4 (Mar 19, 2021) - Updated: Internal cache structure refactor.