From a574ef4b81d9912f2428f7fdabdc953c6349ccf9 Mon Sep 17 00:00:00 2001 From: Eli Bishop Date: Wed, 7 Mar 2018 12:53:58 -0800 Subject: [PATCH 01/67] initial commit of event summary logic, with naive synchronization --- build.gradle | 1 + .../com/launchdarkly/client/CustomEvent.java | 7 +- .../java/com/launchdarkly/client/Event.java | 9 +- .../com/launchdarkly/client/EventFactory.java | 41 ++++ .../launchdarkly/client/EventProcessor.java | 177 +++++++++++++++- .../launchdarkly/client/EventSummarizer.java | 199 ++++++++++++++++++ .../com/launchdarkly/client/FeatureFlag.java | 75 ++++--- .../client/FeatureFlagBuilder.java | 17 +- .../client/FeatureRequestEvent.java | 15 +- .../launchdarkly/client/IdentifyEvent.java | 4 + .../com/launchdarkly/client/LDClient.java | 31 +-- .../com/launchdarkly/client/LDConfig.java | 41 +++- .../client/EventProcessorTest.java | 196 +++++++++++++++++ .../client/EventSummarizerTest.java | 161 ++++++++++++++ .../launchdarkly/client/FeatureFlagTest.java | 26 +-- 15 files changed, 928 insertions(+), 72 deletions(-) create mode 100644 src/main/java/com/launchdarkly/client/EventFactory.java create mode 100644 src/main/java/com/launchdarkly/client/EventSummarizer.java create mode 100644 src/test/java/com/launchdarkly/client/EventProcessorTest.java create mode 100644 src/test/java/com/launchdarkly/client/EventSummarizerTest.java diff --git a/build.gradle b/build.gradle index daaec7c81..dda8588d9 100644 --- a/build.gradle +++ b/build.gradle @@ -41,6 +41,7 @@ libraries.unshaded = [ ] libraries.testCompile = [ + "com.squareup.okhttp3:mockwebserver:3.10.0", "org.easymock:easymock:3.4", "junit:junit:4.12" ] diff --git a/src/main/java/com/launchdarkly/client/CustomEvent.java b/src/main/java/com/launchdarkly/client/CustomEvent.java index 069c6a92d..ce71db6ae 100644 --- a/src/main/java/com/launchdarkly/client/CustomEvent.java +++ b/src/main/java/com/launchdarkly/client/CustomEvent.java @@ -3,10 +3,15 @@ import com.google.gson.JsonElement; class CustomEvent extends Event { - private final JsonElement data; + final JsonElement data; CustomEvent(String key, LDUser user, JsonElement data) { super("custom", key, user); this.data = data; } + + CustomEvent(long timestamp, String key, LDUser user, JsonElement data) { + super(timestamp, "custom", key, user); + this.data = data; + } } diff --git a/src/main/java/com/launchdarkly/client/Event.java b/src/main/java/com/launchdarkly/client/Event.java index f2608cbb9..345189d7e 100644 --- a/src/main/java/com/launchdarkly/client/Event.java +++ b/src/main/java/com/launchdarkly/client/Event.java @@ -2,7 +2,7 @@ class Event { - Long creationDate; + long creationDate; String key; String kind; LDUser user; @@ -13,4 +13,11 @@ class Event { this.kind = kind; this.user = user; } + + Event(long creationDate, String kind, String key, LDUser user) { + this.creationDate = creationDate; + this.key = key; + this.kind = kind; + this.user = user; + } } \ No newline at end of file diff --git a/src/main/java/com/launchdarkly/client/EventFactory.java b/src/main/java/com/launchdarkly/client/EventFactory.java new file mode 100644 index 000000000..360939457 --- /dev/null +++ b/src/main/java/com/launchdarkly/client/EventFactory.java @@ -0,0 +1,41 @@ +package com.launchdarkly.client; + +import com.google.gson.JsonElement; + +abstract class EventFactory { + public static final EventFactory DEFAULT = new DefaultEventFactory(); + + protected abstract long getTimestamp(); + + public FeatureRequestEvent newFeatureRequestEvent(FeatureFlag flag, LDUser user, FeatureFlag.VariationAndValue result, JsonElement defaultVal) { + return new FeatureRequestEvent(getTimestamp(), flag.getKey(), user, flag.getVersion(), + result == null ? null : result.getVariation(), result == null ? null : result.getValue(), + defaultVal, null, flag.isTrackEvents(), flag.getDebugEventsUntilDate()); + } + + public FeatureRequestEvent newUnknownFeatureRequestEvent(String key, LDUser user, JsonElement defaultValue) { + return new FeatureRequestEvent(getTimestamp(), key, user, null, null, defaultValue, defaultValue, null, false, null); + } + + public FeatureRequestEvent newPrerequisiteFeatureRequestEvent(FeatureFlag prereqFlag, LDUser user, FeatureFlag.VariationAndValue result, + FeatureFlag prereqOf) { + return new FeatureRequestEvent(getTimestamp(), prereqFlag.getKey(), user, prereqFlag.getVersion(), + result == null ? null : result.getVariation(), result == null ? null : result.getValue(), + null, prereqOf.getKey(), prereqFlag.isTrackEvents(), prereqFlag.getDebugEventsUntilDate()); + } + + public CustomEvent newCustomEvent(String key, LDUser user, JsonElement data) { + return new CustomEvent(getTimestamp(), key, user, data); + } + + public IdentifyEvent newIdentifyEvent(LDUser user) { + return new IdentifyEvent(getTimestamp(), user.getKeyAsString(), user); + } + + public static class DefaultEventFactory extends EventFactory { + @Override + protected long getTimestamp() { + return System.currentTimeMillis(); + } + } +} diff --git a/src/main/java/com/launchdarkly/client/EventProcessor.java b/src/main/java/com/launchdarkly/client/EventProcessor.java index e83a3df4c..fed292875 100644 --- a/src/main/java/com/launchdarkly/client/EventProcessor.java +++ b/src/main/java/com/launchdarkly/client/EventProcessor.java @@ -1,6 +1,9 @@ package com.launchdarkly.client; import com.google.common.util.concurrent.ThreadFactoryBuilder; +import com.google.gson.JsonElement; +import com.google.gson.annotations.SerializedName; + import okhttp3.MediaType; import okhttp3.Request; import okhttp3.RequestBody; @@ -15,34 +18,91 @@ import java.util.Random; import java.util.concurrent.*; import java.util.concurrent.atomic.AtomicBoolean; +import java.util.concurrent.locks.ReentrantLock; class EventProcessor implements Closeable { private final ScheduledExecutorService scheduler; private final Random random = new Random(); - private final BlockingQueue queue; + private final BlockingQueue queue; private final String sdkKey; private final LDConfig config; private final Consumer consumer; - + private final EventSummarizer summarizer; + private final ReentrantLock lock = new ReentrantLock(); + EventProcessor(String sdkKey, LDConfig config) { this.sdkKey = sdkKey; this.queue = new ArrayBlockingQueue<>(config.capacity); this.consumer = new Consumer(config); + this.summarizer = new EventSummarizer(config); this.config = config; ThreadFactory threadFactory = new ThreadFactoryBuilder() .setDaemon(true) .setNameFormat("LaunchDarkly-EventProcessor-%d") .build(); this.scheduler = Executors.newSingleThreadScheduledExecutor(threadFactory); + this.scheduler.scheduleAtFixedRate(consumer, 0, config.flushInterval, TimeUnit.SECONDS); + Runnable userKeysFlusher = new Runnable() { + public void run() { + lock.lock(); + try { + summarizer.resetUsers(); + } finally { + lock.unlock(); + } + } + }; + this.scheduler.scheduleAtFixedRate(userKeysFlusher, 0, config.userKeysFlushInterval, TimeUnit.SECONDS); } boolean sendEvent(Event e) { - if (config.samplingInterval > 0 && random.nextInt(config.samplingInterval) != 0) { - return true; + lock.lock(); + try { + if (!(e instanceof IdentifyEvent)) { + // identify events don't get deduplicated and always include the full user data + if (e.user != null && !summarizer.noticeUser(e.user)) { + IndexEventOutput ie = new IndexEventOutput(e.creationDate, e.user); + if (!queue.offer(ie)) { + return false; + } + } + } + + if (summarizer.summarizeEvent(e)) { + return true; + } + + if (config.samplingInterval > 0 && random.nextInt(config.samplingInterval) != 0) { + return true; + } + + EventOutput eventOutput = createEventOutput(e); + if (eventOutput == null) { + return false; + } else { + return queue.offer(eventOutput); + } + } finally { + lock.unlock(); + } + } + + private EventOutput createEventOutput(Event e) { + String userKey = e.user == null ? null : e.user.getKeyAsString(); + if (e instanceof FeatureRequestEvent) { + FeatureRequestEvent fe = (FeatureRequestEvent)e; + return new FeatureRequestEventOutput(fe.creationDate, fe.key, userKey, + fe.variation, fe.version, fe.value, fe.defaultVal, fe.prereqOf, + (!fe.trackEvents && fe.debugEventsUntilDate != null) ? Boolean.TRUE : null); + } else if (e instanceof IdentifyEvent) { + return new IdentifyEventOutput(e.creationDate, e.user); + } else if (e instanceof CustomEvent) { + CustomEvent ce = (CustomEvent)e; + return new CustomEventOutput(e.creationDate, ce.key, userKey, ce.data); + } else { + return null; } - - return queue.offer(e); } @Override @@ -71,14 +131,25 @@ public void run() { } public void flush() { - List events = new ArrayList<>(queue.size()); - queue.drainTo(events); + List events = new ArrayList<>(queue.size()); + EventSummarizer.SummaryOutput summary; + lock.lock(); + try { + queue.drainTo(events); + summary = summarizer.flush(); + } finally { + lock.unlock(); + } + if (!summary.counters.isEmpty()) { + SummaryEventOutput seo = new SummaryEventOutput(summary.startDate, summary.endDate, summary.counters); + events.add(seo); + } if (!events.isEmpty() && !shutdown.get()) { postEvents(events); } } - private void postEvents(List events) { + private void postEvents(List events) { String json = config.gson.toJson(events); logger.debug("Posting " + events.size() + " event(s) to " + config.eventsURI + " with payload: " + json); @@ -109,4 +180,92 @@ private void postEvents(List events) { } } } + + private static interface EventOutput { } + + @SuppressWarnings("unused") + private static class FeatureRequestEventOutput implements EventOutput { + private final String kind; + private final long creationDate; + private final String key; + private final String userKey; + private final Integer variation; + private final Integer version; + private final JsonElement value; + @SerializedName("default") private final JsonElement defaultVal; + private final String prereqOf; + private final Boolean debug; + + FeatureRequestEventOutput(long creationDate, String key, String userKey, Integer variation, + Integer version, JsonElement value, JsonElement defaultVal, String prereqOf, Boolean debug) { + this.kind = "feature"; + this.creationDate = creationDate; + this.key = key; + this.userKey = userKey; + this.variation = variation; + this.version = version; + this.value = value; + this.defaultVal = defaultVal; + this.prereqOf = prereqOf; + this.debug = debug; + } + } + + @SuppressWarnings("unused") + private static class IdentifyEventOutput implements EventOutput { + private final String kind; + private final long creationDate; + private final LDUser user; + + IdentifyEventOutput(long creationDate, LDUser user) { + this.kind = "identify"; + this.creationDate = creationDate; + this.user = user; + } + } + + @SuppressWarnings("unused") + private static class CustomEventOutput implements EventOutput { + private final String kind; + private final long creationDate; + private final String key; + private final String userKey; + private final JsonElement data; + + CustomEventOutput(long creationDate, String key, String userKey, JsonElement data) { + this.kind = "custom"; + this.creationDate = creationDate; + this.key = key; + this.userKey = userKey; + this.data = data; + } + } + + @SuppressWarnings("unused") + private static class IndexEventOutput implements EventOutput { + private final String kind; + private final long creationDate; + private final LDUser user; + + IndexEventOutput(long creationDate, LDUser user) { + this.kind = "index"; + this.creationDate = creationDate; + this.user = user; + } + } + + @SuppressWarnings("unused") + private static class SummaryEventOutput implements EventOutput { + private final String kind; + private final long startDate; + private final long endDate; + private final List counters; + + SummaryEventOutput(long startDate, long endDate, List counters) { + this.kind = "summary"; + this.startDate = startDate; + this.endDate = endDate; + this.counters = counters; + } + } } diff --git a/src/main/java/com/launchdarkly/client/EventSummarizer.java b/src/main/java/com/launchdarkly/client/EventSummarizer.java new file mode 100644 index 000000000..5da766e36 --- /dev/null +++ b/src/main/java/com/launchdarkly/client/EventSummarizer.java @@ -0,0 +1,199 @@ +package com.launchdarkly.client; + +import com.google.gson.JsonElement; + +import java.util.ArrayList; +import java.util.HashMap; +import java.util.HashSet; +import java.util.List; +import java.util.Map; +import java.util.Set; + +/** + * Manages the state of summarizable information for the EventProcessor, including the + * event counters and user deduplication. + */ +class EventSummarizer { + private Map counters; + private long startDate; + private long endDate; + private long lastKnownPastTime; + private Set userKeysSeen; + private int userKeysCapacity; + + EventSummarizer(LDConfig config) { + this.counters = new HashMap<>(); + this.startDate = 0; + this.endDate = 0; + this.userKeysSeen = new HashSet<>(); + this.userKeysCapacity = config.userKeysCapacity; + } + + /** + * Add to the set of users we've noticed, and return true if the user was already known to us. + * @param user a user + * @return true if we've already seen this user key + */ + boolean noticeUser(LDUser user) { + if (user == null || user.getKey() == null) { + return false; + } + String key = user.getKeyAsString(); + if (userKeysSeen.contains(key)) { + return true; + } + if (userKeysSeen.size() < userKeysCapacity) { + userKeysSeen.add(key); + } + return false; + } + + /** + * Reset the set of users we've seen. + */ + void resetUsers() { + userKeysSeen.clear(); + } + + /** + * Check whether this is a kind of event that we should summarize; if so, add it to our + * counters and return true. False means that the event should be sent individually. + * @param event an event + * @return true if we summarized the event + */ + boolean summarizeEvent(Event event) { + if (!(event instanceof FeatureRequestEvent)) { + return false; + } + FeatureRequestEvent fe = (FeatureRequestEvent)event; + if (fe.trackEvents) { + return false; + } + + if (fe.debugEventsUntilDate != null) { + // The "last known past time" comes from the last HTTP response we got from the server. + // In case the client's time is set wrong, at least we know that any expiration date + // earlier than that point is definitely in the past. + if (fe.debugEventsUntilDate > lastKnownPastTime && + fe.debugEventsUntilDate > System.currentTimeMillis()) { + return false; + } + } + + CounterKey key = new CounterKey(fe.key, (fe.variation == null) ? 0 : fe.variation.intValue(), + (fe.version == null) ? 0 : fe.version.intValue()); + + CounterValue value = counters.get(key); + if (value != null) { + value.increment(); + } else { + counters.put(key, new CounterValue(1, fe.value)); + } + + if (startDate == 0 || fe.creationDate < startDate) { + startDate = fe.creationDate; + } + if (fe.creationDate > endDate) { + endDate = fe.creationDate; + } + + return true; + } + + /** + * Marks the given timestamp (received from the server) as being in the past, in case the + * client-side time is unreliable. + * @param t a timestamp + */ + void setLastKnownPastTime(long t) { + if (lastKnownPastTime < t) { + lastKnownPastTime = t; + } + } + + SummaryOutput flush() { + List countersOut = new ArrayList<>(counters.size()); + for (Map.Entry entry: counters.entrySet()) { + CounterData c = new CounterData(entry.getKey().key, + entry.getValue().flagValue, + entry.getKey().version == 0 ? null : entry.getKey().version, + entry.getValue().count, + entry.getKey().version == 0 ? true : null); + countersOut.add(c); + } + counters.clear(); + + SummaryOutput ret = new SummaryOutput(startDate, endDate, countersOut); + startDate = 0; + endDate = 0; + return ret; + } + + private static class CounterKey { + private final String key; + private final int variation; + private final int version; + + CounterKey(String key, int variation, int version) { + this.key = key; + this.variation = variation; + this.version = version; + } + + @Override + public boolean equals(Object other) { + if (other instanceof CounterKey) { + CounterKey o = (CounterKey)other; + return o.key.equals(this.key) && o.variation == this.variation && o.version == this.version; + } + return false; + } + + @Override + public int hashCode() { + return key.hashCode() + (variation + (version * 31) * 31); + } + } + + private static class CounterValue { + private int count; + private JsonElement flagValue; + + CounterValue(int count, JsonElement flagValue) { + this.count = count; + this.flagValue = flagValue; + } + + void increment() { + count = count + 1; + } + } + + static class CounterData { + final String key; + final JsonElement value; + final Integer version; + final int count; + final Boolean unknown; + + private CounterData(String key, JsonElement value, Integer version, int count, Boolean unknown) { + this.key = key; + this.value = value; + this.version = version; + this.count = count; + this.unknown = unknown; + } + } + + static class SummaryOutput { + final long startDate; + final long endDate; + final List counters; + + private SummaryOutput(long startDate, long endDate, List counters) { + this.startDate = startDate; + this.endDate = endDate; + this.counters = counters; + } + } +} diff --git a/src/main/java/com/launchdarkly/client/FeatureFlag.java b/src/main/java/com/launchdarkly/client/FeatureFlag.java index a25aa3fe6..f0738efb4 100644 --- a/src/main/java/com/launchdarkly/client/FeatureFlag.java +++ b/src/main/java/com/launchdarkly/client/FeatureFlag.java @@ -28,6 +28,8 @@ class FeatureFlag implements VersionedData { private VariationOrRollout fallthrough; private Integer offVariation; //optional private List variations; + private boolean trackEvents; + private Long debugEventsUntilDate; private boolean deleted; static FeatureFlag fromJson(LDConfig config, String json) { @@ -41,7 +43,9 @@ static Map fromJsonMap(LDConfig config, String json) { // We need this so Gson doesn't complain in certain java environments that restrict unsafe allocation FeatureFlag() {} - FeatureFlag(String key, int version, boolean on, List prerequisites, String salt, List targets, List rules, VariationOrRollout fallthrough, Integer offVariation, List variations, boolean deleted) { + FeatureFlag(String key, int version, boolean on, List prerequisites, String salt, List targets, + List rules, VariationOrRollout fallthrough, Integer offVariation, List variations, + boolean trackEvents, Long debugEventsUntilDate, boolean deleted) { this.key = key; this.version = version; this.on = on; @@ -52,10 +56,12 @@ static Map fromJsonMap(LDConfig config, String json) { this.fallthrough = fallthrough; this.offVariation = offVariation; this.variations = variations; + this.trackEvents = trackEvents; + this.debugEventsUntilDate = debugEventsUntilDate; this.deleted = deleted; } - EvalResult evaluate(LDUser user, FeatureStore featureStore) throws EvaluationException { + EvalResult evaluate(LDUser user, FeatureStore featureStore, EventFactory eventFactory) throws EvaluationException { List prereqEvents = new ArrayList<>(); if (user == null || user.getKey() == null) { @@ -64,45 +70,40 @@ EvalResult evaluate(LDUser user, FeatureStore featureStore) throws EvaluationExc } if (isOn()) { - JsonElement value = evaluate(user, featureStore, prereqEvents); - if (value != null) { - return new EvalResult(value, prereqEvents); + VariationAndValue result = evaluate(user, featureStore, prereqEvents, eventFactory); + if (result != null) { + return new EvalResult(result, prereqEvents); } } - JsonElement offVariation = getOffVariationValue(); - return new EvalResult(offVariation, prereqEvents); + return new EvalResult(new VariationAndValue(offVariation, getOffVariationValue()), prereqEvents); } // Returning either a JsonElement or null indicating prereq failure/error. - private JsonElement evaluate(LDUser user, FeatureStore featureStore, List events) throws EvaluationException { + private VariationAndValue evaluate(LDUser user, FeatureStore featureStore, List events, + EventFactory eventFactory) throws EvaluationException { boolean prereqOk = true; if (prerequisites != null) { for (Prerequisite prereq : prerequisites) { FeatureFlag prereqFeatureFlag = featureStore.get(FEATURES, prereq.getKey()); - JsonElement prereqEvalResult = null; + VariationAndValue prereqEvalResult = null; if (prereqFeatureFlag == null) { logger.error("Could not retrieve prerequisite flag: " + prereq.getKey() + " when evaluating: " + key); return null; } else if (prereqFeatureFlag.isOn()) { - prereqEvalResult = prereqFeatureFlag.evaluate(user, featureStore, events); - try { - JsonElement variation = prereqFeatureFlag.getVariation(prereq.getVariation()); - if (prereqEvalResult == null || variation == null || !prereqEvalResult.equals(variation)) { - prereqOk = false; - } - } catch (EvaluationException err) { - logger.warn("Error evaluating prerequisites: " + err.getMessage()); + prereqEvalResult = prereqFeatureFlag.evaluate(user, featureStore, events, eventFactory); + if (prereqEvalResult == null || prereqEvalResult.getVariation() != prereq.getVariation()) { prereqOk = false; } } else { prereqOk = false; } //We don't short circuit and also send events for each prereq. - events.add(new FeatureRequestEvent(prereqFeatureFlag.getKey(), user, prereqEvalResult, null, prereqFeatureFlag.getVersion(), key)); + events.add(eventFactory.newPrerequisiteFeatureRequestEvent(prereqFeatureFlag, user, prereqEvalResult, this)); } } if (prereqOk) { - return getVariation(evaluateIndex(user, featureStore)); + Integer index = evaluateIndex(user, featureStore); + return new VariationAndValue(index, getVariation(index)); } return null; } @@ -166,6 +167,14 @@ public String getKey() { return key; } + public boolean isTrackEvents() { + return trackEvents; + } + + public Long getDebugEventsUntilDate() { + return debugEventsUntilDate; + } + public boolean isDeleted() { return deleted; } @@ -200,18 +209,36 @@ List getVariations() { Integer getOffVariation() { return offVariation; } - static class EvalResult { + static class VariationAndValue { + private final Integer variation; private final JsonElement value; - private final List prerequisiteEvents; - private EvalResult(JsonElement value, List prerequisiteEvents) { + VariationAndValue(Integer variation, JsonElement value) { + this.variation = variation; this.value = value; - this.prerequisiteEvents = prerequisiteEvents; } - + + Integer getVariation() { + return variation; + } + JsonElement getValue() { return value; } + } + + static class EvalResult { + private final VariationAndValue result; + private final List prerequisiteEvents; + + private EvalResult(VariationAndValue result, List prerequisiteEvents) { + this.result = result; + this.prerequisiteEvents = prerequisiteEvents; + } + + VariationAndValue getResult() { + return result; + } List getPrerequisiteEvents() { return prerequisiteEvents; diff --git a/src/main/java/com/launchdarkly/client/FeatureFlagBuilder.java b/src/main/java/com/launchdarkly/client/FeatureFlagBuilder.java index 120764862..e3f2ee941 100644 --- a/src/main/java/com/launchdarkly/client/FeatureFlagBuilder.java +++ b/src/main/java/com/launchdarkly/client/FeatureFlagBuilder.java @@ -16,6 +16,8 @@ class FeatureFlagBuilder { private VariationOrRollout fallthrough; private Integer offVariation; private List variations = new ArrayList<>(); + private boolean trackEvents; + private Long debugEventsUntilDate; private boolean deleted; FeatureFlagBuilder(String key) { @@ -34,6 +36,8 @@ class FeatureFlagBuilder { this.fallthrough = f.getFallthrough(); this.offVariation = f.getOffVariation(); this.variations = f.getVariations(); + this.trackEvents = f.isTrackEvents(); + this.debugEventsUntilDate = f.getDebugEventsUntilDate(); this.deleted = f.isDeleted(); } } @@ -84,12 +88,23 @@ FeatureFlagBuilder variations(List variations) { return this; } + FeatureFlagBuilder trackEvents(boolean trackEvents) { + this.trackEvents = trackEvents; + return this; + } + + FeatureFlagBuilder debugEventsUntilDate(Long debugEventsUntilDate) { + this.debugEventsUntilDate = debugEventsUntilDate; + return this; + } + FeatureFlagBuilder deleted(boolean deleted) { this.deleted = deleted; return this; } FeatureFlag build() { - return new FeatureFlag(key, version, on, prerequisites, salt, targets, rules, fallthrough, offVariation, variations, deleted); + return new FeatureFlag(key, version, on, prerequisites, salt, targets, rules, fallthrough, offVariation, variations, + trackEvents, debugEventsUntilDate, deleted); } } \ No newline at end of file diff --git a/src/main/java/com/launchdarkly/client/FeatureRequestEvent.java b/src/main/java/com/launchdarkly/client/FeatureRequestEvent.java index d62e8c146..d563acd00 100644 --- a/src/main/java/com/launchdarkly/client/FeatureRequestEvent.java +++ b/src/main/java/com/launchdarkly/client/FeatureRequestEvent.java @@ -4,6 +4,8 @@ import com.google.gson.annotations.SerializedName; class FeatureRequestEvent extends Event { + Integer variation; + JsonElement value; @SerializedName("default") JsonElement defaultVal; @@ -14,11 +16,18 @@ class FeatureRequestEvent extends Event { @SerializedName("prereqOf") String prereqOf; - FeatureRequestEvent(String key, LDUser user, JsonElement value, JsonElement defaultVal, Integer version, String prereqOf) { - super("feature", key, user); + boolean trackEvents; + Long debugEventsUntilDate; + + FeatureRequestEvent(long timestamp, String key, LDUser user, Integer version, Integer variation, JsonElement value, + JsonElement defaultVal, String prereqOf, boolean trackEvents, Long debugEventsUntilDate) { + super(timestamp, "feature", key, user); + this.version = version; + this.variation = variation; this.value = value; this.defaultVal = defaultVal; - this.version = version; this.prereqOf = prereqOf; + this.trackEvents = trackEvents; + this.debugEventsUntilDate = debugEventsUntilDate; } } diff --git a/src/main/java/com/launchdarkly/client/IdentifyEvent.java b/src/main/java/com/launchdarkly/client/IdentifyEvent.java index 6ccfa70b5..aa040e5c0 100644 --- a/src/main/java/com/launchdarkly/client/IdentifyEvent.java +++ b/src/main/java/com/launchdarkly/client/IdentifyEvent.java @@ -5,4 +5,8 @@ class IdentifyEvent extends Event { IdentifyEvent(LDUser user) { super("identify", user.getKeyAsString(), user); } + + IdentifyEvent(long timestamp, String key, LDUser user) { + super(timestamp, "identify", key, user); + } } diff --git a/src/main/java/com/launchdarkly/client/LDClient.java b/src/main/java/com/launchdarkly/client/LDClient.java index 2562a1f37..87a7b017c 100644 --- a/src/main/java/com/launchdarkly/client/LDClient.java +++ b/src/main/java/com/launchdarkly/client/LDClient.java @@ -40,6 +40,7 @@ public class LDClient implements LDClientInterface { private final String sdkKey; private final FeatureRequestor requestor; private final EventProcessor eventProcessor; + private final EventFactory eventFactory = EventFactory.DEFAULT; private UpdateProcessor updateProcessor; private final AtomicBoolean eventCapacityExceeded = new AtomicBoolean(false); @@ -132,7 +133,7 @@ public void track(String eventName, LDUser user, JsonElement data) { if (user == null || user.getKey() == null) { logger.warn("Track called with null user or null user key!"); } - sendEvent(new CustomEvent(eventName, user, data)); + sendEvent(eventFactory.newCustomEvent(eventName, user, data)); } @Override @@ -148,12 +149,12 @@ public void identify(LDUser user) { if (user == null || user.getKey() == null) { logger.warn("Identify called with null user or null user key!"); } - sendEvent(new IdentifyEvent(user)); + sendEvent(eventFactory.newIdentifyEvent(user)); } - private void sendFlagRequestEvent(String featureKey, LDUser user, JsonElement value, JsonElement defaultValue, Integer version) { - if (sendEvent(new FeatureRequestEvent(featureKey, user, value, defaultValue, version, null))) { - NewRelicReflector.annotateTransaction(featureKey, String.valueOf(value)); + private void sendFlagRequestEvent(FeatureRequestEvent event) { + if (sendEvent(event)) { + NewRelicReflector.annotateTransaction(event.key, String.valueOf(event.value)); } } @@ -196,7 +197,7 @@ public Map allFlags(LDUser user) { for (Map.Entry entry : flags.entrySet()) { try { - JsonElement evalResult = entry.getValue().evaluate(user, config.featureStore).getValue(); + JsonElement evalResult = entry.getValue().evaluate(user, config.featureStore, eventFactory).getResult().getValue(); result.put(entry.getKey(), evalResult); } catch (EvaluationException e) { @@ -261,7 +262,7 @@ public boolean isFlagKnown(String featureKey) { private JsonElement evaluate(String featureKey, LDUser user, JsonElement defaultValue, VariationType expectedType) { if (user == null || user.getKey() == null) { logger.warn("Null user or null user key when evaluating flag: " + featureKey + "; returning default value"); - sendFlagRequestEvent(featureKey, user, defaultValue, defaultValue, null); + sendFlagRequestEvent(eventFactory.newUnknownFeatureRequestEvent(featureKey, user, defaultValue)); return defaultValue; } if (user.getKeyAsString().isEmpty()) { @@ -272,7 +273,7 @@ private JsonElement evaluate(String featureKey, LDUser user, JsonElement default logger.warn("Evaluation called before client initialized for feature flag " + featureKey + "; using last known values from feature store"); } else { logger.warn("Evaluation called before client initialized for feature flag " + featureKey + "; feature store unavailable, returning default value"); - sendFlagRequestEvent(featureKey, user, defaultValue, defaultValue, null); + sendFlagRequestEvent(eventFactory.newUnknownFeatureRequestEvent(featureKey, user, defaultValue)); return defaultValue; } } @@ -281,22 +282,22 @@ private JsonElement evaluate(String featureKey, LDUser user, JsonElement default FeatureFlag featureFlag = config.featureStore.get(FEATURES, featureKey); if (featureFlag == null) { logger.info("Unknown feature flag " + featureKey + "; returning default value"); - sendFlagRequestEvent(featureKey, user, defaultValue, defaultValue, null); + sendFlagRequestEvent(eventFactory.newUnknownFeatureRequestEvent(featureKey, user, defaultValue)); return defaultValue; } - FeatureFlag.EvalResult evalResult = featureFlag.evaluate(user, config.featureStore); + FeatureFlag.EvalResult evalResult = featureFlag.evaluate(user, config.featureStore, eventFactory); for (FeatureRequestEvent event : evalResult.getPrerequisiteEvents()) { sendEvent(event); } - if (evalResult.getValue() != null) { - expectedType.assertResultType(evalResult.getValue()); - sendFlagRequestEvent(featureKey, user, evalResult.getValue(), defaultValue, featureFlag.getVersion()); - return evalResult.getValue(); + if (evalResult.getResult() != null && evalResult.getResult().getValue() != null) { + expectedType.assertResultType(evalResult.getResult().getValue()); + sendFlagRequestEvent(eventFactory.newFeatureRequestEvent(featureFlag, user, evalResult.getResult(), defaultValue)); + return evalResult.getResult().getValue(); } } catch (Exception e) { logger.error("Encountered exception in LaunchDarkly client", e); } - sendFlagRequestEvent(featureKey, user, defaultValue, defaultValue, null); + sendFlagRequestEvent(eventFactory.newUnknownFeatureRequestEvent(featureKey, user, defaultValue)); return defaultValue; } diff --git a/src/main/java/com/launchdarkly/client/LDConfig.java b/src/main/java/com/launchdarkly/client/LDConfig.java index 9703a0701..6b42579b0 100644 --- a/src/main/java/com/launchdarkly/client/LDConfig.java +++ b/src/main/java/com/launchdarkly/client/LDConfig.java @@ -41,7 +41,8 @@ public final class LDConfig { private static final long MIN_POLLING_INTERVAL_MILLIS = 30000L; private static final long DEFAULT_START_WAIT_MILLIS = 5000L; private static final int DEFAULT_SAMPLING_INTERVAL = 0; - + private static final int DEFAULT_USER_KEYS_CAPACITY = 1000; + private static final int DEFAULT_USER_KEYS_FLUSH_INTERVAL_SECONDS = 60 * 5; private static final long DEFAULT_RECONNECT_TIME_MILLIS = 1000; private static final long MAX_HTTP_CACHE_SIZE_BYTES = 10 * 1024 * 1024; // 10 MB @@ -68,7 +69,9 @@ public final class LDConfig { final long startWaitMillis; final int samplingInterval; final long reconnectTimeMs; - + final int userKeysCapacity; + final int userKeysFlushInterval; + protected LDConfig(Builder builder) { this.baseURI = builder.baseURI; this.eventsURI = builder.eventsURI; @@ -94,9 +97,9 @@ protected LDConfig(Builder builder) { this.startWaitMillis = builder.startWaitMillis; this.samplingInterval = builder.samplingInterval; this.reconnectTimeMs = builder.reconnectTimeMillis; - - - + this.userKeysCapacity = builder.userKeysCapacity; + this.userKeysFlushInterval = builder.userKeysFlushInterval; + OkHttpClient.Builder httpClientBuilder = new OkHttpClient.Builder() .connectionPool(new ConnectionPool(5, 5, TimeUnit.SECONDS)) .connectTimeout(connectTimeoutMillis, TimeUnit.MILLISECONDS) @@ -166,7 +169,9 @@ public static class Builder { private int samplingInterval = DEFAULT_SAMPLING_INTERVAL; private long reconnectTimeMillis = DEFAULT_RECONNECT_TIME_MILLIS; private Set privateAttrNames = new HashSet<>(); - + private int userKeysCapacity = DEFAULT_USER_KEYS_CAPACITY; + private int userKeysFlushInterval = DEFAULT_USER_KEYS_FLUSH_INTERVAL_SECONDS; + /** * Creates a builder with all configuration parameters set to the default */ @@ -469,6 +474,30 @@ public Builder privateAttributeNames(String... names) { return this; } + /** + * Sets the number of user keys that the event processor can remember at any one time, so that + * duplicate user details will not be sent in analytics events. + * + * @param capacity the maximum number of user keys to remember + * @return the builder + */ + public Builder userKeysCapacity(int capacity) { + this.userKeysCapacity = capacity; + return this; + } + + /** + * Set the interval in seconds at which the event processor will reset its set of known user keys. The + * default value is five minutes. + * + * @param flushInterval the flush interval in seconds + * @return the builder + */ + public Builder userKeysFlushInterval(int flushInterval) { + this.flushIntervalSeconds = flushInterval; + return this; + } + // returns null if none of the proxy bits were configured. Minimum required part: port. Proxy proxy() { if (this.proxyPort == -1) { diff --git a/src/test/java/com/launchdarkly/client/EventProcessorTest.java b/src/test/java/com/launchdarkly/client/EventProcessorTest.java new file mode 100644 index 000000000..fc2b025c2 --- /dev/null +++ b/src/test/java/com/launchdarkly/client/EventProcessorTest.java @@ -0,0 +1,196 @@ +package com.launchdarkly.client; + +import com.google.gson.Gson; +import com.google.gson.JsonArray; +import com.google.gson.JsonElement; +import com.google.gson.JsonObject; +import com.google.gson.JsonPrimitive; +import com.google.gson.reflect.TypeToken; + +import org.junit.After; +import org.junit.Before; +import org.junit.Test; + +import java.lang.reflect.Type; +import java.util.List; +import java.util.Map; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertNull; + +import okhttp3.mockwebserver.MockResponse; +import okhttp3.mockwebserver.MockWebServer; +import okhttp3.mockwebserver.RecordedRequest; + +public class EventProcessorTest { + private static final String SDK_KEY = "SDK_KEY"; + private static final LDUser user = new LDUser.Builder("userkey").name("Red").build(); + private static final Gson gson = new Gson(); + private static final Type listOfMapsType = new TypeToken>>() { + }.getType(); + + private final LDConfig.Builder configBuilder = new LDConfig.Builder(); + private final MockWebServer server = new MockWebServer(); + private EventProcessor ep; + + @Before + public void setup() throws Exception { + server.start(); + configBuilder.eventsURI(server.url("/").uri()); + } + + @After + public void teardown() throws Exception { + if (ep != null) { + ep.close(); + } + server.shutdown(); + } + + @Test + public void testIdentifyEventIsQueued() throws Exception { + ep = new EventProcessor(SDK_KEY, configBuilder.build()); + Event e = EventFactory.DEFAULT.newIdentifyEvent(user); + ep.sendEvent(e); + + List> output = flushAndGetEvents(); + assertEquals(1, output.size()); + Map ieo = output.get(0); + assertEquals(new JsonPrimitive("identify"), ieo.get("kind")); + assertEquals(new JsonPrimitive((double)e.creationDate), ieo.get("creationDate")); + assertUserMatches(user, ieo.get("user")); + } + + @Test + public void testIndividualFeatureEventIsQueuedWithIndexEvent() throws Exception { + ep = new EventProcessor(SDK_KEY, configBuilder.build()); + FeatureFlag flag = new FeatureFlagBuilder("flagkey").version(11).trackEvents(true).build(); + Event fe = EventFactory.DEFAULT.newFeatureRequestEvent(flag, user, + new FeatureFlag.VariationAndValue(new Integer(1), new JsonPrimitive("value")), null); + ep.sendEvent(fe); + + List> output = flushAndGetEvents(); + assertEquals(2, output.size()); + assertIndexEventMatches(output.get(0), fe); + assertFeatureEventMatches(output.get(1), fe, flag, false); + } + + @Test + public void testDebugFlagIsSetIfFlagIsTemporarilyInDebugMode() throws Exception { + ep = new EventProcessor(SDK_KEY, configBuilder.build()); + long futureTime = System.currentTimeMillis() + 1000000; + FeatureFlag flag = new FeatureFlagBuilder("flagkey").version(11).debugEventsUntilDate(futureTime).build(); + Event fe = EventFactory.DEFAULT.newFeatureRequestEvent(flag, user, + new FeatureFlag.VariationAndValue(new Integer(1), new JsonPrimitive("value")), null); + ep.sendEvent(fe); + + List> output = flushAndGetEvents(); + assertEquals(2, output.size()); + assertIndexEventMatches(output.get(0), fe); + assertFeatureEventMatches(output.get(1), fe, flag, true); + } + + @Test + public void testTwoFeatureEventsForSameUserGenerateOnlyOneIndexEvent() throws Exception { + ep = new EventProcessor(SDK_KEY, configBuilder.build()); + FeatureFlag flag1 = new FeatureFlagBuilder("flagkey1").version(11).trackEvents(true).build(); + FeatureFlag flag2 = new FeatureFlagBuilder("flagkey2").version(22).trackEvents(true).build(); + Event fe1 = EventFactory.DEFAULT.newFeatureRequestEvent(flag1, user, + new FeatureFlag.VariationAndValue(new Integer(1), new JsonPrimitive("value")), null); + Event fe2 = EventFactory.DEFAULT.newFeatureRequestEvent(flag2, user, + new FeatureFlag.VariationAndValue(new Integer(1), new JsonPrimitive("value")), null); + ep.sendEvent(fe1); + ep.sendEvent(fe2); + + List> output = flushAndGetEvents(); + assertEquals(3, output.size()); + assertIndexEventMatches(output.get(0), fe1); + assertFeatureEventMatches(output.get(1), fe1, flag1, false); + assertFeatureEventMatches(output.get(2), fe2, flag2, false); + } + + @Test + public void nonTrackedEventsAreSummarized() throws Exception { + ep = new EventProcessor(SDK_KEY, configBuilder.build()); + FeatureFlag flag1 = new FeatureFlagBuilder("flagkey1").version(11).build(); + FeatureFlag flag2 = new FeatureFlagBuilder("flagkey2").version(22).build(); + Event fe1 = EventFactory.DEFAULT.newFeatureRequestEvent(flag1, user, + new FeatureFlag.VariationAndValue(new Integer(1), new JsonPrimitive("value")), null); + Event fe2 = EventFactory.DEFAULT.newFeatureRequestEvent(flag2, user, + new FeatureFlag.VariationAndValue(new Integer(1), new JsonPrimitive("value")), null); + ep.sendEvent(fe1); + ep.sendEvent(fe2); + + List> output = flushAndGetEvents(); + assertEquals(2, output.size()); + assertIndexEventMatches(output.get(0), fe1); + Map seo = output.get(1); + assertEquals(new JsonPrimitive("summary"), seo.get("kind")); + assertEquals(new JsonPrimitive((double)fe1.creationDate), seo.get("startDate")); + assertEquals(new JsonPrimitive((double)fe2.creationDate), seo.get("endDate")); + JsonArray counters = seo.get("counters").getAsJsonArray(); + assertEquals(2, counters.size()); + } + + @Test + public void customEventIsQueuedWithUser() throws Exception { + ep = new EventProcessor(SDK_KEY, configBuilder.build()); + JsonObject data = new JsonObject(); + data.addProperty("thing", "stuff"); + Event ce = EventFactory.DEFAULT.newCustomEvent("eventkey", user, data); + ep.sendEvent(ce); + + List> output = flushAndGetEvents(); + assertEquals(2, output.size()); + assertIndexEventMatches(output.get(0), ce); + Map ceo = output.get(1); + assertEquals(new JsonPrimitive("custom"), ceo.get("kind")); + assertEquals(new JsonPrimitive((double)ce.creationDate), ceo.get("creationDate")); + assertEquals(new JsonPrimitive("eventkey"), ceo.get("key")); + assertEquals(new JsonPrimitive(user.getKeyAsString()), ceo.get("userKey")); + } + + @Test + public void sdkKeyIsSent() throws Exception { + ep = new EventProcessor(SDK_KEY, configBuilder.build()); + Event e = EventFactory.DEFAULT.newIdentifyEvent(user); + ep.sendEvent(e); + + server.enqueue(new MockResponse()); + ep.flush(); + RecordedRequest req = server.takeRequest(); + + assertEquals(SDK_KEY, req.getHeader("Authorization")); + } + + private List> flushAndGetEvents() throws Exception { + server.enqueue(new MockResponse()); + ep.flush(); + RecordedRequest req = server.takeRequest(); + return gson.fromJson(req.getBody().readUtf8(), listOfMapsType); + } + + private void assertUserMatches(LDUser user, JsonElement userJson) { + assertEquals(configBuilder.build().gson.toJsonTree(user), userJson); + } + + private void assertIndexEventMatches(Map eventOutput, Event sourceEvent) { + assertEquals(new JsonPrimitive("index"), eventOutput.get("kind")); + assertEquals(new JsonPrimitive((double)sourceEvent.creationDate), eventOutput.get("creationDate")); + assertUserMatches(sourceEvent.user, eventOutput.get("user")); + } + + private void assertFeatureEventMatches(Map eventOutput, Event sourceEvent, FeatureFlag flag, boolean debug) { + assertEquals(new JsonPrimitive("feature"), eventOutput.get("kind")); + assertEquals(new JsonPrimitive((double)sourceEvent.creationDate), eventOutput.get("creationDate")); + assertEquals(new JsonPrimitive(flag.getKey()), eventOutput.get("key")); + assertEquals(new JsonPrimitive((double)flag.getVersion()), eventOutput.get("version")); + assertEquals(new JsonPrimitive("value"), eventOutput.get("value")); + assertEquals(new JsonPrimitive(sourceEvent.user.getKeyAsString()), eventOutput.get("userKey")); + if (debug) { + assertEquals(new JsonPrimitive(true), eventOutput.get("debug")); + } else { + assertNull(eventOutput.get("debug")); + } + } +} diff --git a/src/test/java/com/launchdarkly/client/EventSummarizerTest.java b/src/test/java/com/launchdarkly/client/EventSummarizerTest.java new file mode 100644 index 000000000..dde4108ee --- /dev/null +++ b/src/test/java/com/launchdarkly/client/EventSummarizerTest.java @@ -0,0 +1,161 @@ +package com.launchdarkly.client; + +import com.google.gson.JsonPrimitive; + +import org.junit.Test; + +import java.util.Objects; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertNotNull; +import static org.junit.Assert.assertNull; +import static org.junit.Assert.assertTrue; + +public class EventSummarizerTest { + private static final LDConfig defaultConfig = new LDConfig.Builder().userKeysCapacity(100).build(); + private static final LDUser user = new LDUser.Builder("key").build(); + + private long eventTimestamp; + private EventFactory eventFactory = new EventFactory() { + @Override + protected long getTimestamp() { + return eventTimestamp; + } + }; + + @Test + public void noticeUserReturnsFalseForNeverSeenUser() { + EventSummarizer es = new EventSummarizer(defaultConfig); + assertFalse(es.noticeUser(user)); + } + + @Test + public void noticeUserReturnsTrueForPreviouslySeenUser() { + EventSummarizer es = new EventSummarizer(defaultConfig); + es.noticeUser(user); + LDUser user2 = new LDUser.Builder(user).build(); + assertTrue(es.noticeUser(user2)); + } + + @Test + public void usersNotDeduplicatedIfCapacityExceeded() { + LDConfig config = new LDConfig.Builder().userKeysCapacity(2).build(); + EventSummarizer es = new EventSummarizer(config); + LDUser user1 = new LDUser.Builder("key1").build(); + LDUser user2 = new LDUser.Builder("key2").build(); + LDUser user3 = new LDUser.Builder("key3").build(); + es.noticeUser(user1); + es.noticeUser(user2); + es.noticeUser(user3); + assertFalse(es.noticeUser(user3)); + } + + @Test + public void summarizeEventReturnsFalseForIdentifyEvent() { + EventSummarizer es = new EventSummarizer(defaultConfig); + Event event = new IdentifyEvent(user); + assertFalse(es.summarizeEvent(event)); + } + + @Test + public void summarizeEventReturnsFalseForCustomEvent() { + EventSummarizer es = new EventSummarizer(defaultConfig); + Event event = new CustomEvent("whatever", user, null); + assertFalse(es.summarizeEvent(event)); + } + + @Test + public void summarizeEventReturnsTrueForFeatureEventWithTrackEventsFalse() { + EventSummarizer es = new EventSummarizer(defaultConfig); + FeatureFlag flag = new FeatureFlagBuilder("key").build(); + Event event = eventFactory.newFeatureRequestEvent(flag, user, null, null); + assertTrue(es.summarizeEvent(event)); + } + + @Test + public void summarizeEventReturnsFalseForFeatureEventWithTrackEventsTrue() { + EventSummarizer es = new EventSummarizer(defaultConfig); + FeatureFlag flag = new FeatureFlagBuilder("key").trackEvents(true).build(); + Event event = eventFactory.newFeatureRequestEvent(flag, user, null, null); + assertFalse(es.summarizeEvent(event)); + } + + @Test + public void summarizeEventSetsStartAndEndDates() { + EventSummarizer es = new EventSummarizer(defaultConfig); + FeatureFlag flag = new FeatureFlagBuilder("key").build(); + eventTimestamp = 2000; + Event event1 = eventFactory.newFeatureRequestEvent(flag, user, null, null); + eventTimestamp = 1000; + Event event2 = eventFactory.newFeatureRequestEvent(flag, user, null, null); + eventTimestamp = 1500; + Event event3 = eventFactory.newFeatureRequestEvent(flag, user, null, null); + es.summarizeEvent(event1); + es.summarizeEvent(event2); + es.summarizeEvent(event3); + EventSummarizer.SummaryOutput data = es.flush(); + + assertEquals(1000, data.startDate); + assertEquals(2000, data.endDate); + } + + @Test + public void summarizeEventIncrementsCounters() { + EventSummarizer es = new EventSummarizer(defaultConfig); + FeatureFlag flag1 = new FeatureFlagBuilder("key1").version(11).build(); + FeatureFlag flag2 = new FeatureFlagBuilder("key2").version(22).build(); + String unknownFlagKey = "badkey"; + Event event1 = eventFactory.newFeatureRequestEvent(flag1, user, + new FeatureFlag.VariationAndValue(1, new JsonPrimitive("value1")), null); + Event event2 = eventFactory.newFeatureRequestEvent(flag1, user, + new FeatureFlag.VariationAndValue(2, new JsonPrimitive("value2")), null); + Event event3 = eventFactory.newFeatureRequestEvent(flag2, user, + new FeatureFlag.VariationAndValue(1, new JsonPrimitive("value99")), null); + Event event4 = eventFactory.newFeatureRequestEvent(flag1, user, + new FeatureFlag.VariationAndValue(1, new JsonPrimitive("value1")), null); + Event event5 = eventFactory.newUnknownFeatureRequestEvent(unknownFlagKey, user, null); + es.summarizeEvent(event1); + es.summarizeEvent(event2); + es.summarizeEvent(event3); + es.summarizeEvent(event4); + es.summarizeEvent(event5); + EventSummarizer.SummaryOutput data = es.flush(); + + assertEquals(4, data.counters.size()); + EventSummarizer.CounterData result1 = findCounter(data.counters, flag1.getKey(), "value1"); + assertNotNull(result1); + assertEquals(flag1.getKey(), result1.key); + assertEquals(new Integer(flag1.getVersion()), result1.version); + assertEquals(2, result1.count); + assertNull(result1.unknown); + EventSummarizer.CounterData result2 = findCounter(data.counters, flag1.getKey(), "value2"); + assertNotNull(result2); + assertEquals(flag1.getKey(), result2.key); + assertEquals(new Integer(flag1.getVersion()), result2.version); + assertEquals(1, result2.count); + assertNull(result2.unknown); + EventSummarizer.CounterData result3 = findCounter(data.counters, flag2.getKey(), "value99"); + assertNotNull(result3); + assertEquals(flag2.getKey(), result3.key); + assertEquals(new Integer(flag2.getVersion()), result3.version); + assertEquals(1, result3.count); + assertNull(result3.unknown); + EventSummarizer.CounterData result4 = findCounter(data.counters, unknownFlagKey, null); + assertNotNull(result4); + assertEquals(unknownFlagKey, result4.key); + assertNull(result4.version); + assertEquals(1, result4.count); + assertEquals(Boolean.TRUE, result4.unknown); + } + + private EventSummarizer.CounterData findCounter(Iterable counters, String key, String value) { + JsonPrimitive jv = value == null ? null : new JsonPrimitive(value); + for (EventSummarizer.CounterData c: counters) { + if (c.key.equals(key) && Objects.equals(c.value, jv)) { + return c; + } + } + return null; + } +} diff --git a/src/test/java/com/launchdarkly/client/FeatureFlagTest.java b/src/test/java/com/launchdarkly/client/FeatureFlagTest.java index 9f4c568d8..5872a08ee 100644 --- a/src/test/java/com/launchdarkly/client/FeatureFlagTest.java +++ b/src/test/java/com/launchdarkly/client/FeatureFlagTest.java @@ -30,9 +30,9 @@ public void testPrereqDoesNotExist() throws EvaluationException { featureStore.upsert(FEATURES, f1); LDUser user = new LDUser.Builder("userKey").build(); - FeatureFlag.EvalResult actual = f1.evaluate(user, featureStore); + FeatureFlag.EvalResult actual = f1.evaluate(user, featureStore, EventFactory.DEFAULT); - Assert.assertNull(actual.getValue()); + Assert.assertNull(actual.getResult().getValue()); Assert.assertNotNull(actual.getPrerequisiteEvents()); Assert.assertEquals(0, actual.getPrerequisiteEvents().size()); } @@ -52,19 +52,19 @@ public void testPrereqCollectsEventsForPrereqs() throws EvaluationException { LDUser user = new LDUser.Builder("userKey").build(); - FeatureFlag.EvalResult flagAResult = flagA.evaluate(user, featureStore); + FeatureFlag.EvalResult flagAResult = flagA.evaluate(user, featureStore, EventFactory.DEFAULT); Assert.assertNotNull(flagAResult); - Assert.assertNull(flagAResult.getValue()); + Assert.assertNull(flagAResult.getResult().getValue()); Assert.assertEquals(2, flagAResult.getPrerequisiteEvents().size()); - FeatureFlag.EvalResult flagBResult = flagB.evaluate(user, featureStore); + FeatureFlag.EvalResult flagBResult = flagB.evaluate(user, featureStore, EventFactory.DEFAULT); Assert.assertNotNull(flagBResult); - Assert.assertNull(flagBResult.getValue()); + Assert.assertNull(flagBResult.getResult().getValue()); Assert.assertEquals(1, flagBResult.getPrerequisiteEvents().size()); - FeatureFlag.EvalResult flagCResult = flagC.evaluate(user, featureStore); + FeatureFlag.EvalResult flagCResult = flagC.evaluate(user, featureStore, EventFactory.DEFAULT); Assert.assertNotNull(flagCResult); - Assert.assertEquals(null, flagCResult.getValue()); + Assert.assertNull(null, flagCResult.getResult().getValue()); Assert.assertEquals(0, flagCResult.getPrerequisiteEvents().size()); } @@ -79,8 +79,9 @@ public void testSegmentMatchClauseRetrievesSegmentFromStore() throws Exception { FeatureFlag flag = segmentMatchBooleanFlag("segkey"); LDUser user = new LDUser.Builder("foo").build(); - FeatureFlag.EvalResult result = flag.evaluate(user, featureStore); - Assert.assertEquals(new JsonPrimitive(true), result.getValue()); + FeatureFlag.EvalResult result = flag.evaluate(user, featureStore, EventFactory.DEFAULT); + Assert.assertNotNull(result.getResult()); + Assert.assertEquals(new JsonPrimitive(true), result.getResult().getValue()); } @Test @@ -88,8 +89,9 @@ public void testSegmentMatchClauseFallsThroughIfSegmentNotFound() throws Excepti FeatureFlag flag = segmentMatchBooleanFlag("segkey"); LDUser user = new LDUser.Builder("foo").build(); - FeatureFlag.EvalResult result = flag.evaluate(user, featureStore); - Assert.assertEquals(new JsonPrimitive(false), result.getValue()); + FeatureFlag.EvalResult result = flag.evaluate(user, featureStore, EventFactory.DEFAULT); + Assert.assertNotNull(result.getResult()); + Assert.assertEquals(new JsonPrimitive(false), result.getResult().getValue()); } private FeatureFlag newFlagWithPrereq(String featureKey, String prereqKey) { From 4f96e343c79526bb9f8efe48504cc83d450f8d24 Mon Sep 17 00:00:00 2001 From: Eli Bishop Date: Wed, 7 Mar 2018 20:38:24 -0800 Subject: [PATCH 02/67] use LRU cache for user keys --- .../launchdarkly/client/EventSummarizer.java | 37 +++++++++++-------- .../client/EventSummarizerTest.java | 6 ++- 2 files changed, 26 insertions(+), 17 deletions(-) diff --git a/src/main/java/com/launchdarkly/client/EventSummarizer.java b/src/main/java/com/launchdarkly/client/EventSummarizer.java index 5da766e36..e0e633759 100644 --- a/src/main/java/com/launchdarkly/client/EventSummarizer.java +++ b/src/main/java/com/launchdarkly/client/EventSummarizer.java @@ -4,29 +4,26 @@ import java.util.ArrayList; import java.util.HashMap; -import java.util.HashSet; +import java.util.LinkedHashMap; import java.util.List; import java.util.Map; -import java.util.Set; /** * Manages the state of summarizable information for the EventProcessor, including the * event counters and user deduplication. */ class EventSummarizer { - private Map counters; + private final Map counters; private long startDate; private long endDate; private long lastKnownPastTime; - private Set userKeysSeen; - private int userKeysCapacity; + private final SimpleLRUCache userKeys; EventSummarizer(LDConfig config) { this.counters = new HashMap<>(); this.startDate = 0; this.endDate = 0; - this.userKeysSeen = new HashSet<>(); - this.userKeysCapacity = config.userKeysCapacity; + this.userKeys = new SimpleLRUCache(config.userKeysCapacity); } /** @@ -39,20 +36,14 @@ boolean noticeUser(LDUser user) { return false; } String key = user.getKeyAsString(); - if (userKeysSeen.contains(key)) { - return true; - } - if (userKeysSeen.size() < userKeysCapacity) { - userKeysSeen.add(key); - } - return false; + return userKeys.put(key, key) != null; } /** * Reset the set of users we've seen. */ void resetUsers() { - userKeysSeen.clear(); + userKeys.clear(); } /** @@ -155,6 +146,22 @@ public int hashCode() { } } + @SuppressWarnings("serial") + private static class SimpleLRUCache extends LinkedHashMap { + // http://chriswu.me/blog/a-lru-cache-in-10-lines-of-java/ + private final int capacity; + + SimpleLRUCache(int capacity) { + super(16, 0.75f, true); + this.capacity = capacity; + } + + @Override + protected boolean removeEldestEntry(Map.Entry eldest) { + return size() > capacity; + } + } + private static class CounterValue { private int count; private JsonElement flagValue; diff --git a/src/test/java/com/launchdarkly/client/EventSummarizerTest.java b/src/test/java/com/launchdarkly/client/EventSummarizerTest.java index dde4108ee..91d3e21f6 100644 --- a/src/test/java/com/launchdarkly/client/EventSummarizerTest.java +++ b/src/test/java/com/launchdarkly/client/EventSummarizerTest.java @@ -39,7 +39,7 @@ public void noticeUserReturnsTrueForPreviouslySeenUser() { } @Test - public void usersNotDeduplicatedIfCapacityExceeded() { + public void oldestUserForgottenIfCapacityExceeded() { LDConfig config = new LDConfig.Builder().userKeysCapacity(2).build(); EventSummarizer es = new EventSummarizer(config); LDUser user1 = new LDUser.Builder("key1").build(); @@ -48,7 +48,9 @@ public void usersNotDeduplicatedIfCapacityExceeded() { es.noticeUser(user1); es.noticeUser(user2); es.noticeUser(user3); - assertFalse(es.noticeUser(user3)); + assertTrue(es.noticeUser(user3)); + assertTrue(es.noticeUser(user2)); + assertFalse(es.noticeUser(user1)); } @Test From da4647d638302e055c09ba30c8580aaf00521e2b Mon Sep 17 00:00:00 2001 From: Eli Bishop Date: Wed, 7 Mar 2018 20:49:01 -0800 Subject: [PATCH 03/67] do summary output generation outside of mutex --- .../launchdarkly/client/EventProcessor.java | 7 +- .../launchdarkly/client/EventSummarizer.java | 85 ++++++++++++------- .../client/EventSummarizerTest.java | 4 +- 3 files changed, 60 insertions(+), 36 deletions(-) diff --git a/src/main/java/com/launchdarkly/client/EventProcessor.java b/src/main/java/com/launchdarkly/client/EventProcessor.java index fed292875..91e3f08e2 100644 --- a/src/main/java/com/launchdarkly/client/EventProcessor.java +++ b/src/main/java/com/launchdarkly/client/EventProcessor.java @@ -132,15 +132,16 @@ public void run() { public void flush() { List events = new ArrayList<>(queue.size()); - EventSummarizer.SummaryOutput summary; + EventSummarizer.EventsState snapshot; lock.lock(); try { queue.drainTo(events); - summary = summarizer.flush(); + snapshot = summarizer.snapshot(); } finally { lock.unlock(); } - if (!summary.counters.isEmpty()) { + if (!snapshot.counters.isEmpty()) { + EventSummarizer.SummaryOutput summary = summarizer.output(snapshot); SummaryEventOutput seo = new SummaryEventOutput(summary.startDate, summary.endDate, summary.counters); events.add(seo); } diff --git a/src/main/java/com/launchdarkly/client/EventSummarizer.java b/src/main/java/com/launchdarkly/client/EventSummarizer.java index e0e633759..07e5c2596 100644 --- a/src/main/java/com/launchdarkly/client/EventSummarizer.java +++ b/src/main/java/com/launchdarkly/client/EventSummarizer.java @@ -13,16 +13,12 @@ * event counters and user deduplication. */ class EventSummarizer { - private final Map counters; - private long startDate; - private long endDate; + private EventsState eventsState; private long lastKnownPastTime; private final SimpleLRUCache userKeys; EventSummarizer(LDConfig config) { - this.counters = new HashMap<>(); - this.startDate = 0; - this.endDate = 0; + this.eventsState = new EventsState(); this.userKeys = new SimpleLRUCache(config.userKeysCapacity); } @@ -70,23 +66,9 @@ boolean summarizeEvent(Event event) { return false; } } - - CounterKey key = new CounterKey(fe.key, (fe.variation == null) ? 0 : fe.variation.intValue(), - (fe.version == null) ? 0 : fe.version.intValue()); - - CounterValue value = counters.get(key); - if (value != null) { - value.increment(); - } else { - counters.put(key, new CounterValue(1, fe.value)); - } - - if (startDate == 0 || fe.creationDate < startDate) { - startDate = fe.creationDate; - } - if (fe.creationDate > endDate) { - endDate = fe.creationDate; - } + + eventsState.incrementCounter(fe.key, fe.variation, fe.version, fe.value); + eventsState.noteTimestamp(fe.creationDate); return true; } @@ -102,9 +84,24 @@ void setLastKnownPastTime(long t) { } } - SummaryOutput flush() { - List countersOut = new ArrayList<>(counters.size()); - for (Map.Entry entry: counters.entrySet()) { + /** + * Returns a snapshot of the current summarized event data, and resets this state. + * @return the previous event state + */ + EventsState snapshot() { + EventsState ret = eventsState; + eventsState = new EventsState(); + return ret; + } + + /** + * Transforms the summary data into the format used for event sending. + * @param snapshot the data obtained from {@link #snapshot()} + * @return the formatted output + */ + SummaryOutput output(EventsState snapshot) { + List countersOut = new ArrayList<>(snapshot.counters.size()); + for (Map.Entry entry: snapshot.counters.entrySet()) { CounterData c = new CounterData(entry.getKey().key, entry.getValue().flagValue, entry.getKey().version == 0 ? null : entry.getKey().version, @@ -112,12 +109,38 @@ SummaryOutput flush() { entry.getKey().version == 0 ? true : null); countersOut.add(c); } - counters.clear(); + return new SummaryOutput(snapshot.startDate, snapshot.endDate, countersOut); + } + + static class EventsState { + final Map counters; + long startDate; + long endDate; - SummaryOutput ret = new SummaryOutput(startDate, endDate, countersOut); - startDate = 0; - endDate = 0; - return ret; + EventsState() { + counters = new HashMap(); + } + + void incrementCounter(String flagKey, Integer variation, Integer version, JsonElement flagValue) { + CounterKey key = new CounterKey(flagKey, (variation == null) ? 0 : variation.intValue(), + (version == null) ? 0 : version.intValue()); + + CounterValue value = counters.get(key); + if (value != null) { + value.increment(); + } else { + counters.put(key, new CounterValue(1, flagValue)); + } + } + + void noteTimestamp(long time) { + if (startDate == 0 || time < startDate) { + startDate = time; + } + if (time > endDate) { + endDate = time; + } + } } private static class CounterKey { diff --git a/src/test/java/com/launchdarkly/client/EventSummarizerTest.java b/src/test/java/com/launchdarkly/client/EventSummarizerTest.java index 91d3e21f6..c080ad208 100644 --- a/src/test/java/com/launchdarkly/client/EventSummarizerTest.java +++ b/src/test/java/com/launchdarkly/client/EventSummarizerTest.java @@ -96,7 +96,7 @@ public void summarizeEventSetsStartAndEndDates() { es.summarizeEvent(event1); es.summarizeEvent(event2); es.summarizeEvent(event3); - EventSummarizer.SummaryOutput data = es.flush(); + EventSummarizer.SummaryOutput data = es.output(es.snapshot()); assertEquals(1000, data.startDate); assertEquals(2000, data.endDate); @@ -122,7 +122,7 @@ public void summarizeEventIncrementsCounters() { es.summarizeEvent(event3); es.summarizeEvent(event4); es.summarizeEvent(event5); - EventSummarizer.SummaryOutput data = es.flush(); + EventSummarizer.SummaryOutput data = es.output(es.snapshot()); assertEquals(4, data.counters.size()); EventSummarizer.CounterData result1 = findCounter(data.counters, flag1.getKey(), "value1"); From d99fbcbad8179ece02abc6bac1247db75d9ae967 Mon Sep 17 00:00:00 2001 From: Eli Bishop Date: Wed, 7 Mar 2018 21:04:52 -0800 Subject: [PATCH 04/67] test cleanup --- .../client/EventProcessorTest.java | 90 ++++++++++--------- 1 file changed, 48 insertions(+), 42 deletions(-) diff --git a/src/test/java/com/launchdarkly/client/EventProcessorTest.java b/src/test/java/com/launchdarkly/client/EventProcessorTest.java index fc2b025c2..73235bcd0 100644 --- a/src/test/java/com/launchdarkly/client/EventProcessorTest.java +++ b/src/test/java/com/launchdarkly/client/EventProcessorTest.java @@ -5,18 +5,14 @@ import com.google.gson.JsonElement; import com.google.gson.JsonObject; import com.google.gson.JsonPrimitive; -import com.google.gson.reflect.TypeToken; import org.junit.After; import org.junit.Before; import org.junit.Test; -import java.lang.reflect.Type; -import java.util.List; -import java.util.Map; - import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertNull; +import static org.junit.Assert.assertTrue; import okhttp3.mockwebserver.MockResponse; import okhttp3.mockwebserver.MockWebServer; @@ -26,8 +22,6 @@ public class EventProcessorTest { private static final String SDK_KEY = "SDK_KEY"; private static final LDUser user = new LDUser.Builder("userkey").name("Red").build(); private static final Gson gson = new Gson(); - private static final Type listOfMapsType = new TypeToken>>() { - }.getType(); private final LDConfig.Builder configBuilder = new LDConfig.Builder(); private final MockWebServer server = new MockWebServer(); @@ -53,12 +47,13 @@ public void testIdentifyEventIsQueued() throws Exception { Event e = EventFactory.DEFAULT.newIdentifyEvent(user); ep.sendEvent(e); - List> output = flushAndGetEvents(); + JsonArray output = flushAndGetEvents(); assertEquals(1, output.size()); - Map ieo = output.get(0); - assertEquals(new JsonPrimitive("identify"), ieo.get("kind")); - assertEquals(new JsonPrimitive((double)e.creationDate), ieo.get("creationDate")); - assertUserMatches(user, ieo.get("user")); + JsonObject expected = new JsonObject(); + expected.addProperty("kind", "identify"); + expected.addProperty("creationDate", e.creationDate); + expected.add("user", makeUserJson(user)); + assertEquals(expected, output.get(0)); } @Test @@ -69,7 +64,7 @@ public void testIndividualFeatureEventIsQueuedWithIndexEvent() throws Exception new FeatureFlag.VariationAndValue(new Integer(1), new JsonPrimitive("value")), null); ep.sendEvent(fe); - List> output = flushAndGetEvents(); + JsonArray output = flushAndGetEvents(); assertEquals(2, output.size()); assertIndexEventMatches(output.get(0), fe); assertFeatureEventMatches(output.get(1), fe, flag, false); @@ -84,7 +79,7 @@ public void testDebugFlagIsSetIfFlagIsTemporarilyInDebugMode() throws Exception new FeatureFlag.VariationAndValue(new Integer(1), new JsonPrimitive("value")), null); ep.sendEvent(fe); - List> output = flushAndGetEvents(); + JsonArray output = flushAndGetEvents(); assertEquals(2, output.size()); assertIndexEventMatches(output.get(0), fe); assertFeatureEventMatches(output.get(1), fe, flag, true); @@ -102,7 +97,7 @@ public void testTwoFeatureEventsForSameUserGenerateOnlyOneIndexEvent() throws Ex ep.sendEvent(fe1); ep.sendEvent(fe2); - List> output = flushAndGetEvents(); + JsonArray output = flushAndGetEvents(); assertEquals(3, output.size()); assertIndexEventMatches(output.get(0), fe1); assertFeatureEventMatches(output.get(1), fe1, flag1, false); @@ -121,13 +116,14 @@ public void nonTrackedEventsAreSummarized() throws Exception { ep.sendEvent(fe1); ep.sendEvent(fe2); - List> output = flushAndGetEvents(); + JsonArray output = flushAndGetEvents(); assertEquals(2, output.size()); assertIndexEventMatches(output.get(0), fe1); - Map seo = output.get(1); - assertEquals(new JsonPrimitive("summary"), seo.get("kind")); - assertEquals(new JsonPrimitive((double)fe1.creationDate), seo.get("startDate")); - assertEquals(new JsonPrimitive((double)fe2.creationDate), seo.get("endDate")); + + JsonObject seo = output.get(1).getAsJsonObject(); + assertEquals("summary", seo.get("kind").getAsString()); + assertEquals(fe1.creationDate, seo.get("startDate").getAsLong()); + assertEquals(fe2.creationDate, seo.get("endDate").getAsLong()); JsonArray counters = seo.get("counters").getAsJsonArray(); assertEquals(2, counters.size()); } @@ -140,14 +136,17 @@ public void customEventIsQueuedWithUser() throws Exception { Event ce = EventFactory.DEFAULT.newCustomEvent("eventkey", user, data); ep.sendEvent(ce); - List> output = flushAndGetEvents(); + JsonArray output = flushAndGetEvents(); assertEquals(2, output.size()); assertIndexEventMatches(output.get(0), ce); - Map ceo = output.get(1); - assertEquals(new JsonPrimitive("custom"), ceo.get("kind")); - assertEquals(new JsonPrimitive((double)ce.creationDate), ceo.get("creationDate")); - assertEquals(new JsonPrimitive("eventkey"), ceo.get("key")); - assertEquals(new JsonPrimitive(user.getKeyAsString()), ceo.get("userKey")); + + JsonObject expected = new JsonObject(); + expected.addProperty("kind", "custom"); + expected.addProperty("creationDate", ce.creationDate); + expected.addProperty("key", "eventkey"); + expected.addProperty("userKey", user.getKeyAsString()); + expected.add("data", data); + assertEquals(expected, output.get(1)); } @Test @@ -163,34 +162,41 @@ public void sdkKeyIsSent() throws Exception { assertEquals(SDK_KEY, req.getHeader("Authorization")); } - private List> flushAndGetEvents() throws Exception { + private JsonArray flushAndGetEvents() throws Exception { server.enqueue(new MockResponse()); ep.flush(); RecordedRequest req = server.takeRequest(); - return gson.fromJson(req.getBody().readUtf8(), listOfMapsType); + return gson.fromJson(req.getBody().readUtf8(), JsonElement.class).getAsJsonArray(); + } + + private JsonElement makeUserJson(LDUser user) { + // need to use the gson instance from the config object, which has a custom serializer + return configBuilder.build().gson.toJsonTree(user); } private void assertUserMatches(LDUser user, JsonElement userJson) { - assertEquals(configBuilder.build().gson.toJsonTree(user), userJson); + assertEquals(makeUserJson(user), userJson); } - private void assertIndexEventMatches(Map eventOutput, Event sourceEvent) { - assertEquals(new JsonPrimitive("index"), eventOutput.get("kind")); - assertEquals(new JsonPrimitive((double)sourceEvent.creationDate), eventOutput.get("creationDate")); - assertUserMatches(sourceEvent.user, eventOutput.get("user")); + private void assertIndexEventMatches(JsonElement eventOutput, Event sourceEvent) { + JsonObject o = eventOutput.getAsJsonObject(); + assertEquals("index", o.get("kind").getAsString()); + assertEquals(sourceEvent.creationDate, o.get("creationDate").getAsLong()); + assertUserMatches(sourceEvent.user, o.get("user")); } - private void assertFeatureEventMatches(Map eventOutput, Event sourceEvent, FeatureFlag flag, boolean debug) { - assertEquals(new JsonPrimitive("feature"), eventOutput.get("kind")); - assertEquals(new JsonPrimitive((double)sourceEvent.creationDate), eventOutput.get("creationDate")); - assertEquals(new JsonPrimitive(flag.getKey()), eventOutput.get("key")); - assertEquals(new JsonPrimitive((double)flag.getVersion()), eventOutput.get("version")); - assertEquals(new JsonPrimitive("value"), eventOutput.get("value")); - assertEquals(new JsonPrimitive(sourceEvent.user.getKeyAsString()), eventOutput.get("userKey")); + private void assertFeatureEventMatches(JsonElement eventOutput, Event sourceEvent, FeatureFlag flag, boolean debug) { + JsonObject o = eventOutput.getAsJsonObject(); + assertEquals("feature", o.get("kind").getAsString()); + assertEquals(sourceEvent.creationDate, o.get("creationDate").getAsLong()); + assertEquals(flag.getKey(), o.get("key").getAsString()); + assertEquals(flag.getVersion(), o.get("version").getAsInt()); + assertEquals("value", o.get("value").getAsString()); + assertEquals(sourceEvent.user.getKeyAsString(), o.get("userKey").getAsString()); if (debug) { - assertEquals(new JsonPrimitive(true), eventOutput.get("debug")); + assertTrue(o.get("debug").getAsBoolean()); } else { - assertNull(eventOutput.get("debug")); + assertNull(o.get("debug")); } } } From b236380cd7d2eafd634657268b085072d97a425e Mon Sep 17 00:00:00 2001 From: Eli Bishop Date: Fri, 9 Mar 2018 11:36:35 -0800 Subject: [PATCH 05/67] make summarization independent of whether event is tracked in detail --- .../launchdarkly/client/EventProcessor.java | 69 +++++++-- .../launchdarkly/client/EventSummarizer.java | 138 ++++++++++-------- 2 files changed, 134 insertions(+), 73 deletions(-) diff --git a/src/main/java/com/launchdarkly/client/EventProcessor.java b/src/main/java/com/launchdarkly/client/EventProcessor.java index 91e3f08e2..0e84c5f5b 100644 --- a/src/main/java/com/launchdarkly/client/EventProcessor.java +++ b/src/main/java/com/launchdarkly/client/EventProcessor.java @@ -13,14 +13,18 @@ import java.io.Closeable; import java.io.IOException; +import java.text.SimpleDateFormat; import java.util.ArrayList; import java.util.List; import java.util.Random; import java.util.concurrent.*; import java.util.concurrent.atomic.AtomicBoolean; +import java.util.concurrent.atomic.AtomicLong; import java.util.concurrent.locks.ReentrantLock; class EventProcessor implements Closeable { + private static final SimpleDateFormat HTTP_DATE_FORMAT = new SimpleDateFormat("EEE, dd MMM yyyy HH:mm:ss zzz"); + private final ScheduledExecutorService scheduler; private final Random random = new Random(); private final BlockingQueue queue; @@ -28,6 +32,7 @@ class EventProcessor implements Closeable { private final LDConfig config; private final Consumer consumer; private final EventSummarizer summarizer; + private AtomicLong lastKnownPastTime = new AtomicLong(0); private final ReentrantLock lock = new ReentrantLock(); EventProcessor(String sdkKey, LDConfig config) { @@ -59,9 +64,10 @@ public void run() { boolean sendEvent(Event e) { lock.lock(); try { - if (!(e instanceof IdentifyEvent)) { - // identify events don't get deduplicated and always include the full user data - if (e.user != null && !summarizer.noticeUser(e.user)) { + // For each user we haven't seen before, we add an index event - unless this is already + // an identify event for that user. + if (e.user != null && !summarizer.noticeUser(e.user)) { + if (!(e instanceof IdentifyEvent)) { IndexEventOutput ie = new IndexEventOutput(e.creationDate, e.user); if (!queue.offer(ie)) { return false; @@ -69,25 +75,49 @@ boolean sendEvent(Event e) { } } - if (summarizer.summarizeEvent(e)) { - return true; - } - - if (config.samplingInterval > 0 && random.nextInt(config.samplingInterval) != 0) { - return true; - } + // Always record the event in the summarizer. + summarizer.summarizeEvent(e); + + if (shouldTrackFullEvent(e)) { + if (config.samplingInterval > 0 && random.nextInt(config.samplingInterval) != 0) { + return true; + } - EventOutput eventOutput = createEventOutput(e); - if (eventOutput == null) { - return false; - } else { - return queue.offer(eventOutput); - } + EventOutput eventOutput = createEventOutput(e); + if (eventOutput == null) { + return false; + } else { + return queue.offer(eventOutput); + } + } + return true; } finally { lock.unlock(); } } + private boolean shouldTrackFullEvent(Event e) { + if (e instanceof FeatureRequestEvent) { + FeatureRequestEvent fe = (FeatureRequestEvent)e; + if (fe.trackEvents) { + return true; + } + if (fe.debugEventsUntilDate != null) { + // The "last known past time" comes from the last HTTP response we got from the server. + // In case the client's time is set wrong, at least we know that any expiration date + // earlier than that point is definitely in the past. + long lastPast = lastKnownPastTime.get(); + if ((lastPast != 0 && fe.debugEventsUntilDate > lastPast) || + fe.debugEventsUntilDate > System.currentTimeMillis()) { + return true; + } + } + return false; + } else { + return true; + } + } + private EventOutput createEventOutput(Event e) { String userKey = e.user == null ? null : e.user.getKeyAsString(); if (e instanceof FeatureRequestEvent) { @@ -175,6 +205,13 @@ private void postEvents(List events) { } } else { logger.debug("Events Response: " + response.code()); + try { + String dateStr = response.header("Date"); + if (dateStr != null) { + lastKnownPastTime.set(HTTP_DATE_FORMAT.parse(dateStr).getTime()); + } + } catch (Exception e) { + } } } catch (IOException e) { logger.info("Unhandled exception in LaunchDarkly client when posting events to URL: " + request.url(), e); diff --git a/src/main/java/com/launchdarkly/client/EventSummarizer.java b/src/main/java/com/launchdarkly/client/EventSummarizer.java index 07e5c2596..8c2d3000d 100644 --- a/src/main/java/com/launchdarkly/client/EventSummarizer.java +++ b/src/main/java/com/launchdarkly/client/EventSummarizer.java @@ -7,6 +7,7 @@ import java.util.LinkedHashMap; import java.util.List; import java.util.Map; +import java.util.Objects; /** * Manages the state of summarizable information for the EventProcessor, including the @@ -14,7 +15,6 @@ */ class EventSummarizer { private EventsState eventsState; - private long lastKnownPastTime; private final SimpleLRUCache userKeys; EventSummarizer(LDConfig config) { @@ -43,44 +43,14 @@ void resetUsers() { } /** - * Check whether this is a kind of event that we should summarize; if so, add it to our - * counters and return true. False means that the event should be sent individually. + * Adds this event to our counters, if it is a type of event we need to count. * @param event an event - * @return true if we summarized the event */ - boolean summarizeEvent(Event event) { - if (!(event instanceof FeatureRequestEvent)) { - return false; - } - FeatureRequestEvent fe = (FeatureRequestEvent)event; - if (fe.trackEvents) { - return false; - } - - if (fe.debugEventsUntilDate != null) { - // The "last known past time" comes from the last HTTP response we got from the server. - // In case the client's time is set wrong, at least we know that any expiration date - // earlier than that point is definitely in the past. - if (fe.debugEventsUntilDate > lastKnownPastTime && - fe.debugEventsUntilDate > System.currentTimeMillis()) { - return false; - } - } - - eventsState.incrementCounter(fe.key, fe.variation, fe.version, fe.value); - eventsState.noteTimestamp(fe.creationDate); - - return true; - } - - /** - * Marks the given timestamp (received from the server) as being in the past, in case the - * client-side time is unreliable. - * @param t a timestamp - */ - void setLastKnownPastTime(long t) { - if (lastKnownPastTime < t) { - lastKnownPastTime = t; + void summarizeEvent(Event event) { + if (event instanceof FeatureRequestEvent) { + FeatureRequestEvent fe = (FeatureRequestEvent)event; + eventsState.incrementCounter(fe.key, fe.variation, fe.version, fe.value); + eventsState.noteTimestamp(fe.creationDate); } } @@ -112,6 +82,22 @@ SummaryOutput output(EventsState snapshot) { return new SummaryOutput(snapshot.startDate, snapshot.endDate, countersOut); } + @SuppressWarnings("serial") + private static class SimpleLRUCache extends LinkedHashMap { + // http://chriswu.me/blog/a-lru-cache-in-10-lines-of-java/ + private final int capacity; + + SimpleLRUCache(int capacity) { + super(16, 0.75f, true); + this.capacity = capacity; + } + + @Override + protected boolean removeEldestEntry(Map.Entry eldest) { + return size() > capacity; + } + } + static class EventsState { final Map counters; long startDate; @@ -141,8 +127,22 @@ void noteTimestamp(long time) { endDate = time; } } + + @Override + public boolean equals(Object other) { + if (other instanceof EventsState) { + EventsState o = (EventsState)other; + return o.counters.equals(counters) && startDate == o.startDate && endDate == o.endDate; + } + return true; + } + + @Override + public int hashCode() { + return counters.hashCode() + 31 * ((int)startDate + 31 * (int)endDate); + } } - + private static class CounterKey { private final String key; private final int variation; @@ -165,23 +165,7 @@ public boolean equals(Object other) { @Override public int hashCode() { - return key.hashCode() + (variation + (version * 31) * 31); - } - } - - @SuppressWarnings("serial") - private static class SimpleLRUCache extends LinkedHashMap { - // http://chriswu.me/blog/a-lru-cache-in-10-lines-of-java/ - private final int capacity; - - SimpleLRUCache(int capacity) { - super(16, 0.75f, true); - this.capacity = capacity; - } - - @Override - protected boolean removeEldestEntry(Map.Entry eldest) { - return size() > capacity; + return key.hashCode() + 31 * (variation + 31 * version); } } @@ -206,13 +190,34 @@ static class CounterData { final int count; final Boolean unknown; - private CounterData(String key, JsonElement value, Integer version, int count, Boolean unknown) { + CounterData(String key, JsonElement value, Integer version, int count, Boolean unknown) { this.key = key; this.value = value; this.version = version; this.count = count; this.unknown = unknown; } + + @Override + public boolean equals(Object other) { + if (other instanceof CounterData) { + CounterData o = (CounterData)other; + return o.key.equals(key) && Objects.equals(value, o.value) && Objects.equals(version, o.version) && + o.count == count && Objects.deepEquals(unknown, o.unknown); + } + return false; + } + + @Override + public int hashCode() { + return key.hashCode() + 31 * (Objects.hashCode(value) + 31 * (Objects.hashCode(version) + 31 * + (count + 31 * (Objects.hashCode(unknown))))); + } + + @Override + public String toString() { + return "{" + key + ", " + value + ", " + version + ", " + count + ", " + unknown + "}"; + } } static class SummaryOutput { @@ -220,10 +225,29 @@ static class SummaryOutput { final long endDate; final List counters; - private SummaryOutput(long startDate, long endDate, List counters) { + SummaryOutput(long startDate, long endDate, List counters) { this.startDate = startDate; this.endDate = endDate; this.counters = counters; } + + @Override + public boolean equals(Object other) { + if (other instanceof SummaryOutput) { + SummaryOutput o = (SummaryOutput)other; + return o.startDate == startDate && o.endDate == endDate && o.counters.equals(counters); + } + return false; + } + + @Override + public int hashCode() { + return counters.hashCode() + 31 * ((int)startDate + 31 * (int)endDate); + } + + @Override + public String toString() { + return "{" + startDate + ", " + endDate + ", " + counters + "}"; + } } } From 27b412f04d66f0b65703ddc7d8dd0744ada74bf3 Mon Sep 17 00:00:00 2001 From: Eli Bishop Date: Fri, 9 Mar 2018 11:36:47 -0800 Subject: [PATCH 06/67] test improvements --- build.gradle | 1 + .../client/EventProcessorTest.java | 167 +++++++++++------- .../client/EventSummarizerTest.java | 82 +++------ .../com/launchdarkly/client/TestUtils.java | 74 ++++++++ 4 files changed, 196 insertions(+), 128 deletions(-) create mode 100644 src/test/java/com/launchdarkly/client/TestUtils.java diff --git a/build.gradle b/build.gradle index dda8588d9..192251f42 100644 --- a/build.gradle +++ b/build.gradle @@ -42,6 +42,7 @@ libraries.unshaded = [ libraries.testCompile = [ "com.squareup.okhttp3:mockwebserver:3.10.0", + "org.hamcrest:hamcrest-all:1.3", "org.easymock:easymock:3.4", "junit:junit:4.12" ] diff --git a/src/test/java/com/launchdarkly/client/EventProcessorTest.java b/src/test/java/com/launchdarkly/client/EventProcessorTest.java index 73235bcd0..e6a639251 100644 --- a/src/test/java/com/launchdarkly/client/EventProcessorTest.java +++ b/src/test/java/com/launchdarkly/client/EventProcessorTest.java @@ -6,13 +6,18 @@ import com.google.gson.JsonObject; import com.google.gson.JsonPrimitive; +import org.hamcrest.Matcher; import org.junit.After; import org.junit.Before; import org.junit.Test; -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertNull; -import static org.junit.Assert.assertTrue; +import static com.launchdarkly.client.TestUtils.hasJsonProperty; +import static com.launchdarkly.client.TestUtils.isJsonArray; +import static org.hamcrest.MatcherAssert.assertThat; +import static org.hamcrest.Matchers.allOf; +import static org.hamcrest.Matchers.equalTo; +import static org.hamcrest.Matchers.hasItem; +import static org.hamcrest.Matchers.hasItems; import okhttp3.mockwebserver.MockResponse; import okhttp3.mockwebserver.MockWebServer; @@ -41,6 +46,7 @@ public void teardown() throws Exception { server.shutdown(); } + @SuppressWarnings("unchecked") @Test public void testIdentifyEventIsQueued() throws Exception { ep = new EventProcessor(SDK_KEY, configBuilder.build()); @@ -48,86 +54,98 @@ public void testIdentifyEventIsQueued() throws Exception { ep.sendEvent(e); JsonArray output = flushAndGetEvents(); - assertEquals(1, output.size()); - JsonObject expected = new JsonObject(); - expected.addProperty("kind", "identify"); - expected.addProperty("creationDate", e.creationDate); - expected.add("user", makeUserJson(user)); - assertEquals(expected, output.get(0)); + assertThat(output, hasItems( + allOf( + hasJsonProperty("kind", "identify"), + hasJsonProperty("creationDate", (double)e.creationDate), + hasJsonProperty("user", makeUserJson(user)) + ))); } + @SuppressWarnings("unchecked") @Test public void testIndividualFeatureEventIsQueuedWithIndexEvent() throws Exception { ep = new EventProcessor(SDK_KEY, configBuilder.build()); FeatureFlag flag = new FeatureFlagBuilder("flagkey").version(11).trackEvents(true).build(); - Event fe = EventFactory.DEFAULT.newFeatureRequestEvent(flag, user, + FeatureRequestEvent fe = EventFactory.DEFAULT.newFeatureRequestEvent(flag, user, new FeatureFlag.VariationAndValue(new Integer(1), new JsonPrimitive("value")), null); ep.sendEvent(fe); JsonArray output = flushAndGetEvents(); - assertEquals(2, output.size()); - assertIndexEventMatches(output.get(0), fe); - assertFeatureEventMatches(output.get(1), fe, flag, false); + assertThat(output, hasItems( + isIndexEvent(fe), + isFeatureEvent(fe, flag, false) + )); } + @SuppressWarnings("unchecked") @Test public void testDebugFlagIsSetIfFlagIsTemporarilyInDebugMode() throws Exception { ep = new EventProcessor(SDK_KEY, configBuilder.build()); long futureTime = System.currentTimeMillis() + 1000000; FeatureFlag flag = new FeatureFlagBuilder("flagkey").version(11).debugEventsUntilDate(futureTime).build(); - Event fe = EventFactory.DEFAULT.newFeatureRequestEvent(flag, user, + FeatureRequestEvent fe = EventFactory.DEFAULT.newFeatureRequestEvent(flag, user, new FeatureFlag.VariationAndValue(new Integer(1), new JsonPrimitive("value")), null); ep.sendEvent(fe); JsonArray output = flushAndGetEvents(); - assertEquals(2, output.size()); - assertIndexEventMatches(output.get(0), fe); - assertFeatureEventMatches(output.get(1), fe, flag, true); + assertThat(output, hasItems( + isIndexEvent(fe), + isFeatureEvent(fe, flag, true) + )); } + @SuppressWarnings("unchecked") @Test public void testTwoFeatureEventsForSameUserGenerateOnlyOneIndexEvent() throws Exception { ep = new EventProcessor(SDK_KEY, configBuilder.build()); FeatureFlag flag1 = new FeatureFlagBuilder("flagkey1").version(11).trackEvents(true).build(); FeatureFlag flag2 = new FeatureFlagBuilder("flagkey2").version(22).trackEvents(true).build(); - Event fe1 = EventFactory.DEFAULT.newFeatureRequestEvent(flag1, user, - new FeatureFlag.VariationAndValue(new Integer(1), new JsonPrimitive("value")), null); - Event fe2 = EventFactory.DEFAULT.newFeatureRequestEvent(flag2, user, - new FeatureFlag.VariationAndValue(new Integer(1), new JsonPrimitive("value")), null); + JsonElement value = new JsonPrimitive("value"); + FeatureRequestEvent fe1 = EventFactory.DEFAULT.newFeatureRequestEvent(flag1, user, + new FeatureFlag.VariationAndValue(new Integer(1), value), null); + FeatureRequestEvent fe2 = EventFactory.DEFAULT.newFeatureRequestEvent(flag2, user, + new FeatureFlag.VariationAndValue(new Integer(1), value), null); ep.sendEvent(fe1); ep.sendEvent(fe2); JsonArray output = flushAndGetEvents(); - assertEquals(3, output.size()); - assertIndexEventMatches(output.get(0), fe1); - assertFeatureEventMatches(output.get(1), fe1, flag1, false); - assertFeatureEventMatches(output.get(2), fe2, flag2, false); + assertThat(output, hasItems( + isIndexEvent(fe1), + isFeatureEvent(fe1, flag1, false), + isFeatureEvent(fe2, flag2, false), + isSummaryEvent(fe1.creationDate, fe2.creationDate) + )); } + @SuppressWarnings("unchecked") @Test public void nonTrackedEventsAreSummarized() throws Exception { ep = new EventProcessor(SDK_KEY, configBuilder.build()); FeatureFlag flag1 = new FeatureFlagBuilder("flagkey1").version(11).build(); FeatureFlag flag2 = new FeatureFlagBuilder("flagkey2").version(22).build(); + JsonElement value = new JsonPrimitive("value"); Event fe1 = EventFactory.DEFAULT.newFeatureRequestEvent(flag1, user, - new FeatureFlag.VariationAndValue(new Integer(1), new JsonPrimitive("value")), null); + new FeatureFlag.VariationAndValue(new Integer(1), value), null); Event fe2 = EventFactory.DEFAULT.newFeatureRequestEvent(flag2, user, - new FeatureFlag.VariationAndValue(new Integer(1), new JsonPrimitive("value")), null); + new FeatureFlag.VariationAndValue(new Integer(1), value), null); ep.sendEvent(fe1); ep.sendEvent(fe2); JsonArray output = flushAndGetEvents(); - assertEquals(2, output.size()); - assertIndexEventMatches(output.get(0), fe1); - - JsonObject seo = output.get(1).getAsJsonObject(); - assertEquals("summary", seo.get("kind").getAsString()); - assertEquals(fe1.creationDate, seo.get("startDate").getAsLong()); - assertEquals(fe2.creationDate, seo.get("endDate").getAsLong()); - JsonArray counters = seo.get("counters").getAsJsonArray(); - assertEquals(2, counters.size()); + assertThat(output, hasItems( + isIndexEvent(fe1), + allOf( + isSummaryEvent(fe1.creationDate, fe2.creationDate), + hasSummaryCounters(allOf( + hasItem(isSummaryEventCounter(flag1, value, 1)), + hasItem(isSummaryEventCounter(flag2, value, 1)) + ) + )) + )); } + @SuppressWarnings("unchecked") @Test public void customEventIsQueuedWithUser() throws Exception { ep = new EventProcessor(SDK_KEY, configBuilder.build()); @@ -137,16 +155,16 @@ public void customEventIsQueuedWithUser() throws Exception { ep.sendEvent(ce); JsonArray output = flushAndGetEvents(); - assertEquals(2, output.size()); - assertIndexEventMatches(output.get(0), ce); - - JsonObject expected = new JsonObject(); - expected.addProperty("kind", "custom"); - expected.addProperty("creationDate", ce.creationDate); - expected.addProperty("key", "eventkey"); - expected.addProperty("userKey", user.getKeyAsString()); - expected.add("data", data); - assertEquals(expected, output.get(1)); + assertThat(output, hasItems( + isIndexEvent(ce), + allOf( + hasJsonProperty("kind", "custom"), + hasJsonProperty("creationDate", (double)ce.creationDate), + hasJsonProperty("key", "eventkey"), + hasJsonProperty("userkey", user.getKeyAsString()), + hasJsonProperty("data", data) + ) + )); } @Test @@ -159,7 +177,7 @@ public void sdkKeyIsSent() throws Exception { ep.flush(); RecordedRequest req = server.takeRequest(); - assertEquals(SDK_KEY, req.getHeader("Authorization")); + assertThat(req.getHeader("Authorization"), equalTo(SDK_KEY)); } private JsonArray flushAndGetEvents() throws Exception { @@ -174,29 +192,44 @@ private JsonElement makeUserJson(LDUser user) { return configBuilder.build().gson.toJsonTree(user); } - private void assertUserMatches(LDUser user, JsonElement userJson) { - assertEquals(makeUserJson(user), userJson); + private Matcher isIndexEvent(Event sourceEvent) { + return allOf( + hasJsonProperty("kind", "index"), + hasJsonProperty("creationDate", (double)sourceEvent.creationDate), + hasJsonProperty("user", makeUserJson(sourceEvent.user)) + ); + } + + @SuppressWarnings("unchecked") + private Matcher isFeatureEvent(FeatureRequestEvent sourceEvent, FeatureFlag flag, boolean debug) { + return allOf( + hasJsonProperty("kind", "feature"), + hasJsonProperty("creationDate", (double)sourceEvent.creationDate), + hasJsonProperty("key", flag.getKey()), + hasJsonProperty("version", (double)flag.getVersion()), + hasJsonProperty("value", sourceEvent.value), + hasJsonProperty("userKey", sourceEvent.user.getKeyAsString()), + hasJsonProperty("debug", debug ? new JsonPrimitive(true) : null) + ); } - private void assertIndexEventMatches(JsonElement eventOutput, Event sourceEvent) { - JsonObject o = eventOutput.getAsJsonObject(); - assertEquals("index", o.get("kind").getAsString()); - assertEquals(sourceEvent.creationDate, o.get("creationDate").getAsLong()); - assertUserMatches(sourceEvent.user, o.get("user")); + private Matcher isSummaryEvent(long startDate, long endDate) { + return allOf( + hasJsonProperty("kind", "summary"), + hasJsonProperty("startDate", (double)startDate), + hasJsonProperty("endDate", (double)endDate) + ); } - private void assertFeatureEventMatches(JsonElement eventOutput, Event sourceEvent, FeatureFlag flag, boolean debug) { - JsonObject o = eventOutput.getAsJsonObject(); - assertEquals("feature", o.get("kind").getAsString()); - assertEquals(sourceEvent.creationDate, o.get("creationDate").getAsLong()); - assertEquals(flag.getKey(), o.get("key").getAsString()); - assertEquals(flag.getVersion(), o.get("version").getAsInt()); - assertEquals("value", o.get("value").getAsString()); - assertEquals(sourceEvent.user.getKeyAsString(), o.get("userKey").getAsString()); - if (debug) { - assertTrue(o.get("debug").getAsBoolean()); - } else { - assertNull(o.get("debug")); - } + private Matcher hasSummaryCounters(Matcher> matcher) { + return hasJsonProperty("counters", isJsonArray(matcher)); + } + + private Matcher isSummaryEventCounter(FeatureFlag flag, JsonElement value, int count) { + return allOf( + hasJsonProperty("version", (double)flag.getVersion()), + hasJsonProperty("value", value), + hasJsonProperty("count", (double)count) + ); } } diff --git a/src/test/java/com/launchdarkly/client/EventSummarizerTest.java b/src/test/java/com/launchdarkly/client/EventSummarizerTest.java index c080ad208..8ba8fab63 100644 --- a/src/test/java/com/launchdarkly/client/EventSummarizerTest.java +++ b/src/test/java/com/launchdarkly/client/EventSummarizerTest.java @@ -4,12 +4,10 @@ import org.junit.Test; -import java.util.Objects; - +import static org.hamcrest.MatcherAssert.assertThat; +import static org.hamcrest.Matchers.containsInAnyOrder; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertFalse; -import static org.junit.Assert.assertNotNull; -import static org.junit.Assert.assertNull; import static org.junit.Assert.assertTrue; public class EventSummarizerTest { @@ -54,33 +52,21 @@ public void oldestUserForgottenIfCapacityExceeded() { } @Test - public void summarizeEventReturnsFalseForIdentifyEvent() { - EventSummarizer es = new EventSummarizer(defaultConfig); - Event event = new IdentifyEvent(user); - assertFalse(es.summarizeEvent(event)); - } - - @Test - public void summarizeEventReturnsFalseForCustomEvent() { + public void summarizeEventDoesNothingForIdentifyEvent() { EventSummarizer es = new EventSummarizer(defaultConfig); - Event event = new CustomEvent("whatever", user, null); - assertFalse(es.summarizeEvent(event)); - } - - @Test - public void summarizeEventReturnsTrueForFeatureEventWithTrackEventsFalse() { - EventSummarizer es = new EventSummarizer(defaultConfig); - FeatureFlag flag = new FeatureFlagBuilder("key").build(); - Event event = eventFactory.newFeatureRequestEvent(flag, user, null, null); - assertTrue(es.summarizeEvent(event)); + EventSummarizer.EventsState snapshot = es.snapshot(); + es.summarizeEvent(eventFactory.newIdentifyEvent(user)); + + assertEquals(snapshot, es.snapshot()); } @Test - public void summarizeEventReturnsFalseForFeatureEventWithTrackEventsTrue() { + public void summarizeEventDoesNothingForCustomEvent() { EventSummarizer es = new EventSummarizer(defaultConfig); - FeatureFlag flag = new FeatureFlagBuilder("key").trackEvents(true).build(); - Event event = eventFactory.newFeatureRequestEvent(flag, user, null, null); - assertFalse(es.summarizeEvent(event)); + EventSummarizer.EventsState snapshot = es.snapshot(); + es.summarizeEvent(eventFactory.newCustomEvent("whatever", user, null)); + + assertEquals(snapshot, es.snapshot()); } @Test @@ -124,40 +110,14 @@ public void summarizeEventIncrementsCounters() { es.summarizeEvent(event5); EventSummarizer.SummaryOutput data = es.output(es.snapshot()); - assertEquals(4, data.counters.size()); - EventSummarizer.CounterData result1 = findCounter(data.counters, flag1.getKey(), "value1"); - assertNotNull(result1); - assertEquals(flag1.getKey(), result1.key); - assertEquals(new Integer(flag1.getVersion()), result1.version); - assertEquals(2, result1.count); - assertNull(result1.unknown); - EventSummarizer.CounterData result2 = findCounter(data.counters, flag1.getKey(), "value2"); - assertNotNull(result2); - assertEquals(flag1.getKey(), result2.key); - assertEquals(new Integer(flag1.getVersion()), result2.version); - assertEquals(1, result2.count); - assertNull(result2.unknown); - EventSummarizer.CounterData result3 = findCounter(data.counters, flag2.getKey(), "value99"); - assertNotNull(result3); - assertEquals(flag2.getKey(), result3.key); - assertEquals(new Integer(flag2.getVersion()), result3.version); - assertEquals(1, result3.count); - assertNull(result3.unknown); - EventSummarizer.CounterData result4 = findCounter(data.counters, unknownFlagKey, null); - assertNotNull(result4); - assertEquals(unknownFlagKey, result4.key); - assertNull(result4.version); - assertEquals(1, result4.count); - assertEquals(Boolean.TRUE, result4.unknown); - } - - private EventSummarizer.CounterData findCounter(Iterable counters, String key, String value) { - JsonPrimitive jv = value == null ? null : new JsonPrimitive(value); - for (EventSummarizer.CounterData c: counters) { - if (c.key.equals(key) && Objects.equals(c.value, jv)) { - return c; - } - } - return null; + EventSummarizer.CounterData expected1 = new EventSummarizer.CounterData(flag1.getKey(), + new JsonPrimitive("value1"), flag1.getVersion(), 2, null); + EventSummarizer.CounterData expected2 = new EventSummarizer.CounterData(flag1.getKey(), + new JsonPrimitive("value2"), flag1.getVersion(), 1, null); + EventSummarizer.CounterData expected3 = new EventSummarizer.CounterData(flag2.getKey(), + new JsonPrimitive("value99"), flag2.getVersion(), 1, null); + EventSummarizer.CounterData expected4 = new EventSummarizer.CounterData(unknownFlagKey, + null, null, 1, true); + assertThat(data.counters, containsInAnyOrder(expected1, expected2, expected3, expected4)); } } diff --git a/src/test/java/com/launchdarkly/client/TestUtils.java b/src/test/java/com/launchdarkly/client/TestUtils.java new file mode 100644 index 000000000..b5d0ebaeb --- /dev/null +++ b/src/test/java/com/launchdarkly/client/TestUtils.java @@ -0,0 +1,74 @@ +package com.launchdarkly.client; + +import com.google.gson.JsonArray; +import com.google.gson.JsonElement; +import com.google.gson.JsonPrimitive; + +import org.hamcrest.Description; +import org.hamcrest.Matcher; +import org.hamcrest.TypeSafeDiagnosingMatcher; + +import static org.hamcrest.Matchers.equalTo; + +public class TestUtils { + + public static Matcher hasJsonProperty(final String name, JsonElement value) { + return hasJsonProperty(name, equalTo(value)); + } + + public static Matcher hasJsonProperty(final String name, String value) { + return hasJsonProperty(name, new JsonPrimitive(value)); + } + + public static Matcher hasJsonProperty(final String name, int value) { + return hasJsonProperty(name, new JsonPrimitive(value)); + } + + public static Matcher hasJsonProperty(final String name, double value) { + return hasJsonProperty(name, new JsonPrimitive(value)); + } + + public static Matcher hasJsonProperty(final String name, boolean value) { + return hasJsonProperty(name, new JsonPrimitive(value)); + } + + public static Matcher hasJsonProperty(final String name, final Matcher matcher) { + return new TypeSafeDiagnosingMatcher() { + @Override + public void describeTo(Description description) { + description.appendText(name + ": "); + matcher.describeTo(description); + } + + @Override + protected boolean matchesSafely(JsonElement item, Description mismatchDescription) { + JsonElement value = item.getAsJsonObject().get(name); + if (!matcher.matches(value)) { + matcher.describeMismatch(value, mismatchDescription); + return false; + } + return true; + } + }; + } + + public static Matcher isJsonArray(final Matcher> matcher) { + return new TypeSafeDiagnosingMatcher() { + @Override + public void describeTo(Description description) { + description.appendText("array: "); + matcher.describeTo(description); + } + + @Override + protected boolean matchesSafely(JsonElement item, Description mismatchDescription) { + JsonArray value = item.getAsJsonArray(); + if (!matcher.matches(value)) { + matcher.describeMismatch(value, mismatchDescription); + return false; + } + return true; + } + }; + } +} From a74f1b9623befc39a9bb079b70e12cf27d2a2762 Mon Sep 17 00:00:00 2001 From: Eli Bishop Date: Fri, 9 Mar 2018 11:40:42 -0800 Subject: [PATCH 07/67] typo --- src/test/java/com/launchdarkly/client/EventProcessorTest.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/test/java/com/launchdarkly/client/EventProcessorTest.java b/src/test/java/com/launchdarkly/client/EventProcessorTest.java index e6a639251..474d4ba0a 100644 --- a/src/test/java/com/launchdarkly/client/EventProcessorTest.java +++ b/src/test/java/com/launchdarkly/client/EventProcessorTest.java @@ -161,7 +161,7 @@ public void customEventIsQueuedWithUser() throws Exception { hasJsonProperty("kind", "custom"), hasJsonProperty("creationDate", (double)ce.creationDate), hasJsonProperty("key", "eventkey"), - hasJsonProperty("userkey", user.getKeyAsString()), + hasJsonProperty("userKey", user.getKeyAsString()), hasJsonProperty("data", data) ) )); From 9f71981c30d61b4dc545a1a006ef88e758d5a38e Mon Sep 17 00:00:00 2001 From: Eli Bishop Date: Fri, 9 Mar 2018 14:30:35 -0800 Subject: [PATCH 08/67] improve synchronization, clean up event classes --- .../com/launchdarkly/client/CustomEvent.java | 9 +- .../java/com/launchdarkly/client/Event.java | 14 +- .../com/launchdarkly/client/EventFactory.java | 2 +- .../launchdarkly/client/EventProcessor.java | 316 ++++++++++++------ .../client/FeatureRequestEvent.java | 24 +- .../launchdarkly/client/IdentifyEvent.java | 8 +- .../com/launchdarkly/client/LDClient.java | 8 +- 7 files changed, 236 insertions(+), 145 deletions(-) diff --git a/src/main/java/com/launchdarkly/client/CustomEvent.java b/src/main/java/com/launchdarkly/client/CustomEvent.java index ce71db6ae..b2f88ef09 100644 --- a/src/main/java/com/launchdarkly/client/CustomEvent.java +++ b/src/main/java/com/launchdarkly/client/CustomEvent.java @@ -3,15 +3,12 @@ import com.google.gson.JsonElement; class CustomEvent extends Event { + final String key; final JsonElement data; - CustomEvent(String key, LDUser user, JsonElement data) { - super("custom", key, user); - this.data = data; - } - CustomEvent(long timestamp, String key, LDUser user, JsonElement data) { - super(timestamp, "custom", key, user); + super(timestamp, user); + this.key = key; this.data = data; } } diff --git a/src/main/java/com/launchdarkly/client/Event.java b/src/main/java/com/launchdarkly/client/Event.java index 345189d7e..0ce69fe8d 100644 --- a/src/main/java/com/launchdarkly/client/Event.java +++ b/src/main/java/com/launchdarkly/client/Event.java @@ -1,23 +1,11 @@ package com.launchdarkly.client; - class Event { long creationDate; - String key; - String kind; LDUser user; - Event(String kind, String key, LDUser user) { - this.creationDate = System.currentTimeMillis(); - this.key = key; - this.kind = kind; - this.user = user; - } - - Event(long creationDate, String kind, String key, LDUser user) { + Event(long creationDate, LDUser user) { this.creationDate = creationDate; - this.key = key; - this.kind = kind; this.user = user; } } \ No newline at end of file diff --git a/src/main/java/com/launchdarkly/client/EventFactory.java b/src/main/java/com/launchdarkly/client/EventFactory.java index 360939457..85b581858 100644 --- a/src/main/java/com/launchdarkly/client/EventFactory.java +++ b/src/main/java/com/launchdarkly/client/EventFactory.java @@ -29,7 +29,7 @@ public CustomEvent newCustomEvent(String key, LDUser user, JsonElement data) { } public IdentifyEvent newIdentifyEvent(LDUser user) { - return new IdentifyEvent(getTimestamp(), user.getKeyAsString(), user); + return new IdentifyEvent(getTimestamp(), user); } public static class DefaultEventFactory extends EventFactory { diff --git a/src/main/java/com/launchdarkly/client/EventProcessor.java b/src/main/java/com/launchdarkly/client/EventProcessor.java index 0e84c5f5b..75ff0a72e 100644 --- a/src/main/java/com/launchdarkly/client/EventProcessor.java +++ b/src/main/java/com/launchdarkly/client/EventProcessor.java @@ -4,10 +4,6 @@ import com.google.gson.JsonElement; import com.google.gson.annotations.SerializedName; -import okhttp3.MediaType; -import okhttp3.Request; -import okhttp3.RequestBody; -import okhttp3.Response; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -17,83 +13,164 @@ import java.util.ArrayList; import java.util.List; import java.util.Random; -import java.util.concurrent.*; +import java.util.concurrent.ArrayBlockingQueue; +import java.util.concurrent.BlockingQueue; +import java.util.concurrent.Executors; +import java.util.concurrent.ScheduledExecutorService; +import java.util.concurrent.ThreadFactory; +import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicBoolean; import java.util.concurrent.atomic.AtomicLong; -import java.util.concurrent.locks.ReentrantLock; + +import okhttp3.MediaType; +import okhttp3.Request; +import okhttp3.RequestBody; +import okhttp3.Response; class EventProcessor implements Closeable { + private static final Logger logger = LoggerFactory.getLogger(EventProcessor.class); private static final SimpleDateFormat HTTP_DATE_FORMAT = new SimpleDateFormat("EEE, dd MMM yyyy HH:mm:ss zzz"); + private static final int CHANNEL_BLOCK_MILLIS = 1000; private final ScheduledExecutorService scheduler; - private final Random random = new Random(); - private final BlockingQueue queue; + private final Thread mainThread; + private final BlockingQueue inputChannel; + private final ArrayList buffer; private final String sdkKey; private final LDConfig config; - private final Consumer consumer; private final EventSummarizer summarizer; - private AtomicLong lastKnownPastTime = new AtomicLong(0); - private final ReentrantLock lock = new ReentrantLock(); + private final Random random = new Random(); + private final AtomicLong lastKnownPastTime = new AtomicLong(0); + private final AtomicBoolean capacityExceeded = new AtomicBoolean(false); + private final AtomicBoolean stopped = new AtomicBoolean(false); EventProcessor(String sdkKey, LDConfig config) { this.sdkKey = sdkKey; - this.queue = new ArrayBlockingQueue<>(config.capacity); - this.consumer = new Consumer(config); + this.inputChannel = new ArrayBlockingQueue<>(config.capacity); + this.buffer = new ArrayList<>(config.capacity); this.summarizer = new EventSummarizer(config); this.config = config; + ThreadFactory threadFactory = new ThreadFactoryBuilder() .setDaemon(true) .setNameFormat("LaunchDarkly-EventProcessor-%d") .build(); - this.scheduler = Executors.newSingleThreadScheduledExecutor(threadFactory); - this.scheduler.scheduleAtFixedRate(consumer, 0, config.flushInterval, TimeUnit.SECONDS); + this.scheduler = Executors.newSingleThreadScheduledExecutor(threadFactory); + Runnable flusher = new Runnable() { + public void run() { + postMessageAsync(MessageType.FLUSH, null); + } + }; + this.scheduler.scheduleAtFixedRate(flusher, config.flushInterval, config.flushInterval, TimeUnit.SECONDS); Runnable userKeysFlusher = new Runnable() { public void run() { - lock.lock(); - try { - summarizer.resetUsers(); - } finally { - lock.unlock(); - } + postMessageAsync(MessageType.FLUSH_USERS, null); } }; - this.scheduler.scheduleAtFixedRate(userKeysFlusher, 0, config.userKeysFlushInterval, TimeUnit.SECONDS); + this.scheduler.scheduleAtFixedRate(userKeysFlusher, config.userKeysFlushInterval, config.userKeysFlushInterval, + TimeUnit.SECONDS); + + mainThread = threadFactory.newThread(new MainLoop()); + mainThread.start(); } - + + void sendEventAsync(Event e) { + postMessageAsync(MessageType.EVENT, e); + } + boolean sendEvent(Event e) { - lock.lock(); - try { - // For each user we haven't seen before, we add an index event - unless this is already - // an identify event for that user. - if (e.user != null && !summarizer.noticeUser(e.user)) { - if (!(e instanceof IdentifyEvent)) { - IndexEventOutput ie = new IndexEventOutput(e.creationDate, e.user); - if (!queue.offer(ie)) { - return false; + return postMessageAndWait(MessageType.EVENT, e); + } + + /** + * This task drains the input queue as quickly as possible. Everything here is done on a single + * thread so we don't have to synchronize on our internal structures; when it's time to flush, + * dispatchFlush will fire off another task to do the part that takes longer. + */ + private class MainLoop implements Runnable { + public void run() { + while (!stopped.get()) { + try { + EventProcessorMessage message = inputChannel.take(); + logger.debug("Processing: {}", message); + switch(message.type) { + case EVENT: + message.setResult(dispatchEvent(message.event)); + break; + case FLUSH: + dispatchFlush(message); + case FLUSH_USERS: + summarizer.resetUsers(); } + } catch (InterruptedException e) { } } - - // Always record the event in the summarizer. - summarizer.summarizeEvent(e); - - if (shouldTrackFullEvent(e)) { - if (config.samplingInterval > 0 && random.nextInt(config.samplingInterval) != 0) { - return true; + } + } + + private void postMessageAsync(MessageType type, Event event) { + postToChannel(new EventProcessorMessage(type, event, null)); + } + + private boolean postMessageAndWait(MessageType type, Event event) { + EventProcessorMessage message = new EventProcessorMessage(type, event, new AtomicBoolean()); + postToChannel(message); + return message.waitForResult(); + } + + private void postToChannel(EventProcessorMessage message) { + while (true) { + try { + if (inputChannel.offer(message, CHANNEL_BLOCK_MILLIS, TimeUnit.MILLISECONDS)) { + break; + } else { + // This doesn't mean that the output event buffer is full, but rather that the main thread is + // seriously backed up with not-yet-processed events. We shouldn't see this. + logger.warn("Events are being produced faster than they can be processed"); } + } catch (InterruptedException ex) { + } + } + } - EventOutput eventOutput = createEventOutput(e); - if (eventOutput == null) { + boolean dispatchEvent(Event e) { + // For each user we haven't seen before, we add an index event - unless this is already + // an identify event for that user. + if (e.user != null && !summarizer.noticeUser(e.user)) { + if (!(e instanceof IdentifyEvent)) { + IndexEvent ie = new IndexEvent(e.creationDate, e.user); + if (!queueEvent(ie)) { return false; - } else { - return queue.offer(eventOutput); } } - return true; - } finally { - lock.unlock(); } + + // Always record the event in the summarizer. + summarizer.summarizeEvent(e); + + if (shouldTrackFullEvent(e)) { + // Sampling interval applies only to fully-tracked events. + if (config.samplingInterval > 0 && random.nextInt(config.samplingInterval) != 0) { + return true; + } + // Queue the event as-is; we'll transform it into an output event when we're flushing + // (to avoid doing that work on our main thread). + return queueEvent(e); + } + return true; + } + + private boolean queueEvent(Event e) { + if (buffer.size() >= config.capacity) { + if (capacityExceeded.compareAndSet(false, true)) { + logger.warn("Exceeded event queue capacity. Increase capacity to avoid dropping events."); + } + return false; + } + capacityExceeded.set(false); + buffer.add(e); + return true; } private boolean shouldTrackFullEvent(Event e) { @@ -122,14 +199,17 @@ private EventOutput createEventOutput(Event e) { String userKey = e.user == null ? null : e.user.getKeyAsString(); if (e instanceof FeatureRequestEvent) { FeatureRequestEvent fe = (FeatureRequestEvent)e; + boolean isDebug = (!fe.trackEvents && fe.debugEventsUntilDate != null); return new FeatureRequestEventOutput(fe.creationDate, fe.key, userKey, fe.variation, fe.version, fe.value, fe.defaultVal, fe.prereqOf, - (!fe.trackEvents && fe.debugEventsUntilDate != null) ? Boolean.TRUE : null); + isDebug ? Boolean.TRUE : null); } else if (e instanceof IdentifyEvent) { return new IdentifyEventOutput(e.creationDate, e.user); } else if (e instanceof CustomEvent) { CustomEvent ce = (CustomEvent)e; - return new CustomEventOutput(e.creationDate, ce.key, userKey, ce.data); + return new CustomEventOutput(ce.creationDate, ce.key, userKey, ce.data); + } else if (e instanceof IndexEvent) { + return (IndexEvent)e; } else { return null; } @@ -137,69 +217,67 @@ private EventOutput createEventOutput(Event e) { @Override public void close() throws IOException { - scheduler.shutdown(); this.flush(); + scheduler.shutdown(); + stopped.set(true); + mainThread.interrupt(); } public void flush() { - this.consumer.flush(); + postMessageAndWait(MessageType.FLUSH, null); } - class Consumer implements Runnable { - private final Logger logger = LoggerFactory.getLogger(Consumer.class); - private final LDConfig config; - private final AtomicBoolean shutdown; - - Consumer(LDConfig config) { - this.config = config; - this.shutdown = new AtomicBoolean(false); + private void dispatchFlush(EventProcessorMessage message) { + Event[] events = buffer.toArray(new Event[buffer.size()]); + buffer.clear(); + EventSummarizer.EventsState snapshot = summarizer.snapshot(); + this.scheduler.schedule(new FlushTask(events, snapshot, message), 0, TimeUnit.SECONDS); + } + + class FlushTask implements Runnable { + private final Logger logger = LoggerFactory.getLogger(FlushTask.class); + private final Event[] events; + private final EventSummarizer.EventsState snapshot; + private final EventProcessorMessage message; + + FlushTask(Event[] events, EventSummarizer.EventsState snapshot, EventProcessorMessage message) { + this.events = events; + this.snapshot = snapshot; + this.message = message; } - - @Override + public void run() { - flush(); - } - - public void flush() { - List events = new ArrayList<>(queue.size()); - EventSummarizer.EventsState snapshot; - lock.lock(); - try { - queue.drainTo(events); - snapshot = summarizer.snapshot(); - } finally { - lock.unlock(); + List eventsOut = new ArrayList<>(events.length + 1); + for (Event event: events) { + eventsOut.add(createEventOutput(event)); } if (!snapshot.counters.isEmpty()) { EventSummarizer.SummaryOutput summary = summarizer.output(snapshot); SummaryEventOutput seo = new SummaryEventOutput(summary.startDate, summary.endDate, summary.counters); - events.add(seo); + eventsOut.add(seo); } - if (!events.isEmpty() && !shutdown.get()) { - postEvents(events); + if (!eventsOut.isEmpty()) { + postEvents(eventsOut); } + message.setResult(true); } - - private void postEvents(List events) { - - String json = config.gson.toJson(events); - logger.debug("Posting " + events.size() + " event(s) to " + config.eventsURI + " with payload: " + json); - - String content = config.gson.toJson(events); + + private void postEvents(List eventsOut) { + String json = config.gson.toJson(eventsOut); + logger.debug("Posting {} event(s) to {} with payload: {}", + eventsOut.size(), config.eventsURI, json); Request request = config.getRequestBuilder(sdkKey) .url(config.eventsURI.toString() + "/bulk") - .post(RequestBody.create(MediaType.parse("application/json; charset=utf-8"), content)) + .post(RequestBody.create(MediaType.parse("application/json; charset=utf-8"), json)) .addHeader("Content-Type", "application/json") .build(); - logger.debug("Posting " + events.size() + " event(s) using request: " + request); - try (Response response = config.httpClient.newCall(request).execute()) { if (!response.isSuccessful()) { logger.info("Got unexpected response when posting events: " + response); if (response.code() == 401) { - shutdown.set(true); + stopped.set(true); logger.error("Received 401 error, no further events will be posted since SDK key is invalid"); close(); } @@ -219,6 +297,54 @@ private void postEvents(List events) { } } + private static enum MessageType { + EVENT, + FLUSH, + FLUSH_USERS + } + + private static class EventProcessorMessage { + private final MessageType type; + private final Event event; + private final AtomicBoolean reply; // if non-null, used to signal back to caller + + private EventProcessorMessage(MessageType type, Event event, AtomicBoolean reply) { + this.type = type; + this.event = event; + this.reply = reply; + } + + void setResult(boolean result) { + if (reply != null) { + synchronized(reply) { + reply.set(result); + reply.notifyAll(); + } + } + } + + boolean waitForResult() { + if (reply == null) { + return false; + } + while (true) { + try { + synchronized(reply) { + reply.wait(); + return reply.get(); + } + } + catch (InterruptedException ex) { + } + } + } + + @Override + public String toString() { + return (event == null) ? type.toString() : (type + ": " + event.getClass().getSimpleName()); + } + } + private static interface EventOutput { } @SuppressWarnings("unused") @@ -250,15 +376,14 @@ private static class FeatureRequestEventOutput implements EventOutput { } @SuppressWarnings("unused") - private static class IdentifyEventOutput implements EventOutput { + private static class IdentifyEventOutput extends Event implements EventOutput { private final String kind; - private final long creationDate; - private final LDUser user; + private final String key; IdentifyEventOutput(long creationDate, LDUser user) { + super(creationDate, user); this.kind = "identify"; - this.creationDate = creationDate; - this.user = user; + this.key = user.getKeyAsString(); } } @@ -280,15 +405,12 @@ private static class CustomEventOutput implements EventOutput { } @SuppressWarnings("unused") - private static class IndexEventOutput implements EventOutput { + private static class IndexEvent extends Event implements EventOutput { private final String kind; - private final long creationDate; - private final LDUser user; - IndexEventOutput(long creationDate, LDUser user) { + IndexEvent(long creationDate, LDUser user) { + super(creationDate, user); this.kind = "index"; - this.creationDate = creationDate; - this.user = user; } } diff --git a/src/main/java/com/launchdarkly/client/FeatureRequestEvent.java b/src/main/java/com/launchdarkly/client/FeatureRequestEvent.java index d563acd00..72914d4cc 100644 --- a/src/main/java/com/launchdarkly/client/FeatureRequestEvent.java +++ b/src/main/java/com/launchdarkly/client/FeatureRequestEvent.java @@ -1,27 +1,21 @@ package com.launchdarkly.client; import com.google.gson.JsonElement; -import com.google.gson.annotations.SerializedName; class FeatureRequestEvent extends Event { - Integer variation; - - JsonElement value; - @SerializedName("default") - JsonElement defaultVal; - - @SerializedName("version") - Integer version; - - @SerializedName("prereqOf") - String prereqOf; - + final String key; + final Integer variation; + final JsonElement value; + final JsonElement defaultVal; + final Integer version; + final String prereqOf; boolean trackEvents; - Long debugEventsUntilDate; + final Long debugEventsUntilDate; FeatureRequestEvent(long timestamp, String key, LDUser user, Integer version, Integer variation, JsonElement value, JsonElement defaultVal, String prereqOf, boolean trackEvents, Long debugEventsUntilDate) { - super(timestamp, "feature", key, user); + super(timestamp, user); + this.key = key; this.version = version; this.variation = variation; this.value = value; diff --git a/src/main/java/com/launchdarkly/client/IdentifyEvent.java b/src/main/java/com/launchdarkly/client/IdentifyEvent.java index aa040e5c0..4b2c8d1c5 100644 --- a/src/main/java/com/launchdarkly/client/IdentifyEvent.java +++ b/src/main/java/com/launchdarkly/client/IdentifyEvent.java @@ -2,11 +2,7 @@ class IdentifyEvent extends Event { - IdentifyEvent(LDUser user) { - super("identify", user.getKeyAsString(), user); - } - - IdentifyEvent(long timestamp, String key, LDUser user) { - super(timestamp, "identify", key, user); + IdentifyEvent(long timestamp, LDUser user) { + super(timestamp, user); } } diff --git a/src/main/java/com/launchdarkly/client/LDClient.java b/src/main/java/com/launchdarkly/client/LDClient.java index 87a7b017c..bffe8d3d2 100644 --- a/src/main/java/com/launchdarkly/client/LDClient.java +++ b/src/main/java/com/launchdarkly/client/LDClient.java @@ -162,13 +162,7 @@ private boolean sendEvent(Event event) { if (isOffline() || !config.sendEvents) { return false; } - - boolean processed = eventProcessor.sendEvent(event); - if (processed) { - eventCapacityExceeded.compareAndSet(true, false); - } else if (eventCapacityExceeded.compareAndSet(false, true)) { - logger.warn("Exceeded event queue capacity. Increase capacity to avoid dropping events."); - } + eventProcessor.sendEventAsync(event); return true; } From 39398f46f1b8695b7e9bd3e91e29f8354ca873c5 Mon Sep 17 00:00:00 2001 From: Eli Bishop Date: Fri, 9 Mar 2018 16:40:13 -0800 Subject: [PATCH 09/67] fix test method names --- .../java/com/launchdarkly/client/EventProcessorTest.java | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/test/java/com/launchdarkly/client/EventProcessorTest.java b/src/test/java/com/launchdarkly/client/EventProcessorTest.java index 474d4ba0a..27e1ea125 100644 --- a/src/test/java/com/launchdarkly/client/EventProcessorTest.java +++ b/src/test/java/com/launchdarkly/client/EventProcessorTest.java @@ -48,7 +48,7 @@ public void teardown() throws Exception { @SuppressWarnings("unchecked") @Test - public void testIdentifyEventIsQueued() throws Exception { + public void identifyEventIsQueued() throws Exception { ep = new EventProcessor(SDK_KEY, configBuilder.build()); Event e = EventFactory.DEFAULT.newIdentifyEvent(user); ep.sendEvent(e); @@ -64,7 +64,7 @@ public void testIdentifyEventIsQueued() throws Exception { @SuppressWarnings("unchecked") @Test - public void testIndividualFeatureEventIsQueuedWithIndexEvent() throws Exception { + public void individualFeatureEventIsQueuedWithIndexEvent() throws Exception { ep = new EventProcessor(SDK_KEY, configBuilder.build()); FeatureFlag flag = new FeatureFlagBuilder("flagkey").version(11).trackEvents(true).build(); FeatureRequestEvent fe = EventFactory.DEFAULT.newFeatureRequestEvent(flag, user, @@ -80,7 +80,7 @@ public void testIndividualFeatureEventIsQueuedWithIndexEvent() throws Exception @SuppressWarnings("unchecked") @Test - public void testDebugFlagIsSetIfFlagIsTemporarilyInDebugMode() throws Exception { + public void debugFlagIsSetIfFlagIsTemporarilyInDebugMode() throws Exception { ep = new EventProcessor(SDK_KEY, configBuilder.build()); long futureTime = System.currentTimeMillis() + 1000000; FeatureFlag flag = new FeatureFlagBuilder("flagkey").version(11).debugEventsUntilDate(futureTime).build(); @@ -97,7 +97,7 @@ public void testDebugFlagIsSetIfFlagIsTemporarilyInDebugMode() throws Exception @SuppressWarnings("unchecked") @Test - public void testTwoFeatureEventsForSameUserGenerateOnlyOneIndexEvent() throws Exception { + public void twoFeatureEventsForSameUserGenerateOnlyOneIndexEvent() throws Exception { ep = new EventProcessor(SDK_KEY, configBuilder.build()); FeatureFlag flag1 = new FeatureFlagBuilder("flagkey1").version(11).trackEvents(true).build(); FeatureFlag flag2 = new FeatureFlagBuilder("flagkey2").version(22).trackEvents(true).build(); From 5d075626fa787cdd12877395ee53c68fe559913a Mon Sep 17 00:00:00 2001 From: Eli Bishop Date: Fri, 9 Mar 2018 16:49:38 -0800 Subject: [PATCH 10/67] fix client tests --- .../com/launchdarkly/client/LDClientTest.java | 48 +++++++++++-------- 1 file changed, 28 insertions(+), 20 deletions(-) diff --git a/src/test/java/com/launchdarkly/client/LDClientTest.java b/src/test/java/com/launchdarkly/client/LDClientTest.java index 30b2646ec..95dc3b1b9 100644 --- a/src/test/java/com/launchdarkly/client/LDClientTest.java +++ b/src/test/java/com/launchdarkly/client/LDClientTest.java @@ -6,6 +6,7 @@ import junit.framework.AssertionFailedError; +import org.easymock.EasyMock; import org.easymock.EasyMockSupport; import org.junit.Before; import org.junit.Test; @@ -22,6 +23,7 @@ import static com.launchdarkly.client.VersionedDataKind.SEGMENTS; import static org.easymock.EasyMock.anyObject; import static org.easymock.EasyMock.expect; +import static org.easymock.EasyMock.expectLastCall; import static org.junit.Assert.*; public class LDClientTest extends EasyMockSupport { @@ -68,7 +70,7 @@ public void testTestFeatureStoreSetFeatureTrue() throws IOException, Interrupted expect(initFuture.get(10L, TimeUnit.MILLISECONDS)).andReturn(new Object()); expect(pollingProcessor.start()).andReturn(initFuture); expect(pollingProcessor.initialized()).andReturn(true).times(1); - expect(eventProcessor.sendEvent(anyObject(Event.class))).andReturn(true); + expectEventsSent(1); replayAll(); client = createMockClient(config); @@ -108,7 +110,7 @@ public void testTestFeatureStoreSetFalse() throws IOException, InterruptedExcept expect(initFuture.get(10L, TimeUnit.MILLISECONDS)).andReturn(new Object()); expect(pollingProcessor.start()).andReturn(initFuture); expect(pollingProcessor.initialized()).andReturn(true).times(1); - expect(eventProcessor.sendEvent(anyObject(Event.class))).andReturn(true); + expectEventsSent(1); replayAll(); client = createMockClient(config); @@ -131,7 +133,7 @@ public void testTestFeatureStoreFlagTrueThenFalse() throws IOException, Interrup expect(initFuture.get(10L, TimeUnit.MILLISECONDS)).andReturn(new Object()); expect(pollingProcessor.start()).andReturn(initFuture); expect(pollingProcessor.initialized()).andReturn(true).times(2); - expect(eventProcessor.sendEvent(anyObject(Event.class))).andReturn(true).times(2); + expectEventsSent(2); replayAll(); client = createMockClient(config); @@ -158,7 +160,7 @@ public void testTestFeatureStoreIntegerVariation() throws Exception { expect(initFuture.get(10L, TimeUnit.MILLISECONDS)).andReturn(new Object()); expect(pollingProcessor.start()).andReturn(initFuture); expect(pollingProcessor.initialized()).andReturn(true).times(2); - expect(eventProcessor.sendEvent(anyObject(Event.class))).andReturn(true).times(2); + expectEventsSent(2); replayAll(); client = createMockClient(config); @@ -183,7 +185,7 @@ public void testTestFeatureStoreDoubleVariation() throws Exception { expect(initFuture.get(10L, TimeUnit.MILLISECONDS)).andReturn(new Object()); expect(pollingProcessor.start()).andReturn(initFuture); expect(pollingProcessor.initialized()).andReturn(true).times(2); - expect(eventProcessor.sendEvent(anyObject(Event.class))).andReturn(true).times(2); + expectEventsSent(2); replayAll(); client = createMockClient(config); @@ -208,7 +210,7 @@ public void testTestFeatureStoreStringVariation() throws Exception { expect(initFuture.get(10L, TimeUnit.MILLISECONDS)).andReturn(new Object()); expect(pollingProcessor.start()).andReturn(initFuture); expect(pollingProcessor.initialized()).andReturn(true).times(2); - expect(eventProcessor.sendEvent(anyObject(Event.class))).andReturn(true).times(2); + expectEventsSent(2); replayAll(); client = createMockClient(config); @@ -233,7 +235,7 @@ public void testTestFeatureStoreJsonVariationPrimitive() throws Exception { expect(initFuture.get(10L, TimeUnit.MILLISECONDS)).andReturn(new Object()); expect(pollingProcessor.start()).andReturn(initFuture); expect(pollingProcessor.initialized()).andReturn(true).times(4); - expect(eventProcessor.sendEvent(anyObject(Event.class))).andReturn(true).times(4); + expectEventsSent(4); replayAll(); client = createMockClient(config); @@ -265,7 +267,7 @@ public void testTestFeatureStoreJsonVariationArray() throws Exception { expect(initFuture.get(10L, TimeUnit.MILLISECONDS)).andReturn(new Object()); expect(pollingProcessor.start()).andReturn(initFuture); expect(pollingProcessor.initialized()).andReturn(true).times(2); - expect(eventProcessor.sendEvent(anyObject(Event.class))).andReturn(true).times(2); + expectEventsSent(2); replayAll(); client = createMockClient(config); @@ -365,7 +367,7 @@ public void testFeatureMatchesUserBySegment() throws Exception { expect(initFuture.get(10L, TimeUnit.MILLISECONDS)).andReturn(new Object()); expect(pollingProcessor.start()).andReturn(initFuture); expect(pollingProcessor.initialized()).andReturn(true).times(1); - expect(eventProcessor.sendEvent(anyObject(Event.class))).andReturn(true).times(1); + expectEventsSent(1); replayAll(); client = createMockClient(config); @@ -405,7 +407,7 @@ public void testUseLdd() throws IOException { client = createMockClient(config); // Asserting 2 things here: no pollingProcessor or streamingProcessor activity // and sending of event: - expect(eventProcessor.sendEvent(anyObject(Event.class))).andReturn(true); + expectEventsSent(1); replayAll(); assertDefaultValueIsReturned(); @@ -422,7 +424,7 @@ public void testStreamingNoWait() throws IOException { expect(streamProcessor.start()).andReturn(initFuture); expect(streamProcessor.initialized()).andReturn(false); - expect(eventProcessor.sendEvent(anyObject(Event.class))).andReturn(true); + expectEventsSent(1); replayAll(); client = createMockClient(config); @@ -455,7 +457,7 @@ public void testPollingNoWait() throws IOException { expect(pollingProcessor.start()).andReturn(initFuture); expect(pollingProcessor.initialized()).andReturn(false); - expect(eventProcessor.sendEvent(anyObject(Event.class))).andReturn(true); + expectEventsSent(1); replayAll(); client = createMockClient(config); @@ -473,7 +475,7 @@ public void testPollingWait() throws Exception { expect(pollingProcessor.start()).andReturn(initFuture); expect(initFuture.get(10L, TimeUnit.MILLISECONDS)).andThrow(new TimeoutException()); - expect(eventProcessor.sendEvent(anyObject(Event.class))).andReturn(true); + expectEventsSent(1); expect(pollingProcessor.initialized()).andReturn(false); replayAll(); @@ -502,8 +504,7 @@ public void testNoFeatureEventsAreSentWhenSendEventsIsFalse() throws Exception { expect(initFuture.get(5000L, TimeUnit.MILLISECONDS)).andThrow(new TimeoutException()); expect(pollingProcessor.start()).andReturn(initFuture); expect(pollingProcessor.initialized()).andReturn(true).anyTimes(); - expect(eventProcessor.sendEvent(anyObject(Event.class))) - .andThrow(new AssertionFailedError("should not have queued an event")).anyTimes(); + expectEventsSent(0); replayAll(); client = createMockClient(config); @@ -522,8 +523,7 @@ public void testNoIdentifyEventsAreSentWhenSendEventsIsFalse() throws Exception expect(initFuture.get(5000L, TimeUnit.MILLISECONDS)).andThrow(new TimeoutException()); expect(pollingProcessor.start()).andReturn(initFuture); expect(pollingProcessor.initialized()).andReturn(true).anyTimes(); - expect(eventProcessor.sendEvent(anyObject(Event.class))) - .andThrow(new AssertionFailedError("should not have queued an event")).anyTimes(); + expectEventsSent(0); replayAll(); client = createMockClient(config); @@ -542,8 +542,7 @@ public void testNoCustomEventsAreSentWhenSendEventsIsFalse() throws Exception { expect(initFuture.get(5000L, TimeUnit.MILLISECONDS)).andThrow(new TimeoutException()); expect(pollingProcessor.start()).andReturn(initFuture); expect(pollingProcessor.initialized()).andReturn(true).anyTimes(); - expect(eventProcessor.sendEvent(anyObject(Event.class))) - .andThrow(new AssertionFailedError("should not have queued an event")).anyTimes(); + expectEventsSent(0); replayAll(); client = createMockClient(config); @@ -564,7 +563,7 @@ public void testEvaluationCanUseFeatureStoreIfInitializationTimesOut() throws IO expect(streamProcessor.start()).andReturn(initFuture); expect(streamProcessor.initialized()).andReturn(false); - expect(eventProcessor.sendEvent(anyObject(Event.class))).andReturn(true); + expectEventsSent(1); replayAll(); client = createMockClient(config); @@ -580,6 +579,15 @@ private void assertDefaultValueIsReturned() { assertEquals(true, result); } + private void expectEventsSent(int count) { + eventProcessor.sendEventAsync(anyObject(Event.class)); + if (count > 0) { + expectLastCall().times(count); + } else { + expectLastCall().andThrow(new AssertionFailedError("should not have queued an event")).anyTimes(); + } + } + private LDClientInterface createMockClient(LDConfig config) { return new LDClient("SDK_KEY", config) { @Override From 04757448c0f648bab4ae42d1794d49dc902c0952 Mon Sep 17 00:00:00 2001 From: Eli Bishop Date: Fri, 9 Mar 2018 16:58:11 -0800 Subject: [PATCH 11/67] add debugging --- src/main/java/com/launchdarkly/client/EventProcessor.java | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/main/java/com/launchdarkly/client/EventProcessor.java b/src/main/java/com/launchdarkly/client/EventProcessor.java index 75ff0a72e..55250e196 100644 --- a/src/main/java/com/launchdarkly/client/EventProcessor.java +++ b/src/main/java/com/launchdarkly/client/EventProcessor.java @@ -217,6 +217,7 @@ private EventOutput createEventOutput(Event e) { @Override public void close() throws IOException { + logger.debug("Shutting down event processor"); this.flush(); scheduler.shutdown(); stopped.set(true); @@ -341,7 +342,8 @@ boolean waitForResult() { @Override public String toString() { - return (event == null) ? type.toString() : (type + ": " + event.getClass().getSimpleName()); + return ((event == null) ? type.toString() : (type + ": " + event.getClass().getSimpleName())) + + (reply == null ? "" : " (sync)"); } } From ea4c7fd2b1c88a87faac5905b563ac17338ce44e Mon Sep 17 00:00:00 2001 From: Eli Bishop Date: Fri, 9 Mar 2018 17:26:56 -0800 Subject: [PATCH 12/67] more debugging --- src/main/java/com/launchdarkly/client/EventProcessor.java | 1 + 1 file changed, 1 insertion(+) diff --git a/src/main/java/com/launchdarkly/client/EventProcessor.java b/src/main/java/com/launchdarkly/client/EventProcessor.java index 55250e196..430f68f7e 100644 --- a/src/main/java/com/launchdarkly/client/EventProcessor.java +++ b/src/main/java/com/launchdarkly/client/EventProcessor.java @@ -317,6 +317,7 @@ private EventProcessorMessage(MessageType type, Event event, AtomicBoolean reply void setResult(boolean result) { if (reply != null) { + logger.debug("completed: " + this); synchronized(reply) { reply.set(result); reply.notifyAll(); From 74ca6f713a164f1c0730698daa7a1ee1f1c59acd Mon Sep 17 00:00:00 2001 From: Eli Bishop Date: Fri, 9 Mar 2018 17:37:04 -0800 Subject: [PATCH 13/67] more debugging --- src/main/java/com/launchdarkly/client/EventProcessor.java | 1 + 1 file changed, 1 insertion(+) diff --git a/src/main/java/com/launchdarkly/client/EventProcessor.java b/src/main/java/com/launchdarkly/client/EventProcessor.java index 430f68f7e..6ff21f3c2 100644 --- a/src/main/java/com/launchdarkly/client/EventProcessor.java +++ b/src/main/java/com/launchdarkly/client/EventProcessor.java @@ -221,6 +221,7 @@ public void close() throws IOException { this.flush(); scheduler.shutdown(); stopped.set(true); + logger.debug("Stopped: " + stopped.get()); mainThread.interrupt(); } From 647cba8bcd9b418971f8009a6d2c920f60748864 Mon Sep 17 00:00:00 2001 From: Eli Bishop Date: Fri, 9 Mar 2018 17:50:07 -0800 Subject: [PATCH 14/67] use semaphore + more debugging --- .../launchdarkly/client/EventProcessor.java | 33 ++++++++++--------- 1 file changed, 18 insertions(+), 15 deletions(-) diff --git a/src/main/java/com/launchdarkly/client/EventProcessor.java b/src/main/java/com/launchdarkly/client/EventProcessor.java index 6ff21f3c2..12cc979fa 100644 --- a/src/main/java/com/launchdarkly/client/EventProcessor.java +++ b/src/main/java/com/launchdarkly/client/EventProcessor.java @@ -17,6 +17,7 @@ import java.util.concurrent.BlockingQueue; import java.util.concurrent.Executors; import java.util.concurrent.ScheduledExecutorService; +import java.util.concurrent.Semaphore; import java.util.concurrent.ThreadFactory; import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicBoolean; @@ -110,11 +111,11 @@ public void run() { } private void postMessageAsync(MessageType type, Event event) { - postToChannel(new EventProcessorMessage(type, event, null)); + postToChannel(new EventProcessorMessage(type, event, false)); } private boolean postMessageAndWait(MessageType type, Event event) { - EventProcessorMessage message = new EventProcessorMessage(type, event, new AtomicBoolean()); + EventProcessorMessage message = new EventProcessorMessage(type, event, true); postToChannel(message); return message.waitForResult(); } @@ -219,7 +220,11 @@ private EventOutput createEventOutput(Event e) { public void close() throws IOException { logger.debug("Shutting down event processor"); this.flush(); - scheduler.shutdown(); + try { + scheduler.shutdown(); + } catch (Exception e) { + logger.warn("failed to shut down scheduler: " + e); + } stopped.set(true); logger.debug("Stopped: " + stopped.get()); mainThread.interrupt(); @@ -308,21 +313,20 @@ private static enum MessageType { private static class EventProcessorMessage { private final MessageType type; private final Event event; - private final AtomicBoolean reply; // if non-null, used to signal back to caller + private final AtomicBoolean result = new AtomicBoolean(false); + private final Semaphore reply; - private EventProcessorMessage(MessageType type, Event event, AtomicBoolean reply) { + private EventProcessorMessage(MessageType type, Event event, boolean sync) { this.type = type; this.event = event; - this.reply = reply; + reply = sync ? new Semaphore(0) : null; } - void setResult(boolean result) { + void setResult(boolean value) { + result.set(value); if (reply != null) { logger.debug("completed: " + this); - synchronized(reply) { - reply.set(result); - reply.notifyAll(); - } + reply.release(); } } @@ -332,10 +336,9 @@ boolean waitForResult() { } while (true) { try { - synchronized(reply) { - reply.wait(); - return reply.get(); - } + reply.acquire(); + logger.debug("received result for " + this); + return result.get(); } catch (InterruptedException ex) { } From bf5a63c25e10e5009c5b731bf1b1072533f900c3 Mon Sep 17 00:00:00 2001 From: Eli Bishop Date: Fri, 9 Mar 2018 18:01:41 -0800 Subject: [PATCH 15/67] less debugging --- .../java/com/launchdarkly/client/EventProcessor.java | 11 +---------- 1 file changed, 1 insertion(+), 10 deletions(-) diff --git a/src/main/java/com/launchdarkly/client/EventProcessor.java b/src/main/java/com/launchdarkly/client/EventProcessor.java index 12cc979fa..767c0f5b7 100644 --- a/src/main/java/com/launchdarkly/client/EventProcessor.java +++ b/src/main/java/com/launchdarkly/client/EventProcessor.java @@ -94,7 +94,6 @@ public void run() { while (!stopped.get()) { try { EventProcessorMessage message = inputChannel.take(); - logger.debug("Processing: {}", message); switch(message.type) { case EVENT: message.setResult(dispatchEvent(message.event)); @@ -218,15 +217,9 @@ private EventOutput createEventOutput(Event e) { @Override public void close() throws IOException { - logger.debug("Shutting down event processor"); this.flush(); - try { - scheduler.shutdown(); - } catch (Exception e) { - logger.warn("failed to shut down scheduler: " + e); - } + scheduler.shutdown(); stopped.set(true); - logger.debug("Stopped: " + stopped.get()); mainThread.interrupt(); } @@ -325,7 +318,6 @@ private EventProcessorMessage(MessageType type, Event event, boolean sync) { void setResult(boolean value) { result.set(value); if (reply != null) { - logger.debug("completed: " + this); reply.release(); } } @@ -337,7 +329,6 @@ boolean waitForResult() { while (true) { try { reply.acquire(); - logger.debug("received result for " + this); return result.get(); } catch (InterruptedException ex) { From 00ea8467456c0afeb624cf2d641c85281bad2519 Mon Sep 17 00:00:00 2001 From: Eli Bishop Date: Fri, 9 Mar 2018 18:06:22 -0800 Subject: [PATCH 16/67] increase timeouts on PollingProcessor tests --- .../java/com/launchdarkly/client/PollingProcessorTest.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/test/java/com/launchdarkly/client/PollingProcessorTest.java b/src/test/java/com/launchdarkly/client/PollingProcessorTest.java index 30d06bb99..fef22cb31 100644 --- a/src/test/java/com/launchdarkly/client/PollingProcessorTest.java +++ b/src/test/java/com/launchdarkly/client/PollingProcessorTest.java @@ -26,7 +26,7 @@ public void testConnectionOk() throws Exception { replayAll(); Future initFuture = pollingProcessor.start(); - initFuture.get(100, TimeUnit.MILLISECONDS); + initFuture.get(1000, TimeUnit.MILLISECONDS); assertTrue(pollingProcessor.initialized()); pollingProcessor.close(); verifyAll(); @@ -44,7 +44,7 @@ public void testConnectionProblem() throws Exception { Future initFuture = pollingProcessor.start(); try { - initFuture.get(100L, TimeUnit.MILLISECONDS); + initFuture.get(200L, TimeUnit.MILLISECONDS); fail("Expected Timeout, instead initFuture.get() returned."); } catch (TimeoutException ignored) { } From d47e364fc9e8ed67b5d735dd9e56a49c19d85ced Mon Sep 17 00:00:00 2001 From: Eli Bishop Date: Fri, 9 Mar 2018 18:06:22 -0800 Subject: [PATCH 17/67] increase timeouts on PollingProcessor tests --- .../java/com/launchdarkly/client/PollingProcessorTest.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/test/java/com/launchdarkly/client/PollingProcessorTest.java b/src/test/java/com/launchdarkly/client/PollingProcessorTest.java index 30d06bb99..fef22cb31 100644 --- a/src/test/java/com/launchdarkly/client/PollingProcessorTest.java +++ b/src/test/java/com/launchdarkly/client/PollingProcessorTest.java @@ -26,7 +26,7 @@ public void testConnectionOk() throws Exception { replayAll(); Future initFuture = pollingProcessor.start(); - initFuture.get(100, TimeUnit.MILLISECONDS); + initFuture.get(1000, TimeUnit.MILLISECONDS); assertTrue(pollingProcessor.initialized()); pollingProcessor.close(); verifyAll(); @@ -44,7 +44,7 @@ public void testConnectionProblem() throws Exception { Future initFuture = pollingProcessor.start(); try { - initFuture.get(100L, TimeUnit.MILLISECONDS); + initFuture.get(200L, TimeUnit.MILLISECONDS); fail("Expected Timeout, instead initFuture.get() returned."); } catch (TimeoutException ignored) { } From edf5e26d9dc0484db4fc70807983fc3a7fd2ad81 Mon Sep 17 00:00:00 2001 From: Eli Bishop Date: Fri, 9 Mar 2018 18:19:26 -0800 Subject: [PATCH 18/67] comment --- src/main/java/com/launchdarkly/client/EventSummarizer.java | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/main/java/com/launchdarkly/client/EventSummarizer.java b/src/main/java/com/launchdarkly/client/EventSummarizer.java index 8c2d3000d..8843862e0 100644 --- a/src/main/java/com/launchdarkly/client/EventSummarizer.java +++ b/src/main/java/com/launchdarkly/client/EventSummarizer.java @@ -11,7 +11,9 @@ /** * Manages the state of summarizable information for the EventProcessor, including the - * event counters and user deduplication. + * event counters and user deduplication. Note that the methods of this class are + * deliberately not thread-safe, because they should always be called from EventProcessor's + * single event-processing thread. */ class EventSummarizer { private EventsState eventsState; From 2d996055d21ac9c2e7601dff98207c94f1774a91 Mon Sep 17 00:00:00 2001 From: Eli Bishop Date: Mon, 12 Mar 2018 11:44:31 -0700 Subject: [PATCH 19/67] add "debug" event kind --- src/main/java/com/launchdarkly/client/EventProcessor.java | 8 +++----- .../java/com/launchdarkly/client/EventProcessorTest.java | 8 +++----- 2 files changed, 6 insertions(+), 10 deletions(-) diff --git a/src/main/java/com/launchdarkly/client/EventProcessor.java b/src/main/java/com/launchdarkly/client/EventProcessor.java index 767c0f5b7..9f883efd6 100644 --- a/src/main/java/com/launchdarkly/client/EventProcessor.java +++ b/src/main/java/com/launchdarkly/client/EventProcessor.java @@ -202,7 +202,7 @@ private EventOutput createEventOutput(Event e) { boolean isDebug = (!fe.trackEvents && fe.debugEventsUntilDate != null); return new FeatureRequestEventOutput(fe.creationDate, fe.key, userKey, fe.variation, fe.version, fe.value, fe.defaultVal, fe.prereqOf, - isDebug ? Boolean.TRUE : null); + isDebug); } else if (e instanceof IdentifyEvent) { return new IdentifyEventOutput(e.creationDate, e.user); } else if (e instanceof CustomEvent) { @@ -356,11 +356,10 @@ private static class FeatureRequestEventOutput implements EventOutput { private final JsonElement value; @SerializedName("default") private final JsonElement defaultVal; private final String prereqOf; - private final Boolean debug; FeatureRequestEventOutput(long creationDate, String key, String userKey, Integer variation, - Integer version, JsonElement value, JsonElement defaultVal, String prereqOf, Boolean debug) { - this.kind = "feature"; + Integer version, JsonElement value, JsonElement defaultVal, String prereqOf, boolean debug) { + this.kind = debug ? "debug" : "feature"; this.creationDate = creationDate; this.key = key; this.userKey = userKey; @@ -369,7 +368,6 @@ private static class FeatureRequestEventOutput implements EventOutput { this.value = value; this.defaultVal = defaultVal; this.prereqOf = prereqOf; - this.debug = debug; } } diff --git a/src/test/java/com/launchdarkly/client/EventProcessorTest.java b/src/test/java/com/launchdarkly/client/EventProcessorTest.java index 27e1ea125..5842019d5 100644 --- a/src/test/java/com/launchdarkly/client/EventProcessorTest.java +++ b/src/test/java/com/launchdarkly/client/EventProcessorTest.java @@ -80,7 +80,7 @@ public void individualFeatureEventIsQueuedWithIndexEvent() throws Exception { @SuppressWarnings("unchecked") @Test - public void debugFlagIsSetIfFlagIsTemporarilyInDebugMode() throws Exception { + public void eventKindIsDebugIfFlagIsTemporarilyInDebugMode() throws Exception { ep = new EventProcessor(SDK_KEY, configBuilder.build()); long futureTime = System.currentTimeMillis() + 1000000; FeatureFlag flag = new FeatureFlagBuilder("flagkey").version(11).debugEventsUntilDate(futureTime).build(); @@ -200,16 +200,14 @@ private Matcher isIndexEvent(Event sourceEvent) { ); } - @SuppressWarnings("unchecked") private Matcher isFeatureEvent(FeatureRequestEvent sourceEvent, FeatureFlag flag, boolean debug) { return allOf( - hasJsonProperty("kind", "feature"), + hasJsonProperty("kind", debug ? "debug" : "feature"), hasJsonProperty("creationDate", (double)sourceEvent.creationDate), hasJsonProperty("key", flag.getKey()), hasJsonProperty("version", (double)flag.getVersion()), hasJsonProperty("value", sourceEvent.value), - hasJsonProperty("userKey", sourceEvent.user.getKeyAsString()), - hasJsonProperty("debug", debug ? new JsonPrimitive(true) : null) + hasJsonProperty("userKey", sourceEvent.user.getKeyAsString()) ); } From 603eb21147cf029cf37b52a728ca44b020cf976d Mon Sep 17 00:00:00 2001 From: Eli Bishop Date: Mon, 12 Mar 2018 17:13:27 -0700 Subject: [PATCH 20/67] update for latest summary event schema with features map --- .../launchdarkly/client/EventProcessor.java | 9 ++- .../launchdarkly/client/EventSummarizer.java | 80 +++++++++++++------ .../client/EventProcessorTest.java | 24 +++--- .../client/EventSummarizerTest.java | 48 ++++++++--- 4 files changed, 113 insertions(+), 48 deletions(-) diff --git a/src/main/java/com/launchdarkly/client/EventProcessor.java b/src/main/java/com/launchdarkly/client/EventProcessor.java index 9f883efd6..268135274 100644 --- a/src/main/java/com/launchdarkly/client/EventProcessor.java +++ b/src/main/java/com/launchdarkly/client/EventProcessor.java @@ -12,6 +12,7 @@ import java.text.SimpleDateFormat; import java.util.ArrayList; import java.util.List; +import java.util.Map; import java.util.Random; import java.util.concurrent.ArrayBlockingQueue; import java.util.concurrent.BlockingQueue; @@ -253,7 +254,7 @@ public void run() { } if (!snapshot.counters.isEmpty()) { EventSummarizer.SummaryOutput summary = summarizer.output(snapshot); - SummaryEventOutput seo = new SummaryEventOutput(summary.startDate, summary.endDate, summary.counters); + SummaryEventOutput seo = new SummaryEventOutput(summary.startDate, summary.endDate, summary.features); eventsOut.add(seo); } if (!eventsOut.isEmpty()) { @@ -415,13 +416,13 @@ private static class SummaryEventOutput implements EventOutput { private final String kind; private final long startDate; private final long endDate; - private final List counters; + private final Map features; - SummaryEventOutput(long startDate, long endDate, List counters) { + SummaryEventOutput(long startDate, long endDate, Map features) { this.kind = "summary"; this.startDate = startDate; this.endDate = endDate; - this.counters = counters; + this.features = features; } } } diff --git a/src/main/java/com/launchdarkly/client/EventSummarizer.java b/src/main/java/com/launchdarkly/client/EventSummarizer.java index 8843862e0..46bb87b34 100644 --- a/src/main/java/com/launchdarkly/client/EventSummarizer.java +++ b/src/main/java/com/launchdarkly/client/EventSummarizer.java @@ -1,6 +1,7 @@ package com.launchdarkly.client; import com.google.gson.JsonElement; +import com.google.gson.annotations.SerializedName; import java.util.ArrayList; import java.util.HashMap; @@ -51,7 +52,7 @@ void resetUsers() { void summarizeEvent(Event event) { if (event instanceof FeatureRequestEvent) { FeatureRequestEvent fe = (FeatureRequestEvent)event; - eventsState.incrementCounter(fe.key, fe.variation, fe.version, fe.value); + eventsState.incrementCounter(fe.key, fe.variation, fe.version, fe.value, fe.defaultVal); eventsState.noteTimestamp(fe.creationDate); } } @@ -72,16 +73,20 @@ EventsState snapshot() { * @return the formatted output */ SummaryOutput output(EventsState snapshot) { - List countersOut = new ArrayList<>(snapshot.counters.size()); + Map flagsOut = new HashMap<>(); for (Map.Entry entry: snapshot.counters.entrySet()) { - CounterData c = new CounterData(entry.getKey().key, - entry.getValue().flagValue, + FlagSummaryData fsd = flagsOut.get(entry.getKey().key); + if (fsd == null) { + fsd = new FlagSummaryData(entry.getValue().defaultVal, new ArrayList()); + flagsOut.put(entry.getKey().key, fsd); + } + CounterData c = new CounterData(entry.getValue().flagValue, entry.getKey().version == 0 ? null : entry.getKey().version, entry.getValue().count, entry.getKey().version == 0 ? true : null); - countersOut.add(c); + fsd.counters.add(c); } - return new SummaryOutput(snapshot.startDate, snapshot.endDate, countersOut); + return new SummaryOutput(snapshot.startDate, snapshot.endDate, flagsOut); } @SuppressWarnings("serial") @@ -109,7 +114,7 @@ static class EventsState { counters = new HashMap(); } - void incrementCounter(String flagKey, Integer variation, Integer version, JsonElement flagValue) { + void incrementCounter(String flagKey, Integer variation, Integer version, JsonElement flagValue, JsonElement defaultVal) { CounterKey key = new CounterKey(flagKey, (variation == null) ? 0 : variation.intValue(), (version == null) ? 0 : version.intValue()); @@ -117,7 +122,7 @@ void incrementCounter(String flagKey, Integer variation, Integer version, JsonEl if (value != null) { value.increment(); } else { - counters.put(key, new CounterValue(1, flagValue)); + counters.put(key, new CounterValue(1, flagValue, defaultVal)); } } @@ -173,11 +178,13 @@ public int hashCode() { private static class CounterValue { private int count; - private JsonElement flagValue; + private final JsonElement flagValue; + private final JsonElement defaultVal; - CounterValue(int count, JsonElement flagValue) { + CounterValue(int count, JsonElement flagValue, JsonElement defaultVal) { this.count = count; this.flagValue = flagValue; + this.defaultVal = defaultVal; } void increment() { @@ -185,15 +192,42 @@ void increment() { } } + static class FlagSummaryData { + @SerializedName("default") final JsonElement defaultVal; + final List counters; + + FlagSummaryData(JsonElement defaultVal, List counters) { + this.defaultVal = defaultVal; + this.counters = counters; + } + + @Override + public boolean equals(Object other) { + if (other instanceof FlagSummaryData) { + FlagSummaryData o = (FlagSummaryData)other; + return Objects.equals(defaultVal, o.defaultVal) && counters.equals(o.counters); + } + return false; + } + + @Override + public int hashCode() { + return Objects.hashCode(defaultVal) + 31 * counters.hashCode(); + } + + @Override + public String toString() { + return "{" + defaultVal + ", " + counters + "}"; + } + } + static class CounterData { - final String key; final JsonElement value; final Integer version; final int count; final Boolean unknown; - CounterData(String key, JsonElement value, Integer version, int count, Boolean unknown) { - this.key = key; + CounterData(JsonElement value, Integer version, int count, Boolean unknown) { this.value = value; this.version = version; this.count = count; @@ -204,7 +238,7 @@ static class CounterData { public boolean equals(Object other) { if (other instanceof CounterData) { CounterData o = (CounterData)other; - return o.key.equals(key) && Objects.equals(value, o.value) && Objects.equals(version, o.version) && + return Objects.equals(value, o.value) && Objects.equals(version, o.version) && o.count == count && Objects.deepEquals(unknown, o.unknown); } return false; @@ -212,44 +246,44 @@ public boolean equals(Object other) { @Override public int hashCode() { - return key.hashCode() + 31 * (Objects.hashCode(value) + 31 * (Objects.hashCode(version) + 31 * - (count + 31 * (Objects.hashCode(unknown))))); + return Objects.hashCode(value) + 31 * (Objects.hashCode(version) + 31 * + (count + 31 * (Objects.hashCode(unknown)))); } @Override public String toString() { - return "{" + key + ", " + value + ", " + version + ", " + count + ", " + unknown + "}"; + return "{" + value + ", " + version + ", " + count + ", " + unknown + "}"; } } static class SummaryOutput { final long startDate; final long endDate; - final List counters; + final Map features; - SummaryOutput(long startDate, long endDate, List counters) { + SummaryOutput(long startDate, long endDate, Map features) { this.startDate = startDate; this.endDate = endDate; - this.counters = counters; + this.features = features; } @Override public boolean equals(Object other) { if (other instanceof SummaryOutput) { SummaryOutput o = (SummaryOutput)other; - return o.startDate == startDate && o.endDate == endDate && o.counters.equals(counters); + return o.startDate == startDate && o.endDate == endDate && o.features.equals(features); } return false; } @Override public int hashCode() { - return counters.hashCode() + 31 * ((int)startDate + 31 * (int)endDate); + return features.hashCode() + 31 * ((int)startDate + 31 * (int)endDate); } @Override public String toString() { - return "{" + startDate + ", " + endDate + ", " + counters + "}"; + return "{" + startDate + ", " + endDate + ", " + features + "}"; } } } diff --git a/src/test/java/com/launchdarkly/client/EventProcessorTest.java b/src/test/java/com/launchdarkly/client/EventProcessorTest.java index 5842019d5..ae5b97705 100644 --- a/src/test/java/com/launchdarkly/client/EventProcessorTest.java +++ b/src/test/java/com/launchdarkly/client/EventProcessorTest.java @@ -125,10 +125,12 @@ public void nonTrackedEventsAreSummarized() throws Exception { FeatureFlag flag1 = new FeatureFlagBuilder("flagkey1").version(11).build(); FeatureFlag flag2 = new FeatureFlagBuilder("flagkey2").version(22).build(); JsonElement value = new JsonPrimitive("value"); + JsonElement default1 = new JsonPrimitive("default1"); + JsonElement default2 = new JsonPrimitive("default2"); Event fe1 = EventFactory.DEFAULT.newFeatureRequestEvent(flag1, user, - new FeatureFlag.VariationAndValue(new Integer(1), value), null); + new FeatureFlag.VariationAndValue(new Integer(1), value), default1); Event fe2 = EventFactory.DEFAULT.newFeatureRequestEvent(flag2, user, - new FeatureFlag.VariationAndValue(new Integer(1), value), null); + new FeatureFlag.VariationAndValue(new Integer(1), value), default2); ep.sendEvent(fe1); ep.sendEvent(fe2); @@ -137,11 +139,11 @@ public void nonTrackedEventsAreSummarized() throws Exception { isIndexEvent(fe1), allOf( isSummaryEvent(fe1.creationDate, fe2.creationDate), - hasSummaryCounters(allOf( - hasItem(isSummaryEventCounter(flag1, value, 1)), - hasItem(isSummaryEventCounter(flag2, value, 1)) - ) - )) + hasSummaryFlag(flag1.getKey(), default1, + hasItem(isSummaryEventCounter(flag1, value, 1))), + hasSummaryFlag(flag2.getKey(), default2, + hasItem(isSummaryEventCounter(flag2, value, 1))) + ) )); } @@ -219,8 +221,12 @@ private Matcher isSummaryEvent(long startDate, long endDate) { ); } - private Matcher hasSummaryCounters(Matcher> matcher) { - return hasJsonProperty("counters", isJsonArray(matcher)); + private Matcher hasSummaryFlag(String key, JsonElement defaultVal, Matcher> counters) { + return hasJsonProperty("features", + hasJsonProperty(key, allOf( + hasJsonProperty("default", defaultVal), + hasJsonProperty("counters", isJsonArray(counters)) + ))); } private Matcher isSummaryEventCounter(FeatureFlag flag, JsonElement value, int count) { diff --git a/src/test/java/com/launchdarkly/client/EventSummarizerTest.java b/src/test/java/com/launchdarkly/client/EventSummarizerTest.java index 8ba8fab63..fc86f4eed 100644 --- a/src/test/java/com/launchdarkly/client/EventSummarizerTest.java +++ b/src/test/java/com/launchdarkly/client/EventSummarizerTest.java @@ -1,11 +1,18 @@ package com.launchdarkly.client; +import com.google.gson.JsonElement; import com.google.gson.JsonPrimitive; +import com.launchdarkly.client.EventSummarizer.CounterData; import org.junit.Test; +import java.util.Arrays; +import java.util.Comparator; +import java.util.HashMap; +import java.util.Map; + import static org.hamcrest.MatcherAssert.assertThat; -import static org.hamcrest.Matchers.containsInAnyOrder; +import static org.hamcrest.Matchers.equalTo; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertFalse; import static org.junit.Assert.assertTrue; @@ -94,15 +101,18 @@ public void summarizeEventIncrementsCounters() { FeatureFlag flag1 = new FeatureFlagBuilder("key1").version(11).build(); FeatureFlag flag2 = new FeatureFlagBuilder("key2").version(22).build(); String unknownFlagKey = "badkey"; + JsonElement default1 = new JsonPrimitive("default1"); + JsonElement default2 = new JsonPrimitive("default2"); + JsonElement default3 = new JsonPrimitive("default3"); Event event1 = eventFactory.newFeatureRequestEvent(flag1, user, - new FeatureFlag.VariationAndValue(1, new JsonPrimitive("value1")), null); + new FeatureFlag.VariationAndValue(1, new JsonPrimitive("value1")), default1); Event event2 = eventFactory.newFeatureRequestEvent(flag1, user, - new FeatureFlag.VariationAndValue(2, new JsonPrimitive("value2")), null); + new FeatureFlag.VariationAndValue(2, new JsonPrimitive("value2")), default1); Event event3 = eventFactory.newFeatureRequestEvent(flag2, user, - new FeatureFlag.VariationAndValue(1, new JsonPrimitive("value99")), null); + new FeatureFlag.VariationAndValue(1, new JsonPrimitive("value99")), default2); Event event4 = eventFactory.newFeatureRequestEvent(flag1, user, - new FeatureFlag.VariationAndValue(1, new JsonPrimitive("value1")), null); - Event event5 = eventFactory.newUnknownFeatureRequestEvent(unknownFlagKey, user, null); + new FeatureFlag.VariationAndValue(1, new JsonPrimitive("value1")), default1); + Event event5 = eventFactory.newUnknownFeatureRequestEvent(unknownFlagKey, user, default3); es.summarizeEvent(event1); es.summarizeEvent(event2); es.summarizeEvent(event3); @@ -110,14 +120,28 @@ public void summarizeEventIncrementsCounters() { es.summarizeEvent(event5); EventSummarizer.SummaryOutput data = es.output(es.snapshot()); - EventSummarizer.CounterData expected1 = new EventSummarizer.CounterData(flag1.getKey(), + data.features.get(flag1.getKey()).counters.sort(new CounterValueComparator()); + EventSummarizer.CounterData expected1 = new EventSummarizer.CounterData( new JsonPrimitive("value1"), flag1.getVersion(), 2, null); - EventSummarizer.CounterData expected2 = new EventSummarizer.CounterData(flag1.getKey(), + EventSummarizer.CounterData expected2 = new EventSummarizer.CounterData( new JsonPrimitive("value2"), flag1.getVersion(), 1, null); - EventSummarizer.CounterData expected3 = new EventSummarizer.CounterData(flag2.getKey(), + EventSummarizer.CounterData expected3 = new EventSummarizer.CounterData( new JsonPrimitive("value99"), flag2.getVersion(), 1, null); - EventSummarizer.CounterData expected4 = new EventSummarizer.CounterData(unknownFlagKey, - null, null, 1, true); - assertThat(data.counters, containsInAnyOrder(expected1, expected2, expected3, expected4)); + EventSummarizer.CounterData expected4 = new EventSummarizer.CounterData( + default3, null, 1, true); + Map expectedFeatures = new HashMap<>(); + expectedFeatures.put(flag1.getKey(), new EventSummarizer.FlagSummaryData(default1, + Arrays.asList(expected1, expected2))); + expectedFeatures.put(flag2.getKey(), new EventSummarizer.FlagSummaryData(default2, + Arrays.asList(expected3))); + expectedFeatures.put(unknownFlagKey, new EventSummarizer.FlagSummaryData(default3, + Arrays.asList(expected4))); + assertThat(data.features, equalTo(expectedFeatures)); + } + + private static class CounterValueComparator implements Comparator { + public int compare(CounterData o1, CounterData o2) { + return o1.value.getAsString().compareTo(o2.value.getAsString()); + } } } From 72881ca38693b0bf09cd145f03035b370ce3036d Mon Sep 17 00:00:00 2001 From: Eli Bishop Date: Tue, 13 Mar 2018 16:28:01 -0700 Subject: [PATCH 21/67] update for change in spec: inline users can now be retained optionally --- .../launchdarkly/client/EventProcessor.java | 51 ++++++++----- .../com/launchdarkly/client/LDConfig.java | 15 ++++ .../java/com/launchdarkly/client/LDUser.java | 5 ++ .../client/EventProcessorTest.java | 72 +++++++++++++++---- 4 files changed, 112 insertions(+), 31 deletions(-) diff --git a/src/main/java/com/launchdarkly/client/EventProcessor.java b/src/main/java/com/launchdarkly/client/EventProcessor.java index 268135274..e5e72d831 100644 --- a/src/main/java/com/launchdarkly/client/EventProcessor.java +++ b/src/main/java/com/launchdarkly/client/EventProcessor.java @@ -105,6 +105,9 @@ public void run() { summarizer.resetUsers(); } } catch (InterruptedException e) { + } catch (Exception e) { + logger.error("Unexpected error in event processor: " + e); + logger.debug(e.getMessage(), e); } } } @@ -138,7 +141,7 @@ private void postToChannel(EventProcessorMessage message) { boolean dispatchEvent(Event e) { // For each user we haven't seen before, we add an index event - unless this is already // an identify event for that user. - if (e.user != null && !summarizer.noticeUser(e.user)) { + if (!config.inlineUsersInEvents && e.user != null && !summarizer.noticeUser(e.user)) { if (!(e instanceof IdentifyEvent)) { IndexEvent ie = new IndexEvent(e.creationDate, e.user); if (!queueEvent(ie)) { @@ -201,14 +204,19 @@ private EventOutput createEventOutput(Event e) { if (e instanceof FeatureRequestEvent) { FeatureRequestEvent fe = (FeatureRequestEvent)e; boolean isDebug = (!fe.trackEvents && fe.debugEventsUntilDate != null); - return new FeatureRequestEventOutput(fe.creationDate, fe.key, userKey, + return new FeatureRequestEventOutput(fe.creationDate, fe.key, + config.inlineUsersInEvents ? null : userKey, + config.inlineUsersInEvents ? e.user : null, fe.variation, fe.version, fe.value, fe.defaultVal, fe.prereqOf, isDebug); } else if (e instanceof IdentifyEvent) { return new IdentifyEventOutput(e.creationDate, e.user); } else if (e instanceof CustomEvent) { CustomEvent ce = (CustomEvent)e; - return new CustomEventOutput(ce.creationDate, ce.key, userKey, ce.data); + return new CustomEventOutput(ce.creationDate, ce.key, + config.inlineUsersInEvents ? null : userKey, + config.inlineUsersInEvents ? e.user : null, + ce.data); } else if (e instanceof IndexEvent) { return (IndexEvent)e; } else { @@ -248,19 +256,24 @@ class FlushTask implements Runnable { } public void run() { - List eventsOut = new ArrayList<>(events.length + 1); - for (Event event: events) { - eventsOut.add(createEventOutput(event)); - } - if (!snapshot.counters.isEmpty()) { - EventSummarizer.SummaryOutput summary = summarizer.output(snapshot); - SummaryEventOutput seo = new SummaryEventOutput(summary.startDate, summary.endDate, summary.features); - eventsOut.add(seo); - } - if (!eventsOut.isEmpty()) { - postEvents(eventsOut); + try { + List eventsOut = new ArrayList<>(events.length + 1); + for (Event event: events) { + eventsOut.add(createEventOutput(event)); + } + if (!snapshot.counters.isEmpty()) { + EventSummarizer.SummaryOutput summary = summarizer.output(snapshot); + SummaryEventOutput seo = new SummaryEventOutput(summary.startDate, summary.endDate, summary.features); + eventsOut.add(seo); + } + if (!eventsOut.isEmpty()) { + postEvents(eventsOut); + } + message.setResult(true); + } catch (Exception e) { + logger.error("Unexpected error in event processor: " + e); + logger.debug(e.getMessage(), e); } - message.setResult(true); } private void postEvents(List eventsOut) { @@ -352,18 +365,20 @@ private static class FeatureRequestEventOutput implements EventOutput { private final long creationDate; private final String key; private final String userKey; + private final LDUser user; private final Integer variation; private final Integer version; private final JsonElement value; @SerializedName("default") private final JsonElement defaultVal; private final String prereqOf; - FeatureRequestEventOutput(long creationDate, String key, String userKey, Integer variation, + FeatureRequestEventOutput(long creationDate, String key, String userKey, LDUser user, Integer variation, Integer version, JsonElement value, JsonElement defaultVal, String prereqOf, boolean debug) { this.kind = debug ? "debug" : "feature"; this.creationDate = creationDate; this.key = key; this.userKey = userKey; + this.user = user; this.variation = variation; this.version = version; this.value = value; @@ -390,13 +405,15 @@ private static class CustomEventOutput implements EventOutput { private final long creationDate; private final String key; private final String userKey; + private final LDUser user; private final JsonElement data; - CustomEventOutput(long creationDate, String key, String userKey, JsonElement data) { + CustomEventOutput(long creationDate, String key, String userKey, LDUser user, JsonElement data) { this.kind = "custom"; this.creationDate = creationDate; this.key = key; this.userKey = userKey; + this.user = user; this.data = data; } } diff --git a/src/main/java/com/launchdarkly/client/LDConfig.java b/src/main/java/com/launchdarkly/client/LDConfig.java index 6b42579b0..4adda9462 100644 --- a/src/main/java/com/launchdarkly/client/LDConfig.java +++ b/src/main/java/com/launchdarkly/client/LDConfig.java @@ -71,6 +71,7 @@ public final class LDConfig { final long reconnectTimeMs; final int userKeysCapacity; final int userKeysFlushInterval; + final boolean inlineUsersInEvents; protected LDConfig(Builder builder) { this.baseURI = builder.baseURI; @@ -99,6 +100,7 @@ protected LDConfig(Builder builder) { this.reconnectTimeMs = builder.reconnectTimeMillis; this.userKeysCapacity = builder.userKeysCapacity; this.userKeysFlushInterval = builder.userKeysFlushInterval; + this.inlineUsersInEvents = builder.inlineUsersInEvents; OkHttpClient.Builder httpClientBuilder = new OkHttpClient.Builder() .connectionPool(new ConnectionPool(5, 5, TimeUnit.SECONDS)) @@ -171,6 +173,7 @@ public static class Builder { private Set privateAttrNames = new HashSet<>(); private int userKeysCapacity = DEFAULT_USER_KEYS_CAPACITY; private int userKeysFlushInterval = DEFAULT_USER_KEYS_FLUSH_INTERVAL_SECONDS; + private boolean inlineUsersInEvents = false; /** * Creates a builder with all configuration parameters set to the default @@ -498,6 +501,18 @@ public Builder userKeysFlushInterval(int flushInterval) { return this; } + /** + * Sets whether to include full user details in every analytics event. The default is false (events will + * only include the user key, except for one "index" event that provides the full details for the user). + * + * @param inlineUsersInEvents true if you want full user details in each event + * @return the builder + */ + public Builder inlineUsersInEvents(boolean inlineUsersInEvents) { + this.inlineUsersInEvents = inlineUsersInEvents; + return this; + } + // returns null if none of the proxy bits were configured. Minimum required part: port. Proxy proxy() { if (this.proxyPort == -1) { diff --git a/src/main/java/com/launchdarkly/client/LDUser.java b/src/main/java/com/launchdarkly/client/LDUser.java index 0e14205c7..d526a783c 100644 --- a/src/main/java/com/launchdarkly/client/LDUser.java +++ b/src/main/java/com/launchdarkly/client/LDUser.java @@ -182,6 +182,11 @@ public UserAdapter(LDConfig config) { @Override public void write(JsonWriter out, LDUser user) throws IOException { + if (user == null) { + out.value((String)null); + return; + } + // Collect the private attribute names Set privateAttributeNames = new HashSet(config.privateAttrNames); diff --git a/src/test/java/com/launchdarkly/client/EventProcessorTest.java b/src/test/java/com/launchdarkly/client/EventProcessorTest.java index ae5b97705..e3bd51cf1 100644 --- a/src/test/java/com/launchdarkly/client/EventProcessorTest.java +++ b/src/test/java/com/launchdarkly/client/EventProcessorTest.java @@ -18,6 +18,7 @@ import static org.hamcrest.Matchers.equalTo; import static org.hamcrest.Matchers.hasItem; import static org.hamcrest.Matchers.hasItems; +import static org.hamcrest.Matchers.nullValue; import okhttp3.mockwebserver.MockResponse; import okhttp3.mockwebserver.MockWebServer; @@ -74,7 +75,23 @@ public void individualFeatureEventIsQueuedWithIndexEvent() throws Exception { JsonArray output = flushAndGetEvents(); assertThat(output, hasItems( isIndexEvent(fe), - isFeatureEvent(fe, flag, false) + isFeatureEvent(fe, flag, false, false) + )); + } + + @SuppressWarnings("unchecked") + @Test + public void featureEventCanContainInlineUser() throws Exception { + configBuilder.inlineUsersInEvents(true); + ep = new EventProcessor(SDK_KEY, configBuilder.build()); + FeatureFlag flag = new FeatureFlagBuilder("flagkey").version(11).trackEvents(true).build(); + FeatureRequestEvent fe = EventFactory.DEFAULT.newFeatureRequestEvent(flag, user, + new FeatureFlag.VariationAndValue(new Integer(1), new JsonPrimitive("value")), null); + ep.sendEvent(fe); + + JsonArray output = flushAndGetEvents(); + assertThat(output, hasItems( + isFeatureEvent(fe, flag, false, true) )); } @@ -91,7 +108,7 @@ public void eventKindIsDebugIfFlagIsTemporarilyInDebugMode() throws Exception { JsonArray output = flushAndGetEvents(); assertThat(output, hasItems( isIndexEvent(fe), - isFeatureEvent(fe, flag, true) + isFeatureEvent(fe, flag, true, false) )); } @@ -112,8 +129,8 @@ public void twoFeatureEventsForSameUserGenerateOnlyOneIndexEvent() throws Except JsonArray output = flushAndGetEvents(); assertThat(output, hasItems( isIndexEvent(fe1), - isFeatureEvent(fe1, flag1, false), - isFeatureEvent(fe2, flag2, false), + isFeatureEvent(fe1, flag1, false, false), + isFeatureEvent(fe2, flag2, false, false), isSummaryEvent(fe1.creationDate, fe2.creationDate) )); } @@ -153,19 +170,29 @@ public void customEventIsQueuedWithUser() throws Exception { ep = new EventProcessor(SDK_KEY, configBuilder.build()); JsonObject data = new JsonObject(); data.addProperty("thing", "stuff"); - Event ce = EventFactory.DEFAULT.newCustomEvent("eventkey", user, data); + CustomEvent ce = EventFactory.DEFAULT.newCustomEvent("eventkey", user, data); ep.sendEvent(ce); JsonArray output = flushAndGetEvents(); assertThat(output, hasItems( isIndexEvent(ce), - allOf( - hasJsonProperty("kind", "custom"), - hasJsonProperty("creationDate", (double)ce.creationDate), - hasJsonProperty("key", "eventkey"), - hasJsonProperty("userKey", user.getKeyAsString()), - hasJsonProperty("data", data) - ) + isCustomEvent(ce, false) + )); + } + + @SuppressWarnings("unchecked") + @Test + public void customEventCanContainInlineUser() throws Exception { + configBuilder.inlineUsersInEvents(true); + ep = new EventProcessor(SDK_KEY, configBuilder.build()); + JsonObject data = new JsonObject(); + data.addProperty("thing", "stuff"); + CustomEvent ce = EventFactory.DEFAULT.newCustomEvent("eventkey", user, data); + ep.sendEvent(ce); + + JsonArray output = flushAndGetEvents(); + assertThat(output, hasItems( + isCustomEvent(ce, true) )); } @@ -202,17 +229,34 @@ private Matcher isIndexEvent(Event sourceEvent) { ); } - private Matcher isFeatureEvent(FeatureRequestEvent sourceEvent, FeatureFlag flag, boolean debug) { + @SuppressWarnings("unchecked") + private Matcher isFeatureEvent(FeatureRequestEvent sourceEvent, FeatureFlag flag, boolean debug, boolean inlineUsers) { return allOf( hasJsonProperty("kind", debug ? "debug" : "feature"), hasJsonProperty("creationDate", (double)sourceEvent.creationDate), hasJsonProperty("key", flag.getKey()), hasJsonProperty("version", (double)flag.getVersion()), hasJsonProperty("value", sourceEvent.value), - hasJsonProperty("userKey", sourceEvent.user.getKeyAsString()) + inlineUsers ? hasJsonProperty("userKey", nullValue(JsonElement.class)) : + hasJsonProperty("userKey", sourceEvent.user.getKeyAsString()), + inlineUsers ? hasJsonProperty("user", makeUserJson(sourceEvent.user)) : + hasJsonProperty("user", nullValue(JsonElement.class)) ); } + private Matcher isCustomEvent(CustomEvent sourceEvent, boolean inlineUsers) { + return allOf( + hasJsonProperty("kind", "custom"), + hasJsonProperty("creationDate", (double)sourceEvent.creationDate), + hasJsonProperty("key", "eventkey"), + inlineUsers ? hasJsonProperty("userKey", nullValue(JsonElement.class)) : + hasJsonProperty("userKey", sourceEvent.user.getKeyAsString()), + inlineUsers ? hasJsonProperty("user", makeUserJson(sourceEvent.user)) : + hasJsonProperty("user", nullValue(JsonElement.class)), + hasJsonProperty("data", sourceEvent.data) + ); + } + private Matcher isSummaryEvent(long startDate, long endDate) { return allOf( hasJsonProperty("kind", "summary"), From ec7170920d6bdf11e8097a958aff9a634f04050e Mon Sep 17 00:00:00 2001 From: Eli Bishop Date: Tue, 13 Mar 2018 21:11:33 -0700 Subject: [PATCH 22/67] don't include variation index in feature event --- src/main/java/com/launchdarkly/client/EventProcessor.java | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/src/main/java/com/launchdarkly/client/EventProcessor.java b/src/main/java/com/launchdarkly/client/EventProcessor.java index e5e72d831..49c64503a 100644 --- a/src/main/java/com/launchdarkly/client/EventProcessor.java +++ b/src/main/java/com/launchdarkly/client/EventProcessor.java @@ -207,8 +207,7 @@ private EventOutput createEventOutput(Event e) { return new FeatureRequestEventOutput(fe.creationDate, fe.key, config.inlineUsersInEvents ? null : userKey, config.inlineUsersInEvents ? e.user : null, - fe.variation, fe.version, fe.value, fe.defaultVal, fe.prereqOf, - isDebug); + fe.version, fe.value, fe.defaultVal, fe.prereqOf, isDebug); } else if (e instanceof IdentifyEvent) { return new IdentifyEventOutput(e.creationDate, e.user); } else if (e instanceof CustomEvent) { @@ -366,20 +365,18 @@ private static class FeatureRequestEventOutput implements EventOutput { private final String key; private final String userKey; private final LDUser user; - private final Integer variation; private final Integer version; private final JsonElement value; @SerializedName("default") private final JsonElement defaultVal; private final String prereqOf; - FeatureRequestEventOutput(long creationDate, String key, String userKey, LDUser user, Integer variation, + FeatureRequestEventOutput(long creationDate, String key, String userKey, LDUser user, Integer version, JsonElement value, JsonElement defaultVal, String prereqOf, boolean debug) { this.kind = debug ? "debug" : "feature"; this.creationDate = creationDate; this.key = key; this.userKey = userKey; this.user = user; - this.variation = variation; this.version = version; this.value = value; this.defaultVal = defaultVal; From 96406315d5fcfb2eec95d2cdf24be594ef9abd4a Mon Sep 17 00:00:00 2001 From: Eli Bishop Date: Tue, 13 Mar 2018 21:11:40 -0700 Subject: [PATCH 23/67] typo --- src/main/java/com/launchdarkly/client/LDConfig.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/main/java/com/launchdarkly/client/LDConfig.java b/src/main/java/com/launchdarkly/client/LDConfig.java index 4adda9462..1ee6ac423 100644 --- a/src/main/java/com/launchdarkly/client/LDConfig.java +++ b/src/main/java/com/launchdarkly/client/LDConfig.java @@ -490,7 +490,7 @@ public Builder userKeysCapacity(int capacity) { } /** - * Set the interval in seconds at which the event processor will reset its set of known user keys. The + * Sets the interval in seconds at which the event processor will reset its set of known user keys. The * default value is five minutes. * * @param flushInterval the flush interval in seconds From 1782ac6229488d1147512fa72055cd1de758f1f0 Mon Sep 17 00:00:00 2001 From: Eli Bishop Date: Thu, 15 Mar 2018 16:07:14 -0700 Subject: [PATCH 24/67] misc cleanup --- .../launchdarkly/client/EventProcessor.java | 16 ++++++++----- .../launchdarkly/client/EventSummarizer.java | 24 +++++++++++-------- .../client/EventSummarizerTest.java | 4 ++-- 3 files changed, 26 insertions(+), 18 deletions(-) diff --git a/src/main/java/com/launchdarkly/client/EventProcessor.java b/src/main/java/com/launchdarkly/client/EventProcessor.java index 49c64503a..d27791af7 100644 --- a/src/main/java/com/launchdarkly/client/EventProcessor.java +++ b/src/main/java/com/launchdarkly/client/EventProcessor.java @@ -238,17 +238,21 @@ public void flush() { private void dispatchFlush(EventProcessorMessage message) { Event[] events = buffer.toArray(new Event[buffer.size()]); buffer.clear(); - EventSummarizer.EventsState snapshot = summarizer.snapshot(); - this.scheduler.schedule(new FlushTask(events, snapshot, message), 0, TimeUnit.SECONDS); + EventSummarizer.SummaryState snapshot = summarizer.snapshot(); + if (events.length > 0 || !snapshot.isEmpty()) { + this.scheduler.schedule(new FlushTask(events, snapshot, message), 0, TimeUnit.SECONDS); + } else { + message.setResult(true); + } } class FlushTask implements Runnable { private final Logger logger = LoggerFactory.getLogger(FlushTask.class); private final Event[] events; - private final EventSummarizer.EventsState snapshot; + private final EventSummarizer.SummaryState snapshot; private final EventProcessorMessage message; - FlushTask(Event[] events, EventSummarizer.EventsState snapshot, EventProcessorMessage message) { + FlushTask(Event[] events, EventSummarizer.SummaryState snapshot, EventProcessorMessage message) { this.events = events; this.snapshot = snapshot; this.message = message; @@ -260,7 +264,7 @@ public void run() { for (Event event: events) { eventsOut.add(createEventOutput(event)); } - if (!snapshot.counters.isEmpty()) { + if (!snapshot.isEmpty()) { EventSummarizer.SummaryOutput summary = summarizer.output(snapshot); SummaryEventOutput seo = new SummaryEventOutput(summary.startDate, summary.endDate, summary.features); eventsOut.add(seo); @@ -268,11 +272,11 @@ public void run() { if (!eventsOut.isEmpty()) { postEvents(eventsOut); } - message.setResult(true); } catch (Exception e) { logger.error("Unexpected error in event processor: " + e); logger.debug(e.getMessage(), e); } + message.setResult(true); } private void postEvents(List eventsOut) { diff --git a/src/main/java/com/launchdarkly/client/EventSummarizer.java b/src/main/java/com/launchdarkly/client/EventSummarizer.java index 46bb87b34..659593353 100644 --- a/src/main/java/com/launchdarkly/client/EventSummarizer.java +++ b/src/main/java/com/launchdarkly/client/EventSummarizer.java @@ -17,11 +17,11 @@ * single event-processing thread. */ class EventSummarizer { - private EventsState eventsState; + private SummaryState eventsState; private final SimpleLRUCache userKeys; EventSummarizer(LDConfig config) { - this.eventsState = new EventsState(); + this.eventsState = new SummaryState(); this.userKeys = new SimpleLRUCache(config.userKeysCapacity); } @@ -61,9 +61,9 @@ void summarizeEvent(Event event) { * Returns a snapshot of the current summarized event data, and resets this state. * @return the previous event state */ - EventsState snapshot() { - EventsState ret = eventsState; - eventsState = new EventsState(); + SummaryState snapshot() { + SummaryState ret = eventsState; + eventsState = new SummaryState(); return ret; } @@ -72,7 +72,7 @@ EventsState snapshot() { * @param snapshot the data obtained from {@link #snapshot()} * @return the formatted output */ - SummaryOutput output(EventsState snapshot) { + SummaryOutput output(SummaryState snapshot) { Map flagsOut = new HashMap<>(); for (Map.Entry entry: snapshot.counters.entrySet()) { FlagSummaryData fsd = flagsOut.get(entry.getKey().key); @@ -105,15 +105,19 @@ protected boolean removeEldestEntry(Map.Entry eldest) { } } - static class EventsState { + static class SummaryState { final Map counters; long startDate; long endDate; - EventsState() { + SummaryState() { counters = new HashMap(); } + boolean isEmpty() { + return counters.isEmpty(); + } + void incrementCounter(String flagKey, Integer variation, Integer version, JsonElement flagValue, JsonElement defaultVal) { CounterKey key = new CounterKey(flagKey, (variation == null) ? 0 : variation.intValue(), (version == null) ? 0 : version.intValue()); @@ -137,8 +141,8 @@ void noteTimestamp(long time) { @Override public boolean equals(Object other) { - if (other instanceof EventsState) { - EventsState o = (EventsState)other; + if (other instanceof SummaryState) { + SummaryState o = (SummaryState)other; return o.counters.equals(counters) && startDate == o.startDate && endDate == o.endDate; } return true; diff --git a/src/test/java/com/launchdarkly/client/EventSummarizerTest.java b/src/test/java/com/launchdarkly/client/EventSummarizerTest.java index fc86f4eed..c5a29bbc4 100644 --- a/src/test/java/com/launchdarkly/client/EventSummarizerTest.java +++ b/src/test/java/com/launchdarkly/client/EventSummarizerTest.java @@ -61,7 +61,7 @@ public void oldestUserForgottenIfCapacityExceeded() { @Test public void summarizeEventDoesNothingForIdentifyEvent() { EventSummarizer es = new EventSummarizer(defaultConfig); - EventSummarizer.EventsState snapshot = es.snapshot(); + EventSummarizer.SummaryState snapshot = es.snapshot(); es.summarizeEvent(eventFactory.newIdentifyEvent(user)); assertEquals(snapshot, es.snapshot()); @@ -70,7 +70,7 @@ public void summarizeEventDoesNothingForIdentifyEvent() { @Test public void summarizeEventDoesNothingForCustomEvent() { EventSummarizer es = new EventSummarizer(defaultConfig); - EventSummarizer.EventsState snapshot = es.snapshot(); + EventSummarizer.SummaryState snapshot = es.snapshot(); es.summarizeEvent(eventFactory.newCustomEvent("whatever", user, null)); assertEquals(snapshot, es.snapshot()); From db5a4e9df66b2770c9785eab302fc757909de81f Mon Sep 17 00:00:00 2001 From: Eli Bishop Date: Thu, 15 Mar 2018 16:35:41 -0700 Subject: [PATCH 25/67] make EventProcessor into an interface; always send events asynchronously --- .../client/DefaultEventProcessor.java | 457 ++++++++++++++++++ .../launchdarkly/client/EventProcessor.java | 452 +---------------- .../com/launchdarkly/client/LDClient.java | 42 +- .../client/NullEventProcessor.java | 18 + .../client/EventProcessorTest.java | 20 +- .../com/launchdarkly/client/LDClientTest.java | 12 +- 6 files changed, 523 insertions(+), 478 deletions(-) create mode 100644 src/main/java/com/launchdarkly/client/DefaultEventProcessor.java create mode 100644 src/main/java/com/launchdarkly/client/NullEventProcessor.java diff --git a/src/main/java/com/launchdarkly/client/DefaultEventProcessor.java b/src/main/java/com/launchdarkly/client/DefaultEventProcessor.java new file mode 100644 index 000000000..7b4b1d362 --- /dev/null +++ b/src/main/java/com/launchdarkly/client/DefaultEventProcessor.java @@ -0,0 +1,457 @@ +package com.launchdarkly.client; + +import com.google.common.util.concurrent.ThreadFactoryBuilder; +import com.google.gson.JsonElement; +import com.google.gson.annotations.SerializedName; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.io.IOException; +import java.text.SimpleDateFormat; +import java.util.ArrayList; +import java.util.List; +import java.util.Map; +import java.util.Random; +import java.util.concurrent.ArrayBlockingQueue; +import java.util.concurrent.BlockingQueue; +import java.util.concurrent.Executors; +import java.util.concurrent.ScheduledExecutorService; +import java.util.concurrent.Semaphore; +import java.util.concurrent.ThreadFactory; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicBoolean; +import java.util.concurrent.atomic.AtomicLong; + +import okhttp3.MediaType; +import okhttp3.Request; +import okhttp3.RequestBody; +import okhttp3.Response; + +class DefaultEventProcessor implements EventProcessor { + private static final Logger logger = LoggerFactory.getLogger(DefaultEventProcessor.class); + private static final SimpleDateFormat HTTP_DATE_FORMAT = new SimpleDateFormat("EEE, dd MMM yyyy HH:mm:ss zzz"); + private static final int CHANNEL_BLOCK_MILLIS = 1000; + + private final ScheduledExecutorService scheduler; + private final Thread mainThread; + private final BlockingQueue inputChannel; + private final ArrayList buffer; + private final String sdkKey; + private final LDConfig config; + private final EventSummarizer summarizer; + private final Random random = new Random(); + private final AtomicLong lastKnownPastTime = new AtomicLong(0); + private final AtomicBoolean capacityExceeded = new AtomicBoolean(false); + private final AtomicBoolean stopped = new AtomicBoolean(false); + + DefaultEventProcessor(String sdkKey, LDConfig config) { + this.sdkKey = sdkKey; + this.inputChannel = new ArrayBlockingQueue<>(config.capacity); + this.buffer = new ArrayList<>(config.capacity); + this.summarizer = new EventSummarizer(config); + this.config = config; + + ThreadFactory threadFactory = new ThreadFactoryBuilder() + .setDaemon(true) + .setNameFormat("LaunchDarkly-EventProcessor-%d") + .build(); + + this.scheduler = Executors.newSingleThreadScheduledExecutor(threadFactory); + Runnable flusher = new Runnable() { + public void run() { + postMessageAsync(MessageType.FLUSH, null); + } + }; + this.scheduler.scheduleAtFixedRate(flusher, config.flushInterval, config.flushInterval, TimeUnit.SECONDS); + Runnable userKeysFlusher = new Runnable() { + public void run() { + postMessageAsync(MessageType.FLUSH_USERS, null); + } + }; + this.scheduler.scheduleAtFixedRate(userKeysFlusher, config.userKeysFlushInterval, config.userKeysFlushInterval, + TimeUnit.SECONDS); + + mainThread = threadFactory.newThread(new MainLoop()); + mainThread.start(); + } + + @Override + public void sendEvent(Event e) { + postMessageAsync(MessageType.EVENT, e); + } + + public boolean sendEventAndWait(Event e) { + return postMessageAndWait(MessageType.EVENT, e); + } + + @Override + public void flush() { + postMessageAndWait(MessageType.FLUSH, null); + } + + @Override + public void close() throws IOException { + this.flush(); + scheduler.shutdown(); + stopped.set(true); + mainThread.interrupt(); + } + + /** + * This task drains the input queue as quickly as possible. Everything here is done on a single + * thread so we don't have to synchronize on our internal structures; when it's time to flush, + * dispatchFlush will fire off another task to do the part that takes longer. + */ + private class MainLoop implements Runnable { + public void run() { + while (!stopped.get()) { + try { + EventProcessorMessage message = inputChannel.take(); + switch(message.type) { + case EVENT: + message.setResult(dispatchEvent(message.event)); + break; + case FLUSH: + dispatchFlush(message); + case FLUSH_USERS: + summarizer.resetUsers(); + } + } catch (InterruptedException e) { + } catch (Exception e) { + logger.error("Unexpected error in event processor: " + e); + logger.debug(e.getMessage(), e); + } + } + } + } + + private void postMessageAsync(MessageType type, Event event) { + postToChannel(new EventProcessorMessage(type, event, false)); + } + + private boolean postMessageAndWait(MessageType type, Event event) { + EventProcessorMessage message = new EventProcessorMessage(type, event, true); + postToChannel(message); + return message.waitForResult(); + } + + private void postToChannel(EventProcessorMessage message) { + while (true) { + try { + if (inputChannel.offer(message, CHANNEL_BLOCK_MILLIS, TimeUnit.MILLISECONDS)) { + break; + } else { + // This doesn't mean that the output event buffer is full, but rather that the main thread is + // seriously backed up with not-yet-processed events. We shouldn't see this. + logger.warn("Events are being produced faster than they can be processed"); + } + } catch (InterruptedException ex) { + } + } + } + + private boolean dispatchEvent(Event e) { + // For each user we haven't seen before, we add an index event - unless this is already + // an identify event for that user. + if (!config.inlineUsersInEvents && e.user != null && !summarizer.noticeUser(e.user)) { + if (!(e instanceof IdentifyEvent)) { + IndexEvent ie = new IndexEvent(e.creationDate, e.user); + if (!queueEvent(ie)) { + return false; + } + } + } + + // Always record the event in the summarizer. + summarizer.summarizeEvent(e); + + if (shouldTrackFullEvent(e)) { + // Sampling interval applies only to fully-tracked events. + if (config.samplingInterval > 0 && random.nextInt(config.samplingInterval) != 0) { + return true; + } + // Queue the event as-is; we'll transform it into an output event when we're flushing + // (to avoid doing that work on our main thread). + return queueEvent(e); + } + return true; + } + + private boolean queueEvent(Event e) { + if (buffer.size() >= config.capacity) { + if (capacityExceeded.compareAndSet(false, true)) { + logger.warn("Exceeded event queue capacity. Increase capacity to avoid dropping events."); + } + return false; + } + capacityExceeded.set(false); + buffer.add(e); + return true; + } + + private boolean shouldTrackFullEvent(Event e) { + if (e instanceof FeatureRequestEvent) { + FeatureRequestEvent fe = (FeatureRequestEvent)e; + if (fe.trackEvents) { + return true; + } + if (fe.debugEventsUntilDate != null) { + // The "last known past time" comes from the last HTTP response we got from the server. + // In case the client's time is set wrong, at least we know that any expiration date + // earlier than that point is definitely in the past. + long lastPast = lastKnownPastTime.get(); + if ((lastPast != 0 && fe.debugEventsUntilDate > lastPast) || + fe.debugEventsUntilDate > System.currentTimeMillis()) { + return true; + } + } + return false; + } else { + return true; + } + } + + private EventOutput createEventOutput(Event e) { + String userKey = e.user == null ? null : e.user.getKeyAsString(); + if (e instanceof FeatureRequestEvent) { + FeatureRequestEvent fe = (FeatureRequestEvent)e; + boolean isDebug = (!fe.trackEvents && fe.debugEventsUntilDate != null); + return new FeatureRequestEventOutput(fe.creationDate, fe.key, + config.inlineUsersInEvents ? null : userKey, + config.inlineUsersInEvents ? e.user : null, + fe.version, fe.value, fe.defaultVal, fe.prereqOf, isDebug); + } else if (e instanceof IdentifyEvent) { + return new IdentifyEventOutput(e.creationDate, e.user); + } else if (e instanceof CustomEvent) { + CustomEvent ce = (CustomEvent)e; + return new CustomEventOutput(ce.creationDate, ce.key, + config.inlineUsersInEvents ? null : userKey, + config.inlineUsersInEvents ? e.user : null, + ce.data); + } else if (e instanceof IndexEvent) { + return new IndexEventOutput(e.creationDate, e.user); + } else { + return null; + } + } + + private void dispatchFlush(EventProcessorMessage message) { + Event[] events = buffer.toArray(new Event[buffer.size()]); + buffer.clear(); + EventSummarizer.SummaryState snapshot = summarizer.snapshot(); + if (events.length > 0 || !snapshot.isEmpty()) { + this.scheduler.schedule(new FlushTask(events, snapshot, message), 0, TimeUnit.SECONDS); + } else { + message.setResult(true); + } + } + + private class FlushTask implements Runnable { + private final Logger logger = LoggerFactory.getLogger(FlushTask.class); + private final Event[] events; + private final EventSummarizer.SummaryState snapshot; + private final EventProcessorMessage message; + + FlushTask(Event[] events, EventSummarizer.SummaryState snapshot, EventProcessorMessage message) { + this.events = events; + this.snapshot = snapshot; + this.message = message; + } + + public void run() { + try { + List eventsOut = new ArrayList<>(events.length + 1); + for (Event event: events) { + eventsOut.add(createEventOutput(event)); + } + if (!snapshot.isEmpty()) { + EventSummarizer.SummaryOutput summary = summarizer.output(snapshot); + SummaryEventOutput seo = new SummaryEventOutput(summary.startDate, summary.endDate, summary.features); + eventsOut.add(seo); + } + if (!eventsOut.isEmpty()) { + postEvents(eventsOut); + } + } catch (Exception e) { + logger.error("Unexpected error in event processor: " + e); + logger.debug(e.getMessage(), e); + } + message.setResult(true); + } + + private void postEvents(List eventsOut) { + String json = config.gson.toJson(eventsOut); + logger.debug("Posting {} event(s) to {} with payload: {}", + eventsOut.size(), config.eventsURI, json); + + Request request = config.getRequestBuilder(sdkKey) + .url(config.eventsURI.toString() + "/bulk") + .post(RequestBody.create(MediaType.parse("application/json; charset=utf-8"), json)) + .addHeader("Content-Type", "application/json") + .build(); + + try (Response response = config.httpClient.newCall(request).execute()) { + if (!response.isSuccessful()) { + logger.info("Got unexpected response when posting events: " + response); + if (response.code() == 401) { + stopped.set(true); + logger.error("Received 401 error, no further events will be posted since SDK key is invalid"); + close(); + } + } else { + logger.debug("Events Response: " + response.code()); + try { + String dateStr = response.header("Date"); + if (dateStr != null) { + lastKnownPastTime.set(HTTP_DATE_FORMAT.parse(dateStr).getTime()); + } + } catch (Exception e) { + } + } + } catch (IOException e) { + logger.info("Unhandled exception in LaunchDarkly client when posting events to URL: " + request.url(), e); + } + } + } + + private static enum MessageType { + EVENT, + FLUSH, + FLUSH_USERS + } + + private static class EventProcessorMessage { + private final MessageType type; + private final Event event; + private final AtomicBoolean result = new AtomicBoolean(false); + private final Semaphore reply; + + private EventProcessorMessage(MessageType type, Event event, boolean sync) { + this.type = type; + this.event = event; + reply = sync ? new Semaphore(0) : null; + } + + void setResult(boolean value) { + result.set(value); + if (reply != null) { + reply.release(); + } + } + + boolean waitForResult() { + if (reply == null) { + return false; + } + while (true) { + try { + reply.acquire(); + return result.get(); + } + catch (InterruptedException ex) { + } + } + } + + @Override + public String toString() { + return ((event == null) ? type.toString() : (type + ": " + event.getClass().getSimpleName())) + + (reply == null ? "" : " (sync)"); + } + } + + private static interface EventOutput { } + + @SuppressWarnings("unused") + private static class FeatureRequestEventOutput implements EventOutput { + private final String kind; + private final long creationDate; + private final String key; + private final String userKey; + private final LDUser user; + private final Integer version; + private final JsonElement value; + @SerializedName("default") private final JsonElement defaultVal; + private final String prereqOf; + + FeatureRequestEventOutput(long creationDate, String key, String userKey, LDUser user, + Integer version, JsonElement value, JsonElement defaultVal, String prereqOf, boolean debug) { + this.kind = debug ? "debug" : "feature"; + this.creationDate = creationDate; + this.key = key; + this.userKey = userKey; + this.user = user; + this.version = version; + this.value = value; + this.defaultVal = defaultVal; + this.prereqOf = prereqOf; + } + } + + @SuppressWarnings("unused") + private static class IdentifyEventOutput extends Event implements EventOutput { + private final String kind; + private final String key; + + IdentifyEventOutput(long creationDate, LDUser user) { + super(creationDate, user); + this.kind = "identify"; + this.key = user.getKeyAsString(); + } + } + + @SuppressWarnings("unused") + private static class CustomEventOutput implements EventOutput { + private final String kind; + private final long creationDate; + private final String key; + private final String userKey; + private final LDUser user; + private final JsonElement data; + + CustomEventOutput(long creationDate, String key, String userKey, LDUser user, JsonElement data) { + this.kind = "custom"; + this.creationDate = creationDate; + this.key = key; + this.userKey = userKey; + this.user = user; + this.data = data; + } + } + + @SuppressWarnings("unused") + private static class IndexEvent extends Event { + IndexEvent(long creationDate, LDUser user) { + super(creationDate, user); + } + } + + @SuppressWarnings("unused") + private static class IndexEventOutput implements EventOutput { + private final String kind; + private final long creationDate; + private final LDUser user; + + public IndexEventOutput(long creationDate, LDUser user) { + this.kind = "index"; + this.creationDate = creationDate; + this.user = user; + } + } + + @SuppressWarnings("unused") + private static class SummaryEventOutput implements EventOutput { + private final String kind; + private final long startDate; + private final long endDate; + private final Map features; + + SummaryEventOutput(long startDate, long endDate, Map features) { + this.kind = "summary"; + this.startDate = startDate; + this.endDate = endDate; + this.features = features; + } + } +} diff --git a/src/main/java/com/launchdarkly/client/EventProcessor.java b/src/main/java/com/launchdarkly/client/EventProcessor.java index d27791af7..10d502d86 100644 --- a/src/main/java/com/launchdarkly/client/EventProcessor.java +++ b/src/main/java/com/launchdarkly/client/EventProcessor.java @@ -1,446 +1,22 @@ package com.launchdarkly.client; -import com.google.common.util.concurrent.ThreadFactoryBuilder; -import com.google.gson.JsonElement; -import com.google.gson.annotations.SerializedName; - -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - import java.io.Closeable; -import java.io.IOException; -import java.text.SimpleDateFormat; -import java.util.ArrayList; -import java.util.List; -import java.util.Map; -import java.util.Random; -import java.util.concurrent.ArrayBlockingQueue; -import java.util.concurrent.BlockingQueue; -import java.util.concurrent.Executors; -import java.util.concurrent.ScheduledExecutorService; -import java.util.concurrent.Semaphore; -import java.util.concurrent.ThreadFactory; -import java.util.concurrent.TimeUnit; -import java.util.concurrent.atomic.AtomicBoolean; -import java.util.concurrent.atomic.AtomicLong; - -import okhttp3.MediaType; -import okhttp3.Request; -import okhttp3.RequestBody; -import okhttp3.Response; - -class EventProcessor implements Closeable { - private static final Logger logger = LoggerFactory.getLogger(EventProcessor.class); - private static final SimpleDateFormat HTTP_DATE_FORMAT = new SimpleDateFormat("EEE, dd MMM yyyy HH:mm:ss zzz"); - private static final int CHANNEL_BLOCK_MILLIS = 1000; - - private final ScheduledExecutorService scheduler; - private final Thread mainThread; - private final BlockingQueue inputChannel; - private final ArrayList buffer; - private final String sdkKey; - private final LDConfig config; - private final EventSummarizer summarizer; - private final Random random = new Random(); - private final AtomicLong lastKnownPastTime = new AtomicLong(0); - private final AtomicBoolean capacityExceeded = new AtomicBoolean(false); - private final AtomicBoolean stopped = new AtomicBoolean(false); - - EventProcessor(String sdkKey, LDConfig config) { - this.sdkKey = sdkKey; - this.inputChannel = new ArrayBlockingQueue<>(config.capacity); - this.buffer = new ArrayList<>(config.capacity); - this.summarizer = new EventSummarizer(config); - this.config = config; - - ThreadFactory threadFactory = new ThreadFactoryBuilder() - .setDaemon(true) - .setNameFormat("LaunchDarkly-EventProcessor-%d") - .build(); - - this.scheduler = Executors.newSingleThreadScheduledExecutor(threadFactory); - Runnable flusher = new Runnable() { - public void run() { - postMessageAsync(MessageType.FLUSH, null); - } - }; - this.scheduler.scheduleAtFixedRate(flusher, config.flushInterval, config.flushInterval, TimeUnit.SECONDS); - Runnable userKeysFlusher = new Runnable() { - public void run() { - postMessageAsync(MessageType.FLUSH_USERS, null); - } - }; - this.scheduler.scheduleAtFixedRate(userKeysFlusher, config.userKeysFlushInterval, config.userKeysFlushInterval, - TimeUnit.SECONDS); - - mainThread = threadFactory.newThread(new MainLoop()); - mainThread.start(); - } - - void sendEventAsync(Event e) { - postMessageAsync(MessageType.EVENT, e); - } - - boolean sendEvent(Event e) { - return postMessageAndWait(MessageType.EVENT, e); - } +/** + * Interface for an object that can send or store analytics events. + */ +interface EventProcessor extends Closeable { /** - * This task drains the input queue as quickly as possible. Everything here is done on a single - * thread so we don't have to synchronize on our internal structures; when it's time to flush, - * dispatchFlush will fire off another task to do the part that takes longer. + * Processes an event. This method is asynchronous; the event may be sent later in the background + * at an interval set by {@link LDConfig#flushInterval}, or due to a call to {@link #flush()}. + * @param e an event */ - private class MainLoop implements Runnable { - public void run() { - while (!stopped.get()) { - try { - EventProcessorMessage message = inputChannel.take(); - switch(message.type) { - case EVENT: - message.setResult(dispatchEvent(message.event)); - break; - case FLUSH: - dispatchFlush(message); - case FLUSH_USERS: - summarizer.resetUsers(); - } - } catch (InterruptedException e) { - } catch (Exception e) { - logger.error("Unexpected error in event processor: " + e); - logger.debug(e.getMessage(), e); - } - } - } - } + void sendEvent(Event e); - private void postMessageAsync(MessageType type, Event event) { - postToChannel(new EventProcessorMessage(type, event, false)); - } - - private boolean postMessageAndWait(MessageType type, Event event) { - EventProcessorMessage message = new EventProcessorMessage(type, event, true); - postToChannel(message); - return message.waitForResult(); - } - - private void postToChannel(EventProcessorMessage message) { - while (true) { - try { - if (inputChannel.offer(message, CHANNEL_BLOCK_MILLIS, TimeUnit.MILLISECONDS)) { - break; - } else { - // This doesn't mean that the output event buffer is full, but rather that the main thread is - // seriously backed up with not-yet-processed events. We shouldn't see this. - logger.warn("Events are being produced faster than they can be processed"); - } - } catch (InterruptedException ex) { - } - } - } - - boolean dispatchEvent(Event e) { - // For each user we haven't seen before, we add an index event - unless this is already - // an identify event for that user. - if (!config.inlineUsersInEvents && e.user != null && !summarizer.noticeUser(e.user)) { - if (!(e instanceof IdentifyEvent)) { - IndexEvent ie = new IndexEvent(e.creationDate, e.user); - if (!queueEvent(ie)) { - return false; - } - } - } - - // Always record the event in the summarizer. - summarizer.summarizeEvent(e); - - if (shouldTrackFullEvent(e)) { - // Sampling interval applies only to fully-tracked events. - if (config.samplingInterval > 0 && random.nextInt(config.samplingInterval) != 0) { - return true; - } - // Queue the event as-is; we'll transform it into an output event when we're flushing - // (to avoid doing that work on our main thread). - return queueEvent(e); - } - return true; - } - - private boolean queueEvent(Event e) { - if (buffer.size() >= config.capacity) { - if (capacityExceeded.compareAndSet(false, true)) { - logger.warn("Exceeded event queue capacity. Increase capacity to avoid dropping events."); - } - return false; - } - capacityExceeded.set(false); - buffer.add(e); - return true; - } - - private boolean shouldTrackFullEvent(Event e) { - if (e instanceof FeatureRequestEvent) { - FeatureRequestEvent fe = (FeatureRequestEvent)e; - if (fe.trackEvents) { - return true; - } - if (fe.debugEventsUntilDate != null) { - // The "last known past time" comes from the last HTTP response we got from the server. - // In case the client's time is set wrong, at least we know that any expiration date - // earlier than that point is definitely in the past. - long lastPast = lastKnownPastTime.get(); - if ((lastPast != 0 && fe.debugEventsUntilDate > lastPast) || - fe.debugEventsUntilDate > System.currentTimeMillis()) { - return true; - } - } - return false; - } else { - return true; - } - } - - private EventOutput createEventOutput(Event e) { - String userKey = e.user == null ? null : e.user.getKeyAsString(); - if (e instanceof FeatureRequestEvent) { - FeatureRequestEvent fe = (FeatureRequestEvent)e; - boolean isDebug = (!fe.trackEvents && fe.debugEventsUntilDate != null); - return new FeatureRequestEventOutput(fe.creationDate, fe.key, - config.inlineUsersInEvents ? null : userKey, - config.inlineUsersInEvents ? e.user : null, - fe.version, fe.value, fe.defaultVal, fe.prereqOf, isDebug); - } else if (e instanceof IdentifyEvent) { - return new IdentifyEventOutput(e.creationDate, e.user); - } else if (e instanceof CustomEvent) { - CustomEvent ce = (CustomEvent)e; - return new CustomEventOutput(ce.creationDate, ce.key, - config.inlineUsersInEvents ? null : userKey, - config.inlineUsersInEvents ? e.user : null, - ce.data); - } else if (e instanceof IndexEvent) { - return (IndexEvent)e; - } else { - return null; - } - } - - @Override - public void close() throws IOException { - this.flush(); - scheduler.shutdown(); - stopped.set(true); - mainThread.interrupt(); - } - - public void flush() { - postMessageAndWait(MessageType.FLUSH, null); - } - - private void dispatchFlush(EventProcessorMessage message) { - Event[] events = buffer.toArray(new Event[buffer.size()]); - buffer.clear(); - EventSummarizer.SummaryState snapshot = summarizer.snapshot(); - if (events.length > 0 || !snapshot.isEmpty()) { - this.scheduler.schedule(new FlushTask(events, snapshot, message), 0, TimeUnit.SECONDS); - } else { - message.setResult(true); - } - } - - class FlushTask implements Runnable { - private final Logger logger = LoggerFactory.getLogger(FlushTask.class); - private final Event[] events; - private final EventSummarizer.SummaryState snapshot; - private final EventProcessorMessage message; - - FlushTask(Event[] events, EventSummarizer.SummaryState snapshot, EventProcessorMessage message) { - this.events = events; - this.snapshot = snapshot; - this.message = message; - } - - public void run() { - try { - List eventsOut = new ArrayList<>(events.length + 1); - for (Event event: events) { - eventsOut.add(createEventOutput(event)); - } - if (!snapshot.isEmpty()) { - EventSummarizer.SummaryOutput summary = summarizer.output(snapshot); - SummaryEventOutput seo = new SummaryEventOutput(summary.startDate, summary.endDate, summary.features); - eventsOut.add(seo); - } - if (!eventsOut.isEmpty()) { - postEvents(eventsOut); - } - } catch (Exception e) { - logger.error("Unexpected error in event processor: " + e); - logger.debug(e.getMessage(), e); - } - message.setResult(true); - } - - private void postEvents(List eventsOut) { - String json = config.gson.toJson(eventsOut); - logger.debug("Posting {} event(s) to {} with payload: {}", - eventsOut.size(), config.eventsURI, json); - - Request request = config.getRequestBuilder(sdkKey) - .url(config.eventsURI.toString() + "/bulk") - .post(RequestBody.create(MediaType.parse("application/json; charset=utf-8"), json)) - .addHeader("Content-Type", "application/json") - .build(); - - try (Response response = config.httpClient.newCall(request).execute()) { - if (!response.isSuccessful()) { - logger.info("Got unexpected response when posting events: " + response); - if (response.code() == 401) { - stopped.set(true); - logger.error("Received 401 error, no further events will be posted since SDK key is invalid"); - close(); - } - } else { - logger.debug("Events Response: " + response.code()); - try { - String dateStr = response.header("Date"); - if (dateStr != null) { - lastKnownPastTime.set(HTTP_DATE_FORMAT.parse(dateStr).getTime()); - } - } catch (Exception e) { - } - } - } catch (IOException e) { - logger.info("Unhandled exception in LaunchDarkly client when posting events to URL: " + request.url(), e); - } - } - } - - private static enum MessageType { - EVENT, - FLUSH, - FLUSH_USERS - } - - private static class EventProcessorMessage { - private final MessageType type; - private final Event event; - private final AtomicBoolean result = new AtomicBoolean(false); - private final Semaphore reply; - - private EventProcessorMessage(MessageType type, Event event, boolean sync) { - this.type = type; - this.event = event; - reply = sync ? new Semaphore(0) : null; - } - - void setResult(boolean value) { - result.set(value); - if (reply != null) { - reply.release(); - } - } - - boolean waitForResult() { - if (reply == null) { - return false; - } - while (true) { - try { - reply.acquire(); - return result.get(); - } - catch (InterruptedException ex) { - } - } - } - - @Override - public String toString() { - return ((event == null) ? type.toString() : (type + ": " + event.getClass().getSimpleName())) + - (reply == null ? "" : " (sync)"); - } - } - - private static interface EventOutput { } - - @SuppressWarnings("unused") - private static class FeatureRequestEventOutput implements EventOutput { - private final String kind; - private final long creationDate; - private final String key; - private final String userKey; - private final LDUser user; - private final Integer version; - private final JsonElement value; - @SerializedName("default") private final JsonElement defaultVal; - private final String prereqOf; - - FeatureRequestEventOutput(long creationDate, String key, String userKey, LDUser user, - Integer version, JsonElement value, JsonElement defaultVal, String prereqOf, boolean debug) { - this.kind = debug ? "debug" : "feature"; - this.creationDate = creationDate; - this.key = key; - this.userKey = userKey; - this.user = user; - this.version = version; - this.value = value; - this.defaultVal = defaultVal; - this.prereqOf = prereqOf; - } - } - - @SuppressWarnings("unused") - private static class IdentifyEventOutput extends Event implements EventOutput { - private final String kind; - private final String key; - - IdentifyEventOutput(long creationDate, LDUser user) { - super(creationDate, user); - this.kind = "identify"; - this.key = user.getKeyAsString(); - } - } - - @SuppressWarnings("unused") - private static class CustomEventOutput implements EventOutput { - private final String kind; - private final long creationDate; - private final String key; - private final String userKey; - private final LDUser user; - private final JsonElement data; - - CustomEventOutput(long creationDate, String key, String userKey, LDUser user, JsonElement data) { - this.kind = "custom"; - this.creationDate = creationDate; - this.key = key; - this.userKey = userKey; - this.user = user; - this.data = data; - } - } - - @SuppressWarnings("unused") - private static class IndexEvent extends Event implements EventOutput { - private final String kind; - - IndexEvent(long creationDate, LDUser user) { - super(creationDate, user); - this.kind = "index"; - } - } - - @SuppressWarnings("unused") - private static class SummaryEventOutput implements EventOutput { - private final String kind; - private final long startDate; - private final long endDate; - private final Map features; - - SummaryEventOutput(long startDate, long endDate, Map features) { - this.kind = "summary"; - this.startDate = startDate; - this.endDate = endDate; - this.features = features; - } - } + /** + * Finishes processing any events that have been buffered. In the default implementation, this means + * sending the events to LaunchDarkly. This method is synchronous; when it returns, you can assume + * that all events queued prior to the {@link #flush()} have now been delivered. + */ + void flush(); } diff --git a/src/main/java/com/launchdarkly/client/LDClient.java b/src/main/java/com/launchdarkly/client/LDClient.java index bffe8d3d2..22f6de549 100644 --- a/src/main/java/com/launchdarkly/client/LDClient.java +++ b/src/main/java/com/launchdarkly/client/LDClient.java @@ -1,18 +1,13 @@ package com.launchdarkly.client; - import com.google.common.annotations.VisibleForTesting; import com.google.gson.JsonElement; import com.google.gson.JsonPrimitive; + import org.apache.commons.codec.binary.Hex; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import javax.crypto.Mac; -import javax.crypto.spec.SecretKeySpec; - -import static com.launchdarkly.client.VersionedDataKind.FEATURES; - import java.io.IOException; import java.io.UnsupportedEncodingException; import java.net.URL; @@ -23,10 +18,14 @@ import java.util.concurrent.Future; import java.util.concurrent.TimeUnit; import java.util.concurrent.TimeoutException; -import java.util.concurrent.atomic.AtomicBoolean; import java.util.jar.Attributes; import java.util.jar.Manifest; +import javax.crypto.Mac; +import javax.crypto.spec.SecretKeySpec; + +import static com.launchdarkly.client.VersionedDataKind.FEATURES; + /** * A client for the LaunchDarkly API. Client instances are thread-safe. Applications should instantiate * a single {@code LDClient} for the lifetime of their application. @@ -43,8 +42,6 @@ public class LDClient implements LDClientInterface { private final EventFactory eventFactory = EventFactory.DEFAULT; private UpdateProcessor updateProcessor; - private final AtomicBoolean eventCapacityExceeded = new AtomicBoolean(false); - /** * Creates a new client instance that connects to LaunchDarkly with the default configuration. In most * cases, you should use this constructor. @@ -66,7 +63,11 @@ public LDClient(String sdkKey, LDConfig config) { this.config = config; this.sdkKey = sdkKey; this.requestor = createFeatureRequestor(sdkKey, config); - this.eventProcessor = createEventProcessor(sdkKey, config); + if (config.offline || !config.sendEvents) { + this.eventProcessor = new NullEventProcessor(); + } else { + this.eventProcessor = createEventProcessor(sdkKey, config); + } if (config.offline) { logger.info("Starting LaunchDarkly client in offline mode"); @@ -112,7 +113,7 @@ protected FeatureRequestor createFeatureRequestor(String sdkKey, LDConfig config @VisibleForTesting protected EventProcessor createEventProcessor(String sdkKey, LDConfig config) { - return new EventProcessor(sdkKey, config); + return new DefaultEventProcessor(sdkKey, config); } @VisibleForTesting @@ -133,7 +134,7 @@ public void track(String eventName, LDUser user, JsonElement data) { if (user == null || user.getKey() == null) { logger.warn("Track called with null user or null user key!"); } - sendEvent(eventFactory.newCustomEvent(eventName, user, data)); + eventProcessor.sendEvent(eventFactory.newCustomEvent(eventName, user, data)); } @Override @@ -149,21 +150,12 @@ public void identify(LDUser user) { if (user == null || user.getKey() == null) { logger.warn("Identify called with null user or null user key!"); } - sendEvent(eventFactory.newIdentifyEvent(user)); + eventProcessor.sendEvent(eventFactory.newIdentifyEvent(user)); } private void sendFlagRequestEvent(FeatureRequestEvent event) { - if (sendEvent(event)) { - NewRelicReflector.annotateTransaction(event.key, String.valueOf(event.value)); - } - } - - private boolean sendEvent(Event event) { - if (isOffline() || !config.sendEvents) { - return false; - } - eventProcessor.sendEventAsync(event); - return true; + eventProcessor.sendEvent(event); + NewRelicReflector.annotateTransaction(event.key, String.valueOf(event.value)); } @Override @@ -281,7 +273,7 @@ private JsonElement evaluate(String featureKey, LDUser user, JsonElement default } FeatureFlag.EvalResult evalResult = featureFlag.evaluate(user, config.featureStore, eventFactory); for (FeatureRequestEvent event : evalResult.getPrerequisiteEvents()) { - sendEvent(event); + eventProcessor.sendEvent(event); } if (evalResult.getResult() != null && evalResult.getResult().getValue() != null) { expectedType.assertResultType(evalResult.getResult().getValue()); diff --git a/src/main/java/com/launchdarkly/client/NullEventProcessor.java b/src/main/java/com/launchdarkly/client/NullEventProcessor.java new file mode 100644 index 000000000..137d198f6 --- /dev/null +++ b/src/main/java/com/launchdarkly/client/NullEventProcessor.java @@ -0,0 +1,18 @@ +package com.launchdarkly.client; + +/** + * Stub implementation of {@link EventProcessor} for when we don't want to send any events. + */ +class NullEventProcessor implements EventProcessor { + @Override + public void sendEvent(Event e) { + } + + @Override + public void flush() { + } + + @Override + public void close() { + } +} diff --git a/src/test/java/com/launchdarkly/client/EventProcessorTest.java b/src/test/java/com/launchdarkly/client/EventProcessorTest.java index e3bd51cf1..c57bc7975 100644 --- a/src/test/java/com/launchdarkly/client/EventProcessorTest.java +++ b/src/test/java/com/launchdarkly/client/EventProcessorTest.java @@ -31,7 +31,7 @@ public class EventProcessorTest { private final LDConfig.Builder configBuilder = new LDConfig.Builder(); private final MockWebServer server = new MockWebServer(); - private EventProcessor ep; + private DefaultEventProcessor ep; @Before public void setup() throws Exception { @@ -50,7 +50,7 @@ public void teardown() throws Exception { @SuppressWarnings("unchecked") @Test public void identifyEventIsQueued() throws Exception { - ep = new EventProcessor(SDK_KEY, configBuilder.build()); + ep = new DefaultEventProcessor(SDK_KEY, configBuilder.build()); Event e = EventFactory.DEFAULT.newIdentifyEvent(user); ep.sendEvent(e); @@ -66,7 +66,7 @@ public void identifyEventIsQueued() throws Exception { @SuppressWarnings("unchecked") @Test public void individualFeatureEventIsQueuedWithIndexEvent() throws Exception { - ep = new EventProcessor(SDK_KEY, configBuilder.build()); + ep = new DefaultEventProcessor(SDK_KEY, configBuilder.build()); FeatureFlag flag = new FeatureFlagBuilder("flagkey").version(11).trackEvents(true).build(); FeatureRequestEvent fe = EventFactory.DEFAULT.newFeatureRequestEvent(flag, user, new FeatureFlag.VariationAndValue(new Integer(1), new JsonPrimitive("value")), null); @@ -83,7 +83,7 @@ public void individualFeatureEventIsQueuedWithIndexEvent() throws Exception { @Test public void featureEventCanContainInlineUser() throws Exception { configBuilder.inlineUsersInEvents(true); - ep = new EventProcessor(SDK_KEY, configBuilder.build()); + ep = new DefaultEventProcessor(SDK_KEY, configBuilder.build()); FeatureFlag flag = new FeatureFlagBuilder("flagkey").version(11).trackEvents(true).build(); FeatureRequestEvent fe = EventFactory.DEFAULT.newFeatureRequestEvent(flag, user, new FeatureFlag.VariationAndValue(new Integer(1), new JsonPrimitive("value")), null); @@ -98,7 +98,7 @@ public void featureEventCanContainInlineUser() throws Exception { @SuppressWarnings("unchecked") @Test public void eventKindIsDebugIfFlagIsTemporarilyInDebugMode() throws Exception { - ep = new EventProcessor(SDK_KEY, configBuilder.build()); + ep = new DefaultEventProcessor(SDK_KEY, configBuilder.build()); long futureTime = System.currentTimeMillis() + 1000000; FeatureFlag flag = new FeatureFlagBuilder("flagkey").version(11).debugEventsUntilDate(futureTime).build(); FeatureRequestEvent fe = EventFactory.DEFAULT.newFeatureRequestEvent(flag, user, @@ -115,7 +115,7 @@ public void eventKindIsDebugIfFlagIsTemporarilyInDebugMode() throws Exception { @SuppressWarnings("unchecked") @Test public void twoFeatureEventsForSameUserGenerateOnlyOneIndexEvent() throws Exception { - ep = new EventProcessor(SDK_KEY, configBuilder.build()); + ep = new DefaultEventProcessor(SDK_KEY, configBuilder.build()); FeatureFlag flag1 = new FeatureFlagBuilder("flagkey1").version(11).trackEvents(true).build(); FeatureFlag flag2 = new FeatureFlagBuilder("flagkey2").version(22).trackEvents(true).build(); JsonElement value = new JsonPrimitive("value"); @@ -138,7 +138,7 @@ public void twoFeatureEventsForSameUserGenerateOnlyOneIndexEvent() throws Except @SuppressWarnings("unchecked") @Test public void nonTrackedEventsAreSummarized() throws Exception { - ep = new EventProcessor(SDK_KEY, configBuilder.build()); + ep = new DefaultEventProcessor(SDK_KEY, configBuilder.build()); FeatureFlag flag1 = new FeatureFlagBuilder("flagkey1").version(11).build(); FeatureFlag flag2 = new FeatureFlagBuilder("flagkey2").version(22).build(); JsonElement value = new JsonPrimitive("value"); @@ -167,7 +167,7 @@ public void nonTrackedEventsAreSummarized() throws Exception { @SuppressWarnings("unchecked") @Test public void customEventIsQueuedWithUser() throws Exception { - ep = new EventProcessor(SDK_KEY, configBuilder.build()); + ep = new DefaultEventProcessor(SDK_KEY, configBuilder.build()); JsonObject data = new JsonObject(); data.addProperty("thing", "stuff"); CustomEvent ce = EventFactory.DEFAULT.newCustomEvent("eventkey", user, data); @@ -184,7 +184,7 @@ public void customEventIsQueuedWithUser() throws Exception { @Test public void customEventCanContainInlineUser() throws Exception { configBuilder.inlineUsersInEvents(true); - ep = new EventProcessor(SDK_KEY, configBuilder.build()); + ep = new DefaultEventProcessor(SDK_KEY, configBuilder.build()); JsonObject data = new JsonObject(); data.addProperty("thing", "stuff"); CustomEvent ce = EventFactory.DEFAULT.newCustomEvent("eventkey", user, data); @@ -198,7 +198,7 @@ public void customEventCanContainInlineUser() throws Exception { @Test public void sdkKeyIsSent() throws Exception { - ep = new EventProcessor(SDK_KEY, configBuilder.build()); + ep = new DefaultEventProcessor(SDK_KEY, configBuilder.build()); Event e = EventFactory.DEFAULT.newIdentifyEvent(user); ep.sendEvent(e); diff --git a/src/test/java/com/launchdarkly/client/LDClientTest.java b/src/test/java/com/launchdarkly/client/LDClientTest.java index 95dc3b1b9..4923e81a6 100644 --- a/src/test/java/com/launchdarkly/client/LDClientTest.java +++ b/src/test/java/com/launchdarkly/client/LDClientTest.java @@ -4,9 +4,6 @@ import com.google.gson.JsonElement; import com.google.gson.JsonPrimitive; -import junit.framework.AssertionFailedError; - -import org.easymock.EasyMock; import org.easymock.EasyMockSupport; import org.junit.Before; import org.junit.Test; @@ -24,7 +21,12 @@ import static org.easymock.EasyMock.anyObject; import static org.easymock.EasyMock.expect; import static org.easymock.EasyMock.expectLastCall; -import static org.junit.Assert.*; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertNotNull; +import static org.junit.Assert.assertTrue; + +import junit.framework.AssertionFailedError; public class LDClientTest extends EasyMockSupport { private FeatureRequestor requestor; @@ -580,7 +582,7 @@ private void assertDefaultValueIsReturned() { } private void expectEventsSent(int count) { - eventProcessor.sendEventAsync(anyObject(Event.class)); + eventProcessor.sendEvent(anyObject(Event.class)); if (count > 0) { expectLastCall().times(count); } else { From 0a923465e669bffcc2fd6dc45d5c84e09ad2f9f1 Mon Sep 17 00:00:00 2001 From: Eli Bishop Date: Thu, 15 Mar 2018 16:38:11 -0700 Subject: [PATCH 26/67] make all event fields final --- src/main/java/com/launchdarkly/client/Event.java | 4 ++-- .../java/com/launchdarkly/client/FeatureRequestEvent.java | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/src/main/java/com/launchdarkly/client/Event.java b/src/main/java/com/launchdarkly/client/Event.java index 0ce69fe8d..9e9ecfdac 100644 --- a/src/main/java/com/launchdarkly/client/Event.java +++ b/src/main/java/com/launchdarkly/client/Event.java @@ -1,8 +1,8 @@ package com.launchdarkly.client; class Event { - long creationDate; - LDUser user; + final long creationDate; + final LDUser user; Event(long creationDate, LDUser user) { this.creationDate = creationDate; diff --git a/src/main/java/com/launchdarkly/client/FeatureRequestEvent.java b/src/main/java/com/launchdarkly/client/FeatureRequestEvent.java index 72914d4cc..b1a640631 100644 --- a/src/main/java/com/launchdarkly/client/FeatureRequestEvent.java +++ b/src/main/java/com/launchdarkly/client/FeatureRequestEvent.java @@ -9,7 +9,7 @@ class FeatureRequestEvent extends Event { final JsonElement defaultVal; final Integer version; final String prereqOf; - boolean trackEvents; + final boolean trackEvents; final Long debugEventsUntilDate; FeatureRequestEvent(long timestamp, String key, LDUser user, Integer version, Integer variation, JsonElement value, From 545460ab4feec5da01a195688e0e32e092ca6d13 Mon Sep 17 00:00:00 2001 From: Eli Bishop Date: Thu, 15 Mar 2018 16:41:10 -0700 Subject: [PATCH 27/67] rm unused method --- .../java/com/launchdarkly/client/DefaultEventProcessor.java | 4 ---- 1 file changed, 4 deletions(-) diff --git a/src/main/java/com/launchdarkly/client/DefaultEventProcessor.java b/src/main/java/com/launchdarkly/client/DefaultEventProcessor.java index 7b4b1d362..2e7b7a8b6 100644 --- a/src/main/java/com/launchdarkly/client/DefaultEventProcessor.java +++ b/src/main/java/com/launchdarkly/client/DefaultEventProcessor.java @@ -81,10 +81,6 @@ public void sendEvent(Event e) { postMessageAsync(MessageType.EVENT, e); } - public boolean sendEventAndWait(Event e) { - return postMessageAndWait(MessageType.EVENT, e); - } - @Override public void flush() { postMessageAndWait(MessageType.FLUSH, null); From 5f0faa1c82bcf90d537c6ce89e4cc77499dbabc4 Mon Sep 17 00:00:00 2001 From: Eli Bishop Date: Fri, 16 Mar 2018 14:33:44 -0700 Subject: [PATCH 28/67] rm unnecessary result state --- .../client/DefaultEventProcessor.java | 21 +++++++++---------- 1 file changed, 10 insertions(+), 11 deletions(-) diff --git a/src/main/java/com/launchdarkly/client/DefaultEventProcessor.java b/src/main/java/com/launchdarkly/client/DefaultEventProcessor.java index 2e7b7a8b6..1600aee71 100644 --- a/src/main/java/com/launchdarkly/client/DefaultEventProcessor.java +++ b/src/main/java/com/launchdarkly/client/DefaultEventProcessor.java @@ -106,7 +106,8 @@ public void run() { EventProcessorMessage message = inputChannel.take(); switch(message.type) { case EVENT: - message.setResult(dispatchEvent(message.event)); + dispatchEvent(message.event); + message.completed(); break; case FLUSH: dispatchFlush(message); @@ -126,10 +127,10 @@ private void postMessageAsync(MessageType type, Event event) { postToChannel(new EventProcessorMessage(type, event, false)); } - private boolean postMessageAndWait(MessageType type, Event event) { + private void postMessageAndWait(MessageType type, Event event) { EventProcessorMessage message = new EventProcessorMessage(type, event, true); postToChannel(message); - return message.waitForResult(); + message.waitForCompletion(); } private void postToChannel(EventProcessorMessage message) { @@ -239,7 +240,7 @@ private void dispatchFlush(EventProcessorMessage message) { if (events.length > 0 || !snapshot.isEmpty()) { this.scheduler.schedule(new FlushTask(events, snapshot, message), 0, TimeUnit.SECONDS); } else { - message.setResult(true); + message.completed(); } } @@ -273,7 +274,7 @@ public void run() { logger.error("Unexpected error in event processor: " + e); logger.debug(e.getMessage(), e); } - message.setResult(true); + message.completed(); } private void postEvents(List eventsOut) { @@ -320,7 +321,6 @@ private static enum MessageType { private static class EventProcessorMessage { private final MessageType type; private final Event event; - private final AtomicBoolean result = new AtomicBoolean(false); private final Semaphore reply; private EventProcessorMessage(MessageType type, Event event, boolean sync) { @@ -329,21 +329,20 @@ private EventProcessorMessage(MessageType type, Event event, boolean sync) { reply = sync ? new Semaphore(0) : null; } - void setResult(boolean value) { - result.set(value); + void completed() { if (reply != null) { reply.release(); } } - boolean waitForResult() { + void waitForCompletion() { if (reply == null) { - return false; + return; } while (true) { try { reply.acquire(); - return result.get(); + return; } catch (InterruptedException ex) { } From 5e9b575465d7489284b6cbfb1bd469b9c49135dd Mon Sep 17 00:00:00 2001 From: Eli Bishop Date: Fri, 16 Mar 2018 14:34:42 -0700 Subject: [PATCH 29/67] don't return early if index event can't be queued --- .../client/DefaultEventProcessor.java | 20 ++++++++----------- 1 file changed, 8 insertions(+), 12 deletions(-) diff --git a/src/main/java/com/launchdarkly/client/DefaultEventProcessor.java b/src/main/java/com/launchdarkly/client/DefaultEventProcessor.java index 1600aee71..f4830ad4e 100644 --- a/src/main/java/com/launchdarkly/client/DefaultEventProcessor.java +++ b/src/main/java/com/launchdarkly/client/DefaultEventProcessor.java @@ -148,15 +148,13 @@ private void postToChannel(EventProcessorMessage message) { } } - private boolean dispatchEvent(Event e) { + private void dispatchEvent(Event e) { // For each user we haven't seen before, we add an index event - unless this is already // an identify event for that user. if (!config.inlineUsersInEvents && e.user != null && !summarizer.noticeUser(e.user)) { if (!(e instanceof IdentifyEvent)) { IndexEvent ie = new IndexEvent(e.creationDate, e.user); - if (!queueEvent(ie)) { - return false; - } + queueEvent(ie); } } @@ -166,25 +164,23 @@ private boolean dispatchEvent(Event e) { if (shouldTrackFullEvent(e)) { // Sampling interval applies only to fully-tracked events. if (config.samplingInterval > 0 && random.nextInt(config.samplingInterval) != 0) { - return true; + return; } // Queue the event as-is; we'll transform it into an output event when we're flushing // (to avoid doing that work on our main thread). - return queueEvent(e); + queueEvent(e); } - return true; } - private boolean queueEvent(Event e) { + private void queueEvent(Event e) { if (buffer.size() >= config.capacity) { if (capacityExceeded.compareAndSet(false, true)) { logger.warn("Exceeded event queue capacity. Increase capacity to avoid dropping events."); } - return false; + } else { + capacityExceeded.set(false); + buffer.add(e); } - capacityExceeded.set(false); - buffer.add(e); - return true; } private boolean shouldTrackFullEvent(Event e) { From 9d37263d724c10431d6d5a1d5411917e2d965d28 Mon Sep 17 00:00:00 2001 From: Eli Bishop Date: Fri, 16 Mar 2018 14:45:56 -0700 Subject: [PATCH 30/67] don't need AtomicBoolean --- .../com/launchdarkly/client/DefaultEventProcessor.java | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/src/main/java/com/launchdarkly/client/DefaultEventProcessor.java b/src/main/java/com/launchdarkly/client/DefaultEventProcessor.java index f4830ad4e..472dd8183 100644 --- a/src/main/java/com/launchdarkly/client/DefaultEventProcessor.java +++ b/src/main/java/com/launchdarkly/client/DefaultEventProcessor.java @@ -42,8 +42,8 @@ class DefaultEventProcessor implements EventProcessor { private final EventSummarizer summarizer; private final Random random = new Random(); private final AtomicLong lastKnownPastTime = new AtomicLong(0); - private final AtomicBoolean capacityExceeded = new AtomicBoolean(false); private final AtomicBoolean stopped = new AtomicBoolean(false); + private boolean capacityExceeded = false; DefaultEventProcessor(String sdkKey, LDConfig config) { this.sdkKey = sdkKey; @@ -174,11 +174,12 @@ private void dispatchEvent(Event e) { private void queueEvent(Event e) { if (buffer.size() >= config.capacity) { - if (capacityExceeded.compareAndSet(false, true)) { + if (!capacityExceeded) { // don't need AtomicBoolean, this is only checked on one thread + capacityExceeded = true; logger.warn("Exceeded event queue capacity. Increase capacity to avoid dropping events."); } } else { - capacityExceeded.set(false); + capacityExceeded = false; buffer.add(e); } } From 83eb47bd907db4ae8c990d2631cedc07ee8c467a Mon Sep 17 00:00:00 2001 From: Eli Bishop Date: Fri, 16 Mar 2018 15:08:09 -0700 Subject: [PATCH 31/67] fix test class name --- .../{EventProcessorTest.java => DefaultEventProcessorTest.java} | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) rename src/test/java/com/launchdarkly/client/{EventProcessorTest.java => DefaultEventProcessorTest.java} (99%) diff --git a/src/test/java/com/launchdarkly/client/EventProcessorTest.java b/src/test/java/com/launchdarkly/client/DefaultEventProcessorTest.java similarity index 99% rename from src/test/java/com/launchdarkly/client/EventProcessorTest.java rename to src/test/java/com/launchdarkly/client/DefaultEventProcessorTest.java index c57bc7975..2fac268e0 100644 --- a/src/test/java/com/launchdarkly/client/EventProcessorTest.java +++ b/src/test/java/com/launchdarkly/client/DefaultEventProcessorTest.java @@ -24,7 +24,7 @@ import okhttp3.mockwebserver.MockWebServer; import okhttp3.mockwebserver.RecordedRequest; -public class EventProcessorTest { +public class DefaultEventProcessorTest { private static final String SDK_KEY = "SDK_KEY"; private static final LDUser user = new LDUser.Builder("userkey").name("Red").build(); private static final Gson gson = new Gson(); From b392392586cab9c9e71a1eca5eade378b3248c09 Mon Sep 17 00:00:00 2001 From: Eli Bishop Date: Fri, 16 Mar 2018 15:09:28 -0700 Subject: [PATCH 32/67] add test for flushing empty queue --- .../launchdarkly/client/DefaultEventProcessorTest.java | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/src/test/java/com/launchdarkly/client/DefaultEventProcessorTest.java b/src/test/java/com/launchdarkly/client/DefaultEventProcessorTest.java index 2fac268e0..ad122eb93 100644 --- a/src/test/java/com/launchdarkly/client/DefaultEventProcessorTest.java +++ b/src/test/java/com/launchdarkly/client/DefaultEventProcessorTest.java @@ -19,6 +19,7 @@ import static org.hamcrest.Matchers.hasItem; import static org.hamcrest.Matchers.hasItems; import static org.hamcrest.Matchers.nullValue; +import static org.junit.Assert.assertEquals; import okhttp3.mockwebserver.MockResponse; import okhttp3.mockwebserver.MockWebServer; @@ -196,6 +197,14 @@ public void customEventCanContainInlineUser() throws Exception { )); } + @Test + public void nothingIsSentIfThereAreNoEvents() throws Exception { + ep = new DefaultEventProcessor(SDK_KEY, configBuilder.build()); + ep.flush(); + + assertEquals(0, server.getRequestCount()); + } + @Test public void sdkKeyIsSent() throws Exception { ep = new DefaultEventProcessor(SDK_KEY, configBuilder.build()); From 2fd75b0129a8490143c2927de6ed56ce23dd64b4 Mon Sep 17 00:00:00 2001 From: Eli Bishop Date: Mon, 19 Mar 2018 16:12:16 -0700 Subject: [PATCH 33/67] revise the "last known server time" logic - it was the reverse of what we want --- .../client/DefaultEventProcessor.java | 7 +- .../client/DefaultEventProcessorTest.java | 67 ++++++++++++++++++- 2 files changed, 70 insertions(+), 4 deletions(-) diff --git a/src/main/java/com/launchdarkly/client/DefaultEventProcessor.java b/src/main/java/com/launchdarkly/client/DefaultEventProcessor.java index 472dd8183..9af6cdf06 100644 --- a/src/main/java/com/launchdarkly/client/DefaultEventProcessor.java +++ b/src/main/java/com/launchdarkly/client/DefaultEventProcessor.java @@ -30,7 +30,7 @@ class DefaultEventProcessor implements EventProcessor { private static final Logger logger = LoggerFactory.getLogger(DefaultEventProcessor.class); - private static final SimpleDateFormat HTTP_DATE_FORMAT = new SimpleDateFormat("EEE, dd MMM yyyy HH:mm:ss zzz"); + static final SimpleDateFormat HTTP_DATE_FORMAT = new SimpleDateFormat("EEE, dd MMM yyyy HH:mm:ss zzz"); private static final int CHANNEL_BLOCK_MILLIS = 1000; private final ScheduledExecutorService scheduler; @@ -193,9 +193,10 @@ private boolean shouldTrackFullEvent(Event e) { if (fe.debugEventsUntilDate != null) { // The "last known past time" comes from the last HTTP response we got from the server. // In case the client's time is set wrong, at least we know that any expiration date - // earlier than that point is definitely in the past. + // earlier than that point is definitely in the past. If there's any discrepancy, we + // want to err on the side of cutting off event debugging sooner. long lastPast = lastKnownPastTime.get(); - if ((lastPast != 0 && fe.debugEventsUntilDate > lastPast) || + if (fe.debugEventsUntilDate > lastPast && fe.debugEventsUntilDate > System.currentTimeMillis()) { return true; } diff --git a/src/test/java/com/launchdarkly/client/DefaultEventProcessorTest.java b/src/test/java/com/launchdarkly/client/DefaultEventProcessorTest.java index ad122eb93..cf173ed61 100644 --- a/src/test/java/com/launchdarkly/client/DefaultEventProcessorTest.java +++ b/src/test/java/com/launchdarkly/client/DefaultEventProcessorTest.java @@ -11,6 +11,8 @@ import org.junit.Before; import org.junit.Test; +import java.util.Date; + import static com.launchdarkly.client.TestUtils.hasJsonProperty; import static com.launchdarkly.client.TestUtils.isJsonArray; import static org.hamcrest.MatcherAssert.assertThat; @@ -32,6 +34,7 @@ public class DefaultEventProcessorTest { private final LDConfig.Builder configBuilder = new LDConfig.Builder(); private final MockWebServer server = new MockWebServer(); + private Long serverTimestamp; private DefaultEventProcessor ep; @Before @@ -113,6 +116,64 @@ public void eventKindIsDebugIfFlagIsTemporarilyInDebugMode() throws Exception { )); } + @SuppressWarnings("unchecked") + @Test + public void debugModeExpiresBasedOnClientTimeIfClientTimeIsLaterThanServerTime() throws Exception { + ep = new DefaultEventProcessor(SDK_KEY, configBuilder.build()); + + // Pick a server time that is somewhat behind the client time + long serverTime = System.currentTimeMillis() - 20000; + + // Send and flush an event we don't care about, just to set the last server time + serverTimestamp = serverTime; + ep.sendEvent(EventFactory.DEFAULT.newIdentifyEvent(new LDUser.Builder("otherUser").build())); + flushAndGetEvents(); + + // Now send an event with debug mode on, with a "debug until" time that is further in + // the future than the server time, but in the past compared to the client. + long debugUntil = serverTime + 1000; + FeatureFlag flag = new FeatureFlagBuilder("flagkey").version(11).debugEventsUntilDate(debugUntil).build(); + FeatureRequestEvent fe = EventFactory.DEFAULT.newFeatureRequestEvent(flag, user, + new FeatureFlag.VariationAndValue(new Integer(1), new JsonPrimitive("value")), null); + ep.sendEvent(fe); + + // Should get a summary event only, not a full feature event + JsonArray output = flushAndGetEvents(); + assertThat(output, hasItems( + isIndexEvent(fe), + isSummaryEvent(fe.creationDate, fe.creationDate) + )); + } + + @SuppressWarnings("unchecked") + @Test + public void debugModeExpiresBasedOnServerTimeIfServerTimeIsLaterThanClientTime() throws Exception { + ep = new DefaultEventProcessor(SDK_KEY, configBuilder.build()); + + // Pick a server time that is somewhat ahead of the client time + long serverTime = System.currentTimeMillis() + 20000; + + // Send and flush an event we don't care about, just to set the last server time + serverTimestamp = serverTime; + ep.sendEvent(EventFactory.DEFAULT.newIdentifyEvent(new LDUser.Builder("otherUser").build())); + flushAndGetEvents(); + + // Now send an event with debug mode on, with a "debug until" time that is further in + // the future than the client time, but in the past compared to the server. + long debugUntil = serverTime - 1000; + FeatureFlag flag = new FeatureFlagBuilder("flagkey").version(11).debugEventsUntilDate(debugUntil).build(); + FeatureRequestEvent fe = EventFactory.DEFAULT.newFeatureRequestEvent(flag, user, + new FeatureFlag.VariationAndValue(new Integer(1), new JsonPrimitive("value")), null); + ep.sendEvent(fe); + + // Should get a summary event only, not a full feature event + JsonArray output = flushAndGetEvents(); + assertThat(output, hasItems( + isIndexEvent(fe), + isSummaryEvent(fe.creationDate, fe.creationDate) + )); + } + @SuppressWarnings("unchecked") @Test public void twoFeatureEventsForSameUserGenerateOnlyOneIndexEvent() throws Exception { @@ -219,7 +280,11 @@ public void sdkKeyIsSent() throws Exception { } private JsonArray flushAndGetEvents() throws Exception { - server.enqueue(new MockResponse()); + MockResponse response = new MockResponse(); + if (serverTimestamp != null) { + response.addHeader("Date", DefaultEventProcessor.HTTP_DATE_FORMAT.format(new Date(serverTimestamp))); + } + server.enqueue(response); ep.flush(); RecordedRequest req = server.takeRequest(); return gson.fromJson(req.getBody().readUtf8(), JsonElement.class).getAsJsonArray(); From ad381dbef057eb36f5fbfa20d1b7b4a3901c04af Mon Sep 17 00:00:00 2001 From: Eli Bishop Date: Wed, 21 Mar 2018 17:37:55 -0700 Subject: [PATCH 34/67] rm duplicate class --- .../client/DefaultEventProcessorTest.java | 4 +- .../com/launchdarkly/client/TestUtil.java | 68 +++++++++++++++++ .../com/launchdarkly/client/TestUtils.java | 74 ------------------- 3 files changed, 70 insertions(+), 76 deletions(-) delete mode 100644 src/test/java/com/launchdarkly/client/TestUtils.java diff --git a/src/test/java/com/launchdarkly/client/DefaultEventProcessorTest.java b/src/test/java/com/launchdarkly/client/DefaultEventProcessorTest.java index cf173ed61..9c33a753e 100644 --- a/src/test/java/com/launchdarkly/client/DefaultEventProcessorTest.java +++ b/src/test/java/com/launchdarkly/client/DefaultEventProcessorTest.java @@ -13,8 +13,8 @@ import java.util.Date; -import static com.launchdarkly.client.TestUtils.hasJsonProperty; -import static com.launchdarkly.client.TestUtils.isJsonArray; +import static com.launchdarkly.client.TestUtil.hasJsonProperty; +import static com.launchdarkly.client.TestUtil.isJsonArray; import static org.hamcrest.MatcherAssert.assertThat; import static org.hamcrest.Matchers.allOf; import static org.hamcrest.Matchers.equalTo; diff --git a/src/test/java/com/launchdarkly/client/TestUtil.java b/src/test/java/com/launchdarkly/client/TestUtil.java index af0c39ad9..277afc2fa 100644 --- a/src/test/java/com/launchdarkly/client/TestUtil.java +++ b/src/test/java/com/launchdarkly/client/TestUtil.java @@ -1,9 +1,17 @@ package com.launchdarkly.client; +import com.google.gson.JsonArray; +import com.google.gson.JsonElement; import com.google.gson.JsonPrimitive; +import org.hamcrest.Description; +import org.hamcrest.Matcher; +import org.hamcrest.TypeSafeDiagnosingMatcher; + import java.util.Arrays; +import static org.hamcrest.Matchers.equalTo; + public class TestUtil { public static JsonPrimitive js(String s) { @@ -32,4 +40,64 @@ public static FeatureFlag booleanFlagWithClauses(Clause... clauses) { .variations(jbool(false), jbool(true)) .build(); } + + public static Matcher hasJsonProperty(final String name, JsonElement value) { + return hasJsonProperty(name, equalTo(value)); + } + + public static Matcher hasJsonProperty(final String name, String value) { + return hasJsonProperty(name, new JsonPrimitive(value)); + } + + public static Matcher hasJsonProperty(final String name, int value) { + return hasJsonProperty(name, new JsonPrimitive(value)); + } + + public static Matcher hasJsonProperty(final String name, double value) { + return hasJsonProperty(name, new JsonPrimitive(value)); + } + + public static Matcher hasJsonProperty(final String name, boolean value) { + return hasJsonProperty(name, new JsonPrimitive(value)); + } + + public static Matcher hasJsonProperty(final String name, final Matcher matcher) { + return new TypeSafeDiagnosingMatcher() { + @Override + public void describeTo(Description description) { + description.appendText(name + ": "); + matcher.describeTo(description); + } + + @Override + protected boolean matchesSafely(JsonElement item, Description mismatchDescription) { + JsonElement value = item.getAsJsonObject().get(name); + if (!matcher.matches(value)) { + matcher.describeMismatch(value, mismatchDescription); + return false; + } + return true; + } + }; + } + + public static Matcher isJsonArray(final Matcher> matcher) { + return new TypeSafeDiagnosingMatcher() { + @Override + public void describeTo(Description description) { + description.appendText("array: "); + matcher.describeTo(description); + } + + @Override + protected boolean matchesSafely(JsonElement item, Description mismatchDescription) { + JsonArray value = item.getAsJsonArray(); + if (!matcher.matches(value)) { + matcher.describeMismatch(value, mismatchDescription); + return false; + } + return true; + } + }; + } } diff --git a/src/test/java/com/launchdarkly/client/TestUtils.java b/src/test/java/com/launchdarkly/client/TestUtils.java deleted file mode 100644 index b5d0ebaeb..000000000 --- a/src/test/java/com/launchdarkly/client/TestUtils.java +++ /dev/null @@ -1,74 +0,0 @@ -package com.launchdarkly.client; - -import com.google.gson.JsonArray; -import com.google.gson.JsonElement; -import com.google.gson.JsonPrimitive; - -import org.hamcrest.Description; -import org.hamcrest.Matcher; -import org.hamcrest.TypeSafeDiagnosingMatcher; - -import static org.hamcrest.Matchers.equalTo; - -public class TestUtils { - - public static Matcher hasJsonProperty(final String name, JsonElement value) { - return hasJsonProperty(name, equalTo(value)); - } - - public static Matcher hasJsonProperty(final String name, String value) { - return hasJsonProperty(name, new JsonPrimitive(value)); - } - - public static Matcher hasJsonProperty(final String name, int value) { - return hasJsonProperty(name, new JsonPrimitive(value)); - } - - public static Matcher hasJsonProperty(final String name, double value) { - return hasJsonProperty(name, new JsonPrimitive(value)); - } - - public static Matcher hasJsonProperty(final String name, boolean value) { - return hasJsonProperty(name, new JsonPrimitive(value)); - } - - public static Matcher hasJsonProperty(final String name, final Matcher matcher) { - return new TypeSafeDiagnosingMatcher() { - @Override - public void describeTo(Description description) { - description.appendText(name + ": "); - matcher.describeTo(description); - } - - @Override - protected boolean matchesSafely(JsonElement item, Description mismatchDescription) { - JsonElement value = item.getAsJsonObject().get(name); - if (!matcher.matches(value)) { - matcher.describeMismatch(value, mismatchDescription); - return false; - } - return true; - } - }; - } - - public static Matcher isJsonArray(final Matcher> matcher) { - return new TypeSafeDiagnosingMatcher() { - @Override - public void describeTo(Description description) { - description.appendText("array: "); - matcher.describeTo(description); - } - - @Override - protected boolean matchesSafely(JsonElement item, Description mismatchDescription) { - JsonArray value = item.getAsJsonArray(); - if (!matcher.matches(value)) { - matcher.describeMismatch(value, mismatchDescription); - return false; - } - return true; - } - }; - } -} From c0fcacbe37e70d13be08b0b12943b32d4b15e16e Mon Sep 17 00:00:00 2001 From: Eli Bishop Date: Wed, 21 Mar 2018 18:14:55 -0700 Subject: [PATCH 35/67] add tests for user filtering in events --- .../client/DefaultEventProcessorTest.java | 77 +++++++++++++++---- 1 file changed, 64 insertions(+), 13 deletions(-) diff --git a/src/test/java/com/launchdarkly/client/DefaultEventProcessorTest.java b/src/test/java/com/launchdarkly/client/DefaultEventProcessorTest.java index 9c33a753e..17217232e 100644 --- a/src/test/java/com/launchdarkly/client/DefaultEventProcessorTest.java +++ b/src/test/java/com/launchdarkly/client/DefaultEventProcessorTest.java @@ -31,6 +31,8 @@ public class DefaultEventProcessorTest { private static final String SDK_KEY = "SDK_KEY"; private static final LDUser user = new LDUser.Builder("userkey").name("Red").build(); private static final Gson gson = new Gson(); + private static final JsonElement userJson = gson.fromJson("{\"key\":\"userkey\",\"name\":\"Red\"}", JsonElement.class); + private static final JsonElement filteredUserJson = gson.fromJson("{\"key\":\"userkey\",\"privateAttrs\":[\"name\"]}", JsonElement.class); private final LDConfig.Builder configBuilder = new LDConfig.Builder(); private final MockWebServer server = new MockWebServer(); @@ -67,6 +69,23 @@ public void identifyEventIsQueued() throws Exception { ))); } + @SuppressWarnings("unchecked") + @Test + public void userIsFilteredInIdentifyEvent() throws Exception { + configBuilder.allAttributesPrivate(true); + ep = new DefaultEventProcessor(SDK_KEY, configBuilder.build()); + Event e = EventFactory.DEFAULT.newIdentifyEvent(user); + ep.sendEvent(e); + + JsonArray output = flushAndGetEvents(); + assertThat(output, hasItems( + allOf( + hasJsonProperty("kind", "identify"), + hasJsonProperty("creationDate", (double)e.creationDate), + hasJsonProperty("user", filteredUserJson) + ))); + } + @SuppressWarnings("unchecked") @Test public void individualFeatureEventIsQueuedWithIndexEvent() throws Exception { @@ -79,7 +98,7 @@ public void individualFeatureEventIsQueuedWithIndexEvent() throws Exception { JsonArray output = flushAndGetEvents(); assertThat(output, hasItems( isIndexEvent(fe), - isFeatureEvent(fe, flag, false, false) + isFeatureEvent(fe, flag, false, null) )); } @@ -95,10 +114,26 @@ public void featureEventCanContainInlineUser() throws Exception { JsonArray output = flushAndGetEvents(); assertThat(output, hasItems( - isFeatureEvent(fe, flag, false, true) + isFeatureEvent(fe, flag, false, userJson) )); } + @SuppressWarnings("unchecked") + @Test + public void userIsFilteredInFeatureEvent() throws Exception { + configBuilder.inlineUsersInEvents(true).allAttributesPrivate(true); + ep = new DefaultEventProcessor(SDK_KEY, configBuilder.build()); + FeatureFlag flag = new FeatureFlagBuilder("flagkey").version(11).trackEvents(true).build(); + FeatureRequestEvent fe = EventFactory.DEFAULT.newFeatureRequestEvent(flag, user, + new FeatureFlag.VariationAndValue(new Integer(1), new JsonPrimitive("value")), null); + ep.sendEvent(fe); + + JsonArray output = flushAndGetEvents(); + assertThat(output, hasItems( + isFeatureEvent(fe, flag, false, filteredUserJson) + )); + } + @SuppressWarnings("unchecked") @Test public void eventKindIsDebugIfFlagIsTemporarilyInDebugMode() throws Exception { @@ -112,7 +147,7 @@ public void eventKindIsDebugIfFlagIsTemporarilyInDebugMode() throws Exception { JsonArray output = flushAndGetEvents(); assertThat(output, hasItems( isIndexEvent(fe), - isFeatureEvent(fe, flag, true, false) + isFeatureEvent(fe, flag, true, null) )); } @@ -191,8 +226,8 @@ public void twoFeatureEventsForSameUserGenerateOnlyOneIndexEvent() throws Except JsonArray output = flushAndGetEvents(); assertThat(output, hasItems( isIndexEvent(fe1), - isFeatureEvent(fe1, flag1, false, false), - isFeatureEvent(fe2, flag2, false, false), + isFeatureEvent(fe1, flag1, false, null), + isFeatureEvent(fe2, flag2, false, null), isSummaryEvent(fe1.creationDate, fe2.creationDate) )); } @@ -238,7 +273,7 @@ public void customEventIsQueuedWithUser() throws Exception { JsonArray output = flushAndGetEvents(); assertThat(output, hasItems( isIndexEvent(ce), - isCustomEvent(ce, false) + isCustomEvent(ce, null) )); } @@ -254,7 +289,23 @@ public void customEventCanContainInlineUser() throws Exception { JsonArray output = flushAndGetEvents(); assertThat(output, hasItems( - isCustomEvent(ce, true) + isCustomEvent(ce, userJson) + )); + } + + @SuppressWarnings("unchecked") + @Test + public void userIsFilteredInCustomEvent() throws Exception { + configBuilder.inlineUsersInEvents(true).allAttributesPrivate(true); + ep = new DefaultEventProcessor(SDK_KEY, configBuilder.build()); + JsonObject data = new JsonObject(); + data.addProperty("thing", "stuff"); + CustomEvent ce = EventFactory.DEFAULT.newCustomEvent("eventkey", user, data); + ep.sendEvent(ce); + + JsonArray output = flushAndGetEvents(); + assertThat(output, hasItems( + isCustomEvent(ce, filteredUserJson) )); } @@ -304,28 +355,28 @@ private Matcher isIndexEvent(Event sourceEvent) { } @SuppressWarnings("unchecked") - private Matcher isFeatureEvent(FeatureRequestEvent sourceEvent, FeatureFlag flag, boolean debug, boolean inlineUsers) { + private Matcher isFeatureEvent(FeatureRequestEvent sourceEvent, FeatureFlag flag, boolean debug, JsonElement inlineUser) { return allOf( hasJsonProperty("kind", debug ? "debug" : "feature"), hasJsonProperty("creationDate", (double)sourceEvent.creationDate), hasJsonProperty("key", flag.getKey()), hasJsonProperty("version", (double)flag.getVersion()), hasJsonProperty("value", sourceEvent.value), - inlineUsers ? hasJsonProperty("userKey", nullValue(JsonElement.class)) : + (inlineUser != null) ? hasJsonProperty("userKey", nullValue(JsonElement.class)) : hasJsonProperty("userKey", sourceEvent.user.getKeyAsString()), - inlineUsers ? hasJsonProperty("user", makeUserJson(sourceEvent.user)) : + (inlineUser != null) ? hasJsonProperty("user", inlineUser) : hasJsonProperty("user", nullValue(JsonElement.class)) ); } - private Matcher isCustomEvent(CustomEvent sourceEvent, boolean inlineUsers) { + private Matcher isCustomEvent(CustomEvent sourceEvent, JsonElement inlineUser) { return allOf( hasJsonProperty("kind", "custom"), hasJsonProperty("creationDate", (double)sourceEvent.creationDate), hasJsonProperty("key", "eventkey"), - inlineUsers ? hasJsonProperty("userKey", nullValue(JsonElement.class)) : + (inlineUser != null) ? hasJsonProperty("userKey", nullValue(JsonElement.class)) : hasJsonProperty("userKey", sourceEvent.user.getKeyAsString()), - inlineUsers ? hasJsonProperty("user", makeUserJson(sourceEvent.user)) : + (inlineUser != null) ? hasJsonProperty("user", inlineUser) : hasJsonProperty("user", nullValue(JsonElement.class)), hasJsonProperty("data", sourceEvent.data) ); From 141df39d579f7008660c7ae89cdb375438aebe8c Mon Sep 17 00:00:00 2001 From: Eli Bishop Date: Wed, 21 Mar 2018 20:30:26 -0700 Subject: [PATCH 36/67] fix unit tests to verify that a summary event is present --- .../client/DefaultEventProcessorTest.java | 18 +++++++++++++----- 1 file changed, 13 insertions(+), 5 deletions(-) diff --git a/src/test/java/com/launchdarkly/client/DefaultEventProcessorTest.java b/src/test/java/com/launchdarkly/client/DefaultEventProcessorTest.java index 17217232e..d7b4041f5 100644 --- a/src/test/java/com/launchdarkly/client/DefaultEventProcessorTest.java +++ b/src/test/java/com/launchdarkly/client/DefaultEventProcessorTest.java @@ -98,7 +98,8 @@ public void individualFeatureEventIsQueuedWithIndexEvent() throws Exception { JsonArray output = flushAndGetEvents(); assertThat(output, hasItems( isIndexEvent(fe), - isFeatureEvent(fe, flag, false, null) + isFeatureEvent(fe, flag, false, null), + isSummaryEvent() )); } @@ -114,7 +115,8 @@ public void featureEventCanContainInlineUser() throws Exception { JsonArray output = flushAndGetEvents(); assertThat(output, hasItems( - isFeatureEvent(fe, flag, false, userJson) + isFeatureEvent(fe, flag, false, userJson), + isSummaryEvent() )); } @@ -130,7 +132,8 @@ public void userIsFilteredInFeatureEvent() throws Exception { JsonArray output = flushAndGetEvents(); assertThat(output, hasItems( - isFeatureEvent(fe, flag, false, filteredUserJson) + isFeatureEvent(fe, flag, false, filteredUserJson), + isSummaryEvent() )); } @@ -147,7 +150,8 @@ public void eventKindIsDebugIfFlagIsTemporarilyInDebugMode() throws Exception { JsonArray output = flushAndGetEvents(); assertThat(output, hasItems( isIndexEvent(fe), - isFeatureEvent(fe, flag, true, null) + isFeatureEvent(fe, flag, true, null), + isSummaryEvent() )); } @@ -381,7 +385,11 @@ private Matcher isCustomEvent(CustomEvent sourceEvent, JsonElement hasJsonProperty("data", sourceEvent.data) ); } - + + private Matcher isSummaryEvent() { + return hasJsonProperty("kind", "summary"); + } + private Matcher isSummaryEvent(long startDate, long endDate) { return allOf( hasJsonProperty("kind", "summary"), From 5ec76f52a406571f1a75e5da5005213a2c929f0d Mon Sep 17 00:00:00 2001 From: Eli Bishop Date: Thu, 22 Mar 2018 11:57:57 -0700 Subject: [PATCH 37/67] misc fixes --- .../client/DefaultEventProcessor.java | 8 +- .../client/DefaultEventProcessorTest.java | 100 +++++++++++------- 2 files changed, 68 insertions(+), 40 deletions(-) diff --git a/src/main/java/com/launchdarkly/client/DefaultEventProcessor.java b/src/main/java/com/launchdarkly/client/DefaultEventProcessor.java index 9af6cdf06..69f6acc35 100644 --- a/src/main/java/com/launchdarkly/client/DefaultEventProcessor.java +++ b/src/main/java/com/launchdarkly/client/DefaultEventProcessor.java @@ -78,12 +78,16 @@ public void run() { @Override public void sendEvent(Event e) { - postMessageAsync(MessageType.EVENT, e); + if (!stopped.get()) { + postMessageAsync(MessageType.EVENT, e); + } } @Override public void flush() { - postMessageAndWait(MessageType.FLUSH, null); + if (!stopped.get()) { + postMessageAndWait(MessageType.FLUSH, null); + } } @Override diff --git a/src/test/java/com/launchdarkly/client/DefaultEventProcessorTest.java b/src/test/java/com/launchdarkly/client/DefaultEventProcessorTest.java index d7b4041f5..9297263fd 100644 --- a/src/test/java/com/launchdarkly/client/DefaultEventProcessorTest.java +++ b/src/test/java/com/launchdarkly/client/DefaultEventProcessorTest.java @@ -12,6 +12,7 @@ import org.junit.Test; import java.util.Date; +import java.util.concurrent.TimeUnit; import static com.launchdarkly.client.TestUtil.hasJsonProperty; import static com.launchdarkly.client.TestUtil.isJsonArray; @@ -36,7 +37,6 @@ public class DefaultEventProcessorTest { private final LDConfig.Builder configBuilder = new LDConfig.Builder(); private final MockWebServer server = new MockWebServer(); - private Long serverTimestamp; private DefaultEventProcessor ep; @Before @@ -60,12 +60,12 @@ public void identifyEventIsQueued() throws Exception { Event e = EventFactory.DEFAULT.newIdentifyEvent(user); ep.sendEvent(e); - JsonArray output = flushAndGetEvents(); + JsonArray output = flushAndGetEvents(new MockResponse()); assertThat(output, hasItems( allOf( hasJsonProperty("kind", "identify"), hasJsonProperty("creationDate", (double)e.creationDate), - hasJsonProperty("user", makeUserJson(user)) + hasJsonProperty("user", userJson) ))); } @@ -77,7 +77,7 @@ public void userIsFilteredInIdentifyEvent() throws Exception { Event e = EventFactory.DEFAULT.newIdentifyEvent(user); ep.sendEvent(e); - JsonArray output = flushAndGetEvents(); + JsonArray output = flushAndGetEvents(new MockResponse()); assertThat(output, hasItems( allOf( hasJsonProperty("kind", "identify"), @@ -95,14 +95,32 @@ public void individualFeatureEventIsQueuedWithIndexEvent() throws Exception { new FeatureFlag.VariationAndValue(new Integer(1), new JsonPrimitive("value")), null); ep.sendEvent(fe); - JsonArray output = flushAndGetEvents(); + JsonArray output = flushAndGetEvents(new MockResponse()); assertThat(output, hasItems( - isIndexEvent(fe), + isIndexEvent(fe, userJson), isFeatureEvent(fe, flag, false, null), isSummaryEvent() )); } + @SuppressWarnings("unchecked") + @Test + public void userIsFilteredInIndexEvent() throws Exception { + configBuilder.allAttributesPrivate(true); + ep = new DefaultEventProcessor(SDK_KEY, configBuilder.build()); + FeatureFlag flag = new FeatureFlagBuilder("flagkey").version(11).trackEvents(true).build(); + FeatureRequestEvent fe = EventFactory.DEFAULT.newFeatureRequestEvent(flag, user, + new FeatureFlag.VariationAndValue(new Integer(1), new JsonPrimitive("value")), null); + ep.sendEvent(fe); + + JsonArray output = flushAndGetEvents(new MockResponse()); + assertThat(output, hasItems( + isIndexEvent(fe, filteredUserJson), + isFeatureEvent(fe, flag, false, null), + isSummaryEvent() + )); + } + @SuppressWarnings("unchecked") @Test public void featureEventCanContainInlineUser() throws Exception { @@ -113,7 +131,7 @@ public void featureEventCanContainInlineUser() throws Exception { new FeatureFlag.VariationAndValue(new Integer(1), new JsonPrimitive("value")), null); ep.sendEvent(fe); - JsonArray output = flushAndGetEvents(); + JsonArray output = flushAndGetEvents(new MockResponse()); assertThat(output, hasItems( isFeatureEvent(fe, flag, false, userJson), isSummaryEvent() @@ -130,7 +148,7 @@ public void userIsFilteredInFeatureEvent() throws Exception { new FeatureFlag.VariationAndValue(new Integer(1), new JsonPrimitive("value")), null); ep.sendEvent(fe); - JsonArray output = flushAndGetEvents(); + JsonArray output = flushAndGetEvents(new MockResponse()); assertThat(output, hasItems( isFeatureEvent(fe, flag, false, filteredUserJson), isSummaryEvent() @@ -147,9 +165,9 @@ public void eventKindIsDebugIfFlagIsTemporarilyInDebugMode() throws Exception { new FeatureFlag.VariationAndValue(new Integer(1), new JsonPrimitive("value")), null); ep.sendEvent(fe); - JsonArray output = flushAndGetEvents(); + JsonArray output = flushAndGetEvents(new MockResponse()); assertThat(output, hasItems( - isIndexEvent(fe), + isIndexEvent(fe, userJson), isFeatureEvent(fe, flag, true, null), isSummaryEvent() )); @@ -164,9 +182,8 @@ public void debugModeExpiresBasedOnClientTimeIfClientTimeIsLaterThanServerTime() long serverTime = System.currentTimeMillis() - 20000; // Send and flush an event we don't care about, just to set the last server time - serverTimestamp = serverTime; ep.sendEvent(EventFactory.DEFAULT.newIdentifyEvent(new LDUser.Builder("otherUser").build())); - flushAndGetEvents(); + flushAndGetEvents(addDateHeader(new MockResponse(), serverTime)); // Now send an event with debug mode on, with a "debug until" time that is further in // the future than the server time, but in the past compared to the client. @@ -177,9 +194,9 @@ public void debugModeExpiresBasedOnClientTimeIfClientTimeIsLaterThanServerTime() ep.sendEvent(fe); // Should get a summary event only, not a full feature event - JsonArray output = flushAndGetEvents(); + JsonArray output = flushAndGetEvents(new MockResponse()); assertThat(output, hasItems( - isIndexEvent(fe), + isIndexEvent(fe, userJson), isSummaryEvent(fe.creationDate, fe.creationDate) )); } @@ -193,9 +210,8 @@ public void debugModeExpiresBasedOnServerTimeIfServerTimeIsLaterThanClientTime() long serverTime = System.currentTimeMillis() + 20000; // Send and flush an event we don't care about, just to set the last server time - serverTimestamp = serverTime; ep.sendEvent(EventFactory.DEFAULT.newIdentifyEvent(new LDUser.Builder("otherUser").build())); - flushAndGetEvents(); + flushAndGetEvents(addDateHeader(new MockResponse(), serverTime)); // Now send an event with debug mode on, with a "debug until" time that is further in // the future than the client time, but in the past compared to the server. @@ -206,9 +222,9 @@ public void debugModeExpiresBasedOnServerTimeIfServerTimeIsLaterThanClientTime() ep.sendEvent(fe); // Should get a summary event only, not a full feature event - JsonArray output = flushAndGetEvents(); + JsonArray output = flushAndGetEvents(new MockResponse()); assertThat(output, hasItems( - isIndexEvent(fe), + isIndexEvent(fe, userJson), isSummaryEvent(fe.creationDate, fe.creationDate) )); } @@ -227,9 +243,9 @@ public void twoFeatureEventsForSameUserGenerateOnlyOneIndexEvent() throws Except ep.sendEvent(fe1); ep.sendEvent(fe2); - JsonArray output = flushAndGetEvents(); + JsonArray output = flushAndGetEvents(new MockResponse()); assertThat(output, hasItems( - isIndexEvent(fe1), + isIndexEvent(fe1, userJson), isFeatureEvent(fe1, flag1, false, null), isFeatureEvent(fe2, flag2, false, null), isSummaryEvent(fe1.creationDate, fe2.creationDate) @@ -252,9 +268,9 @@ public void nonTrackedEventsAreSummarized() throws Exception { ep.sendEvent(fe1); ep.sendEvent(fe2); - JsonArray output = flushAndGetEvents(); + JsonArray output = flushAndGetEvents(new MockResponse()); assertThat(output, hasItems( - isIndexEvent(fe1), + isIndexEvent(fe1, userJson), allOf( isSummaryEvent(fe1.creationDate, fe2.creationDate), hasSummaryFlag(flag1.getKey(), default1, @@ -274,9 +290,9 @@ public void customEventIsQueuedWithUser() throws Exception { CustomEvent ce = EventFactory.DEFAULT.newCustomEvent("eventkey", user, data); ep.sendEvent(ce); - JsonArray output = flushAndGetEvents(); + JsonArray output = flushAndGetEvents(new MockResponse()); assertThat(output, hasItems( - isIndexEvent(ce), + isIndexEvent(ce, userJson), isCustomEvent(ce, null) )); } @@ -291,7 +307,7 @@ public void customEventCanContainInlineUser() throws Exception { CustomEvent ce = EventFactory.DEFAULT.newCustomEvent("eventkey", user, data); ep.sendEvent(ce); - JsonArray output = flushAndGetEvents(); + JsonArray output = flushAndGetEvents(new MockResponse()); assertThat(output, hasItems( isCustomEvent(ce, userJson) )); @@ -307,7 +323,7 @@ public void userIsFilteredInCustomEvent() throws Exception { CustomEvent ce = EventFactory.DEFAULT.newCustomEvent("eventkey", user, data); ep.sendEvent(ce); - JsonArray output = flushAndGetEvents(); + JsonArray output = flushAndGetEvents(new MockResponse()); assertThat(output, hasItems( isCustomEvent(ce, filteredUserJson) )); @@ -334,27 +350,35 @@ public void sdkKeyIsSent() throws Exception { assertThat(req.getHeader("Authorization"), equalTo(SDK_KEY)); } - private JsonArray flushAndGetEvents() throws Exception { - MockResponse response = new MockResponse(); - if (serverTimestamp != null) { - response.addHeader("Date", DefaultEventProcessor.HTTP_DATE_FORMAT.format(new Date(serverTimestamp))); - } + @Test + public void noMorePayloadsAreSentAfter401Error() throws Exception { + ep = new DefaultEventProcessor(SDK_KEY, configBuilder.build()); + Event e = EventFactory.DEFAULT.newIdentifyEvent(user); + ep.sendEvent(e); + flushAndGetEvents(new MockResponse().setResponseCode(401)); + + ep.sendEvent(e); + ep.flush(); + RecordedRequest req = server.takeRequest(0, TimeUnit.SECONDS); + assertThat(req, nullValue(RecordedRequest.class)); + } + + private MockResponse addDateHeader(MockResponse response, long timestamp) { + return response.addHeader("Date", DefaultEventProcessor.HTTP_DATE_FORMAT.format(new Date(timestamp))); + } + + private JsonArray flushAndGetEvents(MockResponse response) throws Exception { server.enqueue(response); ep.flush(); RecordedRequest req = server.takeRequest(); return gson.fromJson(req.getBody().readUtf8(), JsonElement.class).getAsJsonArray(); } - private JsonElement makeUserJson(LDUser user) { - // need to use the gson instance from the config object, which has a custom serializer - return configBuilder.build().gson.toJsonTree(user); - } - - private Matcher isIndexEvent(Event sourceEvent) { + private Matcher isIndexEvent(Event sourceEvent, JsonElement user) { return allOf( hasJsonProperty("kind", "index"), hasJsonProperty("creationDate", (double)sourceEvent.creationDate), - hasJsonProperty("user", makeUserJson(sourceEvent.user)) + hasJsonProperty("user", user) ); } From cea6cf959c6369fb3ce23686d6abb6a0a30ea857 Mon Sep 17 00:00:00 2001 From: Eli Bishop Date: Sat, 24 Mar 2018 18:22:58 -0700 Subject: [PATCH 38/67] reorganize inner classes, make event processor responsible for summary event --- .../client/DefaultEventProcessor.java | 60 ++++++- .../launchdarkly/client/EventSummarizer.java | 158 ++++-------------- .../client/EventSummarizerTest.java | 61 +++---- 3 files changed, 101 insertions(+), 178 deletions(-) diff --git a/src/main/java/com/launchdarkly/client/DefaultEventProcessor.java b/src/main/java/com/launchdarkly/client/DefaultEventProcessor.java index 69f6acc35..e185bd275 100644 --- a/src/main/java/com/launchdarkly/client/DefaultEventProcessor.java +++ b/src/main/java/com/launchdarkly/client/DefaultEventProcessor.java @@ -3,6 +3,8 @@ import com.google.common.util.concurrent.ThreadFactoryBuilder; import com.google.gson.JsonElement; import com.google.gson.annotations.SerializedName; +import com.launchdarkly.client.EventSummarizer.CounterKey; +import com.launchdarkly.client.EventSummarizer.CounterValue; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -10,6 +12,7 @@ import java.io.IOException; import java.text.SimpleDateFormat; import java.util.ArrayList; +import java.util.HashMap; import java.util.List; import java.util.Map; import java.util.Random; @@ -235,10 +238,27 @@ private EventOutput createEventOutput(Event e) { } } + private EventOutput createSummaryEvent(EventSummarizer.EventSummary summary) { + Map flagsOut = new HashMap<>(); + for (Map.Entry entry: summary.counters.entrySet()) { + SummaryEventFlag fsd = flagsOut.get(entry.getKey().key); + if (fsd == null) { + fsd = new SummaryEventFlag(entry.getValue().defaultVal, new ArrayList()); + flagsOut.put(entry.getKey().key, fsd); + } + SummaryEventCounter c = new SummaryEventCounter(entry.getValue().flagValue, + entry.getKey().version == 0 ? null : entry.getKey().version, + entry.getValue().count, + entry.getKey().version == 0 ? true : null); + fsd.counters.add(c); + } + return new SummaryEventOutput(summary.startDate, summary.endDate, flagsOut); + } + private void dispatchFlush(EventProcessorMessage message) { Event[] events = buffer.toArray(new Event[buffer.size()]); buffer.clear(); - EventSummarizer.SummaryState snapshot = summarizer.snapshot(); + EventSummarizer.EventSummary snapshot = summarizer.snapshot(); if (events.length > 0 || !snapshot.isEmpty()) { this.scheduler.schedule(new FlushTask(events, snapshot, message), 0, TimeUnit.SECONDS); } else { @@ -249,10 +269,10 @@ private void dispatchFlush(EventProcessorMessage message) { private class FlushTask implements Runnable { private final Logger logger = LoggerFactory.getLogger(FlushTask.class); private final Event[] events; - private final EventSummarizer.SummaryState snapshot; + private final EventSummarizer.EventSummary snapshot; private final EventProcessorMessage message; - FlushTask(Event[] events, EventSummarizer.SummaryState snapshot, EventProcessorMessage message) { + FlushTask(Event[] events, EventSummarizer.EventSummary snapshot, EventProcessorMessage message) { this.events = events; this.snapshot = snapshot; this.message = message; @@ -265,9 +285,7 @@ public void run() { eventsOut.add(createEventOutput(event)); } if (!snapshot.isEmpty()) { - EventSummarizer.SummaryOutput summary = summarizer.output(snapshot); - SummaryEventOutput seo = new SummaryEventOutput(summary.startDate, summary.endDate, summary.features); - eventsOut.add(seo); + eventsOut.add(createSummaryEvent(snapshot)); } if (!eventsOut.isEmpty()) { postEvents(eventsOut); @@ -442,13 +460,39 @@ private static class SummaryEventOutput implements EventOutput { private final String kind; private final long startDate; private final long endDate; - private final Map features; + private final Map features; - SummaryEventOutput(long startDate, long endDate, Map features) { + SummaryEventOutput(long startDate, long endDate, Map features) { this.kind = "summary"; this.startDate = startDate; this.endDate = endDate; this.features = features; } } + + @SuppressWarnings("unused") + private static class SummaryEventFlag { + @SerializedName("default") final JsonElement defaultVal; + final List counters; + + SummaryEventFlag(JsonElement defaultVal, List counters) { + this.defaultVal = defaultVal; + this.counters = counters; + } + } + + @SuppressWarnings("unused") + private static class SummaryEventCounter { + final JsonElement value; + final Integer version; + final int count; + final Boolean unknown; + + SummaryEventCounter(JsonElement value, Integer version, int count, Boolean unknown) { + this.value = value; + this.version = version; + this.count = count; + this.unknown = unknown; + } + } } diff --git a/src/main/java/com/launchdarkly/client/EventSummarizer.java b/src/main/java/com/launchdarkly/client/EventSummarizer.java index 659593353..3ba24df93 100644 --- a/src/main/java/com/launchdarkly/client/EventSummarizer.java +++ b/src/main/java/com/launchdarkly/client/EventSummarizer.java @@ -1,12 +1,9 @@ package com.launchdarkly.client; import com.google.gson.JsonElement; -import com.google.gson.annotations.SerializedName; -import java.util.ArrayList; import java.util.HashMap; import java.util.LinkedHashMap; -import java.util.List; import java.util.Map; import java.util.Objects; @@ -17,11 +14,11 @@ * single event-processing thread. */ class EventSummarizer { - private SummaryState eventsState; + private EventSummary eventsState; private final SimpleLRUCache userKeys; EventSummarizer(LDConfig config) { - this.eventsState = new SummaryState(); + this.eventsState = new EventSummary(); this.userKeys = new SimpleLRUCache(config.userKeysCapacity); } @@ -61,34 +58,12 @@ void summarizeEvent(Event event) { * Returns a snapshot of the current summarized event data, and resets this state. * @return the previous event state */ - SummaryState snapshot() { - SummaryState ret = eventsState; - eventsState = new SummaryState(); + EventSummary snapshot() { + EventSummary ret = eventsState; + eventsState = new EventSummary(); return ret; } - /** - * Transforms the summary data into the format used for event sending. - * @param snapshot the data obtained from {@link #snapshot()} - * @return the formatted output - */ - SummaryOutput output(SummaryState snapshot) { - Map flagsOut = new HashMap<>(); - for (Map.Entry entry: snapshot.counters.entrySet()) { - FlagSummaryData fsd = flagsOut.get(entry.getKey().key); - if (fsd == null) { - fsd = new FlagSummaryData(entry.getValue().defaultVal, new ArrayList()); - flagsOut.put(entry.getKey().key, fsd); - } - CounterData c = new CounterData(entry.getValue().flagValue, - entry.getKey().version == 0 ? null : entry.getKey().version, - entry.getValue().count, - entry.getKey().version == 0 ? true : null); - fsd.counters.add(c); - } - return new SummaryOutput(snapshot.startDate, snapshot.endDate, flagsOut); - } - @SuppressWarnings("serial") private static class SimpleLRUCache extends LinkedHashMap { // http://chriswu.me/blog/a-lru-cache-in-10-lines-of-java/ @@ -105,12 +80,12 @@ protected boolean removeEldestEntry(Map.Entry eldest) { } } - static class SummaryState { + static class EventSummary { final Map counters; long startDate; long endDate; - SummaryState() { + EventSummary() { counters = new HashMap(); } @@ -141,8 +116,8 @@ void noteTimestamp(long time) { @Override public boolean equals(Object other) { - if (other instanceof SummaryState) { - SummaryState o = (SummaryState)other; + if (other instanceof EventSummary) { + EventSummary o = (EventSummary)other; return o.counters.equals(counters) && startDate == o.startDate && endDate == o.endDate; } return true; @@ -154,10 +129,10 @@ public int hashCode() { } } - private static class CounterKey { - private final String key; - private final int variation; - private final int version; + static class CounterKey { + final String key; + final int variation; + final int version; CounterKey(String key, int variation, int version) { this.key = key; @@ -178,12 +153,17 @@ public boolean equals(Object other) { public int hashCode() { return key.hashCode() + 31 * (variation + 31 * version); } + + @Override + public String toString() { + return "(" + key + "," + variation + "," + version + ")"; + } } - private static class CounterValue { - private int count; - private final JsonElement flagValue; - private final JsonElement defaultVal; + static class CounterValue { + int count; + final JsonElement flagValue; + final JsonElement defaultVal; CounterValue(int count, JsonElement flagValue, JsonElement defaultVal) { this.count = count; @@ -194,100 +174,20 @@ private static class CounterValue { void increment() { count = count + 1; } - } - - static class FlagSummaryData { - @SerializedName("default") final JsonElement defaultVal; - final List counters; - - FlagSummaryData(JsonElement defaultVal, List counters) { - this.defaultVal = defaultVal; - this.counters = counters; - } @Override - public boolean equals(Object other) { - if (other instanceof FlagSummaryData) { - FlagSummaryData o = (FlagSummaryData)other; - return Objects.equals(defaultVal, o.defaultVal) && counters.equals(o.counters); + public boolean equals(Object other) + { + if (other instanceof CounterValue) { + CounterValue o = (CounterValue)other; + return count == o.count && Objects.equals(flagValue, o.flagValue) && + Objects.equals(defaultVal, o.defaultVal); } return false; } - - @Override - public int hashCode() { - return Objects.hashCode(defaultVal) + 31 * counters.hashCode(); - } - - @Override - public String toString() { - return "{" + defaultVal + ", " + counters + "}"; - } - } - - static class CounterData { - final JsonElement value; - final Integer version; - final int count; - final Boolean unknown; - - CounterData(JsonElement value, Integer version, int count, Boolean unknown) { - this.value = value; - this.version = version; - this.count = count; - this.unknown = unknown; - } - - @Override - public boolean equals(Object other) { - if (other instanceof CounterData) { - CounterData o = (CounterData)other; - return Objects.equals(value, o.value) && Objects.equals(version, o.version) && - o.count == count && Objects.deepEquals(unknown, o.unknown); - } - return false; - } - - @Override - public int hashCode() { - return Objects.hashCode(value) + 31 * (Objects.hashCode(version) + 31 * - (count + 31 * (Objects.hashCode(unknown)))); - } - - @Override - public String toString() { - return "{" + value + ", " + version + ", " + count + ", " + unknown + "}"; - } - } - - static class SummaryOutput { - final long startDate; - final long endDate; - final Map features; - - SummaryOutput(long startDate, long endDate, Map features) { - this.startDate = startDate; - this.endDate = endDate; - this.features = features; - } - - @Override - public boolean equals(Object other) { - if (other instanceof SummaryOutput) { - SummaryOutput o = (SummaryOutput)other; - return o.startDate == startDate && o.endDate == endDate && o.features.equals(features); - } - return false; - } - - @Override - public int hashCode() { - return features.hashCode() + 31 * ((int)startDate + 31 * (int)endDate); - } - @Override public String toString() { - return "{" + startDate + ", " + endDate + ", " + features + "}"; + return "(" + count + "," + flagValue + "," + defaultVal + ")"; } } } diff --git a/src/test/java/com/launchdarkly/client/EventSummarizerTest.java b/src/test/java/com/launchdarkly/client/EventSummarizerTest.java index c5a29bbc4..cfc0468a3 100644 --- a/src/test/java/com/launchdarkly/client/EventSummarizerTest.java +++ b/src/test/java/com/launchdarkly/client/EventSummarizerTest.java @@ -1,16 +1,11 @@ package com.launchdarkly.client; -import com.google.gson.JsonElement; -import com.google.gson.JsonPrimitive; -import com.launchdarkly.client.EventSummarizer.CounterData; - import org.junit.Test; -import java.util.Arrays; -import java.util.Comparator; import java.util.HashMap; import java.util.Map; +import static com.launchdarkly.client.TestUtil.js; import static org.hamcrest.MatcherAssert.assertThat; import static org.hamcrest.Matchers.equalTo; import static org.junit.Assert.assertEquals; @@ -61,7 +56,7 @@ public void oldestUserForgottenIfCapacityExceeded() { @Test public void summarizeEventDoesNothingForIdentifyEvent() { EventSummarizer es = new EventSummarizer(defaultConfig); - EventSummarizer.SummaryState snapshot = es.snapshot(); + EventSummarizer.EventSummary snapshot = es.snapshot(); es.summarizeEvent(eventFactory.newIdentifyEvent(user)); assertEquals(snapshot, es.snapshot()); @@ -70,7 +65,7 @@ public void summarizeEventDoesNothingForIdentifyEvent() { @Test public void summarizeEventDoesNothingForCustomEvent() { EventSummarizer es = new EventSummarizer(defaultConfig); - EventSummarizer.SummaryState snapshot = es.snapshot(); + EventSummarizer.EventSummary snapshot = es.snapshot(); es.summarizeEvent(eventFactory.newCustomEvent("whatever", user, null)); assertEquals(snapshot, es.snapshot()); @@ -89,7 +84,7 @@ public void summarizeEventSetsStartAndEndDates() { es.summarizeEvent(event1); es.summarizeEvent(event2); es.summarizeEvent(event3); - EventSummarizer.SummaryOutput data = es.output(es.snapshot()); + EventSummarizer.EventSummary data = es.snapshot(); assertEquals(1000, data.startDate); assertEquals(2000, data.endDate); @@ -101,47 +96,31 @@ public void summarizeEventIncrementsCounters() { FeatureFlag flag1 = new FeatureFlagBuilder("key1").version(11).build(); FeatureFlag flag2 = new FeatureFlagBuilder("key2").version(22).build(); String unknownFlagKey = "badkey"; - JsonElement default1 = new JsonPrimitive("default1"); - JsonElement default2 = new JsonPrimitive("default2"); - JsonElement default3 = new JsonPrimitive("default3"); Event event1 = eventFactory.newFeatureRequestEvent(flag1, user, - new FeatureFlag.VariationAndValue(1, new JsonPrimitive("value1")), default1); + new FeatureFlag.VariationAndValue(1, js("value1")), js("default1")); Event event2 = eventFactory.newFeatureRequestEvent(flag1, user, - new FeatureFlag.VariationAndValue(2, new JsonPrimitive("value2")), default1); + new FeatureFlag.VariationAndValue(2, js("value2")), js("default1")); Event event3 = eventFactory.newFeatureRequestEvent(flag2, user, - new FeatureFlag.VariationAndValue(1, new JsonPrimitive("value99")), default2); + new FeatureFlag.VariationAndValue(1, js("value99")), js("default2")); Event event4 = eventFactory.newFeatureRequestEvent(flag1, user, - new FeatureFlag.VariationAndValue(1, new JsonPrimitive("value1")), default1); - Event event5 = eventFactory.newUnknownFeatureRequestEvent(unknownFlagKey, user, default3); + new FeatureFlag.VariationAndValue(1, js("value1")), js("default1")); + Event event5 = eventFactory.newUnknownFeatureRequestEvent(unknownFlagKey, user, js("default3")); es.summarizeEvent(event1); es.summarizeEvent(event2); es.summarizeEvent(event3); es.summarizeEvent(event4); es.summarizeEvent(event5); - EventSummarizer.SummaryOutput data = es.output(es.snapshot()); + EventSummarizer.EventSummary data = es.snapshot(); - data.features.get(flag1.getKey()).counters.sort(new CounterValueComparator()); - EventSummarizer.CounterData expected1 = new EventSummarizer.CounterData( - new JsonPrimitive("value1"), flag1.getVersion(), 2, null); - EventSummarizer.CounterData expected2 = new EventSummarizer.CounterData( - new JsonPrimitive("value2"), flag1.getVersion(), 1, null); - EventSummarizer.CounterData expected3 = new EventSummarizer.CounterData( - new JsonPrimitive("value99"), flag2.getVersion(), 1, null); - EventSummarizer.CounterData expected4 = new EventSummarizer.CounterData( - default3, null, 1, true); - Map expectedFeatures = new HashMap<>(); - expectedFeatures.put(flag1.getKey(), new EventSummarizer.FlagSummaryData(default1, - Arrays.asList(expected1, expected2))); - expectedFeatures.put(flag2.getKey(), new EventSummarizer.FlagSummaryData(default2, - Arrays.asList(expected3))); - expectedFeatures.put(unknownFlagKey, new EventSummarizer.FlagSummaryData(default3, - Arrays.asList(expected4))); - assertThat(data.features, equalTo(expectedFeatures)); - } - - private static class CounterValueComparator implements Comparator { - public int compare(CounterData o1, CounterData o2) { - return o1.value.getAsString().compareTo(o2.value.getAsString()); - } + Map expected = new HashMap<>(); + expected.put(new EventSummarizer.CounterKey(flag1.getKey(), 1, flag1.getVersion()), + new EventSummarizer.CounterValue(2, js("value1"), js("default1"))); + expected.put(new EventSummarizer.CounterKey(flag1.getKey(), 2, flag1.getVersion()), + new EventSummarizer.CounterValue(1, js("value2"), js("default1"))); + expected.put(new EventSummarizer.CounterKey(flag2.getKey(), 1, flag2.getVersion()), + new EventSummarizer.CounterValue(1, js("value99"), js("default2"))); + expected.put(new EventSummarizer.CounterKey(unknownFlagKey, 0, 0), + new EventSummarizer.CounterValue(1, js("default3"), js("default3"))); + assertThat(data.counters, equalTo(expected)); } } From 87103c61566c85358974836ff2c97b591d4dcf74 Mon Sep 17 00:00:00 2001 From: Eli Bishop Date: Sun, 25 Mar 2018 11:39:31 -0700 Subject: [PATCH 39/67] more reorganization - breaking up DefaultEventProcessor into 3 classes --- .../client/DefaultEventProcessor.java | 431 ++++++++++-------- 1 file changed, 245 insertions(+), 186 deletions(-) diff --git a/src/main/java/com/launchdarkly/client/DefaultEventProcessor.java b/src/main/java/com/launchdarkly/client/DefaultEventProcessor.java index e185bd275..002330e4b 100644 --- a/src/main/java/com/launchdarkly/client/DefaultEventProcessor.java +++ b/src/main/java/com/launchdarkly/client/DefaultEventProcessor.java @@ -36,31 +36,22 @@ class DefaultEventProcessor implements EventProcessor { static final SimpleDateFormat HTTP_DATE_FORMAT = new SimpleDateFormat("EEE, dd MMM yyyy HH:mm:ss zzz"); private static final int CHANNEL_BLOCK_MILLIS = 1000; - private final ScheduledExecutorService scheduler; - private final Thread mainThread; private final BlockingQueue inputChannel; - private final ArrayList buffer; - private final String sdkKey; - private final LDConfig config; - private final EventSummarizer summarizer; - private final Random random = new Random(); - private final AtomicLong lastKnownPastTime = new AtomicLong(0); - private final AtomicBoolean stopped = new AtomicBoolean(false); - private boolean capacityExceeded = false; - + private final EventConsumer consumer; + private final ThreadFactory threadFactory; + private final ScheduledExecutorService scheduler; + DefaultEventProcessor(String sdkKey, LDConfig config) { - this.sdkKey = sdkKey; - this.inputChannel = new ArrayBlockingQueue<>(config.capacity); - this.buffer = new ArrayList<>(config.capacity); - this.summarizer = new EventSummarizer(config); - this.config = config; - - ThreadFactory threadFactory = new ThreadFactoryBuilder() + inputChannel = new ArrayBlockingQueue<>(config.capacity); + + threadFactory = new ThreadFactoryBuilder() .setDaemon(true) .setNameFormat("LaunchDarkly-EventProcessor-%d") .build(); - - this.scheduler = Executors.newSingleThreadScheduledExecutor(threadFactory); + scheduler = Executors.newSingleThreadScheduledExecutor(threadFactory); + + consumer = new EventConsumer(sdkKey, config, inputChannel, threadFactory, scheduler); + Runnable flusher = new Runnable() { public void run() { postMessageAsync(MessageType.FLUSH, null); @@ -74,60 +65,23 @@ public void run() { }; this.scheduler.scheduleAtFixedRate(userKeysFlusher, config.userKeysFlushInterval, config.userKeysFlushInterval, TimeUnit.SECONDS); - - mainThread = threadFactory.newThread(new MainLoop()); - mainThread.start(); } @Override public void sendEvent(Event e) { - if (!stopped.get()) { - postMessageAsync(MessageType.EVENT, e); - } + postMessageAsync(MessageType.EVENT, e); } @Override public void flush() { - if (!stopped.get()) { - postMessageAndWait(MessageType.FLUSH, null); - } + postMessageAndWait(MessageType.FLUSH, null); } @Override public void close() throws IOException { this.flush(); + consumer.close(); scheduler.shutdown(); - stopped.set(true); - mainThread.interrupt(); - } - - /** - * This task drains the input queue as quickly as possible. Everything here is done on a single - * thread so we don't have to synchronize on our internal structures; when it's time to flush, - * dispatchFlush will fire off another task to do the part that takes longer. - */ - private class MainLoop implements Runnable { - public void run() { - while (!stopped.get()) { - try { - EventProcessorMessage message = inputChannel.take(); - switch(message.type) { - case EVENT: - dispatchEvent(message.event); - message.completed(); - break; - case FLUSH: - dispatchFlush(message); - case FLUSH_USERS: - summarizer.resetUsers(); - } - } catch (InterruptedException e) { - } catch (Exception e) { - logger.error("Unexpected error in event processor: " + e); - logger.debug(e.getMessage(), e); - } - } - } } private void postMessageAsync(MessageType type, Event event) { @@ -155,149 +109,227 @@ private void postToChannel(EventProcessorMessage message) { } } - private void dispatchEvent(Event e) { - // For each user we haven't seen before, we add an index event - unless this is already - // an identify event for that user. - if (!config.inlineUsersInEvents && e.user != null && !summarizer.noticeUser(e.user)) { - if (!(e instanceof IdentifyEvent)) { - IndexEvent ie = new IndexEvent(e.creationDate, e.user); - queueEvent(ie); + /** + * Takes messages from the input queue, updating the event buffer and summary counters + * on its own thread. + */ + private static class EventConsumer { + private final String sdkKey; + private final LDConfig config; + private final BlockingQueue inputChannel; + private final ScheduledExecutorService scheduler; + private final Thread mainThread; + private final ArrayList buffer; + private final EventSummarizer summarizer; + private final Random random = new Random(); + private final AtomicLong lastKnownPastTime = new AtomicLong(0); + private final AtomicBoolean disabled = new AtomicBoolean(false); + private final AtomicBoolean shutdown = new AtomicBoolean(false); + private boolean capacityExceeded = false; + + private EventConsumer(String sdkKey, LDConfig config, + BlockingQueue inputChannel, + ThreadFactory threadFactory, ScheduledExecutorService scheduler) { + this.sdkKey = sdkKey; + this.config = config; + this.inputChannel = inputChannel; + this.scheduler = scheduler; + this.buffer = new ArrayList<>(config.capacity); + this.summarizer = new EventSummarizer(config); + + mainThread = threadFactory.newThread(new Runnable() { + public void run() { + runMainLoop(); + } + }); + mainThread.start(); + } + + void close() { + shutdown.set(true); + mainThread.interrupt(); + } + + /** + * This task drains the input queue as quickly as possible. Everything here is done on a single + * thread so we don't have to synchronize on our internal structures; when it's time to flush, + * dispatchFlush will fire off another task to do the part that takes longer. + */ + private void runMainLoop() { + while (!shutdown.get()) { + try { + EventProcessorMessage message = inputChannel.take(); + switch(message.type) { + case EVENT: + dispatchEvent(message.event); + message.completed(); + break; + case FLUSH: + dispatchFlush(message); + case FLUSH_USERS: + summarizer.resetUsers(); + } + } catch (InterruptedException e) { + } catch (Exception e) { + logger.error("Unexpected error in event processor: " + e); + logger.debug(e.getMessage(), e); + } } } - // Always record the event in the summarizer. - summarizer.summarizeEvent(e); - - if (shouldTrackFullEvent(e)) { - // Sampling interval applies only to fully-tracked events. - if (config.samplingInterval > 0 && random.nextInt(config.samplingInterval) != 0) { + private void dispatchEvent(Event e) { + if (disabled.get()) { return; } - // Queue the event as-is; we'll transform it into an output event when we're flushing - // (to avoid doing that work on our main thread). - queueEvent(e); - } - } - - private void queueEvent(Event e) { - if (buffer.size() >= config.capacity) { - if (!capacityExceeded) { // don't need AtomicBoolean, this is only checked on one thread - capacityExceeded = true; - logger.warn("Exceeded event queue capacity. Increase capacity to avoid dropping events."); + + // For each user we haven't seen before, we add an index event - unless this is already + // an identify event for that user. + if (!config.inlineUsersInEvents && e.user != null && !summarizer.noticeUser(e.user)) { + if (!(e instanceof IdentifyEvent)) { + IndexEvent ie = new IndexEvent(e.creationDate, e.user); + addToBuffer(ie); + } + } + + // Always record the event in the summarizer. + summarizer.summarizeEvent(e); + + if (shouldTrackFullEvent(e)) { + // Sampling interval applies only to fully-tracked events. + if (config.samplingInterval > 0 && random.nextInt(config.samplingInterval) != 0) { + return; + } + // Queue the event as-is; we'll transform it into an output event when we're flushing + // (to avoid doing that work on our main thread). + addToBuffer(e); } - } else { - capacityExceeded = false; - buffer.add(e); } - } - - private boolean shouldTrackFullEvent(Event e) { - if (e instanceof FeatureRequestEvent) { - FeatureRequestEvent fe = (FeatureRequestEvent)e; - if (fe.trackEvents) { - return true; + + private void addToBuffer(Event e) { + if (buffer.size() >= config.capacity) { + if (!capacityExceeded) { // don't need AtomicBoolean, this is only checked on one thread + capacityExceeded = true; + logger.warn("Exceeded event queue capacity. Increase capacity to avoid dropping events."); + } + } else { + capacityExceeded = false; + buffer.add(e); } - if (fe.debugEventsUntilDate != null) { - // The "last known past time" comes from the last HTTP response we got from the server. - // In case the client's time is set wrong, at least we know that any expiration date - // earlier than that point is definitely in the past. If there's any discrepancy, we - // want to err on the side of cutting off event debugging sooner. - long lastPast = lastKnownPastTime.get(); - if (fe.debugEventsUntilDate > lastPast && - fe.debugEventsUntilDate > System.currentTimeMillis()) { + } + + private boolean shouldTrackFullEvent(Event e) { + if (e instanceof FeatureRequestEvent) { + FeatureRequestEvent fe = (FeatureRequestEvent)e; + if (fe.trackEvents) { return true; } + if (fe.debugEventsUntilDate != null) { + // The "last known past time" comes from the last HTTP response we got from the server. + // In case the client's time is set wrong, at least we know that any expiration date + // earlier than that point is definitely in the past. If there's any discrepancy, we + // want to err on the side of cutting off event debugging sooner. + long lastPast = lastKnownPastTime.get(); + if (fe.debugEventsUntilDate > lastPast && + fe.debugEventsUntilDate > System.currentTimeMillis()) { + return true; + } + } + return false; + } else { + return true; } - return false; - } else { - return true; - } - } - - private EventOutput createEventOutput(Event e) { - String userKey = e.user == null ? null : e.user.getKeyAsString(); - if (e instanceof FeatureRequestEvent) { - FeatureRequestEvent fe = (FeatureRequestEvent)e; - boolean isDebug = (!fe.trackEvents && fe.debugEventsUntilDate != null); - return new FeatureRequestEventOutput(fe.creationDate, fe.key, - config.inlineUsersInEvents ? null : userKey, - config.inlineUsersInEvents ? e.user : null, - fe.version, fe.value, fe.defaultVal, fe.prereqOf, isDebug); - } else if (e instanceof IdentifyEvent) { - return new IdentifyEventOutput(e.creationDate, e.user); - } else if (e instanceof CustomEvent) { - CustomEvent ce = (CustomEvent)e; - return new CustomEventOutput(ce.creationDate, ce.key, - config.inlineUsersInEvents ? null : userKey, - config.inlineUsersInEvents ? e.user : null, - ce.data); - } else if (e instanceof IndexEvent) { - return new IndexEventOutput(e.creationDate, e.user); - } else { - return null; } - } - private EventOutput createSummaryEvent(EventSummarizer.EventSummary summary) { - Map flagsOut = new HashMap<>(); - for (Map.Entry entry: summary.counters.entrySet()) { - SummaryEventFlag fsd = flagsOut.get(entry.getKey().key); - if (fsd == null) { - fsd = new SummaryEventFlag(entry.getValue().defaultVal, new ArrayList()); - flagsOut.put(entry.getKey().key, fsd); + private void dispatchFlush(EventProcessorMessage message) { + if (disabled.get()) { + message.completed(); + return; + } + + Event[] events = buffer.toArray(new Event[buffer.size()]); + buffer.clear(); + EventSummarizer.EventSummary snapshot = summarizer.snapshot(); + if (events.length == 0 && snapshot.isEmpty()) { + message.completed(); + } else { + EventResponseListener listener = new EventResponseListener() { + public void onEventResponseReceived(Response response) { + handleResponse(response); + } + }; + EventPayloadSender task = new EventPayloadSender(events, snapshot, message, + listener, sdkKey, config); + scheduler.schedule(task, 0, TimeUnit.SECONDS); } - SummaryEventCounter c = new SummaryEventCounter(entry.getValue().flagValue, - entry.getKey().version == 0 ? null : entry.getKey().version, - entry.getValue().count, - entry.getKey().version == 0 ? true : null); - fsd.counters.add(c); } - return new SummaryEventOutput(summary.startDate, summary.endDate, flagsOut); - } - - private void dispatchFlush(EventProcessorMessage message) { - Event[] events = buffer.toArray(new Event[buffer.size()]); - buffer.clear(); - EventSummarizer.EventSummary snapshot = summarizer.snapshot(); - if (events.length > 0 || !snapshot.isEmpty()) { - this.scheduler.schedule(new FlushTask(events, snapshot, message), 0, TimeUnit.SECONDS); - } else { - message.completed(); + + private void handleResponse(Response response) { + logger.debug("Events Response: " + response.code()); + try { + String dateStr = response.header("Date"); + if (dateStr != null) { + lastKnownPastTime.set(HTTP_DATE_FORMAT.parse(dateStr).getTime()); + } + } catch (Exception e) { + } + if (!response.isSuccessful()) { + logger.info("Got unexpected response when posting events: " + response); + if (response.code() == 401) { + disabled.set(true); + logger.error("Received 401 error, no further events will be posted since SDK key is invalid"); + } + } } } + + private interface EventResponseListener { + void onEventResponseReceived(Response response); + } - private class FlushTask implements Runnable { - private final Logger logger = LoggerFactory.getLogger(FlushTask.class); + /** + * Transforms the internal event data into the JSON event payload format and sends it off. + * This is done on a separate worker thread. + */ + private static class EventPayloadSender implements Runnable { + private static final Logger logger = LoggerFactory.getLogger(EventPayloadSender.class); + private final Event[] events; - private final EventSummarizer.EventSummary snapshot; + private final EventSummarizer.EventSummary summary; private final EventProcessorMessage message; + private final EventResponseListener listener; + private final String sdkKey; + private final LDConfig config; - FlushTask(Event[] events, EventSummarizer.EventSummary snapshot, EventProcessorMessage message) { + EventPayloadSender(Event[] events, EventSummarizer.EventSummary summary, EventProcessorMessage message, + EventResponseListener listener, String sdkKey, LDConfig config) { this.events = events; - this.snapshot = snapshot; + this.summary = summary; this.message = message; + this.listener = listener; + this.sdkKey = sdkKey; + this.config = config; } public void run() { try { - List eventsOut = new ArrayList<>(events.length + 1); - for (Event event: events) { - eventsOut.add(createEventOutput(event)); - } - if (!snapshot.isEmpty()) { - eventsOut.add(createSummaryEvent(snapshot)); - } - if (!eventsOut.isEmpty()) { - postEvents(eventsOut); - } + doSend(); } catch (Exception e) { logger.error("Unexpected error in event processor: " + e); logger.debug(e.getMessage(), e); + } finally { + message.completed(); } - message.completed(); } - private void postEvents(List eventsOut) { + private void doSend() throws Exception { + List eventsOut = new ArrayList<>(events.length + 1); + for (Event event: events) { + eventsOut.add(createEventOutput(event)); + } + if (!summary.isEmpty()) { + eventsOut.add(createSummaryEvent(summary)); + } + String json = config.gson.toJson(eventsOut); logger.debug("Posting {} event(s) to {} with payload: {}", eventsOut.size(), config.eventsURI, json); @@ -309,27 +341,54 @@ private void postEvents(List eventsOut) { .build(); try (Response response = config.httpClient.newCall(request).execute()) { - if (!response.isSuccessful()) { - logger.info("Got unexpected response when posting events: " + response); - if (response.code() == 401) { - stopped.set(true); - logger.error("Received 401 error, no further events will be posted since SDK key is invalid"); - close(); - } - } else { - logger.debug("Events Response: " + response.code()); - try { - String dateStr = response.header("Date"); - if (dateStr != null) { - lastKnownPastTime.set(HTTP_DATE_FORMAT.parse(dateStr).getTime()); - } - } catch (Exception e) { - } + if (listener != null) { + listener.onEventResponseReceived(response); } } catch (IOException e) { logger.info("Unhandled exception in LaunchDarkly client when posting events to URL: " + request.url(), e); } } + + private EventOutput createEventOutput(Event e) { + String userKey = e.user == null ? null : e.user.getKeyAsString(); + if (e instanceof FeatureRequestEvent) { + FeatureRequestEvent fe = (FeatureRequestEvent)e; + boolean isDebug = (!fe.trackEvents && fe.debugEventsUntilDate != null); + return new FeatureRequestEventOutput(fe.creationDate, fe.key, + config.inlineUsersInEvents ? null : userKey, + config.inlineUsersInEvents ? e.user : null, + fe.version, fe.value, fe.defaultVal, fe.prereqOf, isDebug); + } else if (e instanceof IdentifyEvent) { + return new IdentifyEventOutput(e.creationDate, e.user); + } else if (e instanceof CustomEvent) { + CustomEvent ce = (CustomEvent)e; + return new CustomEventOutput(ce.creationDate, ce.key, + config.inlineUsersInEvents ? null : userKey, + config.inlineUsersInEvents ? e.user : null, + ce.data); + } else if (e instanceof IndexEvent) { + return new IndexEventOutput(e.creationDate, e.user); + } else { + return null; + } + } + + private EventOutput createSummaryEvent(EventSummarizer.EventSummary summary) { + Map flagsOut = new HashMap<>(); + for (Map.Entry entry: summary.counters.entrySet()) { + SummaryEventFlag fsd = flagsOut.get(entry.getKey().key); + if (fsd == null) { + fsd = new SummaryEventFlag(entry.getValue().defaultVal, new ArrayList()); + flagsOut.put(entry.getKey().key, fsd); + } + SummaryEventCounter c = new SummaryEventCounter(entry.getValue().flagValue, + entry.getKey().version == 0 ? null : entry.getKey().version, + entry.getValue().count, + entry.getKey().version == 0 ? true : null); + fsd.counters.add(c); + } + return new SummaryEventOutput(summary.startDate, summary.endDate, flagsOut); + } } private static enum MessageType { From 09f73ef1196f2582c8a9da56546fc8a7909dec32 Mon Sep 17 00:00:00 2001 From: Eli Bishop Date: Mon, 26 Mar 2018 12:39:32 -0700 Subject: [PATCH 40/67] avoid repeating log warning --- .../com/launchdarkly/client/DefaultEventProcessor.java | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/src/main/java/com/launchdarkly/client/DefaultEventProcessor.java b/src/main/java/com/launchdarkly/client/DefaultEventProcessor.java index 002330e4b..e25a6987a 100644 --- a/src/main/java/com/launchdarkly/client/DefaultEventProcessor.java +++ b/src/main/java/com/launchdarkly/client/DefaultEventProcessor.java @@ -40,7 +40,8 @@ class DefaultEventProcessor implements EventProcessor { private final EventConsumer consumer; private final ThreadFactory threadFactory; private final ScheduledExecutorService scheduler; - + private final AtomicBoolean inputCapacityExceeded = new AtomicBoolean(false); + DefaultEventProcessor(String sdkKey, LDConfig config) { inputChannel = new ArrayBlockingQueue<>(config.capacity); @@ -98,11 +99,14 @@ private void postToChannel(EventProcessorMessage message) { while (true) { try { if (inputChannel.offer(message, CHANNEL_BLOCK_MILLIS, TimeUnit.MILLISECONDS)) { + inputCapacityExceeded.set(false); break; } else { // This doesn't mean that the output event buffer is full, but rather that the main thread is // seriously backed up with not-yet-processed events. We shouldn't see this. - logger.warn("Events are being produced faster than they can be processed"); + if (inputCapacityExceeded.compareAndSet(false, true)) { + logger.warn("Events are being produced faster than they can be processed"); + } } } catch (InterruptedException ex) { } From 3254df4ff259d84ecabb4f5b7d2611fdaf8555f9 Mon Sep 17 00:00:00 2001 From: Eli Bishop Date: Tue, 27 Mar 2018 20:29:40 -0700 Subject: [PATCH 41/67] reorganize event processor code; make flushes async, except during Close --- .../client/DefaultEventProcessor.java | 174 +++++++++++------- .../launchdarkly/client/EventProcessor.java | 27 ++- .../launchdarkly/client/EventSummarizer.java | 64 ++----- .../com/launchdarkly/client/LDClient.java | 2 +- .../client/NullEventProcessor.java | 18 -- .../launchdarkly/client/SimpleLRUCache.java | 24 +++ .../client/DefaultEventProcessorTest.java | 63 ++++--- .../client/EventSummarizerTest.java | 42 +---- .../client/SimpleLRUCacheTest.java | 57 ++++++ 9 files changed, 266 insertions(+), 205 deletions(-) delete mode 100644 src/main/java/com/launchdarkly/client/NullEventProcessor.java create mode 100644 src/main/java/com/launchdarkly/client/SimpleLRUCache.java create mode 100644 src/test/java/com/launchdarkly/client/SimpleLRUCacheTest.java diff --git a/src/main/java/com/launchdarkly/client/DefaultEventProcessor.java b/src/main/java/com/launchdarkly/client/DefaultEventProcessor.java index e25a6987a..743d9eef2 100644 --- a/src/main/java/com/launchdarkly/client/DefaultEventProcessor.java +++ b/src/main/java/com/launchdarkly/client/DefaultEventProcessor.java @@ -1,5 +1,6 @@ package com.launchdarkly.client; +import com.google.common.annotations.VisibleForTesting; import com.google.common.util.concurrent.ThreadFactoryBuilder; import com.google.gson.JsonElement; import com.google.gson.annotations.SerializedName; @@ -18,12 +19,14 @@ import java.util.Random; import java.util.concurrent.ArrayBlockingQueue; import java.util.concurrent.BlockingQueue; +import java.util.concurrent.ExecutorService; import java.util.concurrent.Executors; import java.util.concurrent.ScheduledExecutorService; import java.util.concurrent.Semaphore; import java.util.concurrent.ThreadFactory; import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicBoolean; +import java.util.concurrent.atomic.AtomicInteger; import java.util.concurrent.atomic.AtomicLong; import okhttp3.MediaType; @@ -37,9 +40,9 @@ class DefaultEventProcessor implements EventProcessor { private static final int CHANNEL_BLOCK_MILLIS = 1000; private final BlockingQueue inputChannel; - private final EventConsumer consumer; private final ThreadFactory threadFactory; private final ScheduledExecutorService scheduler; + private final AtomicBoolean closed = new AtomicBoolean(false); private final AtomicBoolean inputCapacityExceeded = new AtomicBoolean(false); DefaultEventProcessor(String sdkKey, LDConfig config) { @@ -51,7 +54,7 @@ class DefaultEventProcessor implements EventProcessor { .build(); scheduler = Executors.newSingleThreadScheduledExecutor(threadFactory); - consumer = new EventConsumer(sdkKey, config, inputChannel, threadFactory, scheduler); + new EventConsumer(sdkKey, config, inputChannel, threadFactory); Runnable flusher = new Runnable() { public void run() { @@ -75,14 +78,22 @@ public void sendEvent(Event e) { @Override public void flush() { - postMessageAndWait(MessageType.FLUSH, null); + postMessageAsync(MessageType.FLUSH, null); } @Override public void close() throws IOException { - this.flush(); - consumer.close(); - scheduler.shutdown(); + if (closed.compareAndSet(false, true)) { + postMessageAsync(MessageType.FLUSH, null); + postMessageAndWait(MessageType.SHUTDOWN, null); + scheduler.shutdown(); + } + } + + @VisibleForTesting + void waitUntilInactive() throws IOException { + // Waits until there are no pending events or flushes + postMessageAndWait(MessageType.SYNC, null); } private void postMessageAsync(MessageType type, Event event) { @@ -118,29 +129,34 @@ private void postToChannel(EventProcessorMessage message) { * on its own thread. */ private static class EventConsumer { + private static final int MAX_FLUSH_THREADS = 5; + private final String sdkKey; private final LDConfig config; private final BlockingQueue inputChannel; - private final ScheduledExecutorService scheduler; + private final ExecutorService flushWorkersPool; + private final AtomicInteger flushWorkersActive; private final Thread mainThread; private final ArrayList buffer; private final EventSummarizer summarizer; + private final SimpleLRUCache userKeys; private final Random random = new Random(); private final AtomicLong lastKnownPastTime = new AtomicLong(0); private final AtomicBoolean disabled = new AtomicBoolean(false); - private final AtomicBoolean shutdown = new AtomicBoolean(false); private boolean capacityExceeded = false; private EventConsumer(String sdkKey, LDConfig config, BlockingQueue inputChannel, - ThreadFactory threadFactory, ScheduledExecutorService scheduler) { + ThreadFactory threadFactory) { this.sdkKey = sdkKey; this.config = config; this.inputChannel = inputChannel; - this.scheduler = scheduler; + this.flushWorkersPool = Executors.newFixedThreadPool(MAX_FLUSH_THREADS, threadFactory); + this.flushWorkersActive = new AtomicInteger(0); this.buffer = new ArrayList<>(config.capacity); - this.summarizer = new EventSummarizer(config); - + this.summarizer = new EventSummarizer(); + this.userKeys = new SimpleLRUCache(config.userKeysCapacity); + mainThread = threadFactory.newThread(new Runnable() { public void run() { runMainLoop(); @@ -149,30 +165,34 @@ public void run() { mainThread.start(); } - void close() { - shutdown.set(true); - mainThread.interrupt(); - } - /** * This task drains the input queue as quickly as possible. Everything here is done on a single * thread so we don't have to synchronize on our internal structures; when it's time to flush, * dispatchFlush will fire off another task to do the part that takes longer. */ private void runMainLoop() { - while (!shutdown.get()) { + boolean running = true; + while (running) { try { EventProcessorMessage message = inputChannel.take(); switch(message.type) { case EVENT: - dispatchEvent(message.event); - message.completed(); + processEvent(message.event); break; case FLUSH: - dispatchFlush(message); + startFlush(); + break; case FLUSH_USERS: - summarizer.resetUsers(); + userKeys.clear(); + break; + case SYNC: + synchronizeForTesting(); + break; + case SHUTDOWN: + finishAllPendingFlushes(); + running = false; } + message.completed(); } catch (InterruptedException e) { } catch (Exception e) { logger.error("Unexpected error in event processor: " + e); @@ -181,14 +201,38 @@ private void runMainLoop() { } } - private void dispatchEvent(Event e) { + private void finishAllPendingFlushes() { + flushWorkersPool.shutdown(); + try { + while (!flushWorkersPool.awaitTermination(CHANNEL_BLOCK_MILLIS, TimeUnit.MILLISECONDS)) { + logger.debug("Waiting for event flush tasks to terminate"); + } + } catch (InterruptedException e) { + } + } + + private void synchronizeForTesting() { + while (true) { + try { + synchronized(flushWorkersActive) { + if (flushWorkersActive.get() == 0) { + return; + } else { + flushWorkersActive.wait(); + } + } + } catch (InterruptedException e) {} + } + } + + private void processEvent(Event e) { if (disabled.get()) { return; } // For each user we haven't seen before, we add an index event - unless this is already // an identify event for that user. - if (!config.inlineUsersInEvents && e.user != null && !summarizer.noticeUser(e.user)) { + if (!config.inlineUsersInEvents && e.user != null && !noticeUser(e.user)) { if (!(e instanceof IdentifyEvent)) { IndexEvent ie = new IndexEvent(e.creationDate, e.user); addToBuffer(ie); @@ -209,6 +253,15 @@ private void dispatchEvent(Event e) { } } + // Add to the set of users we've noticed, and return true if the user was already known to us. + private boolean noticeUser(LDUser user) { + if (user == null || user.getKey() == null) { + return false; + } + String key = user.getKeyAsString(); + return userKeys.put(key, key) != null; + } + private void addToBuffer(Event e) { if (buffer.size() >= config.capacity) { if (!capacityExceeded) { // don't need AtomicBoolean, this is only checked on one thread @@ -244,26 +297,19 @@ private boolean shouldTrackFullEvent(Event e) { } } - private void dispatchFlush(EventProcessorMessage message) { + private void startFlush() { if (disabled.get()) { - message.completed(); return; } Event[] events = buffer.toArray(new Event[buffer.size()]); buffer.clear(); EventSummarizer.EventSummary snapshot = summarizer.snapshot(); - if (events.length == 0 && snapshot.isEmpty()) { - message.completed(); - } else { - EventResponseListener listener = new EventResponseListener() { - public void onEventResponseReceived(Response response) { - handleResponse(response); - } - }; - EventPayloadSender task = new EventPayloadSender(events, snapshot, message, - listener, sdkKey, config); - scheduler.schedule(task, 0, TimeUnit.SECONDS); + + if (events.length != 0 || !snapshot.isEmpty()) { + flushWorkersActive.incrementAndGet(); + EventPayloadSender task = new EventPayloadSender(this, events, snapshot); + flushWorkersPool.execute(task); } } @@ -286,10 +332,6 @@ private void handleResponse(Response response) { } } - private interface EventResponseListener { - void onEventResponseReceived(Response response); - } - /** * Transforms the internal event data into the JSON event payload format and sends it off. * This is done on a separate worker thread. @@ -297,21 +339,14 @@ private interface EventResponseListener { private static class EventPayloadSender implements Runnable { private static final Logger logger = LoggerFactory.getLogger(EventPayloadSender.class); + private final EventConsumer consumer; private final Event[] events; private final EventSummarizer.EventSummary summary; - private final EventProcessorMessage message; - private final EventResponseListener listener; - private final String sdkKey; - private final LDConfig config; - EventPayloadSender(Event[] events, EventSummarizer.EventSummary summary, EventProcessorMessage message, - EventResponseListener listener, String sdkKey, LDConfig config) { + EventPayloadSender(EventConsumer consumer, Event[] events, EventSummarizer.EventSummary summary) { + this.consumer = consumer; this.events = events; this.summary = summary; - this.message = message; - this.listener = listener; - this.sdkKey = sdkKey; - this.config = config; } public void run() { @@ -321,7 +356,10 @@ public void run() { logger.error("Unexpected error in event processor: " + e); logger.debug(e.getMessage(), e); } finally { - message.completed(); + synchronized(consumer.flushWorkersActive) { + consumer.flushWorkersActive.decrementAndGet(); + consumer.flushWorkersActive.notify(); + } } } @@ -334,20 +372,18 @@ private void doSend() throws Exception { eventsOut.add(createSummaryEvent(summary)); } - String json = config.gson.toJson(eventsOut); + String json = consumer.config.gson.toJson(eventsOut); logger.debug("Posting {} event(s) to {} with payload: {}", - eventsOut.size(), config.eventsURI, json); + eventsOut.size(), consumer.config.eventsURI, json); - Request request = config.getRequestBuilder(sdkKey) - .url(config.eventsURI.toString() + "/bulk") + Request request = consumer.config.getRequestBuilder(consumer.sdkKey) + .url(consumer.config.eventsURI.toString() + "/bulk") .post(RequestBody.create(MediaType.parse("application/json; charset=utf-8"), json)) .addHeader("Content-Type", "application/json") .build(); - try (Response response = config.httpClient.newCall(request).execute()) { - if (listener != null) { - listener.onEventResponseReceived(response); - } + try (Response response = consumer.config.httpClient.newCall(request).execute()) { + consumer.handleResponse(response); } catch (IOException e) { logger.info("Unhandled exception in LaunchDarkly client when posting events to URL: " + request.url(), e); } @@ -359,16 +395,16 @@ private EventOutput createEventOutput(Event e) { FeatureRequestEvent fe = (FeatureRequestEvent)e; boolean isDebug = (!fe.trackEvents && fe.debugEventsUntilDate != null); return new FeatureRequestEventOutput(fe.creationDate, fe.key, - config.inlineUsersInEvents ? null : userKey, - config.inlineUsersInEvents ? e.user : null, + consumer.config.inlineUsersInEvents ? null : userKey, + consumer.config.inlineUsersInEvents ? e.user : null, fe.version, fe.value, fe.defaultVal, fe.prereqOf, isDebug); } else if (e instanceof IdentifyEvent) { return new IdentifyEventOutput(e.creationDate, e.user); } else if (e instanceof CustomEvent) { CustomEvent ce = (CustomEvent)e; return new CustomEventOutput(ce.creationDate, ce.key, - config.inlineUsersInEvents ? null : userKey, - config.inlineUsersInEvents ? e.user : null, + consumer.config.inlineUsersInEvents ? null : userKey, + consumer.config.inlineUsersInEvents ? e.user : null, ce.data); } else if (e instanceof IndexEvent) { return new IndexEventOutput(e.creationDate, e.user); @@ -386,9 +422,9 @@ private EventOutput createSummaryEvent(EventSummarizer.EventSummary summary) { flagsOut.put(entry.getKey().key, fsd); } SummaryEventCounter c = new SummaryEventCounter(entry.getValue().flagValue, - entry.getKey().version == 0 ? null : entry.getKey().version, + entry.getKey().version, entry.getValue().count, - entry.getKey().version == 0 ? true : null); + entry.getKey().version == null ? true : null); fsd.counters.add(c); } return new SummaryEventOutput(summary.startDate, summary.endDate, flagsOut); @@ -398,7 +434,9 @@ private EventOutput createSummaryEvent(EventSummarizer.EventSummary summary) { private static enum MessageType { EVENT, FLUSH, - FLUSH_USERS + FLUSH_USERS, + SYNC, + SHUTDOWN } private static class EventProcessorMessage { @@ -498,7 +536,6 @@ private static class CustomEventOutput implements EventOutput { } } - @SuppressWarnings("unused") private static class IndexEvent extends Event { IndexEvent(long creationDate, LDUser user) { super(creationDate, user); @@ -533,7 +570,6 @@ private static class SummaryEventOutput implements EventOutput { } } - @SuppressWarnings("unused") private static class SummaryEventFlag { @SerializedName("default") final JsonElement defaultVal; final List counters; diff --git a/src/main/java/com/launchdarkly/client/EventProcessor.java b/src/main/java/com/launchdarkly/client/EventProcessor.java index 10d502d86..1517269c8 100644 --- a/src/main/java/com/launchdarkly/client/EventProcessor.java +++ b/src/main/java/com/launchdarkly/client/EventProcessor.java @@ -7,16 +7,33 @@ */ interface EventProcessor extends Closeable { /** - * Processes an event. This method is asynchronous; the event may be sent later in the background - * at an interval set by {@link LDConfig#flushInterval}, or due to a call to {@link #flush()}. + * Records an event asynchronously. * @param e an event */ void sendEvent(Event e); /** - * Finishes processing any events that have been buffered. In the default implementation, this means - * sending the events to LaunchDarkly. This method is synchronous; when it returns, you can assume - * that all events queued prior to the {@link #flush()} have now been delivered. + * Specifies that any buffered events should be sent as soon as possible, rather than waiting + * for the next flush interval. This method is asynchronous, so events still may not be sent + * until a later time. However, calling {@link Closeable#close()} will synchronously deliver + * any events that were not yet delivered prior to shutting down. */ void flush(); + + /** + * Stub implementation of {@link EventProcessor} for when we don't want to send any events. + */ + static class NullEventProcessor implements EventProcessor { + @Override + public void sendEvent(Event e) { + } + + @Override + public void flush() { + } + + @Override + public void close() { + } + } } diff --git a/src/main/java/com/launchdarkly/client/EventSummarizer.java b/src/main/java/com/launchdarkly/client/EventSummarizer.java index 3ba24df93..50e336845 100644 --- a/src/main/java/com/launchdarkly/client/EventSummarizer.java +++ b/src/main/java/com/launchdarkly/client/EventSummarizer.java @@ -3,43 +3,19 @@ import com.google.gson.JsonElement; import java.util.HashMap; -import java.util.LinkedHashMap; import java.util.Map; import java.util.Objects; /** - * Manages the state of summarizable information for the EventProcessor, including the - * event counters and user deduplication. Note that the methods of this class are - * deliberately not thread-safe, because they should always be called from EventProcessor's - * single event-processing thread. + * Manages the state of summarizable information for the EventProcessor. Note that the + * methods of this class are deliberately not thread-safe, because they should always + * be called from EventProcessor's single message-processing thread. */ class EventSummarizer { private EventSummary eventsState; - private final SimpleLRUCache userKeys; - EventSummarizer(LDConfig config) { + EventSummarizer() { this.eventsState = new EventSummary(); - this.userKeys = new SimpleLRUCache(config.userKeysCapacity); - } - - /** - * Add to the set of users we've noticed, and return true if the user was already known to us. - * @param user a user - * @return true if we've already seen this user key - */ - boolean noticeUser(LDUser user) { - if (user == null || user.getKey() == null) { - return false; - } - String key = user.getKeyAsString(); - return userKeys.put(key, key) != null; - } - - /** - * Reset the set of users we've seen. - */ - void resetUsers() { - userKeys.clear(); } /** @@ -63,23 +39,7 @@ EventSummary snapshot() { eventsState = new EventSummary(); return ret; } - - @SuppressWarnings("serial") - private static class SimpleLRUCache extends LinkedHashMap { - // http://chriswu.me/blog/a-lru-cache-in-10-lines-of-java/ - private final int capacity; - - SimpleLRUCache(int capacity) { - super(16, 0.75f, true); - this.capacity = capacity; - } - - @Override - protected boolean removeEldestEntry(Map.Entry eldest) { - return size() > capacity; - } - } - + static class EventSummary { final Map counters; long startDate; @@ -94,8 +54,7 @@ boolean isEmpty() { } void incrementCounter(String flagKey, Integer variation, Integer version, JsonElement flagValue, JsonElement defaultVal) { - CounterKey key = new CounterKey(flagKey, (variation == null) ? 0 : variation.intValue(), - (version == null) ? 0 : version.intValue()); + CounterKey key = new CounterKey(flagKey, variation, version); CounterValue value = counters.get(key); if (value != null) { @@ -131,10 +90,10 @@ public int hashCode() { static class CounterKey { final String key; - final int variation; - final int version; + final Integer variation; + final Integer version; - CounterKey(String key, int variation, int version) { + CounterKey(String key, Integer variation, Integer version) { this.key = key; this.variation = variation; this.version = version; @@ -144,14 +103,15 @@ static class CounterKey { public boolean equals(Object other) { if (other instanceof CounterKey) { CounterKey o = (CounterKey)other; - return o.key.equals(this.key) && o.variation == this.variation && o.version == this.version; + return o.key.equals(this.key) && Objects.equals(o.variation, this.variation) && + Objects.equals(o.version, this.version); } return false; } @Override public int hashCode() { - return key.hashCode() + 31 * (variation + 31 * version); + return key.hashCode() + 31 * (Objects.hashCode(variation) + 31 * Objects.hashCode(version)); } @Override diff --git a/src/main/java/com/launchdarkly/client/LDClient.java b/src/main/java/com/launchdarkly/client/LDClient.java index 22f6de549..a8f75952b 100644 --- a/src/main/java/com/launchdarkly/client/LDClient.java +++ b/src/main/java/com/launchdarkly/client/LDClient.java @@ -64,7 +64,7 @@ public LDClient(String sdkKey, LDConfig config) { this.sdkKey = sdkKey; this.requestor = createFeatureRequestor(sdkKey, config); if (config.offline || !config.sendEvents) { - this.eventProcessor = new NullEventProcessor(); + this.eventProcessor = new EventProcessor.NullEventProcessor(); } else { this.eventProcessor = createEventProcessor(sdkKey, config); } diff --git a/src/main/java/com/launchdarkly/client/NullEventProcessor.java b/src/main/java/com/launchdarkly/client/NullEventProcessor.java deleted file mode 100644 index 137d198f6..000000000 --- a/src/main/java/com/launchdarkly/client/NullEventProcessor.java +++ /dev/null @@ -1,18 +0,0 @@ -package com.launchdarkly.client; - -/** - * Stub implementation of {@link EventProcessor} for when we don't want to send any events. - */ -class NullEventProcessor implements EventProcessor { - @Override - public void sendEvent(Event e) { - } - - @Override - public void flush() { - } - - @Override - public void close() { - } -} diff --git a/src/main/java/com/launchdarkly/client/SimpleLRUCache.java b/src/main/java/com/launchdarkly/client/SimpleLRUCache.java new file mode 100644 index 000000000..a048e9a06 --- /dev/null +++ b/src/main/java/com/launchdarkly/client/SimpleLRUCache.java @@ -0,0 +1,24 @@ +package com.launchdarkly.client; + +import java.util.LinkedHashMap; +import java.util.Map; + +/** + * A very basic implementation of a LRU cache with a fixed capacity. Note that in this + * implementation, entries only become new again when written, not when read. + * See: http://chriswu.me/blog/a-lru-cache-in-10-lines-of-java/ + */ +@SuppressWarnings("serial") +class SimpleLRUCache extends LinkedHashMap { + private final int capacity; + + SimpleLRUCache(int capacity) { + super(16, 0.75f, true); + this.capacity = capacity; + } + + @Override + protected boolean removeEldestEntry(Map.Entry eldest) { + return size() > capacity; + } +} diff --git a/src/test/java/com/launchdarkly/client/DefaultEventProcessorTest.java b/src/test/java/com/launchdarkly/client/DefaultEventProcessorTest.java index 9297263fd..a82d8d4d1 100644 --- a/src/test/java/com/launchdarkly/client/DefaultEventProcessorTest.java +++ b/src/test/java/com/launchdarkly/client/DefaultEventProcessorTest.java @@ -23,6 +23,7 @@ import static org.hamcrest.Matchers.hasItems; import static org.hamcrest.Matchers.nullValue; import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertNotNull; import okhttp3.mockwebserver.MockResponse; import okhttp3.mockwebserver.MockWebServer; @@ -32,8 +33,10 @@ public class DefaultEventProcessorTest { private static final String SDK_KEY = "SDK_KEY"; private static final LDUser user = new LDUser.Builder("userkey").name("Red").build(); private static final Gson gson = new Gson(); - private static final JsonElement userJson = gson.fromJson("{\"key\":\"userkey\",\"name\":\"Red\"}", JsonElement.class); - private static final JsonElement filteredUserJson = gson.fromJson("{\"key\":\"userkey\",\"privateAttrs\":[\"name\"]}", JsonElement.class); + private static final JsonElement userJson = + gson.fromJson("{\"key\":\"userkey\",\"name\":\"Red\"}", JsonElement.class); + private static final JsonElement filteredUserJson = + gson.fromJson("{\"key\":\"userkey\",\"privateAttrs\":[\"name\"]}", JsonElement.class); private final LDConfig.Builder configBuilder = new LDConfig.Builder(); private final MockWebServer server = new MockWebServer(); @@ -61,12 +64,7 @@ public void identifyEventIsQueued() throws Exception { ep.sendEvent(e); JsonArray output = flushAndGetEvents(new MockResponse()); - assertThat(output, hasItems( - allOf( - hasJsonProperty("kind", "identify"), - hasJsonProperty("creationDate", (double)e.creationDate), - hasJsonProperty("user", userJson) - ))); + assertThat(output, hasItems(isIdentifyEvent(e, userJson))); } @SuppressWarnings("unchecked") @@ -78,12 +76,7 @@ public void userIsFilteredInIdentifyEvent() throws Exception { ep.sendEvent(e); JsonArray output = flushAndGetEvents(new MockResponse()); - assertThat(output, hasItems( - allOf( - hasJsonProperty("kind", "identify"), - hasJsonProperty("creationDate", (double)e.creationDate), - hasJsonProperty("user", filteredUserJson) - ))); + assertThat(output, hasItems(isIdentifyEvent(e, filteredUserJson))); } @SuppressWarnings("unchecked") @@ -308,9 +301,7 @@ public void customEventCanContainInlineUser() throws Exception { ep.sendEvent(ce); JsonArray output = flushAndGetEvents(new MockResponse()); - assertThat(output, hasItems( - isCustomEvent(ce, userJson) - )); + assertThat(output, hasItems(isCustomEvent(ce, userJson))); } @SuppressWarnings("unchecked") @@ -324,15 +315,26 @@ public void userIsFilteredInCustomEvent() throws Exception { ep.sendEvent(ce); JsonArray output = flushAndGetEvents(new MockResponse()); - assertThat(output, hasItems( - isCustomEvent(ce, filteredUserJson) - )); + assertThat(output, hasItems(isCustomEvent(ce, filteredUserJson))); + } + + @SuppressWarnings("unchecked") + @Test + public void closingEventProcessorForcesSynchronousFlush() throws Exception { + ep = new DefaultEventProcessor(SDK_KEY, configBuilder.build()); + Event e = EventFactory.DEFAULT.newIdentifyEvent(user); + ep.sendEvent(e); + + server.enqueue(new MockResponse()); + ep.close(); + JsonArray output = getEventsFromLastRequest(); + assertThat(output, hasItems(isIdentifyEvent(e, userJson))); } @Test public void nothingIsSentIfThereAreNoEvents() throws Exception { ep = new DefaultEventProcessor(SDK_KEY, configBuilder.build()); - ep.flush(); + ep.close(); assertEquals(0, server.getRequestCount()); } @@ -344,7 +346,7 @@ public void sdkKeyIsSent() throws Exception { ep.sendEvent(e); server.enqueue(new MockResponse()); - ep.flush(); + ep.close(); RecordedRequest req = server.takeRequest(); assertThat(req.getHeader("Authorization"), equalTo(SDK_KEY)); @@ -359,6 +361,7 @@ public void noMorePayloadsAreSentAfter401Error() throws Exception { ep.sendEvent(e); ep.flush(); + ep.waitUntilInactive(); RecordedRequest req = server.takeRequest(0, TimeUnit.SECONDS); assertThat(req, nullValue(RecordedRequest.class)); } @@ -370,10 +373,24 @@ private MockResponse addDateHeader(MockResponse response, long timestamp) { private JsonArray flushAndGetEvents(MockResponse response) throws Exception { server.enqueue(response); ep.flush(); - RecordedRequest req = server.takeRequest(); + ep.waitUntilInactive(); + return getEventsFromLastRequest(); + } + + private JsonArray getEventsFromLastRequest() throws Exception { + RecordedRequest req = server.takeRequest(0, TimeUnit.MILLISECONDS); + assertNotNull(req); return gson.fromJson(req.getBody().readUtf8(), JsonElement.class).getAsJsonArray(); } + private Matcher isIdentifyEvent(Event sourceEvent, JsonElement user) { + return allOf( + hasJsonProperty("kind", "identify"), + hasJsonProperty("creationDate", (double)sourceEvent.creationDate), + hasJsonProperty("user", user) + ); + } + private Matcher isIndexEvent(Event sourceEvent, JsonElement user) { return allOf( hasJsonProperty("kind", "index"), diff --git a/src/test/java/com/launchdarkly/client/EventSummarizerTest.java b/src/test/java/com/launchdarkly/client/EventSummarizerTest.java index cfc0468a3..6a7a83875 100644 --- a/src/test/java/com/launchdarkly/client/EventSummarizerTest.java +++ b/src/test/java/com/launchdarkly/client/EventSummarizerTest.java @@ -9,11 +9,8 @@ import static org.hamcrest.MatcherAssert.assertThat; import static org.hamcrest.Matchers.equalTo; import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertFalse; -import static org.junit.Assert.assertTrue; public class EventSummarizerTest { - private static final LDConfig defaultConfig = new LDConfig.Builder().userKeysCapacity(100).build(); private static final LDUser user = new LDUser.Builder("key").build(); private long eventTimestamp; @@ -24,38 +21,9 @@ protected long getTimestamp() { } }; - @Test - public void noticeUserReturnsFalseForNeverSeenUser() { - EventSummarizer es = new EventSummarizer(defaultConfig); - assertFalse(es.noticeUser(user)); - } - - @Test - public void noticeUserReturnsTrueForPreviouslySeenUser() { - EventSummarizer es = new EventSummarizer(defaultConfig); - es.noticeUser(user); - LDUser user2 = new LDUser.Builder(user).build(); - assertTrue(es.noticeUser(user2)); - } - - @Test - public void oldestUserForgottenIfCapacityExceeded() { - LDConfig config = new LDConfig.Builder().userKeysCapacity(2).build(); - EventSummarizer es = new EventSummarizer(config); - LDUser user1 = new LDUser.Builder("key1").build(); - LDUser user2 = new LDUser.Builder("key2").build(); - LDUser user3 = new LDUser.Builder("key3").build(); - es.noticeUser(user1); - es.noticeUser(user2); - es.noticeUser(user3); - assertTrue(es.noticeUser(user3)); - assertTrue(es.noticeUser(user2)); - assertFalse(es.noticeUser(user1)); - } - @Test public void summarizeEventDoesNothingForIdentifyEvent() { - EventSummarizer es = new EventSummarizer(defaultConfig); + EventSummarizer es = new EventSummarizer(); EventSummarizer.EventSummary snapshot = es.snapshot(); es.summarizeEvent(eventFactory.newIdentifyEvent(user)); @@ -64,7 +32,7 @@ public void summarizeEventDoesNothingForIdentifyEvent() { @Test public void summarizeEventDoesNothingForCustomEvent() { - EventSummarizer es = new EventSummarizer(defaultConfig); + EventSummarizer es = new EventSummarizer(); EventSummarizer.EventSummary snapshot = es.snapshot(); es.summarizeEvent(eventFactory.newCustomEvent("whatever", user, null)); @@ -73,7 +41,7 @@ public void summarizeEventDoesNothingForCustomEvent() { @Test public void summarizeEventSetsStartAndEndDates() { - EventSummarizer es = new EventSummarizer(defaultConfig); + EventSummarizer es = new EventSummarizer(); FeatureFlag flag = new FeatureFlagBuilder("key").build(); eventTimestamp = 2000; Event event1 = eventFactory.newFeatureRequestEvent(flag, user, null, null); @@ -92,7 +60,7 @@ public void summarizeEventSetsStartAndEndDates() { @Test public void summarizeEventIncrementsCounters() { - EventSummarizer es = new EventSummarizer(defaultConfig); + EventSummarizer es = new EventSummarizer(); FeatureFlag flag1 = new FeatureFlagBuilder("key1").version(11).build(); FeatureFlag flag2 = new FeatureFlagBuilder("key2").version(22).build(); String unknownFlagKey = "badkey"; @@ -119,7 +87,7 @@ public void summarizeEventIncrementsCounters() { new EventSummarizer.CounterValue(1, js("value2"), js("default1"))); expected.put(new EventSummarizer.CounterKey(flag2.getKey(), 1, flag2.getVersion()), new EventSummarizer.CounterValue(1, js("value99"), js("default2"))); - expected.put(new EventSummarizer.CounterKey(unknownFlagKey, 0, 0), + expected.put(new EventSummarizer.CounterKey(unknownFlagKey, null, null), new EventSummarizer.CounterValue(1, js("default3"), js("default3"))); assertThat(data.counters, equalTo(expected)); } diff --git a/src/test/java/com/launchdarkly/client/SimpleLRUCacheTest.java b/src/test/java/com/launchdarkly/client/SimpleLRUCacheTest.java new file mode 100644 index 000000000..996d51c29 --- /dev/null +++ b/src/test/java/com/launchdarkly/client/SimpleLRUCacheTest.java @@ -0,0 +1,57 @@ +package com.launchdarkly.client; + +import org.junit.Test; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertNull; + +public class SimpleLRUCacheTest { + @Test + public void getReturnsNullForNeverSeenValue() { + SimpleLRUCache cache = new SimpleLRUCache<>(10); + assertNull(cache.get("a")); + } + + @Test + public void putReturnsNullForNeverSeenValue() { + SimpleLRUCache cache = new SimpleLRUCache<>(10); + assertNull(cache.put("a", "1")); + } + + @Test + public void putReturnsPreviousValueForAlreadySeenValue() { + SimpleLRUCache cache = new SimpleLRUCache<>(10); + cache.put("a", "1"); + assertEquals("1", cache.put("a", "2")); + } + + @Test + public void oldestValueIsDiscardedWhenCapacityIsExceeded() { + SimpleLRUCache cache = new SimpleLRUCache<>(2); + cache.put("a", "1"); + cache.put("b", "2"); + cache.put("c", "3"); + assertEquals("3", cache.get("c")); + assertEquals("2", cache.get("b")); + assertNull(cache.get("a")); + } + + @Test + public void reAddingValueMakesItNewAgain() { + SimpleLRUCache cache = new SimpleLRUCache<>(2); + cache.put("a", "1"); + cache.put("b", "2"); + cache.put("c", "3"); + cache.put("a", "1"); + assertEquals("3", cache.get("c")); + assertEquals("1", cache.get("a")); + assertNull(cache.get("b")); + } + + @Test + public void zeroLengthCacheTreatsValuesAsNew() { + SimpleLRUCache cache = new SimpleLRUCache<>(0); + cache.put("a", "1"); + assertNull(cache.put("a", "2")); + } +} From 4f2c9166a6dacdb8e69ae189a22ba427fb4ca0e7 Mon Sep 17 00:00:00 2001 From: Eli Bishop Date: Wed, 28 Mar 2018 12:28:16 -0700 Subject: [PATCH 42/67] reorganize & expand LDClient tests; simplify logic for creating update processor --- .../com/launchdarkly/client/LDClient.java | 75 +-- .../launchdarkly/client/TestFeatureStore.java | 26 +- .../launchdarkly/client/UpdateProcessor.java | 18 +- .../client/LDClientEvaluationTest.java | 125 ++++ .../client/LDClientEventTest.java | 250 ++++++++ .../client/LDClientLddModeTest.java | 53 ++ .../client/LDClientOfflineTest.java | 82 +++ .../com/launchdarkly/client/LDClientTest.java | 538 ++++-------------- .../com/launchdarkly/client/TestUtil.java | 4 + 9 files changed, 674 insertions(+), 497 deletions(-) create mode 100644 src/test/java/com/launchdarkly/client/LDClientEvaluationTest.java create mode 100644 src/test/java/com/launchdarkly/client/LDClientEventTest.java create mode 100644 src/test/java/com/launchdarkly/client/LDClientLddModeTest.java create mode 100644 src/test/java/com/launchdarkly/client/LDClientOfflineTest.java diff --git a/src/main/java/com/launchdarkly/client/LDClient.java b/src/main/java/com/launchdarkly/client/LDClient.java index a8f75952b..4f4e8f81d 100644 --- a/src/main/java/com/launchdarkly/client/LDClient.java +++ b/src/main/java/com/launchdarkly/client/LDClient.java @@ -37,10 +37,9 @@ public class LDClient implements LDClientInterface { private final LDConfig config; private final String sdkKey; - private final FeatureRequestor requestor; - private final EventProcessor eventProcessor; + final EventProcessor eventProcessor; + final UpdateProcessor updateProcessor; private final EventFactory eventFactory = EventFactory.DEFAULT; - private UpdateProcessor updateProcessor; /** * Creates a new client instance that connects to LaunchDarkly with the default configuration. In most @@ -62,34 +61,18 @@ public LDClient(String sdkKey) { public LDClient(String sdkKey, LDConfig config) { this.config = config; this.sdkKey = sdkKey; - this.requestor = createFeatureRequestor(sdkKey, config); - if (config.offline || !config.sendEvents) { - this.eventProcessor = new EventProcessor.NullEventProcessor(); - } else { - this.eventProcessor = createEventProcessor(sdkKey, config); - } - + + this.eventProcessor = createEventProcessor(sdkKey, config); + if (config.offline) { logger.info("Starting LaunchDarkly client in offline mode"); - return; - } - - if (config.useLdd) { + } else if (config.useLdd) { logger.info("Starting LaunchDarkly in LDD mode. Skipping direct feature retrieval."); - return; - } - - if (config.stream) { - logger.info("Enabling streaming API"); - this.updateProcessor = createStreamProcessor(sdkKey, config, requestor); - } else { - logger.info("Disabling streaming API"); - logger.warn("You should only disable the streaming API if instructed to do so by LaunchDarkly support"); - this.updateProcessor = createPollingProcessor(config); } + this.updateProcessor = createUpdateProcessor(sdkKey, config); Future startFuture = updateProcessor.start(); - if (config.startWaitMillis > 0L) { + if (!config.offline && !config.useLdd && config.startWaitMillis > 0L) { logger.info("Waiting up to " + config.startWaitMillis + " milliseconds for LaunchDarkly client to start..."); try { startFuture.get(config.startWaitMillis, TimeUnit.MILLISECONDS); @@ -103,29 +86,35 @@ public LDClient(String sdkKey, LDConfig config) { @Override public boolean initialized() { - return isOffline() || config.useLdd || updateProcessor.initialized(); - } - - @VisibleForTesting - protected FeatureRequestor createFeatureRequestor(String sdkKey, LDConfig config) { - return new FeatureRequestor(sdkKey, config); + return updateProcessor.initialized(); } - + @VisibleForTesting protected EventProcessor createEventProcessor(String sdkKey, LDConfig config) { - return new DefaultEventProcessor(sdkKey, config); - } - - @VisibleForTesting - protected StreamProcessor createStreamProcessor(String sdkKey, LDConfig config, FeatureRequestor requestor) { - return new StreamProcessor(sdkKey, config, requestor); + if (config.offline || !config.sendEvents) { + return new EventProcessor.NullEventProcessor(); + } else { + return new DefaultEventProcessor(sdkKey, config); + } } @VisibleForTesting - protected PollingProcessor createPollingProcessor(LDConfig config) { - return new PollingProcessor(config, requestor); + protected UpdateProcessor createUpdateProcessor(String sdkKey, LDConfig config) { + if (config.offline || config.useLdd) { + return new UpdateProcessor.NullUpdateProcessor(); + } else { + FeatureRequestor requestor = new FeatureRequestor(sdkKey, config); + if (config.stream) { + logger.info("Enabling streaming API"); + return new StreamProcessor(sdkKey, config, requestor); + } else { + logger.info("Disabling streaming API"); + logger.warn("You should only disable the streaming API if instructed to do so by LaunchDarkly support"); + return new PollingProcessor(config, requestor); + } + } } - + @Override public void track(String eventName, LDUser user, JsonElement data) { if (isOffline()) { @@ -291,9 +280,7 @@ private JsonElement evaluate(String featureKey, LDUser user, JsonElement default public void close() throws IOException { logger.info("Closing LaunchDarkly Client"); this.eventProcessor.close(); - if (this.updateProcessor != null) { - this.updateProcessor.close(); - } + this.updateProcessor.close(); if (this.config.httpClient != null) { if (this.config.httpClient.dispatcher() != null && this.config.httpClient.dispatcher().executorService() != null) { this.config.httpClient.dispatcher().cancelAll(); diff --git a/src/main/java/com/launchdarkly/client/TestFeatureStore.java b/src/main/java/com/launchdarkly/client/TestFeatureStore.java index d69e06d2a..b6bfb188f 100644 --- a/src/main/java/com/launchdarkly/client/TestFeatureStore.java +++ b/src/main/java/com/launchdarkly/client/TestFeatureStore.java @@ -30,7 +30,7 @@ public class TestFeatureStore extends InMemoryFeatureStore { * @param key the key of the feature flag * @param value the new value of the feature flag */ - public void setBooleanValue(String key, Boolean value) { + public FeatureFlag setBooleanValue(String key, Boolean value) { FeatureFlag newFeature = new FeatureFlagBuilder(key) .on(false) .offVariation(value ? 0 : 1) @@ -38,6 +38,7 @@ public void setBooleanValue(String key, Boolean value) { .version(version.incrementAndGet()) .build(); upsert(FEATURES, newFeature); + return newFeature; } /** @@ -46,8 +47,8 @@ public void setBooleanValue(String key, Boolean value) { * * @param key the key of the feature flag to evaluate to true */ - public void setFeatureTrue(String key) { - setBooleanValue(key, true); + public FeatureFlag setFeatureTrue(String key) { + return setBooleanValue(key, true); } /** @@ -56,8 +57,8 @@ public void setFeatureTrue(String key) { * * @param key the key of the feature flag to evaluate to false */ - public void setFeatureFalse(String key) { - setBooleanValue(key, false); + public FeatureFlag setFeatureFalse(String key) { + return setBooleanValue(key, false); } /** @@ -65,8 +66,8 @@ public void setFeatureFalse(String key) { * @param key the key of the flag * @param value the new value of the flag */ - public void setIntegerValue(String key, Integer value) { - setJsonValue(key, new JsonPrimitive(value)); + public FeatureFlag setIntegerValue(String key, Integer value) { + return setJsonValue(key, new JsonPrimitive(value)); } /** @@ -74,8 +75,8 @@ public void setIntegerValue(String key, Integer value) { * @param key the key of the flag * @param value the new value of the flag */ - public void setDoubleValue(String key, Double value) { - setJsonValue(key, new JsonPrimitive(value)); + public FeatureFlag setDoubleValue(String key, Double value) { + return setJsonValue(key, new JsonPrimitive(value)); } /** @@ -83,8 +84,8 @@ public void setDoubleValue(String key, Double value) { * @param key the key of the flag * @param value the new value of the flag */ - public void setStringValue(String key, String value) { - setJsonValue(key, new JsonPrimitive(value)); + public FeatureFlag setStringValue(String key, String value) { + return setJsonValue(key, new JsonPrimitive(value)); } /** @@ -92,7 +93,7 @@ public void setStringValue(String key, String value) { * @param key the key of the flag * @param value the new value of the flag */ - public void setJsonValue(String key, JsonElement value) { + public FeatureFlag setJsonValue(String key, JsonElement value) { FeatureFlag newFeature = new FeatureFlagBuilder(key) .on(false) .offVariation(0) @@ -100,6 +101,7 @@ public void setJsonValue(String key, JsonElement value) { .version(version.incrementAndGet()) .build(); upsert(FEATURES, newFeature); + return newFeature; } @Override diff --git a/src/main/java/com/launchdarkly/client/UpdateProcessor.java b/src/main/java/com/launchdarkly/client/UpdateProcessor.java index 5a240260d..288e0b4f5 100644 --- a/src/main/java/com/launchdarkly/client/UpdateProcessor.java +++ b/src/main/java/com/launchdarkly/client/UpdateProcessor.java @@ -4,6 +4,8 @@ import java.io.IOException; import java.util.concurrent.Future; +import static com.google.common.util.concurrent.Futures.immediateFuture; + interface UpdateProcessor extends Closeable { /** @@ -18,6 +20,20 @@ interface UpdateProcessor extends Closeable { */ boolean initialized(); - void close() throws IOException; + + static class NullUpdateProcessor implements UpdateProcessor { + @Override + public Future start() { + return immediateFuture(null); + } + + @Override + public boolean initialized() { + return true; + } + + @Override + public void close() throws IOException {} + } } diff --git a/src/test/java/com/launchdarkly/client/LDClientEvaluationTest.java b/src/test/java/com/launchdarkly/client/LDClientEvaluationTest.java new file mode 100644 index 000000000..30ee44a96 --- /dev/null +++ b/src/test/java/com/launchdarkly/client/LDClientEvaluationTest.java @@ -0,0 +1,125 @@ +package com.launchdarkly.client; + +import com.google.gson.JsonElement; +import com.google.gson.JsonObject; + +import org.easymock.EasyMockSupport; +import org.junit.Test; + +import java.util.Arrays; + +import static com.launchdarkly.client.TestUtil.jint; +import static com.launchdarkly.client.TestUtil.js; +import static com.launchdarkly.client.VersionedDataKind.FEATURES; +import static com.launchdarkly.client.VersionedDataKind.SEGMENTS; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertTrue; + +public class LDClientEvaluationTest extends EasyMockSupport { + private static final LDUser user = new LDUser("userkey"); + + private TestFeatureStore featureStore = new TestFeatureStore(); + private LDConfig config = new LDConfig.Builder().featureStore(featureStore).build(); + private LDClientInterface client = createTestClient(config); + + @Test + public void boolVariationReturnsFlagValue() throws Exception { + featureStore.setFeatureTrue("key"); + + assertTrue(client.boolVariation("key", user, false)); + } + + @Test + public void boolVariationReturnsDefaultValueForUnknownFlag() throws Exception { + assertFalse(client.boolVariation("key", user, false)); + } + + @Test + public void intVariationReturnsFlagValue() throws Exception { + featureStore.setIntegerValue("key", 2); + + assertEquals(new Integer(2), client.intVariation("key", user, 1)); + } + + @Test + public void intVariationReturnsDefaultValueForUnknownFlag() throws Exception { + assertEquals(new Integer(1), client.intVariation("key", user, 1)); + } + + @Test + public void doubleVariationReturnsFlagValue() throws Exception { + featureStore.setDoubleValue("key", 2.5d); + + assertEquals(new Double(2.5d), client.doubleVariation("key", user, 1.0d)); + } + + @Test + public void doubleVariationReturnsDefaultValueForUnknownFlag() throws Exception { + assertEquals(new Double(1.0d), client.doubleVariation("key", user, 1.0d)); + } + + @Test + public void stringVariationReturnsFlagValue() throws Exception { + featureStore.setStringValue("key", "b"); + + assertEquals("b", client.stringVariation("key", user, "a")); + } + + @Test + public void stringVariationReturnsDefaultValueForUnknownFlag() throws Exception { + assertEquals("a", client.stringVariation("key", user, "a")); + } + + @Test + public void jsonVariationReturnsFlagValue() throws Exception { + JsonObject data = new JsonObject(); + data.addProperty("thing", "stuff"); + featureStore.setJsonValue("key", data); + + assertEquals(data, client.jsonVariation("key", user, jint(42))); + } + + @Test + public void jsonVariationReturnsDefaultValueForUnknownFlag() throws Exception { + JsonElement defaultVal = jint(42); + assertEquals(defaultVal, client.jsonVariation("key", user, defaultVal)); + } + + @Test + public void canMatchUserBySegment() throws Exception { + // This is similar to one of the tests in FeatureFlagTest, but more end-to-end + Segment segment = new Segment.Builder("segment1") + .version(1) + .included(Arrays.asList(user.getKeyAsString())) + .build(); + featureStore.upsert(SEGMENTS, segment); + + Clause clause = new Clause("", Operator.segmentMatch, Arrays.asList(js("segment1")), false); + Rule rule = new Rule(Arrays.asList(clause), 0, null); + FeatureFlag feature = new FeatureFlagBuilder("test-feature") + .version(1) + .rules(Arrays.asList(rule)) + .variations(TestFeatureStore.TRUE_FALSE_VARIATIONS) + .on(true) + .fallthrough(new VariationOrRollout(1, null)) + .build(); + featureStore.upsert(FEATURES, feature); + + assertTrue(client.boolVariation("test-feature", user, false)); + } + + private LDClientInterface createTestClient(LDConfig config) { + return new LDClient("SDK_KEY", config) { + @Override + protected UpdateProcessor createUpdateProcessor(String sdkKey, LDConfig config) { + return new UpdateProcessor.NullUpdateProcessor(); + } + + @Override + protected EventProcessor createEventProcessor(String sdkKey, LDConfig config) { + return new EventProcessor.NullEventProcessor(); + } + }; + } +} diff --git a/src/test/java/com/launchdarkly/client/LDClientEventTest.java b/src/test/java/com/launchdarkly/client/LDClientEventTest.java new file mode 100644 index 000000000..965289653 --- /dev/null +++ b/src/test/java/com/launchdarkly/client/LDClientEventTest.java @@ -0,0 +1,250 @@ +package com.launchdarkly.client; + +import com.google.gson.JsonElement; +import com.google.gson.JsonObject; +import com.google.gson.JsonPrimitive; + +import org.junit.Test; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.List; + +import static com.launchdarkly.client.TestUtil.fallthroughVariation; +import static com.launchdarkly.client.TestUtil.jbool; +import static com.launchdarkly.client.TestUtil.jdouble; +import static com.launchdarkly.client.TestUtil.jint; +import static com.launchdarkly.client.TestUtil.js; +import static com.launchdarkly.client.VersionedDataKind.FEATURES; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertNull; + +public class LDClientEventTest { + private static final LDUser user = new LDUser("userkey"); + + private TestFeatureStore featureStore = new TestFeatureStore(); + private TestEventProcessor eventSink = new TestEventProcessor(); + private LDConfig config = new LDConfig.Builder().featureStore(featureStore).build(); + private LDClientInterface client = createTestClient(config); + + @Test + public void identifySendsEvent() throws Exception { + client.identify(user); + + assertEquals(1, eventSink.events.size()); + Event e = eventSink.events.get(0); + assertEquals(IdentifyEvent.class, e.getClass()); + IdentifyEvent ie = (IdentifyEvent)e; + assertEquals(user.getKey(), ie.user.getKey()); + } + + @Test + public void trackSendsEventWithoutData() throws Exception { + client.track("eventkey", user); + + assertEquals(1, eventSink.events.size()); + Event e = eventSink.events.get(0); + assertEquals(CustomEvent.class, e.getClass()); + CustomEvent ce = (CustomEvent)e; + assertEquals(user.getKey(), ce.user.getKey()); + assertEquals("eventkey", ce.key); + assertNull(ce.data); + } + + @Test + public void trackSendsEventWithData() throws Exception { + JsonObject data = new JsonObject(); + data.addProperty("thing", "stuff"); + client.track("eventkey", user, data); + + assertEquals(1, eventSink.events.size()); + Event e = eventSink.events.get(0); + assertEquals(CustomEvent.class, e.getClass()); + CustomEvent ce = (CustomEvent)e; + assertEquals(user.getKey(), ce.user.getKey()); + assertEquals("eventkey", ce.key); + assertEquals(data, ce.data); + } + + @Test + public void boolVariationSendsEvent() throws Exception { + FeatureFlag flag = featureStore.setFeatureTrue("key"); + + client.boolVariation("key", user, false); + assertEquals(1, eventSink.events.size()); + checkFeatureEvent(eventSink.events.get(0), flag, jbool(true), jbool(false), null); + } + + @Test + public void boolVariationSendsEventForUnknownFlag() throws Exception { + client.boolVariation("key", user, false); + assertEquals(1, eventSink.events.size()); + checkUnknownFeatureEvent(eventSink.events.get(0), "key", jbool(false), null); + } + + @Test + public void intVariationSendsEvent() throws Exception { + FeatureFlag flag = featureStore.setIntegerValue("key", 2); + + client.intVariation("key", user, 1); + assertEquals(1, eventSink.events.size()); + checkFeatureEvent(eventSink.events.get(0), flag, jint(2), jint(1), null); + } + + @Test + public void intVariationSendsEventForUnknownFlag() throws Exception { + client.intVariation("key", user, 1); + assertEquals(1, eventSink.events.size()); + checkUnknownFeatureEvent(eventSink.events.get(0), "key", jint(1), null); + } + + @Test + public void doubleVariationSendsEvent() throws Exception { + FeatureFlag flag = featureStore.setDoubleValue("key", 2.5d); + + client.doubleVariation("key", user, 1.0d); + assertEquals(1, eventSink.events.size()); + checkFeatureEvent(eventSink.events.get(0), flag, jdouble(2.5d), jdouble(1.0d), null); + } + + @Test + public void doubleVariationSendsEventForUnknownFlag() throws Exception { + client.doubleVariation("key", user, 1.0d); + assertEquals(1, eventSink.events.size()); + checkUnknownFeatureEvent(eventSink.events.get(0), "key", jdouble(1.0), null); + } + + @Test + public void stringVariationSendsEvent() throws Exception { + FeatureFlag flag = featureStore.setStringValue("key", "b"); + + client.stringVariation("key", user, "a"); + assertEquals(1, eventSink.events.size()); + checkFeatureEvent(eventSink.events.get(0), flag, js("b"), js("a"), null); + } + + @Test + public void stringVariationSendsEventForUnknownFlag() throws Exception { + client.stringVariation("key", user, "a"); + assertEquals(1, eventSink.events.size()); + checkUnknownFeatureEvent(eventSink.events.get(0), "key", js("a"), null); + } + + @Test + public void jsonVariationSendsEvent() throws Exception { + JsonObject data = new JsonObject(); + data.addProperty("thing", "stuff"); + FeatureFlag flag = featureStore.setJsonValue("key", data); + JsonElement defaultVal = new JsonPrimitive(42); + + client.jsonVariation("key", user, defaultVal); + assertEquals(1, eventSink.events.size()); + checkFeatureEvent(eventSink.events.get(0), flag, data, defaultVal, null); + } + + @Test + public void jsonVariationSendsEventForUnknownFlag() throws Exception { + JsonElement defaultVal = new JsonPrimitive(42); + + client.jsonVariation("key", user, defaultVal); + assertEquals(1, eventSink.events.size()); + checkUnknownFeatureEvent(eventSink.events.get(0), "key", defaultVal, null); + } + + @Test + public void eventIsSentForExistingPrererequisiteFlag() throws Exception { + FeatureFlag f0 = new FeatureFlagBuilder("feature0") + .on(true) + .prerequisites(Arrays.asList(new Prerequisite("feature1", 1))) + .fallthrough(fallthroughVariation(0)) + .offVariation(1) + .variations(js("fall"), js("off"), js("on")) + .version(1) + .build(); + FeatureFlag f1 = new FeatureFlagBuilder("feature1") + .on(true) + .fallthrough(fallthroughVariation(1)) + .variations(js("nogo"), js("go")) + .version(2) + .build(); + featureStore.upsert(FEATURES, f0); + featureStore.upsert(FEATURES, f1); + + client.stringVariation("feature0", user, "default"); + + assertEquals(2, eventSink.events.size()); + checkFeatureEvent(eventSink.events.get(0), f1, js("go"), null, "feature0"); + checkFeatureEvent(eventSink.events.get(1), f0, js("fall"), js("default"), null); + } + + @Test + public void eventIsNotSentForUnknownPrererequisiteFlag() throws Exception { + FeatureFlag f0 = new FeatureFlagBuilder("feature0") + .on(true) + .prerequisites(Arrays.asList(new Prerequisite("feature1", 1))) + .fallthrough(fallthroughVariation(0)) + .offVariation(1) + .variations(js("fall"), js("off"), js("on")) + .version(1) + .build(); + featureStore.upsert(FEATURES, f0); + + client.stringVariation("feature0", user, "default"); + + assertEquals(1, eventSink.events.size()); + checkFeatureEvent(eventSink.events.get(0), f0, js("off"), js("default"), null); + } + + private void checkFeatureEvent(Event e, FeatureFlag flag, JsonElement value, JsonElement defaultVal, + String prereqOf) { + assertEquals(FeatureRequestEvent.class, e.getClass()); + FeatureRequestEvent fe = (FeatureRequestEvent)e; + assertEquals(flag.getKey(), fe.key); + assertEquals(user.getKey(), fe.user.getKey()); + assertEquals(new Integer(flag.getVersion()), fe.version); + assertEquals(value, fe.value); + assertEquals(defaultVal, fe.defaultVal); + assertEquals(prereqOf, fe.prereqOf); + } + + private void checkUnknownFeatureEvent(Event e, String key, JsonElement defaultVal, String prereqOf) { + assertEquals(FeatureRequestEvent.class, e.getClass()); + FeatureRequestEvent fe = (FeatureRequestEvent)e; + assertEquals(key, fe.key); + assertEquals(user.getKey(), fe.user.getKey()); + assertNull(fe.version); + assertEquals(defaultVal, fe.value); + assertEquals(defaultVal, fe.defaultVal); + assertEquals(prereqOf, fe.prereqOf); + } + + private static class TestEventProcessor implements EventProcessor { + List events = new ArrayList<>(); + + @Override + public void close() throws IOException {} + + @Override + public void sendEvent(Event e) { + events.add(e); + } + + @Override + public void flush() {} + } + + private LDClientInterface createTestClient(LDConfig config) { + return new LDClient("SDK_KEY", config) { + @Override + protected UpdateProcessor createUpdateProcessor(String sdkKey, LDConfig config) { + return new UpdateProcessor.NullUpdateProcessor(); + } + + @Override + protected EventProcessor createEventProcessor(String sdkKey, LDConfig config) { + return eventSink; + } + }; + } +} diff --git a/src/test/java/com/launchdarkly/client/LDClientLddModeTest.java b/src/test/java/com/launchdarkly/client/LDClientLddModeTest.java new file mode 100644 index 000000000..db37855f0 --- /dev/null +++ b/src/test/java/com/launchdarkly/client/LDClientLddModeTest.java @@ -0,0 +1,53 @@ +package com.launchdarkly.client; + +import org.junit.Test; + +import java.io.IOException; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertTrue; + +public class LDClientLddModeTest { + @Test + public void lddModeClientHasNullUpdateProcessor() throws IOException { + LDConfig config = new LDConfig.Builder() + .useLdd(true) + .build(); + try (LDClient client = new LDClient("SDK_KEY", config)) { + assertEquals(UpdateProcessor.NullUpdateProcessor.class, client.updateProcessor.getClass()); + } + } + + @Test + public void lddModeClientHasDefaultEventProcessor() throws IOException { + LDConfig config = new LDConfig.Builder() + .useLdd(true) + .build(); + try (LDClient client = new LDClient("SDK_KEY", config)) { + assertEquals(DefaultEventProcessor.class, client.eventProcessor.getClass()); + } + } + + @Test + public void lddModeClientIsInitialized() throws IOException { + LDConfig config = new LDConfig.Builder() + .useLdd(true) + .build(); + try (LDClient client = new LDClient("SDK_KEY", config)) { + assertTrue(client.initialized()); + } + } + + @Test + public void lddModeClientGetsFlagFromFeatureStore() throws IOException { + TestFeatureStore testFeatureStore = new TestFeatureStore(); + LDConfig config = new LDConfig.Builder() + .useLdd(true) + .featureStore(testFeatureStore) + .build(); + testFeatureStore.setFeatureTrue("key"); + try (LDClient client = new LDClient("SDK_KEY", config)) { + assertTrue(client.boolVariation("key", new LDUser("user"), false)); + } + } +} \ No newline at end of file diff --git a/src/test/java/com/launchdarkly/client/LDClientOfflineTest.java b/src/test/java/com/launchdarkly/client/LDClientOfflineTest.java new file mode 100644 index 000000000..57821117e --- /dev/null +++ b/src/test/java/com/launchdarkly/client/LDClientOfflineTest.java @@ -0,0 +1,82 @@ +package com.launchdarkly.client; + +import com.google.gson.JsonElement; +import com.google.gson.JsonPrimitive; + +import org.junit.Test; + +import java.io.IOException; +import java.util.Map; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertNotNull; +import static org.junit.Assert.assertTrue; + +public class LDClientOfflineTest { + @Test + public void offlineClientHasNullUpdateProcessor() throws IOException { + LDConfig config = new LDConfig.Builder() + .offline(true) + .build(); + try (LDClient client = new LDClient("SDK_KEY", config)) { + assertEquals(UpdateProcessor.NullUpdateProcessor.class, client.updateProcessor.getClass()); + } + } + + @Test + public void offlineClientHasNullEventProcessor() throws IOException { + LDConfig config = new LDConfig.Builder() + .offline(true) + .build(); + try (LDClient client = new LDClient("SDK_KEY", config)) { + assertEquals(EventProcessor.NullEventProcessor.class, client.eventProcessor.getClass()); + } + } + + @Test + public void offlineClientIsInitialized() throws IOException { + LDConfig config = new LDConfig.Builder() + .offline(true) + .build(); + try (LDClient client = new LDClient("SDK_KEY", config)) { + assertTrue(client.initialized()); + } + } + + @Test + public void offlineClientReturnsDefaultValue() throws IOException { + LDConfig config = new LDConfig.Builder() + .offline(true) + .build(); + try (LDClient client = new LDClient("SDK_KEY", config)) { + assertEquals("x", client.stringVariation("key", new LDUser("user"), "x")); + } + } + + @Test + public void offlineClientGetsAllFlagsFromFeatureStore() throws IOException { + TestFeatureStore testFeatureStore = new TestFeatureStore(); + LDConfig config = new LDConfig.Builder() + .offline(true) + .featureStore(testFeatureStore) + .build(); + testFeatureStore.setFeatureTrue("key"); + try (LDClient client = new LDClient("SDK_KEY", config)) { + Map allFlags = client.allFlags(new LDUser("user")); + assertNotNull(allFlags); + assertEquals(1, allFlags.size()); + assertEquals(new JsonPrimitive(true), allFlags.get("key")); + } + } + + @Test + public void testSecureModeHash() throws IOException { + LDConfig config = new LDConfig.Builder() + .offline(true) + .build(); + try (LDClientInterface client = new LDClient("secret", config)) { + LDUser user = new LDUser.Builder("Message").build(); + assertEquals("aa747c502a898200f9e4fa21bac68136f886a0e27aec70ba06daf2e2a5cb5597", client.secureModeHash(user)); + } + } +} \ No newline at end of file diff --git a/src/test/java/com/launchdarkly/client/LDClientTest.java b/src/test/java/com/launchdarkly/client/LDClientTest.java index 4923e81a6..b2701a34e 100644 --- a/src/test/java/com/launchdarkly/client/LDClientTest.java +++ b/src/test/java/com/launchdarkly/client/LDClientTest.java @@ -1,570 +1,243 @@ package com.launchdarkly.client; -import com.google.gson.JsonArray; -import com.google.gson.JsonElement; -import com.google.gson.JsonPrimitive; - import org.easymock.EasyMockSupport; import org.junit.Before; import org.junit.Test; import java.io.IOException; -import java.util.Arrays; -import java.util.Map; -import java.util.concurrent.ExecutionException; +import java.net.URI; import java.util.concurrent.Future; import java.util.concurrent.TimeUnit; import java.util.concurrent.TimeoutException; -import static com.launchdarkly.client.VersionedDataKind.FEATURES; -import static com.launchdarkly.client.VersionedDataKind.SEGMENTS; import static org.easymock.EasyMock.anyObject; import static org.easymock.EasyMock.expect; import static org.easymock.EasyMock.expectLastCall; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertFalse; -import static org.junit.Assert.assertNotNull; import static org.junit.Assert.assertTrue; import junit.framework.AssertionFailedError; +/** + * See also LDClientEvaluationTest, etc. This file contains mostly tests for the startup logic. + */ public class LDClientTest extends EasyMockSupport { - private FeatureRequestor requestor; - private StreamProcessor streamProcessor; - private PollingProcessor pollingProcessor; + private UpdateProcessor updateProcessor; private EventProcessor eventProcessor; - private Future initFuture; + private Future initFuture; private LDClientInterface client; + @SuppressWarnings("unchecked") @Before public void before() { - requestor = createStrictMock(FeatureRequestor.class); - streamProcessor = createStrictMock(StreamProcessor.class); - pollingProcessor = createStrictMock(PollingProcessor.class); + updateProcessor = createStrictMock(UpdateProcessor.class); eventProcessor = createStrictMock(EventProcessor.class); initFuture = createStrictMock(Future.class); } @Test - public void testOffline() throws IOException { + public void clientHasDefaultEventProcessorIfSendEventsIsTrue() throws Exception { LDConfig config = new LDConfig.Builder() - .offline(true) + .stream(false) + .baseURI(URI.create("/fake")) + .startWaitMillis(0) + .sendEvents(true) .build(); - - client = createMockClient(config); - replayAll(); - - assertDefaultValueIsReturned(); - assertTrue(client.initialized()); - verifyAll(); - } - - @Test - public void testTestFeatureStoreSetFeatureTrue() throws IOException, InterruptedException, ExecutionException, TimeoutException { - TestFeatureStore testFeatureStore = new TestFeatureStore(); - testFeatureStore.setInitialized(true); - LDConfig config = new LDConfig.Builder() - .startWaitMillis(10L) - .stream(false) - .featureStore(testFeatureStore) - .build(); - - expect(initFuture.get(10L, TimeUnit.MILLISECONDS)).andReturn(new Object()); - expect(pollingProcessor.start()).andReturn(initFuture); - expect(pollingProcessor.initialized()).andReturn(true).times(1); - expectEventsSent(1); - replayAll(); - - client = createMockClient(config); - testFeatureStore.setFeatureTrue("key"); - assertTrue("Test flag should be true, but was not.", client.boolVariation("key", new LDUser("user"), false)); - - verifyAll(); + try (LDClient client = new LDClient("SDK_KEY", config)) { + assertEquals(DefaultEventProcessor.class, client.eventProcessor.getClass()); + } } - + @Test - public void testTestOfflineModeAllFlags() throws IOException, InterruptedException, ExecutionException, TimeoutException { - TestFeatureStore testFeatureStore = new TestFeatureStore(); + public void clientHasNullEventProcessorIfSendEventsIsFalse() throws IOException { LDConfig config = new LDConfig.Builder() - .startWaitMillis(10L) - .offline(true) - .featureStore(testFeatureStore) + .stream(false) + .baseURI(URI.create("/fake")) + .startWaitMillis(0) + .sendEvents(false) .build(); - - client = new LDClient("", config);//createMockClient(config); - testFeatureStore.setFeatureTrue("key"); - Map allFlags = client.allFlags(new LDUser("user")); - assertNotNull("Expected non-nil response from allFlags() when offline mode is set to true", allFlags); - assertEquals("Didn't get expected flag count from allFlags() in offline mode", 1, allFlags.size()); - assertTrue("Test flag should be true, but was not.", allFlags.get("key").getAsBoolean()); + try (LDClient client = new LDClient("SDK_KEY", config)) { + assertEquals(EventProcessor.NullEventProcessor.class, client.eventProcessor.getClass()); + } } - + @Test - public void testTestFeatureStoreSetFalse() throws IOException, InterruptedException, ExecutionException, TimeoutException { - TestFeatureStore testFeatureStore = new TestFeatureStore(); - testFeatureStore.setInitialized(true); + public void streamingClientHasStreamProcessor() throws Exception { LDConfig config = new LDConfig.Builder() - .startWaitMillis(10L) - .stream(false) - .featureStore(testFeatureStore) - .build(); - - expect(initFuture.get(10L, TimeUnit.MILLISECONDS)).andReturn(new Object()); - expect(pollingProcessor.start()).andReturn(initFuture); - expect(pollingProcessor.initialized()).andReturn(true).times(1); - expectEventsSent(1); - replayAll(); - - client = createMockClient(config); - testFeatureStore.setFeatureFalse("key"); - assertFalse("Test flag should be false, but was on (the default).", client.boolVariation("key", new LDUser("user"), true)); - - verifyAll(); + .stream(true) + .streamURI(URI.create("/fake")) + .startWaitMillis(0) + .build(); + try (LDClient client = new LDClient("SDK_KEY", config)) { + assertEquals(StreamProcessor.class, client.updateProcessor.getClass()); + } } @Test - public void testTestFeatureStoreFlagTrueThenFalse() throws IOException, InterruptedException, ExecutionException, TimeoutException { - TestFeatureStore testFeatureStore = new TestFeatureStore(); - testFeatureStore.setInitialized(true); + public void pollingClientHasPollingProcessor() throws IOException { LDConfig config = new LDConfig.Builder() - .startWaitMillis(10L) - .stream(false) - .featureStore(testFeatureStore) - .build(); - - expect(initFuture.get(10L, TimeUnit.MILLISECONDS)).andReturn(new Object()); - expect(pollingProcessor.start()).andReturn(initFuture); - expect(pollingProcessor.initialized()).andReturn(true).times(2); - expectEventsSent(2); - replayAll(); - - client = createMockClient(config); - - testFeatureStore.setFeatureTrue("key"); - assertTrue("Test flag should be true, but was not.", client.boolVariation("key", new LDUser("user"), false)); - - testFeatureStore.setFeatureFalse("key"); - assertFalse("Test flag should be false, but was on (the default).", client.boolVariation("key", new LDUser("user"), true)); - - verifyAll(); + .stream(false) + .baseURI(URI.create("/fake")) + .startWaitMillis(0) + .build(); + try (LDClient client = new LDClient("SDK_KEY", config)) { + assertEquals(PollingProcessor.class, client.updateProcessor.getClass()); + } } @Test - public void testTestFeatureStoreIntegerVariation() throws Exception { - TestFeatureStore testFeatureStore = new TestFeatureStore(); - testFeatureStore.setInitialized(true); + public void noWaitForUpdateProcessorIfWaitMillisIsZero() throws Exception { LDConfig config = new LDConfig.Builder() - .startWaitMillis(10L) - .stream(false) - .featureStore(testFeatureStore) - .build(); + .startWaitMillis(0L) + .build(); - expect(initFuture.get(10L, TimeUnit.MILLISECONDS)).andReturn(new Object()); - expect(pollingProcessor.start()).andReturn(initFuture); - expect(pollingProcessor.initialized()).andReturn(true).times(2); - expectEventsSent(2); + expect(updateProcessor.start()).andReturn(initFuture); + expect(updateProcessor.initialized()).andReturn(false); replayAll(); client = createMockClient(config); + assertFalse(client.initialized()); - testFeatureStore.setIntegerValue("key", 1); - assertEquals(new Integer(1), client.intVariation("key", new LDUser("user"), 0)); - testFeatureStore.setIntegerValue("key", 42); - assertEquals(new Integer(42), client.intVariation("key", new LDUser("user"), 1)); verifyAll(); } @Test - public void testTestFeatureStoreDoubleVariation() throws Exception { - TestFeatureStore testFeatureStore = new TestFeatureStore(); - testFeatureStore.setInitialized(true); + public void willWaitForUpdateProcessorIfWaitMillisIsNonZero() throws Exception { LDConfig config = new LDConfig.Builder() - .startWaitMillis(10L) - .stream(false) - .featureStore(testFeatureStore) - .build(); + .startWaitMillis(10L) + .build(); - expect(initFuture.get(10L, TimeUnit.MILLISECONDS)).andReturn(new Object()); - expect(pollingProcessor.start()).andReturn(initFuture); - expect(pollingProcessor.initialized()).andReturn(true).times(2); - expectEventsSent(2); + expect(updateProcessor.start()).andReturn(initFuture); + expect(initFuture.get(10L, TimeUnit.MILLISECONDS)).andReturn(null); + expect(updateProcessor.initialized()).andReturn(false); replayAll(); client = createMockClient(config); + assertFalse(client.initialized()); - testFeatureStore.setDoubleValue("key", 1d); - assertEquals(new Double(1), client.doubleVariation("key", new LDUser("user"), 0d)); - testFeatureStore.setDoubleValue("key", 42d); - assertEquals(new Double(42), client.doubleVariation("key", new LDUser("user"), 1d)); verifyAll(); } @Test - public void testTestFeatureStoreStringVariation() throws Exception { - TestFeatureStore testFeatureStore = new TestFeatureStore(); - testFeatureStore.setInitialized(true); + public void updateProcessorCanTimeOut() throws Exception { LDConfig config = new LDConfig.Builder() - .startWaitMillis(10L) - .stream(false) - .featureStore(testFeatureStore) - .build(); + .startWaitMillis(10L) + .build(); - expect(initFuture.get(10L, TimeUnit.MILLISECONDS)).andReturn(new Object()); - expect(pollingProcessor.start()).andReturn(initFuture); - expect(pollingProcessor.initialized()).andReturn(true).times(2); - expectEventsSent(2); + expect(updateProcessor.start()).andReturn(initFuture); + expect(initFuture.get(10L, TimeUnit.MILLISECONDS)).andThrow(new TimeoutException()); + expect(updateProcessor.initialized()).andReturn(false); replayAll(); client = createMockClient(config); + assertFalse(client.initialized()); - testFeatureStore.setStringValue("key", "apples"); - assertEquals("apples", client.stringVariation("key", new LDUser("user"), "oranges")); - testFeatureStore.setStringValue("key", "bananas"); - assertEquals("bananas", client.stringVariation("key", new LDUser("user"), "apples")); verifyAll(); } - + @Test - public void testTestFeatureStoreJsonVariationPrimitive() throws Exception { - TestFeatureStore testFeatureStore = new TestFeatureStore(); - testFeatureStore.setInitialized(true); + public void clientCatchesRuntimeExceptionFromUpdateProcessor() throws Exception { LDConfig config = new LDConfig.Builder() - .startWaitMillis(10L) - .stream(false) - .featureStore(testFeatureStore) - .build(); + .startWaitMillis(10L) + .build(); - expect(initFuture.get(10L, TimeUnit.MILLISECONDS)).andReturn(new Object()); - expect(pollingProcessor.start()).andReturn(initFuture); - expect(pollingProcessor.initialized()).andReturn(true).times(4); - expectEventsSent(4); + expect(updateProcessor.start()).andReturn(initFuture); + expect(initFuture.get(10L, TimeUnit.MILLISECONDS)).andThrow(new RuntimeException()); + expect(updateProcessor.initialized()).andReturn(false); replayAll(); client = createMockClient(config); + assertFalse(client.initialized()); - // Character - testFeatureStore.setJsonValue("key", new JsonPrimitive('a')); - assertEquals(new JsonPrimitive('a'), client.jsonVariation("key", new LDUser("user"), new JsonPrimitive('b'))); - testFeatureStore.setJsonValue("key", new JsonPrimitive('b')); - assertEquals(new JsonPrimitive('b'), client.jsonVariation("key", new LDUser("user"), new JsonPrimitive('z'))); - - // Long - testFeatureStore.setJsonValue("key", new JsonPrimitive(1L)); - assertEquals(new JsonPrimitive(1l), client.jsonVariation("key", new LDUser("user"), new JsonPrimitive(0L))); - testFeatureStore.setJsonValue("key", new JsonPrimitive(42L)); - assertEquals(new JsonPrimitive(42L), client.jsonVariation("key", new LDUser("user"), new JsonPrimitive(0L))); verifyAll(); } @Test - public void testTestFeatureStoreJsonVariationArray() throws Exception { + public void isFlagKnownReturnsTrueForExistingFlag() throws Exception { TestFeatureStore testFeatureStore = new TestFeatureStore(); testFeatureStore.setInitialized(true); LDConfig config = new LDConfig.Builder() - .startWaitMillis(10L) - .stream(false) + .startWaitMillis(0) .featureStore(testFeatureStore) .build(); - - expect(initFuture.get(10L, TimeUnit.MILLISECONDS)).andReturn(new Object()); - expect(pollingProcessor.start()).andReturn(initFuture); - expect(pollingProcessor.initialized()).andReturn(true).times(2); - expectEventsSent(2); + expect(updateProcessor.start()).andReturn(initFuture); + expect(updateProcessor.initialized()).andReturn(true).times(1); replayAll(); client = createMockClient(config); - // JsonArray - JsonArray array = new JsonArray(); - array.add("red"); - array.add("blue"); - array.add("green"); - testFeatureStore.setJsonValue("key", array); - assertEquals(array, client.jsonVariation("key", new LDUser("user"), new JsonArray())); - - JsonArray array2 = new JsonArray(); - array2.addAll(array); - array2.add("yellow"); - testFeatureStore.setJsonValue("key", array2); - assertEquals(array2, client.jsonVariation("key", new LDUser("user"), new JsonArray())); + testFeatureStore.setIntegerValue("key", 1); + assertTrue(client.isFlagKnown("key")); verifyAll(); } @Test - public void testIsFlagKnown() throws Exception { + public void isFlagKnownReturnsFalseForUnknownFlag() throws Exception { TestFeatureStore testFeatureStore = new TestFeatureStore(); testFeatureStore.setInitialized(true); LDConfig config = new LDConfig.Builder() - .startWaitMillis(10L) - .stream(false) + .startWaitMillis(0) .featureStore(testFeatureStore) .build(); - - expect(initFuture.get(10L, TimeUnit.MILLISECONDS)).andReturn(new Object()); - expect(pollingProcessor.start()).andReturn(initFuture); - expect(pollingProcessor.initialized()).andReturn(true).times(2); + expect(updateProcessor.start()).andReturn(initFuture); + expect(updateProcessor.initialized()).andReturn(true).times(1); replayAll(); client = createMockClient(config); - testFeatureStore.setIntegerValue("key", 1); - assertTrue("Flag is known", client.isFlagKnown("key")); - assertFalse("Flag is unknown", client.isFlagKnown("unKnownKey")); + assertFalse(client.isFlagKnown("key")); verifyAll(); } @Test - public void testIsFlagKnownCallBeforeInitialization() throws Exception { + public void isFlagKnownReturnsFalseIfStoreAndClientAreNotInitialized() throws Exception { TestFeatureStore testFeatureStore = new TestFeatureStore(); + testFeatureStore.setInitialized(false); LDConfig config = new LDConfig.Builder() - .startWaitMillis(10L) - .stream(false) + .startWaitMillis(0) .featureStore(testFeatureStore) .build(); - - expect(initFuture.get(10L, TimeUnit.MILLISECONDS)).andReturn(new Object()); - expect(pollingProcessor.start()).andReturn(initFuture); - expect(pollingProcessor.initialized()).andReturn(false).times(1); + expect(updateProcessor.start()).andReturn(initFuture); + expect(updateProcessor.initialized()).andReturn(false).times(1); replayAll(); client = createMockClient(config); testFeatureStore.setIntegerValue("key", 1); - assertFalse("Flag is marked as unknown", client.isFlagKnown("key")); + assertFalse(client.isFlagKnown("key")); verifyAll(); } @Test - public void testIsFlagKnownCallBeforeInitializationButFeatureStoreIsInited() throws Exception { + public void isFlagKnownUsesStoreIfStoreIsInitializedButClientIsNot() throws Exception { TestFeatureStore testFeatureStore = new TestFeatureStore(); testFeatureStore.setInitialized(true); LDConfig config = new LDConfig.Builder() - .startWaitMillis(10L) - .stream(false) + .startWaitMillis(0) .featureStore(testFeatureStore) .build(); - - expect(initFuture.get(10L, TimeUnit.MILLISECONDS)).andReturn(new Object()); - expect(pollingProcessor.start()).andReturn(initFuture); - expect(pollingProcessor.initialized()).andReturn(false).times(1); + expect(updateProcessor.start()).andReturn(initFuture); + expect(updateProcessor.initialized()).andReturn(false).times(1); replayAll(); client = createMockClient(config); testFeatureStore.setIntegerValue("key", 1); - assertTrue("Flag is marked as known", client.isFlagKnown("key")); - verifyAll(); - } - - @Test - public void testFeatureMatchesUserBySegment() throws Exception { - TestFeatureStore testFeatureStore = new TestFeatureStore(); - testFeatureStore.setInitialized(true); - LDConfig config = new LDConfig.Builder() - .startWaitMillis(10L) - .stream(false) - .featureStore(testFeatureStore) - .build(); - - expect(initFuture.get(10L, TimeUnit.MILLISECONDS)).andReturn(new Object()); - expect(pollingProcessor.start()).andReturn(initFuture); - expect(pollingProcessor.initialized()).andReturn(true).times(1); - expectEventsSent(1); - replayAll(); - - client = createMockClient(config); - - Segment segment = new Segment.Builder("segment1") - .version(1) - .included(Arrays.asList("user")) - .build(); - testFeatureStore.upsert(SEGMENTS, segment); - - Clause clause = new Clause( - "", - Operator.segmentMatch, - Arrays.asList(new JsonPrimitive("segment1")), - false); - Rule rule = new Rule(Arrays.asList(clause), 0, null); - FeatureFlag feature = new FeatureFlagBuilder("test-feature") - .version(1) - .rules(Arrays.asList(rule)) - .variations(TestFeatureStore.TRUE_FALSE_VARIATIONS) - .on(true) - .fallthrough(new VariationOrRollout(1, null)) - .build(); - testFeatureStore.upsert(FEATURES, feature); - - assertTrue(client.boolVariation("test-feature", new LDUser("user"), false)); - - verifyAll(); - } - - @Test - public void testUseLdd() throws IOException { - LDConfig config = new LDConfig.Builder() - .useLdd(true) - .build(); - - client = createMockClient(config); - // Asserting 2 things here: no pollingProcessor or streamingProcessor activity - // and sending of event: - expectEventsSent(1); - replayAll(); - - assertDefaultValueIsReturned(); - assertTrue(client.initialized()); - verifyAll(); - } - - @Test - public void testStreamingNoWait() throws IOException { - LDConfig config = new LDConfig.Builder() - .startWaitMillis(0L) - .stream(true) - .build(); - - expect(streamProcessor.start()).andReturn(initFuture); - expect(streamProcessor.initialized()).andReturn(false); - expectEventsSent(1); - replayAll(); - - client = createMockClient(config); - assertDefaultValueIsReturned(); - - verifyAll(); - } - - @Test - public void testStreamingWait() throws Exception { - LDConfig config = new LDConfig.Builder() - .startWaitMillis(10L) - .stream(true) - .build(); - - expect(streamProcessor.start()).andReturn(initFuture); - expect(initFuture.get(10L, TimeUnit.MILLISECONDS)).andThrow(new TimeoutException()); - replayAll(); - - client = createMockClient(config); - verifyAll(); - } - - @Test - public void testPollingNoWait() throws IOException { - LDConfig config = new LDConfig.Builder() - .startWaitMillis(0L) - .stream(false) - .build(); - - expect(pollingProcessor.start()).andReturn(initFuture); - expect(pollingProcessor.initialized()).andReturn(false); - expectEventsSent(1); - replayAll(); - - client = createMockClient(config); - assertDefaultValueIsReturned(); - - verifyAll(); - } - - @Test - public void testPollingWait() throws Exception { - LDConfig config = new LDConfig.Builder() - .startWaitMillis(10L) - .stream(false) - .build(); - - expect(pollingProcessor.start()).andReturn(initFuture); - expect(initFuture.get(10L, TimeUnit.MILLISECONDS)).andThrow(new TimeoutException()); - expectEventsSent(1); - expect(pollingProcessor.initialized()).andReturn(false); - replayAll(); - - client = createMockClient(config); - assertDefaultValueIsReturned(); - verifyAll(); - } - - @Test - public void testSecureModeHash() { - LDConfig config = new LDConfig.Builder() - .offline(true) - .build(); - LDClientInterface client = new LDClient("secret", config); - LDUser user = new LDUser.Builder("Message").build(); - assertEquals("aa747c502a898200f9e4fa21bac68136f886a0e27aec70ba06daf2e2a5cb5597", client.secureModeHash(user)); - } - - @Test - public void testNoFeatureEventsAreSentWhenSendEventsIsFalse() throws Exception { - LDConfig config = new LDConfig.Builder() - .sendEvents(false) - .stream(false) - .build(); - - expect(initFuture.get(5000L, TimeUnit.MILLISECONDS)).andThrow(new TimeoutException()); - expect(pollingProcessor.start()).andReturn(initFuture); - expect(pollingProcessor.initialized()).andReturn(true).anyTimes(); - expectEventsSent(0); - replayAll(); - - client = createMockClient(config); - client.boolVariation("test", new LDUser("test.key"), true); - - verifyAll(); - } - - @Test - public void testNoIdentifyEventsAreSentWhenSendEventsIsFalse() throws Exception { - LDConfig config = new LDConfig.Builder() - .sendEvents(false) - .stream(false) - .build(); - - expect(initFuture.get(5000L, TimeUnit.MILLISECONDS)).andThrow(new TimeoutException()); - expect(pollingProcessor.start()).andReturn(initFuture); - expect(pollingProcessor.initialized()).andReturn(true).anyTimes(); - expectEventsSent(0); - replayAll(); - - client = createMockClient(config); - client.identify(new LDUser("test.key")); - + assertTrue(client.isFlagKnown("key")); verifyAll(); } @Test - public void testNoCustomEventsAreSentWhenSendEventsIsFalse() throws Exception { - LDConfig config = new LDConfig.Builder() - .sendEvents(false) - .stream(false) - .build(); - - expect(initFuture.get(5000L, TimeUnit.MILLISECONDS)).andThrow(new TimeoutException()); - expect(pollingProcessor.start()).andReturn(initFuture); - expect(pollingProcessor.initialized()).andReturn(true).anyTimes(); - expectEventsSent(0); - replayAll(); - - client = createMockClient(config); - client.track("test", new LDUser("test.key")); - - verifyAll(); - } - - @Test - public void testEvaluationCanUseFeatureStoreIfInitializationTimesOut() throws IOException { + public void evaluationUsesStoreIfStoreIsInitializedButClientIsNot() throws Exception { TestFeatureStore testFeatureStore = new TestFeatureStore(); testFeatureStore.setInitialized(true); LDConfig config = new LDConfig.Builder() .featureStore(testFeatureStore) .startWaitMillis(0L) - .stream(true) .build(); - - expect(streamProcessor.start()).andReturn(initFuture); - expect(streamProcessor.initialized()).andReturn(false); + expect(updateProcessor.start()).andReturn(initFuture); + expect(updateProcessor.initialized()).andReturn(false); expectEventsSent(1); replayAll(); @@ -576,11 +249,6 @@ public void testEvaluationCanUseFeatureStoreIfInitializationTimesOut() throws IO verifyAll(); } - private void assertDefaultValueIsReturned() { - boolean result = client.boolVariation("test", new LDUser("test.key"), true); - assertEquals(true, result); - } - private void expectEventsSent(int count) { eventProcessor.sendEvent(anyObject(Event.class)); if (count > 0) { @@ -593,23 +261,13 @@ private void expectEventsSent(int count) { private LDClientInterface createMockClient(LDConfig config) { return new LDClient("SDK_KEY", config) { @Override - protected FeatureRequestor createFeatureRequestor(String sdkKey, LDConfig config) { - return requestor; - } - - @Override - protected StreamProcessor createStreamProcessor(String sdkKey, LDConfig config, FeatureRequestor requestor) { - return streamProcessor; - } - - @Override - protected PollingProcessor createPollingProcessor(LDConfig config) { - return pollingProcessor; + protected UpdateProcessor createUpdateProcessor(String sdkKey, LDConfig config) { + return LDClientTest.this.updateProcessor; } @Override protected EventProcessor createEventProcessor(String sdkKey, LDConfig config) { - return eventProcessor; + return LDClientTest.this.eventProcessor; } }; } diff --git a/src/test/java/com/launchdarkly/client/TestUtil.java b/src/test/java/com/launchdarkly/client/TestUtil.java index 277afc2fa..6dd034fc7 100644 --- a/src/test/java/com/launchdarkly/client/TestUtil.java +++ b/src/test/java/com/launchdarkly/client/TestUtil.java @@ -21,6 +21,10 @@ public static JsonPrimitive js(String s) { public static JsonPrimitive jint(int n) { return new JsonPrimitive(n); } + + public static JsonPrimitive jdouble(double d) { + return new JsonPrimitive(d); + } public static JsonPrimitive jbool(boolean b) { return new JsonPrimitive(b); From 50fdfba92447486c79b360d17490ea2bf55b2a13 Mon Sep 17 00:00:00 2001 From: Eli Bishop Date: Wed, 28 Mar 2018 12:54:48 -0700 Subject: [PATCH 43/67] misc cleanup --- .../java/com/launchdarkly/client/DefaultEventProcessor.java | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/src/main/java/com/launchdarkly/client/DefaultEventProcessor.java b/src/main/java/com/launchdarkly/client/DefaultEventProcessor.java index 743d9eef2..a657babbf 100644 --- a/src/main/java/com/launchdarkly/client/DefaultEventProcessor.java +++ b/src/main/java/com/launchdarkly/client/DefaultEventProcessor.java @@ -189,7 +189,7 @@ private void runMainLoop() { synchronizeForTesting(); break; case SHUTDOWN: - finishAllPendingFlushes(); + doShutdown(); running = false; } message.completed(); @@ -201,7 +201,7 @@ private void runMainLoop() { } } - private void finishAllPendingFlushes() { + private void doShutdown() { flushWorkersPool.shutdown(); try { while (!flushWorkersPool.awaitTermination(CHANNEL_BLOCK_MILLIS, TimeUnit.MILLISECONDS)) { @@ -209,6 +209,8 @@ private void finishAllPendingFlushes() { } } catch (InterruptedException e) { } + // Note that we don't close the HTTP client here, because it's shared by other components + // via the LDConfig. The LDClient will dispose of it. } private void synchronizeForTesting() { From 9d540760dfe4d62bb828aee6aeac7d7f67591214 Mon Sep 17 00:00:00 2001 From: Eli Bishop Date: Thu, 29 Mar 2018 12:56:54 -0700 Subject: [PATCH 44/67] migrate to Circle 2 --- .circleci/config.yml | 24 ++++++++++++++++++++++++ circle.yml | 16 ---------------- 2 files changed, 24 insertions(+), 16 deletions(-) create mode 100644 .circleci/config.yml delete mode 100644 circle.yml diff --git a/.circleci/config.yml b/.circleci/config.yml new file mode 100644 index 000000000..c2bfdacaf --- /dev/null +++ b/.circleci/config.yml @@ -0,0 +1,24 @@ +version: 2 +jobs: + build: + branches: + ignore: + - gh-pages + docker: + - image: circleci/java + - image: redis + steps: + - checkout + - run: cp gradle.properties.example gradle.properties + - run: ./gradlew dependencies + - run: ./gradlew test + - run: + name: Save test results + command: | + mkdir -p ~/junit/ + find . -type f -regex ".*/build/test-results/.*xml" -exec cp {} ~/junit/ \; + when: always + - store_test_results: + path: ~/junit + - store_artifacts: + path: ~/junit diff --git a/circle.yml b/circle.yml deleted file mode 100644 index 97c7b8d3d..000000000 --- a/circle.yml +++ /dev/null @@ -1,16 +0,0 @@ ---- -machine: - services: - - redis -dependencies: - pre: - - "cp gradle.properties.example gradle.properties" -experimental: - notify: - branches: - ignore: - - gh-pages -general: - branches: - ignore: - - gh-pages From 6c53d6bfcb6252db4bb75c9eae427cc7af19a5a8 Mon Sep 17 00:00:00 2001 From: Eli Bishop Date: Thu, 29 Mar 2018 22:58:04 -0700 Subject: [PATCH 45/67] fix merge --- src/test/java/com/launchdarkly/client/FeatureFlagTest.java | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/src/test/java/com/launchdarkly/client/FeatureFlagTest.java b/src/test/java/com/launchdarkly/client/FeatureFlagTest.java index dd260d1e0..f32c754be 100644 --- a/src/test/java/com/launchdarkly/client/FeatureFlagTest.java +++ b/src/test/java/com/launchdarkly/client/FeatureFlagTest.java @@ -1,11 +1,8 @@ package com.launchdarkly.client; import com.google.gson.Gson; -import com.google.gson.JsonArray; import com.google.gson.JsonElement; -import com.google.gson.JsonObject; -import org.junit.Assert; import org.junit.Before; import org.junit.Test; @@ -267,7 +264,7 @@ public void clauseWithNullOperatorDoesNotMatch() throws Exception { FeatureFlag f = booleanFlagWithClauses(badClause); LDUser user = new LDUser.Builder("key").name("Bob").build(); - assertEquals(jbool(false), f.evaluate(user, featureStore).getValue()); + assertEquals(jbool(false), f.evaluate(user, featureStore, EventFactory.DEFAULT).getResult().getValue()); } @Test @@ -285,7 +282,7 @@ public void clauseWithNullOperatorDoesNotStopSubsequentRuleFromMatching() throws .build(); LDUser user = new LDUser.Builder("key").name("Bob").build(); - assertEquals(jbool(true), f.evaluate(user, featureStore).getValue()); + assertEquals(jbool(true), f.evaluate(user, featureStore, EventFactory.DEFAULT).getResult().getValue()); } @Test From d0e0ce51d629869eff0c29b80c583292240807e1 Mon Sep 17 00:00:00 2001 From: Eli Bishop Date: Fri, 30 Mar 2018 13:17:14 -0700 Subject: [PATCH 46/67] use a fixed pool of flush workers; move some of the flush logic back into EventConsumer --- .../client/DefaultEventProcessor.java | 220 +++++++++++------- .../launchdarkly/client/EventSummarizer.java | 21 +- 2 files changed, 149 insertions(+), 92 deletions(-) diff --git a/src/main/java/com/launchdarkly/client/DefaultEventProcessor.java b/src/main/java/com/launchdarkly/client/DefaultEventProcessor.java index a657babbf..c4c56a53d 100644 --- a/src/main/java/com/launchdarkly/client/DefaultEventProcessor.java +++ b/src/main/java/com/launchdarkly/client/DefaultEventProcessor.java @@ -6,11 +6,13 @@ import com.google.gson.annotations.SerializedName; import com.launchdarkly.client.EventSummarizer.CounterKey; import com.launchdarkly.client.EventSummarizer.CounterValue; +import com.launchdarkly.client.EventSummarizer.EventSummary; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import java.io.IOException; +import java.text.ParseException; import java.text.SimpleDateFormat; import java.util.ArrayList; import java.util.HashMap; @@ -73,20 +75,24 @@ public void run() { @Override public void sendEvent(Event e) { - postMessageAsync(MessageType.EVENT, e); + if (!closed.get()) { + postMessageAsync(MessageType.EVENT, e); + } } @Override public void flush() { - postMessageAsync(MessageType.FLUSH, null); + if (!closed.get()) { + postMessageAsync(MessageType.FLUSH, null); + } } @Override public void close() throws IOException { if (closed.compareAndSet(false, true)) { + scheduler.shutdown(); postMessageAsync(MessageType.FLUSH, null); postMessageAndWait(MessageType.SHUTDOWN, null); - scheduler.shutdown(); } } @@ -134,15 +140,17 @@ private static class EventConsumer { private final String sdkKey; private final LDConfig config; private final BlockingQueue inputChannel; - private final ExecutorService flushWorkersPool; private final AtomicInteger flushWorkersActive; private final Thread mainThread; + private final List flushWorkerThreads; + private final BlockingQueue payloadQueue; private final ArrayList buffer; private final EventSummarizer summarizer; private final SimpleLRUCache userKeys; private final Random random = new Random(); private final AtomicLong lastKnownPastTime = new AtomicLong(0); private final AtomicBoolean disabled = new AtomicBoolean(false); + private final AtomicBoolean shuttingDown = new AtomicBoolean(false); private boolean capacityExceeded = false; private EventConsumer(String sdkKey, LDConfig config, @@ -151,18 +159,35 @@ private EventConsumer(String sdkKey, LDConfig config, this.sdkKey = sdkKey; this.config = config; this.inputChannel = inputChannel; - this.flushWorkersPool = Executors.newFixedThreadPool(MAX_FLUSH_THREADS, threadFactory); this.flushWorkersActive = new AtomicInteger(0); this.buffer = new ArrayList<>(config.capacity); this.summarizer = new EventSummarizer(); this.userKeys = new SimpleLRUCache(config.userKeysCapacity); + // This queue only holds one element; it represents a flush task that has not yet been + // picked up by any worker, so if we try to push another one and are refused, it means + // all the workers are busy. + this.payloadQueue = new ArrayBlockingQueue<>(1); + mainThread = threadFactory.newThread(new Runnable() { public void run() { runMainLoop(); } }); + mainThread.setDaemon(true); mainThread.start(); + + flushWorkerThreads = new ArrayList<>(); + for (int i = 0; i < MAX_FLUSH_THREADS; i++) { + Thread workerThread = threadFactory.newThread(new Runnable() { + public void run() { + runFlushWorker(); + } + }); + workerThread.setDaemon(true); + workerThread.start(); + flushWorkerThreads.add(workerThread); + } } /** @@ -171,8 +196,7 @@ public void run() { * dispatchFlush will fire off another task to do the part that takes longer. */ private void runMainLoop() { - boolean running = true; - while (running) { + while (true) { try { EventProcessorMessage message = inputChannel.take(); switch(message.type) { @@ -180,17 +204,19 @@ private void runMainLoop() { processEvent(message.event); break; case FLUSH: - startFlush(); + triggerFlush(); break; case FLUSH_USERS: userKeys.clear(); break; case SYNC: - synchronizeForTesting(); + waitUntilAllFlushWorkersInactive(); + message.completed(); break; case SHUTDOWN: doShutdown(); - running = false; + message.completed(); + return; } message.completed(); } catch (InterruptedException e) { @@ -202,18 +228,47 @@ private void runMainLoop() { } private void doShutdown() { - flushWorkersPool.shutdown(); - try { - while (!flushWorkersPool.awaitTermination(CHANNEL_BLOCK_MILLIS, TimeUnit.MILLISECONDS)) { - logger.debug("Waiting for event flush tasks to terminate"); - } - } catch (InterruptedException e) { + waitUntilAllFlushWorkersInactive(); + disabled.set(true); // In case there are any more messages, we want to ignore them + shuttingDown.set(true); + for (Thread t: flushWorkerThreads) { + // Wake up the worker if it's waiting, so it'll notice shuttingDown is true and terminate + t.interrupt(); } // Note that we don't close the HTTP client here, because it's shared by other components // via the LDConfig. The LDClient will dispose of it. } - private void synchronizeForTesting() { + private void runFlushWorker() { + while (!shuttingDown.get()) { + FlushPayload payload = null; + try { + payload = payloadQueue.take(); + } catch (InterruptedException e) { + continue; + } + try { + doFlush(payload); + } catch (Exception e) { + logger.error("Unexpected error in event processor: " + e); + logger.debug(e.getMessage(), e); + } + flushWorkerFinishedWork(); + } + } + + private void flushWorkerStartingWork() { + flushWorkersActive.incrementAndGet(); + } + + private void flushWorkerFinishedWork() { + synchronized(flushWorkersActive) { + flushWorkersActive.decrementAndGet(); + flushWorkersActive.notify(); + } + } + + private void waitUntilAllFlushWorkersInactive() { while (true) { try { synchronized(flushWorkersActive) { @@ -299,93 +354,57 @@ private boolean shouldTrackFullEvent(Event e) { } } - private void startFlush() { + private void triggerFlush() { if (disabled.get()) { return; } Event[] events = buffer.toArray(new Event[buffer.size()]); - buffer.clear(); - EventSummarizer.EventSummary snapshot = summarizer.snapshot(); + EventSummarizer.EventSummary summary = summarizer.snapshot(); - if (events.length != 0 || !snapshot.isEmpty()) { - flushWorkersActive.incrementAndGet(); - EventPayloadSender task = new EventPayloadSender(this, events, snapshot); - flushWorkersPool.execute(task); - } - } - - private void handleResponse(Response response) { - logger.debug("Events Response: " + response.code()); - try { - String dateStr = response.header("Date"); - if (dateStr != null) { - lastKnownPastTime.set(HTTP_DATE_FORMAT.parse(dateStr).getTime()); - } - } catch (Exception e) { - } - if (!response.isSuccessful()) { - logger.info("Got unexpected response when posting events: " + response); - if (response.code() == 401) { - disabled.set(true); - logger.error("Received 401 error, no further events will be posted since SDK key is invalid"); + if (events.length != 0 || !summary.isEmpty()) { + flushWorkerStartingWork(); + FlushPayload payload = new FlushPayload(events, summary); + if (payloadQueue.offer(payload)) { + // These events now belong to the next available flush worker, so drop them from the consumer + buffer.clear(); + summarizer.clear(); + } else { + logger.debug("Skipped flushing because all workers are busy"); + // All the workers are busy so we can't flush now; keep the events in the consumer + flushWorkerFinishedWork(); } } } - } - - /** - * Transforms the internal event data into the JSON event payload format and sends it off. - * This is done on a separate worker thread. - */ - private static class EventPayloadSender implements Runnable { - private static final Logger logger = LoggerFactory.getLogger(EventPayloadSender.class); - - private final EventConsumer consumer; - private final Event[] events; - private final EventSummarizer.EventSummary summary; - EventPayloadSender(EventConsumer consumer, Event[] events, EventSummarizer.EventSummary summary) { - this.consumer = consumer; - this.events = events; - this.summary = summary; - } - - public void run() { - try { - doSend(); - } catch (Exception e) { - logger.error("Unexpected error in event processor: " + e); - logger.debug(e.getMessage(), e); - } finally { - synchronized(consumer.flushWorkersActive) { - consumer.flushWorkersActive.decrementAndGet(); - consumer.flushWorkersActive.notify(); - } + // Runs on a flush worker thread + private void doFlush(FlushPayload payload) { + if (payload.events.length == 0 && payload.summary.isEmpty()) { + return; } - } - - private void doSend() throws Exception { - List eventsOut = new ArrayList<>(events.length + 1); - for (Event event: events) { + List eventsOut = new ArrayList<>(payload.events.length + 1); + for (Event event: payload.events) { eventsOut.add(createEventOutput(event)); } - if (!summary.isEmpty()) { - eventsOut.add(createSummaryEvent(summary)); + if (!payload.summary.isEmpty()) { + eventsOut.add(createSummaryEvent(payload.summary)); } - String json = consumer.config.gson.toJson(eventsOut); + String json = config.gson.toJson(eventsOut); logger.debug("Posting {} event(s) to {} with payload: {}", - eventsOut.size(), consumer.config.eventsURI, json); + eventsOut.size(), config.eventsURI, json); - Request request = consumer.config.getRequestBuilder(consumer.sdkKey) - .url(consumer.config.eventsURI.toString() + "/bulk") + Request request = config.getRequestBuilder(sdkKey) + .url(config.eventsURI.toString() + "/bulk") .post(RequestBody.create(MediaType.parse("application/json; charset=utf-8"), json)) .addHeader("Content-Type", "application/json") .build(); - try (Response response = consumer.config.httpClient.newCall(request).execute()) { - consumer.handleResponse(response); + long startTime = System.currentTimeMillis(); + try (Response response = config.httpClient.newCall(request).execute()) { + long endTime = System.currentTimeMillis(); + logger.debug("Event delivery took {} ms, response status {}", endTime - startTime, response.code()); + handleResponse(response); } catch (IOException e) { logger.info("Unhandled exception in LaunchDarkly client when posting events to URL: " + request.url(), e); } @@ -397,16 +416,16 @@ private EventOutput createEventOutput(Event e) { FeatureRequestEvent fe = (FeatureRequestEvent)e; boolean isDebug = (!fe.trackEvents && fe.debugEventsUntilDate != null); return new FeatureRequestEventOutput(fe.creationDate, fe.key, - consumer.config.inlineUsersInEvents ? null : userKey, - consumer.config.inlineUsersInEvents ? e.user : null, + config.inlineUsersInEvents ? null : userKey, + config.inlineUsersInEvents ? e.user : null, fe.version, fe.value, fe.defaultVal, fe.prereqOf, isDebug); } else if (e instanceof IdentifyEvent) { return new IdentifyEventOutput(e.creationDate, e.user); } else if (e instanceof CustomEvent) { CustomEvent ce = (CustomEvent)e; return new CustomEventOutput(ce.creationDate, ce.key, - consumer.config.inlineUsersInEvents ? null : userKey, - consumer.config.inlineUsersInEvents ? e.user : null, + config.inlineUsersInEvents ? null : userKey, + config.inlineUsersInEvents ? e.user : null, ce.data); } else if (e instanceof IndexEvent) { return new IndexEventOutput(e.creationDate, e.user); @@ -431,6 +450,23 @@ private EventOutput createSummaryEvent(EventSummarizer.EventSummary summary) { } return new SummaryEventOutput(summary.startDate, summary.endDate, flagsOut); } + + private void handleResponse(Response response) { + String dateStr = response.header("Date"); + if (dateStr != null) { + try { + lastKnownPastTime.set(HTTP_DATE_FORMAT.parse(dateStr).getTime()); + } catch (ParseException e) { + } + } + if (!response.isSuccessful()) { + logger.info("Got unexpected response when posting events: " + response); + if (response.code() == 401) { + disabled.set(true); + logger.error("Received 401 error, no further events will be posted since SDK key is invalid"); + } + } + } } private static enum MessageType { @@ -479,6 +515,16 @@ public String toString() { } } + private static class FlushPayload { + final Event[] events; + final EventSummary summary; + + FlushPayload(Event[] events, EventSummary summary) { + this.events = events; + this.summary = summary; + } + } + private static interface EventOutput { } @SuppressWarnings("unused") diff --git a/src/main/java/com/launchdarkly/client/EventSummarizer.java b/src/main/java/com/launchdarkly/client/EventSummarizer.java index 50e336845..d605204dd 100644 --- a/src/main/java/com/launchdarkly/client/EventSummarizer.java +++ b/src/main/java/com/launchdarkly/client/EventSummarizer.java @@ -31,13 +31,18 @@ void summarizeEvent(Event event) { } /** - * Returns a snapshot of the current summarized event data, and resets this state. - * @return the previous event state + * Returns a snapshot of the current summarized event data. + * @return the summary state */ EventSummary snapshot() { - EventSummary ret = eventsState; + return new EventSummary(eventsState); + } + + /** + * Resets the summary counters. + */ + void clear() { eventsState = new EventSummary(); - return ret; } static class EventSummary { @@ -46,7 +51,13 @@ static class EventSummary { long endDate; EventSummary() { - counters = new HashMap(); + counters = new HashMap<>(); + } + + EventSummary(EventSummary from) { + counters = new HashMap<>(from.counters); + startDate = from.startDate; + endDate = from.endDate; } boolean isEmpty() { From 8a0742a75f5431533ade3648eef919c82834c996 Mon Sep 17 00:00:00 2001 From: Eli Bishop Date: Mon, 2 Apr 2018 18:27:32 -0700 Subject: [PATCH 47/67] big refactoring --- .../com/launchdarkly/client/CustomEvent.java | 14 - .../client/DefaultEventProcessor.java | 540 +++++++----------- .../java/com/launchdarkly/client/Event.java | 53 ++ .../com/launchdarkly/client/EventFactory.java | 20 +- .../com/launchdarkly/client/EventOutput.java | 195 +++++++ .../launchdarkly/client/EventSummarizer.java | 12 +- .../com/launchdarkly/client/FeatureFlag.java | 10 +- .../client/FeatureRequestEvent.java | 27 - .../launchdarkly/client/IdentifyEvent.java | 8 - .../com/launchdarkly/client/LDClient.java | 4 +- .../client/DefaultEventProcessorTest.java | 28 +- .../launchdarkly/client/FeatureFlagTest.java | 8 +- 12 files changed, 492 insertions(+), 427 deletions(-) delete mode 100644 src/main/java/com/launchdarkly/client/CustomEvent.java create mode 100644 src/main/java/com/launchdarkly/client/EventOutput.java delete mode 100644 src/main/java/com/launchdarkly/client/FeatureRequestEvent.java delete mode 100644 src/main/java/com/launchdarkly/client/IdentifyEvent.java diff --git a/src/main/java/com/launchdarkly/client/CustomEvent.java b/src/main/java/com/launchdarkly/client/CustomEvent.java deleted file mode 100644 index b2f88ef09..000000000 --- a/src/main/java/com/launchdarkly/client/CustomEvent.java +++ /dev/null @@ -1,14 +0,0 @@ -package com.launchdarkly.client; - -import com.google.gson.JsonElement; - -class CustomEvent extends Event { - final String key; - final JsonElement data; - - CustomEvent(long timestamp, String key, LDUser user, JsonElement data) { - super(timestamp, user); - this.key = key; - this.data = data; - } -} diff --git a/src/main/java/com/launchdarkly/client/DefaultEventProcessor.java b/src/main/java/com/launchdarkly/client/DefaultEventProcessor.java index c4c56a53d..d3f1ae5f6 100644 --- a/src/main/java/com/launchdarkly/client/DefaultEventProcessor.java +++ b/src/main/java/com/launchdarkly/client/DefaultEventProcessor.java @@ -2,10 +2,6 @@ import com.google.common.annotations.VisibleForTesting; import com.google.common.util.concurrent.ThreadFactoryBuilder; -import com.google.gson.JsonElement; -import com.google.gson.annotations.SerializedName; -import com.launchdarkly.client.EventSummarizer.CounterKey; -import com.launchdarkly.client.EventSummarizer.CounterValue; import com.launchdarkly.client.EventSummarizer.EventSummary; import org.slf4j.Logger; @@ -15,13 +11,10 @@ import java.text.ParseException; import java.text.SimpleDateFormat; import java.util.ArrayList; -import java.util.HashMap; import java.util.List; -import java.util.Map; import java.util.Random; import java.util.concurrent.ArrayBlockingQueue; import java.util.concurrent.BlockingQueue; -import java.util.concurrent.ExecutorService; import java.util.concurrent.Executors; import java.util.concurrent.ScheduledExecutorService; import java.util.concurrent.Semaphore; @@ -36,13 +29,12 @@ import okhttp3.RequestBody; import okhttp3.Response; -class DefaultEventProcessor implements EventProcessor { +final class DefaultEventProcessor implements EventProcessor { private static final Logger logger = LoggerFactory.getLogger(DefaultEventProcessor.class); static final SimpleDateFormat HTTP_DATE_FORMAT = new SimpleDateFormat("EEE, dd MMM yyyy HH:mm:ss zzz"); private static final int CHANNEL_BLOCK_MILLIS = 1000; private final BlockingQueue inputChannel; - private final ThreadFactory threadFactory; private final ScheduledExecutorService scheduler; private final AtomicBoolean closed = new AtomicBoolean(false); private final AtomicBoolean inputCapacityExceeded = new AtomicBoolean(false); @@ -50,13 +42,13 @@ class DefaultEventProcessor implements EventProcessor { DefaultEventProcessor(String sdkKey, LDConfig config) { inputChannel = new ArrayBlockingQueue<>(config.capacity); - threadFactory = new ThreadFactoryBuilder() + ThreadFactory threadFactory = new ThreadFactoryBuilder() .setDaemon(true) .setNameFormat("LaunchDarkly-EventProcessor-%d") .build(); scheduler = Executors.newSingleThreadScheduledExecutor(threadFactory); - new EventConsumer(sdkKey, config, inputChannel, threadFactory); + new EventDispatcher(sdkKey, config, inputChannel, threadFactory); Runnable flusher = new Runnable() { public void run() { @@ -129,64 +121,99 @@ private void postToChannel(EventProcessorMessage message) { } } } + + private static enum MessageType { + EVENT, + FLUSH, + FLUSH_USERS, + SYNC, + SHUTDOWN + } + + private static final class EventProcessorMessage { + private final MessageType type; + private final Event event; + private final Semaphore reply; + + private EventProcessorMessage(MessageType type, Event event, boolean sync) { + this.type = type; + this.event = event; + reply = sync ? new Semaphore(0) : null; + } + + void completed() { + if (reply != null) { + reply.release(); + } + } + + void waitForCompletion() { + if (reply == null) { + return; + } + while (true) { + try { + reply.acquire(); + return; + } + catch (InterruptedException ex) { + } + } + } + + @Override + public String toString() { + return ((event == null) ? type.toString() : (type + ": " + event.getClass().getSimpleName())) + + (reply == null ? "" : " (sync)"); + } + } /** * Takes messages from the input queue, updating the event buffer and summary counters * on its own thread. */ - private static class EventConsumer { + private static final class EventDispatcher { private static final int MAX_FLUSH_THREADS = 5; - private final String sdkKey; private final LDConfig config; - private final BlockingQueue inputChannel; - private final AtomicInteger flushWorkersActive; - private final Thread mainThread; - private final List flushWorkerThreads; - private final BlockingQueue payloadQueue; - private final ArrayList buffer; - private final EventSummarizer summarizer; - private final SimpleLRUCache userKeys; + private final List flushWorkers; + private final AtomicInteger activeFlushWorkersCount; private final Random random = new Random(); private final AtomicLong lastKnownPastTime = new AtomicLong(0); private final AtomicBoolean disabled = new AtomicBoolean(false); - private final AtomicBoolean shuttingDown = new AtomicBoolean(false); - private boolean capacityExceeded = false; - private EventConsumer(String sdkKey, LDConfig config, - BlockingQueue inputChannel, - ThreadFactory threadFactory) { - this.sdkKey = sdkKey; + private EventDispatcher(String sdkKey, LDConfig config, + final BlockingQueue inputChannel, + ThreadFactory threadFactory) { this.config = config; - this.inputChannel = inputChannel; - this.flushWorkersActive = new AtomicInteger(0); - this.buffer = new ArrayList<>(config.capacity); - this.summarizer = new EventSummarizer(); - this.userKeys = new SimpleLRUCache(config.userKeysCapacity); + this.activeFlushWorkersCount = new AtomicInteger(0); // This queue only holds one element; it represents a flush task that has not yet been // picked up by any worker, so if we try to push another one and are refused, it means // all the workers are busy. - this.payloadQueue = new ArrayBlockingQueue<>(1); + final BlockingQueue payloadQueue = new ArrayBlockingQueue<>(1); - mainThread = threadFactory.newThread(new Runnable() { + final EventBuffer buffer = new EventBuffer(config.capacity); + final SimpleLRUCache userKeys = new SimpleLRUCache(config.userKeysCapacity); + + Thread mainThread = threadFactory.newThread(new Runnable() { public void run() { - runMainLoop(); + runMainLoop(inputChannel, buffer, userKeys, payloadQueue); } }); mainThread.setDaemon(true); mainThread.start(); - flushWorkerThreads = new ArrayList<>(); - for (int i = 0; i < MAX_FLUSH_THREADS; i++) { - Thread workerThread = threadFactory.newThread(new Runnable() { - public void run() { - runFlushWorker(); + flushWorkers = new ArrayList<>(); + EventResponseListener listener = new EventResponseListener() { + public void handleResponse(Response response) { + EventDispatcher.this.handleResponse(response); } - }); - workerThread.setDaemon(true); - workerThread.start(); - flushWorkerThreads.add(workerThread); + }; + for (int i = 0; i < MAX_FLUSH_THREADS; i++) { + SendEventsTask task = new SendEventsTask(sdkKey, config, listener, payloadQueue, + activeFlushWorkersCount, threadFactory); + flushWorkers.add(task); } } @@ -195,16 +222,18 @@ public void run() { * thread so we don't have to synchronize on our internal structures; when it's time to flush, * dispatchFlush will fire off another task to do the part that takes longer. */ - private void runMainLoop() { + private void runMainLoop(BlockingQueue inputChannel, + EventBuffer buffer, SimpleLRUCache userKeys, + BlockingQueue payloadQueue) { while (true) { try { EventProcessorMessage message = inputChannel.take(); switch(message.type) { case EVENT: - processEvent(message.event); + processEvent(message.event, userKeys, buffer); break; case FLUSH: - triggerFlush(); + triggerFlush(buffer, payloadQueue); break; case FLUSH_USERS: userKeys.clear(); @@ -230,74 +259,43 @@ private void runMainLoop() { private void doShutdown() { waitUntilAllFlushWorkersInactive(); disabled.set(true); // In case there are any more messages, we want to ignore them - shuttingDown.set(true); - for (Thread t: flushWorkerThreads) { - // Wake up the worker if it's waiting, so it'll notice shuttingDown is true and terminate - t.interrupt(); + for (SendEventsTask task: flushWorkers) { + task.stop(); } // Note that we don't close the HTTP client here, because it's shared by other components // via the LDConfig. The LDClient will dispose of it. } - private void runFlushWorker() { - while (!shuttingDown.get()) { - FlushPayload payload = null; - try { - payload = payloadQueue.take(); - } catch (InterruptedException e) { - continue; - } - try { - doFlush(payload); - } catch (Exception e) { - logger.error("Unexpected error in event processor: " + e); - logger.debug(e.getMessage(), e); - } - flushWorkerFinishedWork(); - } - } - - private void flushWorkerStartingWork() { - flushWorkersActive.incrementAndGet(); - } - - private void flushWorkerFinishedWork() { - synchronized(flushWorkersActive) { - flushWorkersActive.decrementAndGet(); - flushWorkersActive.notify(); - } - } - private void waitUntilAllFlushWorkersInactive() { while (true) { try { - synchronized(flushWorkersActive) { - if (flushWorkersActive.get() == 0) { + synchronized(activeFlushWorkersCount) { + if (activeFlushWorkersCount.get() == 0) { return; } else { - flushWorkersActive.wait(); + activeFlushWorkersCount.wait(); } } } catch (InterruptedException e) {} } } - private void processEvent(Event e) { + private void processEvent(Event e, SimpleLRUCache userKeys, EventBuffer buffer) { if (disabled.get()) { return; } // For each user we haven't seen before, we add an index event - unless this is already // an identify event for that user. - if (!config.inlineUsersInEvents && e.user != null && !noticeUser(e.user)) { - if (!(e instanceof IdentifyEvent)) { - IndexEvent ie = new IndexEvent(e.creationDate, e.user); - addToBuffer(ie); + if (!config.inlineUsersInEvents && e.user != null && !noticeUser(e.user, userKeys)) { + if (!(e instanceof Event.Identify)) { + Event.Index ie = new Event.Index(e.creationDate, e.user); + buffer.add(ie); } } // Always record the event in the summarizer. - summarizer.summarizeEvent(e); + buffer.addToSummary(e); if (shouldTrackFullEvent(e)) { // Sampling interval applies only to fully-tracked events. @@ -306,12 +304,12 @@ private void processEvent(Event e) { } // Queue the event as-is; we'll transform it into an output event when we're flushing // (to avoid doing that work on our main thread). - addToBuffer(e); + buffer.add(e); } } // Add to the set of users we've noticed, and return true if the user was already known to us. - private boolean noticeUser(LDUser user) { + private boolean noticeUser(LDUser user, SimpleLRUCache userKeys) { if (user == null || user.getKey() == null) { return false; } @@ -319,21 +317,9 @@ private boolean noticeUser(LDUser user) { return userKeys.put(key, key) != null; } - private void addToBuffer(Event e) { - if (buffer.size() >= config.capacity) { - if (!capacityExceeded) { // don't need AtomicBoolean, this is only checked on one thread - capacityExceeded = true; - logger.warn("Exceeded event queue capacity. Increase capacity to avoid dropping events."); - } - } else { - capacityExceeded = false; - buffer.add(e); - } - } - private boolean shouldTrackFullEvent(Event e) { - if (e instanceof FeatureRequestEvent) { - FeatureRequestEvent fe = (FeatureRequestEvent)e; + if (e instanceof Event.FeatureRequest) { + Event.FeatureRequest fe = (Event.FeatureRequest)e; if (fe.trackEvents) { return true; } @@ -354,101 +340,23 @@ private boolean shouldTrackFullEvent(Event e) { } } - private void triggerFlush() { - if (disabled.get()) { + private void triggerFlush(EventBuffer buffer, BlockingQueue payloadQueue) { + if (disabled.get() || buffer.isEmpty()) { return; } - - Event[] events = buffer.toArray(new Event[buffer.size()]); - EventSummarizer.EventSummary summary = summarizer.snapshot(); - - if (events.length != 0 || !summary.isEmpty()) { - flushWorkerStartingWork(); - FlushPayload payload = new FlushPayload(events, summary); - if (payloadQueue.offer(payload)) { - // These events now belong to the next available flush worker, so drop them from the consumer - buffer.clear(); - summarizer.clear(); - } else { - logger.debug("Skipped flushing because all workers are busy"); - // All the workers are busy so we can't flush now; keep the events in the consumer - flushWorkerFinishedWork(); - } - } - } - - // Runs on a flush worker thread - private void doFlush(FlushPayload payload) { - if (payload.events.length == 0 && payload.summary.isEmpty()) { - return; - } - List eventsOut = new ArrayList<>(payload.events.length + 1); - for (Event event: payload.events) { - eventsOut.add(createEventOutput(event)); - } - if (!payload.summary.isEmpty()) { - eventsOut.add(createSummaryEvent(payload.summary)); - } - - String json = config.gson.toJson(eventsOut); - logger.debug("Posting {} event(s) to {} with payload: {}", - eventsOut.size(), config.eventsURI, json); - - Request request = config.getRequestBuilder(sdkKey) - .url(config.eventsURI.toString() + "/bulk") - .post(RequestBody.create(MediaType.parse("application/json; charset=utf-8"), json)) - .addHeader("Content-Type", "application/json") - .build(); - - long startTime = System.currentTimeMillis(); - try (Response response = config.httpClient.newCall(request).execute()) { - long endTime = System.currentTimeMillis(); - logger.debug("Event delivery took {} ms, response status {}", endTime - startTime, response.code()); - handleResponse(response); - } catch (IOException e) { - logger.info("Unhandled exception in LaunchDarkly client when posting events to URL: " + request.url(), e); - } - } - - private EventOutput createEventOutput(Event e) { - String userKey = e.user == null ? null : e.user.getKeyAsString(); - if (e instanceof FeatureRequestEvent) { - FeatureRequestEvent fe = (FeatureRequestEvent)e; - boolean isDebug = (!fe.trackEvents && fe.debugEventsUntilDate != null); - return new FeatureRequestEventOutput(fe.creationDate, fe.key, - config.inlineUsersInEvents ? null : userKey, - config.inlineUsersInEvents ? e.user : null, - fe.version, fe.value, fe.defaultVal, fe.prereqOf, isDebug); - } else if (e instanceof IdentifyEvent) { - return new IdentifyEventOutput(e.creationDate, e.user); - } else if (e instanceof CustomEvent) { - CustomEvent ce = (CustomEvent)e; - return new CustomEventOutput(ce.creationDate, ce.key, - config.inlineUsersInEvents ? null : userKey, - config.inlineUsersInEvents ? e.user : null, - ce.data); - } else if (e instanceof IndexEvent) { - return new IndexEventOutput(e.creationDate, e.user); + FlushPayload payload = buffer.getPayload(); + activeFlushWorkersCount.incrementAndGet(); + if (payloadQueue.offer(payload)) { + // These events now belong to the next available flush worker, so drop them from our state + buffer.clear(); } else { - return null; - } - } - - private EventOutput createSummaryEvent(EventSummarizer.EventSummary summary) { - Map flagsOut = new HashMap<>(); - for (Map.Entry entry: summary.counters.entrySet()) { - SummaryEventFlag fsd = flagsOut.get(entry.getKey().key); - if (fsd == null) { - fsd = new SummaryEventFlag(entry.getValue().defaultVal, new ArrayList()); - flagsOut.put(entry.getKey().key, fsd); + logger.debug("Skipped flushing because all workers are busy"); + // All the workers are busy so we can't flush now; keep the events in our state + synchronized(activeFlushWorkersCount) { + activeFlushWorkersCount.decrementAndGet(); + activeFlushWorkersCount.notify(); } - SummaryEventCounter c = new SummaryEventCounter(entry.getValue().flagValue, - entry.getKey().version, - entry.getValue().count, - entry.getKey().version == null ? true : null); - fsd.counters.add(c); } - return new SummaryEventOutput(summary.startDate, summary.endDate, flagsOut); } private void handleResponse(Response response) { @@ -469,177 +377,135 @@ private void handleResponse(Response response) { } } - private static enum MessageType { - EVENT, - FLUSH, - FLUSH_USERS, - SYNC, - SHUTDOWN - } - - private static class EventProcessorMessage { - private final MessageType type; - private final Event event; - private final Semaphore reply; + private static final class EventBuffer { + final List events = new ArrayList<>(); + final EventSummarizer summarizer = new EventSummarizer(); + private final int capacity; + private boolean capacityExceeded = false; - private EventProcessorMessage(MessageType type, Event event, boolean sync) { - this.type = type; - this.event = event; - reply = sync ? new Semaphore(0) : null; + EventBuffer(int capacity) { + this.capacity = capacity; } - void completed() { - if (reply != null) { - reply.release(); - } - } - - void waitForCompletion() { - if (reply == null) { - return; - } - while (true) { - try { - reply.acquire(); - return; - } - catch (InterruptedException ex) { + void add(Event e) { + if (events.size() >= capacity) { + if (!capacityExceeded) { // don't need AtomicBoolean, this is only checked on one thread + capacityExceeded = true; + logger.warn("Exceeded event queue capacity. Increase capacity to avoid dropping events."); } + } else { + capacityExceeded = false; + events.add(e); } } - @Override - public String toString() { - return ((event == null) ? type.toString() : (type + ": " + event.getClass().getSimpleName())) + - (reply == null ? "" : " (sync)"); + void addToSummary(Event e) { + summarizer.summarizeEvent(e); } - } - - private static class FlushPayload { - final Event[] events; - final EventSummary summary; - FlushPayload(Event[] events, EventSummary summary) { - this.events = events; - this.summary = summary; + boolean isEmpty() { + return events.isEmpty() && summarizer.snapshot().isEmpty(); } - } - - private static interface EventOutput { } - - @SuppressWarnings("unused") - private static class FeatureRequestEventOutput implements EventOutput { - private final String kind; - private final long creationDate; - private final String key; - private final String userKey; - private final LDUser user; - private final Integer version; - private final JsonElement value; - @SerializedName("default") private final JsonElement defaultVal; - private final String prereqOf; - FeatureRequestEventOutput(long creationDate, String key, String userKey, LDUser user, - Integer version, JsonElement value, JsonElement defaultVal, String prereqOf, boolean debug) { - this.kind = debug ? "debug" : "feature"; - this.creationDate = creationDate; - this.key = key; - this.userKey = userKey; - this.user = user; - this.version = version; - this.value = value; - this.defaultVal = defaultVal; - this.prereqOf = prereqOf; + FlushPayload getPayload() { + Event[] eventsOut = events.toArray(new Event[events.size()]); + EventSummarizer.EventSummary summary = summarizer.snapshot(); + return new FlushPayload(eventsOut, summary); } - } - - @SuppressWarnings("unused") - private static class IdentifyEventOutput extends Event implements EventOutput { - private final String kind; - private final String key; - IdentifyEventOutput(long creationDate, LDUser user) { - super(creationDate, user); - this.kind = "identify"; - this.key = user.getKeyAsString(); + void clear() { + events.clear(); + summarizer.clear(); } } - @SuppressWarnings("unused") - private static class CustomEventOutput implements EventOutput { - private final String kind; - private final long creationDate; - private final String key; - private final String userKey; - private final LDUser user; - private final JsonElement data; + private static final class FlushPayload { + final Event[] events; + final EventSummary summary; - CustomEventOutput(long creationDate, String key, String userKey, LDUser user, JsonElement data) { - this.kind = "custom"; - this.creationDate = creationDate; - this.key = key; - this.userKey = userKey; - this.user = user; - this.data = data; + FlushPayload(Event[] events, EventSummary summary) { + this.events = events; + this.summary = summary; } } - private static class IndexEvent extends Event { - IndexEvent(long creationDate, LDUser user) { - super(creationDate, user); - } + private static interface EventResponseListener { + void handleResponse(Response response); } - @SuppressWarnings("unused") - private static class IndexEventOutput implements EventOutput { - private final String kind; - private final long creationDate; - private final LDUser user; + private static final class SendEventsTask implements Runnable { + private final String sdkKey; + private final LDConfig config; + private final EventResponseListener responseListener; + private final BlockingQueue payloadQueue; + private final AtomicInteger activeFlushWorkersCount; + private final AtomicBoolean stopping; + private final EventOutput.Formatter formatter; + private final Thread thread; - public IndexEventOutput(long creationDate, LDUser user) { - this.kind = "index"; - this.creationDate = creationDate; - this.user = user; + SendEventsTask(String sdkKey, LDConfig config, EventResponseListener responseListener, + BlockingQueue payloadQueue, AtomicInteger activeFlushWorkersCount, + ThreadFactory threadFactory) { + this.sdkKey = sdkKey; + this.config = config; + this.formatter = new EventOutput.Formatter(config.inlineUsersInEvents); + this.responseListener = responseListener; + this.payloadQueue = payloadQueue; + this.activeFlushWorkersCount = activeFlushWorkersCount; + this.stopping = new AtomicBoolean(false); + thread = threadFactory.newThread(this); + thread.setDaemon(true); + thread.start(); } - } - - @SuppressWarnings("unused") - private static class SummaryEventOutput implements EventOutput { - private final String kind; - private final long startDate; - private final long endDate; - private final Map features; - SummaryEventOutput(long startDate, long endDate, Map features) { - this.kind = "summary"; - this.startDate = startDate; - this.endDate = endDate; - this.features = features; + public void run() { + while (!stopping.get()) { + FlushPayload payload = null; + try { + payload = payloadQueue.take(); + } catch (InterruptedException e) { + continue; + } + try { + List eventsOut = formatter.makeOutputEvents(payload.events, payload.summary); + if (!eventsOut.isEmpty()) { + postEvents(eventsOut); + } + } catch (Exception e) { + logger.error("Unexpected error in event processor: " + e); + logger.debug(e.getMessage(), e); + } + synchronized (activeFlushWorkersCount) { + activeFlushWorkersCount.decrementAndGet(); + activeFlushWorkersCount.notifyAll(); + } + } } - } - - private static class SummaryEventFlag { - @SerializedName("default") final JsonElement defaultVal; - final List counters; - SummaryEventFlag(JsonElement defaultVal, List counters) { - this.defaultVal = defaultVal; - this.counters = counters; + void stop() { + stopping.set(true); + thread.interrupt(); } - } - - @SuppressWarnings("unused") - private static class SummaryEventCounter { - final JsonElement value; - final Integer version; - final int count; - final Boolean unknown; - SummaryEventCounter(JsonElement value, Integer version, int count, Boolean unknown) { - this.value = value; - this.version = version; - this.count = count; - this.unknown = unknown; + private void postEvents(List eventsOut) { + String json = config.gson.toJson(eventsOut); + logger.debug("Posting {} event(s) to {} with payload: {}", + eventsOut.size(), config.eventsURI, json); + + Request request = config.getRequestBuilder(sdkKey) + .url(config.eventsURI.toString() + "/bulk") + .post(RequestBody.create(MediaType.parse("application/json; charset=utf-8"), json)) + .addHeader("Content-Type", "application/json") + .build(); + + long startTime = System.currentTimeMillis(); + try (Response response = config.httpClient.newCall(request).execute()) { + long endTime = System.currentTimeMillis(); + logger.debug("Event delivery took {} ms, response status {}", endTime - startTime, response.code()); + responseListener.handleResponse(response); + } catch (IOException e) { + logger.info("Unhandled exception in LaunchDarkly client when posting events to URL: " + request.url(), e); + } } } } diff --git a/src/main/java/com/launchdarkly/client/Event.java b/src/main/java/com/launchdarkly/client/Event.java index 9e9ecfdac..7d429554c 100644 --- a/src/main/java/com/launchdarkly/client/Event.java +++ b/src/main/java/com/launchdarkly/client/Event.java @@ -1,5 +1,10 @@ package com.launchdarkly.client; +import com.google.gson.JsonElement; + +/** + * Base class for all analytics events that are generated by the client. Also defines all of its own subclasses. + */ class Event { final long creationDate; final LDUser user; @@ -8,4 +13,52 @@ class Event { this.creationDate = creationDate; this.user = user; } + + static final class Custom extends Event { + final String key; + final JsonElement data; + + Custom(long timestamp, String key, LDUser user, JsonElement data) { + super(timestamp, user); + this.key = key; + this.data = data; + } + } + + static final class Identify extends Event { + Identify(long timestamp, LDUser user) { + super(timestamp, user); + } + } + + static final class Index extends Event { + Index(long timestamp, LDUser user) { + super(timestamp, user); + } + } + + static final class FeatureRequest extends Event { + final String key; + final Integer variation; + final JsonElement value; + final JsonElement defaultVal; + final Integer version; + final String prereqOf; + final boolean trackEvents; + final Long debugEventsUntilDate; + + FeatureRequest(long timestamp, String key, LDUser user, Integer version, Integer variation, JsonElement value, + JsonElement defaultVal, String prereqOf, boolean trackEvents, Long debugEventsUntilDate) { + super(timestamp, user); + this.key = key; + this.version = version; + this.variation = variation; + this.value = value; + this.defaultVal = defaultVal; + this.prereqOf = prereqOf; + this.trackEvents = trackEvents; + this.debugEventsUntilDate = debugEventsUntilDate; + } + } + } \ No newline at end of file diff --git a/src/main/java/com/launchdarkly/client/EventFactory.java b/src/main/java/com/launchdarkly/client/EventFactory.java index 85b581858..a5e6aa036 100644 --- a/src/main/java/com/launchdarkly/client/EventFactory.java +++ b/src/main/java/com/launchdarkly/client/EventFactory.java @@ -7,29 +7,29 @@ abstract class EventFactory { protected abstract long getTimestamp(); - public FeatureRequestEvent newFeatureRequestEvent(FeatureFlag flag, LDUser user, FeatureFlag.VariationAndValue result, JsonElement defaultVal) { - return new FeatureRequestEvent(getTimestamp(), flag.getKey(), user, flag.getVersion(), + public Event.FeatureRequest newFeatureRequestEvent(FeatureFlag flag, LDUser user, FeatureFlag.VariationAndValue result, JsonElement defaultVal) { + return new Event.FeatureRequest(getTimestamp(), flag.getKey(), user, flag.getVersion(), result == null ? null : result.getVariation(), result == null ? null : result.getValue(), defaultVal, null, flag.isTrackEvents(), flag.getDebugEventsUntilDate()); } - public FeatureRequestEvent newUnknownFeatureRequestEvent(String key, LDUser user, JsonElement defaultValue) { - return new FeatureRequestEvent(getTimestamp(), key, user, null, null, defaultValue, defaultValue, null, false, null); + public Event.FeatureRequest newUnknownFeatureRequestEvent(String key, LDUser user, JsonElement defaultValue) { + return new Event.FeatureRequest(getTimestamp(), key, user, null, null, defaultValue, defaultValue, null, false, null); } - public FeatureRequestEvent newPrerequisiteFeatureRequestEvent(FeatureFlag prereqFlag, LDUser user, FeatureFlag.VariationAndValue result, + public Event.FeatureRequest newPrerequisiteFeatureRequestEvent(FeatureFlag prereqFlag, LDUser user, FeatureFlag.VariationAndValue result, FeatureFlag prereqOf) { - return new FeatureRequestEvent(getTimestamp(), prereqFlag.getKey(), user, prereqFlag.getVersion(), + return new Event.FeatureRequest(getTimestamp(), prereqFlag.getKey(), user, prereqFlag.getVersion(), result == null ? null : result.getVariation(), result == null ? null : result.getValue(), null, prereqOf.getKey(), prereqFlag.isTrackEvents(), prereqFlag.getDebugEventsUntilDate()); } - public CustomEvent newCustomEvent(String key, LDUser user, JsonElement data) { - return new CustomEvent(getTimestamp(), key, user, data); + public Event.Custom newCustomEvent(String key, LDUser user, JsonElement data) { + return new Event.Custom(getTimestamp(), key, user, data); } - public IdentifyEvent newIdentifyEvent(LDUser user) { - return new IdentifyEvent(getTimestamp(), user); + public Event.Identify newIdentifyEvent(LDUser user) { + return new Event.Identify(getTimestamp(), user); } public static class DefaultEventFactory extends EventFactory { diff --git a/src/main/java/com/launchdarkly/client/EventOutput.java b/src/main/java/com/launchdarkly/client/EventOutput.java new file mode 100644 index 000000000..67468ed7b --- /dev/null +++ b/src/main/java/com/launchdarkly/client/EventOutput.java @@ -0,0 +1,195 @@ +package com.launchdarkly.client; + +import com.google.gson.JsonElement; +import com.google.gson.annotations.SerializedName; +import com.launchdarkly.client.EventSummarizer.CounterKey; +import com.launchdarkly.client.EventSummarizer.CounterValue; + +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +/** + * Subclass for data structures that we send in an event payload. Also defines all of its own + * subclasses and the class that constructs them. + */ +abstract class EventOutput { + @SuppressWarnings("unused") + private final String kind; + + protected EventOutput(String kind) { + this.kind = kind; + } + + static class EventOutputWithTimestamp extends EventOutput { + @SuppressWarnings("unused") + private final long creationDate; + + protected EventOutputWithTimestamp(String kind, long creationDate) { + super(kind); + this.creationDate = creationDate; + } + } + + @SuppressWarnings("unused") + static final class FeatureRequest extends EventOutputWithTimestamp { + private final String key; + private final String userKey; + private final LDUser user; + private final Integer version; + private final JsonElement value; + @SerializedName("default") private final JsonElement defaultVal; + private final String prereqOf; + + FeatureRequest(long creationDate, String key, String userKey, LDUser user, + Integer version, JsonElement value, JsonElement defaultVal, String prereqOf, boolean debug) { + super(debug ? "debug" : "feature", creationDate); + this.key = key; + this.userKey = userKey; + this.user = user; + this.version = version; + this.value = value; + this.defaultVal = defaultVal; + this.prereqOf = prereqOf; + } + } + + @SuppressWarnings("unused") + static final class Identify extends EventOutputWithTimestamp { + private final LDUser user; + private final String key; + + Identify(long creationDate, LDUser user) { + super("identify", creationDate); + this.user = user; + this.key = user.getKeyAsString(); + } + } + + @SuppressWarnings("unused") + static final class Custom extends EventOutputWithTimestamp { + private final String key; + private final String userKey; + private final LDUser user; + private final JsonElement data; + + Custom(long creationDate, String key, String userKey, LDUser user, JsonElement data) { + super("custom", creationDate); + this.key = key; + this.userKey = userKey; + this.user = user; + this.data = data; + } + } + + @SuppressWarnings("unused") + static final class Index extends EventOutputWithTimestamp { + private final LDUser user; + + public Index(long creationDate, LDUser user) { + super("index", creationDate); + this.user = user; + } + } + + @SuppressWarnings("unused") + static final class Summary extends EventOutput { + private final long startDate; + private final long endDate; + private final Map features; + + Summary(long startDate, long endDate, Map features) { + super("summary"); + this.startDate = startDate; + this.endDate = endDate; + this.features = features; + } + } + + static final class SummaryEventFlag { + @SerializedName("default") final JsonElement defaultVal; + final List counters; + + @SuppressWarnings("unused") + SummaryEventFlag(JsonElement defaultVal, List counters) { + this.defaultVal = defaultVal; + this.counters = counters; + } + } + + @SuppressWarnings("unused") + static final class SummaryEventCounter { + final JsonElement value; + final Integer version; + final int count; + final Boolean unknown; + + SummaryEventCounter(JsonElement value, Integer version, int count, Boolean unknown) { + this.value = value; + this.version = version; + this.count = count; + this.unknown = unknown; + } + } + + static final class Formatter { + private final boolean inlineUsers; + + Formatter(boolean inlineUsers) { + this.inlineUsers = inlineUsers; + } + + List makeOutputEvents(Event[] events, EventSummarizer.EventSummary summary) { + List eventsOut = new ArrayList<>(events.length + 1); + for (Event event: events) { + eventsOut.add(createEventOutput(event)); + } + if (!summary.isEmpty()) { + eventsOut.add(createSummaryEvent(summary)); + } + return eventsOut; + } + + private EventOutput createEventOutput(Event e) { + String userKey = e.user == null ? null : e.user.getKeyAsString(); + if (e instanceof Event.FeatureRequest) { + Event.FeatureRequest fe = (Event.FeatureRequest)e; + boolean isDebug = (!fe.trackEvents && fe.debugEventsUntilDate != null); + return new EventOutput.FeatureRequest(fe.creationDate, fe.key, + inlineUsers ? null : userKey, + inlineUsers ? e.user : null, + fe.version, fe.value, fe.defaultVal, fe.prereqOf, isDebug); + } else if (e instanceof Event.Identify) { + return new EventOutput.Identify(e.creationDate, e.user); + } else if (e instanceof Event.Custom) { + Event.Custom ce = (Event.Custom)e; + return new EventOutput.Custom(ce.creationDate, ce.key, + inlineUsers ? null : userKey, + inlineUsers ? e.user : null, + ce.data); + } else if (e instanceof Event.Index) { + return new EventOutput.Index(e.creationDate, e.user); + } else { + return null; + } + } + + private EventOutput createSummaryEvent(EventSummarizer.EventSummary summary) { + Map flagsOut = new HashMap<>(); + for (Map.Entry entry: summary.counters.entrySet()) { + SummaryEventFlag fsd = flagsOut.get(entry.getKey().key); + if (fsd == null) { + fsd = new SummaryEventFlag(entry.getValue().defaultVal, new ArrayList()); + flagsOut.put(entry.getKey().key, fsd); + } + SummaryEventCounter c = new SummaryEventCounter(entry.getValue().flagValue, + entry.getKey().version, + entry.getValue().count, + entry.getKey().version == null ? true : null); + fsd.counters.add(c); + } + return new EventOutput.Summary(summary.startDate, summary.endDate, flagsOut); + } + } +} diff --git a/src/main/java/com/launchdarkly/client/EventSummarizer.java b/src/main/java/com/launchdarkly/client/EventSummarizer.java index d605204dd..97fb10daa 100644 --- a/src/main/java/com/launchdarkly/client/EventSummarizer.java +++ b/src/main/java/com/launchdarkly/client/EventSummarizer.java @@ -11,7 +11,7 @@ * methods of this class are deliberately not thread-safe, because they should always * be called from EventProcessor's single message-processing thread. */ -class EventSummarizer { +final class EventSummarizer { private EventSummary eventsState; EventSummarizer() { @@ -23,8 +23,8 @@ class EventSummarizer { * @param event an event */ void summarizeEvent(Event event) { - if (event instanceof FeatureRequestEvent) { - FeatureRequestEvent fe = (FeatureRequestEvent)event; + if (event instanceof Event.FeatureRequest) { + Event.FeatureRequest fe = (Event.FeatureRequest)event; eventsState.incrementCounter(fe.key, fe.variation, fe.version, fe.value, fe.defaultVal); eventsState.noteTimestamp(fe.creationDate); } @@ -45,7 +45,7 @@ void clear() { eventsState = new EventSummary(); } - static class EventSummary { + static final class EventSummary { final Map counters; long startDate; long endDate; @@ -99,7 +99,7 @@ public int hashCode() { } } - static class CounterKey { + static final class CounterKey { final String key; final Integer variation; final Integer version; @@ -131,7 +131,7 @@ public String toString() { } } - static class CounterValue { + static final class CounterValue { int count; final JsonElement flagValue; final JsonElement defaultVal; diff --git a/src/main/java/com/launchdarkly/client/FeatureFlag.java b/src/main/java/com/launchdarkly/client/FeatureFlag.java index f0738efb4..faff33c6b 100644 --- a/src/main/java/com/launchdarkly/client/FeatureFlag.java +++ b/src/main/java/com/launchdarkly/client/FeatureFlag.java @@ -62,7 +62,7 @@ static Map fromJsonMap(LDConfig config, String json) { } EvalResult evaluate(LDUser user, FeatureStore featureStore, EventFactory eventFactory) throws EvaluationException { - List prereqEvents = new ArrayList<>(); + List prereqEvents = new ArrayList<>(); if (user == null || user.getKey() == null) { logger.warn("Null user or null user key when evaluating flag: " + key + "; returning null"); @@ -79,7 +79,7 @@ EvalResult evaluate(LDUser user, FeatureStore featureStore, EventFactory eventFa } // Returning either a JsonElement or null indicating prereq failure/error. - private VariationAndValue evaluate(LDUser user, FeatureStore featureStore, List events, + private VariationAndValue evaluate(LDUser user, FeatureStore featureStore, List events, EventFactory eventFactory) throws EvaluationException { boolean prereqOk = true; if (prerequisites != null) { @@ -229,9 +229,9 @@ JsonElement getValue() { static class EvalResult { private final VariationAndValue result; - private final List prerequisiteEvents; + private final List prerequisiteEvents; - private EvalResult(VariationAndValue result, List prerequisiteEvents) { + private EvalResult(VariationAndValue result, List prerequisiteEvents) { this.result = result; this.prerequisiteEvents = prerequisiteEvents; } @@ -240,7 +240,7 @@ VariationAndValue getResult() { return result; } - List getPrerequisiteEvents() { + List getPrerequisiteEvents() { return prerequisiteEvents; } } diff --git a/src/main/java/com/launchdarkly/client/FeatureRequestEvent.java b/src/main/java/com/launchdarkly/client/FeatureRequestEvent.java deleted file mode 100644 index b1a640631..000000000 --- a/src/main/java/com/launchdarkly/client/FeatureRequestEvent.java +++ /dev/null @@ -1,27 +0,0 @@ -package com.launchdarkly.client; - -import com.google.gson.JsonElement; - -class FeatureRequestEvent extends Event { - final String key; - final Integer variation; - final JsonElement value; - final JsonElement defaultVal; - final Integer version; - final String prereqOf; - final boolean trackEvents; - final Long debugEventsUntilDate; - - FeatureRequestEvent(long timestamp, String key, LDUser user, Integer version, Integer variation, JsonElement value, - JsonElement defaultVal, String prereqOf, boolean trackEvents, Long debugEventsUntilDate) { - super(timestamp, user); - this.key = key; - this.version = version; - this.variation = variation; - this.value = value; - this.defaultVal = defaultVal; - this.prereqOf = prereqOf; - this.trackEvents = trackEvents; - this.debugEventsUntilDate = debugEventsUntilDate; - } -} diff --git a/src/main/java/com/launchdarkly/client/IdentifyEvent.java b/src/main/java/com/launchdarkly/client/IdentifyEvent.java deleted file mode 100644 index 4b2c8d1c5..000000000 --- a/src/main/java/com/launchdarkly/client/IdentifyEvent.java +++ /dev/null @@ -1,8 +0,0 @@ -package com.launchdarkly.client; - -class IdentifyEvent extends Event { - - IdentifyEvent(long timestamp, LDUser user) { - super(timestamp, user); - } -} diff --git a/src/main/java/com/launchdarkly/client/LDClient.java b/src/main/java/com/launchdarkly/client/LDClient.java index a8f75952b..1567eaf61 100644 --- a/src/main/java/com/launchdarkly/client/LDClient.java +++ b/src/main/java/com/launchdarkly/client/LDClient.java @@ -153,7 +153,7 @@ public void identify(LDUser user) { eventProcessor.sendEvent(eventFactory.newIdentifyEvent(user)); } - private void sendFlagRequestEvent(FeatureRequestEvent event) { + private void sendFlagRequestEvent(Event.FeatureRequest event) { eventProcessor.sendEvent(event); NewRelicReflector.annotateTransaction(event.key, String.valueOf(event.value)); } @@ -272,7 +272,7 @@ private JsonElement evaluate(String featureKey, LDUser user, JsonElement default return defaultValue; } FeatureFlag.EvalResult evalResult = featureFlag.evaluate(user, config.featureStore, eventFactory); - for (FeatureRequestEvent event : evalResult.getPrerequisiteEvents()) { + for (Event.FeatureRequest event : evalResult.getPrerequisiteEvents()) { eventProcessor.sendEvent(event); } if (evalResult.getResult() != null && evalResult.getResult().getValue() != null) { diff --git a/src/test/java/com/launchdarkly/client/DefaultEventProcessorTest.java b/src/test/java/com/launchdarkly/client/DefaultEventProcessorTest.java index a82d8d4d1..e39d6ee86 100644 --- a/src/test/java/com/launchdarkly/client/DefaultEventProcessorTest.java +++ b/src/test/java/com/launchdarkly/client/DefaultEventProcessorTest.java @@ -84,7 +84,7 @@ public void userIsFilteredInIdentifyEvent() throws Exception { public void individualFeatureEventIsQueuedWithIndexEvent() throws Exception { ep = new DefaultEventProcessor(SDK_KEY, configBuilder.build()); FeatureFlag flag = new FeatureFlagBuilder("flagkey").version(11).trackEvents(true).build(); - FeatureRequestEvent fe = EventFactory.DEFAULT.newFeatureRequestEvent(flag, user, + Event.FeatureRequest fe = EventFactory.DEFAULT.newFeatureRequestEvent(flag, user, new FeatureFlag.VariationAndValue(new Integer(1), new JsonPrimitive("value")), null); ep.sendEvent(fe); @@ -102,7 +102,7 @@ public void userIsFilteredInIndexEvent() throws Exception { configBuilder.allAttributesPrivate(true); ep = new DefaultEventProcessor(SDK_KEY, configBuilder.build()); FeatureFlag flag = new FeatureFlagBuilder("flagkey").version(11).trackEvents(true).build(); - FeatureRequestEvent fe = EventFactory.DEFAULT.newFeatureRequestEvent(flag, user, + Event.FeatureRequest fe = EventFactory.DEFAULT.newFeatureRequestEvent(flag, user, new FeatureFlag.VariationAndValue(new Integer(1), new JsonPrimitive("value")), null); ep.sendEvent(fe); @@ -120,7 +120,7 @@ public void featureEventCanContainInlineUser() throws Exception { configBuilder.inlineUsersInEvents(true); ep = new DefaultEventProcessor(SDK_KEY, configBuilder.build()); FeatureFlag flag = new FeatureFlagBuilder("flagkey").version(11).trackEvents(true).build(); - FeatureRequestEvent fe = EventFactory.DEFAULT.newFeatureRequestEvent(flag, user, + Event.FeatureRequest fe = EventFactory.DEFAULT.newFeatureRequestEvent(flag, user, new FeatureFlag.VariationAndValue(new Integer(1), new JsonPrimitive("value")), null); ep.sendEvent(fe); @@ -137,7 +137,7 @@ public void userIsFilteredInFeatureEvent() throws Exception { configBuilder.inlineUsersInEvents(true).allAttributesPrivate(true); ep = new DefaultEventProcessor(SDK_KEY, configBuilder.build()); FeatureFlag flag = new FeatureFlagBuilder("flagkey").version(11).trackEvents(true).build(); - FeatureRequestEvent fe = EventFactory.DEFAULT.newFeatureRequestEvent(flag, user, + Event.FeatureRequest fe = EventFactory.DEFAULT.newFeatureRequestEvent(flag, user, new FeatureFlag.VariationAndValue(new Integer(1), new JsonPrimitive("value")), null); ep.sendEvent(fe); @@ -154,7 +154,7 @@ public void eventKindIsDebugIfFlagIsTemporarilyInDebugMode() throws Exception { ep = new DefaultEventProcessor(SDK_KEY, configBuilder.build()); long futureTime = System.currentTimeMillis() + 1000000; FeatureFlag flag = new FeatureFlagBuilder("flagkey").version(11).debugEventsUntilDate(futureTime).build(); - FeatureRequestEvent fe = EventFactory.DEFAULT.newFeatureRequestEvent(flag, user, + Event.FeatureRequest fe = EventFactory.DEFAULT.newFeatureRequestEvent(flag, user, new FeatureFlag.VariationAndValue(new Integer(1), new JsonPrimitive("value")), null); ep.sendEvent(fe); @@ -182,7 +182,7 @@ public void debugModeExpiresBasedOnClientTimeIfClientTimeIsLaterThanServerTime() // the future than the server time, but in the past compared to the client. long debugUntil = serverTime + 1000; FeatureFlag flag = new FeatureFlagBuilder("flagkey").version(11).debugEventsUntilDate(debugUntil).build(); - FeatureRequestEvent fe = EventFactory.DEFAULT.newFeatureRequestEvent(flag, user, + Event.FeatureRequest fe = EventFactory.DEFAULT.newFeatureRequestEvent(flag, user, new FeatureFlag.VariationAndValue(new Integer(1), new JsonPrimitive("value")), null); ep.sendEvent(fe); @@ -210,7 +210,7 @@ public void debugModeExpiresBasedOnServerTimeIfServerTimeIsLaterThanClientTime() // the future than the client time, but in the past compared to the server. long debugUntil = serverTime - 1000; FeatureFlag flag = new FeatureFlagBuilder("flagkey").version(11).debugEventsUntilDate(debugUntil).build(); - FeatureRequestEvent fe = EventFactory.DEFAULT.newFeatureRequestEvent(flag, user, + Event.FeatureRequest fe = EventFactory.DEFAULT.newFeatureRequestEvent(flag, user, new FeatureFlag.VariationAndValue(new Integer(1), new JsonPrimitive("value")), null); ep.sendEvent(fe); @@ -229,9 +229,9 @@ public void twoFeatureEventsForSameUserGenerateOnlyOneIndexEvent() throws Except FeatureFlag flag1 = new FeatureFlagBuilder("flagkey1").version(11).trackEvents(true).build(); FeatureFlag flag2 = new FeatureFlagBuilder("flagkey2").version(22).trackEvents(true).build(); JsonElement value = new JsonPrimitive("value"); - FeatureRequestEvent fe1 = EventFactory.DEFAULT.newFeatureRequestEvent(flag1, user, + Event.FeatureRequest fe1 = EventFactory.DEFAULT.newFeatureRequestEvent(flag1, user, new FeatureFlag.VariationAndValue(new Integer(1), value), null); - FeatureRequestEvent fe2 = EventFactory.DEFAULT.newFeatureRequestEvent(flag2, user, + Event.FeatureRequest fe2 = EventFactory.DEFAULT.newFeatureRequestEvent(flag2, user, new FeatureFlag.VariationAndValue(new Integer(1), value), null); ep.sendEvent(fe1); ep.sendEvent(fe2); @@ -280,7 +280,7 @@ public void customEventIsQueuedWithUser() throws Exception { ep = new DefaultEventProcessor(SDK_KEY, configBuilder.build()); JsonObject data = new JsonObject(); data.addProperty("thing", "stuff"); - CustomEvent ce = EventFactory.DEFAULT.newCustomEvent("eventkey", user, data); + Event.Custom ce = EventFactory.DEFAULT.newCustomEvent("eventkey", user, data); ep.sendEvent(ce); JsonArray output = flushAndGetEvents(new MockResponse()); @@ -297,7 +297,7 @@ public void customEventCanContainInlineUser() throws Exception { ep = new DefaultEventProcessor(SDK_KEY, configBuilder.build()); JsonObject data = new JsonObject(); data.addProperty("thing", "stuff"); - CustomEvent ce = EventFactory.DEFAULT.newCustomEvent("eventkey", user, data); + Event.Custom ce = EventFactory.DEFAULT.newCustomEvent("eventkey", user, data); ep.sendEvent(ce); JsonArray output = flushAndGetEvents(new MockResponse()); @@ -311,7 +311,7 @@ public void userIsFilteredInCustomEvent() throws Exception { ep = new DefaultEventProcessor(SDK_KEY, configBuilder.build()); JsonObject data = new JsonObject(); data.addProperty("thing", "stuff"); - CustomEvent ce = EventFactory.DEFAULT.newCustomEvent("eventkey", user, data); + Event.Custom ce = EventFactory.DEFAULT.newCustomEvent("eventkey", user, data); ep.sendEvent(ce); JsonArray output = flushAndGetEvents(new MockResponse()); @@ -400,7 +400,7 @@ private Matcher isIndexEvent(Event sourceEvent, JsonElement user) { } @SuppressWarnings("unchecked") - private Matcher isFeatureEvent(FeatureRequestEvent sourceEvent, FeatureFlag flag, boolean debug, JsonElement inlineUser) { + private Matcher isFeatureEvent(Event.FeatureRequest sourceEvent, FeatureFlag flag, boolean debug, JsonElement inlineUser) { return allOf( hasJsonProperty("kind", debug ? "debug" : "feature"), hasJsonProperty("creationDate", (double)sourceEvent.creationDate), @@ -414,7 +414,7 @@ private Matcher isFeatureEvent(FeatureRequestEvent sourceEvent, Fea ); } - private Matcher isCustomEvent(CustomEvent sourceEvent, JsonElement inlineUser) { + private Matcher isCustomEvent(Event.Custom sourceEvent, JsonElement inlineUser) { return allOf( hasJsonProperty("kind", "custom"), hasJsonProperty("creationDate", (double)sourceEvent.creationDate), diff --git a/src/test/java/com/launchdarkly/client/FeatureFlagTest.java b/src/test/java/com/launchdarkly/client/FeatureFlagTest.java index f32c754be..70bc94c9f 100644 --- a/src/test/java/com/launchdarkly/client/FeatureFlagTest.java +++ b/src/test/java/com/launchdarkly/client/FeatureFlagTest.java @@ -94,7 +94,7 @@ public void flagReturnsOffVariationAndEventIfPrerequisiteIsNotMet() throws Excep assertEquals(js("off"), result.getResult().getValue()); assertEquals(1, result.getPrerequisiteEvents().size()); - FeatureRequestEvent event = result.getPrerequisiteEvents().get(0); + Event.FeatureRequest event = result.getPrerequisiteEvents().get(0); assertEquals(f1.getKey(), event.key); assertEquals(js("nogo"), event.value); assertEquals(f1.getVersion(), event.version.intValue()); @@ -123,7 +123,7 @@ public void flagReturnsFallthroughVariationAndEventIfPrerequisiteIsMetAndThereAr assertEquals(js("fall"), result.getResult().getValue()); assertEquals(1, result.getPrerequisiteEvents().size()); - FeatureRequestEvent event = result.getPrerequisiteEvents().get(0); + Event.FeatureRequest event = result.getPrerequisiteEvents().get(0); assertEquals(f1.getKey(), event.key); assertEquals(js("go"), event.value); assertEquals(f1.getVersion(), event.version.intValue()); @@ -160,13 +160,13 @@ public void multipleLevelsOfPrerequisitesProduceMultipleEvents() throws Exceptio assertEquals(js("fall"), result.getResult().getValue()); assertEquals(2, result.getPrerequisiteEvents().size()); - FeatureRequestEvent event0 = result.getPrerequisiteEvents().get(0); + Event.FeatureRequest event0 = result.getPrerequisiteEvents().get(0); assertEquals(f2.getKey(), event0.key); assertEquals(js("go"), event0.value); assertEquals(f2.getVersion(), event0.version.intValue()); assertEquals(f1.getKey(), event0.prereqOf); - FeatureRequestEvent event1 = result.getPrerequisiteEvents().get(1); + Event.FeatureRequest event1 = result.getPrerequisiteEvents().get(1); assertEquals(f1.getKey(), event1.key); assertEquals(js("go"), event1.value); assertEquals(f1.getVersion(), event1.version.intValue()); From 99ed6858603b1600117ccdb128b6b22b418eb986 Mon Sep 17 00:00:00 2001 From: Eli Bishop Date: Tue, 3 Apr 2018 13:51:26 -0700 Subject: [PATCH 48/67] misc cleanup --- .../client/DefaultEventProcessor.java | 31 ++++++++++--------- 1 file changed, 16 insertions(+), 15 deletions(-) diff --git a/src/main/java/com/launchdarkly/client/DefaultEventProcessor.java b/src/main/java/com/launchdarkly/client/DefaultEventProcessor.java index d3f1ae5f6..53aadb8e0 100644 --- a/src/main/java/com/launchdarkly/client/DefaultEventProcessor.java +++ b/src/main/java/com/launchdarkly/client/DefaultEventProcessor.java @@ -90,7 +90,6 @@ public void close() throws IOException { @VisibleForTesting void waitUntilInactive() throws IOException { - // Waits until there are no pending events or flushes postMessageAndWait(MessageType.SYNC, null); } @@ -162,7 +161,7 @@ void waitForCompletion() { } @Override - public String toString() { + public String toString() { // for debugging only return ((event == null) ? type.toString() : (type + ": " + event.getClass().getSimpleName())) + (reply == null ? "" : " (sync)"); } @@ -177,7 +176,7 @@ private static final class EventDispatcher { private final LDConfig config; private final List flushWorkers; - private final AtomicInteger activeFlushWorkersCount; + private final AtomicInteger busyFlushWorkersCount; private final Random random = new Random(); private final AtomicLong lastKnownPastTime = new AtomicLong(0); private final AtomicBoolean disabled = new AtomicBoolean(false); @@ -186,7 +185,7 @@ private EventDispatcher(String sdkKey, LDConfig config, final BlockingQueue inputChannel, ThreadFactory threadFactory) { this.config = config; - this.activeFlushWorkersCount = new AtomicInteger(0); + this.busyFlushWorkersCount = new AtomicInteger(0); // This queue only holds one element; it represents a flush task that has not yet been // picked up by any worker, so if we try to push another one and are refused, it means @@ -212,7 +211,7 @@ public void handleResponse(Response response) { }; for (int i = 0; i < MAX_FLUSH_THREADS; i++) { SendEventsTask task = new SendEventsTask(sdkKey, config, listener, payloadQueue, - activeFlushWorkersCount, threadFactory); + busyFlushWorkersCount, threadFactory); flushWorkers.add(task); } } @@ -220,7 +219,7 @@ public void handleResponse(Response response) { /** * This task drains the input queue as quickly as possible. Everything here is done on a single * thread so we don't have to synchronize on our internal structures; when it's time to flush, - * dispatchFlush will fire off another task to do the part that takes longer. + * triggerFlush will hand the events off to another task. */ private void runMainLoop(BlockingQueue inputChannel, EventBuffer buffer, SimpleLRUCache userKeys, @@ -269,11 +268,11 @@ private void doShutdown() { private void waitUntilAllFlushWorkersInactive() { while (true) { try { - synchronized(activeFlushWorkersCount) { - if (activeFlushWorkersCount.get() == 0) { + synchronized(busyFlushWorkersCount) { + if (busyFlushWorkersCount.get() == 0) { return; } else { - activeFlushWorkersCount.wait(); + busyFlushWorkersCount.wait(); } } } catch (InterruptedException e) {} @@ -345,16 +344,16 @@ private void triggerFlush(EventBuffer buffer, BlockingQueue payloa return; } FlushPayload payload = buffer.getPayload(); - activeFlushWorkersCount.incrementAndGet(); + busyFlushWorkersCount.incrementAndGet(); if (payloadQueue.offer(payload)) { // These events now belong to the next available flush worker, so drop them from our state buffer.clear(); } else { logger.debug("Skipped flushing because all workers are busy"); // All the workers are busy so we can't flush now; keep the events in our state - synchronized(activeFlushWorkersCount) { - activeFlushWorkersCount.decrementAndGet(); - activeFlushWorkersCount.notify(); + synchronized(busyFlushWorkersCount) { + busyFlushWorkersCount.decrementAndGet(); + busyFlushWorkersCount.notify(); } } } @@ -489,11 +488,13 @@ void stop() { private void postEvents(List eventsOut) { String json = config.gson.toJson(eventsOut); + String uriStr = config.eventsURI.toString() + "/bulk"; + logger.debug("Posting {} event(s) to {} with payload: {}", - eventsOut.size(), config.eventsURI, json); + eventsOut.size(), uriStr, json); Request request = config.getRequestBuilder(sdkKey) - .url(config.eventsURI.toString() + "/bulk") + .url(uriStr) .post(RequestBody.create(MediaType.parse("application/json; charset=utf-8"), json)) .addHeader("Content-Type", "application/json") .build(); From 238c9dfa5879a294541ec17dc644b028d08522bc Mon Sep 17 00:00:00 2001 From: Eli Bishop Date: Tue, 3 Apr 2018 14:21:45 -0700 Subject: [PATCH 49/67] comment --- src/main/java/com/launchdarkly/client/EventOutput.java | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/src/main/java/com/launchdarkly/client/EventOutput.java b/src/main/java/com/launchdarkly/client/EventOutput.java index 67468ed7b..35754a97d 100644 --- a/src/main/java/com/launchdarkly/client/EventOutput.java +++ b/src/main/java/com/launchdarkly/client/EventOutput.java @@ -11,8 +11,9 @@ import java.util.Map; /** - * Subclass for data structures that we send in an event payload. Also defines all of its own - * subclasses and the class that constructs them. + * Base class for data structures that we send in an event payload, which are somewhat + * different in shape from the originating events. Also defines all of its own subclasses + * and the class that constructs them. */ abstract class EventOutput { @SuppressWarnings("unused") From 65f7b64f21b42b6cefa2f0e022335caf10bd7513 Mon Sep 17 00:00:00 2001 From: Eli Bishop Date: Tue, 3 Apr 2018 14:27:03 -0700 Subject: [PATCH 50/67] misc cleanup --- src/main/java/com/launchdarkly/client/EventOutput.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/main/java/com/launchdarkly/client/EventOutput.java b/src/main/java/com/launchdarkly/client/EventOutput.java index 35754a97d..e4ae49716 100644 --- a/src/main/java/com/launchdarkly/client/EventOutput.java +++ b/src/main/java/com/launchdarkly/client/EventOutput.java @@ -144,7 +144,7 @@ static final class Formatter { List makeOutputEvents(Event[] events, EventSummarizer.EventSummary summary) { List eventsOut = new ArrayList<>(events.length + 1); for (Event event: events) { - eventsOut.add(createEventOutput(event)); + eventsOut.add(createOutputEvent(event)); } if (!summary.isEmpty()) { eventsOut.add(createSummaryEvent(summary)); @@ -152,7 +152,7 @@ List makeOutputEvents(Event[] events, EventSummarizer.EventSummary return eventsOut; } - private EventOutput createEventOutput(Event e) { + private EventOutput createOutputEvent(Event e) { String userKey = e.user == null ? null : e.user.getKeyAsString(); if (e instanceof Event.FeatureRequest) { Event.FeatureRequest fe = (Event.FeatureRequest)e; From ad099e7d2abd5d617c5fa58d53016e68978ed392 Mon Sep 17 00:00:00 2001 From: Eli Bishop Date: Tue, 3 Apr 2018 15:04:12 -0700 Subject: [PATCH 51/67] better error message --- .../java/com/launchdarkly/client/DefaultEventProcessor.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/main/java/com/launchdarkly/client/DefaultEventProcessor.java b/src/main/java/com/launchdarkly/client/DefaultEventProcessor.java index 53aadb8e0..da578ff1e 100644 --- a/src/main/java/com/launchdarkly/client/DefaultEventProcessor.java +++ b/src/main/java/com/launchdarkly/client/DefaultEventProcessor.java @@ -367,7 +367,7 @@ private void handleResponse(Response response) { } } if (!response.isSuccessful()) { - logger.info("Got unexpected response when posting events: " + response); + logger.warn("Unexpected response status when posting events: {}", response.code()); if (response.code() == 401) { disabled.set(true); logger.error("Received 401 error, no further events will be posted since SDK key is invalid"); From df1f96079a55a325a4855ef1b6808eb4113fbaeb Mon Sep 17 00:00:00 2001 From: Eli Bishop Date: Tue, 3 Apr 2018 15:16:53 -0700 Subject: [PATCH 52/67] misc cleanup --- .../java/com/launchdarkly/client/DefaultEventProcessor.java | 4 ++-- src/main/java/com/launchdarkly/client/EventOutput.java | 3 ++- .../com/launchdarkly/client/DefaultEventProcessorTest.java | 3 ++- 3 files changed, 6 insertions(+), 4 deletions(-) diff --git a/src/main/java/com/launchdarkly/client/DefaultEventProcessor.java b/src/main/java/com/launchdarkly/client/DefaultEventProcessor.java index da578ff1e..34ee3fe92 100644 --- a/src/main/java/com/launchdarkly/client/DefaultEventProcessor.java +++ b/src/main/java/com/launchdarkly/client/DefaultEventProcessor.java @@ -31,7 +31,6 @@ final class DefaultEventProcessor implements EventProcessor { private static final Logger logger = LoggerFactory.getLogger(DefaultEventProcessor.class); - static final SimpleDateFormat HTTP_DATE_FORMAT = new SimpleDateFormat("EEE, dd MMM yyyy HH:mm:ss zzz"); private static final int CHANNEL_BLOCK_MILLIS = 1000; private final BlockingQueue inputChannel; @@ -171,8 +170,9 @@ public String toString() { // for debugging only * Takes messages from the input queue, updating the event buffer and summary counters * on its own thread. */ - private static final class EventDispatcher { + static final class EventDispatcher { private static final int MAX_FLUSH_THREADS = 5; + static final SimpleDateFormat HTTP_DATE_FORMAT = new SimpleDateFormat("EEE, dd MMM yyyy HH:mm:ss zzz"); private final LDConfig config; private final List flushWorkers; diff --git a/src/main/java/com/launchdarkly/client/EventOutput.java b/src/main/java/com/launchdarkly/client/EventOutput.java index e4ae49716..2d839853d 100644 --- a/src/main/java/com/launchdarkly/client/EventOutput.java +++ b/src/main/java/com/launchdarkly/client/EventOutput.java @@ -13,7 +13,8 @@ /** * Base class for data structures that we send in an event payload, which are somewhat * different in shape from the originating events. Also defines all of its own subclasses - * and the class that constructs them. + * and the class that constructs them. These are implementation details used only by + * DefaultEventProcessor and related classes, so they are all package-private. */ abstract class EventOutput { @SuppressWarnings("unused") diff --git a/src/test/java/com/launchdarkly/client/DefaultEventProcessorTest.java b/src/test/java/com/launchdarkly/client/DefaultEventProcessorTest.java index e39d6ee86..28a045271 100644 --- a/src/test/java/com/launchdarkly/client/DefaultEventProcessorTest.java +++ b/src/test/java/com/launchdarkly/client/DefaultEventProcessorTest.java @@ -5,6 +5,7 @@ import com.google.gson.JsonElement; import com.google.gson.JsonObject; import com.google.gson.JsonPrimitive; +import com.launchdarkly.client.DefaultEventProcessor.EventDispatcher; import org.hamcrest.Matcher; import org.junit.After; @@ -367,7 +368,7 @@ public void noMorePayloadsAreSentAfter401Error() throws Exception { } private MockResponse addDateHeader(MockResponse response, long timestamp) { - return response.addHeader("Date", DefaultEventProcessor.HTTP_DATE_FORMAT.format(new Date(timestamp))); + return response.addHeader("Date", EventDispatcher.HTTP_DATE_FORMAT.format(new Date(timestamp))); } private JsonArray flushAndGetEvents(MockResponse response) throws Exception { From d6002cc841e79308e663acd1d45e216f14d605a5 Mon Sep 17 00:00:00 2001 From: Eli Bishop Date: Tue, 3 Apr 2018 15:17:49 -0700 Subject: [PATCH 53/67] misc cleanup --- src/main/java/com/launchdarkly/client/EventOutput.java | 2 -- 1 file changed, 2 deletions(-) diff --git a/src/main/java/com/launchdarkly/client/EventOutput.java b/src/main/java/com/launchdarkly/client/EventOutput.java index 2d839853d..acf3d5cfc 100644 --- a/src/main/java/com/launchdarkly/client/EventOutput.java +++ b/src/main/java/com/launchdarkly/client/EventOutput.java @@ -113,14 +113,12 @@ static final class SummaryEventFlag { @SerializedName("default") final JsonElement defaultVal; final List counters; - @SuppressWarnings("unused") SummaryEventFlag(JsonElement defaultVal, List counters) { this.defaultVal = defaultVal; this.counters = counters; } } - @SuppressWarnings("unused") static final class SummaryEventCounter { final JsonElement value; final Integer version; From d59bbf9aa22ad7613f94c5b38f8904f660c91de7 Mon Sep 17 00:00:00 2001 From: Eli Bishop Date: Mon, 9 Apr 2018 15:54:00 -0700 Subject: [PATCH 54/67] add event schema version header --- .../java/com/launchdarkly/client/DefaultEventProcessor.java | 3 +++ 1 file changed, 3 insertions(+) diff --git a/src/main/java/com/launchdarkly/client/DefaultEventProcessor.java b/src/main/java/com/launchdarkly/client/DefaultEventProcessor.java index 34ee3fe92..fe9cf4343 100644 --- a/src/main/java/com/launchdarkly/client/DefaultEventProcessor.java +++ b/src/main/java/com/launchdarkly/client/DefaultEventProcessor.java @@ -32,6 +32,8 @@ final class DefaultEventProcessor implements EventProcessor { private static final Logger logger = LoggerFactory.getLogger(DefaultEventProcessor.class); private static final int CHANNEL_BLOCK_MILLIS = 1000; + private static final String EVENT_SCHEMA_HEADER = "X-LaunchDarkly-Event-Schema"; + private static final String EVENT_SCHEMA_VERSION = "2"; private final BlockingQueue inputChannel; private final ScheduledExecutorService scheduler; @@ -497,6 +499,7 @@ private void postEvents(List eventsOut) { .url(uriStr) .post(RequestBody.create(MediaType.parse("application/json; charset=utf-8"), json)) .addHeader("Content-Type", "application/json") + .addHeader(EVENT_SCHEMA_HEADER, EVENT_SCHEMA_VERSION) .build(); long startTime = System.currentTimeMillis(); From de45c4a967f9bb2910bb1e1694bb131dc079c554 Mon Sep 17 00:00:00 2001 From: Eli Bishop Date: Mon, 9 Apr 2018 15:55:37 -0700 Subject: [PATCH 55/67] move getRequestBuilder out of Config, it's not an instance method --- .../launchdarkly/client/DefaultEventProcessor.java | 4 +++- .../com/launchdarkly/client/FeatureRequestor.java | 14 ++++++++------ .../java/com/launchdarkly/client/LDConfig.java | 6 ------ src/main/java/com/launchdarkly/client/Util.java | 8 ++++++++ 4 files changed, 19 insertions(+), 13 deletions(-) diff --git a/src/main/java/com/launchdarkly/client/DefaultEventProcessor.java b/src/main/java/com/launchdarkly/client/DefaultEventProcessor.java index fe9cf4343..755166265 100644 --- a/src/main/java/com/launchdarkly/client/DefaultEventProcessor.java +++ b/src/main/java/com/launchdarkly/client/DefaultEventProcessor.java @@ -24,6 +24,8 @@ import java.util.concurrent.atomic.AtomicInteger; import java.util.concurrent.atomic.AtomicLong; +import static com.launchdarkly.client.Util.getRequestBuilder; + import okhttp3.MediaType; import okhttp3.Request; import okhttp3.RequestBody; @@ -495,7 +497,7 @@ private void postEvents(List eventsOut) { logger.debug("Posting {} event(s) to {} with payload: {}", eventsOut.size(), uriStr, json); - Request request = config.getRequestBuilder(sdkKey) + Request request = getRequestBuilder(sdkKey) .url(uriStr) .post(RequestBody.create(MediaType.parse("application/json; charset=utf-8"), json)) .addHeader("Content-Type", "application/json") diff --git a/src/main/java/com/launchdarkly/client/FeatureRequestor.java b/src/main/java/com/launchdarkly/client/FeatureRequestor.java index 0ae25e78e..377f773c7 100644 --- a/src/main/java/com/launchdarkly/client/FeatureRequestor.java +++ b/src/main/java/com/launchdarkly/client/FeatureRequestor.java @@ -1,17 +1,19 @@ package com.launchdarkly.client; -import okhttp3.Request; -import okhttp3.Response; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import static com.launchdarkly.client.VersionedDataKind.FEATURES; -import static com.launchdarkly.client.VersionedDataKind.SEGMENTS; - import java.io.IOException; import java.util.HashMap; import java.util.Map; +import static com.launchdarkly.client.Util.getRequestBuilder; +import static com.launchdarkly.client.VersionedDataKind.FEATURES; +import static com.launchdarkly.client.VersionedDataKind.SEGMENTS; + +import okhttp3.Request; +import okhttp3.Response; + class FeatureRequestor { private static final Logger logger = LoggerFactory.getLogger(FeatureRequestor.class); private static final String GET_LATEST_FLAGS_PATH = "/sdk/latest-flags"; @@ -68,7 +70,7 @@ AllData getAllData() throws IOException, InvalidSDKKeyException { } private String get(String path) throws IOException, InvalidSDKKeyException { - Request request = config.getRequestBuilder(sdkKey) + Request request = getRequestBuilder(sdkKey) .url(config.baseURI.toString() + path) .get() .build(); diff --git a/src/main/java/com/launchdarkly/client/LDConfig.java b/src/main/java/com/launchdarkly/client/LDConfig.java index 1ee6ac423..767347589 100644 --- a/src/main/java/com/launchdarkly/client/LDConfig.java +++ b/src/main/java/com/launchdarkly/client/LDConfig.java @@ -132,12 +132,6 @@ protected LDConfig(Builder builder) { .build(); } - Request.Builder getRequestBuilder(String sdkKey) { - return new Request.Builder() - .addHeader("Authorization", sdkKey) - .addHeader("User-Agent", "JavaClient/" + LDClient.CLIENT_VERSION); - } - /** * A builder that helps construct {@link com.launchdarkly.client.LDConfig} objects. Builder * calls can be chained, enabling the following pattern: diff --git a/src/main/java/com/launchdarkly/client/Util.java b/src/main/java/com/launchdarkly/client/Util.java index c3a2643a2..3fca7e3c7 100644 --- a/src/main/java/com/launchdarkly/client/Util.java +++ b/src/main/java/com/launchdarkly/client/Util.java @@ -4,6 +4,8 @@ import org.joda.time.DateTime; import org.joda.time.DateTimeZone; +import okhttp3.Request; + class Util { /** * Converts either a unix epoch millis number or RFC3339/ISO8601 timestamp as {@link JsonPrimitive} to a {@link DateTime} object. @@ -24,4 +26,10 @@ static DateTime jsonPrimitiveToDateTime(JsonPrimitive maybeDate) { return null; } } + + static Request.Builder getRequestBuilder(String sdkKey) { + return new Request.Builder() + .addHeader("Authorization", sdkKey) + .addHeader("User-Agent", "JavaClient/" + LDClient.CLIENT_VERSION); + } } From 2cab9944b56e8a623e4fd10431b2174d5ee53a15 Mon Sep 17 00:00:00 2001 From: Eli Bishop Date: Mon, 9 Apr 2018 16:14:09 -0700 Subject: [PATCH 56/67] debug events should be in addition to regular events, & should always include inline user --- .../client/DefaultEventProcessor.java | 78 +++++++++++-------- .../java/com/launchdarkly/client/Event.java | 4 +- .../com/launchdarkly/client/EventFactory.java | 11 ++- .../com/launchdarkly/client/EventOutput.java | 8 +- .../client/DefaultEventProcessorTest.java | 22 +++++- 5 files changed, 80 insertions(+), 43 deletions(-) diff --git a/src/main/java/com/launchdarkly/client/DefaultEventProcessor.java b/src/main/java/com/launchdarkly/client/DefaultEventProcessor.java index 755166265..ec5222e81 100644 --- a/src/main/java/com/launchdarkly/client/DefaultEventProcessor.java +++ b/src/main/java/com/launchdarkly/client/DefaultEventProcessor.java @@ -287,28 +287,42 @@ private void processEvent(Event e, SimpleLRUCache userKeys, Even if (disabled.get()) { return; } + + // Always record the event in the summarizer. + buffer.addToSummary(e); + + // Decide whether to add the event to the payload. Feature events may be added twice, once for + // the event (if tracked) and once for debugging. + boolean willAddFullEvent = false; + Event debugEvent = null; + if (e instanceof Event.FeatureRequest) { + if (shouldSampleEvent()) { + Event.FeatureRequest fe = (Event.FeatureRequest)e; + willAddFullEvent = fe.trackEvents; + if (shouldDebugEvent(fe)) { + debugEvent = EventFactory.DEFAULT.newDebugEvent(fe); + } + } + } else { + willAddFullEvent = shouldSampleEvent(); + } // For each user we haven't seen before, we add an index event - unless this is already // an identify event for that user. - if (!config.inlineUsersInEvents && e.user != null && !noticeUser(e.user, userKeys)) { - if (!(e instanceof Event.Identify)) { - Event.Index ie = new Event.Index(e.creationDate, e.user); - buffer.add(ie); + if (!(willAddFullEvent && config.inlineUsersInEvents)) { + if (e.user != null && !noticeUser(e.user, userKeys)) { + if (!(e instanceof Event.Identify)) { + Event.Index ie = new Event.Index(e.creationDate, e.user); + buffer.add(ie); + } } } - - // Always record the event in the summarizer. - buffer.addToSummary(e); - - if (shouldTrackFullEvent(e)) { - // Sampling interval applies only to fully-tracked events. - if (config.samplingInterval > 0 && random.nextInt(config.samplingInterval) != 0) { - return; - } - // Queue the event as-is; we'll transform it into an output event when we're flushing - // (to avoid doing that work on our main thread). + if (willAddFullEvent) { buffer.add(e); } + if (debugEvent != null) { + buffer.add(debugEvent); + } } // Add to the set of users we've noticed, and return true if the user was already known to us. @@ -320,29 +334,25 @@ private boolean noticeUser(LDUser user, SimpleLRUCache userKeys) return userKeys.put(key, key) != null; } - private boolean shouldTrackFullEvent(Event e) { - if (e instanceof Event.FeatureRequest) { - Event.FeatureRequest fe = (Event.FeatureRequest)e; - if (fe.trackEvents) { + private boolean shouldSampleEvent() { + return config.samplingInterval <= 0 || random.nextInt(config.samplingInterval) == 0; + } + + private boolean shouldDebugEvent(Event.FeatureRequest fe) { + if (fe.debugEventsUntilDate != null) { + // The "last known past time" comes from the last HTTP response we got from the server. + // In case the client's time is set wrong, at least we know that any expiration date + // earlier than that point is definitely in the past. If there's any discrepancy, we + // want to err on the side of cutting off event debugging sooner. + long lastPast = lastKnownPastTime.get(); + if (fe.debugEventsUntilDate > lastPast && + fe.debugEventsUntilDate > System.currentTimeMillis()) { return true; } - if (fe.debugEventsUntilDate != null) { - // The "last known past time" comes from the last HTTP response we got from the server. - // In case the client's time is set wrong, at least we know that any expiration date - // earlier than that point is definitely in the past. If there's any discrepancy, we - // want to err on the side of cutting off event debugging sooner. - long lastPast = lastKnownPastTime.get(); - if (fe.debugEventsUntilDate > lastPast && - fe.debugEventsUntilDate > System.currentTimeMillis()) { - return true; - } - } - return false; - } else { - return true; } + return false; } - + private void triggerFlush(EventBuffer buffer, BlockingQueue payloadQueue) { if (disabled.get() || buffer.isEmpty()) { return; diff --git a/src/main/java/com/launchdarkly/client/Event.java b/src/main/java/com/launchdarkly/client/Event.java index 7d429554c..376a53ef3 100644 --- a/src/main/java/com/launchdarkly/client/Event.java +++ b/src/main/java/com/launchdarkly/client/Event.java @@ -46,9 +46,10 @@ static final class FeatureRequest extends Event { final String prereqOf; final boolean trackEvents; final Long debugEventsUntilDate; + final boolean debug; FeatureRequest(long timestamp, String key, LDUser user, Integer version, Integer variation, JsonElement value, - JsonElement defaultVal, String prereqOf, boolean trackEvents, Long debugEventsUntilDate) { + JsonElement defaultVal, String prereqOf, boolean trackEvents, Long debugEventsUntilDate, boolean debug) { super(timestamp, user); this.key = key; this.version = version; @@ -58,6 +59,7 @@ static final class FeatureRequest extends Event { this.prereqOf = prereqOf; this.trackEvents = trackEvents; this.debugEventsUntilDate = debugEventsUntilDate; + this.debug = debug; } } diff --git a/src/main/java/com/launchdarkly/client/EventFactory.java b/src/main/java/com/launchdarkly/client/EventFactory.java index a5e6aa036..b1c03491b 100644 --- a/src/main/java/com/launchdarkly/client/EventFactory.java +++ b/src/main/java/com/launchdarkly/client/EventFactory.java @@ -10,18 +10,23 @@ abstract class EventFactory { public Event.FeatureRequest newFeatureRequestEvent(FeatureFlag flag, LDUser user, FeatureFlag.VariationAndValue result, JsonElement defaultVal) { return new Event.FeatureRequest(getTimestamp(), flag.getKey(), user, flag.getVersion(), result == null ? null : result.getVariation(), result == null ? null : result.getValue(), - defaultVal, null, flag.isTrackEvents(), flag.getDebugEventsUntilDate()); + defaultVal, null, flag.isTrackEvents(), flag.getDebugEventsUntilDate(), false); } public Event.FeatureRequest newUnknownFeatureRequestEvent(String key, LDUser user, JsonElement defaultValue) { - return new Event.FeatureRequest(getTimestamp(), key, user, null, null, defaultValue, defaultValue, null, false, null); + return new Event.FeatureRequest(getTimestamp(), key, user, null, null, defaultValue, defaultValue, null, false, null, false); } public Event.FeatureRequest newPrerequisiteFeatureRequestEvent(FeatureFlag prereqFlag, LDUser user, FeatureFlag.VariationAndValue result, FeatureFlag prereqOf) { return new Event.FeatureRequest(getTimestamp(), prereqFlag.getKey(), user, prereqFlag.getVersion(), result == null ? null : result.getVariation(), result == null ? null : result.getValue(), - null, prereqOf.getKey(), prereqFlag.isTrackEvents(), prereqFlag.getDebugEventsUntilDate()); + null, prereqOf.getKey(), prereqFlag.isTrackEvents(), prereqFlag.getDebugEventsUntilDate(), false); + } + + public Event.FeatureRequest newDebugEvent(Event.FeatureRequest from) { + return new Event.FeatureRequest(from.creationDate, from.key, from.user, from.version, from.variation, from.value, + from.defaultVal, from.prereqOf, from.trackEvents, from.debugEventsUntilDate, true); } public Event.Custom newCustomEvent(String key, LDUser user, JsonElement data) { diff --git a/src/main/java/com/launchdarkly/client/EventOutput.java b/src/main/java/com/launchdarkly/client/EventOutput.java index acf3d5cfc..3058c23a9 100644 --- a/src/main/java/com/launchdarkly/client/EventOutput.java +++ b/src/main/java/com/launchdarkly/client/EventOutput.java @@ -155,11 +155,11 @@ private EventOutput createOutputEvent(Event e) { String userKey = e.user == null ? null : e.user.getKeyAsString(); if (e instanceof Event.FeatureRequest) { Event.FeatureRequest fe = (Event.FeatureRequest)e; - boolean isDebug = (!fe.trackEvents && fe.debugEventsUntilDate != null); + boolean inlineThisUser = inlineUsers || fe.debug; return new EventOutput.FeatureRequest(fe.creationDate, fe.key, - inlineUsers ? null : userKey, - inlineUsers ? e.user : null, - fe.version, fe.value, fe.defaultVal, fe.prereqOf, isDebug); + inlineThisUser ? null : userKey, + inlineThisUser ? e.user : null, + fe.version, fe.value, fe.defaultVal, fe.prereqOf, fe.debug); } else if (e instanceof Event.Identify) { return new EventOutput.Identify(e.creationDate, e.user); } else if (e instanceof Event.Custom) { diff --git a/src/test/java/com/launchdarkly/client/DefaultEventProcessorTest.java b/src/test/java/com/launchdarkly/client/DefaultEventProcessorTest.java index 28a045271..c6c0265f3 100644 --- a/src/test/java/com/launchdarkly/client/DefaultEventProcessorTest.java +++ b/src/test/java/com/launchdarkly/client/DefaultEventProcessorTest.java @@ -162,7 +162,27 @@ public void eventKindIsDebugIfFlagIsTemporarilyInDebugMode() throws Exception { JsonArray output = flushAndGetEvents(new MockResponse()); assertThat(output, hasItems( isIndexEvent(fe, userJson), - isFeatureEvent(fe, flag, true, null), + isFeatureEvent(fe, flag, true, userJson), + isSummaryEvent() + )); + } + + @SuppressWarnings("unchecked") + @Test + public void eventCanBeBothTrackedAndDebugged() throws Exception { + ep = new DefaultEventProcessor(SDK_KEY, configBuilder.build()); + long futureTime = System.currentTimeMillis() + 1000000; + FeatureFlag flag = new FeatureFlagBuilder("flagkey").version(11).trackEvents(true) + .debugEventsUntilDate(futureTime).build(); + Event.FeatureRequest fe = EventFactory.DEFAULT.newFeatureRequestEvent(flag, user, + new FeatureFlag.VariationAndValue(new Integer(1), new JsonPrimitive("value")), null); + ep.sendEvent(fe); + + JsonArray output = flushAndGetEvents(new MockResponse()); + assertThat(output, hasItems( + isIndexEvent(fe, userJson), + isFeatureEvent(fe, flag, false, null), + isFeatureEvent(fe, flag, true, userJson), isSummaryEvent() )); } From 613b8e90e848789f064ea14b32af5545ddc60b9f Mon Sep 17 00:00:00 2001 From: Eli Bishop Date: Mon, 9 Apr 2018 16:37:46 -0700 Subject: [PATCH 57/67] add unit test --- .../client/DefaultEventProcessorTest.java | 19 ++++++++++++++++++- 1 file changed, 18 insertions(+), 1 deletion(-) diff --git a/src/test/java/com/launchdarkly/client/DefaultEventProcessorTest.java b/src/test/java/com/launchdarkly/client/DefaultEventProcessorTest.java index c6c0265f3..8bfced280 100644 --- a/src/test/java/com/launchdarkly/client/DefaultEventProcessorTest.java +++ b/src/test/java/com/launchdarkly/client/DefaultEventProcessorTest.java @@ -131,7 +131,7 @@ public void featureEventCanContainInlineUser() throws Exception { isSummaryEvent() )); } - + @SuppressWarnings("unchecked") @Test public void userIsFilteredInFeatureEvent() throws Exception { @@ -149,6 +149,23 @@ public void userIsFilteredInFeatureEvent() throws Exception { )); } + @SuppressWarnings("unchecked") + @Test + public void indexEventIsStillGeneratedIfInlineUsersIsTrueButFeatureEventIsNotTracked() throws Exception { + configBuilder.inlineUsersInEvents(true); + ep = new DefaultEventProcessor(SDK_KEY, configBuilder.build()); + FeatureFlag flag = new FeatureFlagBuilder("flagkey").version(11).trackEvents(false).build(); + Event.FeatureRequest fe = EventFactory.DEFAULT.newFeatureRequestEvent(flag, user, + new FeatureFlag.VariationAndValue(new Integer(1), new JsonPrimitive("value")), null); + ep.sendEvent(fe); + + JsonArray output = flushAndGetEvents(new MockResponse()); + assertThat(output, hasItems( + isIndexEvent(fe, userJson), + isSummaryEvent() + )); + } + @SuppressWarnings("unchecked") @Test public void eventKindIsDebugIfFlagIsTemporarilyInDebugMode() throws Exception { From 9331456113e4faac212bc12e05dc046393c4594d Mon Sep 17 00:00:00 2001 From: Eli Bishop Date: Tue, 10 Apr 2018 14:19:22 -0700 Subject: [PATCH 58/67] retry event posting once on exception or 5xx error --- .../client/DefaultEventProcessor.java | 53 ++++++++++++------- .../client/DefaultEventProcessorTest.java | 21 ++++++++ 2 files changed, 54 insertions(+), 20 deletions(-) diff --git a/src/main/java/com/launchdarkly/client/DefaultEventProcessor.java b/src/main/java/com/launchdarkly/client/DefaultEventProcessor.java index ec5222e81..b08af074a 100644 --- a/src/main/java/com/launchdarkly/client/DefaultEventProcessor.java +++ b/src/main/java/com/launchdarkly/client/DefaultEventProcessor.java @@ -380,12 +380,9 @@ private void handleResponse(Response response) { } catch (ParseException e) { } } - if (!response.isSuccessful()) { - logger.warn("Unexpected response status when posting events: {}", response.code()); - if (response.code() == 401) { - disabled.set(true); - logger.error("Received 401 error, no further events will be posted since SDK key is invalid"); - } + if (response.code() == 401) { + disabled.set(true); + logger.error("Received 401 error, no further events will be posted since SDK key is invalid"); } } } @@ -507,20 +504,36 @@ private void postEvents(List eventsOut) { logger.debug("Posting {} event(s) to {} with payload: {}", eventsOut.size(), uriStr, json); - Request request = getRequestBuilder(sdkKey) - .url(uriStr) - .post(RequestBody.create(MediaType.parse("application/json; charset=utf-8"), json)) - .addHeader("Content-Type", "application/json") - .addHeader(EVENT_SCHEMA_HEADER, EVENT_SCHEMA_VERSION) - .build(); - - long startTime = System.currentTimeMillis(); - try (Response response = config.httpClient.newCall(request).execute()) { - long endTime = System.currentTimeMillis(); - logger.debug("Event delivery took {} ms, response status {}", endTime - startTime, response.code()); - responseListener.handleResponse(response); - } catch (IOException e) { - logger.info("Unhandled exception in LaunchDarkly client when posting events to URL: " + request.url(), e); + for (int attempt = 0; attempt < 2; attempt++) { + if (attempt == 1) { + logger.warn("Will retry posting events after 1 second"); + try { + Thread.sleep(1000); + } catch (InterruptedException e) {} + } + Request request = getRequestBuilder(sdkKey) + .url(uriStr) + .post(RequestBody.create(MediaType.parse("application/json; charset=utf-8"), json)) + .addHeader("Content-Type", "application/json") + .addHeader(EVENT_SCHEMA_HEADER, EVENT_SCHEMA_VERSION) + .build(); + + long startTime = System.currentTimeMillis(); + try (Response response = config.httpClient.newCall(request).execute()) { + long endTime = System.currentTimeMillis(); + logger.debug("Event delivery took {} ms, response status {}", endTime - startTime, response.code()); + if (!response.isSuccessful()) { + logger.warn("Unexpected response status when posting events: {}", response.code()); + if (response.code() >= 500) { + continue; + } + } + responseListener.handleResponse(response); + break; + } catch (IOException e) { + logger.warn("Unhandled exception in LaunchDarkly client when posting events to URL: " + request.url(), e); + continue; + } } } } diff --git a/src/test/java/com/launchdarkly/client/DefaultEventProcessorTest.java b/src/test/java/com/launchdarkly/client/DefaultEventProcessorTest.java index 8bfced280..a3c04a698 100644 --- a/src/test/java/com/launchdarkly/client/DefaultEventProcessorTest.java +++ b/src/test/java/com/launchdarkly/client/DefaultEventProcessorTest.java @@ -8,6 +8,7 @@ import com.launchdarkly.client.DefaultEventProcessor.EventDispatcher; import org.hamcrest.Matcher; +import org.hamcrest.Matchers; import org.junit.After; import org.junit.Before; import org.junit.Test; @@ -22,6 +23,7 @@ import static org.hamcrest.Matchers.equalTo; import static org.hamcrest.Matchers.hasItem; import static org.hamcrest.Matchers.hasItems; +import static org.hamcrest.Matchers.notNullValue; import static org.hamcrest.Matchers.nullValue; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertNotNull; @@ -404,6 +406,25 @@ public void noMorePayloadsAreSentAfter401Error() throws Exception { assertThat(req, nullValue(RecordedRequest.class)); } + @Test + public void flushIsRetriedOnceAfter5xxError() throws Exception { + ep = new DefaultEventProcessor(SDK_KEY, configBuilder.build()); + Event e = EventFactory.DEFAULT.newIdentifyEvent(user); + ep.sendEvent(e); + + server.enqueue(new MockResponse().setResponseCode(503)); + server.enqueue(new MockResponse().setResponseCode(503)); + + ep.flush(); + ep.waitUntilInactive(); + RecordedRequest req = server.takeRequest(0, TimeUnit.SECONDS); + assertThat(req, notNullValue(RecordedRequest.class)); + req = server.takeRequest(0, TimeUnit.SECONDS); + assertThat(req, notNullValue(RecordedRequest.class)); + req = server.takeRequest(0, TimeUnit.SECONDS); + assertThat(req, nullValue(RecordedRequest.class)); // only 2 requests total + } + private MockResponse addDateHeader(MockResponse response, long timestamp) { return response.addHeader("Date", EventDispatcher.HTTP_DATE_FORMAT.format(new Date(timestamp))); } From ab03f47d1d6ba8e41bf454abf0535685820fa0cf Mon Sep 17 00:00:00 2001 From: Eli Bishop Date: Wed, 11 Apr 2018 18:03:24 -0700 Subject: [PATCH 59/67] misc cleanup --- .../client/DefaultEventProcessor.java | 22 ++++++++++++------- 1 file changed, 14 insertions(+), 8 deletions(-) diff --git a/src/main/java/com/launchdarkly/client/DefaultEventProcessor.java b/src/main/java/com/launchdarkly/client/DefaultEventProcessor.java index ec5222e81..bed364b8e 100644 --- a/src/main/java/com/launchdarkly/client/DefaultEventProcessor.java +++ b/src/main/java/com/launchdarkly/client/DefaultEventProcessor.java @@ -293,31 +293,37 @@ private void processEvent(Event e, SimpleLRUCache userKeys, Even // Decide whether to add the event to the payload. Feature events may be added twice, once for // the event (if tracked) and once for debugging. - boolean willAddFullEvent = false; + boolean addIndexEvent = false, + addFullEvent = false; Event debugEvent = null; + if (e instanceof Event.FeatureRequest) { if (shouldSampleEvent()) { Event.FeatureRequest fe = (Event.FeatureRequest)e; - willAddFullEvent = fe.trackEvents; + addFullEvent = fe.trackEvents; if (shouldDebugEvent(fe)) { debugEvent = EventFactory.DEFAULT.newDebugEvent(fe); } } } else { - willAddFullEvent = shouldSampleEvent(); + addFullEvent = shouldSampleEvent(); } // For each user we haven't seen before, we add an index event - unless this is already // an identify event for that user. - if (!(willAddFullEvent && config.inlineUsersInEvents)) { - if (e.user != null && !noticeUser(e.user, userKeys)) { + if (!addFullEvent || !config.inlineUsersInEvents) { + if (e.user != null && e.user.getKey() != null && !noticeUser(e.user, userKeys)) { if (!(e instanceof Event.Identify)) { - Event.Index ie = new Event.Index(e.creationDate, e.user); - buffer.add(ie); + addIndexEvent = true; } } } - if (willAddFullEvent) { + + if (addIndexEvent) { + Event.Index ie = new Event.Index(e.creationDate, e.user); + buffer.add(ie); + } + if (addFullEvent) { buffer.add(e); } if (debugEvent != null) { From fd9b02d442140c332225d66998f4ed982f99f917 Mon Sep 17 00:00:00 2001 From: Eli Bishop Date: Wed, 25 Apr 2018 12:13:05 -0700 Subject: [PATCH 60/67] send as much of a feature event as possible even if user is invalid --- .../com/launchdarkly/client/EventFactory.java | 5 +++++ .../com/launchdarkly/client/LDClient.java | 19 +++++++++++-------- 2 files changed, 16 insertions(+), 8 deletions(-) diff --git a/src/main/java/com/launchdarkly/client/EventFactory.java b/src/main/java/com/launchdarkly/client/EventFactory.java index b1c03491b..fce94a49c 100644 --- a/src/main/java/com/launchdarkly/client/EventFactory.java +++ b/src/main/java/com/launchdarkly/client/EventFactory.java @@ -13,6 +13,11 @@ public Event.FeatureRequest newFeatureRequestEvent(FeatureFlag flag, LDUser user defaultVal, null, flag.isTrackEvents(), flag.getDebugEventsUntilDate(), false); } + public Event.FeatureRequest newDefaultFeatureRequestEvent(FeatureFlag flag, LDUser user, JsonElement defaultValue) { + return new Event.FeatureRequest(getTimestamp(), flag.getKey(), user, flag.getVersion(), + null, defaultValue, defaultValue, null, flag.isTrackEvents(), flag.getDebugEventsUntilDate(), false); + } + public Event.FeatureRequest newUnknownFeatureRequestEvent(String key, LDUser user, JsonElement defaultValue) { return new Event.FeatureRequest(getTimestamp(), key, user, null, null, defaultValue, defaultValue, null, false, null, false); } diff --git a/src/main/java/com/launchdarkly/client/LDClient.java b/src/main/java/com/launchdarkly/client/LDClient.java index 1567eaf61..6ff83efac 100644 --- a/src/main/java/com/launchdarkly/client/LDClient.java +++ b/src/main/java/com/launchdarkly/client/LDClient.java @@ -246,14 +246,6 @@ public boolean isFlagKnown(String featureKey) { } private JsonElement evaluate(String featureKey, LDUser user, JsonElement defaultValue, VariationType expectedType) { - if (user == null || user.getKey() == null) { - logger.warn("Null user or null user key when evaluating flag: " + featureKey + "; returning default value"); - sendFlagRequestEvent(eventFactory.newUnknownFeatureRequestEvent(featureKey, user, defaultValue)); - return defaultValue; - } - if (user.getKeyAsString().isEmpty()) { - logger.warn("User key is blank. Flag evaluation will proceed, but the user will not be stored in LaunchDarkly"); - } if (!initialized()) { if (config.featureStore.initialized()) { logger.warn("Evaluation called before client initialized for feature flag " + featureKey + "; using last known values from feature store"); @@ -271,6 +263,14 @@ private JsonElement evaluate(String featureKey, LDUser user, JsonElement default sendFlagRequestEvent(eventFactory.newUnknownFeatureRequestEvent(featureKey, user, defaultValue)); return defaultValue; } + if (user == null || user.getKey() == null) { + logger.warn("Null user or null user key when evaluating flag: " + featureKey + "; returning default value"); + sendFlagRequestEvent(eventFactory.newDefaultFeatureRequestEvent(featureFlag, user, defaultValue)); + return defaultValue; + } + if (user.getKeyAsString().isEmpty()) { + logger.warn("User key is blank. Flag evaluation will proceed, but the user will not be stored in LaunchDarkly"); + } FeatureFlag.EvalResult evalResult = featureFlag.evaluate(user, config.featureStore, eventFactory); for (Event.FeatureRequest event : evalResult.getPrerequisiteEvents()) { eventProcessor.sendEvent(event); @@ -279,6 +279,9 @@ private JsonElement evaluate(String featureKey, LDUser user, JsonElement default expectedType.assertResultType(evalResult.getResult().getValue()); sendFlagRequestEvent(eventFactory.newFeatureRequestEvent(featureFlag, user, evalResult.getResult(), defaultValue)); return evalResult.getResult().getValue(); + } else { + sendFlagRequestEvent(eventFactory.newDefaultFeatureRequestEvent(featureFlag, user, defaultValue)); + return defaultValue; } } catch (Exception e) { logger.error("Encountered exception in LaunchDarkly client", e); From 739aefea0491ff948c4461371e3bdf545a3403dc Mon Sep 17 00:00:00 2001 From: Eli Bishop Date: Wed, 25 Apr 2018 12:49:03 -0700 Subject: [PATCH 61/67] delay should apply to all attempts after the first one --- .../java/com/launchdarkly/client/DefaultEventProcessor.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/main/java/com/launchdarkly/client/DefaultEventProcessor.java b/src/main/java/com/launchdarkly/client/DefaultEventProcessor.java index fde2127c0..9db6a7aa2 100644 --- a/src/main/java/com/launchdarkly/client/DefaultEventProcessor.java +++ b/src/main/java/com/launchdarkly/client/DefaultEventProcessor.java @@ -511,7 +511,7 @@ private void postEvents(List eventsOut) { eventsOut.size(), uriStr, json); for (int attempt = 0; attempt < 2; attempt++) { - if (attempt == 1) { + if (attempt > 0) { logger.warn("Will retry posting events after 1 second"); try { Thread.sleep(1000); From 7e460813729fac6cde27d728bfa3c1c0be664322 Mon Sep 17 00:00:00 2001 From: Eli Bishop Date: Wed, 25 Apr 2018 20:01:36 -0700 Subject: [PATCH 62/67] use factories for all pluggable components --- .../com/launchdarkly/client/Components.java | 132 +++++++ .../launchdarkly/client/EventProcessor.java | 5 +- .../client/EventProcessorFactory.java | 16 + .../client/FeatureStoreFactory.java | 14 + .../com/launchdarkly/client/LDClient.java | 70 ++-- .../com/launchdarkly/client/LDConfig.java | 56 ++- .../launchdarkly/client/PollingProcessor.java | 4 +- .../client/RedisFeatureStore.java | 47 +-- .../client/RedisFeatureStoreBuilder.java | 336 ++++++++++-------- .../launchdarkly/client/StreamProcessor.java | 4 +- .../launchdarkly/client/UpdateProcessor.java | 10 +- .../client/UpdateProcessorFactory.java | 17 + .../client/LDClientEvaluationTest.java | 26 +- .../client/LDClientEventTest.java | 44 +-- .../client/LDClientLddModeTest.java | 3 +- .../client/LDClientOfflineTest.java | 3 +- .../com/launchdarkly/client/LDClientTest.java | 64 ++-- .../com/launchdarkly/client/LDUserTest.java | 1 - .../client/PollingProcessorTest.java | 4 +- .../client/RedisFeatureStoreBuilderTest.java | 140 ++++---- .../client/RedisFeatureStoreTest.java | 2 +- .../client/StreamProcessorTest.java | 5 +- .../com/launchdarkly/client/TestUtil.java | 42 +++ 23 files changed, 661 insertions(+), 384 deletions(-) create mode 100644 src/main/java/com/launchdarkly/client/Components.java create mode 100644 src/main/java/com/launchdarkly/client/EventProcessorFactory.java create mode 100644 src/main/java/com/launchdarkly/client/FeatureStoreFactory.java create mode 100644 src/main/java/com/launchdarkly/client/UpdateProcessorFactory.java diff --git a/src/main/java/com/launchdarkly/client/Components.java b/src/main/java/com/launchdarkly/client/Components.java new file mode 100644 index 000000000..fa6569186 --- /dev/null +++ b/src/main/java/com/launchdarkly/client/Components.java @@ -0,0 +1,132 @@ +package com.launchdarkly.client; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.net.URI; + +/** + * Provides factories for the standard implementations of LaunchDarkly component interfaces. + * @since 4.0.0 + */ +public abstract class Components { + private static final FeatureStoreFactory inMemoryFeatureStoreFactory = new InMemoryFeatureStoreFactory(); + private static final EventProcessorFactory defaultEventProcessorFactory = new DefaultEventProcessorFactory(); + private static final EventProcessorFactory nullEventProcessorFactory = new NullEventProcessorFactory(); + private static final UpdateProcessorFactory defaultUpdateProcessorFactory = new DefaultUpdateProcessorFactory(); + private static final UpdateProcessorFactory nullUpdateProcessorFactory = new NullUpdateProcessorFactory(); + + /** + * Returns a factory for the default in-memory implementation of {@link FeatureStore}. + */ + public static FeatureStoreFactory inMemoryFeatureStore() { + return inMemoryFeatureStoreFactory; + } + + /** + * Returns a factory with builder methods for creating a Redis-backed implementation of {@link FeatureStore}, + * using {@link RedisFeatureStoreBuilder#DEFAULT_URI}. + */ + public static RedisFeatureStoreBuilder redisFeatureStore() { + return new RedisFeatureStoreBuilder(); + } + + /** + * Returns a factory with builder methods for creating a Redis-backed implementation of {@link FeatureStore}, + * specifying the Redis URI. + */ + public static RedisFeatureStoreBuilder redisFeatureStore(URI redisUri) { + return new RedisFeatureStoreBuilder(redisUri); + } + + /** + * Returns a factory for the default implementation of {@link EventProcessor}, which + * forwards all analytics events to LaunchDarkly (unless the client is offline or you have + * set {@link LDConfig.Builder#sendEvents(boolean)} to {@code false}). + */ + public static EventProcessorFactory defaultEventProcessor() { + return defaultEventProcessorFactory; + } + + /** + * Returns a factory for a null implementation of {@link EventProcessor}, which will discard + * all analytics events and not send them to LaunchDarkly, regardless of any other configuration. + */ + public static EventProcessorFactory nullEventProcessor() { + return nullEventProcessorFactory; + } + + /** + * Returns a factory for the default implementation of {@link UpdateProcessor}, which receives + * feature flag data from LaunchDarkly using either streaming or polling as configured (or does + * nothing if the client is offline, or in LDD mode). + */ + public static UpdateProcessorFactory defaultUpdateProcessor() { + return defaultUpdateProcessorFactory; + } + + /** + * Returns a factory for a null implementation of {@link UpdateProcessor}, which does not + * connect to LaunchDarkly, regardless of any other configuration. + */ + public static UpdateProcessorFactory nullUpdateProcessor() { + return nullUpdateProcessorFactory; + } + + private static final class InMemoryFeatureStoreFactory implements FeatureStoreFactory { + @Override + public FeatureStore createFeatureStore() { + return new InMemoryFeatureStore(); + } + } + + private static final class DefaultEventProcessorFactory implements EventProcessorFactory { + @Override + public EventProcessor createEventProcessor(String sdkKey, LDConfig config) { + if (config.offline || !config.sendEvents) { + return new EventProcessor.NullEventProcessor(); + } else { + return new DefaultEventProcessor(sdkKey, config); + } + } + } + + private static final class NullEventProcessorFactory implements EventProcessorFactory { + public EventProcessor createEventProcessor(String sdkKey, LDConfig config) { + return new EventProcessor.NullEventProcessor(); + } + } + + private static final class DefaultUpdateProcessorFactory implements UpdateProcessorFactory { + // Note, logger uses LDClient class name for backward compatibility + private static final Logger logger = LoggerFactory.getLogger(LDClient.class); + + @Override + public UpdateProcessor createUpdateProcessor(String sdkKey, LDConfig config, FeatureStore featureStore) { + if (config.offline) { + logger.info("Starting LaunchDarkly client in offline mode"); + return new UpdateProcessor.NullUpdateProcessor(); + } else if (config.useLdd) { + logger.info("Starting LaunchDarkly in LDD mode. Skipping direct feature retrieval."); + return new UpdateProcessor.NullUpdateProcessor(); + } else { + FeatureRequestor requestor = new FeatureRequestor(sdkKey, config); + if (config.stream) { + logger.info("Enabling streaming API"); + return new StreamProcessor(sdkKey, config, requestor, featureStore); + } else { + logger.info("Disabling streaming API"); + logger.warn("You should only disable the streaming API if instructed to do so by LaunchDarkly support"); + return new PollingProcessor(config, requestor, featureStore); + } + } + } + } + + private static final class NullUpdateProcessorFactory implements UpdateProcessorFactory { + @Override + public UpdateProcessor createUpdateProcessor(String sdkKey, LDConfig config, FeatureStore featureStore) { + return new UpdateProcessor.NullUpdateProcessor(); + } + } +} diff --git a/src/main/java/com/launchdarkly/client/EventProcessor.java b/src/main/java/com/launchdarkly/client/EventProcessor.java index 1517269c8..fd75598ef 100644 --- a/src/main/java/com/launchdarkly/client/EventProcessor.java +++ b/src/main/java/com/launchdarkly/client/EventProcessor.java @@ -4,8 +4,9 @@ /** * Interface for an object that can send or store analytics events. + * @since 4.0.0 */ -interface EventProcessor extends Closeable { +public interface EventProcessor extends Closeable { /** * Records an event asynchronously. * @param e an event @@ -23,7 +24,7 @@ interface EventProcessor extends Closeable { /** * Stub implementation of {@link EventProcessor} for when we don't want to send any events. */ - static class NullEventProcessor implements EventProcessor { + static final class NullEventProcessor implements EventProcessor { @Override public void sendEvent(Event e) { } diff --git a/src/main/java/com/launchdarkly/client/EventProcessorFactory.java b/src/main/java/com/launchdarkly/client/EventProcessorFactory.java new file mode 100644 index 000000000..3d76b5aad --- /dev/null +++ b/src/main/java/com/launchdarkly/client/EventProcessorFactory.java @@ -0,0 +1,16 @@ +package com.launchdarkly.client; + +/** + * Interface for a factory that creates some implementation of {@link EventProcessor}. + * @see Components + * @since 4.0.0 + */ +public interface EventProcessorFactory { + /** + * Creates an implementation instance. + * @param sdkKey the SDK key for your LaunchDarkly environment + * @param config the LaunchDarkly configuration + * @return an {@link EventProcessor} + */ + EventProcessor createEventProcessor(String sdkKey, LDConfig config); +} diff --git a/src/main/java/com/launchdarkly/client/FeatureStoreFactory.java b/src/main/java/com/launchdarkly/client/FeatureStoreFactory.java new file mode 100644 index 000000000..c019de9c9 --- /dev/null +++ b/src/main/java/com/launchdarkly/client/FeatureStoreFactory.java @@ -0,0 +1,14 @@ +package com.launchdarkly.client; + +/** + * Interface for a factory that creates some implementation of {@link FeatureStore}. + * @see Components + * @since 4.0.0 + */ +public interface FeatureStoreFactory { + /** + * Creates an implementation instance. + * @return a {@link FeatureStore} + */ + FeatureStore createFeatureStore(); +} diff --git a/src/main/java/com/launchdarkly/client/LDClient.java b/src/main/java/com/launchdarkly/client/LDClient.java index 4f4e8f81d..b37573706 100644 --- a/src/main/java/com/launchdarkly/client/LDClient.java +++ b/src/main/java/com/launchdarkly/client/LDClient.java @@ -39,6 +39,8 @@ public class LDClient implements LDClientInterface { private final String sdkKey; final EventProcessor eventProcessor; final UpdateProcessor updateProcessor; + final FeatureStore featureStore; + final boolean shouldCloseFeatureStore; private final EventFactory eventFactory = EventFactory.DEFAULT; /** @@ -62,6 +64,21 @@ public LDClient(String sdkKey, LDConfig config) { this.config = config; this.sdkKey = sdkKey; + if (config.deprecatedFeatureStore != null) { + this.featureStore = config.deprecatedFeatureStore; + // The following line is for backward compatibility with the obsolete mechanism by which the + // caller could pass in a FeatureStore implementation instance that we did not create. We + // were not disposing of that instance when the client was closed, so we should continue not + // doing so until the next major version eliminates that mechanism. We will always dispose + // of instances that we created ourselves from a factory. + this.shouldCloseFeatureStore = false; + } else { + FeatureStoreFactory factory = config.featureStoreFactory == null ? + Components.inMemoryFeatureStore() : config.featureStoreFactory; + this.featureStore = factory.createFeatureStore(); + this.shouldCloseFeatureStore = true; + } + this.eventProcessor = createEventProcessor(sdkKey, config); if (config.offline) { @@ -70,10 +87,12 @@ public LDClient(String sdkKey, LDConfig config) { logger.info("Starting LaunchDarkly in LDD mode. Skipping direct feature retrieval."); } - this.updateProcessor = createUpdateProcessor(sdkKey, config); + this.updateProcessor = createUpdateProcessor(sdkKey, config, featureStore); Future startFuture = updateProcessor.start(); - if (!config.offline && !config.useLdd && config.startWaitMillis > 0L) { - logger.info("Waiting up to " + config.startWaitMillis + " milliseconds for LaunchDarkly client to start..."); + if (config.startWaitMillis > 0L) { + if (!config.offline && !config.useLdd) { + logger.info("Waiting up to " + config.startWaitMillis + " milliseconds for LaunchDarkly client to start..."); + } try { startFuture.get(config.startWaitMillis, TimeUnit.MILLISECONDS); } catch (TimeoutException e) { @@ -91,28 +110,16 @@ public boolean initialized() { @VisibleForTesting protected EventProcessor createEventProcessor(String sdkKey, LDConfig config) { - if (config.offline || !config.sendEvents) { - return new EventProcessor.NullEventProcessor(); - } else { - return new DefaultEventProcessor(sdkKey, config); - } + EventProcessorFactory factory = config.eventProcessorFactory == null ? + Components.defaultEventProcessor() : config.eventProcessorFactory; + return factory.createEventProcessor(sdkKey, config); } @VisibleForTesting - protected UpdateProcessor createUpdateProcessor(String sdkKey, LDConfig config) { - if (config.offline || config.useLdd) { - return new UpdateProcessor.NullUpdateProcessor(); - } else { - FeatureRequestor requestor = new FeatureRequestor(sdkKey, config); - if (config.stream) { - logger.info("Enabling streaming API"); - return new StreamProcessor(sdkKey, config, requestor); - } else { - logger.info("Disabling streaming API"); - logger.warn("You should only disable the streaming API if instructed to do so by LaunchDarkly support"); - return new PollingProcessor(config, requestor); - } - } + protected UpdateProcessor createUpdateProcessor(String sdkKey, LDConfig config, FeatureStore featureStore) { + UpdateProcessorFactory factory = config.updateProcessorFactory == null ? + Components.defaultUpdateProcessor() : config.updateProcessorFactory; + return factory.createUpdateProcessor(sdkKey, config, featureStore); } @Override @@ -154,7 +161,7 @@ public Map allFlags(LDUser user) { } if (!initialized()) { - if (config.featureStore.initialized()) { + if (featureStore.initialized()) { logger.warn("allFlags() was called before client initialized; using last known values from feature store"); } else { logger.warn("allFlags() was called before client initialized; feature store unavailable, returning null"); @@ -167,12 +174,12 @@ public Map allFlags(LDUser user) { return null; } - Map flags = this.config.featureStore.all(FEATURES); + Map flags = featureStore.all(FEATURES); Map result = new HashMap<>(); for (Map.Entry entry : flags.entrySet()) { try { - JsonElement evalResult = entry.getValue().evaluate(user, config.featureStore, eventFactory).getResult().getValue(); + JsonElement evalResult = entry.getValue().evaluate(user, featureStore, eventFactory).getResult().getValue(); result.put(entry.getKey(), evalResult); } catch (EvaluationException e) { @@ -215,7 +222,7 @@ public JsonElement jsonVariation(String featureKey, LDUser user, JsonElement def @Override public boolean isFlagKnown(String featureKey) { if (!initialized()) { - if (config.featureStore.initialized()) { + if (featureStore.initialized()) { logger.warn("isFlagKnown called before client initialized for feature flag " + featureKey + "; using last known values from feature store"); } else { logger.warn("isFlagKnown called before client initialized for feature flag " + featureKey + "; feature store unavailable, returning false"); @@ -224,7 +231,7 @@ public boolean isFlagKnown(String featureKey) { } try { - if (config.featureStore.get(FEATURES, featureKey) != null) { + if (featureStore.get(FEATURES, featureKey) != null) { return true; } } catch (Exception e) { @@ -244,7 +251,7 @@ private JsonElement evaluate(String featureKey, LDUser user, JsonElement default logger.warn("User key is blank. Flag evaluation will proceed, but the user will not be stored in LaunchDarkly"); } if (!initialized()) { - if (config.featureStore.initialized()) { + if (featureStore.initialized()) { logger.warn("Evaluation called before client initialized for feature flag " + featureKey + "; using last known values from feature store"); } else { logger.warn("Evaluation called before client initialized for feature flag " + featureKey + "; feature store unavailable, returning default value"); @@ -254,13 +261,13 @@ private JsonElement evaluate(String featureKey, LDUser user, JsonElement default } try { - FeatureFlag featureFlag = config.featureStore.get(FEATURES, featureKey); + FeatureFlag featureFlag = featureStore.get(FEATURES, featureKey); if (featureFlag == null) { logger.info("Unknown feature flag " + featureKey + "; returning default value"); sendFlagRequestEvent(eventFactory.newUnknownFeatureRequestEvent(featureKey, user, defaultValue)); return defaultValue; } - FeatureFlag.EvalResult evalResult = featureFlag.evaluate(user, config.featureStore, eventFactory); + FeatureFlag.EvalResult evalResult = featureFlag.evaluate(user, featureStore, eventFactory); for (FeatureRequestEvent event : evalResult.getPrerequisiteEvents()) { eventProcessor.sendEvent(event); } @@ -279,6 +286,9 @@ private JsonElement evaluate(String featureKey, LDUser user, JsonElement default @Override public void close() throws IOException { logger.info("Closing LaunchDarkly Client"); + if (shouldCloseFeatureStore) { // see comment in constructor about this variable + this.featureStore.close(); + } this.eventProcessor.close(); this.updateProcessor.close(); if (this.config.httpClient != null) { diff --git a/src/main/java/com/launchdarkly/client/LDConfig.java b/src/main/java/com/launchdarkly/client/LDConfig.java index 1ee6ac423..7e42db4ff 100644 --- a/src/main/java/com/launchdarkly/client/LDConfig.java +++ b/src/main/java/com/launchdarkly/client/LDConfig.java @@ -59,7 +59,10 @@ public final class LDConfig { final Authenticator proxyAuthenticator; final OkHttpClient httpClient; final boolean stream; - final FeatureStore featureStore; + final FeatureStore deprecatedFeatureStore; + final FeatureStoreFactory featureStoreFactory; + final EventProcessorFactory eventProcessorFactory; + final UpdateProcessorFactory updateProcessorFactory; final boolean useLdd; final boolean offline; final boolean allAttributesPrivate; @@ -84,7 +87,10 @@ protected LDConfig(Builder builder) { this.proxyAuthenticator = builder.proxyAuthenticator(); this.streamURI = builder.streamURI; this.stream = builder.stream; - this.featureStore = builder.featureStore; + this.deprecatedFeatureStore = builder.featureStore; + this.featureStoreFactory = builder.featureStoreFactory; + this.eventProcessorFactory = builder.eventProcessorFactory; + this.updateProcessorFactory = builder.updateProcessorFactory; this.useLdd = builder.useLdd; this.offline = builder.offline; this.allAttributesPrivate = builder.allAttributesPrivate; @@ -166,7 +172,10 @@ public static class Builder { private boolean allAttributesPrivate = false; private boolean sendEvents = true; private long pollingIntervalMillis = MIN_POLLING_INTERVAL_MILLIS; - private FeatureStore featureStore = new InMemoryFeatureStore(); + private FeatureStore featureStore = null; + private FeatureStoreFactory featureStoreFactory = Components.inMemoryFeatureStore(); + private EventProcessorFactory eventProcessorFactory = Components.defaultEventProcessor(); + private UpdateProcessorFactory updateProcessorFactory = Components.defaultUpdateProcessor(); private long startWaitMillis = DEFAULT_START_WAIT_MILLIS; private int samplingInterval = DEFAULT_SAMPLING_INTERVAL; private long reconnectTimeMillis = DEFAULT_RECONNECT_TIME_MILLIS; @@ -220,12 +229,53 @@ public Builder streamURI(URI streamURI) { * you may use {@link RedisFeatureStore} or a custom implementation. * @param store the feature store implementation * @return the builder + * @deprecated Please use {@link #featureStoreFactory}. */ public Builder featureStore(FeatureStore store) { this.featureStore = store; return this; } + /** + * Sets the implementation of {@link FeatureStore} to be used for holding feature flags and + * related data received from LaunchDarkly, using a factory object. The default is + * {@link Components#inMemoryFeatureStore()}, but you may use {@link Components#redisFeatureStore()} + * or a custom implementation. + * @param factory the factory object + * @return the builder + * @since 4.0.0 + */ + public Builder featureStoreFactory(FeatureStoreFactory factory) { + this.featureStoreFactory = factory; + return this; + } + + /** + * Sets the implementation of {@link EventProcessor} to be used for processing analytics events, + * using a factory object. The default is {@link Components#defaultEventProcessor()}, but + * you may choose to use a custom implementation (for instance, a test fixture). + * @param factory the factory object + * @return the builder + * @since 4.0.0 + */ + public Builder eventProcessorFactory(EventProcessorFactory factory) { + this.eventProcessorFactory = factory; + return this; + } + + /** + * Sets the implementation of {@link UpdateProcessor} to be used for receiving feature flag data, + * using a factory object. The default is {@link Components#defaultUpdateProcessor()}, but + * you may choose to use a custom implementation (for instance, a test fixture). + * @param factory the factory object + * @return the builder + * @since 4.0.0 + */ + public Builder updateProcessorFactory(UpdateProcessorFactory factory) { + this.updateProcessorFactory = factory; + return this; + } + /** * Set whether streaming mode should be enabled. By default, streaming is enabled. It should only be * disabled on the advice of LaunchDarkly support. diff --git a/src/main/java/com/launchdarkly/client/PollingProcessor.java b/src/main/java/com/launchdarkly/client/PollingProcessor.java index e3a1748df..1797ab474 100644 --- a/src/main/java/com/launchdarkly/client/PollingProcessor.java +++ b/src/main/java/com/launchdarkly/client/PollingProcessor.java @@ -18,10 +18,10 @@ class PollingProcessor implements UpdateProcessor { private AtomicBoolean initialized = new AtomicBoolean(false); private ScheduledExecutorService scheduler = null; - PollingProcessor(LDConfig config, FeatureRequestor requestor) { + PollingProcessor(LDConfig config, FeatureRequestor requestor, FeatureStore featureStore) { this.requestor = requestor; this.config = config; - this.store = config.featureStore; + this.store = featureStore; } @Override diff --git a/src/main/java/com/launchdarkly/client/RedisFeatureStore.java b/src/main/java/com/launchdarkly/client/RedisFeatureStore.java index 90e91b3f3..997c99fc7 100644 --- a/src/main/java/com/launchdarkly/client/RedisFeatureStore.java +++ b/src/main/java/com/launchdarkly/client/RedisFeatureStore.java @@ -1,19 +1,5 @@ package com.launchdarkly.client; -import static com.launchdarkly.client.VersionedDataKind.FEATURES; - -import java.io.IOException; -import java.net.URI; -import java.util.HashMap; -import java.util.Map; -import java.util.concurrent.ExecutorService; -import java.util.concurrent.Executors; -import java.util.concurrent.ThreadFactory; -import java.util.concurrent.TimeUnit; - -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - import com.google.common.base.Optional; import com.google.common.cache.CacheBuilder; import com.google.common.cache.CacheLoader; @@ -24,6 +10,19 @@ import com.google.common.util.concurrent.ThreadFactoryBuilder; import com.google.gson.Gson; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.io.IOException; +import java.util.HashMap; +import java.util.Map; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; +import java.util.concurrent.ThreadFactory; +import java.util.concurrent.TimeUnit; + +import static com.launchdarkly.client.VersionedDataKind.FEATURES; + import redis.clients.jedis.Jedis; import redis.clients.jedis.JedisPool; import redis.clients.jedis.JedisPoolConfig; @@ -35,7 +34,6 @@ */ public class RedisFeatureStore implements FeatureStore { private static final Logger logger = LoggerFactory.getLogger(RedisFeatureStore.class); - private static final String DEFAULT_PREFIX = "launchdarkly"; private static final String INIT_KEY = "$initialized$"; private static final String CACHE_REFRESH_THREAD_POOL_NAME_FORMAT = "RedisFeatureStore-cache-refresher-pool-%d"; private final JedisPool pool; @@ -82,28 +80,19 @@ protected RedisFeatureStore(RedisFeatureStoreBuilder builder) { } else { this.pool = new JedisPool(builder.poolConfig, builder.uri, builder.connectTimeout, builder.socketTimeout); } - setPrefix(builder.prefix); + this.prefix = (builder.prefix == null || builder.prefix.isEmpty()) ? + RedisFeatureStoreBuilder.DEFAULT_PREFIX : + builder.prefix; createCache(builder.cacheTimeSecs, builder.refreshStaleValues, builder.asyncRefresh); } /** * Creates a new store instance that connects to Redis with a default connection (localhost port 6379) and no in-memory cache. + * @deprecated Please use {@link Components#redisFeatureStore()} instead. */ public RedisFeatureStore() { pool = new JedisPool(getPoolConfig(), "localhost"); - this.prefix = DEFAULT_PREFIX; - } - - private void setPrefix(String prefix) { - if (prefix == null || prefix.isEmpty()) { - this.prefix = DEFAULT_PREFIX; - } else { - this.prefix = prefix; - } - } - - private void createCache(long cacheTimeSecs) { - createCache(cacheTimeSecs, false, false); + this.prefix = RedisFeatureStoreBuilder.DEFAULT_PREFIX; } private void createCache(long cacheTimeSecs, boolean refreshStaleValues, boolean asyncRefresh) { diff --git a/src/main/java/com/launchdarkly/client/RedisFeatureStoreBuilder.java b/src/main/java/com/launchdarkly/client/RedisFeatureStoreBuilder.java index d5c59d101..7cb9ed5f6 100644 --- a/src/main/java/com/launchdarkly/client/RedisFeatureStoreBuilder.java +++ b/src/main/java/com/launchdarkly/client/RedisFeatureStoreBuilder.java @@ -23,162 +23,200 @@ * .build() * */ -public class RedisFeatureStoreBuilder { - private static final Logger logger = LoggerFactory.getLogger(RedisFeatureStoreBuilder.class); - protected boolean refreshStaleValues = false; - protected boolean asyncRefresh = false; - protected URI uri; - protected String prefix; - protected int connectTimeout = Protocol.DEFAULT_TIMEOUT; - protected int socketTimeout = Protocol.DEFAULT_TIMEOUT; - protected long cacheTimeSecs; - protected JedisPoolConfig poolConfig; +public final class RedisFeatureStoreBuilder implements FeatureStoreFactory { + private static final Logger logger = LoggerFactory.getLogger(RedisFeatureStoreBuilder.class); + + /** + * The default value for the Redis URI: {@code redis://localhost:6379} + * @since 4.0.0 + */ + public static final URI DEFAULT_URI = URI.create("redis://localhost:6379"); + + /** + * The default value for {@link #prefix(String)}. + * @since 4.0.0 + */ + public static final String DEFAULT_PREFIX = "launchdarkly"; + + /** + * The default value for {@link #cacheTime(long, TimeUnit)} (in seconds). + * @since 4.0.0 + */ + public static final long DEFAULT_CACHE_TIME_SECONDS = 15; + + final URI uri; + boolean refreshStaleValues = false; + boolean asyncRefresh = false; + String prefix = DEFAULT_PREFIX; + int connectTimeout = Protocol.DEFAULT_TIMEOUT; + int socketTimeout = Protocol.DEFAULT_TIMEOUT; + long cacheTimeSecs = DEFAULT_CACHE_TIME_SECONDS; + JedisPoolConfig poolConfig = null; - /** - * The constructor accepts the mandatory fields that must be specified at a minimum to construct a {@link com.launchdarkly.client.RedisFeatureStore}. - * - * @param uri the uri of the Redis resource to connect to. - * @param cacheTimeSecs the cache time in seconds. See {@link RedisFeatureStoreBuilder#cacheTime(long, TimeUnit)} for more information. - */ - public RedisFeatureStoreBuilder(URI uri, long cacheTimeSecs) { - this.uri = uri; - this.cacheTimeSecs = cacheTimeSecs; - } + // These constructors are called only from Implementations + RedisFeatureStoreBuilder() { + this.uri = DEFAULT_URI; + } + + RedisFeatureStoreBuilder(URI uri) { + this.uri = uri; + } + + /** + * The constructor accepts the mandatory fields that must be specified at a minimum to construct a {@link com.launchdarkly.client.RedisFeatureStore}. + * + * @param uri the uri of the Redis resource to connect to. + * @param cacheTimeSecs the cache time in seconds. See {@link RedisFeatureStoreBuilder#cacheTime(long, TimeUnit)} for more information. + * @deprecated Please use {@link Components#redisFeatureStore(java.net.URI)}. + */ + public RedisFeatureStoreBuilder(URI uri, long cacheTimeSecs) { + this.uri = uri; + this.cacheTime(cacheTimeSecs, TimeUnit.SECONDS); + } - /** - * The constructor accepts the mandatory fields that must be specified at a minimum to construct a {@link com.launchdarkly.client.RedisFeatureStore}. - * - * @param scheme the URI scheme to use - * @param host the hostname to connect to - * @param port the port to connect to - * @param cacheTimeSecs the cache time in seconds. See {@link RedisFeatureStoreBuilder#cacheTime(long, TimeUnit)} for more information. - * @throws URISyntaxException if the URI is not valid - */ - public RedisFeatureStoreBuilder(String scheme, String host, int port, long cacheTimeSecs) throws URISyntaxException { - this.uri = new URI(scheme, null, host, port, null, null, null); - this.cacheTimeSecs = cacheTimeSecs; - } + /** + * The constructor accepts the mandatory fields that must be specified at a minimum to construct a {@link com.launchdarkly.client.RedisFeatureStore}. + * + * @param scheme the URI scheme to use + * @param host the hostname to connect to + * @param port the port to connect to + * @param cacheTimeSecs the cache time in seconds. See {@link RedisFeatureStoreBuilder#cacheTime(long, TimeUnit)} for more information. + * @throws URISyntaxException if the URI is not valid + * @deprecated Please use {@link Components#redisFeatureStore(java.net.URI)}. + */ + public RedisFeatureStoreBuilder(String scheme, String host, int port, long cacheTimeSecs) throws URISyntaxException { + this.uri = new URI(scheme, null, host, port, null, null, null); + this.cacheTime(cacheTimeSecs, TimeUnit.SECONDS); + } + + /** + * Optionally set the {@link RedisFeatureStore} local cache to refresh stale values instead of evicting them (the default behaviour). + * + * When enabled; the cache refreshes stale values instead of completely evicting them. This mode returns the previously cached, stale values if + * anything goes wrong during the refresh phase (for example a connection timeout). If there was no previously cached value then the store will + * return null (resulting in the default value being returned). This is useful if you prefer the most recently cached feature rule set to be returned + * for evaluation over the default value when updates go wrong. + * + * When disabled; results in a behaviour which evicts stale values from the local cache and retrieves the latest value from Redis. If the updated value + * can not be returned for whatever reason then a null is returned (resulting in the default value being returned). + * + * This property has no effect if the cache time is set to 0. See {@link RedisFeatureStoreBuilder#cacheTime(long, TimeUnit)} for details. + * + * See: CacheBuilder for more specific information on cache semantics. + * + * @param enabled turns on lazy refresh of cached values. + * @return the builder + */ + public RedisFeatureStoreBuilder refreshStaleValues(boolean enabled) { + this.refreshStaleValues = enabled; + return this; + } - /** - * Optionally set the {@link RedisFeatureStore} local cache to refresh stale values instead of evicting them (the default behaviour). - * - * When enabled; the cache refreshes stale values instead of completely evicting them. This mode returns the previously cached, stale values if - * anything goes wrong during the refresh phase (for example a connection timeout). If there was no previously cached value then the store will - * return null (resulting in the default value being returned). This is useful if you prefer the most recently cached feature rule set to be returned - * for evaluation over the default value when updates go wrong. - * - * When disabled; results in a behaviour which evicts stale values from the local cache and retrieves the latest value from Redis. If the updated value - * can not be returned for whatever reason then a null is returned (resulting in the default value being returned). - * - * This property has no effect if the cache time is set to 0. See {@link RedisFeatureStoreBuilder#cacheTime(long, TimeUnit)} for details. - * - * See: CacheBuilder for more specific information on cache semantics. - * - * @param enabled turns on lazy refresh of cached values. - * @return the builder - */ - public RedisFeatureStoreBuilder refreshStaleValues(boolean enabled) { - this.refreshStaleValues = enabled; - return this; - } + /** + * Optionally make cache refresh mode asynchronous. This setting only works if {@link RedisFeatureStoreBuilder#refreshStaleValues(boolean)} has been enabled + * and has no effect otherwise. + * + * Upon hitting a stale value in the local cache; the refresh of the value will be asynchronous which will return the previously cached value in a + * non-blocking fashion to threads requesting the stale key. This internally will utilize a {@link java.util.concurrent.Executor} to asynchronously + * refresh the stale value upon the first read request for the stale value in the cache. + * + * If there was no previously cached value then the feature store returns null (resulting in the default value being returned). Any exception + * encountered during the asynchronous reload will simply keep the previously cached value instead. + * + * This setting is ideal to enable when you desire high performance reads and can accept returning stale values for the period of the async refresh. For + * example configuring this feature store with a very low cache time and enabling this feature would see great performance benefit by decoupling calls + * from network I/O. + * + * This property has no effect if the cache time is set to 0. See {@link RedisFeatureStoreBuilder#cacheTime(long, TimeUnit)} for details. + * + * @param enabled turns on asychronous refreshes on. + * @return the builder + */ + public RedisFeatureStoreBuilder asyncRefresh(boolean enabled) { + this.asyncRefresh = enabled; + return this; + } - /** - * Optionally make cache refresh mode asynchronous. This setting only works if {@link RedisFeatureStoreBuilder#refreshStaleValues(boolean)} has been enabled - * and has no effect otherwise. - * - * Upon hitting a stale value in the local cache; the refresh of the value will be asynchronous which will return the previously cached value in a - * non-blocking fashion to threads requesting the stale key. This internally will utilize a {@link java.util.concurrent.Executor} to asynchronously - * refresh the stale value upon the first read request for the stale value in the cache. - * - * If there was no previously cached value then the feature store returns null (resulting in the default value being returned). Any exception - * encountered during the asynchronous reload will simply keep the previously cached value instead. - * - * This setting is ideal to enable when you desire high performance reads and can accept returning stale values for the period of the async refresh. For - * example configuring this feature store with a very low cache time and enabling this feature would see great performance benefit by decoupling calls - * from network I/O. - * - * This property has no effect if the cache time is set to 0. See {@link RedisFeatureStoreBuilder#cacheTime(long, TimeUnit)} for details. - * - * @param enabled turns on asychronous refreshes on. - * @return the builder - */ - public RedisFeatureStoreBuilder asyncRefresh(boolean enabled) { - this.asyncRefresh = enabled; - return this; - } + /** + * Optionally configures the namespace prefix for all keys stored in Redis. + * + * @param prefix the namespace prefix + * @return the builder + */ + public RedisFeatureStoreBuilder prefix(String prefix) { + this.prefix = prefix; + return this; + } - /** - * Optionally configures the namespace prefix for all keys stored in Redis. - * - * @param prefix the namespace prefix - * @return the builder - */ - public RedisFeatureStoreBuilder prefix(String prefix) { - this.prefix = prefix; - return this; - } + /** + * A mandatory field which configures the amount of time the store should internally cache the value before being marked invalid. + * + * The eviction strategy of stale values is determined by the configuration picked. See {@link RedisFeatureStoreBuilder#refreshStaleValues(boolean)} for + * more information on stale value updating strategies. + * + * If this value is set to 0 then it effectively disables local caching altogether. + * + * @param cacheTime the time value to cache for + * @param timeUnit the time unit for the time value. This is used to convert your time value to seconds. + * @return the builder + */ + public RedisFeatureStoreBuilder cacheTime(long cacheTime, TimeUnit timeUnit) { + this.cacheTimeSecs = timeUnit.toSeconds(cacheTime); + return this; + } - /** - * A mandatory field which configures the amount of time the store should internally cache the value before being marked invalid. - * - * The eviction strategy of stale values is determined by the configuration picked. See {@link RedisFeatureStoreBuilder#refreshStaleValues(boolean)} for - * more information on stale value updating strategies. - * - * If this value is set to 0 then it effectively disables local caching altogether. - * - * @param cacheTime the time value to cache for - * @param timeUnit the time unit for the time value. This is used to convert your time value to seconds. - * @return the builder - */ - public RedisFeatureStoreBuilder cacheTime(long cacheTime, TimeUnit timeUnit) { - this.cacheTimeSecs = timeUnit.toSeconds(cacheTime); - return this; - } + /** + * Optional override if you wish to specify your own configuration to the underlying Jedis pool. + * + * @param poolConfig the Jedis pool configuration. + * @return the builder + */ + public RedisFeatureStoreBuilder poolConfig(JedisPoolConfig poolConfig) { + this.poolConfig = poolConfig; + return this; + } - /** - * Optional override if you wish to specify your own configuration to the underlying Jedis pool. - * - * @param poolConfig the Jedis pool configuration. - * @return the builder - */ - public RedisFeatureStoreBuilder poolConfig(JedisPoolConfig poolConfig) { - this.poolConfig = poolConfig; - return this; - } + /** + * Optional override which sets the connection timeout for the underlying Jedis pool which otherwise defaults to + * {@link redis.clients.jedis.Protocol#DEFAULT_TIMEOUT} + * + * @param connectTimeout the timeout + * @param timeUnit the time unit for the timeout + * @return the builder + */ + public RedisFeatureStoreBuilder connectTimeout(int connectTimeout, TimeUnit timeUnit) { + this.connectTimeout = (int) timeUnit.toMillis(connectTimeout); + return this; + } - /** - * Optional override which sets the connection timeout for the underlying Jedis pool which otherwise defaults to - * {@link redis.clients.jedis.Protocol#DEFAULT_TIMEOUT} - * - * @param connectTimeout the timeout - * @param timeUnit the time unit for the timeout - * @return the builder - */ - public RedisFeatureStoreBuilder connectTimeout(int connectTimeout, TimeUnit timeUnit) { - this.connectTimeout = (int) timeUnit.toMillis(connectTimeout); - return this; - } + /** + * Optional override which sets the connection timeout for the underlying Jedis pool which otherwise defaults to + * {@link redis.clients.jedis.Protocol#DEFAULT_TIMEOUT} + * + * @param socketTimeout the socket timeout + * @param timeUnit the time unit for the timeout + * @return the builder + */ + public RedisFeatureStoreBuilder socketTimeout(int socketTimeout, TimeUnit timeUnit) { + this.socketTimeout = (int) timeUnit.toMillis(socketTimeout); + return this; + } - /** - * Optional override which sets the connection timeout for the underlying Jedis pool which otherwise defaults to - * {@link redis.clients.jedis.Protocol#DEFAULT_TIMEOUT} - * - * @param socketTimeout the socket timeout - * @param timeUnit the time unit for the timeout - * @return the builder - */ - public RedisFeatureStoreBuilder socketTimeout(int socketTimeout, TimeUnit timeUnit) { - this.socketTimeout = (int) timeUnit.toMillis(socketTimeout); - return this; - } - - /** - * Build a {@link RedisFeatureStore} based on the currently configured builder object. - * - * @return the {@link RedisFeatureStore} configured by this builder. - */ - public RedisFeatureStore build() { - logger.info("Creating RedisFeatureStore with uri: " + uri + " and prefix: " + prefix); - return new RedisFeatureStore(this); - } + /** + * Build a {@link RedisFeatureStore} based on the currently configured builder object. + * @return the {@link RedisFeatureStore} configured by this builder. + */ + public RedisFeatureStore build() { + logger.info("Creating RedisFeatureStore with uri: " + uri + " and prefix: " + prefix); + return new RedisFeatureStore(this); + } + + /** + * Synonym for {@link #build()}. + * @return the {@link RedisFeatureStore} configured by this builder. + * @since 4.0.0 + */ + public RedisFeatureStore createFeatureStore() { + return build(); + } } diff --git a/src/main/java/com/launchdarkly/client/StreamProcessor.java b/src/main/java/com/launchdarkly/client/StreamProcessor.java index 15fce2a04..713e928e3 100644 --- a/src/main/java/com/launchdarkly/client/StreamProcessor.java +++ b/src/main/java/com/launchdarkly/client/StreamProcessor.java @@ -40,8 +40,8 @@ class StreamProcessor implements UpdateProcessor { private AtomicBoolean initialized = new AtomicBoolean(false); - StreamProcessor(String sdkKey, LDConfig config, FeatureRequestor requestor) { - this.store = config.featureStore; + StreamProcessor(String sdkKey, LDConfig config, FeatureRequestor requestor, FeatureStore featureStore) { + this.store = featureStore; this.config = config; this.sdkKey = sdkKey; this.requestor = requestor; diff --git a/src/main/java/com/launchdarkly/client/UpdateProcessor.java b/src/main/java/com/launchdarkly/client/UpdateProcessor.java index 288e0b4f5..6d959a032 100644 --- a/src/main/java/com/launchdarkly/client/UpdateProcessor.java +++ b/src/main/java/com/launchdarkly/client/UpdateProcessor.java @@ -6,8 +6,12 @@ import static com.google.common.util.concurrent.Futures.immediateFuture; -interface UpdateProcessor extends Closeable { - +/** + * Interface for an object that receives updates to feature flags, user segments, and anything + * else that might come from LaunchDarkly, and passes them to a {@link FeatureStore}. + * @since 4.0.0 + */ +public interface UpdateProcessor extends Closeable { /** * Starts the client. * @return {@link Future}'s completion status indicates the client has been initialized. @@ -22,7 +26,7 @@ interface UpdateProcessor extends Closeable { void close() throws IOException; - static class NullUpdateProcessor implements UpdateProcessor { + static final class NullUpdateProcessor implements UpdateProcessor { @Override public Future start() { return immediateFuture(null); diff --git a/src/main/java/com/launchdarkly/client/UpdateProcessorFactory.java b/src/main/java/com/launchdarkly/client/UpdateProcessorFactory.java new file mode 100644 index 000000000..1b3a73e8d --- /dev/null +++ b/src/main/java/com/launchdarkly/client/UpdateProcessorFactory.java @@ -0,0 +1,17 @@ +package com.launchdarkly.client; + +/** + * Interface for a factory that creates some implementation of {@link UpdateProcessor}. + * @see Components + * @since 4.0.0 + */ +public interface UpdateProcessorFactory { + /** + * Creates an implementation instance. + * @param sdkKey the SDK key for your LaunchDarkly environment + * @param config the LaunchDarkly configuration + * @param featureStore the {@link FeatureStore} to use for storing the latest flag state + * @return an {@link UpdateProcessor} + */ + public UpdateProcessor createUpdateProcessor(String sdkKey, LDConfig config, FeatureStore featureStore); +} diff --git a/src/test/java/com/launchdarkly/client/LDClientEvaluationTest.java b/src/test/java/com/launchdarkly/client/LDClientEvaluationTest.java index 30ee44a96..f6d175dc0 100644 --- a/src/test/java/com/launchdarkly/client/LDClientEvaluationTest.java +++ b/src/test/java/com/launchdarkly/client/LDClientEvaluationTest.java @@ -3,25 +3,29 @@ import com.google.gson.JsonElement; import com.google.gson.JsonObject; -import org.easymock.EasyMockSupport; import org.junit.Test; import java.util.Arrays; import static com.launchdarkly.client.TestUtil.jint; import static com.launchdarkly.client.TestUtil.js; +import static com.launchdarkly.client.TestUtil.specificFeatureStore; import static com.launchdarkly.client.VersionedDataKind.FEATURES; import static com.launchdarkly.client.VersionedDataKind.SEGMENTS; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertFalse; import static org.junit.Assert.assertTrue; -public class LDClientEvaluationTest extends EasyMockSupport { +public class LDClientEvaluationTest { private static final LDUser user = new LDUser("userkey"); private TestFeatureStore featureStore = new TestFeatureStore(); - private LDConfig config = new LDConfig.Builder().featureStore(featureStore).build(); - private LDClientInterface client = createTestClient(config); + private LDConfig config = new LDConfig.Builder() + .featureStoreFactory(specificFeatureStore(featureStore)) + .eventProcessorFactory(Components.nullEventProcessor()) + .updateProcessorFactory(Components.nullUpdateProcessor()) + .build(); + private LDClientInterface client = new LDClient("SDK_KEY", config); @Test public void boolVariationReturnsFlagValue() throws Exception { @@ -108,18 +112,4 @@ public void canMatchUserBySegment() throws Exception { assertTrue(client.boolVariation("test-feature", user, false)); } - - private LDClientInterface createTestClient(LDConfig config) { - return new LDClient("SDK_KEY", config) { - @Override - protected UpdateProcessor createUpdateProcessor(String sdkKey, LDConfig config) { - return new UpdateProcessor.NullUpdateProcessor(); - } - - @Override - protected EventProcessor createEventProcessor(String sdkKey, LDConfig config) { - return new EventProcessor.NullEventProcessor(); - } - }; - } } diff --git a/src/test/java/com/launchdarkly/client/LDClientEventTest.java b/src/test/java/com/launchdarkly/client/LDClientEventTest.java index 965289653..990557e32 100644 --- a/src/test/java/com/launchdarkly/client/LDClientEventTest.java +++ b/src/test/java/com/launchdarkly/client/LDClientEventTest.java @@ -6,16 +6,15 @@ import org.junit.Test; -import java.io.IOException; -import java.util.ArrayList; import java.util.Arrays; -import java.util.List; import static com.launchdarkly.client.TestUtil.fallthroughVariation; import static com.launchdarkly.client.TestUtil.jbool; import static com.launchdarkly.client.TestUtil.jdouble; import static com.launchdarkly.client.TestUtil.jint; import static com.launchdarkly.client.TestUtil.js; +import static com.launchdarkly.client.TestUtil.specificEventProcessor; +import static com.launchdarkly.client.TestUtil.specificFeatureStore; import static com.launchdarkly.client.VersionedDataKind.FEATURES; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertNull; @@ -24,9 +23,13 @@ public class LDClientEventTest { private static final LDUser user = new LDUser("userkey"); private TestFeatureStore featureStore = new TestFeatureStore(); - private TestEventProcessor eventSink = new TestEventProcessor(); - private LDConfig config = new LDConfig.Builder().featureStore(featureStore).build(); - private LDClientInterface client = createTestClient(config); + private TestUtil.TestEventProcessor eventSink = new TestUtil.TestEventProcessor(); + private LDConfig config = new LDConfig.Builder() + .featureStoreFactory(specificFeatureStore(featureStore)) + .eventProcessorFactory(specificEventProcessor(eventSink)) + .updateProcessorFactory(Components.nullUpdateProcessor()) + .build(); + private LDClientInterface client = new LDClient("SDK_KEY", config); @Test public void identifySendsEvent() throws Exception { @@ -218,33 +221,4 @@ private void checkUnknownFeatureEvent(Event e, String key, JsonElement defaultVa assertEquals(defaultVal, fe.defaultVal); assertEquals(prereqOf, fe.prereqOf); } - - private static class TestEventProcessor implements EventProcessor { - List events = new ArrayList<>(); - - @Override - public void close() throws IOException {} - - @Override - public void sendEvent(Event e) { - events.add(e); - } - - @Override - public void flush() {} - } - - private LDClientInterface createTestClient(LDConfig config) { - return new LDClient("SDK_KEY", config) { - @Override - protected UpdateProcessor createUpdateProcessor(String sdkKey, LDConfig config) { - return new UpdateProcessor.NullUpdateProcessor(); - } - - @Override - protected EventProcessor createEventProcessor(String sdkKey, LDConfig config) { - return eventSink; - } - }; - } } diff --git a/src/test/java/com/launchdarkly/client/LDClientLddModeTest.java b/src/test/java/com/launchdarkly/client/LDClientLddModeTest.java index db37855f0..62a584c35 100644 --- a/src/test/java/com/launchdarkly/client/LDClientLddModeTest.java +++ b/src/test/java/com/launchdarkly/client/LDClientLddModeTest.java @@ -4,6 +4,7 @@ import java.io.IOException; +import static com.launchdarkly.client.TestUtil.specificFeatureStore; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertTrue; @@ -43,7 +44,7 @@ public void lddModeClientGetsFlagFromFeatureStore() throws IOException { TestFeatureStore testFeatureStore = new TestFeatureStore(); LDConfig config = new LDConfig.Builder() .useLdd(true) - .featureStore(testFeatureStore) + .featureStoreFactory(specificFeatureStore(testFeatureStore)) .build(); testFeatureStore.setFeatureTrue("key"); try (LDClient client = new LDClient("SDK_KEY", config)) { diff --git a/src/test/java/com/launchdarkly/client/LDClientOfflineTest.java b/src/test/java/com/launchdarkly/client/LDClientOfflineTest.java index 57821117e..5a8369f49 100644 --- a/src/test/java/com/launchdarkly/client/LDClientOfflineTest.java +++ b/src/test/java/com/launchdarkly/client/LDClientOfflineTest.java @@ -8,6 +8,7 @@ import java.io.IOException; import java.util.Map; +import static com.launchdarkly.client.TestUtil.specificFeatureStore; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertNotNull; import static org.junit.Assert.assertTrue; @@ -58,7 +59,7 @@ public void offlineClientGetsAllFlagsFromFeatureStore() throws IOException { TestFeatureStore testFeatureStore = new TestFeatureStore(); LDConfig config = new LDConfig.Builder() .offline(true) - .featureStore(testFeatureStore) + .featureStoreFactory(specificFeatureStore(testFeatureStore)) .build(); testFeatureStore.setFeatureTrue("key"); try (LDClient client = new LDClient("SDK_KEY", config)) { diff --git a/src/test/java/com/launchdarkly/client/LDClientTest.java b/src/test/java/com/launchdarkly/client/LDClientTest.java index b2701a34e..18add9a1a 100644 --- a/src/test/java/com/launchdarkly/client/LDClientTest.java +++ b/src/test/java/com/launchdarkly/client/LDClientTest.java @@ -10,6 +10,7 @@ import java.util.concurrent.TimeUnit; import java.util.concurrent.TimeoutException; +import static com.launchdarkly.client.TestUtil.specificFeatureStore; import static org.easymock.EasyMock.anyObject; import static org.easymock.EasyMock.expect; import static org.easymock.EasyMock.expectLastCall; @@ -88,9 +89,8 @@ public void pollingClientHasPollingProcessor() throws IOException { @Test public void noWaitForUpdateProcessorIfWaitMillisIsZero() throws Exception { - LDConfig config = new LDConfig.Builder() - .startWaitMillis(0L) - .build(); + LDConfig.Builder config = new LDConfig.Builder() + .startWaitMillis(0L); expect(updateProcessor.start()).andReturn(initFuture); expect(updateProcessor.initialized()).andReturn(false); @@ -104,9 +104,8 @@ public void noWaitForUpdateProcessorIfWaitMillisIsZero() throws Exception { @Test public void willWaitForUpdateProcessorIfWaitMillisIsNonZero() throws Exception { - LDConfig config = new LDConfig.Builder() - .startWaitMillis(10L) - .build(); + LDConfig.Builder config = new LDConfig.Builder() + .startWaitMillis(10L); expect(updateProcessor.start()).andReturn(initFuture); expect(initFuture.get(10L, TimeUnit.MILLISECONDS)).andReturn(null); @@ -121,9 +120,8 @@ public void willWaitForUpdateProcessorIfWaitMillisIsNonZero() throws Exception { @Test public void updateProcessorCanTimeOut() throws Exception { - LDConfig config = new LDConfig.Builder() - .startWaitMillis(10L) - .build(); + LDConfig.Builder config = new LDConfig.Builder() + .startWaitMillis(10L); expect(updateProcessor.start()).andReturn(initFuture); expect(initFuture.get(10L, TimeUnit.MILLISECONDS)).andThrow(new TimeoutException()); @@ -138,9 +136,8 @@ public void updateProcessorCanTimeOut() throws Exception { @Test public void clientCatchesRuntimeExceptionFromUpdateProcessor() throws Exception { - LDConfig config = new LDConfig.Builder() - .startWaitMillis(10L) - .build(); + LDConfig.Builder config = new LDConfig.Builder() + .startWaitMillis(10L); expect(updateProcessor.start()).andReturn(initFuture); expect(initFuture.get(10L, TimeUnit.MILLISECONDS)).andThrow(new RuntimeException()); @@ -157,10 +154,9 @@ public void clientCatchesRuntimeExceptionFromUpdateProcessor() throws Exception public void isFlagKnownReturnsTrueForExistingFlag() throws Exception { TestFeatureStore testFeatureStore = new TestFeatureStore(); testFeatureStore.setInitialized(true); - LDConfig config = new LDConfig.Builder() + LDConfig.Builder config = new LDConfig.Builder() .startWaitMillis(0) - .featureStore(testFeatureStore) - .build(); + .featureStoreFactory(specificFeatureStore(testFeatureStore)); expect(updateProcessor.start()).andReturn(initFuture); expect(updateProcessor.initialized()).andReturn(true).times(1); replayAll(); @@ -176,10 +172,9 @@ public void isFlagKnownReturnsTrueForExistingFlag() throws Exception { public void isFlagKnownReturnsFalseForUnknownFlag() throws Exception { TestFeatureStore testFeatureStore = new TestFeatureStore(); testFeatureStore.setInitialized(true); - LDConfig config = new LDConfig.Builder() + LDConfig.Builder config = new LDConfig.Builder() .startWaitMillis(0) - .featureStore(testFeatureStore) - .build(); + .featureStoreFactory(specificFeatureStore(testFeatureStore)); expect(updateProcessor.start()).andReturn(initFuture); expect(updateProcessor.initialized()).andReturn(true).times(1); replayAll(); @@ -194,10 +189,9 @@ public void isFlagKnownReturnsFalseForUnknownFlag() throws Exception { public void isFlagKnownReturnsFalseIfStoreAndClientAreNotInitialized() throws Exception { TestFeatureStore testFeatureStore = new TestFeatureStore(); testFeatureStore.setInitialized(false); - LDConfig config = new LDConfig.Builder() + LDConfig.Builder config = new LDConfig.Builder() .startWaitMillis(0) - .featureStore(testFeatureStore) - .build(); + .featureStoreFactory(specificFeatureStore(testFeatureStore)); expect(updateProcessor.start()).andReturn(initFuture); expect(updateProcessor.initialized()).andReturn(false).times(1); replayAll(); @@ -213,10 +207,9 @@ public void isFlagKnownReturnsFalseIfStoreAndClientAreNotInitialized() throws Ex public void isFlagKnownUsesStoreIfStoreIsInitializedButClientIsNot() throws Exception { TestFeatureStore testFeatureStore = new TestFeatureStore(); testFeatureStore.setInitialized(true); - LDConfig config = new LDConfig.Builder() + LDConfig.Builder config = new LDConfig.Builder() .startWaitMillis(0) - .featureStore(testFeatureStore) - .build(); + .featureStoreFactory(specificFeatureStore(testFeatureStore)); expect(updateProcessor.start()).andReturn(initFuture); expect(updateProcessor.initialized()).andReturn(false).times(1); replayAll(); @@ -232,10 +225,9 @@ public void isFlagKnownUsesStoreIfStoreIsInitializedButClientIsNot() throws Exce public void evaluationUsesStoreIfStoreIsInitializedButClientIsNot() throws Exception { TestFeatureStore testFeatureStore = new TestFeatureStore(); testFeatureStore.setInitialized(true); - LDConfig config = new LDConfig.Builder() - .featureStore(testFeatureStore) - .startWaitMillis(0L) - .build(); + LDConfig.Builder config = new LDConfig.Builder() + .featureStoreFactory(specificFeatureStore(testFeatureStore)) + .startWaitMillis(0L); expect(updateProcessor.start()).andReturn(initFuture); expect(updateProcessor.initialized()).andReturn(false); expectEventsSent(1); @@ -258,17 +250,9 @@ private void expectEventsSent(int count) { } } - private LDClientInterface createMockClient(LDConfig config) { - return new LDClient("SDK_KEY", config) { - @Override - protected UpdateProcessor createUpdateProcessor(String sdkKey, LDConfig config) { - return LDClientTest.this.updateProcessor; - } - - @Override - protected EventProcessor createEventProcessor(String sdkKey, LDConfig config) { - return LDClientTest.this.eventProcessor; - } - }; + private LDClientInterface createMockClient(LDConfig.Builder config) { + config.updateProcessorFactory(TestUtil.specificUpdateProcessor(updateProcessor)); + config.eventProcessorFactory(TestUtil.specificEventProcessor(eventProcessor)); + return new LDClient("SDK_KEY", config.build()); } } \ No newline at end of file diff --git a/src/test/java/com/launchdarkly/client/LDUserTest.java b/src/test/java/com/launchdarkly/client/LDUserTest.java index f80184dce..f57ce8d05 100644 --- a/src/test/java/com/launchdarkly/client/LDUserTest.java +++ b/src/test/java/com/launchdarkly/client/LDUserTest.java @@ -5,7 +5,6 @@ import com.google.gson.JsonPrimitive; import com.google.gson.reflect.TypeToken; -import org.junit.Assert; import org.junit.Test; import java.lang.reflect.Type; diff --git a/src/test/java/com/launchdarkly/client/PollingProcessorTest.java b/src/test/java/com/launchdarkly/client/PollingProcessorTest.java index fef22cb31..e42a26dd7 100644 --- a/src/test/java/com/launchdarkly/client/PollingProcessorTest.java +++ b/src/test/java/com/launchdarkly/client/PollingProcessorTest.java @@ -18,7 +18,7 @@ public class PollingProcessorTest extends EasyMockSupport { @Test public void testConnectionOk() throws Exception { FeatureRequestor requestor = createStrictMock(FeatureRequestor.class); - PollingProcessor pollingProcessor = new PollingProcessor(LDConfig.DEFAULT, requestor); + PollingProcessor pollingProcessor = new PollingProcessor(LDConfig.DEFAULT, requestor, new InMemoryFeatureStore()); expect(requestor.getAllData()) .andReturn(new FeatureRequestor.AllData(new HashMap(), new HashMap())) @@ -35,7 +35,7 @@ public void testConnectionOk() throws Exception { @Test public void testConnectionProblem() throws Exception { FeatureRequestor requestor = createStrictMock(FeatureRequestor.class); - PollingProcessor pollingProcessor = new PollingProcessor(LDConfig.DEFAULT, requestor); + PollingProcessor pollingProcessor = new PollingProcessor(LDConfig.DEFAULT, requestor, new InMemoryFeatureStore()); expect(requestor.getAllData()) .andThrow(new IOException("This exception is part of a test and yes you should be seeing it.")) diff --git a/src/test/java/com/launchdarkly/client/RedisFeatureStoreBuilderTest.java b/src/test/java/com/launchdarkly/client/RedisFeatureStoreBuilderTest.java index c11e1979d..e83f0a2ac 100644 --- a/src/test/java/com/launchdarkly/client/RedisFeatureStoreBuilderTest.java +++ b/src/test/java/com/launchdarkly/client/RedisFeatureStoreBuilderTest.java @@ -1,8 +1,6 @@ package com.launchdarkly.client; import org.junit.Test; -import redis.clients.jedis.JedisPoolConfig; -import redis.clients.jedis.Protocol; import java.net.URI; import java.net.URISyntaxException; @@ -11,75 +9,91 @@ import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertNull; -public class RedisFeatureStoreBuilderTest { +import redis.clients.jedis.JedisPoolConfig; +import redis.clients.jedis.Protocol; - @Test - public void testDefaultValues() throws URISyntaxException { - RedisFeatureStoreBuilder conf = new RedisFeatureStoreBuilder("http", "host", 1234, 1); - assertEquals(Protocol.DEFAULT_TIMEOUT, conf.connectTimeout); - assertEquals(Protocol.DEFAULT_TIMEOUT, conf.socketTimeout); - assertEquals(false, conf.refreshStaleValues); - assertEquals(false, conf.asyncRefresh); - assertNull(conf.poolConfig); - } +public class RedisFeatureStoreBuilderTest { + @Test + public void testDefaultValues() { + RedisFeatureStoreBuilder conf = new RedisFeatureStoreBuilder(); + assertEquals(RedisFeatureStoreBuilder.DEFAULT_URI, conf.uri); + assertEquals(RedisFeatureStoreBuilder.DEFAULT_CACHE_TIME_SECONDS, conf.cacheTimeSecs); + assertEquals(Protocol.DEFAULT_TIMEOUT, conf.connectTimeout); + assertEquals(Protocol.DEFAULT_TIMEOUT, conf.socketTimeout); + assertEquals(false, conf.refreshStaleValues); + assertEquals(false, conf.asyncRefresh); + assertEquals(RedisFeatureStoreBuilder.DEFAULT_PREFIX, conf.prefix); + assertNull(conf.poolConfig); + } - @Test - public void testMandatoryFields() throws URISyntaxException { - RedisFeatureStoreBuilder conf = new RedisFeatureStoreBuilder("http", "host", 1234, 1); - assertEquals(new URI("http://host:1234"), conf.uri); - assertEquals(1, conf.cacheTimeSecs); - } + @Test + public void testConstructorSpecifyingUri() { + URI uri = URI.create("redis://host:1234"); + RedisFeatureStoreBuilder conf = new RedisFeatureStoreBuilder(uri); + assertEquals(uri, conf.uri); + assertEquals(RedisFeatureStoreBuilder.DEFAULT_CACHE_TIME_SECONDS, conf.cacheTimeSecs); + assertEquals(Protocol.DEFAULT_TIMEOUT, conf.connectTimeout); + assertEquals(Protocol.DEFAULT_TIMEOUT, conf.socketTimeout); + assertEquals(false, conf.refreshStaleValues); + assertEquals(false, conf.asyncRefresh); + assertEquals(RedisFeatureStoreBuilder.DEFAULT_PREFIX, conf.prefix); + assertNull(conf.poolConfig); + } - @Test - public void testMandatoryFieldsWithAlternateConstructor() throws URISyntaxException { - URI expectedURI = new URI("http://host:1234"); - RedisFeatureStoreBuilder conf = new RedisFeatureStoreBuilder(expectedURI, 1); - assertEquals(expectedURI, conf.uri); - assertEquals(1, conf.cacheTimeSecs); - } + @SuppressWarnings("deprecation") + @Test + public void testDeprecatedUriBuildingConstructor() throws URISyntaxException { + RedisFeatureStoreBuilder conf = new RedisFeatureStoreBuilder("badscheme", "example", 1234, 100); + assertEquals(URI.create("badscheme://example:1234"), conf.uri); + assertEquals(100, conf.cacheTimeSecs); + assertEquals(Protocol.DEFAULT_TIMEOUT, conf.connectTimeout); + assertEquals(Protocol.DEFAULT_TIMEOUT, conf.socketTimeout); + assertEquals(false, conf.refreshStaleValues); + assertEquals(false, conf.asyncRefresh); + assertEquals(RedisFeatureStoreBuilder.DEFAULT_PREFIX, conf.prefix); + assertNull(conf.poolConfig); + } - @Test - public void testRefreshStaleValues() throws URISyntaxException { - URI expectedURI = new URI("http://host:1234"); - RedisFeatureStoreBuilder conf = new RedisFeatureStoreBuilder(expectedURI, 1).refreshStaleValues(true); - assertEquals(true, conf.refreshStaleValues); - } + @Test + public void testRefreshStaleValues() throws URISyntaxException { + RedisFeatureStoreBuilder conf = new RedisFeatureStoreBuilder().refreshStaleValues(true); + assertEquals(true, conf.refreshStaleValues); + } - @Test - public void testAsyncRefresh() throws URISyntaxException { - URI expectedURI = new URI("http://host:1234"); - RedisFeatureStoreBuilder conf = new RedisFeatureStoreBuilder(expectedURI, 1).asyncRefresh(true); - assertEquals(true, conf.asyncRefresh); - } + @Test + public void testAsyncRefresh() throws URISyntaxException { + RedisFeatureStoreBuilder conf = new RedisFeatureStoreBuilder().asyncRefresh(true); + assertEquals(true, conf.asyncRefresh); + } - @Test - public void testPrefixConfigured() throws URISyntaxException { - RedisFeatureStoreBuilder conf = new RedisFeatureStoreBuilder("http", "host", 1234, 1).prefix("prefix"); - assertEquals("prefix", conf.prefix); - } + @Test + public void testPrefixConfigured() throws URISyntaxException { + RedisFeatureStoreBuilder conf = new RedisFeatureStoreBuilder().prefix("prefix"); + assertEquals("prefix", conf.prefix); + } - @Test - public void testConnectTimeoutConfigured() throws URISyntaxException { - RedisFeatureStoreBuilder conf = new RedisFeatureStoreBuilder("http", "host", 1234, 1).connectTimeout(1, TimeUnit.SECONDS); - assertEquals(1000, conf.connectTimeout); - } + @Test + public void testConnectTimeoutConfigured() throws URISyntaxException { + RedisFeatureStoreBuilder conf = new RedisFeatureStoreBuilder().connectTimeout(1, TimeUnit.SECONDS); + assertEquals(1000, conf.connectTimeout); + } - @Test - public void testSocketTimeoutConfigured() throws URISyntaxException { - RedisFeatureStoreBuilder conf = new RedisFeatureStoreBuilder("http", "host", 1234, 1).socketTimeout(1, TimeUnit.SECONDS); - assertEquals(1000, conf.socketTimeout); - } + @Test + public void testSocketTimeoutConfigured() throws URISyntaxException { + RedisFeatureStoreBuilder conf = new RedisFeatureStoreBuilder().socketTimeout(1, TimeUnit.SECONDS); + assertEquals(1000, conf.socketTimeout); + } - @Test - public void testCacheTimeConfiguredInSeconds() throws URISyntaxException { - RedisFeatureStoreBuilder conf = new RedisFeatureStoreBuilder("http", "host", 1234, 1).cacheTime(2000, TimeUnit.MILLISECONDS); - assertEquals(2, conf.cacheTimeSecs); - } + @Test + public void testCacheTimeConfiguredInSeconds() throws URISyntaxException { + RedisFeatureStoreBuilder conf = new RedisFeatureStoreBuilder().cacheTime(2000, TimeUnit.MILLISECONDS); + assertEquals(2, conf.cacheTimeSecs); + } - @Test - public void testPoolConfigConfigured() throws URISyntaxException { - JedisPoolConfig poolConfig = new JedisPoolConfig(); - RedisFeatureStoreBuilder conf = new RedisFeatureStoreBuilder("http", "host", 1234, 1).poolConfig(poolConfig); - assertEquals(poolConfig, conf.poolConfig); - } + @Test + public void testPoolConfigConfigured() throws URISyntaxException { + JedisPoolConfig poolConfig = new JedisPoolConfig(); + RedisFeatureStoreBuilder conf = new RedisFeatureStoreBuilder().poolConfig(poolConfig); + assertEquals(poolConfig, conf.poolConfig); + } } diff --git a/src/test/java/com/launchdarkly/client/RedisFeatureStoreTest.java b/src/test/java/com/launchdarkly/client/RedisFeatureStoreTest.java index 90b194f76..ec30740cd 100644 --- a/src/test/java/com/launchdarkly/client/RedisFeatureStoreTest.java +++ b/src/test/java/com/launchdarkly/client/RedisFeatureStoreTest.java @@ -8,6 +8,6 @@ public class RedisFeatureStoreTest extends FeatureStoreTestBase events = new ArrayList<>(); + + @Override + public void close() throws IOException {} + + @Override + public void sendEvent(Event e) { + events.add(e); + } + + @Override + public void flush() {} + } + public static JsonPrimitive js(String s) { return new JsonPrimitive(s); } From 89633ef40ae7dbdef46520dc777a0b9d7c34cfd8 Mon Sep 17 00:00:00 2001 From: Eli Bishop Date: Wed, 25 Apr 2018 20:09:43 -0700 Subject: [PATCH 63/67] make LDClient final, remove methods that were overrideable only for testing --- .../com/launchdarkly/client/LDClient.java | 25 ++++++------------- 1 file changed, 7 insertions(+), 18 deletions(-) diff --git a/src/main/java/com/launchdarkly/client/LDClient.java b/src/main/java/com/launchdarkly/client/LDClient.java index b37573706..51e5851e6 100644 --- a/src/main/java/com/launchdarkly/client/LDClient.java +++ b/src/main/java/com/launchdarkly/client/LDClient.java @@ -1,6 +1,5 @@ package com.launchdarkly.client; -import com.google.common.annotations.VisibleForTesting; import com.google.gson.JsonElement; import com.google.gson.JsonPrimitive; @@ -30,7 +29,7 @@ * A client for the LaunchDarkly API. Client instances are thread-safe. Applications should instantiate * a single {@code LDClient} for the lifetime of their application. */ -public class LDClient implements LDClientInterface { +public final class LDClient implements LDClientInterface { private static final Logger logger = LoggerFactory.getLogger(LDClient.class); private static final String HMAC_ALGORITHM = "HmacSHA256"; static final String CLIENT_VERSION = getClientVersion(); @@ -79,7 +78,9 @@ public LDClient(String sdkKey, LDConfig config) { this.shouldCloseFeatureStore = true; } - this.eventProcessor = createEventProcessor(sdkKey, config); + EventProcessorFactory epFactory = config.eventProcessorFactory == null ? + Components.defaultEventProcessor() : config.eventProcessorFactory; + this.eventProcessor = epFactory.createEventProcessor(sdkKey, config); if (config.offline) { logger.info("Starting LaunchDarkly client in offline mode"); @@ -87,7 +88,9 @@ public LDClient(String sdkKey, LDConfig config) { logger.info("Starting LaunchDarkly in LDD mode. Skipping direct feature retrieval."); } - this.updateProcessor = createUpdateProcessor(sdkKey, config, featureStore); + UpdateProcessorFactory upFactory = config.updateProcessorFactory == null ? + Components.defaultUpdateProcessor() : config.updateProcessorFactory; + this.updateProcessor = upFactory.createUpdateProcessor(sdkKey, config, featureStore); Future startFuture = updateProcessor.start(); if (config.startWaitMillis > 0L) { if (!config.offline && !config.useLdd) { @@ -108,20 +111,6 @@ public boolean initialized() { return updateProcessor.initialized(); } - @VisibleForTesting - protected EventProcessor createEventProcessor(String sdkKey, LDConfig config) { - EventProcessorFactory factory = config.eventProcessorFactory == null ? - Components.defaultEventProcessor() : config.eventProcessorFactory; - return factory.createEventProcessor(sdkKey, config); - } - - @VisibleForTesting - protected UpdateProcessor createUpdateProcessor(String sdkKey, LDConfig config, FeatureStore featureStore) { - UpdateProcessorFactory factory = config.updateProcessorFactory == null ? - Components.defaultUpdateProcessor() : config.updateProcessorFactory; - return factory.createUpdateProcessor(sdkKey, config, featureStore); - } - @Override public void track(String eventName, LDUser user, JsonElement data) { if (isOffline()) { From 2b935e88e3fee1c5e52c3e3b0bcf117cc575442b Mon Sep 17 00:00:00 2001 From: Eli Bishop Date: Wed, 25 Apr 2018 20:33:27 -0700 Subject: [PATCH 64/67] rm duplicate logging --- src/main/java/com/launchdarkly/client/LDClient.java | 6 ------ 1 file changed, 6 deletions(-) diff --git a/src/main/java/com/launchdarkly/client/LDClient.java b/src/main/java/com/launchdarkly/client/LDClient.java index 51e5851e6..072e6d85e 100644 --- a/src/main/java/com/launchdarkly/client/LDClient.java +++ b/src/main/java/com/launchdarkly/client/LDClient.java @@ -82,12 +82,6 @@ public LDClient(String sdkKey, LDConfig config) { Components.defaultEventProcessor() : config.eventProcessorFactory; this.eventProcessor = epFactory.createEventProcessor(sdkKey, config); - if (config.offline) { - logger.info("Starting LaunchDarkly client in offline mode"); - } else if (config.useLdd) { - logger.info("Starting LaunchDarkly in LDD mode. Skipping direct feature retrieval."); - } - UpdateProcessorFactory upFactory = config.updateProcessorFactory == null ? Components.defaultUpdateProcessor() : config.updateProcessorFactory; this.updateProcessor = upFactory.createUpdateProcessor(sdkKey, config, featureStore); From a6757abd437c6f2d09e5d3fbae49e7f9449907a3 Mon Sep 17 00:00:00 2001 From: Eli Bishop Date: Fri, 27 Apr 2018 16:30:34 -0700 Subject: [PATCH 65/67] add variation index to feature events and summary counters --- .../java/com/launchdarkly/client/EventOutput.java | 14 ++++++++++---- .../client/DefaultEventProcessorTest.java | 12 +++++++----- 2 files changed, 17 insertions(+), 9 deletions(-) diff --git a/src/main/java/com/launchdarkly/client/EventOutput.java b/src/main/java/com/launchdarkly/client/EventOutput.java index 3058c23a9..016e52f32 100644 --- a/src/main/java/com/launchdarkly/client/EventOutput.java +++ b/src/main/java/com/launchdarkly/client/EventOutput.java @@ -40,16 +40,18 @@ static final class FeatureRequest extends EventOutputWithTimestamp { private final String userKey; private final LDUser user; private final Integer version; + private final Integer variation; private final JsonElement value; @SerializedName("default") private final JsonElement defaultVal; private final String prereqOf; FeatureRequest(long creationDate, String key, String userKey, LDUser user, - Integer version, JsonElement value, JsonElement defaultVal, String prereqOf, boolean debug) { + Integer version, Integer variation, JsonElement value, JsonElement defaultVal, String prereqOf, boolean debug) { super(debug ? "debug" : "feature", creationDate); this.key = key; this.userKey = userKey; this.user = user; + this.variation = variation; this.version = version; this.value = value; this.defaultVal = defaultVal; @@ -120,12 +122,14 @@ static final class SummaryEventFlag { } static final class SummaryEventCounter { + final Integer variation; final JsonElement value; final Integer version; final int count; final Boolean unknown; - SummaryEventCounter(JsonElement value, Integer version, int count, Boolean unknown) { + SummaryEventCounter(Integer variation, JsonElement value, Integer version, int count, Boolean unknown) { + this.variation = variation; this.value = value; this.version = version; this.count = count; @@ -159,7 +163,7 @@ private EventOutput createOutputEvent(Event e) { return new EventOutput.FeatureRequest(fe.creationDate, fe.key, inlineThisUser ? null : userKey, inlineThisUser ? e.user : null, - fe.version, fe.value, fe.defaultVal, fe.prereqOf, fe.debug); + fe.version, fe.variation, fe.value, fe.defaultVal, fe.prereqOf, fe.debug); } else if (e instanceof Event.Identify) { return new EventOutput.Identify(e.creationDate, e.user); } else if (e instanceof Event.Custom) { @@ -183,7 +187,9 @@ private EventOutput createSummaryEvent(EventSummarizer.EventSummary summary) { fsd = new SummaryEventFlag(entry.getValue().defaultVal, new ArrayList()); flagsOut.put(entry.getKey().key, fsd); } - SummaryEventCounter c = new SummaryEventCounter(entry.getValue().flagValue, + SummaryEventCounter c = new SummaryEventCounter( + entry.getKey().variation, + entry.getValue().flagValue, entry.getKey().version, entry.getValue().count, entry.getKey().version == null ? true : null); diff --git a/src/test/java/com/launchdarkly/client/DefaultEventProcessorTest.java b/src/test/java/com/launchdarkly/client/DefaultEventProcessorTest.java index a3c04a698..8cc9eca54 100644 --- a/src/test/java/com/launchdarkly/client/DefaultEventProcessorTest.java +++ b/src/test/java/com/launchdarkly/client/DefaultEventProcessorTest.java @@ -295,9 +295,9 @@ public void nonTrackedEventsAreSummarized() throws Exception { JsonElement default1 = new JsonPrimitive("default1"); JsonElement default2 = new JsonPrimitive("default2"); Event fe1 = EventFactory.DEFAULT.newFeatureRequestEvent(flag1, user, - new FeatureFlag.VariationAndValue(new Integer(1), value), default1); + new FeatureFlag.VariationAndValue(new Integer(2), value), default1); Event fe2 = EventFactory.DEFAULT.newFeatureRequestEvent(flag2, user, - new FeatureFlag.VariationAndValue(new Integer(1), value), default2); + new FeatureFlag.VariationAndValue(new Integer(2), value), default2); ep.sendEvent(fe1); ep.sendEvent(fe2); @@ -307,9 +307,9 @@ public void nonTrackedEventsAreSummarized() throws Exception { allOf( isSummaryEvent(fe1.creationDate, fe2.creationDate), hasSummaryFlag(flag1.getKey(), default1, - hasItem(isSummaryEventCounter(flag1, value, 1))), + hasItem(isSummaryEventCounter(flag1, 2, value, 1))), hasSummaryFlag(flag2.getKey(), default2, - hasItem(isSummaryEventCounter(flag2, value, 1))) + hasItem(isSummaryEventCounter(flag2, 2, value, 1))) ) )); } @@ -465,6 +465,7 @@ private Matcher isFeatureEvent(Event.FeatureRequest sourceEvent, Fe hasJsonProperty("creationDate", (double)sourceEvent.creationDate), hasJsonProperty("key", flag.getKey()), hasJsonProperty("version", (double)flag.getVersion()), + hasJsonProperty("variation", sourceEvent.variation), hasJsonProperty("value", sourceEvent.value), (inlineUser != null) ? hasJsonProperty("userKey", nullValue(JsonElement.class)) : hasJsonProperty("userKey", sourceEvent.user.getKeyAsString()), @@ -506,8 +507,9 @@ private Matcher hasSummaryFlag(String key, JsonElement defaultVal, ))); } - private Matcher isSummaryEventCounter(FeatureFlag flag, JsonElement value, int count) { + private Matcher isSummaryEventCounter(FeatureFlag flag, Integer variation, JsonElement value, int count) { return allOf( + hasJsonProperty("variation", variation), hasJsonProperty("version", (double)flag.getVersion()), hasJsonProperty("value", value), hasJsonProperty("count", (double)count) From dec738e41c5aa0f82620b0ccb3f8ba0a49b2c84a Mon Sep 17 00:00:00 2001 From: Eli Bishop Date: Mon, 30 Apr 2018 10:40:21 -0700 Subject: [PATCH 66/67] Event classes should be public --- .../java/com/launchdarkly/client/Event.java | 20 +++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/src/main/java/com/launchdarkly/client/Event.java b/src/main/java/com/launchdarkly/client/Event.java index 7d429554c..568646bd1 100644 --- a/src/main/java/com/launchdarkly/client/Event.java +++ b/src/main/java/com/launchdarkly/client/Event.java @@ -5,39 +5,39 @@ /** * Base class for all analytics events that are generated by the client. Also defines all of its own subclasses. */ -class Event { +public class Event { final long creationDate; final LDUser user; - Event(long creationDate, LDUser user) { + public Event(long creationDate, LDUser user) { this.creationDate = creationDate; this.user = user; } - static final class Custom extends Event { + public static final class Custom extends Event { final String key; final JsonElement data; - Custom(long timestamp, String key, LDUser user, JsonElement data) { + public Custom(long timestamp, String key, LDUser user, JsonElement data) { super(timestamp, user); this.key = key; this.data = data; } } - static final class Identify extends Event { - Identify(long timestamp, LDUser user) { + public static final class Identify extends Event { + public Identify(long timestamp, LDUser user) { super(timestamp, user); } } - static final class Index extends Event { - Index(long timestamp, LDUser user) { + public static final class Index extends Event { + public Index(long timestamp, LDUser user) { super(timestamp, user); } } - static final class FeatureRequest extends Event { + public static final class FeatureRequest extends Event { final String key; final Integer variation; final JsonElement value; @@ -47,7 +47,7 @@ static final class FeatureRequest extends Event { final boolean trackEvents; final Long debugEventsUntilDate; - FeatureRequest(long timestamp, String key, LDUser user, Integer version, Integer variation, JsonElement value, + public FeatureRequest(long timestamp, String key, LDUser user, Integer version, Integer variation, JsonElement value, JsonElement defaultVal, String prereqOf, boolean trackEvents, Long debugEventsUntilDate) { super(timestamp, user); this.key = key; From dd1ee9c7a8338f4e1315012a64daf99a4b6a58d5 Mon Sep 17 00:00:00 2001 From: Eli Bishop Date: Wed, 9 May 2018 15:14:41 -0700 Subject: [PATCH 67/67] deprecate TestFeatureStore --- src/main/java/com/launchdarkly/client/TestFeatureStore.java | 3 +++ 1 file changed, 3 insertions(+) diff --git a/src/main/java/com/launchdarkly/client/TestFeatureStore.java b/src/main/java/com/launchdarkly/client/TestFeatureStore.java index b6bfb188f..afebbad2f 100644 --- a/src/main/java/com/launchdarkly/client/TestFeatureStore.java +++ b/src/main/java/com/launchdarkly/client/TestFeatureStore.java @@ -14,7 +14,10 @@ * A decorated {@link InMemoryFeatureStore} which provides functionality to create (or override) true or false feature flags for all users. *

* Using this store is useful for testing purposes when you want to have runtime support for turning specific features on or off. + * + * @deprecated Will be replaced by a file-based test fixture. */ +@Deprecated public class TestFeatureStore extends InMemoryFeatureStore { static List TRUE_FALSE_VARIATIONS = Arrays.asList( (JsonElement) (new JsonPrimitive(true)),