Skip to content

HTTPS clone URL

Subversion checkout URL

You can clone with
or
.
Download ZIP
Browse files

add new Counting experiement type allowing multiple conversions per user

  • Loading branch information...
commit bad6d51cb92dcd530a159c223cefc3833178ee77 1 parent 5638261
@kohlmeier kohlmeier authored
View
5 cache.py
@@ -355,7 +355,10 @@ def participate_in(self, experiment_name):
self.dirty = True
def convert_in(self, experiment_name):
- self.converted_tests[experiment_name] = 1
+ if experiment_name not in self.converted_tests:
+ self.converted_tests[experiment_name] = 1
+ else:
+ self.converted_tests[experiment_name] += 1
self.dirty = True
def bingo_and_identity_cache():
View
13 gae_bingo.py
@@ -7,10 +7,10 @@
from google.appengine.api import memcache
from .cache import BingoCache, bingo_and_identity_cache
-from .models import create_experiment_and_alternatives
+from .models import create_experiment_and_alternatives, ConversionTypes
from .identity import identity
-def ab_test(canonical_name, alternative_params = None, conversion_name = None):
+def ab_test(canonical_name, alternative_params = None, conversion_name = None, conversion_type = ConversionTypes.Binary):
bingo_cache, bingo_identity_cache = bingo_and_identity_cache()
@@ -54,7 +54,8 @@ def ab_test(canonical_name, alternative_params = None, conversion_name = None):
unique_experiment_name,
canonical_name,
alternative_params,
- conversion_name
+ conversion_name,
+ conversion_type
)
bingo_cache.add_experiment(exp, alts)
@@ -131,15 +132,15 @@ def score_conversion(experiment_name, canonical_name):
if experiment_name not in bingo_identity_cache.participating_tests:
return
- if experiment_name in bingo_identity_cache.converted_tests:
- return
-
experiment = bingo_cache.get_experiment(experiment_name)
if not experiment or not experiment.live:
# Don't count conversions for short-circuited experiments that are no longer live
return
+ if experiment_name in bingo_identity_cache.converted_tests and experiment.conversion_type!=ConversionTypes.Counting:
+ return
+
alternative = find_alternative_for_user(canonical_name, bingo_cache.get_alternatives(experiment_name))
alternative.increment_conversions()
View
18 models.py
@@ -14,11 +14,21 @@
class GAEBingoIdentityModel(db.Model):
gae_bingo_identity = db.StringProperty()
+class ConversionTypes():
+ Binary = "binary"
+ Counting = "counting"
+ @staticmethod
+ def get_all_as_list():
+ return [ConversionTypes.Binary, ConversionTypes.Counting]
+ def __setattr__(self, attr, value):
+ pass
+
class _GAEBingoExperiment(db.Model):
name = db.StringProperty()
# Not necessarily unique. Experiments "monkeys" and "monkeys (2)" both have canonical_name "monkeys"
canonical_name = db.StringProperty()
conversion_name = db.StringProperty()
+ conversion_type = db.StringProperty(default=ConversionTypes.Binary, choices=set(ConversionTypes.get_all_as_list()))
live = db.BooleanProperty(default = True)
dt_started = db.DateTimeProperty(auto_now_add = True)
short_circuit_pickled_content = db.BlobProperty()
@@ -68,7 +78,10 @@ def conversion_rate(self):
@property
def pretty_conversion_rate(self):
- return "%4.2f%%" % (self.conversion_rate * 100)
+ if self.conversion_rate > 1.0:
+ return "%.2f/participant" % (self.conversion_rate)
+ else:
+ return "%4.2f%%" % (self.conversion_rate * 100)
def key_for_self(self):
return _GAEBingoAlternative.key_for_experiment_name_and_number(self.experiment_name, self.number)
@@ -119,7 +132,7 @@ def load(identity):
return None
-def create_experiment_and_alternatives(experiment_name, canonical_name, alternative_params = None, conversion_name = None):
+def create_experiment_and_alternatives(experiment_name, canonical_name, alternative_params = None, conversion_name = None, conversion_type = ConversionTypes.Binary):
if not experiment_name:
raise Exception("gae_bingo experiments must be named.")
@@ -135,6 +148,7 @@ def create_experiment_and_alternatives(experiment_name, canonical_name, alternat
name = experiment_name,
canonical_name = canonical_name,
conversion_name = conversion_name,
+ conversion_type = conversion_type,
live = True,
)
View
8 plots.py
@@ -5,7 +5,7 @@
from google.appengine.ext.webapp import template, RequestHandler
from .cache import BingoCache
-from .models import _GAEBingoSnapshotLog
+from .models import _GAEBingoSnapshotLog, ConversionTypes
class TimeSeries:
def __init__(self, name):
@@ -22,6 +22,9 @@ def get(self):
bingo_cache = BingoCache.get()
experiment = bingo_cache.get_experiment(experiment_name)
+
+ y_axis_title = "Average Conversions per Participant" if experiment.conversion_type==ConversionTypes.Counting else "Conversions (%)"
+ y_scale_multiplier = 1.0 if experiment.conversion_type==ConversionTypes.Counting else 100.0
query = _GAEBingoSnapshotLog.all().ancestor(experiment)
query.order('-time_recorded')
@@ -46,7 +49,7 @@ def get_alternative_content_str(alt_num):
conv_rate = 0.0
if snapshot.participants > 0:
- conv_rate = float(snapshot.conversions) / float(snapshot.participants) * 100.0
+ conv_rate = float(snapshot.conversions) / float(snapshot.participants) * y_scale_multiplier
conv_rate = round(conv_rate, 1)
utc_time = time.mktime(snapshot.time_recorded.timetuple()) * 1000
@@ -57,6 +60,7 @@ def get_alternative_content_str(alt_num):
self.response.out.write(
template.render(path, {
"experiment": experiment,
+ "y_axis_title": y_axis_title,
"experiment_data": experiment_data,
})
)
View
2  templates/timeline.html
@@ -34,7 +34,7 @@
},
yAxis: {
title: {
- text: 'Conversion Rate (%)'
+ text: '{{ y_axis_title }}'
},
min: 0
},
View
7 tests/__init__.py
@@ -9,6 +9,7 @@
from gae_bingo.cache import BingoCache, BingoIdentityCache
from gae_bingo.config import can_control_experiments
from gae_bingo.dashboard import ControlExperiment
+from gae_bingo.models import ConversionTypes
# See gae_bingo/tests/run_tests.py for the full explanation/sequence of these tests
@@ -34,6 +35,8 @@ def get(self):
v = self.participate_in_chimpanzees()
elif step == "participate_in_crocodiles":
v = self.participate_in_crocodiles()
+ elif step == "participate_in_hippos":
+ v = self.participate_in_hippos()
elif step == "convert_in":
v = self.convert_in()
elif step == "count_participants_in":
@@ -76,6 +79,10 @@ def participate_in_chimpanzees(self):
def participate_in_crocodiles(self):
# Weighted test
return ab_test("crocodiles", {"a": 100, "b": 200, "c": 400})
+
+ def participate_in_hippos(self):
+ # Multiple conversions test
+ return ab_test("hippos", conversion_type=ConversionTypes.Counting)
def convert_in(self):
bingo(self.request.get("conversion_name"))
View
21 tests/run_tests.py
@@ -37,7 +37,7 @@ def run_tests():
# Refresh bot's identity record so it doesn't pollute tests
assert(test_response("refresh_identity_record", bot=True) == True)
-
+
# Participate in experiment A, check for correct alternative values being returned,
for i in range(0, 20):
assert(test_response("participate_in_monkeys") in [True, False])
@@ -102,6 +102,7 @@ def run_tests():
for key in dict_conversions:
assert(dict_conversions[str(key).lower()] == dict_conversions_server[str(key).lower()])
+
# Participate in experiment B, using cookies to maintain identity
# and making sure alternatives for B are stable per identity
last_response = None
@@ -136,6 +137,13 @@ def run_tests():
# Make sure conversions for the 1st conversion type of this experiment are empty
dict_conversions_server = test_response("count_conversions_in", {"experiment_name": "chimpanzees"})
assert(0 == reduce(lambda a, b: a + b, map(lambda key: dict_conversions_server[key], dict_conversions_server)))
+
+ # Test that calling bingo multiple times for a signle user creates only one conversion (for a BINARY conversion type)
+ assert(test_response("participate_in_chimpanzees") in [True, False])
+ assert(test_response("convert_in", {"conversion_name": "chimps_conversion_1"}, use_last_cookies=True) == True)
+ assert(test_response("convert_in", {"conversion_name": "chimps_conversion_1"}, use_last_cookies=True) == True)
+ dict_conversions_server = test_response("count_conversions_in", {"experiment_name": "chimpanzees"})
+ assert(1 == reduce(lambda a, b: a + b, map(lambda key: dict_conversions_server[key], dict_conversions_server)))
# End experiment C, choosing a short-circuit alternative
test_response("end_and_choose", {"experiment_name": "chimpanzees", "alternative_number": 1})
@@ -144,6 +152,13 @@ def run_tests():
for i in range(0, 5):
assert(test_response("participate_in_chimpanzees") == False)
+ # Test an experiment with a Counting type conversion by converting multiple times for a single user
+ assert(test_response("participate_in_hippos") in [True, False])
+ for i in range(0, 5):
+ assert(test_response("convert_in", {"conversion_name": "hippos"}, use_last_cookies=True) == True)
+ dict_conversions_server = test_response("count_conversions_in", {"experiment_name": "hippos"})
+ assert(5 == reduce(lambda a, b: a + b, map(lambda key: dict_conversions_server[key], dict_conversions_server)))
+
# Participate in experiment D (weight alternatives), keeping track of alternative returned count.
dict_alternatives = {}
for i in range(0, 75):
@@ -164,14 +179,14 @@ def run_tests():
assert(dict_alternatives.get("b", 0) < dict_alternatives.get("c", 0))
# Check experiments count
- assert(test_response("count_experiments") == 5)
+ assert(test_response("count_experiments") == 6)
# Test persist and load from DS
assert(test_response("persist") == True)
assert(test_response("flush_memcache") == True)
# Check experiments and converion counts remain after persist and memcache flush
- assert(test_response("count_experiments") == 5)
+ assert(test_response("count_experiments") == 6)
dict_conversions_server = test_response("count_conversions_in", {"experiment_name": "chimpanzees (2)"})
assert(expected_conversions == reduce(lambda a, b: a + b, map(lambda key: dict_conversions_server[key], dict_conversions_server)))
Please sign in to comment.
Something went wrong with that request. Please try again.