Conditional Contextual Bandit

cs/cli/vw_cbutil.cpp

@@ -1,17 +1,17 @@
-/*
-Copyright (c) by respective owners including Yahoo!, Microsoft, and
-individual contributors. All rights reserved.  Released under a BSD (revised)
-license as described in the file LICENSE.
-*/
-
-#include "vw_cbutil.h"
-#include "cb_algs.h"
-
-namespace VW
-{
-float VowpalWabbitContextualBanditUtil::GetUnbiasedCost(uint32_t actionObservered, uint32_t actionTaken, float cost, float probability)
-{ CB::cb_class observation = { cost, actionObservered, probability };
-
-  return CB_ALGS::get_unbiased_cost(&observation, actionTaken);
-}
-}
+/*
+Copyright (c) by respective owners including Yahoo!, Microsoft, and
+individual contributors. All rights reserved.  Released under a BSD (revised)
+license as described in the file LICENSE.
+*/
+
+#include "vw_cbutil.h"
+#include "cb_algs.h"
+
+namespace VW
+{
+float VowpalWabbitContextualBanditUtil::GetUnbiasedCost(uint32_t actionObservered, uint32_t actionTaken, float cost, float probability)
+{ CB::cb_class observation = { cost, actionObservered, probability };
+
+  return CB_ALGS::get_cost_estimate(&observation, actionTaken);
+}
+}

explore/explore_internal.h

@@ -229,6 +229,8 @@ namespace exploration
 	  return enforce_minimum_probability(minimum_uniform, update_zero_elements, pdf_first, pdf_last, pdf_category());
   }
 
+  // Warning: `seed` must be sufficiently random for the PRNG to produce uniform random values. Using sequential seeds will result in a very biased distribution.
+  // If unsure how to update seed between calls, merand48 (in rand48.h) can be used to inplace mutate it.
   template<typename It>
   int sample_after_normalizing(uint64_t seed, It pdf_first, It pdf_last, uint32_t& chosen_index, std::input_iterator_tag /* pdf_category */)
   {
@@ -277,20 +279,26 @@ namespace exploration
     return S_EXPLORATION_OK;
   }
 
+  // Warning: `seed` must be sufficiently random for the PRNG to produce uniform random values. Using sequential seeds will result in a very biased distribution.
+  // If unsure how to update seed between calls, merand48 (in rand48.h) can be used to inplace mutate it.
   template<typename It>
   int sample_after_normalizing(uint64_t seed, It pdf_first, It pdf_last, uint32_t& chosen_index)
   {
 	  typedef typename std::iterator_traits<It>::iterator_category pdf_category;
     return sample_after_normalizing(seed, pdf_first, pdf_last, chosen_index, pdf_category());
   }
 
+  // Warning: `seed` must be sufficiently random for the PRNG to produce uniform random values. Using sequential seeds will result in a very biased distribution.
+  // If unsure how to update seed between calls, merand48 (in rand48.h) can be used to inplace mutate it.
   template<typename It>
   int sample_after_normalizing(const char* seed, It pdf_first, It pdf_last, uint32_t& chosen_index, std::random_access_iterator_tag pdf_category)
   {
     uint64_t seed_hash = uniform_hash(seed, strlen(seed), 0);
     return sample_after_normalizing(seed_hash, pdf_first, pdf_last, chosen_index, pdf_category);
   }
 
+  // Warning: `seed` must be sufficiently random for the PRNG to produce uniform random values. Using sequential seeds will result in a very biased distribution.
+  // If unsure how to update seed between calls, merand48 (in rand48.h) can be used to inplace mutate it.
   template<typename It>
   int sample_after_normalizing(const char* seed, It pdf_first, It pdf_last, uint32_t& chosen_index)
   {

java/src/main/c++/jni_spark_vw.cc

@@ -194,6 +194,7 @@ JNIEXPORT jlong JNICALL Java_org_vowpalwabbit_spark_VowpalWabbitExample_initiali
   try
   {
     example* ex = VW::alloc_examples(0, 1);
+    ex->interactions = &all->interactions;
 
     if (isEmpty)
     {
@@ -419,4 +420,4 @@ JNIEXPORT jobject JNICALL Java_org_vowpalwabbit_spark_VowpalWabbitExample_predic
   {
     rethrow_cpp_exception_as_java_exception(env);
   }
-}
+}

python/pylibvw.cc

@@ -148,6 +148,7 @@ example* my_empty_example0(vw_ptr vw, size_t labelType)
 { label_parser* lp = get_label_parser(&*vw, labelType);
   example* ec = VW::alloc_examples(lp->label_size, 1);
   lp->default_label(&ec->l);
+  ec->interactions = &vw->interactions;
   if (labelType == lCOST_SENSITIVE)
   { COST_SENSITIVE::wclass zero = { 0., 1, 0., 0. };
     ec->l.cs.costs.push_back(zero);

test/RunTests

@@ -1731,12 +1731,13 @@ printf '3 |f a b c |e x y z\n2 |f a y c |e x\n' | {VW} --oaa 3 -q ef --audit
 {VW} -d train-sets/malformed.dat --onethread
     train-sets/ref/malformed.stderr
 
-# Test 196: online contextual memory tree
-{VW} -d train-sets/rcv1_smaller.dat --memory_tree 10 --learn_at_leaf --max_number_of_labels 2 --dream_at_update 0 --dream_repeats 3 --online --leaf_example_multiplier 10 --alpha 0.1 -l 0.001 -b 15 --passes 1 --loss_function squared --holdout_off
-    train-sets/ref/cmt_rcv1_smaller_online.stderr
-
-# Test 197: offline contextual memory tree
-{VW} -d train-sets/rcv1_smaller.dat --memory_tree 10 --learn_at_leaf --max_number_of_labels 2 --dream_at_update 0 --dream_repeats 3 --leaf_example_multiplier 10 --alpha 0.1 -l 0.001 -b 15 -c --passes 2 --loss_function squared --holdout_off
-    train-sets/ref/cmt_rcv1_smaller_offline.stderr
+# Test 196: test cb_sample
+{VW} --cb_sample --cb_explore_adf -d test-sets/cb_sample_seed.data -p cb_sample_seed.predict --random_seed 1234
+    pred-sets/ref/cb_sample_seed.predict
+
+# Test 197: CCB train then test
+{VW} -d train-sets/ccb_test.dat --ccb_explore_adf -p ccb_test.predict
+    train-sets/ref/ccb_test.stderr
+    train-sets/ref/ccb_test.predict
 
 # Do not delete this line or the empty line above it

test/pred-sets/ref/cb_sample_seed.predict

@@ -0,0 +1,4 @@
+1:0.5,0:0.5
+
+0:0.5,1:0.5
+

test/test-sets/cb_sample_seed.data

@@ -0,0 +1,7 @@
+shared seed=1234| s_1 s_2
+| a:1 b:1 c:1
+| a:0.5 b:2 c:1
+
+shared seed=1234| s_1 s_2
+0:1.0:0.5 | a:1 b:1 c:1
+| a:0.5 b:2 c:1

test/train-sets/ccb_test.dat

@@ -0,0 +1,240 @@
+ccb shared |User b
+ccb action |Action d
+ccb action |Action e
+ccb action |Action f
+ccb action |Action ff
+ccb action |Action fff
+ccb slot 0:0:0.2 |Slot h
+ccb slot 1:0:0.25 |Slot i
+ccb slot 2:0:0.333333 |Slot j
+
+ccb shared |User b
+ccb action |Action d
+ccb action |Action e
+ccb action |Action f
+ccb action |Action ff
+ccb action |Action fff
+ccb slot 3:0:0.2 |Slot h
+ccb slot 4:0:0.25 |Slot i
+ccb slot 0:0:0.333333 |Slot j
+
+ccb shared |User c
+ccb action |Action d
+ccb action |Action e
+ccb action |Action f
+ccb action |Action ff
+ccb action |Action fff
+ccb slot 4:0:0.2 |Slot h
+ccb slot 3:0:0.25 |Slot i
+ccb slot 2:0:0.333333 |Slot j
+
+ccb shared |User a
+ccb action |Action d
+ccb action |Action e
+ccb action |Action f
+ccb action |Action ff
+ccb action |Action fff
+ccb slot 1:0:0.2 |Slot h
+ccb slot 4:-1:0.25 |Slot i
+ccb slot 2:0:0.333333 |Slot j
+
+ccb shared |User a
+ccb action |Action d
+ccb action |Action e
+ccb action |Action f
+ccb action |Action ff
+ccb action |Action fff
+ccb slot 4:0:0.84 |Slot h
+ccb slot 3:0:0.316667 |Slot i
+ccb slot 1:-1:0.466667 |Slot j
+
+ccb shared |User b
+ccb action |Action d
+ccb action |Action e
+ccb action |Action f
+ccb action |Action ff
+ccb action |Action fff
+ccb slot 1:0:0.04 |Slot h
+ccb slot 4:-1:0.85 |Slot i
+ccb slot 0:0:0.866667 |Slot j
+
+ccb shared |User c
+ccb action |Action d
+ccb action |Action e
+ccb action |Action f
+ccb action |Action ff
+ccb action |Action fff
+ccb slot 4:-1:0.84 |Slot h
+ccb slot 1:0:0.85 |Slot i
+ccb slot 2:0:0.0666667 |Slot j
+
+ccb shared |User c
+ccb action |Action d
+ccb action |Action e
+ccb action |Action f
+ccb action |Action ff
+ccb action |Action fff
+ccb slot 4:-1:0.84 |Slot h
+ccb slot 1:0:0.85 |Slot i
+ccb slot 0:0:0.866667 |Slot j
+
+ccb shared |User a
+ccb action |Action d
+ccb action |Action e
+ccb action |Action f
+ccb action |Action ff
+ccb action |Action fff
+ccb slot 4:0:0.84 |Slot h
+ccb slot 1:0:0.85 |Slot i
+ccb slot 2:0:0.0666667 |Slot j
+
+ccb shared |User c
+ccb action |Action d
+ccb action |Action e
+ccb action |Action f
+ccb action |Action ff
+ccb action |Action fff
+ccb slot 4:0:0.84 |Slot h
+ccb slot 1:0:0.85 |Slot i
+ccb slot 0:0:0.0666667 |Slot j
+
+ccb shared |User b
+ccb action |Action d
+ccb action |Action e
+ccb action |Action f
+ccb action |Action ff
+ccb action |Action fff
+ccb slot 4:0:0.84 |Slot h
+ccb slot 1:0:0.85 |Slot i
+ccb slot 3:0:0.866667 |Slot j
+
+ccb shared |User b
+ccb action |Action d
+ccb action |Action e
+ccb action |Action f
+ccb action |Action ff
+ccb action |Action fff
+ccb slot 4:0:0.84 |Slot h
+ccb slot 1:0:0.85 |Slot i
+ccb slot 2:0:0.0666667 |Slot j
+
+ccb shared |User a
+ccb action |Action d
+ccb action |Action e
+ccb action |Action f
+ccb action |Action ff
+ccb action |Action fff
+ccb slot 4:0:0.84 |Slot h
+ccb slot 1:0:0.85 |Slot i
+ccb slot 3:0:0.866667 |Slot j
+
+ccb shared |User a
+ccb action |Action d
+ccb action |Action e
+ccb action |Action f
+ccb action |Action ff
+ccb action |Action fff
+ccb slot 4:0:0.84 |Slot h
+ccb slot 2:-1:0.05 |Slot i
+ccb slot 1:0:0.866667 |Slot j
+
+ccb shared |User b
+ccb action |Action d
+ccb action |Action e
+ccb action |Action f
+ccb action |Action ff
+ccb action |Action fff
+ccb slot 2:0:0.84 |Slot h
+ccb slot 4:0:0.85 |Slot i
+ccb slot 1:0:0.866667 |Slot j
+
+ccb shared |User a
+ccb action |Action d
+ccb action |Action e
+ccb action |Action f
+ccb action |Action ff
+ccb action |Action fff
+ccb slot |Slot h
+ccb slot |Slot i
+ccb slot |Slot j
+
+ccb shared |User b
+ccb action |Action d
+ccb action |Action e
+ccb action |Action f
+ccb action |Action ff
+ccb action |Action fff
+ccb slot |Slot h
+ccb slot |Slot i
+ccb slot |Slot j
+
+ccb shared |User c
+ccb action |Action d
+ccb action |Action e
+ccb action |Action f
+ccb action |Action ff
+ccb action |Action fff
+ccb slot |Slot h
+ccb slot |Slot i
+ccb slot |Slot j
+
+ccb shared |User b
+ccb action |Action d
+ccb action |Action e
+ccb action |Action f
+ccb action |Action ff
+ccb action |Action fff
+ccb slot |Slot h
+ccb slot |Slot i
+ccb slot |Slot j
+
+ccb shared |User a
+ccb action |Action d
+ccb action |Action e
+ccb action |Action f
+ccb action |Action ff
+ccb action |Action fff
+ccb slot |Slot h
+ccb slot |Slot i
+ccb slot |Slot j
+
+ccb shared |User c
+ccb action |Action d
+ccb action |Action e
+ccb action |Action f
+ccb action |Action ff
+ccb action |Action fff
+ccb slot |Slot h
+ccb slot |Slot i
+ccb slot |Slot j
+
+ccb shared |User b
+ccb action |Action d
+ccb action |Action e
+ccb action |Action f
+ccb action |Action ff
+ccb action |Action fff
+ccb slot |Slot h
+ccb slot |Slot i
+ccb slot |Slot j
+
+ccb shared |User b
+ccb action |Action d
+ccb action |Action e
+ccb action |Action f
+ccb action |Action ff
+ccb action |Action fff
+ccb slot |Slot h
+ccb slot |Slot i
+ccb slot |Slot j
+
+ccb shared |User a
+ccb action |Action d
+ccb action |Action e
+ccb action |Action f
+ccb action |Action ff
+ccb action |Action fff
+ccb slot |Slot h
+ccb slot |Slot i
+ccb slot |Slot j
+