Implement finish_example, update scripts to match (#1837)

python/examples/VW_in_Python.ipynb

@@ -322,8 +322,8 @@
    },
    "outputs": [],
    "source": [
-    "ex.finish()\n",
-    "ex2.finish()"
+    "vw.finish_example(ex)\n",
+    "vw.finish_example(ex2)"
    ]
   },
   {

python/examples/mini_vw.py

@@ -20,7 +20,7 @@ def mini_vw(inputFile, numPasses, otherArgs):
             else:
                 ex = vw.example(l)
                 vw.learn(ex)
-                ex.finish()
+                vw.finish_example(ex)
 
         h.close()
     vw.finish()

python/examples/test.py

@@ -35,7 +35,7 @@ def ensure_close(a,b,eps=1e-6):
 print('     my partial prediction =', my_pred)
 ensure_close(updated_pred, my_pred)
 print('')
-ex.finish()
+vw.finish_example(ex)
 
 ###############################################################################
 print('# make our own example from scratch')
@@ -54,7 +54,8 @@ def ensure_close(a,b,eps=1e-6):
 print('  final partial prediction =', ex.get_updated_prediction())
 ensure_close(ex.get_updated_prediction(), my_predict(vw,ex))
 print('')
-ex.finish()
+vw.finish_example(ex)
+
 
 ###############################################################################
 exList = []
@@ -64,15 +65,16 @@ def ensure_close(a,b,eps=1e-6):
 
 # this is the safe way to delete the examples for VW to reuse:
 for ex in exList:
-    ex.finish()
+    vw.finish_example(ex)
 
 exList = [] # this should __del__ the examples, we hope :)
 for i in range(120):    # note: if this is >=129, we hang!!!
     ex = vw.example()
     exList.append(ex)
 
 for ex in exList:
-    ex.finish()
+    vw.finish_example(ex)
+
 
 ###############################################################################
 
@@ -84,7 +86,8 @@ def ensure_close(a,b,eps=1e-6):
     print('loss =', ex.get_loss())
 
     print('label =', ex.get_label())
-    ex.finish()
+    vw.finish_example(ex)
+
 
 
 # to be safe, finish explicity (should happen by default anyway)
@@ -98,7 +101,8 @@ def ensure_close(a,b,eps=1e-6):
 ex.learn() ; ex.learn() ; ex.learn() ; ex.learn()
 before_save = ex.get_updated_prediction()
 print('before saving, prediction =', before_save)
-ex.finish()
+vw.finish_example(ex)
+
 vw.finish()   # this should create the file
 
 # now re-start vw by loading that model
@@ -107,7 +111,8 @@ def ensure_close(a,b,eps=1e-6):
 ex.learn()
 after_save = ex.get_partial_prediction()
 print(' after saving, prediction =', after_save)
-ex.finish()
+vw.finish_example(ex)
+
 ensure_close(before_save, after_save)
 vw.finish()   # this should create the file
 

python/examples/word_alignment.py

@@ -87,7 +87,7 @@ def _run(self, alignedSentence):
                                     oracle    = oracle,
                                     condition = [ (i, 'p'), (i-1, 'q') ] )
 
-            for ex in examples: ex.finish()
+            self.vw.finish_example(examples)
 
             output.append( spans[pred][2] )
             for j in spans[pred][2]:

python/pylibvw.cc

@@ -175,8 +175,30 @@ example_ptr my_existing_example(vw_ptr all, size_t labelType, example_ptr existi
   return boost::shared_ptr<example>(existing_example);
 }
 
+multi_ex unwrap_example_list(py::list& ec)
+{
+  multi_ex ex_coll;
+  for (ssize_t i = 0; i<len(ec); i++)
+  {
+    py::object eci = ec[i];
+    py::extract<example_ptr> get_ex(eci);
+    example_ptr ecp;
+    if (get_ex.check())
+      ecp = get_ex();
+    ex_coll.push_back(ecp.get());
+  }
+  return ex_coll;
+}
+
 void my_finish_example(vw_ptr all, example_ptr ec)
-{ // TODO
+{
+  as_singleline(all->l)->finish_example(*all, *ec);
+}
+
+void my_finish_multi_ex(vw_ptr& all, py::list& ec)
+{
+  auto ex_col = unwrap_example_list(ec);
+  as_multiline(all->l)->finish_example(*all, ex_col);
 }
 
 void my_learn(vw_ptr all, example_ptr ec)
@@ -200,15 +222,8 @@ bool my_is_multiline(vw_ptr all)
 
 template<bool learn>
 void predict_or_learn(vw_ptr& all, py::list& ec)
-{ multi_ex ex_coll;
-  for (ssize_t i = 0; i<len(ec); i++)
-  { py::object eci = ec[i];
-    py::extract<example_ptr> get_ex(eci);
-    example_ptr ecp;
-    if (get_ex.check())
-      ecp = get_ex();
-    ex_coll.push_back(ecp.get());
-  }
+{
+  multi_ex ex_coll = unwrap_example_list(ec);
   if (learn) all->learn(ex_coll);
   else as_multiline(all->l)->predict(ex_coll);
 }
@@ -716,7 +731,8 @@ BOOST_PYTHON_MODULE(pylibvw)
   .def("predict", &my_predict, "given a pyvw example, predict on that example")
   .def("hash_space", &VW::hash_space, "given a namespace (as a string), compute the hash of that namespace")
   .def("hash_feature", &VW::hash_feature, "given a feature string (arg2) and a hashed namespace (arg3), hash that feature")
-  .def("finish_example", &my_finish_example, "tell VW that you're done with a given example")
+  .def("_finish_example", &my_finish_example, "tell VW that you're done with a given example")
+  .def("_finish_example_multi_ex", &my_finish_multi_ex, "tell VW that you're done with the given examples")
   .def("setup_example", &my_setup_example, "given an example that you've created by hand, prepare it for learning (eg, compute quadratic feature)")
   .def("unsetup_example", &unsetup_example, "reverse the process of setup, so that you can go back and modify this example")
 

python/vowpalwabbit/pyvw.py

@@ -13,7 +13,7 @@ def __init__(self, vw, sch, num_actions):
         self.bogus_example = [self.vw.example("1 | x")]
 
     def __del__(self):
-        self.bogus_example[0].finish()
+        self.vw.finish_examples(bogus_example)
 
     def _run(self, your_own_input_example):
         pass
@@ -125,6 +125,23 @@ def parse(self, str_ex, labelType=pylibvw.vw.lDefault):
                 raise TypeError('expecting single line example, got multi_ex of len %i' % len(ec))
         return ec
 
+    def finish_example(self, ex):
+        """Should only be used in conjunction with the parse method"""
+
+        if isinstance(ex, example):
+            if self._is_multiline():
+                raise ValueError('Learner is multiline but single example was passed to finish_example. Use the list of examples instead?')
+            if not ex.finished:
+                pylibvw.vw._finish_example(self, ex)
+                ex.finished = True
+        elif isinstance(ex, list):
+            if not self._is_multiline():
+                raise ValueError('Learner is singleline but multi example was passed to finish_example. Use a single example instead?')
+            if all(x.finished == False for x in ex):
+                pylibvw.vw._finish_example_multi_ex(self, ex)
+                for x in ex:
+                    x.finished = True
+
     def num_weights(self):
         """Get length of weight vector."""
         return pylibvw.vw.num_weights(self)
@@ -154,10 +171,7 @@ def learn(self, ec):
             raise TypeError('expecting string or example object as ec argument for learn, got %s' % type(ec))
 
         if new_example:
-            if isinstance(ec, list):
-                map(lambda x: x.finish(), ec)
-            else:
-                ec.finish()
+            self.finish_example(ec)
 
     def predict(self, ec, prediction_type=None):
         """Just make a prediction on this example; ec can either be an example
@@ -197,10 +211,7 @@ def predict(self, ec, prediction_type=None):
             prediction = get_prediction(ec[0], prediction_type)
 
         if new_example:
-            if isinstance(ec, list):
-                map(lambda x: x.finish(), ec)
-            else:
-                ec.finish()
+            self.finish_example(ec)
 
         return prediction
 
@@ -591,16 +602,9 @@ def __init__(self, vw, initStringOrDictOrRawExample=None, labelType=pylibvw.vw.l
         self.finished = False
         self.labelType = labelType
 
-    def __del__(self):
-        self.finish()
-
     def __enter__(self):
         return self
 
-    def __exit__(self,typ,value,traceback):
-        self.finish()
-        return typ is None
-
     def get_ns(self, id):
         """Construct a namespace_id from either an integer or string
         (or, if a namespace_id is fed it, just return it directly)."""
@@ -746,13 +750,6 @@ def push_features(self, ns, featureList):
         #         raise Exception('malformed feature to push of type: ' + str(type(feature)))
         #     self.push_feature(ns, f, v, ns_hash)
 
-    def finish(self):
-        """Tell VW that you're done with this example and it can
-        recycle it for later use."""
-        if not self.finished:
-            self.vw.finish_example(self)
-            self.finished = True
-
     def iter_features(self):
         """Iterate over all feature/value pairs in this example (all
         namespace included)."""