[multistep] master merge (VowpalWabbit#395)

* [Binary parser] reward functions for ccb format (VowpalWabbit#361)

* [Binary parser] refactoring rewards (VowpalWabbit#366)

* Example gen add ccb loop for e2e testing (VowpalWabbit#368)

* [Binary parser] add e2e ccb test, compare dsjson and fb logged files create the same model (VowpalWabbit#369)

* minor binary parser cleanup (VowpalWabbit#370)

* [Binary parser] add external parser test for apprentice mode cb (VowpalWabbit#374)

* CCB apprentice reward (VowpalWabbit#373)

* [Binary parser] add metrics for cb (VowpalWabbit#371)

* [Binary parser] don't log when skip learn, more tests, skip over unknown msg type (VowpalWabbit#375)

* [binary parser] ccb skip learn (VowpalWabbit#376)

* refactor: add error message to fix config file (VowpalWabbit#377)

* Fix CI's after flatbuffer version update to 2.0 (VowpalWabbit#390)

* try set fb span minimal

* add to preprocessor definitions

* add to unit_test project file

* Revert "mac ci: continue on error true (VowpalWabbit#327)" (VowpalWabbit#385)

* Fix python build path on windows, and formatting. (VowpalWabbit#383)

* Update build_docs.yml (VowpalWabbit#391)

* only convert timestamp to string before exiting (VowpalWabbit#382)

* ntohl is a define on osx, rename the function. (VowpalWabbit#386)

* Add bunch of nice to haves CLI options and fix FB 2.0 compat. (VowpalWabbit#387)

* our build requires CMP0074 due to usage of PackageName_ROOT variables. (VowpalWabbit#393)

* our build requires CMP0074 due to usage of PackageName_ROOT variables.

* try to use cmake_policy

* Activations in multistep: first PR with schema changes only (VowpalWabbit#392)

* deferred action to multistep schema

* Multistep to problem type

* try to set cmake policy for CMP0074

* OLD -> NEW

* try default policy for cmp0074

* build fix

Co-authored-by: cheng-tan <chengtan2013@gmail.com>
Co-authored-by: olgavrou <olgavrou@gmail.com>
Co-authored-by: Griffin Bassman <griffinbassman@gmail.com>
Co-authored-by: Eduardo Salinas <edus@microsoft.com>
Co-authored-by: zwd-ms <71728747+zwd-ms@users.noreply.github.com>
Co-authored-by: Rodrigo Kumpera <kumpera@users.noreply.github.com>

.pipelines/build-macos.yml

@@ -11,14 +11,10 @@ pool:
 steps:
 - checkout: self
   submodules: true
-  continueOnError: true
 - bash: .scripts/macos/restore.sh
   displayName: 'Restore dependencies'
-  continueOnError: true
 - bash: .scripts/macos/build.sh
   displayName: 'Build'
-  continueOnError: true
 - bash: .scripts/macos/test.sh
   displayName: 'Run tests'
-  continueOnError: true
 - task: PublishTestResults@2

external_parser/event_processors/joined_event.h

@@ -154,6 +154,43 @@ struct cb_joined_event : public typed_joined_event {
   float get_sum_original_reward() const override {
     return original_reward;
   }
+
+  bool should_calculate_reward(
+      const std::vector<reward::outcome_event> &outcome_events) {
+    return outcome_events.size() > 0 &&
+           std::any_of(outcome_events.begin(), outcome_events.end(),
+                       [](const reward::outcome_event &o) {
+                         return o.action_taken != true;
+                       });
+  }
+
+  void calc_and_set_cost(
+      v_array<example *> &examples, float default_reward,
+      reward::RewardFunctionType reward_function,
+      const metadata::event_metadata_info &interaction_metadata,
+      std::vector<reward::outcome_event> &outcome_events) override {
+    reward = default_reward;
+    // original reward is used to record the observed reward of apprentice mode
+    original_reward = default_reward;
+
+    if (should_calculate_reward(outcome_events)) {
+      original_reward = reward_function(outcome_events);
+
+      if (interaction_metadata.learning_mode == v2::LearningModeType_Apprentice) {
+        set_apprentice_reward();
+      } else {
+        reward = original_reward;
+      }
+    }
+
+    set_cost(examples, reward);
+  }
+
+  void calculate_metrics(dsjson_metrics* metrics) override {
+    if (metrics && interaction_data.actions.size() == 0) {
+      metrics->NumberOfEventsZeroActions++;
+    }
+  }
 };
 
 struct ccb_joined_event : public typed_joined_event {

external_parser/event_processors/metadata.h

@@ -17,4 +17,4 @@ struct event_metadata_info {
   std::string event_id;
   v2::LearningModeType learning_mode;
 };
-} // namespace metadata
+} // namespace metadata

external_parser/joiners/example_joiner.h

@@ -70,6 +70,7 @@ class example_joiner : public i_joiner {
    * in the metadata)
    */
   bool process_joined(v_array<example *> &examples) override;
+
   // true if there are still event-groups to be processed from a deserialized
   // batch
   bool processing_batch() override;

external_parser/parse_example_binary.cc

@@ -17,7 +17,6 @@
 #include "joiners/example_joiner.h"
 #include "memory.h"
 #include "parse_example_binary.h"
-#include "joiners/example_joiner.h"
 
 // TODO need to check if errors will be detected from stderr/stdout/other and
 // use appropriate logger

external_parser/parse_example_external.cc

@@ -109,6 +109,10 @@ parser::get_external_parser(vw *all, const input_options &parsed_options) {
       all->example_parser->metrics = VW::make_unique<dsjson_metrics>();
     }
 
+    if (all->options->was_supplied("extra_metrics")) {
+      all->example_parser->metrics = VW::make_unique<dsjson_metrics>();
+    }
+
     return VW::make_unique<binary_parser>(std::move(joiner));
   }
   throw std::runtime_error("external parser type not recognised");

rlclientlib/extensions/onnx/src/onnx_model.h

@@ -32,6 +32,11 @@ namespace reinforcement_learning { namespace onnx {
       return error_code::not_supported;
     }
 
+    int choose_rank_multistep(uint64_t rnd_seed, const char* features, const episode_history* history, std::vector<int>& action_ids, std::vector<float>& action_pdf, std::string& model_version, api_status* status = nullptr) override
+    {
+      return error_code::not_supported;
+    }
+
     model_management::model_type_t model_type() const { return model_management::model_type_t::CB; }
 
   private:

setup.py

@@ -14,6 +14,7 @@
     "win-arm64": "ARM64",
 }
 
+
 class CMakeDistribution(Distribution):
     global_options = Distribution.global_options
 
@@ -29,6 +30,7 @@ def __init__(self, attrs=None):
         self.cmake_options = None
         Distribution.__init__(self, attrs)
 
+
 # A CMakeExtension needs a sourcedir instead of a file list.
 # The name must be the _single_ output extension from the CMake build.
 # If you need multiple extensions, see scikit-build.

test_tools/log_parser/joiner.py

@@ -174,7 +174,6 @@ def mk_offsets_vector(builder, arr, startFun):
     startFun(builder, len(arr))
     for i in reversed(range(len(arr))):
         builder.PrependUOffsetTRelative(arr[i])
-
     return end_vector_shim(builder, len(arr))
 
 def mk_bytes_vector(builder, arr):

unit_test/unit_test.vcxproj

@@ -241,4 +241,4 @@
       <Command Condition="'$(Platform)'=='Win32'">xcopy /Y $(SolutionDir)packages\boost_unit_test_framework-vc141.1.70.0.0\lib\native\*.dll $(OutDir)</Command>
     </PostBuildEvent>
   </ItemDefinitionGroup>
-</Project>
+</Project>