From 4dc4503bd83a626418063fa18fa4e427cd8f8c4b Mon Sep 17 00:00:00 2001 From: Jack Gerrits Date: Wed, 13 Apr 2022 09:54:12 -0700 Subject: [PATCH] refactor: refactor flatbuffer parser into a lib (#3864) * refactor: refactor flatbuffer parser into a lib * dont remove io and allreduce * move fb tests * fmt * increase LINK_INTERFACE_MULTIPLICITY of fb parser * dont use latest std on windows * remove multiplicity * build fix * change target name --- .github/workflows/build_windows_cmake.yml | 2 +- .scripts/build.cmd | 1 - CMakeLists.txt | 4 - cmake/FindFlatbuffers.cmake | 39 +++++++++ cmake/FlatbufferUtils.cmake | 48 +++++++++++ ext_libs/ext_libs.cmake | 14 +++- test/unit_test/CMakeLists.txt | 4 - utl/flatbuffer/CMakeLists.txt | 2 +- utl/flatbuffer/vw_to_flat.h | 2 +- vowpalwabbit/CMakeLists.txt | 24 +----- vowpalwabbit/fb_parser/CMakeLists.txt | 36 ++++++++ .../vw/fb_parser}/parse_example_flatbuffer.h | 23 +++-- .../schema/example.fbs | 0 .../src}/parse_example_flatbuffer.cc | 13 +-- .../src}/parse_label.cc | 18 ++-- .../tests}/flatbuffer_parser_test.cc | 84 ++++++++++--------- vowpalwabbit/global_data.cc | 2 +- vowpalwabbit/parser.cc | 2 +- vowpalwabbit/parser/flatbuffer/CMakeLists.txt | 5 -- 19 files changed, 219 insertions(+), 104 deletions(-) create mode 100644 cmake/FindFlatbuffers.cmake create mode 100644 cmake/FlatbufferUtils.cmake create mode 100644 vowpalwabbit/fb_parser/CMakeLists.txt rename vowpalwabbit/{parser/flatbuffer => fb_parser/include/vw/fb_parser}/parse_example_flatbuffer.h (79%) rename vowpalwabbit/{parser/flatbuffer => fb_parser}/schema/example.fbs (100%) rename vowpalwabbit/{parser/flatbuffer => fb_parser/src}/parse_example_flatbuffer.cc (97%) rename vowpalwabbit/{parser/flatbuffer => fb_parser/src}/parse_label.cc (95%) rename {test/unit_test => vowpalwabbit/fb_parser/tests}/flatbuffer_parser_test.cc (57%) delete mode 100644 vowpalwabbit/parser/flatbuffer/CMakeLists.txt diff --git a/.github/workflows/build_windows_cmake.yml b/.github/workflows/build_windows_cmake.yml index bcd693bfdf8..927bf57856f 100644 --- a/.github/workflows/build_windows_cmake.yml +++ b/.github/workflows/build_windows_cmake.yml @@ -52,7 +52,7 @@ jobs: run: ${{ env.VCPKG_ROOT }}/vcpkg.exe --triplet x64-windows install zlib boost-test flatbuffers - name: Generate project files run: | - cmake -S "${{ env.SOURCE_DIR }}" -B "${{ env.CMAKE_BUILD_DIR }}" -A "x64" -DCMAKE_TOOLCHAIN_FILE="${{ env.VCPKG_ROOT }}/scripts/buildsystems/vcpkg.cmake" -DUSE_LATEST_STD=On -DBUILD_FLATBUFFERS=On -Dvw_BUILD_NET_FRAMEWORK=On + cmake -S "${{ env.SOURCE_DIR }}" -B "${{ env.CMAKE_BUILD_DIR }}" -A "x64" -DCMAKE_TOOLCHAIN_FILE="${{ env.VCPKG_ROOT }}/scripts/buildsystems/vcpkg.cmake" -DBUILD_FLATBUFFERS=On -Dvw_BUILD_NET_FRAMEWORK=On - name: Build project run: | cmake --build "${{ env.CMAKE_BUILD_DIR }}" --config ${{ matrix.build_config }} diff --git a/.scripts/build.cmd b/.scripts/build.cmd index 01e559c3675..3f07b135077 100644 --- a/.scripts/build.cmd +++ b/.scripts/build.cmd @@ -15,7 +15,6 @@ REM CMAKE_PROGRAM_PATH is for nuget and texttransform cmake -S "%vwRoot%" -B "%vwRoot%\build" -G "Visual Studio 16 2019" -A "x64" ^ -DCMAKE_TOOLCHAIN_FILE="%VCPKG_INSTALLATION_ROOT%\scripts\buildsystems\vcpkg.cmake" ^ -Dvw_BUILD_NET_FRAMEWORK=On ^ - -DUSE_LATEST_STD=On ^ -DBUILD_FLATBUFFERS=On ^ -Dvw_BUILD_NET_FRAMEWORK=On ^ -DRAPIDJSON_SYS_DEP=Off ^ diff --git a/CMakeLists.txt b/CMakeLists.txt index d27128646a6..965dd4cb5f1 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -238,10 +238,6 @@ if(STATIC_LINK_VW) endif() endif() -if(BUILD_FLATBUFFERS) -find_package(flatbuffers REQUIRED) -endif() - # This provides the variables such as CMAKE_INSTALL_LIBDIR for installation paths. include(GNUInstallDirs) diff --git a/cmake/FindFlatbuffers.cmake b/cmake/FindFlatbuffers.cmake new file mode 100644 index 00000000000..10d581536bc --- /dev/null +++ b/cmake/FindFlatbuffers.cmake @@ -0,0 +1,39 @@ +# Copyright 2014 Stefan.Eilemann@epfl.ch +# Copyright 2014 Google Inc. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Find the flatbuffers schema compiler +# +# Output Variables: +# * FLATBUFFERS_FLATC_EXECUTABLE the flatc compiler executable +# * FLATBUFFERS_FOUND +# + +# File modified from original contents to remove FLATBUFFERS_GENERATE_C_HEADERS and inclusion of BuildFlatBuffers.cmake + +set(FLATBUFFERS_CMAKE_DIR ${CMAKE_CURRENT_LIST_DIR}) + +find_program(FLATBUFFERS_FLATC_EXECUTABLE NAMES flatc) +find_path(FLATBUFFERS_INCLUDE_DIR NAMES flatbuffers/flatbuffers.h) + +include(FindPackageHandleStandardArgs) +find_package_handle_standard_args(Flatbuffers + DEFAULT_MSG FLATBUFFERS_FLATC_EXECUTABLE FLATBUFFERS_INCLUDE_DIR) + +if(FLATBUFFERS_FOUND) + set(FLATBUFFERS_INCLUDE_DIRS ${FLATBUFFERS_INCLUDE_DIR}) + include_directories(${CMAKE_BINARY_DIR}) +else() + set(FLATBUFFERS_INCLUDE_DIR) +endif() diff --git a/cmake/FlatbufferUtils.cmake b/cmake/FlatbufferUtils.cmake new file mode 100644 index 00000000000..88abf0ed10c --- /dev/null +++ b/cmake/FlatbufferUtils.cmake @@ -0,0 +1,48 @@ +include(CMakeParseArguments) + +function(add_flatbuffer_schema) + cmake_parse_arguments(ADD_FB_SCHEMA_ARGS + "" + "TARGET;FLATC_EXE;OUTPUT_DIR;FLATC_EXTRA_SCHEMA_ARGS" + "SCHEMAS" + ${ARGN} + ) + + if(NOT DEFINED ADD_FB_SCHEMA_ARGS_TARGET) + message(FATAL_ERROR "Missing TARGET argument to build_flatbuffers") + endif() + + if(NOT DEFINED ADD_FB_SCHEMA_ARGS_FLATC_EXE) + message(FATAL_ERROR "Missing FLATC_EXE argument to build_flatbuffers") + endif() + + if(NOT EXISTS ${ADD_FB_SCHEMA_ARGS_FLATC_EXE}) + message(FATAL_ERROR "FLATC_EXE ${ADD_FB_SCHEMA_ARGS_FLATC_EXE} does not exist") + endif() + + if(NOT DEFINED ADD_FB_SCHEMA_ARGS_OUTPUT_DIR) + message(FATAL_ERROR "Missing OUTPUT_DIR argument to build_flatbuffers") + endif() + + if(NOT DEFINED ADD_FB_SCHEMA_ARGS_SCHEMAS) + message(FATAL_ERROR "Missing SCHEMAS argument to build_flatbuffers") + endif() + + set(FLATC_SCHEMA_ARGS --gen-mutable) + if(DEFINED ADD_FB_SCHEMA_ARGS_FLATC_EXTRA_SCHEMA_ARGS) + set(FLATC_SCHEMA_ARGS ${ADD_FB_SCHEMA_ARGS_FLATC_EXTRA_SCHEMA_ARGS} ${FLATC_SCHEMA_ARGS}) + endif() + + foreach(schema IN ITEMS ${ADD_FB_SCHEMA_ARGS_SCHEMAS}) + get_filename_component(filename ${schema} NAME_WE) + set(generated_file_name ${ADD_FB_SCHEMA_ARGS_OUTPUT_DIR}/${filename}_generated.h) + add_custom_command( + OUTPUT ${generated_file_name} + COMMAND ${ADD_FB_SCHEMA_ARGS_FLATC_EXE} ${FLATC_SCHEMA_ARGS} -o ${ADD_FB_SCHEMA_ARGS_OUTPUT_DIR} -c ${schema} + DEPENDS ${schema} + ) + list(APPEND ALL_SCHEMAS ${generated_file_name}) + endforeach() + + add_custom_target(${ADD_FB_SCHEMA_ARGS_TARGET} DEPENDS ${ALL_SCHEMAS}) +endfunction() diff --git a/ext_libs/ext_libs.cmake b/ext_libs/ext_libs.cmake index a6835d79dc5..edd7c08fb9b 100644 --- a/ext_libs/ext_libs.cmake +++ b/ext_libs/ext_libs.cmake @@ -90,4 +90,16 @@ else() endif() endif() -add_subdirectory(${CMAKE_CURRENT_LIST_DIR}/string-view-lite) \ No newline at end of file +add_subdirectory(${CMAKE_CURRENT_LIST_DIR}/string-view-lite) + +if(BUILD_FLATBUFFERS) + find_package(Flatbuffers CONFIG QUIET) + if(FLATBUFFERS_FOUND) + get_property(flatc_location TARGET flatbuffers::flatc PROPERTY LOCATION) + else() + # Fallback to the old version + find_package(Flatbuffers MODULE REQUIRED) + set(flatc_location ${FLATBUFFERS_FLATC_EXECUTABLE}) + endif() + include(FlatbufferUtils) +endif() \ No newline at end of file diff --git a/test/unit_test/CMakeLists.txt b/test/unit_test/CMakeLists.txt index bfd2708f7df..9159044fa96 100644 --- a/test/unit_test/CMakeLists.txt +++ b/test/unit_test/CMakeLists.txt @@ -55,10 +55,6 @@ add_executable(vw-unit-test.out weights_test.cc ) -if(BUILD_FLATBUFFERS) - target_sources(vw-unit-test.out PRIVATE flatbuffer_parser_test.cc) -endif() - # Add the include directories from vw target for testing target_include_directories(vw-unit-test.out PRIVATE $) target_link_libraries(vw-unit-test.out PRIVATE vw Boost::unit_test_framework) diff --git a/utl/flatbuffer/CMakeLists.txt b/utl/flatbuffer/CMakeLists.txt index ed8b766c27c..40014eda449 100644 --- a/utl/flatbuffer/CMakeLists.txt +++ b/utl/flatbuffer/CMakeLists.txt @@ -1,2 +1,2 @@ add_executable(to_flatbuff txt_to_flat.cc vw_to_flat.cc vw_to_flat.h) -target_link_libraries(to_flatbuff PUBLIC VowpalWabbit::vw explore FlatbuffersTarget) \ No newline at end of file +target_link_libraries(to_flatbuff PUBLIC vw explore vw_fb_parser) diff --git a/utl/flatbuffer/vw_to_flat.h b/utl/flatbuffer/vw_to_flat.h index eed706a7176..b7e257d207c 100644 --- a/utl/flatbuffer/vw_to_flat.h +++ b/utl/flatbuffer/vw_to_flat.h @@ -7,7 +7,7 @@ #include "vw.h" VW_WARNING_STATE_PUSH VW_WARNING_DISABLE_BADLY_FORMED_XML -#include "parser/flatbuffer/generated/example_generated.h" +#include "vw/fb_parser/generated/example_generated.h" VW_WARNING_STATE_POP #include "named_labels.h" #include "simple_label.h" diff --git a/vowpalwabbit/CMakeLists.txt b/vowpalwabbit/CMakeLists.txt index 5c855cd855c..ece37181e51 100644 --- a/vowpalwabbit/CMakeLists.txt +++ b/vowpalwabbit/CMakeLists.txt @@ -4,18 +4,13 @@ add_subdirectory(allreduce) add_subdirectory(active_interactor) add_subdirectory(config) add_subdirectory(slim) - +if(BUILD_FLATBUFFERS) + add_subdirectory(fb_parser) +endif() configure_file(config.h.in config.h) # Use position independent code for all targets in this directory set(CMAKE_POSITION_INDEPENDENT_CODE ON) -if(BUILD_FLATBUFFERS) - add_subdirectory(parser/flatbuffer) - add_library(FlatbuffersTarget INTERFACE) - target_include_directories(FlatbuffersTarget INTERFACE ${FLATBUFFERS_INCLUDE_DIR}) - add_dependencies(FlatbuffersTarget fbschemas) -endif() - set(vw_all_headers accumulate.h action_score.h @@ -202,9 +197,6 @@ set(vw_all_headers vwdll.h ) -if(BUILD_FLATBUFFERS) - set(vw_all_headers ${vw_all_headers} parser/flatbuffer/parse_example_flatbuffer.h) -endif() if(BUILD_EXTERNAL_PARSER) set(vw_all_headers ${vw_all_headers} ${external_parser_headers}) @@ -354,12 +346,6 @@ set(vw_all_sources vw_validate.cc ) -if(BUILD_FLATBUFFERS) - set(vw_all_sources ${vw_all_sources} - parser/flatbuffer/parse_example_flatbuffer.cc - parser/flatbuffer/parse_label.cc) -endif() - if(BUILD_EXTERNAL_PARSER) set(vw_all_sources ${vw_all_sources} ${external_parser_sources}) endif() @@ -405,9 +391,7 @@ if (BUILD_PRIVACY_ACTIVATION) endif() if(BUILD_FLATBUFFERS) - target_link_libraries(vw - PRIVATE - $) + target_link_libraries(vw PRIVATE vw_fb_parser) target_compile_definitions(vw PUBLIC BUILD_FLATBUFFERS) endif() diff --git a/vowpalwabbit/fb_parser/CMakeLists.txt b/vowpalwabbit/fb_parser/CMakeLists.txt new file mode 100644 index 00000000000..93a1ba4e214 --- /dev/null +++ b/vowpalwabbit/fb_parser/CMakeLists.txt @@ -0,0 +1,36 @@ +# Generate header file +add_flatbuffer_schema( + TARGET fbschemas + SCHEMAS "${CMAKE_CURRENT_LIST_DIR}/schema/example.fbs" + OUTPUT_DIR "${CMAKE_CURRENT_BINARY_DIR}/include/vw/fb_parser/generated/" + FLATC_EXE ${flatc_location} +) +add_library(fb_generate_headers INTERFACE) +add_dependencies(fb_generate_headers fbschemas) +target_include_directories(fb_generate_headers INTERFACE ${CMAKE_CURRENT_BINARY_DIR}/include/) + +set(vw_fb_parser_sources + include/vw/fb_parser/parse_example_flatbuffer.h + src/parse_example_flatbuffer.cc + src/parse_label.cc +) + +vw_add_library( + NAME "fb_parser" + TYPE "STATIC_ONLY" + SOURCES ${vw_fb_parser_sources} + PUBLIC_DEPS VowpalWabbit::vw $ +) +target_include_directories(vw_fb_parser PUBLIC ${FLATBUFFERS_INCLUDE_DIR}) + +# Install the generated header +# if(VW_INSTALL) +# install(DIRECTORY ${CMAKE_CURRENT_BINARY_DIR}/include/ +# DESTINATION ${CMAKE_INSTALL_INCLUDEDIR}) +# endif() + +vw_add_test_executable( + FOR_LIB "fb_parser" + SOURCES "tests/flatbuffer_parser_test.cc" + EXTRA_DEPS VowpalWabbit::vw +) \ No newline at end of file diff --git a/vowpalwabbit/parser/flatbuffer/parse_example_flatbuffer.h b/vowpalwabbit/fb_parser/include/vw/fb_parser/parse_example_flatbuffer.h similarity index 79% rename from vowpalwabbit/parser/flatbuffer/parse_example_flatbuffer.h rename to vowpalwabbit/fb_parser/include/vw/fb_parser/parse_example_flatbuffer.h index a733c9d1b9b..c8a6f78671f 100644 --- a/vowpalwabbit/parser/flatbuffer/parse_example_flatbuffer.h +++ b/vowpalwabbit/fb_parser/include/vw/fb_parser/parse_example_flatbuffer.h @@ -4,11 +4,17 @@ #pragma once -#include "../../vw.h" -VW_WARNING_STATE_PUSH -VW_WARNING_DISABLE_BADLY_FORMED_XML -#include "generated/example_generated.h" -VW_WARNING_STATE_POP +#include "v_array.h" +#include "vw/fb_parser/generated/example_generated.h" + +class io_buf; +namespace VW +{ +struct workspace; +struct example; +struct polylabel; +class reduction_features; +} // namespace VW namespace VW { @@ -16,14 +22,15 @@ namespace parsers { namespace flatbuffer { -int flatbuffer_to_examples(VW::workspace* all, io_buf& buf, v_array& examples); +int flatbuffer_to_examples(VW::workspace* all, io_buf& buf, VW::v_array& examples); class parser { public: parser() = default; const VW::parsers::flatbuffer::ExampleRoot* data(); - bool parse_examples(VW::workspace* all, io_buf& buf, v_array& examples, uint8_t* buffer_pointer = nullptr); + bool parse_examples( + VW::workspace* all, io_buf& buf, VW::v_array& examples, uint8_t* buffer_pointer = nullptr); private: const VW::parsers::flatbuffer::ExampleRoot* _data; @@ -38,7 +45,7 @@ class parser uint64_t _c_hash = 0; bool parse(io_buf& buf, uint8_t* buffer_pointer = nullptr); - void process_collection_item(VW::workspace* all, v_array& examples); + void process_collection_item(VW::workspace* all, VW::v_array& examples); void parse_example(VW::workspace* all, example* ae, const Example* eg); void parse_multi_example(VW::workspace* all, example* ae, const MultiExample* eg); void parse_namespaces(VW::workspace* all, example* ae, const Namespace* ns); diff --git a/vowpalwabbit/parser/flatbuffer/schema/example.fbs b/vowpalwabbit/fb_parser/schema/example.fbs similarity index 100% rename from vowpalwabbit/parser/flatbuffer/schema/example.fbs rename to vowpalwabbit/fb_parser/schema/example.fbs diff --git a/vowpalwabbit/parser/flatbuffer/parse_example_flatbuffer.cc b/vowpalwabbit/fb_parser/src/parse_example_flatbuffer.cc similarity index 97% rename from vowpalwabbit/parser/flatbuffer/parse_example_flatbuffer.cc rename to vowpalwabbit/fb_parser/src/parse_example_flatbuffer.cc index 50e872e654a..f68c774d795 100644 --- a/vowpalwabbit/parser/flatbuffer/parse_example_flatbuffer.cc +++ b/vowpalwabbit/fb_parser/src/parse_example_flatbuffer.cc @@ -2,13 +2,14 @@ // individual contributors. All rights reserved. Released under a BSD (revised) // license as described in the file LICENSE. -#include "parse_example_flatbuffer.h" +#include "vw/fb_parser/parse_example_flatbuffer.h" -#include "../../action_score.h" -#include "../../best_constant.h" -#include "../../cb.h" -#include "../../constant.h" -#include "../../global_data.h" +#include "action_score.h" +#include "best_constant.h" +#include "cb.h" +#include "constant.h" +#include "global_data.h" +#include "parser.h" #include #include diff --git a/vowpalwabbit/parser/flatbuffer/parse_label.cc b/vowpalwabbit/fb_parser/src/parse_label.cc similarity index 95% rename from vowpalwabbit/parser/flatbuffer/parse_label.cc rename to vowpalwabbit/fb_parser/src/parse_label.cc index 0f6215306cc..af8d2c4bdb5 100644 --- a/vowpalwabbit/parser/flatbuffer/parse_label.cc +++ b/vowpalwabbit/fb_parser/src/parse_label.cc @@ -2,15 +2,15 @@ // individual contributors. All rights reserved. Released under a BSD (revised) // license as described in the file LICENSE. -#include "../../action_score.h" -#include "../../best_constant.h" -#include "../../cb.h" -#include "../../constant.h" -#include "../../example.h" -#include "../../global_data.h" -#include "../../named_labels.h" -#include "../../slates_label.h" -#include "parse_example_flatbuffer.h" +#include "action_score.h" +#include "best_constant.h" +#include "cb.h" +#include "constant.h" +#include "example.h" +#include "global_data.h" +#include "named_labels.h" +#include "slates_label.h" +#include "vw/fb_parser/parse_example_flatbuffer.h" #include #include diff --git a/test/unit_test/flatbuffer_parser_test.cc b/vowpalwabbit/fb_parser/tests/flatbuffer_parser_test.cc similarity index 57% rename from test/unit_test/flatbuffer_parser_test.cc rename to vowpalwabbit/fb_parser/tests/flatbuffer_parser_test.cc index 9cd16d14351..5e48a13e265 100644 --- a/test/unit_test/flatbuffer_parser_test.cc +++ b/vowpalwabbit/fb_parser/tests/flatbuffer_parser_test.cc @@ -2,15 +2,17 @@ // individual contributors. All rights reserved. Released under a BSD (revised) // license as described in the file LICENSE. -#include -#include +#include "constant.h" +#include "feature_group.h" +#include "parse_example.h" +#include "vw.h" +#include "vw/fb_parser/parse_example_flatbuffer.h" -#include "test_common.h" +#include +#include +#include #include -#include "parser/flatbuffer/parse_example_flatbuffer.h" -#include "constant.h" -#include "feature_group.h" flatbuffers::Offset get_label(flatbuffers::FlatBufferBuilder& builder, VW::parsers::flatbuffer::Label label_type) { @@ -53,7 +55,7 @@ flatbuffers::Offset sample_flatbuffer( return CreateExampleRoot(builder, VW::parsers::flatbuffer::ExampleType_Example, example.Union()); } -BOOST_AUTO_TEST_CASE(test_flatbuffer_standalone_example) +TEST(flatbuffer_parser_tests, test_flatbuffer_standalone_example) { auto all = VW::initialize("--no_stdin --quiet --flatbuffer", nullptr, false, nullptr, nullptr); @@ -71,33 +73,33 @@ BOOST_AUTO_TEST_CASE(test_flatbuffer_standalone_example) all->flat_converter->parse_examples(all, unused_buffer, examples, buf); auto example = all->flat_converter->data()->example_obj_as_Example(); - BOOST_CHECK_EQUAL(example->namespaces()->size(), 1); - BOOST_CHECK_EQUAL(example->namespaces()->Get(0)->features()->size(), 1); - BOOST_CHECK_CLOSE(example->label_as_SimpleLabel()->label(), 0.0, FLOAT_TOL); - BOOST_CHECK_CLOSE(example->label_as_SimpleLabel()->weight(), 1.0, FLOAT_TOL); - BOOST_CHECK_EQUAL(example->namespaces()->Get(0)->hash(), constant_namespace); - BOOST_CHECK_EQUAL(example->namespaces()->Get(0)->full_hash(), constant_namespace); - BOOST_CHECK_EQUAL(example->namespaces()->Get(0)->features()->Get(0)->name()->c_str(), "hello"); - BOOST_CHECK_EQUAL(example->namespaces()->Get(0)->features()->Get(0)->hash(), constant); - BOOST_CHECK_CLOSE(example->namespaces()->Get(0)->features()->Get(0)->value(), 2.23, FLOAT_TOL); + EXPECT_EQ(example->namespaces()->size(), 1); + EXPECT_EQ(example->namespaces()->Get(0)->features()->size(), 1); + EXPECT_FLOAT_EQ(example->label_as_SimpleLabel()->label(), 0.0); + EXPECT_FLOAT_EQ(example->label_as_SimpleLabel()->weight(), 1.0); + EXPECT_EQ(example->namespaces()->Get(0)->hash(), constant_namespace); + EXPECT_EQ(example->namespaces()->Get(0)->full_hash(), constant_namespace); + EXPECT_STREQ(example->namespaces()->Get(0)->features()->Get(0)->name()->c_str(), "hello"); + EXPECT_EQ(example->namespaces()->Get(0)->features()->Get(0)->hash(), constant); + EXPECT_FLOAT_EQ(example->namespaces()->Get(0)->features()->Get(0)->value(), 2.23); // Check vw example - BOOST_CHECK_EQUAL(examples.size(), 1); - BOOST_CHECK_CLOSE(examples[0]->l.simple.label, 0.f, FLOAT_TOL); + EXPECT_EQ(examples.size(), 1); + EXPECT_FLOAT_EQ(examples[0]->l.simple.label, 0.f); const auto& red_features = examples[0]->_reduction_features.template get(); - BOOST_CHECK_CLOSE(red_features.weight, 1.f, FLOAT_TOL); + EXPECT_FLOAT_EQ(red_features.weight, 1.f); - BOOST_CHECK_EQUAL(examples[0]->indices[0], constant_namespace); - BOOST_CHECK_CLOSE(examples[0]->feature_space[examples[0]->indices[0]].values[0], 2.23f, FLOAT_TOL); - BOOST_CHECK_EQUAL(examples[0]->feature_space[examples[0]->indices[0]].namespace_extents.size(), 1); - BOOST_CHECK_EQUAL(examples[0]->feature_space[examples[0]->indices[0]].namespace_extents[0], + EXPECT_EQ(examples[0]->indices[0], constant_namespace); + EXPECT_FLOAT_EQ(examples[0]->feature_space[examples[0]->indices[0]].values[0], 2.23f); + EXPECT_EQ(examples[0]->feature_space[examples[0]->indices[0]].namespace_extents.size(), 1); + EXPECT_EQ(examples[0]->feature_space[examples[0]->indices[0]].namespace_extents[0], (VW::namespace_extent{0, 1, constant_namespace})); VW::finish_example(*all, *examples[0]); VW::finish(*all); } -BOOST_AUTO_TEST_CASE(test_flatbuffer_collection) +TEST(flatbuffer_parser_tests, test_flatbuffer_collection) { auto all = VW::initialize("--no_stdin --quiet --flatbuffer", nullptr, false, nullptr, nullptr); @@ -115,27 +117,27 @@ BOOST_AUTO_TEST_CASE(test_flatbuffer_collection) all->flat_converter->parse_examples(all, unused_buffer, examples, buf); auto collection_examples = all->flat_converter->data()->example_obj_as_ExampleCollection()->examples(); - BOOST_CHECK_EQUAL(collection_examples->size(), 1); - BOOST_CHECK_EQUAL(collection_examples->Get(0)->namespaces()->size(), 1); - BOOST_CHECK_EQUAL(collection_examples->Get(0)->namespaces()->Get(0)->features()->size(), 1); - BOOST_CHECK_CLOSE(collection_examples->Get(0)->label_as_SimpleLabel()->label(), 0.0, FLOAT_TOL); - BOOST_CHECK_CLOSE(collection_examples->Get(0)->label_as_SimpleLabel()->weight(), 1.0, FLOAT_TOL); - BOOST_CHECK_EQUAL(collection_examples->Get(0)->namespaces()->Get(0)->hash(), constant_namespace); - BOOST_CHECK_EQUAL(collection_examples->Get(0)->namespaces()->Get(0)->full_hash(), constant_namespace); - BOOST_CHECK_EQUAL(collection_examples->Get(0)->namespaces()->Get(0)->features()->Get(0)->name()->c_str(), "hello"); - BOOST_CHECK_EQUAL(collection_examples->Get(0)->namespaces()->Get(0)->features()->Get(0)->hash(), constant); - BOOST_CHECK_CLOSE(collection_examples->Get(0)->namespaces()->Get(0)->features()->Get(0)->value(), 2.23, FLOAT_TOL); + EXPECT_EQ(collection_examples->size(), 1); + EXPECT_EQ(collection_examples->Get(0)->namespaces()->size(), 1); + EXPECT_EQ(collection_examples->Get(0)->namespaces()->Get(0)->features()->size(), 1); + EXPECT_FLOAT_EQ(collection_examples->Get(0)->label_as_SimpleLabel()->label(), 0.0); + EXPECT_FLOAT_EQ(collection_examples->Get(0)->label_as_SimpleLabel()->weight(), 1.0); + EXPECT_EQ(collection_examples->Get(0)->namespaces()->Get(0)->hash(), constant_namespace); + EXPECT_EQ(collection_examples->Get(0)->namespaces()->Get(0)->full_hash(), constant_namespace); + EXPECT_STREQ(collection_examples->Get(0)->namespaces()->Get(0)->features()->Get(0)->name()->c_str(), "hello"); + EXPECT_EQ(collection_examples->Get(0)->namespaces()->Get(0)->features()->Get(0)->hash(), constant); + EXPECT_FLOAT_EQ(collection_examples->Get(0)->namespaces()->Get(0)->features()->Get(0)->value(), 2.23); // check vw example - BOOST_CHECK_EQUAL(examples.size(), 1); - BOOST_CHECK_CLOSE(examples[0]->l.simple.label, 0.f, FLOAT_TOL); + EXPECT_EQ(examples.size(), 1); + EXPECT_FLOAT_EQ(examples[0]->l.simple.label, 0.f); const auto& red_features = examples[0]->_reduction_features.template get(); - BOOST_CHECK_CLOSE(red_features.weight, 1.f, FLOAT_TOL); + EXPECT_FLOAT_EQ(red_features.weight, 1.f); - BOOST_CHECK_EQUAL(examples[0]->indices[0], constant_namespace); - BOOST_CHECK_CLOSE(examples[0]->feature_space[examples[0]->indices[0]].values[0], 2.23f, FLOAT_TOL); - BOOST_CHECK_EQUAL(examples[0]->feature_space[examples[0]->indices[0]].namespace_extents.size(), 1); - BOOST_CHECK_EQUAL(examples[0]->feature_space[examples[0]->indices[0]].namespace_extents[0], + EXPECT_EQ(examples[0]->indices[0], constant_namespace); + EXPECT_FLOAT_EQ(examples[0]->feature_space[examples[0]->indices[0]].values[0], 2.23f); + EXPECT_EQ(examples[0]->feature_space[examples[0]->indices[0]].namespace_extents.size(), 1); + EXPECT_EQ(examples[0]->feature_space[examples[0]->indices[0]].namespace_extents[0], (VW::namespace_extent{0, 1, constant_namespace})); VW::finish_example(*all, *examples[0]); diff --git a/vowpalwabbit/global_data.cc b/vowpalwabbit/global_data.cc index 7ddeabea2e7..71b723ed8d5 100644 --- a/vowpalwabbit/global_data.cc +++ b/vowpalwabbit/global_data.cc @@ -37,7 +37,7 @@ #include #ifdef BUILD_FLATBUFFERS -# include "parser/flatbuffer/parse_example_flatbuffer.h" +# include "vw/fb_parser/parse_example_flatbuffer.h" #endif #ifdef BUILD_EXTERNAL_PARSER # include "parse_example_external.h" diff --git a/vowpalwabbit/parser.cc b/vowpalwabbit/parser.cc index 53e7799466d..42c5b0596c4 100644 --- a/vowpalwabbit/parser.cc +++ b/vowpalwabbit/parser.cc @@ -73,7 +73,7 @@ int VW_getpid() { return (int)::GetCurrentProcessId(); } #include #include #ifdef BUILD_FLATBUFFERS -# include "parser/flatbuffer/parse_example_flatbuffer.h" +# include "vw/fb_parser/parse_example_flatbuffer.h" #endif #ifdef BUILD_EXTERNAL_PARSER diff --git a/vowpalwabbit/parser/flatbuffer/CMakeLists.txt b/vowpalwabbit/parser/flatbuffer/CMakeLists.txt deleted file mode 100644 index 2b7f0fb00d3..00000000000 --- a/vowpalwabbit/parser/flatbuffer/CMakeLists.txt +++ /dev/null @@ -1,5 +0,0 @@ -set(EXAMPLE_SCHEMA - "${CMAKE_CURRENT_SOURCE_DIR}/schema/example.fbs") - -build_flatbuffers("${EXAMPLE_SCHEMA}" "" fbschemas "" "${CMAKE_CURRENT_SOURCE_DIR}/generated" "" "") -