refactor: refactor flatbuffer parser into a lib (#3864)

* refactor: refactor flatbuffer parser into a lib

* dont remove io and allreduce

* move fb tests

* fmt

* increase LINK_INTERFACE_MULTIPLICITY of fb parser

* dont use latest std on windows

* remove multiplicity

* build fix

* change target name

.github/workflows/build_windows_cmake.yml

@@ -52,7 +52,7 @@ jobs:
         run: ${{ env.VCPKG_ROOT }}/vcpkg.exe --triplet x64-windows install zlib boost-test flatbuffers
       - name: Generate project files
         run: |
-          cmake -S "${{ env.SOURCE_DIR }}" -B "${{ env.CMAKE_BUILD_DIR }}" -A "x64" -DCMAKE_TOOLCHAIN_FILE="${{ env.VCPKG_ROOT }}/scripts/buildsystems/vcpkg.cmake" -DUSE_LATEST_STD=On -DBUILD_FLATBUFFERS=On -Dvw_BUILD_NET_FRAMEWORK=On
+          cmake -S "${{ env.SOURCE_DIR }}" -B "${{ env.CMAKE_BUILD_DIR }}" -A "x64" -DCMAKE_TOOLCHAIN_FILE="${{ env.VCPKG_ROOT }}/scripts/buildsystems/vcpkg.cmake" -DBUILD_FLATBUFFERS=On -Dvw_BUILD_NET_FRAMEWORK=On
       - name: Build project
         run: |
           cmake --build "${{ env.CMAKE_BUILD_DIR }}" --config ${{ matrix.build_config }}

.scripts/build.cmd

@@ -15,7 +15,6 @@ REM CMAKE_PROGRAM_PATH is for nuget and texttransform
 cmake -S "%vwRoot%" -B "%vwRoot%\build" -G "Visual Studio 16 2019" -A "x64" ^
     -DCMAKE_TOOLCHAIN_FILE="%VCPKG_INSTALLATION_ROOT%\scripts\buildsystems\vcpkg.cmake" ^
     -Dvw_BUILD_NET_FRAMEWORK=On ^
-    -DUSE_LATEST_STD=On ^
     -DBUILD_FLATBUFFERS=On ^
     -Dvw_BUILD_NET_FRAMEWORK=On ^
     -DRAPIDJSON_SYS_DEP=Off ^

CMakeLists.txt

@@ -238,10 +238,6 @@ if(STATIC_LINK_VW)
   endif()
 endif()
 
-if(BUILD_FLATBUFFERS)
-find_package(flatbuffers REQUIRED)
-endif()
-
 # This provides the variables such as CMAKE_INSTALL_LIBDIR for installation paths.
 include(GNUInstallDirs)
 

cmake/FindFlatbuffers.cmake

@@ -0,0 +1,39 @@
+# Copyright 2014 Stefan.Eilemann@epfl.ch
+# Copyright 2014 Google Inc. All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# Find the flatbuffers schema compiler
+#
+# Output Variables:
+# * FLATBUFFERS_FLATC_EXECUTABLE the flatc compiler executable
+# * FLATBUFFERS_FOUND
+#
+
+# File modified from original contents to remove FLATBUFFERS_GENERATE_C_HEADERS and inclusion of BuildFlatBuffers.cmake
+
+set(FLATBUFFERS_CMAKE_DIR ${CMAKE_CURRENT_LIST_DIR})
+
+find_program(FLATBUFFERS_FLATC_EXECUTABLE NAMES flatc)
+find_path(FLATBUFFERS_INCLUDE_DIR NAMES flatbuffers/flatbuffers.h)
+
+include(FindPackageHandleStandardArgs)
+find_package_handle_standard_args(Flatbuffers
+  DEFAULT_MSG FLATBUFFERS_FLATC_EXECUTABLE FLATBUFFERS_INCLUDE_DIR)
+
+if(FLATBUFFERS_FOUND)
+  set(FLATBUFFERS_INCLUDE_DIRS ${FLATBUFFERS_INCLUDE_DIR})
+  include_directories(${CMAKE_BINARY_DIR})
+else()
+  set(FLATBUFFERS_INCLUDE_DIR)
+endif()

cmake/FlatbufferUtils.cmake

@@ -0,0 +1,48 @@
+include(CMakeParseArguments)
+
+function(add_flatbuffer_schema)
+  cmake_parse_arguments(ADD_FB_SCHEMA_ARGS
+    ""
+    "TARGET;FLATC_EXE;OUTPUT_DIR;FLATC_EXTRA_SCHEMA_ARGS"
+    "SCHEMAS"
+    ${ARGN}
+  )
+
+  if(NOT DEFINED ADD_FB_SCHEMA_ARGS_TARGET)
+    message(FATAL_ERROR "Missing TARGET argument to build_flatbuffers")
+  endif()
+
+  if(NOT DEFINED ADD_FB_SCHEMA_ARGS_FLATC_EXE)
+    message(FATAL_ERROR "Missing FLATC_EXE argument to build_flatbuffers")
+  endif()
+
+  if(NOT EXISTS ${ADD_FB_SCHEMA_ARGS_FLATC_EXE})
+    message(FATAL_ERROR "FLATC_EXE ${ADD_FB_SCHEMA_ARGS_FLATC_EXE} does not exist")
+  endif()
+
+  if(NOT DEFINED ADD_FB_SCHEMA_ARGS_OUTPUT_DIR)
+    message(FATAL_ERROR "Missing OUTPUT_DIR argument to build_flatbuffers")
+  endif()
+
+  if(NOT DEFINED ADD_FB_SCHEMA_ARGS_SCHEMAS)
+    message(FATAL_ERROR "Missing SCHEMAS argument to build_flatbuffers")
+  endif()
+
+  set(FLATC_SCHEMA_ARGS --gen-mutable)
+  if(DEFINED ADD_FB_SCHEMA_ARGS_FLATC_EXTRA_SCHEMA_ARGS)
+    set(FLATC_SCHEMA_ARGS ${ADD_FB_SCHEMA_ARGS_FLATC_EXTRA_SCHEMA_ARGS} ${FLATC_SCHEMA_ARGS})
+  endif()
+
+  foreach(schema IN ITEMS ${ADD_FB_SCHEMA_ARGS_SCHEMAS})
+    get_filename_component(filename ${schema} NAME_WE)
+    set(generated_file_name ${ADD_FB_SCHEMA_ARGS_OUTPUT_DIR}/${filename}_generated.h)
+    add_custom_command(
+        OUTPUT ${generated_file_name}
+        COMMAND ${ADD_FB_SCHEMA_ARGS_FLATC_EXE} ${FLATC_SCHEMA_ARGS} -o ${ADD_FB_SCHEMA_ARGS_OUTPUT_DIR} -c ${schema}
+        DEPENDS ${schema}
+    )
+    list(APPEND ALL_SCHEMAS ${generated_file_name})
+  endforeach()
+
+  add_custom_target(${ADD_FB_SCHEMA_ARGS_TARGET} DEPENDS ${ALL_SCHEMAS})
+endfunction()

ext_libs/ext_libs.cmake

@@ -90,4 +90,16 @@ else()
   endif()
 endif()
 
-add_subdirectory(${CMAKE_CURRENT_LIST_DIR}/string-view-lite)
+add_subdirectory(${CMAKE_CURRENT_LIST_DIR}/string-view-lite)
+
+if(BUILD_FLATBUFFERS)
+  find_package(Flatbuffers CONFIG QUIET)
+  if(FLATBUFFERS_FOUND)
+    get_property(flatc_location TARGET flatbuffers::flatc PROPERTY LOCATION)
+  else()
+    # Fallback to the old version
+    find_package(Flatbuffers MODULE REQUIRED)
+    set(flatc_location ${FLATBUFFERS_FLATC_EXECUTABLE})
+  endif()
+  include(FlatbufferUtils)
+endif()

test/unit_test/CMakeLists.txt

@@ -55,10 +55,6 @@ add_executable(vw-unit-test.out
   weights_test.cc
 )
 
-if(BUILD_FLATBUFFERS)
-  target_sources(vw-unit-test.out PRIVATE flatbuffer_parser_test.cc)
-endif()
-
 # Add the include directories from vw target for testing
 target_include_directories(vw-unit-test.out PRIVATE $<TARGET_PROPERTY:vw,INCLUDE_DIRECTORIES>)
 target_link_libraries(vw-unit-test.out PRIVATE vw Boost::unit_test_framework)

utl/flatbuffer/CMakeLists.txt

@@ -1,2 +1,2 @@
 add_executable(to_flatbuff txt_to_flat.cc vw_to_flat.cc vw_to_flat.h)
-target_link_libraries(to_flatbuff PUBLIC VowpalWabbit::vw explore FlatbuffersTarget)
+target_link_libraries(to_flatbuff PUBLIC vw explore vw_fb_parser)

utl/flatbuffer/vw_to_flat.h

@@ -7,7 +7,7 @@
 #include "vw.h"
 VW_WARNING_STATE_PUSH
 VW_WARNING_DISABLE_BADLY_FORMED_XML
-#include "parser/flatbuffer/generated/example_generated.h"
+#include "vw/fb_parser/generated/example_generated.h"
 VW_WARNING_STATE_POP
 #include "named_labels.h"
 #include "simple_label.h"

vowpalwabbit/CMakeLists.txt

@@ -4,18 +4,13 @@ add_subdirectory(allreduce)
 add_subdirectory(active_interactor)
 add_subdirectory(config)
 add_subdirectory(slim)
-
+if(BUILD_FLATBUFFERS)
+  add_subdirectory(fb_parser)
+endif()
 configure_file(config.h.in config.h)
 # Use position independent code for all targets in this directory
 set(CMAKE_POSITION_INDEPENDENT_CODE ON)
 
-if(BUILD_FLATBUFFERS)
-  add_subdirectory(parser/flatbuffer)
-  add_library(FlatbuffersTarget INTERFACE)
-  target_include_directories(FlatbuffersTarget INTERFACE ${FLATBUFFERS_INCLUDE_DIR})
-  add_dependencies(FlatbuffersTarget fbschemas)
-endif()
-
 set(vw_all_headers
   accumulate.h
   action_score.h
@@ -202,9 +197,6 @@ set(vw_all_headers
   vwdll.h
 )
 
-if(BUILD_FLATBUFFERS)
-  set(vw_all_headers ${vw_all_headers} parser/flatbuffer/parse_example_flatbuffer.h)
-endif()
 
 if(BUILD_EXTERNAL_PARSER)
   set(vw_all_headers ${vw_all_headers} ${external_parser_headers})
@@ -354,12 +346,6 @@ set(vw_all_sources
   vw_validate.cc
 )
 
-if(BUILD_FLATBUFFERS)
-  set(vw_all_sources ${vw_all_sources}
-    parser/flatbuffer/parse_example_flatbuffer.cc
-    parser/flatbuffer/parse_label.cc)
-endif()
-
 if(BUILD_EXTERNAL_PARSER)
   set(vw_all_sources ${vw_all_sources} ${external_parser_sources})
 endif()
@@ -405,9 +391,7 @@ if (BUILD_PRIVACY_ACTIVATION)
 endif()
 
 if(BUILD_FLATBUFFERS)
-  target_link_libraries(vw
-      PRIVATE
-      $<BUILD_INTERFACE:FlatbuffersTarget>)
+  target_link_libraries(vw PRIVATE vw_fb_parser)
   target_compile_definitions(vw PUBLIC BUILD_FLATBUFFERS)
 endif()
 

vowpalwabbit/fb_parser/CMakeLists.txt

@@ -0,0 +1,36 @@
+# Generate header file
+add_flatbuffer_schema(
+  TARGET fbschemas
+  SCHEMAS "${CMAKE_CURRENT_LIST_DIR}/schema/example.fbs"
+  OUTPUT_DIR "${CMAKE_CURRENT_BINARY_DIR}/include/vw/fb_parser/generated/"
+  FLATC_EXE ${flatc_location}
+)
+add_library(fb_generate_headers INTERFACE)
+add_dependencies(fb_generate_headers fbschemas)
+target_include_directories(fb_generate_headers INTERFACE ${CMAKE_CURRENT_BINARY_DIR}/include/)
+
+set(vw_fb_parser_sources
+    include/vw/fb_parser/parse_example_flatbuffer.h
+    src/parse_example_flatbuffer.cc
+    src/parse_label.cc
+)
+
+vw_add_library(
+    NAME "fb_parser"
+    TYPE "STATIC_ONLY"
+    SOURCES ${vw_fb_parser_sources}
+    PUBLIC_DEPS VowpalWabbit::vw $<BUILD_INTERFACE:fb_generate_headers>
+)
+target_include_directories(vw_fb_parser PUBLIC ${FLATBUFFERS_INCLUDE_DIR})
+
+# Install the generated header
+# if(VW_INSTALL)
+#   install(DIRECTORY ${CMAKE_CURRENT_BINARY_DIR}/include/
+#     DESTINATION ${CMAKE_INSTALL_INCLUDEDIR})
+# endif()
+
+vw_add_test_executable(
+  FOR_LIB "fb_parser"
+  SOURCES "tests/flatbuffer_parser_test.cc"
+  EXTRA_DEPS VowpalWabbit::vw
+)

vowpalwabbit/parser/flatbuffer/parse_example_flatbuffer.h → vowpalwabbit/fb_parser/include/vw/fb_parser/parse_example_flatbuffer.h

@@ -4,26 +4,33 @@
 
 #pragma once
 
-#include "../../vw.h"
-VW_WARNING_STATE_PUSH
-VW_WARNING_DISABLE_BADLY_FORMED_XML
-#include "generated/example_generated.h"
-VW_WARNING_STATE_POP
+#include "v_array.h"
+#include "vw/fb_parser/generated/example_generated.h"
+
+class io_buf;
+namespace VW
+{
+struct workspace;
+struct example;
+struct polylabel;
+class reduction_features;
+}  // namespace VW
 
 namespace VW
 {
 namespace parsers
 {
 namespace flatbuffer
 {
-int flatbuffer_to_examples(VW::workspace* all, io_buf& buf, v_array<example*>& examples);
+int flatbuffer_to_examples(VW::workspace* all, io_buf& buf, VW::v_array<example*>& examples);
 
 class parser
 {
 public:
   parser() = default;
   const VW::parsers::flatbuffer::ExampleRoot* data();
-  bool parse_examples(VW::workspace* all, io_buf& buf, v_array<example*>& examples, uint8_t* buffer_pointer = nullptr);
+  bool parse_examples(
+      VW::workspace* all, io_buf& buf, VW::v_array<example*>& examples, uint8_t* buffer_pointer = nullptr);
 
 private:
   const VW::parsers::flatbuffer::ExampleRoot* _data;
@@ -38,7 +45,7 @@ class parser
   uint64_t _c_hash = 0;
 
   bool parse(io_buf& buf, uint8_t* buffer_pointer = nullptr);
-  void process_collection_item(VW::workspace* all, v_array<example*>& examples);
+  void process_collection_item(VW::workspace* all, VW::v_array<example*>& examples);
   void parse_example(VW::workspace* all, example* ae, const Example* eg);
   void parse_multi_example(VW::workspace* all, example* ae, const MultiExample* eg);
   void parse_namespaces(VW::workspace* all, example* ae, const Namespace* ns);

vowpalwabbit/parser/flatbuffer/parse_example_flatbuffer.cc → vowpalwabbit/fb_parser/src/parse_example_flatbuffer.cc

@@ -2,13 +2,14 @@
 // individual contributors. All rights reserved. Released under a BSD (revised)
 // license as described in the file LICENSE.
 
-#include "parse_example_flatbuffer.h"
+#include "vw/fb_parser/parse_example_flatbuffer.h"
 
-#include "../../action_score.h"
-#include "../../best_constant.h"
-#include "../../cb.h"
-#include "../../constant.h"
-#include "../../global_data.h"
+#include "action_score.h"
+#include "best_constant.h"
+#include "cb.h"
+#include "constant.h"
+#include "global_data.h"
+#include "parser.h"
 
 #include <cfloat>
 #include <fstream>

vowpalwabbit/parser/flatbuffer/parse_label.cc → vowpalwabbit/fb_parser/src/parse_label.cc

@@ -2,15 +2,15 @@
 // individual contributors. All rights reserved. Released under a BSD (revised)
 // license as described in the file LICENSE.
 
-#include "../../action_score.h"
-#include "../../best_constant.h"
-#include "../../cb.h"
-#include "../../constant.h"
-#include "../../example.h"
-#include "../../global_data.h"
-#include "../../named_labels.h"
-#include "../../slates_label.h"
-#include "parse_example_flatbuffer.h"
+#include "action_score.h"
+#include "best_constant.h"
+#include "cb.h"
+#include "constant.h"
+#include "example.h"
+#include "global_data.h"
+#include "named_labels.h"
+#include "slates_label.h"
+#include "vw/fb_parser/parse_example_flatbuffer.h"
 
 #include <cfloat>
 #include <fstream>