Generate headers from schema files

.github/workflows/linux.yml

@@ -0,0 +1,36 @@
+name: Build and test sparrow-ipc
+
+on:
+  workflow_dispatch:
+  pull_request:
+  push:
+    branches: [main]
+
+defaults:
+  run:
+    shell: bash -l -eo pipefail {0}
+
+jobs:
+  build:
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@v4
+      - name: Create build environment
+        uses: mamba-org/setup-micromamba@v2
+        with:
+          environment-file: ./environment-dev.yml
+          environment-name: build_env
+          cache-environment: true
+      - name: Build sparrow-ipc
+        run: |
+          cmake -B build/ -G Ninja \
+            -DCMAKE_BUILD_TYPE=Release \
+            -DCMAKE_INSTALL_PREFIX=$CONDA_PREFIX \
+            -DCMAKE_PREFIX_PATH=$CONDA_PREFIX \
+            -DBUILD_TESTS=ON
+          cmake --build build/ --parallel
+      - name: Run tests
+        run: |
+          cd build
+          ctest --output-on-failure

.gitignore

@@ -30,3 +30,6 @@
 *.exe
 *.out
 *.app
+
+# Build directories
+/build*/

CMakeLists.txt

@@ -0,0 +1,86 @@
+cmake_minimum_required(VERSION 3.28)
+
+project(sparrow-ipc CXX)
+
+set(CMAKE_CXX_STANDARD 20)
+set(CMAKE_CXX_STANDARD_REQUIRED ON)
+set(CMAKE_CXX_SCAN_FOR_MODULES OFF)
+
+# Build options
+# =============
+OPTION(BUILD_TESTS "Build sparrow-ipc test suite" OFF)
+MESSAGE(STATUS "🔧 Build tests: ${BUILD_TESTS}")
+
+set(SCHEMA_DIR ${CMAKE_BINARY_DIR}/format)
+set(FLATBUFFERS_GENERATED_DIR ${CMAKE_BINARY_DIR}/generated)
+
+find_program(FLATC_EXECUTABLE flatc)
+
+if(NOT FLATC_EXECUTABLE)
+    message(FATAL_ERROR "flatc not found. Please install Flatbuffers.")
+endif()
+
+# Fetch schemas from apache arrow
+set(SCHEMA_URLS
+    "https://raw.githubusercontent.com/apache/arrow/refs/heads/main/format/File.fbs"
+    "https://raw.githubusercontent.com/apache/arrow/refs/heads/main/format/Message.fbs"
+    "https://raw.githubusercontent.com/apache/arrow/refs/heads/main/format/Schema.fbs"
+    "https://raw.githubusercontent.com/apache/arrow/refs/heads/main/format/Tensor.fbs"
+    "https://raw.githubusercontent.com/apache/arrow/refs/heads/main/format/SparseTensor.fbs"
+    # TODO what about feather.fbs?
+)
+
+file(MAKE_DIRECTORY ${SCHEMA_DIR})
+
+# Download schemas
+set(FLATBUFFERS_SCHEMAS "")
+foreach(url IN LISTS SCHEMA_URLS)
+    get_filename_component(filename ${url} NAME)
+    message(STATUS "Downloading schema: ${url}")
+    file(DOWNLOAD ${url} ${SCHEMA_DIR}/${filename}
+        STATUS status
+        SHOW_PROGRESS)
+    list(APPEND FLATBUFFERS_SCHEMAS ${SCHEMA_DIR}/${filename})
+endforeach()
+
+# Generate Flatbuffers C++ headers from the schemas
+file(MAKE_DIRECTORY ${FLATBUFFERS_GENERATED_DIR})
+
+# Generate output files list
+set(FLATBUFFERS_GENERATED_HEADERS "")
+foreach(fbs_file IN LISTS FLATBUFFERS_SCHEMAS)
+    # Generate the corresponding header file name
+    get_filename_component(header_name ${fbs_file} NAME_WE)
+    list(APPEND FLATBUFFERS_GENERATED_HEADERS "${FLATBUFFERS_GENERATED_DIR}/${header_name}_generated.h")
+endforeach()
+
+add_custom_command(
+    OUTPUT ${FLATBUFFERS_GENERATED_HEADERS}
+    COMMAND ${FLATC_EXECUTABLE} --cpp -o ${FLATBUFFERS_GENERATED_DIR} --cpp-std c++17 --scoped-enums ${FLATBUFFERS_SCHEMAS}
+    DEPENDS ${FLATBUFFERS_SCHEMAS}
+    COMMENT "Generating FlatBuffers C++ headers from schemas"
+)
+
+add_custom_target(generate_flatbuffers_headers
+    DEPENDS ${FLATBUFFERS_GENERATED_HEADERS}
+)
+
+# Interface target for generated headers
+add_library(flatbuffers_interface INTERFACE)
+target_include_directories(flatbuffers_interface INTERFACE ${FLATBUFFERS_GENERATED_DIR})
+add_dependencies(flatbuffers_interface generate_flatbuffers_headers)
+
+find_package(FlatBuffers CONFIG REQUIRED)
+find_package(sparrow CONFIG REQUIRED)
+
+# TODO Handle shared/static build later (after more code is available)
+add_library(sparrow-ipc STATIC src/sparrow-ipc.cpp)
+target_link_libraries(sparrow-ipc PRIVATE flatbuffers_interface flatbuffers::flatbuffers sparrow)
+
+add_dependencies(sparrow-ipc generate_flatbuffers_headers)
+
+if(BUILD_TESTS)
+    message(STATUS "🧪 Create tests targets")
+    enable_testing()
+    add_subdirectory(tests)
+endif()

environment-dev.yml

@@ -0,0 +1,14 @@
+name: sparrow-ipc
+channels:
+  - conda-forge
+dependencies:
+  # Build dependencies
+  - cmake
+  - make
+  - ninja
+  - cxx-compiler
+  # Libraries dependencies
+  - flatbuffers
+  - sparrow
+  # Tests
+  - doctest

src/sparrow-ipc.cpp

@@ -0,0 +1,4 @@
+#include "sparrow/sparrow.hpp"
+
+#include "../generated/Schema_generated.h"
+

tests/CMakeLists.txt

@@ -0,0 +1,18 @@
+cmake_minimum_required(VERSION 3.28)
+
+find_package(doctest CONFIG REQUIRED)
+
+set(test_target "test_sparrow_ipc_lib")
+
+add_executable(${test_target} test.cpp)
+target_link_libraries(${test_target}
+    PRIVATE
+        sparrow-ipc
+        doctest::doctest
+)
+target_include_directories(${test_target}
+    PRIVATE
+        ${CMAKE_BINARY_DIR}/generated
+)
+add_dependencies(${test_target} generate_flatbuffers_headers)
+add_test(NAME sparrow-ipc-tests COMMAND ${test_target})

tests/test.cpp

@@ -0,0 +1,25 @@
+#define DOCTEST_CONFIG_IMPLEMENT_WITH_MAIN
+
+#include "sparrow/sparrow.hpp"
+#include "doctest/doctest.h"
+
+#include "../generated/Schema_generated.h"
+
+// NOTE this is just testing sparrow internals usability,
+// for now we are not testing anything with serialization/deserialization
+TEST_CASE("Use sparrow primitive_array")
+{
+    namespace sp = sparrow;
+
+    sp::primitive_array<int> ar = { 1, 3, 5, 7, 9 };
+    CHECK_EQ(ar.size(), 5);
+
+    auto [arrow_array, arrow_schema] = sp::extract_arrow_structures(std::move(ar));
+    CHECK_EQ(arrow_array.length, 5);
+
+    // Serialize
+    // Deserialize
+
+    arrow_array.release(&arrow_array);
+    arrow_schema.release(&arrow_schema);
+}