From f8d25e66897750cef45984640f105ab12a1a6d0e Mon Sep 17 00:00:00 2001 From: Hind Montassif Date: Wed, 27 Aug 2025 14:03:18 +0200 Subject: [PATCH 1/8] Add conanfile --- conanfile.py | 95 ++++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 95 insertions(+) create mode 100644 conanfile.py diff --git a/conanfile.py b/conanfile.py new file mode 100644 index 0000000..1dfdc44 --- /dev/null +++ b/conanfile.py @@ -0,0 +1,95 @@ +from conan import ConanFile +from conan.errors import ConanInvalidConfiguration +from conan.tools.build.cppstd import check_min_cppstd +from conan.tools.cmake import CMake, CMakeToolchain, cmake_layout +from conan.tools.files import copy +from conan.tools.scm import Version +import os + +required_conan_version = ">=2.0" + +class SparrowIPCRecipe(ConanFile): + name = "sparrow-ipc" + description = "C++20 library for memory-mapped serialization of Apache Arrow data." + license = "BSD-3-Clause" + author = "QuantStack" + url = "https://github.com/QuantStack/sparrow-ipc" + homepage = "https://github.com/QuantStack/sparrow-ipc" + topics = ("arrow", "apache arrow", "columnar format", "dataframe", "ipc", "serialization", "deserialization", "flatbuffers") + package_type = "library" + settings = "os", "arch", "compiler", "build_type" + generators = "CMakeDeps" + exports_sources = "include/*", "src/*", "cmake/*", "CMakeLists.txt", "LICENSE" + options = { + "shared": [True, False], + "fPIC": [True, False], + "build_tests": [True, False], + } + default_options = { + "shared": False, + "fPIC": True, + "build_tests": False, + } + + def config_options(self): + if self.settings.os == "Windows": + del self.options.fPIC + + def configure(self): + if self.options.shared: + self.options.rm_safe("fPIC") + + def requirements(self): + self.requires("sparrow/1.0.0") + self.requires("flatbuffers/25.2.10") + if self.options.get_safe("build_tests"): + self.test_requires("doctest/2.4.12") + + @property + def _min_cppstd(self): + return 20 + + @property + def _compilers_minimum_version(self): + return { + "apple-clang": "16", + "clang": "18", + "gcc": "12", + "msvc": "194" + } + + def validate(self): + if self.settings.compiler.get_safe("cppstd"): + check_min_cppstd(self, self._min_cppstd) + + minimum_version = self._compilers_minimum_version.get( + str(self.settings.compiler), False) + if minimum_version and Version(self.settings.compiler.version) < minimum_version: + raise ConanInvalidConfiguration( + f"{self.ref} requires C++{self._min_cppstd}, which your compiler does not support." + ) + + def layout(self): + cmake_layout(self, src_folder=".") + + def generate(self): + tc = CMakeToolchain(self) + tc.variables["SPARROW_IPC_BUILD_TESTS"] = self.options.build_tests + tc.generate() + + def build(self): + cmake = CMake(self) + cmake.configure() + cmake.build() + + def package(self): + copy(self, "LICENSE", + dst=os.path.join(self.package_folder, "licenses"), + src=self.source_folder) + cmake = CMake(self) + cmake.install() + + def package_info(self): + self.cpp_info.libs = ["sparrow-ipc"] + self.cpp_info.set_property("cmake_file_name", "sparrow-ipc") + self.cpp_info.set_property("cmake_target_name", "sparrow-ipc::sparrow-ipc") From 3fe3ae08b8c1b6188f8bcbed3565a8fbab56d7bd Mon Sep 17 00:00:00 2001 From: Hind Montassif Date: Wed, 27 Aug 2025 14:05:57 +0200 Subject: [PATCH 2/8] Add conan workflow --- .github/workflows/conan.yaml | 39 ++++++++++++++++++++++++++++++++++++ conanfile.py | 10 +++++++-- 2 files changed, 47 insertions(+), 2 deletions(-) create mode 100644 .github/workflows/conan.yaml diff --git a/.github/workflows/conan.yaml b/.github/workflows/conan.yaml new file mode 100644 index 0000000..855d1a1 --- /dev/null +++ b/.github/workflows/conan.yaml @@ -0,0 +1,39 @@ +name: Conan +on: + workflow_dispatch: + pull_request: + push: + branches: [main] + +jobs: + conan-package: + runs-on: ubuntu-24.04 + + steps: + - name: Checkout repository + uses: actions/checkout@v4 + + - name: Setup Conan Environment + uses: hankhsu1996/setup-conan@v1.0.0 + with: + cache-dependencies: true + cache-tool: true + + - name: Install conan dependencies + run: | + conan profile detect --force + conan install . --output-folder=build --build=missing -s:a compiler.cppstd=20 -o:a build_tests=True + + - name: CMake configuration + run: | + cmake --preset conan-release \ + -DSPARROW_IPC_BUILD_TESTS=ON \ + -DFETCH_DEPENDENCIES_WITH_CMAKE=MISSING + + - name: Build + working-directory: build/build/Release + run: cmake --build . --config Release --target test_sparrow_ipc_lib + + - name: Run tests + working-directory: build/build/Release + run: cmake --build . --config Release --target run_tests_with_junit_report diff --git a/conanfile.py b/conanfile.py index 1dfdc44..a293b8c 100644 --- a/conanfile.py +++ b/conanfile.py @@ -1,7 +1,7 @@ from conan import ConanFile from conan.errors import ConanInvalidConfiguration from conan.tools.build.cppstd import check_min_cppstd -from conan.tools.cmake import CMake, CMakeToolchain, cmake_layout +from conan.tools.cmake import CMake, CMakeDeps, CMakeToolchain, cmake_layout from conan.tools.files import copy from conan.tools.scm import Version import os @@ -18,7 +18,6 @@ class SparrowIPCRecipe(ConanFile): topics = ("arrow", "apache arrow", "columnar format", "dataframe", "ipc", "serialization", "deserialization", "flatbuffers") package_type = "library" settings = "os", "arch", "compiler", "build_type" - generators = "CMakeDeps" exports_sources = "include/*", "src/*", "cmake/*", "CMakeLists.txt", "LICENSE" options = { "shared": [True, False], @@ -45,6 +44,9 @@ def requirements(self): if self.options.get_safe("build_tests"): self.test_requires("doctest/2.4.12") + def build_requirements(self): + self.tool_requires("cmake/[>=3.28.1 <4.2.0]") + @property def _min_cppstd(self): return 20 @@ -73,7 +75,11 @@ def layout(self): cmake_layout(self, src_folder=".") def generate(self): + deps = CMakeDeps(self) + deps.generate() + tc = CMakeToolchain(self) + tc.variables["SPARROW_IPC_BUILD_SHARED"] = self.options.shared tc.variables["SPARROW_IPC_BUILD_TESTS"] = self.options.build_tests tc.generate() From 6363264fc8d09538a1054f760208a79612acd0ca Mon Sep 17 00:00:00 2001 From: Hind Montassif Date: Wed, 27 Aug 2025 14:11:20 +0200 Subject: [PATCH 3/8] Use available flatbuffers version --- conanfile.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/conanfile.py b/conanfile.py index a293b8c..ecbcdf2 100644 --- a/conanfile.py +++ b/conanfile.py @@ -40,7 +40,7 @@ def configure(self): def requirements(self): self.requires("sparrow/1.0.0") - self.requires("flatbuffers/25.2.10") + self.requires("flatbuffers/24.12.23") if self.options.get_safe("build_tests"): self.test_requires("doctest/2.4.12") From 655001f12a7c9b881ec274bd409d7fcbba061836 Mon Sep 17 00:00:00 2001 From: Hind Montassif Date: Wed, 27 Aug 2025 14:21:59 +0200 Subject: [PATCH 4/8] Add flatbuffers as build requirement --- conanfile.py | 1 + 1 file changed, 1 insertion(+) diff --git a/conanfile.py b/conanfile.py index ecbcdf2..a8ff6a9 100644 --- a/conanfile.py +++ b/conanfile.py @@ -46,6 +46,7 @@ def requirements(self): def build_requirements(self): self.tool_requires("cmake/[>=3.28.1 <4.2.0]") + self.tool_requires("flatbuffers/24.12.23") @property def _min_cppstd(self): From 23f0222572590fb00e929a23da4e6a13b8820fd3 Mon Sep 17 00:00:00 2001 From: Hind Montassif Date: Wed, 27 Aug 2025 15:33:56 +0200 Subject: [PATCH 5/8] Handle conan package name and flatc target --- .github/workflows/conan.yaml | 2 +- CMakeLists.txt | 8 +++++++- cmake/external_dependencies.cmake | 18 +++++++++++++----- 3 files changed, 21 insertions(+), 7 deletions(-) diff --git a/.github/workflows/conan.yaml b/.github/workflows/conan.yaml index 855d1a1..f9a313d 100644 --- a/.github/workflows/conan.yaml +++ b/.github/workflows/conan.yaml @@ -30,7 +30,7 @@ jobs: -DSPARROW_IPC_BUILD_TESTS=ON \ -DFETCH_DEPENDENCIES_WITH_CMAKE=MISSING - - name: Build + - name: Build tests working-directory: build/build/Release run: cmake --build . --config Release --target test_sparrow_ipc_lib diff --git a/CMakeLists.txt b/CMakeLists.txt index 4cd4ac1..3e44c46 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -99,9 +99,15 @@ foreach(fbs_file IN LISTS FLATBUFFERS_SCHEMAS) list(APPEND FLATBUFFERS_GENERATED_HEADERS "${FLATBUFFERS_GENERATED_DIR}/${header_name}_generated.h") endforeach() +if(TARGET flatbuffers::flatc) + set(FLATC_EXECUTABLE flatbuffers::flatc) +else() + set(FLATC_EXECUTABLE flatc) +endif() + add_custom_command( OUTPUT ${FLATBUFFERS_GENERATED_HEADERS} - COMMAND flatc --cpp -o ${FLATBUFFERS_GENERATED_DIR} --cpp-std c++17 --scoped-enums ${FLATBUFFERS_SCHEMAS} + COMMAND ${FLATC_EXECUTABLE} --cpp -o ${FLATBUFFERS_GENERATED_DIR} --cpp-std c++17 --scoped-enums ${FLATBUFFERS_SCHEMAS} DEPENDS ${FLATBUFFERS_SCHEMAS} COMMENT "Generating FlatBuffers C++ headers from schemas" ) diff --git a/cmake/external_dependencies.cmake b/cmake/external_dependencies.cmake index 742a3c9..e1ff392 100644 --- a/cmake/external_dependencies.cmake +++ b/cmake/external_dependencies.cmake @@ -11,16 +11,23 @@ endif() function(find_package_or_fetch) set(options) - set(oneValueArgs PACKAGE_NAME VERSION GIT_REPOSITORY TAG) + set(oneValueArgs CONAN_PKG_NAME PACKAGE_NAME VERSION GIT_REPOSITORY TAG) set(multiValueArgs) cmake_parse_arguments(PARSE_ARGV 0 arg "${options}" "${oneValueArgs}" "${multiValueArgs}" ) + + set(actual_pkg_name ${arg_PACKAGE_NAME}) + if(arg_CONAN_PKG_NAME) + set(actual_pkg_name ${arg_CONAN_PKG_NAME}) + endif() + if(NOT FETCH_DEPENDENCIES_WITH_CMAKE STREQUAL "ON") - find_package(${arg_PACKAGE_NAME} ${FIND_PACKAGE_OPTIONS}) + find_package(${actual_pkg_name} ${FIND_PACKAGE_OPTIONS}) endif() + if(FETCH_DEPENDENCIES_WITH_CMAKE STREQUAL "ON" OR FETCH_DEPENDENCIES_WITH_CMAKE STREQUAL "MISSING") - if(NOT ${arg_PACKAGE_NAME}_FOUND) + if(NOT ${actual_pkg_name}_FOUND) message(STATUS "📦 Fetching ${arg_PACKAGE_NAME}") FetchContent_Declare( ${arg_PACKAGE_NAME} @@ -33,7 +40,7 @@ function(find_package_or_fetch) FetchContent_MakeAvailable(${arg_PACKAGE_NAME}) message(STATUS "\t✅ Fetched ${arg_PACKAGE_NAME}") else() - message(STATUS "📦 ${arg_PACKAGE_NAME} found here: ${arg_PACKAGE_NAME}_DIR") + message(STATUS "📦 ${actual_pkg_name} found here: ${actual_pkg_name}_DIR") endif() endif() endfunction() @@ -53,6 +60,7 @@ endif() set(FLATBUFFERS_BUILD_TESTS OFF) set(FLATBUFFERS_BUILD_SHAREDLIB ${SPARROW_IPC_BUILD_SHARED}) find_package_or_fetch( + CONAN_PKG_NAME flatbuffers PACKAGE_NAME FlatBuffers VERSION v25.2.10 GIT_REPOSITORY https://github.com/google/flatbuffers.git @@ -71,4 +79,4 @@ if(SPARROW_IPC_BUILD_TESTS) GIT_REPOSITORY https://github.com/doctest/doctest.git TAG v2.4.12 ) -endif() \ No newline at end of file +endif() From 5d9acc3fe14817d6bc3bb96b6fb1bee8e27717f5 Mon Sep 17 00:00:00 2001 From: Hind Montassif Date: Wed, 27 Aug 2025 16:03:01 +0200 Subject: [PATCH 6/8] Use var for flatbuffers version --- conanfile.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/conanfile.py b/conanfile.py index a8ff6a9..17336f7 100644 --- a/conanfile.py +++ b/conanfile.py @@ -30,6 +30,8 @@ class SparrowIPCRecipe(ConanFile): "build_tests": False, } + _flatbuffers_version = "24.12.23" + def config_options(self): if self.settings.os == "Windows": del self.options.fPIC @@ -40,13 +42,13 @@ def configure(self): def requirements(self): self.requires("sparrow/1.0.0") - self.requires("flatbuffers/24.12.23") + self.requires(f"flatbuffers/{self._flatbuffers_version}") if self.options.get_safe("build_tests"): self.test_requires("doctest/2.4.12") def build_requirements(self): self.tool_requires("cmake/[>=3.28.1 <4.2.0]") - self.tool_requires("flatbuffers/24.12.23") + self.tool_requires(f"flatbuffers/{self._flatbuffers_version}") @property def _min_cppstd(self): From 1d74156cfc63470c72ff7718dcecfa8bb884bd43 Mon Sep 17 00:00:00 2001 From: Hind Montassif Date: Wed, 27 Aug 2025 16:22:56 +0200 Subject: [PATCH 7/8] Use hankhsu1996/setup-conan@v1.1.0 --- .github/workflows/conan.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/conan.yaml b/.github/workflows/conan.yaml index f9a313d..fc8462d 100644 --- a/.github/workflows/conan.yaml +++ b/.github/workflows/conan.yaml @@ -14,7 +14,7 @@ jobs: uses: actions/checkout@v4 - name: Setup Conan Environment - uses: hankhsu1996/setup-conan@v1.0.0 + uses: hankhsu1996/setup-conan@v1.1.0 with: cache-dependencies: true cache-tool: true From 59c00fabc2a4974256cf07d9564ad35afece7f67 Mon Sep 17 00:00:00 2001 From: Hind Montassif Date: Wed, 27 Aug 2025 16:50:42 +0200 Subject: [PATCH 8/8] Use sccache --- .github/workflows/conan.yaml | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/.github/workflows/conan.yaml b/.github/workflows/conan.yaml index fc8462d..12082d8 100644 --- a/.github/workflows/conan.yaml +++ b/.github/workflows/conan.yaml @@ -10,6 +10,9 @@ jobs: runs-on: ubuntu-24.04 steps: + - name: Run sccache-cache + uses: mozilla-actions/sccache-action@v0.0.9 + - name: Checkout repository uses: actions/checkout@v4 @@ -28,7 +31,9 @@ jobs: run: | cmake --preset conan-release \ -DSPARROW_IPC_BUILD_TESTS=ON \ - -DFETCH_DEPENDENCIES_WITH_CMAKE=MISSING + -DFETCH_DEPENDENCIES_WITH_CMAKE=MISSING \ + -DCMAKE_C_COMPILER_LAUNCHER=sccache \ + -DCMAKE_CXX_COMPILER_LAUNCHER=sccache - name: Build tests working-directory: build/build/Release @@ -37,3 +42,7 @@ jobs: - name: Run tests working-directory: build/build/Release run: cmake --build . --config Release --target run_tests_with_junit_report + + - name: Run sccache stat for check + shell: bash + run: ${SCCACHE_PATH} --show-stats