Add support for non-vcpkg based builds

.github/workflows/build.yml

@@ -10,15 +10,15 @@ on:
       - LICENSE
       - NOTICE
       - README.md
-      - docs
+      - "docs/wiki/**"
   pull_request:
     branches:
       - main
     paths-ignore:
       - LICENSE
       - NOTICE
       - README.md
-      - docs
+      - "docs/wiki/**"
   workflow_dispatch:
     inputs:
       ci-full:
@@ -585,7 +585,7 @@ jobs:
             - "sqlalchemy>=2"
             - "sqlalchemy<2"
             - "numpy==1.19.5"
-            
+
       runs-on: ${{ matrix.os }}
 
       steps:
@@ -666,7 +666,7 @@ jobs:
           run: ls -R ./dist
 
         - name: Publish to github releases
-          uses: softprops/action-gh-release@v1
+          uses: softprops/action-gh-release@v2
           with:
             draft: true
             generate_release_notes: true

.github/workflows/conda.yml

@@ -0,0 +1,61 @@
+name: Conda End-to-end Test
+
+on:
+  push:
+    branches:
+      - main
+    paths-ignore:
+      - LICENSE
+      - NOTICE
+      - README.md
+      - docs
+  pull_request:
+    branches:
+      - main
+    paths-ignore:
+      - LICENSE
+      - NOTICE
+      - README.md
+      - "docs/wiki/**"
+  workflow_dispatch:
+
+concurrency:
+  group: ${{ github.workflow }}-${{ github.head_ref || github.run_id }}
+  cancel-in-progress: true
+
+permissions:
+  contents: write
+  checks: write
+
+jobs:
+    build:
+      runs-on: ubuntu-22.04
+      steps:
+        - name: Checkout
+          uses: actions/checkout@v4
+
+        - uses: mamba-org/setup-micromamba@v1
+          with:
+            micromamba-version: '1.5.7-0'
+            environment-file: conda/dev-environment-linux.yml
+            init-shell: >-
+              bash
+            cache-environment: true
+            post-cleanup: 'all'
+
+        - name: Set up Caches
+          uses: ./.github/actions/setup-caches
+          with:
+            cibuildwheel: 'cp311'
+
+        - name: Python Lint Steps (Linux)
+          run: make lint
+          shell: micromamba-shell {0}
+
+        - name: Python Build Steps (Linux)
+          run: make build-conda
+          shell: micromamba-shell {0}
+
+        - name: Python Test Steps (Linux)
+          run: make test
+          shell: micromamba-shell {0}

.github/workflows/publish_wiki.yml → .github/workflows/wiki.yml

@@ -19,7 +19,9 @@ jobs:
   build:
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/checkout@v1
+      - name: Checkout
+        uses: actions/checkout@v4
+
       - name: Upload Documentation to Wiki
         uses: Andrew-Chen-Wang/github-wiki-action@v4
         with:

MANIFEST.in

@@ -1,6 +1,6 @@
 # Text files
 include LICENSE
-include NOTICE 
+include NOTICE
 include README.md
 
 # Python files
@@ -22,6 +22,7 @@ prune .git
 prune .github
 prune .vscode
 prune ci
+prune conda
 prune examples
 exclude .clang-format
 exclude .gitattributes

Makefile

@@ -11,7 +11,7 @@ endif
 #########
 # BUILD #
 #########
-.PHONY: develop build-py build install
+.PHONY: requirements develop build build-conda install
 
 requirements:  ## install python dev and runtime dependencies
 	python -m pip install toml
@@ -21,10 +21,11 @@ requirements:  ## install python dev and runtime dependencies
 develop: requirements  ## install dependencies and build library
 	python -m pip install -e .[develop]
 
-build-py:  ## build the python library
+build:  ## build the library
 	python setup.py build build_ext --inplace -- -- -j$(NPROC)
 
-build: build-py  ## build the library
+build-conda:  ## build the library in Conda
+	CSP_USE_VCPKG=0 python setup.py build build_ext --inplace -- -- -j$(NPROC)
 
 install:  ## install library
 	python -m pip install .

dev-environment.yml → conda/dev-environment-linux.yml

@@ -3,38 +3,47 @@ channels:
   - conda-forge
   - nodefaults
 dependencies:
+ - bison
+ - brotli
+ - bump2version>=1
+ - cmake
+ - codespell
  - compilers
  - cyrus-sasl
+ - exprtk
  - flex
- - cmake
- - ninja
- - bison
- - tar
- - unzip
- - zip
- - numpy
- - pyarrow>=7.0.0
- - ruamel.yaml
- - scikit-build
- - psutil
- - sqlalchemy
- - bump2version>=1.0.0
  - python-graphviz
  - httpx
  - isort
- - ruff
- - twine
- - wheel
+ - libarrow=15
+ - librdkafka
+ - lz4-c
+ - mamba
+ - mdformat
+ - ninja
+ - numpy
+ - psutil
+ - pyarrow=15
  - pandas
+ - pillow
  - polars
  - pytz
  - pytest
  - pytest-cov
  - pytest-sugar
  - pytest-asyncio
  - python<3.12
- - tornado
  - python-rapidjson
- - pillow
- - mdformat
- - codespell
+ - rapidjson
+ - requests
+ - ruamel.yaml
+ - ruff
+ - scikit-build
+ - slack-sdk
+ - sqlalchemy
+ - tar
+ - tornado
+ - twine
+ - unzip
+ - wheel
+ - zip

cpp/cmake/modules/FindBrotli.cmake

@@ -49,17 +49,17 @@ find_path( BROTLI_INCLUDE_DIR
   message("Include: ${BROTLI_INCLUDE_DIR}")
 
 find_library( BROTLI_STATIC_LIB_ENC
-  NAMES libbrotlienc.a libbrotlienc-static.a
+  NAMES libbrotlienc.a libbrotlienc-static.a brotlienc
   PATHS ${_brotli_roots}
   PATH_SUFFIXES "" "lib" )
 
 find_library( BROTLI_STATIC_LIB_DEC
-  NAMES libbrotlidec.a libbrotlidec-static.a
+  NAMES libbrotlidec.a libbrotlidec-static.a brotlidec
   PATHS ${_brotli_roots}
   PATH_SUFFIXES "" "lib" )
 
 find_library( BROTLI_STATIC_LIB_COMMON
-  NAMES libbrotlicommon.a libbrotlicommon-static.a
+  NAMES libbrotlicommon.a libbrotlicommon-static.a brotlicommon
   PATHS ${_brotli_roots}
   PATH_SUFFIXES "lib/${CMAKE_LIBRARY_ARCHITECTURE}" "lib" )
 
@@ -101,4 +101,4 @@ mark_as_advanced(
   BROTLI_LIBS
   BROTLI_LIBRARIES
   BROTLI_STATIC_LIB
-)
+)

cpp/cmake/modules/FindDepsBase.cmake

@@ -12,4 +12,4 @@ include_directories(${RapidJSON_INCLUDE_DIRS})
 find_path(EXPRTK_INCLUDE_DIRS "exprtk.hpp")
 
 # For adapter utils
-find_package(protobuf CONFIG REQUIRED)
+find_package(Protobuf REQUIRED)

cpp/cmake/modules/FindDepsKafkaAdapter.cmake

@@ -1,3 +1,7 @@
 cmake_minimum_required(VERSION 3.7.2)
 
-find_package(RdKafka CONFIG REQUIRED)
+if (CSP_USE_VCPKG)
+  find_package(RdKafka CONFIG REQUIRED)
+else()
+  find_package(RdKafka REQUIRED)
+endif()

cpp/cmake/modules/FindDepsParquetAdapter.cmake

@@ -11,6 +11,7 @@ include_directories(${PARQUET_INCLUDE_DIR})
 # Other deps
 find_package(Thrift REQUIRED)
 find_package(Brotli REQUIRED)
+find_package(Snappy REQUIRED)
 # find_package(unofficial-brotli CONFIG REQUIRED)
 find_package(utf8proc REQUIRED)
 # find_package(unofficial-utf8proc CONFIG REQUIRED)

cpp/cmake/modules/FindRdKafka.cmake

@@ -0,0 +1,20 @@
+# NOTE: this only assumes that rdkafka is a shared library
+find_path(RdKafka_INCLUDE_DIR NAMES librdkafka/rdkafkacpp.h)
+find_library(RdKafka_LIBRARY NAMES rdkafka++ librdkafkacpp)
+find_library(RdKafka_C_LIBRARY NAMES rdkafka librdkafka)
+
+if (NOT TARGET RdKafka::rdkafka)
+  add_library(RdKafka::rdkafka STATIC IMPORTED)
+  set_property(TARGET RdKafka::rdkafka PROPERTY
+          IMPORTED_LOCATION "${RdKafka_C_LIBRARY}")
+  target_include_directories(RdKafka::rdkafka INTERFACE ${RdKafka_INCLUDE_DIR})
+
+  add_library(RdKafka::rdkafka++ STATIC IMPORTED)
+  set_property(TARGET RdKafka::rdkafka++ PROPERTY
+          IMPORTED_LOCATION "${RdKafka_LIBRARY}")
+  target_include_directories(RdKafka::rdkafka++ INTERFACE ${RdKafka_INCLUDE_DIR})
+endif()
+
+include(FindPackageHandleStandardArgs)
+find_package_handle_standard_args(RdKafka DEFAULT_MSG RdKafka_LIBRARY RdKafka_C_LIBRARY RdKafka_INCLUDE_DIR)
+mark_as_advanced(RdKafka_INCLUDE_DIR RdKafka_LIBRARY RdKafka_C_LIBRARY RdKafka::rdkafka++ RdKafka::rdkafka)

cpp/cmake/modules/Findlz4.cmake

@@ -0,0 +1,12 @@
+find_path(Lz4_INCLUDE_DIR lz4.h)
+find_library(Lz4_LIBRARY NAMES lz4 liblz4)
+
+if (NOT TARGET lz4::lz4)
+  add_library(lz4::lz4 STATIC IMPORTED)
+  set_property(TARGET lz4::lz4 PROPERTY IMPORTED_LOCATION "${Lz4_LIBRARY}")
+  target_include_directories(lz4::lz4 INTERFACE ${Lz4_INCLUDE_DIR})
+endif()
+
+include(FindPackageHandleStandardArgs)
+find_package_handle_standard_args(lz4 DEFAULT_MSG Lz4_LIBRARY Lz4_INCLUDE_DIR)
+mark_as_advanced(Lz4_INCLUDE_DIR Lz4_LIBRARY)

cpp/csp/adapters/kafka/CMakeLists.txt

@@ -20,8 +20,9 @@ set(KAFKA_SOURCE_FILES
 add_library(csp_kafka_adapter STATIC ${KAFKA_SOURCE_FILES})
 set_target_properties(csp_kafka_adapter PROPERTIES PUBLIC_HEADER "${KAFKA_HEADER_FILES}")
 
-find_package(RdKafka REQUIRED)
-target_link_libraries(csp_kafka_adapter csp_adapter_utils RdKafka::rdkafka RdKafka::rdkafka++)
+find_package(DepsKafkaAdapter REQUIRED)
+
+target_link_libraries(csp_kafka_adapter PRIVATE csp_adapter_utils RdKafka::rdkafka RdKafka::rdkafka++)
 
 install(TARGETS csp_kafka_adapter
         PUBLIC_HEADER DESTINATION include/csp/adapters/kafka

cpp/csp/adapters/parquet/ArrowIPCFileWriterWrapper.cpp

@@ -19,7 +19,7 @@ void ArrowIPCFileWriterWrapper::openImpl( const std::string &fileName, const std
     writeOptions.codec  = resolveCompressionCodec( compression );
 
     STATUS_OK_OR_THROW_RUNTIME(
-            ::arrow::ipc::NewStreamWriter( m_outputStream.get(), getSchema(), writeOptions ).Value(&m_fileWriter),
+            ::arrow::ipc::MakeStreamWriter( m_outputStream.get(), getSchema(), writeOptions ).Value(&m_fileWriter),
             "Failed to open arrow file writer" );
 }
 

cpp/csp/adapters/parquet/CMakeLists.txt

@@ -50,7 +50,15 @@ find_package(lz4 REQUIRED)
 find_package(utf8proc REQUIRED)
 find_package(Brotli REQUIRED)
 
-target_link_libraries(csp_parquet_adapter PRIVATE csp_adapter_utils parquet_static arrow_static thrift::thrift lz4::lz4 utf8proc::utf8proc ${BROTLI_STATIC_LIB})
+if(CSP_USE_VCPKG)
+  # use staic variants
+  set(ARROW_PACKAGES_TO_LINK parquet_static arrow_static)
+else()
+  # use dynamic variants
+  set(ARROW_PACKAGES_TO_LINK parquet arrow)
+endif()
+
+target_link_libraries(csp_parquet_adapter PRIVATE csp_adapter_utils thrift::thrift lz4::lz4 utf8proc::utf8proc ${BROTLI_STATIC_LIB} ${ARROW_PACKAGES_TO_LINK})
 
 install(TARGETS csp_parquet_adapter
         PUBLIC_HEADER DESTINATION include/csp/adapters/parquet

cpp/csp/adapters/parquet/ParquetReaderColumnAdapter.cpp

@@ -777,14 +777,14 @@ void ListColumnAdapter<arrow::StringArray, std::string>::readCurValue()
         for( int64_t i = 0; i < typedValues -> length(); ++i )
         {
 
-            maxStringLength = std::max( ( uint32_t ) ArrayValidValueProvider<arrow::util::string_view>::getValue(typedValues, i).length(), maxStringLength );
+            maxStringLength = std::max( ( uint32_t ) ArrayValidValueProvider<std::string_view>::getValue(typedValues, i).length(), maxStringLength );
         }
 
         auto arrayValue = m_listReader -> create( typedValues -> length(), maxStringLength );
 
         for( int64_t i = 0; i < typedValues -> length(); ++i )
         {
-            m_listReader -> setValue( arrayValue, i, typedValues -> GetView( i ).to_string() );
+            m_listReader -> setValue( arrayValue, i, std::string(typedValues -> GetView( i )));
         }
         this -> m_curValue = std::move( arrayValue );
     }

cpp/csp/adapters/utils/ProtobufHelper.h

@@ -5,6 +5,7 @@
 #include <google/protobuf/compiler/importer.h>
 #include <google/protobuf/dynamic_message.h>
 #include <memory>
+#include <mutex>
 #include <string>
 
 namespace csp::adapters::utils
@@ -46,7 +47,7 @@ class ProtobufStructMapper
 public:
     ProtobufStructMapper() {}
 
-    void init( const CspTypePtr & type, const Dictionary & fieldMap, 
+    void init( const CspTypePtr & type, const Dictionary & fieldMap,
                const google::protobuf::Descriptor * protoDesc );
 
     void mapProtoToStruct( StructPtr & struct_, const google::protobuf::Message & protoMsg )