Building gluten with a velox backend in mac can sometimes run into this issue & fail to create the libgluten.dylib file.
[74/94] Linking CXX shared library releases/libgluten.dylib; Checking ld result of libgluten.so; Checking ldd result of libgluten.so
Undefined symbols for architecture arm64:
"_LZ4F_compressBegin", referenced from:
arrow::util::internal::(anonymous namespace)::LZ4Compressor::Compress(long long, unsigned char const*, long long, unsigned char*) in libarrow.a[169](compression_lz4.cc.o)
arrow::util::internal::(anonymous namespace)::LZ4Compressor::Flush(long long, unsigned char*) in libarrow.a[169](compression_lz4.cc.o)
arrow::util::internal::(anonymous namespace)::LZ4Compressor::End(long long, unsigned char*) in libarrow.a[169](compression_lz4.cc.o)
"_LZ4F_compressBound", referenced from:
arrow::util::internal::(anonymous namespace)::LZ4Compressor::Compress(long long, unsigned char const*, long long, unsigned char*) in libarrow.a[169](compression_lz4.cc.o)
arrow::util::internal::(anonymous namespace)::LZ4Compressor::Flush(long long, unsigned char*) in libarrow.a[169](compression_lz4.cc.o)
arrow::util::internal::(anonymous namespace)::LZ4Compressor::End(long long, unsigned char*) in libarrow.a[169](compression_lz4.cc.o)
"_LZ4F_compressEnd", referenced from:
arrow::util::internal::(anonymous namespace)::LZ4Compressor::End(long long, unsigned char*) in libarrow.a[169](compression_lz4.cc.o)
"_LZ4F_compressFrame", referenced from:
arrow::util::internal::(anonymous namespace)::Lz4FrameCodec::Compress(long long, unsigned char const*, long long, unsigned char*) in libarrow.a[169](compression_lz4.cc.o)
"_LZ4F_compressFrameBound", referenced from:
arrow::util::internal::(anonymous namespace)::Lz4FrameCodec::MaxCompressedLen(long long, unsigned char const*) in libarrow.a[169](compression_lz4.cc.o)
"_LZ4F_compressUpdate", referenced from:
arrow::util::internal::(anonymous namespace)::LZ4Compressor::Compress(long long, unsigned char const*, long long, unsigned char*) in libarrow.a[169](compression_lz4.cc.o)
"_LZ4F_compressionLevel_max", referenced from:
arrow::util::internal::(anonymous namespace)::Lz4FrameCodec::maximum_compression_level() const in libarrow.a[169](compression_lz4.cc.o)
arrow::util::internal::(anonymous namespace)::Lz4Codec::maximum_compression_level() const in libarrow.a[169](compression_lz4.cc.o)
"_LZ4F_createCompressionContext", referenced from:
arrow::util::internal::(anonymous namespace)::Lz4FrameCodec::MakeCompressor() in libarrow.a[169](compression_lz4.cc.o)
"_LZ4F_createDecompressionContext", referenced from:
arrow::util::internal::(anonymous namespace)::Lz4FrameCodec::MakeDecompressor() in libarrow.a[169](compression_lz4.cc.o)
"_LZ4F_decompress", referenced from:
arrow::util::internal::(anonymous namespace)::LZ4Decompressor::Decompress(long long, unsigned char const*, long long, unsigned char*) in libarrow.a[169](compression_lz4.cc.o)
"_LZ4F_flush", referenced from:
arrow::util::internal::(anonymous namespace)::LZ4Compressor::Flush(long long, unsigned char*) in libarrow.a[169](compression_lz4.cc.o)
"_LZ4F_freeCompressionContext", referenced from:
arrow::util::internal::(anonymous namespace)::LZ4Compressor::~LZ4Compressor() in libarrow.a[169](compression_lz4.cc.o)
arrow::util::internal::(anonymous namespace)::LZ4Compressor::~LZ4Compressor() in libarrow.a[169](compression_lz4.cc.o)
"_LZ4F_freeDecompressionContext", referenced from:
arrow::util::internal::(anonymous namespace)::LZ4Decompressor::~LZ4Decompressor() in libarrow.a[169](compression_lz4.cc.o)
arrow::util::internal::(anonymous namespace)::LZ4Decompressor::~LZ4Decompressor() in libarrow.a[169](compression_lz4.cc.o)
"_LZ4F_getErrorName", referenced from:
arrow::util::internal::(anonymous namespace)::LZ4Error(unsigned long, char const*) in libarrow.a[169](compression_lz4.cc.o)
"_LZ4F_isError", referenced from:
arrow::util::internal::(anonymous namespace)::Lz4FrameCodec::Compress(long long, unsigned char const*, long long, unsigned char*) in libarrow.a[169](compression_lz4.cc.o)
arrow::util::internal::(anonymous namespace)::Lz4FrameCodec::MakeCompressor() in libarrow.a[169](compression_lz4.cc.o)
arrow::util::internal::(anonymous namespace)::Lz4FrameCodec::MakeDecompressor() in libarrow.a[169](compression_lz4.cc.o)
arrow::util::internal::(anonymous namespace)::LZ4Compressor::Compress(long long, unsigned char const*, long long, unsigned char*) in libarrow.a[169](compression_lz4.cc.o)
arrow::util::internal::(anonymous namespace)::LZ4Compressor::Compress(long long, unsigned char const*, long long, unsigned char*) in libarrow.a[169](compression_lz4.cc.o)
arrow::util::internal::(anonymous namespace)::LZ4Compressor::Flush(long long, unsigned char*) in libarrow.a[169](compression_lz4.cc.o)
arrow::util::internal::(anonymous namespace)::LZ4Compressor::Flush(long long, unsigned char*) in libarrow.a[169](compression_lz4.cc.o)
...
"_LZ4F_resetDecompressionContext", referenced from:
arrow::util::internal::(anonymous namespace)::LZ4Decompressor::Reset() in libarrow.a[169](compression_lz4.cc.o)
"_LZ4_compressBound", referenced from:
arrow::util::internal::(anonymous namespace)::Lz4HadoopCodec::MaxCompressedLen(long long, unsigned char const*) in libarrow.a[169](compression_lz4.cc.o)
arrow::util::internal::(anonymous namespace)::Lz4Codec::MaxCompressedLen(long long, unsigned char const*) in libarrow.a[169](compression_lz4.cc.o)
"_LZ4_compress_HC", referenced from:
arrow::util::internal::(anonymous namespace)::Lz4Codec::Compress(long long, unsigned char const*, long long, unsigned char*) in libarrow.a[169](compression_lz4.cc.o)
"_LZ4_compress_default", referenced from:
arrow::util::internal::(anonymous namespace)::Lz4Codec::Compress(long long, unsigned char const*, long long, unsigned char*) in libarrow.a[169](compression_lz4.cc.o)
"_LZ4_decompress_safe", referenced from:
arrow::util::internal::(anonymous namespace)::Lz4HadoopCodec::Decompress(long long, unsigned char const*, long long, unsigned char*) in libarrow.a[169](compression_lz4.cc.o)
arrow::util::internal::(anonymous namespace)::Lz4HadoopCodec::Decompress(long long, unsigned char const*, long long, unsigned char*) in libarrow.a[169](compression_lz4.cc.o)
arrow::util::internal::(anonymous namespace)::Lz4Codec::Decompress(long long, unsigned char const*, long long, unsigned char*) in libarrow.a[169](compression_lz4.cc.o)
"_ZSTD_compress", referenced from:
arrow::util::internal::(anonymous namespace)::ZSTDCodec::Compress(long long, unsigned char const*, long long, unsigned char*) in libarrow.a[172](compression_zstd.cc.o)
"_ZSTD_compressBound", referenced from:
arrow::util::internal::(anonymous namespace)::ZSTDCodec::MaxCompressedLen(long long, unsigned char const*) in libarrow.a[172](compression_zstd.cc.o)
"_ZSTD_compressStream", referenced from:
arrow::util::internal::(anonymous namespace)::ZSTDCompressor::Compress(long long, unsigned char const*, long long, unsigned char*) in libarrow.a[172](compression_zstd.cc.o)
"_ZSTD_createCStream", referenced from:
arrow::util::internal::(anonymous namespace)::ZSTDCodec::MakeCompressor() in libarrow.a[172](compression_zstd.cc.o)
"_ZSTD_createDStream", referenced from:
arrow::util::internal::(anonymous namespace)::ZSTDCodec::MakeDecompressor() in libarrow.a[172](compression_zstd.cc.o)
"_ZSTD_decompress", referenced from:
arrow::util::internal::(anonymous namespace)::ZSTDCodec::Decompress(long long, unsigned char const*, long long, unsigned char*) in libarrow.a[172](compression_zstd.cc.o)
"_ZSTD_decompressStream", referenced from:
arrow::util::internal::(anonymous namespace)::ZSTDDecompressor::Decompress(long long, unsigned char const*, long long, unsigned char*) in libarrow.a[172](compression_zstd.cc.o)
"_ZSTD_endStream", referenced from:
arrow::util::internal::(anonymous namespace)::ZSTDCompressor::End(long long, unsigned char*) in libarrow.a[172](compression_zstd.cc.o)
"_ZSTD_flushStream", referenced from:
arrow::util::internal::(anonymous namespace)::ZSTDCompressor::Flush(long long, unsigned char*) in libarrow.a[172](compression_zstd.cc.o)
"_ZSTD_freeCStream", referenced from:
arrow::util::internal::(anonymous namespace)::ZSTDCompressor::~ZSTDCompressor() in libarrow.a[172](compression_zstd.cc.o)
arrow::util::internal::(anonymous namespace)::ZSTDCompressor::~ZSTDCompressor() in libarrow.a[172](compression_zstd.cc.o)
"_ZSTD_freeDStream", referenced from:
arrow::util::internal::(anonymous namespace)::ZSTDDecompressor::~ZSTDDecompressor() in libarrow.a[172](compression_zstd.cc.o)
arrow::util::internal::(anonymous namespace)::ZSTDDecompressor::~ZSTDDecompressor() in libarrow.a[172](compression_zstd.cc.o)
"_ZSTD_getErrorName", referenced from:
arrow::util::internal::(anonymous namespace)::ZSTDError(unsigned long, char const*) in libarrow.a[172](compression_zstd.cc.o)
"_ZSTD_initCStream", referenced from:
arrow::util::internal::(anonymous namespace)::ZSTDCodec::MakeCompressor() in libarrow.a[172](compression_zstd.cc.o)
"_ZSTD_initDStream", referenced from:
arrow::util::internal::(anonymous namespace)::ZSTDCodec::MakeDecompressor() in libarrow.a[172](compression_zstd.cc.o)
arrow::util::internal::(anonymous namespace)::ZSTDDecompressor::Reset() in libarrow.a[172](compression_zstd.cc.o)
"_ZSTD_isError", referenced from:
arrow::util::internal::(anonymous namespace)::ZSTDCodec::Decompress(long long, unsigned char const*, long long, unsigned char*) in libarrow.a[172](compression_zstd.cc.o)
arrow::util::internal::(anonymous namespace)::ZSTDCodec::Compress(long long, unsigned char const*, long long, unsigned char*) in libarrow.a[172](compression_zstd.cc.o)
arrow::util::internal::(anonymous namespace)::ZSTDCodec::MakeCompressor() in libarrow.a[172](compression_zstd.cc.o)
arrow::util::internal::(anonymous namespace)::ZSTDCodec::MakeDecompressor() in libarrow.a[172](compression_zstd.cc.o)
arrow::util::internal::(anonymous namespace)::ZSTDCompressor::Compress(long long, unsigned char const*, long long, unsigned char*) in libarrow.a[172](compression_zstd.cc.o)
arrow::util::internal::(anonymous namespace)::ZSTDCompressor::Flush(long long, unsigned char*) in libarrow.a[172](compression_zstd.cc.o)
arrow::util::internal::(anonymous namespace)::ZSTDCompressor::End(long long, unsigned char*) in libarrow.a[172](compression_zstd.cc.o)
...
"_ZSTD_maxCLevel", referenced from:
arrow::util::internal::(anonymous namespace)::ZSTDCodec::maximum_compression_level() const in libarrow.a[172](compression_zstd.cc.o)
"_ZSTD_minCLevel", referenced from:
arrow::util::internal::(anonymous namespace)::ZSTDCodec::minimum_compression_level() const in libarrow.a[172](compression_zstd.cc.o)
"arrow::RecordBatch::Make(std::__1::shared_ptr<arrow::Schema>, long long, std::__1::vector<std::__1::shared_ptr<arrow::Array>, std::__1::allocator<std::__1::shared_ptr<arrow::Array>>>)", referenced from:
gluten::createZeroColumnBatch(int) in ColumnarBatch.cc.o
"snappy::RawCompress(char const*, unsigned long, char*, unsigned long*)", referenced from:
arrow::util::internal::(anonymous namespace)::SnappyCodec::Compress(long long, unsigned char const*, long long, unsigned char*) in libarrow.a[170](compression_snappy.cc.o)
"snappy::RawUncompress(char const*, unsigned long, char*)", referenced from:
arrow::util::internal::(anonymous namespace)::SnappyCodec::Decompress(long long, unsigned char const*, long long, unsigned char*) in libarrow.a[170](compression_snappy.cc.o)
"snappy::MaxCompressedLength(unsigned long)", referenced from:
arrow::util::internal::(anonymous namespace)::SnappyCodec::MaxCompressedLen(long long, unsigned char const*) in libarrow.a[170](compression_snappy.cc.o)
"snappy::GetUncompressedLength(char const*, unsigned long, unsigned long*)", referenced from:
arrow::util::internal::(anonymous namespace)::SnappyCodec::Decompress(long long, unsigned char const*, long long, unsigned char*) in libarrow.a[170](compression_snappy.cc.o)
"_deflate", referenced from:
arrow::util::internal::(anonymous namespace)::GZipCodec::Compress(long long, unsigned char const*, long long, unsigned char*) in libarrow.a[171](compression_zlib.cc.o)
arrow::util::internal::(anonymous namespace)::GZipCompressor::Compress(long long, unsigned char const*, long long, unsigned char*) in libarrow.a[171](compression_zlib.cc.o)
arrow::util::internal::(anonymous namespace)::GZipCompressor::Flush(long long, unsigned char*) in libarrow.a[171](compression_zlib.cc.o)
arrow::util::internal::(anonymous namespace)::GZipCompressor::End(long long, unsigned char*) in libarrow.a[171](compression_zlib.cc.o)
"_deflateBound", referenced from:
arrow::util::internal::(anonymous namespace)::GZipCodec::MaxCompressedLen(long long, unsigned char const*) in libarrow.a[171](compression_zlib.cc.o)
"_deflateEnd", referenced from:
arrow::util::internal::(anonymous namespace)::GZipCodec::~GZipCodec() in libarrow.a[171](compression_zlib.cc.o)
arrow::util::internal::(anonymous namespace)::GZipCodec::~GZipCodec() in libarrow.a[171](compression_zlib.cc.o)
arrow::util::internal::(anonymous namespace)::GZipCodec::InitDecompressor() in libarrow.a[171](compression_zlib.cc.o)
arrow::util::internal::(anonymous namespace)::GZipCompressor::~GZipCompressor() in libarrow.a[171](compression_zlib.cc.o)
arrow::util::internal::(anonymous namespace)::GZipCompressor::~GZipCompressor() in libarrow.a[171](compression_zlib.cc.o)
arrow::util::internal::(anonymous namespace)::GZipCompressor::End(long long, unsigned char*) in libarrow.a[171](compression_zlib.cc.o)
"_deflateInit2_", referenced from:
arrow::util::internal::(anonymous namespace)::GZipCodec::MakeCompressor() in libarrow.a[171](compression_zlib.cc.o)
arrow::util::internal::(anonymous namespace)::GZipCodec::InitCompressor() in libarrow.a[171](compression_zlib.cc.o)
"_deflateReset", referenced from:
arrow::util::internal::(anonymous namespace)::GZipCodec::Compress(long long, unsigned char const*, long long, unsigned char*) in libarrow.a[171](compression_zlib.cc.o)
"_inflate", referenced from:
arrow::util::internal::(anonymous namespace)::GZipCodec::Decompress(long long, unsigned char const*, long long, unsigned char*) in libarrow.a[171](compression_zlib.cc.o)
arrow::util::internal::(anonymous namespace)::GZipDecompressor::Decompress(long long, unsigned char const*, long long, unsigned char*) in libarrow.a[171](compression_zlib.cc.o)
"_inflateEnd", referenced from:
arrow::util::internal::(anonymous namespace)::GZipCodec::~GZipCodec() in libarrow.a[171](compression_zlib.cc.o)
arrow::util::internal::(anonymous namespace)::GZipCodec::~GZipCodec() in libarrow.a[171](compression_zlib.cc.o)
arrow::util::internal::(anonymous namespace)::GZipCodec::InitCompressor() in libarrow.a[171](compression_zlib.cc.o)
arrow::util::internal::(anonymous namespace)::GZipDecompressor::~GZipDecompressor() in libarrow.a[171](compression_zlib.cc.o)
arrow::util::internal::(anonymous namespace)::GZipDecompressor::~GZipDecompressor() in libarrow.a[171](compression_zlib.cc.o)
"_inflateInit2_", referenced from:
arrow::util::internal::(anonymous namespace)::GZipCodec::MakeDecompressor() in libarrow.a[171](compression_zlib.cc.o)
arrow::util::internal::(anonymous namespace)::GZipCodec::InitDecompressor() in libarrow.a[171](compression_zlib.cc.o)
"_inflateReset", referenced from:
arrow::util::internal::(anonymous namespace)::GZipCodec::Decompress(long long, unsigned char const*, long long, unsigned char*) in libarrow.a[171](compression_zlib.cc.o)
arrow::util::internal::(anonymous namespace)::GZipDecompressor::Reset() in libarrow.a[171](compression_zlib.cc.o)
ld: symbol(s) not found for architecture arm64
clang++: error: linker command failed with exit code 1 (use -v to see invocation)
/bin/sh: ldd: command not found
Problem description
Building gluten with a velox backend in mac can sometimes run into this issue & fail to create the libgluten.dylib file.
System information
hazmi@infs-MacBook-Pro incubator-gluten % ./dev/info.sh
./dev/info.sh: line 47: lscpu: command not found
Gluten Version: 1.7.0-SNAPSHOT
Commit: 2887032
CMake Version: 4.3.1
System: Darwin-24.6.0
Arch: arm64
CPU Name:
C++ Compiler: /usr/bin/c++
C++ Compiler Version: 17.0.0.17000013
17.0.0.17000013
C Compiler: /usr/bin/cc
C Compiler Version: 17.0.0.17000013
17.0.0.17000013
CMake Prefix Path: /Applications/Xcode-16.4.0.app/Contents/Developer/Platforms/MacOSX.platform/Developer/SDKs/MacOSX.sdk/usr;/opt/homebrew;/usr/local;/usr;/;/opt/homebrew;/usr/local;/usr/X11R6;/usr/pkg;/opt;/sw;/opt/local
CMake log