Nulls are not handled well in Iceberg tables.
CREATE TABLE t0(c0 Nullable(Int)) ENGINE = IcebergLocal(local, format = 'Parquet', path = '/var/lib/clickhouse/user_files/lakehouse/t0/');
INSERT INTO TABLE t0 (c0) VALUES (NULL), (1);
SELECT count() FROM t0 WHERE c0 IS NULL; --1, ok
SELECT c0 FROM t0 WHERE c0 IS NULL; --empty, wrong, it should be 1 row with NULL
Another issue. When the table is partitioned by a nullable column, the insert fails.
<Error> TCPHandler: Code: 36. DB::Exception: Unsupported type to write into avro file Null. (BAD_ARGUMENTS), Stack trace (when copying this message, always include the lines below):
0. contrib/llvm-project/libcxx/include/__exception/exception.h:113: Poco::Exception::Exception(String const&, int) @ 0x00000000273c5f32
1. ci/tmp/build/./src/Common/Exception.cpp:128: DB::Exception::Exception(DB::Exception::MessageMasked&&, int, bool) @ 0x00000000144149a9
2. src/Common/Exception.h:123: DB::Exception::Exception(String&&, int, String, bool) @ 0x000000000d0370ce
3. src/Common/Exception.h:58: DB::Exception::Exception(PreformattedMessage&&, int) @ 0x000000000d036ad1
4. src/Common/Exception.h:141: DB::Exception::Exception<DB::Field::Types::Which>(int, FormatStringHelperImpl<std::type_identity<DB::Field::Types::Which>::type>, DB::Field::Types::Which&&) @ 0x0000000019ac1136
5. ci/tmp/build/./src/Storages/ObjectStorage/DataLakes/Iceberg/IcebergWrites.cpp:567: DB::generateManifestFile(Poco::SharedPtr<Poco::JSON::Object, Poco::ReferenceCounter, Poco::ReleasePolicy<Poco::JSON::Object>>, std::vector<String, std::allocator<String>> const&, std::vector<DB::Field, std::allocator<DB::Field>> const&, std::vector<std::shared_ptr<DB::IDataType const>, std::allocator<std::shared_ptr<DB::IDataType const>>> const&, std::vector<String, std::allocator<String>> const&, std::optional<DB::DataFileStatistics> const&, std::shared_ptr<DB::Block const>, Poco::SharedPtr<Poco::JSON::Object, Poco::ReferenceCounter, Poco::ReleasePolicy<Poco::JSON::Object>>, String const&, Poco::SharedPtr<Poco::JSON::Object, Poco::ReferenceCounter, Poco::ReleasePolicy<Poco::JSON::Object>>, long, DB::WriteBuffer&, DB::Iceberg::FileContentType) @ 0x0000000019aa6641
6. ci/tmp/build/./src/Storages/ObjectStorage/DataLakes/Iceberg/IcebergWrites.cpp:1475: DB::IcebergStorageSink::initializeMetadata() @ 0x0000000019abe072
7. ci/tmp/build/./src/Storages/ObjectStorage/DataLakes/Iceberg/IcebergWrites.cpp:1406: DB::IcebergStorageSink::finalizeBuffers() @ 0x0000000019abd588
8. ci/tmp/build/./src/Storages/ObjectStorage/DataLakes/Iceberg/IcebergWrites.cpp:1391: DB::IcebergStorageSink::onFinish() @ 0x0000000019abd3e7
9. contrib/llvm-project/libcxx/include/__functional/function.h:716: ? @ 0x000000001f561143
10. ci/tmp/build/./src/Processors/Transforms/ExceptionKeepingTransform.cpp:172: DB::ExceptionKeepingTransform::work() @ 0x000000001f560cfd
11. ci/tmp/build/./src/Processors/Executors/ExecutionThreadContext.cpp:53: DB::ExecutionThreadContext::executeTask() @ 0x000000001f2c0a50
12. ci/tmp/build/./src/Processors/Executors/PipelineExecutor.cpp:350: DB::PipelineExecutor::executeStepImpl(unsigned long, DB::IAcquiredSlot*, std::atomic<bool>*) @ 0x000000001f2b3785
13. ci/tmp/build/./src/Processors/Executors/PipelineExecutor.cpp:179: DB::PipelineExecutor::executeStep(std::atomic<bool>*) @ 0x000000001f2b2ece
14. ci/tmp/build/./src/Processors/Executors/PushingPipelineExecutor.cpp:131: DB::PushingPipelineExecutor::finish() @ 0x000000001f2cccdf
15. ci/tmp/build/./src/Server/TCPHandler.cpp:1205: DB::TCPHandler::processInsertQuery(DB::QueryState&) @ 0x000000001f1e0936
16. ci/tmp/build/./src/Server/TCPHandler.cpp:748: DB::TCPHandler::runImpl() @ 0x000000001f1d5733
17. ci/tmp/build/./src/Server/TCPHandler.cpp:2740: DB::TCPHandler::run() @ 0x000000001f1ef4a4
18. ci/tmp/build/./base/poco/Net/src/TCPServerConnection.cpp:40: Poco::Net::TCPServerConnection::start() @ 0x000000002747c907
19. ci/tmp/build/./base/poco/Net/src/TCPServerDispatcher.cpp:115: Poco::Net::TCPServerDispatcher::run() @ 0x000000002747cee5
20. ci/tmp/build/./base/poco/Foundation/src/ThreadPool.cpp:205: Poco::PooledThread::run() @ 0x000000002741c0bf
21. base/poco/Foundation/src/Thread_POSIX.cpp:341: Poco::ThreadImpl::runnableEntry(void*) @ 0x0000000027419791
22. start_thread @ 0x00000000000a27f1
23. __GI___clone3 @ 0x0000000000133c9c
Describe the bug
Nulls are not handled well in Iceberg tables.
How to reproduce
In ClickHouse:
Another issue. When the table is partitioned by a nullable column, the insert fails.
Error message and/or stacktrace
Stack trace for last error: