deltalake/iceberg/hudi improvements

docker/test/integration/runner/Dockerfile

@@ -32,6 +32,7 @@ RUN apt-get update \
     libssl-dev \
     libcurl4-openssl-dev \
     gdb \
+    default-jdk \
     software-properties-common \
     libkrb5-dev \
     krb5-user \
@@ -92,15 +93,24 @@ RUN python3 -m pip install \
     tzlocal==2.1 \
     urllib3 \
     requests-kerberos \
+    pyspark==3.3.2 \
+    delta-spark==2.2.0 \
     pyhdfs \
     azure-storage-blob \
-    meilisearch==0.18.3 
+    meilisearch==0.18.3
 
 COPY modprobe.sh /usr/local/bin/modprobe
 COPY dockerd-entrypoint.sh /usr/local/bin/
 COPY compose/ /compose/
 COPY misc/ /misc/
 
+RUN wget https://dlcdn.apache.org/spark/spark-3.3.2/spark-3.3.2-bin-hadoop3.tgz \
+    && tar xzvf spark-3.3.2-bin-hadoop3.tgz -C /
+
+# download spark and packages
+# if you change packages, don't forget to update them in tests/integration/helpers/cluster.py
+RUN echo ":quit" | /spark-3.3.2-bin-hadoop3/bin/spark-shell --packages "org.apache.hudi:hudi-spark3.3-bundle_2.12:0.13.0,io.delta:delta-core_2.12:2.2.0,org.apache.iceberg:iceberg-spark-runtime-3.3_2.12:1.1.0" > /dev/null
+
 RUN set -x \
   && addgroup --system dockremap \
     && adduser --system dockremap \

docker/test/integration/runner/dockerd-entrypoint.sh

@@ -37,6 +37,12 @@ set -e
     docker ps --all --quiet | xargs --no-run-if-empty docker rm || true
 }
 
+java_path="$(update-alternatives --config java | sed -n 's/.*(providing \/usr\/bin\/java): //p')"
+export JAVA_PATH=$java_path
+export SPARK_HOME="/spark-3.3.2-bin-hadoop3"
+export PATH=$SPARK_HOME/bin:$PATH
+export JAVA_TOOL_OPTIONS="-Djdk.attach.allowAttachSelf=true"
+
 echo "Start tests"
 export CLICKHOUSE_TESTS_SERVER_BIN_PATH=/clickhouse
 export CLICKHOUSE_TESTS_CLIENT_BIN_PATH=/clickhouse

src/CMakeLists.txt

@@ -103,6 +103,7 @@ if (TARGET ch_contrib::nats_io)
     add_headers_and_sources(dbms Storages/NATS)
 endif()
 
+add_headers_and_sources(dbms Storages/DataLakes)
 add_headers_and_sources(dbms Storages/MeiliSearch)
 add_headers_and_sources(dbms Common/NamedCollections)
 

src/Core/NamesAndTypes.cpp

@@ -159,6 +159,18 @@ DataTypes NamesAndTypesList::getTypes() const
     return res;
 }
 
+void NamesAndTypesList::filterColumns(const NameSet & names)
+{
+    for (auto it = begin(); it != end();)
+    {
+        const auto & column = *it;
+        if (names.contains(column.name))
+            ++it;
+        else
+            it = erase(it);
+    }
+}
+
 NamesAndTypesList NamesAndTypesList::filter(const NameSet & names) const
 {
     NamesAndTypesList res;

src/Core/NamesAndTypes.h

@@ -102,6 +102,9 @@ class NamesAndTypesList : public std::list<NameAndTypePair>
     Names getNames() const;
     DataTypes getTypes() const;
 
+    /// Remove columns which names are not in the `names`.
+    void filterColumns(const NameSet & names);
+
     /// Leave only the columns whose names are in the `names`. In `names` there can be superfluous columns.
     NamesAndTypesList filter(const NameSet & names) const;