Merge branch 'main' into bz/iter-vnode-in-bitmap

.gitignore

@@ -23,6 +23,9 @@ cmake-build-debug/
 *.app
 build/
 
+# Python
+*.pyc
+
 # Golang
 go/bin/
 
@@ -58,6 +61,7 @@ ft.txt
 src/tests/regress/output/*
 src/tests/sync_point/slt/e2e_test
 # generated e2e tests
-e2e_test/generated
+e2e_test/generated/*
+!e2e_test/generated/README.md
 # generated scale tests by `cargo nextest`
 scale-test.tar.zst

Cargo.toml

@@ -119,6 +119,14 @@ incremental = false
 [patch.crates-io]
 quanta = { git = "https://github.com/madsim-rs/quanta.git", rev = "a819877" }
 getrandom = { git = "https://github.com/madsim-rs/getrandom.git", rev = "cc95ee3" }
+tokio-stream = { git = "https://github.com/madsim-rs/tokio.git", rev = "0c25710" }
 tokio-retry = { git = "https://github.com/madsim-rs/rust-tokio-retry.git", rev = "95e2fd3" }
 tokio-postgres = { git = "https://github.com/madsim-rs/rust-postgres.git", rev = "87ca1dc" }
 postgres-types = { git = "https://github.com/madsim-rs/rust-postgres.git", rev = "87ca1dc" }
+
+# TODO: remove these patches when arrow releases v34
+# we need this commit for handling batch with no columns
+# https://github.com/apache/arrow-rs/commit/ea48b9571f88bfbced60f9790ae2a7102502870e
+arrow-array = { git = "https://github.com/apache/arrow-rs.git", rev = "9a6c516" }
+arrow-schema = { git = "https://github.com/apache/arrow-rs.git", rev = "9a6c516" }
+arrow-flight = { git = "https://github.com/apache/arrow-rs.git", rev = "9a6c516" }

ci/Dockerfile

@@ -5,7 +5,7 @@ ENV LANG en_US.utf8
 ARG RUST_TOOLCHAIN
 
 RUN apt-get update -yy && \
-    DEBIAN_FRONTEND=noninteractive apt-get -y install make build-essential cmake protobuf-compiler curl parallel \
+    DEBIAN_FRONTEND=noninteractive apt-get -y install make build-essential cmake protobuf-compiler curl parallel python3 python3-pip \
     openssl libssl-dev libsasl2-dev libcurl4-openssl-dev pkg-config bash openjdk-11-jdk wget unzip git tmux lld postgresql-client kafkacat netcat mysql-client \
     maven -yy \
     && rm -rf /var/lib/{apt,dpkg,cache,log}/
@@ -23,6 +23,9 @@ WORKDIR /risingwave
 
 ENV PATH /root/.cargo/bin/:$PATH
 
+# install python dependencies
+RUN pip3 install pyarrow
+
 # add required rustup components
 RUN rustup component add rustfmt llvm-tools-preview clippy
 

ci/build-ci-image.sh

@@ -14,7 +14,7 @@ export RUST_TOOLCHAIN=$(cat ../rust-toolchain)
 # !!! CHANGE THIS WHEN YOU WANT TO BUMP CI IMAGE !!! #
 #          AND ALSO docker-compose.yml               #
 ######################################################
-export BUILD_ENV_VERSION=v20230220
+export BUILD_ENV_VERSION=v20230221_01
 
 export BUILD_TAG="public.ecr.aws/x5u3w5h6/rw-build-env:${BUILD_ENV_VERSION}"
 

ci/docker-compose.yml

@@ -24,34 +24,38 @@ services:
       - MYSQL_USER=mysqluser
       - MYSQL_PASSWORD=mysqlpw
     healthcheck:
-      test: [ "CMD-SHELL", "mysqladmin ping -h 127.0.0.1 -u root -p123456" ]
+      test:
+        [
+          "CMD-SHELL",
+          "mysqladmin ping -h 127.0.0.1 -u root -p123456"
+        ]
       interval: 5s
       timeout: 5s
       retries: 5
 
   source-test-env:
-    image: public.ecr.aws/x5u3w5h6/rw-build-env:v20230220
+    image: public.ecr.aws/x5u3w5h6/rw-build-env:v20230221_01
     depends_on:
       - mysql
       - db
     volumes:
       - ..:/risingwave
 
   sink-test-env:
-    image: public.ecr.aws/x5u3w5h6/rw-build-env:v20230220
+    image: public.ecr.aws/x5u3w5h6/rw-build-env:v20230221_01
     depends_on:
       - mysql
       - db
     volumes:
       - ..:/risingwave
 
   rw-build-env:
-    image: public.ecr.aws/x5u3w5h6/rw-build-env:v20230220
+    image: public.ecr.aws/x5u3w5h6/rw-build-env:v20230221_01
     volumes:
       - ..:/risingwave
 
   regress-test-env:
-    image: public.ecr.aws/x5u3w5h6/rw-build-env:v20230220
+    image: public.ecr.aws/x5u3w5h6/rw-build-env:v20230221_01
     depends_on:
       db:
         condition: service_healthy

ci/scripts/e2e-iceberg-sink-test.sh

@@ -52,7 +52,7 @@ sleep 1
 # prepare minio iceberg sink
 echo "--- preparing iceberg"
 .risingwave/bin/mcli -C .risingwave/config/mcli mb hummock-minio/iceberg
-wget https://archive.apache.org/dist/spark/spark-3.3.1/spark-3.3.1-bin-hadoop3.tgz
+wget https://iceberg-ci-spark-dist.s3.amazonaws.com/spark-3.3.1-bin-hadoop3.tgz
 tar -xf spark-3.3.1-bin-hadoop3.tgz --no-same-owner
 DEPENDENCIES=org.apache.iceberg:iceberg-spark-runtime-3.3_2.12:1.0.0,org.apache.hadoop:hadoop-aws:3.3.2
 spark-3.3.1-bin-hadoop3/bin/spark-sql --packages $DEPENDENCIES \

ci/scripts/run-e2e-test.sh

@@ -55,6 +55,12 @@ sqllogictest -p 4566 -d dev './e2e_test/batch/**/*.slt' --junit "batch-${profile
 sqllogictest -p 4566 -d dev './e2e_test/database/prepare.slt'
 sqllogictest -p 4566 -d test './e2e_test/database/test.slt'
 
+echo "--- e2e, ci-3cn-1fe, udf"
+python3 e2e_test/udf/test.py &
+sleep 2
+sqllogictest -p 4566 -d dev './e2e_test/udf/python.slt'
+pkill python3
+
 echo "--- Kill cluster"
 cargo make ci-kill