From ca4f94dd10216b87cbc505e52d6c20ce700e6a25 Mon Sep 17 00:00:00 2001 From: Gezi-lzq Date: Tue, 4 Aug 2026 19:04:05 +0000 Subject: [PATCH] [build] Add local development kit Add a Docker Compose and just based development kit that runs locally built Fluss artifacts and supports core, Iceberg, Paimon, Lance, and Hudi profiles with manual tiering validation commands. Run Fluss processes under Docker init, enable JDWP and Prometheus metrics, and propagate configured Hadoop settings to the Hudi write client so the S3A-backed Hudi profile retains its endpoint and credentials. --- devkit/.gitignore | 1 + devkit/README.md | 140 +++++++ devkit/compose/iceberg.yml | 40 ++ devkit/compose/lake-common.yml | 62 +++ devkit/compose/lance.yml | 24 ++ devkit/compose/tiering.yml | 114 ++++++ devkit/docker-compose.yml | 170 ++++++++ devkit/justfile | 363 ++++++++++++++++++ devkit/profiles/core/server.yaml | 19 + devkit/profiles/hudi/compose.files | 17 + devkit/profiles/hudi/flink.urls | 17 + devkit/profiles/hudi/jars.urls | 17 + devkit/profiles/hudi/server.urls | 21 + devkit/profiles/hudi/server.yaml | 41 ++ devkit/profiles/iceberg/compose.files | 18 + devkit/profiles/iceberg/flink.urls | 18 + devkit/profiles/iceberg/jars.urls | 20 + devkit/profiles/iceberg/server.yaml | 41 ++ devkit/profiles/lance/compose.files | 18 + devkit/profiles/lance/server.yaml | 37 ++ devkit/profiles/paimon/compose.files | 17 + devkit/profiles/paimon/flink.urls | 18 + devkit/profiles/paimon/jars.urls | 17 + devkit/profiles/paimon/server.yaml | 35 ++ devkit/tests/hudi-create-table.sql | 32 ++ devkit/tests/hudi-query-lake.sql | 40 ++ devkit/tests/hudi-write-data.sql | 27 ++ devkit/tests/iceberg-create-table.sql | 34 ++ devkit/tests/iceberg-query-lake.sql | 36 ++ devkit/tests/iceberg-write-data.sql | 30 ++ devkit/tests/lance-create-table.sql | 31 ++ devkit/tests/lance-write-data.sql | 27 ++ devkit/tests/paimon-create-table.sql | 33 ++ devkit/tests/paimon-query-lake.sql | 35 ++ devkit/tests/paimon-write-data.sql | 29 ++ .../lake/hudi/tiering/HudiWriteTableInfo.java | 5 +- 36 files changed, 1643 insertions(+), 1 deletion(-) create mode 100644 devkit/.gitignore create mode 100644 devkit/README.md create mode 100644 devkit/compose/iceberg.yml create mode 100644 devkit/compose/lake-common.yml create mode 100644 devkit/compose/lance.yml create mode 100644 devkit/compose/tiering.yml create mode 100644 devkit/docker-compose.yml create mode 100644 devkit/justfile create mode 100644 devkit/profiles/core/server.yaml create mode 100644 devkit/profiles/hudi/compose.files create mode 100644 devkit/profiles/hudi/flink.urls create mode 100644 devkit/profiles/hudi/jars.urls create mode 100644 devkit/profiles/hudi/server.urls create mode 100644 devkit/profiles/hudi/server.yaml create mode 100644 devkit/profiles/iceberg/compose.files create mode 100644 devkit/profiles/iceberg/flink.urls create mode 100644 devkit/profiles/iceberg/jars.urls create mode 100644 devkit/profiles/iceberg/server.yaml create mode 100644 devkit/profiles/lance/compose.files create mode 100644 devkit/profiles/lance/server.yaml create mode 100644 devkit/profiles/paimon/compose.files create mode 100644 devkit/profiles/paimon/flink.urls create mode 100644 devkit/profiles/paimon/jars.urls create mode 100644 devkit/profiles/paimon/server.yaml create mode 100644 devkit/tests/hudi-create-table.sql create mode 100644 devkit/tests/hudi-query-lake.sql create mode 100644 devkit/tests/hudi-write-data.sql create mode 100644 devkit/tests/iceberg-create-table.sql create mode 100644 devkit/tests/iceberg-query-lake.sql create mode 100644 devkit/tests/iceberg-write-data.sql create mode 100644 devkit/tests/lance-create-table.sql create mode 100644 devkit/tests/lance-write-data.sql create mode 100644 devkit/tests/paimon-create-table.sql create mode 100644 devkit/tests/paimon-query-lake.sql create mode 100644 devkit/tests/paimon-write-data.sql diff --git a/devkit/.gitignore b/devkit/.gitignore new file mode 100644 index 0000000000..321b2a2a67 --- /dev/null +++ b/devkit/.gitignore @@ -0,0 +1 @@ +.deps/ diff --git a/devkit/README.md b/devkit/README.md new file mode 100644 index 0000000000..ab069d5fab --- /dev/null +++ b/devkit/README.md @@ -0,0 +1,140 @@ +# Fluss DevKit + +Fluss DevKit starts a local Fluss environment with Docker Compose and `just`. It uses upstream +runtime images and mounts the local `../build-target` directory read-only, so testing source changes +never requires building a Docker image. + +## Requirements + +- JDK 11 or later +- Docker with Docker Compose +- [just](https://github.com/casey/just) + +## Quick Start + +Build and start the core development environment: + +```bash +cd devkit +just build +just up +``` + +For a lake profile, build the tiering artifacts and select a format: + +```bash +just build-tiering +just up iceberg +``` + +`just up` waits for the Fluss and Flink clusters and, for lake profiles, the tiering job to become +ready. Run `just --list` to see all available commands. + +## Profiles + +| Profile | Services | +|---|---| +| `core` | ZooKeeper, Fluss, and Flink | +| `iceberg` | Core, RustFS, PostgreSQL JDBC Catalog, Flink, and Iceberg tiering | +| `paimon` | Core, RustFS, Flink, and Paimon tiering | +| `lance` | Core, RustFS, Flink, and Lance tiering | +| `hudi` | Core, RustFS, Flink, and Hudi tiering | + +Profiles start one TabletServer by default. Pass `3` as the second argument to start three: + +```bash +just up core 3 +just up paimon 3 +``` + +## Validate Lake Tiering + +The validation workflow is intentionally split into commands that can be run and inspected one at +a time: + +```bash +just build-tiering +just up paimon + +format=paimon +table="${format}_manual_$(date +%s)" + +just create-table "$format" "$table" +just write-data "$format" "$table" +just tiering-status +just query-lake "$format" "$table" +``` + +Use `iceberg`, `paimon`, or `hudi` for a complete create, write, tier, and lake-query workflow. +Tiering is asynchronous, so repeat `query-lake` if the first query runs before the lake commit is +visible. + +The sample writes three rows. A successful lake query reports row count `3`, ID sum `6`, three +distinct payloads, and the payload range `alpha` to `gamma`. Use a new table name for each run: +dropping a non-empty Fluss table does not remove its lake table. + +Lance supports `create-table`, `write-data`, and tiering, but the DevKit does not bundle a Flink SQL +reader for `query-lake`. Inspect the generated objects in RustFS when validating Lance. + +## Operations + +```bash +just status +just logs +just logs tablet-server-0 200 +just exec tablet-server-0 java -version +just run-sql ./my-query.sql +just down +just clean +``` + +Start a profile with `just up` before using `run-sql`. SQL files are read from the host and passed to +the Flink SQL Client. Relative paths are resolved from the current working directory. + +`just up` replaces containers from the previous profile but preserves named volumes. `down` removes +containers and keeps data; `clean` also removes Compose volumes. Downloaded JARs remain in the +ignored `.deps` cache. + +Default endpoints: + +| Service | Address | +|---|---| +| CoordinatorServer | `localhost:9123` | +| TabletServer 0 | `localhost:9124` | +| TabletServers 1 and 2 | `localhost:9125` and `localhost:9126` (three-node mode) | +| CoordinatorServer JDWP | `localhost:15005` | +| TabletServer JDWP | `localhost:15006` to `localhost:15008` | +| CoordinatorServer metrics | `http://localhost:9249/metrics` | +| TabletServer metrics | `http://localhost:9250/metrics` to `http://localhost:9252/metrics` | +| Flink UI | `http://localhost:8083` | +| RustFS S3 API | `http://localhost:9000` | +| RustFS Console | `http://localhost:9001` | + +JDWP is enabled for every Fluss process with `suspend=n`, so a remote debugger can attach without +blocking startup. Prometheus export is also enabled by default and can be checked directly, for +example with `curl http://localhost:9249/metrics`. + +The default Fluss runtime is `eclipse-temurin:17-jre-noble`; set `FLUSS_DEVKIT_IMAGE` to use +another Java 17 runtime. Port environment variables in the Compose files can override the default +addresses, for example `COORDINATOR_DEBUG_PORT=5005 just up` or +`TABLET_SERVER_0_METRICS_PORT=9300 just up`. + +## Adding a Profile + +Create a directory under `profiles/`. The directory contains one required file and up to four +optional files: + +| File | Purpose | +|---|---| +| `server.yaml` | Fluss and lake configuration | +| `jars.urls` | JARs used by both Fluss Server and Flink | +| `server.urls` | Additional Fluss Server-only JARs | +| `flink.urls` | Additional Flink-only JARs | +| `compose.files` | Compose overlays, relative to `devkit/`, for extra local services | + +Put one URL or Compose path on each line; empty lines and `#` comments are ignored. If `server.yaml` +contains `datalake.format`, `just up` automatically stages the matching locally built lake plugin +and starts Flink tiering. Use `compose.files` only when the profile needs additional local services +or runtime overrides. + +Start the new profile with `just up `. No `justfile` change is required. diff --git a/devkit/compose/iceberg.yml b/devkit/compose/iceberg.yml new file mode 100644 index 0000000000..27cb9bcb10 --- /dev/null +++ b/devkit/compose/iceberg.yml @@ -0,0 +1,40 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +services: + postgres: + image: postgres:17 + restart: unless-stopped + environment: + POSTGRES_USER: iceberg + POSTGRES_PASSWORD: iceberg + POSTGRES_DB: iceberg + volumes: + - iceberg-postgres-data:/var/lib/postgresql/data + healthcheck: + test: ["CMD-SHELL", "pg_isready -U iceberg -d iceberg"] + interval: 3s + timeout: 3s + retries: 20 + start_period: 5s + + coordinator-server: + depends_on: + postgres: + condition: service_healthy + +volumes: + iceberg-postgres-data: diff --git a/devkit/compose/lake-common.yml b/devkit/compose/lake-common.yml new file mode 100644 index 0000000000..13821f797d --- /dev/null +++ b/devkit/compose/lake-common.yml @@ -0,0 +1,62 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +services: + rustfs: + image: rustfs/rustfs:1.0.0-alpha.83 + restart: unless-stopped + command: /data + environment: + RUSTFS_ACCESS_KEY: rustfsadmin + RUSTFS_SECRET_KEY: rustfsadmin + RUSTFS_CONSOLE_ENABLE: "true" + ports: + - "${RUSTFS_API_PORT:-9000}:9000" + - "${RUSTFS_CONSOLE_PORT:-9001}:9001" + volumes: + - rustfs-data:/data + healthcheck: + test: ["CMD-SHELL", "nc -z 127.0.0.1 9000"] + interval: 5s + timeout: 5s + retries: 20 + start_period: 5s + + rustfs-init: + image: minio/mc + depends_on: + rustfs: + condition: service_healthy + entrypoint: + - /bin/sh + - -c + - | + until mc alias set rustfs http://rustfs:9000 rustfsadmin rustfsadmin; do sleep 2; done + mc mb --ignore-existing rustfs/fluss + printf '' | mc pipe rustfs/fluss/hudi/.keep + + coordinator-server: + depends_on: + rustfs-init: + condition: service_completed_successfully + + jobmanager: + depends_on: + rustfs-init: + condition: service_completed_successfully + +volumes: + rustfs-data: diff --git a/devkit/compose/lance.yml b/devkit/compose/lance.yml new file mode 100644 index 0000000000..efd52061c2 --- /dev/null +++ b/devkit/compose/lance.yml @@ -0,0 +1,24 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +services: + taskmanager: + environment: + FLINK_PROPERTIES: | + jobmanager.rpc.address: jobmanager + taskmanager.memory.process.size: 2048m + taskmanager.memory.task.off-heap.size: 512m + taskmanager.numberOfTaskSlots: 4 diff --git a/devkit/compose/tiering.yml b/devkit/compose/tiering.yml new file mode 100644 index 0000000000..61ead3a446 --- /dev/null +++ b/devkit/compose/tiering.yml @@ -0,0 +1,114 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +x-flink-common: &flink-common + image: flink:1.20.0-scala_2.12-java17 + entrypoint: + - /bin/bash + - -c + - | + cp /opt/devkit-jars/lib/*.jar /opt/flink/lib/ + chmod a+r /opt/flink/lib/*.jar + exec /docker-entrypoint.sh "$$@" + - -- + volumes: + - ./.deps/flink/active:/opt/devkit-jars:ro + - ${FLUSS_DEVKIT_CONFIG:-./profiles/core/server.yaml}:/opt/devkit-config/server.yaml:ro + - ./tests:/opt/devkit-tests:ro + - shared-data:/tmp/fluss + +services: + jobmanager: + <<: *flink-common + profiles: [flink] + command: ["jobmanager"] + depends_on: + coordinator-server: + condition: service_healthy + environment: + FLINK_PROPERTIES: | + jobmanager.rpc.address: jobmanager + jobmanager.memory.process.size: 1024m + parallelism.default: 1 + rest.address: jobmanager + ports: + - "${FLINK_REST_PORT:-8083}:8081" + healthcheck: + test: ["CMD-SHELL", "curl -fsS http://localhost:8081/overview >/dev/null"] + interval: 5s + timeout: 5s + retries: 24 + start_period: 10s + + taskmanager: + <<: *flink-common + profiles: [flink] + command: ["taskmanager"] + depends_on: + jobmanager: + condition: service_healthy + environment: + FLINK_PROPERTIES: | + jobmanager.rpc.address: jobmanager + taskmanager.memory.process.size: 1536m + taskmanager.numberOfTaskSlots: 4 + healthcheck: + test: + - CMD-SHELL + - curl -fsS http://jobmanager:8081/taskmanagers | grep -q '"id"' + interval: 5s + timeout: 5s + retries: 24 + start_period: 10s + + tiering-submit: + <<: *flink-common + profiles: [tiering] + restart: "no" + depends_on: + taskmanager: + condition: service_healthy + environment: + FLINK_PROPERTIES: | + jobmanager.rpc.address: jobmanager + rest.address: jobmanager + command: + - /bin/bash + - -c + - | + ARGS=() + HAS_BOOTSTRAP=false + while IFS= read -r LINE || [ -n "$$LINE" ]; do + LINE="$${LINE%%#*}" + [[ "$$LINE" == *": "* ]] || continue + KEY="$${LINE%%: *}" + VALUE="$${LINE#*: }" + KEY="$${KEY#"$${KEY%%[![:space:]]*}"}" + KEY="$${KEY%"$${KEY##*[![:space:]]}"}" + VALUE="$${VALUE#"$${VALUE%%[![:space:]]*}"}" + VALUE="$${VALUE%"$${VALUE##*[![:space:]]}"}" + case "$$KEY" in + datalake.*|lake.tiering.*|config.providers*|fluss.*) + ARGS+=("--$$KEY" "$$VALUE") + [ "$$KEY" = "fluss.bootstrap.servers" ] && HAS_BOOTSTRAP=true + ;; + esac + done < /opt/devkit-config/server.yaml + if [ "$$HAS_BOOTSTRAP" = false ]; then + ARGS+=(--fluss.bootstrap.servers coordinator-server:19123) + fi + exec /opt/flink/bin/flink run -d \ + /opt/devkit-jars/jobs/fluss-flink-tiering.jar "$${ARGS[@]}" diff --git a/devkit/docker-compose.yml b/devkit/docker-compose.yml new file mode 100644 index 0000000000..035a0b96df --- /dev/null +++ b/devkit/docker-compose.yml @@ -0,0 +1,170 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +name: fluss-devkit + +x-fluss-common: &fluss-common + image: ${FLUSS_DEVKIT_IMAGE:-eclipse-temurin:17-jre-noble} + init: true + restart: unless-stopped + stop_grace_period: 2m + volumes: + - ../build-target:/opt/fluss:ro + - ${FLUSS_DEVKIT_CONFIG:-./profiles/core/server.yaml}:/opt/fluss/conf/server.yaml:ro + - shared-data:/tmp/fluss + environment: + FLUSS_LOG_DIR: /tmp/fluss-logs + logging: + driver: json-file + options: + max-size: 10m + max-file: "3" + +x-tablet-common: &tablet-common + <<: *fluss-common + entrypoint: ["/opt/fluss/bin/tablet-server.sh", "start-foreground"] + depends_on: + coordinator-server: + condition: service_healthy + environment: + FLUSS_LOG_DIR: /tmp/fluss-logs + READINESS_TCP_PORT: "9123" + READINESS_HEALTH_CHECK_TIMEOUT_SECONDS: "180" + healthcheck: + test: ["CMD-SHELL", "bash /opt/fluss/bin/readiness-check.sh"] + interval: 5s + timeout: 15s + retries: 36 + start_period: 10s + +services: + zookeeper: + image: zookeeper:3.9.2 + restart: unless-stopped + healthcheck: + test: ["CMD-SHELL", "zkServer.sh status"] + interval: 5s + timeout: 5s + retries: 20 + start_period: 5s + volumes: + - zookeeper-data:/data + - zookeeper-log:/datalog + + coordinator-server: + <<: *fluss-common + entrypoint: ["/opt/fluss/bin/coordinator-server.sh", "start-foreground"] + depends_on: + zookeeper: + condition: service_healthy + command: + - -Dzookeeper.address=zookeeper:2181 + - -Dbind.listeners=INTERNAL://coordinator-server:0,CLIENT://coordinator-server:9123,DEVKIT://coordinator-server:19123 + - -Dadvertised.listeners=CLIENT://localhost:${COORDINATOR_PORT:-9123} + - -Dinternal.listener.name=INTERNAL + - -Ddefault.replication.factor=${FLUSS_REPLICATION_FACTOR:-1} + - -Dmetrics.reporters=prometheus + - -Dmetrics.reporter.prometheus.port=9249 + environment: + FLUSS_LOG_DIR: /tmp/fluss-logs + FLUSS_ENV_JAVA_OPTS: -Xms512m -Xmx512m -agentlib:jdwp=transport=dt_socket,server=y,suspend=n,address=*:5005 + ports: + - "${COORDINATOR_PORT:-9123}:9123" + - "${COORDINATOR_DEBUG_PORT:-15005}:5005" + - "${COORDINATOR_METRICS_PORT:-9249}:9249" + healthcheck: + test: ["CMD-SHELL", "bash -c 'echo > /dev/tcp/coordinator-server/9123'"] + interval: 5s + timeout: 5s + retries: 24 + start_period: 10s + + tablet-server-0: + <<: *tablet-common + profiles: [single, cluster] + command: + - -Dzookeeper.address=zookeeper:2181 + - -Dbind.listeners=INTERNAL://tablet-server-0:0,CLIENT://tablet-server-0:9123,DEVKIT://tablet-server-0:19123 + - -Dadvertised.listeners=CLIENT://localhost:${TABLET_SERVER_0_PORT:-9124} + - -Dinternal.listener.name=INTERNAL + - -Dtablet-server.id=0 + - -Dkv.snapshot.interval=0s + - -Ddata.dir=/tmp/fluss/data/tablet-server-0 + - -Dmetrics.reporters=prometheus + - -Dmetrics.reporter.prometheus.port=9249 + environment: + FLUSS_LOG_DIR: /tmp/fluss-logs + FLUSS_ENV_JAVA_OPTS: -Xms512m -Xmx512m -agentlib:jdwp=transport=dt_socket,server=y,suspend=n,address=*:5005 + READINESS_TCP_HOST: tablet-server-0 + READINESS_TCP_PORT: "9123" + READINESS_HEALTH_CHECK_TIMEOUT_SECONDS: "180" + ports: + - "${TABLET_SERVER_0_PORT:-9124}:9123" + - "${TABLET_SERVER_0_DEBUG_PORT:-15006}:5005" + - "${TABLET_SERVER_0_METRICS_PORT:-9250}:9249" + + tablet-server-1: + <<: *tablet-common + profiles: [cluster] + command: + - -Dzookeeper.address=zookeeper:2181 + - -Dbind.listeners=INTERNAL://tablet-server-1:0,CLIENT://tablet-server-1:9123,DEVKIT://tablet-server-1:19123 + - -Dadvertised.listeners=CLIENT://localhost:${TABLET_SERVER_1_PORT:-9125} + - -Dinternal.listener.name=INTERNAL + - -Dtablet-server.id=1 + - -Dkv.snapshot.interval=0s + - -Ddata.dir=/tmp/fluss/data/tablet-server-1 + - -Dmetrics.reporters=prometheus + - -Dmetrics.reporter.prometheus.port=9249 + environment: + FLUSS_LOG_DIR: /tmp/fluss-logs + FLUSS_ENV_JAVA_OPTS: -Xms512m -Xmx512m -agentlib:jdwp=transport=dt_socket,server=y,suspend=n,address=*:5005 + READINESS_TCP_HOST: tablet-server-1 + READINESS_TCP_PORT: "9123" + READINESS_HEALTH_CHECK_TIMEOUT_SECONDS: "180" + ports: + - "${TABLET_SERVER_1_PORT:-9125}:9123" + - "${TABLET_SERVER_1_DEBUG_PORT:-15007}:5005" + - "${TABLET_SERVER_1_METRICS_PORT:-9251}:9249" + + tablet-server-2: + <<: *tablet-common + profiles: [cluster] + command: + - -Dzookeeper.address=zookeeper:2181 + - -Dbind.listeners=INTERNAL://tablet-server-2:0,CLIENT://tablet-server-2:9123,DEVKIT://tablet-server-2:19123 + - -Dadvertised.listeners=CLIENT://localhost:${TABLET_SERVER_2_PORT:-9126} + - -Dinternal.listener.name=INTERNAL + - -Dtablet-server.id=2 + - -Dkv.snapshot.interval=0s + - -Ddata.dir=/tmp/fluss/data/tablet-server-2 + - -Dmetrics.reporters=prometheus + - -Dmetrics.reporter.prometheus.port=9249 + environment: + FLUSS_LOG_DIR: /tmp/fluss-logs + FLUSS_ENV_JAVA_OPTS: -Xms512m -Xmx512m -agentlib:jdwp=transport=dt_socket,server=y,suspend=n,address=*:5005 + READINESS_TCP_HOST: tablet-server-2 + READINESS_TCP_PORT: "9123" + READINESS_HEALTH_CHECK_TIMEOUT_SECONDS: "180" + ports: + - "${TABLET_SERVER_2_PORT:-9126}:9123" + - "${TABLET_SERVER_2_DEBUG_PORT:-15008}:5005" + - "${TABLET_SERVER_2_METRICS_PORT:-9252}:9249" + +volumes: + shared-data: + zookeeper-data: + zookeeper-log: diff --git a/devkit/justfile b/devkit/justfile new file mode 100644 index 0000000000..9c5978884f --- /dev/null +++ b/devkit/justfile @@ -0,0 +1,363 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +set dotenv-load := false + +# --- Paths -------------------------------------------------------------------- + +DEVKIT_DIR := justfile_directory() +FLUSS_DEV_HOME := DEVKIT_DIR + "/.." +PROFILES_DIR := DEVKIT_DIR + "/profiles" +DEPS_DIR := DEVKIT_DIR + "/.deps" +CACHE_DIR := DEPS_DIR + "/cache" + +# --- Docker Compose commands -------------------------------------------------- + +COMPOSE := "docker compose -f " + DEVKIT_DIR + "/docker-compose.yml" +COMPOSE_TIERING := COMPOSE + " -f " + DEVKIT_DIR + "/compose/tiering.yml" +ALL_COMPOSE := COMPOSE_TIERING + " -f " + DEVKIT_DIR + "/compose/lake-common.yml -f " + DEVKIT_DIR + "/compose/iceberg.yml" +ALL_PROFILES := "--profile single --profile cluster --profile flink --profile tiering" + +# --- Build commands ----------------------------------------------------------- + +BUILD_CMD := env_var_or_default("DEVKIT_BUILD_CMD", "./mvnw -DskipTests -pl fluss-dist,fluss-flink/fluss-flink-1.20 -am package") +TIERING_BUILD_CMD := env_var_or_default("DEVKIT_TIERING_BUILD_CMD", "./mvnw -DskipTests -pl fluss-dist,fluss-flink/fluss-flink-1.20,fluss-flink/fluss-flink-tiering -am package") + +default: + @just --list + +[doc("Build the local Fluss distribution and Flink connector")] +build: + cd "{{ FLUSS_DEV_HOME }}" && {{ BUILD_CMD }} + +[doc("Build the local distribution and lake tiering artifacts")] +build-tiering: + cd "{{ FLUSS_DEV_HOME }}" && {{ TIERING_BUILD_CMD }} + +[private] +ensure-dist: + #!/usr/bin/env bash + set -e + if [ ! -x "{{ FLUSS_DEV_HOME }}/build-target/bin/coordinator-server.sh" ]; then + echo "Local distribution not found. Run: just build" + exit 1 + fi + +[private] +validate-profile profile: + #!/usr/bin/env bash + set -e + PROFILE={{ quote(profile) }} + if [[ ! "$PROFILE" =~ ^[a-z0-9][a-z0-9-]*$ ]]; then + echo "profile must match [a-z0-9][a-z0-9-]*" + exit 1 + fi + if [ ! -f "{{ PROFILES_DIR }}/$PROFILE/server.yaml" ]; then + echo "Unknown profile: $PROFILE" + exit 1 + fi + +[private] +profile-format profile: + #!/usr/bin/env bash + PROFILE={{ quote(profile) }} + sed -n 's/^[[:space:]]*datalake\.format[[:space:]]*: \([^#]*\).*$/\1/p' \ + "{{ PROFILES_DIR }}/$PROFILE/server.yaml" | sed 's/[[:space:]]*$//' | tail -n 1 + +[doc("Download the JARs declared by a profile")] +deps profile: + #!/usr/bin/env bash + set -e + PROFILE={{ quote(profile) }} + just validate-profile "$PROFILE" + PROFILE_DIR="{{ PROFILES_DIR }}/$PROFILE" + mkdir -p "{{ CACHE_DIR }}" + for FILE in "$PROFILE_DIR"/{jars,server,flink}.urls; do + [ -f "$FILE" ] || continue + while IFS= read -r URL; do + NAME="$(basename "${URL%%\?*}")" + TARGET="{{ CACHE_DIR }}/$NAME" + [ -s "$TARGET" ] && continue + echo "Downloading $(basename "$TARGET")..." + curl -fL --retry 3 -o "${TARGET}.tmp" "$URL" + mv "${TARGET}.tmp" "$TARGET" + done < <(sed 's/#.*//; s/[[:space:]]//g; /^$/d' "$FILE") + done + +[private] +ensure-flink: + #!/usr/bin/env bash + set -e + ACTIVE_DIR="{{ DEPS_DIR }}/flink/active" + MATCHES=( {{ FLUSS_DEV_HOME }}/fluss-flink/fluss-flink-1.20/target/fluss-flink-1.20-*.jar ) + if [ "${#MATCHES[@]}" -ne 1 ] || [ ! -f "${MATCHES[0]}" ]; then + echo "Expected one Flink connector artifact" + echo "Run: just build" + exit 1 + fi + mkdir -p "$ACTIVE_DIR/lib" "$ACTIVE_DIR/jobs" + rm -f "$ACTIVE_DIR/lib/"*.jar "$ACTIVE_DIR/jobs/"*.jar + cp "${MATCHES[0]}" "$ACTIVE_DIR/lib/fluss-flink-1.20.jar" + chmod -R a+rX "$ACTIVE_DIR" + +[private] +ensure-tiering profile: + #!/usr/bin/env bash + set -e + PROFILE={{ quote(profile) }} + just validate-profile "$PROFILE" + PROFILE_DIR="{{ PROFILES_DIR }}/$PROFILE" + FORMAT=$(just profile-format "$PROFILE") + if [ -z "$FORMAT" ]; then + echo "Profile $PROFILE does not configure datalake.format" + exit 1 + fi + stage() { + local PATTERN="$1" + local TARGET="$2" + MATCHES=( $PATTERN ) + if [ "${#MATCHES[@]}" -ne 1 ] || [ ! -f "${MATCHES[0]}" ]; then + echo "Expected one build artifact: $PATTERN" + echo "Run: just build-tiering" + exit 1 + fi + cp "${MATCHES[0]}" "$TARGET" + } + copy_urls() { + local FILE="$1" + shift + local URL NAME TARGET + [ -f "$FILE" ] || return 0 + while IFS= read -r URL; do + NAME="$(basename "${URL%%\?*}")" + for TARGET in "$@"; do + cp "{{ CACHE_DIR }}/$NAME" "$TARGET/" + done + done < <(sed 's/#.*//; s/[[:space:]]//g; /^$/d' "$FILE") + } + ACTIVE_DIR="{{ DEPS_DIR }}/flink/active" + just deps "$PROFILE" + stage "{{ FLUSS_DEV_HOME }}/fluss-lake/fluss-lake-$FORMAT/target/fluss-lake-$FORMAT-*.jar" "$ACTIVE_DIR/lib/fluss-lake-$FORMAT.jar" + stage "{{ FLUSS_DEV_HOME }}/fluss-flink/fluss-flink-tiering/target/fluss-flink-tiering-*.jar" "$ACTIVE_DIR/jobs/fluss-flink-tiering.jar" + + SERVER_PLUGIN_DIR="{{ FLUSS_DEV_HOME }}/build-target/plugins/$FORMAT" + mkdir -p "$SERVER_PLUGIN_DIR" + # Filesystem plugins must stay isolated from lake plugin classpaths. + rm -f "$SERVER_PLUGIN_DIR"/fluss-fs-*.jar + REMOTE_DATA_DIR=$(sed -n 's/^[[:space:]]*remote\.data\.dir[[:space:]]*: \([^#]*\).*$/\1/p' "$PROFILE_DIR/server.yaml" | sed 's/[[:space:]]*$//' | tail -n 1) + SCHEME="${REMOTE_DATA_DIR%%:*}" + if [ "$SCHEME" != "$REMOTE_DATA_DIR" ] && [ -d "{{ FLUSS_DEV_HOME }}/build-target/plugins/$SCHEME" ]; then + cp "{{ FLUSS_DEV_HOME }}/build-target/plugins/$SCHEME/"*.jar "$ACTIVE_DIR/lib/" + fi + + copy_urls "$PROFILE_DIR/jars.urls" "$SERVER_PLUGIN_DIR" "$ACTIVE_DIR/lib" + copy_urls "$PROFILE_DIR/server.urls" "$SERVER_PLUGIN_DIR" + copy_urls "$PROFILE_DIR/flink.urls" "$ACTIVE_DIR/lib" + chmod -R a+rX "$ACTIVE_DIR" + +[doc("Start a profile from the profiles directory")] +up profile="core" nodes="1": ensure-dist ensure-flink + #!/usr/bin/env bash + set -e + PROFILE={{ quote(profile) }} + just validate-profile "$PROFILE" + case "{{ nodes }}" in + 1) TOPOLOGY=single ;; + 3) TOPOLOGY=cluster ;; + *) echo "nodes must be 1 or 3"; exit 1 ;; + esac + PROFILE_DIR="{{ PROFILES_DIR }}/$PROFILE" + FORMAT=$(just profile-format "$PROFILE") + COMPOSE_ARGS=(-f "{{ DEVKIT_DIR }}/docker-compose.yml" -f "{{ DEVKIT_DIR }}/compose/tiering.yml") + PROFILES=(--profile "$TOPOLOGY" --profile flink) + if [ -n "$FORMAT" ]; then + just ensure-tiering "$PROFILE" + PROFILES+=(--profile tiering) + if [ -f "$PROFILE_DIR/compose.files" ]; then + while IFS= read -r FILE; do + FILE="${FILE%%#*}" + FILE="${FILE//[[:space:]]/}" + [ -n "$FILE" ] || continue + if [ ! -f "{{ DEVKIT_DIR }}/$FILE" ]; then + echo "Compose file not found: $FILE" + exit 1 + fi + COMPOSE_ARGS+=(-f "{{ DEVKIT_DIR }}/$FILE") + done < "$PROFILE_DIR/compose.files" + fi + fi + export FLUSS_DEVKIT_CONFIG="$PROFILE_DIR/server.yaml" + export FLUSS_REPLICATION_FACTOR="{{ nodes }}" + {{ ALL_COMPOSE }} {{ ALL_PROFILES }} down --remove-orphans >/dev/null 2>&1 || true + docker compose "${COMPOSE_ARGS[@]}" "${PROFILES[@]}" up -d + just check "$PROFILE" + +[doc("Stop services and keep data")] +down: + @{{ ALL_COMPOSE }} {{ ALL_PROFILES }} down --remove-orphans + +[doc("Stop services and remove data")] +clean: + @{{ ALL_COMPOSE }} {{ ALL_PROFILES }} down -v --remove-orphans + +[doc("Show service status")] +status: + @{{ ALL_COMPOSE }} {{ ALL_PROFILES }} ps -a + +[doc("Show log output. Example: just logs tablet-server-0 200")] +logs service="" tail="100": + #!/usr/bin/env bash + if [ -z "{{ service }}" ]; then + {{ ALL_COMPOSE }} {{ ALL_PROFILES }} logs --tail="{{ tail }}" + else + {{ ALL_COMPOSE }} {{ ALL_PROFILES }} logs --tail="{{ tail }}" "{{ service }}" + fi + +[doc("Run a command in a service. Example: just exec tablet-server-0 java -version")] +exec service *CMD: + @{{ ALL_COMPOSE }} {{ ALL_PROFILES }} exec "{{ service }}" {{ CMD }} + +[private] +check profile: check-cluster check-flink + #!/usr/bin/env bash + FORMAT=$(just profile-format {{ quote(profile) }}) + if [ -n "$FORMAT" ]; then just check-tiering "$FORMAT"; fi + +[private] +check-cluster: + #!/usr/bin/env bash + set -e + ATTEMPT=0 + until {{ COMPOSE }} --profile single --profile cluster exec -T tablet-server-0 java -Xmx64m -classpath '/opt/fluss/lib/*' org.apache.fluss.server.tools.ClusterHealthReadinessCheck --host tablet-server-0 --port 9123 --timeoutMs 10000; do + ATTEMPT=$((ATTEMPT + 1)) + if [ "$ATTEMPT" -ge 20 ]; then + echo "Cluster health did not become ready" + exit 1 + fi + sleep 3 + done + +[private] +check-flink: + #!/usr/bin/env bash + set -e + for _ in {1..40}; do + if {{ COMPOSE_TIERING }} --profile flink exec -T jobmanager curl -fsS --max-time 5 http://localhost:8081/taskmanagers | grep -q '"id"'; then + exit 0 + fi + sleep 3 + done + {{ COMPOSE_TIERING }} --profile flink logs --tail=200 jobmanager taskmanager + echo "Flink cluster did not become ready" + exit 1 + +[private] +check-tiering format: + #!/usr/bin/env bash + set -e + for _ in {1..40}; do + JOB=$({{ COMPOSE_TIERING }} --profile flink --profile tiering exec -T jobmanager curl -fsS http://localhost:8081/jobs/overview | grep -o '"name":"Fluss Lake Tiering Service - {{ format }}"[^}]*' || true) + STATE=$(printf '%s' "$JOB" | sed -n 's/.*"state":"\([^"]*\)".*/\1/p') + case "$STATE" in + RUNNING) + echo "{{ format }} tiering job is running" + exit 0 + ;; + FAILING|FAILED|CANCELLING|CANCELED|FINISHED|RESTARTING|SUSPENDED) + {{ COMPOSE_TIERING }} --profile flink --profile tiering logs --tail=200 tiering-submit jobmanager taskmanager + echo "Tiering job entered state $STATE" + exit 1 + ;; + esac + sleep 3 + done + {{ COMPOSE_TIERING }} --profile flink --profile tiering logs --tail=200 tiering-submit jobmanager taskmanager + echo "Tiering job did not start" + exit 1 + +[private] +validate-table table: + #!/usr/bin/env bash + if [[ ! "{{ table }}" =~ ^[A-Za-z_][A-Za-z0-9_]*$ ]]; then + echo "table must match [A-Za-z_][A-Za-z0-9_]*" + exit 1 + fi + +[doc("Run a local SQL file with Flink SQL Client")] +run-sql file: + #!/usr/bin/env bash + set -e + FILE={{ quote(file) }} + if [[ "$FILE" != /* ]]; then + FILE={{ quote(invocation_directory()) }}/"$FILE" + fi + if [ ! -f "$FILE" ]; then + echo "SQL file not found: $FILE" + exit 1 + fi + set +e + OUTPUT=$({{ COMPOSE_TIERING }} --profile flink exec -T jobmanager /opt/flink/bin/sql-client.sh -f /dev/stdin < "$FILE" 2>&1) + STATUS=$? + set -e + printf '%s\n' "$OUTPUT" + if [ "$STATUS" -ne 0 ] || printf '%s\n' "$OUTPUT" | grep -Eq '\[ERROR\]|Could not execute SQL statement'; then + exit 1 + fi + +[private] +run-demo-sql file table marker="": + #!/usr/bin/env bash + set -e + just validate-table "{{ table }}" + set +e + OUTPUT=$({{ ALL_COMPOSE }} {{ ALL_PROFILES }} exec -T -e DEVKIT_TABLE="{{ table }}" jobmanager /bin/bash -c "envsubst '\${DEVKIT_TABLE}' < /opt/devkit-tests/{{ file }} > /tmp/{{ file }} && /opt/flink/bin/sql-client.sh -f /tmp/{{ file }}" 2>&1) + STATUS=$? + set -e + printf '%s\n' "$OUTPUT" + if [ "$STATUS" -ne 0 ] || printf '%s\n' "$OUTPUT" | grep -Eq '\[ERROR\]|Could not execute SQL statement'; then + exit 1 + fi + if [ -n "{{ marker }}" ] && ! printf '%s\n' "$OUTPUT" | grep -Fq "{{ marker }}"; then + echo "SQL command did not report expected marker: {{ marker }}" + exit 1 + fi + +[private] +run-format-sql operation format table marker="": + #!/usr/bin/env bash + just validate-profile "{{ format }}" + FILE="{{ format }}-{{ operation }}.sql" + if [ ! -f "{{ DEVKIT_DIR }}/tests/$FILE" ]; then + echo "{{ operation }} is not available for {{ format }}" + exit 1 + fi + just run-demo-sql "$FILE" "{{ table }}" "{{ marker }}" + +[doc("Create a lake-enabled Fluss table")] +create-table format table: + @just run-format-sql create-table "{{ format }}" "{{ table }}" + +[doc("Write three sample rows to a Fluss table")] +write-data format table: + @just run-format-sql write-data "{{ format }}" "{{ table }}" "Job ID:" + +[doc("Show running Flink jobs")] +tiering-status: + @{{ ALL_COMPOSE }} {{ ALL_PROFILES }} exec -T jobmanager /opt/flink/bin/flink list -r + +[doc("Query a table through its lake table")] +query-lake format table: + @just run-format-sql query-lake "{{ format }}" "{{ table }}" diff --git a/devkit/profiles/core/server.yaml b/devkit/profiles/core/server.yaml new file mode 100644 index 0000000000..a71718012e --- /dev/null +++ b/devkit/profiles/core/server.yaml @@ -0,0 +1,19 @@ +################################################################################ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +################################################################################ + +remote.data.dir: file:///tmp/fluss/remote-data diff --git a/devkit/profiles/hudi/compose.files b/devkit/profiles/hudi/compose.files new file mode 100644 index 0000000000..9619bb10bf --- /dev/null +++ b/devkit/profiles/hudi/compose.files @@ -0,0 +1,17 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +compose/lake-common.yml diff --git a/devkit/profiles/hudi/flink.urls b/devkit/profiles/hudi/flink.urls new file mode 100644 index 0000000000..dab108432e --- /dev/null +++ b/devkit/profiles/hudi/flink.urls @@ -0,0 +1,17 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +https://repo.maven.apache.org/maven2/io/trino/hadoop/hadoop-apache/3.3.5-2/hadoop-apache-3.3.5-2.jar diff --git a/devkit/profiles/hudi/jars.urls b/devkit/profiles/hudi/jars.urls new file mode 100644 index 0000000000..1d6ff80ffb --- /dev/null +++ b/devkit/profiles/hudi/jars.urls @@ -0,0 +1,17 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +https://repo.maven.apache.org/maven2/org/apache/hudi/hudi-flink1.20-bundle/1.1.0/hudi-flink1.20-bundle-1.1.0.jar diff --git a/devkit/profiles/hudi/server.urls b/devkit/profiles/hudi/server.urls new file mode 100644 index 0000000000..a00613c8fc --- /dev/null +++ b/devkit/profiles/hudi/server.urls @@ -0,0 +1,21 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +https://repo.maven.apache.org/maven2/io/trino/hadoop/hadoop-apache/3.3.5-2/hadoop-apache-3.3.5-2.jar +https://repo.maven.apache.org/maven2/org/apache/flink/flink-core/1.20.1/flink-core-1.20.1.jar +https://repo.maven.apache.org/maven2/org/apache/flink/flink-table-common/1.20.1/flink-table-common-1.20.1.jar +https://repo.maven.apache.org/maven2/org/apache/flink/flink-table-api-java/1.20.1/flink-table-api-java-1.20.1.jar +https://repo.maven.apache.org/maven2/org/apache/flink/flink-table-runtime/1.20.1/flink-table-runtime-1.20.1.jar diff --git a/devkit/profiles/hudi/server.yaml b/devkit/profiles/hudi/server.yaml new file mode 100644 index 0000000000..0e5a33d7cf --- /dev/null +++ b/devkit/profiles/hudi/server.yaml @@ -0,0 +1,41 @@ +################################################################################ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +################################################################################ + +remote.data.dir: s3://fluss/remote-data +s3.endpoint: http://rustfs:9000 +s3.access-key: rustfsadmin +s3.secret-key: rustfsadmin +s3.region: us-east-1 +s3.path-style-access: true +s3.assumed.role.arn: arn:aws:iam::000000000000:role/rustfsadmin +s3.assumed.role.sts.endpoint: http://rustfs:9000 + +datalake.enabled: true +datalake.format: hudi +datalake.hudi.mode: dfs +datalake.hudi.catalog.path: s3a://fluss/hudi +datalake.hudi.hadoop.fs.s3a.endpoint: http://rustfs:9000 +datalake.hudi.hadoop.fs.s3a.access.key: rustfsadmin +datalake.hudi.hadoop.fs.s3a.secret.key: rustfsadmin +datalake.hudi.hadoop.fs.s3a.path.style.access: true +datalake.hudi.hadoop.fs.s3a.connection.ssl.enabled: false +datalake.hudi.hadoop.fs.s3a.impl.disable.cache: true +datalake.hudi.hadoop.fs.s3a.aws.credentials.provider: org.apache.hadoop.fs.s3a.SimpleAWSCredentialsProvider +datalake.hudi.hoodie.write.lock.provider: org.apache.hudi.client.transaction.lock.InProcessLockProvider + +fluss.tiering.poll.table.interval: 1s diff --git a/devkit/profiles/iceberg/compose.files b/devkit/profiles/iceberg/compose.files new file mode 100644 index 0000000000..081f3d6812 --- /dev/null +++ b/devkit/profiles/iceberg/compose.files @@ -0,0 +1,18 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +compose/lake-common.yml +compose/iceberg.yml diff --git a/devkit/profiles/iceberg/flink.urls b/devkit/profiles/iceberg/flink.urls new file mode 100644 index 0000000000..9cd9964a6d --- /dev/null +++ b/devkit/profiles/iceberg/flink.urls @@ -0,0 +1,18 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +https://repo.maven.apache.org/maven2/io/trino/hadoop/hadoop-apache/3.3.5-2/hadoop-apache-3.3.5-2.jar +https://repo.maven.apache.org/maven2/org/apache/iceberg/iceberg-flink-runtime-1.20/1.10.1/iceberg-flink-runtime-1.20-1.10.1.jar diff --git a/devkit/profiles/iceberg/jars.urls b/devkit/profiles/iceberg/jars.urls new file mode 100644 index 0000000000..08339249be --- /dev/null +++ b/devkit/profiles/iceberg/jars.urls @@ -0,0 +1,20 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +https://repo.maven.apache.org/maven2/org/apache/iceberg/iceberg-aws/1.10.1/iceberg-aws-1.10.1.jar +https://repo.maven.apache.org/maven2/org/apache/iceberg/iceberg-aws-bundle/1.10.1/iceberg-aws-bundle-1.10.1.jar +https://repo.maven.apache.org/maven2/dev/failsafe/failsafe/3.3.2/failsafe-3.3.2.jar +https://repo.maven.apache.org/maven2/org/postgresql/postgresql/42.7.4/postgresql-42.7.4.jar diff --git a/devkit/profiles/iceberg/server.yaml b/devkit/profiles/iceberg/server.yaml new file mode 100644 index 0000000000..9ba4f1d7ba --- /dev/null +++ b/devkit/profiles/iceberg/server.yaml @@ -0,0 +1,41 @@ +################################################################################ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +################################################################################ + +remote.data.dir: s3://fluss/remote-data +s3.endpoint: http://rustfs:9000 +s3.access-key: rustfsadmin +s3.secret-key: rustfsadmin +s3.region: us-east-1 +s3.path-style-access: true +s3.assumed.role.arn: arn:aws:iam::000000000000:role/rustfsadmin +s3.assumed.role.sts.endpoint: http://rustfs:9000 + +datalake.enabled: true +datalake.format: iceberg +datalake.iceberg.catalog-impl: org.apache.iceberg.jdbc.JdbcCatalog +datalake.iceberg.name: fluss_catalog +datalake.iceberg.uri: jdbc:postgresql://postgres:5432/iceberg +datalake.iceberg.jdbc.user: iceberg +datalake.iceberg.jdbc.password: iceberg +datalake.iceberg.warehouse: s3://fluss/iceberg +datalake.iceberg.io-impl: org.apache.iceberg.aws.s3.S3FileIO +datalake.iceberg.s3.endpoint: http://rustfs:9000 +datalake.iceberg.s3.access-key-id: rustfsadmin +datalake.iceberg.s3.secret-access-key: rustfsadmin +datalake.iceberg.s3.path-style-access: true +datalake.iceberg.client.region: us-east-1 diff --git a/devkit/profiles/lance/compose.files b/devkit/profiles/lance/compose.files new file mode 100644 index 0000000000..04ef1ab87d --- /dev/null +++ b/devkit/profiles/lance/compose.files @@ -0,0 +1,18 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +compose/lake-common.yml +compose/lance.yml diff --git a/devkit/profiles/lance/server.yaml b/devkit/profiles/lance/server.yaml new file mode 100644 index 0000000000..691b56a33b --- /dev/null +++ b/devkit/profiles/lance/server.yaml @@ -0,0 +1,37 @@ +################################################################################ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +################################################################################ + +remote.data.dir: s3://fluss/remote-data +s3.endpoint: http://rustfs:9000 +s3.access-key: rustfsadmin +s3.secret-key: rustfsadmin +s3.region: us-east-1 +s3.path-style-access: true +s3.assumed.role.arn: arn:aws:iam::000000000000:role/rustfsadmin +s3.assumed.role.sts.endpoint: http://rustfs:9000 + +datalake.enabled: true +datalake.format: lance +datalake.lance.warehouse: s3://fluss/lance +datalake.lance.endpoint: http://rustfs:9000 +datalake.lance.allow_http: true +datalake.lance.region: us-east-1 +datalake.lance.access_key_id: rustfsadmin +datalake.lance.secret_access_key: rustfsadmin + +fluss.tiering.poll.table.interval: 1s diff --git a/devkit/profiles/paimon/compose.files b/devkit/profiles/paimon/compose.files new file mode 100644 index 0000000000..9619bb10bf --- /dev/null +++ b/devkit/profiles/paimon/compose.files @@ -0,0 +1,17 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +compose/lake-common.yml diff --git a/devkit/profiles/paimon/flink.urls b/devkit/profiles/paimon/flink.urls new file mode 100644 index 0000000000..7beac802f2 --- /dev/null +++ b/devkit/profiles/paimon/flink.urls @@ -0,0 +1,18 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +https://repo.maven.apache.org/maven2/io/trino/hadoop/hadoop-apache/3.3.5-2/hadoop-apache-3.3.5-2.jar +https://repo.maven.apache.org/maven2/org/apache/paimon/paimon-flink-1.20/1.3.1/paimon-flink-1.20-1.3.1.jar diff --git a/devkit/profiles/paimon/jars.urls b/devkit/profiles/paimon/jars.urls new file mode 100644 index 0000000000..83f6838ea8 --- /dev/null +++ b/devkit/profiles/paimon/jars.urls @@ -0,0 +1,17 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +https://repo.maven.apache.org/maven2/org/apache/paimon/paimon-s3/1.3.1/paimon-s3-1.3.1.jar diff --git a/devkit/profiles/paimon/server.yaml b/devkit/profiles/paimon/server.yaml new file mode 100644 index 0000000000..91cfc344b4 --- /dev/null +++ b/devkit/profiles/paimon/server.yaml @@ -0,0 +1,35 @@ +################################################################################ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +################################################################################ + +remote.data.dir: s3://fluss/remote-data +s3.endpoint: http://rustfs:9000 +s3.access-key: rustfsadmin +s3.secret-key: rustfsadmin +s3.region: us-east-1 +s3.path-style-access: true +s3.assumed.role.arn: arn:aws:iam::000000000000:role/rustfsadmin +s3.assumed.role.sts.endpoint: http://rustfs:9000 + +datalake.enabled: true +datalake.format: paimon +datalake.paimon.metastore: filesystem +datalake.paimon.warehouse: s3://fluss/paimon +datalake.paimon.s3.endpoint: http://rustfs:9000 +datalake.paimon.s3.access-key: rustfsadmin +datalake.paimon.s3.secret-key: rustfsadmin +datalake.paimon.s3.path.style.access: true diff --git a/devkit/tests/hudi-create-table.sql b/devkit/tests/hudi-create-table.sql new file mode 100644 index 0000000000..a12e01a658 --- /dev/null +++ b/devkit/tests/hudi-create-table.sql @@ -0,0 +1,32 @@ +-- Licensed to the Apache Software Foundation (ASF) under one +-- or more contributor license agreements. See the NOTICE file +-- distributed with this work for additional information +-- regarding copyright ownership. The ASF licenses this file +-- to you under the Apache License, Version 2.0 (the +-- "License"); you may not use this file except in compliance +-- with the License. You may obtain a copy of the License at +-- +-- http://www.apache.org/licenses/LICENSE-2.0 +-- +-- Unless required by applicable law or agreed to in writing, software +-- distributed under the License is distributed on an "AS IS" BASIS, +-- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +-- See the License for the specific language governing permissions and +-- limitations under the License. + +CREATE CATALOG fluss_catalog WITH ( + 'type' = 'fluss', + 'bootstrap.servers' = 'coordinator-server:19123' +); + +USE CATALOG fluss_catalog; + +CREATE TABLE ${DEVKIT_TABLE} ( + id BIGINT, + payload STRING +) WITH ( + 'bucket.num' = '1', + 'table.datalake.enabled' = 'true', + 'table.datalake.freshness' = '5s', + 'hudi.hoodie.datasource.write.recordkey.field' = 'id' +); diff --git a/devkit/tests/hudi-query-lake.sql b/devkit/tests/hudi-query-lake.sql new file mode 100644 index 0000000000..0318d87a24 --- /dev/null +++ b/devkit/tests/hudi-query-lake.sql @@ -0,0 +1,40 @@ +-- Licensed to the Apache Software Foundation (ASF) under one +-- or more contributor license agreements. See the NOTICE file +-- distributed with this work for additional information +-- regarding copyright ownership. The ASF licenses this file +-- to you under the Apache License, Version 2.0 (the +-- "License"); you may not use this file except in compliance +-- with the License. You may obtain a copy of the License at +-- +-- http://www.apache.org/licenses/LICENSE-2.0 +-- +-- Unless required by applicable law or agreed to in writing, software +-- distributed under the License is distributed on an "AS IS" BASIS, +-- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +-- See the License for the specific language governing permissions and +-- limitations under the License. + +SET 'execution.runtime-mode' = 'batch'; +SET 'sql-client.execution.result-mode' = 'tableau'; + +CREATE CATALOG hudi_catalog WITH ( + 'type' = 'hudi', + 'mode' = 'dfs', + 'catalog.path' = 's3a://fluss/hudi', + 'hadoop.fs.s3a.endpoint' = 'http://rustfs:9000', + 'hadoop.fs.s3a.access.key' = 'rustfsadmin', + 'hadoop.fs.s3a.secret.key' = 'rustfsadmin', + 'hadoop.fs.s3a.path.style.access' = 'true', + 'hadoop.fs.s3a.connection.ssl.enabled' = 'false', + 'hadoop.fs.s3a.impl.disable.cache' = 'true' +); + +USE CATALOG hudi_catalog; + +SELECT + COUNT(*) AS row_count, + SUM(id) AS id_sum, + COUNT(DISTINCT payload) AS payload_count, + MIN(payload) AS first_payload, + MAX(payload) AS last_payload +FROM fluss.${DEVKIT_TABLE}; diff --git a/devkit/tests/hudi-write-data.sql b/devkit/tests/hudi-write-data.sql new file mode 100644 index 0000000000..70a924cad8 --- /dev/null +++ b/devkit/tests/hudi-write-data.sql @@ -0,0 +1,27 @@ +-- Licensed to the Apache Software Foundation (ASF) under one +-- or more contributor license agreements. See the NOTICE file +-- distributed with this work for additional information +-- regarding copyright ownership. The ASF licenses this file +-- to you under the Apache License, Version 2.0 (the +-- "License"); you may not use this file except in compliance +-- with the License. You may obtain a copy of the License at +-- +-- http://www.apache.org/licenses/LICENSE-2.0 +-- +-- Unless required by applicable law or agreed to in writing, software +-- distributed under the License is distributed on an "AS IS" BASIS, +-- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +-- See the License for the specific language governing permissions and +-- limitations under the License. + +CREATE CATALOG fluss_catalog WITH ( + 'type' = 'fluss', + 'bootstrap.servers' = 'coordinator-server:19123' +); + +USE CATALOG fluss_catalog; + +INSERT INTO ${DEVKIT_TABLE} VALUES + (1, 'alpha'), + (2, 'beta'), + (3, 'gamma'); diff --git a/devkit/tests/iceberg-create-table.sql b/devkit/tests/iceberg-create-table.sql new file mode 100644 index 0000000000..064478ed36 --- /dev/null +++ b/devkit/tests/iceberg-create-table.sql @@ -0,0 +1,34 @@ +-- Licensed to the Apache Software Foundation (ASF) under one +-- or more contributor license agreements. See the NOTICE file +-- distributed with this work for additional information +-- regarding copyright ownership. The ASF licenses this file +-- to you under the Apache License, Version 2.0 (the +-- "License"); you may not use this file except in compliance +-- with the License. You may obtain a copy of the License at +-- +-- http://www.apache.org/licenses/LICENSE-2.0 +-- +-- Unless required by applicable law or agreed to in writing, software +-- distributed under the License is distributed on an "AS IS" BASIS, +-- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +-- See the License for the specific language governing permissions and +-- limitations under the License. + +CREATE CATALOG fluss_catalog WITH ( + 'type' = 'fluss', + 'bootstrap.servers' = 'coordinator-server:19123', + 'iceberg.jdbc.password' = 'iceberg', + 'iceberg.s3.access-key-id' = 'rustfsadmin', + 'iceberg.s3.secret-access-key' = 'rustfsadmin' +); + +USE CATALOG fluss_catalog; + +CREATE TABLE ${DEVKIT_TABLE} ( + id BIGINT, + payload STRING +) WITH ( + 'bucket.num' = '1', + 'table.datalake.enabled' = 'true', + 'table.datalake.freshness' = '5s' +); diff --git a/devkit/tests/iceberg-query-lake.sql b/devkit/tests/iceberg-query-lake.sql new file mode 100644 index 0000000000..f9be90a218 --- /dev/null +++ b/devkit/tests/iceberg-query-lake.sql @@ -0,0 +1,36 @@ +-- Licensed to the Apache Software Foundation (ASF) under one +-- or more contributor license agreements. See the NOTICE file +-- distributed with this work for additional information +-- regarding copyright ownership. The ASF licenses this file +-- to you under the Apache License, Version 2.0 (the +-- "License"); you may not use this file except in compliance +-- with the License. You may obtain a copy of the License at +-- +-- http://www.apache.org/licenses/LICENSE-2.0 +-- +-- Unless required by applicable law or agreed to in writing, software +-- distributed under the License is distributed on an "AS IS" BASIS, +-- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +-- See the License for the specific language governing permissions and +-- limitations under the License. + +SET 'execution.runtime-mode' = 'batch'; +SET 'sql-client.execution.result-mode' = 'tableau'; + +CREATE CATALOG fluss_catalog WITH ( + 'type' = 'fluss', + 'bootstrap.servers' = 'coordinator-server:19123', + 'iceberg.jdbc.password' = 'iceberg', + 'iceberg.s3.access-key-id' = 'rustfsadmin', + 'iceberg.s3.secret-access-key' = 'rustfsadmin' +); + +USE CATALOG fluss_catalog; + +SELECT + COUNT(*) AS row_count, + SUM(id) AS id_sum, + COUNT(DISTINCT payload) AS payload_count, + MIN(payload) AS first_payload, + MAX(payload) AS last_payload +FROM `${DEVKIT_TABLE}$lake`; diff --git a/devkit/tests/iceberg-write-data.sql b/devkit/tests/iceberg-write-data.sql new file mode 100644 index 0000000000..b8b823e981 --- /dev/null +++ b/devkit/tests/iceberg-write-data.sql @@ -0,0 +1,30 @@ +-- Licensed to the Apache Software Foundation (ASF) under one +-- or more contributor license agreements. See the NOTICE file +-- distributed with this work for additional information +-- regarding copyright ownership. The ASF licenses this file +-- to you under the Apache License, Version 2.0 (the +-- "License"); you may not use this file except in compliance +-- with the License. You may obtain a copy of the License at +-- +-- http://www.apache.org/licenses/LICENSE-2.0 +-- +-- Unless required by applicable law or agreed to in writing, software +-- distributed under the License is distributed on an "AS IS" BASIS, +-- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +-- See the License for the specific language governing permissions and +-- limitations under the License. + +CREATE CATALOG fluss_catalog WITH ( + 'type' = 'fluss', + 'bootstrap.servers' = 'coordinator-server:19123', + 'iceberg.jdbc.password' = 'iceberg', + 'iceberg.s3.access-key-id' = 'rustfsadmin', + 'iceberg.s3.secret-access-key' = 'rustfsadmin' +); + +USE CATALOG fluss_catalog; + +INSERT INTO ${DEVKIT_TABLE} VALUES + (1, 'alpha'), + (2, 'beta'), + (3, 'gamma'); diff --git a/devkit/tests/lance-create-table.sql b/devkit/tests/lance-create-table.sql new file mode 100644 index 0000000000..82b8876f82 --- /dev/null +++ b/devkit/tests/lance-create-table.sql @@ -0,0 +1,31 @@ +-- Licensed to the Apache Software Foundation (ASF) under one +-- or more contributor license agreements. See the NOTICE file +-- distributed with this work for additional information +-- regarding copyright ownership. The ASF licenses this file +-- to you under the Apache License, Version 2.0 (the +-- "License"); you may not use this file except in compliance +-- with the License. You may obtain a copy of the License at +-- +-- http://www.apache.org/licenses/LICENSE-2.0 +-- +-- Unless required by applicable law or agreed to in writing, software +-- distributed under the License is distributed on an "AS IS" BASIS, +-- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +-- See the License for the specific language governing permissions and +-- limitations under the License. + +CREATE CATALOG fluss_catalog WITH ( + 'type' = 'fluss', + 'bootstrap.servers' = 'coordinator-server:19123' +); + +USE CATALOG fluss_catalog; + +CREATE TABLE ${DEVKIT_TABLE} ( + id BIGINT, + payload STRING +) WITH ( + 'bucket.num' = '1', + 'table.datalake.enabled' = 'true', + 'table.datalake.freshness' = '5s' +); diff --git a/devkit/tests/lance-write-data.sql b/devkit/tests/lance-write-data.sql new file mode 100644 index 0000000000..70a924cad8 --- /dev/null +++ b/devkit/tests/lance-write-data.sql @@ -0,0 +1,27 @@ +-- Licensed to the Apache Software Foundation (ASF) under one +-- or more contributor license agreements. See the NOTICE file +-- distributed with this work for additional information +-- regarding copyright ownership. The ASF licenses this file +-- to you under the Apache License, Version 2.0 (the +-- "License"); you may not use this file except in compliance +-- with the License. You may obtain a copy of the License at +-- +-- http://www.apache.org/licenses/LICENSE-2.0 +-- +-- Unless required by applicable law or agreed to in writing, software +-- distributed under the License is distributed on an "AS IS" BASIS, +-- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +-- See the License for the specific language governing permissions and +-- limitations under the License. + +CREATE CATALOG fluss_catalog WITH ( + 'type' = 'fluss', + 'bootstrap.servers' = 'coordinator-server:19123' +); + +USE CATALOG fluss_catalog; + +INSERT INTO ${DEVKIT_TABLE} VALUES + (1, 'alpha'), + (2, 'beta'), + (3, 'gamma'); diff --git a/devkit/tests/paimon-create-table.sql b/devkit/tests/paimon-create-table.sql new file mode 100644 index 0000000000..c3d2110f7e --- /dev/null +++ b/devkit/tests/paimon-create-table.sql @@ -0,0 +1,33 @@ +-- Licensed to the Apache Software Foundation (ASF) under one +-- or more contributor license agreements. See the NOTICE file +-- distributed with this work for additional information +-- regarding copyright ownership. The ASF licenses this file +-- to you under the Apache License, Version 2.0 (the +-- "License"); you may not use this file except in compliance +-- with the License. You may obtain a copy of the License at +-- +-- http://www.apache.org/licenses/LICENSE-2.0 +-- +-- Unless required by applicable law or agreed to in writing, software +-- distributed under the License is distributed on an "AS IS" BASIS, +-- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +-- See the License for the specific language governing permissions and +-- limitations under the License. + +CREATE CATALOG fluss_catalog WITH ( + 'type' = 'fluss', + 'bootstrap.servers' = 'coordinator-server:19123', + 'paimon.s3.access-key' = 'rustfsadmin', + 'paimon.s3.secret-key' = 'rustfsadmin' +); + +USE CATALOG fluss_catalog; + +CREATE TABLE ${DEVKIT_TABLE} ( + id BIGINT, + payload STRING +) WITH ( + 'bucket.num' = '1', + 'table.datalake.enabled' = 'true', + 'table.datalake.freshness' = '5s' +); diff --git a/devkit/tests/paimon-query-lake.sql b/devkit/tests/paimon-query-lake.sql new file mode 100644 index 0000000000..491c9feefe --- /dev/null +++ b/devkit/tests/paimon-query-lake.sql @@ -0,0 +1,35 @@ +-- Licensed to the Apache Software Foundation (ASF) under one +-- or more contributor license agreements. See the NOTICE file +-- distributed with this work for additional information +-- regarding copyright ownership. The ASF licenses this file +-- to you under the Apache License, Version 2.0 (the +-- "License"); you may not use this file except in compliance +-- with the License. You may obtain a copy of the License at +-- +-- http://www.apache.org/licenses/LICENSE-2.0 +-- +-- Unless required by applicable law or agreed to in writing, software +-- distributed under the License is distributed on an "AS IS" BASIS, +-- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +-- See the License for the specific language governing permissions and +-- limitations under the License. + +SET 'execution.runtime-mode' = 'batch'; +SET 'sql-client.execution.result-mode' = 'tableau'; + +CREATE CATALOG fluss_catalog WITH ( + 'type' = 'fluss', + 'bootstrap.servers' = 'coordinator-server:19123', + 'paimon.s3.access-key' = 'rustfsadmin', + 'paimon.s3.secret-key' = 'rustfsadmin' +); + +USE CATALOG fluss_catalog; + +SELECT + COUNT(*) AS row_count, + SUM(id) AS id_sum, + COUNT(DISTINCT payload) AS payload_count, + MIN(payload) AS first_payload, + MAX(payload) AS last_payload +FROM `${DEVKIT_TABLE}$lake`; diff --git a/devkit/tests/paimon-write-data.sql b/devkit/tests/paimon-write-data.sql new file mode 100644 index 0000000000..1248e51281 --- /dev/null +++ b/devkit/tests/paimon-write-data.sql @@ -0,0 +1,29 @@ +-- Licensed to the Apache Software Foundation (ASF) under one +-- or more contributor license agreements. See the NOTICE file +-- distributed with this work for additional information +-- regarding copyright ownership. The ASF licenses this file +-- to you under the Apache License, Version 2.0 (the +-- "License"); you may not use this file except in compliance +-- with the License. You may obtain a copy of the License at +-- +-- http://www.apache.org/licenses/LICENSE-2.0 +-- +-- Unless required by applicable law or agreed to in writing, software +-- distributed under the License is distributed on an "AS IS" BASIS, +-- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +-- See the License for the specific language governing permissions and +-- limitations under the License. + +CREATE CATALOG fluss_catalog WITH ( + 'type' = 'fluss', + 'bootstrap.servers' = 'coordinator-server:19123', + 'paimon.s3.access-key' = 'rustfsadmin', + 'paimon.s3.secret-key' = 'rustfsadmin' +); + +USE CATALOG fluss_catalog; + +INSERT INTO ${DEVKIT_TABLE} VALUES + (1, 'alpha'), + (2, 'beta'), + (3, 'gamma'); diff --git a/fluss-lake/fluss-lake-hudi/src/main/java/org/apache/fluss/lake/hudi/tiering/HudiWriteTableInfo.java b/fluss-lake/fluss-lake-hudi/src/main/java/org/apache/fluss/lake/hudi/tiering/HudiWriteTableInfo.java index d4711f261a..7c76878ce9 100644 --- a/fluss-lake/fluss-lake-hudi/src/main/java/org/apache/fluss/lake/hudi/tiering/HudiWriteTableInfo.java +++ b/fluss-lake/fluss-lake-hudi/src/main/java/org/apache/fluss/lake/hudi/tiering/HudiWriteTableInfo.java @@ -140,7 +140,10 @@ public static HudiWriteTableInfo create( HoodieEngineContext engineContext = new HoodieLocalEngineContext(metaClient.getStorageConf()); HoodieFlinkWriteClient writeClient = - new HoodieFlinkWriteClient<>(HoodieFlinkEngineContext.DEFAULT, writeConfig); + new HoodieFlinkWriteClient<>( + new HoodieFlinkEngineContext( + HadoopConfigurations.getHadoopConf(flinkConfig)), + writeConfig); Schema avroSchema = Schema.parse(writeConfig.getSchema()); RowType rowType = (RowType) AvroSchemaConverter.convertToDataType(avroSchema).getLogicalType();