diff --git a/devkit/.gitignore b/devkit/.gitignore new file mode 100644 index 0000000000..321b2a2a67 --- /dev/null +++ b/devkit/.gitignore @@ -0,0 +1 @@ +.deps/ diff --git a/devkit/README.md b/devkit/README.md new file mode 100644 index 0000000000..ab069d5fab --- /dev/null +++ b/devkit/README.md @@ -0,0 +1,140 @@ +# Fluss DevKit + +Fluss DevKit starts a local Fluss environment with Docker Compose and `just`. It uses upstream +runtime images and mounts the local `../build-target` directory read-only, so testing source changes +never requires building a Docker image. + +## Requirements + +- JDK 11 or later +- Docker with Docker Compose +- [just](https://github.com/casey/just) + +## Quick Start + +Build and start the core development environment: + +```bash +cd devkit +just build +just up +``` + +For a lake profile, build the tiering artifacts and select a format: + +```bash +just build-tiering +just up iceberg +``` + +`just up` waits for the Fluss and Flink clusters and, for lake profiles, the tiering job to become +ready. Run `just --list` to see all available commands. + +## Profiles + +| Profile | Services | +|---|---| +| `core` | ZooKeeper, Fluss, and Flink | +| `iceberg` | Core, RustFS, PostgreSQL JDBC Catalog, Flink, and Iceberg tiering | +| `paimon` | Core, RustFS, Flink, and Paimon tiering | +| `lance` | Core, RustFS, Flink, and Lance tiering | +| `hudi` | Core, RustFS, Flink, and Hudi tiering | + +Profiles start one TabletServer by default. Pass `3` as the second argument to start three: + +```bash +just up core 3 +just up paimon 3 +``` + +## Validate Lake Tiering + +The validation workflow is intentionally split into commands that can be run and inspected one at +a time: + +```bash +just build-tiering +just up paimon + +format=paimon +table="${format}_manual_$(date +%s)" + +just create-table "$format" "$table" +just write-data "$format" "$table" +just tiering-status +just query-lake "$format" "$table" +``` + +Use `iceberg`, `paimon`, or `hudi` for a complete create, write, tier, and lake-query workflow. +Tiering is asynchronous, so repeat `query-lake` if the first query runs before the lake commit is +visible. + +The sample writes three rows. A successful lake query reports row count `3`, ID sum `6`, three +distinct payloads, and the payload range `alpha` to `gamma`. Use a new table name for each run: +dropping a non-empty Fluss table does not remove its lake table. + +Lance supports `create-table`, `write-data`, and tiering, but the DevKit does not bundle a Flink SQL +reader for `query-lake`. Inspect the generated objects in RustFS when validating Lance. + +## Operations + +```bash +just status +just logs +just logs tablet-server-0 200 +just exec tablet-server-0 java -version +just run-sql ./my-query.sql +just down +just clean +``` + +Start a profile with `just up` before using `run-sql`. SQL files are read from the host and passed to +the Flink SQL Client. Relative paths are resolved from the current working directory. + +`just up` replaces containers from the previous profile but preserves named volumes. `down` removes +containers and keeps data; `clean` also removes Compose volumes. Downloaded JARs remain in the +ignored `.deps` cache. + +Default endpoints: + +| Service | Address | +|---|---| +| CoordinatorServer | `localhost:9123` | +| TabletServer 0 | `localhost:9124` | +| TabletServers 1 and 2 | `localhost:9125` and `localhost:9126` (three-node mode) | +| CoordinatorServer JDWP | `localhost:15005` | +| TabletServer JDWP | `localhost:15006` to `localhost:15008` | +| CoordinatorServer metrics | `http://localhost:9249/metrics` | +| TabletServer metrics | `http://localhost:9250/metrics` to `http://localhost:9252/metrics` | +| Flink UI | `http://localhost:8083` | +| RustFS S3 API | `http://localhost:9000` | +| RustFS Console | `http://localhost:9001` | + +JDWP is enabled for every Fluss process with `suspend=n`, so a remote debugger can attach without +blocking startup. Prometheus export is also enabled by default and can be checked directly, for +example with `curl http://localhost:9249/metrics`. + +The default Fluss runtime is `eclipse-temurin:17-jre-noble`; set `FLUSS_DEVKIT_IMAGE` to use +another Java 17 runtime. Port environment variables in the Compose files can override the default +addresses, for example `COORDINATOR_DEBUG_PORT=5005 just up` or +`TABLET_SERVER_0_METRICS_PORT=9300 just up`. + +## Adding a Profile + +Create a directory under `profiles/`. The directory contains one required file and up to four +optional files: + +| File | Purpose | +|---|---| +| `server.yaml` | Fluss and lake configuration | +| `jars.urls` | JARs used by both Fluss Server and Flink | +| `server.urls` | Additional Fluss Server-only JARs | +| `flink.urls` | Additional Flink-only JARs | +| `compose.files` | Compose overlays, relative to `devkit/`, for extra local services | + +Put one URL or Compose path on each line; empty lines and `#` comments are ignored. If `server.yaml` +contains `datalake.format`, `just up` automatically stages the matching locally built lake plugin +and starts Flink tiering. Use `compose.files` only when the profile needs additional local services +or runtime overrides. + +Start the new profile with `just up `. No `justfile` change is required. diff --git a/devkit/compose/iceberg.yml b/devkit/compose/iceberg.yml new file mode 100644 index 0000000000..27cb9bcb10 --- /dev/null +++ b/devkit/compose/iceberg.yml @@ -0,0 +1,40 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +services: + postgres: + image: postgres:17 + restart: unless-stopped + environment: + POSTGRES_USER: iceberg + POSTGRES_PASSWORD: iceberg + POSTGRES_DB: iceberg + volumes: + - iceberg-postgres-data:/var/lib/postgresql/data + healthcheck: + test: ["CMD-SHELL", "pg_isready -U iceberg -d iceberg"] + interval: 3s + timeout: 3s + retries: 20 + start_period: 5s + + coordinator-server: + depends_on: + postgres: + condition: service_healthy + +volumes: + iceberg-postgres-data: diff --git a/devkit/compose/lake-common.yml b/devkit/compose/lake-common.yml new file mode 100644 index 0000000000..13821f797d --- /dev/null +++ b/devkit/compose/lake-common.yml @@ -0,0 +1,62 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +services: + rustfs: + image: rustfs/rustfs:1.0.0-alpha.83 + restart: unless-stopped + command: /data + environment: + RUSTFS_ACCESS_KEY: rustfsadmin + RUSTFS_SECRET_KEY: rustfsadmin + RUSTFS_CONSOLE_ENABLE: "true" + ports: + - "${RUSTFS_API_PORT:-9000}:9000" + - "${RUSTFS_CONSOLE_PORT:-9001}:9001" + volumes: + - rustfs-data:/data + healthcheck: + test: ["CMD-SHELL", "nc -z 127.0.0.1 9000"] + interval: 5s + timeout: 5s + retries: 20 + start_period: 5s + + rustfs-init: + image: minio/mc + depends_on: + rustfs: + condition: service_healthy + entrypoint: + - /bin/sh + - -c + - | + until mc alias set rustfs http://rustfs:9000 rustfsadmin rustfsadmin; do sleep 2; done + mc mb --ignore-existing rustfs/fluss + printf '' | mc pipe rustfs/fluss/hudi/.keep + + coordinator-server: + depends_on: + rustfs-init: + condition: service_completed_successfully + + jobmanager: + depends_on: + rustfs-init: + condition: service_completed_successfully + +volumes: + rustfs-data: diff --git a/devkit/compose/lance.yml b/devkit/compose/lance.yml new file mode 100644 index 0000000000..efd52061c2 --- /dev/null +++ b/devkit/compose/lance.yml @@ -0,0 +1,24 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +services: + taskmanager: + environment: + FLINK_PROPERTIES: | + jobmanager.rpc.address: jobmanager + taskmanager.memory.process.size: 2048m + taskmanager.memory.task.off-heap.size: 512m + taskmanager.numberOfTaskSlots: 4 diff --git a/devkit/compose/tiering.yml b/devkit/compose/tiering.yml new file mode 100644 index 0000000000..61ead3a446 --- /dev/null +++ b/devkit/compose/tiering.yml @@ -0,0 +1,114 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +x-flink-common: &flink-common + image: flink:1.20.0-scala_2.12-java17 + entrypoint: + - /bin/bash + - -c + - | + cp /opt/devkit-jars/lib/*.jar /opt/flink/lib/ + chmod a+r /opt/flink/lib/*.jar + exec /docker-entrypoint.sh "$$@" + - -- + volumes: + - ./.deps/flink/active:/opt/devkit-jars:ro + - ${FLUSS_DEVKIT_CONFIG:-./profiles/core/server.yaml}:/opt/devkit-config/server.yaml:ro + - ./tests:/opt/devkit-tests:ro + - shared-data:/tmp/fluss + +services: + jobmanager: + <<: *flink-common + profiles: [flink] + command: ["jobmanager"] + depends_on: + coordinator-server: + condition: service_healthy + environment: + FLINK_PROPERTIES: | + jobmanager.rpc.address: jobmanager + jobmanager.memory.process.size: 1024m + parallelism.default: 1 + rest.address: jobmanager + ports: + - "${FLINK_REST_PORT:-8083}:8081" + healthcheck: + test: ["CMD-SHELL", "curl -fsS http://localhost:8081/overview >/dev/null"] + interval: 5s + timeout: 5s + retries: 24 + start_period: 10s + + taskmanager: + <<: *flink-common + profiles: [flink] + command: ["taskmanager"] + depends_on: + jobmanager: + condition: service_healthy + environment: + FLINK_PROPERTIES: | + jobmanager.rpc.address: jobmanager + taskmanager.memory.process.size: 1536m + taskmanager.numberOfTaskSlots: 4 + healthcheck: + test: + - CMD-SHELL + - curl -fsS http://jobmanager:8081/taskmanagers | grep -q '"id"' + interval: 5s + timeout: 5s + retries: 24 + start_period: 10s + + tiering-submit: + <<: *flink-common + profiles: [tiering] + restart: "no" + depends_on: + taskmanager: + condition: service_healthy + environment: + FLINK_PROPERTIES: | + jobmanager.rpc.address: jobmanager + rest.address: jobmanager + command: + - /bin/bash + - -c + - | + ARGS=() + HAS_BOOTSTRAP=false + while IFS= read -r LINE || [ -n "$$LINE" ]; do + LINE="$${LINE%%#*}" + [[ "$$LINE" == *": "* ]] || continue + KEY="$${LINE%%: *}" + VALUE="$${LINE#*: }" + KEY="$${KEY#"$${KEY%%[![:space:]]*}"}" + KEY="$${KEY%"$${KEY##*[![:space:]]}"}" + VALUE="$${VALUE#"$${VALUE%%[![:space:]]*}"}" + VALUE="$${VALUE%"$${VALUE##*[![:space:]]}"}" + case "$$KEY" in + datalake.*|lake.tiering.*|config.providers*|fluss.*) + ARGS+=("--$$KEY" "$$VALUE") + [ "$$KEY" = "fluss.bootstrap.servers" ] && HAS_BOOTSTRAP=true + ;; + esac + done < /opt/devkit-config/server.yaml + if [ "$$HAS_BOOTSTRAP" = false ]; then + ARGS+=(--fluss.bootstrap.servers coordinator-server:19123) + fi + exec /opt/flink/bin/flink run -d \ + /opt/devkit-jars/jobs/fluss-flink-tiering.jar "$${ARGS[@]}" diff --git a/devkit/docker-compose.yml b/devkit/docker-compose.yml new file mode 100644 index 0000000000..035a0b96df --- /dev/null +++ b/devkit/docker-compose.yml @@ -0,0 +1,170 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +name: fluss-devkit + +x-fluss-common: &fluss-common + image: ${FLUSS_DEVKIT_IMAGE:-eclipse-temurin:17-jre-noble} + init: true + restart: unless-stopped + stop_grace_period: 2m + volumes: + - ../build-target:/opt/fluss:ro + - ${FLUSS_DEVKIT_CONFIG:-./profiles/core/server.yaml}:/opt/fluss/conf/server.yaml:ro + - shared-data:/tmp/fluss + environment: + FLUSS_LOG_DIR: /tmp/fluss-logs + logging: + driver: json-file + options: + max-size: 10m + max-file: "3" + +x-tablet-common: &tablet-common + <<: *fluss-common + entrypoint: ["/opt/fluss/bin/tablet-server.sh", "start-foreground"] + depends_on: + coordinator-server: + condition: service_healthy + environment: + FLUSS_LOG_DIR: /tmp/fluss-logs + READINESS_TCP_PORT: "9123" + READINESS_HEALTH_CHECK_TIMEOUT_SECONDS: "180" + healthcheck: + test: ["CMD-SHELL", "bash /opt/fluss/bin/readiness-check.sh"] + interval: 5s + timeout: 15s + retries: 36 + start_period: 10s + +services: + zookeeper: + image: zookeeper:3.9.2 + restart: unless-stopped + healthcheck: + test: ["CMD-SHELL", "zkServer.sh status"] + interval: 5s + timeout: 5s + retries: 20 + start_period: 5s + volumes: + - zookeeper-data:/data + - zookeeper-log:/datalog + + coordinator-server: + <<: *fluss-common + entrypoint: ["/opt/fluss/bin/coordinator-server.sh", "start-foreground"] + depends_on: + zookeeper: + condition: service_healthy + command: + - -Dzookeeper.address=zookeeper:2181 + - -Dbind.listeners=INTERNAL://coordinator-server:0,CLIENT://coordinator-server:9123,DEVKIT://coordinator-server:19123 + - -Dadvertised.listeners=CLIENT://localhost:${COORDINATOR_PORT:-9123} + - -Dinternal.listener.name=INTERNAL + - -Ddefault.replication.factor=${FLUSS_REPLICATION_FACTOR:-1} + - -Dmetrics.reporters=prometheus + - -Dmetrics.reporter.prometheus.port=9249 + environment: + FLUSS_LOG_DIR: /tmp/fluss-logs + FLUSS_ENV_JAVA_OPTS: -Xms512m -Xmx512m -agentlib:jdwp=transport=dt_socket,server=y,suspend=n,address=*:5005 + ports: + - "${COORDINATOR_PORT:-9123}:9123" + - "${COORDINATOR_DEBUG_PORT:-15005}:5005" + - "${COORDINATOR_METRICS_PORT:-9249}:9249" + healthcheck: + test: ["CMD-SHELL", "bash -c 'echo > /dev/tcp/coordinator-server/9123'"] + interval: 5s + timeout: 5s + retries: 24 + start_period: 10s + + tablet-server-0: + <<: *tablet-common + profiles: [single, cluster] + command: + - -Dzookeeper.address=zookeeper:2181 + - -Dbind.listeners=INTERNAL://tablet-server-0:0,CLIENT://tablet-server-0:9123,DEVKIT://tablet-server-0:19123 + - -Dadvertised.listeners=CLIENT://localhost:${TABLET_SERVER_0_PORT:-9124} + - -Dinternal.listener.name=INTERNAL + - -Dtablet-server.id=0 + - -Dkv.snapshot.interval=0s + - -Ddata.dir=/tmp/fluss/data/tablet-server-0 + - -Dmetrics.reporters=prometheus + - -Dmetrics.reporter.prometheus.port=9249 + environment: + FLUSS_LOG_DIR: /tmp/fluss-logs + FLUSS_ENV_JAVA_OPTS: -Xms512m -Xmx512m -agentlib:jdwp=transport=dt_socket,server=y,suspend=n,address=*:5005 + READINESS_TCP_HOST: tablet-server-0 + READINESS_TCP_PORT: "9123" + READINESS_HEALTH_CHECK_TIMEOUT_SECONDS: "180" + ports: + - "${TABLET_SERVER_0_PORT:-9124}:9123" + - "${TABLET_SERVER_0_DEBUG_PORT:-15006}:5005" + - "${TABLET_SERVER_0_METRICS_PORT:-9250}:9249" + + tablet-server-1: + <<: *tablet-common + profiles: [cluster] + command: + - -Dzookeeper.address=zookeeper:2181 + - -Dbind.listeners=INTERNAL://tablet-server-1:0,CLIENT://tablet-server-1:9123,DEVKIT://tablet-server-1:19123 + - -Dadvertised.listeners=CLIENT://localhost:${TABLET_SERVER_1_PORT:-9125} + - -Dinternal.listener.name=INTERNAL + - -Dtablet-server.id=1 + - -Dkv.snapshot.interval=0s + - -Ddata.dir=/tmp/fluss/data/tablet-server-1 + - -Dmetrics.reporters=prometheus + - -Dmetrics.reporter.prometheus.port=9249 + environment: + FLUSS_LOG_DIR: /tmp/fluss-logs + FLUSS_ENV_JAVA_OPTS: -Xms512m -Xmx512m -agentlib:jdwp=transport=dt_socket,server=y,suspend=n,address=*:5005 + READINESS_TCP_HOST: tablet-server-1 + READINESS_TCP_PORT: "9123" + READINESS_HEALTH_CHECK_TIMEOUT_SECONDS: "180" + ports: + - "${TABLET_SERVER_1_PORT:-9125}:9123" + - "${TABLET_SERVER_1_DEBUG_PORT:-15007}:5005" + - "${TABLET_SERVER_1_METRICS_PORT:-9251}:9249" + + tablet-server-2: + <<: *tablet-common + profiles: [cluster] + command: + - -Dzookeeper.address=zookeeper:2181 + - -Dbind.listeners=INTERNAL://tablet-server-2:0,CLIENT://tablet-server-2:9123,DEVKIT://tablet-server-2:19123 + - -Dadvertised.listeners=CLIENT://localhost:${TABLET_SERVER_2_PORT:-9126} + - -Dinternal.listener.name=INTERNAL + - -Dtablet-server.id=2 + - -Dkv.snapshot.interval=0s + - -Ddata.dir=/tmp/fluss/data/tablet-server-2 + - -Dmetrics.reporters=prometheus + - -Dmetrics.reporter.prometheus.port=9249 + environment: + FLUSS_LOG_DIR: /tmp/fluss-logs + FLUSS_ENV_JAVA_OPTS: -Xms512m -Xmx512m -agentlib:jdwp=transport=dt_socket,server=y,suspend=n,address=*:5005 + READINESS_TCP_HOST: tablet-server-2 + READINESS_TCP_PORT: "9123" + READINESS_HEALTH_CHECK_TIMEOUT_SECONDS: "180" + ports: + - "${TABLET_SERVER_2_PORT:-9126}:9123" + - "${TABLET_SERVER_2_DEBUG_PORT:-15008}:5005" + - "${TABLET_SERVER_2_METRICS_PORT:-9252}:9249" + +volumes: + shared-data: + zookeeper-data: + zookeeper-log: diff --git a/devkit/justfile b/devkit/justfile new file mode 100644 index 0000000000..9c5978884f --- /dev/null +++ b/devkit/justfile @@ -0,0 +1,363 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +set dotenv-load := false + +# --- Paths -------------------------------------------------------------------- + +DEVKIT_DIR := justfile_directory() +FLUSS_DEV_HOME := DEVKIT_DIR + "/.." +PROFILES_DIR := DEVKIT_DIR + "/profiles" +DEPS_DIR := DEVKIT_DIR + "/.deps" +CACHE_DIR := DEPS_DIR + "/cache" + +# --- Docker Compose commands -------------------------------------------------- + +COMPOSE := "docker compose -f " + DEVKIT_DIR + "/docker-compose.yml" +COMPOSE_TIERING := COMPOSE + " -f " + DEVKIT_DIR + "/compose/tiering.yml" +ALL_COMPOSE := COMPOSE_TIERING + " -f " + DEVKIT_DIR + "/compose/lake-common.yml -f " + DEVKIT_DIR + "/compose/iceberg.yml" +ALL_PROFILES := "--profile single --profile cluster --profile flink --profile tiering" + +# --- Build commands ----------------------------------------------------------- + +BUILD_CMD := env_var_or_default("DEVKIT_BUILD_CMD", "./mvnw -DskipTests -pl fluss-dist,fluss-flink/fluss-flink-1.20 -am package") +TIERING_BUILD_CMD := env_var_or_default("DEVKIT_TIERING_BUILD_CMD", "./mvnw -DskipTests -pl fluss-dist,fluss-flink/fluss-flink-1.20,fluss-flink/fluss-flink-tiering -am package") + +default: + @just --list + +[doc("Build the local Fluss distribution and Flink connector")] +build: + cd "{{ FLUSS_DEV_HOME }}" && {{ BUILD_CMD }} + +[doc("Build the local distribution and lake tiering artifacts")] +build-tiering: + cd "{{ FLUSS_DEV_HOME }}" && {{ TIERING_BUILD_CMD }} + +[private] +ensure-dist: + #!/usr/bin/env bash + set -e + if [ ! -x "{{ FLUSS_DEV_HOME }}/build-target/bin/coordinator-server.sh" ]; then + echo "Local distribution not found. Run: just build" + exit 1 + fi + +[private] +validate-profile profile: + #!/usr/bin/env bash + set -e + PROFILE={{ quote(profile) }} + if [[ ! "$PROFILE" =~ ^[a-z0-9][a-z0-9-]*$ ]]; then + echo "profile must match [a-z0-9][a-z0-9-]*" + exit 1 + fi + if [ ! -f "{{ PROFILES_DIR }}/$PROFILE/server.yaml" ]; then + echo "Unknown profile: $PROFILE" + exit 1 + fi + +[private] +profile-format profile: + #!/usr/bin/env bash + PROFILE={{ quote(profile) }} + sed -n 's/^[[:space:]]*datalake\.format[[:space:]]*: \([^#]*\).*$/\1/p' \ + "{{ PROFILES_DIR }}/$PROFILE/server.yaml" | sed 's/[[:space:]]*$//' | tail -n 1 + +[doc("Download the JARs declared by a profile")] +deps profile: + #!/usr/bin/env bash + set -e + PROFILE={{ quote(profile) }} + just validate-profile "$PROFILE" + PROFILE_DIR="{{ PROFILES_DIR }}/$PROFILE" + mkdir -p "{{ CACHE_DIR }}" + for FILE in "$PROFILE_DIR"/{jars,server,flink}.urls; do + [ -f "$FILE" ] || continue + while IFS= read -r URL; do + NAME="$(basename "${URL%%\?*}")" + TARGET="{{ CACHE_DIR }}/$NAME" + [ -s "$TARGET" ] && continue + echo "Downloading $(basename "$TARGET")..." + curl -fL --retry 3 -o "${TARGET}.tmp" "$URL" + mv "${TARGET}.tmp" "$TARGET" + done < <(sed 's/#.*//; s/[[:space:]]//g; /^$/d' "$FILE") + done + +[private] +ensure-flink: + #!/usr/bin/env bash + set -e + ACTIVE_DIR="{{ DEPS_DIR }}/flink/active" + MATCHES=( {{ FLUSS_DEV_HOME }}/fluss-flink/fluss-flink-1.20/target/fluss-flink-1.20-*.jar ) + if [ "${#MATCHES[@]}" -ne 1 ] || [ ! -f "${MATCHES[0]}" ]; then + echo "Expected one Flink connector artifact" + echo "Run: just build" + exit 1 + fi + mkdir -p "$ACTIVE_DIR/lib" "$ACTIVE_DIR/jobs" + rm -f "$ACTIVE_DIR/lib/"*.jar "$ACTIVE_DIR/jobs/"*.jar + cp "${MATCHES[0]}" "$ACTIVE_DIR/lib/fluss-flink-1.20.jar" + chmod -R a+rX "$ACTIVE_DIR" + +[private] +ensure-tiering profile: + #!/usr/bin/env bash + set -e + PROFILE={{ quote(profile) }} + just validate-profile "$PROFILE" + PROFILE_DIR="{{ PROFILES_DIR }}/$PROFILE" + FORMAT=$(just profile-format "$PROFILE") + if [ -z "$FORMAT" ]; then + echo "Profile $PROFILE does not configure datalake.format" + exit 1 + fi + stage() { + local PATTERN="$1" + local TARGET="$2" + MATCHES=( $PATTERN ) + if [ "${#MATCHES[@]}" -ne 1 ] || [ ! -f "${MATCHES[0]}" ]; then + echo "Expected one build artifact: $PATTERN" + echo "Run: just build-tiering" + exit 1 + fi + cp "${MATCHES[0]}" "$TARGET" + } + copy_urls() { + local FILE="$1" + shift + local URL NAME TARGET + [ -f "$FILE" ] || return 0 + while IFS= read -r URL; do + NAME="$(basename "${URL%%\?*}")" + for TARGET in "$@"; do + cp "{{ CACHE_DIR }}/$NAME" "$TARGET/" + done + done < <(sed 's/#.*//; s/[[:space:]]//g; /^$/d' "$FILE") + } + ACTIVE_DIR="{{ DEPS_DIR }}/flink/active" + just deps "$PROFILE" + stage "{{ FLUSS_DEV_HOME }}/fluss-lake/fluss-lake-$FORMAT/target/fluss-lake-$FORMAT-*.jar" "$ACTIVE_DIR/lib/fluss-lake-$FORMAT.jar" + stage "{{ FLUSS_DEV_HOME }}/fluss-flink/fluss-flink-tiering/target/fluss-flink-tiering-*.jar" "$ACTIVE_DIR/jobs/fluss-flink-tiering.jar" + + SERVER_PLUGIN_DIR="{{ FLUSS_DEV_HOME }}/build-target/plugins/$FORMAT" + mkdir -p "$SERVER_PLUGIN_DIR" + # Filesystem plugins must stay isolated from lake plugin classpaths. + rm -f "$SERVER_PLUGIN_DIR"/fluss-fs-*.jar + REMOTE_DATA_DIR=$(sed -n 's/^[[:space:]]*remote\.data\.dir[[:space:]]*: \([^#]*\).*$/\1/p' "$PROFILE_DIR/server.yaml" | sed 's/[[:space:]]*$//' | tail -n 1) + SCHEME="${REMOTE_DATA_DIR%%:*}" + if [ "$SCHEME" != "$REMOTE_DATA_DIR" ] && [ -d "{{ FLUSS_DEV_HOME }}/build-target/plugins/$SCHEME" ]; then + cp "{{ FLUSS_DEV_HOME }}/build-target/plugins/$SCHEME/"*.jar "$ACTIVE_DIR/lib/" + fi + + copy_urls "$PROFILE_DIR/jars.urls" "$SERVER_PLUGIN_DIR" "$ACTIVE_DIR/lib" + copy_urls "$PROFILE_DIR/server.urls" "$SERVER_PLUGIN_DIR" + copy_urls "$PROFILE_DIR/flink.urls" "$ACTIVE_DIR/lib" + chmod -R a+rX "$ACTIVE_DIR" + +[doc("Start a profile from the profiles directory")] +up profile="core" nodes="1": ensure-dist ensure-flink + #!/usr/bin/env bash + set -e + PROFILE={{ quote(profile) }} + just validate-profile "$PROFILE" + case "{{ nodes }}" in + 1) TOPOLOGY=single ;; + 3) TOPOLOGY=cluster ;; + *) echo "nodes must be 1 or 3"; exit 1 ;; + esac + PROFILE_DIR="{{ PROFILES_DIR }}/$PROFILE" + FORMAT=$(just profile-format "$PROFILE") + COMPOSE_ARGS=(-f "{{ DEVKIT_DIR }}/docker-compose.yml" -f "{{ DEVKIT_DIR }}/compose/tiering.yml") + PROFILES=(--profile "$TOPOLOGY" --profile flink) + if [ -n "$FORMAT" ]; then + just ensure-tiering "$PROFILE" + PROFILES+=(--profile tiering) + if [ -f "$PROFILE_DIR/compose.files" ]; then + while IFS= read -r FILE; do + FILE="${FILE%%#*}" + FILE="${FILE//[[:space:]]/}" + [ -n "$FILE" ] || continue + if [ ! -f "{{ DEVKIT_DIR }}/$FILE" ]; then + echo "Compose file not found: $FILE" + exit 1 + fi + COMPOSE_ARGS+=(-f "{{ DEVKIT_DIR }}/$FILE") + done < "$PROFILE_DIR/compose.files" + fi + fi + export FLUSS_DEVKIT_CONFIG="$PROFILE_DIR/server.yaml" + export FLUSS_REPLICATION_FACTOR="{{ nodes }}" + {{ ALL_COMPOSE }} {{ ALL_PROFILES }} down --remove-orphans >/dev/null 2>&1 || true + docker compose "${COMPOSE_ARGS[@]}" "${PROFILES[@]}" up -d + just check "$PROFILE" + +[doc("Stop services and keep data")] +down: + @{{ ALL_COMPOSE }} {{ ALL_PROFILES }} down --remove-orphans + +[doc("Stop services and remove data")] +clean: + @{{ ALL_COMPOSE }} {{ ALL_PROFILES }} down -v --remove-orphans + +[doc("Show service status")] +status: + @{{ ALL_COMPOSE }} {{ ALL_PROFILES }} ps -a + +[doc("Show log output. Example: just logs tablet-server-0 200")] +logs service="" tail="100": + #!/usr/bin/env bash + if [ -z "{{ service }}" ]; then + {{ ALL_COMPOSE }} {{ ALL_PROFILES }} logs --tail="{{ tail }}" + else + {{ ALL_COMPOSE }} {{ ALL_PROFILES }} logs --tail="{{ tail }}" "{{ service }}" + fi + +[doc("Run a command in a service. Example: just exec tablet-server-0 java -version")] +exec service *CMD: + @{{ ALL_COMPOSE }} {{ ALL_PROFILES }} exec "{{ service }}" {{ CMD }} + +[private] +check profile: check-cluster check-flink + #!/usr/bin/env bash + FORMAT=$(just profile-format {{ quote(profile) }}) + if [ -n "$FORMAT" ]; then just check-tiering "$FORMAT"; fi + +[private] +check-cluster: + #!/usr/bin/env bash + set -e + ATTEMPT=0 + until {{ COMPOSE }} --profile single --profile cluster exec -T tablet-server-0 java -Xmx64m -classpath '/opt/fluss/lib/*' org.apache.fluss.server.tools.ClusterHealthReadinessCheck --host tablet-server-0 --port 9123 --timeoutMs 10000; do + ATTEMPT=$((ATTEMPT + 1)) + if [ "$ATTEMPT" -ge 20 ]; then + echo "Cluster health did not become ready" + exit 1 + fi + sleep 3 + done + +[private] +check-flink: + #!/usr/bin/env bash + set -e + for _ in {1..40}; do + if {{ COMPOSE_TIERING }} --profile flink exec -T jobmanager curl -fsS --max-time 5 http://localhost:8081/taskmanagers | grep -q '"id"'; then + exit 0 + fi + sleep 3 + done + {{ COMPOSE_TIERING }} --profile flink logs --tail=200 jobmanager taskmanager + echo "Flink cluster did not become ready" + exit 1 + +[private] +check-tiering format: + #!/usr/bin/env bash + set -e + for _ in {1..40}; do + JOB=$({{ COMPOSE_TIERING }} --profile flink --profile tiering exec -T jobmanager curl -fsS http://localhost:8081/jobs/overview | grep -o '"name":"Fluss Lake Tiering Service - {{ format }}"[^}]*' || true) + STATE=$(printf '%s' "$JOB" | sed -n 's/.*"state":"\([^"]*\)".*/\1/p') + case "$STATE" in + RUNNING) + echo "{{ format }} tiering job is running" + exit 0 + ;; + FAILING|FAILED|CANCELLING|CANCELED|FINISHED|RESTARTING|SUSPENDED) + {{ COMPOSE_TIERING }} --profile flink --profile tiering logs --tail=200 tiering-submit jobmanager taskmanager + echo "Tiering job entered state $STATE" + exit 1 + ;; + esac + sleep 3 + done + {{ COMPOSE_TIERING }} --profile flink --profile tiering logs --tail=200 tiering-submit jobmanager taskmanager + echo "Tiering job did not start" + exit 1 + +[private] +validate-table table: + #!/usr/bin/env bash + if [[ ! "{{ table }}" =~ ^[A-Za-z_][A-Za-z0-9_]*$ ]]; then + echo "table must match [A-Za-z_][A-Za-z0-9_]*" + exit 1 + fi + +[doc("Run a local SQL file with Flink SQL Client")] +run-sql file: + #!/usr/bin/env bash + set -e + FILE={{ quote(file) }} + if [[ "$FILE" != /* ]]; then + FILE={{ quote(invocation_directory()) }}/"$FILE" + fi + if [ ! -f "$FILE" ]; then + echo "SQL file not found: $FILE" + exit 1 + fi + set +e + OUTPUT=$({{ COMPOSE_TIERING }} --profile flink exec -T jobmanager /opt/flink/bin/sql-client.sh -f /dev/stdin < "$FILE" 2>&1) + STATUS=$? + set -e + printf '%s\n' "$OUTPUT" + if [ "$STATUS" -ne 0 ] || printf '%s\n' "$OUTPUT" | grep -Eq '\[ERROR\]|Could not execute SQL statement'; then + exit 1 + fi + +[private] +run-demo-sql file table marker="": + #!/usr/bin/env bash + set -e + just validate-table "{{ table }}" + set +e + OUTPUT=$({{ ALL_COMPOSE }} {{ ALL_PROFILES }} exec -T -e DEVKIT_TABLE="{{ table }}" jobmanager /bin/bash -c "envsubst '\${DEVKIT_TABLE}' < /opt/devkit-tests/{{ file }} > /tmp/{{ file }} && /opt/flink/bin/sql-client.sh -f /tmp/{{ file }}" 2>&1) + STATUS=$? + set -e + printf '%s\n' "$OUTPUT" + if [ "$STATUS" -ne 0 ] || printf '%s\n' "$OUTPUT" | grep -Eq '\[ERROR\]|Could not execute SQL statement'; then + exit 1 + fi + if [ -n "{{ marker }}" ] && ! printf '%s\n' "$OUTPUT" | grep -Fq "{{ marker }}"; then + echo "SQL command did not report expected marker: {{ marker }}" + exit 1 + fi + +[private] +run-format-sql operation format table marker="": + #!/usr/bin/env bash + just validate-profile "{{ format }}" + FILE="{{ format }}-{{ operation }}.sql" + if [ ! -f "{{ DEVKIT_DIR }}/tests/$FILE" ]; then + echo "{{ operation }} is not available for {{ format }}" + exit 1 + fi + just run-demo-sql "$FILE" "{{ table }}" "{{ marker }}" + +[doc("Create a lake-enabled Fluss table")] +create-table format table: + @just run-format-sql create-table "{{ format }}" "{{ table }}" + +[doc("Write three sample rows to a Fluss table")] +write-data format table: + @just run-format-sql write-data "{{ format }}" "{{ table }}" "Job ID:" + +[doc("Show running Flink jobs")] +tiering-status: + @{{ ALL_COMPOSE }} {{ ALL_PROFILES }} exec -T jobmanager /opt/flink/bin/flink list -r + +[doc("Query a table through its lake table")] +query-lake format table: + @just run-format-sql query-lake "{{ format }}" "{{ table }}" diff --git a/devkit/profiles/core/server.yaml b/devkit/profiles/core/server.yaml new file mode 100644 index 0000000000..a71718012e --- /dev/null +++ b/devkit/profiles/core/server.yaml @@ -0,0 +1,19 @@ +################################################################################ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +################################################################################ + +remote.data.dir: file:///tmp/fluss/remote-data diff --git a/devkit/profiles/hudi/compose.files b/devkit/profiles/hudi/compose.files new file mode 100644 index 0000000000..9619bb10bf --- /dev/null +++ b/devkit/profiles/hudi/compose.files @@ -0,0 +1,17 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +compose/lake-common.yml diff --git a/devkit/profiles/hudi/flink.urls b/devkit/profiles/hudi/flink.urls new file mode 100644 index 0000000000..dab108432e --- /dev/null +++ b/devkit/profiles/hudi/flink.urls @@ -0,0 +1,17 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +https://repo.maven.apache.org/maven2/io/trino/hadoop/hadoop-apache/3.3.5-2/hadoop-apache-3.3.5-2.jar diff --git a/devkit/profiles/hudi/jars.urls b/devkit/profiles/hudi/jars.urls new file mode 100644 index 0000000000..1d6ff80ffb --- /dev/null +++ b/devkit/profiles/hudi/jars.urls @@ -0,0 +1,17 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +https://repo.maven.apache.org/maven2/org/apache/hudi/hudi-flink1.20-bundle/1.1.0/hudi-flink1.20-bundle-1.1.0.jar diff --git a/devkit/profiles/hudi/server.urls b/devkit/profiles/hudi/server.urls new file mode 100644 index 0000000000..a00613c8fc --- /dev/null +++ b/devkit/profiles/hudi/server.urls @@ -0,0 +1,21 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +https://repo.maven.apache.org/maven2/io/trino/hadoop/hadoop-apache/3.3.5-2/hadoop-apache-3.3.5-2.jar +https://repo.maven.apache.org/maven2/org/apache/flink/flink-core/1.20.1/flink-core-1.20.1.jar +https://repo.maven.apache.org/maven2/org/apache/flink/flink-table-common/1.20.1/flink-table-common-1.20.1.jar +https://repo.maven.apache.org/maven2/org/apache/flink/flink-table-api-java/1.20.1/flink-table-api-java-1.20.1.jar +https://repo.maven.apache.org/maven2/org/apache/flink/flink-table-runtime/1.20.1/flink-table-runtime-1.20.1.jar diff --git a/devkit/profiles/hudi/server.yaml b/devkit/profiles/hudi/server.yaml new file mode 100644 index 0000000000..0e5a33d7cf --- /dev/null +++ b/devkit/profiles/hudi/server.yaml @@ -0,0 +1,41 @@ +################################################################################ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +################################################################################ + +remote.data.dir: s3://fluss/remote-data +s3.endpoint: http://rustfs:9000 +s3.access-key: rustfsadmin +s3.secret-key: rustfsadmin +s3.region: us-east-1 +s3.path-style-access: true +s3.assumed.role.arn: arn:aws:iam::000000000000:role/rustfsadmin +s3.assumed.role.sts.endpoint: http://rustfs:9000 + +datalake.enabled: true +datalake.format: hudi +datalake.hudi.mode: dfs +datalake.hudi.catalog.path: s3a://fluss/hudi +datalake.hudi.hadoop.fs.s3a.endpoint: http://rustfs:9000 +datalake.hudi.hadoop.fs.s3a.access.key: rustfsadmin +datalake.hudi.hadoop.fs.s3a.secret.key: rustfsadmin +datalake.hudi.hadoop.fs.s3a.path.style.access: true +datalake.hudi.hadoop.fs.s3a.connection.ssl.enabled: false +datalake.hudi.hadoop.fs.s3a.impl.disable.cache: true +datalake.hudi.hadoop.fs.s3a.aws.credentials.provider: org.apache.hadoop.fs.s3a.SimpleAWSCredentialsProvider +datalake.hudi.hoodie.write.lock.provider: org.apache.hudi.client.transaction.lock.InProcessLockProvider + +fluss.tiering.poll.table.interval: 1s diff --git a/devkit/profiles/iceberg/compose.files b/devkit/profiles/iceberg/compose.files new file mode 100644 index 0000000000..081f3d6812 --- /dev/null +++ b/devkit/profiles/iceberg/compose.files @@ -0,0 +1,18 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +compose/lake-common.yml +compose/iceberg.yml diff --git a/devkit/profiles/iceberg/flink.urls b/devkit/profiles/iceberg/flink.urls new file mode 100644 index 0000000000..9cd9964a6d --- /dev/null +++ b/devkit/profiles/iceberg/flink.urls @@ -0,0 +1,18 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +https://repo.maven.apache.org/maven2/io/trino/hadoop/hadoop-apache/3.3.5-2/hadoop-apache-3.3.5-2.jar +https://repo.maven.apache.org/maven2/org/apache/iceberg/iceberg-flink-runtime-1.20/1.10.1/iceberg-flink-runtime-1.20-1.10.1.jar diff --git a/devkit/profiles/iceberg/jars.urls b/devkit/profiles/iceberg/jars.urls new file mode 100644 index 0000000000..08339249be --- /dev/null +++ b/devkit/profiles/iceberg/jars.urls @@ -0,0 +1,20 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +https://repo.maven.apache.org/maven2/org/apache/iceberg/iceberg-aws/1.10.1/iceberg-aws-1.10.1.jar +https://repo.maven.apache.org/maven2/org/apache/iceberg/iceberg-aws-bundle/1.10.1/iceberg-aws-bundle-1.10.1.jar +https://repo.maven.apache.org/maven2/dev/failsafe/failsafe/3.3.2/failsafe-3.3.2.jar +https://repo.maven.apache.org/maven2/org/postgresql/postgresql/42.7.4/postgresql-42.7.4.jar diff --git a/devkit/profiles/iceberg/server.yaml b/devkit/profiles/iceberg/server.yaml new file mode 100644 index 0000000000..9ba4f1d7ba --- /dev/null +++ b/devkit/profiles/iceberg/server.yaml @@ -0,0 +1,41 @@ +################################################################################ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +################################################################################ + +remote.data.dir: s3://fluss/remote-data +s3.endpoint: http://rustfs:9000 +s3.access-key: rustfsadmin +s3.secret-key: rustfsadmin +s3.region: us-east-1 +s3.path-style-access: true +s3.assumed.role.arn: arn:aws:iam::000000000000:role/rustfsadmin +s3.assumed.role.sts.endpoint: http://rustfs:9000 + +datalake.enabled: true +datalake.format: iceberg +datalake.iceberg.catalog-impl: org.apache.iceberg.jdbc.JdbcCatalog +datalake.iceberg.name: fluss_catalog +datalake.iceberg.uri: jdbc:postgresql://postgres:5432/iceberg +datalake.iceberg.jdbc.user: iceberg +datalake.iceberg.jdbc.password: iceberg +datalake.iceberg.warehouse: s3://fluss/iceberg +datalake.iceberg.io-impl: org.apache.iceberg.aws.s3.S3FileIO +datalake.iceberg.s3.endpoint: http://rustfs:9000 +datalake.iceberg.s3.access-key-id: rustfsadmin +datalake.iceberg.s3.secret-access-key: rustfsadmin +datalake.iceberg.s3.path-style-access: true +datalake.iceberg.client.region: us-east-1 diff --git a/devkit/profiles/lance/compose.files b/devkit/profiles/lance/compose.files new file mode 100644 index 0000000000..04ef1ab87d --- /dev/null +++ b/devkit/profiles/lance/compose.files @@ -0,0 +1,18 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +compose/lake-common.yml +compose/lance.yml diff --git a/devkit/profiles/lance/server.yaml b/devkit/profiles/lance/server.yaml new file mode 100644 index 0000000000..691b56a33b --- /dev/null +++ b/devkit/profiles/lance/server.yaml @@ -0,0 +1,37 @@ +################################################################################ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +################################################################################ + +remote.data.dir: s3://fluss/remote-data +s3.endpoint: http://rustfs:9000 +s3.access-key: rustfsadmin +s3.secret-key: rustfsadmin +s3.region: us-east-1 +s3.path-style-access: true +s3.assumed.role.arn: arn:aws:iam::000000000000:role/rustfsadmin +s3.assumed.role.sts.endpoint: http://rustfs:9000 + +datalake.enabled: true +datalake.format: lance +datalake.lance.warehouse: s3://fluss/lance +datalake.lance.endpoint: http://rustfs:9000 +datalake.lance.allow_http: true +datalake.lance.region: us-east-1 +datalake.lance.access_key_id: rustfsadmin +datalake.lance.secret_access_key: rustfsadmin + +fluss.tiering.poll.table.interval: 1s diff --git a/devkit/profiles/paimon/compose.files b/devkit/profiles/paimon/compose.files new file mode 100644 index 0000000000..9619bb10bf --- /dev/null +++ b/devkit/profiles/paimon/compose.files @@ -0,0 +1,17 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +compose/lake-common.yml diff --git a/devkit/profiles/paimon/flink.urls b/devkit/profiles/paimon/flink.urls new file mode 100644 index 0000000000..7beac802f2 --- /dev/null +++ b/devkit/profiles/paimon/flink.urls @@ -0,0 +1,18 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +https://repo.maven.apache.org/maven2/io/trino/hadoop/hadoop-apache/3.3.5-2/hadoop-apache-3.3.5-2.jar +https://repo.maven.apache.org/maven2/org/apache/paimon/paimon-flink-1.20/1.3.1/paimon-flink-1.20-1.3.1.jar diff --git a/devkit/profiles/paimon/jars.urls b/devkit/profiles/paimon/jars.urls new file mode 100644 index 0000000000..83f6838ea8 --- /dev/null +++ b/devkit/profiles/paimon/jars.urls @@ -0,0 +1,17 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +https://repo.maven.apache.org/maven2/org/apache/paimon/paimon-s3/1.3.1/paimon-s3-1.3.1.jar diff --git a/devkit/profiles/paimon/server.yaml b/devkit/profiles/paimon/server.yaml new file mode 100644 index 0000000000..91cfc344b4 --- /dev/null +++ b/devkit/profiles/paimon/server.yaml @@ -0,0 +1,35 @@ +################################################################################ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +################################################################################ + +remote.data.dir: s3://fluss/remote-data +s3.endpoint: http://rustfs:9000 +s3.access-key: rustfsadmin +s3.secret-key: rustfsadmin +s3.region: us-east-1 +s3.path-style-access: true +s3.assumed.role.arn: arn:aws:iam::000000000000:role/rustfsadmin +s3.assumed.role.sts.endpoint: http://rustfs:9000 + +datalake.enabled: true +datalake.format: paimon +datalake.paimon.metastore: filesystem +datalake.paimon.warehouse: s3://fluss/paimon +datalake.paimon.s3.endpoint: http://rustfs:9000 +datalake.paimon.s3.access-key: rustfsadmin +datalake.paimon.s3.secret-key: rustfsadmin +datalake.paimon.s3.path.style.access: true diff --git a/devkit/tests/hudi-create-table.sql b/devkit/tests/hudi-create-table.sql new file mode 100644 index 0000000000..a12e01a658 --- /dev/null +++ b/devkit/tests/hudi-create-table.sql @@ -0,0 +1,32 @@ +-- Licensed to the Apache Software Foundation (ASF) under one +-- or more contributor license agreements. See the NOTICE file +-- distributed with this work for additional information +-- regarding copyright ownership. The ASF licenses this file +-- to you under the Apache License, Version 2.0 (the +-- "License"); you may not use this file except in compliance +-- with the License. You may obtain a copy of the License at +-- +-- http://www.apache.org/licenses/LICENSE-2.0 +-- +-- Unless required by applicable law or agreed to in writing, software +-- distributed under the License is distributed on an "AS IS" BASIS, +-- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +-- See the License for the specific language governing permissions and +-- limitations under the License. + +CREATE CATALOG fluss_catalog WITH ( + 'type' = 'fluss', + 'bootstrap.servers' = 'coordinator-server:19123' +); + +USE CATALOG fluss_catalog; + +CREATE TABLE ${DEVKIT_TABLE} ( + id BIGINT, + payload STRING +) WITH ( + 'bucket.num' = '1', + 'table.datalake.enabled' = 'true', + 'table.datalake.freshness' = '5s', + 'hudi.hoodie.datasource.write.recordkey.field' = 'id' +); diff --git a/devkit/tests/hudi-query-lake.sql b/devkit/tests/hudi-query-lake.sql new file mode 100644 index 0000000000..0318d87a24 --- /dev/null +++ b/devkit/tests/hudi-query-lake.sql @@ -0,0 +1,40 @@ +-- Licensed to the Apache Software Foundation (ASF) under one +-- or more contributor license agreements. See the NOTICE file +-- distributed with this work for additional information +-- regarding copyright ownership. The ASF licenses this file +-- to you under the Apache License, Version 2.0 (the +-- "License"); you may not use this file except in compliance +-- with the License. You may obtain a copy of the License at +-- +-- http://www.apache.org/licenses/LICENSE-2.0 +-- +-- Unless required by applicable law or agreed to in writing, software +-- distributed under the License is distributed on an "AS IS" BASIS, +-- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +-- See the License for the specific language governing permissions and +-- limitations under the License. + +SET 'execution.runtime-mode' = 'batch'; +SET 'sql-client.execution.result-mode' = 'tableau'; + +CREATE CATALOG hudi_catalog WITH ( + 'type' = 'hudi', + 'mode' = 'dfs', + 'catalog.path' = 's3a://fluss/hudi', + 'hadoop.fs.s3a.endpoint' = 'http://rustfs:9000', + 'hadoop.fs.s3a.access.key' = 'rustfsadmin', + 'hadoop.fs.s3a.secret.key' = 'rustfsadmin', + 'hadoop.fs.s3a.path.style.access' = 'true', + 'hadoop.fs.s3a.connection.ssl.enabled' = 'false', + 'hadoop.fs.s3a.impl.disable.cache' = 'true' +); + +USE CATALOG hudi_catalog; + +SELECT + COUNT(*) AS row_count, + SUM(id) AS id_sum, + COUNT(DISTINCT payload) AS payload_count, + MIN(payload) AS first_payload, + MAX(payload) AS last_payload +FROM fluss.${DEVKIT_TABLE}; diff --git a/devkit/tests/hudi-write-data.sql b/devkit/tests/hudi-write-data.sql new file mode 100644 index 0000000000..70a924cad8 --- /dev/null +++ b/devkit/tests/hudi-write-data.sql @@ -0,0 +1,27 @@ +-- Licensed to the Apache Software Foundation (ASF) under one +-- or more contributor license agreements. See the NOTICE file +-- distributed with this work for additional information +-- regarding copyright ownership. The ASF licenses this file +-- to you under the Apache License, Version 2.0 (the +-- "License"); you may not use this file except in compliance +-- with the License. You may obtain a copy of the License at +-- +-- http://www.apache.org/licenses/LICENSE-2.0 +-- +-- Unless required by applicable law or agreed to in writing, software +-- distributed under the License is distributed on an "AS IS" BASIS, +-- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +-- See the License for the specific language governing permissions and +-- limitations under the License. + +CREATE CATALOG fluss_catalog WITH ( + 'type' = 'fluss', + 'bootstrap.servers' = 'coordinator-server:19123' +); + +USE CATALOG fluss_catalog; + +INSERT INTO ${DEVKIT_TABLE} VALUES + (1, 'alpha'), + (2, 'beta'), + (3, 'gamma'); diff --git a/devkit/tests/iceberg-create-table.sql b/devkit/tests/iceberg-create-table.sql new file mode 100644 index 0000000000..064478ed36 --- /dev/null +++ b/devkit/tests/iceberg-create-table.sql @@ -0,0 +1,34 @@ +-- Licensed to the Apache Software Foundation (ASF) under one +-- or more contributor license agreements. See the NOTICE file +-- distributed with this work for additional information +-- regarding copyright ownership. The ASF licenses this file +-- to you under the Apache License, Version 2.0 (the +-- "License"); you may not use this file except in compliance +-- with the License. You may obtain a copy of the License at +-- +-- http://www.apache.org/licenses/LICENSE-2.0 +-- +-- Unless required by applicable law or agreed to in writing, software +-- distributed under the License is distributed on an "AS IS" BASIS, +-- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +-- See the License for the specific language governing permissions and +-- limitations under the License. + +CREATE CATALOG fluss_catalog WITH ( + 'type' = 'fluss', + 'bootstrap.servers' = 'coordinator-server:19123', + 'iceberg.jdbc.password' = 'iceberg', + 'iceberg.s3.access-key-id' = 'rustfsadmin', + 'iceberg.s3.secret-access-key' = 'rustfsadmin' +); + +USE CATALOG fluss_catalog; + +CREATE TABLE ${DEVKIT_TABLE} ( + id BIGINT, + payload STRING +) WITH ( + 'bucket.num' = '1', + 'table.datalake.enabled' = 'true', + 'table.datalake.freshness' = '5s' +); diff --git a/devkit/tests/iceberg-query-lake.sql b/devkit/tests/iceberg-query-lake.sql new file mode 100644 index 0000000000..f9be90a218 --- /dev/null +++ b/devkit/tests/iceberg-query-lake.sql @@ -0,0 +1,36 @@ +-- Licensed to the Apache Software Foundation (ASF) under one +-- or more contributor license agreements. See the NOTICE file +-- distributed with this work for additional information +-- regarding copyright ownership. The ASF licenses this file +-- to you under the Apache License, Version 2.0 (the +-- "License"); you may not use this file except in compliance +-- with the License. You may obtain a copy of the License at +-- +-- http://www.apache.org/licenses/LICENSE-2.0 +-- +-- Unless required by applicable law or agreed to in writing, software +-- distributed under the License is distributed on an "AS IS" BASIS, +-- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +-- See the License for the specific language governing permissions and +-- limitations under the License. + +SET 'execution.runtime-mode' = 'batch'; +SET 'sql-client.execution.result-mode' = 'tableau'; + +CREATE CATALOG fluss_catalog WITH ( + 'type' = 'fluss', + 'bootstrap.servers' = 'coordinator-server:19123', + 'iceberg.jdbc.password' = 'iceberg', + 'iceberg.s3.access-key-id' = 'rustfsadmin', + 'iceberg.s3.secret-access-key' = 'rustfsadmin' +); + +USE CATALOG fluss_catalog; + +SELECT + COUNT(*) AS row_count, + SUM(id) AS id_sum, + COUNT(DISTINCT payload) AS payload_count, + MIN(payload) AS first_payload, + MAX(payload) AS last_payload +FROM `${DEVKIT_TABLE}$lake`; diff --git a/devkit/tests/iceberg-write-data.sql b/devkit/tests/iceberg-write-data.sql new file mode 100644 index 0000000000..b8b823e981 --- /dev/null +++ b/devkit/tests/iceberg-write-data.sql @@ -0,0 +1,30 @@ +-- Licensed to the Apache Software Foundation (ASF) under one +-- or more contributor license agreements. See the NOTICE file +-- distributed with this work for additional information +-- regarding copyright ownership. The ASF licenses this file +-- to you under the Apache License, Version 2.0 (the +-- "License"); you may not use this file except in compliance +-- with the License. You may obtain a copy of the License at +-- +-- http://www.apache.org/licenses/LICENSE-2.0 +-- +-- Unless required by applicable law or agreed to in writing, software +-- distributed under the License is distributed on an "AS IS" BASIS, +-- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +-- See the License for the specific language governing permissions and +-- limitations under the License. + +CREATE CATALOG fluss_catalog WITH ( + 'type' = 'fluss', + 'bootstrap.servers' = 'coordinator-server:19123', + 'iceberg.jdbc.password' = 'iceberg', + 'iceberg.s3.access-key-id' = 'rustfsadmin', + 'iceberg.s3.secret-access-key' = 'rustfsadmin' +); + +USE CATALOG fluss_catalog; + +INSERT INTO ${DEVKIT_TABLE} VALUES + (1, 'alpha'), + (2, 'beta'), + (3, 'gamma'); diff --git a/devkit/tests/lance-create-table.sql b/devkit/tests/lance-create-table.sql new file mode 100644 index 0000000000..82b8876f82 --- /dev/null +++ b/devkit/tests/lance-create-table.sql @@ -0,0 +1,31 @@ +-- Licensed to the Apache Software Foundation (ASF) under one +-- or more contributor license agreements. See the NOTICE file +-- distributed with this work for additional information +-- regarding copyright ownership. The ASF licenses this file +-- to you under the Apache License, Version 2.0 (the +-- "License"); you may not use this file except in compliance +-- with the License. You may obtain a copy of the License at +-- +-- http://www.apache.org/licenses/LICENSE-2.0 +-- +-- Unless required by applicable law or agreed to in writing, software +-- distributed under the License is distributed on an "AS IS" BASIS, +-- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +-- See the License for the specific language governing permissions and +-- limitations under the License. + +CREATE CATALOG fluss_catalog WITH ( + 'type' = 'fluss', + 'bootstrap.servers' = 'coordinator-server:19123' +); + +USE CATALOG fluss_catalog; + +CREATE TABLE ${DEVKIT_TABLE} ( + id BIGINT, + payload STRING +) WITH ( + 'bucket.num' = '1', + 'table.datalake.enabled' = 'true', + 'table.datalake.freshness' = '5s' +); diff --git a/devkit/tests/lance-write-data.sql b/devkit/tests/lance-write-data.sql new file mode 100644 index 0000000000..70a924cad8 --- /dev/null +++ b/devkit/tests/lance-write-data.sql @@ -0,0 +1,27 @@ +-- Licensed to the Apache Software Foundation (ASF) under one +-- or more contributor license agreements. See the NOTICE file +-- distributed with this work for additional information +-- regarding copyright ownership. The ASF licenses this file +-- to you under the Apache License, Version 2.0 (the +-- "License"); you may not use this file except in compliance +-- with the License. You may obtain a copy of the License at +-- +-- http://www.apache.org/licenses/LICENSE-2.0 +-- +-- Unless required by applicable law or agreed to in writing, software +-- distributed under the License is distributed on an "AS IS" BASIS, +-- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +-- See the License for the specific language governing permissions and +-- limitations under the License. + +CREATE CATALOG fluss_catalog WITH ( + 'type' = 'fluss', + 'bootstrap.servers' = 'coordinator-server:19123' +); + +USE CATALOG fluss_catalog; + +INSERT INTO ${DEVKIT_TABLE} VALUES + (1, 'alpha'), + (2, 'beta'), + (3, 'gamma'); diff --git a/devkit/tests/paimon-create-table.sql b/devkit/tests/paimon-create-table.sql new file mode 100644 index 0000000000..c3d2110f7e --- /dev/null +++ b/devkit/tests/paimon-create-table.sql @@ -0,0 +1,33 @@ +-- Licensed to the Apache Software Foundation (ASF) under one +-- or more contributor license agreements. See the NOTICE file +-- distributed with this work for additional information +-- regarding copyright ownership. The ASF licenses this file +-- to you under the Apache License, Version 2.0 (the +-- "License"); you may not use this file except in compliance +-- with the License. You may obtain a copy of the License at +-- +-- http://www.apache.org/licenses/LICENSE-2.0 +-- +-- Unless required by applicable law or agreed to in writing, software +-- distributed under the License is distributed on an "AS IS" BASIS, +-- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +-- See the License for the specific language governing permissions and +-- limitations under the License. + +CREATE CATALOG fluss_catalog WITH ( + 'type' = 'fluss', + 'bootstrap.servers' = 'coordinator-server:19123', + 'paimon.s3.access-key' = 'rustfsadmin', + 'paimon.s3.secret-key' = 'rustfsadmin' +); + +USE CATALOG fluss_catalog; + +CREATE TABLE ${DEVKIT_TABLE} ( + id BIGINT, + payload STRING +) WITH ( + 'bucket.num' = '1', + 'table.datalake.enabled' = 'true', + 'table.datalake.freshness' = '5s' +); diff --git a/devkit/tests/paimon-query-lake.sql b/devkit/tests/paimon-query-lake.sql new file mode 100644 index 0000000000..491c9feefe --- /dev/null +++ b/devkit/tests/paimon-query-lake.sql @@ -0,0 +1,35 @@ +-- Licensed to the Apache Software Foundation (ASF) under one +-- or more contributor license agreements. See the NOTICE file +-- distributed with this work for additional information +-- regarding copyright ownership. The ASF licenses this file +-- to you under the Apache License, Version 2.0 (the +-- "License"); you may not use this file except in compliance +-- with the License. You may obtain a copy of the License at +-- +-- http://www.apache.org/licenses/LICENSE-2.0 +-- +-- Unless required by applicable law or agreed to in writing, software +-- distributed under the License is distributed on an "AS IS" BASIS, +-- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +-- See the License for the specific language governing permissions and +-- limitations under the License. + +SET 'execution.runtime-mode' = 'batch'; +SET 'sql-client.execution.result-mode' = 'tableau'; + +CREATE CATALOG fluss_catalog WITH ( + 'type' = 'fluss', + 'bootstrap.servers' = 'coordinator-server:19123', + 'paimon.s3.access-key' = 'rustfsadmin', + 'paimon.s3.secret-key' = 'rustfsadmin' +); + +USE CATALOG fluss_catalog; + +SELECT + COUNT(*) AS row_count, + SUM(id) AS id_sum, + COUNT(DISTINCT payload) AS payload_count, + MIN(payload) AS first_payload, + MAX(payload) AS last_payload +FROM `${DEVKIT_TABLE}$lake`; diff --git a/devkit/tests/paimon-write-data.sql b/devkit/tests/paimon-write-data.sql new file mode 100644 index 0000000000..1248e51281 --- /dev/null +++ b/devkit/tests/paimon-write-data.sql @@ -0,0 +1,29 @@ +-- Licensed to the Apache Software Foundation (ASF) under one +-- or more contributor license agreements. See the NOTICE file +-- distributed with this work for additional information +-- regarding copyright ownership. The ASF licenses this file +-- to you under the Apache License, Version 2.0 (the +-- "License"); you may not use this file except in compliance +-- with the License. You may obtain a copy of the License at +-- +-- http://www.apache.org/licenses/LICENSE-2.0 +-- +-- Unless required by applicable law or agreed to in writing, software +-- distributed under the License is distributed on an "AS IS" BASIS, +-- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +-- See the License for the specific language governing permissions and +-- limitations under the License. + +CREATE CATALOG fluss_catalog WITH ( + 'type' = 'fluss', + 'bootstrap.servers' = 'coordinator-server:19123', + 'paimon.s3.access-key' = 'rustfsadmin', + 'paimon.s3.secret-key' = 'rustfsadmin' +); + +USE CATALOG fluss_catalog; + +INSERT INTO ${DEVKIT_TABLE} VALUES + (1, 'alpha'), + (2, 'beta'), + (3, 'gamma'); diff --git a/fluss-lake/fluss-lake-hudi/src/main/java/org/apache/fluss/lake/hudi/tiering/HudiWriteTableInfo.java b/fluss-lake/fluss-lake-hudi/src/main/java/org/apache/fluss/lake/hudi/tiering/HudiWriteTableInfo.java index d4711f261a..7c76878ce9 100644 --- a/fluss-lake/fluss-lake-hudi/src/main/java/org/apache/fluss/lake/hudi/tiering/HudiWriteTableInfo.java +++ b/fluss-lake/fluss-lake-hudi/src/main/java/org/apache/fluss/lake/hudi/tiering/HudiWriteTableInfo.java @@ -140,7 +140,10 @@ public static HudiWriteTableInfo create( HoodieEngineContext engineContext = new HoodieLocalEngineContext(metaClient.getStorageConf()); HoodieFlinkWriteClient writeClient = - new HoodieFlinkWriteClient<>(HoodieFlinkEngineContext.DEFAULT, writeConfig); + new HoodieFlinkWriteClient<>( + new HoodieFlinkEngineContext( + HadoopConfigurations.getHadoopConf(flinkConfig)), + writeConfig); Schema avroSchema = Schema.parse(writeConfig.getSchema()); RowType rowType = (RowType) AvroSchemaConverter.convertToDataType(avroSchema).getLogicalType();