[feat](docker) run the 4.1 all-in-one image as multi-node and cloud compose clusters

docker/runtime/all-in-one/4.1/Dockerfile

@@ -18,9 +18,10 @@
 # ============================================================================
 # Apache Doris all-in-one image for integration testing -- branch-4.1 only.
 #
-# A single container running one FE and one BE, sized for use as a test
-# fixture in downstream CI. Build it with ./build.sh; the commands below are
-# what that script ends up running.
+# One image, three ways to run it: a single container with FE and BE (the
+# default entrypoint role), the multi-node compose file, and the cloud
+# (compute-storage separated) compose file. See README.md. Build it with
+# ./build.sh; the commands below are what that script ends up running.
 #
 #   base flavor (default):
 #     docker buildx build --build-arg DORIS_VERSION=4.1.3 \
@@ -30,23 +31,27 @@
 #   full flavor (adds hudi / trino / maxcompute):
 #     ... --build-arg FLAVOR=full -t apache/doris:all-in-one-4.1.3-full .
 #
-# The build context is the repository root (so ARTIFACT_SOURCE=local can pick
-# up ./output); Dockerfile.dockerignore narrows it down to a few KB.
+# The build context is the repository root; Dockerfile.dockerignore narrows it
+# down to the resource directory. A local ./output or an extracted release
+# tarball comes in through named contexts (ARTIFACT_SOURCE=dir), which
+# build.sh sets up.
 # ============================================================================
 
 # ---- global args: must be declared before the first FROM ----
 ARG DORIS_VERSION=4.1.3
 ARG ARTIFACT_SOURCE=image
 ARG FE_IMAGE=apache/doris:fe-${DORIS_VERSION}
 ARG BE_IMAGE=apache/doris:be-${DORIS_VERSION}
+ARG MS_IMAGE=apache/doris:ms-${DORIS_VERSION}
 ARG BASE_IMAGE=ubuntu:22.04
 
 # ============================ artifact sources ==============================
-# Three interchangeable ways to get fe/ and be/ into /artifacts. Everything
+# Two interchangeable ways to get fe/, be/ and ms/ into /artifacts. Everything
 # downstream only knows about /artifacts, never about where it came from.
 
 FROM ${FE_IMAGE} AS fe-src
 FROM ${BE_IMAGE} AS be-src
+FROM ${MS_IMAGE} AS ms-src
 
 # strip(1) lives here, so this is ubuntu rather than busybox.
 FROM ${BASE_IMAGE} AS artifacts-base
@@ -59,18 +64,16 @@ RUN apt-get update -y && \
 FROM artifacts-base AS artifacts-image
 COPY --from=fe-src /opt/apache-doris/fe /artifacts/fe
 COPY --from=be-src /opt/apache-doris/be /artifacts/be
+COPY --from=ms-src /opt/apache-doris/ms /artifacts/ms
 
-# B. a locally built ./output, for developers testing their own build
-FROM artifacts-base AS artifacts-local
-ARG LOCAL_OUTPUT=output
-COPY ${LOCAL_OUTPUT}/fe /artifacts/fe
-COPY ${LOCAL_OUTPUT}/be /artifacts/be
-
-# C. an extracted release tarball
-FROM artifacts-base AS artifacts-tarball
-ARG TARBALL_DIR
-COPY ${TARBALL_DIR}/fe /artifacts/fe
-COPY ${TARBALL_DIR}/be /artifacts/be
+# B. directories: a locally built ./output or an extracted release tarball.
+# Each part is a named build context (--build-context doris-fe=<dir>/fe ...),
+# which keeps them out of the main context and its dockerignore. A build
+# without a meta-service points doris-ms at an empty directory.
+FROM artifacts-base AS artifacts-dir
+COPY --from=doris-fe / /artifacts/fe
+COPY --from=doris-be / /artifacts/be
+COPY --from=doris-ms / /artifacts/ms
 
 # ========================= strip + flavor pruning ===========================
 # Both have to happen HERE, before the final COPY. Doing them in the runtime
@@ -79,28 +82,33 @@ COPY ${TARBALL_DIR}/be /artifacts/be
 FROM artifacts-${ARTIFACT_SOURCE} AS artifacts
 ARG CTX_PREFIX=docker/runtime/all-in-one/4.1
 ARG FLAVOR=base
-ARG STRIP_BE=debug
+ARG STRIP_MODE=debug
 
 COPY ${CTX_PREFIX}/resource/prune.txt /tmp/prune.txt
 
 RUN set -eux; \
     case "${FLAVOR}" in base|full) ;; *) echo "bad FLAVOR=${FLAVOR}" >&2; exit 1 ;; esac; \
-    # 4.1.3 ships doris_be with debug info: 2213M -> 430M, and --strip-debug
+    # 4.1.3 ships doris_be with debug info: 2213M -> 430M, and the meta-service
+    # payload is 664M of which 456M is an unstripped libfdb_c.so. --strip-debug
     # keeps .symtab so crash backtraces still resolve function names.
-    case "${STRIP_BE}" in \
-      debug) strip --strip-debug /artifacts/be/lib/doris_be ;; \
-      full)  strip -s            /artifacts/be/lib/doris_be ;; \
+    bins="/artifacts/be/lib/doris_be"; \
+    if [ -f /artifacts/ms/lib/doris_cloud ]; then \
+        bins="${bins} /artifacts/ms/lib/doris_cloud $(find /artifacts/ms/lib -name 'libfdb_c.so')"; \
+    fi; \
+    case "${STRIP_MODE}" in \
+      debug) strip --strip-debug ${bins} ;; \
+      full)  strip -s            ${bins} ;; \
       none)  : ;; \
-      *) echo "bad STRIP_BE=${STRIP_BE}" >&2; exit 1 ;; \
+      *) echo "bad STRIP_MODE=${STRIP_MODE}" >&2; exit 1 ;; \
     esac; \
     # "always" rows apply to both flavors, "base" rows only to the base flavor.
     awk -v f="${FLAVOR}" '$1=="always" || $1==f {print $2}' /tmp/prune.txt \
       | while IFS= read -r p; do echo "prune: ${p}"; rm -rf "/artifacts/${p}"; done; \
     rm -f /tmp/prune.txt; \
-    rm -rf /artifacts/fe/log /artifacts/be/log; \
-    mkdir -p /artifacts/fe/log /artifacts/be/log \
+    rm -rf /artifacts/fe/log /artifacts/be/log /artifacts/ms/log; \
+    mkdir -p /artifacts/fe/log /artifacts/be/log /artifacts/ms \
              /artifacts/fe/doris-meta /artifacts/be/storage; \
-    echo "=== artifact size (${FLAVOR}) ==="; du -sh /artifacts/fe /artifacts/be
+    echo "=== artifact size (${FLAVOR}) ==="; du -sh /artifacts/fe /artifacts/be /artifacts/ms
 
 # ================================ runtime ===================================
 
@@ -111,7 +119,8 @@ ARG FLAVOR=base
 ARG CTX_PREFIX=docker/runtime/all-in-one/4.1
 ARG JDK_PKG=openjdk-17-jre-headless
 # FE ships -Xmx8192m, far more than a CI runner can spare; BE ships -Xmx2048m
-# for its JNI-side JVM. Both are rewritten in place below.
+# for its JNI-side JVM. Both are rewritten in place below, and again at
+# container start when FE_HEAP / BE_HEAP are set.
 ARG FE_HEAP=2048m
 ARG BE_HEAP=1024m
 
@@ -144,12 +153,14 @@ ENV JAVA_HOME=/usr/lib/jvm/java \
 
 COPY --from=artifacts /artifacts/fe ${DORIS_HOME}/fe
 COPY --from=artifacts /artifacts/be ${DORIS_HOME}/be
+COPY --from=artifacts /artifacts/ms ${DORIS_HOME}/ms
 COPY ${CTX_PREFIX}/resource/ ${CI_HOME}/
 
 RUN set -eux; \
     # Integration-test defaults are appended, never edited in: the upstream
     # values stay visible above them and the last assignment wins for both the
     # shell-sourced ALL_CAPS vars and the property-style lowercase keys.
+    # Topology keys are not in these files; entrypoint.sh adds them per role.
     cat "${CI_HOME}/conf/fe_ci.conf" >> "${DORIS_HOME}/fe/conf/fe.conf"; \
     cat "${CI_HOME}/conf/be_ci.conf" >> "${DORIS_HOME}/be/conf/be.conf"; \
     # The heap lives inside a long JAVA_OPTS_FOR_JDK_17 line that also carries
@@ -162,14 +173,15 @@ RUN set -eux; \
     chmod +x "${CI_HOME}"/*.sh
 
 LABEL org.opencontainers.image.title="Apache Doris all-in-one" \
-      org.opencontainers.image.description="Single-container Doris (FE+BE) for integration testing" \
+      org.opencontainers.image.description="Single-container Doris (FE+BE) for integration testing; also the node image of the multi-node and cloud compose files" \
       org.opencontainers.image.version="${DORIS_VERSION}" \
       org.opencontainers.image.source="https://github.com/apache/doris" \
       org.apache.doris.allinone.flavor="${FLAVOR}" \
       org.apache.doris.allinone.branch="4.1"
 
 # 8030 FE http | 9030 FE mysql | 8040 BE http (stream load) | 9050 BE heartbeat
-EXPOSE 8030 9030 8040 9050
+# 5000 meta-service (DORIS_ROLE=ms / recycler)
+EXPOSE 8030 9030 8040 9050 5000
 
 WORKDIR ${DORIS_HOME}
 

docker/runtime/all-in-one/4.1/Dockerfile.dockerignore

@@ -18,10 +18,9 @@
 # BuildKit reads this in preference to the repository root .dockerignore when
 # building with -f docker/runtime/all-in-one/4.1/Dockerfile.
 #
-# The context has to be the repository root so that ARTIFACT_SOURCE=local can
-# reach ./output, but nothing else in the tree is needed. Exclude everything,
-# then add back the two paths the build actually reads.
+# The context is the repository root, but the only thing the build reads from
+# it is the resource directory. Local builds and release tarballs arrive as
+# named build contexts (see build.sh), not through this context, so nothing
+# else needs to be let in.
 *
 !docker/runtime/all-in-one/4.1/resource
-!output/fe
-!output/be

docker/runtime/all-in-one/4.1/README.md

@@ -20,25 +20,31 @@ under the License.
 # Doris all-in-one image (branch-4.1)
 
 One FE and one BE in a single container, sized to be a test fixture in a
-downstream project's CI. It is assembled from the official `apache/doris:fe-*`
-and `apache/doris:be-*` images, so a new Doris release needs no repackaging of
+downstream project's CI. It is assembled from the official `apache/doris:fe-*`,
+`be-*` and `ms-*` images, so a new Doris release needs no repackaging of
 anything here.
 
+The same image also runs one Doris process per container, which is what the
+two compose files under `compose/` do: a three-FE / three-BE cluster, and a
+compute-storage separated (cloud) cluster with its meta-service, FoundationDB
+and MinIO. See [Multi-node and cloud clusters](#multi-node-and-cloud-clusters).
+
 This directory targets the **4.1 release line only**. Other lines differ enough
 in payload layout to deserve their own directory rather than a version switch.
 
 ## Two tags
 
 | tag | covers | size |
 |---|---|---|
-| `apache/doris:all-in-one-<version>` | internal tables, Hive, Iceberg (including system tables), Paimon, JDBC catalogs, external-table writeback, Java UDF | 2.46 GB |
-| `apache/doris:all-in-one-<version>-full` | the above plus Hudi, Trino connector, MaxCompute | 2.99 GB |
+| `apache/doris:all-in-one-<version>` | internal tables, Hive, Iceberg (including system tables), Paimon, JDBC catalogs, external-table writeback, Java UDF | 2.68 GB |
+| `apache/doris:all-in-one-<version>-full` | the above plus Hudi, Trino connector, MaxCompute | 3.21 GB |
 
 Pick `-full` only if the tests touch Hudi, the Trino connector or MaxCompute.
 
 Both come up `healthy` in under 20 seconds. Sizes are the uncompressed layer
-sum measured on 4.1.3/arm64, against 4.9 GB for the same payload untouched.
-`docker image inspect --format '{{.Size}}'` reports 1.61 / 2.06 GiB for them;
+sum measured on 4.1.3/arm64, against 5.6 GB for the same payload untouched;
+about 0.2 GB of each is the meta-service, which only the cloud compose file
+uses. `docker image inspect --format '{{.Size}}'` reports 1.73 / 2.18 GiB;
 `docker images` can print a much larger figure when the containerd image store
 is enabled, because it adds the compressed blobs to the unpacked snapshot.
 
@@ -54,12 +60,19 @@ A plain run builds for the host architecture only. Multi-arch needs an explicit
 # base tag only, then smoke test it
 ./build.sh -v 4.1.3 -f base -t
 
-# from a locally built ./output instead
+# from a locally built ./output instead (build.sh --fe --be --cloud; without
+# --cloud there is no ms/ and the image cannot serve the cloud compose file)
 ./build.sh -v dev -s local
+
+# from an extracted release tarball, e.g. while the component images for a
+# new release are not on Docker Hub yet
+./build.sh -v 4.1.4 -s tarball --tarball-dir ~/apache-doris-4.1.4-bin-arm64
 ```
 
 `./build.sh --help` lists the rest. The build context is the repository root,
-narrowed to a few KB by `Dockerfile.dockerignore`; run the script from anywhere.
+narrowed to a few KB by `Dockerfile.dockerignore`; a local `./output` or a
+tarball directory is passed as BuildKit named contexts instead, so it can live
+anywhere. Run the script from anywhere.
 
 ## Multi-architecture
 
@@ -208,6 +221,7 @@ deliberately omits them and says so.
 | | base | -full |
 |---|---|---|
 | `strip --strip-debug` on `doris_be` | 2213 MB → 430 MB (450 MB on arm64) | same |
+| `strip --strip-debug` on the meta-service (`doris_cloud` and its two `libfdb_c.so`) | 664 MB → 183 MB | same |
 | `be/lib/meta_tool`, `be/lib/cdc_client`, `fe/arthas` | removed | removed |
 | hudi / trino / maxcompute scanners | removed | kept |
 
@@ -235,17 +249,117 @@ It does **not** read a real external table. To check that end of things, point
 the built image at the fixtures under `docker/thirdparties` and run an Iceberg
 or Hive query by hand.
 
+## Multi-node and cloud clusters
+
+The image's entrypoint takes a `DORIS_ROLE`. The default, `all`, is the
+single container described above; the other roles run one process each and
+are what the compose files are built from:
+
+| `DORIS_ROLE` | runs | needs |
+|---|---|---|
+| `all` | FE + BE on loopback | nothing |
+| `fe` | one FE; bootstraps the cluster when `FE_MASTER` is empty, otherwise registers with it as `FE_ROLE` (`follower` \| `observer`) and joins | `FE_MASTER` |
+| `be` | one BE, registered with `FE_MASTER`; in cloud mode into `COMPUTE_GROUP` | `FE_MASTER` |
+| `ms`, `recycler` | the cloud meta-service / recycler | `FDB_CLUSTER` |
+| `cloud-init` | one-shot: creates the cloud instance on an S3-compatible store, then exits | `MS_ENDPOINT`, `INSTANCE_ID`, `S3_*` |
+| `client` | waits for `EXPECT_FE` / `EXPECT_BE` live nodes, then idles with `mysql` and `curl` | `FE_MASTER` |
+
+`DEPLOY_MODE=cloud` turns `fe` and `be` into cloud nodes (`deploy_mode`,
+`meta_service_endpoint`, file cache); the FE takes `INSTANCE_ID` as its
+`cluster_id` and manages nodes by SQL, so no `cloud_unique_id` has to be handed
+out. `FE_MASTER` may list several FEs (`fe-1,fe-2,fe-3`): the first one that
+answers is used, which keeps a node restart from waiting on the one FE that is
+down. An FE that already has metadata rejoins on its own. Every role honours
+`FE_CONFIG_EXTRA` / `BE_CONFIG_EXTRA` / `MS_CONFIG_EXTRA`, `FE_HEAP` / `BE_HEAP`
+and drops a ready flag that the image `HEALTHCHECK` keys on.
+
+Topology-dependent settings (`priority_networks`, replica count, balancing,
+cloud keys) are written by the entrypoint at container start; the conf files
+baked into the image only carry the size-related defaults.
+
+### `compose/multi-node.yml` — three FEs, three BEs
+
+```shell
+cd docker/runtime/all-in-one/4.1/compose
+docker compose -f multi-node.yml up --wait        # ~35 s to healthy
+docker compose -f multi-node.yml exec client mysql -uroot -hfe-1 -P9030
+docker compose -f multi-node.yml kill fe-1        # a new master in a few seconds
+docker compose -f multi-node.yml start fe-1       # rejoins as a follower
+docker compose -f multi-node.yml down             # nothing persists
+```
+
+Tables default to three replicas, one per BE. `fe-3` becomes an observer with
+`FE3_ROLE=observer`. Host ports: `9030`/`8030` for `fe-1`, `9031`/`8031` and
+`9032`/`8032` for the other two, `8040` for `be-1`.
+
+### `compose/cloud.yml` — compute-storage separated
+
+```shell
+docker compose -f cloud.yml up --wait             # ~50 s to healthy
+docker compose -f cloud.yml --profile ha up --wait   # ... plus two follower FEs
+docker compose -f cloud.yml exec client mysql -uroot -hfe-1 -P9030
+```
+
+```
+fdb ─ fdb-init ─┬─ ms ─ cloud-init ─ fe-1 ─┬─ be-1, be-2  (compute group cg_a)
+                └─ recycler                 ├─ be-3        (compute group cg_b)
+minio ─ minio-init ─┘                       └─ client
+```
+
+`cloud-init` creates the instance in storage-vault mode with MinIO as the
+vault (path-style, plain HTTP), and `fe-1` marks `built_in_storage_vault` as
+the default, so `CREATE TABLE` works as is. Both are idempotent, so
+`docker compose stop` / `up` keeps the data; `down` wipes everything. The
+MinIO console is on `9001` (`minioadmin` / `minioadmin`), the meta-service
+HTTP API on `15000` (`5000` is taken by AirPlay on macOS).
+
+What this is for: `use @cg_b`, `SHOW COMPUTE GROUPS`, `ALTER SYSTEM ADD
+BACKEND ... ("tag.compute_group_name" = ...)`, storage vaults, warm-up, FE
+failover in cloud mode, watching objects land in the bucket. What it is not:
+a stand-in for the cloud regression pipelines, and the `docker()` suites in
+`regression-test/suites/cloud_p0` still need `doris-compose`.
+
+### Things to know
+
+- **Addresses.** Nodes get fixed IPs on a private subnet (`SUBNET`, default
+  `172.31.80` for cloud and `172.31.81` for multi-node), so a restarted
+  container keeps the identity Doris knows it by. Both files can run at once
+  if one of them is given other host ports.
+- **From the host.** Use the published ports. On Docker Desktop the container
+  addresses are not routable from the host, so a stream load from the host
+  cannot follow FE's redirect to a BE; run it from the `client` service
+  instead, which is inside the network. Linux hosts can reach the nodes
+  directly.
+- **Memory.** Defaults are `FE_HEAP=1024m` and `mem_limit = 25%` per BE. A
+  full cloud cluster is around 9 GB; give Docker Desktop 12 GB or more, and do
+  not expect both topologies to fit side by side on a 16 GB VM.
+- **FoundationDB on arm64.** The upstream `foundationdb/foundationdb:7.1.x`
+  images are amd64 only, so on an Apple Silicon host `fdb` runs under
+  emulation. It is fine for this purpose; `FDB_IMAGE` / `FDB_PLATFORM` switch
+  it out. The meta-service links the 7.1 client, so stay on a 7.1 server.
+- **Smoke test.** `compose/smoke-test.sh <multi-node|cloud> [image:tag]`
+  brings the topology up under its own project name, subnet and host ports,
+  checks replicas or compute groups and the vault, stream loads through the
+  client, kills the master FE, restarts a BE while the old master is down,
+  brings the master back, and tears everything down. Two minutes each.
+
 ## Layout
 
 ```
 4.1/
-├── Dockerfile                 three artifact sources -> strip + prune -> runtime
+├── Dockerfile                 artifact sources -> strip + prune -> runtime
 ├── Dockerfile.dockerignore    keeps the repo-root context to a few KB
 ├── build.sh                   the only entry point you need
+├── compose/
+│   ├── multi-node.yml         3 FE + 3 BE, storage and compute together
+│   ├── cloud.yml              fdb + ms + recycler + minio + 1..3 FE + 3 BE in 2 compute groups
+│   └── smoke-test.sh          guards the two compose files
 └── resource/
-    ├── entrypoint.sh          FE -> readiness -> BE -> register -> fail-fast wait
-    ├── health_check.sh        backs HEALTHCHECK
-    ├── smoke-test.sh          guards prune.txt
+    ├── entrypoint.sh          one entrypoint, dispatched on DORIS_ROLE
+    ├── lib.sh                 shared by entrypoint.sh, health_check.sh and cloud_init.sh
+    ├── cloud_init.sh          creates the cloud instance (DORIS_ROLE=cloud-init)
+    ├── health_check.sh        backs HEALTHCHECK, per role
+    ├── smoke-test.sh          guards prune.txt and the single-container role
     ├── prune.txt              what each flavor drops, and what must never be dropped
     └── conf/{fe_ci.conf,be_ci.conf}   appended to the upstream conf at build time
 ```