Merge pull request #2372 from ZhiHanZ/retry

[Docker/Release]fix docker build and release issues

.github/workflows/databend-release.yml

@@ -79,9 +79,8 @@ jobs:
         run: |
           aws s3 cp databend-${{ steps.get_version.outputs.VERSION }}-${{ matrix.config.target }}.tar.gz s3://repo.databend.rs/databend/${{ steps.get_version.outputs.VERSION }}/databend-${{ steps.get_version.outputs.VERSION }}-${{ matrix.config.target }}.tar.gz
           aws s3 cp bendctl-${{ steps.get_version.outputs.VERSION }}-${{ matrix.config.target }}.tar.gz s3://repo.databend.rs/databend/${{ steps.get_version.outputs.VERSION }}/bendctl-${{ steps.get_version.outputs.VERSION }}-${{ matrix.config.target }}.tar.gz
-          aws s3 cp s3://repo.databend.rs/databend/tags.json ./tags.json
-          cat ./tags.json | jq '[{"name": "${{ steps.get_version.outputs.VERSION }}"}] + .' > ./tags.json
-          cat ./tags.json
+          curl https://api.github.com/repos/datafuselabs/databend/tags > tags.json
+          cat tags.json
           aws s3 cp ./tags.json s3://repo.databend.rs/databend/tags.json
   release-docker:
     name: Release assets
@@ -117,8 +116,8 @@ jobs:
         with:
           push: true
           tags: ${{ secrets.DOCKERHUB_NAMESPACE }}/databend:${{ steps.get_version.outputs.VERSION }}
-          platforms: linux/amd64,linux/arm64
+          platforms: linux/amd64
           context: .
           file: ./docker/release/Dockerfile
           build-args: |
-            version=${{ steps.get_version.outputs.VERSION }}
+            VERSION=${{ steps.get_version.outputs.VERSION }}

Makefile

@@ -1,6 +1,6 @@
 HUB ?= datafuselabs
 TAG ?= latest
-PLATFORM ?= linux/amd64,linux/arm64,linux/arm/v7,linux/arm/v6
+PLATFORM ?= linux/amd64
 VERSION ?= latest
 
 # Setup dev toolchain
@@ -84,11 +84,11 @@ docker:
 	docker build --network host -f docker/Dockerfile -t ${HUB}/databend-query:${TAG} .
 
 docker_release:
-	docker buildx build . -f ./docker/release/Dockerfile  --platform ${PLATFORM} --allow network.host --builder host -t ${HUB}/databend:${TAG} --build-arg version=$VERSION --push
+	docker buildx build . -f ./docker/release/Dockerfile  --platform ${PLATFORM} --allow network.host --builder host -t ${HUB}/databend:${TAG} --build-arg VERSION=${VERSION}--push
 
 # experiment feature: take a look at docker/README.md for detailed multi architecture image build support
 dockerx:
-	docker buildx build . -f ./docker/Dockerfile  --platform ${PLATFORM} --allow network.host --builder host -t ${HUB}/databend-query:${TAG} --push
+	docker buildx build . -f ./docker/Dockerfile  --platform ${PLATFORM} --allow network.host --builder host -t ${HUB}/databend-query:${TAG} --build-arg VERSION=${VERSION} --push
 
 build-perf-tool:
 	cargo build --target x86_64-unknown-linux-gnu --bin databend-benchmark

cli/Cargo.toml

@@ -30,6 +30,7 @@ flate2 = "1.0.22"
 indicatif = "0.16.2"
 run_script = "^0.9.0"
 rustyline = "9.0.0"
+futures = "0.3"
 serde = { version = "1.0", features = ["derive"] }
 serde_json = "1.0"
 sha2 = "0.9.8"
@@ -48,6 +49,8 @@ libc = "0.2"
 lexical-util = "0.8.1"
 async-trait = "0.1"
 webbrowser = "0.5.5"
+async-std = "1.10.0"
+tryhard = "0.4.0"
 
 [dev-dependencies]
 pretty_assertions = "1.0"

cli/src/cmds/packages/fetch.rs

@@ -59,6 +59,7 @@ pub fn get_go_architecture() -> Result<(String, String)> {
     let goarch = match arch {
         "x86_64" => "amd64".to_string(),
         "aarch_64" => "arm64".to_string(),
+        "aarch64" => "arm64".to_string(),
         _ => {
             return Err(CliError::Unknown(format!(
                 "Unsupported go architecture {}",

docker/bootstrap.sh

@@ -2,6 +2,8 @@
 
 /databend-meta --single true &> /tmp/databend-meta.log  &
 P1=$!
+# add health check to remove the race condition issue during databend-query bootstrap
+sleep 1
 /databend-query -c databend-query.toml &> /tmp/databend-query.log  &
 P2=$!
 

docker/databend-query-docker.toml

@@ -2,23 +2,6 @@
 # databend-query -c databend_query_config_spec.toml
 
 # Log
-[log]
-log_level = "DEBUG"
-log_dir = "./_logs"
-
-# Meta
-[meta]
-meta_address = "0.0.0.0:9191"
-meta_username = "root"
-meta_password = "root"
-
-# Store
-[store]
-store_address = "0.0.0.0:9191"
-store_username = "root"
-store_password = "root"
-
-# Query
 [query]
 max_active_sessions = 256
 
@@ -41,3 +24,26 @@ clickhouse_handler_host = "0.0.0.0"
 clickhouse_handler_port = 9001
 
 namespace = "test_cluster"
+
+# Log
+[log]
+log_level = "DEBUG"
+log_dir = "./_logs"
+
+# Meta service.
+[meta]
+meta_address = "0.0.0.0:9191"
+meta_username = "root"
+meta_password = "root"
+meta_client_timeout_in_second = 60
+
+# Storage config.
+[storage]
+# disk|s3
+storage_type = ""
+
+# DISK storage.
+[storage.disk]
+
+# S3 storage.
+[storage.s3]

docker/release/Dockerfile

@@ -1,19 +1,23 @@
 FROM debian as installer
-ARG version
-COPY ./scripts/installer/install-databend.sh /install.sh
+ARG VERSION
+ENV VERSION ${VERSION}
+COPY ./scripts/installer/install-bendctl.sh /install.sh
 RUN apt update \
     && apt install curl tar -y
 # hack ssl issue in curl
 RUN update-ca-certificates -f
 RUN chmod +x /install.sh
-RUN /install.sh $version
+RUN /install.sh -v ${VERSION}
 # double check whether binaries could run on host architecture
-RUN timeout 3 /root/.databend/bin/databend-query || true
-RUN timeout 3 /root/.databend/bin/databend-meta || true
+RUN /root/.databend/bin/bendctl package fetch --databend_dir /root/.databend/ ${VERSION}
+RUN timeout 3 /root/.databend/test/bin/${VERSION}/databend-query || true
+RUN timeout 3 /root/.databend/test/bin/${VERSION}/databend-query || true
 # CI environment build image under GLIBC 2.29 and debian:buster only run GLIBC 2.28
 FROM debian:bullseye
-COPY --from=installer /root/.databend/bin/databend-query  /databend-query
-COPY --from=installer /root/.databend/bin/databend-meta /databend-meta
+ARG VERSION
+ENV VERSION ${VERSION}
+COPY --from=installer /root/.databend/test/bin/${VERSION}/databend-query  /databend-query
+COPY --from=installer /root/.databend/test/bin/${VERSION}/databend-meta  /databend-meta
 COPY ./docker/bootstrap.sh /bootstrap.sh
 COPY ./docker/databend-query-docker.toml  /databend-query.toml
 ENTRYPOINT ["/bootstrap.sh"]