diff --git a/.github/workflows/docker-images.yml b/.github/workflows/docker-images.yml index bc4a48ce4d..52e7326041 100644 --- a/.github/workflows/docker-images.yml +++ b/.github/workflows/docker-images.yml @@ -160,7 +160,7 @@ jobs: && echo "AMORO_VERSION=${AMORO_VERSION}" >> $GITHUB_OUTPUT - name: Build optimizer module with Maven - run: mvn clean package -pl 'amoro-ams/amoro-ams-optimizer/flink-optimizer' -am -e ${OPTIMIZER_FLINK} -DskipTests -B -ntp + run: mvn clean package -pl 'amoro-ams/amoro-ams-optimizer/amoro-optimizer-flink' -am -e ${OPTIMIZER_FLINK} -DskipTests -B -ntp - name: Build and Push Flink Optimizer Docker Image uses: docker/build-push-action@v4 @@ -174,7 +174,7 @@ jobs: tags: ${{ steps.meta.outputs.tags }} build-args: | FLINK_VERSION=${{ matrix.flink }} - OPTIMIZER_JOB=amoro-ams/amoro-ams-optimizer/flink-optimizer/target/flink-optimizer-${{ env.AMORO_VERSION }}-jar-with-dependencies.jar + OPTIMIZER_JOB=amoro-ams/amoro-ams-optimizer/amoro-optimizer-flink/target/amoro-optimizer-flink-${{ env.AMORO_VERSION }}-jar-with-dependencies.jar docker-optimizer-spark: name: Push Amoro Optimizer-Spark Docker Image to Docker Hub @@ -232,7 +232,7 @@ jobs: && echo "AMORO_VERSION=${AMORO_VERSION}" >> $GITHUB_OUTPUT - name: Build optimizer module with Maven - run: mvn clean package -pl 'amoro-ams/amoro-ams-optimizer/spark-optimizer' -am -e ${OPTIMIZER_SPARK} -DskipTests -B -ntp + run: mvn clean package -pl 'amoro-ams/amoro-ams-optimizer/amoro-optimizer-spark' -am -e ${OPTIMIZER_SPARK} -DskipTests -B -ntp - name: Build and Push Spark Optimizer Docker Image uses: docker/build-push-action@v4 @@ -246,7 +246,7 @@ jobs: tags: ${{ steps.meta.outputs.tags }} build-args: | SPARK_VERSION=${{ matrix.spark }} - OPTIMIZER_JOB=amoro-ams/amoro-ams-optimizer/spark-optimizer/target/spark-optimizer-${{ env.AMORO_VERSION }}-jar-with-dependencies.jar + OPTIMIZER_JOB=amoro-ams/amoro-ams-optimizer/amoro-optimizer-spark/target/amoro-optimizer-spark-${{ env.AMORO_VERSION }}-jar-with-dependencies.jar docker-quickdemo: diff --git a/.gitignore b/.gitignore index f161009284..f20c59c19b 100644 --- a/.gitignore +++ b/.gitignore @@ -55,7 +55,7 @@ core/derby.log target/ # keberos configs -*arctic_krb_conf/ +*amoro_krb_conf/ # local config conf/ @@ -72,4 +72,4 @@ amoro-ams/amoro-ams-dashboard/src/main/resources/static/ amoro-ams/amoro-ams-dashboard/node/ !amoro-ams/dist/src/main/arctic-bin/bin/ -!amoro-ams/dist/src/main/arctic-bin/conf/ \ No newline at end of file +!amoro-ams/dist/src/main/arctic-bin/conf/ diff --git a/amoro-ams/amoro-ams-optimizer/common/pom.xml b/amoro-ams/amoro-ams-optimizer/amoro-optimizer-common/pom.xml similarity index 97% rename from amoro-ams/amoro-ams-optimizer/common/pom.xml rename to amoro-ams/amoro-ams-optimizer/amoro-optimizer-common/pom.xml index 370a73a9ce..11851ae144 100644 --- a/amoro-ams/amoro-ams-optimizer/common/pom.xml +++ b/amoro-ams/amoro-ams-optimizer/amoro-optimizer-common/pom.xml @@ -28,7 +28,7 @@ 4.0.0 - optimizer-common + amoro-optimizer-common Amoro Project AMS Optimizer Common https://amoro.apache.org diff --git a/amoro-ams/amoro-ams-optimizer/common/src/main/java/org/apache/amoro/optimizer/common/AbstractOptimizerOperator.java b/amoro-ams/amoro-ams-optimizer/amoro-optimizer-common/src/main/java/org/apache/amoro/optimizer/common/AbstractOptimizerOperator.java similarity index 100% rename from amoro-ams/amoro-ams-optimizer/common/src/main/java/org/apache/amoro/optimizer/common/AbstractOptimizerOperator.java rename to amoro-ams/amoro-ams-optimizer/amoro-optimizer-common/src/main/java/org/apache/amoro/optimizer/common/AbstractOptimizerOperator.java diff --git a/amoro-ams/amoro-ams-optimizer/common/src/main/java/org/apache/amoro/optimizer/common/Optimizer.java b/amoro-ams/amoro-ams-optimizer/amoro-optimizer-common/src/main/java/org/apache/amoro/optimizer/common/Optimizer.java similarity index 100% rename from amoro-ams/amoro-ams-optimizer/common/src/main/java/org/apache/amoro/optimizer/common/Optimizer.java rename to amoro-ams/amoro-ams-optimizer/amoro-optimizer-common/src/main/java/org/apache/amoro/optimizer/common/Optimizer.java diff --git a/amoro-ams/amoro-ams-optimizer/common/src/main/java/org/apache/amoro/optimizer/common/OptimizerConfig.java b/amoro-ams/amoro-ams-optimizer/amoro-optimizer-common/src/main/java/org/apache/amoro/optimizer/common/OptimizerConfig.java similarity index 100% rename from amoro-ams/amoro-ams-optimizer/common/src/main/java/org/apache/amoro/optimizer/common/OptimizerConfig.java rename to amoro-ams/amoro-ams-optimizer/amoro-optimizer-common/src/main/java/org/apache/amoro/optimizer/common/OptimizerConfig.java diff --git a/amoro-ams/amoro-ams-optimizer/common/src/main/java/org/apache/amoro/optimizer/common/OptimizerExecutor.java b/amoro-ams/amoro-ams-optimizer/amoro-optimizer-common/src/main/java/org/apache/amoro/optimizer/common/OptimizerExecutor.java similarity index 100% rename from amoro-ams/amoro-ams-optimizer/common/src/main/java/org/apache/amoro/optimizer/common/OptimizerExecutor.java rename to amoro-ams/amoro-ams-optimizer/amoro-optimizer-common/src/main/java/org/apache/amoro/optimizer/common/OptimizerExecutor.java diff --git a/amoro-ams/amoro-ams-optimizer/common/src/main/java/org/apache/amoro/optimizer/common/OptimizerToucher.java b/amoro-ams/amoro-ams-optimizer/amoro-optimizer-common/src/main/java/org/apache/amoro/optimizer/common/OptimizerToucher.java similarity index 100% rename from amoro-ams/amoro-ams-optimizer/common/src/main/java/org/apache/amoro/optimizer/common/OptimizerToucher.java rename to amoro-ams/amoro-ams-optimizer/amoro-optimizer-common/src/main/java/org/apache/amoro/optimizer/common/OptimizerToucher.java diff --git a/amoro-ams/amoro-ams-optimizer/common/src/test/java/org/apache/amoro/optimizer/common/OptimizerTestBase.java b/amoro-ams/amoro-ams-optimizer/amoro-optimizer-common/src/test/java/org/apache/amoro/optimizer/common/OptimizerTestBase.java similarity index 100% rename from amoro-ams/amoro-ams-optimizer/common/src/test/java/org/apache/amoro/optimizer/common/OptimizerTestBase.java rename to amoro-ams/amoro-ams-optimizer/amoro-optimizer-common/src/test/java/org/apache/amoro/optimizer/common/OptimizerTestBase.java diff --git a/amoro-ams/amoro-ams-optimizer/common/src/test/java/org/apache/amoro/optimizer/common/OptimizerTestHelpers.java b/amoro-ams/amoro-ams-optimizer/amoro-optimizer-common/src/test/java/org/apache/amoro/optimizer/common/OptimizerTestHelpers.java similarity index 100% rename from amoro-ams/amoro-ams-optimizer/common/src/test/java/org/apache/amoro/optimizer/common/OptimizerTestHelpers.java rename to amoro-ams/amoro-ams-optimizer/amoro-optimizer-common/src/test/java/org/apache/amoro/optimizer/common/OptimizerTestHelpers.java diff --git a/amoro-ams/amoro-ams-optimizer/common/src/test/java/org/apache/amoro/optimizer/common/TestOptimizer.java b/amoro-ams/amoro-ams-optimizer/amoro-optimizer-common/src/test/java/org/apache/amoro/optimizer/common/TestOptimizer.java similarity index 100% rename from amoro-ams/amoro-ams-optimizer/common/src/test/java/org/apache/amoro/optimizer/common/TestOptimizer.java rename to amoro-ams/amoro-ams-optimizer/amoro-optimizer-common/src/test/java/org/apache/amoro/optimizer/common/TestOptimizer.java diff --git a/amoro-ams/amoro-ams-optimizer/common/src/test/java/org/apache/amoro/optimizer/common/TestOptimizerConfig.java b/amoro-ams/amoro-ams-optimizer/amoro-optimizer-common/src/test/java/org/apache/amoro/optimizer/common/TestOptimizerConfig.java similarity index 100% rename from amoro-ams/amoro-ams-optimizer/common/src/test/java/org/apache/amoro/optimizer/common/TestOptimizerConfig.java rename to amoro-ams/amoro-ams-optimizer/amoro-optimizer-common/src/test/java/org/apache/amoro/optimizer/common/TestOptimizerConfig.java diff --git a/amoro-ams/amoro-ams-optimizer/common/src/test/java/org/apache/amoro/optimizer/common/TestOptimizerExecutor.java b/amoro-ams/amoro-ams-optimizer/amoro-optimizer-common/src/test/java/org/apache/amoro/optimizer/common/TestOptimizerExecutor.java similarity index 100% rename from amoro-ams/amoro-ams-optimizer/common/src/test/java/org/apache/amoro/optimizer/common/TestOptimizerExecutor.java rename to amoro-ams/amoro-ams-optimizer/amoro-optimizer-common/src/test/java/org/apache/amoro/optimizer/common/TestOptimizerExecutor.java diff --git a/amoro-ams/amoro-ams-optimizer/common/src/test/java/org/apache/amoro/optimizer/common/TestOptimizerToucher.java b/amoro-ams/amoro-ams-optimizer/amoro-optimizer-common/src/test/java/org/apache/amoro/optimizer/common/TestOptimizerToucher.java similarity index 100% rename from amoro-ams/amoro-ams-optimizer/common/src/test/java/org/apache/amoro/optimizer/common/TestOptimizerToucher.java rename to amoro-ams/amoro-ams-optimizer/amoro-optimizer-common/src/test/java/org/apache/amoro/optimizer/common/TestOptimizerToucher.java diff --git a/amoro-ams/amoro-ams-optimizer/flink-optimizer/pom.xml b/amoro-ams/amoro-ams-optimizer/amoro-optimizer-flink/pom.xml similarity index 98% rename from amoro-ams/amoro-ams-optimizer/flink-optimizer/pom.xml rename to amoro-ams/amoro-ams-optimizer/amoro-optimizer-flink/pom.xml index 56696e31ba..98cadb2fc3 100644 --- a/amoro-ams/amoro-ams-optimizer/flink-optimizer/pom.xml +++ b/amoro-ams/amoro-ams-optimizer/amoro-optimizer-flink/pom.xml @@ -28,7 +28,7 @@ 4.0.0 - flink-optimizer + amoro-optimizer-flink Amoro Project AMS Flink Optimizer https://amoro.apache.org @@ -42,7 +42,7 @@ org.apache.amoro - optimizer-common + amoro-optimizer-common ${project.version} diff --git a/amoro-ams/amoro-ams-optimizer/flink-optimizer/src/main/java/org/apache/amoro/optimizer/flink/FlinkExecutor.java b/amoro-ams/amoro-ams-optimizer/amoro-optimizer-flink/src/main/java/org/apache/amoro/optimizer/flink/FlinkExecutor.java similarity index 100% rename from amoro-ams/amoro-ams-optimizer/flink-optimizer/src/main/java/org/apache/amoro/optimizer/flink/FlinkExecutor.java rename to amoro-ams/amoro-ams-optimizer/amoro-optimizer-flink/src/main/java/org/apache/amoro/optimizer/flink/FlinkExecutor.java diff --git a/amoro-ams/amoro-ams-optimizer/flink-optimizer/src/main/java/org/apache/amoro/optimizer/flink/FlinkOptimizer.java b/amoro-ams/amoro-ams-optimizer/amoro-optimizer-flink/src/main/java/org/apache/amoro/optimizer/flink/FlinkOptimizer.java similarity index 100% rename from amoro-ams/amoro-ams-optimizer/flink-optimizer/src/main/java/org/apache/amoro/optimizer/flink/FlinkOptimizer.java rename to amoro-ams/amoro-ams-optimizer/amoro-optimizer-flink/src/main/java/org/apache/amoro/optimizer/flink/FlinkOptimizer.java diff --git a/amoro-ams/amoro-ams-optimizer/flink-optimizer/src/main/java/org/apache/amoro/optimizer/flink/FlinkOptimizerExecutor.java b/amoro-ams/amoro-ams-optimizer/amoro-optimizer-flink/src/main/java/org/apache/amoro/optimizer/flink/FlinkOptimizerExecutor.java similarity index 100% rename from amoro-ams/amoro-ams-optimizer/flink-optimizer/src/main/java/org/apache/amoro/optimizer/flink/FlinkOptimizerExecutor.java rename to amoro-ams/amoro-ams-optimizer/amoro-optimizer-flink/src/main/java/org/apache/amoro/optimizer/flink/FlinkOptimizerExecutor.java diff --git a/amoro-ams/amoro-ams-optimizer/flink-optimizer/src/main/java/org/apache/amoro/optimizer/flink/FlinkToucher.java b/amoro-ams/amoro-ams-optimizer/amoro-optimizer-flink/src/main/java/org/apache/amoro/optimizer/flink/FlinkToucher.java similarity index 100% rename from amoro-ams/amoro-ams-optimizer/flink-optimizer/src/main/java/org/apache/amoro/optimizer/flink/FlinkToucher.java rename to amoro-ams/amoro-ams-optimizer/amoro-optimizer-flink/src/main/java/org/apache/amoro/optimizer/flink/FlinkToucher.java diff --git a/amoro-ams/amoro-ams-optimizer/spark-optimizer/pom.xml b/amoro-ams/amoro-ams-optimizer/amoro-optimizer-spark/pom.xml similarity index 98% rename from amoro-ams/amoro-ams-optimizer/spark-optimizer/pom.xml rename to amoro-ams/amoro-ams-optimizer/amoro-optimizer-spark/pom.xml index 8fd5f0fa60..8ad91d0e6d 100644 --- a/amoro-ams/amoro-ams-optimizer/spark-optimizer/pom.xml +++ b/amoro-ams/amoro-ams-optimizer/amoro-optimizer-spark/pom.xml @@ -28,7 +28,7 @@ 4.0.0 - spark-optimizer + amoro-optimizer-spark Amoro Project AMS Spark Optimizer https://amoro.apache.org @@ -40,7 +40,7 @@ org.apache.amoro - optimizer-common + amoro-optimizer-common ${project.version} @@ -183,4 +183,4 @@ - \ No newline at end of file + diff --git a/amoro-ams/amoro-ams-optimizer/spark-optimizer/src/main/java/org/apache/amoro/optimizer/spark/SparkOptimizer.java b/amoro-ams/amoro-ams-optimizer/amoro-optimizer-spark/src/main/java/org/apache/amoro/optimizer/spark/SparkOptimizer.java similarity index 100% rename from amoro-ams/amoro-ams-optimizer/spark-optimizer/src/main/java/org/apache/amoro/optimizer/spark/SparkOptimizer.java rename to amoro-ams/amoro-ams-optimizer/amoro-optimizer-spark/src/main/java/org/apache/amoro/optimizer/spark/SparkOptimizer.java diff --git a/amoro-ams/amoro-ams-optimizer/spark-optimizer/src/main/java/org/apache/amoro/optimizer/spark/SparkOptimizerExecutor.java b/amoro-ams/amoro-ams-optimizer/amoro-optimizer-spark/src/main/java/org/apache/amoro/optimizer/spark/SparkOptimizerExecutor.java similarity index 100% rename from amoro-ams/amoro-ams-optimizer/spark-optimizer/src/main/java/org/apache/amoro/optimizer/spark/SparkOptimizerExecutor.java rename to amoro-ams/amoro-ams-optimizer/amoro-optimizer-spark/src/main/java/org/apache/amoro/optimizer/spark/SparkOptimizerExecutor.java diff --git a/amoro-ams/amoro-ams-optimizer/spark-optimizer/src/main/java/org/apache/amoro/optimizer/spark/SparkOptimizingTaskFunction.java b/amoro-ams/amoro-ams-optimizer/amoro-optimizer-spark/src/main/java/org/apache/amoro/optimizer/spark/SparkOptimizingTaskFunction.java similarity index 100% rename from amoro-ams/amoro-ams-optimizer/spark-optimizer/src/main/java/org/apache/amoro/optimizer/spark/SparkOptimizingTaskFunction.java rename to amoro-ams/amoro-ams-optimizer/amoro-optimizer-spark/src/main/java/org/apache/amoro/optimizer/spark/SparkOptimizingTaskFunction.java diff --git a/amoro-ams/amoro-ams-optimizer/standalone-optimizer/pom.xml b/amoro-ams/amoro-ams-optimizer/amoro-optimizer-standalone/pom.xml similarity index 95% rename from amoro-ams/amoro-ams-optimizer/standalone-optimizer/pom.xml rename to amoro-ams/amoro-ams-optimizer/amoro-optimizer-standalone/pom.xml index 5407ee6e43..de575faf2e 100644 --- a/amoro-ams/amoro-ams-optimizer/standalone-optimizer/pom.xml +++ b/amoro-ams/amoro-ams-optimizer/amoro-optimizer-standalone/pom.xml @@ -28,7 +28,7 @@ 4.0.0 - standalone-optimizer + amoro-optimizer-standalone Amoro Project AMS Standalone Optimizer https://amoro.apache.org @@ -36,7 +36,7 @@ org.apache.amoro - optimizer-common + amoro-optimizer-common ${project.version} diff --git a/amoro-ams/amoro-ams-optimizer/standalone-optimizer/src/main/java/org/apache/amoro/optimizer/standalone/StandaloneOptimizer.java b/amoro-ams/amoro-ams-optimizer/amoro-optimizer-standalone/src/main/java/org/apache/amoro/optimizer/standalone/StandaloneOptimizer.java similarity index 100% rename from amoro-ams/amoro-ams-optimizer/standalone-optimizer/src/main/java/org/apache/amoro/optimizer/standalone/StandaloneOptimizer.java rename to amoro-ams/amoro-ams-optimizer/amoro-optimizer-standalone/src/main/java/org/apache/amoro/optimizer/standalone/StandaloneOptimizer.java diff --git a/amoro-ams/amoro-ams-optimizer/pom.xml b/amoro-ams/amoro-ams-optimizer/pom.xml index 2da0e62575..b520437e07 100644 --- a/amoro-ams/amoro-ams-optimizer/pom.xml +++ b/amoro-ams/amoro-ams-optimizer/pom.xml @@ -33,10 +33,10 @@ https://amoro.apache.org - flink-optimizer - spark-optimizer - standalone-optimizer - common + amoro-optimizer-common + amoro-optimizer-flink + amoro-optimizer-spark + amoro-optimizer-standalone diff --git a/amoro-ams/amoro-ams-server/pom.xml b/amoro-ams/amoro-ams-server/pom.xml index 481fc72c81..bdf6650703 100644 --- a/amoro-ams/amoro-ams-server/pom.xml +++ b/amoro-ams/amoro-ams-server/pom.xml @@ -300,7 +300,7 @@ org.apache.amoro - standalone-optimizer + amoro-optimizer-standalone ${project.version} runtime diff --git a/amoro-ams/dist/pom.xml b/amoro-ams/dist/pom.xml index 9152baeccd..659c3c0652 100644 --- a/amoro-ams/dist/pom.xml +++ b/amoro-ams/dist/pom.xml @@ -43,18 +43,18 @@ org.apache.amoro - standalone-optimizer + amoro-optimizer-standalone ${project.version} org.apache.amoro - flink-optimizer + amoro-optimizer-flink ${project.version} compile org.apache.amoro - spark-optimizer + amoro-optimizer-spark ${project.version} compile diff --git a/amoro-ams/dist/src/main/assemblies/bin.xml b/amoro-ams/dist/src/main/assemblies/bin.xml index b3cdddaa27..8630e1cdd1 100644 --- a/amoro-ams/dist/src/main/assemblies/bin.xml +++ b/amoro-ams/dist/src/main/assemblies/bin.xml @@ -30,7 +30,7 @@ - ../amoro-ams-optimizer/flink-optimizer/target/flink-optimizer-${project.version}-jar-with-dependencies.jar + ../amoro-ams-optimizer/amoro-optimizer-flink/target/amoro-optimizer-flink-${project.version}-jar-with-dependencies.jar plugin/optimizer/flink optimizer-job.jar @@ -38,7 +38,7 @@ - ../amoro-ams-optimizer/spark-optimizer/target/spark-optimizer-${project.version}-jar-with-dependencies.jar + ../amoro-ams-optimizer/amoro-optimizer-spark/target/amoro-optimizer-spark-${project.version}-jar-with-dependencies.jar plugin/optimizer/spark optimizer-job.jar diff --git a/docker/README.md b/docker/README.md index 4732f2908f..6849e02d36 100644 --- a/docker/README.md +++ b/docker/README.md @@ -19,13 +19,16 @@ We provide a bash script to help you build docker image easier. -You can build all images via the script in current dir. +You can control which image is built by the follow parameters : -```shell -./build.sh all -``` +- amoro +- optimizer-flink +- optimizer-spark +- quickdemo +- namenode +- datanode -or just build only one image. +e.g. ```shell ./build.sh amoro diff --git a/docker/build.sh b/docker/build.sh index 7b84ab8b08..7cc3182d71 100755 --- a/docker/build.sh +++ b/docker/build.sh @@ -29,9 +29,9 @@ SPARK_VERSION=3.3.3 HADOOP_VERSION=2.10.2 DEBIAN_MIRROR=http://deb.debian.org APACHE_ARCHIVE=https://archive.apache.org/dist -FLINK_OPTIMIZER_JOB_PATH=ams/optimizer/flink-optimizer/target/flink-optimizer-${AMORO_VERSION}-jar-with-dependencies.jar +FLINK_OPTIMIZER_JOB_PATH=amoro-ams/amoro-ams-optimizer/amoro-optimizer-flink/target/amoro-optimizer-flink-${AMORO_VERSION}-jar-with-dependencies.jar FLINK_OPTIMIZER_JOB=${PROJECT_HOME}/${FLINK_OPTIMIZER_JOB_PATH} -SPARK_OPTIMIZER_JOB_PATH=ams/optimizer/spark-optimizer/target/spark-optimizer-${AMORO_VERSION}-jar-with-dependencies.jar +SPARK_OPTIMIZER_JOB_PATH=amoro-ams/amoro-ams-optimizer/amoro-optimizer-spark/target/amoro-optimizer-spark-${AMORO_VERSION}-jar-with-dependencies.jar SPARK_OPTIMIZER_JOB=${PROJECT_HOME}/${SPARK_OPTIMIZER_JOB_PATH} AMORO_TAG=$AMORO_VERSION ALSO_MAKE=true @@ -210,7 +210,7 @@ function build_optimizer_flink() { OPTIMIZER_JOB=${FLINK_OPTIMIZER_JOB} if [ ! -f "${OPTIMIZER_JOB}" ]; then - BUILD_CMD="mvn clean package -pl ams/optimizer/flink-optimizer -am -e -DskipTests" + BUILD_CMD="mvn clean package -pl amoro-ams/amoro-ams-optimizer/amoro-optimizer-flink -am -e -DskipTests" echo "flink optimizer job not exists in ${OPTIMIZER_JOB}" echo "please check the file or run '${BUILD_CMD}' first. " exit 1 @@ -233,7 +233,7 @@ function build_optimizer_spark() { OPTIMIZER_JOB=${SPARK_OPTIMIZER_JOB} if [ ! -f "${OPTIMIZER_JOB}" ]; then - BUILD_CMD="mvn clean package -pl ams/optimizer/spark-optimizer -am -e -DskipTests" + BUILD_CMD="mvn clean package -pl amoro-ams/amoro-ams-optimizer/amoro-optimizer-spark -am -e -DskipTests" echo "spark optimizer job not exists in ${OPTIMIZER_JOB}" echo "please check the file or run '${BUILD_CMD}' first. " exit 1 @@ -253,7 +253,7 @@ function build_amoro() { local IMAGE_TAG=$AMORO_TAG print_image $IMAGE_REF $IMAGE_TAG - local DIST_FILE=${PROJECT_HOME}/ams/dist/target/amoro-${AMORO_VERSION}-bin.zip + local DIST_FILE=${PROJECT_HOME}/amoro-ams/dist/target/amoro-${AMORO_VERSION}-bin.zip if [ ! -f "${DIST_FILE}" ]; then local BUILD_CMD="mvn clean package -am -e -pl ams/dist -DskipTests " @@ -272,8 +272,8 @@ function build_quickdemo() { local IMAGE_REF=arctic163/quickdemo local IMAGE_TAG=$AMORO_TAG - local FLINK_CONNECTOR_BINARY=${PROJECT_HOME}/mixed/flink/v${FLINK_MAJOR_VERSION}/flink-runtime/target/amoro-mixed-flink-runtime-${FLINK_MAJOR_VERSION}-${AMORO_VERSION}.jar - local SPARK_CONNECTOR_BINARY=${PROJECT_HOME}/mixed/spark/v${SPARK_MAJOR_VERSION}/spark-runtime/target/amoro-mixed-spark-runtime-${SPARK_MAJOR_VERSION}-${AMORO_VERSION}.jar + local FLINK_CONNECTOR_BINARY=${PROJECT_HOME}/amoro-mixed-format/amoro-mixed-format-flink/v${FLINK_MAJOR_VERSION}/amoro-mixed-format-flink-runtime-${FLINK_MAJOR_VERSION}/target/amoro-mixed-format-flink-runtime-${FLINK_MAJOR_VERSION}-${AMORO_VERSION}.jar + local SPARK_CONNECTOR_BINARY=${PROJECT_HOME}/amoro-mixed-format/amoro-mixed-format-spark/v${SPARK_MAJOR_VERSION}/amoro-mixed-format-spark-${SPARK_MAJOR_VERSION}-runtime/target/amoro-mixed-format-spark-${SPARK_MAJOR_VERSION}-runtime-${AMORO_VERSION}.jar if [ ! -f "${FLINK_CONNECTOR_BINARY}" ]; then echo "amoro-mixed-flink-connector not exists in ${FLINK_CONNECTOR_BINARY}, run 'mvn clean package -pl !mixed/trino' first. " @@ -338,4 +338,4 @@ case "$ACTION" in echo "Unknown image type: $ACTION" exit 1 ;; -esac \ No newline at end of file +esac diff --git a/docker/quickdemo/Dockerfile b/docker/quickdemo/Dockerfile index 24836556e8..eef22c5044 100644 --- a/docker/quickdemo/Dockerfile +++ b/docker/quickdemo/Dockerfile @@ -34,8 +34,8 @@ WORKDIR /workspace/amoro RUN AMORO_VERSION=`cat pom.xml | grep 'amoro-parent' -C 3 | grep -Eo '.*' | awk -F'[><]' '{print $3}'` \ && FLINK_VERSION=${FLINK_VERSION} \ && FLINK_MAJOR_VERSION="${FLINK_VERSION%.*}" \ - && cp mixed/flink/v${FLINK_MAJOR_VERSION}/flink-runtime/target/amoro-mixed-flink-runtime-${FLINK_MAJOR_VERSION}-${AMORO_VERSION}.jar \ - /opt/amoro-mixed-flink-runtime-${FLINK_MAJOR_VERSION}-${AMORO_VERSION}.jar \ + && cp amoro-mixed-format/amoro-mixed-format-flink/v${FLINK_MAJOR_VERSION}/amoro-mixed-format-flink-runtime-${FLINK_MAJOR_VERSION}/target/amoro-mixed-format-flink-runtime-${FLINK_MAJOR_VERSION}-${AMORO_VERSION}.jar \ + /opt/amoro-mixed-format-flink-runtime-${FLINK_MAJOR_VERSION}-${AMORO_VERSION}.jar \ && rm -rf /workspace/amoro @@ -72,6 +72,3 @@ WORKDIR ${FLINK_HOME} COPY --from=builder /opt/*.jar ${FLINK_HOME}/lib/ COPY ./docker/quickdemo/flink-conf.yaml ${FLINK_HOME}/conf/flink-conf.yaml RUN ln -s ${FLINK_HOME} /usr/local/flink - - - diff --git a/docs/admin-guides/deployment.md b/docs/admin-guides/deployment.md index 5c4efb46cf..368b0a2cc1 100644 --- a/docs/admin-guides/deployment.md +++ b/docs/admin-guides/deployment.md @@ -59,7 +59,7 @@ amoro-mixed-format-spark-3.2-runtime-x.y.z-sources.jar original-amoro-mixed-format-spark-3.2-runtime-x.y.z.jar ``` -If the Flink version in the ams/optimizer/flink-optimizer module you compiled is lower than 1.15, you must add the `-Pflink-pre-1.15` parameter before mvn. +If the Flink version in the amoro-ams/amoro-ams-optimizer/amoro-optimizer-flink module you compiled is lower than 1.15, you must add the `-Pflink-pre-1.15` parameter before mvn. for example `mvn clean package -Pflink-pre-1.15 -Dflink-optimizer.flink-version=1.14.6 -DskipTests` to compile. If you need to compile the Trino module at the same time, you need to install jdk17 locally and configure `toolchains.xml` in the user's `${user.home}/.m2/` directory,