diff --git a/.github/actions/dllib-scala-ut-action/action.yml b/.github/actions/dllib-scala-ut-action/action.yml index 0ba0a81fb7f..830a5e77d2a 100644 --- a/.github/actions/dllib-scala-ut-action/action.yml +++ b/.github/actions/dllib-scala-ut-action/action.yml @@ -9,14 +9,14 @@ runs: export SPARK_LOCAL_HOSTNAME=localhost export KERAS_BACKEND=tensorflow cd scala - mvn "-DwildcardSuites=com.intel.analytics.bigdl.dllib.optim" "-Dtest=com.intel.analytics.bigdl.dllib.optim.*Test" test -P spark_3.x -Dspark.version=3.4.1 - mvn "-DwildcardSuites=com.intel.analytics.bigdl.dllib.models" "-Dtest=com.intel.analytics.bigdl.dllib.models.*Test" test -P spark_3.x -Dspark.version=3.4.1 - mvn "-DwildcardSuites=com.intel.analytics.bigdl.dllib.estimator" "-Dtest=com.intel.analytics.bigdl.dllib.estimator.*Test" test -P spark_3.x -Dspark.version=3.4.1 - mvn "-DwildcardSuites=com.intel.analytics.bigdl.dllib.nnframes" "-Dtest=com.intel.analytics.bigdl.dllib.nnframes.*Test" test -P spark_3.x -Dspark.version=3.4.1 - mvn "-DwildcardSuites=com.intel.analytics.bigdl.dllib.feature" "-Dtest=com.intel.analytics.bigdl.dllib.feature.*Test" test -P spark_3.x -Dspark.version=3.4.1 - mvn "-DwildcardSuites=com.intel.analytics.bigdl.dllib.utils.intermediate" "-Dtest=com.intel.analytics.bigdl.dllib.utils.intermediate.*Test" test -P spark_3.x -Dspark.version=3.4.1 - mvn "-DwildcardSuites=com.intel.analytics.bigdl.dllib.utils.tf" "-Dtest=com.intel.analytics.bigdl.dllib.utils.tf.*Test" test -P spark_3.x -Dspark.version=3.4.1 - mvn "-DwildcardSuites=com.intel.analytics.bigdl.dllib.utils.python.api" "-Dtest=com.intel.analytics.bigdl.dllib.utils.python.api.*Test" test -P spark_3.x -Dspark.version=3.4.1 + mvn "-DwildcardSuites=com.intel.analytics.bigdl.dllib.optim" "-Dtest=com.intel.analytics.bigdl.dllib.optim.*Test" test -P spark_3.x -Dspark.version=3.1.3 + mvn "-DwildcardSuites=com.intel.analytics.bigdl.dllib.models" "-Dtest=com.intel.analytics.bigdl.dllib.models.*Test" test -P spark_3.x -Dspark.version=3.1.3 + mvn "-DwildcardSuites=com.intel.analytics.bigdl.dllib.estimator" "-Dtest=com.intel.analytics.bigdl.dllib.estimator.*Test" test -P spark_3.x -Dspark.version=3.1.3 + mvn "-DwildcardSuites=com.intel.analytics.bigdl.dllib.nnframes" "-Dtest=com.intel.analytics.bigdl.dllib.nnframes.*Test" test -P spark_3.x -Dspark.version=3.1.3 + mvn "-DwildcardSuites=com.intel.analytics.bigdl.dllib.feature" "-Dtest=com.intel.analytics.bigdl.dllib.feature.*Test" test -P spark_3.x -Dspark.version=3.1.3 + mvn "-DwildcardSuites=com.intel.analytics.bigdl.dllib.utils.intermediate" "-Dtest=com.intel.analytics.bigdl.dllib.utils.intermediate.*Test" test -P spark_3.x -Dspark.version=3.1.3 + mvn "-DwildcardSuites=com.intel.analytics.bigdl.dllib.utils.tf" "-Dtest=com.intel.analytics.bigdl.dllib.utils.tf.*Test" test -P spark_3.x -Dspark.version=3.1.3 + mvn "-DwildcardSuites=com.intel.analytics.bigdl.dllib.utils.python.api" "-Dtest=com.intel.analytics.bigdl.dllib.utils.python.api.*Test" test -P spark_3.x -Dspark.version=3.1.3 #need python requirements #mvn "-DwildcardSuites=com.intel.analytics.bigdl.dllib.keras" "-Dtest=com.intel.analytics.bigdl.dllib.keras.*Test" test -P spark_3.x -Dspark.version=3.1.3 #mvn "-DwildcardSuites=com.intel.analytics.bigdl.dllib.nn.mkldnn" "-Dtest=com.intel.analytics.bigdl.dllib.nn.mkldnn.*Test" test -P spark_3.x -Dspark.version=3.1.3 diff --git a/.github/actions/friesian-scala-ut-action/action.yml b/.github/actions/friesian-scala-ut-action/action.yml index 66cef1c813b..74cee432b68 100644 --- a/.github/actions/friesian-scala-ut-action/action.yml +++ b/.github/actions/friesian-scala-ut-action/action.yml @@ -39,4 +39,4 @@ runs: export KERAS_BACKEND=tensorflow cd scala mvn "-DwildcardSuites=com.intel.analytics.bigdl.friesian" "-Dtest=com.intel.analytics.bigdl.friesian.*Test" test -P serving,spark_2.x -Dspark.version=2.4.6 "-Dspark.master=local[*]" -pl "!ppml" - mvn "-DwildcardSuites=com.intel.analytics.bigdl.friesian" "-Dtest=com.intel.analytics.bigdl.friesian.*Test" test -P serving,spark_3.x -Dspark.version=3.4.1 "-Dspark.master=local[*]" + mvn "-DwildcardSuites=com.intel.analytics.bigdl.friesian" "-Dtest=com.intel.analytics.bigdl.friesian.*Test" test -P serving,spark_3.x -Dspark.version=3.1.3 "-Dspark.master=local[*]" diff --git a/.github/actions/friesian/friesian-python-exampletest-feature-tf1-py37-spark3/action.yml b/.github/actions/friesian/friesian-python-exampletest-feature-tf1-py37-spark3/action.yml index da71ee2d7a9..00acb2d7f6d 100644 --- a/.github/actions/friesian/friesian-python-exampletest-feature-tf1-py37-spark3/action.yml +++ b/.github/actions/friesian/friesian-python-exampletest-feature-tf1-py37-spark3/action.yml @@ -7,17 +7,17 @@ runs: shell: bash run: | cd scala - ./make-dist.sh -P spark_3.x -Dspark.version=3.4.1 + ./make-dist.sh -P spark_3.x -Dspark.version=3.1.3 cd - - wget https://archive.apache.org/dist/spark/spark-3.4.1/spark-3.4.1-bin-hadoop3.tgz -O ${{ github.workspace }}/spark-3.4.1-bin-hadoop3.tgz - tar -xf spark-3.4.1-bin-hadoop3.tgz -C ${{ github.workspace }}/ - export SPARK_HOME=${{ github.workspace }}/spark-3.4.1-bin-hadoop3 + wget https://archive.apache.org/dist/spark/spark-3.1.3/spark-3.1.3-bin-hadoop2.7.tgz -O ${{ github.workspace }}/spark-3.1.3-bin-hadoop2.7.tgz + tar -xf spark-3.1.3-bin-hadoop2.7.tgz -C ${{ github.workspace }}/ + export SPARK_HOME=${{ github.workspace }}/spark-3.1.3-bin-hadoop2.7 source activate py37 source python/friesian/dev/prepare_env.sh export SPARK_LOCAL_HOSTNAME=localhost - pip install pyspark==3.4.1 + pip install pyspark==3.1.3 pip install -i https://pypi.org/simple bigdl-math bigdl-tf pip install -i ${GONDOLIN_PIP_MIRROR} --trusted-host ${GONDOLIN_TRUSTED_HOST} grpcio==1.43.0 diff --git a/.github/actions/friesian/friesian-python-exampletest-tf2-py37-spark3/action.yml b/.github/actions/friesian/friesian-python-exampletest-tf2-py37-spark3/action.yml index 3b7a1720e67..0df8a3c5bf1 100644 --- a/.github/actions/friesian/friesian-python-exampletest-tf2-py37-spark3/action.yml +++ b/.github/actions/friesian/friesian-python-exampletest-tf2-py37-spark3/action.yml @@ -7,17 +7,17 @@ runs: shell: bash run: | cd scala - ./make-dist.sh -P spark_3.x -Dspark.version=3.4.1 + ./make-dist.sh -P spark_3.x -Dspark.version=3.1.3 cd - - wget https://archive.apache.org/dist/spark/spark-3.4.1/spark-3.4.1-bin-hadoop3.tgz -O ${{ github.workspace }}/spark-3.4.1-bin-hadoop3.tgz - tar -xf spark-3.4.1-bin-hadoop3.tgz -C ${{ github.workspace }}/ - export SPARK_HOME=${{ github.workspace }}/spark-3.4.1-bin-hadoop3 + wget https://archive.apache.org/dist/spark/spark-3.1.3/spark-3.1.3-bin-hadoop2.7.tgz -O ${{ github.workspace }}/spark-3.1.3-bin-hadoop2.7.tgz + tar -xf spark-3.1.3-bin-hadoop2.7.tgz -C ${{ github.workspace }}/ + export SPARK_HOME=${{ github.workspace }}/spark-3.1.3-bin-hadoop2.7 source activate py37 source python/friesian/dev/prepare_env.sh export SPARK_LOCAL_HOSTNAME=localhost - pip install pyspark==3.4.1 + pip install pyspark==3.1.3 pip install -i ${GONDOLIN_PIP_MIRROR} --trusted-host ${GONDOLIN_TRUSTED_HOST} grpcio==1.43.0 cd python/friesian/dev/test diff --git a/.github/actions/friesian/friesian-python-py37-spark3-pip/action.yml b/.github/actions/friesian/friesian-python-py37-spark3-pip/action.yml index c10ca36ea63..45d7334780e 100644 --- a/.github/actions/friesian/friesian-python-py37-spark3-pip/action.yml +++ b/.github/actions/friesian/friesian-python-py37-spark3-pip/action.yml @@ -7,7 +7,7 @@ runs: shell: bash run: | cd scala - mvn clean package -DskipTests -P spark_3.x -Dspark.version=3.4.1 -U + mvn clean package -DskipTests -P spark_3.x -Dspark.version=3.1.3 -U cd - #wget https://archive.apache.org/dist/spark/spark-3.1.3/spark-3.1.3-bin-hadoop2.7.tgz -O ${{ github.workspace }}/spark-3.1.3-bin-hadoop2.7.tgz @@ -16,7 +16,7 @@ runs: source activate py37 export SPARK_LOCAL_HOSTNAME=localhost - pip install pyspark==3.4.1 + pip install pyspark==3.1.3 chmod a+x python/dev/release_default_linux_spark3.sh ./python/dev/release_default_linux_spark3.sh default false false diff --git a/.github/actions/friesian/friesian-python-py37-spark3/action.yml b/.github/actions/friesian/friesian-python-py37-spark3/action.yml index b20814f629e..8be71dbc56e 100644 --- a/.github/actions/friesian/friesian-python-py37-spark3/action.yml +++ b/.github/actions/friesian/friesian-python-py37-spark3/action.yml @@ -7,16 +7,16 @@ runs: shell: bash run: | cd scala - ./make-dist.sh -P spark_3.x -Dspark.version=3.4.1 + ./make-dist.sh -P spark_3.x -Dspark.version=3.1.3 cd - - wget https://archive.apache.org/dist/spark/spark-3.4.1/spark-3.4.1-bin-hadoop3.tgz -O ${{ github.workspace }}/spark-3.4.1-bin-hadoop3.tgz - tar -xf spark-3.4.1-bin-hadoop3.tgz -C ${{ github.workspace }}/ - export SPARK_HOME=${{ github.workspace }}/spark-3.4.1-bin-hadoop3 + wget https://archive.apache.org/dist/spark/spark-3.1.3/spark-3.1.3-bin-hadoop2.7.tgz -O ${{ github.workspace }}/spark-3.1.3-bin-hadoop2.7.tgz + tar -xf spark-3.1.3-bin-hadoop2.7.tgz -C ${{ github.workspace }}/ + export SPARK_HOME=${{ github.workspace }}/spark-3.1.3-bin-hadoop2.7 source activate py37 export SPARK_LOCAL_HOSTNAME=localhost - pip install pyspark==3.4.1 + pip install pyspark==3.1.3 chmod a+x python/friesian/dev/test/run-pytests-friesian python/friesian/dev/test/run-pytests-friesian source deactivate diff --git a/.github/actions/mac/dllib/mac-dllib-scalatest-spark3/action.yml b/.github/actions/mac/dllib/mac-dllib-scalatest-spark3/action.yml index 896b292ae65..c2b72e41972 100644 --- a/.github/actions/mac/dllib/mac-dllib-scalatest-spark3/action.yml +++ b/.github/actions/mac/dllib/mac-dllib-scalatest-spark3/action.yml @@ -1,5 +1,5 @@ -name: 'Mac Dllib ScalaTest Spark3.4' -description: 'Mac Dllib ScalaTest Spark3.4' +name: 'Mac Dllib ScalaTest Spark3.1' +description: 'Mac Dllib ScalaTest Spark3.1' runs: using: "composite" steps: @@ -8,13 +8,13 @@ runs: run: | source ~/.bash_profile source activate py37 - export SPARK_HOME=/opt/work/spark-3.4.1 + export SPARK_HOME=/opt/work/spark-3.1.3 export KERAS_BACKEND=tensorflow # rm /var/root/.m2/repository/io/netty/netty-common/4.1.50.Final/netty-common-4.1.50.Final.jar cd scala - mvn "-DwildcardSuites=com.intel.analytics.bigdl.dllib.estimator" test -P spark_3.x -Dspark.version=3.4.1 -pl '!orca,!friesian,!grpc,!serving,!ppml,!assembly' && \ - mvn "-DwildcardSuites=com.intel.analytics.bigdl.dllib.nnframes" test -P spark_3.x -Dspark.version=3.4.1 -pl '!orca,!friesian,!grpc,!serving,!ppml,!assembly' && \ - mvn "-DwildcardSuites=com.intel.analytics.bigdl.dllib.keras" test -P spark_3.x -Dspark.version=3.4.1 -pl '!orca,!friesian,!grpc,!serving,!ppml,!assembly' + mvn "-DwildcardSuites=com.intel.analytics.bigdl.dllib.estimator" test -P spark_3.x -Dspark.version=3.1.3 -pl '!orca,!friesian,!grpc,!serving,!ppml,!assembly' && \ + mvn "-DwildcardSuites=com.intel.analytics.bigdl.dllib.nnframes" test -P spark_3.x -Dspark.version=3.1.3 -pl '!orca,!friesian,!grpc,!serving,!ppml,!assembly' && \ + mvn "-DwildcardSuites=com.intel.analytics.bigdl.dllib.keras" test -P spark_3.x -Dspark.version=3.1.3 -pl '!orca,!friesian,!grpc,!serving,!ppml,!assembly' cd - conda deactivate env: diff --git a/.github/actions/mac/friesian/mac-friesian-python-exampletest-feature-tf1-pip-spark3/action.yml b/.github/actions/mac/friesian/mac-friesian-python-exampletest-feature-tf1-pip-spark3/action.yml index f7ebad994d6..600c81419b2 100644 --- a/.github/actions/mac/friesian/mac-friesian-python-exampletest-feature-tf1-pip-spark3/action.yml +++ b/.github/actions/mac/friesian/mac-friesian-python-exampletest-feature-tf1-pip-spark3/action.yml @@ -10,23 +10,23 @@ runs: source activate py37 pip uninstall -y bigdl-dllib bigdl-orca bigdl-chronos bigdl-orca-spark3 bigdl-dllib-spark3 bigdl-chronos-spark3 pyspark export KERAS_BACKEND=tensorflow - export SPARK_HOME=/opt/work/spark-3.4.1 + export SPARK_HOME=/opt/work/spark-3.1.3 PATH="/usr/local/opt/gnu-tar/libexec/gnubin:$PATH" - gsed -i "s/'pyspark=='+PYSPARK_VERSION/'pyspark==3.4.1'/g" python/dllib/src/setup.py + gsed -i "s/'pyspark=='+PYSPARK_VERSION/'pyspark==3.1.3'/g" python/dllib/src/setup.py gsed -i "s/name='bigdl-dllib'/name='bigdl-dllib-spark3'/g" python/dllib/src/setup.py gsed -i "s/dist\/bigdl_dllib-/dist\/bigdl_dllib_spark3-/g" python/dllib/dev/release/release.sh - bash python/dllib/dev/release/release.sh mac default false false -Dspark.version=3.4.1 -P spark_3.x ${profiles} + bash python/dllib/dev/release/release.sh mac default false false -Dspark.version=3.1.3 -P spark_3.x ${profiles} gsed -i "s/bigdl-dllib==/bigdl-dllib-spark3==/g" python/orca/src/setup.py gsed -i "s/name='bigdl-orca'/name='bigdl-orca-spark3'/g" python/orca/src/setup.py gsed -i "s/dist\/bigdl_orca-/dist\/bigdl_orca_spark3-/g" python/orca/dev/release/release.sh - bash python/orca/dev/release/release.sh mac default true false -Dspark.version=3.4.1 -P spark_3.x + bash python/orca/dev/release/release.sh mac default true false -Dspark.version=3.1.3 -P spark_3.x gsed -i "s/bigdl-orca==/bigdl-orca-spark3==/g" python/friesian/src/setup.py gsed -i "s/name='bigdl-friesian'/name='bigdl-friesian-spark3'/g" python/friesian/src/setup.py gsed -i "s/dist\/bigdl_friesian-/dist\/bigdl_friesian_spark3-/g" python/friesian/dev/release/release.sh - bash python/friesian/dev/release/release.sh mac default false false -Dspark.version=3.4.1 -P spark_3.x + bash python/friesian/dev/release/release.sh mac default false false -Dspark.version=3.1.3 -P spark_3.x echo "Running py37 tests" # install dllib diff --git a/.github/actions/mac/friesian/mac-friesian-python-exampletest-feature-tf1-spark3/action.yml b/.github/actions/mac/friesian/mac-friesian-python-exampletest-feature-tf1-spark3/action.yml index 6f36274d40b..74bfabc3a1d 100644 --- a/.github/actions/mac/friesian/mac-friesian-python-exampletest-feature-tf1-spark3/action.yml +++ b/.github/actions/mac/friesian/mac-friesian-python-exampletest-feature-tf1-spark3/action.yml @@ -7,16 +7,16 @@ runs: shell: bash run: | source ~/.bash_profile - export SPARK_HOME=/opt/work/spark-3.4.1 + export SPARK_HOME=/opt/work/spark-3.1.3 PATH="/usr/local/opt/gnu-tar/libexec/gnubin:$PATH" cd scala - ./make-dist.sh -P spark_3.x -Dspark.version=3.4.1 + ./make-dist.sh -P spark_3.x -Dspark.version=3.1.3 cd - source python/friesian/dev/prepare_env.sh source activate py37 pip uninstall -y bigdl-dllib bigdl-orca bigdl-chronos bigdl-orca-spark3 bigdl-dllib-spark3 bigdl-chronos-spark3 pyspark - pip install pyspark==3.4.1 + pip install pyspark==3.1.3 pip install -i https://pypi.python.org/simple bigdl-tf bigdl-math echo "Running py37 tests" cd python/friesian/dev/test diff --git a/.github/actions/mac/friesian/mac-friesian-python-exampletest-tf2-pip-spark3/action.yml b/.github/actions/mac/friesian/mac-friesian-python-exampletest-tf2-pip-spark3/action.yml index de5693c842f..2227d104805 100644 --- a/.github/actions/mac/friesian/mac-friesian-python-exampletest-tf2-pip-spark3/action.yml +++ b/.github/actions/mac/friesian/mac-friesian-python-exampletest-tf2-pip-spark3/action.yml @@ -10,22 +10,22 @@ runs: source activate py37-tf2 pip uninstall -y bigdl-dllib bigdl-orca bigdl-chronos bigdl-orca-spark3 bigdl-dllib-spark3 bigdl-chronos-spark3 bigdl-friesian-spark3 pyspark export KERAS_BACKEND=tensorflow - export SPARK_HOME=/opt/work/spark-3.4.1 + export SPARK_HOME=/opt/work/spark-3.1.3 PATH="/usr/local/opt/gnu-tar/libexec/gnubin:$PATH" - gsed -i "s/'pyspark=='+PYSPARK_VERSION/'pyspark==3.4.1'/g" python/dllib/src/setup.py + gsed -i "s/'pyspark=='+PYSPARK_VERSION/'pyspark==3.1.3'/g" python/dllib/src/setup.py gsed -i "s/name='bigdl-dllib'/name='bigdl-dllib-spark3'/g" python/dllib/src/setup.py gsed -i "s/dist\/bigdl_dllib-/dist\/bigdl_dllib_spark3-/g" python/dllib/dev/release/release.sh - bash python/dllib/dev/release/release.sh mac default false false -Dspark.version=3.4.1 -P spark_3.x ${profiles} + bash python/dllib/dev/release/release.sh mac default false false -Dspark.version=3.1.3 -P spark_3.x ${profiles} gsed -i "s/bigdl-dllib==/bigdl-dllib-spark3==/g" python/orca/src/setup.py gsed -i "s/name='bigdl-orca'/name='bigdl-orca-spark3'/g" python/orca/src/setup.py gsed -i "s/dist\/bigdl_orca-/dist\/bigdl_orca_spark3-/g" python/orca/dev/release/release.sh - bash python/orca/dev/release/release.sh mac default true false -Dspark.version=3.4.1 -P spark_3.x + bash python/orca/dev/release/release.sh mac default true false -Dspark.version=3.1.3 -P spark_3.x gsed -i "s/bigdl-orca==/bigdl-orca-spark3==/g" python/friesian/src/setup.py gsed -i "s/name='bigdl-friesian'/name='bigdl-friesian-spark3'/g" python/friesian/src/setup.py gsed -i "s/dist\/bigdl_friesian-/dist\/bigdl_friesian_spark3-/g" python/friesian/dev/release/release.sh - bash python/friesian/dev/release/release.sh mac default false false -Dspark.version=3.4.1 -P spark_3.x + bash python/friesian/dev/release/release.sh mac default false false -Dspark.version=3.1.3 -P spark_3.x echo "Running py37 tests" # install dllib diff --git a/.github/actions/mac/friesian/mac-friesian-python-exampletest-tf2-spark3/action.yml b/.github/actions/mac/friesian/mac-friesian-python-exampletest-tf2-spark3/action.yml index 73793906194..289b48e6886 100644 --- a/.github/actions/mac/friesian/mac-friesian-python-exampletest-tf2-spark3/action.yml +++ b/.github/actions/mac/friesian/mac-friesian-python-exampletest-tf2-spark3/action.yml @@ -7,16 +7,16 @@ runs: shell: bash run: | source ~/.bash_profile - export SPARK_HOME=/opt/work/spark-3.4.1 + export SPARK_HOME=/opt/work/spark-3.1.3 PATH="/usr/local/opt/gnu-tar/libexec/gnubin:$PATH" cd scala - ./make-dist.sh -P spark_3.x -Dspark.version=3.4.1 + ./make-dist.sh -P spark_3.x -Dspark.version=3.1.3 cd - source python/friesian/dev/prepare_env.sh source activate py37-tf2 pip uninstall -y bigdl-dllib bigdl-orca bigdl-chronos bigdl-orca-spark3 bigdl-dllib-spark3 bigdl-chronos-spark3 pyspark - pip install pyspark==3.4.1 + pip install pyspark==3.1.3 echo "Running py37 tests" cd python/friesian/dev/test gsed -i "s/sed/gsed/g" run-example-tests-tf2.6.0.sh diff --git a/.github/actions/mac/friesian/mac-friesian-python-ut-pip-spark3/action.yml b/.github/actions/mac/friesian/mac-friesian-python-ut-pip-spark3/action.yml index 5e087da5cd0..27712be26b8 100644 --- a/.github/actions/mac/friesian/mac-friesian-python-ut-pip-spark3/action.yml +++ b/.github/actions/mac/friesian/mac-friesian-python-ut-pip-spark3/action.yml @@ -10,22 +10,22 @@ runs: source activate py37 pip uninstall -y bigdl-dllib bigdl-orca bigdl-chronos bigdl-orca-spark3 bigdl-dllib-spark3 bigdl-chronos-spark3 pyspark export KERAS_BACKEND=tensorflow - export SPARK_HOME=/opt/work/spark-3.4.1 + export SPARK_HOME=/opt/work/spark-3.1.3 - gsed -i "s/'pyspark=='+PYSPARK_VERSION/'pyspark==3.4.1'/g" python/dllib/src/setup.py + gsed -i "s/'pyspark=='+PYSPARK_VERSION/'pyspark==3.1.3'/g" python/dllib/src/setup.py gsed -i "s/name='bigdl-dllib'/name='bigdl-dllib-spark3'/g" python/dllib/src/setup.py gsed -i "s/dist\/bigdl_dllib-/dist\/bigdl_dllib_spark3-/g" python/dllib/dev/release/release.sh - bash python/dllib/dev/release/release.sh mac default false false -Dspark.version=3.4.1 -P spark_3.x ${profiles} + bash python/dllib/dev/release/release.sh mac default false false -Dspark.version=3.1.3 -P spark_3.x ${profiles} gsed -i "s/bigdl-dllib==/bigdl-dllib-spark3==/g" python/orca/src/setup.py gsed -i "s/name='bigdl-orca'/name='bigdl-orca-spark3'/g" python/orca/src/setup.py gsed -i "s/dist\/bigdl_orca-/dist\/bigdl_orca_spark3-/g" python/orca/dev/release/release.sh - bash python/orca/dev/release/release.sh mac default true false -Dspark.version=3.4.1 -P spark_3.x + bash python/orca/dev/release/release.sh mac default true false -Dspark.version=3.1.3 -P spark_3.x gsed -i "s/bigdl-orca==/bigdl-orca-spark3==/g" python/friesian/src/setup.py gsed -i "s/name='bigdl-friesian'/name='bigdl-friesian-spark3'/g" python/friesian/src/setup.py gsed -i "s/dist\/bigdl_friesian-/dist\/bigdl_friesian_spark3-/g" python/friesian/dev/release/release.sh - bash python/friesian/dev/release/release.sh mac default false false -Dspark.version=3.4.1 -P spark_3.x + bash python/friesian/dev/release/release.sh mac default false false -Dspark.version=3.1.3 -P spark_3.x echo "Running py37 tests" # install dllib diff --git a/.github/actions/mac/friesian/mac-friesian-python-ut-spark3/action.yml b/.github/actions/mac/friesian/mac-friesian-python-ut-spark3/action.yml index 89dfb1b2387..988f77224ea 100644 --- a/.github/actions/mac/friesian/mac-friesian-python-ut-spark3/action.yml +++ b/.github/actions/mac/friesian/mac-friesian-python-ut-spark3/action.yml @@ -10,11 +10,11 @@ runs: source activate py37 pip uninstall -y bigdl-dllib bigdl-orca bigdl-chronos bigdl-orca-spark3 bigdl-dllib-spark3 bigdl-chronos-spark3 pyspark export KERAS_BACKEND=tensorflow - export SPARK_HOME=/opt/work/spark-3.4.1 + export SPARK_HOME=/opt/work/spark-3.1.3 cd scala - ./make-dist.sh -P spark_3.x -Dspark.version=3.4.1 + ./make-dist.sh -P spark_3.x -Dspark.version=3.1.3 cd - - pip install pyspark==3.4.1 + pip install pyspark==3.1.3 echo "Running py37 tests" python/friesian/dev/test/run-pytests-friesian diff --git a/.github/actions/mac/orca/mac-orca-python-ut-spark3/action.yml b/.github/actions/mac/orca/mac-orca-python-ut-spark3/action.yml index 489d4d7efd8..8d0f7a19a80 100644 --- a/.github/actions/mac/orca/mac-orca-python-ut-spark3/action.yml +++ b/.github/actions/mac/orca/mac-orca-python-ut-spark3/action.yml @@ -10,7 +10,7 @@ runs: source activate py37 pip uninstall -y bigdl-dllib bigdl-orca bigdl-chronos bigdl-orca-spark3 bigdl-dllib-spark3 bigdl-chronos-spark3 pyspark export KERAS_BACKEND=tensorflow - #export SPARK_HOME=/opt/work/spark-3.4.1 + #export SPARK_HOME=/opt/work/spark-3.1.3 export MASTER=local[4] export HDFS_URI=/Users/arda/data export PATH=/opt/maven/apache-maven-3.6.3/bin:$PATH diff --git a/.github/workflows/chronos-example-python-spark31.yml b/.github/workflows/chronos-example-python-spark31.yml index 1e514e82939..76df2e2e813 100644 --- a/.github/workflows/chronos-example-python-spark31.yml +++ b/.github/workflows/chronos-example-python-spark31.yml @@ -53,7 +53,7 @@ jobs: apt-get install -y libgl1 apt-get install patchelf pip uninstall -y bigdl-friesian bigdl-friesian-spark3 bigdl-dllib bigdl-dllib-spark3 bigdl-orca pyspark bigdl-orca-spark3 bigdl-chronos bigdl-chronos-spark3 bigdl-nano bigdl-friesian bigdl-friesian-spark3 - sed -i "s/pyspark==2.4.6/pyspark==3.4.1/g" python/dllib/src/setup.py + sed -i "s/pyspark==2.4.6/pyspark==3.1.3/g" python/dllib/src/setup.py wget https://raw.githubusercontent.com/analytics-zoo/gha-cicd-env/main/python-requirements/requirements-chronos-python-ut.txt -O ${{ github.workspace }}/requirements-chronos-python-ut.txt pip install -i ${GONDOLIN_PIP_MIRROR} --trusted-host ${GONDOLIN_TRUSTED_HOST} -r ${{ github.workspace }}/requirements-chronos-python-ut.txt bash python/dev/release_default_linux_spark3.sh default false false diff --git a/.github/workflows/chronos-howto-guides-python-spark31.yml b/.github/workflows/chronos-howto-guides-python-spark31.yml index 55fb1fcdcc8..ca494906d9f 100644 --- a/.github/workflows/chronos-howto-guides-python-spark31.yml +++ b/.github/workflows/chronos-howto-guides-python-spark31.yml @@ -48,7 +48,7 @@ jobs: apt-get update apt-get install -y libgl1 pip uninstall -y bigdl-friesian bigdl-friesian-spark3 bigdl-dllib bigdl-dllib-spark3 bigdl-orca pyspark bigdl-orca-spark3 bigdl-chronos bigdl-chronos-spark3 bigdl-nano bigdl-friesian bigdl-friesian-spark3 - sed -i "s/pyspark==2.4.6/pyspark==3.4.1/g" python/dllib/src/setup.py + sed -i "s/pyspark==2.4.6/pyspark==3.1.3/g" python/dllib/src/setup.py wget https://raw.githubusercontent.com/analytics-zoo/gha-cicd-env/main/python-requirements/requirements-chronos-python-ut.txt -O ${{ github.workspace }}/requirements-chronos-python-ut.txt pip install -i ${GONDOLIN_PIP_MIRROR} --trusted-host ${GONDOLIN_TRUSTED_HOST} -r ${{ github.workspace }}/requirements-chronos-python-ut.txt bash python/dev/release_default_linux_spark3.sh default false false @@ -102,7 +102,7 @@ jobs: apt-get install patchelf pip uninstall -y bigdl-friesian bigdl-friesian-spark3 bigdl-dllib bigdl-dllib-spark3 bigdl-orca pyspark bigdl-orca-spark3 bigdl-chronos bigdl-chronos-spark3 bigdl-nano bigdl-friesian bigdl-friesian-spark3 wget https://raw.githubusercontent.com/analytics-zoo/gha-cicd-env/main/python-requirements/requirements-chronos-python-ut.txt -O ${{ github.workspace }}/requirements-chronos-python-ut.txt - sed -i "s/pyspark==2.4.6/pyspark==3.4.1/g" python/dllib/src/setup.py + sed -i "s/pyspark==2.4.6/pyspark==3.1.3/g" python/dllib/src/setup.py bash python/dev/release_default_linux_spark3.sh default false false bash python/nano/dev/build_and_install.sh linux default false pytorch --force-reinstall --no-deps -U whl_name=`ls python/nano/dist/` diff --git a/.github/workflows/chronos-prvn-python-spark31.yml b/.github/workflows/chronos-prvn-python-spark31.yml index 8275a9f4b18..2fe38a611eb 100644 --- a/.github/workflows/chronos-prvn-python-spark31.yml +++ b/.github/workflows/chronos-prvn-python-spark31.yml @@ -55,7 +55,7 @@ jobs: apt-get install -y libgl1 apt-get install patchelf pip uninstall -y bigdl-friesian bigdl-friesian-spark3 bigdl-dllib bigdl-dllib-spark3 bigdl-orca pyspark bigdl-orca-spark3 bigdl-chronos bigdl-chronos-spark3 bigdl-nano bigdl-friesian bigdl-friesian-spark3 - sed -i "s/pyspark==2.4.6/pyspark==3.4.1/g" python/dllib/src/setup.py + sed -i "s/pyspark==2.4.6/pyspark==3.1.3/g" python/dllib/src/setup.py wget https://raw.githubusercontent.com/analytics-zoo/gha-cicd-env/main/python-requirements/requirements-chronos-python-ut.txt -O ${{ github.workspace }}/requirements-chronos-python-ut.txt pip install -i ${GONDOLIN_PIP_MIRROR} --trusted-host ${GONDOLIN_TRUSTED_HOST} -r ${{ github.workspace }}/requirements-chronos-python-ut.txt bash python/dev/release_default_linux_spark3.sh default false false @@ -109,7 +109,7 @@ jobs: apt-get install -y libgl1 apt-get install patchelf pip uninstall -y bigdl-friesian bigdl-friesian-spark3 bigdl-dllib bigdl-dllib-spark3 bigdl-orca pyspark bigdl-orca-spark3 bigdl-chronos bigdl-chronos-spark3 bigdl-nano bigdl-friesian bigdl-friesian-spark3 - sed -i "s/pyspark==2.4.6/pyspark==3.4.1/g" python/dllib/src/setup.py + sed -i "s/pyspark==2.4.6/pyspark==3.1.3/g" python/dllib/src/setup.py wget https://raw.githubusercontent.com/analytics-zoo/gha-cicd-env/main/python-requirements/requirements-chronos-python-ut.txt -O ${{ github.workspace }}/requirements-chronos-python-ut.txt pip install -i ${GONDOLIN_PIP_MIRROR} --trusted-host ${GONDOLIN_TRUSTED_HOST} -r ${{ github.workspace }}/requirements-chronos-python-ut.txt bash python/dev/release_default_linux_spark3.sh default false false @@ -163,7 +163,7 @@ jobs: apt-get install -y libgl1 apt-get install patchelf pip uninstall -y bigdl-friesian bigdl-friesian-spark3 bigdl-dllib bigdl-dllib-spark3 bigdl-orca pyspark bigdl-orca-spark3 bigdl-chronos bigdl-chronos-spark3 bigdl-nano bigdl-friesian bigdl-friesian-spark3 - sed -i "s/pyspark==2.4.6/pyspark==3.4.1/g" python/dllib/src/setup.py + sed -i "s/pyspark==2.4.6/pyspark==3.1.3/g" python/dllib/src/setup.py wget https://raw.githubusercontent.com/analytics-zoo/gha-cicd-env/main/python-requirements/requirements-chronos-python-ut.txt -O ${{ github.workspace }}/requirements-chronos-python-ut.txt pip install -i ${GONDOLIN_PIP_MIRROR} --trusted-host ${GONDOLIN_TRUSTED_HOST} -r ${{ github.workspace }}/requirements-chronos-python-ut.txt bash python/dev/release_default_linux_spark3.sh default false false @@ -217,7 +217,7 @@ jobs: apt-get install -y libgl1 apt-get install patchelf pip uninstall -y bigdl-friesian bigdl-friesian-spark3 bigdl-dllib bigdl-dllib-spark3 bigdl-orca pyspark bigdl-orca-spark3 bigdl-chronos bigdl-chronos-spark3 bigdl-nano bigdl-friesian bigdl-friesian-spark3 - sed -i "s/pyspark==2.4.6/pyspark==3.4.1/g" python/dllib/src/setup.py + sed -i "s/pyspark==2.4.6/pyspark==3.1.3/g" python/dllib/src/setup.py wget https://raw.githubusercontent.com/analytics-zoo/gha-cicd-env/main/python-requirements/requirements-chronos-python-ut.txt -O ${{ github.workspace }}/requirements-chronos-python-ut.txt pip install -i ${GONDOLIN_PIP_MIRROR} --trusted-host ${GONDOLIN_TRUSTED_HOST} -r ${{ github.workspace }}/requirements-chronos-python-ut.txt bash python/dev/release_default_linux_spark3.sh default false false diff --git a/.github/workflows/license-scan.yml b/.github/workflows/license-scan.yml index 1e3d06604d1..a69e3762407 100644 --- a/.github/workflows/license-scan.yml +++ b/.github/workflows/license-scan.yml @@ -44,26 +44,26 @@ jobs: if: ${{ failure() }} run: | ls - #spark3.4.1 + #spark3.1.3 sed -i 's/${spark-version.project}<\/artifactId>/${spark-version.project}-${SPARK_PLATFORM}<\/artifactId>/' scala/dllib/pom.xml sed -i 's/3.0<\/artifactId>/3.0-${SPARK_PLATFORM}<\/artifactId>/' scala/common/spark-version/3.0/pom.xml - sed -i 's/bigdl-parent-spark_${spark.version}<\/artifactId>/bigdl-parent-spark_3.4.1<\/artifactId>/' scala/pom.xml - sed -i 's/bigdl-parent-spark_${spark.version}<\/artifactId>/bigdl-parent-spark_3.4.1<\/artifactId>/' scala/common/spark-version/pom.xml - sed -i 's/bigdl-parent-spark_${spark.version}<\/artifactId>/bigdl-parent-spark_3.4.1<\/artifactId>/' scala/common/spark-version/3.0/pom.xml - sed -i 's/bigdl-parent-spark_${spark.version}<\/artifactId>/bigdl-parent-spark_3.4.1<\/artifactId>/' scala/dllib/pom.xml - sed -i 's/bigdl-parent-spark_${spark.version}<\/artifactId>/bigdl-parent-spark_3.4.1<\/artifactId>/' scala/orca/pom.xml - sed -i 's/bigdl-parent-spark_${spark.version}<\/artifactId>/bigdl-parent-spark_3.4.1<\/artifactId>/' scala/friesian/pom.xml - sed -i 's/bigdl-parent-spark_${spark.version}<\/artifactId>/bigdl-parent-spark_3.4.1<\/artifactId>/' scala/grpc/pom.xml - sed -i 's/bigdl-parent-spark_${spark.version}<\/artifactId>/bigdl-parent-spark_3.4.1<\/artifactId>/' scala/serving/pom.xml - sed -i 's/bigdl-parent-spark_${spark.version}<\/artifactId>/bigdl-parent-spark_3.4.1<\/artifactId>/' scala/ppml/pom.xml - sed -i 's/bigdl-parent-spark_${spark.version}<\/artifactId>/bigdl-parent-spark_3.4.1<\/artifactId>/' scala/assembly/pom.xml - mvn dependency:tree -Dhttp.proxyHost=${{ secrets.HTTP_PROXY_HOST_2 }} -Dhttp.proxyPort=${{ secrets.HTTP_PROXY_PORT_2 }} -Dhttps.proxyHost=${{ secrets.HTTP_PROXY_HOST_2 }} -Dhttps.proxyPort=${{ secrets.HTTP_PROXY_PORT_3 }} -Dspark.version=3.4.1 -DSPARK_PLATFORM=SPARK_3.4 -P spark_3.x --file scala/pom.xml - mvn clean package -Dhttp.proxyHost=${{ secrets.HTTP_PROXY_HOST_2 }} -Dhttp.proxyPort=${{ secrets.HTTP_PROXY_PORT_2 }} -Dhttps.proxyHost=${{ secrets.HTTP_PROXY_HOST_2 }} -Dhttps.proxyPort=${{ secrets.HTTP_PROXY_PORT_3 }} -DskipTests -Dspark.version=3.4.1 -DSPARK_PLATFORM=SPARK_3.4 -P spark_3.x --file scala/pom.xml + sed -i 's/bigdl-parent-spark_${spark.version}<\/artifactId>/bigdl-parent-spark_3.1.3<\/artifactId>/' scala/pom.xml + sed -i 's/bigdl-parent-spark_${spark.version}<\/artifactId>/bigdl-parent-spark_3.1.3<\/artifactId>/' scala/common/spark-version/pom.xml + sed -i 's/bigdl-parent-spark_${spark.version}<\/artifactId>/bigdl-parent-spark_3.1.3<\/artifactId>/' scala/common/spark-version/3.0/pom.xml + sed -i 's/bigdl-parent-spark_${spark.version}<\/artifactId>/bigdl-parent-spark_3.1.3<\/artifactId>/' scala/dllib/pom.xml + sed -i 's/bigdl-parent-spark_${spark.version}<\/artifactId>/bigdl-parent-spark_3.1.3<\/artifactId>/' scala/orca/pom.xml + sed -i 's/bigdl-parent-spark_${spark.version}<\/artifactId>/bigdl-parent-spark_3.1.3<\/artifactId>/' scala/friesian/pom.xml + sed -i 's/bigdl-parent-spark_${spark.version}<\/artifactId>/bigdl-parent-spark_3.1.3<\/artifactId>/' scala/grpc/pom.xml + sed -i 's/bigdl-parent-spark_${spark.version}<\/artifactId>/bigdl-parent-spark_3.1.3<\/artifactId>/' scala/serving/pom.xml + sed -i 's/bigdl-parent-spark_${spark.version}<\/artifactId>/bigdl-parent-spark_3.1.3<\/artifactId>/' scala/ppml/pom.xml + sed -i 's/bigdl-parent-spark_${spark.version}<\/artifactId>/bigdl-parent-spark_3.1.3<\/artifactId>/' scala/assembly/pom.xml + mvn dependency:tree -Dhttp.proxyHost=${{ secrets.HTTP_PROXY_HOST_2 }} -Dhttp.proxyPort=${{ secrets.HTTP_PROXY_PORT_2 }} -Dhttps.proxyHost=${{ secrets.HTTP_PROXY_HOST_2 }} -Dhttps.proxyPort=${{ secrets.HTTP_PROXY_PORT_3 }} -Dspark.version=3.1.3 -DSPARK_PLATFORM=SPARK_3.1 -P spark_3.x --file scala/pom.xml + mvn clean package -Dhttp.proxyHost=${{ secrets.HTTP_PROXY_HOST_2 }} -Dhttp.proxyPort=${{ secrets.HTTP_PROXY_PORT_2 }} -Dhttps.proxyHost=${{ secrets.HTTP_PROXY_HOST_2 }} -Dhttps.proxyPort=${{ secrets.HTTP_PROXY_PORT_3 }} -DskipTests -Dspark.version=3.1.3 -DSPARK_PLATFORM=SPARK_3.1 -P spark_3.x --file scala/pom.xml echo "ls -d scala/assembly/target/*" ls -d scala/assembly/target/* echo "" - echo "zipinfo -1 scala/assembly/target/bigdl-assembly-spark_3.4.1-2.4.0-SNAPSHOT-dist-all.zip 'jars/*.jar'" - zipinfo -1 scala/assembly/target/bigdl-assembly-spark_3.4.1-2.4.0-SNAPSHOT-dist-all.zip 'jars/*.jar' + echo "zipinfo -1 scala/assembly/target/bigdl-assembly-spark_3.1.3-2.4.0-SNAPSHOT-dist-all.zip 'jars/*.jar'" + zipinfo -1 scala/assembly/target/bigdl-assembly-spark_3.1.3-2.4.0-SNAPSHOT-dist-all.zip 'jars/*.jar' - name: Create Job Badge if: ${{ always() }} diff --git a/.github/workflows/nightly_build.yml b/.github/workflows/nightly_build.yml index a57581944a3..a74d236c49b 100644 --- a/.github/workflows/nightly_build.yml +++ b/.github/workflows/nightly_build.yml @@ -92,7 +92,7 @@ jobs: - name: Build with Maven run: | ls - #spark3.4.1 + #spark3.1.3 cp scala/pom.xml scala/pom.xml.origin cp scala/common/spark-version/pom.xml scala/common/spark-version/pom.xml.origin cp scala/common/spark-version/3.0/pom.xml scala/common/spark-version/3.0/pom.xml.origin @@ -106,17 +106,17 @@ jobs: sed -i 's/${spark-version.project}<\/artifactId>/${spark-version.project}-${SPARK_PLATFORM}<\/artifactId>/' scala/dllib/pom.xml sed -i 's/3.0<\/artifactId>/3.0-${SPARK_PLATFORM}<\/artifactId>/' scala/common/spark-version/3.0/pom.xml - sed -i 's/bigdl-parent-spark_${spark.version}<\/artifactId>/bigdl-parent-spark_3.4.1<\/artifactId>/' scala/pom.xml - sed -i 's/bigdl-parent-spark_${spark.version}<\/artifactId>/bigdl-parent-spark_3.4.1<\/artifactId>/' scala/common/spark-version/pom.xml - sed -i 's/bigdl-parent-spark_${spark.version}<\/artifactId>/bigdl-parent-spark_3.4.1<\/artifactId>/' scala/common/spark-version/3.0/pom.xml - sed -i 's/bigdl-parent-spark_${spark.version}<\/artifactId>/bigdl-parent-spark_3.4.1<\/artifactId>/' scala/dllib/pom.xml - sed -i 's/bigdl-parent-spark_${spark.version}<\/artifactId>/bigdl-parent-spark_3.4.1<\/artifactId>/' scala/orca/pom.xml - sed -i 's/bigdl-parent-spark_${spark.version}<\/artifactId>/bigdl-parent-spark_3.4.1<\/artifactId>/' scala/friesian/pom.xml - sed -i 's/bigdl-parent-spark_${spark.version}<\/artifactId>/bigdl-parent-spark_3.4.1<\/artifactId>/' scala/grpc/pom.xml - sed -i 's/bigdl-parent-spark_${spark.version}<\/artifactId>/bigdl-parent-spark_3.4.1<\/artifactId>/' scala/serving/pom.xml - sed -i 's/bigdl-parent-spark_${spark.version}<\/artifactId>/bigdl-parent-spark_3.4.1<\/artifactId>/' scala/ppml/pom.xml - sed -i 's/bigdl-parent-spark_${spark.version}<\/artifactId>/bigdl-parent-spark_3.4.1<\/artifactId>/' scala/assembly/pom.xml - mvn -Dhttp.proxyHost=${{ secrets.HTTP_PROXY_HOST_2 }} -Dhttp.proxyPort=${{ secrets.HTTP_PROXY_PORT_2 }} -Dhttps.proxyHost=${{ secrets.HTTP_PROXY_HOST_2 }} -Dhttps.proxyPort=${{ secrets.HTTP_PROXY_PORT_3 }} clean deploy -DskipTests -Dspark.version=3.4.1 -DSPARK_PLATFORM=SPARK_3.4 -P spark_3.x -P sign --file scala/pom.xml + sed -i 's/bigdl-parent-spark_${spark.version}<\/artifactId>/bigdl-parent-spark_3.1.3<\/artifactId>/' scala/pom.xml + sed -i 's/bigdl-parent-spark_${spark.version}<\/artifactId>/bigdl-parent-spark_3.1.3<\/artifactId>/' scala/common/spark-version/pom.xml + sed -i 's/bigdl-parent-spark_${spark.version}<\/artifactId>/bigdl-parent-spark_3.1.3<\/artifactId>/' scala/common/spark-version/3.0/pom.xml + sed -i 's/bigdl-parent-spark_${spark.version}<\/artifactId>/bigdl-parent-spark_3.1.3<\/artifactId>/' scala/dllib/pom.xml + sed -i 's/bigdl-parent-spark_${spark.version}<\/artifactId>/bigdl-parent-spark_3.1.3<\/artifactId>/' scala/orca/pom.xml + sed -i 's/bigdl-parent-spark_${spark.version}<\/artifactId>/bigdl-parent-spark_3.1.3<\/artifactId>/' scala/friesian/pom.xml + sed -i 's/bigdl-parent-spark_${spark.version}<\/artifactId>/bigdl-parent-spark_3.1.3<\/artifactId>/' scala/grpc/pom.xml + sed -i 's/bigdl-parent-spark_${spark.version}<\/artifactId>/bigdl-parent-spark_3.1.3<\/artifactId>/' scala/serving/pom.xml + sed -i 's/bigdl-parent-spark_${spark.version}<\/artifactId>/bigdl-parent-spark_3.1.3<\/artifactId>/' scala/ppml/pom.xml + sed -i 's/bigdl-parent-spark_${spark.version}<\/artifactId>/bigdl-parent-spark_3.1.3<\/artifactId>/' scala/assembly/pom.xml + mvn -Dhttp.proxyHost=${{ secrets.HTTP_PROXY_HOST_2 }} -Dhttp.proxyPort=${{ secrets.HTTP_PROXY_PORT_2 }} -Dhttps.proxyHost=${{ secrets.HTTP_PROXY_HOST_2 }} -Dhttps.proxyPort=${{ secrets.HTTP_PROXY_PORT_3 }} clean deploy -DskipTests -Dspark.version=3.1.3 -DSPARK_PLATFORM=SPARK_3.1 -P spark_3.x -P sign --file scala/pom.xml mv scala/pom.xml.origin scala/pom.xml mv scala/common/spark-version/pom.xml.origin scala/common/spark-version/pom.xml @@ -198,54 +198,52 @@ jobs: export IMAGE=intelanalytics/bigdl cd docker/bigdl echo "########################################" - echo "################# bigdl 3.4.1 #######" + echo "################# bigdl 3.1.3 #######" echo "########################################" docker build \ --build-arg http_proxy=${HTTP_PROXY} \ --build-arg https_proxy=${HTTPS_PROXY} \ - --build-arg SPARK_VERSION=3.4.1 \ - --build-arg HADOOP_VERSION=3 \ + --build-arg SPARK_VERSION=3.1.3 \ --build-arg JDK_VERSION=8u192 \ --build-arg JDK_URL=${JDK_URL} \ --build-arg no_proxy=${NO_PROXY} \ - --rm --no-cache -t $IMAGE-spark-3.4.1:${TAG} . + --rm --no-cache -t $IMAGE-spark-3.1.3:${TAG} . # tag 'latest' - docker push ${IMAGE}-spark-3.4.1:${TAG} - docker tag ${IMAGE}-spark-3.4.1:${TAG} 10.239.45.10/arda/${IMAGE}-spark-3.4.1:${TAG} - docker push 10.239.45.10/arda/${IMAGE}-spark-3.4.1:${TAG} + docker push ${IMAGE}-spark-3.1.3:${TAG} + docker tag ${IMAGE}-spark-3.1.3:${TAG} 10.239.45.10/arda/${IMAGE}-spark-3.1.3:${TAG} + docker push 10.239.45.10/arda/${IMAGE}-spark-3.1.3:${TAG} # tag SNAPSHOT export TAG_SNAPSHOT=2.4.0-SNAPSHOT - docker tag 10.239.45.10/arda/${IMAGE}-spark-3.4.1:${TAG} ${IMAGE}-spark-3.4.1:${TAG_SNAPSHOT} - docker push ${IMAGE}-spark-3.4.1:${TAG_SNAPSHOT} - docker rmi -f ${IMAGE}-spark-3.4.1:${TAG} 10.239.45.10/arda/${IMAGE}-spark-3.4.1:${TAG} ${IMAGE}-spark-3.4.1:${TAG_SNAPSHOT} + docker tag 10.239.45.10/arda/${IMAGE}-spark-3.1.3:${TAG} ${IMAGE}-spark-3.1.3:${TAG_SNAPSHOT} + docker push ${IMAGE}-spark-3.1.3:${TAG_SNAPSHOT} + docker rmi -f ${IMAGE}-spark-3.1.3:${TAG} 10.239.45.10/arda/${IMAGE}-spark-3.1.3:${TAG} ${IMAGE}-spark-3.1.3:${TAG_SNAPSHOT} - name: docker deploy bigdl-k8s run: | cd docker/bigdl-k8s export IMAGE=intelanalytics/bigdl-k8s export TAG=latest echo "########################################" - echo "################# bigdl-k8s 3.4.1 #######" + echo "################# bigdl-k8s 3.1.3 #######" echo "########################################" docker build \ --build-arg http_proxy=${HTTP_PROXY} \ --build-arg https_proxy=${HTTPS_PROXY} \ - --build-arg SPARK_VERSION=3.4.1 \ - --build-arg HADOOP_VERSION=3 \ + --build-arg SPARK_VERSION=3.1.3 \ --build-arg JDK_VERSION=8u192 \ --build-arg JDK_URL=${JDK_URL} \ --build-arg no_proxy=${NO_PROXY} \ - --rm --no-cache -t ${IMAGE}-spark-3.4.1:${TAG} . + --rm --no-cache -t ${IMAGE}-spark-3.1.3:${TAG} . # tag 'latest' - docker push ${IMAGE}-spark-3.4.1:${TAG} - docker tag ${IMAGE}-spark-3.4.1:${TAG} ${IMAGE}:${TAG} + docker push ${IMAGE}-spark-3.1.3:${TAG} + docker tag ${IMAGE}-spark-3.1.3:${TAG} ${IMAGE}:${TAG} docker push ${IMAGE}:${TAG} - docker tag ${IMAGE}:${TAG} 10.239.45.10/arda/${IMAGE}-spark-3.4.1:${TAG} - docker push 10.239.45.10/arda/${IMAGE}-spark-3.4.1:${TAG} + docker tag ${IMAGE}:${TAG} 10.239.45.10/arda/${IMAGE}-spark-3.1.3:${TAG} + docker push 10.239.45.10/arda/${IMAGE}-spark-3.1.3:${TAG} # tag SNAPSHOT export TAG_SNAPSHOT=2.4.0-SNAPSHOT - docker tag 10.239.45.10/arda/${IMAGE}-spark-3.4.1:${TAG} ${IMAGE}:${TAG_SNAPSHOT} + docker tag 10.239.45.10/arda/${IMAGE}-spark-3.1.3:${TAG} ${IMAGE}:${TAG_SNAPSHOT} docker push ${IMAGE}:${TAG_SNAPSHOT} - docker rmi -f ${IMAGE}-spark-3.4.1:${TAG} ${IMAGE}:${TAG} 10.239.45.10/arda/${IMAGE}-spark-3.4.1:${TAG} ${IMAGE}:${TAG_SNAPSHOT} + docker rmi -f ${IMAGE}-spark-3.1.3:${TAG} ${IMAGE}:${TAG} 10.239.45.10/arda/${IMAGE}-spark-3.1.3:${TAG} ${IMAGE}:${TAG_SNAPSHOT} - name: docker deploy bigdl-chronos run: | mv docker/chronos-nightly/Dockerfile ./Dockerfile @@ -282,8 +280,7 @@ jobs: --build-arg JDK_VERSION=8u192 \ --build-arg JDK_URL=${JDK_URL} \ --build-arg no_proxy=${NO_PROXY} \ - --build-arg SPARK_VERSION=3.4.1 \ - --build-arg HADOOP_VERSION=3 \ + --build-arg SPARK_VERSION=3.1.3 \ --build-arg PY4J_VERSION=0.10.9 \ --rm --no-cache -t ${IMAGE}:${TAG} . # tag 'latest' diff --git a/.github/workflows/sdl_snyk_scala.yml b/.github/workflows/sdl_snyk_scala.yml index d6c67b790f0..60d39e15ffd 100644 --- a/.github/workflows/sdl_snyk_scala.yml +++ b/.github/workflows/sdl_snyk_scala.yml @@ -31,14 +31,14 @@ jobs: cd scala snyk monitor --all-projects cd - - - name: "run Snyk test scan Spark 3.4.1" + - name: "run Snyk test scan Spark 3.1.3" env: no_proxy: snyk.devtools.intel.com, intel.com run: | sed -i 's/2.11<\/scala.major.version>/2.12<\/scala.major.version>/' scala/pom.xml sed -i 's/2.11.12<\/scala.version>/2.12.10<\/scala.version>/' scala/pom.xml sed -i 's/2.0<\/spark-version.project>/3.0<\/spark-version.project>/' scala/pom.xml - sed -i 's/2.4.6<\/spark.version>/3.4.1<\/spark.version>/' scala/pom.xml + sed -i 's/2.4.6<\/spark.version>/3.1.3<\/spark.version>/' scala/pom.xml cd scala snyk monitor --all-projects cd - diff --git a/docker/bigdl-k8s/Dockerfile b/docker/bigdl-k8s/Dockerfile index c18a41fc5a1..3ea26ed33ea 100644 --- a/docker/bigdl-k8s/Dockerfile +++ b/docker/bigdl-k8s/Dockerfile @@ -63,7 +63,7 @@ RUN apt-get update --fix-missing && \ COPY ./log4j2.xml ${SPARK_HOME}/conf/log4j2.xml RUN ln -fs /bin/bash /bin/sh -RUN if [ $SPARK_VERSION = "3.4.1" ]; then \ +RUN if [ $SPARK_VERSION = "3.1.3" ]; then \ rm $SPARK_HOME/jars/okhttp-*.jar && \ wget -P $SPARK_HOME/jars https://repo1.maven.org/maven2/com/squareup/okhttp3/okhttp/3.8.0/okhttp-3.8.0.jar; \ elif [ $SPARK_VERSION = "2.4.6" ]; then \ diff --git a/docker/bigdl/Dockerfile b/docker/bigdl/Dockerfile index 814e5932395..3c3bcf871d0 100644 --- a/docker/bigdl/Dockerfile +++ b/docker/bigdl/Dockerfile @@ -60,7 +60,7 @@ RUN apt-get update --fix-missing && \ COPY ./log4j2.xml ${SPARK_HOME}/conf/log4j2.xml COPY ./spark-defaults.conf ${SPARK_HOME}/conf/spark-defaults.conf RUN ln -fs /bin/bash /bin/sh -RUN if [ $SPARK_VERSION = "3.4.1" ]; then \ +RUN if [ $SPARK_VERSION = "3.1.3" ]; then \ rm $SPARK_HOME/jars/okhttp-*.jar && \ wget -P $SPARK_HOME/jars https://repo1.maven.org/maven2/com/squareup/okhttp3/okhttp/3.8.0/okhttp-3.8.0.jar; \ elif [ $SPARK_VERSION = "2.4.6" ]; then \ diff --git a/docker/orca/Dockerfile b/docker/orca/Dockerfile index 9b81a5a65f3..2d08a26a3fb 100644 --- a/docker/orca/Dockerfile +++ b/docker/orca/Dockerfile @@ -62,7 +62,7 @@ RUN apt-get update --fix-missing && \ COPY ./log4j2.xml ${SPARK_HOME}/conf/log4j2.xml COPY ./spark-defaults.conf ${SPARK_HOME}/conf/spark-defaults.conf SHELL ["/bin/bash", "-c"] -RUN if [ $SPARK_VERSION = "3.4.1" ]; then \ +RUN if [ $SPARK_VERSION = "3.1.3" ]; then \ rm $SPARK_HOME/jars/okhttp-*.jar && \ wget --no-check-certificate -qP $SPARK_HOME/jars https://repo1.maven.org/maven2/com/squareup/okhttp3/okhttp/3.8.0/okhttp-3.8.0.jar; \ elif [ $SPARK_VERSION = "2.4.6" ]; then \ diff --git a/python/dev/release_default_mac_spark3.sh b/python/dev/release_default_mac_spark3.sh index 9558ad473b1..c241b0fd2b2 100755 --- a/python/dev/release_default_mac_spark3.sh +++ b/python/dev/release_default_mac_spark3.sh @@ -17,7 +17,7 @@ # # This is the default script with maven parameters to release all the bigdl sub-packages -# built on top of Spark3 (currently 3.4.1) for mac. +# built on top of Spark3 (currently 3.1.3) for mac. set -e RUN_SCRIPT_DIR=$(cd $(dirname $0) ; pwd) @@ -44,4 +44,4 @@ else profiles=${*:5} fi -bash ${RUN_SCRIPT_DIR}/release_default_spark.sh mac ${version} ${quick} ${upload} 3.4.1 ${suffix} ${profiles} +bash ${RUN_SCRIPT_DIR}/release_default_spark.sh mac ${version} ${quick} ${upload} 3.1.3 ${suffix} ${profiles} diff --git a/scala/common/spark-version/2.0/pom.xml b/scala/common/spark-version/2.0/pom.xml index 9fcd2cb892e..ca3b05c878d 100644 --- a/scala/common/spark-version/2.0/pom.xml +++ b/scala/common/spark-version/2.0/pom.xml @@ -17,7 +17,7 @@ org.apache.hadoop hadoop-client - ${hadoop.version} + 3.2.4 provided @@ -61,7 +61,7 @@ org.apache.hadoop hadoop-common - ${hadoop.version} + 3.2.4 provided diff --git a/scala/common/spark-version/3.0/pom.xml b/scala/common/spark-version/3.0/pom.xml index a82c117d13d..a7b25066b38 100644 --- a/scala/common/spark-version/3.0/pom.xml +++ b/scala/common/spark-version/3.0/pom.xml @@ -17,7 +17,7 @@ org.apache.hadoop hadoop-client - ${hadoop.version} + 3.2.4 provided @@ -56,16 +56,12 @@ org.apache.hadoop hadoop-hdfs-client - - com.fasterxml.jackson.core - jackson-databind - org.apache.hadoop hadoop-common - ${hadoop.version} + 3.2.4 provided @@ -100,14 +96,6 @@ org.codehaus.jettison jettison - - org.apache.commons - commons-lang3 - - - org.apache.commons - commons-text - diff --git a/scala/dllib/pom.xml b/scala/dllib/pom.xml index ac762e94bf4..cf6a7d51060 100644 --- a/scala/dllib/pom.xml +++ b/scala/dllib/pom.xml @@ -115,14 +115,6 @@ net.minidev json-smart - - org.apache.commons - commons-lang3 - - - org.apache.commons - commons-text - @@ -149,14 +141,6 @@ log4j log4j - - io.netty - netty-handler - - - io.netty - netty-transport-native-epoll - @@ -243,14 +227,6 @@ org.apache.ivy ivy - - org.apache.commons - commons-lang3 - - - org.apache.commons - commons-text - @@ -316,18 +292,6 @@ commons-compress 1.21 - - org.apache.commons - commons-lang3 - 3.12.0 - ${spark-scope} - - - org.apache.commons - commons-text - 1.10.0 - ${spark-scope} - org.scalanlp breeze_${scala.major.version} diff --git a/scala/friesian/pom.xml b/scala/friesian/pom.xml index e70efa38ad7..298e54e833d 100644 --- a/scala/friesian/pom.xml +++ b/scala/friesian/pom.xml @@ -137,7 +137,7 @@ org.apache.hadoop hadoop-common - ${hadoop.version} + 3.2.4 provided @@ -234,7 +234,7 @@ io.lettuce lettuce-core - 6.2.4.RELEASE + 6.1.6.RELEASE ${serving.scope} diff --git a/scala/pom.xml b/scala/pom.xml index c33e6868f9c..21015246251 100644 --- a/scala/pom.xml +++ b/scala/pom.xml @@ -109,9 +109,9 @@ 1.7 1.7 - 2.11 - 2.11.12 - 2.1.0 + 2.12 + 2.12.15 + 2.1.1 3.0.7 @@ -160,10 +160,9 @@ 1.0-alpha-8 3.0.0-M3 - 3.3.5 - 32.0.1-jre - 3.7.0-M11 - 1.1.2 + 3.2.4 + 32.0.1-jre + 1.1.2 thrift 0.9.2 1.7.7 @@ -189,7 +188,7 @@ http://download.tensorflow.org provided 1.53.0 - 1.12.3 + 1.12.2 @@ -814,9 +813,10 @@ 1.8 1.8 + 3.1.3 3.0 - 3.4.1 - 2.12 + 2.12.10 + 2.1.0 2.12.15 2.1.1 1.6.2 @@ -897,8 +897,8 @@ scala_2.12 2.12 - 2.12.15 - 2.1.1 + 2.12.10 + 2.1.0 diff --git a/scala/ppml/pom.xml b/scala/ppml/pom.xml index 6acdd0c94b6..babaf22eb68 100644 --- a/scala/ppml/pom.xml +++ b/scala/ppml/pom.xml @@ -114,10 +114,6 @@ org.apache.ivy ivy - - org.scala-lang.modules - scala-parser-combinators_${scala.major.version} - @@ -125,17 +121,6 @@ commons-text 1.10.0 - - org.scala-lang.modules - scala-parser-combinators_${scala.major.version} - 1.1.2 - - - org.apache.hadoop - hadoop-client - ${hadoop.version} - provided - org.apache.commons commons-compress diff --git a/scala/serving/pom.xml b/scala/serving/pom.xml index 20683c0cca1..4911336413d 100644 --- a/scala/serving/pom.xml +++ b/scala/serving/pom.xml @@ -44,7 +44,7 @@ org.json4s json4s-jackson_${scala.major.version} - ${json4s.version} + 3.7.0-M5 com.google.protobuf