-
Notifications
You must be signed in to change notification settings - Fork 12
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
Browse files
Browse the repository at this point in the history
* [ML-35] Restrict printNumericTable to first 10 eigenvalues with first 20 dimensions (#36) * restrict PCA printNumericTable to first 10 eigenvalues with first 20 dimensions * fix ALS printNumericTable * [ML-44] [PIP] Update to oneAPI 2021.2 and Rework examples for validation (#47) * remove hibench examples * Fix tbb linking * Add data * Add env.sh.template * Revise examples * Add ALS scala and modify als-pyspark.py * nit * Add build-all & run-all * remove test-cluster/workloads and use examples for validation * add setup-python3 and setup-cluster, fix paths * fix java home * add config ssh * add config ssh * add config ssh * add config ssh * fix config-ssh * fix config-ssh * fix config-ssh * fix config-ssh * fix config-ssh * fix config-ssh * set strict modes no * clear out comments * Update oneCCL and oneDAL to oneAPI 2021.2.0, don't build oneCCL from source * nit * Fix install oneapi and source setvars * nit * Add spark.driver.host * Add ci-build * nit * Update * Update * Add --ccl-configuration=cpu_icc * Update * Update * revert to build oneCCL from source and package related so * nit * nit * Add ci-test-cluster * update * update * update * update * update * Add check: OneCCL doesn't support loopback IP * update * update * update * update * update * update * update * update * update * update * update * update README * update * update * update * update * Update README and nit changes
- Loading branch information
Showing
54 changed files
with
761 additions
and
1,092 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,46 @@ | ||
| # == OAP MLlib users to customize the following environments for running examples ======= # | ||
|
|
||
| # ============== Minimum Settings ============= # | ||
|
|
||
| # Set OAP MLlib version (e.g. 1.1.0) | ||
| OAP_MLLIB_VERSION=x.x.x | ||
| # Set Spark master | ||
| SPARK_MASTER=yarn | ||
| # Set Hadoop home path | ||
| export HADOOP_HOME=/path/to/your/hadoop/home | ||
| # Set Spark home path | ||
| export SPARK_HOME=/path/to/your/spark/home | ||
| # Set HDFS Root, should be hdfs://xxx or file://xxx | ||
| export HDFS_ROOT=hdfs://localhost:8020 | ||
| # Set OAP MLlib source code root directory | ||
| export OAP_MLLIB_ROOT=/path/to/oap-mllib/home | ||
|
|
||
| # ============================================= # | ||
|
|
||
| # Set HADOOP_CONF_DIR for Spark | ||
| export HADOOP_CONF_DIR=$HADOOP_HOME/etc/hadoop | ||
|
|
||
| # Set JAR name & path | ||
| OAP_MLLIB_JAR_NAME=oap-mllib-$OAP_MLLIB_VERSION.jar | ||
| OAP_MLLIB_JAR=$OAP_MLLIB_ROOT/mllib-dal/target/$OAP_MLLIB_JAR_NAME | ||
| # Set Spark driver & executor classpaths, | ||
| # absolute path for driver, relative path for executor | ||
| SPARK_DRIVER_CLASSPATH=$OAP_MLLIB_JAR | ||
| SPARK_EXECUTOR_CLASSPATH=./$OAP_MLLIB_JAR_NAME | ||
|
|
||
| # Set Spark resources, can be overwritten in example | ||
| SPARK_DRIVER_MEMORY=1G | ||
| SPARK_NUM_EXECUTORS=2 | ||
| SPARK_EXECUTOR_CORES=1 | ||
| SPARK_EXECUTOR_MEMORY=1G | ||
| SPARK_DEFAULT_PARALLELISM=$(expr $SPARK_NUM_EXECUTORS '*' $SPARK_EXECUTOR_CORES '*' 2) | ||
|
|
||
| # Checks | ||
|
|
||
| for dir in $SPARK_HOME $HADOOP_HOME $OAP_MLLIB_JAR | ||
| do | ||
| if [[ ! -e $dir ]]; then | ||
| echo $dir does not exist! | ||
| exit 1 | ||
| fi | ||
| done |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,43 @@ | ||
| #!/usr/bin/env bash | ||
|
|
||
| # Setup building envs | ||
| source /opt/intel/oneapi/setvars.sh | ||
| source /tmp/oneCCL/build/_install/env/setvars.sh | ||
|
|
||
| # Check envs for building | ||
| if [[ -z $JAVA_HOME ]]; then | ||
| echo JAVA_HOME not defined! | ||
| exit 1 | ||
| fi | ||
|
|
||
| if [[ -z $(which mvn) ]]; then | ||
| echo Maven not found! | ||
| exit 1 | ||
| fi | ||
|
|
||
| if [[ -z $DAALROOT ]]; then | ||
| echo DAALROOT not defined! | ||
| exit 1 | ||
| fi | ||
|
|
||
| if [[ -z $TBBROOT ]]; then | ||
| echo TBBROOT not defined! | ||
| exit 1 | ||
| fi | ||
|
|
||
| if [[ -z $CCL_ROOT ]]; then | ||
| echo CCL_ROOT not defined! | ||
| exit 1 | ||
| fi | ||
|
|
||
| echo === Building Environments === | ||
| echo JAVA_HOME=$JAVA_HOME | ||
| echo DAALROOT=$DAALROOT | ||
| echo TBBROOT=$TBBROOT | ||
| echo CCL_ROOT=$CCL_ROOT | ||
| echo Maven Version: $(mvn -v | head -n 1 | cut -f3 -d" ") | ||
| echo Clang Version: $(clang -dumpversion) | ||
| echo ============================= | ||
|
|
||
| cd $GITHUB_WORKSPACE/mllib-dal | ||
| mvn --no-transfer-progress -DskipTests clean package |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,61 @@ | ||
| #!/usr/bin/env bash | ||
|
|
||
| # Setup building envs | ||
| source /opt/intel/oneapi/setvars.sh | ||
| source /tmp/oneCCL/build/_install/env/setvars.sh | ||
|
|
||
| # Check envs for building | ||
| if [[ -z $JAVA_HOME ]]; then | ||
| echo JAVA_HOME not defined! | ||
| exit 1 | ||
| fi | ||
|
|
||
| if [[ -z $(which mvn) ]]; then | ||
| echo Maven not found! | ||
| exit 1 | ||
| fi | ||
|
|
||
| if [[ -z $DAALROOT ]]; then | ||
| echo DAALROOT not defined! | ||
| exit 1 | ||
| fi | ||
|
|
||
| if [[ -z $TBBROOT ]]; then | ||
| echo TBBROOT not defined! | ||
| exit 1 | ||
| fi | ||
|
|
||
| if [[ -z $CCL_ROOT ]]; then | ||
| echo CCL_ROOT not defined! | ||
| exit 1 | ||
| fi | ||
|
|
||
| echo === Testing Environments === | ||
| echo JAVA_HOME=$JAVA_HOME | ||
| echo DAALROOT=$DAALROOT | ||
| echo TBBROOT=$TBBROOT | ||
| echo CCL_ROOT=$CCL_ROOT | ||
| echo Maven Version: $(mvn -v | head -n 1 | cut -f3 -d" ") | ||
| echo Clang Version: $(clang -dumpversion) | ||
| echo ============================= | ||
|
|
||
| cd $GITHUB_WORKSPACE/mllib-dal | ||
|
|
||
| # Build test | ||
| $GITHUB_WORKSPACE/dev/ci-build.sh | ||
|
|
||
| # Enable signal chaining support for JNI | ||
| # export LD_PRELOAD=$JAVA_HOME/jre/lib/amd64/libjsig.so | ||
|
|
||
| # -Dtest=none to turn off the Java tests | ||
|
|
||
| # Test all | ||
| # mvn -Dtest=none -Dmaven.test.skip=false test | ||
|
|
||
| # Individual test | ||
| mvn --no-transfer-progress -Dtest=none -DwildcardSuites=org.apache.spark.ml.clustering.IntelKMeansSuite test | ||
| mvn --no-transfer-progress -Dtest=none -DwildcardSuites=org.apache.spark.ml.feature.IntelPCASuite test | ||
| # mvn -Dtest=none -DwildcardSuites=org.apache.spark.ml.recommendation.IntelALSSuite test | ||
|
|
||
| # Yarn cluster test | ||
| $GITHUB_WORKSPACE/dev/test-cluster/ci-test-cluster.sh |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -1,32 +1,25 @@ | ||
| #!/usr/bin/env bash | ||
|
|
||
| if [ ! -f /opt/intel/oneapi ]; then | ||
| if [ ! -d /opt/intel/oneapi ]; then | ||
| echo "Installing oneAPI components ..." | ||
| cd /tmp | ||
| wget https://apt.repos.intel.com/intel-gpg-keys/GPG-PUB-KEY-INTEL-SW-PRODUCTS-2023.PUB | ||
| sudo apt-key add GPG-PUB-KEY-INTEL-SW-PRODUCTS-2023.PUB | ||
| rm GPG-PUB-KEY-INTEL-SW-PRODUCTS-2023.PUB | ||
| echo "deb https://apt.repos.intel.com/oneapi all main" | sudo tee /etc/apt/sources.list.d/oneAPI.list | ||
| sudo apt-get update | ||
| sudo apt-get install intel-oneapi-dal-devel-2021.1.1 intel-oneapi-tbb-devel-2021.1.1 | ||
| # sudo apt-get install -y build-essential cmake | ||
| sudo apt-get install -y intel-oneapi-dpcpp-cpp-2021.2.0 intel-oneapi-dal-devel-2021.2.0 intel-oneapi-tbb-devel-2021.2.0 | ||
| else | ||
| echo "oneAPI components already installed!" | ||
| fi | ||
| fi | ||
|
|
||
| echo "Building oneCCL ..." | ||
| cd /tmp | ||
| rm -rf oneCCL | ||
| git clone https://github.com/oneapi-src/oneCCL | ||
| cd oneCCL | ||
| git checkout 2021.1 | ||
| git checkout 2021.2 | ||
| mkdir build && cd build | ||
| cmake .. | ||
| make -j 2 install | ||
|
|
||
| # | ||
| # Setup building environments manually: | ||
| # | ||
| # export ONEAPI_ROOT=/opt/intel/oneapi | ||
| # source /opt/intel/oneapi/dal/latest/env/vars.sh | ||
| # source /opt/intel/oneapi/tbb/latest/env/vars.sh | ||
| # source /tmp/oneCCL/build/_install/env/setvars.sh | ||
| # |
Oops, something went wrong.