update CI actions and reduce Linear dataset

Signed-off-by: minmingzhu <minming.zhu@intel.com>
Intel-bigdata · Jul 14, 2023 · 921635f · 921635f
1 parent c7313ff
commit 921635f
Show file tree

Hide file tree

Showing 7 changed files with 15 additions and 9 deletions.
diff --git a/.github/workflows/hibench_ci_spark2.4_hadoop_2.7.yml b/.github/workflows/hibench_ci_spark2.4_hadoop_2.7.yml
@@ -7,13 +7,13 @@ jobs:
     runs-on: ubuntu-20.04
 
     steps:
-      - uses: actions/checkout@v2
+      - uses: actions/checkout@v3
       - name: Set up JDK 1.8
         uses: actions/setup-java@v1
         with:
           java-version: 1.8
       - name: Restore cached dependencies
-        uses: actions/cache@v2
+        uses: actions/cache@v3
         with:
           path: |
             # /var/cache/apt/archives/*.deb

diff --git a/.github/workflows/hibench_ci_spark3.0_hadoop_3.2.yml b/.github/workflows/hibench_ci_spark3.0_hadoop_3.2.yml
@@ -7,13 +7,13 @@ jobs:
     runs-on: ubuntu-20.04
 
     steps:
-      - uses: actions/checkout@v2
+      - uses: actions/checkout@v3
       - name: Set up JDK 1.8
         uses: actions/setup-java@v1
         with:
           java-version: 1.8
       - name: Restore cached dependencies
-        uses: actions/cache@v2
+        uses: actions/cache@v3
         with:
           path: |
             # /var/cache/apt/archives/*.deb

diff --git a/.github/workflows/hibench_ci_spark3.1_hadoop_3.2.yml b/.github/workflows/hibench_ci_spark3.1_hadoop_3.2.yml
@@ -7,13 +7,13 @@ jobs:
     runs-on: ubuntu-20.04
 
     steps:
-      - uses: actions/checkout@v2
+      - uses: actions/checkout@v3
       - name: Set up JDK 1.8
         uses: actions/setup-java@v1
         with:
           java-version: 1.8
       - name: Restore cached dependencies
-        uses: actions/cache@v2
+        uses: actions/cache@v3
         with:
           path: |
             # /var/cache/apt/archives/*.deb

diff --git a/actions/build_and_run_all_spark_3.1_hadoop_3.2.sh b/actions/build_and_run_all_spark_3.1_hadoop_3.2.sh
@@ -29,3 +29,6 @@ echo "========================================="
 
 # run all examples
 source $GITHUB_WORKSPACE/bin/run_all.sh
+
+ls -l $HADOOP_HOME/logs/
+cat $HADOOP_HOME/logs/*.log
diff --git a/actions/spark.conf b/actions/spark.conf
@@ -9,8 +9,8 @@ hibench.yarn.executor.num     2
 hibench.yarn.executor.cores   1
 
 # executor and driver memory in standalone & YARN mode
-spark.executor.memory  4g
-spark.driver.memory    1g
+spark.executor.memory  2g
+spark.driver.memory    2g
 
 # set spark parallelism property according to hibench's parallelism value
 spark.default.parallelism     ${hibench.default.map.parallelism}

diff --git a/actions/test-cluster/setup-cluster.sh b/actions/test-cluster/setup-cluster.sh
@@ -59,10 +59,13 @@ mkdir -p /tmp/run/hdfs/datanode
 # hdfs format
 $HADOOP_HOME/bin/hdfs namenode -format
 
+wget -P $HADOOP_HOME/share/hadoop/yarn/lib/ https://repo1.maven.org/maven2/javax/activation/activation/1.1.1/activation-1.1.1.jar
+
 # start hdfs and yarn
 $HADOOP_HOME/sbin/start-dfs.sh
 $HADOOP_HOME/sbin/start-yarn.sh
 
 sleep 10
+
 $HADOOP_HOME/bin/hadoop fs -ls /
 $HADOOP_HOME/bin/yarn node -list 2
diff --git a/conf/workloads/ml/linear.conf b/conf/workloads/ml/linear.conf
@@ -1,5 +1,5 @@
 hibench.linear.tiny.examples           50000
-hibench.linear.tiny.features           10000
+hibench.linear.tiny.features           1000
 
 hibench.linear.small.examples          100000
 hibench.linear.small.features          20000