Skip to content
This repository was archived by the owner on Dec 15, 2025. It is now read-only.

supporting multi benchmarks and modules flexible build&&add sort test case #313

Merged
carsonwang merged 10 commits into from
Sep 23, 2016
Merged

supporting multi benchmarks and modules flexible build&&add sort test case #313

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
10 commits
Select commit Hold shift + click to select a range
54cd7be
support multi benchmark/module and more flexible spark version
zybing Sep 21, 2016
652a403
sparkbench's pom for multi spark module build
zybing Sep 21, 2016
51030fe
modify parent and remove mr profiles
zybing Sep 21, 2016
c0344e8
modify parent remove mr profiles
zybing Sep 21, 2016
2344cca
add basic graph pom file
zybing Sep 21, 2016
a035622
add basic pom file
zybing Sep 21, 2016
bef5e96
add basic pom file for other benchmark
zybing Sep 21, 2016
b5f8f26
add sort run.sh
zybing Sep 22, 2016
8039e00
add sort.conf
zybing Sep 22, 2016
8340598
flexibl specify benchmark and module
zybing Sep 22, 2016
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
34 changes: 34 additions & 0 deletions bin/workloads/micro/sort/hadoop/run.sh
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,34 @@
#!/bin/bash
# Licensed to the Apache Software Foundation (ASF) under one or more
# contributor license agreements. See the NOTICE file distributed with
# this work for additional information regarding copyright ownership.
# The ASF licenses this file to You under the Apache License, Version 2.0
# (the "License"); you may not use this file except in compliance with
# the License. You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

current_dir=`dirname "$0"`
root_dir=${current_dir}/../../../../../
workload_config=${root_dir}/conf/workloads/micro/sort.conf
. "${root_dir}/bin/functions/load-bench-config.sh"

enter_bench HadoopSort ${workload_config}
show_bannar start

rmr-hdfs $OUTPUT_HDFS || true

SIZE=`dir_size $INPUT_HDFS`
START_TIME=`timestamp`
run-hadoop-job ${HADOOP_EXAMPLES_JAR} sort -outKey org.apache.hadoop.io.Text -outValue org.apache.hadoop.io.Text -r ${NUM_REDS} ${INPUT_HDFS} ${OUTPUT_HDFS}

END_TIME=`timestamp`
gen_report ${START_TIME} ${END_TIME} ${SIZE}
show_bannar finish
leave_bench
36 changes: 36 additions & 0 deletions bin/workloads/micro/sort/prepare/prepare.sh
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,36 @@
#!/bin/bash
# Licensed to the Apache Software Foundation (ASF) under one or more
# contributor license agreements. See the NOTICE file distributed with
# this work for additional information regarding copyright ownership.
# The ASF licenses this file to You under the Apache License, Version 2.0
# (the "License"); you may not use this file except in compliance with
# the License. You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

current_dir=`dirname "$0"`
root_dir=${current_dir}/../../../../../
workload_config=${root_dir}/conf/workloads/micro/sort.conf
. "${root_dir}/bin/functions/load-bench-config.sh"

enter_bench HadoopPrepareSort ${workload_config}
show_bannar start

rmr-hdfs $INPUT_HDFS || true
START_TIME=`timestamp`

run-hadoop-job ${HADOOP_EXAMPLES_JAR} randomtextwriter \
-D mapreduce.randomtextwriter.totalbytes=${DATASIZE} \
-D mapreduce.randomtextwriter.bytespermap=$(( ${DATASIZE} / ${NUM_MAPS} )) \
-D mapreduce.job.maps=${NUM_MAPS} \
-D mapreduce.job.reduces=${NUM_REDS} \
${INPUT_HDFS}
END_TIME=`timestamp`
show_bannar finish
leave_bench
34 changes: 34 additions & 0 deletions bin/workloads/micro/sort/spark/run.sh
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,34 @@
#!/bin/bash
# Licensed to the Apache Software Foundation (ASF) under one or more
# contributor license agreements. See the NOTICE file distributed with
# this work for additional information regarding copyright ownership.
# The ASF licenses this file to You under the Apache License, Version 2.0
# (the "License"); you may not use this file except in compliance with
# the License. You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

current_dir=`dirname "$0"`
root_dir=${current_dir}/../../../../../
workload_config=${root_dir}/conf/workloads/micro/sort.conf
. "${root_dir}/bin/functions/load-bench-config.sh"

enter_bench ScalaSparkSort ${workload_config}
show_bannar start

rmr-hdfs $OUTPUT_HDFS || true

SIZE=`dir_size $INPUT_HDFS`
START_TIME=`timestamp`
run-spark-job com.intel.hibench.sparkbench.micro.ScalaSort $INPUT_HDFS $OUTPUT_HDFS
END_TIME=`timestamp`

gen_report ${START_TIME} ${END_TIME} ${SIZE}
show_bannar finish
leave_bench
16 changes: 16 additions & 0 deletions conf/workloads/micro/sort.conf
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,16 @@
#Sort
# datasize=> bytes of data
hibench.sort.tiny.datasize 32000
hibench.sort.small.datasize 3200000
hibench.sort.large.datasize 320000000
hibench.sort.huge.datasize 3200000000
hibench.sort.gigantic.datasize 32000000000
hibench.sort.bigdata.datasize 300000000000

hibench.workload.datasize ${hibench.sort.${hibench.scale.profile}.datasize}
hibench.sparkbench.jar ${hibench.home}/sparkbench/micro/target/sparkbench-micro-6.0-SNAPSHOT-${hibench.spark.version}-jar-with-dependencies.jar

# export for shell script
hibench.workload.input ${hibench.hdfs.data.dir}/Sort/Input
hibench.workload.output ${hibench.hdfs.data.dir}/Sort/Output

15 changes: 15 additions & 0 deletions flinkbench/pom.xml
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,15 @@
<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/maven-v4_0_0.xsd">
<modelVersion>4.0.0</modelVersion>
<parent>
<groupId>com.intel.hibench</groupId>
<artifactId>hibench</artifactId>
<version>6.0-SNAPSHOT</version>
</parent>
<groupId>com.intel.hibench</groupId>
<artifactId>flinkbench</artifactId>
<packaging>pom</packaging>
<version>6.0-SNAPSHOT</version>
<name>flinkbench</name>
</project>

14 changes: 14 additions & 0 deletions gearpumpbench/pom.xml
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,14 @@
<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/maven-v4_0_0.xsd">
<modelVersion>4.0.0</modelVersion>
<parent>
<groupId>com.intel.hibench</groupId>
<artifactId>hibench</artifactId>
<version>6.0-SNAPSHOT</version>
</parent>
<groupId>com.intel.hibench</groupId>
<artifactId>gearpumpbench</artifactId>
<packaging>pom</packaging>
<version>6.0-SNAPSHOT</version>
<name>gearpumpbench</name>
</project>
14 changes: 14 additions & 0 deletions hadoopbench/pom.xml
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,14 @@
<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/maven-v4_0_0.xsd">
<modelVersion>4.0.0</modelVersion>
<parent>
<groupId>com.intel.hibench</groupId>
<artifactId>hibench</artifactId>
<version>6.0-SNAPSHOT</version>
</parent>
<groupId>com.intel.hibench</groupId>
<artifactId>hadoopbench</artifactId>
<packaging>pom</packaging>
<version>6.0-SNAPSHOT</version>
<name>hadoopbench</name>
</project>
71 changes: 50 additions & 21 deletions pom.xml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -106,34 +106,63 @@
</plugins>
</build>

<modules>
<module>sparkbench/common</module>
<module>sparkbench/micro</module>
</modules>

<profiles>
<profile>
<id>spark1.6</id>
<properties>
<env>spark1.6</env>
<spark.version>1.6.0</spark.version>
<spark.bin.version>1.6</spark.bin.version>
</properties>
<activation>
<property><name>spark1.6</name></property>
</activation>
<id>sparkbench</id>
<modules>
<module>sparkbench</module>
</modules>
</profile>

<profile>
<id>flinkbench</id>
<modules>
<module>flinkbench</module>
</modules>
</profile>

<profile>
<id>spark2.0</id>
<properties>
<env>spark2.0</env>
<spark.version>2.0.0</spark.version>
<spark.bin.version>2.0</spark.bin.version>
</properties>
<id>gearpumpbench</id>
<modules>
<module>gearpumpbench</module>
</modules>
</profile>

<profile>
<id>hadoopbench</id>
<modules>
<module>hadoopbench</module>
</modules>
</profile>

<profile>
<id>samzabench</id>
<modules>
<module>samzabench</module>
</modules>
</profile>

<profile>
<id>stormbench</id>
<modules>
<module>stormbench</module>
</modules>
</profile>

<profile>
<id>defaultbench</id>
<modules>
<module>sparkbench</module>
<module>flinkbench</module>
<module>gearpumpbench</module>
<module>hadoopbench</module>
<module>samzabench</module>
<module>stormbench</module>
</modules>
<activation>
<property><name>spark2.0</name></property>
<activeByDefault>true</activeByDefault>
</activation>
</profile>

</profiles>
</project>
14 changes: 14 additions & 0 deletions samzabench/pom.xml
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,14 @@
<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/maven-v4_0_0.xsd">
<modelVersion>4.0.0</modelVersion>
<parent>
<groupId>com.intel.hibench</groupId>
<artifactId>hibench</artifactId>
<version>6.0-SNAPSHOT</version>
</parent>
<groupId>com.intel.hibench</groupId>
<artifactId>samzabench</artifactId>
<packaging>pom</packaging>
<version>6.0-SNAPSHOT</version>
<name>samzabench</name>
</project>
74 changes: 45 additions & 29 deletions sparkbench/common/pom.xml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -6,9 +6,8 @@

<parent>
<groupId>com.intel.hibench</groupId>
<artifactId>hibench</artifactId>
<artifactId>sparkbench</artifactId>
<version>6.0-SNAPSHOT</version>
<relativePath>../../pom.xml</relativePath>
</parent>

<groupId>com.intel.hibench.sparkbench</groupId>
Expand Down Expand Up @@ -169,30 +168,47 @@

</plugins>
</build>
<profiles>
<profile>
<id>MR2</id>
<dependencies>
<dependency>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-mapreduce-examples</artifactId>
<version>${hadoop.mr2.version}</version>
</dependency>
<dependency>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-client</artifactId>
<version>${hadoop.mr2.version}</version>
<exclusions>
<exclusion>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-hdfs</artifactId>
</exclusion>
</exclusions>
</dependency>
</dependencies>
<activation>
<property><name>!MR1</name></property>
</activation>
</profile>
</profiles>
</project>
<profiles>
<profile>
<id>spark1.6</id>
<properties>
<spark.version>1.6.0</spark.version>
<spark.bin.version>1.6</spark.bin.version>
</properties>
<activation>
<property>
<name>spark</name>
<value>1.6</value>
</property>
</activation>
</profile>

<profile>
<id>spark2.0</id>
<properties>
<spark.version>2.0.0</spark.version>
<spark.bin.version>2.0</spark.bin.version>
</properties>
<activation>
<property>
<name>spark</name>
<value>2.0</value>
</property>
</activation>
</profile>

<profile>
<id>defaultspark</id>
<properties>
<spark.version>1.6.0</spark.version>
<spark.bin.version>1.6</spark.bin.version>
</properties>
<activation>
<property>
<name>!spark</name>
</property>
</activation>
</profile>

</profiles>
</project>
16 changes: 16 additions & 0 deletions sparkbench/graph/pom.xml
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,16 @@
<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/maven-v4_0_0.xsd">
<modelVersion>4.0.0</modelVersion>

<parent>
<groupId>com.intel.hibench</groupId>
<artifactId>sparkbench</artifactId>
<version>6.0-SNAPSHOT</version>
</parent>

<groupId>com.intel.hibench</groupId>
<artifactId>sparkbench-graph</artifactId>
<packaging>jar</packaging>
<version>6.0-SNAPSHOT</version>
<name>sparkbench-graph</name>
</project>
Loading