Skip to content
This repository
Browse code

Merge remote-tracking branch 'jfishman/master'

Builds a bundled mstress jar and tar file, excluded from the ALL target of the main build, using ant
  • Loading branch information...
commit eef3dc4dece787a7eecd03d560074e40685e20e2 2 parents 3148dd7 + 5e08401
Jeremy R. Fishman authored
73 benchmarks/mstress/CMakeLists.txt
@@ -19,7 +19,8 @@
19 19 # Build the C++ mstress benchmark
20 20 #
21 21
22   -add_executable (mstress_client mstress_client.cc)
  22 +add_executable (mstress_client EXCLUDE_FROM_ALL mstress_client.cc)
  23 +
23 24 IF (USE_STATIC_LIB_LINKAGE)
24 25 add_dependencies (mstress_client kfsClient)
25 26 target_link_libraries (mstress_client kfsClient qcdio pthread)
@@ -32,5 +33,71 @@ IF (NOT APPLE)
32 33 target_link_libraries (rt)
33 34 ENDIF (NOT APPLE)
34 35
35   -install (TARGETS mstress_client
36   - RUNTIME DESTINATION bin/benchmarks)
  36 +add_custom_command (
  37 + OUTPUT mstress.jar
  38 + COMMAND ant -f ${CMAKE_CURRENT_SOURCE_DIR}/build.xml
  39 + DEPENDS MStress_Client.java
  40 + COMMENT The HDFS mstress client bundled as a jar.
  41 + VERBATIM
  42 +)
  43 +
  44 +add_custom_target (mstress DEPENDS mstress_client mstress.jar)
  45 +
  46 +set (mstress_scripts
  47 + mstress_plan.py
  48 + mstress_run.py
  49 + mstress.py
  50 + mstress_cleanup.py
  51 + mstress_install.sh
  52 +)
  53 +
  54 +foreach (script ${mstress_scripts})
  55 + add_custom_command (
  56 + OUTPUT ${script}
  57 + COMMAND ${CMAKE_COMMAND} -E copy ${CMAKE_CURRENT_SOURCE_DIR}/${script} ${CMAKE_CURRENT_BINARY_DIR}/
  58 + )
  59 +endforeach(script)
  60 +
  61 +add_custom_target (
  62 + mstress-scripts
  63 + DEPENDS ${mstress_scripts}
  64 + COMMENT Copy mstress scripts to build directory.
  65 +)
  66 +
  67 +add_custom_target (
  68 + mstress-tarball
  69 + COMMAND cd .. && ${CMAKE_COMMAND} -E tar czvf mstress.tgz
  70 + mstress/mstress_client
  71 + mstress/mstress.jar
  72 + mstress/*.py
  73 + mstress/*.sh
  74 + DEPENDS mstress_client mstress.jar
  75 + COMMENT Bundle mstress files in a tar archive.
  76 +)
  77 +add_dependencies(mstress-tarball mstress-scripts)
  78 +
  79 +get_property (metaserver_location TARGET metaserver PROPERTY LOCATION)
  80 +get_property (chunkserver_location TARGET chunkserver PROPERTY LOCATION)
  81 +add_custom_target (
  82 + mstress-bootstrap
  83 + COMMAND ${CMAKE_COMMAND} -E make_directory ${CMAKE_CURRENT_BINARY_DIR}/bin/
  84 + COMMAND ${CMAKE_COMMAND} -E copy ${metaserver_location} ${CMAKE_CURRENT_BINARY_DIR}/bin/
  85 + COMMAND ${CMAKE_COMMAND} -E copy ${chunkserver_location} ${CMAKE_CURRENT_BINARY_DIR}/bin/
  86 + COMMAND ${CMAKE_COMMAND} -E copy_directory ${CMAKE_SOURCE_DIR}/webui ${CMAKE_CURRENT_BINARY_DIR}/webui
  87 + COMMAND ${CMAKE_COMMAND} -E copy ${CMAKE_SOURCE_DIR}/examples/sampleservers/sample_setup.py ${CMAKE_CURRENT_BINARY_DIR}/setup.py
  88 + COMMAND cd .. && ${CMAKE_COMMAND} -E tar czvf mstress-bootstrap.tgz
  89 + mstress/mstress_client
  90 + mstress/mstress.jar
  91 + mstress/*.py
  92 + mstress/*.sh
  93 + mstress/bin/metaserver
  94 + mstress/bin/chunkserver
  95 + mstress/webui
  96 + DEPENDS mstress_client mstress.jar metaserver chunkserver
  97 + COMMENT Bundle mstress files along with QFS binaries.
  98 +)
  99 +add_dependencies(mstress-bootstrap mstress-scripts)
  100 +
  101 +set_directory_properties (PROPERTIES
  102 + ADDITIONAL_MAKE_CLEAN_FILES "../mstress.tgz;mstress.jar;${mstress_scripts}"
  103 +)
2  benchmarks/mstress/MStress_Client.java
@@ -17,7 +17,7 @@
17 17 * License for the specific language governing permissions and limitations
18 18 * under the License.
19 19 *
20   - * This Java client performs filesystem meta opetarions on the Hadoop namenode
  20 + * This Java client performs filesystem meta operations on the Hadoop namenode
21 21 * using HDFS DFSClient.
22 22 */
23 23
93 benchmarks/mstress/README
@@ -74,28 +74,27 @@ specified by the benchmark plan.
74 74 [2] Files
75 75 =========
76 76
77   - - mstress_initialize.sh
78   - Helper script to be used before compiling the source and deploying the
79   - mstress bundle.
80   - Do ./mstress_initialize.sh --h to see options.
  77 + - CMakeLists.txt
  78 + Builds the QFS stress client (C++) and HDFS stress client (Java) along with
  79 + the main release build.
81 80
82   - - Makefile
83   - Used to build the QFS stress client (C++) and HDFS stress client (Java).
84   - Ensure that $JAVA_HOME is set correctly.
  81 + - build.xml, ivy.xml, ivysettings.xml
  82 + Ant build file and Ivy dependency and settings files, used by CMakeLists.txt
  83 + to build the HDFS stress client (Java). Ensure that $JAVA_HOME is set.
85 84
86 85 - mstress_client.cc
87 86 Produces the mstress_client binary that actually drives the QFS metaserver.
88   - Build using the provided Makefile ('make ccclient')
  87 + Builds to $GIT_DIR/build/release/bin/benchmarks/mstress_client
89 88 See 'Benchmarking Procedure' below for details.
90 89
91 90 - MStress_Client.java
92   - Produces the java MStress_Client for HDFS namenode.
93   - Build using the provided Makefile ('make javaclient')
  91 + Produces the java MStress_Client for HDFS namenode. Built using ant to
  92 + $GIT_DIR/build/release/bin/bin/benchmarks/mstress.jar
94 93 See 'Benchmarking Procedure' below for details.
95 94
96   - - mstress_prepare_master_clients.sh
97   - Helper script used to copy the mstress directory to a list of hosts. To be
98   - used after running make.
  95 + - mstress_install.sh
  96 + Helper script used to deploy mstress to a list of hosts. Will invoke cmake
  97 + and make under ./build/.
99 98
100 99 - mstress_plan.py
101 100 Used to generate a plan file for benchmarking.
@@ -141,65 +140,20 @@ on the same box, "localhost".
141 140 (1) Setup the QFS metaserver and HDFS namenode with the help of
142 141 section [4] "Setting up DFS metaserver/namenode" below.
143 142
144   -
145 143 (2) You should have SSH key authentication set up on the hosts involved so
146 144 that the scripts can do password/passphrase-less login.
147 145
  146 +(3) On the build host, compile and install QFS using the steps described in
  147 + https://github.com/quantcast/qfs/wiki/Developer-Documentation.
148 148
149   -(3) On the build machine, ensure that you have the Cloudera HDFS client jars.
150   - This is typically at /usr/lib/hadoop/client/*.jars.
151   - If you don't have them, install them by,
152   - 1. Add the following to /etc/yum.repos.d/thirdparty.repo (sudo needed)
153   - -----------------------------------
154   - [cloudera-cdh4]
155   - name=Cloudera's Distribution for Hadoop, Version 4
156   - baseurl=http://archive.cloudera.com/cdh4/redhat/6/x86_64/cdh/4/
157   - gpgkey = http://archive.cloudera.com/cdh4/redhat/6/x86_64/cdh/RPM-GPG-KEY-cloudera
158   - gpgcheck = 1
159   - -----------------------------------
160   - 2. sudo yum install hadoop-client
161   -
162   -
163   -(4) On the build host, execute 'mstress_initialize.sh' to set up jar paths.
164   - ./mstress_initialize.sh /usr/lib/hadoop/client/
165   -
166   -
167   -(5) On the build host, compile and install QFS using the steps described in
168   - https://github.com/quantcast/qfs/wiki/Developer-Documentation. Then change
169   - directory to benchmarks/mstress, and just issuing 'make' should build the
170   - Java/C++ clients.
171   -
172   - To manually build C++ client:
173   - - Assuming the QFS code is in ~/code/qfs, compile and install QFS using
174   - the steps described in
175   - https://github.com/quantcast/qfs/wiki/Developer-Documentation
176   - - cd ~/code/qfs/benchmarks/mstress
177   - - QFS_BUILD_INCLUDE=~/code/qfs/build/include \
178   - QFS_BUILD_STATLIB=~/code/qfs/build/lib/static \
179   - BOOST_LIBRARY_DIR=/opt/local/lib \
180   - make
181   - If you encounter any build problem, ensure that your QFS_BUILD_INCLUDE etc.
182   - refer to valid paths.
183   -
184   -
185   - To manually build MStress_Client.java
186   - - Compile MStress_client.java with hadoop-client jars in the class path.
187   - theCP=$(echo mstress_hdfs_client_jars/*.jar | sed 's/ /:/g')
188   - javac -cp $theCP MStress_Client.java
189   -
190   -
191   -(6) Determine the master and load generating client hosts that you want to use
  149 +(4) Determine the master and load generating client hosts that you want to use
192 150 to connect to the DFS server. This could just be "localhost" if you want to
193 151 run the benchmark locally.
194 152
  153 +(5) From the build host in this directory, run `./mstress_install.sh hosts..`
  154 + to deploy mstress files to the participating hosts under ~/mstress.
195 155
196   -(7) From the build host, use "mstress_prepare_master_clients.sh" to copy your
197   - mstress directory to the participating hosts.
198   - Note: Do './mstress_prepare_master_clients.sh localhost' localhost-only run.
199   - The mstress directory paths should be the same on master and client hosts.
200   -
201   -
202   -(8) On the master host change directory to ~/mstress
  156 +(6) On the master host change directory to ~/mstress
203 157 Create a plan file using mstress_plan.py.
204 158 Do ./mstress_plan.py --help to see example usage.
205 159 Eg:
@@ -216,19 +170,18 @@ on the same box, "localhost".
216 170 The plan file gets copied to the /tmp directory where you run it. It will
217 171 also get copied to the participating client hosts in the '-c' option.
218 172
219   -
220   -(9) Checklist: check the presence of,
221   - - the plan file on master host and client hosts (step 8 does this for you)
  173 +(7) Checklist: check the presence of,
  174 + - the plan file on master host and client hosts (step 6 does this for you)
222 175 - the mstress_client binaries (QFS and HDFS clients) on master and all
223   - client hosts (step 7).
  176 + client hosts (step 5).
224 177
225   -(10) Run the benchmark from the master with mstress.py.
  178 +(8) Run the benchmark from the master with mstress.py.
226 179 Do ./mstress.py --help to see options.
227 180 Eg:
228 181 ./mstress.py -f qfs -s <metahost> -p <metaport> -a </tmp/something.plan>
229 182 ./mstress.py -f hdfs -s <namehost> -p <nameport> -a </tmp/something.plan>
230 183
231   -(11) The benchmark name, progress, and time taken will be printed out.
  184 +(9) The benchmark name, progress, and time taken will be printed out.
232 185
233 186
234 187 [4] DFS Server Setup
68 benchmarks/mstress/build.xml
... ... @@ -1,15 +1,42 @@
1   -<project name="qfs_mstress" default="compile" xmlns:ivy="antlib:org.apache.ivy.ant">
  1 +<!--
  2 +# $Id$
  3 +#
  4 +# Copyright 2012 Quantcast Corp.
  5 +#
  6 +# This file is part of Quantcast File System (QFS).
  7 +#
  8 +# Licensed under the Apache License, Version 2.0
  9 +# (the "License"); you may not use this file except in compliance with
  10 +# the License. You may obtain a copy of the License at
  11 +#
  12 +# http://www.apache.org/licenses/LICENSE-2.0
  13 +#
  14 +# Unless required by applicable law or agreed to in writing, software
  15 +# distributed under the License is distributed on an "AS IS" BASIS,
  16 +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
  17 +# implied. See the License for the specific language governing
  18 +# permissions and limitations under the License.
  19 +#
  20 +# Build the Java mstress benchmark against HDFS.
  21 +#
  22 +-->
  23 +<project name="mstress" default="build" xmlns:ivy="antlib:org.apache.ivy.ant">
2 24
3   - <property name="ivy.install.version" value="2.3.0" />
4   - <property name="ivy.jar.dir" value="${basedir}/.ivy" />
  25 + <property name="src.dir" value="${basedir}" />
  26 + <property name="dist.dir" value="${user.dir}" />
  27 + <property name="build.dir" value="${user.dir}/build" />
  28 +
  29 + <property name="ivy.jar.dir" value="${user.dir}/.ivy" />
5 30 <property name="ivy.jar.file" value="${ivy.jar.dir}/ivy.jar" />
6   - <property name="build.dir" value="build/output" />
7   - <property name="src.dir" value="." />
  31 + <property name="ivy.install.version" value="2.3.0" />
  32 +
  33 + <property name="hadoop.build.version" value="2.0.2-alpha" />
8 34
9 35 <!--=======================
10 36 target: get-ivy
11   - ===========================-->
12   - <target name="get-ivy" unless="skip.download">
  37 + ========================-->
  38 + <available property="ivy.jar.present" file="${ivy.jar.file}"/>
  39 + <target name="get-ivy" unless="ivy.jar.present">
13 40 <mkdir dir="${ivy.jar.dir}"/>
14 41 <echo message="installing ivy..."/>
15 42 <get src="http://repo1.maven.org/maven2/org/apache/ivy/ivy/${ivy.install.version}/ivy-${ivy.install.version}.jar"
@@ -18,7 +45,7 @@
18 45
19 46 <!--=======================
20 47 target: setup-ivy
21   - ===========================-->
  48 + ========================-->
22 49 <target name="setup-ivy" depends="get-ivy" description="==> install ivy">
23 50 <path id="ivy.lib.path">
24 51 <fileset dir="${ivy.jar.dir}" includes="*.jar"/>
@@ -28,22 +55,35 @@
28 55
29 56 <!--=======================
30 57 target: resolve
31   - ===========================-->
  58 + ========================-->
32 59 <target name="resolve" depends="setup-ivy" description="==> resolve ivy dependencies">
33 60 <ivy:cachepath pathid="ivy.lib.path" conf="compile" />
  61 + <ivy:cachefileset setid="runtime.classpath" conf="runtime" type="jar,bundle" />
34 62 </target>
35 63
36 64 <!--=======================
37 65 target: compile
38   - ===========================-->
  66 + ========================-->
39 67 <target name="compile" depends="resolve" description="==> compile sources">
40 68 <mkdir dir="${build.dir}" />
41 69 <javac srcdir="${src.dir}" destdir="${build.dir}" classpathref="ivy.lib.path" />
42 70 </target>
43 71
44 72 <!--=======================
  73 + target: build
  74 + ========================-->
  75 + <target name="build" depends="compile" description="==> package files into a jar">
  76 + <jar jarfile="${dist.dir}/mstress.jar" basedir="${build.dir}" compress="true">
  77 + <zipgroupfileset refid="runtime.classpath" />
  78 + <manifest>
  79 + <attribute name="Main-Class" value="MStress_Client" />
  80 + </manifest>
  81 + </jar>
  82 + </target>
  83 +
  84 + <!--=======================
45 85 target: run
46   - ===========================-->
  86 + ========================-->
47 87 <target name="run" depends="compile, input-runargs" description="==> run the client">
48 88 <java classname="MStress_Client">
49 89 <arg line="${runArgs}" />
@@ -59,7 +99,7 @@
59 99
60 100 <!--=======================
61 101 target: clean
62   - ===========================-->
  102 + ========================-->
63 103 <target name="clean" description="==> clean the project">
64 104 <delete includeemptydirs="true" quiet="true">
65 105 <fileset dir="${build.dir}" />
@@ -68,14 +108,14 @@
68 108
69 109 <!--=======================
70 110 target: clean-ivy
71   - ===========================-->
  111 + ========================-->
72 112 <target name="clean-ivy" description="==> clean the ivy installation">
73 113 <delete dir="${ivy.jar.dir}"/>
74 114 </target>
75 115
76 116 <!--=======================
77 117 target: clean-cache
78   - ===========================-->
  118 + ========================-->
79 119 <target name="clean-cache" depends="setup-ivy" description="==> clean the ivy cache">
80 120 <ivy:cleancache />
81 121 </target>
106 benchmarks/mstress/curl_run_mstress.sh
... ... @@ -0,0 +1,106 @@
  1 +#!/bin/bash
  2 +PORT=40000 # metaserver client port
  3 +CLIENTS=5
  4 +LEVELS=3
  5 +INODES=10
  6 +STATS=1000
  7 +
  8 +url=$1
  9 +tarfile=${2:-"mstress.tgz"}
  10 +if [ -z "$url" ]
  11 +then
  12 + echo >&2 "Usage: $0 url [tgz]"
  13 + exit 64
  14 +fi
  15 +
  16 +date=`date +%Y%m%d%H%M%S`
  17 +exec > >(tee mstress.log.$date)
  18 +exec 2>&1
  19 +
  20 +echo "Downloading: $url"
  21 +tmpfile=`mktemp $tarfile.XXXXXX` || exit $?
  22 +if [ -f "$tarfile" ]
  23 +then
  24 + code=`curl -sf "$url" -z "$tarfile" -o "$tmpfile" -w '%{http_code}'`
  25 +else
  26 + code=`curl -sf "$url" -o "$tmpfile" -w '%{http_code}'`
  27 +fi
  28 +
  29 +if [ $? -ne 0 ]
  30 +then
  31 + echo >&2 "Failed to curl $url"
  32 + rm "$tmpfile"
  33 +elif [ "$code" = 200 ]
  34 +then
  35 + mv "$tmpfile" "$tarfile"
  36 +elif [ "$code" = 304 ]
  37 +then
  38 + echo " not modified"
  39 + rm "$tmpfile"
  40 +fi
  41 +
  42 +if [ ! -f "$tarfile" ]
  43 +then
  44 + exit 1
  45 +fi
  46 +
  47 +echo "Unpacking $tarfile"
  48 +contents=`tar xzfv "$tarfile"` || exit $?
  49 +echo -n "$contents" | sed 's/^/ /'
  50 +
  51 +echo "Installing client files"
  52 +./mstress/mstress_install.sh localhost || exit $?
  53 +
  54 +echo "Configuring meta and chunk servers"
  55 +cd mstress
  56 +cat >setup.cfg <<EOF
  57 +[metaserver]
  58 +hostname = localhost
  59 +rundir = ~/qfsbase/meta
  60 +clientport = $PORT
  61 +chunkport = `expr $PORT + 100`
  62 +clusterkey = myTestCluster
  63 +
  64 +[chunkserver1]
  65 +hostname = localhost
  66 +rundir = ~/qfsbase/chunk1
  67 +chunkport = `expr $PORT + 1000`
  68 +# in practice, have a chunkdir per disk.
  69 +chunkdirs = ~/qfsbase/chunk1/chunkdir11 ~/qfsbase/chunk1/chunkdir12
  70 +
  71 +[chunkserver2]
  72 +hostname = localhost
  73 +rundir = ~/qfsbase/chunk2
  74 +chunkport = `expr $PORT + 1001`
  75 +# in practice, have a chunkdir per disk.
  76 +chunkdirs = ~/qfsbase/chunk2/chunkdir21
  77 +
  78 +[webui]
  79 +hostname = localhost
  80 +rundir = ~/qfsbase/web
  81 +webport = 42000
  82 +EOF
  83 +./setup.py -c setup.cfg -r . -s . -a install || exit $?
  84 +
  85 +echo "Running benchmark"
  86 +(
  87 + ./mstress_plan.py -c localhost -n $CLIENTS -t file -l $LEVELS -i $INODES -s $STATS -o mstress.plan &&
  88 + ./mstress.py -f qfs -s localhost -p $PORT -a mstress.plan
  89 + # disabled - unknown hanging error on create
  90 + #./mstress.py -f hdfs -s localhost -p $PORT -a mstress.plan
  91 + )
  92 +ret=$?
  93 +
  94 +uid=`id -u`
  95 +pid=`pgrep -u $uid metaserver`
  96 +ctime=`awk '{print $14,$15}' /proc/$pid/stat`
  97 +rev=`strings bin/metaserver | grep -B1 KFS_BUILD_INFO_END | head -1 | cut -d@ -f2 | cut -c-16`
  98 +
  99 +./setup.py -c setup.cfg -r . -s . -a uninstall || exit $?
  100 +
  101 +if [ $ret -eq 0 ]
  102 +then
  103 + echo "Metaserver cpu usage (date gitrev utime stime):"
  104 + echo "$date $rev $ctime"
  105 +fi
  106 +exit $ret
13 benchmarks/mstress/ivy.xml
... ... @@ -1,22 +1,21 @@
1 1 <ivy-module version="1.0" xmlns:e="http://ant.apache.org/ivy/extra"
2 2 xmlns:m="http://ant.apache.org/ivy/maven">
3   - <info organisation="com.quantcast" module="qfs_mstress"/>
  3 + <info organisation="com.quantcast" module="mstress"/>
4 4
5 5 <configurations>
6 6 <conf name="compile" description="Dependencies needed to compile" />
7 7 <conf name="master" visibility="public" description="The artifacts published by this project" />
8   - <conf name="source" visibility="public" />
9   - <conf name="javadoc" visibility="public" />
  8 + <conf name="runtime" description="All dependendies needed to run the client" />
10 9 </configurations>
11 10
12 11 <publications>
13 12 <artifact type="jar" conf="master" ext="jar"/>
14   - <artifact m:classifier="sources" type="source" conf="source" ext="zip"/>
15   - <artifact m:classifier="javadoc" type="javadoc" conf="javadoc" ext="zip"/>
16 13 </publications>
17 14
18 15 <dependencies defaultconfmapping="*->default">
19   - <dependency org="org.apache.hadoop" name="hadoop-common" rev="latest.integration" conf="compile->master"/>
20   - <dependency org="org.apache.hadoop" name="hadoop-hdfs" rev="latest.integration" conf="compile->master"/>
  16 + <dependency org="org.apache.hadoop" name="hadoop-common" rev="${hadoop.build.version}"
  17 + conf="compile->master; runtime->runtime,master" />
  18 + <dependency org="org.apache.hadoop" name="hadoop-hdfs" rev="${hadoop.build.version}"
  19 + conf="compile->master; runtime->runtime,master" />
21 20 </dependencies>
22 21 </ivy-module>
12 benchmarks/mstress/ivysettings.xml
... ... @@ -0,0 +1,12 @@
  1 +<ivysettings>
  2 + <settings defaultResolver="main"/>
  3 + <include url="${ivy.default.settings.dir}/ivysettings-local.xml"/>
  4 + <resolvers>
  5 + <!-- set checkconsistency="false" to bypass POM error in commons-daemon -->
  6 + <ibiblio name="public" m2compatible="true" checkconsistency="false" />
  7 + <chain name="main" dual="true">
  8 + <resolver ref="local"/>
  9 + <resolver ref="public"/>
  10 + </chain>
  11 + </resolvers>
  12 +</ivysettings>
3  benchmarks/mstress/mstress.py
@@ -407,8 +407,7 @@ def SetGlobalPaths(opts):
407 407 Globals.SERVER_CMD = Globals.KFS_SERVER_CMD
408 408 Globals.SERVER_KEYWORD = Globals.KFS_SERVER_KEYWORD
409 409 elif opts.filesystem == 'hdfs':
410   - hdfsjars = commands.getoutput("echo %s/mstress_hdfs_client_jars/*.jar | sed 's/ /:/g'" % mydir)
411   - Globals.CLIENT_PATH = 'java -Xmx256m -cp %s:%s MStress_Client' % (mydir,hdfsjars)
  410 + Globals.CLIENT_PATH = 'java -Xmx256m -jar %s/mstress.jar' % mydir
412 411 Globals.SERVER_CMD = Globals.HDFS_SERVER_CMD
413 412 Globals.SERVER_KEYWORD = Globals.HDFS_SERVER_KEYWORD
414 413 else:
77 benchmarks/mstress/mstress_initialize.sh
... ... @@ -1,77 +0,0 @@
1   -#!/bin/sh
2   -
3   -# $Id$
4   -#
5   -# Author: Thilee Subramaniam
6   -#
7   -# Copyright 2012 Quantcast Corp.
8   -#
9   -# Licensed under the Apache License, Version 2.0 (the "License"); you may not
10   -# use this file except in compliance with the License. You may obtain a copy
11   -# of the License at
12   -#
13   -# http://www.apache.org/licenses/LICENSE-2.0
14   -#
15   -# Unless required by applicable law or agreed to in writing, software
16   -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
17   -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
18   -# License for the specific language governing permissions and limitations
19   -# under the License.
20   -#
21   -#
22   -# The mstress runs stress tests on QFS metaserver and HDFS namenode. To be able
23   -# to work with the namenode, the java HDFS client needs to be compiled and run
24   -# with hadoop hdfs jars.
25   -#
26   -# To make this process uniform across mstress master and slaves, we copy the
27   -# jars to a local directory and copy it around to all participating hosts at
28   -# a fixed location.
29   -# (Without this, each client would need to add entries to /etc/yum.repos.d/,
30   -# and require root access to install hadoop client rpms).
31   -#
32   -# This script packages the jars locally on the build node, so that the
33   -# 'mstress_prepare_clients.sh' script can copy it over to master and clients.
34   -#
35   -# Run this program with the path of hadoop client jars as argument (default
36   -# /usr/lib/hadoop/client/), and it will create a "mstress_hdfs_client_jars"
37   -# directory containing the jars.
38   -#
39   -
40   -if [[ "$1" = -* ]]
41   -then
42   - echo "Usage: $0 [ path/to/hadoop/client/jars ]"
43   - echo " This prepares the build environment on mstress build host"
44   - echo " Default path is '/usr/lib/hadoop/client/'"
45   - exit
46   -fi
47   -
48   -JARS_SOURCE=${1:-"/usr/lib/hadoop/client/"}
49   -
50   -DIR="$( cd "$( dirname "$0" )" && pwd )"
51   -JARS_TARGET=${DIR}/mstress_hdfs_client_jars
52   -
53   -if [ ! -d "$JARS_SOURCE" ] || [ -z "$(ls -A "$JARS_SOURCE"/*.jar)" ]; then
54   - echo ""$JARS_SOURCE" is not a directory or does not have the jars."
55   - exit 1
56   -fi
57   -
58   -if [ -d "$JARS_TARGET" ]; then
59   - if [ "$(ls -A "$JARS_TARGET"/*.jar 2> /dev/null)" ]; then
60   - echo ""$JARS_TARGET" already has the jars. Nothing to do."
61   - exit 0
62   - fi
63   -fi
64   -
65   -mkdir -p "$JARS_TARGET"
66   -cp $JARS_SOURCE/*.jar "$JARS_TARGET"
67   -
68   -if [ $? -ne 0 ]
69   -then
70   - echo "Failed to copy jars."
71   - exit 1
72   -fi
73   -
74   -echo "Hadoop client jars from $JARS_SOURCE copied to $JARS_TARGET."
75   -
76   -exit 0
77   -
43 benchmarks/mstress/mstress_prepare_master_clients.sh → benchmarks/mstress/mstress_install.sh
... ... @@ -1,10 +1,7 @@
1 1 #!/bin/sh
2   -
3 2 #
4 3 # $Id$
5 4 #
6   -# Author: Thilee Subramaniam
7   -#
8 5 # Copyright 2012 Quantcast Corp.
9 6 #
10 7 # Licensed under the Apache License, Version 2.0 (the "License"); you may not
@@ -28,35 +25,33 @@
28 25 # and untar + unzip them for usage.
29 26 #
30 27
31   -if [ -z $1 ] || [[ "$1" = -* ]]
  28 +TAR=${TAR:-"tar"}
  29 +
  30 +if [ -z "$BOOTSTRAP" ]
32 31 then
33   - echo "Usage: $0 <comma-separated hosts>"
34   - echo " This copies the mstress bundle to master and client hosts."
35   - exit
  32 + tarfile="mstress.tgz"
  33 + target="mstress-tarball"
  34 +else
  35 + tarfile="mstress-bootstrap.tgz"
  36 + target="mstress-bootstrap"
36 37 fi
37 38
38   -which tar &>/dev/null
39   -if [ $? -ne 0 ]
  39 +if [ $# -lt 1 ]
40 40 then
41   - echo "tar command not found."
42   - exit 1
  41 + echo "Usage: $0 <comma-separated hosts>"
  42 + echo " This copies the mstress bundle to master and client hosts."
  43 + exit
43 44 fi
44 45
45   -script_dir=$(dirname "$0")
46   -
47   -cd $script_dir/.. && tar cvfz mstress.tgz mstress
48   -if [ $? -ne 0 ]
  46 +if [ ! -f "$tarfile" ]
49 47 then
50   - echo "failed to create archive."
51   - cd -
52   - exit 1
  48 + [ -d build ] || mkdir build
  49 + (cd build && cmake ../../.. && make "$target" && cp "benchmarks/$tarfile" ..) || exit 1
53 50 fi
54 51
55   -cd -
56   -for v in `echo "$@"|sed 's/,/ /g'`
  52 +while [ $# -ne 0 ]
57 53 do
58   - ssh $v "rm -rf ~/mstress*"
59   - scp $script_dir/../mstress.tgz $v:~
60   - ssh $v "tar xvfz mstress.tgz"
  54 + echo "Deploying mstress tarball to $1"
  55 + ssh $1 "$TAR xzv || echo >&2 failed to untar on '$1'" < "$tarfile"
  56 + shift
61 57 done
62   -
8 examples/sampleservers/sample_setup.py
@@ -128,8 +128,8 @@ def kill_running_program(binaryPath):
128 128 checkPath = os.path.split(binaryPath)[1]
129 129 if not checkPath:
130 130 return
131   - cmd = ('ps -ef | grep %s | grep -v grep | awk \'{print $2}\''
132   - % checkPath)
  131 + cmd = ('ps -ef | grep %s | grep %s | grep -v grep | awk \'{print $2}\''
  132 + % (os.getlogin(), checkPath))
133 133 res = subprocess.Popen(cmd, shell=True,
134 134 stdout=subprocess.PIPE).communicate()
135 135 pids = res[0].split('\n')
@@ -138,8 +138,8 @@ def kill_running_program(binaryPath):
138 138 os.kill(int(pid.strip()), signal.SIGTERM)
139 139 else:
140 140 if binaryPath.find('qfsstatus') >= 0:
141   - cmd = ('ps -ef | grep /qfsbase/ | grep %s | grep -v grep | awk \'{print $2}\''
142   - % binaryPath)
  141 + cmd = ('ps -ef | grep %s | grep /qfsbase/ | grep %s | grep -v grep | awk \'{print $2}\''
  142 + % (os.getlogin(), binaryPath))
143 143 res = subprocess.Popen(cmd, shell=True,
144 144 stdout=subprocess.PIPE).communicate()
145 145 pids = res[0].split('\n')

0 comments on commit eef3dc4

Please sign in to comment.
Something went wrong with that request. Please try again.