Skip to content
Permalink
Browse files
Replace use of pssh with parallel-ssh (#179)
* Replaced pssh and pscp
* Fall back to parallel-ssh when pssh unavailable
* Fix quoting in sudo commands in agitator
* Preserve PSSH environment over sudo commands for agitator
* Properly set PSSH in perl agitator code from process environment
* Fix a couple typos in READMEs

Co-authored-by: Christopher Tubbs <ctubbsii@apache.org>
  • Loading branch information
DomGarguilo and ctubbsii committed Dec 16, 2021
1 parent 1ac45a0 commit e74a1f5904bc165d4054ea1a0fd93d8eacc82606
Showing 11 changed files with 46 additions and 20 deletions.
@@ -31,7 +31,14 @@ EOF
}

function start_agitator() {
hash pssh 2>/dev/null || { echo >&2 "The agitator requires pssh to be installed. Aborting."; exit 1; }
## check that pssh is installed, falling back to parallel-ssh if needed
if hash pssh 2>/dev/null; then
PSSH=pssh
elif hash parallel-ssh 2>/dev/null; then
PSSH=parallel-ssh
else
echo >&2 "The agitator requires pssh/parallel-ssh to be installed. Aborting."; exit 1;
fi

mkdir -p "${at_home}/logs"
log_base="${at_home}/logs/$(date +%Y%m%d%H%M%S)_$(hostname)"
@@ -50,15 +57,15 @@ function start_agitator() {
$tserver_cmd > "${tserver_log}.out" 2> "${tserver_log}.err" &
else
echo "Running master-agitator and tserver-agitator as $AGTR_ACCUMULO_USER using sudo."
sudo -u "$AGTR_ACCUMULO_USER" "$master_cmd" > "${master_log}.out" 2> "${master_log}.err" &
sudo -u "$AGTR_ACCUMULO_USER" "$tserver_cmd" > "${tserver_log}.out" 2> "${tserver_log}.err" &
sudo --preserve-env=PSSH -u "$AGTR_ACCUMULO_USER" $master_cmd > "${master_log}.out" 2> "${master_log}.err" &
sudo -u "$AGTR_ACCUMULO_USER" $tserver_cmd > "${tserver_log}.out" 2> "${tserver_log}.err" &
fi
if [[ $AGITATOR_USER == "$AGTR_HDFS_USER" ]]; then
echo "Running datanode-agitator as $AGITATOR_USER"
$datanode_cmd > "${datanode_log}.out" 2> "${datanode_log}.err" &
else
echo "Running datanode-agitator as $AGTR_HDFS_USER using sudo."
sudo -u "$AGTR_HDFS_USER" "$datanode_cmd" > "${datanode_log}.out" 2> "${datanode_log}.err" &
sudo -u "$AGTR_HDFS_USER" $datanode_cmd > "${datanode_log}.out" 2> "${datanode_log}.err" &
fi

if ${AGTR_HDFS:-false} ; then
@@ -30,6 +30,12 @@
print "ERROR: ACCUMULO_HOME needs to be set!";
exit(1);
}
if( defined $ENV{'PSSH'} ){
$PSSH = $ENV{'PSSH'};
} else {
print "ERROR: PSSH needs to be set!";
exit(1);
}

$accumuloConfDir = $accumuloHome . '/conf';

@@ -67,11 +73,11 @@
system($cmd);
}else{
print STDERR "$t Killing all masters\n";
$cmd = "pssh -h $accumuloConfDir/masters \"pkill -f '[ ]org.apache.accumulo.start.*master'\" < /dev/null";
$cmd = "$PSSH -h $accumuloConfDir/masters \"pkill -f '[ ]org.apache.accumulo.start.*master'\" < /dev/null";
print "$t $cmd\n";
system($cmd);

$cmd = "pssh -h $accumuloConfDir/$gcfile \"pkill -f '[ ]org.apache.accumulo.start.*gc'\" < /dev/null";
$cmd = "$PSSH -h $accumuloConfDir/$gcfile \"pkill -f '[ ]org.apache.accumulo.start.*gc'\" < /dev/null";
print "$t $cmd\n";
system($cmd);
}
@@ -80,11 +86,11 @@
$t = strftime "%Y%m%d %H:%M:%S", localtime;
print STDERR "$t Running start-all\n";

$cmd = "pssh -h $accumuloConfDir/masters \"$accumuloHome/bin/accumulo-service master start\" < /dev/null";
$cmd = "$PSSH -h $accumuloConfDir/masters \"$accumuloHome/bin/accumulo-service master start\" < /dev/null";
print "$t $cmd\n";
system($cmd);

$cmd = "pssh -h $accumuloConfDir/$gcfile \"$accumuloHome/bin/accumulo-service gc start\" < /dev/null";
$cmd = "$PSSH -h $accumuloConfDir/$gcfile \"$accumuloHome/bin/accumulo-service gc start\" < /dev/null";
print "$t $cmd\n";
system($cmd);
}
@@ -55,7 +55,7 @@ The 4th Benchmark is Terasort. Run the benchmarks with speed 'slow' to do a ful
Misc
----

These benchmarks create tables in accumulo named 'test_ingest' and 'CloudIngestTest'. These tables are deleted
These benchmarks create tables in accumulo named `test_ingest` and 'CloudIngestTest'. These tables are deleted
at the end of the benchmarks. The benchmarks will also alter user auths while it runs. It is recommended that
a benchmark user is created.

@@ -26,7 +26,7 @@ following command.
japi-compliance-checker.pl -skip-deprecated -old japi-accumulo-1.5.xml -new japi-accumulo-1.6.xml -l accumulo
```

Optionally, you can use the --skip-classes argument with the provided exclude_classes.txt file to skip classes from
Optionally, you can use the --skip-classes argument with the provided `exclude_classes.txt` file to skip classes from
org.apache.accumulo.core.data that aren't in the public API.

This directory should have a library configuration file for each release on supported lines as well as an in-progress
@@ -26,7 +26,7 @@ your Accumulo instance. Do not run these tests on a cluster holding essential
data.

1. Make sure you have both `ACCUMULO_HOME` and `HADOOP_HOME` set in your
`$ACCUMULO_CONF_DIR/accumulo-env.sh.`
`$ACCUMULO_CONF_DIR/accumulo-env.sh`

2. Edit the 'site.conf' file in the `conf` directory containing settings
needed by test nodes to connect to Accumulo, and to guide the tests.
@@ -48,10 +48,11 @@ across hosts.
## Prerequisites

The utilities are run on a normal, initialized Accumulo cluster. By default,
they only work with a table named "stress_test".
they only work with a table named `stress_test`.

The start and stop scripts rely on pssh. Before running them, you will need
to install pssh on the machines that will be controlled by them.
The start and stop scripts rely on pssh/parallel-ssh. Before running them,
you will need to install pssh/parallel-ssh on the machines that will be
controlled by them.

## Running

@@ -35,6 +35,6 @@ if [[ ! -f ${DIR}/readers ]]; then
fi

# Copy environment out
pscp -h "${DIR}/readers" "${DIR}/stress-env.sh" "${DIR}"
$PSCP -h "${DIR}/readers" "${DIR}/stress-env.sh" "${DIR}"

pssh -h "${DIR}/readers" "nohup ${DIR}/reader.sh >${DIR}/reader.out 2>${DIR}/reader.err < /dev/null &"
$PSSH -h "${DIR}/readers" "nohup ${DIR}/reader.sh >${DIR}/reader.out 2>${DIR}/reader.err < /dev/null &"
@@ -35,6 +35,6 @@ if [[ ! -f ${DIR}/writers ]]; then
fi

# Copy environment out
pscp -h "${DIR}/writers" "${DIR}/stress-env.sh" "${DIR}"
$PSCP -h "${DIR}/writers" "${DIR}/stress-env.sh" "${DIR}"

pssh -h "${DIR}/writers" "nohup ${DIR}/writer.sh >${DIR}/writer.out 2>${DIR}/writer.err < /dev/null &"
$PSSH -h "${DIR}/writers" "nohup ${DIR}/writer.sh >${DIR}/writer.out 2>${DIR}/writer.err < /dev/null &"
@@ -33,4 +33,4 @@ if [[ ! -f "${DIR}/readers" ]]; then
echo readers file is missing
exit 1
fi
pssh -h "${DIR}/readers" "pkill -f '[o]rg.apache.accumulo.test.stress.random.Scan'" < /dev/null
$PSSH -h "${DIR}/readers" "pkill -f '[o]rg.apache.accumulo.test.stress.random.Scan'" < /dev/null
@@ -33,4 +33,4 @@ if [[ ! -f "${DIR}/writers" ]]; then
echo writers file is missing
exit 1
fi
pssh -h "${DIR}/writers" "pkill -f '[o]rg.apache.accumulo.test.stress.random.Write'" < /dev/null
$PSSH -h "${DIR}/writers" "pkill -f '[o]rg.apache.accumulo.test.stress.random.Write'" < /dev/null
@@ -15,6 +15,18 @@

ACCUMULO_HOME=${ACCUMULO_HOME:-/opt/accumulo}

## check that pssh and pscp is installed, falling back to parallel-ssh/scp if needed
if hash pssh 2>/dev/null; then
PSSH=pssh
PSCP=pscp
hash pscp.pssh 2>/dev/null && PSCP=pscp.pssh
elif hash parallel-ssh 2>/dev/null; then
PSSH=parallel-ssh
PSCP=parallel-scp
else
echo >&2 "The stress test requires pssh/parallel-ssh and pscp/parallel-scp to be installed. Aborting."; exit 1;
fi

# Edit the credentials to match your system
USERPASS='-u root -p secret'
INSTANCE='-z localhost -i inst'

0 comments on commit e74a1f5

Please sign in to comment.