Permalink
Browse files

merged distributed txn project

  • Loading branch information...
Atreyee Maiti
Atreyee Maiti committed May 22, 2014
2 parents 18dd06f + 3f92739 commit de93be92e5683ff490a28c14cbac218185658a22
Showing with 3,018 additions and 198 deletions.
  1. +6 −0 properties/benchmarks/users.properties
  2. +175 −0 scripts/anticache/users.sh
  3. +164 −0 src/benchmarks/edu/brown/benchmark/users/UsersClient.java
  4. +23 −0 src/benchmarks/edu/brown/benchmark/users/UsersConstants.java
  5. +83 −0 src/benchmarks/edu/brown/benchmark/users/UsersLoader.java
  6. +35 −0 src/benchmarks/edu/brown/benchmark/users/UsersProjectBuilder.java
  7. +94 −0 src/benchmarks/edu/brown/benchmark/users/UsersUtil.java
  8. +19 −0 src/benchmarks/edu/brown/benchmark/users/procedures/GetComments.java
  9. +46 −0 src/benchmarks/edu/brown/benchmark/users/procedures/GetUsers.java
  10. +32 −0 src/benchmarks/edu/brown/benchmark/users/users-ddl.sql
  11. +2 −0 src/benchmarks/edu/brown/benchmark/users/users.mappings
  12. +4 −2 src/ee/anticache/EvictedTupleAccessException.cpp
  13. +2 −1 src/ee/anticache/EvictedTupleAccessException.h
  14. +2 −1 src/ee/executors/indexscanexecutor.cpp
  15. +79 −30 src/frontend/edu/brown/hstore/AntiCacheManager.java
  16. +110 −6 src/frontend/edu/brown/hstore/HStoreCoordinator.java
  17. +14 −2 src/frontend/edu/brown/hstore/HStoreSite.java
  18. +10 −0 src/frontend/edu/brown/hstore/PartitionExecutor.java
  19. +22 −3 src/frontend/edu/brown/hstore/TransactionInitializer.java
  20. +2 −2 src/frontend/edu/brown/hstore/TransactionQueueManager.java
  21. +3 −3 src/frontend/edu/brown/hstore/callbacks/RemoteFinishCallback.java
  22. +5 −2 src/frontend/edu/brown/hstore/callbacks/RemoteWorkCallback.java
  23. +83 −0 src/frontend/edu/brown/hstore/txns/AbstractTransaction.java
  24. +10 −64 src/frontend/edu/brown/hstore/txns/LocalTransaction.java
  25. +14 −0 src/frontend/edu/brown/hstore/txns/RemoteTransaction.java
  26. +1 −0 src/frontend/edu/brown/utils/ProjectType.java
  27. +1 −0 src/frontend/org/voltdb/ClientResponseImpl.java
  28. +0 −1 src/frontend/org/voltdb/VoltProcedure.java
  29. +1 −0 src/frontend/org/voltdb/client/ClientImpl.java
  30. +9 −1 src/frontend/org/voltdb/exceptions/EvictedTupleAccessException.java
  31. +1,196 −52 src/protorpc/edu/brown/hstore/Hstoreservice.java
  32. +22 −0 src/protorpc/edu/brown/hstore/hstoreservice.proto
  33. +25 −0 tests/frontend/edu/brown/hstore/MockAntiCacheManager.java
  34. +27 −0 tests/frontend/edu/brown/hstore/MockHStoreCoordinator.java
  35. +45 −0 tests/frontend/edu/brown/hstore/MockHStoreSite.java
  36. +398 −0 tests/frontend/edu/brown/hstore/TestAntiCacheManagerDistributedTxn.java
  37. +68 −23 tests/frontend/edu/brown/hstore/TestHStoreCoordinator.java
  38. +5 −5 tests/frontend/org/voltdb/regressionsuites/LocalCluster.java
  39. +181 −0 tests/frontend/org/voltdb/regressionsuites/TestUsersSuite.java
@@ -0,0 +1,6 @@
## ------------------------------------------------------------
## H-Store Benchmark Configuration Parameters
## ------------------------------------------------------------
builder = edu.brown.benchmark.users.UsersProjectBuilder
datadir = src/benchmarks/edu/brown/benchmark/users/data
View
@@ -0,0 +1,175 @@
#!/bin/bash
# ---------------------------------------------------------------------
trap onexit 1 2 3 15
function onexit() {
local exit_status=${1:-$?}
pkill -f hstore.tag
exit $exit_status
}
# ---------------------------------------------------------------------
ENABLE_ANTICACHE=true
SITE_HOST="localhost"
CLIENT_HOSTS=( \
"localhost" \
)
BASE_CLIENT_THREADS=1
BASE_SITE_MEMORY=8192
BASE_SITE_MEMORY_PER_PARTITION=1024
BASE_PROJECT="users"
BASE_DIR=`pwd`
OUTPUT_DIR="~/data/users/read-heavy/2/80-20"
ANTICACHE_BLOCK_SIZE=1048576
ANTICACHE_THRESHOLD=.5
BASE_ARGS=( \
# SITE DEBUG
"-Dsite.status_enable=false" \
"-Dsite.status_interval=10000" \
# "-Dsite.status_exec_info=true" \
# "-Dsite.status_check_for_zombies=true" \
# "-Dsite.exec_profiling=true" \
# "-Dsite.profiling=true" \
# "-Dsite.txn_counters=true" \
# "-Dsite.pool_profiling=true" \
# "-Dsite.network_profiling=false" \
# "-Dsite.log_backup=true"\
# "-Dnoshutdown=true" \
# Site Params
"-Dsite.jvm_asserts=false" \
"-Dsite.specexec_enable=false" \
"-Dsite.cpu_affinity_one_partition_per_core=true" \
#"-Dsite.cpu_partition_blacklist=0,2,4,6,8,10,12,14,16,18" \
#"-Dsite.cpu_utility_blacklist=0,2,4,6,8,10,12,14,16,18" \
"-Dsite.network_incoming_limit_txns=50000" \
"-Dsite.commandlog_enable=true" \
"-Dsite.txn_incoming_delay=5" \
"-Dsite.exec_postprocessing_threads=true" \
"-Dsite.anticache_profiling=${ENABLE_ANTICACHE}" \
"-Dsite.anticache_eviction_distribution=even" \
# "-Dsite.queue_allow_decrease=true" \
# "-Dsite.queue_allow_increase=true" \
# "-Dsite.queue_threshold_factor=0.5" \
# Client Params
"-Dclient.scalefactor=1" \
"-Dclient.memory=2048" \
"-Dclient.txnrate=2000" \
"-Dclient.warmup=10000" \
"-Dclient.duration=300000" \
"-Dclient.interval=5000" \
"-Dclient.shared_connection=false" \
"-Dclient.blocking=false" \
"-Dclient.blocking_concurrent=100" \
"-Dclient.throttle_backoff=100" \
"-Dclient.output_interval=5000" \
# "-Dclient.output_anticache_evictions=evictions.csv" \
# "-Dclient.output_memory=memory.csv" \
# Anti-Caching Experiments
"-Dsite.anticache_enable=${ENABLE_ANTICACHE}" \
# "-Dsite.anticache_profiling=true" \
"-Dsite.anticache_reset=false" \
"-Dsite.anticache_block_size=${ANTICACHE_BLOCK_SIZE}" \
"-Dsite.anticache_check_interval=5000" \
"-Dsite.anticache_threshold_mb=1" \
"-Dsite.anticache_blocks_per_eviction=200" \
"-Dsite.anticache_max_evicted_blocks=325" \
# "-Dsite.anticache_evict_size=${ANTICACHE_EVICT_SIZE}" \
"-Dsite.anticache_threshold=${ANTICACHE_THRESHOLD}" \
"-Dclient.anticache_enable=false" \
"-Dclient.anticache_evict_interval=10000" \
"-Dclient.anticache_evict_size=102400" \
"-Dclient.output_csv=results.csv" \
# CLIENT DEBUG
# "-Dclient.output_txn_counters=txncounters.csv" \
"-Dclient.output_clients=false" \
"-Dclient.profiling=false" \
"-Dclient.output_response_status=false" \
# "-Dclient.output_queue_profiling=${BASE_PROJECT}-queue.csv" \
# "-Dclient.output_basepartitions=true" \
# "-Dclient.jvm_args=\"-verbose:gc -XX:+PrintGCDetails -XX:+PrintGCTimeStamps -XX:-TraceClassUnloading\""
)
EVICTABLE_TABLES=( \
"USERS" \
)
EVICTABLES=""
if [ "$ENABLE_ANTICACHE" = "true" ]; then
for t in ${EVICTABLE_TABLES[@]}; do
EVICTABLES="${t},${EVICTABLES}"
done
fi
# Compile
HOSTS_TO_UPDATE=("$SITE_HOST")
for CLIENT_HOST in ${CLIENT_HOSTS[@]}; do
NEED_UPDATE=1
for x in ${HOSTS_TO_UPDATE[@]}; do
if [ "$CLIENT_HOST" = "$x" ]; then
NEED_UPDATE=0
break
fi
done
if [ $NEED_UPDATE = 1 ]; then
HOSTS_TO_UPDATE+=("$CLIENT_HOST")
fi
done
#for HOST in ${HOSTS_TO_UPDATE[@]}; do
# ssh $HOST "cd $BASE_DIR && git pull && ant compile" &
#done
wait
ant compile
HSTORE_HOSTS="${SITE_HOST}:0:0;${SITE_HOST}:1:1"
NUM_CLIENTS=`expr 1 \* $BASE_CLIENT_THREADS`
SITE_MEMORY=`expr $BASE_SITE_MEMORY + \( 1 \* $BASE_SITE_MEMORY_PER_PARTITION \)`
# BUILD PROJECT JAR
ant hstore-prepare \
-Dproject=${BASE_PROJECT} \
-Dhosts=${HSTORE_HOSTS} \
-Devictable=${EVICTABLES}
test -f ${BASE_PROJECT}.jar || exit -1
# UPDATE CLIENTS
CLIENT_COUNT=0
CLIENT_HOSTS_STR=""
for CLIENT_HOST in ${CLIENT_HOSTS[@]}; do
CLIENT_COUNT=`expr $CLIENT_COUNT + 1`
if [ ! -z "$CLIENT_HOSTS_STR" ]; then
CLIENT_HOSTS_STR="${CLIENT_HOSTS_STR},"
fi
CLIENT_HOSTS_STR="${CLIENT_HOSTS_STR}${CLIENT_HOST}"
done
# DISTRIBUTE PROJECT JAR
for HOST in ${HOSTS_TO_UPDATE[@]}; do
if [ "$HOST" != $(hostname) ]; then
scp -r ${BASE_PROJECT}.jar ${HOST}:${BASE_DIR} &
fi
done
wait
# EXECUTE BENCHMARK
ant hstore-benchmark ${BASE_ARGS[@]} \
-Dproject=${BASE_PROJECT} \
-Dkillonzero=false \
-Dclient.threads_per_host=${NUM_CLIENTS} \
-Dsite.memory=${SITE_MEMORY} \
-Dclient.hosts=${CLIENT_HOSTS_STR} \
-Dclient.count=${CLIENT_COUNT}
result=$?
if [ $result != 0 ]; then
exit $result
fi
@@ -0,0 +1,164 @@
package edu.brown.benchmark.users;
import java.io.IOException;
import java.util.Random;
import org.apache.log4j.Logger;
import org.voltdb.client.Client;
import org.voltdb.client.ClientResponse;
import org.voltdb.client.NoConnectionsException;
import org.voltdb.client.ProcedureCallback;
import edu.brown.api.BenchmarkComponent;
import edu.brown.hstore.Hstoreservice.Status;
import edu.brown.logging.LoggerUtil;
import edu.brown.logging.LoggerUtil.LoggerBoolean;
import edu.brown.rand.RandomDistribution.FlatHistogram;
import edu.brown.rand.RandomDistribution.Zipf;
import edu.brown.statistics.Histogram;
import edu.brown.statistics.ObjectHistogram;
public class UsersClient extends BenchmarkComponent {
private static final Logger LOG = Logger.getLogger(UsersClient.class);
private static final LoggerBoolean debug = new LoggerBoolean();
private static final LoggerBoolean trace = new LoggerBoolean();
static {
LoggerUtil.attachObserver(LOG, debug, trace);
}
public static void main(String args[]) {
BenchmarkComponent.main(UsersClient.class, args, false);
}
int run_count = 0;
private FlatHistogram<Transaction> txnWeights;
private final Random rand_gen;
private Zipf readRecord;
private Zipf userRecord;
private long usersSize;
public UsersClient(String[] args) {
super(args);
// Initialize the sampling table
Histogram<Transaction> txns = new ObjectHistogram<Transaction>();
for (Transaction t : Transaction.values()) {
Integer weight = this.getTransactionWeight(t.callName);
if (weight == null) weight = t.weight;
txns.put(t, weight);
} // FOR
assert(txns.getSampleCount() == 100) : txns;
this.rand_gen = new Random();
this.usersSize = Math.round(UsersConstants.USERS_SIZE * this.getScaleFactor());
this.userRecord = new Zipf(this.rand_gen, 0, usersSize, 1.0001);
this.txnWeights = new FlatHistogram<Transaction>(this.rand_gen, txns);
}
public static enum Transaction {
GET_USERS("Get Users", UsersConstants.FREQUENCY_GET_USERS);
/**
* Constructor
*/
private Transaction(String displayName, int weight) {
this.displayName = displayName;
this.callName = displayName.replace(" ", "");
this.weight = weight;
}
public final String displayName;
public final String callName;
public final int weight; // probability (in terms of percentage) the transaction gets executed
} // TRANSCTION ENUM
@Override
protected boolean runOnce() throws IOException {
// pick random transaction to call, weighted by txnWeights
final Transaction target = this.txnWeights.nextValue();
Object params[];
// ProcedureCallback callback = null;
switch (target) {
case GET_USERS: {
long u_id1 = this.userRecord.nextLong();// uid
long u_id2 = this.usersSize - u_id1;// uid
params = new Object[]{ u_id1, u_id2};
break;
}
default:
throw new RuntimeException("Unexpected txn '" + target + "'");
} // SWITCH
assert(params != null);
Callback callback = new Callback(target.ordinal());
boolean val = this.getClientHandle().callProcedure(callback, target.callName, params);
return val;
}
@SuppressWarnings("unused")
@Deprecated
@Override
public void runLoop() {
Client client = this.getClientHandle();
try {
while (true) {
this.runOnce();
client.backpressureBarrier();
} // WHILE
} catch (Exception ex) {
ex.printStackTrace();
}
//
//
// try {
//
// Random rand = new Random();
// int key = -1;
// int scan_count;
// while (true) {
// System.out.println("loop print");
// runOnce();
// this.run_count++;
// }
// }
// catch (IOException e) {
//
// }
}
private class Callback implements ProcedureCallback {
private final int idx;
public Callback(int idx) {
this.idx = idx;
}
@Override
public void clientCallback(ClientResponse clientResponse) {
if (clientResponse.getStatus() == Status.ABORT_UNEXPECTED) {
LOG.error(String.format("Unexpected Error in %s: %s",
this.idx, clientResponse.getStatusString()),
clientResponse.getException());
System.out.println(String.format("Unexpected Error in %s: %s",
this.idx, clientResponse.getStatusString())+ clientResponse.getException());
}
// Increment the BenchmarkComponent's internal counter on the
// number of transactions that have been completed
incrementTransactionCounter(clientResponse, this.idx);
}
} // END CLASS
@Override
public String[] getTransactionDisplayNames() {
// Return an array of transaction names
String procNames[] = new String[UsersProjectBuilder.PROCEDURES.length];
for (int i = 0; i < procNames.length; i++) {
procNames[i] = UsersProjectBuilder.PROCEDURES[i].getSimpleName();
}
return (procNames);
}
}
@@ -0,0 +1,23 @@
package edu.brown.benchmark.users;
public abstract class UsersConstants {
// ----------------------------------------------------------------
// STORED PROCEDURE EXECUTION FREQUENCIES (0-100)
// ----------------------------------------------------------------
public static final int FREQUENCY_GET_USERS = 100;
// ----------------------------------------------------------------
// TABLE NAMES
// ----------------------------------------------------------------
public static final String TABLENAME_USERS = "USERS";
public static final int USERS_SIZE = 100000;
public static final int BATCH_SIZE = 500;
public static final String TABLENAMES[] = {
TABLENAME_USERS
};
}
Oops, something went wrong.

0 comments on commit de93be9

Please sign in to comment.