Permalink
Browse files

Merged in joy's branch. Removed hardcoded nvm directory.

  • Loading branch information...
jarulraj committed Apr 24, 2014
2 parents 0a2daf7 + c3f469f commit 1cb5165356bc48dbc9e7fcd3541f6151127f62bd
Showing with 2,000 additions and 1,404 deletions.
  1. +3 −0 build-common.xml
  2. +1 −1 build.py
  3. +2 −2 buildtools.py
  4. +102 −19 ycsb.sh → experiment.sh
  5. +10 −9 log4j.properties
  6. +2 −2 properties/benchmarks/ycsb.properties
  7. +31 −5 run.sh
  8. +22 −0 run_tpcc.sh
  9. +51 −0 run_ycsb.sh
  10. +1 −1 src/benchmarks/edu/brown/benchmark/ycsb/YCSBClient.java
  11. +5 −4 src/benchmarks/edu/brown/benchmark/ycsb/YCSBConstants.java
  12. +2 −2 src/benchmarks/edu/brown/benchmark/ycsb/YCSBLoader.java
  13. +1 −1 src/benchmarks/org/voltdb/benchmark/tpcc/TPCCClient.java
  14. +13 −10 src/ee/anticache/AntiCacheDB.cpp
  15. +2 −2 src/ee/anticache/AntiCacheEvictionManager.cpp
  16. +0 −1 src/ee/common/StringRef.cpp
  17. +28 −17 src/ee/execution/VoltDBEngine.cpp
  18. +12 −15 src/ee/executors/deleteexecutor.cpp
  19. +50 −28 src/ee/logging/AriesLogProxy.cpp
  20. +4 −2 src/ee/logging/AriesLogProxy.h
  21. +4 −4 src/ee/logging/Logrecord.cpp
  22. +0 −4 src/ee/storage/persistenttable.cpp
  23. +3 −4 src/ee/storage/tablefactory.cpp
  24. +1 −3 src/frontend/edu/brown/hstore/HStore.java
  25. +195 −13 src/frontend/edu/brown/hstore/HStoreSite.java
  26. +30 −16 src/frontend/edu/brown/hstore/PartitionExecutor.java
  27. +14 −1 src/frontend/edu/brown/hstore/cmdlog/CommandLogReader.java
  28. +148 −134 src/frontend/edu/brown/hstore/cmdlog/CommandLogWriter.java
  29. +44 −4 src/frontend/edu/brown/hstore/conf/HStoreConf.java
  30. +6 −6 src/frontend/org/voltdb/AriesLogNative.java
  31. +5 −5 src/frontend/org/voltdb/SnapshotSaveAPI.java
  32. +27 −22 src/frontend/org/voltdb/VoltProcedure.java
  33. +0 −5 src/frontend/org/voltdb/catalog/Host.java
  34. +0 −2 src/frontend/org/voltdb/compiler/VoltCompiler.java
  35. +0 −1 src/frontend/org/voltdb/compiler/VoltProjectBuilder.java
  36. +7 −4 src/frontend/org/voltdb/jni/ExecutionEngineJNI.java
  37. +3 −3 src/frontend/org/voltdb/sysprocs/SnapshotDelete.java
  38. +0 −798 src/frontend/org/voltdb/sysprocs/SnapshotRestoreLocal.java
  39. +1 −1 src/frontend/org/voltdb/sysprocs/SnapshotSave.java
  40. +7 −7 src/frontend/org/voltdb/sysprocs/SnapshotScan.java
  41. +3 −3 src/frontend/org/voltdb/sysprocs/saverestore/PartitionedTableSaveFileState.java
  42. +0 −1 src/frontend/org/voltdb/utils/DBBPool.java
  43. +3 −0 tests/ee/anticache/anticachedb_test.cpp
  44. +4 −7 tests/ee/storage/persistent_table_log_test.cpp
  45. +1 −0 tests/frontend/edu/brown/catalog/TestCatalogCloner.java
  46. +2 −0 tests/frontend/edu/brown/hstore/TestAntiCacheManager.java
  47. +451 −0 tests/frontend/org/voltdb/regressionsuites/TestTPCCLogicalRecovery.java
  48. +238 −0 tests/frontend/org/voltdb/regressionsuites/TestTPCCPhysicalRecovery.java
  49. +212 −235 ...s/frontend/org/voltdb/regressionsuites/{TestSnapshotRecovery.java → TestYCSBLogicalRecovery.java}
  50. +249 −0 tests/frontend/org/voltdb/regressionsuites/TestYCSBPhysicalRecovery.java
View
@@ -163,6 +163,9 @@
<arg value="site.aries=${site.aries}" />
<arg value="site.aries_dir=${site.aries_dir}" />
<arg value="site.aries_reset=${site.aries_reset}" />
<arg value="site.snapshot=${site.snapshot}" />
<arg value="site.snapshot_dir=${site.snapshot_dir}" />
<arg value="site.snapshot_reset=${site.snapshot_reset}" />
<arg value="site.mr_map_blocking=${site.mr_map_blocking}" />
<arg value="site.mr_reduce_blocking=${site.mr_reduce_blocking}" />
<arg value="site.network_heartbeats_interval=${site.network_heartbeats_interval}" />
View
@@ -384,7 +384,7 @@
if CTX.ARIES:
CTX.CPPFLAGS += " -DARIES"
###############################################################################
# ANTI-CACHING
###############################################################################
View
@@ -41,11 +41,11 @@ def __init__(self, args):
self.LOG_LEVEL = "DEBUG"
self.VOLT_LOG_LEVEL = None
self.STORAGE_MMAP = False
self.ANTICACHE_BUILD = False
self.ANTICACHE_BUILD = True
self.ANTICACHE_REVERSIBLE_LRU = True
self.ANTICACHE_NVM = False
self.ANTICACHE_DRAM = False
self.ARIES= False
self.ARIES= True
for arg in [x.strip().upper() for x in args]:
if arg in ["DEBUG", "RELEASE", "MEMCHECK", "MEMCHECK_NOFREELIST"]:
View
@@ -1,4 +1,88 @@
#!/bin/bash
# DAMON Experiments
USAGE="Usage: `basename $0`
[-h]
[-g (get latency)]
[-s (set latency)]
[-t (tpcc project)]
[-y (ycsb project)]
[-r (clean and rebuild)]
[-d (disable anticache)]
"
ENABLE_ANTICACHE=false
REBUILD=false
NVM_LATENCY=110
BASE_PROJECT="ycsb"
# Parse command line options.
while getopts hrgsd:yt OPT; do
case "$OPT" in
h)
echo "$USAGE"
exit 0
;;
r)
REBUILD=true
;;
t)
BASE_PROJECT="tpcc"
;;
y)
BASE_PROJECT="ycsb"
;;
g)
# GO TO SDV directory
cd /data/devel/sdv-tools/sdv-release
# MEASURE LATENCY
sudo ./ivt_pm_sdv.sh --measure
cd `readlink -f /home/user/joy/h-store`
exit 0
;;
s)
# GO TO SDV directory
cd /data/devel/sdv-tools/sdv-release
# MEASURE LATENCY
NVM_LATENCY=$2
echo "SETTING NVM LATENCY : " $NVM_LATENCY
sudo ./ivt_pm_sdv.sh --enable --pm-latency=$NVM_LATENCY
cd `readlink -f /home/user/joy/h-store`
exit 0
;;
\?)
# getopts issues an error message
echo "$USAGE" >&2
exit 1
;;
esac
done
# Remove the switches we parsed above.
shift `expr $OPTIND - 1`
echo "---------------------------------------------------------"
echo "BASE_PROJECT : " $BASE_PROJECT
echo "REBUILD : " $REBUILD
echo "---------------------------------------------------------"
# Access additional arguments as usual through
# variables $@, $*, $1, $2, etc. or using this loop:
for PARAM; do
echo $PARAM
done
# ---------------------------------------------------------------------
@@ -16,19 +100,17 @@ ENABLE_ANTICACHE=false
SITE_HOST="10.212.84.152"
CLIENT_HOSTS=( \
"client2" \
# "client1" \
"client2" \
"10.212.84.152" \
"10.212.84.152" \
)
BASE_CLIENT_THREADS=1
#BASE_SITE_MEMORY=8192
#BASE_SITE_MEMORY_PER_PARTITION=1024
BASE_SITE_MEMORY=8192
BASE_SITE_MEMORY_PER_PARTITION=750
BASE_PROJECT="ycsb"
BASE_DIR=`pwd`
BASE_DIR=`readlink -f /home/user/joy/h-store`
OUTPUT_DIR="~/data/ycsb/read-heavy/2/80-20"
#ANTICACHE_BLOCK_SIZE=131072
@@ -57,8 +139,15 @@ BASE_ARGS=( \
#"-Dsite.cpu_partition_blacklist=0,2,4,6,8,10,12,14,16,18" \
#"-Dsite.cpu_utility_blacklist=0,2,4,6,8,10,12,14,16,18" \
"-Dsite.network_incoming_limit_txns=10000" \
"-Dsite.commandlog_enable=true" \
# ENABLE EITHER ONE OF THESE - NOT BOTH
# COMMAND LOG
"-Dsite.commandlog_enable=false" \
"-Dsite.commandlog_dir=/mnt/pmfs/cmdlog" \
# ARIES
"-Dsite.aries=true" \
"-Dsite.aries_dir=/mnt/pmfs/aries" \
"-Dsite.txn_incoming_delay=5" \
"-Dsite.exec_postprocessing_threads=false" \
"-Dsite.anticache_eviction_distribution=even" \
@@ -71,9 +160,10 @@ BASE_ARGS=( \
"-Dclient.scalefactor=1" \
"-Dclient.memory=2048" \
"-Dclient.txnrate=20000" \
"-Dclient.warmup=60000" \
"-Dclient.duration=60000" \
"-Dclient.interval=20000" \
"-Dclient.warmup=10000" \
"-Dclient.duration=10000" \
"-Dclient.interval=10000" \
"-Dclient.shared_connection=false" \
"-Dclient.blocking=false" \
"-Dclient.blocking_concurrent=100" \
@@ -111,12 +201,6 @@ BASE_ARGS=( \
EVICTABLE_TABLES=( \
"USERTABLE" \
)
EVICTABLES=""
if [ "$ENABLE_ANTICACHE" = "true" ]; then
for t in ${EVICTABLE_TABLES[@]}; do
EVICTABLES="${t},${EVICTABLES}"
done
fi
# Compile
HOSTS_TO_UPDATE=("$SITE_HOST")
@@ -143,12 +227,11 @@ for i in 8; do
HSTORE_HOSTS="${SITE_HOST}:0:0-"`expr $i - 1`
NUM_CLIENTS=`expr $i \* $BASE_CLIENT_THREADS`
SITE_MEMORY=`expr $BASE_SITE_MEMORY + \( $i \* $BASE_SITE_MEMORY_PER_PARTITION \)`
# BUILD PROJECT JAR
ant hstore-prepare \
-Dproject=${BASE_PROJECT} \
-Dhosts=${HSTORE_HOSTS} \
-Devictable=${EVICTABLES}
-Dhosts=${HSTORE_HOSTS}
test -f ${BASE_PROJECT}.jar || exit -1
# UPDATE CLIENTS
@@ -171,10 +254,10 @@ for i in 8; do
wait
# EXECUTE BENCHMARK
ant hstore-benchmark ${BASE_ARGS[@]} \
numactl --membind=2 ant hstore-benchmark ${BASE_ARGS[@]} \
-Dproject=${BASE_PROJECT} \
-Dkillonzero=false \
-Dclient.threads_per_host=4 \
-Dclient.threads_per_host=4 \
-Dsite.memory=${SITE_MEMORY} \
-Dclient.hosts=${CLIENT_HOSTS_STR} \
-Dclient.count=${CLIENT_COUNT}
View
@@ -34,23 +34,24 @@ log4j.logger.edu.brown.hstore.conf=INFO
log4j.logger.edu.brown.catalog=INFO
log4j.logger.edu.brown.utils=INFO
log4j.logger.edu.brown.benchmark.ycsb.YCSBClient=TRACE
## VoltDB Stuff
log4j.logger.org.voltdb.VoltProcedure=WARN
log4j.logger.org.voltdb.VoltProcedure=INFO
log4j.logger.org.voltdb.VoltSystemProcedure=INFO
log4j.logger.org.voltdb.client=INFO
log4j.logger.org.voltdb.compiler=INFO
log4j.logger.org.voltdb.planner=INFO
log4j.logger.org.voltdb.sysprocs=INFO
log4j.logger.org.voltdb.sysprocs.LoadMultipartitionTable=INFO
log4j.logger.org.voltdb.sysprocs.SnapshotSave=INFO
log4j.logger.org.voltdb.SnapshotSaveAPI=INFO
log4j.logger.org.voltdb.SnapshotSiteProcessor=INFO
log4j.logger.org.voltdb.DefaultSnapshotDataTarget=INFO
log4j.logger.org.voltdb.sysprocs.SnapshotDelete=INFO
log4j.logger.org.voltdb.sysprocs.SnapshotRestore=INFO
log4j.logger.org.voltdb.sysprocs.saverestore.PartitionedTableSaveFileState=INFO
log4j.logger.org.voltdb.sysprocs.saverestore.ReplicatedTableSaveFileState=INFO
log4j.logger.org.voltdb.sysprocs.SnapshotSave=WARN
log4j.logger.org.voltdb.SnapshotSaveAPI=WARN
log4j.logger.org.voltdb.SnapshotSiteProcessor=WARN
log4j.logger.org.voltdb.DefaultSnapshotDataTarget=WARN
log4j.logger.org.voltdb.sysprocs.SnapshotDelete=WARN
log4j.logger.org.voltdb.sysprocs.SnapshotRestore=WARN
log4j.logger.org.voltdb.sysprocs.saverestore.PartitionedTableSaveFileState=WARN
log4j.logger.org.voltdb.sysprocs.saverestore.ReplicatedTableSaveFileState=WARN
## Research Projects
@@ -6,8 +6,8 @@ builder = edu.brown.benchmark.ycsb.YCSBProjectBuilder
# If 'fixed_size' is set to true, then then the YCSBLoader will
# only add the number of tuples defined in 'num_records'
fixed_size = true
num_records = 100000
# Zipfian skew factor for tuple access
skew_factor = 1.5
#loadthreads = 16
View
36 run.sh 100644 → 100755
@@ -1,17 +1,43 @@
#!/bin/sh
#!/usr/bin/zsh
# RUN EXPERIMENTS
DEFAULT_LATENCY=110
LOG_DIR=log
SCRIPT=./experiment.sh
for (( i=1; i<=8; i*=2 ))
for ((i=2; i<=8; i*=4))
do
l=$(($i*$DEFAULT_LATENCY))
echo "---------------------------------------------------"
echo "LATENCY" $l
./ycsb.sh -s $l &> $LOG_DIR/$i.log
$SCRIPT -s $l &> $LOG_DIR/$i.log
./ycsb.sh -m &>> $LOG_DIR/$i.log
# RESET SKEW AT START
cp ./src/benchmarks/edu/brown/benchmark/ycsb/YCSBConstants.java.base ./src/benchmarks/edu/brown/benchmark/ycsb/YCSBConstants.java
cp ./properties/benchmarks/ycsb.properties.base ./properties/benchmarks/ycsb.properties
cp results.csv $LOG_DIR/$i.csv
grep "skew_factor =" ./properties/benchmarks/ycsb.properties
grep "ZIPFIAN_CONSTANT =" ./src/benchmarks/edu/brown/benchmark/ycsb/YCSBConstants.java
echo "---------------------------------------------------"
for ((s=1; s<=5; s+=1))
do
p=$((0.25*$s));
q=$((0.25*$(($s+1))));
echo "UPDATING SKEW FROM $p TO $q"
sed -i "s/skew_factor = $p/skew_factor = $q/g" ./properties/benchmarks/ycsb.properties
sed -i "s/ZIPFIAN_CONSTANT = $p/ZIPFIAN_CONSTANT = $q/g" ./src/benchmarks/edu/brown/benchmark/ycsb/YCSBConstants.java
#ant compile
grep "skew_factor =" ./properties/benchmarks/ycsb.properties
grep "ZIPFIAN_CONSTANT =" ./src/benchmarks/edu/brown/benchmark/ycsb/YCSBConstants.java
$SCRIPT &>> $LOG_DIR/$i.log
cp results.csv "$LOG_DIR/""$i""_$s.csv"
done
done
View
@@ -0,0 +1,22 @@
#!/usr/bin/zsh
# RUN EXPERIMENTS
DEFAULT_LATENCY=100
LOG_DIR=log
SCRIPT=./experiment.sh
for ((i=2; i<=8; i*=4))
do
l=$(($i*$DEFAULT_LATENCY))
echo "---------------------------------------------------"
echo "LATENCY" $l
$SCRIPT -s $l &> $LOG_DIR/$i.log
echo "---------------------------------------------------"
$SCRIPT -t &>> $LOG_DIR/$i.log
cp results.csv "$LOG_DIR/$i.csv"
done
View
@@ -0,0 +1,51 @@
#!/usr/bin/zsh
# RUN EXPERIMENTS
DEFAULT_LATENCY=100
LOG_DIR=log
SCRIPT=./experiment.sh
for ((i=2; i<=8; i*=4))
do
l=$(($i*$DEFAULT_LATENCY))
echo "---------------------------------------------------"
echo "LATENCY" $l
$SCRIPT -s $l &> $LOG_DIR/$i.log
# RESET SKEW AT START
cp ./src/benchmarks/edu/brown/benchmark/ycsb/YCSBConstants.java.base ./src/benchmarks/edu/brown/benchmark/ycsb/YCSBConstants.java
cp ./properties/benchmarks/ycsb.properties.base ./properties/benchmarks/ycsb.properties
grep "skew_factor =" ./properties/benchmarks/ycsb.properties
grep "ZIPFIAN_CONSTANT =" ./src/benchmarks/edu/brown/benchmark/ycsb/YCSBConstants.java
echo "---------------------------------------------------"
for ((s=1; s<=5; s+=1))
do
p=$((0.25*$s));
q=$((0.25*$(($s+1))));
echo "UPDATING SKEW FROM $p TO $q"
if [[ $s -eq 4 ]]; then
p=$((1.01));
fi
if [[ $s -eq 3 ]]; then
q=$((1.01));
fi
sed -i "s/skew_factor = $p/skew_factor = $q/g" ./properties/benchmarks/ycsb.properties
sed -i "s/ZIPFIAN_CONSTANT = $p/ZIPFIAN_CONSTANT = $q/g" ./src/benchmarks/edu/brown/benchmark/ycsb/YCSBConstants.java
grep "skew_factor =" ./properties/benchmarks/ycsb.properties
grep "ZIPFIAN_CONSTANT =" ./src/benchmarks/edu/brown/benchmark/ycsb/YCSBConstants.java
$SCRIPT -y &>> $LOG_DIR/$i.log
cp results.csv "$LOG_DIR/""$i""_$s.csv"
done
done
@@ -176,7 +176,7 @@ public void runLoop() {
}
@Override
protected boolean runOnce() throws IOException {
public boolean runOnce() throws IOException {
// pick random transaction to call, weighted by txnWeights
final Transaction target = this.txnWeights.nextValue();
Oops, something went wrong.

0 comments on commit 1cb5165

Please sign in to comment.