From 35c8a1ae79f586a2a7f215dde178208137a65c68 Mon Sep 17 00:00:00 2001 From: Lin Ma Date: Wed, 23 Mar 2016 13:37:08 -0700 Subject: [PATCH] Checkin scripts. --- scripts/multitier/tm1-single-tier-nvm.sh | 15 +- scripts/multitier/voter-single-tier-nvm.sh | 349 +++++++++++++++++++++ scripts/multitier/ycsb-single-tier-nvm.sh | 32 +- 3 files changed, 378 insertions(+), 18 deletions(-) create mode 100755 scripts/multitier/voter-single-tier-nvm.sh diff --git a/scripts/multitier/tm1-single-tier-nvm.sh b/scripts/multitier/tm1-single-tier-nvm.sh index 3dcdbdcf5c..50e3721e65 100755 --- a/scripts/multitier/tm1-single-tier-nvm.sh +++ b/scripts/multitier/tm1-single-tier-nvm.sh @@ -50,7 +50,7 @@ AC_THRESH=500 SCALE=100 #BLOCK_SIZE_KB=256 DURATION_S=600 -WARMUP_S=60 +WARMUP_S=30 INTERVAL_S=2 PARTITIONS=8 @@ -58,7 +58,8 @@ CPU_SITE_BLACKLIST="1,2,4,6,8,10,12,14" for BLK_CON in 500; do for round in 1; do -for DB in 'SSD' 'HDD' 'ALLOCATORNVM'; do +for DB in 'SSD' ; do +#for DB in 'SSD' 'HDD' 'NVM'; do for BLOCK_SIZE in 1024; do #for DB in 'ALLOCATORNVM'; do #for device in 'NVM' 'SSD' 'DRAM'; do @@ -83,9 +84,9 @@ for BLOCKING in 'true';do if [ "$DB" = "SSD" ]; then BLOCK_SIZE=4 fi - if [ "$DB" = "HDD" ]; then - BLOCK_SIZE=16 - fi + #if [ "$DB" = "HDD" ]; then + # BLOCK_SIZE=16 + #fi #for DB in 'NVM' 'ALLOCATORNVM' 'SSD'; do #for DB in 'SSD' 'NVM' 'HDD' 'DRAM'; do for skew in 1.01; do @@ -129,6 +130,10 @@ for BLOCKING in 'true';do AC_DIR="tmp/ac_berk/ycsb-berk-level1" #AC_DIR="tmp/ac_berk/ycsb-berk-level$RANDOM" fi + if [ "$DB" = "SMR" ]; then + AC_DIR="/data2/ac_berk/ycsb-berk-level1" + #AC_DIR="tmp/ac_berk/ycsb-berk-level$RANDOM" + fi if [ "$DB" = "DRAM" ]; then AC_THRESH=5000 fi diff --git a/scripts/multitier/voter-single-tier-nvm.sh b/scripts/multitier/voter-single-tier-nvm.sh new file mode 100755 index 0000000000..058260e810 --- /dev/null +++ b/scripts/multitier/voter-single-tier-nvm.sh @@ -0,0 +1,349 @@ +#!/bin/bash + +#UtilityWorkMessage --------------------------------------------------------------------- + +trap onexit 1 2 3 15 +function onexit() { + local exit_status=${1:-$?} + pkill -f hstore.tag + exit $exit_status +} + +# --------------------------------------------------------------------- + +ENABLE_ANTICACHE=true + +#SITE_HOST="dev3.db.pdl.cmu.local" +SITE_HOST="localhost" + +#CLIENT_HOSTS=( "localhost") + +CLIENT_HOSTS=( \ + "localhost" \ + "localhost" \ + "localhost" \ + "localhost" \ + ) +N_HOSTS=4 +BASE_CLIENT_THREADS=1 +CLIENT_THREADS_PER_HOST=4 +#BASE_SITE_MEMORY=8192 +#BASE_SITE_MEMORY_PER_PARTITION=1024 +BASE_SITE_MEMORY=12288 +BASE_SITE_MEMORY_PER_PARTITION=2024 +BASE_PROJECT="voter" +BASE_DIR=`pwd` +#OUTPUT_DIR_PREFIX="data-sketch/mergeupdate" +#OUTPUT_DIR_PREFIX="data-DRAM/" +#OUTPUT_DIR_PREFIX="data-tpcc-8GB/optimized/" +#OUTPUT_DIR_PREFIX="data-tmp" +#OUTPUT_DIR_PREFIX="data-tm1-3GB/optimized/" +OUTPUT_DIR_PREFIX="data-voter/optimized/" +#OUTPUT_DIR_PREFIX="data-voter/default/" +#OUTPUT_DIR_PREFIX="data-blocksize-throttle-10GB-2/" +#OUTPUT_DIR_PREFIX="data-allocator-10GB/" +#OUTPUT_DIR_PREFIX="data-ALLOCATORNVM-10GB/" +#OUTPUT_DIR_PREFIX="data-blocksize-10GB-HDD/" +#OUTPUT_DIR_PREFIX="data-sketch-125T/" +#BLK_CON=1 +#BLK_EVICT=800 +AC_THRESH=500 +SCALE=100 +#BLOCK_SIZE_KB=256 +DURATION_S=150 +WARMUP_S=3 +INTERVAL_S=2 +PARTITIONS=8 + +CPU_SITE_BLACKLIST="1,2,4,6,8,10,12,14" + +for BLK_CON in 500; do +for round in 1; do +#for DB in 'NVM'; do +for DB in 'SSD' 'HDD' 'ALLOCATORNVM'; do +for BLOCK_SIZE in 1024; do +#for DB in 'ALLOCATORNVM'; do +#for device in 'NVM' 'SSD' 'DRAM'; do +for BLOCK_MERGE in 'false'; do +for BLOCKING in 'true';do + #if [ "$DB" = "HDD" -a "$BLOCK_MERGE" = "false" ]; then + # continue + #fi +#for BLOCK_MERGE in 'false' 'true'; do + #OUTPUT_DIR=${OUTPUT_DIR_PREFIX}${BLK_EVICT}*${BLOCK_SIZE}kb + #mkdir -p $OUTPUT_DIR + + #for DB in 'NVM' ; do + #for latency in 160; do + #for latency in 160 320 640 1280; do + # /data/devel/sdv-tools/sdv-release/ivt_pm_sdv.sh --enable --pm-latency=$latency + for AC_THRESH in 250; do + #for AC_THRESH in 9000; do + #if [ "$DB" = "NVM" ]; then + # BLOCK_SIZE=1 + #fi + if [ "$DB" = "SSD" ]; then + BLOCK_SIZE=4 + fi + if [ "$DB" = "HDD" ]; then + BLOCK_SIZE=16 + fi + #for DB in 'NVM' 'ALLOCATORNVM' 'SSD'; do + #for DB in 'SSD' 'NVM' 'HDD' 'DRAM'; do + for skew in 1.01; do + #for read_percent in 90 100; do + for sketch_thresh in 10; do + #for sketch_thresh in 0 10 40 200; do + #sed -i "55c\ \ \ \ #define SKETCH_THRESH ${sketch_thresh}" src/ee/anticache/AntiCacheEvictionManager.h + #ant ee-build + + #OUTPUT_DIR=${OUTPUT_DIR_PREFIX}${device} + OUTPUT_DIR=${OUTPUT_DIR_PREFIX}${DB} + #OUTPUT_DIR="${OUTPUT_DIR_PREFIX}${DB}/sketch${sketch_thresh}" + sudo -u user mkdir -p $OUTPUT_DIR + echo $BLK_EVICT + + if [ "$BLOCKING" = "true" ]; then + block='sync' + else + block='abrt' + fi + if [ "$BLOCK_MERGE" = "true" ]; then + block_merge='block' + else + block_merge='tuple' + fi + + #AC_THRESH=500 + DB_TYPE="$DB" + if [ "$DB" = "NVM" ]; then + AC_DIR='/mnt/pmfs/aclevel2' + else + #BLOCK_SIZE=4 + if [ "$DB" != "ALLOCATORNVM" ]; then + DB_TYPE="BERKELEY" + fi + if [ "$DB" = "SSD" ]; then + AC_DIR="/data1/ac_berk/ycsb-berk-level1" + #AC_DIR="/data1/ac_berk/ycsb-berk-level$RANDOM" + fi + if [ "$DB" = "HDD" ]; then + AC_DIR="tmp/ac_berk/ycsb-berk-level1" + #AC_DIR="tmp/ac_berk/ycsb-berk-level$RANDOM" + fi + if [ "$DB" = "SMR" ]; then + AC_DIR="/data2/ac_berk/ycsb-berk-level1" + #AC_DIR="tmp/ac_berk/ycsb-berk-level$RANDOM" + fi + if [ "$DB" = "DRAM" ]; then + AC_THRESH=5000 + fi + fi + rm -rf $AC_DIR + + BLOCK_SIZE_KB=$BLOCK_SIZE + BLK_EVICT=$((102400 / $BLOCK_SIZE_KB)) + #if [ "$BLOCK_SIZE" = "256" ]; then + #BLK_EVICT=$((409600 / $BLOCK_SIZE_KB)) + # BLOCKING="false" + #BLOCK_MERGE="true" + #fi + + OUTPUT_PREFIX="$OUTPUT_DIR/$round-tm1-3G-$block-$DB-S$skew-${PARTITIONS}p-${BLK_CON}c-${N_HOSTS}h-${CLIENT_THREADS_PER_HOST}ct-sc$SCALE-${BLOCK_SIZE_KB}kb-${BLK_EVICT}b-${AC_THRESH}th-${DURATION_S}s-${block_merge}-R${read_percent}" + #OUTPUT_PREFIX="$OUTPUT_DIR/$round-ycsb1G-$block-$DB-S$skew-${PARTITIONS}p-${BLK_CON}c-${N_HOSTS}h-${CLIENT_THREADS_PER_HOST}ct-sc$SCALE-${BLOCK_SIZE_KB}kb-${BLK_EVICT}b-${AC_THRESH}th-${DURATION_S}s-${block_merge}" + LOG_PREFIX="logs/ycsb-nvm/$round-ycsb1G-$block-$DB-S$skew-${PARTITIONS}p-${BLK_CON}c-${N_HOSTS}h-${CLIENT_THREADS_PER_HOST}ct-sc$SCALE-${BLOCK_SIZE_KB}kb-${BLK_EVICT}b-${AC_THRESH}th-${DURATION_S}s-${block_merge}" + echo "log = $LOG_PREFIX" + echo $OUTPUT_PREFIX + sed -i '$ d' "properties/benchmarks/ycsb.properties" + echo "skew_factor = $skew" >> "properties/benchmarks/ycsb.properties" + +#ANTICACHE_BLOCK_SIZE=65536 + +#ANTICACHE_BLOCK_SIZE=262144 + ANTICACHE_BLOCK_SIZE=$(($BLOCK_SIZE_KB * 1024)) + if [ "$BLOCK_SIZE" = "1" ]; then + ANTICACHE_BLOCK_SIZE=$(($BLOCK_SIZE_KB * 1100)) + fi + ANTICACHE_THRESHOLD=.5 + DURATION=$((${DURATION_S} * 1000)) + WARMUP=$((${WARMUP_S} * 1000)) + INTERVAL=$((${INTERVAL_S} * 1000)) + + BASE_ARGS=( \ +# SITE DEBUG + "-Dsite.status_enable=false" \ + "-Dsite.status_interval=10000" \ +# "-Dsite.status_exec_info=true" \ +# "-Dsite.status_check_for_zombies=true" \ +# "-Dsite.exec_profiling=true" \ +# "-Dsite.profiling=true" \ +# "-Dsite.txn_counters=true" \ +# "-Dsite.pool_profiling=true" \ +# "-Dsite.network_profiling=false" \ +# "-Dsite.log_backup=true"\ +# "-Dnoshutdown=true" \ + +# Site Params + "-Dsite.jvm_asserts=false" \ + "-Dsite.specexec_enable=false" \ + "-Dsite.cpu_affinity_one_partition_per_core=true" \ +#"-Dsite.cpu_partition_blacklist=0,2,4,6,8,10,12,14,16,18" \ +#"-Dsite.cpu_utility_blacklist=0,2,4,6,8,10,12,14,16,18" \ + "-Dsite.network_incoming_limit_txns=100000" \ + "-Dsite.commandlog_enable=false" \ + "-Dsite.txn_incoming_delay=5" \ + "-Dsite.exec_postprocessing_threads=false" \ + "-Dsite.anticache_eviction_distribution=proportional" \ + #"-Dsite.log_dir=$LOG_PREFIX" \ + #"-Dclient.log_dir=$LOG_PREFIX" \ + "-Dsite.specexec_enable=false" \ + +# "-Dsite.queue_allow_decrease=true" \ +# "-Dsite.queue_allow_increase=true" \ + "-Dsite.queue_threshold_factor=0.02" \ + + "-Dsite.cpu_partition_blacklist=${CPU_SITE_BLACKLIST}" \ + +# Client Params + "-Dclient.scalefactor=${SCALE}" \ + "-Dclient.memory=2048" \ + "-Dclient.txnrate=16000" \ + "-Dclient.warmup=${WARMUP}" \ + "-Dclient.duration=${DURATION}" \ + "-Dclient.interval=${INTERVAL}" \ + "-Dclient.shared_connection=false" \ + "-Dclient.blocking=true" \ + "-Dclient.blocking_concurrent=${BLK_CON}" \ + #"-Dclient.throttle_backoff=100" \ + "-Dclient.output_anticache_evictions=${OUTPUT_PREFIX}-evictions.csv" \ + #"-Dclient.output_anticache_profiling=${OUTPUT_PREFIX}-acprofiling.csv" \ +# "-Dclient.output_anticache_access=${OUTPUT_PREFIX}-accesses.csv" \ + "-Dclient.output_memory_stats=${OUTPUT_PREFIX}-memory.csv" \ + "-Dclient.output_anticache_memory_stats=${OUTPUT_PREFIX}-anticache-memory.csv" \ + #"-Dclient.weights=\"ReadRecord:${read_percent},UpdateRecord:$((100 - ${read_percent})),*:0\"" \ + +# Anti-Caching Experiments + "-Dsite.anticache_enable=${ENABLE_ANTICACHE}" \ + "-Dsite.anticache_timestamps=${ENABLE_TIMESTAMPS}" \ + "-Dsite.anticache_batching=false" \ + "-Dsite.anticache_profiling=false" \ + "-Dsite.anticache_reset=false" \ + "-Dsite.anticache_block_size=${ANTICACHE_BLOCK_SIZE}" \ + "-Dsite.anticache_check_interval=2000" \ + "-Dsite.anticache_threshold_mb=${AC_THRESH}" \ + "-Dsite.anticache_blocks_per_eviction=${BLK_EVICT}" \ + "-Dsite.anticache_max_evicted_blocks=50000000" \ + "-Dsite.anticache_dbsize=10000M" \ + "-Dsite.anticache_db_blocks=$BLOCKING" \ + "-Dsite.anticache_block_merge=$BLOCK_MERGE" \ + "-Dsite.anticache_dbtype=$DB_TYPE" \ +# "-Dsite.anticache_evict_size=${ANTICACHE_EVICT_SIZE}" \ + "-Dsite.anticache_dir=${AC_DIR}" \ + #"-Dsite.anticache_dir=/mnt/pmfs/aclevel0" \ + #"-Dsite.anticache_dir=/data1/berk-level$round" \ + "-Dsite.anticache_threshold=${ANTICACHE_THRESHOLD}" \ + "-Dclient.anticache_enable=false" \ + "-Dclient.anticache_evict_interval=2000" \ + "-Dclient.anticache_evict_size=${ANTICACHE_BLOCK_SIZE}" \ + "-Dclient.output_csv=${OUTPUT_PREFIX}-results.csv" \ + +# CLIENT DEBUG + "-Dclient.output_txn_counters=${OUTPUT_PREFIX}-txncounters.csv" \ + "-Dclient.output_clients=false" \ + "-Dclient.profiling=false" \ + "-Dclient.output_response_status=false" \ + "-Dclient.output_queue_profiling=${OUTPUT_PREFIX}-queue.csv" \ + "-Dclient.output_basepartitions=true" \ +#"-Dclient.jvm_args=\"-verbose:gc -XX:+PrintGCDetails -XX:+PrintGCTimeStamps -XX:-TraceClassUnloading\"" + ) + + EVICTABLE_TABLES=( \ + "votes" \ + ) + EVICTABLES="" + if [ "$ENABLE_ANTICACHE" = "true" ]; then + for t in ${EVICTABLE_TABLES[@]}; do + EVICTABLES="${t},${EVICTABLES}" + done + fi + +# Compile + HOSTS_TO_UPDATE=("$SITE_HOST") + for CLIENT_HOST in ${CLIENT_HOSTS[@]}; do + NEED_UPDATE=1 + for x in ${HOSTS_TO_UPDATE[@]}; do + if [ "$CLIENT_HOST" = "$x" ]; then + NEED_UPDATE=0 + break + fi + done + if [ $NEED_UPDATE = 1 ]; then + HOSTS_TO_UPDATE+=("$CLIENT_HOST") + fi + done +#for HOST in ${HOSTS_TO_UPDATE[@]}; do +#ssh $HOST "cd $BASE_DIR && git pull && ant compile" & +#done + wait + + ant compile +#HSTORE_HOSTS="${SITE_HOST}:0:0-7" + if [ "$PARTITIONS" = "1" ]; then + HSTORE_HOSTS="${SITE_HOST}:0:0" + else + PART_NO=$((${PARTITIONS} - 1)) + HSTORE_HOSTS="${SITE_HOST}:0:0-$PART_NO" + fi + echo "$HSTORE_HOSTS" + + NUM_CLIENTS=$((${PARTITIONS} * ${BASE_CLIENT_THREADS})) + SITE_MEMORY=`expr $BASE_SITE_MEMORY + \( $PARTITIONS \* $BASE_SITE_MEMORY_PER_PARTITION \)` + +# BUILD PROJECT JAR + ant hstore-prepare \ + -Dproject=${BASE_PROJECT} \ + -Dhosts=${HSTORE_HOSTS} \ + -Devictable=${EVICTABLES} + test -f ${BASE_PROJECT}.jar || exit -1 + +# UPDATE CLIENTS + CLIENT_COUNT=0 + CLIENT_HOSTS_STR="" + for CLIENT_HOST in ${CLIENT_HOSTS[@]}; do + CLIENT_COUNT=`expr $CLIENT_COUNT + 1` + if [ ! -z "$CLIENT_HOSTS_STR" ]; then + CLIENT_HOSTS_STR="${CLIENT_HOSTS_STR}", + fi + CLIENT_HOSTS_STR="${CLIENT_HOSTS_STR}${CLIENT_HOST}" + done + +# DISTRIBUTE PROJECT JAR + for HOST in ${HOSTS_TO_UPDATE[@]}; do + if [ "$HOST" != $(hostname) ]; then + sudo -u user scp -r ${BASE_PROJECT}.jar ${HOST}:${BASE_DIR} & + fi + done + wait + + echo "Client count $CLIENT_COUNT client hosts: $CLIENT_HOSTS_STR" +# EXECUTE BENCHMARK + sudo -u user ant hstore-benchmark ${BASE_ARGS[@]} \ + -Dproject=${BASE_PROJECT} \ + -Dkillonzero=false \ + -Dclient.threads_per_host=${CLIENT_THREADS_PER_HOST} \ + -Dsite.memory=${SITE_MEMORY} \ + -Dclient.hosts=${CLIENT_HOSTS_STR} \ + -Dclient.count=${CLIENT_COUNT} + result=$? + if [ $result != 0 ]; then + exit $result + fi + done + done + done +done +done +done +done +done #BLOCK_SIZE +done #BLK_CON diff --git a/scripts/multitier/ycsb-single-tier-nvm.sh b/scripts/multitier/ycsb-single-tier-nvm.sh index 0dc68eb6b4..c455f8df5a 100755 --- a/scripts/multitier/ycsb-single-tier-nvm.sh +++ b/scripts/multitier/ycsb-single-tier-nvm.sh @@ -41,7 +41,7 @@ BASE_DIR=`pwd` #OUTPUT_DIR_PREFIX="data-DRAM/" OUTPUT_DIR_PREFIX="data-sketch-10GB-final/" #OUTPUT_DIR_PREFIX="data-temp/" -#OUTPUT_DIR_PREFIX="data-blocksize-throttle-10GB-2/" +#OUTPUT_DIR_PREFIX="data-blocksize-throttle-10GB-4/" #OUTPUT_DIR_PREFIX="data-allocator-10GB/" #OUTPUT_DIR_PREFIX="data-ALLOCATORNVM-10GB/" #OUTPUT_DIR_PREFIX="data-blocksize-10GB-HDD/" @@ -49,21 +49,22 @@ OUTPUT_DIR_PREFIX="data-sketch-10GB-final/" #BLK_CON=1 #BLK_EVICT=800 AC_THRESH=500 -SCALE=100 +SCALE=10 #BLOCK_SIZE_KB=256 -DURATION_S=780 +DURATION_S=1200 WARMUP_S=60 -INTERVAL_S=2 +INTERVAL_S=1 PARTITIONS=8 for BLK_CON in 500; do for round in 1 2; do -for DB in 'HDD' ; do -for BLOCK_SIZE in 4; do +for DB in 'SMR' ; do +for BLOCK_SIZE in 1024 16; do +#for BLOCK_SIZE in 4 1024 1 16 64 256; do #for DB in 'ALLOCATORNVM'; do #for device in 'NVM' 'SSD' 'DRAM'; do for BLOCK_MERGE in 'false'; do -for BLOCKING in 'true';do +for BLOCKING in 'false' ;do #if [ "$DB" = "HDD" -a "$BLOCK_MERGE" = "false" ]; then # continue #fi @@ -75,8 +76,8 @@ for BLOCKING in 'true';do #for latency in 160; do #for latency in 160 320 640 1280; do # /data/devel/sdv-tools/sdv-release/ivt_pm_sdv.sh --enable --pm-latency=$latency - #for AC_THRESH in 500; do - for AC_THRESH in 1250; do + for AC_THRESH in 500; do + #for AC_THRESH in 1250; do #if [ "$DB" = "NVM" ]; then # BLOCK_SIZE=1 #fi @@ -89,16 +90,17 @@ for BLOCKING in 'true';do fi #for DB in 'NVM' 'ALLOCATORNVM' 'SSD'; do #for DB in 'SSD' 'NVM' 'HDD' 'DRAM'; do + #for skew in 0.5 0.75 1.01 1.25; do for skew in 1.01; do - for read_percent in 90 100; do - #for sketch_thresh in 0; do + for read_percent in 90; do + #for sketch_thresh in 10; do for sketch_thresh in 0 10 40 200; do sed -i "55c\ \ \ \ #define SKETCH_THRESH ${sketch_thresh}" src/ee/anticache/AntiCacheEvictionManager.h ant ee-build #OUTPUT_DIR=${OUTPUT_DIR_PREFIX}${device} #OUTPUT_DIR=${OUTPUT_DIR_PREFIX}${DB} - OUTPUT_DIR="${OUTPUT_DIR_PREFIX}${DB}/sketch${sketch_thresh}" + OUTPUT_DIR="${OUTPUT_DIR_PREFIX}${DB}-${BLOCK_SIZE}/sketch${sketch_thresh}" sudo -u user mkdir -p $OUTPUT_DIR echo $BLK_EVICT @@ -142,6 +144,10 @@ for BLOCKING in 'true';do AC_DIR="tmp/ac_berk/ycsb-berk-level1" #AC_DIR="tmp/ac_berk/ycsb-berk-level$RANDOM" fi + if [ "$DB" = "SMR" ]; then + AC_DIR="/data2/ac_berk/ycsb-berk-level1" + #AC_DIR="tmp/ac_berk/ycsb-berk-level$RANDOM" + fi if [ "$DB" = "DRAM" ]; then AC_THRESH=5000 fi @@ -150,7 +156,7 @@ for BLOCKING in 'true';do rm -rf $AC_DIR BLOCK_SIZE_KB=$BLOCK_SIZE - BLK_EVICT=$((204800 / $BLOCK_SIZE_KB)) + BLK_EVICT=$((40960 / $BLOCK_SIZE_KB)) #if [ "$BLOCK_SIZE" = "256" ]; then #BLK_EVICT=$((409600 / $BLOCK_SIZE_KB)) # BLOCKING="false"