Skip to content

Commit 0702dca

Browse files
apavloapavlo
committed
Merge pull request #211 from huanchenz/master
include the "hybrid index" option
2 parents ef70c46 + c293a7d commit 0702dca

27 files changed

+18259
-2
lines changed

‎scripts/hybrid_index/tpcc.sh

Lines changed: 187 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,187 @@
1+
#!/bin/bash
2+
3+
# ---------------------------------------------------------------------
4+
5+
trap onexit 1 2 3 15
6+
function onexit() {
7+
local exit_status=${1:-$?}
8+
pkill -f hstore.tag
9+
exit $exit_status
10+
}
11+
12+
# ---------------------------------------------------------------------
13+
14+
ENABLE_ANTICACHE=false
15+
16+
SITE_HOST="localhost"
17+
18+
CLIENT_HOSTS=( \
19+
"localhost" \
20+
)
21+
22+
BASE_CLIENT_THREADS=1
23+
BASE_SITE_MEMORY=8192
24+
BASE_SITE_MEMORY_PER_PARTITION=1024
25+
BASE_PROJECT="tpcc"
26+
BASE_DIR=`pwd`
27+
28+
ANTICACHE_BLOCK_SIZE=1048576
29+
ANTICACHE_THRESHOLD=.75
30+
31+
CPU_SITE_BLACKLIST="0,2,4,6,8,10,12,14"
32+
33+
mkdir -p "tpcc-test"
34+
35+
for round in 1; do
36+
#for round in 1 2 3; do
37+
#for round in 1 2 3 4 5 6 7 8 9 10; do
38+
OUTPUT_PREFIX="tpcc-test/$round-tpcc"
39+
echo $OUTPUT_PREFIX
40+
BASE_ARGS=( \
41+
# SITE DEBUG
42+
"-Dsite.status_enable=false" \
43+
"-Dsite.status_interval=20000" \
44+
# "-Dsite.status_exec_info=true" \
45+
# "-Dsite.status_check_for_zombies=true" \
46+
# "-Dsite.exec_profiling=true" \
47+
# "-Dsite.pool_profiling=true" \
48+
# "-Dsite.network_profiling=false" \
49+
# "-Dsite.log_backup=true"\
50+
# "-Dnoshutdown=true" \
51+
"-Dsite.log_dir=logs/$OUTPUT_PREFIX" \
52+
53+
# Site Params
54+
"-Dsite.jvm_asserts=false" \
55+
"-Dsite.specexec_enable=false" \
56+
"-Dsite.cpu_affinity_one_partition_per_core=true" \
57+
"-Dsite.network_incoming_limit_txns=50000" \
58+
"-Dsite.commandlog_enable=false" \
59+
"-Dsite.txn_incoming_delay=5" \
60+
"-Dsite.exec_postprocessing_threads=true" \
61+
"-Dsite.anticache_profiling=false" \
62+
"-Dsite.anticache_eviction_distribution=even" \
63+
64+
"-Dsite.cpu_partition_blacklist=${CPU_SITE_BLACKLIST}" \
65+
66+
# Client Params
67+
"-Dclient.scalefactor=1" \
68+
"-Dclient.memory=512" \
69+
"-Dclient.txnrate=10000" \
70+
"-Dclient.warmup=60000" \
71+
"-Dclient.duration=300000" \
72+
"-Dclient.shared_connection=false" \
73+
"-Dclient.blocking=true" \
74+
"-Dclient.blocking_concurrent=120" \
75+
#"-Dclient.throttle_backoff=100" \
76+
"-Dclient.output_interval=true" \
77+
#"-Dclient.output_anticache_evictions=${OUTPUT_PREFIX}-evictions.csv" \
78+
"-Dclient.output_memory_stats=${OUTPUT_PREFIX}-memory.csv" \
79+
"-Dclient.output_csv=${OUTPUT_PREFIX}-results.csv" \
80+
#"-Dclient.output_index_stats=${OUTPUT_PREFIX}-index.csv" \
81+
82+
# Anti-Caching Experiments
83+
"-Dsite.anticache_enable=${ENABLE_ANTICACHE}" \
84+
"-Dsite.anticache_block_size=${ANTICACHE_BLOCK_SIZE}"
85+
"-Dsite.anticache_check_interval=20000" \
86+
"-Dsite.anticache_threshold_mb=500" \
87+
"-Dsite.anticache_blocks_per_eviction=50" \
88+
"-Dsite.anticache_max_evicted_blocks=100000" \
89+
#"-Dsite.anticache_evict_size=${ANTICACHE_EVICT_SIZE}" \
90+
"-Dsite.anticache_threshold=${ANTICACHE_THRESHOLD}" \
91+
"-Dsite.anticache_eviction_distribution=PROPORTIONAL" \
92+
"-Dclient.interval=5000" \
93+
"-Dclient.anticache_enable=false" \
94+
"-Dclient.anticache_evict_interval=10000" \
95+
"-Dclient.anticache_evict_size=4194304" \
96+
"-Dclient.output_interval=true" \
97+
98+
# CLIENT DEBUG
99+
# "-Dclient.output_txn_counters=txncounters.csv" \
100+
"-Dclient.output_clients=false" \
101+
"-Dclient.profiling=false" \
102+
#"-Dclient.output_response_status=false" \
103+
"-Dclient.output_status=true" \
104+
"-Dclient.output_queue_profiling=${BASE_PROJECT}-queue.csv" \
105+
"-Dclient.output_basepartitions=true" \
106+
# "-Dclient.jvm_args=\"-verbose:gc -XX:+PrintGCDetails -XX:+PrintGCTimeStamps -XX:-TraceClassUnloading\""
107+
)
108+
109+
EVICTABLE_TABLES=( \
110+
"orders" \
111+
"order_line" \
112+
"history" \
113+
)
114+
115+
EVICTABLES=""
116+
if [ "$ENABLE_ANTICACHE" = "true" ]; then
117+
for t in ${EVICTABLE_TABLES[@]}; do
118+
EVICTABLES="${t},${EVICTABLES}"
119+
done
120+
fi
121+
122+
# Compile
123+
HOSTS_TO_UPDATE=("$SITE_HOST")
124+
for CLIENT_HOST in ${CLIENT_HOSTS[@]}; do
125+
NEED_UPDATE=1
126+
for x in ${HOSTS_TO_UPDATE[@]}; do
127+
if [ "$CLIENT_HOST" = "$x" ]; then
128+
NEED_UPDATE=0
129+
break
130+
fi
131+
done
132+
if [ $NEED_UPDATE = 1 ]; then
133+
HOSTS_TO_UPDATE+=("$CLIENT_HOST")
134+
fi
135+
done
136+
#for HOST in ${HOSTS_TO_UPDATE[@]}; do
137+
# ssh $HOST "cd $BASE_DIR && git pull && ant compile" &
138+
#done
139+
wait
140+
141+
ant compile
142+
for i in 8; do
143+
#for i in 4; do
144+
HSTORE_HOSTS="${SITE_HOST}:0:0-"`expr $i - 1`
145+
NUM_CLIENTS=`expr $i \* $BASE_CLIENT_THREADS`
146+
SITE_MEMORY=`expr $BASE_SITE_MEMORY + \( $i \* $BASE_SITE_MEMORY_PER_PARTITION \)`
147+
148+
# BUILD PROJECT JAR
149+
ant hstore-prepare \
150+
-Dproject=${BASE_PROJECT} \
151+
-Dhosts=${HSTORE_HOSTS} \
152+
-Devictable=${EVICTABLES}
153+
test -f ${BASE_PROJECT}.jar || exit -1
154+
155+
# UPDATE CLIENTS
156+
CLIENT_COUNT=0
157+
CLIENT_HOSTS_STR=""
158+
for CLIENT_HOST in ${CLIENT_HOSTS[@]}; do
159+
CLIENT_COUNT=`expr $CLIENT_COUNT + 1`
160+
if [ ! -z "$CLIENT_HOSTS_STR" ]; then
161+
CLIENT_HOSTS_STR="${CLIENT_HOSTS_STR},"
162+
fi
163+
CLIENT_HOSTS_STR="${CLIENT_HOSTS_STR}${CLIENT_HOST}"
164+
done
165+
166+
# DISTRIBUTE PROJECT JAR
167+
for HOST in ${HOSTS_TO_UPDATE[@]}; do
168+
if [ "$HOST" != $(hostname) ]; then
169+
scp -r ${BASE_PROJECT}.jar ${HOST}:${BASE_DIR} &
170+
fi
171+
done
172+
wait
173+
174+
# EXECUTE BENCHMARK
175+
ant hstore-benchmark ${BASE_ARGS[@]} \
176+
-Dproject=${BASE_PROJECT} \
177+
-Dkillonzero=false \
178+
-Dclient.threads_per_host=${NUM_CLIENTS} \
179+
-Dsite.memory=${SITE_MEMORY} \
180+
-Dclient.hosts=${CLIENT_HOSTS_STR} \
181+
-Dclient.count=${CLIENT_COUNT}
182+
result=$?
183+
if [ $result != 0 ]; then
184+
exit $result
185+
fi
186+
done
187+
done

‎scripts/hybrid_index/tpcc_ac.sh

Lines changed: 202 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,202 @@
1+
#!/bin/bash
2+
3+
# ---------------------------------------------------------------------
4+
5+
trap onexit 1 2 3 15
6+
function onexit() {
7+
local exit_status=${1:-$?}
8+
pkill -f hstore.tag
9+
exit $exit_status
10+
}
11+
12+
# ---------------------------------------------------------------------
13+
14+
ENABLE_ANTICACHE=true
15+
16+
SITE_HOST="localhost"
17+
18+
CLIENT_HOSTS=( \
19+
"localhost" \
20+
)
21+
22+
BASE_CLIENT_THREADS=1
23+
BASE_SITE_MEMORY=8192
24+
BASE_SITE_MEMORY_PER_PARTITION=1024
25+
BASE_PROJECT="tpcc"
26+
BASE_DIR=`pwd`
27+
SPACE=" "
28+
29+
CPU_SITE_BLACKLIST="0,2,4,6,8,10,12,14"
30+
31+
for memory in allmt; do
32+
duration=720000
33+
mkdir -p "tpcc-ac-test"
34+
35+
#for round in 1 2 3 4 5; do
36+
for round in 1; do
37+
OUTPUT_PREFIX="tpcc-ac-test/$round-tpcc"
38+
LOG_PREFIX="logs/tpcc"
39+
echo $OUTPUT_PREFIX
40+
41+
ANTICACHE_BLOCK_SIZE=1048576
42+
ANTICACHE_THRESHOLD=.5
43+
44+
BASE_ARGS=( \
45+
# SITE DEBUG
46+
"-Dsite.status_enable=false" \
47+
"-Dsite.status_interval=5000" \
48+
# "-Dsite.status_exec_info=true" \
49+
# "-Dsite.status_check_for_zombies=true" \
50+
# "-Dsite.exec_profiling=true" \
51+
# "-Dsite.profiling=true" \
52+
# "-Dsite.txn_counters=true" \
53+
# "-Dsite.pool_profiling=true" \
54+
# "-Dsite.network_profiling=false" \
55+
# "-Dsite.log_backup=true"\
56+
# "-Dnoshutdown=true" \
57+
58+
# Site Params
59+
"-Dsite.jvm_asserts=false" \
60+
"-Dsite.specexec_enable=false" \
61+
"-Dsite.cpu_affinity_one_partition_per_core=true" \
62+
"-Dsite.network_incoming_limit_txns=50000" \
63+
"-Dsite.commandlog_enable=false" \
64+
"-Dsite.txn_incoming_delay=5" \
65+
"-Dsite.exec_postprocessing_threads=true" \
66+
#"-Dsite.anticache_eviction_distribution=$eviction_distribution" \
67+
"-Dsite.anticache_eviction_distribution=proportional" \
68+
#"-Dsite.log_dir=$LOG_PREFIX" \
69+
"-Dsite.specexec_enable=false"
70+
71+
"-Dsite.cpu_partition_blacklist=${CPU_SITE_BLACKLIST}" \
72+
73+
# "-Dsite.queue_allow_decrease=true" \
74+
# "-Dsite.queue_allow_increase=true" \
75+
# "-Dsite.queue_threshold_factor=0.5" \
76+
77+
# Client Params
78+
"-Dclient.scalefactor=1" \
79+
"-Dclient.memory=512" \
80+
"-Dclient.txnrate=10000" \
81+
"-Dclient.warmup=0000" \
82+
"-Dclient.duration=$duration" \
83+
"-Dclient.interval=2000" \
84+
"-Dclient.shared_connection=false" \
85+
"-Dclient.blocking=true" \
86+
"-Dclient.blocking_concurrent=120" \
87+
#"-Dclient.throttle_backoff=100" \
88+
"-Dclient.output_anticache_evictions=${OUTPUT_PREFIX}-evictions.csv" \
89+
"-Dclient.output_memory_stats=${OUTPUT_PREFIX}-memory.csv" \
90+
#"-Dclient.output_index_stats=${OUTPUT_PREFIX}-indexes.csv" \
91+
#"-Dclient.output_anticache_access=${OUTPUT_PREFIX}-access.csv"\
92+
#"-Dclient.output_txn_profiling=${OUTPUT_PREFIX}-txnprofiler.csv"\
93+
#"-Dclient.output_exec_profiling=${OUTPUT_PREFIX}-execprofiler.csv"\
94+
#"-Dclient.weights=\"ReadRecord:100,UpdateRecord:0,*:0\""\
95+
96+
# Anti-Caching Experiments
97+
"-Dsite.anticache_enable=${ENABLE_ANTICACHE}" \
98+
"-Dsite.anticache_timestamps=${ENABLE_TIMESTAMPS}" \
99+
"-Dsite.anticache_batching=true" \
100+
"-Dsite.anticache_profiling=true" \
101+
"-Dsite.anticache_reset=false" \
102+
"-Dsite.anticache_block_size=${ANTICACHE_BLOCK_SIZE}" \
103+
"-Dsite.anticache_check_interval=5000" \
104+
"-Dsite.anticache_threshold_mb=5000" \
105+
"-Dsite.anticache_blocks_per_eviction=250" \
106+
#"-Dsite.anticache_blocks_per_eviction=$eviction_size" \
107+
"-Dsite.anticache_max_evicted_blocks=1000000" \
108+
# "-Dsite.anticache_evict_size=${ANTICACHE_EVICT_SIZE}" \
109+
"-Dsite.anticache_threshold=${ANTICACHE_THRESHOLD}" \
110+
"-Dclient.anticache_enable=false" \
111+
"-Dclient.anticache_evict_interval=5000" \
112+
"-Dclient.anticache_evict_size=102400" \
113+
"-Dclient.output_csv=${OUTPUT_PREFIX}-results.csv" \
114+
115+
# CLIENT DEBUG
116+
#"-Dclient.output_txn_counters=${OUTPUT_PREFIX}-txncounters.csv" \
117+
"-Dclient.output_clients=false" \
118+
"-Dclient.profiling=false" \
119+
"-Dclient.output_response_status=false" \
120+
#"-Dclient.output_queue_profiling=${OUTPUT_PREFIX}-queue.csv" \
121+
"-Dclient.output_basepartitions=true" \
122+
#"-Dclient.jvm_args=\"-verbose:gc -XX:+PrintGCDetails -XX:+PrintGCTimeStamps -XX:-TraceClassUnloading\""
123+
)
124+
125+
EVICTABLE_TABLES=( \
126+
"orders" \
127+
"order_line" \
128+
"history" \
129+
)
130+
EVICTABLES=""
131+
if [ "$ENABLE_ANTICACHE" = "true" ]; then
132+
for t in ${EVICTABLE_TABLES[@]}; do
133+
EVICTABLES="${t},${EVICTABLES}"
134+
done
135+
fi
136+
137+
# Compile
138+
HOSTS_TO_UPDATE=("$SITE_HOST")
139+
for CLIENT_HOST in ${CLIENT_HOSTS[@]}; do
140+
NEED_UPDATE=1
141+
for x in ${HOSTS_TO_UPDATE[@]}; do
142+
if [ "$CLIENT_HOST" = "$x" ]; then
143+
NEED_UPDATE=0
144+
break
145+
fi
146+
done
147+
if [ $NEED_UPDATE = 1 ]; then
148+
HOSTS_TO_UPDATE+=("$CLIENT_HOST")
149+
fi
150+
done
151+
#for HOST in ${HOSTS_TO_UPDATE[@]}; do
152+
#ssh $HOST "cd $BASE_DIR && git pull && ant compile" &
153+
#done
154+
wait
155+
156+
ant compile
157+
HSTORE_HOSTS="${SITE_HOST}:0:0-7"
158+
NUM_CLIENTS=`expr 8 \* $BASE_CLIENT_THREADS`
159+
SITE_MEMORY=`expr $BASE_SITE_MEMORY + \( 8 \* $BASE_SITE_MEMORY_PER_PARTITION \)`
160+
161+
# BUILD PROJECT JAR
162+
ant hstore-prepare \
163+
-Dproject=${BASE_PROJECT} \
164+
-Dhosts=${HSTORE_HOSTS} \
165+
-Devictable=${EVICTABLES}
166+
test -f ${BASE_PROJECT}.jar || exit -1
167+
168+
# UPDATE CLIENTS
169+
CLIENT_COUNT=0
170+
CLIENT_HOSTS_STR=""
171+
for CLIENT_HOST in ${CLIENT_HOSTS[@]}; do
172+
CLIENT_COUNT=`expr $CLIENT_COUNT + 1`
173+
if [ ! -z "$CLIENT_HOSTS_STR" ]; then
174+
CLIENT_HOSTS_STR="${CLIENT_HOSTS_STR},"
175+
fi
176+
CLIENT_HOSTS_STR="${CLIENT_HOSTS_STR}${CLIENT_HOST}"
177+
done
178+
179+
# DISTRIBUTE PROJECT JAR
180+
for HOST in ${HOSTS_TO_UPDATE[@]}; do
181+
if [ "$HOST" != $(hostname) ]; then
182+
scp -r ${BASE_PROJECT}.jar ${HOST}:${BASE_DIR} &
183+
fi
184+
done
185+
wait
186+
187+
# EXECUTE BENCHMARK
188+
ant hstore-benchmark ${BASE_ARGS[@]} \
189+
-Dproject=${BASE_PROJECT} \
190+
-Dkillonzero=false \
191+
-Dclient.threads_per_host=8 \
192+
-Dsite.memory=${SITE_MEMORY} \
193+
-Dclient.hosts=${CLIENT_HOSTS_STR} \
194+
-Dclient.count=${CLIENT_COUNT}
195+
result=$?
196+
if [ $result != 0 ]; then
197+
exit $result
198+
fi
199+
done
200+
done
201+
#done
202+
#done

0 commit comments

Comments
 (0)