Permalink
Browse files

Merge remote-tracking branch 'upstream/master'

  • Loading branch information...
apavlo committed May 23, 2014
2 parents 4b35b98 + de93be9 commit e51924546740d679188da6d6ba0a751646a1ed22
Showing with 6,640 additions and 919 deletions.
  1. +1 −0 build-common.xml
  2. +40 −3 build.xml
  3. +6 −0 properties/benchmarks/articles.properties
  4. +4 −4 properties/benchmarks/tpcc.properties
  5. +6 −0 properties/benchmarks/users.properties
  6. +1 −1 properties/benchmarks/ycsb.properties
  7. +188 −0 scripts/anticache/articles.sh
  8. +1 −1 scripts/anticache/tpcc.sh
  9. +175 −0 scripts/anticache/users.sh
  10. +15 −23 scripts/anticache/ycsb.sh
  11. +8 −0 src/benchmarks/edu/brown/api/BenchmarkCompiler.java
  12. +6 −1 src/benchmarks/edu/brown/api/BenchmarkConfig.java
  13. +17 −3 src/benchmarks/edu/brown/api/BenchmarkController.java
  14. +1 −1 src/benchmarks/edu/brown/api/PeriodicEvictionThread.java
  15. +216 −0 src/benchmarks/edu/brown/benchmark/articles/ArticlesClient.java
  16. +32 −0 src/benchmarks/edu/brown/benchmark/articles/ArticlesConstants.java
  17. +205 −0 src/benchmarks/edu/brown/benchmark/articles/ArticlesLoader.java
  18. +43 −0 src/benchmarks/edu/brown/benchmark/articles/ArticlesProjectBuilder.java
  19. +94 −0 src/benchmarks/edu/brown/benchmark/articles/ArticlesUtil.java
  20. +88 −0 src/benchmarks/edu/brown/benchmark/articles/articles-ddl.sql
  21. +56 −0 src/benchmarks/edu/brown/benchmark/articles/articles.mappings
  22. +78 −0 src/benchmarks/edu/brown/benchmark/articles/procedures/AddComment.java
  23. +57 −0 src/benchmarks/edu/brown/benchmark/articles/procedures/GetArticle.java
  24. +18 −0 src/benchmarks/edu/brown/benchmark/articles/procedures/GetComments.java
  25. +19 −0 src/benchmarks/edu/brown/benchmark/articles/procedures/UpdateUserInfo.java
  26. +164 −0 src/benchmarks/edu/brown/benchmark/users/UsersClient.java
  27. +23 −0 src/benchmarks/edu/brown/benchmark/users/UsersConstants.java
  28. +83 −0 src/benchmarks/edu/brown/benchmark/users/UsersLoader.java
  29. +35 −0 src/benchmarks/edu/brown/benchmark/users/UsersProjectBuilder.java
  30. +94 −0 src/benchmarks/edu/brown/benchmark/users/UsersUtil.java
  31. +19 −0 src/benchmarks/edu/brown/benchmark/users/procedures/GetComments.java
  32. +46 −0 src/benchmarks/edu/brown/benchmark/users/procedures/GetUsers.java
  33. +32 −0 src/benchmarks/edu/brown/benchmark/users/users-ddl.sql
  34. +2 −0 src/benchmarks/edu/brown/benchmark/users/users.mappings
  35. +1 −0 src/catgen/spec.txt
  36. +79 −19 src/ee/anticache/AntiCacheDB.cpp
  37. +77 −9 src/ee/anticache/AntiCacheDB.h
  38. +763 −7 src/ee/anticache/AntiCacheEvictionManager.cpp
  39. +8 −2 src/ee/anticache/AntiCacheEvictionManager.h
  40. +0 −1 src/ee/anticache/EvictedTable.cpp
  41. +4 −2 src/ee/anticache/EvictedTupleAccessException.cpp
  42. +2 −1 src/ee/anticache/EvictedTupleAccessException.h
  43. +9 −2 src/ee/anticache/UnknownBlockAccessException.cpp
  44. +1 −0 src/ee/anticache/UnknownBlockAccessException.h
  45. +6 −0 src/ee/catalog/table.cpp
  46. +3 −0 src/ee/catalog/table.h
  47. +17 −14 src/ee/common/NValue.hpp
  48. +7 −6 src/ee/common/executorcontext.hpp
  49. +90 −89 src/ee/common/tabletuple.h
  50. +35 −4 src/ee/execution/VoltDBEngine.cpp
  51. +6 −3 src/ee/execution/VoltDBEngine.h
  52. +8 −4 src/ee/executors/indexscanexecutor.cpp
  53. +1 −0 src/ee/storage/TableCatalogDelegate.cpp
  54. +188 −341 src/ee/storage/persistenttable.cpp
  55. +37 −6 src/ee/storage/persistenttable.h
  56. +25 −0 src/ee/voltdbjni.cpp
  57. +18 −0 src/frontend/edu/brown/catalog/CatalogUtil.java
  58. +98 −32 src/frontend/edu/brown/hstore/AntiCacheManager.java
  59. +111 −6 src/frontend/edu/brown/hstore/HStoreCoordinator.java
  60. +31 −5 src/frontend/edu/brown/hstore/HStoreSite.java
  61. +15 −0 src/frontend/edu/brown/hstore/PartitionExecutor.java
  62. +22 −3 src/frontend/edu/brown/hstore/TransactionInitializer.java
  63. +2 −2 src/frontend/edu/brown/hstore/TransactionQueueManager.java
  64. +3 −3 src/frontend/edu/brown/hstore/callbacks/RemoteFinishCallback.java
  65. +5 −2 src/frontend/edu/brown/hstore/callbacks/RemoteWorkCallback.java
  66. +7 −0 src/frontend/edu/brown/hstore/conf/HStoreConf.java
  67. +83 −0 src/frontend/edu/brown/hstore/txns/AbstractTransaction.java
  68. +10 −64 src/frontend/edu/brown/hstore/txns/LocalTransaction.java
  69. +14 −0 src/frontend/edu/brown/hstore/txns/RemoteTransaction.java
  70. +2 −0 src/frontend/edu/brown/utils/ProjectType.java
  71. +190 −0 src/frontend/edu/brown/workload/WorkloadAnalyzer.java
  72. +1 −1 src/frontend/org/voltdb/CatalogContext.java
  73. +1 −0 src/frontend/org/voltdb/ClientResponseImpl.java
  74. +5 −1 src/frontend/org/voltdb/VoltProcedure.java
  75. +13 −0 src/frontend/org/voltdb/catalog/Table.java
  76. +1 −0 src/frontend/org/voltdb/client/ClientImpl.java
  77. +1 −0 src/frontend/org/voltdb/compiler/ProjectFileSchema.xsd
  78. +22 −1 src/frontend/org/voltdb/compiler/VoltCompiler.java
  79. +26 −2 src/frontend/org/voltdb/compiler/VoltProjectBuilder.java
  80. +2 −2 src/frontend/org/voltdb/compiler/projectfile/ClassdependenciesType.java
  81. +30 −4 src/frontend/org/voltdb/compiler/projectfile/DatabaseType.java
  82. +3 −3 src/frontend/org/voltdb/compiler/projectfile/EvictablesType.java
  83. +7 −7 src/frontend/org/voltdb/compiler/projectfile/ExportsType.java
  84. +5 −5 src/frontend/org/voltdb/compiler/projectfile/GroupsType.java
  85. +70 −70 src/frontend/org/voltdb/compiler/projectfile/ObjectFactory.java
  86. +4 −4 src/frontend/org/voltdb/compiler/projectfile/PartitionsType.java
  87. +7 −7 src/frontend/org/voltdb/compiler/projectfile/ProceduresType.java
  88. +2 −2 src/frontend/org/voltdb/compiler/projectfile/ProjectType.java
  89. +3 −3 src/frontend/org/voltdb/compiler/projectfile/SchemasType.java
  90. +3 −3 src/frontend/org/voltdb/compiler/projectfile/SecurityType.java
  91. +6 −6 src/frontend/org/voltdb/compiler/projectfile/SnapshotType.java
  92. +7 −7 src/frontend/org/voltdb/compiler/projectfile/UsersType.java
  93. +4 −4 src/frontend/org/voltdb/compiler/projectfile/VerticalpartitionsType.java
  94. +9 −1 src/frontend/org/voltdb/exceptions/EvictedTupleAccessException.java
  95. +1 −16 src/frontend/org/voltdb/exceptions/UnknownBlockAccessException.java
  96. +20 −2 src/frontend/org/voltdb/jni/ExecutionEngine.java
  97. +7 −0 src/frontend/org/voltdb/jni/ExecutionEngineIPC.java
  98. +30 −1 src/frontend/org/voltdb/jni/ExecutionEngineJNI.java
  99. +10 −0 src/frontend/org/voltdb/jni/MockExecutionEngine.java
  100. +25 −4 src/frontend/org/voltdb/sysprocs/EvictTuples.java
  101. +1,196 −52 src/protorpc/edu/brown/hstore/Hstoreservice.java
  102. +22 −0 src/protorpc/edu/brown/hstore/hstoreservice.proto
  103. +23 −2 tests/ee/anticache/anticachedb_test.cpp
  104. +21 −0 tests/frontend/edu/brown/catalog/TestCatalogUtil2.java
  105. +25 −0 tests/frontend/edu/brown/hstore/MockAntiCacheManager.java
  106. +27 −0 tests/frontend/edu/brown/hstore/MockHStoreCoordinator.java
  107. +45 −0 tests/frontend/edu/brown/hstore/MockHStoreSite.java
  108. +406 −0 tests/frontend/edu/brown/hstore/TestAntiCacheBatching.java
  109. +2 −2 tests/frontend/edu/brown/hstore/TestAntiCacheManager.java
  110. +398 −0 tests/frontend/edu/brown/hstore/TestAntiCacheManagerDistributedTxn.java
  111. +0 −1 tests/frontend/edu/brown/hstore/TestAntiCacheManagerTPCC.java
  112. +13 −13 tests/frontend/edu/brown/hstore/TestAntiCachePerformance.java
  113. +68 −23 tests/frontend/edu/brown/hstore/TestHStoreCoordinator.java
  114. +29 −0 tests/frontend/edu/brown/workload/TestWorkloadAnalyzer.java
  115. +5 −5 tests/frontend/org/voltdb/regressionsuites/LocalCluster.java
  116. +3 −1 tests/frontend/org/voltdb/regressionsuites/TestAntiCacheSuite.java
  117. +181 −0 tests/frontend/org/voltdb/regressionsuites/TestUsersSuite.java
View
@@ -155,6 +155,7 @@
<arg value="site.anticache_max_evicted_blocks=${site.anticache_max_evicted_blocks}" />
<arg value="site.anticache_blocks_per_eviction=${site.anticache_blocks_per_eviction}" />
<arg value="site.anticache_eviction_distribution=${site.anticache_eviction_distribution}" />
<arg value="site.anticache_batching=${site.anticache_batching}" />
<arg value="site.storage_mmap=${site.storage_mmap}" />
<arg value="site.storage_mmap_dir=${site.storage_mmap_dir}" />
<arg value="site.storage_mmap_file_size=${site.storage_mmap_file_size}" />
View
@@ -139,10 +139,24 @@
-->
<!-- Default Project -->
<!--<condition property="project" value="tpcc">
<condition property="project" value="auctionmark">
<not><isset property="project"/></not>
</condition>-->
</condition>
<condition property="target.workload" value="${project}-1">
<not><isset property="target.workload"/></not>
</condition>
<condition property="workload" value="${files.workloads.dir}/${target.workload}.trace.gz">
<not><isset property="workload"/></not>
</condition>
<condition property="stats" value="${benchmark.tablestats}">
<not><isset property="stats"/></not>
</condition>
<condition property="output" value="${files.stats.dir}/${project}.stats">
<not><isset property="output"/></not>
</condition>
<!-- Project Jar File -->
<property name="benchmark.jar" location="${project}.jar" />
<condition property="jar" value="${benchmark.jar}">
@@ -181,6 +195,11 @@
<isset property="regressions"/>
</condition>
<condition property="junitclass" value="TestAntiCacheBatching">
<not><isset property="junitclass"/></not>
</condition>
<!-- Workload Tracer Properties -->
<condition property="workload.trace.class" value="">
<not><isset property="workload.trace.class"/></not>
@@ -1716,7 +1735,7 @@ TEST CASES
<arg value="CATALOG=${catalog}" />
<arg value="DEFERRABLE=${deferrable}" />
<arg value="EVICTABLE=${evictable}" />
<arg value="BATCHEVICTABLE=${batchevictable}" />
<arg value="partitionplan=${partitionplan}" />
<arg value="partitionplan.nosecondary=${partitionplan.nosecondary}" />
<arg value="partitionplan.ignore_missing=${partitionplan.ignore_missing}" />
@@ -2069,6 +2088,24 @@ UTILITIES
</java>
</target>
<target name='workload-analyzer'
description="analyze workload">
<java fork="yes" classname="edu.brown.workload.WorkloadAnalyzer" failonerror='true'>
<jvmarg value="-Xmx${global.memory}m" />
<jvmarg value="-Dlog4j.configuration=${basedir}/log4j.properties"/>
<arg value="catalog.jar=${jar}" />
<arg value="workload=${workload}" />
<arg value="workload.output=${output}" />
<arg value="workload.xactlimit=${limit}" />
<arg value="workload.xactoffset=${offset}" />
<arg value="workload.procexclude=${exclude}" />
<arg value="workload.procinclude=${include}" />
<arg value="workload.procinclude.multiplier=${multiplier}" />
<classpath refid='project.classpath' />
<assertions><enable /></assertions>
</java>
</target>
<!--
******************************************************************************
* MARKOV MODELS
@@ -0,0 +1,6 @@
## ------------------------------------------------------------
## H-Store Benchmark Configuration Parameters
## ------------------------------------------------------------
builder = edu.brown.benchmark.articles.ArticlesProjectBuilder
datadir = src/benchmarks/edu/brown/benchmark/articles/data
@@ -7,12 +7,12 @@ workload.ignore = LoadWarehouse,LoadWarehouseReplicated
## These parameters will make the benchmark use one WAREHOUSE
## and one loader thread per partition in the cluster
warehouse_per_partition = true
warehouse_per_partition = false
loadthread_per_warehouse = true
## Or you can use these parameters to specifically
## control the number of warehouses and/or load threads
# warehouses = 4
warehouses = 100
# loadthreads = 4
## Testing Options
@@ -42,7 +42,7 @@ neworder_only = false
# Whether to allow NewOrder txns to touch multiple partitions
# You can control the percentage of these txns with neworder_multip_mix
neworder_multip = true
neworder_multip = false
# The percentage of NewOrder txns that touch multiple partitions.
# Setting it to -1 will use the default configuration
@@ -64,7 +64,7 @@ neworder_skew_warehouse = false
payment_only = false
# Whether to allow Payment txns to touch multiple partitions.
payment_multip = true
payment_multip = false
# The percentage of Payment txns that touch multiple partitions.
# Setting it to -1 will use the default configuration
@@ -0,0 +1,6 @@
## ------------------------------------------------------------
## H-Store Benchmark Configuration Parameters
## ------------------------------------------------------------
builder = edu.brown.benchmark.users.UsersProjectBuilder
datadir = src/benchmarks/edu/brown/benchmark/users/data
@@ -10,4 +10,4 @@ fixed_size = false
num_records = 100000
# Zipfian skew factor for tuple access
skew_factor = 0.5
skew_factor = 0.5
@@ -0,0 +1,188 @@
#!/bin/bash
# ---------------------------------------------------------------------
trap onexit 1 2 3 15
function onexit() {
local exit_status=${1:-$?}
pkill -f hstore.tag
exit $exit_status
}
# ---------------------------------------------------------------------
ENABLE_ANTICACHE=true
SITE_HOST="localhost"
CLIENT_HOSTS=( \
"localhost" \
)
BASE_CLIENT_THREADS=1
BASE_SITE_MEMORY=8192
BASE_SITE_MEMORY_PER_PARTITION=1024
BASE_PROJECT="articles"
BASE_DIR=`pwd`
OUTPUT_DIR="~/data/articles/read-heavy/2/80-20"
ANTICACHE_BLOCK_SIZE=500000
ANTICACHE_THRESHOLD=.5
BASE_ARGS=( \
# SITE DEBUG
"-Dsite.status_enable=false" \
"-Dsite.status_interval=10000" \
"-Dsite.txn_restart_limit=35" \
# "-Dsite.status_exec_info=true" \
# "-Dsite.status_check_for_zombies=true" \
# "-Dsite.exec_profiling=true" \
# "-Dsite.profiling=true" \
# "-Dsite.txn_counters=true" \
# "-Dsite.pool_profiling=true" \
# "-Dsite.network_profiling=false" \
# "-Dsite.log_backup=true"\
# "-Dnoshutdown=true" \
# Site Params
"-Dsite.jvm_asserts=false" \
"-Dsite.specexec_enable=false" \
"-Dsite.cpu_affinity_one_partition_per_core=true" \
#"-Dsite.cpu_partition_blacklist=0,2,4,6,8,10,12,14,16,18" \
#"-Dsite.cpu_utility_blacklist=0,2,4,6,8,10,12,14,16,18" \
#"-Dsite.network_incoming_limit_txns=50000" \
#"-Dsite.commandlog_enable=true" \
#"-Dsite.txn_incoming_delay=5" \
#"-Dsite.exec_postprocessing_threads=true" \
"-Dsite.anticache_profiling=${ENABLE_ANTICACHE}" \
"-Dsite.anticache_eviction_distribution=even" \
# "-Dsite.queue_allow_decrease=true" \
# "-Dsite.queue_allow_increase=true" \
# "-Dsite.queue_threshold_factor=0.5" \
# Client Params
"-Dclient.scalefactor=1" \
"-Dclient.memory=2048" \
"-Dclient.txnrate=2000" \
"-Dclient.warmup=10000" \
"-Dclient.duration=120000" \
"-Dclient.interval=5000" \
"-Dclient.shared_connection=false" \
"-Dclient.blocking=true" \
"-Dclient.blocking_concurrent=1" \
"-Dclient.throttle_backoff=100" \
"-Dclient.output_interval=5000" \
# "-Dclient.output_anticache_evictions=evictions.csv" \
# "-Dclient.output_memory=memory.csv" \
# Anti-Caching Experiments
"-Dsite.anticache_enable=${ENABLE_ANTICACHE}" \
#"-Dsite.anticache_batching=true" \
# "-Dsite.anticache_profiling=true" \
"-Dsite.anticache_reset=false" \
"-Dsite.anticache_block_size=${ANTICACHE_BLOCK_SIZE}" \
"-Dsite.anticache_check_interval=30000" \
"-Dsite.anticache_threshold_mb=5" \
"-Dsite.anticache_blocks_per_eviction=16" \
"-Dsite.anticache_max_evicted_blocks=1600" \
# "-Dsite.anticache_evict_size=${ANTICACHE_EVICT_SIZE}" \
"-Dsite.anticache_threshold=${ANTICACHE_THRESHOLD}" \
"-Dclient.anticache_enable=false" \
"-Dclient.anticache_evict_interval=10000" \
"-Dclient.anticache_evict_size=102400" \
"-Dclient.output_csv=results.csv" \
# CLIENT DEBUG
"-Dclient.output_txn_counters=txncounters.csv" \
"-Dclient.output_clients=false" \
"-Dclient.profiling=false" \
"-Dclient.output_response_status=false" \
# "-Dclient.output_queue_profiling=${BASE_PROJECT}-queue.csv" \
# "-Dclient.output_basepartitions=true" \
# "-Dclient.jvm_args=\"-verbose:gc -XX:+PrintGCDetails -XX:+PrintGCTimeStamps -XX:-TraceClassUnloading\""
)
EVICTABLE_TABLES=( \
"ARTICLES" \
"COMMENTS" \
)
BATCHEVICTABLE_TABLES=( \
"COMMENTS" \
)
EVICTABLES=""
BATCHEVICTABLES=""
if [ "$ENABLE_ANTICACHE" = "true" ]; then
for t in ${EVICTABLE_TABLES[@]}; do
EVICTABLES="${t},${EVICTABLES}"
done
for t in ${BATCHEVICTABLE_TABLES[@]}; do
BATCHEVICTABLES="${t},${BATCHEVICTABLES}"
done
fi
# Compile
HOSTS_TO_UPDATE=("$SITE_HOST")
for CLIENT_HOST in ${CLIENT_HOSTS[@]}; do
NEED_UPDATE=1
for x in ${HOSTS_TO_UPDATE[@]}; do
if [ "$CLIENT_HOST" = "$x" ]; then
NEED_UPDATE=0
break
fi
done
if [ $NEED_UPDATE = 1 ]; then
HOSTS_TO_UPDATE+=("$CLIENT_HOST")
fi
done
#for HOST in ${HOSTS_TO_UPDATE[@]}; do
# ssh $HOST "cd $BASE_DIR && git pull && ant compile" &
#done
wait
ant compile
HSTORE_HOSTS="${SITE_HOST}:0:0-7"
NUM_CLIENTS=`expr 8 \* $BASE_CLIENT_THREADS`
SITE_MEMORY=`expr $BASE_SITE_MEMORY + \( 8 \* $BASE_SITE_MEMORY_PER_PARTITION \)`
# BUILD PROJECT JAR
ant hstore-prepare \
-Dproject=${BASE_PROJECT} \
-Dhosts=${HSTORE_HOSTS} \
-Devictable=${EVICTABLES}
#-Dbatchevictable=${BATCHEVICTABLES} \
test -f ${BASE_PROJECT}.jar || exit -1
# UPDATE CLIENTS
CLIENT_COUNT=0
CLIENT_HOSTS_STR=""
for CLIENT_HOST in ${CLIENT_HOSTS[@]}; do
CLIENT_COUNT=`expr $CLIENT_COUNT + 1`
if [ ! -z "$CLIENT_HOSTS_STR" ]; then
CLIENT_HOSTS_STR="${CLIENT_HOSTS_STR},"
fi
CLIENT_HOSTS_STR="${CLIENT_HOSTS_STR}${CLIENT_HOST}"
done
# DISTRIBUTE PROJECT JAR
for HOST in ${HOSTS_TO_UPDATE[@]}; do
if [ "$HOST" != $(hostname) ]; then
scp -r ${BASE_PROJECT}.jar ${HOST}:${BASE_DIR} &
fi
done
wait
# EXECUTE BENCHMARK
ant hstore-benchmark ${BASE_ARGS[@]} \
-Dproject=${BASE_PROJECT} \
-Dkillonzero=false \
-Dclient.threads_per_host=${NUM_CLIENTS} \
-Dsite.memory=${SITE_MEMORY} \
-Dclient.hosts=${CLIENT_HOSTS_STR} \
-Dclient.count=${CLIENT_COUNT}
#-Dnoloader=true \
#-Dnostart=true \
result=$?
if [ $result != 0 ]; then
exit $result
fi
View
2 scripts/anticache/tpcc.sh 100755 → 100644
@@ -178,4 +178,4 @@ result=$?
if [ $result != 0 ]; then
exit $result
fi
done
done
Oops, something went wrong.

0 comments on commit e519245

Please sign in to comment.