Applying anti-cache patch.

apavlo · Jul 23, 2013 · e33d394 · e33d394
1 parent 2d61087
commit e33d394
Show file tree

Hide file tree

Showing 36 changed files with 1,689 additions and 615 deletions.
diff --git a/build-common.xml b/build-common.xml
@@ -81,6 +81,7 @@
 <arg value="client.output_planner_profiling_combine=${client.output_planner_profiling_combine}" />
 <arg value="client.output_markov_profiling=${client.output_markov_profiling}" />
 <arg value="client.output_table_stats=${client.output_table_stats}" />
+<arg value="client.output_table_stats_periodically=${client.output_table_stats_periodically}" />
 <arg value="client.output_txn_counters=${client.output_txn_counters}" />
 <arg value="client.output_txn_counters_combine=${client.output_txn_counters_combine}" />
 <arg value="client.profiling=${client.profiling}" />
@@ -146,7 +147,11 @@
 <arg value="site.anticache_block_size=${site.anticache_block_size}" />
 <arg value="site.anticache_reset=${site.anticache_reset}" />
 <arg value="site.anticache_check_interval=${site.anticache_check_interval}" />
-<arg value="site.anticache_threshold=${site.anticache_threshold}" />
+<arg value="site.anticache_threshold_mb=${site.anticache_threshold_mb}" />
+<arg value="site.anticache_threshold_ratio=${site.anticache_threshold_ratio}" />
+<arg value="site.anticache_max_evicted_blocks=${site.anticache_max_evicted_blocks}" />
+<arg value="site.anticache_blocks_per_eviction=${site.anticache_blocks_per_eviction}" />
+<arg value="site.anticache_eviction_distribution=${site.anticache_eviction_distribution}" />
 <arg value="site.mr_map_blocking=${site.mr_map_blocking}" />
 <arg value="site.mr_reduce_blocking=${site.mr_reduce_blocking}" />
 <arg value="site.network_heartbeats_interval=${site.network_heartbeats_interval}" />

diff --git a/build.py b/build.py
@@ -366,6 +366,11 @@
 # ANTI-CACHING
 ###############################################################################
 ENABLE_ANTICACHE = True
+DOUBLE_LINKED_LIST = True
+
+if DOUBLE_LINKED_LIST:
+    CTX.CPPFLAGS += " -DDOUBLE_LINKED_LIST"
+
 if ENABLE_ANTICACHE:
     CTX.CPPFLAGS += " -DANTICACHE"
     CTX.SYSTEM_DIRS.append(os.path.join(CTX.OUTPUT_PREFIX, 'berkeleydb'))

diff --git a/scripts/anticache/ycsb.sh b/scripts/anticache/ycsb.sh
@@ -11,14 +11,14 @@ function onexit() {
 
 # ---------------------------------------------------------------------
 
-ENABLE_ANTICACHE=true
+ENABLE_ANTICACHE=false
 
-SITE_HOST="modis"
+SITE_HOST="istc12"
 
 CLIENT_HOSTS=( \
-        "modis2" \
-        "modis2" \
-        "modis" \
+        "istc12" \
+        "istc13" \
+        "istc13" \
 )
 
 BASE_CLIENT_THREADS=2
@@ -28,14 +28,17 @@ BASE_PROJECT="ycsb"
 BASE_DIR=`pwd`
 OUTPUT_DIR="~/data/ycsb/read-heavy/2/80-20"
 
-ANTICACHE_BLOCK_SIZE=524288
-#ANTICACHE_EVICT_SIZE=268400000
+#ANTICACHE_BLOCK_SIZE=524288
+ANTICACHE_BLOCK_SIZE=1048576
+#ANTICACHE_BLOCK_SIZE=2097152
+#ANTICACHE_BLOCK_SIZE=4194304
+#ANTICACHE_BLOCK_SIZE=131072
 ANTICACHE_THRESHOLD=.5
 
 BASE_ARGS=( \
     # SITE DEBUG
      "-Dsite.status_enable=false" \
-     "-Dsite.status_interval=20000" \
+     "-Dsite.status_interval=10000" \
 #    "-Dsite.status_exec_info=true" \
 #    "-Dsite.status_check_for_zombies=true" \
 #    "-Dsite.exec_profiling=true" \
@@ -56,7 +59,8 @@ BASE_ARGS=( \
     "-Dsite.commandlog_enable=true" \
     "-Dsite.txn_incoming_delay=5" \
     "-Dsite.exec_postprocessing_threads=true" \
-    "-Dsite.anticache_profiling=true" \
+    "-Dsite.anticache_profiling=${ENABLE_ANTICACHE}" \
+    "-Dsite.anticache_eviction_distribution=even" \
 
 #    "-Dsite.queue_allow_decrease=true" \
 #    "-Dsite.queue_allow_increase=true" \
@@ -65,26 +69,31 @@ BASE_ARGS=( \
     # Client Params
     "-Dclient.scalefactor=1" \
     "-Dclient.memory=2048" \
-    "-Dclient.txnrate=1000" \
+    "-Dclient.txnrate=2000" \
     "-Dclient.warmup=120000" \
     "-Dclient.duration=120000" \
-    "-Dclient.interval=20000" \
+    "-Dclient.interval=5000" \
     "-Dclient.shared_connection=false" \
     "-Dclient.blocking=false" \
     "-Dclient.blocking_concurrent=100" \
     "-Dclient.throttle_backoff=100" \
     "-Dclient.output_interval=5000" \
-    "-Dclient.output_anticache_evictions=evictions.csv" \
-
+#    "-Dclient.output_anticache_evictions=evictions.csv" \
+#    "-Dclient.output_memory=memory.csv" \
+
     # Anti-Caching Experiments
     "-Dsite.anticache_enable=${ENABLE_ANTICACHE}" \
+#    "-Dsite.anticache_profiling=true" \
+    "-Dsite.anticache_reset=false" \
     "-Dsite.anticache_block_size=${ANTICACHE_BLOCK_SIZE}" \
     "-Dsite.anticache_check_interval=10000" \
+    "-Dsite.anticache_threshold_mb=200" \
+    "-Dsite.anticache_blocks_per_eviction=200" \
+    "-Dsite.anticache_max_evicted_blocks=325" \
 #    "-Dsite.anticache_evict_size=${ANTICACHE_EVICT_SIZE}" \
     "-Dsite.anticache_threshold=${ANTICACHE_THRESHOLD}" \
-#    "-Dclient.interval=500" \
     "-Dclient.anticache_enable=false" \
-    "-Dclient.anticache_evict_interval=2000" \
+    "-Dclient.anticache_evict_interval=10000" \
     "-Dclient.anticache_evict_size=102400" \
     "-Dclient.output_csv=results.csv" \
 
@@ -99,7 +108,7 @@ BASE_ARGS=( \
 )
 
 EVICTABLE_TABLES=( \
-    "usertable" \
+    "USERTABLE" \
 )
 EVICTABLES=""
 if [ "$ENABLE_ANTICACHE" = "true" ]; then
@@ -128,7 +137,7 @@ done
 wait
 
 ant compile
-for i in 6; do
+for i in 8; do
 
     HSTORE_HOSTS="${SITE_HOST}:0:0-"`expr $i - 1`
     NUM_CLIENTS=`expr $i \* $BASE_CLIENT_THREADS`

diff --git a/src/benchmarks/edu/brown/api/BenchmarkController.java b/src/benchmarks/edu/brown/api/BenchmarkController.java
@@ -110,6 +110,7 @@
 import edu.brown.api.results.CSVResultsPrinter;
 import edu.brown.api.results.JSONResultsPrinter;
 import edu.brown.api.results.MemoryStatsPrinter;
+import edu.brown.api.results.TableStatsPrinter;
 import edu.brown.api.results.ResponseEntries;
 import edu.brown.api.results.ResultsChecker;
 import edu.brown.api.results.ResultsPrinter;
@@ -946,6 +947,15 @@ public void run() {
             File f = new File(hstore_conf.client.output_memory);
             this.registerInterest(new MemoryStatsPrinter(this.getClientConnection(), f));
         }
+
+        // Table Stats Output
+        if (hstore_conf.client.output_table_stats_periodically != null && hstore_conf.client.output_table_stats_periodically.isEmpty() == false) {
+            if (hstore_conf.client.output_table_stats_periodically.equalsIgnoreCase("true")) {
+                LOG.warn("The HStoreConf parameter 'hstore_conf.client.output_table_stats_periodically' should be a file path, not a boolean value");
+            }
+            File f = new File(hstore_conf.client.output_table_stats_periodically);
+            this.registerInterest(new TableStatsPrinter(this.getClientConnection(), f));
+        }
 
         // Kill Benchmark on Zero Results
         if (m_config.killOnZeroResults) {

diff --git a/src/benchmarks/edu/brown/api/results/MemoryStatsPrinter.java b/src/benchmarks/edu/brown/api/results/MemoryStatsPrinter.java
@@ -85,7 +85,6 @@ public String formatFinalResults(BenchmarkResults br) {
     @Override
     public void benchmarkHasUpdated(final BenchmarkResults br) {
         if (this.stop) return;
-
         final int interval = this.intervalCounter++;
 
         // HACK: Skip every other interval
@@ -103,7 +102,6 @@ public void clientCallback(ClientResponse clientResponse) {
 
                 // TOTAL STATS FROM ALL SITES 
                 VoltTable vt = clientResponse.getResults()[0];
-
                 long totals[] = new long[vt.getColumnCount()];
                 Arrays.fill(totals, 0);
                 int colOffsets[] = null;

diff --git a/src/benchmarks/edu/brown/benchmark/ycsb/YCSBClient.java b/src/benchmarks/edu/brown/benchmark/ycsb/YCSBClient.java
@@ -38,7 +38,6 @@
 import org.voltdb.client.ProcedureCallback;
 
 import edu.brown.api.BenchmarkComponent;
-import edu.brown.benchmark.ycsb.distributions.CustomSkewGenerator;
 import edu.brown.benchmark.ycsb.distributions.ZipfianGenerator;
 import edu.brown.logging.LoggerUtil;
 import edu.brown.logging.LoggerUtil.LoggerBoolean;
@@ -82,15 +81,18 @@ private Transaction(String displayName, int weight) {
     } // TRANSCTION ENUM
 
     private final long init_record_count;
-    private final CustomSkewGenerator readRecord; 
+//    private final CustomSkewGenerator readRecord;
+    private final ZipfianGenerator readRecord;
     @SuppressWarnings("unused")
-    private final CustomSkewGenerator insertRecord;
+    private final ZipfianGenerator insertRecord;
+//    private final CustomSkewGenerator insertRecord;
     private final ZipfianGenerator randScan;
     private final FlatHistogram<Transaction> txnWeights;
     private final Random rand_gen;
 
+    int run_count = 0; 
+
     // private ZipfianGenerator readRecord;
-    // private final double SIGMA = 1.4; 
 
     public static void main(String args[]) {
         BenchmarkComponent.main(YCSBClient.class, args, false);
@@ -133,16 +135,18 @@ else if (key.equalsIgnoreCase("num_records")) {
         this.rand_gen = new Random(); 
         this.randScan = new ZipfianGenerator(YCSBConstants.MAX_SCAN);
 
-        // initialize distribution generators 
-        // We must know where to start inserting
-        this.insertRecord = new CustomSkewGenerator(this.rand_gen, this.init_record_count, 
-                                            YCSBConstants.HOT_DATA_WORKLOAD_SKEW, YCSBConstants.HOT_DATA_SIZE, 
-                                            YCSBConstants.WARM_DATA_WORKLOAD_SKEW, YCSBConstants.WARM_DATA_SIZE);
-
-        this.readRecord = new CustomSkewGenerator(this.rand_gen, this.init_record_count, 
-                                            YCSBConstants.HOT_DATA_WORKLOAD_SKEW, YCSBConstants.HOT_DATA_SIZE, 
-                                            YCSBConstants.WARM_DATA_WORKLOAD_SKEW, YCSBConstants.WARM_DATA_SIZE);
+//        // initialize distribution generators 
+//        // We must know where to start inserting
+//        this.insertRecord = new CustomSkewGenerator(this.rand_gen, this.init_record_count, 
+//                                            YCSBConstants.HOT_DATA_WORKLOAD_SKEW, YCSBConstants.HOT_DATA_SIZE, 
+//                                            YCSBConstants.WARM_DATA_WORKLOAD_SKEW, YCSBConstants.WARM_DATA_SIZE);
+//
+//        this.readRecord = new CustomSkewGenerator(this.rand_gen, this.init_record_count, 
+//                                            YCSBConstants.HOT_DATA_WORKLOAD_SKEW, YCSBConstants.HOT_DATA_SIZE, 
+//                                            YCSBConstants.WARM_DATA_WORKLOAD_SKEW, YCSBConstants.WARM_DATA_SIZE);
 
+        this.insertRecord = new ZipfianGenerator(YCSBConstants.NUM_RECORDS, YCSBConstants.ZIPFIAN_CONSTANT);
+        this.readRecord = new ZipfianGenerator(YCSBConstants.NUM_RECORDS, YCSBConstants.ZIPFIAN_CONSTANT);
 
         // Initialize the sampling table
         Histogram<Transaction> txns = new ObjectHistogram<Transaction>(); 
@@ -164,12 +168,9 @@ public void runLoop() {
             Random rand = new Random();
             int key = -1; 
             int scan_count; 
-
-            int run_count = 0; 
-
             while (true) {
-
-                runOnce(); 
+                runOnce();
+                this.run_count++; 
             } 
         } 
         catch (IOException e) {
@@ -191,7 +192,7 @@ protected boolean runOnce() throws IOException {
             }
             case UPDATE_RECORD:
             case INSERT_RECORD: {
-                int key = this.readRecord.nextInt();
+                int key = this.insertRecord.nextInt();
                 String fields[] = new String[YCSBConstants.NUM_COLUMNS];
                 for (int i = 0; i < fields.length; i++) {
                     fields[i] = YCSBUtil.astring(YCSBConstants.COLUMN_LENGTH, YCSBConstants.COLUMN_LENGTH);

diff --git a/src/benchmarks/edu/brown/benchmark/ycsb/YCSBConstants.java b/src/benchmarks/edu/brown/benchmark/ycsb/YCSBConstants.java
@@ -32,9 +32,11 @@
 public abstract class YCSBConstants {
 
     public static final int NUM_RECORDS = 20000000;  // Note: this should match value in YCSB.properties
+
+    public static final double ZIPFIAN_CONSTANT = .5;
 
-    public static final int HOT_DATA_WORKLOAD_SKEW = 90;
-    public static final int HOT_DATA_SIZE = 10;
+    public static final int HOT_DATA_WORKLOAD_SKEW = 50;
+    public static final int HOT_DATA_SIZE = 50;
 
     public static final int WARM_DATA_SIZE = 0;
     public static final int WARM_DATA_WORKLOAD_SKEW = 0;
@@ -48,9 +50,9 @@ public abstract class YCSBConstants {
     public static final int MAX_SCAN = 1000; 
 
     // Transaction frequencies as specified in YCSB
-    public static final int FREQUENCY_INSERT_RECORD = 0; 
+    public static final int FREQUENCY_INSERT_RECORD = 0;
     public static final int FREQUENCY_DELETE_RECORD = 0;
-    public static final int FREQUENCY_READ_RECORD = 100;
+    public static final int FREQUENCY_READ_RECORD = 50;
     public static final int FREQUENCY_SCAN_RECORD = 0;
-    public static final int FREQUENCY_UPDATE_RECORD = 0;
+    public static final int FREQUENCY_UPDATE_RECORD = 50;
 }
diff --git a/src/benchmarks/org/voltdb/benchmark/tpcc/TPCCConstants.java b/src/benchmarks/org/voltdb/benchmark/tpcc/TPCCConstants.java
@@ -94,12 +94,12 @@ public final class TPCCConstants {
     // 2 digits after the decimal point for money types
     public static final int MONEY_DECIMALS = 2;
 
-	// skew constants
-    public static final int HOT_DATA_WORKLOAD_SKEW = 50;
-	public static final int HOT_DATA_SIZE = 50;
-	public static final int WARM_DATA_SIZE = 0;
-	public static final int WARM_DATA_WORKLOAD_SKEW = 0;
-	
+        // skew constants
+    public static final int HOT_DATA_WORKLOAD_SKEW = 100;
+        public static final int HOT_DATA_SIZE = 0;
+        public static final int WARM_DATA_SIZE = 0;
+        public static final int WARM_DATA_WORKLOAD_SKEW = 0;
+        
 	// Percentage of neworder txns that will abort
 	public static final int NEWORDER_ABORT = 1; // 1%
 

diff --git a/src/ee/anticache/AntiCacheDB.cpp b/src/ee/anticache/AntiCacheDB.cpp
@@ -119,7 +119,7 @@ void AntiCacheDB::writeBlock(const std::string tableName,
 
 void AntiCacheDB::flushBlocks()
 {
-    m_db->sync(0); 
+//    m_db->sync(0); 
 }
 
 AntiCacheBlock AntiCacheDB::readBlock(std::string tableName, int16_t blockId) {