Permalink
Browse files

Merge remote-tracking branch 'upstream/master'

  • Loading branch information...
malin1993ml committed Apr 12, 2016
2 parents 393867d + 4351196 commit 550e8dcb7ae42d63e9cb2ade26463b4cbe6c64a4
@@ -0,0 +1,24 @@
#!/bin/bash
NUM_PARTITIONS=32
for b in "tpcc" "seats" "smallbank" ; do # "seats" "smallbank"
echo $b
ant hstore-prepare -Dproject=$b -Dhosts=localhost:0:0-$(expr $NUM_PARTITIONS - 1)
if [ $b == "tpcc" ]; then
workload=files/workloads/$b.${NUM_PARTITIONS}p-1.trace.gz
else
workload=files/workloads/$b-1.trace.gz
fi
for limit in 100 1000 10000 100000; do
echo $limit
ant markov-generate -Dproject=$b \
-Dnumcpus=8 \
-Dglobal=false \
-Dworkload=$workload \
-Doutput=/tmp/$b.markov \
-Dlimit=$limit
done
done
@@ -605,7 +605,7 @@ public void fromJSON(JSONObject json_object, Database catalog_db) throws JSONExc
}
// Target PartitionPlan
if (this.target_plan_path != null && this.target_plan_path != null) {
if (this.target_plan_path != null) {
if (debug.val)
LOG.debug("Loading in target PartitionPlan from '" + this.target_plan_path + "'");
this.target_plan = new PartitionPlan();
@@ -3107,23 +3107,26 @@ else if (ts.getRestartCounter() == 0) {
if (ts.isSpeculative() && status != Status.ABORT_SPECULATIVE) {
TransactionCounter.SPECULATIVE.inc(catalog_proc);
switch (ts.getSpeculationType()) {
case IDLE:
TransactionCounter.SPECULATIVE_IDLE.inc(catalog_proc);
case SP1_IDLE:
TransactionCounter.SPECULATIVE_SP1_IDLE.inc(catalog_proc);
break;
case SP1_LOCAL:
TransactionCounter.SPECULATIVE_SP1.inc(catalog_proc);
TransactionCounter.SPECULATIVE_SP1_LOCAL.inc(catalog_proc);
break;
case SP2_REMOTE_BEFORE:
TransactionCounter.SPECULATIVE_SP2_BEFORE.inc(catalog_proc);
case SP2_LOCAL:
TransactionCounter.SPECULATIVE_SP2.inc(catalog_proc);
break;
case SP2_REMOTE_AFTER:
TransactionCounter.SPECULATIVE_SP2_AFTER.inc(catalog_proc);
case SP3_REMOTE_BEFORE:
TransactionCounter.SPECULATIVE_SP3_BEFORE.inc(catalog_proc);
break;
case SP3_LOCAL:
TransactionCounter.SPECULATIVE_SP3_LOCAL.inc(catalog_proc);
case SP3_REMOTE_AFTER:
TransactionCounter.SPECULATIVE_SP3_AFTER.inc(catalog_proc);
break;
case SP3_REMOTE:
TransactionCounter.SPECULATIVE_SP3_REMOTE.inc(catalog_proc);
case SP4_LOCAL:
TransactionCounter.SPECULATIVE_SP4_LOCAL.inc(catalog_proc);
break;
case SP4_REMOTE:
TransactionCounter.SPECULATIVE_SP4_REMOTE.inc(catalog_proc);
break;
default:
throw new RuntimeException("Unexpected " + ts.getSpeculationType());
@@ -111,7 +111,6 @@
import org.voltdb.utils.DBBPool.BBContainer;
import org.voltdb.utils.Encoder;
import org.voltdb.utils.EstTime;
import org.voltdb.utils.VoltTableUtil;
import com.google.protobuf.ByteString;
import com.google.protobuf.RpcCallback;
@@ -1780,7 +1779,6 @@ private void updateMemoryStats(long time) {
// rollup the table memory stats for this site
while (stats.advanceRow()) {
int idx = 7;
tupleCount += stats.getLong("TUPLE_COUNT");
tupleAccessCount += stats.getLong("TUPLE_ACCESSES");
tupleAllocatedMem += (int) stats.getLong("TUPLE_ALLOCATED_MEMORY");
@@ -1965,26 +1963,29 @@ private SpeculationType calculateSpeculationType() {
// IDLE
if (this.currentDtxn == null) {
specType = SpeculationType.IDLE;
// If the lock queue is not empty, then that doesn't mean we're really idle I suppose...
// if (this.lockQueue.approximateIsEmpty() == false) {
specType = SpeculationType.SP1_IDLE;
}
// LOCAL
else if (this.currentDtxn.getBasePartition() == this.partitionId) {
// Check whether the DTXN has started executing
if (((LocalTransaction)this.currentDtxn).isMarkedControlCodeExecuted() == false) {
specType = SpeculationType.IDLE;
specType = SpeculationType.SP1_LOCAL;
} else if (this.currentDtxn.isMarkedPrepared(this.partitionId)) {
specType = SpeculationType.SP3_LOCAL;
specType = SpeculationType.SP4_LOCAL;
} else {
specType = SpeculationType.SP1_LOCAL;
specType = SpeculationType.SP2_LOCAL;
}
}
// REMOTE
else {
if (this.currentDtxn.isMarkedPrepared(this.partitionId)) {
specType = SpeculationType.SP3_REMOTE;
specType = SpeculationType.SP4_REMOTE;
} else if (this.currentDtxn.hasExecutedWork(this.partitionId) == false) {
specType = SpeculationType.SP2_REMOTE_BEFORE;
specType = SpeculationType.SP3_REMOTE_BEFORE;
} else {
specType = SpeculationType.SP2_REMOTE_AFTER;
specType = SpeculationType.SP3_REMOTE_AFTER;
}
}
@@ -4732,7 +4733,7 @@ private Status prepareTransaction(AbstractTransaction ts,
// Check whether we can quickly ignore this speculative txn because
// it was executed at a stall point where conflicts don't matter.
SpeculationType specType = spec_ts.getSpeculationType();
if (specType != SpeculationType.SP2_REMOTE_AFTER && specType != SpeculationType.SP1_LOCAL) {
if (specType != SpeculationType.SP3_REMOTE_AFTER && specType != SpeculationType.SP2_LOCAL) {
continue;
}
@@ -387,17 +387,18 @@ else if (singlePartition == false) {
// We can execute anything when we are in SP3 (i.e., 2PC) or IDLE
// For SP2, we can execute anything if the txn has not
// executed a query at this partition.
case IDLE:
case SP2_REMOTE_BEFORE:
case SP3_LOCAL:
case SP3_REMOTE: {
case SP1_IDLE:
case SP1_LOCAL:
case SP3_REMOTE_BEFORE:
case SP4_LOCAL:
case SP4_REMOTE: {
break;
}
// Otherwise we have to use the ConflictChecker to determine whether
// it is safe to execute this txn given what the distributed txn
// is expected to execute in the future.
case SP1_LOCAL:
case SP2_REMOTE_AFTER: {
case SP2_LOCAL:
case SP3_REMOTE_AFTER: {
if (this.checker.hasConflictBefore(dtxn, localTxn, this.partitionId)) {
if (debug.val)
LOG.debug(String.format("Skipping %s because it conflicts with current transaction", localTxn));
@@ -67,12 +67,13 @@
// --------------------------------------------------------
// Speculative Execution Stall Points
// --------------------------------------------------------
SPECULATIVE_IDLE,
SPECULATIVE_SP1,
SPECULATIVE_SP2_BEFORE,
SPECULATIVE_SP2_AFTER,
SPECULATIVE_SP3_LOCAL,
SPECULATIVE_SP3_REMOTE,
SPECULATIVE_SP1_IDLE,
SPECULATIVE_SP1_LOCAL,
SPECULATIVE_SP2,
SPECULATIVE_SP3_BEFORE,
SPECULATIVE_SP3_AFTER,
SPECULATIVE_SP4_LOCAL,
SPECULATIVE_SP4_REMOTE,
;
private final FastIntHistogram h = new FastIntHistogram();
@@ -156,12 +157,13 @@ public Double ratio() {
case PREFETCH:
total = RECEIVED.get();
break;
case SPECULATIVE_IDLE:
case SPECULATIVE_SP1:
case SPECULATIVE_SP2_AFTER:
case SPECULATIVE_SP2_BEFORE:
case SPECULATIVE_SP3_LOCAL:
case SPECULATIVE_SP3_REMOTE:
case SPECULATIVE_SP1_IDLE:
case SPECULATIVE_SP1_LOCAL:
case SPECULATIVE_SP2:
case SPECULATIVE_SP3_AFTER:
case SPECULATIVE_SP3_BEFORE:
case SPECULATIVE_SP4_LOCAL:
case SPECULATIVE_SP4_REMOTE:
total = SPECULATIVE.get();
break;
case COMPLETED:
@@ -31,6 +31,7 @@
import edu.brown.markov.containers.MarkovGraphsContainer;
import edu.brown.markov.containers.SEATSMarkovGraphsContainer;
import edu.brown.markov.containers.TPCCMarkovGraphsContainer;
import edu.brown.profilers.ProfileMeasurement;
import edu.brown.utils.ArgumentsParser;
import edu.brown.utils.MathUtil;
import edu.brown.utils.PartitionEstimator;
@@ -702,6 +703,7 @@ public static void main(String vargs[]) throws Exception {
MarkovGraphsContainerUtil.setHasher(markovs_map, p_estimator.getHasher());
}
ProfileMeasurement pm = new ProfileMeasurement().start();
if (markovs_map == null) {
markovs_map = MarkovGraphsContainerUtil.createMarkovGraphsContainers(args.catalogContext,
args.workload,
@@ -714,6 +716,8 @@ public static void main(String vargs[]) throws Exception {
containerClass,
markovs_map);
}
pm.stop();
LOG.info(String.format("Completed processing in %.2f ms", pm.getTotalThinkTimeMS()));
// Save the graphs
assert(markovs_map != null);
@@ -15,34 +15,41 @@
*/
NULL,
/**
* Idle because of empty queue
* This is when the partition executor is allowed to jump ahead in time and execute txns
* assuming that nobody is going to show up with a lower timestamp.
*/
IDLE,
SP1_IDLE,
/**
* This stall point occurs on the base partition when the
* transaction is waiting for a WorkResponse from a remote partition.
* This stall point occurs on the local partition where the dtxn has acquired the lock
* for it's local partition but it's waiting to hear back about getting the locks
* from the other remote partitions.
*/
SP1_LOCAL,
/**
* This stall point occurs on the base partition while the executor is
* waiting for the 2PC-PREPARE responses from all of the dtxn's remote partitions.
* This stall point occurs on the base partition when the
* transaction is waiting for a WorkResponse from a remote partition.
*/
SP3_LOCAL,
SP2_LOCAL,
/**
* This stall point occurs on the remote partition while the executor
* is waiting <b>before</b> the first query request for the distributed transaction
*/
SP2_REMOTE_BEFORE,
SP3_REMOTE_BEFORE,
/**
* This stall point occurs on the remote partition while the executor
* is waiting <b>after</b> the first query request for the distributed transaction
*/
SP2_REMOTE_AFTER,
SP3_REMOTE_AFTER,
/**
* This stall point occurs on the base partition while the executor is
* waiting for the 2PC-PREPARE responses from all of the dtxn's remote partitions.
*/
SP4_LOCAL,
/**
* This stall point occurs on the remote partition while the executor is
* waiting for the 2PC-PREPARE acknowledgment from the dtxn's base partition.
*/
SP3_REMOTE,
SP4_REMOTE,
;
private static final Map<String, SpeculationType> name_lookup = new HashMap<String, SpeculationType>();
@@ -258,7 +258,7 @@ public void testPrepareAbort() throws Exception {
final int remotePartition = BASE_PARTITION+1;
final PartitionExecutor remoteExecutor = hstore_site.getPartitionExecutor(remotePartition);
final AtomicBoolean remotePartitionCheck = new AtomicBoolean(false);
remoteExecutor.getDebugContext().getSpecExecScheduler().ignoreSpeculationType(SpeculationType.SP2_REMOTE_BEFORE);
remoteExecutor.getDebugContext().getSpecExecScheduler().ignoreSpeculationType(SpeculationType.SP3_REMOTE_BEFORE);
hstore_conf.site.exec_early_prepare = false;
// Load in some test data so the txn doesn't abort
@@ -358,7 +358,7 @@ public void testEarly2PCWithQuery() throws Throwable {
// We want to make sure that the PartitionExecutor only spec execs
// at the 2PC stall points.
for (SpeculationType specType : SpeculationType.values()) {
if (specType == SpeculationType.SP3_REMOTE) continue;
if (specType == SpeculationType.SP4_REMOTE) continue;
for (int i = 0; i < this.executors.length; i++) {
this.executorDbgs[i].getSpecExecScheduler().ignoreSpeculationType(specType);
}
Oops, something went wrong.

0 comments on commit 550e8dc

Please sign in to comment.