Permalink
Browse files

add voterwinsstorew100s10withbatch to simulate spark behavior

  • Loading branch information...
hawkwang committed Sep 12, 2014
1 parent efa63d7 commit feba23f9c5f6b749f1ccadd0210ea03548a86aab
Showing with 2,583 additions and 5 deletions.
  1. +5 −0 properties/benchmarks/voterwinsstorew100s10withbatch.properties
  2. +144 −0 sparkbatchvoter
  3. +128 −0 ...enchmarks/edu/brown/benchmark/voterexperiments/winsstore/w100s10withbatch/PhoneCallGenerator.java
  4. +165 −0 ...chmarks/edu/brown/benchmark/voterexperiments/winsstore/w100s10withbatch/VoterWinSStoreClient.java
  5. +51 −0 ...arks/edu/brown/benchmark/voterexperiments/winsstore/w100s10withbatch/VoterWinSStoreConstants.java
  6. +66 −0 ...chmarks/edu/brown/benchmark/voterexperiments/winsstore/w100s10withbatch/VoterWinSStoreLoader.java
  7. +74 −0 ...edu/brown/benchmark/voterexperiments/winsstore/w100s10withbatch/VoterWinSStoreProjectBuilder.java
  8. +130 −0 ...enchmarks/edu/brown/benchmark/voterexperiments/winsstore/w100s10withbatch/VoterWinSStoreUtil.java
  9. +98 −0 ...own/benchmark/voterexperiments/winsstore/w100s10withbatch/procedures/ContestantWinningStates.java
  10. +95 −0 ...s/edu/brown/benchmark/voterexperiments/winsstore/w100s10withbatch/procedures/GetStateHeatmap.java
  11. +116 −0 ...hmarks/edu/brown/benchmark/voterexperiments/winsstore/w100s10withbatch/procedures/Initialize.java
  12. +127 −0 src/benchmarks/edu/brown/benchmark/voterexperiments/winsstore/w100s10withbatch/procedures/Vote.java
  13. +85 −0 ...rown/benchmark/voterexperiments/winsstore/w100s10withbatch/voterwinsstorew100s10withbatch-ddl.sql
  14. +3 −0 ...own/benchmark/voterexperiments/winsstore/w100s10withbatch/voterwinsstorew100s10withbatch.mappings
  15. +5 −3 src/frontend/edu/brown/stream/BatchProducer.java
  16. +21 −2 src/frontend/edu/brown/stream/BatchRunner.java
  17. +841 −0 src/frontend/edu/brown/stream/SparkBatchVoter/BatchRunner.java
  18. +266 −0 src/frontend/edu/brown/stream/SparkBatchVoter/InputClient.java
  19. +162 −0 src/frontend/edu/brown/stream/SparkBatchVoter/VoteProducer.java
  20. +1 −0 src/frontend/edu/brown/utils/ProjectType.java
@@ -0,0 +1,5 @@
## ------------------------------------------------------------
## H-Store Benchmark Configuration Parameters
## ------------------------------------------------------------
builder = edu.brown.benchmark.voterexperiments.winsstore.w100s10withbatch.VoterWinSStoreProjectBuilder
View
@@ -0,0 +1,144 @@
#!/usr/bin/env python
# -*- coding: utf-8 -*-
# -----------------------------------------------------------------------
# Copyright (C) 2012 by H-Store Project
# Brown University
# Massachusetts Institute of Technology
# Yale University
#
# http://hstore.cs.brown.edu/
#
# Permission is hereby granted, free of charge, to any person obtaining
# a copy of this software and associated documentation files (the
# "Software"), to deal in the Software without restriction, including
# without limitation the rights to use, copy, modify, merge, publish,
# distribute, sublicense, and/or sell copies of the Software, and to
# permit persons to whom the Software is furnished to do so, subject to
# the following conditions:
#
# The above copyright notice and this permission notice shall be
# included in all copies or substantial portions of the Software.
#
# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
# MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT
# IN NO EVENT SHALL THE AUTHORS BE LIABLE FOR ANY CLAIM, DAMAGES OR
# OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
# ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
# OTHER DEALINGS IN THE SOFTWARE.
# -----------------------------------------------------------------------
from __future__ import with_statement
import os
import sys
import subprocess
import logging
from glob import glob
## H-Store Third-Party Libraries
realpath = os.path.realpath(__file__)
basedir = os.path.dirname(realpath)
if not os.path.exists(realpath):
cwd = os.getcwd()
basename = os.path.basename(realpath)
if os.path.exists(os.path.join(cwd, basename)):
basedir = cwd
sys.path.append(os.path.realpath(os.path.join(basedir, "third_party/python")))
sys.path.append(os.path.realpath(os.path.join(basedir, "tools")))
import hstore
import argparse
logging.basicConfig(level = logging.INFO,
format="%(asctime)s %(levelname)-5s: %(message)s",
datefmt="%m-%d-%Y %H:%M:%S",
stream = sys.stdout)
THIRD_PARTY_DIR = "third_party/java/jars"
BUILD_DIR = "obj/release/prod"
BUILD_1_DIR = "obj/release/benchmarks"
JVM_ARGS = [
"-client",
"-Xmx2048m",
"-Dlog4j.configuration=log4j.properties",
"-ea",
]
JVM_CLASSPATH = [ BUILD_DIR ] + glob("%s/*.jar" % THIRD_PARTY_DIR)
JVM_TARGET = "edu.brown.stream.SparkBatchVoter.InputClient"
## ==============================================
## main
## ==============================================
if __name__ == '__main__':
aparser = argparse.ArgumentParser(description='S-Store Input Client')
aparser.add_argument('project', help='Benchmark name or Project Jar')
aparser.add_argument('--host', type=str,
help='Hostname of an HStoreSite to connect to.')
aparser.add_argument('--port', type=int,
help='Port number of an HStoreSite to connect to.')
aparser.add_argument('--file', type=str,
help='input file which contains votes.')
aparser.add_argument('--rate', type=int,
help='tuple sending rate.')
aparser.add_argument('--stop', type=bool,
help='stop after all tuple sends out or repeat again.')
aparser.add_argument('--json', type=str,
help='indicate if json result format should be printed out.')
aparser.add_argument('--display', type=str,
help='indicate if the terminal display the statistic result, otherwise, it will display the function result.')
aparser.add_argument('--interval', type=int,
help='batch time interval.')
aparser.add_argument('--rounds', type=int,
help='to indicate how batch rounds we need.')
args = vars(aparser.parse_args())
# Check whether we have a jar file
# Otherwise make sure it's a valid benchmarks
if not os.path.exists(args['project']):
benchmarks = hstore.getBenchmarks()
if not args['project'] in benchmarks:
logging.error("Invalid benchmark name '%s'" % args['project'])
logging.error("Valid benchmarks: %s" % ", ".join(benchmarks))
sys.exit(1)
args['project'] = "%s.jar" % args['project']
if not os.path.exists(args['project']):
logging.error("Project jar '%s' does not exist" % (args['project']))
sys.exit(1)
# Options
options = [ ]
for key in ('host', 'port'):
if key in args and args[key]:
options.append("terminal.%s=%s" % (key, args[key]))
## FOR
for key in ('file', 'rate', 'stop'):
if key in args and args[key]:
options.append("source.%s=%s" % (key, args[key]))
#end for
for key in ('interval', 'rounds'):
if key in args and args[key]:
options.append("batch.%s=%s" % (key, args[key]))
#end for
for key in ('json','display'):
if key in args and args[key]:
options.append("result.%s=%s" % (key, args[key]))
#end for
# Invoke java directly
cmd = "java %s -classpath %s %s catalog.jar=%s %s" % (\
" ".join(JVM_ARGS), \
":".join(JVM_CLASSPATH), \
JVM_TARGET, \
args['project'],
" ".join(options))
#logging.error(cmd)
logging.debug(cmd)
try:
p = subprocess.Popen(cmd, shell=True)
p.wait()
except:
pass
sys.exit(p.returncode)
## MAIN
@@ -0,0 +1,128 @@
/***************************************************************************
* Copyright (C) 2012 by H-Store Project *
* Brown University *
* Massachusetts Institute of Technology *
* Yale University *
* *
* Original By: VoltDB Inc. *
* Ported By: Justin A. DeBrabant (http://www.cs.brown.edu/~debrabant/) *
* *
* *
* Permission is hereby granted, free of charge, to any person obtaining *
* a copy of this software and associated documentation files (the *
* "Software"), to deal in the Software without restriction, including *
* without limitation the rights to use, copy, modify, merge, publish, *
* distribute, sublicense, and/or sell copies of the Software, and to *
* permit persons to whom the Software is furnished to do so, subject to *
* the following conditions: *
* *
* The above copyright notice and this permission notice shall be *
* included in all copies or substantial portions of the Software. *
* *
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, *
* EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF *
* MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. *
* IN NO EVENT SHALL THE AUTHORS BE LIABLE FOR ANY CLAIM, DAMAGES OR *
* OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, *
* ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR *
* OTHER DEALINGS IN THE SOFTWARE. *
***************************************************************************/
package edu.brown.benchmark.voterexperiments.winsstore.w100s10withbatch;
import java.util.Random;
public class PhoneCallGenerator {
private long nextVoteId;
private final int contestantCount;
private final Random rand = new Random();
private final int[] votingMap = new int[AREA_CODES.length];
// Initialize some common constants and variables
private static final String[] AREA_CODE_STRS = ("907,205,256,334,251,870,501,479" +
",480,602,623,928,520,341,764,628,831,925,909,562,661,510,650,949,760" +
",415,951,209,669,408,559,626,442,530,916,627,714,707,310,323,213,424" +
",747,818,858,935,619,805,369,720,303,970,719,860,203,959,475,202,302" +
",689,407,239,850,727,321,754,954,927,352,863,386,904,561,772,786,305" +
",941,813,478,770,470,404,762,706,678,912,229,808,515,319,563,641,712" +
",208,217,872,312,773,464,708,224,847,779,815,618,309,331,630,317,765" +
",574,260,219,812,913,785,316,620,606,859,502,270,504,985,225,318,337" +
",774,508,339,781,857,617,978,351,413,443,410,301,240,207,517,810,278" +
",679,313,586,947,248,734,269,989,906,616,231,612,320,651,763,952,218" +
",507,636,660,975,816,573,314,557,417,769,601,662,228,406,336,252,984" +
",919,980,910,828,704,701,402,308,603,908,848,732,551,201,862,973,609" +
",856,575,957,505,775,702,315,518,646,347,212,718,516,917,845,631,716" +
",585,607,914,216,330,234,567,419,440,380,740,614,283,513,937,918,580" +
",405,503,541,971,814,717,570,878,835,484,610,267,215,724,412,401,843" +
",864,803,605,423,865,931,615,901,731,254,325,713,940,817,430,903,806" +
",737,512,361,210,979,936,409,972,469,214,682,832,281,830,956,432,915" +
",435,801,385,434,804,757,703,571,276,236,540,802,509,360,564,206,425" +
",253,715,920,262,414,608,304,307").split(",");
// convert the area code array to a list of digits
private static final long[] AREA_CODES = new long[AREA_CODE_STRS.length];
static {
for (int i = 0; i < AREA_CODES.length; i++)
AREA_CODES[i] = Long.parseLong(AREA_CODE_STRS[i]);
}
public static class PhoneCall {
public final long voteId;
public final int contestantNumber;
public final long phoneNumber;
protected PhoneCall(long voteId, int contestantNumber, long phoneNumber) {
this.voteId = voteId;
this.contestantNumber = contestantNumber;
this.phoneNumber = phoneNumber;
}
}
public PhoneCallGenerator(int clientId, int contestantCount) {
this.nextVoteId = clientId * 10000000l;
this.contestantCount = contestantCount;
// This is a just a small fudge to make the geographical voting map more interesting for the benchmark!
for(int i = 0; i < votingMap.length; i++) {
votingMap[i] = 1;
if (rand.nextInt(100) >= 30) {
votingMap[i] = (int) (Math.abs(Math.sin(i)* contestantCount) % contestantCount) + 1;
}
}
}
/**
* Receives/generates a simulated voting call
* @return Call details (calling number and contestant to whom the vote is given)
*/
public PhoneCall receive()
{
// For the purpose of a benchmark, issue random voting activity
// (including invalid votes to demonstrate transaction validationg in the database)
// Pick a random area code for the originating phone call
int areaCodeIndex = rand.nextInt(AREA_CODES.length);
// Pick a contestant number
int contestantNumber = votingMap[areaCodeIndex];
if (rand.nextBoolean()) {
contestantNumber = rand.nextInt(contestantCount) + 1;
}
// introduce an invalid contestant every 100 call or so to simulate fraud
// and invalid entries (something the transaction validates against)
if (rand.nextInt(100) == 0) {
contestantNumber = 999;
}
// Build the phone number
long phoneNumber = AREA_CODES[areaCodeIndex] * 10000000L + rand.nextInt(10000000);
// This needs to be globally unique
// Return the generated phone number
return new PhoneCall(this.nextVoteId++, contestantNumber, phoneNumber);
}
}
Oops, something went wrong.

0 comments on commit feba23f

Please sign in to comment.