Permalink
Browse files

articles loader

  • Loading branch information...
Atreyee committed Mar 2, 2014
1 parent 5b2abf1 commit a2990829dff9e18b8fb2b3c1ea90fc382848f76b
@@ -0,0 +1,30 @@
package edu.brown.benchmark.articles;
public abstract class ArticlesConstants {
// ----------------------------------------------------------------
// STORED PROCEDURE EXECUTION FREQUENCIES (0-100)
// ----------------------------------------------------------------
public static final int FREQUENCY_GET_ARTICLE = 40;
public static final int FREQUENCY_GET_ARTICLES = 20;
public static final int FREQUENCY_ADD_COMMENT = 30;
public static final int FREQUENCY_UPDATE_USER_INFO = 10;
// ----------------------------------------------------------------
// TABLE NAMES
// ----------------------------------------------------------------
public static final String TABLENAME_ARTICLES = "ARTICLES";
public static final String TABLENAME_USERS = "USERS";
public static final String TABLENAME_COMMENTS = "COMMENTS";
public static final int ARTICLES_SIZE = 10000;
public static final int USERS_SIZE = 200;
public static final int BATCH_SIZE = 500;
public static final String TABLENAMES[] = { TABLENAME_ARTICLES,
TABLENAME_USERS,
TABLENAME_COMMENTS
};
}
@@ -1,45 +1,203 @@
package edu.brown.benchmark.articles;
import org.voltdb.CatalogContext;
import org.apache.log4j.Logger;
import org.voltdb.VoltTable;
import org.voltdb.catalog.*;
import org.voltdb.client.Client;
import org.voltdb.catalog.Database;
import org.voltdb.catalog.Table;
import org.voltdb.utils.Pair;
import edu.brown.api.BenchmarkComponent;
import edu.brown.api.Loader;
import edu.brown.catalog.CatalogUtil;
import edu.brown.api.BenchmarkComponent;
import edu.brown.utils.EventObservable;
import edu.brown.utils.EventObservableExceptionHandler;
import edu.brown.utils.EventObserver;
public class ArticlesLoader extends Loader{
public static void main(String args[]) throws Exception {
BenchmarkComponent.main(ArticlesLoader.class, args, true);
}
public ArticlesLoader(String[] args) {
super(args);
for (String key : m_extraParams.keySet()) {
// TODO: Retrieve extra configuration parameters
} // FOR
}
@Override
public void load() {
// The catalog contains all the information about the database (e.g., tables, columns, indexes)
// It is loaded from the benchmark's project JAR file
// Catalog
CatalogContext _catalog = this.getCatalogContext();
Catalog catalog = _catalog.catalog;
// Iterate over all of the Table handles in the catalog and generate
// tuples to upload into the database
for (Table catalog_tbl : CatalogUtil.getDatabase(catalog).getTables()) {
// TODO: Create an empty VoltTable handle and then populate it in batches to
// be sent to the DBMS
VoltTable table = CatalogUtil.getVoltTable(catalog_tbl);
// Invoke the BenchmarkComponent's data loading method
// This will upload the contents of the VoltTable into the DBMS cluster
this.loadVoltTable(catalog_tbl.getName(), table);
} // FOR
}
private static final Logger LOG = Logger.getLogger(ArticlesLoader.class);
private static final boolean d = LOG.isDebugEnabled();
private final boolean blocking = false;
private long articlesSize;
private long usersSize;
public static void main(String[] args) {
BenchmarkComponent.main(ArticlesLoader.class, args, true);
}
public ArticlesLoader(String[] args) {
super(args);
this.articlesSize = Math.round(ArticlesConstants.ARTICLES_SIZE * this.getScaleFactor());
this.usersSize = Math.round(ArticlesConstants.USERS_SIZE * this.getScaleFactor());
}
@Override
public void load() {
if (d) LOG.debug("Starting ArticlesLoader");
final Database catalog_db = this.getCatalogContext().database;
final Thread threads[] =
{ new Thread() {
public void run() {
if (d) LOG.debug("Start loading " + ArticlesConstants.TABLENAME_ARTICLES);
Table catalog_tbl = catalog_db.getTables().get(ArticlesConstants.TABLENAME_ARTICLES);
genArticles(catalog_tbl);
if (d) LOG.debug("Finished loading " + ArticlesConstants.TABLENAME_ARTICLES);
}
}, new Thread() {
public void run() {
if (d) LOG.debug("Start loading " + ArticlesConstants.TABLENAME_USERS);
Table catalog_tbl = catalog_db.getTables().get(ArticlesConstants.TABLENAME_USERS);
genUsers(catalog_tbl);
if (d) LOG.debug("Finished loading " + ArticlesConstants.TABLENAME_USERS);
}
}, new Thread() {
public void run() {
if (d) LOG.debug("Start loading " + ArticlesConstants.TABLENAME_COMMENTS);
Table catalog_spe = catalog_db.getTables().get(ArticlesConstants.TABLENAME_COMMENTS);
genComments(catalog_spe);
if (d) LOG.debug("Finished loading " + ArticlesConstants.TABLENAME_COMMENTS);
}
} };
final EventObservableExceptionHandler handler = new EventObservableExceptionHandler();
handler.addObserver(new EventObserver<Pair<Thread,Throwable>>() {
@Override
public void update(EventObservable<Pair<Thread, Throwable>> o, Pair<Thread, Throwable> t) {
for (Thread thread : threads)
thread.interrupt();
}
});
try {
for (Thread t : threads) {
t.setUncaughtExceptionHandler(handler);
t.start();
if (this.blocking)
t.join();
} // FOR
if (!this.blocking) {
for (Thread t : threads)
t.join();
}
this.getClientHandle().drain();
} catch (Exception e) {
throw new RuntimeException(e);
} finally {
if (handler.hasError()) {
throw new RuntimeException("Error while generating table data.", handler.getError());
}
}
}
/**
* Populate Articles table per benchmark spec.
*/
void genArticles(Table catalog_tbl) {
final VoltTable table = CatalogUtil.getVoltTable(catalog_tbl);
Object row[] = new Object[table.getColumnCount()];
long total = 0;
for (long a_id = 0; a_id < this.articlesSize; a_id++) {
int col = 0;
row[col++] = a_id;
row[col++] = ArticlesUtil.astring(100, 100); // title
row[col++] = ArticlesUtil.astring(100, 100); // text
row[col++] = 0; // number of comments
assert col == table.getColumnCount();
table.addRow(row);
total++;
if (table.getRowCount() >= ArticlesConstants.BATCH_SIZE) {
if (d) LOG.debug(String.format("%s: %6d / %d",
ArticlesConstants.TABLENAME_ARTICLES, total, this.articlesSize));
loadVoltTable(ArticlesConstants.TABLENAME_ARTICLES, table);
table.clearRowData();
assert(table.getRowCount() == 0);
}
} // FOR
if (table.getRowCount() > 0) {
if (d) LOG.debug(String.format("%s: %6d / %d",
ArticlesConstants.TABLENAME_ARTICLES, total, this.articlesSize));
loadVoltTable(ArticlesConstants.TABLENAME_ARTICLES, table);
table.clearRowData();
assert(table.getRowCount() == 0);
}
}
/**
* Populate Users table per benchmark spec.
*/
void genUsers(Table catalog_tbl) {
final VoltTable table = CatalogUtil.getVoltTable(catalog_tbl);
/*-- u_firstname User's first name
-- u_lastname User's last name
-- u_password User's password
-- u_email User's email
*
* */
long total = 0;
for (long s_id = 0; s_id < this.usersSize; s_id++) {
Object row[] = new Object[table.getColumnCount()];
row[0] = s_id;
row[1] = ArticlesUtil.astring(3, 3);
row[2] = ArticlesUtil.astring(3, 3);
row[3] = ArticlesUtil.astring(3, 3);
row[4] = ArticlesUtil.astring(3, 3);
table.addRow(row);
total++;
if (table.getRowCount() >= ArticlesConstants.BATCH_SIZE) {
if (d) LOG.debug(String.format("%s: %6d / %d",
ArticlesConstants.TABLENAME_USERS, total, usersSize));
loadVoltTable(ArticlesConstants.TABLENAME_USERS, table);
table.clearRowData();
}
} // WHILE
if (table.getRowCount() > 0) {
if (d) LOG.debug(String.format("%s: %6d / %d",
ArticlesConstants.TABLENAME_USERS, total, usersSize));
loadVoltTable(ArticlesConstants.TABLENAME_USERS, table);
table.clearRowData();
}
}
/**
* Populate Comments table per benchmark spec.
*/
void genComments(Table catalog_comments) {
/*
* -- c_id Comment's ID
-- a_id Article's ID
-- u_id User's ID
-- c_text Actual comment text
* */
VoltTable speTbl = CatalogUtil.getVoltTable(catalog_comments);
long speTotal = 0;
for (long c_id = 0; c_id < this.articlesSize; c_id++) {
Object row_spe[] = new Object[speTbl.getColumnCount()];
row_spe[0] = c_id;
row_spe[1] = ArticlesUtil.number(0, this.articlesSize); // random number from the article id
row_spe[2] = ArticlesUtil.number(0, this.usersSize); // random number from user id
row_spe[3] = ArticlesUtil.astring(5, 5); // comment
speTbl.addRow(row_spe);
speTotal++;
if (speTbl.getRowCount() >= ArticlesConstants.BATCH_SIZE) {
if (d) LOG.debug(String.format("%s: %d", ArticlesConstants.TABLENAME_COMMENTS, speTotal));
loadVoltTable(ArticlesConstants.TABLENAME_COMMENTS, speTbl);
speTbl.clearRowData();
assert(speTbl.getRowCount() == 0);
}
} // WHILE
if (speTbl.getRowCount() > 0) {
if (d) LOG.debug(String.format("%s: %d", ArticlesConstants.TABLENAME_COMMENTS, speTotal));
loadVoltTable(ArticlesConstants.TABLENAME_COMMENTS, speTbl);
speTbl.clearRowData();
assert(speTbl.getRowCount() == 0);
}
}
}
@@ -16,10 +16,15 @@
public static final Class<?> PROCEDURES[] = new Class<?>[] {
GetArticle.class,
};
{
// Transaction Frequencies
addTransactionFrequency(GetArticle.class, ArticlesConstants.FREQUENCY_GET_ARTICLE);
}
public static final String PARTITIONING[][] = new String[][] {
// { "TABLE NAME", "PARTITIONING COLUMN NAME" }
{"TABLEA", "A_ID"},
{"TABLEB", "B_A_ID"},
{ ArticlesConstants.TABLENAME_ARTICLES, "A_ID" },
{ ArticlesConstants.TABLENAME_USERS, "U_ID" },
{ ArticlesConstants.TABLENAME_COMMENTS, "A_ID" }
};
@SuppressWarnings("unchecked")
@@ -29,4 +34,3 @@ public ArticlesProjectBuilder() {
}
}
@@ -0,0 +1,94 @@
package edu.brown.benchmark.articles;
import java.util.Random;
import edu.brown.rand.RandomDistribution.Zipf;
public abstract class ArticlesUtil {
public static final Random rand = new Random();
public static Zipf zipf = null;
public static final double zipf_sigma = 1.001d;
public static int isActive() {
return number(1, 100) < number(86, 100) ? 1 : 0;
}
public static long getSubscriberId(long subscriberSize) {
// We have to initalize the zipfian random distribution the first time
// we are called
// if (zipf == null) {
// zipf = new Zipf(rand, 1, subscriberSize, zipf_sigma);
// }
// return (zipf.nextLong());
return (ArticlesUtil.number(1, subscriberSize));
}
// modified from tpcc.RandomGenerator
/**
* @returns a random alphabetic string with length in range [minimum_length,
* maximum_length].
*/
public static String astring(int minimum_length, int maximum_length) {
return randomString(minimum_length, maximum_length, 'A', 26);
}
// taken from tpcc.RandomGenerator
/**
* @returns a random numeric string with length in range [minimum_length,
* maximum_length].
*/
public static String nstring(int minimum_length, int maximum_length) {
return randomString(minimum_length, maximum_length, '0', 10);
}
// taken from tpcc.RandomGenerator
public static String randomString(int minimum_length, int maximum_length, char base, int numCharacters) {
int length = (int)number(minimum_length, maximum_length);
byte baseByte = (byte) base;
byte[] bytes = new byte[length];
for (int i = 0; i < length; ++i) {
bytes[i] = (byte) (baseByte + number(0, numCharacters - 1));
}
return new String(bytes);
}
// taken from tpcc.RandomGenerator
public static long number(long minimum, long maximum) {
assert minimum <= maximum;
long value = Math.abs(rand.nextLong()) % (maximum - minimum + 1) + minimum;
assert minimum <= value && value <= maximum;
return value;
}
public static String padWithZero(long n) {
return String.format("%015d", n);
}
/**
* Returns sub array of arr, with length in range [min_len, max_len]. Each
* element in arr appears at most once in sub array.
*/
public static int[] subArr(int arr[], int min_len, int max_len) {
assert min_len <= max_len && min_len >= 0;
int sub_len = (int)number(min_len, max_len);
int arr_len = arr.length;
assert sub_len <= arr_len;
int sub[] = new int[sub_len];
for (int i = 0; i < sub_len; i++) {
int j = (int)number(0, arr_len - 1);
sub[i] = arr[j];
// arr[j] put to tail
int tmp = arr[j];
arr[j] = arr[arr_len - 1];
arr[arr_len - 1] = tmp;
arr_len--;
} // FOR
return sub;
}
}
Oops, something went wrong.

0 comments on commit a299082

Please sign in to comment.