Skip to content

Commit

Permalink
articles loader
Browse files Browse the repository at this point in the history
  • Loading branch information
@Atreyee
Atreyee committed Mar 2, 2014
1 parent 5b2abf1 commit a299082
Show file tree
Hide file tree
Showing 5 changed files with 330 additions and 44 deletions.
30 changes: 30 additions & 0 deletions src/benchmarks/edu/brown/benchmark/articles/ArticlesConstants.java
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,30 @@
package edu.brown.benchmark.articles;

public abstract class ArticlesConstants {

// ----------------------------------------------------------------
// STORED PROCEDURE EXECUTION FREQUENCIES (0-100)
// ----------------------------------------------------------------

public static final int FREQUENCY_GET_ARTICLE = 40;
public static final int FREQUENCY_GET_ARTICLES = 20;
public static final int FREQUENCY_ADD_COMMENT = 30;
public static final int FREQUENCY_UPDATE_USER_INFO = 10;

// ----------------------------------------------------------------
// TABLE NAMES
// ----------------------------------------------------------------
public static final String TABLENAME_ARTICLES = "ARTICLES";
public static final String TABLENAME_USERS = "USERS";
public static final String TABLENAME_COMMENTS = "COMMENTS";

public static final int ARTICLES_SIZE = 10000;
public static final int USERS_SIZE = 200;

public static final int BATCH_SIZE = 500;

public static final String TABLENAMES[] = { TABLENAME_ARTICLES,
TABLENAME_USERS,
TABLENAME_COMMENTS
};
}
230 changes: 194 additions & 36 deletions src/benchmarks/edu/brown/benchmark/articles/ArticlesLoader.java
View file
Original file line number Diff line number Diff line change
@@ -1,45 +1,203 @@
package edu.brown.benchmark.articles;
import org.voltdb.CatalogContext;
import org.apache.log4j.Logger;
import org.voltdb.VoltTable;
import org.voltdb.catalog.*;
import org.voltdb.client.Client;
import org.voltdb.catalog.Database;
import org.voltdb.catalog.Table;
import org.voltdb.utils.Pair;

import edu.brown.api.BenchmarkComponent;
import edu.brown.api.Loader;
import edu.brown.catalog.CatalogUtil;
import edu.brown.api.BenchmarkComponent;
import edu.brown.utils.EventObservable;
import edu.brown.utils.EventObservableExceptionHandler;
import edu.brown.utils.EventObserver;

public class ArticlesLoader extends Loader{

public static void main(String args[]) throws Exception {
BenchmarkComponent.main(ArticlesLoader.class, args, true);
}

public ArticlesLoader(String[] args) {
super(args);
for (String key : m_extraParams.keySet()) {
// TODO: Retrieve extra configuration parameters
} // FOR
}

@Override
public void load() {
// The catalog contains all the information about the database (e.g., tables, columns, indexes)
// It is loaded from the benchmark's project JAR file
// Catalog
CatalogContext _catalog = this.getCatalogContext();
Catalog catalog = _catalog.catalog;

// Iterate over all of the Table handles in the catalog and generate
// tuples to upload into the database
for (Table catalog_tbl : CatalogUtil.getDatabase(catalog).getTables()) {
// TODO: Create an empty VoltTable handle and then populate it in batches to
// be sent to the DBMS
VoltTable table = CatalogUtil.getVoltTable(catalog_tbl);

// Invoke the BenchmarkComponent's data loading method
// This will upload the contents of the VoltTable into the DBMS cluster
this.loadVoltTable(catalog_tbl.getName(), table);
} // FOR
}
private static final Logger LOG = Logger.getLogger(ArticlesLoader.class);
private static final boolean d = LOG.isDebugEnabled();

private final boolean blocking = false;
private long articlesSize;
private long usersSize;
public static void main(String[] args) {
BenchmarkComponent.main(ArticlesLoader.class, args, true);
}

public ArticlesLoader(String[] args) {
super(args);
this.articlesSize = Math.round(ArticlesConstants.ARTICLES_SIZE * this.getScaleFactor());
this.usersSize = Math.round(ArticlesConstants.USERS_SIZE * this.getScaleFactor());
}

@Override
public void load() {
if (d) LOG.debug("Starting ArticlesLoader");
final Database catalog_db = this.getCatalogContext().database;

final Thread threads[] =
{ new Thread() {
public void run() {
if (d) LOG.debug("Start loading " + ArticlesConstants.TABLENAME_ARTICLES);
Table catalog_tbl = catalog_db.getTables().get(ArticlesConstants.TABLENAME_ARTICLES);
genArticles(catalog_tbl);
if (d) LOG.debug("Finished loading " + ArticlesConstants.TABLENAME_ARTICLES);
}
}, new Thread() {
public void run() {
if (d) LOG.debug("Start loading " + ArticlesConstants.TABLENAME_USERS);
Table catalog_tbl = catalog_db.getTables().get(ArticlesConstants.TABLENAME_USERS);
genUsers(catalog_tbl);
if (d) LOG.debug("Finished loading " + ArticlesConstants.TABLENAME_USERS);
}
}, new Thread() {
public void run() {
if (d) LOG.debug("Start loading " + ArticlesConstants.TABLENAME_COMMENTS);
Table catalog_spe = catalog_db.getTables().get(ArticlesConstants.TABLENAME_COMMENTS);
genComments(catalog_spe);
if (d) LOG.debug("Finished loading " + ArticlesConstants.TABLENAME_COMMENTS);
}
} };

final EventObservableExceptionHandler handler = new EventObservableExceptionHandler();
handler.addObserver(new EventObserver<Pair<Thread,Throwable>>() {
@Override
public void update(EventObservable<Pair<Thread, Throwable>> o, Pair<Thread, Throwable> t) {
for (Thread thread : threads)
thread.interrupt();
}
});

try {
for (Thread t : threads) {
t.setUncaughtExceptionHandler(handler);
t.start();
if (this.blocking)
t.join();
} // FOR
if (!this.blocking) {
for (Thread t : threads)
t.join();
}
this.getClientHandle().drain();
} catch (Exception e) {
throw new RuntimeException(e);
} finally {
if (handler.hasError()) {
throw new RuntimeException("Error while generating table data.", handler.getError());
}
}

}


/**
* Populate Articles table per benchmark spec.
*/
void genArticles(Table catalog_tbl) {
final VoltTable table = CatalogUtil.getVoltTable(catalog_tbl);
Object row[] = new Object[table.getColumnCount()];
long total = 0;
for (long a_id = 0; a_id < this.articlesSize; a_id++) {
int col = 0;
row[col++] = a_id;
row[col++] = ArticlesUtil.astring(100, 100); // title
row[col++] = ArticlesUtil.astring(100, 100); // text
row[col++] = 0; // number of comments
assert col == table.getColumnCount();
table.addRow(row);
total++;

if (table.getRowCount() >= ArticlesConstants.BATCH_SIZE) {
if (d) LOG.debug(String.format("%s: %6d / %d",
ArticlesConstants.TABLENAME_ARTICLES, total, this.articlesSize));
loadVoltTable(ArticlesConstants.TABLENAME_ARTICLES, table);
table.clearRowData();
assert(table.getRowCount() == 0);
}
} // FOR
if (table.getRowCount() > 0) {
if (d) LOG.debug(String.format("%s: %6d / %d",
ArticlesConstants.TABLENAME_ARTICLES, total, this.articlesSize));
loadVoltTable(ArticlesConstants.TABLENAME_ARTICLES, table);
table.clearRowData();
assert(table.getRowCount() == 0);
}
}

/**
* Populate Users table per benchmark spec.
*/
void genUsers(Table catalog_tbl) {
final VoltTable table = CatalogUtil.getVoltTable(catalog_tbl);

/*-- u_firstname User's first name
-- u_lastname User's last name
-- u_password User's password
-- u_email User's email
*
* */
long total = 0;
for (long s_id = 0; s_id < this.usersSize; s_id++) {
Object row[] = new Object[table.getColumnCount()];
row[0] = s_id;
row[1] = ArticlesUtil.astring(3, 3);
row[2] = ArticlesUtil.astring(3, 3);
row[3] = ArticlesUtil.astring(3, 3);
row[4] = ArticlesUtil.astring(3, 3);
table.addRow(row);
total++;
if (table.getRowCount() >= ArticlesConstants.BATCH_SIZE) {
if (d) LOG.debug(String.format("%s: %6d / %d",
ArticlesConstants.TABLENAME_USERS, total, usersSize));
loadVoltTable(ArticlesConstants.TABLENAME_USERS, table);
table.clearRowData();
}
} // WHILE
if (table.getRowCount() > 0) {
if (d) LOG.debug(String.format("%s: %6d / %d",
ArticlesConstants.TABLENAME_USERS, total, usersSize));
loadVoltTable(ArticlesConstants.TABLENAME_USERS, table);
table.clearRowData();
}
}

/**
* Populate Comments table per benchmark spec.
*/
void genComments(Table catalog_comments) {

/*
* -- c_id Comment's ID
-- a_id Article's ID
-- u_id User's ID
-- c_text Actual comment text
* */
VoltTable speTbl = CatalogUtil.getVoltTable(catalog_comments);

long speTotal = 0;
for (long c_id = 0; c_id < this.articlesSize; c_id++) {
Object row_spe[] = new Object[speTbl.getColumnCount()];
row_spe[0] = c_id;
row_spe[1] = ArticlesUtil.number(0, this.articlesSize); // random number from the article id
row_spe[2] = ArticlesUtil.number(0, this.usersSize); // random number from user id
row_spe[3] = ArticlesUtil.astring(5, 5); // comment
speTbl.addRow(row_spe);
speTotal++;
if (speTbl.getRowCount() >= ArticlesConstants.BATCH_SIZE) {
if (d) LOG.debug(String.format("%s: %d", ArticlesConstants.TABLENAME_COMMENTS, speTotal));
loadVoltTable(ArticlesConstants.TABLENAME_COMMENTS, speTbl);
speTbl.clearRowData();
assert(speTbl.getRowCount() == 0);
}
} // WHILE
if (speTbl.getRowCount() > 0) {
if (d) LOG.debug(String.format("%s: %d", ArticlesConstants.TABLENAME_COMMENTS, speTotal));
loadVoltTable(ArticlesConstants.TABLENAME_COMMENTS, speTbl);
speTbl.clearRowData();
assert(speTbl.getRowCount() == 0);
}
}

}

12 changes: 8 additions & 4 deletions src/benchmarks/edu/brown/benchmark/articles/ArticlesProjectBuilder.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -16,10 +16,15 @@ public class ArticlesProjectBuilder extends AbstractProjectBuilder{
public static final Class<?> PROCEDURES[] = new Class<?>[] {
GetArticle.class,
};
{
// Transaction Frequencies
addTransactionFrequency(GetArticle.class, ArticlesConstants.FREQUENCY_GET_ARTICLE);
}

public static final String PARTITIONING[][] = new String[][] {
// { "TABLE NAME", "PARTITIONING COLUMN NAME" }
{"TABLEA", "A_ID"},
{"TABLEB", "B_A_ID"},
{ ArticlesConstants.TABLENAME_ARTICLES, "A_ID" },
{ ArticlesConstants.TABLENAME_USERS, "U_ID" },
{ ArticlesConstants.TABLENAME_COMMENTS, "A_ID" }
};

@SuppressWarnings("unchecked")
Expand All @@ -29,4 +34,3 @@ public ArticlesProjectBuilder() {
}

}

94 changes: 94 additions & 0 deletions src/benchmarks/edu/brown/benchmark/articles/ArticlesUtil.java
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,94 @@
package edu.brown.benchmark.articles;

import java.util.Random;

import edu.brown.rand.RandomDistribution.Zipf;

public abstract class ArticlesUtil {

public static final Random rand = new Random();
public static Zipf zipf = null;
public static final double zipf_sigma = 1.001d;

public static int isActive() {
return number(1, 100) < number(86, 100) ? 1 : 0;
}

public static long getSubscriberId(long subscriberSize) {
// We have to initalize the zipfian random distribution the first time
// we are called
// if (zipf == null) {
// zipf = new Zipf(rand, 1, subscriberSize, zipf_sigma);
// }
// return (zipf.nextLong());
return (ArticlesUtil.number(1, subscriberSize));
}

// modified from tpcc.RandomGenerator
/**
* @returns a random alphabetic string with length in range [minimum_length,
* maximum_length].
*/
public static String astring(int minimum_length, int maximum_length) {
return randomString(minimum_length, maximum_length, 'A', 26);
}

// taken from tpcc.RandomGenerator
/**
* @returns a random numeric string with length in range [minimum_length,
* maximum_length].
*/
public static String nstring(int minimum_length, int maximum_length) {
return randomString(minimum_length, maximum_length, '0', 10);
}

// taken from tpcc.RandomGenerator
public static String randomString(int minimum_length, int maximum_length, char base, int numCharacters) {
int length = (int)number(minimum_length, maximum_length);
byte baseByte = (byte) base;
byte[] bytes = new byte[length];
for (int i = 0; i < length; ++i) {
bytes[i] = (byte) (baseByte + number(0, numCharacters - 1));
}
return new String(bytes);
}

// taken from tpcc.RandomGenerator
public static long number(long minimum, long maximum) {
assert minimum <= maximum;
long value = Math.abs(rand.nextLong()) % (maximum - minimum + 1) + minimum;
assert minimum <= value && value <= maximum;
return value;
}

public static String padWithZero(long n) {
return String.format("%015d", n);
}

/**
* Returns sub array of arr, with length in range [min_len, max_len]. Each
* element in arr appears at most once in sub array.
*/
public static int[] subArr(int arr[], int min_len, int max_len) {
assert min_len <= max_len && min_len >= 0;
int sub_len = (int)number(min_len, max_len);
int arr_len = arr.length;

assert sub_len <= arr_len;

int sub[] = new int[sub_len];
for (int i = 0; i < sub_len; i++) {
int j = (int)number(0, arr_len - 1);
sub[i] = arr[j];
// arr[j] put to tail
int tmp = arr[j];
arr[j] = arr[arr_len - 1];
arr[arr_len - 1] = tmp;

arr_len--;
} // FOR

return sub;
}

}
Loading

0 comments on commit a299082

Please sign in to comment.