Skip to content

Commit

Permalink
MONDRIAN: fixes for aggregate loading
Browse files Browse the repository at this point in the history 
[git-p4: depot-paths = "//open/mondrian/": change = 3599]
  • Loading branch information
Sherman Wood committed May 18, 2005
1 parent 5a79bb1 commit fb2c367
Showing 1 changed file with 63 additions and 54 deletions.
117 changes: 63 additions & 54 deletions testsrc/main/mondrian/test/loader/MondrianFoodMartLoader.java
View file
Expand Up @@ -29,6 +29,8 @@
import java.util.zip.ZipEntry;
import java.util.zip.ZipFile;

import org.apache.log4j.Logger;

/**
* Utility to load the FoodMart dataset into an arbitrary JDBC database.
*
Expand Down Expand Up @@ -62,6 +64,8 @@
* @version $Id$
*/
public class MondrianFoodMartLoader {
private static final Logger LOGGER = Logger.getLogger(MondrianFoodMartLoader.class);

final Pattern decimalDataTypeRegex = Pattern.compile("DECIMAL\\((.*),(.*)\\)");
final DecimalFormat integerFormatter = new DecimalFormat(decimalFormat(15, 0));
final String dateFormatString = "yyyy-MM-dd";
Expand All @@ -80,7 +84,6 @@ public class MondrianFoodMartLoader {
private boolean indexes = false;
private boolean data = false;
private static final String nl = System.getProperty("line.separator");
private boolean verbose = false;
private boolean jdbcInput = false;
private boolean jdbcOutput = false;
private boolean populationQueries = false;
Expand All @@ -99,15 +102,13 @@ public class MondrianFoodMartLoader {
public MondrianFoodMartLoader(String[] args) {

StringBuffer errorMessage = new StringBuffer();
StringBuffer parametersMessage = new StringBuffer();

for ( int i=0; i<args.length; i++ ) {
if (args[i].equals("-verbose")) {
verbose = true;
} else if (args[i].equals("-tables")) {
if (args[i].equals("-tables")) {
tables = true;
} else if (args[i].equals("-data")) {
data = true;
populationQueries = true;
} else if (args[i].equals("-indexes")) {
indexes = true;
} else if (args[i].equals("-populationQueries")) {
Expand All @@ -133,7 +134,11 @@ public MondrianFoodMartLoader(String[] args) {
} else if (args[i].startsWith("-outputJdbcBatchSize=")) {
inputBatchSize = Integer.parseInt(args[i].substring("-outputJdbcBatchSize=".length()));
} else {
errorMessage.append("unknown arg: " + args[i] + "\n");
errorMessage.append("unknown arg: " + args[i] + nl);
}

if (LOGGER.isInfoEnabled()) {
parametersMessage.append("\t" + args[i] + nl);
}
}
if (inputJdbcURL != null) {
Expand All @@ -149,10 +154,14 @@ public MondrianFoodMartLoader(String[] args) {
usage();
throw MondrianResource.instance().newMissingArg(errorMessage.toString());
}

if (LOGGER.isInfoEnabled()) {
LOGGER.info("Parameters: " + nl + parametersMessage.toString());
}
}

public void usage() {
System.out.println("Usage: MondrianFoodMartLoader [-verbose] [-tables] [-data] [-indexes] [-populationQueries]" +
System.out.println("Usage: MondrianFoodMartLoader [-tables] [-data] [-indexes] [-populationQueries]" +
"-jdbcDrivers=<jdbcDriver> " +
"-outputJdbcURL=<jdbcURL> [-outputJdbcUser=user] [-outputJdbcPassword=password]" +
"[-outputJdbcBatchSize=<batch size>] " +
Expand Down Expand Up @@ -183,13 +192,13 @@ public void usage() {
}

public static void main(String[] args) {
System.out.println("Starting load at: " + (new Date()));
LOGGER.warn("Starting load at: " + (new Date()));
try {
new MondrianFoodMartLoader(args).load();
} catch (Throwable e) {
e.printStackTrace();
LOGGER.error("Main error", e);
}
System.out.println("Finished load at: " + (new Date()));
LOGGER.warn("Finished load at: " + (new Date()));
}

/**
Expand Down Expand Up @@ -219,14 +228,14 @@ private void load() throws Exception {
String productName = metaData.getDatabaseProductName();
String version = metaData.getDatabaseProductVersion();

System.out.println("Output connection is " + productName + ", " + version);
LOGGER.info("Output connection is " + productName + ", " + version);

sqlQuery = new SqlQuery(metaData);
booleanColumnType = "SMALLINT";
if (sqlQuery.isPostgres()) {
booleanColumnType = "BOOLEAN";
} else if (sqlQuery.isMySQL()) {
booleanColumnType = "BIT";
booleanColumnType = "TINYINT(1)";
}

bigIntColumnType = "BIGINT";
Expand All @@ -236,15 +245,14 @@ private void load() throws Exception {

try {
createTables(); // This also initializes tableMetadataToLoad
if (data && !populationQueries) {
if (jdbcInput) {
loadDataFromJdbcInput();
} else {
loadDataFromFile();
}
}

if (data || populationQueries) {
if (data) {
if (!populationQueries) {
if (jdbcInput) {
loadDataFromJdbcInput();
} else {
loadDataFromFile();
}
}
loadFromSQLInserts();
}

Expand Down Expand Up @@ -342,7 +350,7 @@ private void loadDataFromFile() throws Exception {
// If table just changed, flush the previous batch.
if (!tableName.equals(prevTable)) {
if (!prevTable.equals("")) {
System.out.println("Table " + prevTable +
LOGGER.info("Table " + prevTable +
": loaded " + tableRowCount + " rows.");
}
tableRowCount = 0;
Expand Down Expand Up @@ -398,7 +406,7 @@ private void loadDataFromFile() throws Exception {
}
// Print summary of the final table.
if (!prevTable.equals("")) {
System.out.println("Table " + prevTable +
LOGGER.info("Table " + prevTable +
": loaded " + tableRowCount + " rows.");
tableRowCount = 0;
writeBatch(batch, batchSize);
Expand Down Expand Up @@ -498,7 +506,7 @@ private void loadDataFromJdbcInput() throws Exception {
for (Iterator it = tableMetadataToLoad.entrySet().iterator(); it.hasNext(); ) {
Map.Entry tableEntry = (Map.Entry) it.next();
int rowsAdded = loadTable((String) tableEntry.getKey(), (Column[]) tableEntry.getValue());
System.out.println("Table " + (String) tableEntry.getKey() +
LOGGER.info("Table " + (String) tableEntry.getKey() +
": loaded " + rowsAdded + " rows.");
}

Expand Down Expand Up @@ -601,9 +609,8 @@ private int loadTable(String name, Column[] columns) throws Exception {
.append(quoteId(name));
String ddl = buf.toString();
Statement statement = inputConnection.createStatement();
if (verbose) {
System.out.println("Input table SQL: " + ddl);
}
LOGGER.debug("Input table SQL: " + ddl);

ResultSet rs = statement.executeQuery(ddl);

String[] batch = new String[inputBatchSize];
Expand All @@ -616,8 +623,8 @@ private int loadTable(String name, Column[] columns) throws Exception {
*/

String insertStatement = createInsertStatement(rs, name, columns);
if (!displayedInsert && verbose) {
System.out.println("Example Insert statement: " + insertStatement);
if (!displayedInsert && LOGGER.isDebugEnabled()) {
LOGGER.debug("Example Insert statement: " + insertStatement);
displayedInsert = true;
}
batch[batchSize++] = insertStatement;
Expand Down Expand Up @@ -685,7 +692,7 @@ private int writeBatch(String[] batch, int batchSize) throws IOException, SQLExc
if (outputDirectory != null) {
for (int i = 0; i < batchSize; i++) {
fileOutput.write(batch[i]);
fileOutput.write(";\n");
fileOutput.write(";" + nl);
}
} else {
connection.setAutoCommit(false);
Expand All @@ -705,14 +712,14 @@ private int writeBatch(String[] batch, int batchSize) throws IOException, SQLExc
updateCounts = stmt.executeBatch();
} catch (SQLException e) {
for (int i = 0; i < batchSize; i++) {
System.out.println("Error in SQL batch: " + batch[i]);
LOGGER.error("Error in SQL batch: " + batch[i]);
}
throw e;
}
int updates = 0;
for (int i = 0; i < updateCounts.length; updates += updateCounts[i], i++) {
if (updateCounts[i] == 0) {
System.out.println("Error in SQL: " + batch[i]);
LOGGER.error("Error in SQL: " + batch[i]);
}
}
if (updates < batchSize) {
Expand All @@ -736,7 +743,7 @@ private InputStream openInputStream() throws Exception {
final String defaultDataFileName = "FoodMartCreateData.sql";
final File file = (inputFile != null) ? new File(inputFile) : new File("demo", defaultZipFileName);
if (!file.exists()) {
System.out.println("No input file: " + file);
LOGGER.error("No input file: " + file);
return null;
}
if (file.getName().toLowerCase().endsWith(".zip")) {
Expand Down Expand Up @@ -878,7 +885,14 @@ private void createIndex(
String[] columnNames)
{
try {

// Only do aggregate tables
if (populationQueries && !aggregateTableMetadataToLoad.containsKey(tableName)) {
return;
}

StringBuffer buf = new StringBuffer();

// If we're [re]creating tables, no need to drop indexes.
if (jdbcOutput && !tables) {
try {
Expand All @@ -891,14 +905,9 @@ private void createIndex(
final String deleteDDL = buf.toString();
executeDDL(deleteDDL);
} catch (Exception e1) {
System.out.println("Drop failed: but continue");
LOGGER.info("Index Drop failed for " + tableName + ", " + indexName + " : but continue");
}
}

// Only do aggregate tables
if (populationQueries && !aggregateTableMetadataToLoad.containsKey(tableName)) {
return;
}

buf = new StringBuffer();
buf.append(isUnique ? "CREATE UNIQUE INDEX " : "CREATE INDEX ")
Expand Down Expand Up @@ -1359,7 +1368,10 @@ private void createTable(String name, Column[] columns, boolean loadData, boole
}

if (!tables) {
if ((data || (populationQueries && aggregate)) && jdbcOutput) {
if (data && jdbcOutput) {
if (populationQueries && !aggregate) {
return;
}
// We're going to load the data without [re]creating
// the table, so let's remove the data.
try {
Expand All @@ -1379,9 +1391,7 @@ private void createTable(String name, Column[] columns, boolean loadData, boole
try {
executeDDL("DROP TABLE " + quoteId(name));
} catch (Exception e) {
if (verbose) {
System.out.println("Drop of " + name + " failed. Ignored");
}
LOGGER.debug("Drop of " + name + " failed. Ignored");
}

// Define the table.
Expand Down Expand Up @@ -1409,15 +1419,14 @@ private void createTable(String name, Column[] columns, boolean loadData, boole
}

private void executeDDL(String ddl) throws Exception {
if (verbose) {
System.out.println(ddl);
}
LOGGER.info(ddl);

if (jdbcOutput) {
final Statement statement = connection.createStatement();
statement.execute(ddl);
} else {
fileOutput.write(ddl);
fileOutput.write(";\n");
fileOutput.write(";" + nl);
}

}
Expand Down Expand Up @@ -1465,15 +1474,15 @@ private String columnValue(ResultSet rs, Column column) throws Exception {
Double result = (Double) obj;
return integerFormatter.format(result.doubleValue());
} catch (ClassCastException cce) {
System.out.println("CCE: " + column.name + " to Long from: " + obj.getClass().getName() + " - " + obj.toString());
LOGGER.error("CCE: " + column.name + " to Long from: " + obj.getClass().getName() + " - " + obj.toString());
throw cce;
}
} else {
try {
Integer result = (Integer) obj;
return result.toString();
} catch (ClassCastException cce) {
System.out.println("CCE: " + column.name + " to Integer from: " + obj.getClass().getName() + " - " + obj.toString());
LOGGER.error("CCE: " + column.name + " to Integer from: " + obj.getClass().getName() + " - " + obj.toString());
throw cce;
}
}
Expand All @@ -1495,7 +1504,7 @@ private String columnValue(ResultSet rs, Column column) throws Exception {
Integer result = (Integer) obj;
return result.toString();
} catch (ClassCastException cce) {
System.out.println("CCE: " + column.name + " to Integer from: " + obj.getClass().getName() + " - " + obj.toString());
LOGGER.error("CCE: " + column.name + " to Integer from: " + obj.getClass().getName() + " - " + obj.toString());
throw cce;
}
}
Expand All @@ -1509,15 +1518,15 @@ private String columnValue(ResultSet rs, Column column) throws Exception {
Double result = (Double) obj;
return integerFormatter.format(result.doubleValue());
} catch (ClassCastException cce) {
System.out.println("CCE: " + column.name + " to Double from: " + obj.getClass().getName() + " - " + obj.toString());
LOGGER.error("CCE: " + column.name + " to Double from: " + obj.getClass().getName() + " - " + obj.toString());
throw cce;
}
} else {
try {
Long result = (Long) obj;
return result.toString();
} catch (ClassCastException cce) {
System.out.println("CCE: " + column.name + " to Long from: " + obj.getClass().getName() + " - " + obj.toString());
LOGGER.error("CCE: " + column.name + " to Long from: " + obj.getClass().getName() + " - " + obj.toString());
throw cce;
}
}
Expand Down Expand Up @@ -1572,7 +1581,7 @@ private String columnValue(ResultSet rs, Column column) throws Exception {
Double result = (Double) obj;
return formatter.format(result.doubleValue());
} catch (ClassCastException cce) {
System.out.println("CCE: " + column.name + " to Double from: " + obj.getClass().getName() + " - " + obj.toString());
LOGGER.error("CCE: " + column.name + " to Double from: " + obj.getClass().getName() + " - " + obj.toString());
throw cce;
}
} else {
Expand All @@ -1581,7 +1590,7 @@ private String columnValue(ResultSet rs, Column column) throws Exception {
BigDecimal result = (BigDecimal) obj;
return formatter.format(result);
} catch (ClassCastException cce) {
System.out.println("CCE: " + column.name + " to BigDecimal from: " + obj.getClass().getName() + " - " + obj.toString());
LOGGER.error("CCE: " + column.name + " to BigDecimal from: " + obj.getClass().getName() + " - " + obj.toString());
throw cce;
}
}
Expand Down

0 comments on commit fb2c367

Please sign in to comment.