Navigation Menu

Skip to content

Commit

Permalink
DbFill Changes...
Browse files Browse the repository at this point in the history 
- Add support for generating random UUID.
- Added new cascading insert feature.
- Added new table prefix ignore feature.
- Changed the default number of rows inserted to 1.
- Made DbFill more verbose.

Issues addressed:
0000880: Add support for cascading random data to DbFill.
  • Loading branch information
@abrougher
abrougher committed Oct 26, 2012
1 parent 819c773 commit 896290b
Show file tree
Hide file tree
Showing 3 changed files with 113 additions and 32 deletions.
131 changes: 99 additions & 32 deletions symmetric-client/src/main/java/org/jumpmind/symmetric/DbFill.java
View file
Expand Up @@ -5,9 +5,12 @@
import java.util.ArrayList;
import java.util.Date;
import java.util.HashMap;
import java.util.HashSet;
import java.util.List;
import java.util.Map;
import java.util.Random;
import java.util.Set;
import java.util.UUID;

import javax.sql.DataSource;

Expand Down Expand Up @@ -42,7 +45,11 @@ class DbFill {

private boolean ignoreMissingTables;

private int inputLength = 10;
private boolean cascading = false;

private String ignorePrefix = "";

private int inputLength = 1;

public DbFill() {
}
Expand All @@ -62,10 +69,21 @@ public void setDataSource(DataSource dataSource) {
public void fillTables(String... tableNames) {
Table[] tables;
if (tableNames.length == 0) {
// If no tableNames are provided, fill all of the tables in the
// schema with random data.
// If no tableNames are provided look up all tables.
Database db = platform.readDatabase(catalog, schema, null);
tables = db.getTables();
if (!StringUtils.isEmpty(ignorePrefix)) {
// Ignore any tables matching the ignorePrefix. (e.g., "sym_")
List<Table> tableList = new ArrayList<Table>(tables.length);
for (Table table : tables) {
if (table.getName().startsWith(ignorePrefix)) {
System.out.println("Ignore table " + table.getName());
} else {
tableList.add(table);
}
}
tables = tableList.toArray(new Table[tableList.size()]);
}
} else {
ArrayList<Table> tableList = new ArrayList<Table>();
for (String tableName : tableNames) {
Expand All @@ -83,13 +101,50 @@ public void fillTables(String... tableNames) {

fillTables(tables);
}

/**
* Identify the tables not included in the given list that the initial tables have FK relation ships to.
*
* @param tables
*
* @return The table array argument and the tables that the initial table array argument depend on.
*/
public Table[] addFKDependentTables(Table...tables) {
Table[] fkDepTblArray = null;
if (tables != null) {
List<Table> fkDepList = new ArrayList<Table>();
Set<String> tableNames = new HashSet<String>();
for (Table tbl : tables) {
tableNames.add(tbl.getName());
}
for (Table table : tables) {
for (ForeignKey fk : table.getForeignKeys()) {
if (tableNames.add(fk.getForeignTableName())) {
Table tableObj = platform.readTableFromDatabase(getCatalogToUse(), getSchemaToUse(),
fk.getForeignTableName());
fkDepList.add(tableObj);
}
}
}
fkDepTblArray = fkDepList.toArray(new Table[fkDepList.size()]);
fkDepTblArray = (Table[])ArrayUtils.addAll(fkDepTblArray, tables);
if (fkDepList.size()>0) {
fkDepTblArray = addFKDependentTables(fkDepTblArray);
}
}
return fkDepTblArray;
}

private void fillTables(Table... tables) {
ISqlTemplate sqlTemplate = platform.getSqlTemplate();

if (cascading) {
tables = addFKDependentTables(tables);
}

tables = Database.sortByForeignKeys(tables);

// Maintain a map of previously inserted values for foreign key
// constraints
// Maintain a map of previously inserted values for foreign key constraints
Map<String, Object> insertedColumns = new HashMap<String, Object>(tables.length);

// Insert multiple records
Expand All @@ -98,12 +153,7 @@ private void fillTables(Table... tables) {
for (Table table : tables) {
DmlStatement statement = platform.createDmlStatement(DmlType.INSERT, table);

Column[] tableColumns = table.getColumns();
Object[] columnValues = generateRandomValues(insertedColumns, table);
for (int j = 0; j < tableColumns.length; j++) {
insertedColumns.put(table.getName() + "." + tableColumns[j].getName(),
columnValues[j]);
}
generateRandomValues(insertedColumns, table);

Column[] statementColumns = statement.getMetaData();
Object[] statementValues = new Object[statementColumns.length];
Expand All @@ -113,6 +163,7 @@ private void fillTables(Table... tables) {
}
try {
sqlTemplate.update(statement.getSql(), statementValues);
System.out.println("Succesfully inserted values in " + table.getName());
} catch (SqlException ex) {
log.error("Failed to process {} with values of {}", statement.getSql(),
ArrayUtils.toString(statementValues));
Expand All @@ -124,25 +175,20 @@ private void fillTables(Table... tables) {
}
}

private Object[] generateRandomValues(Map<String, Object> insertedColumns, Table table) {

private void generateRandomValues(Map<String, Object> insertedColumns, Table table) {
Column[] columns = table.getColumns();
List<Object> list = new ArrayList<Object>(columns.length);



for_column: for (Column column : columns) {

for_column:
for (Column column : columns) {
Object objectValue = null;

for (ForeignKey fk : table.getForeignKeys()) {
for (Reference ref : fk.getReferences()) {
if (ref.getLocalColumnName().equalsIgnoreCase(column.getName())) {
objectValue = insertedColumns.get(fk.getForeignTableName() + "."
+ ref.getForeignColumnName());

if (objectValue != null) {
list.add(objectValue);
insertedColumns.put(table.getName() + "." + column.getName(),
objectValue);
continue for_column;
} else {
throw new RuntimeException("The foreign key column, " + column.getName()
Expand All @@ -154,14 +200,14 @@ private Object[] generateRandomValues(Map<String, Object> insertedColumns, Table
}

int type = column.getMappedTypeCode();
if (type == Types.DATE || type == Types.TIMESTAMP || type == Types.TIME) {
if (type == Types.DATE || type == Types.TIMESTAMP || type == Types.TIME || type == -101) {
objectValue = randomDate();
} else if (type == Types.CHAR) {
objectValue = randomChar().toString();
} else if (type == Types.INTEGER || type == Types.BIGINT) {
objectValue = randomInt();
} else if (type == Types.BIT) {
objectValue = randomBit();
// } else if (type == Types.BIT) {
// objectValue = randomBit();
} else if (type == Types.SMALLINT) {
objectValue = randomSmallInt();
} else if (type == Types.FLOAT) {
Expand All @@ -173,7 +219,7 @@ private Object[] generateRandomValues(Map<String, Object> insertedColumns, Table
} else if (type == Types.NUMERIC || type == Types.DECIMAL
|| type == Types.REAL) {
objectValue = randomBigDecimal(column.getSizeAsInt(), column.getScale());
} else if (type == Types.BOOLEAN) {
} else if (type == Types.BOOLEAN || type == Types.BIT) {
objectValue = randomBoolean();
} else if (type == Types.BLOB || type == Types.LONGVARBINARY || type == Types.BINARY
|| type == Types.VARBINARY || type == Types.CLOB ||
Expand All @@ -184,13 +230,17 @@ private Object[] generateRandomValues(Map<String, Object> insertedColumns, Table
objectValue = null;
} else if (type == Types.VARCHAR || type == Types.LONGVARCHAR) {
objectValue = randomString(column.getSizeAsInt()>1000?1000:column.getSizeAsInt());
} else if (type == Types.OTHER) {
if ("UUID".equalsIgnoreCase(column.getJdbcTypeName())) {
objectValue = randomUUID();
}
}
if (objectValue == null) {
throw new RuntimeException("No random value generated for the object " + table.getName() + "." + column.getName() + " of type " + type);
}
list.add(objectValue);

insertedColumns.put(table.getName() + "." + column.getName(), objectValue);
}
return list.toArray();
}

private Object randomSmallInt() {
Expand All @@ -203,10 +253,7 @@ private Object randomFloat() {
}

private Object randomDouble() {
final long places = 1000000000l;
double d = Math.random()*places;
long l = Math.round(d);
return ((double)l)/(double)places+2 + (double)randomInt();
return new java.util.Random().nextDouble();
}

private Object randomTinyInt() {
Expand Down Expand Up @@ -255,7 +302,7 @@ private Character randomChar() {
char base = (rnd < 26) ? 'A' : 'a';
return (char) (base + rnd % 26);
}

private Date randomDate() {
// Random date between 1970 and 2020
long l = Math.abs(new java.util.Random().nextLong());
Expand All @@ -270,6 +317,10 @@ private Integer randomInt() {
private Integer randomBit() {
return new Integer(new java.util.Random().nextInt(1));
}

private String randomUUID() {
return UUID.randomUUID().toString();
}

protected String getSchemaToUse() {
if (StringUtils.isBlank(schema)) {
Expand Down Expand Up @@ -307,4 +358,20 @@ public void setSchema(String schema) {
this.schema = schema;
}

public boolean isCascading() {
return cascading;
}

public void setCascading(boolean cascading) {
this.cascading = cascading;
}

public String getIgnorePrefix() {
return ignorePrefix;
}

public void setIgnorePrefix(String ignorePrefix) {
this.ignorePrefix = ignorePrefix;
}

}
12 changes: 12 additions & 0 deletions symmetric-client/src/main/java/org/jumpmind/symmetric/DbFillCommand.java
View file
Expand Up @@ -12,6 +12,10 @@ public class DbFillCommand extends AbstractCommandLauncher {

private static final String OPTION_COUNT = "count";

private static final String OPTION_CASCADE = "cascade";

private static final String OPTION_IGNORE_TABLES = "ignore-prefix";

public DbFillCommand() {
super("dbfill", "[tablename...]", "DbFill.Option.");
}
Expand Down Expand Up @@ -43,6 +47,8 @@ protected void buildOptions(Options options) {
addOption(options, null, OPTION_SCHEMA, true);
addOption(options, null, OPTION_CATALOG, true);
addOption(options, null, OPTION_COUNT, true);
addOption(options, null, OPTION_CASCADE, false);
addOption(options, null, OPTION_IGNORE_TABLES, true);
}

@Override
Expand All @@ -58,6 +64,12 @@ protected boolean executeWithOptions(CommandLine line) throws Exception {
if (line.hasOption(OPTION_COUNT)) {
dbFill.setRecordCount(Integer.parseInt(line.getOptionValue(OPTION_COUNT)));
}
if (line.hasOption(OPTION_CASCADE)) {
dbFill.setCascading(true);
}
if (line.hasOption(OPTION_IGNORE_TABLES)) {
dbFill.setIgnorePrefix(line.getOptionValue(OPTION_IGNORE_TABLES));
}
String[] args = line.getArgs();
dbFill.fillTables(args);

Expand Down
2 changes: 2 additions & 0 deletions symmetric-client/src/main/resources/symmetric-messages.properties
View file
Expand Up @@ -108,3 +108,5 @@ DbImport.Option.filter-classes=A comma separated list of Java classes that imple
DbFill.Option.catalog=Look for tables in catalog.
DbFill.Option.count=The number of rows to generate in each table.
DbFill.Option.schema=Look for tables in schema.
DbFill.Option.cascade=Populate foreign key dependent tables not included in the table list.
DbFill.Option.ignore-prefix=Specify a string prefix to identify tables to ignore. This argument only works when no table names are provided. (e.g., "sym_")

0 comments on commit 896290b

Please sign in to comment.