Skip to content

Commit

Permalink
0003073: When auto.resolve.foreign.key.violation=true and circular fk
Browse files Browse the repository at this point in the history 
dependencies exist SymmetricDS can blow up with an OOM error
  • Loading branch information
@chenson42
chenson42 committed May 1, 2017
1 parent 639fbe9 commit d1df665
Showing 1 changed file with 101 additions and 65 deletions.
166 changes: 101 additions & 65 deletions symmetric-core/src/main/java/org/jumpmind/symmetric/service/impl/DataService.java
View file
Expand Up @@ -1550,7 +1550,7 @@ public void reloadMissingForeignKeyRows(String nodeId, long dataId) {
tableRows.add(new TableRow(table, row, null, null, null));
List<TableRow> foreignTableRows;
try {
foreignTableRows = getForeignTableRows(tableRows);
foreignTableRows = getForeignTableRows(tableRows, new HashSet<TableRow>());
} catch (CloneNotSupportedException e) {
throw new RuntimeException(e);
}
Expand Down Expand Up @@ -1589,83 +1589,87 @@ public void reloadMissingForeignKeyRows(String nodeId, long dataId) {
}
}

protected List<TableRow> getForeignTableRows(List<TableRow> tableRows) throws CloneNotSupportedException {
protected List<TableRow> getForeignTableRows(List<TableRow> tableRows, Set<TableRow> visited) throws CloneNotSupportedException {
List<TableRow> fkDepList = new ArrayList<TableRow>();
for (TableRow tableRow : tableRows) {
for (ForeignKey fk : tableRow.getTable().getForeignKeys()) {
Table table = platform.getTableFromCache(fk.getForeignTableName(), false);
if (table == null) {
table = fk.getForeignTable();
if (table == null) {
table = platform.getTableFromCache(tableRow.getTable().getCatalog(),
tableRow.getTable().getSchema(), fk.getForeignTableName(), false);
}
}
if (table != null) {
Table foreignTable = (Table) table.clone();
for (Column column : foreignTable.getColumns()) {
column.setPrimaryKey(false);
}
Row whereRow = new Row(fk.getReferenceCount());
String referenceColumnName = null;
boolean[] nullValues = new boolean[fk.getReferenceCount()];
int index = 0;
for (Reference ref : fk.getReferences()) {
Column foreignColumn = foreignTable.findColumn(ref.getForeignColumnName());
Object value = tableRow.getRow().get(ref.getLocalColumnName());
nullValues[index++] = value == null;
referenceColumnName = ref.getLocalColumnName();
whereRow.put(foreignColumn.getName(), value);
foreignColumn.setPrimaryKey(true);
}

boolean allNullValues = true;
for (boolean b : nullValues) {
if (!b) {
allNullValues = false;
break;
if (!visited.contains(tableRow)) {
visited.add(tableRow);
for (ForeignKey fk : tableRow.getTable().getForeignKeys()) {
Table table = platform.getTableFromCache(fk.getForeignTableName(), false);
if (table == null) {
table = fk.getForeignTable();
if (table == null) {
table = platform.getTableFromCache(tableRow.getTable().getCatalog(), tableRow.getTable().getSchema(),
fk.getForeignTableName(), false);
}
}

if (!allNullValues) {
DmlStatement whereSt = platform.createDmlStatement(DmlType.WHERE, foreignTable.getCatalog(), foreignTable.getSchema(),
foreignTable.getName(), foreignTable.getPrimaryKeyColumns(), foreignTable.getColumns(), nullValues, null);
String whereSql = whereSt.buildDynamicSql(symmetricDialect.getBinaryEncoding(), whereRow, false, true,
foreignTable.getPrimaryKeyColumns()).substring(6);
String delimiter = platform.getDatabaseInfo().getSqlCommandDelimiter();
if (delimiter != null && delimiter.length() > 0) {
whereSql = whereSql.substring(0, whereSql.length() - delimiter.length());
if (table != null) {
Table foreignTable = (Table) table.clone();
for (Column column : foreignTable.getColumns()) {
column.setPrimaryKey(false);
}
Row whereRow = new Row(fk.getReferenceCount());
String referenceColumnName = null;
boolean[] nullValues = new boolean[fk.getReferenceCount()];
int index = 0;
for (Reference ref : fk.getReferences()) {
Column foreignColumn = foreignTable.findColumn(ref.getForeignColumnName());
Object value = tableRow.getRow().get(ref.getLocalColumnName());
nullValues[index++] = value == null;
referenceColumnName = ref.getLocalColumnName();
whereRow.put(foreignColumn.getName(), value);
foreignColumn.setPrimaryKey(true);
}

Row foreignRow = new Row(foreignTable.getColumnCount());
if (foreignTable.getForeignKeyCount() > 0) {
DmlStatement selectSt = platform.createDmlStatement(DmlType.SELECT, foreignTable, null);
Object[] keys = whereRow.toArray(foreignTable.getPrimaryKeyColumnNames());
Map<String, Object> values = sqlTemplate.queryForMap(selectSt.getSql(), keys);
if (values == null) {
log.warn(
"Unable to reload rows for missing foreign key data for table '{}', parent data not found. Using sql='{}' with keys '{}'",
table.getName(), selectSt.getSql(), keys);
} else {
foreignRow.putAll(values);
boolean allNullValues = true;
for (boolean b : nullValues) {
if (!b) {
allNullValues = false;
break;
}
}

TableRow foreignTableRow = new TableRow(foreignTable, foreignRow, whereSql, referenceColumnName, fk.getName());
fkDepList.add(foreignTableRow);
log.debug("Add foreign table reference '{}' whereSql='{}'", foreignTable.getName(), whereSql);
if (!allNullValues) {
DmlStatement whereSt = platform.createDmlStatement(DmlType.WHERE, foreignTable.getCatalog(),
foreignTable.getSchema(), foreignTable.getName(), foreignTable.getPrimaryKeyColumns(),
foreignTable.getColumns(), nullValues, null);
String whereSql = whereSt.buildDynamicSql(symmetricDialect.getBinaryEncoding(), whereRow, false, true,
foreignTable.getPrimaryKeyColumns()).substring(6);
String delimiter = platform.getDatabaseInfo().getSqlCommandDelimiter();
if (delimiter != null && delimiter.length() > 0) {
whereSql = whereSql.substring(0, whereSql.length() - delimiter.length());
}

Row foreignRow = new Row(foreignTable.getColumnCount());
if (foreignTable.getForeignKeyCount() > 0) {
DmlStatement selectSt = platform.createDmlStatement(DmlType.SELECT, foreignTable, null);
Object[] keys = whereRow.toArray(foreignTable.getPrimaryKeyColumnNames());
Map<String, Object> values = sqlTemplate.queryForMap(selectSt.getSql(), keys);
if (values == null) {
log.warn(
"Unable to reload rows for missing foreign key data for table '{}', parent data not found. Using sql='{}' with keys '{}'",
table.getName(), selectSt.getSql(), keys);
} else {
foreignRow.putAll(values);
}
}

TableRow foreignTableRow = new TableRow(foreignTable, foreignRow, whereSql, referenceColumnName, fk.getName());
fkDepList.add(foreignTableRow);
log.debug("Add foreign table reference '{}' whereSql='{}'", foreignTable.getName(), whereSql);
} else {
log.debug("The foreign table reference was null for {}", foreignTable.getName());
}
} else {
log.debug("The foreign table reference was null for {}", foreignTable.getName());
log.debug("Foreign table '{}' not found for foreign key '{}'", fk.getForeignTableName(), fk.getName());
}
if (fkDepList.size() > 0) {
fkDepList.addAll(getForeignTableRows(fkDepList, visited));
}
} else {
log.debug("Foreign table '{}' not found for foreign key '{}'", fk.getForeignTableName(), fk.getName());
}
if (fkDepList.size() > 0) {
fkDepList.addAll(getForeignTableRows(fkDepList));
}
}
}

return fkDepList;
}

Expand Down Expand Up @@ -2090,6 +2094,7 @@ class TableRow {
String whereSql;
String referenceColumnName;
String fkName;
String fkColumnValues = null;

public TableRow(Table table, Row row, String whereSql, String referenceColumnName, String fkName) {
this.table = table;
Expand All @@ -2098,24 +2103,53 @@ public TableRow(Table table, Row row, String whereSql, String referenceColumnNam
this.referenceColumnName = referenceColumnName;
this.fkName = fkName;
}

protected String getFkColumnValues() {
if (fkColumnValues == null) {
StringBuilder builder = new StringBuilder();
ForeignKey[] keys = table.getForeignKeys();
for (ForeignKey foreignKey : keys) {
if (foreignKey.getName().equals(fkName)) {
Reference[] refs = foreignKey.getReferences();
for (Reference ref : refs) {
Object value = row.get(ref.getLocalColumnName());
if (value != null) {
builder.append("\"").append(value).append("\",");
} else {
builder.append("null,");
}
}
}
}
fkColumnValues = builder.toString();
}
return fkColumnValues;
}

@Override
public int hashCode() {
final int prime = 31;
int result = 1;
result = prime * result + ((table == null) ? 0 : table.hashCode());
result = prime * result + ((whereSql == null) ? 0 : whereSql.hashCode());
result = prime * result + ((getFkColumnValues() == null) ? 0 : getFkColumnValues().hashCode());
return result;
}

@Override
public boolean equals(Object o) {
if (o instanceof TableRow) {
TableRow tr = (TableRow) o;
return tr.table.equals(table) && tr.whereSql.equals(whereSql);
return tr.table.equals(table) && tr.whereSql.equals(whereSql)
&& tr.getFkColumnValues().equals(getFkColumnValues().toString());
}
return false;
}

@Override
public String toString() {
return table.getFullyQualifiedTableName() + ":" + whereSql + ":" + getFkColumnValues();
}

public Table getTable() {
return table;
Expand All @@ -2135,6 +2169,8 @@ public String getFkName() {
return fkName;
}



}

public class DataMapper implements ISqlRowMapper<Data> {
Expand Down

0 comments on commit d1df665

Please sign in to comment.