From fef88527c9660f6ab472a2176f435d48e0f0848a Mon Sep 17 00:00:00 2001 From: chenson42 Date: Sun, 23 Oct 2011 00:35:00 +0000 Subject: [PATCH] SYMMETRICDS-532 - only fallback to update if the error is truly a primary key violation. --- .../symmetric/db/AbstractDbDialect.java | 57 ++++++++++ .../org/jumpmind/symmetric/db/IDbDialect.java | 7 ++ .../symmetric/load/csv/CsvLoader.java | 64 ++++++----- .../transform/TransformDataLoader.java | 54 +++++---- .../org/jumpmind/symmetric/db/db2-zseries.xml | 1 + .../org/jumpmind/symmetric/db/db2.xml | 1 + .../org/jumpmind/symmetric/db/derby.xml | 1 + .../org/jumpmind/symmetric/db/firebird.xml | 3 +- .../org/jumpmind/symmetric/db/greenplum.xml | 107 +----------------- .../org/jumpmind/symmetric/db/h2.xml | 1 + .../org/jumpmind/symmetric/db/hsqldb2.xml | 1 + .../org/jumpmind/symmetric/db/informix.xml | 1 + .../org/jumpmind/symmetric/db/interbase.xml | 1 + .../org/jumpmind/symmetric/db/mssql.xml | 1 + .../org/jumpmind/symmetric/db/mysql.xml | 3 +- .../org/jumpmind/symmetric/db/oracle.xml | 3 +- .../org/jumpmind/symmetric/db/postgresql.xml | 1 + .../org/jumpmind/symmetric/db/sybase.xml | 1 + 18 files changed, 151 insertions(+), 157 deletions(-) diff --git a/symmetric/symmetric-core/src/main/java/org/jumpmind/symmetric/db/AbstractDbDialect.java b/symmetric/symmetric-core/src/main/java/org/jumpmind/symmetric/db/AbstractDbDialect.java index 1163febf00..a5b8961092 100644 --- a/symmetric/symmetric-core/src/main/java/org/jumpmind/symmetric/db/AbstractDbDialect.java +++ b/symmetric/symmetric-core/src/main/java/org/jumpmind/symmetric/db/AbstractDbDialect.java @@ -165,6 +165,10 @@ abstract public class AbstractDbDialect implements IDbDialect { protected int queryTimeoutInSeconds = 300; + protected int[] primaryKeyViolationCodes; + + protected String[] primaryKeyViolationSqlStates; + protected List databaseUpgradeListeners = new ArrayList(); protected AbstractDbDialect() { @@ -1688,5 +1692,58 @@ public StatementBuilder createStatementBuilder(DmlType type, String tableName, C return new StatementBuilder(type, tableName, keys, columns, preFilteredColumns, isDateOverrideToTimestamp(), getIdentifierQuoteString()); + } + + public void setPrimaryKeyViolationCodes(int[] primaryKeyViolationCodes) { + this.primaryKeyViolationCodes = primaryKeyViolationCodes; + } + + public void setPrimaryKeyViolationSqlStates(String[] primaryKeyViolationSqlStates) { + this.primaryKeyViolationSqlStates = primaryKeyViolationSqlStates; + } + + public boolean isPrimaryKeyViolation(Exception ex) { + boolean primaryKeyViolation = false; + if (primaryKeyViolationCodes != null || primaryKeyViolationSqlStates != null) { + SQLException sqlEx = findSQLException(ex); + if (sqlEx != null) { + if (primaryKeyViolationCodes != null) { + int errorCode = sqlEx.getErrorCode(); + for (int primaryKeyViolationCode : primaryKeyViolationCodes) { + if (primaryKeyViolationCode == errorCode) { + primaryKeyViolation = true; + break; + } + } + } + + if (primaryKeyViolationSqlStates != null) { + String sqlState = sqlEx.getSQLState(); + if (sqlState != null) { + for (String primaryKeyViolationSqlState : primaryKeyViolationSqlStates) { + if (primaryKeyViolationSqlState != null + && primaryKeyViolationSqlState.equals(sqlState)) { + primaryKeyViolation = true; + break; + } + } + } + } + } + } + + return primaryKeyViolation; + } + + protected SQLException findSQLException(Throwable ex) { + if (ex instanceof SQLException) { + return (SQLException)ex; + } else { + Throwable cause = ex.getCause(); + if (cause != null && !cause.equals(ex)) { + return findSQLException(ex); + } + } + return null; } } \ No newline at end of file diff --git a/symmetric/symmetric-core/src/main/java/org/jumpmind/symmetric/db/IDbDialect.java b/symmetric/symmetric-core/src/main/java/org/jumpmind/symmetric/db/IDbDialect.java index 61201e7024..5c9ccd5db2 100644 --- a/symmetric/symmetric-core/src/main/java/org/jumpmind/symmetric/db/IDbDialect.java +++ b/symmetric/symmetric-core/src/main/java/org/jumpmind/symmetric/db/IDbDialect.java @@ -334,5 +334,12 @@ public long insertWithGeneratedKey(JdbcTemplate jdbcTemplate, final String sql, public StatementBuilder createStatementBuilder(DmlType type, String tableName, Column[] keys, Column[] columns, Column[] preFilteredColumns); + + /** + * Check to see if the passed in exception (or a nested exception) was caused by a primary key violation. + * @param ex The exception to check + * @return true if the exception was caused by a primary key violation + */ + public boolean isPrimaryKeyViolation(Exception ex); } \ No newline at end of file diff --git a/symmetric/symmetric-core/src/main/java/org/jumpmind/symmetric/load/csv/CsvLoader.java b/symmetric/symmetric-core/src/main/java/org/jumpmind/symmetric/load/csv/CsvLoader.java index 01b6dfb791..1448d41c76 100644 --- a/symmetric/symmetric-core/src/main/java/org/jumpmind/symmetric/load/csv/CsvLoader.java +++ b/symmetric/symmetric-core/src/main/java/org/jumpmind/symmetric/load/csv/CsvLoader.java @@ -58,7 +58,6 @@ import org.jumpmind.symmetric.statistic.StatisticConstants; import org.jumpmind.symmetric.util.AppUtils; import org.jumpmind.symmetric.util.CsvUtils; -import org.springframework.dao.DataIntegrityViolationException; import org.springframework.jdbc.core.JdbcTemplate; import org.springframework.jdbc.datasource.SingleConnectionDataSource; @@ -374,33 +373,44 @@ protected int insert(String[] tokens) { } if (continueToLoad) { - String keyValues[] = context.getTableTemplate().parseKeys(tokens, 1); - boolean enableFallbackUpdate = parameterService.is(ParameterConstants.DATA_LOADER_ENABLE_FALLBACK_UPDATE); - try { - stats.startTimer(); - rows = context.getTableTemplate().insert(context, columnValues, keyValues); - if (rows <= 0) { - throw new DataIntegrityViolationException("Insert was not processed"); - } - } catch (DataIntegrityViolationException e) { - // TODO: modify sql-error-codes.xml for unique constraint vs. foreign key - if (enableFallbackUpdate) { - if (log.isDebugEnabled()) { - log.debug("LoaderInsertingFailedUpdating", context.getTableName(), ArrayUtils.toString(tokens)); - } - stats.incrementFallbackUpdateCount(); - rows = context.getTableTemplate().update(context, columnValues, keyValues); + String keyValues[] = context.getTableTemplate().parseKeys(tokens, 1); + boolean attemptFallbackUpdate = false; + RuntimeException insertException = null; + try { + stats.startTimer(); + try { + rows = context.getTableTemplate().insert(context, columnValues, keyValues); if (rows == 0) { - throw new SymmetricException("LoaderFallbackUpdateFailed", e, context.getTableTemplate().getTable().toVerboseString(), ArrayUtils - .toString(tokens), ArrayUtils.toString(keyValues)); - } - } else { - log.error("LoaderInsertingFailed", context.getTableName(), ArrayUtils.toString(tokens)); - throw e; - } - } finally { - stats.incrementDatabaseMillis(stats.endTimer()); - } + attemptFallbackUpdate = parameterService + .is(ParameterConstants.DATA_LOADER_ENABLE_FALLBACK_UPDATE); + } + } catch (RuntimeException e) { + insertException = e; + attemptFallbackUpdate = dbDialect.isPrimaryKeyViolation(e) + && parameterService + .is(ParameterConstants.DATA_LOADER_ENABLE_FALLBACK_UPDATE); + if (!attemptFallbackUpdate) { + throw e; + } + } + + if (attemptFallbackUpdate) { + if (log.isDebugEnabled()) { + log.debug("LoaderInsertingFailedUpdating", context.getTableName(), + ArrayUtils.toString(tokens)); + } + stats.incrementFallbackUpdateCount(); + rows = context.getTableTemplate().update(context, columnValues, keyValues); + if (rows == 0) { + throw new SymmetricException("LoaderFallbackUpdateFailed", insertException, context + .getTableTemplate().getTable().toVerboseString(), + ArrayUtils.toString(tokens), ArrayUtils.toString(keyValues)); + } + } + + } finally { + stats.incrementDatabaseMillis(stats.endTimer()); + } } return rows; } diff --git a/symmetric/symmetric-core/src/main/java/org/jumpmind/symmetric/transform/TransformDataLoader.java b/symmetric/symmetric-core/src/main/java/org/jumpmind/symmetric/transform/TransformDataLoader.java index 7566faa828..687d898d06 100644 --- a/symmetric/symmetric-core/src/main/java/org/jumpmind/symmetric/transform/TransformDataLoader.java +++ b/symmetric/symmetric-core/src/main/java/org/jumpmind/symmetric/transform/TransformDataLoader.java @@ -32,7 +32,6 @@ import org.jumpmind.symmetric.load.IMissingTableHandler; import org.jumpmind.symmetric.load.StatementBuilder.DmlType; import org.jumpmind.symmetric.load.TableTemplate; -import org.springframework.dao.DataIntegrityViolationException; public class TransformDataLoader extends AbstractTransformer implements IBuiltInExtensionPoint, IDataLoaderFilter, IMissingTableHandler { @@ -109,30 +108,42 @@ protected void apply(IDataLoaderContext context, tableTemplate.setKeyNames(data.getKeyNames()); switch (data.getTargetDmlType()) { case INSERT: - boolean enableFallbackUpdate = parameterService - .is(ParameterConstants.DATA_LOADER_ENABLE_FALLBACK_UPDATE); Table table = tableTemplate.getTable(); + boolean attemptFallbackUpdate = false; + RuntimeException insertException = null; try { - if (data.isGeneratedIdentityNeeded()) { - if (log.isDebugEnabled()) { - log.debug("TransformEnablingGeneratedIdentity", table.getName()); + try { + if (data.isGeneratedIdentityNeeded()) { + if (log.isDebugEnabled()) { + log.debug("TransformEnablingGeneratedIdentity", table.getName()); + } + dbDialect.revertAllowIdentityInserts(context.getJdbcTemplate(), table); + } else if (table.hasAutoIncrementColumn()) { + dbDialect.allowIdentityInserts(context.getJdbcTemplate(), table); } - dbDialect.revertAllowIdentityInserts(context.getJdbcTemplate(), table); - } else if (table.hasAutoIncrementColumn()) { - dbDialect.allowIdentityInserts(context.getJdbcTemplate(), table); - } - if (tableTemplate.insert(context, data.getColumnValues(), data.getKeyValues()) <= 0) { - throw new DataIntegrityViolationException("Insert not executed"); + if (tableTemplate.insert(context, data.getColumnValues(), + data.getKeyValues()) == 0) { + attemptFallbackUpdate = parameterService + .is(ParameterConstants.DATA_LOADER_ENABLE_FALLBACK_UPDATE); + } + } catch (RuntimeException ex) { + insertException = ex; + if (dbDialect.isPrimaryKeyViolation(ex) + && parameterService + .is(ParameterConstants.DATA_LOADER_ENABLE_FALLBACK_UPDATE)) { + attemptFallbackUpdate = true; + } else { + throw ex; + } } - } catch (DataIntegrityViolationException ex) { - if (enableFallbackUpdate) { + if (attemptFallbackUpdate) { List newlyTransformedDatas = transform(DmlType.UPDATE, context, data.getTransformation(), data.getSourceKeyValues(), data.getOldSourceValues(), data.getSourceValues()); for (TransformedData newlyTransformedData : newlyTransformedDatas) { - if (newlyTransformedData.hasSameKeyValues(data.getKeyValues()) || - data.isGeneratedIdentityNeeded()) { + if (newlyTransformedData.hasSameKeyValues(data.getKeyValues()) + || data.isGeneratedIdentityNeeded()) { if (newlyTransformedData.getKeyNames() != null && newlyTransformedData.getKeyNames().length > 0) { tableTemplate.setColumnNames(newlyTransformedData @@ -142,7 +153,8 @@ protected void apply(IDataLoaderContext context, newlyTransformedData.getColumnValues(), newlyTransformedData.getKeyValues())) { throw new SymmetricException("LoaderFallbackUpdateFailed", - ex, tableTemplate.getTable().toVerboseString(), + insertException, tableTemplate.getTable() + .toVerboseString(), ArrayUtils.toString(data.getColumnValues()), ArrayUtils.toString(data.getKeyValues())); } @@ -150,19 +162,19 @@ protected void apply(IDataLoaderContext context, // If not keys are specified we are going to // assume that this is intentional and we // will simply log a warning and not fail. - log.warn("Message", ex.getMessage()); + log.warn("Message", insertException.getMessage()); log.warn("TransformNoPrimaryKeyDefinedNoUpdate", newlyTransformedData.getTransformation() .getTransformId()); } } else { - log.debug("TransformMatchingFallbackNotFound", DmlType.UPDATE.name()); + log.debug("TransformMatchingFallbackNotFound", + DmlType.UPDATE.name()); } } - } else { - throw ex; } + } finally { if (table.hasAutoIncrementColumn()) { dbDialect.revertAllowIdentityInserts(context.getJdbcTemplate(), table); diff --git a/symmetric/symmetric-core/src/main/resources/org/jumpmind/symmetric/db/db2-zseries.xml b/symmetric/symmetric-core/src/main/resources/org/jumpmind/symmetric/db/db2-zseries.xml index 28572deed0..3dd1ea2cc4 100644 --- a/symmetric/symmetric-core/src/main/resources/org/jumpmind/symmetric/db/db2-zseries.xml +++ b/symmetric/symmetric-core/src/main/resources/org/jumpmind/symmetric/db/db2-zseries.xml @@ -10,6 +10,7 @@ + diff --git a/symmetric/symmetric-core/src/main/resources/org/jumpmind/symmetric/db/db2.xml b/symmetric/symmetric-core/src/main/resources/org/jumpmind/symmetric/db/db2.xml index 946a888f6d..6ff81a5fa5 100644 --- a/symmetric/symmetric-core/src/main/resources/org/jumpmind/symmetric/db/db2.xml +++ b/symmetric/symmetric-core/src/main/resources/org/jumpmind/symmetric/db/db2.xml @@ -10,6 +10,7 @@ + diff --git a/symmetric/symmetric-core/src/main/resources/org/jumpmind/symmetric/db/derby.xml b/symmetric/symmetric-core/src/main/resources/org/jumpmind/symmetric/db/derby.xml index 609992957c..05135881f9 100644 --- a/symmetric/symmetric-core/src/main/resources/org/jumpmind/symmetric/db/derby.xml +++ b/symmetric/symmetric-core/src/main/resources/org/jumpmind/symmetric/db/derby.xml @@ -8,6 +8,7 @@ + diff --git a/symmetric/symmetric-core/src/main/resources/org/jumpmind/symmetric/db/firebird.xml b/symmetric/symmetric-core/src/main/resources/org/jumpmind/symmetric/db/firebird.xml index 887486b051..35b5c32c36 100644 --- a/symmetric/symmetric-core/src/main/resources/org/jumpmind/symmetric/db/firebird.xml +++ b/symmetric/symmetric-core/src/main/resources/org/jumpmind/symmetric/db/firebird.xml @@ -7,7 +7,8 @@ http://www.springframework.org/schema/tx http://www.springframework.org/schema/tx/spring-tx-3.0.xsd" default-lazy-init="true"> + scope="prototype"> + diff --git a/symmetric/symmetric-core/src/main/resources/org/jumpmind/symmetric/db/greenplum.xml b/symmetric/symmetric-core/src/main/resources/org/jumpmind/symmetric/db/greenplum.xml index 8f0718e95c..f12f321de3 100644 --- a/symmetric/symmetric-core/src/main/resources/org/jumpmind/symmetric/db/greenplum.xml +++ b/symmetric/symmetric-core/src/main/resources/org/jumpmind/symmetric/db/greenplum.xml @@ -8,6 +8,7 @@ + @@ -127,112 +128,6 @@ - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - diff --git a/symmetric/symmetric-core/src/main/resources/org/jumpmind/symmetric/db/h2.xml b/symmetric/symmetric-core/src/main/resources/org/jumpmind/symmetric/db/h2.xml index c93fae2553..1f6628a256 100644 --- a/symmetric/symmetric-core/src/main/resources/org/jumpmind/symmetric/db/h2.xml +++ b/symmetric/symmetric-core/src/main/resources/org/jumpmind/symmetric/db/h2.xml @@ -16,6 +16,7 @@ + diff --git a/symmetric/symmetric-core/src/main/resources/org/jumpmind/symmetric/db/hsqldb2.xml b/symmetric/symmetric-core/src/main/resources/org/jumpmind/symmetric/db/hsqldb2.xml index f8b531cf2d..728366f0d3 100644 --- a/symmetric/symmetric-core/src/main/resources/org/jumpmind/symmetric/db/hsqldb2.xml +++ b/symmetric/symmetric-core/src/main/resources/org/jumpmind/symmetric/db/hsqldb2.xml @@ -16,6 +16,7 @@ + diff --git a/symmetric/symmetric-core/src/main/resources/org/jumpmind/symmetric/db/informix.xml b/symmetric/symmetric-core/src/main/resources/org/jumpmind/symmetric/db/informix.xml index 799ba724bc..607133d325 100644 --- a/symmetric/symmetric-core/src/main/resources/org/jumpmind/symmetric/db/informix.xml +++ b/symmetric/symmetric-core/src/main/resources/org/jumpmind/symmetric/db/informix.xml @@ -8,6 +8,7 @@ + diff --git a/symmetric/symmetric-core/src/main/resources/org/jumpmind/symmetric/db/interbase.xml b/symmetric/symmetric-core/src/main/resources/org/jumpmind/symmetric/db/interbase.xml index 1c92859d8a..a976520580 100644 --- a/symmetric/symmetric-core/src/main/resources/org/jumpmind/symmetric/db/interbase.xml +++ b/symmetric/symmetric-core/src/main/resources/org/jumpmind/symmetric/db/interbase.xml @@ -8,6 +8,7 @@ + diff --git a/symmetric/symmetric-core/src/main/resources/org/jumpmind/symmetric/db/mssql.xml b/symmetric/symmetric-core/src/main/resources/org/jumpmind/symmetric/db/mssql.xml index 349e2e9f53..8b60c973a4 100644 --- a/symmetric/symmetric-core/src/main/resources/org/jumpmind/symmetric/db/mssql.xml +++ b/symmetric/symmetric-core/src/main/resources/org/jumpmind/symmetric/db/mssql.xml @@ -16,6 +16,7 @@ + diff --git a/symmetric/symmetric-core/src/main/resources/org/jumpmind/symmetric/db/mysql.xml b/symmetric/symmetric-core/src/main/resources/org/jumpmind/symmetric/db/mysql.xml index ef2af3220d..42b205fe11 100644 --- a/symmetric/symmetric-core/src/main/resources/org/jumpmind/symmetric/db/mysql.xml +++ b/symmetric/symmetric-core/src/main/resources/org/jumpmind/symmetric/db/mysql.xml @@ -15,7 +15,8 @@ + scope="prototype"> + diff --git a/symmetric/symmetric-core/src/main/resources/org/jumpmind/symmetric/db/oracle.xml b/symmetric/symmetric-core/src/main/resources/org/jumpmind/symmetric/db/oracle.xml index 63d526631a..8956b8f049 100644 --- a/symmetric/symmetric-core/src/main/resources/org/jumpmind/symmetric/db/oracle.xml +++ b/symmetric/symmetric-core/src/main/resources/org/jumpmind/symmetric/db/oracle.xml @@ -6,7 +6,8 @@ http://www.springframework.org/schema/beans http://www.springframework.org/schema/beans/spring-beans-3.0.xsd http://www.springframework.org/schema/tx http://www.springframework.org/schema/tx/spring-tx-3.0.xsd" default-lazy-init="true"> - + + diff --git a/symmetric/symmetric-core/src/main/resources/org/jumpmind/symmetric/db/postgresql.xml b/symmetric/symmetric-core/src/main/resources/org/jumpmind/symmetric/db/postgresql.xml index 1254bd02d7..6b4debfd1a 100644 --- a/symmetric/symmetric-core/src/main/resources/org/jumpmind/symmetric/db/postgresql.xml +++ b/symmetric/symmetric-core/src/main/resources/org/jumpmind/symmetric/db/postgresql.xml @@ -8,6 +8,7 @@ + diff --git a/symmetric/symmetric-core/src/main/resources/org/jumpmind/symmetric/db/sybase.xml b/symmetric/symmetric-core/src/main/resources/org/jumpmind/symmetric/db/sybase.xml index 75145a3baf..1bf5b2a408 100644 --- a/symmetric/symmetric-core/src/main/resources/org/jumpmind/symmetric/db/sybase.xml +++ b/symmetric/symmetric-core/src/main/resources/org/jumpmind/symmetric/db/sybase.xml @@ -8,6 +8,7 @@ +