From a97e1746a98631db9b3e23654f04fbc98ad1ceb0 Mon Sep 17 00:00:00 2001 From: chenson42 Date: Thu, 30 Jan 2014 14:54:54 +0000 Subject: [PATCH] 0001529: Columns in old data are not transformed causing conflict detection --- symmetric-io/pom.xml | 20 ++ .../io/data/transform/TransformedData.java | 134 +++++++++++-- .../data/transform/TransformedDataTest.java | 179 ++++++++++++++++++ symmetric-parent/pom.xml | 25 +++ 4 files changed, 339 insertions(+), 19 deletions(-) create mode 100644 symmetric-io/src/test/java/org/jumpmind/symmetric/io/data/transform/TransformedDataTest.java diff --git a/symmetric-io/pom.xml b/symmetric-io/pom.xml index 0b75cfe9a6..8768e3f975 100644 --- a/symmetric-io/pom.xml +++ b/symmetric-io/pom.xml @@ -114,5 +114,25 @@ jeval 0.9.4 + + org.mockito + mockito-all + test + + + org.powermock + powermock-api-mockito + test + + + org.powermock + powermock-module-junit4 + test + + + org.powermock + powermock-core + test + diff --git a/symmetric-io/src/main/java/org/jumpmind/symmetric/io/data/transform/TransformedData.java b/symmetric-io/src/main/java/org/jumpmind/symmetric/io/data/transform/TransformedData.java index 12e618b976..18abcdf1c8 100644 --- a/symmetric-io/src/main/java/org/jumpmind/symmetric/io/data/transform/TransformedData.java +++ b/symmetric-io/src/main/java/org/jumpmind/symmetric/io/data/transform/TransformedData.java @@ -53,8 +53,9 @@ public class TransformedData implements Cloneable { protected Map sourceValues; public TransformedData(TransformTable transformation, DataEventType sourceDmlType, - Map sourceKeyValues, Map oldSourceValues, - Map sourceValues) { + Map sourceKeyValues, Map oldSourceValues, + Map sourceValues) { + this.transformation = transformation; this.targetDmlType = sourceDmlType; this.sourceDmlType = sourceDmlType; @@ -64,30 +65,37 @@ public TransformedData(TransformTable transformation, DataEventType sourceDmlTyp } public String getFullyQualifiedTableName() { + return transformation.getFullyQualifiedTargetTableName(); } public DataEventType getTargetDmlType() { + return targetDmlType; } public void setTargetDmlType(DataEventType dmlType) { + this.targetDmlType = dmlType; } public String getTableName() { + return transformation.getTargetTableName(); } public String getCatalogName() { + return transformation.getTargetCatalogName(); } public String getSchemaName() { + return transformation.getTargetSchemaName(); } public void put(TransformColumn column, String columnValue, boolean recordAsKey) { + if (recordAsKey) { if (keysBy == null) { keysBy = new HashMap>( @@ -114,6 +122,7 @@ public void put(TransformColumn column, String columnValue, boolean recordAsKey) protected List retrieve( Map> source, boolean getColumnNames) { + List list = new ArrayList(source == null ? 0 : source.size()); if (source != null) { LinkedHashMap values = source.get(IncludeOnType.ALL); @@ -145,30 +154,36 @@ protected List retrieve( } public String[] getKeyNames() { + List list = retrieve(keysBy, true); return list.toArray(new String[list.size()]); } public String[] getKeyValues() { + List list = retrieve(keysBy, false); return list.toArray(new String[list.size()]); } public String[] getColumnNames() { + List list = retrieve(columnsBy, true); return list.toArray(new String[list.size()]); } public String[] getColumnValues() { + List list = retrieve(columnsBy, false); return list.toArray(new String[list.size()]); } public DataEventType getSourceDmlType() { + return sourceDmlType; } public TransformedData copy() { + try { TransformedData clone = (TransformedData) this.clone(); clone.columnsBy = copy(columnsBy); @@ -180,19 +195,23 @@ public TransformedData copy() { } public TransformTable getTransformation() { + return transformation; } public void setGeneratedIdentityNeeded(boolean generatedIdentityNeeded) { + this.generatedIdentityNeeded = generatedIdentityNeeded; } public boolean isGeneratedIdentityNeeded() { + return generatedIdentityNeeded; } protected Map> copy( Map> toCopy) { + Map> newMap = new HashMap>( toCopy.size()); for (TransformColumn.IncludeOnType key : toCopy.keySet()) { @@ -203,18 +222,22 @@ protected Map> copy } public Map getSourceKeyValues() { + return sourceKeyValues; } public Map getOldSourceValues() { + return oldSourceValues; } public Map getSourceValues() { + return sourceValues; } public boolean hasSameKeyValues(String[] otherKeyValues) { + String[] keyValues = getKeyValues(); if (otherKeyValues != null) { if (keyValues != null) { @@ -236,6 +259,7 @@ public boolean hasSameKeyValues(String[] otherKeyValues) { } public Table buildTargetTable() { + Table table = null; String[] columnNames = getColumnNames(); String[] keyNames = getKeyNames(); @@ -258,6 +282,7 @@ public Table buildTargetTable() { } public CsvData buildTargetCsvData() { + CsvData data = new CsvData(this.targetDmlType); data.putParsedData(CsvData.OLD_DATA, getOldColumnValues()); data.putParsedData(CsvData.ROW_DATA, getColumnValues()); @@ -267,29 +292,100 @@ public CsvData buildTargetCsvData() { } public String[] getOldColumnValues() { + List names = retrieve(columnsBy, true); List keyNames = retrieve(keysBy, true); List keyValues = retrieve(keysBy, false); List values = new ArrayList(); + for (String name : names) { - if (keyNames.contains(name)) { - /* - * Must always use the transformed value for the key else - * deletes and updates won't work. - */ - values.add(keyValues.get(keyNames.indexOf(name))); - } else { - if (oldSourceValues.containsKey(name)) { - values.add(oldSourceValues.get(name)); - } else { - /* - * TODO copy transforms should probably copy the old value - * to the new value. Currently, it will be null. - */ - values.add(null); - } - } + addOldValue(keyNames, keyValues, values, name); } + return values.toArray(new String[values.size()]); } + + private void addOldValue(List keyNames, List keyValues, List values, String name) { + + if (keyNames.contains(name)) { + /* + * Must always use the transformed value for the key else + * deletes and updates won't work. + */ + values.add(keyValues.get(keyNames.indexOf(name))); + return; + } + + TransformColumn transformColumn = findTransformColumn(name); + + if (null == transformColumn) { + if (oldSourceValues.containsKey(name)) { + values.add(oldSourceValues.get(name)); + return; + } + + values.add(null); + return; + } + + String transformType = transformColumn.getTransformType(); + + if (CopyColumnTransform.NAME.equals(transformType)) { + values.add(oldSourceValues.get(transformColumn.getSourceColumnName())); + return; + } else if (RemoveColumnTransform.NAME.equalsIgnoreCase(transformType)) { + values.add(null); + return; + } else if (ConstantColumnTransform.NAME.equalsIgnoreCase(transformType)) { + values.add(transformColumn.getTransformExpression()); + return; + } else if (SubstrColumnTransform.NAME.equalsIgnoreCase(transformType)) { + + String value = oldSourceValues.get(transformColumn.getSourceColumnName()); + + try { + values.add(new SubstrColumnTransform().transform( + null, // IDatabasePlatform + null, // DataContext + transformColumn, + null, // TransformedData + null, // Map sourceValue + value, // String newValue + null // String oldValue + )); + } catch (IgnoreColumnException e) { + throw new RuntimeException(e); + } catch (IgnoreRowException e) { + throw new RuntimeException(e); + } + + return; + } else if (doesSelectedConflictTypeRequireOldData()) { + throw new UnsupportedOperationException( + "'" + transformType + "' transformation of OLD values hasn't been implemented" + ); + } + + values.add(null); + } + + private boolean doesSelectedConflictTypeRequireOldData() { + + // TODO return true if USE_OLD_DATA or USE_CHANGED_DATA + // needed to inject DatabaseWriterSettings and invoke the method Conflict pickConflict(Table table, Batch batch) + + // Conflict.DetectConflict type = DatabaseWriterSettings#pickConflict(table, batch).getDetectType(); + // return type == Conflict.DetectConflict.USE_OLD_DATA || type == Conflict.DetectConflict.USE_CHANGED_DATA; + + return false; + } + + private TransformColumn findTransformColumn(String name) { + + for (TransformColumn tc : transformation.getTransformColumns()) + if (tc.getTargetColumnName().equals(name)) + return tc; + return null; + } + } diff --git a/symmetric-io/src/test/java/org/jumpmind/symmetric/io/data/transform/TransformedDataTest.java b/symmetric-io/src/test/java/org/jumpmind/symmetric/io/data/transform/TransformedDataTest.java new file mode 100644 index 0000000000..ec06bbbadf --- /dev/null +++ b/symmetric-io/src/test/java/org/jumpmind/symmetric/io/data/transform/TransformedDataTest.java @@ -0,0 +1,179 @@ +package org.jumpmind.symmetric.io.data.transform; + +import org.junit.After; +import org.junit.Before; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.powermock.core.classloader.annotations.PrepareForTest; +import org.powermock.modules.junit4.PowerMockRunner; + +import java.util.*; + +import static java.util.Arrays.asList; +import static org.hamcrest.core.IsEqual.equalTo; +import static org.hamcrest.core.IsNull.nullValue; +import static org.jumpmind.symmetric.io.data.DataEventType.UPDATE; +import static org.jumpmind.symmetric.io.data.transform.CopyColumnTransform.NAME; +import static org.junit.Assert.assertThat; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; +import static org.powermock.api.mockito.PowerMockito.whenNew; +import static org.powermock.reflect.Whitebox.invokeMethod; + +@RunWith(PowerMockRunner.class) +@PrepareForTest({TransformedData.class}) +public class TransformedDataTest { + + public static final String ADD_VALUE_METHOD = "addOldValue"; + private TransformedData transformedData; + private TransformTable transformation; + private HashMap sourceKeyValues; + private HashMap oldSourceValues; + private HashMap sourceValues; + private List keyNames; + private List keyValues; + private List values; + public static final String COLUMN_VALUE = String.valueOf(Integer.MAX_VALUE); + private TransformColumn transformColumn; + public static final String TARGET_NAME = "code_store_table"; + public static final String SOURCE_NAME = "code_corp_table"; + + @Before + public void setUp() throws Exception { + + keyNames = new ArrayList(); + keyValues = new ArrayList(); + values = new ArrayList(); + + transformedData = new TransformedData( + transformation = mock(TransformTable.class), + UPDATE, + sourceKeyValues = new HashMap(), + oldSourceValues = new HashMap(), + sourceValues = new HashMap() + ); + + transformColumn = mock(TransformColumn.class); + } + + @SuppressWarnings("rawtypes") + @After + public void tearDown() throws Exception { + + for (Collection c : new Collection[]{keyNames, keyValues, values}) { + c.clear(); + } + + for (Map m : new Map[]{sourceKeyValues, oldSourceValues, sourceValues}) { + m.clear(); + } + } + + @Test + public void addKeyValue() throws Exception { + + String name = "id_store_table"; + String value = String.valueOf(Integer.MAX_VALUE); + + keyNames.add(name); + keyValues.add(value); + + invokeMethod(transformedData, ADD_VALUE_METHOD, keyNames, keyValues, values, name); + + assertThat(values.iterator().next(), equalTo(value)); + } + + @Test + public void addValueWithoutTransformation() throws Exception { + + String name = "column_same_name_on_source_and_target_nodes"; + + oldSourceValues.put(name, COLUMN_VALUE); + + invokeMethod(transformedData, ADD_VALUE_METHOD, keyNames, keyValues, values, name); + + assertThat(values.iterator().next(), equalTo(COLUMN_VALUE)); + } + + @Test + public void addNoValueWithoutTransformation() throws Exception { + + String name = "column_same_name_on_source_and_target_nodes"; + invokeMethod(transformedData, ADD_VALUE_METHOD, keyNames, keyValues, values, name); + + assertThat(values.iterator().next(), nullValue()); + } + + @Test + public void addValueCopy() throws Exception { + + setTransformColumn(TARGET_NAME, SOURCE_NAME, NAME); + + oldSourceValues.put(SOURCE_NAME, COLUMN_VALUE); + + invokeMethod(transformedData, ADD_VALUE_METHOD, keyNames, keyValues, values, TARGET_NAME); + + assertThat(values.iterator().next(), equalTo(COLUMN_VALUE)); + } + + @Test + public void removeTransformedValue() throws Exception { + + setTransformColumn(TARGET_NAME, SOURCE_NAME, "remove"); + + oldSourceValues.put(SOURCE_NAME, COLUMN_VALUE); + + invokeMethod(transformedData, ADD_VALUE_METHOD, keyNames, keyValues, values, TARGET_NAME); + + assertThat(values.iterator().next(), nullValue()); + } + + @Test + public void addTransformedConstant() throws Exception { + + String someConstantValue = "some constant value"; + when(transformColumn.getTransformExpression()).thenReturn(someConstantValue); + setTransformColumn(TARGET_NAME, SOURCE_NAME, "const"); + + oldSourceValues.put(SOURCE_NAME, COLUMN_VALUE); + + invokeMethod(transformedData, ADD_VALUE_METHOD, keyNames, keyValues, values, TARGET_NAME); + + assertThat(values.iterator().next(), equalTo(someConstantValue)); + } + + @Test + public void addTransformedSubstringValue() throws Exception { + + setTransformColumn(TARGET_NAME, SOURCE_NAME, "substr"); + + oldSourceValues.put(SOURCE_NAME, COLUMN_VALUE); + + SubstrColumnTransform substrColumnTransform = mock(SubstrColumnTransform.class); + + String substring = COLUMN_VALUE.substring(1, 2); + when(substrColumnTransform.transform( + null, // IDatabasePlatform + null, // DataContext + transformColumn, + null, // TransformedData + null, // Map sourceValue + COLUMN_VALUE, // String newValue + null // String oldValue + )).thenReturn(substring); + + whenNew(SubstrColumnTransform.class).withNoArguments().thenReturn(substrColumnTransform); + + invokeMethod(transformedData, ADD_VALUE_METHOD, keyNames, keyValues, values, TARGET_NAME); + + assertThat(values.iterator().next(), equalTo(substring)); + } + + private void setTransformColumn(String targetName, String sourceName, String transformType) { + + when(transformColumn.getTargetColumnName()).thenReturn(targetName); + when(transformColumn.getTransformType()).thenReturn(transformType); + when(transformColumn.getSourceColumnName()).thenReturn(sourceName); + when(transformation.getTransformColumns()).thenReturn(asList(transformColumn)); + } +} diff --git a/symmetric-parent/pom.xml b/symmetric-parent/pom.xml index 2eb4c6884b..a9c647e921 100644 --- a/symmetric-parent/pom.xml +++ b/symmetric-parent/pom.xml @@ -27,6 +27,7 @@ 5.1.17 1.1.1 1.6.4 + 1.5.3 h2 h2 UTF-8 @@ -913,6 +914,30 @@ xerial-sqlite ${version.sqlite} + + org.mockito + mockito-all + 1.9.5 + test + + + org.powermock + powermock-api-mockito + ${version.powermock} + test + + + org.powermock + powermock-module-junit4 + ${version.powermock} + test + + + org.powermock + powermock-core + ${version.powermock} + test + \ No newline at end of file