diff --git a/symmetric-core/src/main/java/org/jumpmind/symmetric/route/DataGapFastDetector.java b/symmetric-core/src/main/java/org/jumpmind/symmetric/route/DataGapFastDetector.java index a9dbac560f..648afe4fd9 100644 --- a/symmetric-core/src/main/java/org/jumpmind/symmetric/route/DataGapFastDetector.java +++ b/symmetric-core/src/main/java/org/jumpmind/symmetric/route/DataGapFastDetector.java @@ -26,6 +26,7 @@ import java.util.HashMap; import java.util.HashSet; import java.util.List; +import java.util.ListIterator; import java.util.Map; import java.util.Set; @@ -104,6 +105,7 @@ public void beforeRouting() { maxDataToSelect = parameterService.getLong(ParameterConstants.ROUTING_LARGEST_GAP_SIZE); gaps = dataService.findDataGaps(); processInfo.setStatus(Status.OK); + fixOverlappingGaps(gaps, processInfo); if (contextService.is(ContextConstants.ROUTING_FULL_GAP_ANALYSIS)) { log.info("Full gap analysis is running"); @@ -303,14 +305,6 @@ protected boolean isOkayToAdd(DataGap dataGap) { } else if (dataGap.gapSize() < maxDataToSelect - 1 && dataGap.gapSize() >= (long) (maxDataToSelect * 0.75)) { log.warn("Detected a very large gap range: " + dataGap); isOkay = false; - } else { - for (DataGap gapAdded : gapsAll) { - if (dataGap.overlaps(gapAdded)) { - log.warn("Detected an overlapping data gap: " + dataGap); - isOkay = false; - break; - } - } } if (isOkay) { @@ -341,10 +335,6 @@ protected void queryDataIdMap() { } } - public Long mapRow(Row row) { - return row.getLong("data_id"); - } - protected Map> getDataIdMap() { HashMap> map = new HashMap>(); Collections.sort(dataIds); @@ -365,6 +355,78 @@ protected Map> getDataIdMap() { return map; } + protected void fixOverlappingGaps(List gaps, ProcessInfo processInfo) { + try { + ISqlTransaction transaction = null; + log.debug("Looking for overlapping gaps"); + try { + ISqlTemplate sqlTemplate = symmetricDialect.getPlatform().getSqlTemplate(); + transaction = sqlTemplate.startSqlTransaction(); + DataGap prevGap = null, lastGap = null; + ListIterator iter = gaps.listIterator(); + while (iter.hasNext()) { + DataGap curGap = iter.next(); + if (lastGap != null) { + log.warn("Removing gap found after last gap: " + curGap); + dataService.deleteDataGap(transaction, curGap); + iter.remove(); + } else { + if (curGap.gapSize() >= maxDataToSelect - 1) { + lastGap = curGap; + } + + if (prevGap != null) { + if (prevGap.overlaps(curGap)) { + log.warn("Removing overlapping gaps: " + prevGap + ", " + curGap); + dataService.deleteDataGap(transaction, prevGap); + dataService.deleteDataGap(transaction, curGap); + DataGap newGap = null; + if (curGap.equals(lastGap)) { + newGap = new DataGap(prevGap.getStartId(), prevGap.getStartId() + maxDataToSelect - 1); + } else { + newGap = new DataGap(prevGap.getStartId(), + prevGap.getEndId() > curGap.getEndId() ? prevGap.getEndId() : curGap.getEndId()); + } + log.warn("Inserting new gap to fix overlap: " + newGap); + dataService.insertDataGap(transaction, newGap); + iter.remove(); + iter.previous(); + iter.set(newGap); + if (iter.hasNext()) { + iter.next(); + } + curGap = newGap; + } + } + } + prevGap = curGap; + } + transaction.commit(); + } catch (Error ex) { + if (transaction != null) { + transaction.rollback(); + } + throw ex; + } catch (RuntimeException ex) { + if (transaction != null) { + transaction.rollback(); + } + throw ex; + } finally { + if (transaction != null) { + transaction.close(); + } + } + } catch (RuntimeException ex) { + processInfo.setStatus(Status.ERROR); + throw ex; + } + } + + public Long mapRow(Row row) { + return row.getLong("data_id"); + } + public List getDataGaps() { return gaps; } diff --git a/symmetric-core/src/test/java/org/jumpmind/symmetric/route/DataGapDetectorTest.java b/symmetric-core/src/test/java/org/jumpmind/symmetric/route/DataGapDetectorTest.java index d6afec185a..876a77ef61 100644 --- a/symmetric-core/src/test/java/org/jumpmind/symmetric/route/DataGapDetectorTest.java +++ b/symmetric-core/src/test/java/org/jumpmind/symmetric/route/DataGapDetectorTest.java @@ -436,10 +436,7 @@ public void testGapsBeforeAndAfterFull() throws Exception { verifyNoMoreInteractions(dataService); } - // Commenting out the overlap tests because these would need to be handled by - // a repair routine that runs at the beginning of the gap detection. - - //@Test + @Test public void testGapsOverlap() throws Exception { List dataIds = new ArrayList(); @@ -454,7 +451,7 @@ public void testGapsOverlap() throws Exception { verifyNoMoreInteractions(dataService); } - //@Test + @Test public void testGapsOverlapMultiple() throws Exception { List dataIds = new ArrayList(); @@ -470,34 +467,22 @@ public void testGapsOverlapMultiple() throws Exception { runGapDetector(dataGaps, dataIds, true); verify(dataService).findDataGaps(); - verify(dataService).deleteDataGap(sqlTransaction, new DataGap(1, 10)); + + verify(dataService, VerificationModeFactory.times(6)).deleteDataGap(sqlTransaction, new DataGap(1, 10)); + verify(dataService, VerificationModeFactory.times(5)).insertDataGap(sqlTransaction, new DataGap(1, 10)); + verify(dataService).deleteDataGap(sqlTransaction, new DataGap(3, 8)); - verify(dataService).insertDataGap(sqlTransaction, new DataGap(1, 10)); - - verify(dataService).deleteDataGap(sqlTransaction, new DataGap(1, 10)); verify(dataService).deleteDataGap(sqlTransaction, new DataGap(4, 6)); - verify(dataService).insertDataGap(sqlTransaction, new DataGap(1, 10)); - - verify(dataService).deleteDataGap(sqlTransaction, new DataGap(1, 10)); verify(dataService).deleteDataGap(sqlTransaction, new DataGap(4, 8)); - verify(dataService).insertDataGap(sqlTransaction, new DataGap(1, 10)); - - verify(dataService).deleteDataGap(sqlTransaction, new DataGap(1, 10)); verify(dataService).deleteDataGap(sqlTransaction, new DataGap(4, 5)); - verify(dataService).insertDataGap(sqlTransaction, new DataGap(1, 10)); - - verify(dataService).deleteDataGap(sqlTransaction, new DataGap(1, 10)); verify(dataService).deleteDataGap(sqlTransaction, new DataGap(5, 10)); - verify(dataService).insertDataGap(sqlTransaction, new DataGap(1, 10)); - - verify(dataService).deleteDataGap(sqlTransaction, new DataGap(1, 10)); verify(dataService).deleteDataGap(sqlTransaction, new DataGap(6, 11)); verify(dataService).insertDataGap(sqlTransaction, new DataGap(1, 11)); verifyNoMoreInteractions(dataService); } - //@Test + @Test public void testGapsOverlapThenData() throws Exception { List dataIds = new ArrayList(); dataIds.add(30953883L); @@ -515,7 +500,7 @@ public void testGapsOverlapThenData() throws Exception { verifyNoMoreInteractions(dataService); } - //@Test + @Test public void testGapsOverlapThenDataFull() throws Exception { when(contextService.is(ContextConstants.ROUTING_FULL_GAP_ANALYSIS)).thenReturn(true); List dataIds = new ArrayList(); @@ -545,7 +530,7 @@ public void testGapsOverlapThenDataFull() throws Exception { verifyNoMoreInteractions(dataService); } - //@Test + @Test public void testGapsOverlapAfterLastGap() throws Exception { List dataIds = new ArrayList(); dataIds.add(30953883L); @@ -581,7 +566,9 @@ public void testGapsDuplicateDetection() throws Exception { runGapDetector(dataGaps, dataIds, true); verify(dataService).findDataGaps(); + verify(dataService).deleteDataGap(sqlTransaction, new DataGap(31832440, 81832439)); verify(dataService).deleteDataGap(sqlTransaction, new DataGap(31832439, 81832439)); + verify(dataService).insertDataGap(new DataGap(31832440, 81832439)); verifyNoMoreInteractions(dataService); }