Skip to content

Commit

Permalink
0002526: Improve performance of data gap detection
Browse files Browse the repository at this point in the history 
add routine that fixes overlapping gaps
  • Loading branch information
@erilong
erilong committed May 24, 2016
1 parent e1290ae commit 39e8457
Show file tree
Hide file tree
Showing 2 changed files with 85 additions and 36 deletions.
86 changes: 74 additions & 12 deletions symmetric-core/src/main/java/org/jumpmind/symmetric/route/DataGapFastDetector.java
View file
Expand Up @@ -26,6 +26,7 @@
import java.util.HashMap;
import java.util.HashSet;
import java.util.List;
import java.util.ListIterator;
import java.util.Map;
import java.util.Set;

Expand Down Expand Up @@ -104,6 +105,7 @@ public void beforeRouting() {
maxDataToSelect = parameterService.getLong(ParameterConstants.ROUTING_LARGEST_GAP_SIZE);
gaps = dataService.findDataGaps();
processInfo.setStatus(Status.OK);
fixOverlappingGaps(gaps, processInfo);

if (contextService.is(ContextConstants.ROUTING_FULL_GAP_ANALYSIS)) {
log.info("Full gap analysis is running");
Expand Down Expand Up @@ -303,14 +305,6 @@ protected boolean isOkayToAdd(DataGap dataGap) {
} else if (dataGap.gapSize() < maxDataToSelect - 1 && dataGap.gapSize() >= (long) (maxDataToSelect * 0.75)) {
log.warn("Detected a very large gap range: " + dataGap);
isOkay = false;
} else {
for (DataGap gapAdded : gapsAll) {
if (dataGap.overlaps(gapAdded)) {
log.warn("Detected an overlapping data gap: " + dataGap);
isOkay = false;
break;
}
}
}

if (isOkay) {
Expand Down Expand Up @@ -341,10 +335,6 @@ protected void queryDataIdMap() {
}
}

public Long mapRow(Row row) {
return row.getLong("data_id");
}

protected Map<DataGap, List<Long>> getDataIdMap() {
HashMap<DataGap, List<Long>> map = new HashMap<DataGap, List<Long>>();
Collections.sort(dataIds);
Expand All @@ -365,6 +355,78 @@ protected Map<DataGap, List<Long>> getDataIdMap() {
return map;
}

protected void fixOverlappingGaps(List<DataGap> gaps, ProcessInfo processInfo) {
try {
ISqlTransaction transaction = null;
log.debug("Looking for overlapping gaps");
try {
ISqlTemplate sqlTemplate = symmetricDialect.getPlatform().getSqlTemplate();
transaction = sqlTemplate.startSqlTransaction();
DataGap prevGap = null, lastGap = null;
ListIterator<DataGap> iter = gaps.listIterator();
while (iter.hasNext()) {
DataGap curGap = iter.next();
if (lastGap != null) {
log.warn("Removing gap found after last gap: " + curGap);
dataService.deleteDataGap(transaction, curGap);
iter.remove();
} else {
if (curGap.gapSize() >= maxDataToSelect - 1) {
lastGap = curGap;
}

if (prevGap != null) {
if (prevGap.overlaps(curGap)) {
log.warn("Removing overlapping gaps: " + prevGap + ", " + curGap);
dataService.deleteDataGap(transaction, prevGap);
dataService.deleteDataGap(transaction, curGap);
DataGap newGap = null;
if (curGap.equals(lastGap)) {
newGap = new DataGap(prevGap.getStartId(), prevGap.getStartId() + maxDataToSelect - 1);
} else {
newGap = new DataGap(prevGap.getStartId(),
prevGap.getEndId() > curGap.getEndId() ? prevGap.getEndId() : curGap.getEndId());
}
log.warn("Inserting new gap to fix overlap: " + newGap);
dataService.insertDataGap(transaction, newGap);
iter.remove();
iter.previous();
iter.set(newGap);
if (iter.hasNext()) {
iter.next();
}
curGap = newGap;
}
}
}
prevGap = curGap;
}
transaction.commit();
} catch (Error ex) {
if (transaction != null) {
transaction.rollback();
}
throw ex;
} catch (RuntimeException ex) {
if (transaction != null) {
transaction.rollback();
}
throw ex;
} finally {
if (transaction != null) {
transaction.close();
}
}
} catch (RuntimeException ex) {
processInfo.setStatus(Status.ERROR);
throw ex;
}
}

public Long mapRow(Row row) {
return row.getLong("data_id");
}

public List<DataGap> getDataGaps() {
return gaps;
}
Expand Down
35 changes: 11 additions & 24 deletions symmetric-core/src/test/java/org/jumpmind/symmetric/route/DataGapDetectorTest.java
View file
Expand Up @@ -436,10 +436,7 @@ public void testGapsBeforeAndAfterFull() throws Exception {
verifyNoMoreInteractions(dataService);
}

// Commenting out the overlap tests because these would need to be handled by
// a repair routine that runs at the beginning of the gap detection.

//@Test
@Test
public void testGapsOverlap() throws Exception {
List<Long> dataIds = new ArrayList<Long>();

Expand All @@ -454,7 +451,7 @@ public void testGapsOverlap() throws Exception {
verifyNoMoreInteractions(dataService);
}

//@Test
@Test
public void testGapsOverlapMultiple() throws Exception {
List<Long> dataIds = new ArrayList<Long>();

Expand All @@ -470,34 +467,22 @@ public void testGapsOverlapMultiple() throws Exception {
runGapDetector(dataGaps, dataIds, true);

verify(dataService).findDataGaps();
verify(dataService).deleteDataGap(sqlTransaction, new DataGap(1, 10));

verify(dataService, VerificationModeFactory.times(6)).deleteDataGap(sqlTransaction, new DataGap(1, 10));
verify(dataService, VerificationModeFactory.times(5)).insertDataGap(sqlTransaction, new DataGap(1, 10));

verify(dataService).deleteDataGap(sqlTransaction, new DataGap(3, 8));
verify(dataService).insertDataGap(sqlTransaction, new DataGap(1, 10));

verify(dataService).deleteDataGap(sqlTransaction, new DataGap(1, 10));
verify(dataService).deleteDataGap(sqlTransaction, new DataGap(4, 6));
verify(dataService).insertDataGap(sqlTransaction, new DataGap(1, 10));

verify(dataService).deleteDataGap(sqlTransaction, new DataGap(1, 10));
verify(dataService).deleteDataGap(sqlTransaction, new DataGap(4, 8));
verify(dataService).insertDataGap(sqlTransaction, new DataGap(1, 10));

verify(dataService).deleteDataGap(sqlTransaction, new DataGap(1, 10));
verify(dataService).deleteDataGap(sqlTransaction, new DataGap(4, 5));
verify(dataService).insertDataGap(sqlTransaction, new DataGap(1, 10));

verify(dataService).deleteDataGap(sqlTransaction, new DataGap(1, 10));
verify(dataService).deleteDataGap(sqlTransaction, new DataGap(5, 10));
verify(dataService).insertDataGap(sqlTransaction, new DataGap(1, 10));

verify(dataService).deleteDataGap(sqlTransaction, new DataGap(1, 10));
verify(dataService).deleteDataGap(sqlTransaction, new DataGap(6, 11));
verify(dataService).insertDataGap(sqlTransaction, new DataGap(1, 11));

verifyNoMoreInteractions(dataService);
}

//@Test
@Test
public void testGapsOverlapThenData() throws Exception {
List<Long> dataIds = new ArrayList<Long>();
dataIds.add(30953883L);
Expand All @@ -515,7 +500,7 @@ public void testGapsOverlapThenData() throws Exception {
verifyNoMoreInteractions(dataService);
}

//@Test
@Test
public void testGapsOverlapThenDataFull() throws Exception {
when(contextService.is(ContextConstants.ROUTING_FULL_GAP_ANALYSIS)).thenReturn(true);
List<Long> dataIds = new ArrayList<Long>();
Expand Down Expand Up @@ -545,7 +530,7 @@ public void testGapsOverlapThenDataFull() throws Exception {
verifyNoMoreInteractions(dataService);
}

//@Test
@Test
public void testGapsOverlapAfterLastGap() throws Exception {
List<Long> dataIds = new ArrayList<Long>();
dataIds.add(30953883L);
Expand Down Expand Up @@ -581,7 +566,9 @@ public void testGapsDuplicateDetection() throws Exception {
runGapDetector(dataGaps, dataIds, true);

verify(dataService).findDataGaps();
verify(dataService).deleteDataGap(sqlTransaction, new DataGap(31832440, 81832439));
verify(dataService).deleteDataGap(sqlTransaction, new DataGap(31832439, 81832439));
verify(dataService).insertDataGap(new DataGap(31832440, 81832439));
verifyNoMoreInteractions(dataService);
}

Expand Down

0 comments on commit 39e8457

Please sign in to comment.