From f3bdd938f55a08f90510e48a7fa9f7141d9fb118 Mon Sep 17 00:00:00 2001 From: Eric Long Date: Fri, 3 Jan 2020 09:21:30 -0500 Subject: [PATCH] 0004227: Data gap detector should use batch mode to insert and delete sym_data_gap --- .../symmetric/route/DataGapFastDetector.java | 40 +- .../symmetric/service/IDataService.java | 5 + .../symmetric/service/impl/DataService.java | 53 +- .../symmetric/route/DataGapDetectorTest.java | 549 ++++++------------ 4 files changed, 242 insertions(+), 405 deletions(-) diff --git a/symmetric-core/src/main/java/org/jumpmind/symmetric/route/DataGapFastDetector.java b/symmetric-core/src/main/java/org/jumpmind/symmetric/route/DataGapFastDetector.java index 3f3be55c09..98dd324ae8 100644 --- a/symmetric-core/src/main/java/org/jumpmind/symmetric/route/DataGapFastDetector.java +++ b/symmetric-core/src/main/java/org/jumpmind/symmetric/route/DataGapFastDetector.java @@ -21,7 +21,6 @@ package org.jumpmind.symmetric.route; import java.util.ArrayList; -import java.util.Collection; import java.util.Collections; import java.util.Date; import java.util.HashMap; @@ -292,7 +291,7 @@ public void afterRouting() { } } - printStats = saveDataGaps(ts, printStats); + saveDataGaps(); if (gaps.size() > 0) { lastGap = gaps.get(gaps.size() - 1); } @@ -348,7 +347,7 @@ private void printGapState() { log.info(buff.toString()); } - protected long saveDataGaps(long ts, long printStats) { + protected void saveDataGaps() { ISqlTemplate sqlTemplate = symmetricDialect.getPlatform().getSqlTemplate(); int totalGapChanges = gapsDeleted.size() + gapsAdded.size(); if (totalGapChanges > 0) { @@ -364,7 +363,7 @@ protected long saveDataGaps(long ts, long printStats) { log.info("There are {} data gap changes, which is within the max of {}, so switching to database", totalGapChanges, maxGapChanges); useInMemoryGaps = false; - printStats = insertDataGaps(transaction, ts, printStats, gaps); + dataService.insertDataGaps(transaction, gaps); } else { if (!useInMemoryGaps) { log.info("There are {} data gap changes, which exceeds the max of {}, so switching to in-memory", @@ -375,8 +374,8 @@ protected long saveDataGaps(long ts, long printStats) { dataService.insertDataGap(transaction, newGap); } } else { - printStats = deleteDataGaps(transaction, ts, printStats); - printStats = insertDataGaps(transaction, ts, printStats, gapsAdded); + dataService.deleteDataGaps(transaction, gapsDeleted); + dataService.insertDataGaps(transaction, gapsAdded); } transaction.commit(); } catch (Error ex) { @@ -395,35 +394,6 @@ protected long saveDataGaps(long ts, long printStats) { } } } - return printStats; - } - - protected long deleteDataGaps(ISqlTransaction transaction, long ts, long printStats) { - int counter = 0; - for (DataGap dataGap : gapsDeleted) { - dataService.deleteDataGap(transaction, dataGap); - counter++; - if (System.currentTimeMillis() - printStats > 30000) { - log.info("The data gap detection has been running for {}ms, deleted {} of {} old gaps", new Object[] { - System.currentTimeMillis() - ts, counter, gapsDeleted.size() }); - printStats = System.currentTimeMillis(); - } - } - return printStats; - } - - protected long insertDataGaps(ISqlTransaction transaction, long ts, long printStats, Collection argGaps) { - int counter = 0; - for (DataGap dataGap : argGaps) { - dataService.insertDataGap(transaction, dataGap); - counter++; - if (System.currentTimeMillis() - printStats > 30000) { - log.info("The data gap detection has been running for {}ms, inserted {} of {} new gaps", new Object[] { - System.currentTimeMillis() - ts, counter, gapsDeleted.size() }); - printStats = System.currentTimeMillis(); - } - } - return printStats; } protected void queryDataIdMap() { diff --git a/symmetric-core/src/main/java/org/jumpmind/symmetric/service/IDataService.java b/symmetric-core/src/main/java/org/jumpmind/symmetric/service/IDataService.java index 3f47ac47bc..82818edc48 100644 --- a/symmetric-core/src/main/java/org/jumpmind/symmetric/service/IDataService.java +++ b/symmetric-core/src/main/java/org/jumpmind/symmetric/service/IDataService.java @@ -20,6 +20,7 @@ */ package org.jumpmind.symmetric.service; +import java.util.Collection; import java.util.Date; import java.util.List; import java.util.Map; @@ -183,9 +184,13 @@ public void insertScriptEvent(ISqlTransaction transaction, String channelId, public void insertDataGap(DataGap gap); public void insertDataGap(ISqlTransaction transaction, DataGap gap); + + public void insertDataGaps(ISqlTransaction transaction, Collection gaps); public void deleteDataGap(ISqlTransaction transaction, DataGap gap); + public void deleteDataGaps(ISqlTransaction transaction, Collection gaps); + public void deleteAllDataGaps(ISqlTransaction transaction); public void deleteDataGap(DataGap gap); diff --git a/symmetric-core/src/main/java/org/jumpmind/symmetric/service/impl/DataService.java b/symmetric-core/src/main/java/org/jumpmind/symmetric/service/impl/DataService.java index cd761533aa..ac9bac84b6 100644 --- a/symmetric-core/src/main/java/org/jumpmind/symmetric/service/impl/DataService.java +++ b/symmetric-core/src/main/java/org/jumpmind/symmetric/service/impl/DataService.java @@ -28,6 +28,7 @@ import java.sql.Types; import java.util.ArrayList; import java.util.Arrays; +import java.util.Collection; import java.util.Collections; import java.util.Date; import java.util.HashMap; @@ -2717,7 +2718,33 @@ public void insertDataGap(ISqlTransaction transaction, DataGap gap) { gap.getCreateTime() }, new int[] { Types.VARCHAR, Types.NUMERIC, Types.NUMERIC, Types.TIMESTAMP }); } - + + @Override + public void insertDataGaps(ISqlTransaction transaction, Collection gaps) { + if (gaps.size() > 0) { + int[] types = new int[] { Types.VARCHAR, Types.NUMERIC, Types.NUMERIC, Types.TIMESTAMP }; + int maxRowsToFlush = engine.getParameterService().getInt(ParameterConstants.ROUTING_FLUSH_JDBC_BATCH_SIZE); + long ts = System.currentTimeMillis(); + int flushCount = 0, totalCount = 0; + transaction.setInBatchMode(true); + transaction.prepare(getSql("insertDataGapSql")); + for (DataGap gap : gaps) { + transaction.addRow(gap, new Object[] { engine.getClusterService().getServerId(), gap.getStartId(), gap.getEndId(), + gap.getCreateTime() }, types); + totalCount++; + if (++flushCount >= maxRowsToFlush) { + transaction.flush(); + flushCount = 0; + } + if (System.currentTimeMillis() - ts > 30000) { + log.info("Inserted {} of {} new gaps", totalCount, gaps.size()); + ts = System.currentTimeMillis(); + } + } + transaction.flush(); + } + } + @Override public void deleteDataGap(DataGap gap) { ISqlTransaction transaction = null; @@ -2753,6 +2780,30 @@ public void deleteDataGap(ISqlTransaction transaction, DataGap gap) { } } + @Override + public void deleteDataGaps(ISqlTransaction transaction, Collection gaps) { + if (gaps.size() > 0) { + int[] types = new int[] { symmetricDialect.getSqlTypeForIds(), symmetricDialect.getSqlTypeForIds() }; + int maxRowsToFlush = engine.getParameterService().getInt(ParameterConstants.ROUTING_FLUSH_JDBC_BATCH_SIZE); + long ts = System.currentTimeMillis(); + int flushCount = 0, totalCount = 0; + transaction.setInBatchMode(true); + transaction.prepare(getSql("deleteDataGapSql")); + for (DataGap gap : gaps) { + transaction.addRow(gap, new Object[] { gap.getStartId(), gap.getEndId() }, types); + if (++flushCount >= maxRowsToFlush) { + transaction.flush(); + flushCount = 0; + } + if (System.currentTimeMillis() - ts > 30000) { + log.info("Deleted {} of {} old gaps", totalCount, gaps.size()); + ts = System.currentTimeMillis(); + } + } + transaction.flush(); + } + } + public void deleteAllDataGaps(ISqlTransaction transaction) { transaction.prepareAndExecute(getSql("deleteAllDataGapsSql")); } diff --git a/symmetric-core/src/test/java/org/jumpmind/symmetric/route/DataGapDetectorTest.java b/symmetric-core/src/test/java/org/jumpmind/symmetric/route/DataGapDetectorTest.java index ff110ac670..e9b57a69af 100644 --- a/symmetric-core/src/test/java/org/jumpmind/symmetric/route/DataGapDetectorTest.java +++ b/symmetric-core/src/test/java/org/jumpmind/symmetric/route/DataGapDetectorTest.java @@ -1,7 +1,6 @@ package org.jumpmind.symmetric.route; import static org.mockito.ArgumentMatchers.any; -import static org.mockito.Mockito.doAnswer; import static org.mockito.Mockito.mock; import static org.mockito.Mockito.verify; import static org.mockito.Mockito.verifyNoMoreInteractions; @@ -155,10 +154,16 @@ public void testNewGap() throws Exception { runGapDetector(dataGaps, dataIds, true); + Set deleted = new HashSet(); + deleted.add(new DataGap(4, 50000004)); + + Set inserted = new HashSet(); + inserted.add(new DataGap(4, 99)); + inserted.add(new DataGap(101, 50000100)); + verify(dataService).findDataGaps(); - verify(dataService).deleteDataGap(sqlTransaction, new DataGap(4, 50000004)); - verify(dataService).insertDataGap(sqlTransaction, new DataGap(4, 99)); - verify(dataService).insertDataGap(sqlTransaction, new DataGap(101, 50000100)); + verify(dataService).deleteDataGaps(sqlTransaction, deleted); + verify(dataService).insertDataGaps(sqlTransaction, inserted); verifyNoMoreInteractions(dataService); } @@ -181,10 +186,16 @@ public void testNewGapFull() throws Exception { runGapDetector(dataGaps, new ArrayList(), true); + Set deleted = new HashSet(); + deleted.add(new DataGap(4, 50000004)); + + Set inserted = new HashSet(); + inserted.add(new DataGap(4, 99)); + inserted.add(new DataGap(101, 50000100)); + verify(dataService).findDataGaps(); - verify(dataService).deleteDataGap(sqlTransaction, new DataGap(4, 50000004)); - verify(dataService).insertDataGap(sqlTransaction, new DataGap(4, 99)); - verify(dataService).insertDataGap(sqlTransaction, new DataGap(101, 50000100)); + verify(dataService).deleteDataGaps(sqlTransaction, deleted); + verify(dataService).insertDataGaps(sqlTransaction, inserted); verifyNoMoreInteractions(dataService); } @@ -200,11 +211,17 @@ public void testTwoNewGaps() throws Exception { runGapDetector(dataGaps, dataIds, true); + Set deleted = new HashSet(); + deleted.add(new DataGap(4, 50000004)); + + Set inserted = new HashSet(); + inserted.add(new DataGap(4, 4)); + inserted.add(new DataGap(6, 7)); + inserted.add(new DataGap(9, 50000008)); + verify(dataService).findDataGaps(); - verify(dataService).deleteDataGap(sqlTransaction, new DataGap(4, 50000004)); - verify(dataService).insertDataGap(sqlTransaction, new DataGap(4, 4)); - verify(dataService).insertDataGap(sqlTransaction, new DataGap(6, 7)); - verify(dataService).insertDataGap(sqlTransaction, new DataGap(9, 50000008)); + verify(dataService).deleteDataGaps(sqlTransaction, deleted); + verify(dataService).insertDataGaps(sqlTransaction, inserted); verifyNoMoreInteractions(dataService); } @@ -228,11 +245,17 @@ public void testTwoNewGapsFull() throws Exception { runGapDetector(dataGaps, new ArrayList(), true); + Set deleted = new HashSet(); + deleted.add(new DataGap(4, 50000004)); + + Set inserted = new HashSet(); + inserted.add(new DataGap(4, 4)); + inserted.add(new DataGap(6, 7)); + inserted.add(new DataGap(9, 50000008)); + verify(dataService).findDataGaps(); - verify(dataService).deleteDataGap(sqlTransaction, new DataGap(4, 50000004)); - verify(dataService).insertDataGap(sqlTransaction, new DataGap(4, 4)); - verify(dataService).insertDataGap(sqlTransaction, new DataGap(6, 7)); - verify(dataService).insertDataGap(sqlTransaction, new DataGap(9, 50000008)); + verify(dataService).deleteDataGaps(sqlTransaction, deleted); + verify(dataService).insertDataGaps(sqlTransaction, inserted); verifyNoMoreInteractions(dataService); } @@ -251,16 +274,22 @@ public void testGapInGap() throws Exception { runGapDetector(dataGaps, dataIds, true); + Set deleted = new HashSet(); + deleted.add(new DataGap(5, 10)); + deleted.add(new DataGap(15, 20)); + deleted.add(new DataGap(21, 50000020)); + + Set inserted = new HashSet(); + inserted.add(new DataGap(5, 5)); + inserted.add(new DataGap(7, 10)); + inserted.add(new DataGap(15, 17)); + inserted.add(new DataGap(19, 20)); + inserted.add(new DataGap(21, 22)); + inserted.add(new DataGap(24, 50000023)); + verify(dataService).findDataGaps(); - verify(dataService).deleteDataGap(sqlTransaction, new DataGap(5, 10)); - verify(dataService).insertDataGap(sqlTransaction, new DataGap(5, 5)); - verify(dataService).insertDataGap(sqlTransaction, new DataGap(7, 10)); - verify(dataService).deleteDataGap(sqlTransaction, new DataGap(15, 20)); - verify(dataService).insertDataGap(sqlTransaction, new DataGap(15, 17)); - verify(dataService).insertDataGap(sqlTransaction, new DataGap(19, 20)); - verify(dataService).deleteDataGap(sqlTransaction, new DataGap(21, 50000020)); - verify(dataService).insertDataGap(sqlTransaction, new DataGap(21, 22)); - verify(dataService).insertDataGap(sqlTransaction, new DataGap(24, 50000023)); + verify(dataService).deleteDataGaps(sqlTransaction, deleted); + verify(dataService).insertDataGaps(sqlTransaction, inserted); verifyNoMoreInteractions(dataService); } @@ -296,16 +325,22 @@ public List answer(InvocationOnMock invocation) { runGapDetector(dataGaps, new ArrayList(), true); + Set deleted = new HashSet(); + deleted.add(new DataGap(5, 10)); + deleted.add(new DataGap(15, 20)); + deleted.add(new DataGap(21, 50000020)); + + Set inserted = new HashSet(); + inserted.add(new DataGap(5, 5)); + inserted.add(new DataGap(7, 10)); + inserted.add(new DataGap(15, 17)); + inserted.add(new DataGap(19, 20)); + inserted.add(new DataGap(21, 22)); + inserted.add(new DataGap(24, 50000023)); + verify(dataService).findDataGaps(); - verify(dataService).deleteDataGap(sqlTransaction, new DataGap(5, 10)); - verify(dataService).insertDataGap(sqlTransaction, new DataGap(5, 5)); - verify(dataService).insertDataGap(sqlTransaction, new DataGap(7, 10)); - verify(dataService).deleteDataGap(sqlTransaction, new DataGap(15, 20)); - verify(dataService).insertDataGap(sqlTransaction, new DataGap(15, 17)); - verify(dataService).insertDataGap(sqlTransaction, new DataGap(19, 20)); - verify(dataService).deleteDataGap(sqlTransaction, new DataGap(21, 50000020)); - verify(dataService).insertDataGap(sqlTransaction, new DataGap(21, 22)); - verify(dataService).insertDataGap(sqlTransaction, new DataGap(24, 50000023)); + verify(dataService).deleteDataGaps(sqlTransaction, deleted); + verify(dataService).insertDataGaps(sqlTransaction, inserted); verifyNoMoreInteractions(dataService); } @@ -320,9 +355,15 @@ public void testGapExpire() throws Exception { when(symmetricDialect.getDatabaseTime()).thenReturn(System.currentTimeMillis() + 60001L); runGapDetector(dataGaps, new ArrayList(), true); + Set deleted = new HashSet(); + deleted.add(new DataGap(3, 3)); + deleted.add(new DataGap(5, 6)); + + Set inserted = new HashSet(); + verify(dataService).findDataGaps(); - verify(dataService).deleteDataGap(sqlTransaction, new DataGap(3, 3)); - verify(dataService).deleteDataGap(sqlTransaction, new DataGap(5, 6)); + verify(dataService).deleteDataGaps(sqlTransaction, deleted); + verify(dataService).insertDataGaps(sqlTransaction, inserted); verifyNoMoreInteractions(dataService); } @@ -339,10 +380,16 @@ public void testGapExpireBusyChannel() throws Exception { detector.setLastBusyExpireRunTime(System.currentTimeMillis() - 61000); runGapDetector(dataGaps, new ArrayList(), false); + Set deleted = new HashSet(); + deleted.add(new DataGap(3, 3)); + + Set inserted = new HashSet(); + verify(dataService).findDataGaps(); verify(dataService).countDataInRange(2, 4); verify(dataService).countDataInRange(4, 7); - verify(dataService).deleteDataGap(sqlTransaction, new DataGap(3, 3)); + verify(dataService).deleteDataGaps(sqlTransaction, deleted); + verify(dataService).insertDataGaps(sqlTransaction, inserted); verifyNoMoreInteractions(dataService); } @@ -360,11 +407,17 @@ public void testGapBusyExpireRun() throws Exception { detector.setLastBusyExpireRunTime(System.currentTimeMillis() - 61000); runGapDetector(dataGaps, new ArrayList(), false); + Set deleted = new HashSet(); + deleted.add(new DataGap(3, 3)); + deleted.add(new DataGap(5, 6)); + + Set inserted = new HashSet(); + verify(dataService).findDataGaps(); verify(dataService).countDataInRange(2, 4); verify(dataService).countDataInRange(4, 7); - verify(dataService).deleteDataGap(sqlTransaction, new DataGap(3, 3)); - verify(dataService).deleteDataGap(sqlTransaction, new DataGap(5, 6)); + verify(dataService).deleteDataGaps(sqlTransaction, deleted); + verify(dataService).insertDataGaps(sqlTransaction, inserted); verifyNoMoreInteractions(dataService); } @@ -400,10 +453,16 @@ public void testGapBusyExpireRunMultiple() throws Exception { detector.setLastBusyExpireRunTime(System.currentTimeMillis() - 61000); runGapDetector(dataGaps, new ArrayList(), false); + Set deleted = new HashSet(); + deleted.add(new DataGap(3, 3)); + deleted.add(new DataGap(5, 6)); + + Set inserted = new HashSet(); + verify(dataService).countDataInRange(2, 4); verify(dataService).countDataInRange(4, 7); - verify(dataService).deleteDataGap(sqlTransaction, new DataGap(3, 3)); - verify(dataService).deleteDataGap(sqlTransaction, new DataGap(5, 6)); + verify(dataService).deleteDataGaps(sqlTransaction, deleted); + verify(dataService).insertDataGaps(sqlTransaction, inserted); verifyNoMoreInteractions(dataService); } @@ -433,9 +492,15 @@ public void testGapExpireOracle() throws Exception { when(symmetricDialect.getEarliestTransactionStartTime()).thenReturn(new Date(System.currentTimeMillis() + 60001L)); runGapDetector(dataGaps, new ArrayList(), true); + Set deleted = new HashSet(); + deleted.add(new DataGap(3, 3)); + deleted.add(new DataGap(5, 6)); + + Set inserted = new HashSet(); + verify(dataService).findDataGaps(); - verify(dataService).deleteDataGap(sqlTransaction, new DataGap(3, 3)); - verify(dataService).deleteDataGap(sqlTransaction, new DataGap(5, 6)); + verify(dataService).deleteDataGaps(sqlTransaction, deleted); + verify(dataService).insertDataGaps(sqlTransaction, inserted); verifyNoMoreInteractions(dataService); } @@ -452,10 +517,16 @@ public void testGapExpireOracleBusyChannel() throws Exception { detector.setLastBusyExpireRunTime(System.currentTimeMillis() - 61000); runGapDetector(dataGaps, new ArrayList(), false); + Set deleted = new HashSet(); + deleted.add(new DataGap(3, 3)); + + Set inserted = new HashSet(); + verify(dataService).findDataGaps(); verify(dataService).countDataInRange(2, 4); verify(dataService).countDataInRange(4, 7); - verify(dataService).deleteDataGap(sqlTransaction, new DataGap(3, 3)); + verify(dataService).deleteDataGaps(sqlTransaction, deleted); + verify(dataService).insertDataGaps(sqlTransaction, inserted); verifyNoMoreInteractions(dataService); } @@ -484,12 +555,24 @@ public void testGapsBeforeAndAfterFull() throws Exception { runGapDetector(dataGaps1, dataGaps2, dataIds, true); + Set deleted = new HashSet(); + deleted.add(new DataGap(841, 50000840)); + + Set deleted2 = new HashSet(); + deleted2.add(new DataGap(845, 50000844)); + + Set inserted = new HashSet(); + inserted.add(new DataGap(841, 842)); + inserted.add(new DataGap(845, 50000844)); + + Set inserted2 = new HashSet(); + inserted2.add(new DataGap(846, 50000845)); + verify(dataService).findDataGaps(); - verify(dataService).deleteDataGap(sqlTransaction, new DataGap(841, 50000840)); - verify(dataService).insertDataGap(sqlTransaction, new DataGap(841, 842)); - verify(dataService).insertDataGap(sqlTransaction, new DataGap(845, 50000844)); - verify(dataService).deleteDataGap(sqlTransaction, new DataGap(845, 50000844)); - verify(dataService).insertDataGap(sqlTransaction, new DataGap(846, 50000845)); + verify(dataService).deleteDataGaps(sqlTransaction, deleted); + verify(dataService).insertDataGaps(sqlTransaction, inserted); + verify(dataService).deleteDataGaps(sqlTransaction, deleted2); + verify(dataService).insertDataGaps(sqlTransaction, inserted2); verifyNoMoreInteractions(dataService); } @@ -550,10 +633,16 @@ public void testGapsOverlapThenData() throws Exception { runGapDetector(dataGaps, dataIds, true); + Set deleted = new HashSet(); + deleted.add(new DataGap(30953883, 80953883)); + + Set inserted = new HashSet(); + inserted.add(new DataGap(30953884, 80953883)); + verify(dataService).findDataGaps(); verify(dataService).deleteDataGap(sqlTransaction, new DataGap(30953884, 80953883)); - verify(dataService).deleteDataGap(sqlTransaction, new DataGap(30953883, 80953883)); - verify(dataService).insertDataGap(sqlTransaction, new DataGap(30953884, 80953883)); + verify(dataService).deleteDataGaps(sqlTransaction, deleted); + verify(dataService).insertDataGaps(sqlTransaction, inserted); verifyNoMoreInteractions(dataService); } @@ -581,10 +670,16 @@ public void testGapsOverlapThenDataFull() throws Exception { runGapDetector(dataGaps1, dataGaps2, dataIds, true); + Set deleted = new HashSet(); + deleted.add(new DataGap(30953883, 80953883)); + + Set inserted = new HashSet(); + inserted.add(new DataGap(30953884, 80953883)); + verify(dataService).findDataGaps(); verify(dataService).deleteDataGap(sqlTransaction, new DataGap(30953884, 80953883)); - verify(dataService).deleteDataGap(sqlTransaction, new DataGap(30953883, 80953883)); - verify(dataService).insertDataGap(sqlTransaction, new DataGap(30953884, 80953883)); + verify(dataService).deleteDataGaps(sqlTransaction, deleted); + verify(dataService).insertDataGaps(sqlTransaction, inserted); verifyNoMoreInteractions(dataService); } @@ -601,13 +696,19 @@ public void testGapsOverlapAfterLastGap() throws Exception { runGapDetector(dataGaps, dataIds, true); + Set deleted = new HashSet(); + deleted.add(new DataGap(30953883, 80953883)); + + Set inserted = new HashSet(); + inserted.add(new DataGap(30953884, 80953883)); + verify(dataService).findDataGaps(); verify(dataService).deleteDataGap(sqlTransaction, new DataGap(30953884, 80953883)); verify(dataService).deleteDataGap(sqlTransaction, new DataGap(30953885, 81953883)); verify(dataService).deleteDataGap(sqlTransaction, new DataGap(30953885, 30953885)); - verify(dataService).deleteDataGap(sqlTransaction, new DataGap(30953883, 80953883)); - verify(dataService).insertDataGap(sqlTransaction, new DataGap(30953884, 80953883)); + verify(dataService).deleteDataGaps(sqlTransaction, deleted); + verify(dataService).insertDataGaps(sqlTransaction, inserted); verifyNoMoreInteractions(dataService); } @@ -623,10 +724,17 @@ public void testGapsDuplicateDetection() throws Exception { runGapDetector(dataGaps, dataIds, true); + Set deleted = new HashSet(); + deleted.add(new DataGap(31832439, 81832439)); + + Set inserted = new HashSet(); + inserted.add(new DataGap(31832440, 81832439)); + verify(dataService).findDataGaps(); verify(dataService).deleteDataGap(sqlTransaction, new DataGap(31832440, 81832439)); - verify(dataService).deleteDataGap(sqlTransaction, new DataGap(31832439, 81832439)); - verify(dataService).insertDataGap(sqlTransaction, new DataGap(31832440, 81832439)); + + verify(dataService).deleteDataGaps(sqlTransaction, deleted); + verify(dataService).insertDataGaps(sqlTransaction, inserted); verifyNoMoreInteractions(dataService); } @@ -642,11 +750,18 @@ public void testGapsOverlapDetection() throws Exception { runGapDetector(dataGaps, dataIds, true); + Set deleted = new HashSet(); + deleted.add(new DataGap(31837983, 81837982)); + + Set inserted = new HashSet(); + inserted.add(new DataGap(31837984, 31837988)); + inserted.add(new DataGap(31837990, 81837989)); + verify(dataService).findDataGaps(); - verify(dataService).deleteDataGap(sqlTransaction, new DataGap(31837983, 81837982)); verify(dataService).deleteDataGap(sqlTransaction, new DataGap(31837983, 81837983)); - verify(dataService).insertDataGap(sqlTransaction, new DataGap(31837984, 31837988)); - verify(dataService).insertDataGap(sqlTransaction, new DataGap(31837990, 81837989)); + + verify(dataService).deleteDataGaps(sqlTransaction, deleted); + verify(dataService).insertDataGaps(sqlTransaction, inserted); verifyNoMoreInteractions(dataService); } @@ -677,318 +792,14 @@ public void testGapsFailedToDelete() throws Exception { runGapDetector(dataGaps, dataIds, false); - verify(dataService).deleteAllDataGaps(sqlTransaction); - verify(dataService).insertDataGap(sqlTransaction, new DataGap(1, 1)); - verify(dataService).insertDataGap(sqlTransaction, new DataGap(2, 2)); - verify(dataService).insertDataGap(sqlTransaction, new DataGap(5, 14)); - verifyNoMoreInteractions(dataService); - } - - @Test - public void testRandom() throws Exception { - for (int loop = 0; loop < 5000; loop++) { - List dataGaps = getRandomDataGaps(); - List dataIds = getRandomDatas(dataGaps); - checksOnDataService(dataGaps); - - detector = newGapDetector(); - detector.setFullGapAnalysis(false); - runGapDetector(dataGaps, dataIds, true); - - verify(dataService).findDataGaps(); - verifyInteractions(dataGaps, dataIds, true); - Mockito.reset(dataService); - } - } - - @Test - public void testRandomReuse() throws Exception { - List dataGaps = getRandomDataGaps(); - - for (int loop = 0; loop < 500; loop++) { - List dataIds = getRandomDatas(dataGaps); - checksOnDataService(dataGaps); - - runGapDetector(dataGaps, dataIds, true); - - if (loop == 0) { - verify(dataService).findDataGaps(); - } - verifyInteractions(dataGaps, dataIds, true); - - dataGaps = detector.getDataGaps(); - Mockito.reset(dataService); - } - } + List inserted = new ArrayList(); + inserted.add(new DataGap(1, 1)); + inserted.add(new DataGap(2, 2)); + inserted.add(new DataGap(5, 14)); - private List getRandomDataGaps() { - List dataGaps = new ArrayList(); - long startId = 0, endId = 0; - for (int i = 0; i < rand.nextInt(1, 10); i++) { - startId = rand.nextLong(endId + 1, endId + 10); - endId = rand.nextLong(startId, startId + 10); - dataGaps.add(new DataGap(startId, endId)); - } - return dataGaps; - } - - private List getRandomDatas(List dataGaps) { - List dataIds = new ArrayList(); - for (DataGap dataGap : dataGaps) { - if (rand.nextBoolean()) { - for (long dataId = dataGap.getStartId(); dataId <= dataGap.getEndId() && dataId - dataGap.getStartId() < 100; dataId++) { - if (rand.nextBoolean()) { - dataIds.add(dataId); - } - } - } - } - return dataIds; - } - - private void checksOnDataService(List dataGaps) { - final Set allGaps = new HashSet(dataGaps); - doAnswer(new Answer() { - public Void answer(InvocationOnMock invocation) { - Object[] args = invocation.getArguments(); - DataGap gap = (DataGap) args[1]; - checkDeleteGap(allGaps, gap); - return null; - } - }).when(dataService).deleteDataGap(ArgumentMatchers.eq(sqlTransaction), (DataGap) ArgumentMatchers.any()); - - doAnswer(new Answer() { - public Void answer(InvocationOnMock invocation) { - Object[] args = invocation.getArguments(); - DataGap gap = (DataGap) args[1]; - checkInsertGap(allGaps, gap); - return null; - } - }).when(dataService).insertDataGap(ArgumentMatchers.eq(sqlTransaction), (DataGap) ArgumentMatchers.any()); - - doAnswer(new Answer() { - public Void answer(InvocationOnMock invocation) { - Object[] args = invocation.getArguments(); - DataGap gap = (DataGap) args[0]; - checkInsertGap(allGaps, gap); - return null; - } - }).when(dataService).insertDataGap((DataGap) ArgumentMatchers.any()); - } - - private void verifyInteractions(List dataGaps, List dataIds, boolean verifyDeletes) { - int index = 0; - int lastIndex = dataGaps.size() - 1; - boolean isLastGapInserted = false; - boolean isLastGapModified = false; - long lastGapSize = parameterService.getLong(ParameterConstants.ROUTING_LARGEST_GAP_SIZE); - for (DataGap dataGap : dataGaps) { - long lastDataId = -1; - for (Long dataId : dataIds) { - if (dataId >= dataGap.getStartId() && dataId <= dataGap.getEndId()) { - if (lastDataId == -1) { - if (verifyDeletes) { - verify(dataService).deleteDataGap(sqlTransaction, dataGap); - } - if (dataId > dataGap.getStartId()) { - verify(dataService).insertDataGap(sqlTransaction, new DataGap(dataGap.getStartId(), dataId - 1)); - } - if (index == lastIndex) { - isLastGapModified = true; - } - } else if (dataId > lastDataId + 1) { - verify(dataService).insertDataGap(sqlTransaction, new DataGap(lastDataId + 1, dataId - 1)); - } - lastDataId = dataId; - } else if (dataId > dataGap.getEndId()){ - break; - } - } - if (lastDataId >= dataGap.getStartId() && lastDataId < dataGap.getEndId()) { - if (index == lastIndex) { - isLastGapInserted = true; - verify(dataService).insertDataGap(sqlTransaction, new DataGap(lastDataId + 1, lastDataId + lastGapSize)); - } else { - verify(dataService).insertDataGap(sqlTransaction, new DataGap(lastDataId + 1, dataGap.getEndId())); - } - } - index++; - } - - if (!isLastGapInserted && isLastGapModified) { - DataGap lastGap = dataGaps.get(lastIndex); - if (lastGap.getEndId() - lastGap.getStartId() < lastGapSize - 1) { - verify(dataService).insertDataGap(sqlTransaction, new DataGap(lastGap.getEndId() + 1, lastGap.getEndId() + lastGapSize)); - } - } - - verifyNoMoreInteractions(dataService); - } - - private void checkInsertGap(Set allGaps, DataGap gap) { - checkGap(gap); - if (allGaps.contains(gap)) { - throw new RuntimeException("Detected a duplicate data gap: " + gap); - } - for (DataGap gapAdded : allGaps) { - if (gap.overlaps(gapAdded)) { - throw new RuntimeException("Detected an overlapping data gap: " + gap); - } - } - allGaps.add(gap); - } - - private void checkDeleteGap(Set allGaps, DataGap gap) { - checkGap(gap); - if (!allGaps.contains(gap)) { - throw new RuntimeException("Detected a delete of a non-existent data gap: " + gap); - } - allGaps.remove(gap); - } - - private void checkGap(DataGap gap) { - long lastGapSize = parameterService.getLong(ParameterConstants.ROUTING_LARGEST_GAP_SIZE); - if (gap.getStartId() > gap.getEndId()) { - throw new RuntimeException("Detected an invalid gap range: " + gap); - } else if (gap.gapSize() < lastGapSize - 1 && gap.gapSize() >= (long) (lastGapSize * 0.75)) { - throw new RuntimeException("Detected a very large gap range: " + gap); - } - } - - @Test - public void testRandomReuseInMemory() throws Exception { - when(parameterService.getInt(ParameterConstants.ROUTING_MAX_GAP_CHANGES)).thenReturn(0); - List dataGaps = getRandomDataGaps(); - - for (int loop = 0; loop < 500; loop++) { - List dataIds = null; - do { - dataIds = getRandomDatas(dataGaps); - } while (dataIds.size() == 0); - - runGapDetector(dataGaps, dataIds, true); - - if (loop == 0) { - verify(dataService).findDataGaps(); - } - - List outDataGaps = detector.getDataGaps(); - verifyInteractionsInMemory(dataGaps, dataIds, outDataGaps, false); - - dataGaps = outDataGaps; - Mockito.reset(dataService); - } - } - - @Test - public void testRandomReuseInMemorySwitchDatabase() throws Exception { - boolean useMemory = false, lastUseMemory = false; - List dataGaps = getRandomDataGaps(); - - for (int loop = 0; loop < 500; loop++) { - List dataIds = null; - do { - dataIds = getRandomDatas(dataGaps); - } while (dataIds.size() == 0); - - if (useMemory = rand.nextBoolean()) { - when(parameterService.getInt(ParameterConstants.ROUTING_MAX_GAP_CHANGES)).thenReturn(0); - } else { - when(parameterService.getInt(ParameterConstants.ROUTING_MAX_GAP_CHANGES)).thenReturn(1000); - checksOnDataService(lastUseMemory ? new ArrayList() : dataGaps); - } - - runGapDetector(dataGaps, dataIds, true); - - if (loop == 0) { - verify(dataService).findDataGaps(); - } - - List outDataGaps = detector.getDataGaps(); - - if (useMemory) { - verifyInteractionsInMemory(dataGaps, dataIds, outDataGaps, false); - } else { - if (lastUseMemory) { - verifyInteractionsInMemory(dataGaps, dataIds, outDataGaps, true); - } else { - verifyInteractions(dataGaps, dataIds, true); - } - } - - dataGaps = outDataGaps; - Mockito.reset(dataService); - lastUseMemory = useMemory; - } - } - - private void verifyInteractionsInMemory(List dataGaps, List dataIds, List outDataGaps, boolean switchingToDatabase) { - int index = 0; - int lastIndex = dataGaps.size() - 1; - boolean isLastGapInserted = false; - boolean isLastGapModified = false; - long lastGapSize = parameterService.getLong(ParameterConstants.ROUTING_LARGEST_GAP_SIZE); - HashSet allGaps = new HashSet(outDataGaps); - for (DataGap dataGap : dataGaps) { - long lastDataId = -1; - for (Long dataId : dataIds) { - if (dataId >= dataGap.getStartId() && dataId <= dataGap.getEndId()) { - if (lastDataId == -1) { - checkDeleteGapInMemory(allGaps, dataGap); - if (dataId > dataGap.getStartId()) { - checkInsertGapInMemory(allGaps, new DataGap(dataGap.getStartId(), dataId - 1)); - } - if (index == lastIndex) { - isLastGapModified = true; - } - } else if (dataId > lastDataId + 1) { - checkInsertGapInMemory(allGaps, new DataGap(lastDataId + 1, dataId - 1)); - } - lastDataId = dataId; - } else if (dataId > dataGap.getEndId()){ - break; - } - } - if (lastDataId >= dataGap.getStartId() && lastDataId < dataGap.getEndId()) { - if (index == lastIndex) { - isLastGapInserted = true; - checkInsertGapInMemory(allGaps, new DataGap(lastDataId + 1, lastDataId + lastGapSize)); - } else { - checkInsertGapInMemory(allGaps, new DataGap(lastDataId + 1, dataGap.getEndId())); - } - } - index++; - } - - if (!isLastGapInserted && isLastGapModified) { - DataGap lastGap = dataGaps.get(lastIndex); - if (lastGap.getEndId() - lastGap.getStartId() < lastGapSize - 1) { - checkInsertGapInMemory(allGaps, new DataGap(lastGap.getEndId() + 1, lastGap.getEndId() + lastGapSize)); - } - } - verify(dataService).deleteAllDataGaps(sqlTransaction); - if (switchingToDatabase) { - for (DataGap dataGap : outDataGaps) { - verify(dataService).insertDataGap(sqlTransaction, dataGap); - } - } else { - verify(dataService).insertDataGap(sqlTransaction, new DataGap(outDataGaps.get(0).getStartId(), - outDataGaps.get(outDataGaps.size() - 1).getEndId())); - } - verifyNoMoreInteractions(dataService); - } - - private void checkDeleteGapInMemory(Set allGaps, DataGap gap) { - if (allGaps.contains(gap)) { - throw new RuntimeException("Found gap, but expected it to be deleted: " + gap); - } - } - - private void checkInsertGapInMemory(Set allGaps, DataGap gap) { - if (!allGaps.contains(gap)) { - throw new RuntimeException("Missing gap, but expected it to be inserted: " + gap); - } + verify(dataService).insertDataGaps(sqlTransaction, inserted); + verifyNoMoreInteractions(dataService); } }