[near-cache] MapInvalidationMetaDataMigrationTest failures

[mmedenjak]

https://hazelcast-l337.ci.cloudbees.com/view/Hazelcast/job/Hazelcast-3.x-IbmJDK1.7/1239/#showFailuresLink

com.hazelcast.map.impl.nearcache.invalidation.MapInvalidationMetaDataMigrationTest.uuids_migrated_whenNewlyJoinedNodesShutdown
com.hazelcast.map.impl.nearcache.invalidation.MapInvalidationMetaDataMigrationTest.sequences_migrated_whenOneNodeContinuouslyStartsAndStops
com.hazelcast.map.impl.nearcache.invalidation.MapInvalidationMetaDataMigrationTest.sequences_migrated_whenNewlyJoinedNodesShutdown
com.hazelcast.map.impl.nearcache.invalidation.MapInvalidationMetaDataMigrationTest.sequences_migrated_whenSourceNodeShutdown
com.hazelcast.map.impl.nearcache.invalidation.MapInvalidationMetaDataMigrationTest.uuids_migrated_whenSourceNodeShutdown

[mmedenjak]

https://hazelcast-l337.ci.cloudbees.com/view/Hazelcast/job/Hazelcast-3.x-IbmJDK1.7/1237/

[mmedenjak]

https://hazelcast-l337.ci.cloudbees.com/view/Hazelcast/job/Hazelcast-3.x-IbmJDK1.8/551/

[mmedenjak]

https://hazelcast-l337.ci.cloudbees.com/view/Hazelcast/job/Hazelcast-3.x-OracleJDK1.7/815/

[mmedenjak]

https://hazelcast-l337.ci.cloudbees.com/view/Hazelcast/job/Hazelcast-3.x-ZuluJDK8/537/

com.hazelcast.map.impl.nearcache.invalidation.MapInvalidationMetaDataMigrationTest.sequences_migrated_whenNewlyJoinedNodesShutdown

java.lang.AssertionError: expected:<53> but was:<0>
	at org.junit.Assert.fail(Assert.java:88)
	at org.junit.Assert.failNotEquals(Assert.java:834)
	at org.junit.Assert.assertEquals(Assert.java:118)
	at org.junit.Assert.assertEquals(Assert.java:144)
	at com.hazelcast.map.impl.nearcache.invalidation.MapInvalidationMetaDataMigrationTest.sequences_migrated_whenNewlyJoinedNodesShutdown(MapInvalidationMetaDataMigrationTest.java:92)

[mdogan]

https://hazelcast-l337.ci.cloudbees.com/job/new-lab-fast-pr/8448/

java.lang.AssertionError: expected:<35> but was:<36>
org.junit.Assert.fail(Assert.java:88)
org.junit.Assert.failNotEquals(Assert.java:834)
org.junit.Assert.assertEquals(Assert.java:118)
org.junit.Assert.assertEquals(Assert.java:144)
com.hazelcast.map.impl.nearcache.invalidation.MapInvalidationMetaDataMigrationTest.sequences_migrated_whenNewlyJoinedNodesShutdown(MapInvalidationMetaDataMigrationTest.java:92)

[Donnerbart]

com.hazelcast.map.impl.nearcache.invalidation.MapInvalidationMetaDataMigrationTest.sequences_migrated_whenOneNodeContinuouslyStartsAndStops

java.lang.AssertionError: expected:<{0=36, 1=29, 2=33, 3=35, 4=30, 5=44, 6=29, 7=33, 8=43, 9=45, 10=49, 11=43, 12=35, 13=48, 14=47, 15=35, 17=36, 16=30, 19=44, 18=46, 21=39, 20=39, 23=36, 22=36, 25=37, 24=45, 27=35, 26=48, 29=32, 28=26, 31=33, 30=31, 34=33, 35=30, 32=36, 33=40, 38=34, 39=30, 36=36, 37=35, 42=43, 43=36, 40=23, 41=29, 46=35, 47=43, 44=37, 45=41, 51=49, 50=30, 49=36, 48=44, 55=33, 54=28, 53=38, 52=40, 59=49, 58=46, 57=38, 56=23, 63=27, 62=42, 61=36, 60=32, 68=39, 69=34, 70=43, 71=37, 64=20, 65=37, 66=40, 67=37, 76=40, 77=41, 78=37, 79=30, 72=34, 73=42, 74=45, 75=31, 85=33, 84=38, 87=43, 86=30, 81=35, 80=31, 83=32, 82=34, 93=38, 92=37, 95=35, 94=52, 89=30, 88=39, 91=41, 90=47, 102=37, 103=30, 100=40, 101=32, 98=38, 99=29, 96=41, 97=43, 110=28, 111=27, 108=41, 109=29, 106=40, 107=32, 104=29, 105=45, 119=45, 118=36, 117=44, 116=31, 115=34, 114=35, 113=41, 112=34, 127=44, 126=32, 125=35, 124=31, 123=46, 122=24, 121=46, 120=52, 137=52, 136=36, 139=36, 138=36, 141=40, 140=28, 143=42, 142=38, 129=34, 128=30, 131=36, 130=34, 133=38, 132=42, 135=34, 134=39, 152=54, 153=30, 154=34, 155=23, 156=53, 157=48, 158=35, 159=32, 144=34, 145=35, 146=44, 147=39, 148=34, 149=33, 150=49, 151=45, 171=31, 170=38, 169=28, 168=32, 175=45, 174=34, 173=37, 172=41, 163=35, 162=41, 161=29, 160=33, 167=38, 166=29, 165=39, 164=43, 186=42, 187=32, 184=44, 185=49, 190=39, 191=26, 188=41, 189=34, 178=41, 179=38, 176=27, 177=36, 182=34, 183=29, 180=33, 181=42, 205=30, 204=27, 207=30, 206=42, 201=41, 200=28, 203=26, 202=34, 197=42, 196=40, 199=41, 198=33, 193=32, 192=44, 195=46, 194=32, 220=50, 221=33, 222=48, 223=29, 216=43, 217=35, 218=48, 219=39, 212=40, 213=31, 214=30, 215=34, 208=36, 209=34, 210=34, 211=37, 239=45, 238=34, 237=41, 236=41, 235=37, 234=37, 233=41, 232=35, 231=37, 230=37, 229=42, 228=40, 227=44, 226=31, 225=36, 224=37, 254=35, 255=40, 252=37, 253=35, 250=38, 251=36, 248=32, 249=34, 246=30, 247=41, 244=35, 245=43, 242=38, 243=31, 240=36, 241=42, 258=39, 259=35, 256=35, 257=34, 262=44, 263=30, 260=35, 261=40, 266=38, 267=40, 264=39, 265=34, 270=30, 268=37, 269=32}> but was:<{0=36, 1=29, 2=33, 3=35, 4=30, 5=44, 6=29, 7=33, 8=43, 9=45, 10=49, 11=43, 12=35, 13=48, 14=47, 15=35, 17=36, 16=30, 19=44, 18=46, 21=39, 20=39, 23=36, 22=36, 25=37, 24=45, 27=35, 26=48, 29=32, 28=26, 31=33, 30=31, 34=33, 35=30, 32=36, 33=40, 38=34, 39=30, 36=36, 37=35, 42=43, 43=36, 40=23, 41=29, 46=35, 47=43, 44=37, 45=41, 51=49, 50=30, 49=36, 48=44, 55=33, 54=28, 53=38, 52=40, 59=49, 58=46, 57=38, 56=23, 63=27, 62=42, 61=36, 60=32, 68=39, 69=34, 70=43, 71=37, 64=20, 65=37, 66=40, 67=37, 76=40, 77=41, 78=37, 79=30, 72=34, 73=42, 74=45, 75=31, 85=33, 84=38, 87=43, 86=30, 81=35, 80=31, 83=32, 82=34, 93=38, 92=37, 95=35, 94=52, 89=30, 88=39, 91=41, 90=47, 102=37, 103=30, 100=40, 101=32, 98=38, 99=29, 96=41, 97=43, 110=28, 111=27, 108=41, 109=29, 106=40, 107=32, 104=29, 105=45, 119=45, 118=36, 117=44, 116=31, 115=34, 114=35, 113=41, 112=34, 127=44, 126=32, 125=35, 124=31, 123=46, 122=24, 121=46, 120=52, 137=52, 136=36, 139=36, 138=36, 141=40, 140=28, 143=42, 142=38, 129=34, 128=30, 131=37, 130=34, 133=38, 132=42, 135=34, 134=39, 152=54, 153=30, 154=34, 155=23, 156=53, 157=48, 158=35, 159=32, 144=34, 145=35, 146=45, 147=39, 148=34, 149=33, 150=49, 151=45, 171=31, 170=38, 169=28, 168=32, 175=45, 174=34, 173=37, 172=41, 163=35, 162=41, 161=29, 160=33, 167=38, 166=29, 165=39, 164=43, 186=42, 187=32, 184=44, 185=49, 190=39, 191=26, 188=41, 189=34, 178=41, 179=38, 176=27, 177=36, 182=34, 183=29, 180=33, 181=42, 205=30, 204=27, 207=30, 206=42, 201=41, 200=28, 203=26, 202=34, 197=42, 196=40, 199=41, 198=33, 193=32, 192=44, 195=46, 194=32, 220=50, 221=33, 222=48, 223=29, 216=43, 217=35, 218=48, 219=39, 212=40, 213=31, 214=30, 215=34, 208=36, 209=34, 210=34, 211=37, 239=45, 238=34, 237=41, 236=41, 235=37, 234=37, 233=41, 232=35, 231=37, 230=37, 229=42, 228=40, 227=44, 226=31, 225=36, 224=37, 254=35, 255=40, 252=37, 253=35, 250=38, 251=36, 248=32, 249=34, 246=30, 247=41, 244=35, 245=43, 242=38, 243=31, 240=36, 241=42, 258=39, 259=35, 256=35, 257=34, 262=44, 263=30, 260=35, 261=40, 266=38, 267=40, 264=39, 265=34, 270=30, 268=37, 269=32}>
	at org.junit.Assert.fail(Assert.java:88)
	at org.junit.Assert.failNotEquals(Assert.java:834)
	at org.junit.Assert.assertEquals(Assert.java:118)
	at org.junit.Assert.assertEquals(Assert.java:144)
	at com.hazelcast.map.impl.nearcache.invalidation.MapInvalidationMetaDataMigrationTest.sequences_migrated_whenOneNodeContinuouslyStartsAndStops(MapInvalidationMetaDataMigrationTest.java:164)

https://hazelcast-l337.ci.cloudbees.com/job/new-lab-fast-pr/9012/testReport/junit/com.hazelcast.map.impl.nearcache.invalidation/MapInvalidationMetaDataMigrationTest/sequences_migrated_whenOneNodeContinuouslyStartsAndStops/

[Donnerbart]

@ahmetmircik Do you think this could also be solved via #10900?

[ahmetmircik]

Not sure the root cause of this issue but since the test is not failing for a long time, we can close and wait for next failure.

[mmedenjak]

https://hazelcast-l337.ci.cloudbees.com/job/new-lab-fast-pr/9979

com.hazelcast.map.impl.nearcache.invalidation.MapInvalidationMetaDataMigrationTest.sequences_migrated_whenOneNodeContinuouslyStartsAndStops

expected:<{0=36, 1=29, 2=33, 3=35, 4=30, 5=44, 6=29, 7=33, 8=43, 9=45, 10=49, 11=43, 12=35, 13=48, 14=47, 15=35, 17=36, 16=30, 19=44, 18=46, 21=39, 20=39, 23=36, 22=36, 25=37, 24=45, 27=35, 26=48, 29=32, 28=26, 31=33, 30=31, 34=33, 35=30, 32=36, 33=40, 38=34, 39=30, 36=36, 37=35, 42=43, 43=36, 40=23, 41=29, 46=35, 47=43, 44=37, 45=41, 51=49, 50=30, 49=36, 48=44, 55=33, 54=28, 53=38, 52=40, 59=49, 58=46, 57=38, 56=23, 63=27, 62=42, 61=36, 60=32, 68=39, 69=34, 70=43, 71=37, 64=20, 65=37, 66=40, 67=37, 76=40, 77=41, 78=37, 79=30, 72=34, 73=42, 74=45, 75=31, 85=33, 84=38, 87=43, 86=30, 81=35, 80=31, 83=32, 82=34, 93=38, 92=37, 95=35, 94=52, 89=30, 88=39, 91=41, 90=47, 102=37, 103=30, 100=40, 101=32, 98=38, 99=29, 96=41, 97=43, 110=28, 111=27, 108=41, 109=29, 106=40, 107=32, 104=29, 105=45, 119=45, 118=36, 117=44, 116=31, 115=34, 114=35, 113=41, 112=34, 127=44, 126=32, 125=35, 124=31, 123=46, 122=24, 121=46, 120=52, 137=52, 136=36, 139=36, 138=36, 141=40, 140=28, 143=42, 142=38, 129=34, 128=30, 131=37, 130=34, 133=38, 132=42, 135=34, 134=39, 152=54, 153=30, 154=34, 155=23, 156=53, 157=47, 158=35, 159=32, 144=34, 145=35, 146=45, 147=39, 148=34, 149=33, 150=49, 151=45, 171=31, 170=38, 169=28, 168=32, 175=45, 174=34, 173=37, 172=41, 163=35, 162=41, 161=29, 160=33, 167=38, 166=29, 165=39, 164=43, 186=42, 187=32, 184=44, 185=49, 190=39, 191=26, 188=41, 189=34, 178=41, 179=38, 176=27, 177=36, 182=34, 183=29, 180=33, 181=42, 205=30, 204=27, 207=30, 206=42, 201=41, 200=28, 203=26, 202=34, 197=42, 196=40, 199=41, 198=33, 193=32, 192=44, 195=46, 194=32, 220=50, 221=33, 222=48, 223=29, 216=43, 217=35, 218=48, 219=39, 212=40, 213=31, 214=30, 215=34, 208=36, 209=34, 210=34, 211=37, 239=45, 238=34, 237=41, 236=41, 235=37, 234=37, 233=41, 232=35, 231=37, 230=37, 229=42, 228=40, 227=44, 226=31, 225=36, 224=37, 254=35, 255=40, 252=37, 253=35, 250=38, 251=36, 248=32, 249=34, 246=30, 247=41, 244=35, 245=43, 242=38, 243=31, 240=36, 241=42, 258=39, 259=35, 256=35, 257=34, 262=44, 263=30, 260=35, 261=40, 266=38, 267=40, 264=39, 265=34, 270=30, 268=37, 269=32}> but was:<{0=36, 1=29, 2=33, 3=35, 4=30, 5=44, 6=29, 7=33, 8=43, 9=45, 10=49, 11=43, 12=35, 13=48, 14=47, 15=35, 17=36, 16=30, 19=44, 18=46, 21=39, 20=39, 23=36, 22=36, 25=37, 24=45, 27=35, 26=48, 29=32, 28=26, 31=33, 30=31, 34=33, 35=30, 32=36, 33=40, 38=34, 39=30, 36=36, 37=35, 42=43, 43=36, 40=23, 41=29, 46=35, 47=43, 44=37, 45=41, 51=49, 50=30, 49=36, 48=44, 55=33, 54=28, 53=38, 52=40, 59=49, 58=46, 57=38, 56=23, 63=27, 62=42, 61=36, 60=32, 68=39, 69=34, 70=43, 71=37, 64=20, 65=37, 66=40, 67=37, 76=40, 77=41, 78=37, 79=30, 72=34, 73=42, 74=45, 75=31, 85=33, 84=38, 87=43, 86=30, 81=35, 80=31, 83=32, 82=34, 93=38, 92=37, 95=35, 94=52, 89=30, 88=39, 91=41, 90=47, 102=37, 103=30, 100=40, 101=32, 98=38, 99=29, 96=41, 97=43, 110=28, 111=27, 108=41, 109=29, 106=40, 107=32, 104=29, 105=45, 119=45, 118=36, 117=44, 116=31, 115=34, 114=35, 113=41, 112=34, 127=44, 126=32, 125=35, 124=31, 123=46, 122=24, 121=46, 120=52, 137=52, 136=36, 139=36, 138=36, 141=40, 140=28, 143=42, 142=38, 129=34, 128=30, 131=37, 130=34, 133=38, 132=42, 135=34, 134=39, 152=54, 153=30, 154=34, 155=23, 156=53, 157=48, 158=35, 159=32, 144=34, 145=35, 146=45, 147=39, 148=34, 149=33, 150=49, 151=45, 171=31, 170=38, 169=28, 168=32, 175=45, 174=34, 173=37, 172=41, 163=35, 162=41, 161=29, 160=33, 167=38, 166=29, 165=39, 164=43, 186=42, 187=32, 184=44, 185=49, 190=39, 191=26, 188=41, 189=34, 178=41, 179=38, 176=27, 177=36, 182=34, 183=29, 180=33, 181=42, 205=30, 204=27, 207=30, 206=42, 201=41, 200=28, 203=26, 202=34, 197=42, 196=40, 199=41, 198=33, 193=32, 192=44, 195=46, 194=32, 220=50, 221=33, 222=48, 223=29, 216=43, 217=35, 218=48, 219=39, 212=40, 213=31, 214=30, 215=34, 208=36, 209=34, 210=34, 211=37, 239=45, 238=34, 237=41, 236=41, 235=37, 234=37, 233=41, 232=35, 231=37, 230=37, 229=42, 228=40, 227=44, 226=31, 225=36, 224=37, 254=35, 255=40, 252=37, 253=35, 250=38, 251=36, 248=32, 249=34, 246=30, 247=41, 244=35, 245=43, 242=38, 243=31, 240=36, 241=42, 258=39, 259=35, 256=35, 257=34, 262=44, 263=30, 260=35, 261=40, 266=38, 267=40, 264=39, 265=34, 270=30, 268=37, 269=32}>

com.hazelcast.map.impl.nearcache.invalidation.MapInvalidationMetaDataMigrationTest.sequences_migrated_whenNewlyJoinedNodesShutdown

java.lang.AssertionError: expected:<47> but was:<48>
	at org.junit.Assert.fail(Assert.java:88)
	at org.junit.Assert.failNotEquals(Assert.java:834)
	at org.junit.Assert.assertEquals(Assert.java:118)
	at org.junit.Assert.assertEquals(Assert.java:144)
	at com.hazelcast.map.impl.nearcache.invalidation.MapInvalidationMetaDataMigrationTest.sequences_migrated_whenNewlyJoinedNodesShutdown(MapInvalidationMetaDataMigrationTest.java:92)

[ahmetmircik]

https://hazelcast-l337.ci.cloudbees.com/job/new-lab-fast-pr/10025/testReport/junit/com.hazelcast.map.impl.nearcache.invalidation/MapInvalidationMetaDataMigrationTest/sequences_migrated_whenNewlyJoinedNodesShutdown/

[ahmetmircik]

added assertion to wait expected number of invalidations before progress further
58c8256

closing this now and waiting possible new failures