server crashes with AssertionError

[lorban]

It happens when playing with very small pools and relatively large mappings.

Here are the AssertionErrors that I collected:

java.lang.AssertionError
    at org.terracotta.offheapstore.storage.allocator.LongBestFitAllocator.unlinkSmallChunk(LongBestFitAllocator.java:642)
    at org.terracotta.offheapstore.storage.allocator.LongBestFitAllocator.unlinkChunk(LongBestFitAllocator.java:617)
    at org.terracotta.offheapstore.storage.allocator.LongBestFitAllocator.dlfree(LongBestFitAllocator.java:478)
    at org.terracotta.offheapstore.storage.allocator.LongBestFitAllocator.free(LongBestFitAllocator.java:155)
    at org.terracotta.offheapstore.paging.OffHeapStorageArea.free(OffHeapStorageArea.java:336)
    at org.ehcache.clustered.server.offheap.OffHeapChainStorageEngine$AttachedInternalChain.free(OffHeapChainStorageEngine.java:378)
    at org.ehcache.clustered.server.offheap.OffHeapChainStorageEngine$AttachedInternalChain.access$000(OffHeapChainStorageEngine.java:227)
    at org.ehcache.clustered.server.offheap.OffHeapChainStorageEngine.freeMapping(OffHeapChainStorageEngine.java:83)
    at org.terracotta.offheapstore.OffHeapHashMap.removeAtTableOffset(OffHeapHashMap.java:1646)
    at org.terracotta.offheapstore.AbstractOffHeapClockCache.evict(AbstractOffHeapClockCache.java:240)
    at org.terracotta.offheapstore.eviction.EvictionListeningReadWriteLockedOffHeapClockCache.evict(EvictionListeningReadWriteLockedOffHeapClockCache.java:53)
    at org.terracotta.offheapstore.AbstractOffHeapClockCache.storageEngineFailure(AbstractOffHeapClockCache.java:92)
    at org.terracotta.offheapstore.OffHeapHashMap.writeEntry(OffHeapHashMap.java:689)
    at org.terracotta.offheapstore.OffHeapHashMap.put(OffHeapHashMap.java:471)
    at org.terracotta.offheapstore.AbstractLockedOffHeapHashMap.put(AbstractLockedOffHeapHashMap.java:145)
    at org.terracotta.offheapstore.OffHeapHashMap.put(OffHeapHashMap.java:461)
    at org.terracotta.offheapstore.AbstractLockedOffHeapHashMap.put(AbstractLockedOffHeapHashMap.java:134)
    at org.ehcache.clustered.server.offheap.OffHeapChainMap.getAndAppend(OffHeapChainMap.java:100)
    at org.ehcache.clustered.server.offheap.OffHeapServerStore.getAndAppend(OffHeapServerStore.java:93)
    at org.ehcache.clustered.server.ServerStoreImpl.getAndAppend(ServerStoreImpl.java:70)
    at org.ehcache.clustered.server.EhcacheActiveEntity.invokeServerStoreOperation(EhcacheActiveEntity.java:396)
    at org.ehcache.clustered.server.EhcacheActiveEntity.invoke(EhcacheActiveEntity.java:308)
    at org.ehcache.clustered.server.EhcacheActiveEntity.invoke(EhcacheActiveEntity.java:88)
    at com.tc.objectserver.entity.ManagedEntityImpl.lambda$performAction$38(ManagedEntityImpl.java:610)
    at com.tc.objectserver.entity.ManagedEntityImpl.runWithHelper(ManagedEntityImpl.java:259)
    at com.tc.objectserver.entity.ManagedEntityImpl.performAction(ManagedEntityImpl.java:610)
    at com.tc.objectserver.entity.ManagedEntityImpl.invoke(ManagedEntityImpl.java:358)
    at com.tc.objectserver.entity.ManagedEntityImpl.lambda$processInvokeRequest$31(ManagedEntityImpl.java:204)
    at com.tc.objectserver.entity.ManagedEntityImpl$SchedulingRunnable.run(ManagedEntityImpl.java:837)
    at com.tc.objectserver.entity.RequestProcessor$EntityRequest.invoke(RequestProcessor.java:219)
    at com.tc.objectserver.entity.RequestProcessor$EntityRequest.run(RequestProcessor.java:211)
    at com.tc.objectserver.entity.RequestProcessorHandler.handleEvent(RequestProcessorHandler.java:27)
    at com.tc.objectserver.entity.RequestProcessorHandler.handleEvent(RequestProcessorHandler.java:23)
    at com.tc.async.impl.StageQueueImpl$HandledContext.runWithHandler(StageQueueImpl.java:502)
    at com.tc.async.impl.StageImpl$WorkerThread.run(StageImpl.java:197)

and

java.lang.AssertionError: Failure to move mapping during release
    at org.terracotta.offheapstore.paging.OffHeapStorageArea.moveAddressDown(OffHeapStorageArea.java:607)
    at org.terracotta.offheapstore.paging.OffHeapStorageArea.release(OffHeapStorageArea.java:544)
    at org.terracotta.offheapstore.paging.OffHeapStorageArea.shrink(OffHeapStorageArea.java:646)
    at org.ehcache.clustered.server.offheap.OffHeapChainStorageEngine.shrink(OffHeapChainStorageEngine.java:170)
    at org.terracotta.offheapstore.AbstractLockedOffHeapHashMap.shrink(AbstractLockedOffHeapHashMap.java:562)
    at org.ehcache.clustered.server.offheap.OffHeapChainMap.shrink(OffHeapChainMap.java:310)
    at org.ehcache.clustered.server.offheap.OffHeapServerStore.handleOversizeMappingException(OffHeapServerStore.java:179)
    at org.ehcache.clustered.server.offheap.OffHeapServerStore.getAndAppend(OffHeapServerStore.java:95)
    at org.ehcache.clustered.server.ServerStoreImpl.getAndAppend(ServerStoreImpl.java:70)
    at org.ehcache.clustered.server.EhcacheActiveEntity.invokeServerStoreOperation(EhcacheActiveEntity.java:396)
    at org.ehcache.clustered.server.EhcacheActiveEntity.invoke(EhcacheActiveEntity.java:308)
    at org.ehcache.clustered.server.EhcacheActiveEntity.invoke(EhcacheActiveEntity.java:88)
    at com.tc.objectserver.entity.ManagedEntityImpl.lambda$performAction$38(ManagedEntityImpl.java:610)
    at com.tc.objectserver.entity.ManagedEntityImpl.runWithHelper(ManagedEntityImpl.java:259)
    at com.tc.objectserver.entity.ManagedEntityImpl.performAction(ManagedEntityImpl.java:610)
    at com.tc.objectserver.entity.ManagedEntityImpl.invoke(ManagedEntityImpl.java:358)
    at com.tc.objectserver.entity.ManagedEntityImpl.lambda$processInvokeRequest$31(ManagedEntityImpl.java:204)
    at com.tc.objectserver.entity.ManagedEntityImpl$SchedulingRunnable.run(ManagedEntityImpl.java:837)
    at com.tc.objectserver.entity.RequestProcessor$EntityRequest.invoke(RequestProcessor.java:219)
    at com.tc.objectserver.entity.RequestProcessor$EntityRequest.run(RequestProcessor.java:211)
    at com.tc.objectserver.entity.RequestProcessorHandler.handleEvent(RequestProcessorHandler.java:27)
    at com.tc.objectserver.entity.RequestProcessorHandler.handleEvent(RequestProcessorHandler.java:23)
    at com.tc.async.impl.StageQueueImpl$HandledContext.runWithHandler(StageQueueImpl.java:502)
    at com.tc.async.impl.StageImpl$WorkerThread.run(StageImpl.java:197)

[lorban]

Here's a test that reproduces the crash:

  @Test
  public void heapClusterCrash() throws Exception {
    CacheManagerBuilder<PersistentCacheManager> clusteredCacheManagerBuilder
        = CacheManagerBuilder.newCacheManagerBuilder()
        .withCache("clustered-cache", CacheConfigurationBuilder.newCacheConfigurationBuilder(Long.class, byte[].class,
            ResourcePoolsBuilder.newResourcePoolsBuilder().heap(25, EntryUnit.ENTRIES)
                .with(ClusteredResourcePoolBuilder.clusteredShared("mySharedPool"))
            )
        )
        .with(ClusteringServiceConfigurationBuilder.cluster(URI.create("terracotta://localhost:9510/my-application"))
            .autoCreate()
            .resourcePool("mySharedPool", 1, MemoryUnit.MB, "primary-server-resource")
        );

    PersistentCacheManager cacheManager1 = clusteredCacheManagerBuilder.build(true);

    final Cache<Long, byte[]> cache1 = cacheManager1.getCache("clustered-cache", Long.class, byte[].class);

    final Random random = new Random();
    final int n = 500;

    byte[] value = new byte[1 * 1024];
    long before = System.currentTimeMillis();

    System.out.println("small values run");
    while ((System.currentTimeMillis() - before) < 10000) {
      cache1.put((long) random.nextInt(n), value);
      cache1.get((long) random.nextInt(n));
    }

    value = new byte[100 * 1024];
    before = System.currentTimeMillis();

    System.out.println("large values run");
    while ((System.currentTimeMillis() - before) < 10000) {
      cache1.put((long) random.nextInt(n), value);
      cache1.get((long) random.nextInt(n));
    }

    cacheManager1.close();
  }