Lazy file allocation was implemented

core/src/main/java/com/orientechnologies/orient/core/storage/cache/local/OWOWCache.java

@@ -1808,7 +1808,7 @@ private void convertNameIdMapFromV1ToV2() throws IOException {
 
   private OFileClassic createFileInstance(final String fileName, final int fileId) {
     final String internalFileName = createInternalFileName(fileName, fileId);
-    return new OFileClassic(storagePath.resolve(internalFileName));
+    return new OFileClassic(storagePath.resolve(internalFileName), pageSize);
   }
 
   private static String createInternalFileName(final String fileName, final int fileId) {
@@ -1877,7 +1877,7 @@ private void readNameIdMapV2(FileChannel nameIdMapHolder) throws IOException, In
 
         if (files.get(externalId) == null) {
           final Path path = storagePath.resolve(idFileNameMap.get((nameIdEntry.getValue())));
-          final OFileClassic fileClassic = new OFileClassic(path);
+          final OFileClassic fileClassic = new OFileClassic(path, pageSize);
 
           if (fileClassic.exists()) {
             fileClassic.open();
@@ -1949,7 +1949,7 @@ private void readNameIdMapV1(final FileChannel nameIdMapHolder) throws IOExcepti
         final long externalId = composeFileId(id, nameIdEntry.getValue());
 
         if (files.get(externalId) == null) {
-          final OFileClassic fileClassic = new OFileClassic(storagePath.resolve(nameIdEntry.getKey()));
+          final OFileClassic fileClassic = new OFileClassic(storagePath.resolve(nameIdEntry.getKey()), pageSize);
 
           if (fileClassic.exists()) {
             fileClassic.open();
@@ -2735,7 +2735,12 @@ public void run() {
                 final Map.Entry<Long, TreeSet<PageKey>> lsnEntry = localDirtyPagesBySegment.firstEntry();
 
                 if (lsnEntry != null && lsnEntry.getKey() < endSegment) {
-                  lsnPages += flushChunk(lsnFlushInterval, startSegment, endSegment);
+                  final int flushedPages = flushChunk(lsnFlushInterval, startSegment, endSegment);
+                  lsnPages += flushedPages;
+                  if (flushedPages == 0) {
+                    flushTs = System.nanoTime();
+                    break;
+                  }
                 }
               }
 
@@ -2745,7 +2750,12 @@ public void run() {
                   final TreeSet<PageKey> pages = firstSegment.getValue();
 
                   if (pages.size() >= 1024 * 1024 * 1024 / pageSize) {
-                    lsnPages += flushChunk(lsnFlushInterval, startSegment, endSegment);
+                    final int flushedPages = flushChunk(lsnFlushInterval, startSegment, endSegment);
+                    lsnPages += flushedPages;
+                    if (flushedPages == 0) {
+                      flushTs = System.nanoTime();
+                      break;
+                    }
                   }
                 }
               }

core/src/main/java/com/orientechnologies/orient/core/storage/fs/OFileClassic.java

@@ -71,11 +71,14 @@ public final class OFileClassic implements OClosableItem {
 
   private int version;
 
-  private volatile long size;
+  private final    AtomicLong size          = new AtomicLong();
+  private volatile long       committedSize = 0;
 
   private AllocationMode allocationMode;
   private int            fd;
 
+  private final int pageSize;
+
   /**
    * Map which calculates which files are opened and how many users they have
    */
@@ -91,26 +94,38 @@ public final class OFileClassic implements OClosableItem {
    */
   private final boolean trackFileOpen = OGlobalConfiguration.STORAGE_TRACK_FILE_ACCESS.getValueAsBoolean();
 
-  public OFileClassic(final Path osFile) {
+  public OFileClassic(final Path osFile, final int pageSize) {
     this.osFile = osFile;
+    this.pageSize = pageSize;
   }
 
   public long allocateSpace(final int size) throws IOException {
-    acquireWriteLock();
+    acquireReadLock();
     try {
-      final long currentSize = this.size;
-      assert channel.size() == currentSize + HEADER_SIZE;
+      final long currentSize = this.size.addAndGet(size);
+      final long currentCommittedSize = this.committedSize;
 
-      //noinspection NonAtomicOperationOnVolatileField
-      this.size += size;
+      final int pagesDifference = (int) ((currentSize - currentCommittedSize) / pageSize);
+      if (pagesDifference < 10 || pagesDifference < 0.1 * currentCommittedSize) {
+        return currentSize;
+      }
+    } finally {
+      releaseReadLock();
+    }
 
-      assert this.size >= size;
+    acquireWriteLock();
+    try {
+      final long currentSize = this.size.get();
+      assert channel.size() == this.committedSize + HEADER_SIZE;
+
+      final long sizeDiff = currentSize - this.committedSize;
+      this.committedSize = currentSize;
 
       assert allocationMode != null;
       if (allocationMode == AllocationMode.WRITE) {
-        final long ptr = Native.malloc(size);
+        final long ptr = Native.malloc(sizeDiff);
         try {
-          final ByteBuffer buffer = new Pointer(ptr).getByteBuffer(0, size);
+          final ByteBuffer buffer = new Pointer(ptr).getByteBuffer(0, sizeDiff);
           buffer.position(0);
           OIOUtils.writeByteBuffer(buffer, channel, currentSize + HEADER_SIZE);
         } finally {
@@ -120,7 +135,7 @@ public long allocateSpace(final int size) throws IOException {
         assert fd > 0;
 
         try {
-          ONative.instance().fallocate(fd, currentSize + HEADER_SIZE, size);
+          ONative.instance().fallocate(fd, currentSize + HEADER_SIZE, sizeDiff);
         } catch (final LastErrorException e) {
           OLogManager.instance()
               .debug(this, "Can not allocate space (error %d) for file %s using native Linux API, more slower methods will be used",
@@ -136,9 +151,9 @@ public long allocateSpace(final int size) throws IOException {
                     lee.getErrorCode());
           }
 
-          final long ptr = Native.malloc(size);
+          final long ptr = Native.malloc(sizeDiff);
           try {
-            final ByteBuffer buffer = new Pointer(ptr).getByteBuffer(0, size);
+            final ByteBuffer buffer = new Pointer(ptr).getByteBuffer(0, sizeDiff);
             buffer.position(0);
             OIOUtils.writeByteBuffer(buffer, channel, currentSize + HEADER_SIZE);
           } finally {
@@ -147,12 +162,12 @@ public long allocateSpace(final int size) throws IOException {
         }
 
       } else if (allocationMode == AllocationMode.LENGTH) {
-        frnd.setLength(this.size + HEADER_SIZE);
+        frnd.setLength(currentSize + HEADER_SIZE);
       } else {
         throw new IllegalStateException("Unknown allocation mode");
       }
 
-      assert channel.size() == this.size + HEADER_SIZE;
+      assert channel.size() == currentSize + HEADER_SIZE;
       return currentSize;
     } finally {
       releaseWriteLock();
@@ -167,17 +182,19 @@ public void shrink(final long size) throws IOException {
     try {
       //noinspection resource
       channel.truncate(HEADER_SIZE + size);
-      this.size = size;
+      this.size.set(size);
+      this.committedSize = size;
 
-      assert this.size >= 0;
+      assert this.committedSize >= 0;
+      assert this.size.get() >= 0;
 
     } finally {
       releaseWriteLock();
     }
   }
 
   public long getFileSize() {
-    return size;
+    return size.get();
   }
 
   public void read(long offset, final ByteBuffer buffer, final boolean throwOnEof) throws IOException {
@@ -289,16 +306,16 @@ private void initAllocationMode() {
   /**
    * ALWAYS ADD THE HEADER SIZE BECAUSE ON THIS TYPE IS ALWAYS NEEDED
    */
-  private long checkRegions(final long iOffset, final long iLength) {
+  private long checkRegions(final long offset, final long iLength) {
     acquireReadLock();
     try {
-      if (iOffset < 0 || iOffset + iLength > size) {
+      if (offset < 0 || offset + iLength > size.get()) {
         throw new OIOException(
-            "You cannot access outside the file size (" + size + " bytes). You have requested portion from " + iOffset + "-" + (iOffset
-                + iLength) + " bytes. File: " + this);
+            "You cannot access outside the file size (" + size.get() + " bytes). You have requested portion from " + offset + "-"
+                + (offset + iLength) + " bytes. File: " + this);
       }
 
-      return iOffset + HEADER_SIZE;
+      return offset + HEADER_SIZE;
     } finally {
       releaseReadLock();
     }
@@ -520,8 +537,10 @@ private void openChannel() throws IOException {
   }
 
   private void init() throws IOException {
-    size = channel.size() - HEADER_SIZE;
-    assert size >= 0;
+    size.set(channel.size() - HEADER_SIZE);
+    assert size.get() >= 0;
+
+    this.committedSize = size.get();
 
     final ByteBuffer buffer = ByteBuffer.allocate(1);
 

core/src/test/java/com/orientechnologies/orient/core/storage/index/hashindex/local/cache/WOWCacheTestIT.java

@@ -541,7 +541,7 @@ public void testChecksumFailure() throws IOException {
     wowCache.flush();
 
     final Path path = storagePath.resolve(wowCache.nativeFileNameById(fileId));
-    final OFileClassic file = new OFileClassic(path);
+    final OFileClassic file = new OFileClassic(path, pageSize);
     file.open();
     file.write(ODurablePage.NEXT_FREE_POSITION, ByteBuffer.wrap(new byte[] { 1 }).order(ByteOrder.nativeOrder()));
     file.close();
@@ -574,7 +574,7 @@ public void testMagicFailure() throws IOException {
     wowCache.flush();
 
     final Path path = storagePath.resolve(wowCache.nativeFileNameById(fileId));
-    final OFileClassic file = new OFileClassic(path);
+    final OFileClassic file = new OFileClassic(path, pageSize);
     file.open();
     file.write(0, ByteBuffer.wrap(new byte[] { 1 }).order(ByteOrder.nativeOrder()));
     file.close();
@@ -607,7 +607,7 @@ public void testNoChecksumVerificationIfNotRequested() throws IOException {
     wowCache.flush();
 
     final Path path = storagePath.resolve(wowCache.nativeFileNameById(fileId));
-    final OFileClassic file = new OFileClassic(path);
+    final OFileClassic file = new OFileClassic(path, pageSize);
     file.open();
     file.write(ODurablePage.NEXT_FREE_POSITION, ByteBuffer.wrap(new byte[] { 1 }).order(ByteOrder.nativeOrder()));
     file.close();
@@ -635,7 +635,7 @@ public void testNoChecksumFailureIfVerificationTurnedOff() throws IOException {
     wowCache.flush();
 
     final Path path = storagePath.resolve(wowCache.nativeFileNameById(fileId));
-    final OFileClassic file = new OFileClassic(path);
+    final OFileClassic file = new OFileClassic(path, pageSize);
     file.open();
     file.write(ODurablePage.NEXT_FREE_POSITION, ByteBuffer.wrap(new byte[] { 1 }).order(ByteOrder.nativeOrder()));
     file.close();
@@ -663,7 +663,7 @@ public void testNoChecksumFailureIfVerificationTurnedOffOnLoad() throws IOExcept
     wowCache.flush();
 
     final Path path = storagePath.resolve(wowCache.nativeFileNameById(fileId));
-    final OFileClassic file = new OFileClassic(path);
+    final OFileClassic file = new OFileClassic(path, pageSize);
     file.open();
     file.write(ODurablePage.NEXT_FREE_POSITION, ByteBuffer.wrap(new byte[] { 1 }).order(ByteOrder.nativeOrder()));
     file.close();
@@ -691,7 +691,7 @@ public void testNoChecksumFailureIfNoChecksumProvided() throws IOException {
     wowCache.flush();
 
     final Path path = storagePath.resolve(wowCache.nativeFileNameById(fileId));
-    final OFileClassic file = new OFileClassic(path);
+    final OFileClassic file = new OFileClassic(path, pageSize);
     file.open();
     file.write(ODurablePage.NEXT_FREE_POSITION, ByteBuffer.wrap(new byte[] { 1 }).order(ByteOrder.nativeOrder()));
     file.close();
@@ -701,7 +701,7 @@ public void testNoChecksumFailureIfNoChecksumProvided() throws IOException {
   }
 
   private void assertFile(long pageIndex, byte[] value, OLogSequenceNumber lsn, String fileName) throws IOException {
-    OFileClassic fileClassic = new OFileClassic(storagePath.resolve(fileName));
+    OFileClassic fileClassic = new OFileClassic(storagePath.resolve(fileName), pageSize);
     fileClassic.open();
     byte[] content = new byte[8 + ODurablePage.NEXT_FREE_POSITION];
     fileClassic
@@ -726,7 +726,7 @@ private void assertFile(long pageIndex, byte[] value, OLogSequenceNumber lsn, St
 
   private void assertFileEncrypted(int fileId, int pageIndex, byte[] value, OLogSequenceNumber lsn, String fileName,
       final byte[] aesKey, final byte[] iv) throws Exception {
-    OFileClassic fileClassic = new OFileClassic(storagePath.resolve(fileName));
+    OFileClassic fileClassic = new OFileClassic(storagePath.resolve(fileName), pageSize);
     fileClassic.open();
     byte[] content = new byte[8 + ODurablePage.NEXT_FREE_POSITION];
     fileClassic