Skip to content

Commit

Permalink
Lazy file allocation was implemented
Browse files Browse the repository at this point in the history
  • Loading branch information
@laa
laa committed Aug 12, 2019
1 parent 753439f commit 854d26b
Show file tree
Hide file tree
Showing 3 changed files with 67 additions and 38 deletions.
20 changes: 15 additions & 5 deletions core/src/main/java/com/orientechnologies/orient/core/storage/cache/local/OWOWCache.java
View file
Expand Up @@ -1808,7 +1808,7 @@ private void convertNameIdMapFromV1ToV2() throws IOException {

private OFileClassic createFileInstance(final String fileName, final int fileId) {
final String internalFileName = createInternalFileName(fileName, fileId);
return new OFileClassic(storagePath.resolve(internalFileName));
return new OFileClassic(storagePath.resolve(internalFileName), pageSize);
}

private static String createInternalFileName(final String fileName, final int fileId) {
Expand Down Expand Up @@ -1877,7 +1877,7 @@ private void readNameIdMapV2(FileChannel nameIdMapHolder) throws IOException, In

if (files.get(externalId) == null) {
final Path path = storagePath.resolve(idFileNameMap.get((nameIdEntry.getValue())));
final OFileClassic fileClassic = new OFileClassic(path);
final OFileClassic fileClassic = new OFileClassic(path, pageSize);

if (fileClassic.exists()) {
fileClassic.open();
Expand Down Expand Up @@ -1949,7 +1949,7 @@ private void readNameIdMapV1(final FileChannel nameIdMapHolder) throws IOExcepti
final long externalId = composeFileId(id, nameIdEntry.getValue());

if (files.get(externalId) == null) {
final OFileClassic fileClassic = new OFileClassic(storagePath.resolve(nameIdEntry.getKey()));
final OFileClassic fileClassic = new OFileClassic(storagePath.resolve(nameIdEntry.getKey()), pageSize);

if (fileClassic.exists()) {
fileClassic.open();
Expand Down Expand Up @@ -2735,7 +2735,12 @@ public void run() {
final Map.Entry<Long, TreeSet<PageKey>> lsnEntry = localDirtyPagesBySegment.firstEntry();

if (lsnEntry != null && lsnEntry.getKey() < endSegment) {
lsnPages += flushChunk(lsnFlushInterval, startSegment, endSegment);
final int flushedPages = flushChunk(lsnFlushInterval, startSegment, endSegment);
lsnPages += flushedPages;
if (flushedPages == 0) {
flushTs = System.nanoTime();
break;
}
}
}

Expand All @@ -2745,7 +2750,12 @@ public void run() {
final TreeSet<PageKey> pages = firstSegment.getValue();

if (pages.size() >= 1024 * 1024 * 1024 / pageSize) {
lsnPages += flushChunk(lsnFlushInterval, startSegment, endSegment);
final int flushedPages = flushChunk(lsnFlushInterval, startSegment, endSegment);
lsnPages += flushedPages;
if (flushedPages == 0) {
flushTs = System.nanoTime();
break;
}
}
}
}
Expand Down
69 changes: 44 additions & 25 deletions core/src/main/java/com/orientechnologies/orient/core/storage/fs/OFileClassic.java
View file
Expand Up @@ -71,11 +71,14 @@ public final class OFileClassic implements OClosableItem {

private int version;

private volatile long size;
private final AtomicLong size = new AtomicLong();
private volatile long committedSize = 0;

private AllocationMode allocationMode;
private int fd;

private final int pageSize;

/**
* Map which calculates which files are opened and how many users they have
*/
Expand All @@ -91,26 +94,38 @@ public final class OFileClassic implements OClosableItem {
*/
private final boolean trackFileOpen = OGlobalConfiguration.STORAGE_TRACK_FILE_ACCESS.getValueAsBoolean();

public OFileClassic(final Path osFile) {
public OFileClassic(final Path osFile, final int pageSize) {
this.osFile = osFile;
this.pageSize = pageSize;
}

public long allocateSpace(final int size) throws IOException {
acquireWriteLock();
acquireReadLock();
try {
final long currentSize = this.size;
assert channel.size() == currentSize + HEADER_SIZE;
final long currentSize = this.size.addAndGet(size);
final long currentCommittedSize = this.committedSize;

//noinspection NonAtomicOperationOnVolatileField
this.size += size;
final int pagesDifference = (int) ((currentSize - currentCommittedSize) / pageSize);
if (pagesDifference < 10 || pagesDifference < 0.1 * currentCommittedSize) {
return currentSize;
}
} finally {
releaseReadLock();
}

assert this.size >= size;
acquireWriteLock();
try {
final long currentSize = this.size.get();
assert channel.size() == this.committedSize + HEADER_SIZE;

final long sizeDiff = currentSize - this.committedSize;
this.committedSize = currentSize;

assert allocationMode != null;
if (allocationMode == AllocationMode.WRITE) {
final long ptr = Native.malloc(size);
final long ptr = Native.malloc(sizeDiff);
try {
final ByteBuffer buffer = new Pointer(ptr).getByteBuffer(0, size);
final ByteBuffer buffer = new Pointer(ptr).getByteBuffer(0, sizeDiff);
buffer.position(0);
OIOUtils.writeByteBuffer(buffer, channel, currentSize + HEADER_SIZE);
} finally {
Expand All @@ -120,7 +135,7 @@ public long allocateSpace(final int size) throws IOException {
assert fd > 0;

try {
ONative.instance().fallocate(fd, currentSize + HEADER_SIZE, size);
ONative.instance().fallocate(fd, currentSize + HEADER_SIZE, sizeDiff);
} catch (final LastErrorException e) {
OLogManager.instance()
.debug(this, "Can not allocate space (error %d) for file %s using native Linux API, more slower methods will be used",
Expand All @@ -136,9 +151,9 @@ public long allocateSpace(final int size) throws IOException {
lee.getErrorCode());
}

final long ptr = Native.malloc(size);
final long ptr = Native.malloc(sizeDiff);
try {
final ByteBuffer buffer = new Pointer(ptr).getByteBuffer(0, size);
final ByteBuffer buffer = new Pointer(ptr).getByteBuffer(0, sizeDiff);
buffer.position(0);
OIOUtils.writeByteBuffer(buffer, channel, currentSize + HEADER_SIZE);
} finally {
Expand All @@ -147,12 +162,12 @@ public long allocateSpace(final int size) throws IOException {
}

} else if (allocationMode == AllocationMode.LENGTH) {
frnd.setLength(this.size + HEADER_SIZE);
frnd.setLength(currentSize + HEADER_SIZE);
} else {
throw new IllegalStateException("Unknown allocation mode");
}

assert channel.size() == this.size + HEADER_SIZE;
assert channel.size() == currentSize + HEADER_SIZE;
return currentSize;
} finally {
releaseWriteLock();
Expand All @@ -167,17 +182,19 @@ public void shrink(final long size) throws IOException {
try {
//noinspection resource
channel.truncate(HEADER_SIZE + size);
this.size = size;
this.size.set(size);
this.committedSize = size;

assert this.size >= 0;
assert this.committedSize >= 0;
assert this.size.get() >= 0;

} finally {
releaseWriteLock();
}
}

public long getFileSize() {
return size;
return size.get();
}

public void read(long offset, final ByteBuffer buffer, final boolean throwOnEof) throws IOException {
Expand Down Expand Up @@ -289,16 +306,16 @@ private void initAllocationMode() {
/**
* ALWAYS ADD THE HEADER SIZE BECAUSE ON THIS TYPE IS ALWAYS NEEDED
*/
private long checkRegions(final long iOffset, final long iLength) {
private long checkRegions(final long offset, final long iLength) {
acquireReadLock();
try {
if (iOffset < 0 || iOffset + iLength > size) {
if (offset < 0 || offset + iLength > size.get()) {
throw new OIOException(
"You cannot access outside the file size (" + size + " bytes). You have requested portion from " + iOffset + "-" + (iOffset
+ iLength) + " bytes. File: " + this);
"You cannot access outside the file size (" + size.get() + " bytes). You have requested portion from " + offset + "-"
+ (offset + iLength) + " bytes. File: " + this);
}

return iOffset + HEADER_SIZE;
return offset + HEADER_SIZE;
} finally {
releaseReadLock();
}
Expand Down Expand Up @@ -520,8 +537,10 @@ private void openChannel() throws IOException {
}

private void init() throws IOException {
size = channel.size() - HEADER_SIZE;
assert size >= 0;
size.set(channel.size() - HEADER_SIZE);
assert size.get() >= 0;

this.committedSize = size.get();

final ByteBuffer buffer = ByteBuffer.allocate(1);

Expand Down
16 changes: 8 additions & 8 deletions ...com/orientechnologies/orient/core/storage/index/hashindex/local/cache/WOWCacheTestIT.java
View file
Expand Up @@ -541,7 +541,7 @@ public void testChecksumFailure() throws IOException {
wowCache.flush();

final Path path = storagePath.resolve(wowCache.nativeFileNameById(fileId));
final OFileClassic file = new OFileClassic(path);
final OFileClassic file = new OFileClassic(path, pageSize);
file.open();
file.write(ODurablePage.NEXT_FREE_POSITION, ByteBuffer.wrap(new byte[] { 1 }).order(ByteOrder.nativeOrder()));
file.close();
Expand Down Expand Up @@ -574,7 +574,7 @@ public void testMagicFailure() throws IOException {
wowCache.flush();

final Path path = storagePath.resolve(wowCache.nativeFileNameById(fileId));
final OFileClassic file = new OFileClassic(path);
final OFileClassic file = new OFileClassic(path, pageSize);
file.open();
file.write(0, ByteBuffer.wrap(new byte[] { 1 }).order(ByteOrder.nativeOrder()));
file.close();
Expand Down Expand Up @@ -607,7 +607,7 @@ public void testNoChecksumVerificationIfNotRequested() throws IOException {
wowCache.flush();

final Path path = storagePath.resolve(wowCache.nativeFileNameById(fileId));
final OFileClassic file = new OFileClassic(path);
final OFileClassic file = new OFileClassic(path, pageSize);
file.open();
file.write(ODurablePage.NEXT_FREE_POSITION, ByteBuffer.wrap(new byte[] { 1 }).order(ByteOrder.nativeOrder()));
file.close();
Expand Down Expand Up @@ -635,7 +635,7 @@ public void testNoChecksumFailureIfVerificationTurnedOff() throws IOException {
wowCache.flush();

final Path path = storagePath.resolve(wowCache.nativeFileNameById(fileId));
final OFileClassic file = new OFileClassic(path);
final OFileClassic file = new OFileClassic(path, pageSize);
file.open();
file.write(ODurablePage.NEXT_FREE_POSITION, ByteBuffer.wrap(new byte[] { 1 }).order(ByteOrder.nativeOrder()));
file.close();
Expand Down Expand Up @@ -663,7 +663,7 @@ public void testNoChecksumFailureIfVerificationTurnedOffOnLoad() throws IOExcept
wowCache.flush();

final Path path = storagePath.resolve(wowCache.nativeFileNameById(fileId));
final OFileClassic file = new OFileClassic(path);
final OFileClassic file = new OFileClassic(path, pageSize);
file.open();
file.write(ODurablePage.NEXT_FREE_POSITION, ByteBuffer.wrap(new byte[] { 1 }).order(ByteOrder.nativeOrder()));
file.close();
Expand Down Expand Up @@ -691,7 +691,7 @@ public void testNoChecksumFailureIfNoChecksumProvided() throws IOException {
wowCache.flush();

final Path path = storagePath.resolve(wowCache.nativeFileNameById(fileId));
final OFileClassic file = new OFileClassic(path);
final OFileClassic file = new OFileClassic(path, pageSize);
file.open();
file.write(ODurablePage.NEXT_FREE_POSITION, ByteBuffer.wrap(new byte[] { 1 }).order(ByteOrder.nativeOrder()));
file.close();
Expand All @@ -701,7 +701,7 @@ public void testNoChecksumFailureIfNoChecksumProvided() throws IOException {
}

private void assertFile(long pageIndex, byte[] value, OLogSequenceNumber lsn, String fileName) throws IOException {
OFileClassic fileClassic = new OFileClassic(storagePath.resolve(fileName));
OFileClassic fileClassic = new OFileClassic(storagePath.resolve(fileName), pageSize);
fileClassic.open();
byte[] content = new byte[8 + ODurablePage.NEXT_FREE_POSITION];
fileClassic
Expand All @@ -726,7 +726,7 @@ private void assertFile(long pageIndex, byte[] value, OLogSequenceNumber lsn, St

private void assertFileEncrypted(int fileId, int pageIndex, byte[] value, OLogSequenceNumber lsn, String fileName,
final byte[] aesKey, final byte[] iv) throws Exception {
OFileClassic fileClassic = new OFileClassic(storagePath.resolve(fileName));
OFileClassic fileClassic = new OFileClassic(storagePath.resolve(fileName), pageSize);
fileClassic.open();
byte[] content = new byte[8 + ODurablePage.NEXT_FREE_POSITION];
fileClassic
Expand Down

0 comments on commit 854d26b

Please sign in to comment.