Skip to content

Commit

Permalink
Merge branch 'develop' into laa_develop
Browse files Browse the repository at this point in the history 
# Conflicts:
#	core/src/main/java/com/orientechnologies/orient/core/storage/ridbag/sbtree/OSBTreeCollectionManagerAbstract.java
#	core/src/main/java/com/orientechnologies/orient/core/storage/ridbag/sbtree/OSBTreeCollectionManagerShared.java
  • Loading branch information
@laa
laa committed Aug 1, 2019
2 parents 05c65b2 + 2c86c2a commit f889302
Show file tree
Hide file tree
Showing 17 changed files with 176 additions and 80 deletions.
14 changes: 6 additions & 8 deletions client/src/main/java/com/orientechnologies/orient/client/remote/OSBTreeBonsaiRemote.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -22,14 +22,7 @@

import com.orientechnologies.common.serialization.types.OBinarySerializer;
import com.orientechnologies.common.serialization.types.OByteSerializer;
import com.orientechnologies.orient.client.remote.message.OSBTFetchEntriesMajorRequest;
import com.orientechnologies.orient.client.remote.message.OSBTFetchEntriesMajorResponse;
import com.orientechnologies.orient.client.remote.message.OSBTFirstKeyRequest;
import com.orientechnologies.orient.client.remote.message.OSBTFirstKeyResponse;
import com.orientechnologies.orient.client.remote.message.OSBTGetRealBagSizeRequest;
import com.orientechnologies.orient.client.remote.message.OSBTGetRealBagSizeResponse;
import com.orientechnologies.orient.client.remote.message.OSBTGetRequest;
import com.orientechnologies.orient.client.remote.message.OSBTGetResponse;
import com.orientechnologies.orient.client.remote.message.*;
import com.orientechnologies.orient.core.db.ODatabaseRecordThreadLocal;
import com.orientechnologies.orient.core.db.record.OIdentifiable;
import com.orientechnologies.orient.core.serialization.serializer.binary.OBinarySerializerFactory;
Expand Down Expand Up @@ -220,6 +213,11 @@ public OBinarySerializer<V> getValueSerializer() {
return valueSerializer;
}

@Override
public void markToDelete() {
throw new UnsupportedOperationException();
}

public static class TreeEntry<EK, EV> implements Map.Entry<EK, EV> {
private final EK key;
private final EV value;
Expand Down
3 changes: 3 additions & 0 deletions core/src/main/java/com/orientechnologies/orient/core/config/OGlobalConfiguration.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -482,6 +482,9 @@ public enum OGlobalConfiguration {// ENVIRONMENT
"Amount of values, after which a LINKBAG implementation will use an embedded values container (disabled by default)",
Integer.class, -1, true),

RID_BAG_SBTREEBONSAI_DELETE_DALAY("ridBag.sbtreeBonsaiDeleteDelay",
"How long should pass from last access before delete an already converted ridbag", Integer.class, 30000),

// FILE
@Deprecated TRACK_FILE_CLOSE("file.trackFileClose",
"Log all the cases when files are closed. This is needed only for internal debugging purposes", Boolean.class, false),
Expand Down
4 changes: 4 additions & 0 deletions core/src/main/java/com/orientechnologies/orient/core/db/ODatabaseDocumentInternal.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -39,12 +39,14 @@
import com.orientechnologies.orient.core.sql.executor.OResultSet;
import com.orientechnologies.orient.core.storage.ORecordCallback;
import com.orientechnologies.orient.core.storage.OStorage;
import com.orientechnologies.orient.core.storage.ridbag.sbtree.OBonsaiCollectionPointer;
import com.orientechnologies.orient.core.storage.ridbag.sbtree.OSBTreeCollectionManager;
import com.orientechnologies.orient.core.tx.OTransaction;
import com.orientechnologies.orient.core.tx.OTransactionAbstract;
import com.orientechnologies.orient.core.tx.OTransactionInternal;

import java.util.Map;
import java.util.UUID;
import java.util.concurrent.ExecutionException;

public interface ODatabaseDocumentInternal extends ODatabaseSession, ODatabaseInternal<ORecord> {
Expand Down Expand Up @@ -255,4 +257,6 @@ default OResultSet indexQuery(String indexName, String query, Object... args) {
default boolean isDistributed() {
return false;
}

Map<UUID, OBonsaiCollectionPointer> getCollectionsChanges();
}
11 changes: 10 additions & 1 deletion ...rc/main/java/com/orientechnologies/orient/core/db/document/ODatabaseDocumentAbstract.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -60,6 +60,7 @@
import com.orientechnologies.orient.core.storage.*;
import com.orientechnologies.orient.core.storage.impl.local.OFreezableStorageComponent;
import com.orientechnologies.orient.core.storage.impl.local.OMicroTransaction;
import com.orientechnologies.orient.core.storage.ridbag.sbtree.OBonsaiCollectionPointer;
import com.orientechnologies.orient.core.storage.ridbag.sbtree.OSBTreeCollectionManager;
import com.orientechnologies.orient.core.tx.*;

Expand Down Expand Up @@ -102,7 +103,8 @@ public abstract class ODatabaseDocumentAbstract extends OListenerManger<ODatabas

protected OMicroTransaction microTransaction = null;

protected Map<String, OResultSet> activeQueries = new HashMap<>();
protected Map<String, OResultSet> activeQueries = new HashMap<>();
private Map<UUID, OBonsaiCollectionPointer> collectionsChanges;

protected ODatabaseDocumentAbstract() {
// DO NOTHING IS FOR EXTENDED OBJECTS
Expand Down Expand Up @@ -2473,4 +2475,11 @@ protected void pessimisticLockChecks(ORID recordId) {
}
}

public Map<UUID, OBonsaiCollectionPointer> getCollectionsChanges() {
if (collectionsChanges == null) {
collectionsChanges = new HashMap<>();
}
return collectionsChanges;
}

}
6 changes: 6 additions & 0 deletions core/src/main/java/com/orientechnologies/orient/core/db/document/ODatabaseDocumentTx.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -48,6 +48,7 @@
import com.orientechnologies.orient.core.storage.ORecordCallback;
import com.orientechnologies.orient.core.storage.ORecordMetadata;
import com.orientechnologies.orient.core.storage.OStorage;
import com.orientechnologies.orient.core.storage.ridbag.sbtree.OBonsaiCollectionPointer;
import com.orientechnologies.orient.core.storage.ridbag.sbtree.OSBTreeCollectionManager;
import com.orientechnologies.orient.core.tx.OTransaction;
import com.orientechnologies.orient.core.tx.OTransactionAbstract;
Expand Down Expand Up @@ -1560,4 +1561,9 @@ public void unlock(ORID recordId) throws OLockException {
public <T> T sendSequenceAction(OSequenceAction action) throws ExecutionException, InterruptedException {
throw new UnsupportedOperationException("Not supported yet."); //To change body of generated methods, choose Tools | Templates.
}

@Override
public Map<UUID, OBonsaiCollectionPointer> getCollectionsChanges() {
return internal.getCollectionsChanges();
}
}
39 changes: 14 additions & 25 deletions ...ologies/orient/core/storage/impl/local/paginated/ORidBagDeleteSerializationOperation.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -21,24 +21,27 @@
package com.orientechnologies.orient.core.storage.impl.local.paginated;

import com.orientechnologies.common.exception.OException;
import com.orientechnologies.orient.core.Orient;
import com.orientechnologies.orient.core.db.ODatabaseRecordThreadLocal;
import com.orientechnologies.orient.core.db.record.OIdentifiable;
import com.orientechnologies.orient.core.exception.ODatabaseException;
import com.orientechnologies.orient.core.storage.impl.local.OAbstractPaginatedStorage;
import com.orientechnologies.orient.core.storage.index.sbtreebonsai.local.OSBTreeBonsai;
import com.orientechnologies.orient.core.storage.ridbag.sbtree.OBonsaiCollectionPointer;
import com.orientechnologies.orient.core.storage.ridbag.sbtree.OSBTreeCollectionManager;
import com.orientechnologies.orient.core.storage.ridbag.sbtree.OSBTreeCollectionManagerShared;
import com.orientechnologies.orient.core.storage.ridbag.sbtree.OSBTreeRidBag;

import java.io.IOException;
import java.util.ArrayList;
import java.util.List;

import static com.orientechnologies.orient.core.config.OGlobalConfiguration.RID_BAG_SBTREEBONSAI_DELETE_DALAY;

public class ORidBagDeleteSerializationOperation implements ORecordSerializationOperation {
private final OBonsaiCollectionPointer collectionPointer;

private final OSBTreeCollectionManager collectionManager;
private final OSBTreeRidBag ridBag;
private Runnable deleteTask;

public ORidBagDeleteSerializationOperation(OBonsaiCollectionPointer collectionPointer, OSBTreeRidBag ridBag) {
this.collectionPointer = collectionPointer;
Expand All @@ -50,35 +53,21 @@ public ORidBagDeleteSerializationOperation(OBonsaiCollectionPointer collectionPo
public void execute(OAbstractPaginatedStorage paginatedStorage) {
OSBTreeBonsai<OIdentifiable, Integer> treeBonsai = loadTree();
try {
final OIdentifiable first = treeBonsai.firstKey();
if (first != null) {
final OIdentifiable last = treeBonsai.lastKey();
final List<OIdentifiable> entriesToDelete = new ArrayList<>(64);

treeBonsai.loadEntriesBetween(first, true, last, true, (entry) -> {
final int count = entry.getValue();
final OIdentifiable rid = entry.getKey();

for (int i = 0; i < count; i++) {
entriesToDelete.add(rid);
}

return true;
});

for (final OIdentifiable identifiable : entriesToDelete) {
treeBonsai.remove(identifiable);
}
}

treeBonsai.delete();
treeBonsai.markToDelete();
} catch (IOException e) {
throw OException.wrapException(new ODatabaseException("Error during ridbag deletion"), e);
} finally {
releaseTree();
}

collectionManager.delete(collectionPointer);
long delay = paginatedStorage.getConfiguration().getContextConfiguration().getValueAsInteger(RID_BAG_SBTREEBONSAI_DELETE_DALAY);
long schedule = delay / 3;
deleteTask = () -> {
if (!((OSBTreeCollectionManagerShared) collectionManager).tryDelete(collectionPointer, delay)) {
Orient.instance().scheduleTask(deleteTask, schedule, 0);
}
};
Orient.instance().scheduleTask(deleteTask, schedule, 0);
ridBag.confirmDelete();
}

Expand Down
2 changes: 2 additions & 0 deletions ...ava/com/orientechnologies/orient/core/storage/index/sbtreebonsai/local/OSBTreeBonsai.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -122,4 +122,6 @@ public interface OSBTreeBonsai<K, V> extends OTreeInternal<K, V> {
OBinarySerializer<K> getKeySerializer();

OBinarySerializer<V> getValueSerializer();

void markToDelete() throws IOException;
}
13 changes: 13 additions & 0 deletions ...m/orientechnologies/orient/core/storage/index/sbtreebonsai/local/OSBTreeBonsaiBucket.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -46,6 +46,7 @@ public class OSBTreeBonsaiBucket<K, V> extends OBonsaiBucketAbstract {
*/
private static final byte LEAF = 0x1;
private static final byte DELETED = 0x2;
private static final byte TO_DELETE = 0x4;
private static final int MAX_ENTREE_SIZE = 24576000;
private static final int FREE_POINTER_OFFSET = WAL_POSITION_OFFSET + OLongSerializer.LONG_SIZE;
private static final int SIZE_OFFSET = FREE_POINTER_OFFSET + OIntegerSerializer.INT_SIZE;
Expand Down Expand Up @@ -438,6 +439,18 @@ public boolean isDeleted() {
return (getByteValue(offset + FLAGS_OFFSET) & DELETED) == DELETED;
}

public void setToDelete(boolean toDelete) {
byte value = getByteValue(offset + FLAGS_OFFSET);
if (toDelete)
setByteValue(offset + FLAGS_OFFSET, (byte) (value | TO_DELETE));
else
//REMOVE THE FLAG the &(and) ~(not) is the opreation to remove flags in bits
setByteValue(offset + FLAGS_OFFSET, (byte) (value & (~TO_DELETE)));
}

public boolean isToDelete() {
return (getByteValue(offset + FLAGS_OFFSET) & TO_DELETE) == TO_DELETE;
}

public OBonsaiBucketPointer getLeftSibling() {
return getBucketPointer(offset + LEFT_SIBLING_OFFSET);
Expand Down
27 changes: 27 additions & 0 deletions ...om/orientechnologies/orient/core/storage/index/sbtreebonsai/local/OSBTreeBonsaiLocal.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -1545,4 +1545,31 @@ private void debugPrintBucket(final OBonsaiBucketPointer bucketPointer, final Pr
}

}

@Override
public void markToDelete() throws IOException {
boolean rollback = false;
final OAtomicOperation atomicOperation = startAtomicOperation(true);
try {
final Lock lock = FILE_LOCK_MANAGER.acquireExclusiveLock(fileId);
try {
final OCacheEntry cacheEntry = loadPageForWrite(atomicOperation, fileId, rootBucketPointer.getPageIndex(), false, true);
try {
OSBTreeBonsaiBucket<K, V> rootBucket = new OSBTreeBonsaiBucket<>(cacheEntry, rootBucketPointer.getPageOffset(),
keySerializer, valueSerializer, this);
rootBucket.setToDelete(true);
} finally {
releasePageFromWrite(atomicOperation, cacheEntry);
}
} finally {
lock.unlock();
}
} catch (final Exception e) {
rollback = true;
throw e;
} finally {
endAtomicOperation(rollback);
}

}
}
28 changes: 16 additions & 12 deletions ...orientechnologies/orient/core/storage/ridbag/sbtree/OSBTreeCollectionManagerAbstract.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -84,13 +84,13 @@ public static String generateLockName(int clusterId) {
GLOBAL_LOCKS = locks;
}

private final int evictionThreshold;
private final int cacheMaxSize;
private final int shift;
private final int mask;
private final Object[] locks;
private final ConcurrentLinkedHashMap<CacheKey, SBTreeBonsaiContainer> treeCache;
private final OStorage storage;
private final int evictionThreshold;
private final int cacheMaxSize;
private final int shift;
private final int mask;
private final Object[] locks;
protected final ConcurrentLinkedHashMap<CacheKey, SBTreeBonsaiContainer> treeCache;
private final OStorage storage;

public OSBTreeCollectionManagerAbstract(OStorage storage) {
this(GLOBAL_TREE_CACHE, storage, GLOBAL_EVICTION_THRESHOLD, GLOBAL_CACHE_MAX_SIZE, GLOBAL_LOCKS);
Expand Down Expand Up @@ -171,6 +171,7 @@ public OSBTreeBonsai<OIdentifiable, Integer> loadSBTree(OBonsaiCollectionPointer
SBTreeBonsaiContainer container = treeCache.get(cacheKey);
if (container != null) {
container.usagesCounter++;
container.lastAccessTime = System.currentTimeMillis();
tree = container.tree;
} else {
tree = loadTree(collectionPointer);
Expand All @@ -179,6 +180,7 @@ public OSBTreeBonsai<OIdentifiable, Integer> loadSBTree(OBonsaiCollectionPointer

container = new SBTreeBonsaiContainer(tree);
container.usagesCounter++;
container.lastAccessTime = System.currentTimeMillis();

treeCache.put(cacheKey, container);
}
Expand All @@ -201,6 +203,7 @@ public void releaseSBTree(OBonsaiCollectionPointer collectionPointer) {
assert container != null;
container.usagesCounter--;
assert container.usagesCounter >= 0;
container.lastAccessTime = System.currentTimeMillis();
}

evict();
Expand Down Expand Up @@ -274,23 +277,24 @@ int size() {
return treeCache.size();
}

private Object treesSubsetLock(CacheKey cacheKey) {
protected Object treesSubsetLock(CacheKey cacheKey) {
final int hashCode = cacheKey.hashCode();
final int index = (hashCode >>> shift) & mask;

return locks[index];
}

private static final class SBTreeBonsaiContainer {
private final OSBTreeBonsai<OIdentifiable, Integer> tree;
private int usagesCounter = 0;
protected static final class SBTreeBonsaiContainer {
private final OSBTreeBonsai<OIdentifiable, Integer> tree;
protected volatile int usagesCounter = 0;
protected volatile long lastAccessTime = 0;

private SBTreeBonsaiContainer(OSBTreeBonsai<OIdentifiable, Integer> tree) {
this.tree = tree;
}
}

private static final class CacheKey {
protected static final class CacheKey {
private final OStorage storage;
private final OBonsaiCollectionPointer pointer;

Expand Down

0 comments on commit f889302

Please sign in to comment.