From e1e45c48146cb3484bd0ebf4c59538553e7358cc Mon Sep 17 00:00:00 2001 From: laa Date: Mon, 9 Mar 2015 16:57:54 +0200 Subject: [PATCH] Issue #2874, add file and open file are separated in disk cache. --- .../ridbag/sbtree/OIndexRIDContainer.java | 12 ++- .../hashindex/local/OHashTableDirectory.java | 2 +- .../hashindex/local/OLocalHashTable.java | 6 +- .../hashindex/local/cache/ODiskCache.java | 4 + .../local/cache/OReadWriteDiskCache.java | 23 +++++ .../hashindex/local/cache/OWOWCache.java | 92 +++++++++++++++---- .../core/index/sbtree/local/OSBTree.java | 4 +- .../local/OSBTreeBonsaiLocal.java | 2 +- .../local/paginated/OClusterPositionMap.java | 2 +- .../local/paginated/OPaginatedCluster.java | 2 +- .../memory/ODirectMemoryOnlyDiskCache.java | 25 ++++- 11 files changed, 146 insertions(+), 28 deletions(-) diff --git a/core/src/main/java/com/orientechnologies/orient/core/db/record/ridbag/sbtree/OIndexRIDContainer.java b/core/src/main/java/com/orientechnologies/orient/core/db/record/ridbag/sbtree/OIndexRIDContainer.java index c7b1cafdc5b..bcaa3f127b9 100755 --- a/core/src/main/java/com/orientechnologies/orient/core/db/record/ridbag/sbtree/OIndexRIDContainer.java +++ b/core/src/main/java/com/orientechnologies/orient/core/db/record/ridbag/sbtree/OIndexRIDContainer.java @@ -29,6 +29,7 @@ import com.orientechnologies.orient.core.config.OGlobalConfiguration; import com.orientechnologies.orient.core.db.ODatabaseRecordThreadLocal; import com.orientechnologies.orient.core.db.record.OIdentifiable; +import com.orientechnologies.orient.core.index.hashindex.local.cache.ODiskCache; import com.orientechnologies.orient.core.index.sbtree.local.OSBTreeException; import com.orientechnologies.orient.core.storage.impl.local.OAbstractPaginatedStorage; @@ -49,6 +50,9 @@ public class OIndexRIDContainer implements Set { .getValueAsInteger(); private final boolean durableNonTxMode; + /** + * Should be called inside of lock to ensure uniqueness of entity on disk !!! + */ public OIndexRIDContainer(String name, boolean durableNonTxMode) { fileId = resolveFileIdByName(name + INDEX_FILE_EXTENSION); underlying = new HashSet(); @@ -73,9 +77,13 @@ private long resolveFileIdByName(String fileName) { final OAbstractPaginatedStorage storage = (OAbstractPaginatedStorage) ODatabaseRecordThreadLocal.INSTANCE.get().getStorage() .getUnderlying(); try { - return storage.getDiskCache().openFile(fileName); + ODiskCache diskCache = storage.getDiskCache(); + if (diskCache.exists(fileName)) + return diskCache.openFile(fileName); + + return diskCache.addFile(fileName); } catch (IOException e) { - throw new OSBTreeException("Error creation of sbtree with name" + fileName, e); + throw new OSBTreeException("Error creation of sbtree with name " + fileName, e); } } diff --git a/core/src/main/java/com/orientechnologies/orient/core/index/hashindex/local/OHashTableDirectory.java b/core/src/main/java/com/orientechnologies/orient/core/index/hashindex/local/OHashTableDirectory.java index 84bb8012dc8..bf6e4866ae1 100755 --- a/core/src/main/java/com/orientechnologies/orient/core/index/hashindex/local/OHashTableDirectory.java +++ b/core/src/main/java/com/orientechnologies/orient/core/index/hashindex/local/OHashTableDirectory.java @@ -75,7 +75,7 @@ public void create() throws IOException { startAtomicOperation(); acquireExclusiveLock(); try { - fileId = diskCache.openFile(name + defaultExtension); + fileId = diskCache.addFile(name + defaultExtension); logFileCreation(name + defaultExtension, fileId); init(); endAtomicOperation(false); diff --git a/core/src/main/java/com/orientechnologies/orient/core/index/hashindex/local/OLocalHashTable.java b/core/src/main/java/com/orientechnologies/orient/core/index/hashindex/local/OLocalHashTable.java index 0567ff3c95d..7efab6aede4 100755 --- a/core/src/main/java/com/orientechnologies/orient/core/index/hashindex/local/OLocalHashTable.java +++ b/core/src/main/java/com/orientechnologies/orient/core/index/hashindex/local/OLocalHashTable.java @@ -170,7 +170,7 @@ public void create(String name, OBinarySerializer keySerializer, OBinarySeria OAtomicOperation atomicOperation = startAtomicOperation(); try { - fileStateId = diskCache.openFile(name + metadataConfigurationFileExtension); + fileStateId = diskCache.addFile(name + metadataConfigurationFileExtension); logFileCreation(name + metadataConfigurationFileExtension, fileStateId); directory.create(); @@ -195,7 +195,7 @@ public void create(String name, OBinarySerializer keySerializer, OBinarySeria initHashTreeState(); if (nullKeyIsSupported) { - nullBucketFileId = diskCache.openFile(name + nullBucketFileExtension); + nullBucketFileId = diskCache.addFile(name + nullBucketFileExtension); logFileCreation(name + nullBucketFileExtension, nullBucketFileId); } @@ -326,7 +326,7 @@ public void setValueSerializer(OBinarySerializer valueSerializer) { private void createFileMetadata(int fileLevel, OHashIndexFileLevelMetadataPage page) throws IOException { final String fileName = name + fileLevel + bucketFileExtension; - final long fileId = diskCache.openFile(fileName); + final long fileId = diskCache.addFile(fileName); logFileCreation(fileName, fileId); diff --git a/core/src/main/java/com/orientechnologies/orient/core/index/hashindex/local/cache/ODiskCache.java b/core/src/main/java/com/orientechnologies/orient/core/index/hashindex/local/cache/ODiskCache.java index ad723e42413..04c549bbce7 100755 --- a/core/src/main/java/com/orientechnologies/orient/core/index/hashindex/local/cache/ODiskCache.java +++ b/core/src/main/java/com/orientechnologies/orient/core/index/hashindex/local/cache/ODiskCache.java @@ -63,6 +63,10 @@ * @since 14.03.13 */ public interface ODiskCache { + long addFile(String fileName) throws IOException; + + void addFile(String fileName, long fileId) throws IOException; + long openFile(String fileName) throws IOException; void openFile(long fileId) throws IOException; diff --git a/core/src/main/java/com/orientechnologies/orient/core/index/hashindex/local/cache/OReadWriteDiskCache.java b/core/src/main/java/com/orientechnologies/orient/core/index/hashindex/local/cache/OReadWriteDiskCache.java index 231517c6f1b..a7dcec62e64 100755 --- a/core/src/main/java/com/orientechnologies/orient/core/index/hashindex/local/cache/OReadWriteDiskCache.java +++ b/core/src/main/java/com/orientechnologies/orient/core/index/hashindex/local/cache/OReadWriteDiskCache.java @@ -136,6 +136,18 @@ LRUList getA1in() { return a1in; } + @Override + public long addFile(String fileName) throws IOException { + cacheLock.acquireWriteLock(); + try { + long fileId = writeCache.addFile(fileName); + filePages.put(fileId, Collections.newSetFromMap(new ConcurrentHashMap())); + return fileId; + } finally { + cacheLock.releaseWriteLock(); + } + } + @Override public long openFile(final String fileName) throws IOException { cacheLock.acquireWriteLock(); @@ -193,6 +205,17 @@ else if (existingFileId >= 0) } } + @Override + public void addFile(String fileName, long fileId) throws IOException { + cacheLock.acquireWriteLock(); + try { + writeCache.addFile(fileName, fileId); + filePages.put(fileId, Collections.newSetFromMap(new ConcurrentHashMap())); + } finally { + cacheLock.releaseWriteLock(); + } + } + @Override public boolean exists(final String fileName) { return writeCache.exists(fileName); diff --git a/core/src/main/java/com/orientechnologies/orient/core/index/hashindex/local/cache/OWOWCache.java b/core/src/main/java/com/orientechnologies/orient/core/index/hashindex/local/cache/OWOWCache.java index aed01a2a2b9..60bfc66242a 100755 --- a/core/src/main/java/com/orientechnologies/orient/core/index/hashindex/local/cache/OWOWCache.java +++ b/core/src/main/java/com/orientechnologies/orient/core/index/hashindex/local/cache/OWOWCache.java @@ -228,17 +228,38 @@ public long openFile(String fileName) throws IOException { else fileClassic = files.get(fileId); - if (fileClassic == null) { - fileId = ++fileCounter; + if (fileClassic == null) + throw new OStorageException("File with name " + fileName + " does not exist in storage " + storageLocal.getName()); - fileClassic = createFile(fileName); + openFile(fileClassic); - files.put(fileId, fileClassic); - nameIdMap.put(fileName, fileId); - writeNameIdEntry(new NameFileIdEntry(fileName, fileId), true); - } + return fileId; - openFile(fileClassic); + } finally { + filesLock.releaseWriteLock(); + } + } + + public long addFile(String fileName) throws IOException { + filesLock.acquireWriteLock(); + try { + initNameIdMapping(); + + Long fileId = nameIdMap.get(fileName); + OFileClassic fileClassic; + + if (fileId != null) + throw new OStorageException("File with name " + fileName + " already exists in storage " + storageLocal.getName()); + + fileId = ++fileCounter; + + fileClassic = createFile(fileName); + + files.put(fileId, fileClassic); + nameIdMap.put(fileName, fileId); + writeNameIdEntry(new NameFileIdEntry(fileName, fileId), true); + + addFile(fileClassic); return fileId; @@ -263,17 +284,48 @@ public void openFile(String fileName, long fileId) throws IOException { throw new OStorageException("File with given name already exists but has different id " + existingFileId + " vs. proposed " + fileId); } else { - if (fileCounter < fileId) - fileCounter = fileId; + throw new OStorageException("File with name " + fileName + " does not exist in storage " + storageLocal.getName()); + } - fileClassic = createFile(fileName); + openFile(fileClassic); + } finally { + filesLock.releaseWriteLock(); + } + } - files.put(fileId, fileClassic); - nameIdMap.put(fileName, fileId); - writeNameIdEntry(new NameFileIdEntry(fileName, fileId), true); + public void addFile(String fileName, long fileId) throws IOException { + filesLock.acquireWriteLock(); + try { + initNameIdMapping(); + + OFileClassic fileClassic; + + Long existingFileId = nameIdMap.get(fileName); + + if (existingFileId != null) { + if (existingFileId == fileId) + throw new OStorageException("File with name " + fileName + " already exists in storage " + storageLocal.getName()); + else + throw new OStorageException("File with given name already exists but has different id " + existingFileId + + " vs. proposed " + fileId); } - openFile(fileClassic); + fileClassic = files.get(fileId); + + if (fileClassic != null) + throw new OStorageException("File with given id exists but has different name " + fileClassic.getName() + " vs. proposed " + + fileName); + + if (fileCounter < fileId) + fileCounter = fileId; + + fileClassic = createFile(fileName); + + files.put(fileId, fileClassic); + nameIdMap.put(fileName, fileId); + writeNameIdEntry(new NameFileIdEntry(fileName, fileId), true); + + addFile(fileClassic); } finally { filesLock.releaseWriteLock(); } @@ -772,10 +824,18 @@ private void openFile(OFileClassic fileClassic) throws IOException { if (!fileClassic.isOpen()) fileClassic.open(); } else { + throw new OStorageException("File " + fileClassic + " does not exist."); + } + + } + + private void addFile(OFileClassic fileClassic) throws IOException { + if (!fileClassic.exists()) { fileClassic.create(-1); fileClassic.synch(); + } else { + throw new OStorageException("File " + fileClassic + " already exists."); } - } private void initNameIdMapping() throws IOException { diff --git a/core/src/main/java/com/orientechnologies/orient/core/index/sbtree/local/OSBTree.java b/core/src/main/java/com/orientechnologies/orient/core/index/sbtree/local/OSBTree.java index 3ecd7d5ddb0..b25217bcbf4 100755 --- a/core/src/main/java/com/orientechnologies/orient/core/index/sbtree/local/OSBTree.java +++ b/core/src/main/java/com/orientechnologies/orient/core/index/sbtree/local/OSBTree.java @@ -129,10 +129,10 @@ public void create(String name, OBinarySerializer keySerializer, OBinarySeria this.valueSerializer = valueSerializer; this.nullPointerSupport = nullPointerSupport; - fileId = diskCache.openFile(name + dataFileExtension); + fileId = diskCache.addFile(name + dataFileExtension); if (nullPointerSupport) - nullBucketFileId = diskCache.openFile(name + nullFileExtension); + nullBucketFileId = diskCache.addFile(name + nullFileExtension); initDurableComponent(storageLocal); diff --git a/core/src/main/java/com/orientechnologies/orient/core/index/sbtreebonsai/local/OSBTreeBonsaiLocal.java b/core/src/main/java/com/orientechnologies/orient/core/index/sbtreebonsai/local/OSBTreeBonsaiLocal.java index 10b231d0352..f076c3f9346 100755 --- a/core/src/main/java/com/orientechnologies/orient/core/index/sbtreebonsai/local/OSBTreeBonsaiLocal.java +++ b/core/src/main/java/com/orientechnologies/orient/core/index/sbtreebonsai/local/OSBTreeBonsaiLocal.java @@ -108,7 +108,7 @@ public void create(String name, OBinarySerializer keySerializer, OBinarySeria this.keySerializer = keySerializer; this.valueSerializer = valueSerializer; - this.fileId = diskCache.openFile(name + dataFileExtension); + this.fileId = diskCache.addFile(name + dataFileExtension); this.name = name; initAfterCreate(); diff --git a/core/src/main/java/com/orientechnologies/orient/core/storage/impl/local/paginated/OClusterPositionMap.java b/core/src/main/java/com/orientechnologies/orient/core/storage/impl/local/paginated/OClusterPositionMap.java index 605bb2549b9..bffd2091491 100755 --- a/core/src/main/java/com/orientechnologies/orient/core/storage/impl/local/paginated/OClusterPositionMap.java +++ b/core/src/main/java/com/orientechnologies/orient/core/storage/impl/local/paginated/OClusterPositionMap.java @@ -78,7 +78,7 @@ public void open() throws IOException { public void create() throws IOException { acquireExclusiveLock(); try { - fileId = diskCache.openFile(name + DEF_EXTENSION); + fileId = diskCache.addFile(name + DEF_EXTENSION); } finally { releaseExclusiveLock(); } diff --git a/core/src/main/java/com/orientechnologies/orient/core/storage/impl/local/paginated/OPaginatedCluster.java b/core/src/main/java/com/orientechnologies/orient/core/storage/impl/local/paginated/OPaginatedCluster.java index 2a77adfdc81..0d6b7ad9365 100755 --- a/core/src/main/java/com/orientechnologies/orient/core/storage/impl/local/paginated/OPaginatedCluster.java +++ b/core/src/main/java/com/orientechnologies/orient/core/storage/impl/local/paginated/OPaginatedCluster.java @@ -157,7 +157,7 @@ public void create(int startSize) throws IOException { try { acquireExclusiveLock(); try { - fileId = diskCache.openFile(name + DEF_EXTENSION); + fileId = diskCache.addFile(name + DEF_EXTENSION); startAtomicOperation(); diff --git a/core/src/main/java/com/orientechnologies/orient/core/storage/impl/memory/ODirectMemoryOnlyDiskCache.java b/core/src/main/java/com/orientechnologies/orient/core/storage/impl/memory/ODirectMemoryOnlyDiskCache.java index 39ed7a8e619..de5bca4653e 100755 --- a/core/src/main/java/com/orientechnologies/orient/core/storage/impl/memory/ODirectMemoryOnlyDiskCache.java +++ b/core/src/main/java/com/orientechnologies/orient/core/storage/impl/memory/ODirectMemoryOnlyDiskCache.java @@ -59,7 +59,7 @@ public ODirectMemoryOnlyDiskCache(int pageSize) { } @Override - public long openFile(String fileName) throws IOException { + public long addFile(String fileName) throws IOException { metadataLock.lock(); try { Long fileId = fileNameIdMap.get(fileName); @@ -73,6 +73,24 @@ public long openFile(String fileName) throws IOException { fileId = id; fileIdNameMap.put(fileId, fileName); + } else { + throw new OStorageException(fileName + " already exists."); + } + + return fileId; + } finally { + metadataLock.unlock(); + } + } + + @Override + public long openFile(String fileName) throws IOException { + metadataLock.lock(); + try { + Long fileId = fileNameIdMap.get(fileName); + + if (fileId == null) { + throw new OStorageException("File " + fileName + " does not exist."); } return fileId; @@ -93,6 +111,11 @@ public void openFile(String fileName, long fileId) throws IOException { throw new UnsupportedOperationException(); } + @Override + public void addFile(String fileName, long fileId) throws IOException { + throw new UnsupportedOperationException(); + } + @Override public OCacheEntry load(long fileId, long pageIndex, boolean checkPinnedPages) throws IOException { final MemoryFile memoryFile = getFile(fileId);