Skip to content

Commit

Permalink
Merge pull request #8553 from orientechnologies/issue8471
Browse files Browse the repository at this point in the history 
Issue8471 fix
  • Loading branch information
@markodjurovic
markodjurovic committed Sep 20, 2018
2 parents 173e26c + 295ea68 commit 39c4f4a
Show file tree
Hide file tree
Showing 5 changed files with 112 additions and 38 deletions.
12 changes: 9 additions & 3 deletions core/src/main/java/com/orientechnologies/common/io/OFileUtils.java
View file
Expand Up @@ -149,6 +149,10 @@ public static void checkValidName(final String iFileName) throws IOException {
}

public static void deleteRecursively(final File rootFile) {
deleteRecursively(rootFile, false);
}

public static void deleteRecursively(final File rootFile, boolean onlyDirs) {
if (!rootFile.exists())
return;

Expand All @@ -157,15 +161,17 @@ public static void deleteRecursively(final File rootFile) {
Files.walkFileTree(rootPath, new SimpleFileVisitor<Path>() {
@Override
public FileVisitResult visitFile(Path file, BasicFileAttributes attrs) throws IOException {
if (file != null && file.toFile() != null && file.toFile().exists()){
file.toFile().delete();
if (!onlyDirs) {
if (file != null && file.toFile() != null && file.toFile().exists()) {
file.toFile().delete();
}
}
return FileVisitResult.CONTINUE;
}

@Override
public FileVisitResult postVisitDirectory(Path dir, IOException exc) throws IOException {
if (dir != null && dir.toFile() != null && dir.toFile().exists()){
if (dir != null && dir.toFile() != null && dir.toFile().exists()) {
dir.toFile().delete();
}
return FileVisitResult.CONTINUE;
Expand Down
39 changes: 39 additions & 0 deletions lucene/src/main/java/com/orientechnologies/lucene/engine/OLuceneDirectory.java
View file
@@ -0,0 +1,39 @@
/*
* Copyright 2018 OrientDB.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package com.orientechnologies.lucene.engine;

import org.apache.lucene.store.Directory;

/**
* @author mdjurovi
*/
public class OLuceneDirectory {
private final Directory dir;
private final String path;

public OLuceneDirectory(final Directory dir, final String path) {
this.dir = dir;
this.path = path;
}

public Directory getDirectory() {
return dir;
}

public String getPath() {
return path;
}
}
20 changes: 11 additions & 9 deletions lucene/src/main/java/com/orientechnologies/lucene/engine/OLuceneDirectoryFactory.java
View file
Expand Up @@ -27,37 +27,39 @@ public class OLuceneDirectoryFactory {

public static final String DIRECTORY_PATH = "directory_path";

public Directory createDirectory(ODatabaseDocumentInternal database, String indexName, ODocument metadata) {
public OLuceneDirectory createDirectory(ODatabaseDocumentInternal database, String indexName, ODocument metadata) {

String luceneType = metadata.containsField(DIRECTORY_TYPE) ? metadata.<String>field(DIRECTORY_TYPE) : DIRECTORY_MMAP;

if (database.getStorage().getType().equals("memory") || DIRECTORY_RAM.equals(luceneType)) {
return new RAMDirectory();
Directory dir = new RAMDirectory();
return new OLuceneDirectory(dir, null);
}

return createDirectory(database, indexName, metadata, luceneType);
}

private Directory createDirectory(ODatabaseDocumentInternal database, String indexName, ODocument metadata, String luceneType) {
private OLuceneDirectory createDirectory(ODatabaseDocumentInternal database, String indexName, ODocument metadata,
String luceneType) {
String luceneBasePath = metadata.containsField(DIRECTORY_PATH) ? metadata.<String>field(DIRECTORY_PATH) : OLUCENE_BASE_DIR;

Path luceneIndexPath = Paths.get(database.getStorage().getConfiguration().getDirectory(), luceneBasePath, indexName);
try {

Directory dir = null;
if (DIRECTORY_NIO.equals(luceneType)) {
return new NIOFSDirectory(luceneIndexPath);
}

if (DIRECTORY_MMAP.equals(luceneType)) {
return new MMapDirectory(luceneIndexPath);
dir = new NIOFSDirectory(luceneIndexPath);
} else if (DIRECTORY_MMAP.equals(luceneType)) {
dir = new MMapDirectory(luceneIndexPath);
}

return new OLuceneDirectory(dir, luceneIndexPath.toString());
} catch (IOException e) {
OLogManager.instance().error(this, "unable to create Lucene Directory with type " + luceneType, e);
}

OLogManager.instance().warn(this, "unable to create Lucene Directory, FALL BACK to ramDir");
return new RAMDirectory();
return new OLuceneDirectory(new RAMDirectory(), null);
}

}
71 changes: 49 additions & 22 deletions lucene/src/main/java/com/orientechnologies/lucene/engine/OLuceneIndexEngineAbstract.java
View file
Expand Up @@ -18,6 +18,7 @@

import com.orientechnologies.common.concur.resource.OSharedResourceAdaptiveExternal;
import com.orientechnologies.common.exception.OException;
import com.orientechnologies.common.io.OFileUtils;
import com.orientechnologies.common.log.OLogManager;
import com.orientechnologies.common.serialization.types.OBinarySerializer;
import com.orientechnologies.lucene.analyzer.OLuceneAnalyzerFactory;
Expand Down Expand Up @@ -78,6 +79,10 @@
import static com.orientechnologies.lucene.analyzer.OLuceneAnalyzerFactory.AnalyzerKind.INDEX;
import static com.orientechnologies.lucene.analyzer.OLuceneAnalyzerFactory.AnalyzerKind.QUERY;

import java.io.File;
import java.nio.file.FileSystems;
import java.nio.file.Files;

public abstract class OLuceneIndexEngineAbstract extends OSharedResourceAdaptiveExternal implements OLuceneIndexEngine {

public static final String RID = "RID";
Expand All @@ -91,17 +96,17 @@ public abstract class OLuceneIndexEngineAbstract extends OSharedResourceAdaptive
protected ODocument metadata;
protected Version version;
Map<String, Boolean> collectionFields = new HashMap<>();
private TimerTask commitTask;
private AtomicBoolean closed;
private OStorage storage;
private volatile long reopenToken;
private Analyzer indexAnalyzer;
private Analyzer queryAnalyzer;
private volatile Directory directory;
private IndexWriter indexWriter;
private long flushIndexInterval;
private long closeAfterInterval;
private long firstFlushAfter;
private TimerTask commitTask;
private AtomicBoolean closed;
private OStorage storage;
private volatile long reopenToken;
private Analyzer indexAnalyzer;
private Analyzer queryAnalyzer;
private volatile OLuceneDirectory directory;
private IndexWriter indexWriter;
private long flushIndexInterval;
private long closeAfterInterval;
private long firstFlushAfter;

private Lock openCloseLock;

Expand Down Expand Up @@ -186,7 +191,8 @@ public void run() {
}

private boolean shouldClose() {
return !(directory instanceof RAMDirectory) && System.currentTimeMillis() - lastAccess.get() > closeAfterInterval;
return !(directory.getDirectory() instanceof RAMDirectory)
&& System.currentTimeMillis() - lastAccess.get() > closeAfterInterval;
}

private void checkCollectionIndex(OIndexDefinition indexDefinition) {
Expand All @@ -207,7 +213,7 @@ private void checkCollectionIndex(OIndexDefinition indexDefinition) {

private void reOpen() throws IOException {

if (indexWriter != null && indexWriter.isOpen() && directory instanceof RAMDirectory) {
if (indexWriter != null && indexWriter.isOpen() && directory.getDirectory() instanceof RAMDirectory) {
// don't waste time reopening an in memory index
return;
}
Expand All @@ -221,7 +227,6 @@ protected ODatabaseDocumentInternal getDatabase() {

private void open() throws IOException {


openCloseLock.lock();

if (!closed.get())
Expand All @@ -232,7 +237,7 @@ private void open() throws IOException {

directory = directoryFactory.createDirectory(getDatabase(), name, metadata);

indexWriter = createIndexWriter(directory);
indexWriter = createIndexWriter(directory.getDirectory());
searcherManager = new SearcherManager(indexWriter, true, true, null);

reopenToken = 0;
Expand Down Expand Up @@ -343,26 +348,47 @@ public void delete() {

final OAbstractPaginatedStorage storageLocalAbstract = (OAbstractPaginatedStorage) storage.getUnderlying();
if (storageLocalAbstract instanceof OLocalPaginatedStorage) {
deleteIndexFolder();
OLocalPaginatedStorage localStorage = (OLocalPaginatedStorage) storageLocalAbstract;
File storagePath = localStorage.getStoragePath().toFile();
deleteIndexFolder(storagePath);
}
} catch (IOException e) {
throw OException.wrapException(new OStorageException("Error during deletion of Lucene index " + name), e);
}
}

private void deleteIndexFolder() throws IOException {
final String[] files = directory.listAll();
private void deleteIndexFolder(File baseStoragePath) throws IOException {
final String[] files = directory.getDirectory().listAll();
for (String fileName : files) {
directory.deleteFile(fileName);
directory.getDirectory().deleteFile(fileName);
}
directory.getDirectory().close();
String indexPath = directory.getPath();
if (indexPath != null) {
File indexDir = new File(indexPath);
while (true) {
if (Files.isSameFile(FileSystems.getDefault().getPath(baseStoragePath.getCanonicalPath()),
FileSystems.getDefault().getPath(indexDir.getCanonicalPath()))) {
break;
}
//delete only if dir is empty, otherwise stop deleting process
//last index will remove all upper dirs
if (indexDir.listFiles().length == 0) {
OFileUtils.deleteRecursively(indexDir, true);
indexDir = indexDir.getParentFile();
} else {
break;
}
}
}
directory.close();
}

@Override
public String indexName() {
return name;
}

@Override
public abstract void onRecordAddedToResultSet(OLuceneQueryContext queryContext, OContextualRecordId recordId, Document ret,
ScoreDoc score);

Expand Down Expand Up @@ -494,7 +520,8 @@ private void internalDelete() throws IOException {

final OAbstractPaginatedStorage storageLocalAbstract = (OAbstractPaginatedStorage) storage.getUnderlying();
if (storageLocalAbstract instanceof OLocalPaginatedStorage) {
deleteIndexFolder();
OLocalPaginatedStorage localPaginatedStorage = (OLocalPaginatedStorage) storageLocalAbstract;
deleteIndexFolder(localPaginatedStorage.getStoragePath().toFile());
}
}

Expand Down Expand Up @@ -533,7 +560,7 @@ private void doClose(boolean onDelete) {
commitAndCloseWriter();

if (!onDelete)
directory.close();
directory.getDirectory().close();
} catch (Exception e) {
OLogManager.instance().error(this, "Error on closing Lucene index", e);
}
Expand Down
8 changes: 4 additions & 4 deletions lucene/src/test/java/com/orientechnologies/lucene/engine/OLuceneDirectoryFactoryTest.java
View file
Expand Up @@ -47,7 +47,7 @@ public void shouldCreateNioFsDirectory() throws Exception {

ODatabaseDocumentTx db = dropOrCreate("plocal:./target/testDatabase/" + name.getMethodName(), true);

Directory directory = fc.createDirectory(db, "index.name", meta);
Directory directory = fc.createDirectory(db, "index.name", meta).getDirectory();

assertThat(directory).isInstanceOf(NIOFSDirectory.class);

Expand All @@ -64,7 +64,7 @@ public void shouldCreateMMapFsDirectory() throws Exception {

ODatabaseDocumentTx db = dropOrCreate("plocal:./target/testDatabase/" + name.getMethodName(), true);

Directory directory = fc.createDirectory(db, "index.name", meta);
Directory directory = fc.createDirectory(db, "index.name", meta).getDirectory();

assertThat(directory).isInstanceOf(MMapDirectory.class);

Expand All @@ -81,7 +81,7 @@ public void shouldCreateRamDirectory() throws Exception {

ODatabaseDocumentTx db = dropOrCreate("plocal:./target/testDatabase/" + name.getMethodName(), true);

Directory directory = fc.createDirectory(db, "index.name", meta);
Directory directory = fc.createDirectory(db, "index.name", meta).getDirectory();

assertThat(directory).isInstanceOf(RAMDirectory.class);

Expand All @@ -97,7 +97,7 @@ public void shouldCreateRamDirectoryOnMemoryDatabase() throws Exception {

ODatabaseDocumentTx db = dropOrCreate("memory:" + name.getMethodName(), true);

Directory directory = fc.createDirectory(db, "index.name", meta);
Directory directory = fc.createDirectory(db, "index.name", meta).getDirectory();

assertThat(directory).isInstanceOf(RAMDirectory.class);
}
Expand Down

0 comments on commit 39c4f4a

Please sign in to comment.