From 4bd76f9f646d8796980fb4e8d2db744e27f4bc11 Mon Sep 17 00:00:00 2001 From: Dawid Weiss Date: Thu, 19 Jul 2018 23:14:29 +0200 Subject: [PATCH 01/10] Initial removal of test assertions. --- .../org/apache/lucene/store/BaseDirectoryTestCase.java | 6 +----- .../java/org/apache/lucene/store/MockDirectoryWrapper.java | 7 +++---- 2 files changed, 4 insertions(+), 9 deletions(-) diff --git a/lucene/test-framework/src/java/org/apache/lucene/store/BaseDirectoryTestCase.java b/lucene/test-framework/src/java/org/apache/lucene/store/BaseDirectoryTestCase.java index bb9b7118604b..6e0516fb9c14 100644 --- a/lucene/test-framework/src/java/org/apache/lucene/store/BaseDirectoryTestCase.java +++ b/lucene/test-framework/src/java/org/apache/lucene/store/BaseDirectoryTestCase.java @@ -456,11 +456,7 @@ public void run() { } catch (FileNotFoundException | NoSuchFileException e) { // ignore } catch (IOException e) { - if (e.getMessage() != null && e.getMessage().contains("still open for writing")) { - // ignore - } else { - throw new RuntimeException(e); - } + throw new RuntimeException(e); } if (random().nextBoolean()) { break; diff --git a/lucene/test-framework/src/java/org/apache/lucene/store/MockDirectoryWrapper.java b/lucene/test-framework/src/java/org/apache/lucene/store/MockDirectoryWrapper.java index 019417771ed7..f1ac7ee2544f 100644 --- a/lucene/test-framework/src/java/org/apache/lucene/store/MockDirectoryWrapper.java +++ b/lucene/test-framework/src/java/org/apache/lucene/store/MockDirectoryWrapper.java @@ -650,7 +650,7 @@ public synchronized IndexOutput createOutput(String name, IOContext context) thr } init(); synchronized(this) { - if (createdFiles.contains(name) && !name.equals("segments.gen")) { + if (createdFiles.contains(name)) { throw new IOException("file \"" + name + "\" was already written to"); } } @@ -750,9 +750,8 @@ public synchronized IndexInput openInput(String name, IOContext context) throws throw randomState.nextBoolean() ? new FileNotFoundException(name + " in dir=" + in) : new NoSuchFileException(name + " in dir=" + in); } - // cannot open a file for input if it's still open for - // output, except for segments.gen and segments_N - if (!allowReadingFilesStillOpenForWrite && openFilesForWrite.contains(name) && !name.startsWith("segments")) { + // cannot open a file for input if it's still open for output. + if (!allowReadingFilesStillOpenForWrite && openFilesForWrite.contains(name)) { throw (IOException) fillOpenTrace(new IOException("MockDirectoryWrapper: file \"" + name + "\" is still open for writing"), name, false); } From 386b9ceab13e9982cd2dd4d5fb279cd35580ec42 Mon Sep 17 00:00:00 2001 From: Dawid Weiss Date: Fri, 20 Jul 2018 10:55:26 +0200 Subject: [PATCH 02/10] Directory documentation updated. --- .../org/apache/lucene/store/Directory.java | 194 ++++++++++-------- .../lucene/store/BaseDirectoryTestCase.java | 24 ++- 2 files changed, 133 insertions(+), 85 deletions(-) diff --git a/lucene/core/src/java/org/apache/lucene/store/Directory.java b/lucene/core/src/java/org/apache/lucene/store/Directory.java index 0c6101d14e9e..1e0531960f22 100644 --- a/lucene/core/src/java/org/apache/lucene/store/Directory.java +++ b/lucene/core/src/java/org/apache/lucene/store/Directory.java @@ -27,102 +27,139 @@ import org.apache.lucene.util.IOUtils; -/** A Directory is a flat list of files. Files may be written once, when they - * are created. Once a file is created it may only be opened for read, or - * deleted. Random access is permitted both when reading and writing. +/** + * A {@code Directory} provides an abstraction layer for storing a + * list of files. A directory contains only files (no sub-folder hierarchy). * - *

Java's i/o APIs not used directly, but rather all i/o is - * through this API. This permits things such as:

+ * Implementing classes must comply with the following: * - * Directory locking is implemented by an instance of {@link - * LockFactory}. + * * + * @see FSDirectory + * @see RAMDirectory + * @see FilterDirectory */ public abstract class Directory implements Closeable { - /** - * Returns an array of strings, one for each entry in the directory, in sorted (UTF16, java's String.compare) order. + * Returns names of all files stored in this directory. + * The output must be in sorted (UTF-16, java's {@link String#compareTo}) order. * - * @throws IOException in case of IO error + * @throws IOException in case of I/O error */ public abstract String[] listAll() throws IOException; - /** Removes an existing file in the directory. */ + /** + * Removes an existing file in the directory. + * + * This method must throw either {@link NoSuchFileException} or {@link FileNotFoundException} + * if {@code name} points to a non-existing file. + * + * @param name the name of an existing file. + * @throws IOException in case of I/O error + */ public abstract void deleteFile(String name) throws IOException; /** - * Returns the length of a file in the directory. This method follows the - * following contract: - * - * - * @param name the name of the file for which to return the length. - * @throws IOException if there was an IO error while retrieving the file's - * length. + * Returns the byte length of a file in the directory. + * + * This method must throw either {@link NoSuchFileException} or {@link FileNotFoundException} + * if {@code name} points to a non-existing file. + * + * @param name the name of an existing file. + * @throws IOException in case of I/O error */ public abstract long fileLength(String name) throws IOException; - /** Creates a new, empty file in the directory with the given name. - Returns a stream writing this file. */ + /** + * Creates a new, empty file in the directory and returns an {@link IndexOutput} + * instance for appending data to this file. + * + * This method must throw {@link java.nio.file.FileAlreadyExistsException} if the file + * already exists. + * + * @param name the name of the file to create. + * @throws IOException in case of I/O error + */ public abstract IndexOutput createOutput(String name, IOContext context) throws IOException; - /** Creates a new, empty file for writing in the directory, with a - * temporary file name including prefix and suffix, ending with the - * reserved extension .tmp. Use - * {@link IndexOutput#getName} to see what name was used. */ + /** + * Creates a new, empty, temporary file in the directory and returns an {@link IndexOutput} + * instance for appending data to this file. + * + * The temporary file name (accessible via {@link IndexOutput#getName()}) will start with + * {@code prefix}, end with {@code suffix} and have a reserved file extension {@code .tmp}. + */ public abstract IndexOutput createTempOutput(String prefix, String suffix, IOContext context) throws IOException; /** - * Ensure that any writes to these files are moved to - * stable storage. Lucene uses this to properly commit - * changes to the index, to prevent a machine/OS crash + * Ensures that any writes to these files are moved to + * stable storage. + * + * Lucene uses this to properly commit changes to the index, to prevent a machine/OS crash * from corrupting the index. - *
- * NOTE: Clients may call this method for same files over - * and over again, so some impls might optimize for that. - * For other impls the operation can be a noop, for various - * reasons. + * + * @see #syncMetaData() */ public abstract void sync(Collection names) throws IOException; - + /** - * Renames {@code source} to {@code dest} as an atomic operation, - * where {@code dest} does not yet exist in the directory. - *

- * Notes: This method is used by IndexWriter to publish commits. - * It is ok if this operation is not truly atomic, for example - * both {@code source} and {@code dest} can be visible temporarily. - * It is just important that the contents of {@code dest} appear - * atomically, or an exception is thrown. + * Ensures that directory metadata, such as recent file renames, are moved to stable + * storage. + * + * @see #sync(Collection) */ - public abstract void rename(String source, String dest) throws IOException; + public abstract void syncMetaData() throws IOException; /** - * Ensure that directory metadata, such as recent file renames, are made - * durable. + * Renames {@code source} file to {@code dest} file where + * {@code dest} must not already exist in the directory. + * + * It is permitted for this operation to not be truly atomic, for example + * both {@code source} and {@code dest} can be visible temporarily in {@link #listAll()}. + * However, the implementation of this method must ensure the content of + * {@code dest} appears as the entire {@code source} atomically. So once + * {@code dest} is visible for readers, the entire content of previous {@code source} + * is visible. + * + * This method is used by IndexWriter to publish commits. */ - public abstract void syncMetaData() throws IOException; - - /** Returns a stream reading an existing file. - *

Throws {@link FileNotFoundException} or {@link NoSuchFileException} - * if the file does not exist. + public abstract void rename(String source, String dest) throws IOException; + + /** + * Opens a stream for reading an existing file. + * + * This method must throw either {@link NoSuchFileException} or {@link FileNotFoundException} + * if {@code name} points to a non-existing file. + * + * @param name the name of an existing file. + * @throws IOException in case of I/O error */ public abstract IndexInput openInput(String name, IOContext context) throws IOException; - /** Returns a stream reading an existing file, computing checksum as it reads */ + /** + * Opens a checksum-computing stream for reading an existing file. + * + * This method must throw either {@link NoSuchFileException} or {@link FileNotFoundException} + * if {@code name} points to a non-existing file. + * + * @param name the name of an existing file. + * @throws IOException in case of I/O error + */ public ChecksumIndexInput openChecksumInput(String name, IOContext context) throws IOException { return new BufferedChecksumIndexInput(openInput(name, context)); } /** - * Returns an obtained {@link Lock}. + * Acquires and returns a {@link Lock} for a file with the given name. + * * @param name the name of the lock file * @throws LockObtainFailedException (optional specific exception) if the lock could * not be obtained because it is currently held elsewhere. @@ -130,31 +167,15 @@ public ChecksumIndexInput openChecksumInput(String name, IOContext context) thro */ public abstract Lock obtainLock(String name) throws IOException; - /** Closes the store. */ + /** + * Closes the directory. + */ @Override public abstract void close() throws IOException; - @Override - public String toString() { - return getClass().getSimpleName() + '@' + Integer.toHexString(hashCode()); - } - /** - * Copies the file src in from to this directory under the new - * file name dest. - *

- * If you want to copy the entire source directory to the destination one, you - * can do so like this: - * - * 

-   * Directory to; // the directory to copy to
-   * for (String file : dir.listAll()) {
-   *   to.copyFrom(dir, file, newFile, IOContext.DEFAULT); // newFile can be either file, or a new name
-   * }
-   *
- *

- * NOTE: this method does not check whether dest exist and will - * overwrite it if it does. + * Copies an existing {@code src} file from directory {@code from} + * to a non-existent file {@code dest} in this directory. */ public void copyFrom(Directory from, String src, String dest, IOContext context) throws IOException { boolean success = false; @@ -169,14 +190,21 @@ public void copyFrom(Directory from, String src, String dest, IOContext context) } } + @Override + public String toString() { + return getClass().getSimpleName() + '@' + Integer.toHexString(hashCode()); + } + /** - * @throws AlreadyClosedException if this Directory is closed + * Ensures this directory is still open. + * + * @throws AlreadyClosedException if this directory is closed. */ protected void ensureOpen() throws AlreadyClosedException {} /** - * Returns the current pending deletions in this directory or an emtpy set - * if there are no known pending deletions. + * Returns a set of files currently pending deletion in this directory. + * * @lucene.internal */ public Set getPendingDeletions() throws IOException { diff --git a/lucene/test-framework/src/java/org/apache/lucene/store/BaseDirectoryTestCase.java b/lucene/test-framework/src/java/org/apache/lucene/store/BaseDirectoryTestCase.java index 6e0516fb9c14..5afe28d2d51d 100644 --- a/lucene/test-framework/src/java/org/apache/lucene/store/BaseDirectoryTestCase.java +++ b/lucene/test-framework/src/java/org/apache/lucene/store/BaseDirectoryTestCase.java @@ -19,6 +19,7 @@ import java.io.EOFException; import java.io.FileNotFoundException; import java.io.IOException; +import java.nio.file.FileAlreadyExistsException; import java.nio.file.Files; import java.nio.file.NoSuchFileException; import java.nio.file.Path; @@ -1097,6 +1098,25 @@ public void testCreateTempOutput() throws Throwable { dir.close(); } + public void testCreateOutput() throws IOException { + try (Directory dir = getDirectory(createTempDir())) { + String name = "file"; + try (IndexOutput out = dir.createOutput(name, IOContext.DEFAULT)) { + } + + // Try to create an existing file. + expectThrows(FileAlreadyExistsException.class, () -> { + try (IndexOutput out = dir.createOutput(name, IOContext.DEFAULT)) { + } + }); + + // Delete file and try to recreate it. + dir.deleteFile(name); + try (IndexOutput out = dir.createOutput(name, IOContext.DEFAULT)) { + } + } + } + public void testSeekToEndOfFile() throws IOException { try (Directory dir = getDirectory(createTempDir())) { try (IndexOutput out = dir.createOutput("a", IOContext.DEFAULT)) { @@ -1181,7 +1201,7 @@ public void testListAllIsSorted() throws IOException { try (Directory dir = getDirectory(createTempDir())) { int count = atLeast(20); Set names = new HashSet<>(); - while(names.size() < count) { + while (names.size() < count) { // create a random filename (segment file name style), so it cannot hit windows problem with special filenames ("con", "com1",...): String name = IndexFileNames.segmentFileName(TestUtil.randomSimpleString(random(), 1, 6), TestUtil.randomSimpleString(random()), "test"); if (random().nextInt(5) == 1) { @@ -1197,7 +1217,7 @@ public void testListAllIsSorted() throws IOException { String[] actual = dir.listAll(); String[] expected = actual.clone(); Arrays.sort(expected); - assertEquals(expected, actual); + assertArrayEquals(expected, actual); } } } From 87eaf89a63d10d0c55c184ebf01279646fb87f4f Mon Sep 17 00:00:00 2001 From: Dawid Weiss Date: Fri, 20 Jul 2018 11:22:42 +0200 Subject: [PATCH 03/10] Add contract check for open-before-writes-close. --- .../lucene/store/BaseDirectoryTestCase.java | 26 +++++++++++-------- 1 file changed, 15 insertions(+), 11 deletions(-) diff --git a/lucene/test-framework/src/java/org/apache/lucene/store/BaseDirectoryTestCase.java b/lucene/test-framework/src/java/org/apache/lucene/store/BaseDirectoryTestCase.java index 5afe28d2d51d..1d838c69d5e4 100644 --- a/lucene/test-framework/src/java/org/apache/lucene/store/BaseDirectoryTestCase.java +++ b/lucene/test-framework/src/java/org/apache/lucene/store/BaseDirectoryTestCase.java @@ -1046,15 +1046,6 @@ public void testIndexOutputToString() throws Throwable { dir.close(); } - public void testDoubleCloseDirectory() throws Throwable { - Directory dir = getDirectory(createTempDir()); - IndexOutput out = dir.createOutput("foobar", newIOContext(random())); - out.writeString("testing"); - out.close(); - dir.close(); - dir.close(); // close again - } - public void testDoubleCloseOutput() throws Throwable { Directory dir = getDirectory(createTempDir()); IndexOutput out = dir.createOutput("foobar", newIOContext(random())); @@ -1098,13 +1089,13 @@ public void testCreateTempOutput() throws Throwable { dir.close(); } - public void testCreateOutput() throws IOException { + public void testCreateOutputForExistingFile() throws IOException { try (Directory dir = getDirectory(createTempDir())) { String name = "file"; try (IndexOutput out = dir.createOutput(name, IOContext.DEFAULT)) { } - // Try to create an existing file. + // Try to create an existing file should fail. expectThrows(FileAlreadyExistsException.class, () -> { try (IndexOutput out = dir.createOutput(name, IOContext.DEFAULT)) { } @@ -1117,6 +1108,19 @@ public void testCreateOutput() throws IOException { } } + public void testReadFileOpenForWrites() throws IOException { + try (Directory dir = getDirectory(createTempDir())) { + String name = "file"; + try (IndexOutput out = dir.createOutput(name, IOContext.DEFAULT)) { + // Shouldn't be able to open this file for reading. + expectThrows(IOException.class, () -> { + try (IndexInput in = dir.openInput(name, IOContext.DEFAULT)) { + } + }); + } + } + } + public void testSeekToEndOfFile() throws IOException { try (Directory dir = getDirectory(createTempDir())) { try (IndexOutput out = dir.createOutput("a", IOContext.DEFAULT)) { From c2eb9761414af32d348e1d696c77ce97a8aea3e9 Mon Sep 17 00:00:00 2001 From: Dawid Weiss Date: Fri, 20 Jul 2018 11:58:05 +0200 Subject: [PATCH 04/10] IndexOutput javadoc correction. --- .../java/org/apache/lucene/store/IndexOutput.java | 13 ++++++------- 1 file changed, 6 insertions(+), 7 deletions(-) diff --git a/lucene/core/src/java/org/apache/lucene/store/IndexOutput.java b/lucene/core/src/java/org/apache/lucene/store/IndexOutput.java index e4881e0307fd..14b843f58785 100644 --- a/lucene/core/src/java/org/apache/lucene/store/IndexOutput.java +++ b/lucene/core/src/java/org/apache/lucene/store/IndexOutput.java @@ -20,17 +20,16 @@ import java.io.Closeable; import java.io.IOException; -/** Abstract base class for output to a file in a Directory. A random-access - * output stream. Used for all Lucene index output operations. - - *

{@code IndexOutput} may only be used from one thread, because it is not - * thread safe (it keeps internal state like file position). - +/** + * A {@link DataOutput} for appending data to a file in a {@link Directory}. + * + * Instances of this class are not thread-safe. + * * @see Directory * @see IndexInput */ public abstract class IndexOutput extends DataOutput implements Closeable { - + /** Full description of this output, e.g. which class such as {@code FSIndexOutput}, and the full path to the file */ private final String resourceDescription; From 8dd1e5dfc7f1737c7529d4337fb37ce457d0effa Mon Sep 17 00:00:00 2001 From: Dawid Weiss Date: Mon, 23 Jul 2018 11:03:44 +0200 Subject: [PATCH 05/10] Removed some duplicated code, corrected some typos. --- .../org/apache/lucene/store/Directory.java | 10 +- .../apache/lucene/index/TestIndexWriter.java | 6 +- .../lucene/store/BaseDirectoryTestCase.java | 1630 ++++++++--------- .../apache/lucene/util/LuceneTestCase.java | 33 + .../store/TestMockDirectoryWrapper.java | 9 +- 5 files changed, 827 insertions(+), 861 deletions(-) diff --git a/lucene/core/src/java/org/apache/lucene/store/Directory.java b/lucene/core/src/java/org/apache/lucene/store/Directory.java index 1e0531960f22..505be7fe0f10 100644 --- a/lucene/core/src/java/org/apache/lucene/store/Directory.java +++ b/lucene/core/src/java/org/apache/lucene/store/Directory.java @@ -34,11 +34,11 @@ * Implementing classes must comply with the following: * *

    - *
  • A file in a directory can be created ({@link #createOutput}) and appended - * to, then closed.
    • - *
  • A file open for writing must not be available + *
  • A file in a directory can be created ({@link #createOutput}), appended + * to, then closed.
    • + *
  • A file open for writing may not be available * for read access until the corresponding {@link IndexOutput} is closed.
    • - *
  • Once a file is created it may only be opened for input ({@link #openInput}), or + *
  • Once a file is created it must only be opened for input ({@link #openInput}), or * deleted ({@link #deleteFile}). Calling {@link #createOutput} on an existing file * must throw {@link java.nio.file.FileAlreadyExistsException}.
    • *
@@ -101,7 +101,7 @@ public abstract class Directory implements Closeable { /** * Ensures that any writes to these files are moved to - * stable storage. + * stable storage (made durable). * * Lucene uses this to properly commit changes to the index, to prevent a machine/OS crash * from corrupting the index. diff --git a/lucene/core/src/test/org/apache/lucene/index/TestIndexWriter.java b/lucene/core/src/test/org/apache/lucene/index/TestIndexWriter.java index 8e6de4703e99..0df98bc65203 100644 --- a/lucene/core/src/test/org/apache/lucene/index/TestIndexWriter.java +++ b/lucene/core/src/test/org/apache/lucene/index/TestIndexWriter.java @@ -2730,7 +2730,7 @@ public void testWithPendingDeletions() throws Exception { IndexWriter w = new IndexWriter(dir, iwc); w.commit(); reader = w.getReader(); - // we pull this commit to open it again later to check that we fail if a futur file delete is pending + // we pull this commit to open it again later to check that we fail if a future file delete is pending indexCommit = reader.getIndexCommit(); w.close(); w = new IndexWriter(dir, new IndexWriterConfig(new MockAnalyzer(random())).setIndexDeletionPolicy(NoDeletionPolicy.INSTANCE)); @@ -2740,7 +2740,7 @@ public void testWithPendingDeletions() throws Exception { dir.deleteFile("segments_2"); assertTrue(dir.getPendingDeletions().size() > 0); - // make sure we get NFSF if we try to delete and already-pending-delete file: + // make sure we get NoSuchFileException if we try to delete and already-pending-delete file: expectThrows(NoSuchFileException.class, () -> { dir.deleteFile("segments_2"); }); @@ -2777,7 +2777,7 @@ public void testPendingDeletesAlreadyWrittenFiles() throws IOException { assertTrue(dir.getPendingDeletions().size() > 0); - // make sure we get NFSF if we try to delete and already-pending-delete file: + // make sure we get NoSuchFileException if we try to delete and already-pending-delete file: expectThrows(NoSuchFileException.class, () -> { dir.deleteFile("segments_1"); }); diff --git a/lucene/test-framework/src/java/org/apache/lucene/store/BaseDirectoryTestCase.java b/lucene/test-framework/src/java/org/apache/lucene/store/BaseDirectoryTestCase.java index 1d838c69d5e4..4f196ae2179f 100644 --- a/lucene/test-framework/src/java/org/apache/lucene/store/BaseDirectoryTestCase.java +++ b/lucene/test-framework/src/java/org/apache/lucene/store/BaseDirectoryTestCase.java @@ -19,6 +19,8 @@ import java.io.EOFException; import java.io.FileNotFoundException; import java.io.IOException; +import java.io.UncheckedIOException; +import java.nio.file.AccessDeniedException; import java.nio.file.FileAlreadyExistsException; import java.nio.file.Files; import java.nio.file.NoSuchFileException; @@ -30,201 +32,196 @@ import java.util.HashSet; import java.util.List; import java.util.Map; +import java.util.Random; import java.util.Set; +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.CyclicBarrier; +import java.util.concurrent.atomic.AtomicBoolean; +import java.util.stream.IntStream; import java.util.zip.CRC32; +import com.carrotsearch.randomizedtesting.RandomizedTest; +import com.carrotsearch.randomizedtesting.generators.RandomBytes; +import com.carrotsearch.randomizedtesting.generators.RandomNumbers; +import com.carrotsearch.randomizedtesting.generators.RandomPicks; import org.apache.lucene.index.DirectoryReader; import org.apache.lucene.index.IndexFileNames; import org.apache.lucene.index.IndexNotFoundException; import org.apache.lucene.util.IOUtils; import org.apache.lucene.util.LuceneTestCase; import org.apache.lucene.util.TestUtil; +import org.junit.Assert; +import org.junit.internal.AssumptionViolatedException; -/** Base class for per-Directory tests. */ - +/** + * Base class for per-Directory tests. + */ public abstract class BaseDirectoryTestCase extends LuceneTestCase { - /** Subclass returns the Directory to be tested; if it's + /** A subclass returns the Directory to be tested; if it's * an FS-based directory it should point to the specified * path, else it can ignore it. */ protected abstract Directory getDirectory(Path path) throws IOException; - - // first some basic tests for the directory api - + public void testCopyFrom() throws Exception { - Directory source = getDirectory(createTempDir("testCopy")); - Directory dest = newDirectory(); - - IndexOutput output = source.createOutput("foobar", newIOContext(random())); - int numBytes = random().nextInt(20000); - byte bytes[] = new byte[numBytes]; - random().nextBytes(bytes); - output.writeBytes(bytes, bytes.length); - output.close(); - - dest.copyFrom(source, "foobar", "foobaz", newIOContext(random())); - assertTrue(slowFileExists(dest, "foobaz")); - - IndexInput input = dest.openInput("foobaz", newIOContext(random())); - byte bytes2[] = new byte[numBytes]; - input.readBytes(bytes2, 0, bytes2.length); - assertEquals(input.length(), numBytes); - input.close(); - - assertArrayEquals(bytes, bytes2); - - IOUtils.close(source, dest); + try (Directory source = getDirectory(createTempDir("testCopy")); + Directory dest = newDirectory()) { + runCopyFrom(source, dest); + } + + try (Directory source = newDirectory(); + Directory dest = getDirectory(createTempDir("testCopyDestination"))) { + runCopyFrom(source, dest); + } } - - public void testCopyFromDestination() throws Exception { - Directory source = newDirectory(); - Directory dest = getDirectory(createTempDir("testCopyDestination")); - + + private void runCopyFrom(Directory source, Directory dest) throws IOException { IndexOutput output = source.createOutput("foobar", newIOContext(random())); - int numBytes = random().nextInt(20000); - byte bytes[] = new byte[numBytes]; - random().nextBytes(bytes); + + byte bytes[] = RandomBytes.randomBytesOfLength(random(), 20000); output.writeBytes(bytes, bytes.length); output.close(); - + dest.copyFrom(source, "foobar", "foobaz", newIOContext(random())); assertTrue(slowFileExists(dest, "foobaz")); - + IndexInput input = dest.openInput("foobaz", newIOContext(random())); - byte bytes2[] = new byte[numBytes]; + byte bytes2[] = new byte[bytes.length]; input.readBytes(bytes2, 0, bytes2.length); - assertEquals(input.length(), numBytes); input.close(); - + assertArrayEquals(bytes, bytes2); - - IOUtils.close(source, dest); } - + public void testRename() throws Exception { - Directory dir = getDirectory(createTempDir("testRename")); - - IndexOutput output = dir.createOutput("foobar", newIOContext(random())); - int numBytes = random().nextInt(20000); - byte bytes[] = new byte[numBytes]; - random().nextBytes(bytes); - output.writeBytes(bytes, bytes.length); - output.close(); - - dir.rename("foobar", "foobaz"); - - IndexInput input = dir.openInput("foobaz", newIOContext(random())); - byte bytes2[] = new byte[numBytes]; - input.readBytes(bytes2, 0, bytes2.length); - assertEquals(input.length(), numBytes); - input.close(); - - assertArrayEquals(bytes, bytes2); - - dir.close(); + try (Directory dir = getDirectory(createTempDir("testRename"))) { + IndexOutput output = dir.createOutput("foobar", newIOContext(random())); + int numBytes = random().nextInt(20000); + byte bytes[] = new byte[numBytes]; + random().nextBytes(bytes); + output.writeBytes(bytes, bytes.length); + output.close(); + + dir.rename("foobar", "foobaz"); + + IndexInput input = dir.openInput("foobaz", newIOContext(random())); + byte bytes2[] = new byte[numBytes]; + input.readBytes(bytes2, 0, bytes2.length); + assertEquals(input.length(), numBytes); + input.close(); + + assertArrayEquals(bytes, bytes2); + } } - + public void testDeleteFile() throws Exception { - Directory dir = getDirectory(createTempDir("testDeleteFile")); - int count = dir.listAll().length; - dir.createOutput("foo.txt", IOContext.DEFAULT).close(); - assertEquals(count+1, dir.listAll().length); - dir.deleteFile("foo.txt"); - assertEquals(count, dir.listAll().length); - dir.close(); + try (Directory dir = getDirectory(createTempDir("testDeleteFile"))) { + String file = "foo.txt"; + Assert.assertFalse(Arrays.asList(dir.listAll()).contains(file)); + + dir.createOutput("foo.txt", IOContext.DEFAULT).close(); + Assert.assertTrue(Arrays.asList(dir.listAll()).contains(file)); + + dir.deleteFile("foo.txt"); + Assert.assertFalse(Arrays.asList(dir.listAll()).contains(file)); + + expectThrowsAnyOf(Arrays.asList(NoSuchFileException.class, FileNotFoundException.class), () -> { + dir.deleteFile("foo.txt"); + }); + } } - + public void testByte() throws Exception { - Directory dir = getDirectory(createTempDir("testByte")); - IndexOutput output = dir.createOutput("byte", newIOContext(random())); - output.writeByte((byte)128); - output.close(); - - IndexInput input = dir.openInput("byte", newIOContext(random())); - assertEquals(1, input.length()); - assertEquals((byte)128, input.readByte()); - input.close(); - dir.close(); + try (Directory dir = getDirectory(createTempDir("testByte"))) { + IndexOutput output = dir.createOutput("byte", newIOContext(random())); + output.writeByte((byte) 128); + output.close(); + + IndexInput input = dir.openInput("byte", newIOContext(random())); + assertEquals(1, input.length()); + assertEquals((byte) 128, input.readByte()); + input.close(); + } } - + public void testShort() throws Exception { - Directory dir = getDirectory(createTempDir("testShort")); - IndexOutput output = dir.createOutput("short", newIOContext(random())); - output.writeShort((short)-20); - output.close(); - - IndexInput input = dir.openInput("short", newIOContext(random())); - assertEquals(2, input.length()); - assertEquals((short)-20, input.readShort()); - input.close(); - dir.close(); + try (Directory dir = getDirectory(createTempDir("testShort"))) { + IndexOutput output = dir.createOutput("short", newIOContext(random())); + output.writeShort((short) -20); + output.close(); + + IndexInput input = dir.openInput("short", newIOContext(random())); + assertEquals(2, input.length()); + assertEquals((short) -20, input.readShort()); + input.close(); + } } - + public void testInt() throws Exception { - Directory dir = getDirectory(createTempDir("testInt")); - IndexOutput output = dir.createOutput("int", newIOContext(random())); - output.writeInt(-500); - output.close(); - - IndexInput input = dir.openInput("int", newIOContext(random())); - assertEquals(4, input.length()); - assertEquals(-500, input.readInt()); - input.close(); - dir.close(); + try (Directory dir = getDirectory(createTempDir("testInt"))) { + IndexOutput output = dir.createOutput("int", newIOContext(random())); + output.writeInt(-500); + output.close(); + + IndexInput input = dir.openInput("int", newIOContext(random())); + assertEquals(4, input.length()); + assertEquals(-500, input.readInt()); + input.close(); + } } - + public void testLong() throws Exception { - Directory dir = getDirectory(createTempDir("testLong")); - IndexOutput output = dir.createOutput("long", newIOContext(random())); - output.writeLong(-5000); - output.close(); - - IndexInput input = dir.openInput("long", newIOContext(random())); - assertEquals(8, input.length()); - assertEquals(-5000L, input.readLong()); - input.close(); - dir.close(); + try (Directory dir = getDirectory(createTempDir("testLong"))) { + IndexOutput output = dir.createOutput("long", newIOContext(random())); + output.writeLong(-5000); + output.close(); + + IndexInput input = dir.openInput("long", newIOContext(random())); + assertEquals(8, input.length()); + assertEquals(-5000L, input.readLong()); + input.close(); + } } - + public void testString() throws Exception { - Directory dir = getDirectory(createTempDir("testString")); - IndexOutput output = dir.createOutput("string", newIOContext(random())); - output.writeString("hello!"); - output.close(); - - IndexInput input = dir.openInput("string", newIOContext(random())); - assertEquals("hello!", input.readString()); - assertEquals(7, input.length()); - input.close(); - dir.close(); + try (Directory dir = getDirectory(createTempDir("testString"))) { + IndexOutput output = dir.createOutput("string", newIOContext(random())); + output.writeString("hello!"); + output.close(); + + IndexInput input = dir.openInput("string", newIOContext(random())); + assertEquals("hello!", input.readString()); + assertEquals(7, input.length()); + input.close(); + } } - + public void testVInt() throws Exception { - Directory dir = getDirectory(createTempDir("testVInt")); - IndexOutput output = dir.createOutput("vint", newIOContext(random())); - output.writeVInt(500); - output.close(); - - IndexInput input = dir.openInput("vint", newIOContext(random())); - assertEquals(2, input.length()); - assertEquals(500, input.readVInt()); - input.close(); - dir.close(); + try (Directory dir = getDirectory(createTempDir("testVInt"))) { + IndexOutput output = dir.createOutput("vint", newIOContext(random())); + output.writeVInt(500); + output.close(); + + IndexInput input = dir.openInput("vint", newIOContext(random())); + assertEquals(2, input.length()); + assertEquals(500, input.readVInt()); + input.close(); + } } - + public void testVLong() throws Exception { - Directory dir = getDirectory(createTempDir("testVLong")); - IndexOutput output = dir.createOutput("vlong", newIOContext(random())); - output.writeVLong(Long.MAX_VALUE); - output.close(); - - IndexInput input = dir.openInput("vlong", newIOContext(random())); - assertEquals(9, input.length()); - assertEquals(Long.MAX_VALUE, input.readVLong()); - input.close(); - dir.close(); + try (Directory dir = getDirectory(createTempDir("testVLong"))) { + IndexOutput output = dir.createOutput("vlong", newIOContext(random())); + output.writeVLong(Long.MAX_VALUE); + output.close(); + + IndexInput input = dir.openInput("vlong", newIOContext(random())); + assertEquals(9, input.length()); + assertEquals(Long.MAX_VALUE, input.readVLong()); + input.close(); + } } - + public void testZInt() throws Exception { final int[] ints = new int[random().nextInt(10)]; for (int i = 0; i < ints.length; ++i) { @@ -242,22 +239,23 @@ public void testZInt() throws Exception { throw new AssertionError(); } } - Directory dir = getDirectory(createTempDir("testZInt")); - IndexOutput output = dir.createOutput("zint", newIOContext(random())); - for (int i : ints) { - output.writeZInt(i); - } - output.close(); - - IndexInput input = dir.openInput("zint", newIOContext(random())); - for (int i : ints) { - assertEquals(i, input.readZInt()); + + try (Directory dir = getDirectory(createTempDir("testZInt"))) { + IndexOutput output = dir.createOutput("zint", newIOContext(random())); + for (int i : ints) { + output.writeZInt(i); + } + output.close(); + + IndexInput input = dir.openInput("zint", newIOContext(random())); + for (int i : ints) { + assertEquals(i, input.readZInt()); + } + assertEquals(input.length(), input.getFilePointer()); + input.close(); } - assertEquals(input.length(), input.getFilePointer()); - input.close(); - dir.close(); } - + public void testZLong() throws Exception { final long[] longs = new long[random().nextInt(10)]; for (int i = 0; i < longs.length; ++i) { @@ -275,97 +273,98 @@ public void testZLong() throws Exception { throw new AssertionError(); } } - Directory dir = getDirectory(createTempDir("testZLong")); - IndexOutput output = dir.createOutput("zlong", newIOContext(random())); - for (long l : longs) { - output.writeZLong(l); - } - output.close(); - - IndexInput input = dir.openInput("zlong", newIOContext(random())); - for (long l : longs) { - assertEquals(l, input.readZLong()); + + try (Directory dir = getDirectory(createTempDir("testZLong"))) { + IndexOutput output = dir.createOutput("zlong", newIOContext(random())); + for (long l : longs) { + output.writeZLong(l); + } + output.close(); + + IndexInput input = dir.openInput("zlong", newIOContext(random())); + for (long l : longs) { + assertEquals(l, input.readZLong()); + } + assertEquals(input.length(), input.getFilePointer()); + input.close(); } - assertEquals(input.length(), input.getFilePointer()); - input.close(); - dir.close(); } public void testSetOfStrings() throws Exception { - Directory dir = getDirectory(createTempDir("testSetOfStrings")); - - IndexOutput output = dir.createOutput("stringset", newIOContext(random())); - output.writeSetOfStrings(asSet("test1", "test2")); - output.writeSetOfStrings(Collections.emptySet()); - output.writeSetOfStrings(asSet("test3")); - output.close(); - - IndexInput input = dir.openInput("stringset", newIOContext(random())); - Set set1 = input.readSetOfStrings(); - assertEquals(asSet("test1", "test2"), set1); - // set should be immutable - expectThrows(UnsupportedOperationException.class, () -> { - set1.add("bogus"); - }); - - Set set2 = input.readSetOfStrings(); - assertEquals(Collections.emptySet(), set2); - // set should be immutable - expectThrows(UnsupportedOperationException.class, () -> { - set2.add("bogus"); - }); - - Set set3 = input.readSetOfStrings(); - assertEquals(Collections.singleton("test3"), set3); - // set should be immutable - expectThrows(UnsupportedOperationException.class, () -> { - set3.add("bogus"); - }); - - assertEquals(input.length(), input.getFilePointer()); - input.close(); - dir.close(); + try (Directory dir = getDirectory(createTempDir("testSetOfStrings"))) { + + IndexOutput output = dir.createOutput("stringset", newIOContext(random())); + output.writeSetOfStrings(asSet("test1", "test2")); + output.writeSetOfStrings(Collections.emptySet()); + output.writeSetOfStrings(asSet("test3")); + output.close(); + + IndexInput input = dir.openInput("stringset", newIOContext(random())); + Set set1 = input.readSetOfStrings(); + assertEquals(asSet("test1", "test2"), set1); + // set should be immutable + expectThrows(UnsupportedOperationException.class, () -> { + set1.add("bogus"); + }); + + Set set2 = input.readSetOfStrings(); + assertEquals(Collections.emptySet(), set2); + // set should be immutable + expectThrows(UnsupportedOperationException.class, () -> { + set2.add("bogus"); + }); + + Set set3 = input.readSetOfStrings(); + assertEquals(Collections.singleton("test3"), set3); + // set should be immutable + expectThrows(UnsupportedOperationException.class, () -> { + set3.add("bogus"); + }); + + assertEquals(input.length(), input.getFilePointer()); + input.close(); + } } - + public void testMapOfStrings() throws Exception { Map m = new HashMap<>(); m.put("test1", "value1"); m.put("test2", "value2"); - - Directory dir = getDirectory(createTempDir("testMapOfStrings")); - IndexOutput output = dir.createOutput("stringmap", newIOContext(random())); - output.writeMapOfStrings(m); - output.writeMapOfStrings(Collections.emptyMap()); - output.writeMapOfStrings(Collections.singletonMap("key", "value")); - output.close(); - - IndexInput input = dir.openInput("stringmap", newIOContext(random())); - Map map1 = input.readMapOfStrings(); - assertEquals(m, map1); - // map should be immutable - expectThrows(UnsupportedOperationException.class, () -> { - map1.put("bogus1", "bogus2"); - }); - - Map map2 = input.readMapOfStrings(); - assertEquals(Collections.emptyMap(), map2); - // map should be immutable - expectThrows(UnsupportedOperationException.class, () -> { - map2.put("bogus1", "bogus2"); - }); - - Map map3 = input.readMapOfStrings(); - assertEquals(Collections.singletonMap("key", "value"), map3); - // map should be immutable - expectThrows(UnsupportedOperationException.class, () -> { - map3.put("bogus1", "bogus2"); - }); - - assertEquals(input.length(), input.getFilePointer()); - input.close(); - dir.close(); + + try (Directory dir = getDirectory(createTempDir("testMapOfStrings"))) { + IndexOutput output = dir.createOutput("stringmap", newIOContext(random())); + output.writeMapOfStrings(m); + output.writeMapOfStrings(Collections.emptyMap()); + output.writeMapOfStrings(Collections.singletonMap("key", "value")); + output.close(); + + IndexInput input = dir.openInput("stringmap", newIOContext(random())); + Map map1 = input.readMapOfStrings(); + assertEquals(m, map1); + // map should be immutable + expectThrows(UnsupportedOperationException.class, () -> { + map1.put("bogus1", "bogus2"); + }); + + Map map2 = input.readMapOfStrings(); + assertEquals(Collections.emptyMap(), map2); + // map should be immutable + expectThrows(UnsupportedOperationException.class, () -> { + map2.put("bogus1", "bogus2"); + }); + + Map map3 = input.readMapOfStrings(); + assertEquals(Collections.singletonMap("key", "value"), map3); + // map should be immutable + expectThrows(UnsupportedOperationException.class, () -> { + map3.put("bogus1", "bogus2"); + }); + + assertEquals(input.length(), input.getFilePointer()); + input.close(); + } } - + // TODO: fold in some of the testing of o.a.l.index.TestIndexInput in here! public void testChecksum() throws Exception { CRC32 expected = new CRC32(); @@ -373,646 +372,593 @@ public void testChecksum() throws Exception { byte bytes[] = new byte[numBytes]; random().nextBytes(bytes); expected.update(bytes); - - Directory dir = getDirectory(createTempDir("testChecksum")); - IndexOutput output = dir.createOutput("checksum", newIOContext(random())); - output.writeBytes(bytes, 0, bytes.length); - output.close(); - - ChecksumIndexInput input = dir.openChecksumInput("checksum", newIOContext(random())); - input.skipBytes(numBytes); - - assertEquals(expected.getValue(), input.getChecksum()); - input.close(); - dir.close(); + + try (Directory dir = getDirectory(createTempDir("testChecksum"))) { + IndexOutput output = dir.createOutput("checksum", newIOContext(random())); + output.writeBytes(bytes, 0, bytes.length); + output.close(); + + ChecksumIndexInput input = dir.openChecksumInput("checksum", newIOContext(random())); + input.skipBytes(numBytes); + + assertEquals(expected.getValue(), input.getChecksum()); + input.close(); + } } - + /** Make sure directory throws AlreadyClosedException if * you try to createOutput after closing. */ public void testDetectClose() throws Throwable { Directory dir = getDirectory(createTempDir("testDetectClose")); dir.close(); - expectThrows(AlreadyClosedException.class, () -> { + + expectThrows(AlreadyClosedException.class, () -> { dir.createOutput("test", newIOContext(random())); }); } - - public void testThreadSafety() throws Exception { - final Directory dir = getDirectory(createTempDir("testThreadSafety")); - if (dir instanceof BaseDirectoryWrapper) { - ((BaseDirectoryWrapper)dir).setCheckIndexOnClose(false); // we arent making an index - } - if (dir instanceof MockDirectoryWrapper) { - ((MockDirectoryWrapper)dir).setThrottling(MockDirectoryWrapper.Throttling.NEVER); // makes this test really slow - } - - if (VERBOSE) { - System.out.println(dir); - } - - class TheThread extends Thread { - private String name; - public TheThread(String name) { - this.name = name; + public void testThreadSafetyInListAll() throws Exception { + try (Directory dir = getDirectory(createTempDir("testThreadSafety"))) { + if (dir instanceof BaseDirectoryWrapper) { + // we are not making a real index, just writing, reading files. + ((BaseDirectoryWrapper) dir).setCheckIndexOnClose(false); } - - @Override - public void run() { - for (int i = 0; i < 1000; i++) { - String fileName = this.name + i; - try { - //System.out.println("create:" + fileName); - IndexOutput output = dir.createOutput(fileName, newIOContext(random())); - output.close(); + if (dir instanceof MockDirectoryWrapper) { + // makes this test really slow + ((MockDirectoryWrapper) dir).setThrottling(MockDirectoryWrapper.Throttling.NEVER); + } + + AtomicBoolean stop = new AtomicBoolean(); + Thread writer = new Thread(() -> { + try { + Random rnd = new Random(RandomizedTest.randomLong() + 1); + for (int i = 0, max = RandomizedTest.randomIntBetween(500, 1000); i < max; i++) { + String fileName = "T1-" + i; + try (IndexOutput output = dir.createOutput(fileName, newIOContext(random()))) { + // Add some lags so that the other thread can read the content of the directory. + Thread.yield(); + } assertTrue(slowFileExists(dir, fileName)); - } catch (IOException e) { - throw new RuntimeException(e); } + } catch (IOException e) { + throw new UncheckedIOException(e); + } finally { + stop.set(true); } - } - }; - - class TheThread2 extends Thread { - private String name; - private volatile boolean stop; - - public TheThread2(String name) { - this.name = name; - } - - @Override - public void run() { - while (stop == false) { - try { + }); + + Thread reader = new Thread(() -> { + try { + Random rnd = new Random(RandomizedTest.randomLong()); + while (!stop.get()) { String[] files = dir.listAll(); - for (String file : files) { - if (!file.startsWith(name)) { - continue; - } - //System.out.println("file:" + file); - try { - IndexInput input = dir.openInput(file, newIOContext(random())); - input.close(); - } catch (FileNotFoundException | NoSuchFileException e) { - // ignore + if (files.length == 0) { + continue; + } + + do { + String file = RandomPicks.randomFrom(rnd, files); + try (IndexInput input = dir.openInput(file, newIOContext(random()))) { + // Just open, nothing else. + } catch (AccessDeniedException e) { + // Access denied is allowed for files for which the output is still open. + // Since we don't synchronize with the writer thread, just ignore it. } catch (IOException e) { - throw new RuntimeException(e); - } - if (random().nextBoolean()) { - break; + throw new UncheckedIOException("Something went wrong when opening: " + file, e); } - } - } catch (IOException e) { - throw new RuntimeException(e); + } while (rnd.nextInt(3) != 0); // Sometimes break and list files again. } + } catch (IOException e) { + throw new UncheckedIOException(e); } - } - }; - - TheThread theThread = new TheThread("t1"); - TheThread2 theThread2 = new TheThread2("t2"); - theThread.start(); - theThread2.start(); - - theThread.join(); - - // after first thread is done, no sense in waiting on thread 2 - // to listFiles() and loop over and over - theThread2.stop = true; - theThread2.join(); - - dir.close(); + }); + + reader.start(); + writer.start(); + + writer.join(); + reader.join(); + } } - /** LUCENE-1468: once we create an output, we should see - * it in the dir listing and be able to open it with - * openInput. */ - public void testDirectoryFilter() throws IOException { - String name = "file"; - Directory dir = getDirectory(createTempDir("testDirectoryFilter")); - try { + /** + * LUCENE-1468: once we create an output, we should see + * it in the dir listing. + */ + public void testFileExistsInListAfterCreated() throws IOException { + try (Directory dir = getDirectory(createTempDir("testFileExistsInListAfterCreated"))) { + String name = "file"; dir.createOutput(name, newIOContext(random())).close(); assertTrue(slowFileExists(dir, name)); assertTrue(Arrays.asList(dir.listAll()).contains(name)); - } finally { - dir.close(); } } // LUCENE-2852 public void testSeekToEOFThenBack() throws Exception { - Directory dir = getDirectory(createTempDir("testSeekToEOFThenBack")); - - IndexOutput o = dir.createOutput("out", newIOContext(random())); - byte[] bytes = new byte[3*RAMOutputStream.BUFFER_SIZE]; - o.writeBytes(bytes, 0, bytes.length); - o.close(); - - IndexInput i = dir.openInput("out", newIOContext(random())); - i.seek(2*RAMOutputStream.BUFFER_SIZE-1); - i.seek(3*RAMOutputStream.BUFFER_SIZE); - i.seek(RAMOutputStream.BUFFER_SIZE); - i.readBytes(bytes, 0, 2*RAMOutputStream.BUFFER_SIZE); - i.close(); - dir.close(); + try (Directory dir = getDirectory(createTempDir("testSeekToEOFThenBack"))) { + int bufferLength = 1024; + IndexOutput o = dir.createOutput("out", newIOContext(random())); + byte[] bytes = new byte[3 * bufferLength]; + o.writeBytes(bytes, 0, bytes.length); + o.close(); + + IndexInput i = dir.openInput("out", newIOContext(random())); + i.seek(2 * bufferLength - 1); + i.seek(3 * bufferLength); + i.seek(bufferLength); + i.readBytes(bytes, 0, 2 * bufferLength); + i.close(); + } } // LUCENE-1196 public void testIllegalEOF() throws Exception { - Directory dir = getDirectory(createTempDir("testIllegalEOF")); - IndexOutput o = dir.createOutput("out", newIOContext(random())); - byte[] b = new byte[1024]; - o.writeBytes(b, 0, 1024); - o.close(); - IndexInput i = dir.openInput("out", newIOContext(random())); - i.seek(1024); - i.close(); - dir.close(); + try (Directory dir = getDirectory(createTempDir("testIllegalEOF"))) { + IndexOutput o = dir.createOutput("out", newIOContext(random())); + byte[] b = new byte[1024]; + o.writeBytes(b, 0, 1024); + o.close(); + IndexInput i = dir.openInput("out", newIOContext(random())); + i.seek(1024); + i.close(); + } } public void testSeekPastEOF() throws Exception { - Directory dir = getDirectory(createTempDir("testSeekPastEOF")); - IndexOutput o = dir.createOutput("out", newIOContext(random())); - final int len = random().nextInt(2048); - byte[] b = new byte[len]; - o.writeBytes(b, 0, len); - o.close(); - IndexInput i = dir.openInput("out", newIOContext(random())); - expectThrows(EOFException.class, () -> { - i.seek(len + random().nextInt(2048)); - i.readByte(); - }); + try (Directory dir = getDirectory(createTempDir("testSeekPastEOF"))) { + IndexOutput o = dir.createOutput("out", newIOContext(random())); + final int len = random().nextInt(2048); + byte[] b = new byte[len]; + o.writeBytes(b, 0, len); + o.close(); + IndexInput i = dir.openInput("out", newIOContext(random())); + expectThrows(EOFException.class, () -> { + i.seek(len + random().nextInt(2048)); + i.readByte(); + }); - i.close(); - dir.close(); + i.close(); + } } public void testSliceOutOfBounds() throws Exception { - Directory dir = getDirectory(createTempDir("testSliceOutOfBounds")); - IndexOutput o = dir.createOutput("out", newIOContext(random())); - final int len = random().nextInt(2040) + 8; - byte[] b = new byte[len]; - o.writeBytes(b, 0, len); - o.close(); - IndexInput i = dir.openInput("out", newIOContext(random())); - expectThrows(IllegalArgumentException.class, () -> { - i.slice("slice1", 0, len + 1); - }); + try (Directory dir = getDirectory(createTempDir("testSliceOutOfBounds"))) { + IndexOutput o = dir.createOutput("out", newIOContext(random())); + final int len = random().nextInt(2040) + 8; + byte[] b = new byte[len]; + o.writeBytes(b, 0, len); + o.close(); + IndexInput i = dir.openInput("out", newIOContext(random())); + expectThrows(IllegalArgumentException.class, () -> { + i.slice("slice1", 0, len + 1); + }); - expectThrows(IllegalArgumentException.class, () -> { - i.slice("slice2", -1, len); - }); + expectThrows(IllegalArgumentException.class, () -> { + i.slice("slice2", -1, len); + }); - IndexInput slice = i.slice("slice3", 4, len / 2); - expectThrows(IllegalArgumentException.class, () -> { - slice.slice("slice3sub", 1, len / 2); - }); + IndexInput slice = i.slice("slice3", 4, len / 2); + expectThrows(IllegalArgumentException.class, () -> { + slice.slice("slice3sub", 1, len / 2); + }); - i.close(); - dir.close(); + i.close(); + } } - + // LUCENE-3382 -- make sure we get exception if the directory really does not exist. public void testNoDir() throws Throwable { Path tempDir = createTempDir("doesnotexist"); IOUtils.rm(tempDir); - Directory dir = getDirectory(tempDir); - try { - DirectoryReader.open(dir); - fail("did not hit expected exception"); - } catch (NoSuchFileException | IndexNotFoundException nsde) { - // expected + try (Directory dir = getDirectory(tempDir)) { + expectThrowsAnyOf(Arrays.asList(NoSuchFileException.class, IndexNotFoundException.class), () -> { + DirectoryReader.open(dir); + }); } - dir.close(); } public void testCopyBytes() throws Exception { - testCopyBytes(getDirectory(createTempDir("testCopyBytes"))); - } - - private static byte value(int idx) { - return (byte) ((idx % 256) * (1 + (idx / 256))); - } - - public static void testCopyBytes(Directory dir) throws Exception { - - // make random file - IndexOutput out = dir.createOutput("test", newIOContext(random())); - byte[] bytes = new byte[TestUtil.nextInt(random(), 1, 77777)]; - final int size = TestUtil.nextInt(random(), 1, 1777777); - int upto = 0; - int byteUpto = 0; - while (upto < size) { - bytes[byteUpto++] = value(upto); - upto++; - if (byteUpto == bytes.length) { - out.writeBytes(bytes, 0, bytes.length); - byteUpto = 0; - } - } - - out.writeBytes(bytes, 0, byteUpto); - assertEquals(size, out.getFilePointer()); - out.close(); - assertEquals(size, dir.fileLength("test")); - - // copy from test -> test2 - final IndexInput in = dir.openInput("test", newIOContext(random())); - - out = dir.createOutput("test2", newIOContext(random())); - - upto = 0; - while (upto < size) { - if (random().nextBoolean()) { - out.writeByte(in.readByte()); + try (Directory dir = getDirectory(createTempDir("testCopyBytes"))) { + IndexOutput out = dir.createOutput("test", newIOContext(random())); + byte[] bytes = new byte[TestUtil.nextInt(random(), 1, 77777)]; + final int size = TestUtil.nextInt(random(), 1, 1777777); + int upto = 0; + int byteUpto = 0; + while (upto < size) { + bytes[byteUpto++] = value(upto); upto++; - } else { - final int chunk = Math.min( - TestUtil.nextInt(random(), 1, bytes.length), size - upto); - out.copyBytes(in, chunk); - upto += chunk; + if (byteUpto == bytes.length) { + out.writeBytes(bytes, 0, bytes.length); + byteUpto = 0; + } } - } - assertEquals(size, upto); - out.close(); - in.close(); - - // verify - IndexInput in2 = dir.openInput("test2", newIOContext(random())); - upto = 0; - while (upto < size) { - if (random().nextBoolean()) { - final byte v = in2.readByte(); - assertEquals(value(upto), v); - upto++; - } else { - final int limit = Math.min( - TestUtil.nextInt(random(), 1, bytes.length), size - upto); - in2.readBytes(bytes, 0, limit); - for (int byteIdx = 0; byteIdx < limit; byteIdx++) { - assertEquals(value(upto), bytes[byteIdx]); + + out.writeBytes(bytes, 0, byteUpto); + assertEquals(size, out.getFilePointer()); + out.close(); + assertEquals(size, dir.fileLength("test")); + + // copy from test -> test2 + final IndexInput in = dir.openInput("test", newIOContext(random())); + + out = dir.createOutput("test2", newIOContext(random())); + + upto = 0; + while (upto < size) { + if (random().nextBoolean()) { + out.writeByte(in.readByte()); upto++; + } else { + final int chunk = Math.min( + TestUtil.nextInt(random(), 1, bytes.length), size - upto); + out.copyBytes(in, chunk); + upto += chunk; + } + } + assertEquals(size, upto); + out.close(); + in.close(); + + // verify + IndexInput in2 = dir.openInput("test2", newIOContext(random())); + upto = 0; + while (upto < size) { + if (random().nextBoolean()) { + final byte v = in2.readByte(); + assertEquals(value(upto), v); + upto++; + } else { + final int limit = Math.min( + TestUtil.nextInt(random(), 1, bytes.length), size - upto); + in2.readBytes(bytes, 0, limit); + for (int byteIdx = 0; byteIdx < limit; byteIdx++) { + assertEquals(value(upto), bytes[byteIdx]); + upto++; + } } } + in2.close(); + + dir.deleteFile("test"); + dir.deleteFile("test2"); } - in2.close(); - - dir.deleteFile("test"); - dir.deleteFile("test2"); - dir.close(); } - + + private static byte value(int idx) { + return (byte) ((idx % 256) * (1 + (idx / 256))); + } + // LUCENE-3541 public void testCopyBytesWithThreads() throws Exception { - testCopyBytesWithThreads(getDirectory(createTempDir("testCopyBytesWithThreads"))); - } + try (Directory d = getDirectory(createTempDir("testCopyBytesWithThreads"))) { + byte data[] = RandomBytes.randomBytesOfLengthBetween(random(), 101, 10000); - public static void testCopyBytesWithThreads(Directory d) throws Exception { - int datalen = TestUtil.nextInt(random(), 101, 10000); - byte data[] = new byte[datalen]; - random().nextBytes(data); - - IndexOutput output = d.createOutput("data", IOContext.DEFAULT); - output.writeBytes(data, 0, datalen); - output.close(); - - IndexInput input = d.openInput("data", IOContext.DEFAULT); - IndexOutput outputHeader = d.createOutput("header", IOContext.DEFAULT); - // copy our 100-byte header - outputHeader.copyBytes(input, 100); - outputHeader.close(); - - // now make N copies of the remaining bytes - CopyThread copies[] = new CopyThread[10]; - for (int i = 0; i < copies.length; i++) { - copies[i] = new CopyThread(input.clone(), d.createOutput("copy" + i, IOContext.DEFAULT)); - } - - for (int i = 0; i < copies.length; i++) { - copies[i].start(); - } - - for (int i = 0; i < copies.length; i++) { - copies[i].join(); - } - - for (int i = 0; i < copies.length; i++) { - IndexInput copiedData = d.openInput("copy" + i, IOContext.DEFAULT); - byte[] dataCopy = new byte[datalen]; - System.arraycopy(data, 0, dataCopy, 0, 100); // copy the header for easy testing - copiedData.readBytes(dataCopy, 100, datalen-100); - assertArrayEquals(data, dataCopy); - copiedData.close(); - } - input.close(); - d.close(); - - } - - static class CopyThread extends Thread { - final IndexInput src; - final IndexOutput dst; - - CopyThread(IndexInput src, IndexOutput dst) { - this.src = src; - this.dst = dst; - } + IndexOutput output = d.createOutput("data", IOContext.DEFAULT); + output.writeBytes(data, 0, data.length); + output.close(); - @Override - public void run() { - try { - dst.copyBytes(src, src.length()-100); - dst.close(); - } catch (IOException ex) { - throw new RuntimeException(ex); + IndexInput input = d.openInput("data", IOContext.DEFAULT); + IndexOutput outputHeader = d.createOutput("header", IOContext.DEFAULT); + // copy our 100-byte header + outputHeader.copyBytes(input, 100); + outputHeader.close(); + + // now make N copies of the remaining bytes + int threads = 10; + CyclicBarrier start = new CyclicBarrier(threads); + Thread copies [] = IntStream.range(0, threads) + .mapToObj((i) -> { + IndexInput src = input.clone(); + Thread t = new Thread(() -> { + try { + start.await(); + IndexOutput dst = d.createOutput("copy" + i, IOContext.DEFAULT); + dst.copyBytes(src, src.length() - 100); + dst.close(); + } catch (Exception e) { + throw new RuntimeException(e); + } + }); + t.start(); + return t; + }) + .toArray(Thread[]::new); + + for (Thread t : copies) { + t.join(); + } + + for (int i = 0; i < threads; i++) { + try (IndexInput copiedData = d.openInput("copy" + i, IOContext.DEFAULT)) { + byte[] dataCopy = new byte[data.length]; + System.arraycopy(data, 0, dataCopy, 0, 100); + copiedData.readBytes(dataCopy, 100, data.length - 100); + assertArrayEquals(data, dataCopy); + } } + input.close(); } } - + // this test backdoors the directory via the filesystem. so it must actually use the filesystem - // TODO: somehow change this test to + // TODO: somehow change this test to public void testFsyncDoesntCreateNewFiles() throws Exception { Path path = createTempDir("nocreate"); - Directory fsdir = getDirectory(path); - - // this test backdoors the directory via the filesystem. so it must be an FSDir (for now) - // TODO: figure a way to test this better/clean it up. E.g. we should be testing for FileSwitchDir, - // if it's using two FSdirs and so on - if (fsdir instanceof FSDirectory == false) { - fsdir.close(); - assumeTrue("test only works for FSDirectory subclasses", false); - } - - // write a file - IndexOutput out = fsdir.createOutput("afile", newIOContext(random())); - out.writeString("boo"); - out.close(); - - // delete it - Files.delete(path.resolve("afile")); - - int fileCount = fsdir.listAll().length; - - // fsync it - try { - fsdir.sync(Collections.singleton("afile")); - fail("didn't get expected exception, instead fsync created new files: " + Arrays.asList(fsdir.listAll())); - } catch (FileNotFoundException | NoSuchFileException expected) { - // ok + try (Directory fsdir = getDirectory(path)) { + // this test backdoors the directory via the filesystem. so it must be an FSDir (for now) + // TODO: figure a way to test this better/clean it up. E.g. we should be testing for FileSwitchDir, + // if it's using two FSdirs and so on + if (fsdir instanceof FSDirectory == false) { + assumeTrue("test only works for FSDirectory subclasses", false); + return; + } + + // create a file + IndexOutput out = fsdir.createOutput("afile", newIOContext(random())); + out.writeString("boo"); + out.close(); + + // delete it in the file system. + Files.delete(path.resolve("afile")); + + int fileCount = fsdir.listAll().length; + + // fsync it + expectThrowsAnyOf(Arrays.asList(FileNotFoundException.class, NoSuchFileException.class), () -> { + fsdir.sync(Collections.singleton("afile")); + }); + + // no new files created + assertEquals(fileCount, fsdir.listAll().length); } - - // no new files created - assertEquals(fileCount, fsdir.listAll().length); - - fsdir.close(); } // random access APIs - + public void testRandomLong() throws Exception { - Directory dir = getDirectory(createTempDir("testLongs")); - IndexOutput output = dir.createOutput("longs", newIOContext(random())); - int num = TestUtil.nextInt(random(), 50, 3000); - long longs[] = new long[num]; - for (int i = 0; i < longs.length; i++) { - longs[i] = TestUtil.nextLong(random(), Long.MIN_VALUE, Long.MAX_VALUE); - output.writeLong(longs[i]); - } - output.close(); - - // slice - IndexInput input = dir.openInput("longs", newIOContext(random())); - RandomAccessInput slice = input.randomAccessSlice(0, input.length()); - for (int i = 0; i < longs.length; i++) { - assertEquals(longs[i], slice.readLong(i * 8)); - } - - // subslices - for (int i = 1; i < longs.length; i++) { - long offset = i * 8; - RandomAccessInput subslice = input.randomAccessSlice(offset, input.length() - offset); - for (int j = i; j < longs.length; j++) { - assertEquals(longs[j], subslice.readLong((j - i) * 8)); + try (Directory dir = getDirectory(createTempDir("testLongs"))) { + IndexOutput output = dir.createOutput("longs", newIOContext(random())); + int num = TestUtil.nextInt(random(), 50, 3000); + long longs[] = new long[num]; + for (int i = 0; i < longs.length; i++) { + longs[i] = TestUtil.nextLong(random(), Long.MIN_VALUE, Long.MAX_VALUE); + output.writeLong(longs[i]); } - } - - // with padding - for (int i = 0; i < 7; i++) { - String name = "longs-" + i; - IndexOutput o = dir.createOutput(name, newIOContext(random())); - byte junk[] = new byte[i]; - random().nextBytes(junk); - o.writeBytes(junk, junk.length); - input.seek(0); - o.copyBytes(input, input.length()); - o.close(); - IndexInput padded = dir.openInput(name, newIOContext(random())); - RandomAccessInput whole = padded.randomAccessSlice(i, padded.length() - i); - for (int j = 0; j < longs.length; j++) { - assertEquals(longs[j], whole.readLong(j * 8)); + output.close(); + + // slice + IndexInput input = dir.openInput("longs", newIOContext(random())); + RandomAccessInput slice = input.randomAccessSlice(0, input.length()); + for (int i = 0; i < longs.length; i++) { + assertEquals(longs[i], slice.readLong(i * 8)); + } + + // subslices + for (int i = 1; i < longs.length; i++) { + long offset = i * 8; + RandomAccessInput subslice = input.randomAccessSlice(offset, input.length() - offset); + for (int j = i; j < longs.length; j++) { + assertEquals(longs[j], subslice.readLong((j - i) * 8)); + } + } + + // with padding + for (int i = 0; i < 7; i++) { + String name = "longs-" + i; + IndexOutput o = dir.createOutput(name, newIOContext(random())); + byte junk[] = new byte[i]; + random().nextBytes(junk); + o.writeBytes(junk, junk.length); + input.seek(0); + o.copyBytes(input, input.length()); + o.close(); + IndexInput padded = dir.openInput(name, newIOContext(random())); + RandomAccessInput whole = padded.randomAccessSlice(i, padded.length() - i); + for (int j = 0; j < longs.length; j++) { + assertEquals(longs[j], whole.readLong(j * 8)); + } + padded.close(); } - padded.close(); + + input.close(); } - - input.close(); - dir.close(); } public void testRandomInt() throws Exception { - Directory dir = getDirectory(createTempDir("testInts")); - IndexOutput output = dir.createOutput("ints", newIOContext(random())); - int num = TestUtil.nextInt(random(), 50, 3000); - int ints[] = new int[num]; - for (int i = 0; i < ints.length; i++) { - ints[i] = random().nextInt(); - output.writeInt(ints[i]); - } - output.close(); - - // slice - IndexInput input = dir.openInput("ints", newIOContext(random())); - RandomAccessInput slice = input.randomAccessSlice(0, input.length()); - for (int i = 0; i < ints.length; i++) { - assertEquals(ints[i], slice.readInt(i * 4)); - } - - // subslices - for (int i = 1; i < ints.length; i++) { - long offset = i * 4; - RandomAccessInput subslice = input.randomAccessSlice(offset, input.length() - offset); - for (int j = i; j < ints.length; j++) { - assertEquals(ints[j], subslice.readInt((j - i) * 4)); + try (Directory dir = getDirectory(createTempDir("testInts"))) { + IndexOutput output = dir.createOutput("ints", newIOContext(random())); + int num = TestUtil.nextInt(random(), 50, 3000); + int ints[] = new int[num]; + for (int i = 0; i < ints.length; i++) { + ints[i] = random().nextInt(); + output.writeInt(ints[i]); } - } - - // with padding - for (int i = 0; i < 7; i++) { - String name = "ints-" + i; - IndexOutput o = dir.createOutput(name, newIOContext(random())); - byte junk[] = new byte[i]; - random().nextBytes(junk); - o.writeBytes(junk, junk.length); - input.seek(0); - o.copyBytes(input, input.length()); - o.close(); - IndexInput padded = dir.openInput(name, newIOContext(random())); - RandomAccessInput whole = padded.randomAccessSlice(i, padded.length() - i); - for (int j = 0; j < ints.length; j++) { - assertEquals(ints[j], whole.readInt(j * 4)); + output.close(); + + // slice + IndexInput input = dir.openInput("ints", newIOContext(random())); + RandomAccessInput slice = input.randomAccessSlice(0, input.length()); + for (int i = 0; i < ints.length; i++) { + assertEquals(ints[i], slice.readInt(i * 4)); + } + + // subslices + for (int i = 1; i < ints.length; i++) { + long offset = i * 4; + RandomAccessInput subslice = input.randomAccessSlice(offset, input.length() - offset); + for (int j = i; j < ints.length; j++) { + assertEquals(ints[j], subslice.readInt((j - i) * 4)); + } } - padded.close(); + + // with padding + for (int i = 0; i < 7; i++) { + String name = "ints-" + i; + IndexOutput o = dir.createOutput(name, newIOContext(random())); + byte junk[] = new byte[i]; + random().nextBytes(junk); + o.writeBytes(junk, junk.length); + input.seek(0); + o.copyBytes(input, input.length()); + o.close(); + IndexInput padded = dir.openInput(name, newIOContext(random())); + RandomAccessInput whole = padded.randomAccessSlice(i, padded.length() - i); + for (int j = 0; j < ints.length; j++) { + assertEquals(ints[j], whole.readInt(j * 4)); + } + padded.close(); + } + input.close(); } - input.close(); - dir.close(); } public void testRandomShort() throws Exception { - Directory dir = getDirectory(createTempDir("testShorts")); - IndexOutput output = dir.createOutput("shorts", newIOContext(random())); - int num = TestUtil.nextInt(random(), 50, 3000); - short shorts[] = new short[num]; - for (int i = 0; i < shorts.length; i++) { - shorts[i] = (short) random().nextInt(); - output.writeShort(shorts[i]); - } - output.close(); - - // slice - IndexInput input = dir.openInput("shorts", newIOContext(random())); - RandomAccessInput slice = input.randomAccessSlice(0, input.length()); - for (int i = 0; i < shorts.length; i++) { - assertEquals(shorts[i], slice.readShort(i * 2)); - } - - // subslices - for (int i = 1; i < shorts.length; i++) { - long offset = i * 2; - RandomAccessInput subslice = input.randomAccessSlice(offset, input.length() - offset); - for (int j = i; j < shorts.length; j++) { - assertEquals(shorts[j], subslice.readShort((j - i) * 2)); + try (Directory dir = getDirectory(createTempDir("testShorts"))) { + IndexOutput output = dir.createOutput("shorts", newIOContext(random())); + int num = TestUtil.nextInt(random(), 50, 3000); + short shorts[] = new short[num]; + for (int i = 0; i < shorts.length; i++) { + shorts[i] = (short) random().nextInt(); + output.writeShort(shorts[i]); } - } - - // with padding - for (int i = 0; i < 7; i++) { - String name = "shorts-" + i; - IndexOutput o = dir.createOutput(name, newIOContext(random())); - byte junk[] = new byte[i]; - random().nextBytes(junk); - o.writeBytes(junk, junk.length); - input.seek(0); - o.copyBytes(input, input.length()); - o.close(); - IndexInput padded = dir.openInput(name, newIOContext(random())); - RandomAccessInput whole = padded.randomAccessSlice(i, padded.length() - i); - for (int j = 0; j < shorts.length; j++) { - assertEquals(shorts[j], whole.readShort(j * 2)); + output.close(); + + // slice + IndexInput input = dir.openInput("shorts", newIOContext(random())); + RandomAccessInput slice = input.randomAccessSlice(0, input.length()); + for (int i = 0; i < shorts.length; i++) { + assertEquals(shorts[i], slice.readShort(i * 2)); } - padded.close(); + + // subslices + for (int i = 1; i < shorts.length; i++) { + long offset = i * 2; + RandomAccessInput subslice = input.randomAccessSlice(offset, input.length() - offset); + for (int j = i; j < shorts.length; j++) { + assertEquals(shorts[j], subslice.readShort((j - i) * 2)); + } + } + + // with padding + for (int i = 0; i < 7; i++) { + String name = "shorts-" + i; + IndexOutput o = dir.createOutput(name, newIOContext(random())); + byte junk[] = new byte[i]; + random().nextBytes(junk); + o.writeBytes(junk, junk.length); + input.seek(0); + o.copyBytes(input, input.length()); + o.close(); + IndexInput padded = dir.openInput(name, newIOContext(random())); + RandomAccessInput whole = padded.randomAccessSlice(i, padded.length() - i); + for (int j = 0; j < shorts.length; j++) { + assertEquals(shorts[j], whole.readShort(j * 2)); + } + padded.close(); + } + input.close(); } - input.close(); - dir.close(); } public void testRandomByte() throws Exception { - Directory dir = getDirectory(createTempDir("testBytes")); - IndexOutput output = dir.createOutput("bytes", newIOContext(random())); - int num = TestUtil.nextInt(random(), 50, 3000); - byte bytes[] = new byte[num]; - random().nextBytes(bytes); - for (int i = 0; i < bytes.length; i++) { - output.writeByte(bytes[i]); - } - output.close(); - - // slice - IndexInput input = dir.openInput("bytes", newIOContext(random())); - RandomAccessInput slice = input.randomAccessSlice(0, input.length()); - for (int i = 0; i < bytes.length; i++) { - assertEquals(bytes[i], slice.readByte(i)); - } - - // subslices - for (int i = 1; i < bytes.length; i++) { - long offset = i; - RandomAccessInput subslice = input.randomAccessSlice(offset, input.length() - offset); - for (int j = i; j < bytes.length; j++) { - assertEquals(bytes[j], subslice.readByte(j - i)); + try (Directory dir = getDirectory(createTempDir("testBytes"))) { + IndexOutput output = dir.createOutput("bytes", newIOContext(random())); + int num = TestUtil.nextInt(random(), 50, 3000); + byte bytes[] = new byte[num]; + random().nextBytes(bytes); + for (int i = 0; i < bytes.length; i++) { + output.writeByte(bytes[i]); } - } - - // with padding - for (int i = 0; i < 7; i++) { - String name = "bytes-" + i; - IndexOutput o = dir.createOutput(name, newIOContext(random())); - byte junk[] = new byte[i]; - random().nextBytes(junk); - o.writeBytes(junk, junk.length); - input.seek(0); - o.copyBytes(input, input.length()); - o.close(); - IndexInput padded = dir.openInput(name, newIOContext(random())); - RandomAccessInput whole = padded.randomAccessSlice(i, padded.length() - i); - for (int j = 0; j < bytes.length; j++) { - assertEquals(bytes[j], whole.readByte(j)); + output.close(); + + // slice + IndexInput input = dir.openInput("bytes", newIOContext(random())); + RandomAccessInput slice = input.randomAccessSlice(0, input.length()); + for (int i = 0; i < bytes.length; i++) { + assertEquals(bytes[i], slice.readByte(i)); + } + + // subslices + for (int i = 1; i < bytes.length; i++) { + long offset = i; + RandomAccessInput subslice = input.randomAccessSlice(offset, input.length() - offset); + for (int j = i; j < bytes.length; j++) { + assertEquals(bytes[j], subslice.readByte(j - i)); + } + } + + // with padding + for (int i = 0; i < 7; i++) { + String name = "bytes-" + i; + IndexOutput o = dir.createOutput(name, newIOContext(random())); + byte junk[] = new byte[i]; + random().nextBytes(junk); + o.writeBytes(junk, junk.length); + input.seek(0); + o.copyBytes(input, input.length()); + o.close(); + IndexInput padded = dir.openInput(name, newIOContext(random())); + RandomAccessInput whole = padded.randomAccessSlice(i, padded.length() - i); + for (int j = 0; j < bytes.length; j++) { + assertEquals(bytes[j], whole.readByte(j)); + } + padded.close(); } - padded.close(); + input.close(); } - input.close(); - dir.close(); } /** try to stress slices of slices */ public void testSliceOfSlice() throws Exception { - Directory dir = getDirectory(createTempDir("sliceOfSlice")); - IndexOutput output = dir.createOutput("bytes", newIOContext(random())); - final int num; - if (TEST_NIGHTLY) { - num = TestUtil.nextInt(random(), 250, 2500); - } else { - num = TestUtil.nextInt(random(), 50, 250); - } - byte bytes[] = new byte[num]; - random().nextBytes(bytes); - for (int i = 0; i < bytes.length; i++) { - output.writeByte(bytes[i]); - } - output.close(); - - IndexInput input = dir.openInput("bytes", newIOContext(random())); - // seek to a random spot shouldnt impact slicing. - input.seek(TestUtil.nextLong(random(), 0, input.length())); - for (int i = 0; i < num; i += 16) { - IndexInput slice1 = input.slice("slice1", i, num-i); - assertEquals(0, slice1.getFilePointer()); - assertEquals(num-i, slice1.length()); - + try (Directory dir = getDirectory(createTempDir("sliceOfSlice"))) { + IndexOutput output = dir.createOutput("bytes", newIOContext(random())); + final int num; + if (TEST_NIGHTLY) { + num = TestUtil.nextInt(random(), 250, 2500); + } else { + num = TestUtil.nextInt(random(), 50, 250); + } + byte bytes[] = new byte[num]; + random().nextBytes(bytes); + for (int i = 0; i < bytes.length; i++) { + output.writeByte(bytes[i]); + } + output.close(); + + IndexInput input = dir.openInput("bytes", newIOContext(random())); // seek to a random spot shouldnt impact slicing. - slice1.seek(TestUtil.nextLong(random(), 0, slice1.length())); - for (int j = 0; j < slice1.length(); j += 16) { - IndexInput slice2 = slice1.slice("slice2", j, num-i-j); - assertEquals(0, slice2.getFilePointer()); - assertEquals(num-i-j, slice2.length()); - byte data[] = new byte[num]; - System.arraycopy(bytes, 0, data, 0, i+j); - if (random().nextBoolean()) { - // read the bytes for this slice-of-slice - slice2.readBytes(data, i+j, num-i-j); - } else { - // seek to a random spot in between, read some, seek back and read the rest - long seek = TestUtil.nextLong(random(), 0, slice2.length()); - slice2.seek(seek); - slice2.readBytes(data, (int)(i+j+seek), (int)(num-i-j-seek)); - slice2.seek(0); - slice2.readBytes(data, i+j, (int)seek); + input.seek(TestUtil.nextLong(random(), 0, input.length())); + for (int i = 0; i < num; i += 16) { + IndexInput slice1 = input.slice("slice1", i, num - i); + assertEquals(0, slice1.getFilePointer()); + assertEquals(num - i, slice1.length()); + + // seek to a random spot shouldnt impact slicing. + slice1.seek(TestUtil.nextLong(random(), 0, slice1.length())); + for (int j = 0; j < slice1.length(); j += 16) { + IndexInput slice2 = slice1.slice("slice2", j, num - i - j); + assertEquals(0, slice2.getFilePointer()); + assertEquals(num - i - j, slice2.length()); + byte data[] = new byte[num]; + System.arraycopy(bytes, 0, data, 0, i + j); + if (random().nextBoolean()) { + // read the bytes for this slice-of-slice + slice2.readBytes(data, i + j, num - i - j); + } else { + // seek to a random spot in between, read some, seek back and read the rest + long seek = TestUtil.nextLong(random(), 0, slice2.length()); + slice2.seek(seek); + slice2.readBytes(data, (int) (i + j + seek), (int) (num - i - j - seek)); + slice2.seek(0); + slice2.readBytes(data, i + j, (int) seek); + } + assertArrayEquals(bytes, data); } - assertArrayEquals(bytes, data); } + + input.close(); } - - input.close(); - dir.close(); } /** @@ -1020,73 +966,73 @@ public void testSliceOfSlice() throws Exception { * will correctly increment the file pointer. */ public void testLargeWrites() throws IOException { - Directory dir = getDirectory(createTempDir("largeWrites")); - IndexOutput os = dir.createOutput("testBufferStart.txt", newIOContext(random())); - - byte[] largeBuf = new byte[2048]; - random().nextBytes(largeBuf); - - long currentPos = os.getFilePointer(); - os.writeBytes(largeBuf, largeBuf.length); - - try { - assertEquals(currentPos + largeBuf.length, os.getFilePointer()); - } finally { - os.close(); + try (Directory dir = getDirectory(createTempDir("largeWrites"))) { + IndexOutput os = dir.createOutput("testBufferStart.txt", newIOContext(random())); + + byte[] largeBuf = new byte[2048]; + random().nextBytes(largeBuf); + + long currentPos = os.getFilePointer(); + os.writeBytes(largeBuf, largeBuf.length); + + try { + assertEquals(currentPos + largeBuf.length, os.getFilePointer()); + } finally { + os.close(); + } } - dir.close(); } // LUCENE-6084 public void testIndexOutputToString() throws Throwable { - Directory dir = getDirectory(createTempDir()); - IndexOutput out = dir.createOutput("camelCase.txt", newIOContext(random())); - assertTrue(out.toString(), out.toString().contains("camelCase.txt")); - out.close(); - dir.close(); + try (Directory dir = getDirectory(createTempDir())) { + IndexOutput out = dir.createOutput("camelCase.txt", newIOContext(random())); + assertTrue(out.toString(), out.toString().contains("camelCase.txt")); + out.close(); + } } public void testDoubleCloseOutput() throws Throwable { - Directory dir = getDirectory(createTempDir()); - IndexOutput out = dir.createOutput("foobar", newIOContext(random())); - out.writeString("testing"); - out.close(); - out.close(); // close again - dir.close(); + try (Directory dir = getDirectory(createTempDir())) { + IndexOutput out = dir.createOutput("foobar", newIOContext(random())); + out.writeString("testing"); + out.close(); + out.close(); // close again + } } public void testDoubleCloseInput() throws Throwable { - Directory dir = getDirectory(createTempDir()); - IndexOutput out = dir.createOutput("foobar", newIOContext(random())); - out.writeString("testing"); - out.close(); - IndexInput in = dir.openInput("foobar", newIOContext(random())); - assertEquals("testing", in.readString()); - in.close(); - in.close(); // close again - dir.close(); + try (Directory dir = getDirectory(createTempDir())) { + IndexOutput out = dir.createOutput("foobar", newIOContext(random())); + out.writeString("testing"); + out.close(); + IndexInput in = dir.openInput("foobar", newIOContext(random())); + assertEquals("testing", in.readString()); + in.close(); + in.close(); // close again + } } public void testCreateTempOutput() throws Throwable { - Directory dir = getDirectory(createTempDir()); - List names = new ArrayList<>(); - int iters = atLeast(50); - for(int iter=0;iter names = new ArrayList<>(); + int iters = atLeast(50); + for (int iter = 0; iter < iters; iter++) { + IndexOutput out = dir.createTempOutput("foo", "bar", newIOContext(random())); + names.add(out.getName()); + out.writeVInt(iter); + out.close(); + } + for (int iter = 0; iter < iters; iter++) { + IndexInput in = dir.openInput(names.get(iter), newIOContext(random())); + assertEquals(iter, in.readVInt()); + in.close(); + } + Set files = new HashSet(Arrays.asList(dir.listAll())); + // In case ExtrasFS struck: + files.remove("extra0"); + assertEquals(new HashSet(names), files); } - Set files = new HashSet(Arrays.asList(dir.listAll())); - // In case ExtrasFS struck: - files.remove("extra0"); - assertEquals(new HashSet(names), files); - dir.close(); } public void testCreateOutputForExistingFile() throws IOException { @@ -1103,21 +1049,7 @@ public void testCreateOutputForExistingFile() throws IOException { // Delete file and try to recreate it. dir.deleteFile(name); - try (IndexOutput out = dir.createOutput(name, IOContext.DEFAULT)) { - } - } - } - - public void testReadFileOpenForWrites() throws IOException { - try (Directory dir = getDirectory(createTempDir())) { - String name = "file"; - try (IndexOutput out = dir.createOutput(name, IOContext.DEFAULT)) { - // Shouldn't be able to open this file for reading. - expectThrows(IOException.class, () -> { - try (IndexInput in = dir.openInput(name, IOContext.DEFAULT)) { - } - }); - } + dir.createOutput(name, IOContext.DEFAULT).close(); } } diff --git a/lucene/test-framework/src/java/org/apache/lucene/util/LuceneTestCase.java b/lucene/test-framework/src/java/org/apache/lucene/util/LuceneTestCase.java index 6be8d037fd68..ff3adc3405fa 100644 --- a/lucene/test-framework/src/java/org/apache/lucene/util/LuceneTestCase.java +++ b/lucene/test-framework/src/java/org/apache/lucene/util/LuceneTestCase.java @@ -64,6 +64,7 @@ import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicReference; import java.util.logging.Logger; +import java.util.stream.Collectors; import org.apache.lucene.analysis.Analyzer; import org.apache.lucene.analysis.MockAnalyzer; @@ -2678,6 +2679,38 @@ public static T expectThrows(Class expectedType, Throwi throw new AssertionFailedError("Expected exception " + expectedType.getSimpleName() + " but no exception was thrown"); } + /** Checks a specific exception class is thrown by the given runnable, and returns it. */ + public static T expectThrowsAnyOf(List> expectedTypes, ThrowingRunnable runnable) { + if (expectedTypes.isEmpty()) { + throw new AssertionError("At least one expected exception type is required?"); + } + + Throwable thrown = null; + try { + runnable.run(); + } catch (Throwable e) { + for (Class expectedType : expectedTypes) { + if (expectedType.isInstance(e)) { + return expectedType.cast(e); + } + } + thrown = e; + } + + List exceptionTypes = expectedTypes.stream().map(c -> c.getSimpleName()).collect(Collectors.toList()); + + if (thrown != null) { + AssertionFailedError assertion = new AssertionFailedError("Unexpected exception type, expected any of " + + exceptionTypes + + " but got: " + thrown); + assertion.initCause(thrown); + throw assertion; + } else { + throw new AssertionFailedError("Expected any of the following exception types: " + + exceptionTypes+ " but no exception was thrown."); + } + } + /** * Checks that specific wrapped and outer exception classes are thrown * by the given runnable, and returns the wrapped exception. diff --git a/lucene/test-framework/src/test/org/apache/lucene/store/TestMockDirectoryWrapper.java b/lucene/test-framework/src/test/org/apache/lucene/store/TestMockDirectoryWrapper.java index 79732566644c..2ea6ce0c1acc 100644 --- a/lucene/test-framework/src/test/org/apache/lucene/store/TestMockDirectoryWrapper.java +++ b/lucene/test-framework/src/test/org/apache/lucene/store/TestMockDirectoryWrapper.java @@ -40,11 +40,12 @@ protected Directory getDirectory(Path path) throws IOException { } // we wrap the directory in slow stuff, so only run nightly - @Override @Nightly - public void testThreadSafety() throws Exception { - super.testThreadSafety(); + @Override + @Nightly + public void testThreadSafetyInListAll() throws Exception { + super.testThreadSafetyInListAll(); } - + public void testDiskFull() throws IOException { // test writeBytes MockDirectoryWrapper dir = newMockDirectory(); From fbbea4ef76015cfbd8a43dc74c2718c88162a862 Mon Sep 17 00:00:00 2001 From: Dawid Weiss Date: Mon, 23 Jul 2018 11:16:01 +0200 Subject: [PATCH 06/10] Unused imports. --- .../java/org/apache/lucene/store/BaseDirectoryTestCase.java | 3 --- 1 file changed, 3 deletions(-) diff --git a/lucene/test-framework/src/java/org/apache/lucene/store/BaseDirectoryTestCase.java b/lucene/test-framework/src/java/org/apache/lucene/store/BaseDirectoryTestCase.java index 4f196ae2179f..118e74f04d1d 100644 --- a/lucene/test-framework/src/java/org/apache/lucene/store/BaseDirectoryTestCase.java +++ b/lucene/test-framework/src/java/org/apache/lucene/store/BaseDirectoryTestCase.java @@ -34,7 +34,6 @@ import java.util.Map; import java.util.Random; import java.util.Set; -import java.util.concurrent.CountDownLatch; import java.util.concurrent.CyclicBarrier; import java.util.concurrent.atomic.AtomicBoolean; import java.util.stream.IntStream; @@ -42,7 +41,6 @@ import com.carrotsearch.randomizedtesting.RandomizedTest; import com.carrotsearch.randomizedtesting.generators.RandomBytes; -import com.carrotsearch.randomizedtesting.generators.RandomNumbers; import com.carrotsearch.randomizedtesting.generators.RandomPicks; import org.apache.lucene.index.DirectoryReader; import org.apache.lucene.index.IndexFileNames; @@ -51,7 +49,6 @@ import org.apache.lucene.util.LuceneTestCase; import org.apache.lucene.util.TestUtil; import org.junit.Assert; -import org.junit.internal.AssumptionViolatedException; /** * Base class for per-Directory tests. From bd10c6e59d9ee54a9e7880bfd1cdd3a163a74883 Mon Sep 17 00:00:00 2001 From: Dawid Weiss Date: Mon, 23 Jul 2018 11:50:48 +0200 Subject: [PATCH 07/10] Ignore ExtraFS files. --- .../org/apache/lucene/mockfile/ExtrasFS.java | 10 ++++- .../lucene/store/BaseDirectoryTestCase.java | 43 +++++++++++-------- 2 files changed, 33 insertions(+), 20 deletions(-) diff --git a/lucene/test-framework/src/java/org/apache/lucene/mockfile/ExtrasFS.java b/lucene/test-framework/src/java/org/apache/lucene/mockfile/ExtrasFS.java index 28fc44f45199..dc39977c4f0c 100644 --- a/lucene/test-framework/src/java/org/apache/lucene/mockfile/ExtrasFS.java +++ b/lucene/test-framework/src/java/org/apache/lucene/mockfile/ExtrasFS.java @@ -34,6 +34,8 @@ * All other filesystem operations are passed thru as normal. */ public class ExtrasFS extends FilterFileSystemProvider { + private final static String EXTRA_FILE_NAME = "extra0"; + final boolean active; final boolean createDirectory; @@ -57,7 +59,7 @@ public void createDirectory(Path dir, FileAttribute... attrs) throws IOExcept if (active) { // lets add a bogus file... if this fails, we don't care, its best effort. try { - Path target = dir.resolve("extra0"); + Path target = dir.resolve(EXTRA_FILE_NAME); if (createDirectory) { super.createDirectory(target); } else { @@ -73,4 +75,10 @@ public void createDirectory(Path dir, FileAttribute... attrs) throws IOExcept // our fake files. But this is tricky because its hooked into several places. // Currently MDW has a hack so we don't break disk full tests. + /** + * @return Return true if {@code fileName} is one of the extra files added by this class. + */ + public static boolean isExtra(String fileName) { + return fileName.equals(EXTRA_FILE_NAME); + } } diff --git a/lucene/test-framework/src/java/org/apache/lucene/store/BaseDirectoryTestCase.java b/lucene/test-framework/src/java/org/apache/lucene/store/BaseDirectoryTestCase.java index 118e74f04d1d..4270424b4ad6 100644 --- a/lucene/test-framework/src/java/org/apache/lucene/store/BaseDirectoryTestCase.java +++ b/lucene/test-framework/src/java/org/apache/lucene/store/BaseDirectoryTestCase.java @@ -36,6 +36,7 @@ import java.util.Set; import java.util.concurrent.CyclicBarrier; import java.util.concurrent.atomic.AtomicBoolean; +import java.util.stream.Collectors; import java.util.stream.IntStream; import java.util.zip.CRC32; @@ -45,6 +46,7 @@ import org.apache.lucene.index.DirectoryReader; import org.apache.lucene.index.IndexFileNames; import org.apache.lucene.index.IndexNotFoundException; +import org.apache.lucene.mockfile.ExtrasFS; import org.apache.lucene.util.IOUtils; import org.apache.lucene.util.LuceneTestCase; import org.apache.lucene.util.TestUtil; @@ -410,7 +412,7 @@ public void testThreadSafetyInListAll() throws Exception { try { Random rnd = new Random(RandomizedTest.randomLong() + 1); for (int i = 0, max = RandomizedTest.randomIntBetween(500, 1000); i < max; i++) { - String fileName = "T1-" + i; + String fileName = "file-" + i; try (IndexOutput output = dir.createOutput(fileName, newIOContext(random()))) { // Add some lags so that the other thread can read the content of the directory. Thread.yield(); @@ -428,22 +430,23 @@ public void testThreadSafetyInListAll() throws Exception { try { Random rnd = new Random(RandomizedTest.randomLong()); while (!stop.get()) { - String[] files = dir.listAll(); - if (files.length == 0) { - continue; + String [] files = Arrays.stream(dir.listAll()) + .filter(name -> !ExtrasFS.isExtra(name)) // Ignore anything from ExtraFS. + .toArray(String[]::new); + + if (files.length > 0) { + do { + String file = RandomPicks.randomFrom(rnd, files); + try (IndexInput input = dir.openInput(file, newIOContext(random()))) { + // Just open, nothing else. + } catch (AccessDeniedException e) { + // Access denied is allowed for files for which the output is still open. + // Since we don't synchronize with the writer thread, just ignore it. + } catch (IOException e) { + throw new UncheckedIOException("Something went wrong when opening: " + file, e); + } + } while (rnd.nextInt(3) != 0); // Sometimes break and list files again. } - - do { - String file = RandomPicks.randomFrom(rnd, files); - try (IndexInput input = dir.openInput(file, newIOContext(random()))) { - // Just open, nothing else. - } catch (AccessDeniedException e) { - // Access denied is allowed for files for which the output is still open. - // Since we don't synchronize with the writer thread, just ignore it. - } catch (IOException e) { - throw new UncheckedIOException("Something went wrong when opening: " + file, e); - } - } while (rnd.nextInt(3) != 0); // Sometimes break and list files again. } } catch (IOException e) { throw new UncheckedIOException(e); @@ -1025,9 +1028,11 @@ public void testCreateTempOutput() throws Throwable { assertEquals(iter, in.readVInt()); in.close(); } - Set files = new HashSet(Arrays.asList(dir.listAll())); - // In case ExtrasFS struck: - files.remove("extra0"); + + Set files = Arrays.stream(dir.listAll()) + .filter(file -> !ExtrasFS.isExtra(file)) // remove any ExtrasFS stuff. + .collect(Collectors.toSet()); + assertEquals(new HashSet(names), files); } } From 348702c5704142bd51fb942289732ab7dd350266 Mon Sep 17 00:00:00 2001 From: Dawid Weiss Date: Mon, 23 Jul 2018 12:15:01 +0200 Subject: [PATCH 08/10] ExtraFS 'extra0' refs. --- .../test/org/apache/lucene/index/TestIndexWriter.java | 10 +++++----- .../src/test/org/apache/lucene/util/bkd/TestBKD.java | 7 +++++-- .../src/java/org/apache/lucene/mockfile/ExtrasFS.java | 8 +++++--- 3 files changed, 15 insertions(+), 10 deletions(-) diff --git a/lucene/core/src/test/org/apache/lucene/index/TestIndexWriter.java b/lucene/core/src/test/org/apache/lucene/index/TestIndexWriter.java index 0df98bc65203..31740f9d238f 100644 --- a/lucene/core/src/test/org/apache/lucene/index/TestIndexWriter.java +++ b/lucene/core/src/test/org/apache/lucene/index/TestIndexWriter.java @@ -68,6 +68,7 @@ import org.apache.lucene.document.StringField; import org.apache.lucene.document.TextField; import org.apache.lucene.index.IndexWriterConfig.OpenMode; +import org.apache.lucene.mockfile.ExtrasFS; import org.apache.lucene.mockfile.FilterPath; import org.apache.lucene.mockfile.WindowsFS; import org.apache.lucene.search.DocIdSetIterator; @@ -3348,12 +3349,11 @@ public IndexOutput createOutput(String name, IOContext context) throws IOExcepti private static void assertFiles(IndexWriter writer) throws IOException { Predicate filter = file -> file.startsWith("segments") == false && file.equals("write.lock") == false; // remove segment files we don't know if we have committed and what is kept around - Set segFiles = new HashSet<>(writer.segmentInfos.files(true)).stream() + Set segFiles = writer.segmentInfos.files(true).stream() + .filter(filter).collect(Collectors.toSet()); + Set dirFiles = Arrays.stream(writer.getDirectory().listAll()) + .filter(file -> !ExtrasFS.isExtra(file)) // ExtraFS might add an files, ignore them .filter(filter).collect(Collectors.toSet()); - Set dirFiles = new HashSet<>(Arrays.asList(writer.getDirectory().listAll())) - .stream().filter(filter).collect(Collectors.toSet()); - // ExtraFS might add an extra0 file, ignore it - dirFiles.remove("extra0"); assertEquals(segFiles.size(), dirFiles.size()); } diff --git a/lucene/core/src/test/org/apache/lucene/util/bkd/TestBKD.java b/lucene/core/src/test/org/apache/lucene/util/bkd/TestBKD.java index b9dad6fff800..3cf248b2ccf2 100644 --- a/lucene/core/src/test/org/apache/lucene/util/bkd/TestBKD.java +++ b/lucene/core/src/test/org/apache/lucene/util/bkd/TestBKD.java @@ -29,6 +29,7 @@ import org.apache.lucene.index.PointValues.IntersectVisitor; import org.apache.lucene.index.PointValues.Relation; import org.apache.lucene.index.PointValues; +import org.apache.lucene.mockfile.ExtrasFS; import org.apache.lucene.store.CorruptingIndexOutput; import org.apache.lucene.store.Directory; import org.apache.lucene.store.FilterDirectory; @@ -414,8 +415,10 @@ public void testWithExceptions() throws Exception { } } - String[] files = dir.listAll(); - assertTrue("files=" + Arrays.toString(files), files.length == 0 || Arrays.equals(files, new String[] {"extra0"})); + String[] files = Arrays.stream(dir.listAll()) + .filter(file -> !ExtrasFS.isExtra(file)) + .toArray(String[]::new); + assertTrue("files=" + Arrays.toString(files), files.length == 0); dir.close(); } } diff --git a/lucene/test-framework/src/java/org/apache/lucene/mockfile/ExtrasFS.java b/lucene/test-framework/src/java/org/apache/lucene/mockfile/ExtrasFS.java index dc39977c4f0c..f8bba9c35942 100644 --- a/lucene/test-framework/src/java/org/apache/lucene/mockfile/ExtrasFS.java +++ b/lucene/test-framework/src/java/org/apache/lucene/mockfile/ExtrasFS.java @@ -29,9 +29,11 @@ * create special files themselves (.DS_Store, thumbs.db, .nfsXXX, ...), * so we add them and see what breaks. *

- * When a directory is created, sometimes a file or directory named - * "extra0" will be included with it. - * All other filesystem operations are passed thru as normal. + * When a directory is created, sometimes an "extra" file or directory + * will be included with it (use {@link #isExtra(String)} to check if it's one + * of those files). + * + * All other filesystem operations are delegated as normal. */ public class ExtrasFS extends FilterFileSystemProvider { private final static String EXTRA_FILE_NAME = "extra0"; From c87d4e92984ffa8f39d25377869be0785387424b Mon Sep 17 00:00:00 2001 From: Dawid Weiss Date: Mon, 23 Jul 2018 12:33:35 +0200 Subject: [PATCH 09/10] MockDirectoryWrapper should throw FileAlreadyExists on files already written to. --- .../lucene/store/MockDirectoryWrapper.java | 35 +++++++++---------- 1 file changed, 16 insertions(+), 19 deletions(-) diff --git a/lucene/test-framework/src/java/org/apache/lucene/store/MockDirectoryWrapper.java b/lucene/test-framework/src/java/org/apache/lucene/store/MockDirectoryWrapper.java index f1ac7ee2544f..ec0d4ff2fd1e 100644 --- a/lucene/test-framework/src/java/org/apache/lucene/store/MockDirectoryWrapper.java +++ b/lucene/test-framework/src/java/org/apache/lucene/store/MockDirectoryWrapper.java @@ -19,6 +19,7 @@ import java.io.Closeable; import java.io.FileNotFoundException; import java.io.IOException; +import java.nio.file.FileAlreadyExistsException; import java.nio.file.NoSuchFileException; import java.util.ArrayList; import java.util.Arrays; @@ -649,11 +650,11 @@ public synchronized IndexOutput createOutput(String name, IOContext context) thr throw new IOException("cannot createOutput after crash"); } init(); - synchronized(this) { - if (createdFiles.contains(name)) { - throw new IOException("file \"" + name + "\" was already written to"); - } + + if (createdFiles.contains(name)) { + throw new FileAlreadyExistsException("File \"" + name + "\" was already written to."); } + if (assertNoDeleteOpenFile && openFiles.containsKey(name)) { throw new AssertionError("MockDirectoryWrapper: file \"" + name + "\" is still open: cannot overwrite"); } @@ -666,19 +667,24 @@ public synchronized IndexOutput createOutput(String name, IOContext context) thr final IndexOutput io = new MockIndexOutputWrapper(this, delegateOutput, name); addFileHandle(io, name, Handle.Output); openFilesForWrite.add(name); - + return maybeThrottle(name, io); + + + } + + private IndexOutput maybeThrottle(String name, IndexOutput output) { // throttling REALLY slows down tests, so don't do it very often for SOMETIMES. - if (throttling == Throttling.ALWAYS || + if (throttling == Throttling.ALWAYS || (throttling == Throttling.SOMETIMES && randomState.nextInt(200) == 0)) { if (LuceneTestCase.VERBOSE) { System.out.println("MockDirectoryWrapper: throttling indexOutput (" + name + ")"); } - return throttledOutput.newFromDelegate(io); + return throttledOutput.newFromDelegate(output); } else { - return io; + return output; } } - + @Override public synchronized IndexOutput createTempOutput(String prefix, String suffix, IOContext context) throws IOException { maybeThrowDeterministicException(); @@ -704,16 +710,7 @@ public synchronized IndexOutput createTempOutput(String prefix, String suffix, I addFileHandle(io, name, Handle.Output); openFilesForWrite.add(name); - // throttling REALLY slows down tests, so don't do it very often for SOMETIMES. - if (throttling == Throttling.ALWAYS || - (throttling == Throttling.SOMETIMES && randomState.nextInt(200) == 0)) { - if (LuceneTestCase.VERBOSE) { - System.out.println("MockDirectoryWrapper: throttling indexOutput (" + name + ")"); - } - return throttledOutput.newFromDelegate(io); - } else { - return io; - } + return maybeThrottle(name, io); } private static enum Handle { From 0b46f9bb93ab53379cd0c6ddcbd8c990a0737c84 Mon Sep 17 00:00:00 2001 From: Dawid Weiss Date: Mon, 23 Jul 2018 13:42:13 +0200 Subject: [PATCH 10/10] Changes entry. --- lucene/CHANGES.txt | 2 ++ 1 file changed, 2 insertions(+) diff --git a/lucene/CHANGES.txt b/lucene/CHANGES.txt index 730e24c18187..e324f2c6d42b 100644 --- a/lucene/CHANGES.txt +++ b/lucene/CHANGES.txt @@ -202,6 +202,8 @@ Improvements * LUCENE-8345, GitHub PR #392: Remove instantiation of redundant wrapper classes for primitives; add wrapper class constructors to forbiddenapis. (Michael Braun via Uwe Schindler) +* LUCENE-8415: Clean up Directory contracts and JavaDoc comments. (Dawid Weiss) + Other: * LUCENE-8366: Upgrade to ICU 62.1. Emoji handling now uses Unicode 11's