Skip to content

NIFI-13186 corrected required runtime dep lucene-backward-codecs and … #8782

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Closed
joewitt wants to merge 1 commit into from
Closed

NIFI-13186 corrected required runtime dep lucene-backward-codecs and … #8782

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
15 changes: 9 additions & 6 deletions ...n-bundles/nifi-provenance-repository-bundle/nifi-persistent-provenance-repository/pom.xml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -52,13 +52,16 @@
<artifactId>nifi-repository-encryption</artifactId>
<version>2.0.0-SNAPSHOT</version>
</dependency>

<dependency>
<groupId>org.apache.lucene</groupId>
<artifactId>lucene-core</artifactId>
</dependency>
<dependency>
<groupId>org.apache.lucene</groupId>
<artifactId>lucene-backward-codecs</artifactId>
<scope>runtime</scope>
</dependency>
</dependencies>

<build>
<plugins>
<plugin>
Expand All @@ -68,10 +71,10 @@
<excludes combine.children="append">
<exclude>src/test/resources/lucene-4-prov-repo/0.prov</exclude>
<exclude>src/test/resources/lucene-4-prov-repo/toc/0.toc</exclude>
<exclude>src/test/resources/lucene-4-prov-repo/index-1554304717707/_0.fdt</exclude>
<exclude>src/test/resources/lucene-4-prov-repo/index-1554304717707/_0.fdx</exclude>
<exclude>src/test/resources/lucene-4-prov-repo/index-1554304717707/_1.fdt</exclude>
<exclude>src/test/resources/lucene-4-prov-repo/index-1554304717707/_1.fdx</exclude>
<exclude>src/test/resources/lucene-4-prov-repo/lucene-4-index-1554304717707/_0.fdt</exclude>
<exclude>src/test/resources/lucene-4-prov-repo/lucene-4-index-1554304717707/_0.fdx</exclude>
<exclude>src/test/resources/lucene-4-prov-repo/lucene-4-index-1554304717707/_1.fdt</exclude>
<exclude>src/test/resources/lucene-4-prov-repo/lucene-4-index-1554304717707/_1.fdx</exclude>
</excludes>
</configuration>
</plugin>
Expand Down
2 changes: 1 addition & 1 deletion ...nt-provenance-repository/src/main/java/org/apache/nifi/provenance/IndexConfiguration.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -170,7 +170,7 @@ private File addNewIndex(final File storageDirectory, final File provenanceLogFi
if (firstEntryTime == null) {
firstEntryTime = newIndexTimestamp;
}
return new File(storageDirectory, "lucene-8-index-" + firstEntryTime);
return new File(storageDirectory, "lucene-9-index-" + firstEntryTime);
}

public List<File> getIndexDirectories() {
Expand Down
24 changes: 4 additions & 20 deletions ...pository/src/main/java/org/apache/nifi/provenance/index/lucene/IndexDirectoryManager.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -51,13 +51,6 @@ public class IndexDirectoryManager {
private static final Pattern LUCENE_8_AND_LATER_INDEX_PATTERN = Pattern.compile("lucene-\\d+-index-(.*)");
private static final FileFilter LUCENE_8_AND_LATER_INDEX_DIRECTORY_FILTER = f -> LUCENE_8_AND_LATER_INDEX_PATTERN.matcher(f.getName()).matches();

private static final Pattern INDEX_FILENAME_PATTERN = DirectoryUtils.INDEX_DIRECTORY_NAME_PATTERN;
private static final FileFilter ALL_INDEX_FILE_FILTER = f -> INDEX_FILENAME_PATTERN.matcher(f.getName()).matches();

private static final Pattern LUCENE_4_INDEX_PATTERN = Pattern.compile("index-(.*)");
private static final FileFilter LUCENE_4_INDEX_FILE_FILTER = f -> LUCENE_4_INDEX_PATTERN.matcher(f.getName()).matches();


private final RepositoryConfiguration repoConfig;

// guarded by synchronizing on 'this'
Expand All @@ -82,7 +75,7 @@ public synchronized void initialize() {
}

for (final File indexDir : indexDirs) {
final Matcher matcher = INDEX_FILENAME_PATTERN.matcher(indexDir.getName());
final Matcher matcher = LUCENE_8_AND_LATER_INDEX_PATTERN.matcher(indexDir.getName());
if (!matcher.matches()) {
continue;
}
Expand Down Expand Up @@ -131,19 +124,10 @@ public synchronized void removeDirectory(final File directory) {
}
}

public synchronized List<File> getAllIndexDirectories(final boolean includeLucene4Directories, final boolean includeLaterLuceneDirectories) {
public synchronized List<File> getAllIndexDirectories() {
final List<File> allDirectories = new ArrayList<>();

final FileFilter directoryFilter;
if (includeLucene4Directories && includeLaterLuceneDirectories) {
directoryFilter = ALL_INDEX_FILE_FILTER;
} else if (includeLucene4Directories) {
directoryFilter = LUCENE_4_INDEX_FILE_FILTER;
} else if (includeLaterLuceneDirectories) {
directoryFilter = LUCENE_8_AND_LATER_INDEX_DIRECTORY_FILTER;
} else {
throw new IllegalArgumentException("Cannot list all directoreis but excluded Lucene 4 directories and later directories");
}
final FileFilter directoryFilter = LUCENE_8_AND_LATER_INDEX_DIRECTORY_FILTER;

for (final File storageDir : repoConfig.getStorageDirectories().values()) {
final File[] indexDirs = storageDir.listFiles(directoryFilter);
Expand Down Expand Up @@ -410,7 +394,7 @@ private File createIndex(final long earliestTimestamp, final String partitionNam
.findFirst()
.orElseThrow(() -> new IllegalArgumentException("Invalid Partition: " + partitionName));

final File indexDir = new File(storageDir, "lucene-8-index-" + earliestTimestamp);
final File indexDir = new File(storageDir, "lucene-9-index-" + earliestTimestamp);
return indexDir;
}

Expand Down
31 changes: 1 addition & 30 deletions ...ce-repository/src/main/java/org/apache/nifi/provenance/index/lucene/LuceneEventIndex.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -173,7 +173,7 @@ public void initialize(final EventStore eventStore) {

private void triggerReindexOfDefunctIndices() {
final ExecutorService rebuildIndexExecutor = Executors.newScheduledThreadPool(2, new NamedThreadFactory("Rebuild Defunct Provenance Indices", true));
final List<File> allIndexDirectories = directoryManager.getAllIndexDirectories(true, true);
final List<File> allIndexDirectories = directoryManager.getAllIndexDirectories();
allIndexDirectories.sort(DirectoryUtils.OLDEST_INDEX_FIRST);
final List<File> defunctIndices = detectDefunctIndices(allIndexDirectories);

Expand All @@ -182,14 +182,7 @@ private void triggerReindexOfDefunctIndices() {

for (final File defunctIndex : defunctIndices) {
try {
if (isLucene4IndexPresent(defunctIndex)) {
logger.info("Encountered Lucene 8 index {} and also the corresponding Lucene 4 index; will only trigger rebuilding of one directory.", defunctIndex);
rebuildCount.incrementAndGet();
continue;
}

logger.info("Determined that Lucene Index Directory {} is defunct. Will destroy and rebuild index", defunctIndex);

final Tuple<Long, Long> timeRange = getTimeRange(defunctIndex, allIndexDirectories);
rebuildIndexExecutor.submit(new MigrateDefunctIndex(defunctIndex, indexManager, directoryManager, timeRange.getKey(), timeRange.getValue(),
eventStore, eventReporter, eventConverter, rebuildCount, totalCount));
Expand All @@ -208,28 +201,6 @@ private void triggerReindexOfDefunctIndices() {
}
}

/**
* Returns true if the given Index Directory appears to be a later version of the Lucene Index and there also exists a version 4 Lucene
* Index for the same timestamp
* @param indexDirectory the index directory to check
* @return <code>true</code> if there exists a Lucene 4 index directory for the same timestamp, <code>false</code> otherwise
*/
private boolean isLucene4IndexPresent(final File indexDirectory) {
final String indexName = indexDirectory.getName();
if (indexName.contains("lucene-8-")) {
final int prefixEnd = indexName.indexOf("index-");
final String oldIndexName = indexName.substring(prefixEnd);

final File oldIndexFile = new File(indexDirectory.getParentFile(), oldIndexName);
final boolean oldIndexExists = oldIndexFile.exists();
if (oldIndexExists) {
return true;
}
}

return false;
}

private void triggerCacheWarming() {
final Optional<Integer> warmCacheMinutesOption = config.getWarmCacheFrequencyMinutes();
if (warmCacheMinutesOption.isPresent() && warmCacheMinutesOption.get() > 0) {
Expand Down
13 changes: 7 additions & 6 deletions ...repository/src/main/java/org/apache/nifi/provenance/index/lucene/MigrateDefunctIndex.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -36,8 +36,8 @@
import java.util.concurrent.atomic.AtomicInteger;

public class MigrateDefunctIndex implements Runnable {
private static final String TEMP_FILENAME_PREFIX = "temp-lucene-8-";
private static final String MIGRATED_FILENAME_PREFIX = "lucene-8-";
private static final String TEMP_FILENAME_PREFIX = "temp-lucene-9-";
private static final String MIGRATED_FILENAME_PREFIX = "lucene-9-";
private static final Logger logger = LoggerFactory.getLogger(MigrateDefunctIndex.class);

private final File indexDirectory;
Expand Down Expand Up @@ -71,8 +71,9 @@ public MigrateDefunctIndex(final File indexDirectory, final IndexManager indexMa

@Override
public void run() {
final File tempIndexDir = new File(indexDirectory.getParentFile(), TEMP_FILENAME_PREFIX + indexDirectory.getName());
final File migratedIndexDir = new File(indexDirectory.getParentFile(), MIGRATED_FILENAME_PREFIX + indexDirectory.getName());
final String indexDirSuffix = indexDirectory.getName().substring(MIGRATED_FILENAME_PREFIX.length());
final File tempIndexDir = new File(indexDirectory.getParentFile(), TEMP_FILENAME_PREFIX + indexDirSuffix);
final File migratedIndexDir = new File(indexDirectory.getParentFile(), MIGRATED_FILENAME_PREFIX + indexDirSuffix);

final boolean preconditionsMet = verifyPreconditions(tempIndexDir, migratedIndexDir);
if (!preconditionsMet) {
Expand Down Expand Up @@ -101,7 +102,7 @@ private boolean verifyPreconditions(final File tempIndexDir, final File migrated
try {
FileUtils.deleteFile(tempIndexDir, true);
} catch (final Exception e) {
logger.error("Attempted to rebuild index for {} but there already exists a temporary Lucene 8 index at {}. " +
logger.error("Attempted to rebuild index for {} but there already exists a temporary Lucene 9 index at {}. " +
"Attempted to delete existing temp directory but failed. This index will not be rebuilt.", tempIndexDir, e);
return false;
}
Expand All @@ -112,7 +113,7 @@ private boolean verifyPreconditions(final File tempIndexDir, final File migrated
try {
FileUtils.deleteFile(migratedIndexDir, true);
} catch (final Exception e) {
logger.error("Attempted to rebuild index for {} but there already exists a Lucene 8 index at {}. " +
logger.error("Attempted to rebuild index for {} but there already exists a Lucene 9 index at {}. " +
"Attempted to delete existing Lucene 8 directory but failed. This index will not be rebuilt.", migratedIndexDir, e);
return false;
}
Expand Down
9 changes: 4 additions & 5 deletions ...e-repository/src/test/java/org/apache/nifi/provenance/StartupAgainstOldLuceneIndexIT.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -35,7 +35,6 @@
import static org.junit.jupiter.api.Assertions.assertTrue;

/**
* With NiFi 1.10.0 (?) we changed from Lucene 4.x to Lucene 8.x
* This test is intended to ensure that we can properly startup even when pointing to a Provenance
* Repository that was created against the old Lucene.
*/
Expand All @@ -59,11 +58,11 @@ private void testStartup(final boolean createTempDirectory, final boolean create
final File tempDir = new File("target/" + UUID.randomUUID());

copy(existingRepo, tempDir);
final File oldIndexDir = new File(tempDir, "index-1554304717707");
final File oldIndexDir = new File(tempDir, "lucene-4-index-1554304717707");
assertTrue(oldIndexDir.exists());

if (createTempDirectory) {
final File tempIndexDir = new File(tempDir, "temp-lucene-8-index-1554304717707");
final File tempIndexDir = new File(tempDir, "temp-lucene-9-index-1554304717707");
assertTrue(tempIndexDir.mkdirs());

final File dummyFile = new File(tempIndexDir, "_0.fdt");
Expand All @@ -73,7 +72,7 @@ private void testStartup(final boolean createTempDirectory, final boolean create
}

if (createMigratedDirectory) {
final File migratedDirectory = new File(tempDir, "lucene-8-index-1554304717707");
final File migratedDirectory = new File(tempDir, "lucene-9-index-1554304717707");
assertTrue(migratedDirectory.mkdirs());

final File dummyFile = new File(migratedDirectory, "_0.fdt");
Expand All @@ -99,7 +98,7 @@ private void testStartup(final boolean createTempDirectory, final boolean create

assertFalse(oldIndexDir.exists());

final File newIndexDir = new File(tempDir, "lucene-8-index-1554304717707");
final File newIndexDir = new File(tempDir, "lucene-9-index-1554304717707");
while (!newIndexDir.exists()) {
Thread.sleep(5L);
}
Expand Down
26 changes: 13 additions & 13 deletions ...tory/src/test/java/org/apache/nifi/provenance/index/lucene/TestIndexDirectoryManager.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -42,8 +42,8 @@ public void testGetDirectoriesIncludesMatchingTimestampPlusOne() {

final List<File> directories = IndexDirectoryManager.getDirectories(1000L, 1001L, locations);
assertEquals(2, directories.size());
assertTrue(directories.contains(new File("lucene-8-index-999")));
assertTrue(directories.contains(new File("lucene-8-index-1002")));
assertTrue(directories.contains(new File("lucene-9-index-999")));
assertTrue(directories.contains(new File("lucene-9-index-1002")));
}

@Test
Expand All @@ -53,10 +53,10 @@ public void testGetDirectoriesOnlyObtainsDirectoriesForDesiredPartition() {
final File storageDir1 = config.getStorageDirectories().get("1");
final File storageDir2 = config.getStorageDirectories().get("2");

final File index1 = new File(storageDir1, "lucene-8-index-1");
final File index2 = new File(storageDir1, "lucene-8-index-2");
final File index3 = new File(storageDir2, "lucene-8-index-3");
final File index4 = new File(storageDir2, "lucene-8-index-4");
final File index1 = new File(storageDir1, "lucene-9-index-1");
final File index2 = new File(storageDir1, "lucene-9-index-2");
final File index3 = new File(storageDir2, "lucene-9-index-3");
final File index4 = new File(storageDir2, "lucene-9-index-4");

final File[] allIndices = new File[] {index1, index2, index3, index4};
for (final File file : allIndices) {
Expand Down Expand Up @@ -93,10 +93,10 @@ public void testActiveIndexNotLostWhenSizeExceeded() throws IOException {
final File storageDir1 = config.getStorageDirectories().get("1");
final File storageDir2 = config.getStorageDirectories().get("2");

final File index1 = new File(storageDir1, "lucene-8-index-1");
final File index2 = new File(storageDir1, "lucene-8-index-2");
final File index3 = new File(storageDir2, "lucene-8-index-3");
final File index4 = new File(storageDir2, "lucene-8-index-4");
final File index1 = new File(storageDir1, "lucene-9-index-1");
final File index2 = new File(storageDir1, "lucene-9-index-2");
final File index3 = new File(storageDir2, "lucene-9-index-3");
final File index4 = new File(storageDir2, "lucene-9-index-4");

final File[] allIndices = new File[] {index1, index2, index3, index4};
for (final File file : allIndices) {
Expand Down Expand Up @@ -138,8 +138,8 @@ public void testGetDirectoriesBefore() {

final File storageDir = config.getStorageDirectories().get("1");

final File index1 = new File(storageDir, "lucene-8-index-1");
final File index2 = new File(storageDir, "lucene-8-index-2");
final File index1 = new File(storageDir, "lucene-9-index-1");
final File index2 = new File(storageDir, "lucene-9-index-2");

final File[] allIndices = new File[] {index1, index2};
for (final File file : allIndices) {
Expand Down Expand Up @@ -171,7 +171,7 @@ private IndexLocation createLocation(final long timestamp) {
}

private IndexLocation createLocation(final long timestamp, final String partitionName) {
return new IndexLocation(new File("lucene-8-index-" + timestamp), timestamp, partitionName);
return new IndexLocation(new File("lucene-9-index-" + timestamp), timestamp, partitionName);
}

private RepositoryConfiguration createConfig(final int partitions) {
Expand Down
32 changes: 16 additions & 16 deletions ...epository/src/test/java/org/apache/nifi/provenance/index/lucene/TestLuceneEventIndex.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -83,22 +83,22 @@ public void testGetTimeRange() {
final long now = System.currentTimeMillis();

final List<File> indexFiles = new ArrayList<>();
indexFiles.add(new File("index-1000"));
indexFiles.add(new File("lucene-8-index-3000"));
indexFiles.add(new File("index-4000"));
indexFiles.add(new File("index-5000"));
indexFiles.add(new File("lucene-8-index-6000"));
indexFiles.add(new File("index-7000"));

assertEquals(new Tuple<>(1000L, 3000L), LuceneEventIndex.getTimeRange(new File("index-1000"), indexFiles));

assertEquals(new Tuple<>(3000L, 4000L), LuceneEventIndex.getTimeRange(new File("lucene-8-index-3000"), indexFiles));
assertEquals(new Tuple<>(4000L, 5000L), LuceneEventIndex.getTimeRange(new File("index-4000"), indexFiles));
assertEquals(new Tuple<>(5000L, 6000L), LuceneEventIndex.getTimeRange(new File("index-5000"), indexFiles));
assertEquals(new Tuple<>(6000L, 7000L), LuceneEventIndex.getTimeRange(new File("lucene-8-index-6000"), indexFiles));

assertEquals(7000L, LuceneEventIndex.getTimeRange(new File("index-7000"), indexFiles).getKey().longValue());
assertTrue(LuceneEventIndex.getTimeRange(new File("index-7000"), indexFiles).getValue() >= now);
indexFiles.add(new File("lucene-4-index-1000"));
indexFiles.add(new File("lucene-9-index-3000"));
indexFiles.add(new File("lucene-4-index-4000"));
indexFiles.add(new File("lucene-4-index-5000"));
indexFiles.add(new File("lucene-9-index-6000"));
indexFiles.add(new File("lucene-4-index-7000"));

assertEquals(new Tuple<>(1000L, 3000L), LuceneEventIndex.getTimeRange(new File("lucene-4-index-1000"), indexFiles));

assertEquals(new Tuple<>(3000L, 4000L), LuceneEventIndex.getTimeRange(new File("lucene-9-index-3000"), indexFiles));
assertEquals(new Tuple<>(4000L, 5000L), LuceneEventIndex.getTimeRange(new File("lucene-4-index-4000"), indexFiles));
assertEquals(new Tuple<>(5000L, 6000L), LuceneEventIndex.getTimeRange(new File("lucene-4-index-5000"), indexFiles));
assertEquals(new Tuple<>(6000L, 7000L), LuceneEventIndex.getTimeRange(new File("lucene-9-index-6000"), indexFiles));

assertEquals(7000L, LuceneEventIndex.getTimeRange(new File("lucene-4-index-7000"), indexFiles).getKey().longValue());
assertTrue(LuceneEventIndex.getTimeRange(new File("lucene-4-index-7000"), indexFiles).getValue() >= now);

}

Expand Down
0 ...ne-4-prov-repo/index-1554304717707/_0.fdt → ...-repo/lucene-4-index-1554304717707/_0.fdt
View file
Open in desktop
File renamed without changes.
0 ...ne-4-prov-repo/index-1554304717707/_0.fdx → ...-repo/lucene-4-index-1554304717707/_0.fdx
View file
Open in desktop
File renamed without changes.
0 ...ne-4-prov-repo/index-1554304717707/_1.fdt → ...-repo/lucene-4-index-1554304717707/_1.fdt
View file
Open in desktop
File renamed without changes.
0 ...ne-4-prov-repo/index-1554304717707/_1.fdx → ...-repo/lucene-4-index-1554304717707/_1.fdx
View file
Open in desktop
File renamed without changes.
10 changes: 0 additions & 10 deletions nifi-extension-bundles/nifi-provenance-repository-bundle/pom.xml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -46,16 +46,6 @@
<artifactId>lucene-core</artifactId>
<version>${lucene.version}</version>
</dependency>
<dependency>
<groupId>org.apache.lucene</groupId>
<artifactId>lucene-analyzers-common</artifactId>
<version>${lucene.version}</version>
</dependency>
<dependency>
<groupId>org.apache.lucene</groupId>
<artifactId>lucene-queryparser</artifactId>
<version>${lucene.version}</version>
</dependency>
<dependency>
<groupId>org.apache.lucene</groupId>
<artifactId>lucene-backward-codecs</artifactId>
Expand Down