diff --git a/advanced/neo4j-advanced/LICENSES.txt b/advanced/neo4j-advanced/LICENSES.txt index e3437d2e2eaf6..d575fc223f6ea 100644 --- a/advanced/neo4j-advanced/LICENSES.txt +++ b/advanced/neo4j-advanced/LICENSES.txt @@ -7,6 +7,7 @@ Apache Software License, Version 2.0 ConcurrentLinkedHashMap Lucene Common Analyzers Lucene Core + Lucene Memory Lucene QueryParsers opencsv parboiled-core diff --git a/advanced/neo4j-advanced/NOTICE.txt b/advanced/neo4j-advanced/NOTICE.txt index 757f4b5367ef9..00b5c84942ab8 100644 --- a/advanced/neo4j-advanced/NOTICE.txt +++ b/advanced/neo4j-advanced/NOTICE.txt @@ -29,6 +29,7 @@ Apache Software License, Version 2.0 ConcurrentLinkedHashMap Lucene Common Analyzers Lucene Core + Lucene Memory Lucene QueryParsers opencsv parboiled-core diff --git a/advanced/server-advanced/LICENSES.txt b/advanced/server-advanced/LICENSES.txt index f44165accf8d3..9c0f539a61219 100644 --- a/advanced/server-advanced/LICENSES.txt +++ b/advanced/server-advanced/LICENSES.txt @@ -24,6 +24,7 @@ Apache Software License, Version 2.0 Jetty :: XML utilities Lucene Common Analyzers Lucene Core + Lucene Memory Lucene QueryParsers Netty/All-in-One opencsv diff --git a/advanced/server-advanced/NOTICE.txt b/advanced/server-advanced/NOTICE.txt index 34b16d5001e21..d1810c958dac8 100644 --- a/advanced/server-advanced/NOTICE.txt +++ b/advanced/server-advanced/NOTICE.txt @@ -46,6 +46,7 @@ Apache Software License, Version 2.0 Jetty :: XML utilities Lucene Common Analyzers Lucene Core + Lucene Memory Lucene QueryParsers Netty/All-in-One opencsv diff --git a/community/bolt/LICENSES.txt b/community/bolt/LICENSES.txt index ef8505591c9bf..ef081193880d6 100644 --- a/community/bolt/LICENSES.txt +++ b/community/bolt/LICENSES.txt @@ -7,6 +7,7 @@ Apache Software License, Version 2.0 ConcurrentLinkedHashMap Lucene Common Analyzers Lucene Core + Lucene Memory Lucene QueryParsers Netty/All-in-One opencsv diff --git a/community/bolt/NOTICE.txt b/community/bolt/NOTICE.txt index 2be6dc53bca2e..1b861f0957aa7 100644 --- a/community/bolt/NOTICE.txt +++ b/community/bolt/NOTICE.txt @@ -30,6 +30,7 @@ Apache Software License, Version 2.0 ConcurrentLinkedHashMap Lucene Common Analyzers Lucene Core + Lucene Memory Lucene QueryParsers Netty/All-in-One opencsv diff --git a/community/consistency-check-legacy/LICENSES.txt b/community/consistency-check-legacy/LICENSES.txt index 1a8322c3e2ff5..e5992a36c4207 100644 --- a/community/consistency-check-legacy/LICENSES.txt +++ b/community/consistency-check-legacy/LICENSES.txt @@ -6,6 +6,7 @@ Apache Software License, Version 2.0 Apache Commons Lang Lucene Common Analyzers Lucene Core + Lucene Memory Lucene QueryParsers ------------------------------------------------------------------------------ diff --git a/community/consistency-check-legacy/NOTICE.txt b/community/consistency-check-legacy/NOTICE.txt index 411e529abf3b2..db43e3d869ee0 100644 --- a/community/consistency-check-legacy/NOTICE.txt +++ b/community/consistency-check-legacy/NOTICE.txt @@ -29,5 +29,6 @@ Apache Software License, Version 2.0 Apache Commons Lang Lucene Common Analyzers Lucene Core + Lucene Memory Lucene QueryParsers diff --git a/community/consistency-check/LICENSES.txt b/community/consistency-check/LICENSES.txt index 1a8322c3e2ff5..e5992a36c4207 100644 --- a/community/consistency-check/LICENSES.txt +++ b/community/consistency-check/LICENSES.txt @@ -6,6 +6,7 @@ Apache Software License, Version 2.0 Apache Commons Lang Lucene Common Analyzers Lucene Core + Lucene Memory Lucene QueryParsers ------------------------------------------------------------------------------ diff --git a/community/consistency-check/NOTICE.txt b/community/consistency-check/NOTICE.txt index 411e529abf3b2..db43e3d869ee0 100644 --- a/community/consistency-check/NOTICE.txt +++ b/community/consistency-check/NOTICE.txt @@ -29,5 +29,6 @@ Apache Software License, Version 2.0 Apache Commons Lang Lucene Common Analyzers Lucene Core + Lucene Memory Lucene QueryParsers diff --git a/community/cypher/cypher/LICENSES.txt b/community/cypher/cypher/LICENSES.txt index c1aaf42a02fb2..fb704217d9667 100644 --- a/community/cypher/cypher/LICENSES.txt +++ b/community/cypher/cypher/LICENSES.txt @@ -6,6 +6,7 @@ Apache Software License, Version 2.0 ConcurrentLinkedHashMap Lucene Common Analyzers Lucene Core + Lucene Memory Lucene QueryParsers opencsv parboiled-core diff --git a/community/cypher/cypher/NOTICE.txt b/community/cypher/cypher/NOTICE.txt index b569fd4965858..969473c73b211 100644 --- a/community/cypher/cypher/NOTICE.txt +++ b/community/cypher/cypher/NOTICE.txt @@ -29,6 +29,7 @@ Apache Software License, Version 2.0 ConcurrentLinkedHashMap Lucene Common Analyzers Lucene Core + Lucene Memory Lucene QueryParsers opencsv parboiled-core diff --git a/community/embedded-examples/LICENSES.txt b/community/embedded-examples/LICENSES.txt index ad85a41a453cb..383abdd386dfc 100644 --- a/community/embedded-examples/LICENSES.txt +++ b/community/embedded-examples/LICENSES.txt @@ -7,6 +7,7 @@ Apache Software License, Version 2.0 ConcurrentLinkedHashMap Lucene Common Analyzers Lucene Core + Lucene Memory Lucene QueryParsers opencsv parboiled-core diff --git a/community/embedded-examples/NOTICE.txt b/community/embedded-examples/NOTICE.txt index 037598f22a8be..33c7bd1b34b41 100644 --- a/community/embedded-examples/NOTICE.txt +++ b/community/embedded-examples/NOTICE.txt @@ -30,6 +30,7 @@ Apache Software License, Version 2.0 ConcurrentLinkedHashMap Lucene Common Analyzers Lucene Core + Lucene Memory Lucene QueryParsers opencsv parboiled-core diff --git a/community/graph-algo/LICENSES.txt b/community/graph-algo/LICENSES.txt index fd96cba2c36a7..de7f66b3b397e 100644 --- a/community/graph-algo/LICENSES.txt +++ b/community/graph-algo/LICENSES.txt @@ -4,6 +4,8 @@ libraries. For an overview of the licenses see the NOTICE.txt file. ------------------------------------------------------------------------------ Apache Software License, Version 2.0 Apache Commons Lang + Lucene Core + Lucene Memory ------------------------------------------------------------------------------ Apache License diff --git a/community/graph-algo/NOTICE.txt b/community/graph-algo/NOTICE.txt index fc42d74a24f82..8c2e5c7b849bd 100644 --- a/community/graph-algo/NOTICE.txt +++ b/community/graph-algo/NOTICE.txt @@ -27,4 +27,6 @@ Third-party licenses Apache Software License, Version 2.0 Apache Commons Lang + Lucene Core + Lucene Memory diff --git a/community/graph-matching/LICENSES.txt b/community/graph-matching/LICENSES.txt index fd96cba2c36a7..de7f66b3b397e 100644 --- a/community/graph-matching/LICENSES.txt +++ b/community/graph-matching/LICENSES.txt @@ -4,6 +4,8 @@ libraries. For an overview of the licenses see the NOTICE.txt file. ------------------------------------------------------------------------------ Apache Software License, Version 2.0 Apache Commons Lang + Lucene Core + Lucene Memory ------------------------------------------------------------------------------ Apache License diff --git a/community/graph-matching/NOTICE.txt b/community/graph-matching/NOTICE.txt index fc42d74a24f82..8c2e5c7b849bd 100644 --- a/community/graph-matching/NOTICE.txt +++ b/community/graph-matching/NOTICE.txt @@ -27,4 +27,6 @@ Third-party licenses Apache Software License, Version 2.0 Apache Commons Lang + Lucene Core + Lucene Memory diff --git a/community/graphviz/LICENSES.txt b/community/graphviz/LICENSES.txt index fd96cba2c36a7..de7f66b3b397e 100644 --- a/community/graphviz/LICENSES.txt +++ b/community/graphviz/LICENSES.txt @@ -4,6 +4,8 @@ libraries. For an overview of the licenses see the NOTICE.txt file. ------------------------------------------------------------------------------ Apache Software License, Version 2.0 Apache Commons Lang + Lucene Core + Lucene Memory ------------------------------------------------------------------------------ Apache License diff --git a/community/graphviz/NOTICE.txt b/community/graphviz/NOTICE.txt index fc42d74a24f82..8c2e5c7b849bd 100644 --- a/community/graphviz/NOTICE.txt +++ b/community/graphviz/NOTICE.txt @@ -27,4 +27,6 @@ Third-party licenses Apache Software License, Version 2.0 Apache Commons Lang + Lucene Core + Lucene Memory diff --git a/community/import-tool/LICENSES.txt b/community/import-tool/LICENSES.txt index 95f66d4785419..059a67ba8e64e 100644 --- a/community/import-tool/LICENSES.txt +++ b/community/import-tool/LICENSES.txt @@ -5,6 +5,7 @@ libraries. For an overview of the licenses see the NOTICE.txt file. Apache Software License, Version 2.0 Lucene Common Analyzers Lucene Core + Lucene Memory Lucene QueryParsers ------------------------------------------------------------------------------ diff --git a/community/import-tool/NOTICE.txt b/community/import-tool/NOTICE.txt index c336369cafed4..7b4d28500069a 100644 --- a/community/import-tool/NOTICE.txt +++ b/community/import-tool/NOTICE.txt @@ -28,5 +28,6 @@ Third-party licenses Apache Software License, Version 2.0 Lucene Common Analyzers Lucene Core + Lucene Memory Lucene QueryParsers diff --git a/community/kernel/LICENSES.txt b/community/kernel/LICENSES.txt index fd96cba2c36a7..de7f66b3b397e 100644 --- a/community/kernel/LICENSES.txt +++ b/community/kernel/LICENSES.txt @@ -4,6 +4,8 @@ libraries. For an overview of the licenses see the NOTICE.txt file. ------------------------------------------------------------------------------ Apache Software License, Version 2.0 Apache Commons Lang + Lucene Core + Lucene Memory ------------------------------------------------------------------------------ Apache License diff --git a/community/kernel/NOTICE.txt b/community/kernel/NOTICE.txt index fc42d74a24f82..8c2e5c7b849bd 100644 --- a/community/kernel/NOTICE.txt +++ b/community/kernel/NOTICE.txt @@ -27,4 +27,6 @@ Third-party licenses Apache Software License, Version 2.0 Apache Commons Lang + Lucene Core + Lucene Memory diff --git a/community/kernel/pom.xml b/community/kernel/pom.xml index daf1df331b580..71b8a14aa4e25 100644 --- a/community/kernel/pom.xml +++ b/community/kernel/pom.xml @@ -294,6 +294,11 @@ the relevant Commercial Agreement. neo4j-logging ${project.version} + + org.neo4j + neo4j-lucene-upgrade + ${project.version} + diff --git a/community/kernel/src/main/java/org/neo4j/graphdb/index/IndexImplementation.java b/community/kernel/src/main/java/org/neo4j/graphdb/index/IndexImplementation.java index 59d0008b0dac7..52f98017f79da 100644 --- a/community/kernel/src/main/java/org/neo4j/graphdb/index/IndexImplementation.java +++ b/community/kernel/src/main/java/org/neo4j/graphdb/index/IndexImplementation.java @@ -35,6 +35,9 @@ */ public interface IndexImplementation extends Lifecycle { + + File getStoreDirectory( File storeDir ); + /** * Returns a {@link LegacyIndexProviderTransaction} that keeps transaction state for all * indexes for a given provider in a transaction. diff --git a/community/kernel/src/main/java/org/neo4j/kernel/NeoStoreDataSource.java b/community/kernel/src/main/java/org/neo4j/kernel/NeoStoreDataSource.java index 5636ad7dfe03d..9705986da3bf1 100644 --- a/community/kernel/src/main/java/org/neo4j/kernel/NeoStoreDataSource.java +++ b/community/kernel/src/main/java/org/neo4j/kernel/NeoStoreDataSource.java @@ -90,6 +90,7 @@ import org.neo4j.kernel.impl.store.StoreId; import org.neo4j.kernel.impl.store.UnderlyingStorageException; import org.neo4j.kernel.impl.store.record.SchemaRule; +import org.neo4j.kernel.impl.storemigration.LegacyIndexMigrator; import org.neo4j.kernel.impl.storemigration.StoreMigrator; import org.neo4j.kernel.impl.storemigration.StoreUpgrader; import org.neo4j.kernel.impl.storemigration.StoreVersionCheck; @@ -314,7 +315,7 @@ boolean applicable( DiagnosticsPhase phase ) private Dependencies dependencies; private LifeSupport life; - private SchemaIndexProvider indexProvider; + private SchemaIndexProvider schemaIndexProvider; private File storeDir; private boolean readOnly; @@ -444,7 +445,7 @@ public void start() throws IOException dependencies = new Dependencies(); life = new LifeSupport(); - indexProvider = dependencyResolver.resolveDependency( SchemaIndexProvider.class, + schemaIndexProvider = dependencyResolver.resolveDependency( SchemaIndexProvider.class, HighestSelectionStrategy.getInstance() ); LabelScanStoreProvider labelScanStoreProvider = @@ -464,7 +465,8 @@ public void start() throws IOException life.add( new Lifecycle.Delegate( Lifecycles.multiple( indexProviders.values() ) ) ); // Upgrade the store before we begin - upgradeStore( storeDir, storeMigrationProcess, indexProvider, labelScanStoreProvider ); + upgradeStore( storeDir, storeMigrationProcess, schemaIndexProvider, labelScanStoreProvider, indexProviders, + logProvider ); // Build all modules and their services StorageEngine storageEngine = null; @@ -572,11 +574,13 @@ public void start() throws IOException // By doing this sequence of method calls we can ensure that no dependency cycles exist, and get a clearer view // of the dependency tree, starting at the bottom private void upgradeStore( File storeDir, StoreUpgrader storeMigrationProcess, SchemaIndexProvider indexProvider, - LabelScanStoreProvider labelScanStoreProvider ) + LabelScanStoreProvider labelScanStoreProvider, Map indexProviders, + LogProvider logProvider ) { UpgradableDatabase upgradableDatabase = new UpgradableDatabase( fs, new StoreVersionCheck( pageCache ), new LegacyStoreVersionCheck( fs ) ); storeMigrationProcess.addParticipant( indexProvider.storeMigrationParticipant( fs, pageCache ) ); + storeMigrationProcess.addParticipant( new LegacyIndexMigrator( fs, indexProviders, logProvider) ); storeMigrationProcess.addParticipant( storeMigrator ); storeMigrationProcess.migrateIfNeeded( storeDir, upgradableDatabase, indexProvider, labelScanStoreProvider ); } @@ -592,7 +596,7 @@ private StorageEngine buildStorageEngine( return life.add( new RecordStorageEngine( storeDir, config, idGeneratorFactory, pageCache, fs, logProvider, propertyKeyTokenHolder, labelTokens, relationshipTypeTokens, schemaStateChangeCallback, constraintSemantics, scheduler, - tokenNameLookup, lockService, indexProvider, indexingServiceMonitor, databaseHealth, + tokenNameLookup, lockService, schemaIndexProvider, indexingServiceMonitor, databaseHealth, labelScanStore, legacyIndexProviderLookup, indexConfigStore ) ); } diff --git a/community/kernel/src/main/java/org/neo4j/kernel/impl/storemigration/LegacyIndexMigrator.java b/community/kernel/src/main/java/org/neo4j/kernel/impl/storemigration/LegacyIndexMigrator.java new file mode 100644 index 0000000000000..b4fd5a9ae4fc5 --- /dev/null +++ b/community/kernel/src/main/java/org/neo4j/kernel/impl/storemigration/LegacyIndexMigrator.java @@ -0,0 +1,136 @@ +/* + * Copyright (c) 2002-2015 "Neo Technology," + * Network Engine for Objects in Lund AB [http://neotechnology.com] + * + * This file is part of Neo4j. + * + * Neo4j is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package org.neo4j.kernel.impl.storemigration; + +import java.io.File; +import java.io.IOException; +import java.nio.file.Path; +import java.util.Map; + +import org.neo4j.graphdb.index.IndexImplementation; +import org.neo4j.logging.Log; +import org.neo4j.logging.LogProvider; +import org.neo4j.upgrade.lucene.LegacyIndexMigrationException; +import org.neo4j.upgrade.lucene.LuceneLegacyIndexUpgrader; +import org.neo4j.io.fs.FileSystemAbstraction; +import org.neo4j.kernel.api.index.SchemaIndexProvider; +import org.neo4j.kernel.impl.api.scan.LabelScanStoreProvider; +import org.neo4j.kernel.impl.storemigration.legacystore.v19.Legacy19Store; +import org.neo4j.kernel.impl.storemigration.legacystore.v20.Legacy20Store; +import org.neo4j.kernel.impl.storemigration.legacystore.v21.Legacy21Store; +import org.neo4j.kernel.impl.storemigration.legacystore.v22.Legacy22Store; +import org.neo4j.kernel.impl.storemigration.legacystore.v23.Legacy23Store; + +/** + * Migrates legacy lucene indexes between different neo4j versions. + * Participates in store upgrade as one of the migration participants. + */ +public class LegacyIndexMigrator implements StoreMigrationParticipant +{ + private static final String LUCENE_LEGACY_INDEX_PROVIDER_NAME = "lucene"; + private Map indexProviders; + private final FileSystemAbstraction fileSystem; + private File migrationLegacyIndexesRoot; + private File originalLegacyIndexesRoot; + private Log log; + private boolean legacyIndexMigrated = false; + + public LegacyIndexMigrator( FileSystemAbstraction fileSystem, Map indexProviders, + LogProvider logProvider ) + { + this.fileSystem = fileSystem; + this.indexProviders = indexProviders; + this.log = logProvider.getLog( getClass() ); + } + + @Override + public void migrate( File storeDir, File migrationDir, SchemaIndexProvider schemaIndexProvider, + LabelScanStoreProvider labelScanStoreProvider, String versionToMigrateFrom ) throws IOException + { + IndexImplementation indexImplementation = indexProviders.get( LUCENE_LEGACY_INDEX_PROVIDER_NAME ); + if ( indexImplementation != null ) + { + switch ( versionToMigrateFrom ) + { + case Legacy23Store.LEGACY_VERSION: + case Legacy22Store.LEGACY_VERSION: + case Legacy21Store.LEGACY_VERSION: + case Legacy20Store.LEGACY_VERSION: + case Legacy19Store.LEGACY_VERSION: + originalLegacyIndexesRoot = indexImplementation.getStoreDirectory( storeDir ); + migrationLegacyIndexesRoot = indexImplementation.getStoreDirectory( migrationDir ); + migrateLegacyIndexes(); + legacyIndexMigrated = true; + break; + default: + throw new IllegalStateException( "Unknown version to upgrade from: " + versionToMigrateFrom ); + } + } + else + { + log.debug( "Lucene index provider not found, nothing to migrate." ); + } + } + + @Override + public void moveMigratedFiles( File migrationDir, File storeDir, String versionToMigrateFrom ) + throws IOException + { + if ( legacyIndexMigrated ) + { + fileSystem.deleteRecursively( originalLegacyIndexesRoot ); + fileSystem.copyRecursively( migrationLegacyIndexesRoot, originalLegacyIndexesRoot ); + } + } + + private void migrateLegacyIndexes() throws IOException + { + try + { + fileSystem.copyRecursively( originalLegacyIndexesRoot, migrationLegacyIndexesRoot ); + Path indexRootPath = migrationLegacyIndexesRoot.toPath(); + LuceneLegacyIndexUpgrader indexUpgrader = createLuceneLegacyIndexUpgrader( indexRootPath ); + indexUpgrader.upgradeIndexes(); + } + catch ( LegacyIndexMigrationException lime ) + { + log.error( "Migration of legacy indexes failed. Index: " + lime.getFailedIndexName() + " can't be " + + "migrated.", lime ); + throw new IOException( "Legacy index migration failed.", lime ); + } + } + + LuceneLegacyIndexUpgrader createLuceneLegacyIndexUpgrader( Path indexRootPath ) + { + return new LuceneLegacyIndexUpgrader( indexRootPath ); + } + + @Override + public void rebuildCounts( File storeDir, String versionToMigrateFrom ) throws IOException + { + // nothing to do + } + + @Override + public void cleanup( File migrationDir ) throws IOException + { + // nothing to do + } +} diff --git a/community/kernel/src/main/java/org/neo4j/kernel/impl/storemigration/SchemaIndexMigrator.java b/community/kernel/src/main/java/org/neo4j/kernel/impl/storemigration/SchemaIndexMigrator.java index edfc934f85a1f..9cb9c9e3e7d26 100644 --- a/community/kernel/src/main/java/org/neo4j/kernel/impl/storemigration/SchemaIndexMigrator.java +++ b/community/kernel/src/main/java/org/neo4j/kernel/impl/storemigration/SchemaIndexMigrator.java @@ -31,9 +31,18 @@ import org.neo4j.kernel.impl.storemigration.legacystore.v22.Legacy22Store; import org.neo4j.kernel.impl.storemigration.legacystore.v23.Legacy23Store; +/** + * Migrates schema and label indexes between different neo4j versions. + * Participates in store upgrade as one of the migration participants. + *

+ * Since index format can be completely incompatible between version should be executed before {@link StoreMigrator} + */ public class SchemaIndexMigrator implements StoreMigrationParticipant { private final FileSystemAbstraction fileSystem; + private boolean deleteObsoleteIndexes = false; + private File labelIndexDirectory; + private File schemaIndexDirectory; public SchemaIndexMigrator( FileSystemAbstraction fileSystem ) { @@ -51,18 +60,23 @@ public void migrate( File storeDir, File migrationDir, SchemaIndexProvider schem case Legacy21Store.LEGACY_VERSION: case Legacy22Store.LEGACY_VERSION: case Legacy23Store.LEGACY_VERSION: - deleteIndexes( storeDir, schemaIndexProvider, labelScanStoreProvider ); + schemaIndexDirectory = schemaIndexProvider.getStoreDirectory( storeDir ); + labelIndexDirectory = labelScanStoreProvider.getStoreDirectory( storeDir ); + deleteObsoleteIndexes = true; break; default: throw new IllegalStateException( "Unknown version to upgrade from: " + versionToMigrateFrom ); } } - private void deleteIndexes( File storeDir, SchemaIndexProvider schemaIndexProvider, - LabelScanStoreProvider labelScanStoreProvider ) throws IOException + @Override + public void moveMigratedFiles( File migrationDir, File storeDir, String versionToUpgradeFrom ) throws IOException { - deleteIndexes( schemaIndexProvider.getStoreDirectory( storeDir ) ); - deleteIndexes( labelScanStoreProvider.getStoreDirectory( storeDir ) ); + if ( deleteObsoleteIndexes ) + { + deleteIndexes( schemaIndexDirectory ); + deleteIndexes( labelIndexDirectory ); + } } private void deleteIndexes( File indexRootDirectory ) throws IOException @@ -70,18 +84,15 @@ private void deleteIndexes( File indexRootDirectory ) throws IOException fileSystem.deleteRecursively( indexRootDirectory ); } - @Override - public void moveMigratedFiles( File migrationDir, File storeDir, String versionToUpgradeFrom ) throws IOException - { // nothing to do - } - @Override public void rebuildCounts( File storeDir, String versionToMigrateFrom ) throws IOException - { // nothing to do + { + // nothing to do } @Override public void cleanup( File migrationDir ) throws IOException - { // nothing to do + { + // nothing to do } } diff --git a/community/kernel/src/main/java/org/neo4j/kernel/impl/storemigration/StoreMigrationParticipant.java b/community/kernel/src/main/java/org/neo4j/kernel/impl/storemigration/StoreMigrationParticipant.java index 959adf31ba85a..388f351a74cfc 100644 --- a/community/kernel/src/main/java/org/neo4j/kernel/impl/storemigration/StoreMigrationParticipant.java +++ b/community/kernel/src/main/java/org/neo4j/kernel/impl/storemigration/StoreMigrationParticipant.java @@ -38,7 +38,7 @@ public interface StoreMigrationParticipant * @param storeDir data to migrate. * @param migrationDir place to migrate to. * @param schemaIndexProvider The SchemaIndexProvider for the migrating database. - * @param labelScanStoreProvider The LabelScnaStoreProvider for the migrating database. + * @param labelScanStoreProvider The LabelScanStoreProvider for the migrating database. * @param versionToMigrateFrom the version to migrate from * @throws IOException if there was an error migrating. * @throws UnsatisfiedDependencyException if one or more dependencies were unsatisfied. diff --git a/community/kernel/src/test/java/org/neo4j/kernel/impl/index/DummyIndexImplementation.java b/community/kernel/src/test/java/org/neo4j/kernel/impl/index/DummyIndexImplementation.java index b0e31ba02c242..53416bfcb4a4a 100644 --- a/community/kernel/src/test/java/org/neo4j/kernel/impl/index/DummyIndexImplementation.java +++ b/community/kernel/src/test/java/org/neo4j/kernel/impl/index/DummyIndexImplementation.java @@ -188,6 +188,12 @@ private void mutate() } } + @Override + public File getStoreDirectory( File storeDir ) + { + return storeDir; + } + @Override public LegacyIndexProviderTransaction newTransaction( IndexCommandFactory commandFactory ) { diff --git a/community/kernel/src/test/java/org/neo4j/kernel/impl/storemigration/LegacyIndexMigratorTest.java b/community/kernel/src/test/java/org/neo4j/kernel/impl/storemigration/LegacyIndexMigratorTest.java new file mode 100644 index 0000000000000..e439e6da84e75 --- /dev/null +++ b/community/kernel/src/test/java/org/neo4j/kernel/impl/storemigration/LegacyIndexMigratorTest.java @@ -0,0 +1,162 @@ +/* + * Copyright (c) 2002-2015 "Neo Technology," + * Network Engine for Objects in Lund AB [http://neotechnology.com] + * + * This file is part of Neo4j. + * + * Neo4j is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package org.neo4j.kernel.impl.storemigration; + +import org.junit.Test; + +import java.io.File; +import java.io.IOException; +import java.nio.file.Path; +import java.util.HashMap; +import java.util.Map; + +import org.neo4j.graphdb.index.IndexImplementation; +import org.neo4j.io.fs.FileSystemAbstraction; +import org.neo4j.kernel.api.index.SchemaIndexProvider; +import org.neo4j.kernel.impl.api.scan.LabelScanStoreProvider; +import org.neo4j.kernel.impl.storemigration.legacystore.v23.Legacy23Store; +import org.neo4j.logging.Log; +import org.neo4j.logging.LogProvider; +import org.neo4j.upgrade.lucene.LegacyIndexMigrationException; +import org.neo4j.upgrade.lucene.LuceneLegacyIndexUpgrader; + +import static org.junit.Assert.fail; +import static org.mockito.Matchers.any; +import static org.mockito.Matchers.eq; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.reset; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.when; + +public class LegacyIndexMigratorTest +{ + + private FileSystemAbstraction fs = mock( FileSystemAbstraction.class ); + private LogProvider logProvider = mock( LogProvider.class ); + private SchemaIndexProvider schemaIndexProvider = mock( SchemaIndexProvider.class ); + private LabelScanStoreProvider labelScanStoreProvider = mock( LabelScanStoreProvider.class ); + private File storeDir = mock( File.class ); + private File migrationDir = mock( File.class ); + private File originalIndexStore = mock( File.class ); + private File migratedIndexStore = new File( "." ); + + @Test + public void transferOriginalDataToMigrationDirectory() throws IOException + { + HashMap indexProviders = getIndexProviders(); + LegacyIndexMigrator indexMigrator = new TestLegacyIndexMigrator( fs, indexProviders, logProvider, true ); + + indexMigrator.migrate( storeDir, migrationDir, schemaIndexProvider, labelScanStoreProvider, + Legacy23Store.LEGACY_VERSION ); + + verify( fs ).copyRecursively( originalIndexStore, migratedIndexStore ); + } + + @Test + public void transferMigratedIndexesToStoreDirectory() throws IOException + { + HashMap indexProviders = getIndexProviders(); + LegacyIndexMigrator indexMigrator = new TestLegacyIndexMigrator( fs, indexProviders, logProvider, true ); + + indexMigrator.migrate( storeDir, migrationDir, schemaIndexProvider, labelScanStoreProvider, + Legacy23Store.LEGACY_VERSION ); + reset( fs ); + + indexMigrator.moveMigratedFiles( migrationDir, storeDir, "any" ); + + verify( fs ).deleteRecursively( originalIndexStore ); + verify( fs ).copyRecursively( migratedIndexStore, originalIndexStore ); + } + + @Test + public void logErrorWithIndexNameOnIndexMigrationException() throws IOException + { + Log log = mock( Log.class ); + when( logProvider.getLog( TestLegacyIndexMigrator.class ) ).thenReturn( log ); + + HashMap indexProviders = getIndexProviders(); + try + { + LegacyIndexMigrator indexMigrator = new TestLegacyIndexMigrator( fs, indexProviders, logProvider, false ); + indexMigrator.migrate( storeDir, migrationDir, schemaIndexProvider, labelScanStoreProvider, + Legacy23Store.LEGACY_VERSION ); + + fail("Index migration should fail"); + } + catch ( IOException e ) + { + // ignored + } + + verify( log ).error( eq( "Migration of legacy indexes failed. Index: testIndex can't be migrated." ), + any( Throwable.class ) ); + + } + + private HashMap getIndexProviders() + { + HashMap indexProviders = new HashMap<>(); + IndexImplementation indexImplementation = mock( IndexImplementation.class ); + indexProviders.put( "lucene", indexImplementation ); + + when( indexImplementation.getStoreDirectory( storeDir ) ).thenReturn( originalIndexStore ); + when( indexImplementation.getStoreDirectory( migrationDir ) ).thenReturn( migratedIndexStore ); + return indexProviders; + } + + private class TestLegacyIndexMigrator extends LegacyIndexMigrator + { + + private boolean successfullMigration; + + public TestLegacyIndexMigrator( FileSystemAbstraction fileSystem, + Map indexProviders, LogProvider logProvider, boolean successfullMigration ) + { + super( fileSystem, indexProviders, logProvider ); + this.successfullMigration = successfullMigration; + } + + @Override + LuceneLegacyIndexUpgrader createLuceneLegacyIndexUpgrader( Path indexRootPath ) + { + return new HumbleLegacyIndexUpgrader( indexRootPath, successfullMigration ); + } + } + + private class HumbleLegacyIndexUpgrader extends LuceneLegacyIndexUpgrader + { + private boolean successfulMigration; + + public HumbleLegacyIndexUpgrader( Path indexRootPath, boolean successfulMigration ) + { + super( indexRootPath ); + this.successfulMigration = successfulMigration; + } + + @Override + public void upgradeIndexes() throws LegacyIndexMigrationException + { + if ( !successfulMigration ) + { + throw new LegacyIndexMigrationException( "testIndex", "Index migration failed", null ); + } + } + } +} \ No newline at end of file diff --git a/community/kernel/src/test/java/org/neo4j/kernel/impl/storemigration/SchemaIndexMigratorTest.java b/community/kernel/src/test/java/org/neo4j/kernel/impl/storemigration/SchemaIndexMigratorTest.java index 646287b99d813..ed5a394c804e1 100644 --- a/community/kernel/src/test/java/org/neo4j/kernel/impl/storemigration/SchemaIndexMigratorTest.java +++ b/community/kernel/src/test/java/org/neo4j/kernel/impl/storemigration/SchemaIndexMigratorTest.java @@ -46,7 +46,7 @@ public class SchemaIndexMigratorTest private final SchemaIndexMigrator migrator = new SchemaIndexMigrator( fs ); @Test - public void migrationRemovesAllIndexes() throws IOException + public void schemaAndLabelIndexesRemovedAfterSuccessfulMigration() throws IOException { when( schemaIndexProvider.getProviderDescriptor() ) .thenReturn( new SchemaIndexProvider.Descriptor( "key", "version" ) ); @@ -54,6 +54,8 @@ public void migrationRemovesAllIndexes() throws IOException migrator.migrate( storeDir, migrationDir, schemaIndexProvider, labelScanStoreProvider, Legacy23Store.LEGACY_VERSION ); + migrator.moveMigratedFiles( migrationDir, storeDir, Legacy23Store.LEGACY_VERSION ); + verify( fs ).deleteRecursively( schemaIndexProvider.getStoreDirectory( storeDir ) ); verify( fs ).deleteRecursively( labelScanStoreProvider.getStoreDirectory( storeDir ) ); } diff --git a/community/lucene-index/LICENSES.txt b/community/lucene-index/LICENSES.txt index 1a8322c3e2ff5..e5992a36c4207 100644 --- a/community/lucene-index/LICENSES.txt +++ b/community/lucene-index/LICENSES.txt @@ -6,6 +6,7 @@ Apache Software License, Version 2.0 Apache Commons Lang Lucene Common Analyzers Lucene Core + Lucene Memory Lucene QueryParsers ------------------------------------------------------------------------------ diff --git a/community/lucene-index/NOTICE.txt b/community/lucene-index/NOTICE.txt index 411e529abf3b2..db43e3d869ee0 100644 --- a/community/lucene-index/NOTICE.txt +++ b/community/lucene-index/NOTICE.txt @@ -29,5 +29,6 @@ Apache Software License, Version 2.0 Apache Commons Lang Lucene Common Analyzers Lucene Core + Lucene Memory Lucene QueryParsers diff --git a/community/lucene-index/src/main/java/org/neo4j/index/lucene/legacy/LuceneDataSource.java b/community/lucene-index/src/main/java/org/neo4j/index/lucene/legacy/LuceneDataSource.java index 9994e40a3e19f..4a8a196bec220 100644 --- a/community/lucene-index/src/main/java/org/neo4j/index/lucene/legacy/LuceneDataSource.java +++ b/community/lucene-index/src/main/java/org/neo4j/index/lucene/legacy/LuceneDataSource.java @@ -137,7 +137,12 @@ public void init() closed = false; } - private File baseDirectory( File storeDir ) + public static File getStoreDirectory(File storeDir) + { + return baseDirectory( storeDir ); + } + + private static File baseDirectory( File storeDir ) { return new File( storeDir, "index" ); } diff --git a/community/lucene-index/src/main/java/org/neo4j/index/lucene/legacy/LuceneIndexImplementation.java b/community/lucene-index/src/main/java/org/neo4j/index/lucene/legacy/LuceneIndexImplementation.java index ece19a2b4595a..3e8dff25e9c1b 100644 --- a/community/lucene-index/src/main/java/org/neo4j/index/lucene/legacy/LuceneIndexImplementation.java +++ b/community/lucene-index/src/main/java/org/neo4j/index/lucene/legacy/LuceneIndexImplementation.java @@ -96,6 +96,12 @@ public void shutdown() throws Throwable this.dataSource = null; } + @Override + public File getStoreDirectory(File storeDir) + { + return LuceneDataSource.getStoreDirectory(storeDir); + } + @Override public LegacyIndexProviderTransaction newTransaction( IndexCommandFactory commandFactory ) { diff --git a/community/monitor-logging/LICENSES.txt b/community/monitor-logging/LICENSES.txt index fd96cba2c36a7..de7f66b3b397e 100644 --- a/community/monitor-logging/LICENSES.txt +++ b/community/monitor-logging/LICENSES.txt @@ -4,6 +4,8 @@ libraries. For an overview of the licenses see the NOTICE.txt file. ------------------------------------------------------------------------------ Apache Software License, Version 2.0 Apache Commons Lang + Lucene Core + Lucene Memory ------------------------------------------------------------------------------ Apache License diff --git a/community/monitor-logging/NOTICE.txt b/community/monitor-logging/NOTICE.txt index fc42d74a24f82..8c2e5c7b849bd 100644 --- a/community/monitor-logging/NOTICE.txt +++ b/community/monitor-logging/NOTICE.txt @@ -27,4 +27,6 @@ Third-party licenses Apache Software License, Version 2.0 Apache Commons Lang + Lucene Core + Lucene Memory diff --git a/community/neo4j-harness/LICENSES.txt b/community/neo4j-harness/LICENSES.txt index 1414faab30364..3ff3baa852aed 100644 --- a/community/neo4j-harness/LICENSES.txt +++ b/community/neo4j-harness/LICENSES.txt @@ -24,6 +24,7 @@ Apache Software License, Version 2.0 Jetty :: XML utilities Lucene Common Analyzers Lucene Core + Lucene Memory Lucene QueryParsers Netty/All-in-One opencsv diff --git a/community/neo4j-harness/NOTICE.txt b/community/neo4j-harness/NOTICE.txt index efa2db557d0ca..598245abcd09f 100644 --- a/community/neo4j-harness/NOTICE.txt +++ b/community/neo4j-harness/NOTICE.txt @@ -47,6 +47,7 @@ Apache Software License, Version 2.0 Jetty :: XML utilities Lucene Common Analyzers Lucene Core + Lucene Memory Lucene QueryParsers Netty/All-in-One opencsv diff --git a/community/neo4j/LICENSES.txt b/community/neo4j/LICENSES.txt index e3437d2e2eaf6..d575fc223f6ea 100644 --- a/community/neo4j/LICENSES.txt +++ b/community/neo4j/LICENSES.txt @@ -7,6 +7,7 @@ Apache Software License, Version 2.0 ConcurrentLinkedHashMap Lucene Common Analyzers Lucene Core + Lucene Memory Lucene QueryParsers opencsv parboiled-core diff --git a/community/neo4j/NOTICE.txt b/community/neo4j/NOTICE.txt index 7e64d7e333b3e..5df05620250cc 100644 --- a/community/neo4j/NOTICE.txt +++ b/community/neo4j/NOTICE.txt @@ -30,6 +30,7 @@ Apache Software License, Version 2.0 ConcurrentLinkedHashMap Lucene Common Analyzers Lucene Core + Lucene Memory Lucene QueryParsers opencsv parboiled-core diff --git a/community/pom.xml b/community/pom.xml index 6f18e9d0e0d48..e661196f355ea 100644 --- a/community/pom.xml +++ b/community/pom.xml @@ -61,6 +61,7 @@ consistency-check neo4j-harness licensecheck-config + upgade diff --git a/community/server-api/LICENSES.txt b/community/server-api/LICENSES.txt index 649e5d90b9ada..7ab7be8627f5d 100644 --- a/community/server-api/LICENSES.txt +++ b/community/server-api/LICENSES.txt @@ -9,6 +9,8 @@ Apache Software License, Version 2.0 Commons Digester Commons Lang Commons Logging + Lucene Core + Lucene Memory ------------------------------------------------------------------------------ Apache License diff --git a/community/server-api/NOTICE.txt b/community/server-api/NOTICE.txt index 7c742b911e7d7..ea06a3ec3ec55 100644 --- a/community/server-api/NOTICE.txt +++ b/community/server-api/NOTICE.txt @@ -32,6 +32,8 @@ Apache Software License, Version 2.0 Commons Digester Commons Lang Commons Logging + Lucene Core + Lucene Memory Common Development and Distribution License Version 1.1 jsr311-api diff --git a/community/server-examples/LICENSES.txt b/community/server-examples/LICENSES.txt index 5b5c5a55809b7..f22b2c7e0a111 100644 --- a/community/server-examples/LICENSES.txt +++ b/community/server-examples/LICENSES.txt @@ -11,6 +11,8 @@ Apache Software License, Version 2.0 Commons Logging Data Mapper for Jackson Jackson + Lucene Core + Lucene Memory ------------------------------------------------------------------------------ Apache License diff --git a/community/server-examples/NOTICE.txt b/community/server-examples/NOTICE.txt index 649e42b92325e..99ec0ffccf6ec 100644 --- a/community/server-examples/NOTICE.txt +++ b/community/server-examples/NOTICE.txt @@ -34,6 +34,8 @@ Apache Software License, Version 2.0 Commons Logging Data Mapper for Jackson Jackson + Lucene Core + Lucene Memory Common Development and Distribution License Version 1.1 jersey-client diff --git a/community/server/LICENSES.txt b/community/server/LICENSES.txt index c42008a1b38c9..eed51acbda5dc 100644 --- a/community/server/LICENSES.txt +++ b/community/server/LICENSES.txt @@ -24,6 +24,7 @@ Apache Software License, Version 2.0 Jetty :: XML utilities Lucene Common Analyzers Lucene Core + Lucene Memory Lucene QueryParsers Netty/All-in-One opencsv diff --git a/community/server/NOTICE.txt b/community/server/NOTICE.txt index a29ec930fa55b..6f8c42db59804 100644 --- a/community/server/NOTICE.txt +++ b/community/server/NOTICE.txt @@ -47,6 +47,7 @@ Apache Software License, Version 2.0 Jetty :: XML utilities Lucene Common Analyzers Lucene Core + Lucene Memory Lucene QueryParsers Netty/All-in-One opencsv diff --git a/community/udc/LICENSES.txt b/community/udc/LICENSES.txt index fd96cba2c36a7..de7f66b3b397e 100644 --- a/community/udc/LICENSES.txt +++ b/community/udc/LICENSES.txt @@ -4,6 +4,8 @@ libraries. For an overview of the licenses see the NOTICE.txt file. ------------------------------------------------------------------------------ Apache Software License, Version 2.0 Apache Commons Lang + Lucene Core + Lucene Memory ------------------------------------------------------------------------------ Apache License diff --git a/community/udc/NOTICE.txt b/community/udc/NOTICE.txt index fc42d74a24f82..8c2e5c7b849bd 100644 --- a/community/udc/NOTICE.txt +++ b/community/udc/NOTICE.txt @@ -27,4 +27,6 @@ Third-party licenses Apache Software License, Version 2.0 Apache Commons Lang + Lucene Core + Lucene Memory diff --git a/community/upgade/LICENSES.txt b/community/upgade/LICENSES.txt new file mode 100644 index 0000000000000..39bcd6a643262 --- /dev/null +++ b/community/upgade/LICENSES.txt @@ -0,0 +1,213 @@ +This file contains the full license text of the included third party +libraries. For an overview of the licenses see the NOTICE.txt file. + +------------------------------------------------------------------------------ +Apache Software License, Version 2.0 + Lucene Core + Lucene Memory +------------------------------------------------------------------------------ + + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright [yyyy] [name of copyright owner] + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + + + diff --git a/community/upgade/NOTICE.txt b/community/upgade/NOTICE.txt new file mode 100644 index 0000000000000..9dfe08e497be1 --- /dev/null +++ b/community/upgade/NOTICE.txt @@ -0,0 +1,31 @@ +Neo4j +Copyright © 2002-2015 Network Engine for Objects in Lund AB (referred to +in this notice as "Neo Technology") + [http://neotechnology.com] + +This product includes software ("Software") developed by Neo Technology. + +The copyright in the bundled Neo4j graph database (including the +Software) is owned by Neo Technology. The Software developed and owned +by Neo Technology is licensed under the GNU GENERAL PUBLIC LICENSE +Version 3 (http://www.fsf.org/licensing/licenses/gpl-3.0.html) ("GPL") +to all third parties and that license, as required by the GPL, is +included in the LICENSE.txt file. + +However, if you have executed an End User Software License and Services +Agreement or an OEM Software License and Support Services Agreement, or +another commercial license agreement with Neo Technology or one of its +affiliates (each, a "Commercial Agreement"), the terms of the license in +such Commercial Agreement will supersede the GPL and you may use the +software solely pursuant to the terms of the relevant Commercial +Agreement. + +Full license texts are found in LICENSES.txt. + +Third-party licenses +-------------------- + +Apache Software License, Version 2.0 + Lucene Core + Lucene Memory + diff --git a/community/upgade/pom.xml b/community/upgade/pom.xml new file mode 100644 index 0000000000000..dd56e62f82db3 --- /dev/null +++ b/community/upgade/pom.xml @@ -0,0 +1,114 @@ + + + + org.neo4j + parent + 3.0.0-SNAPSHOT + ../.. + + + + lucene-index-upgrade + GPL-3-header.txt + notice-gpl-prefix.txt + 4.10.4 + 5.3.1 + + + 4.0.0 + neo4j-lucene-upgrade + 3.0.0-SNAPSHOT + + jar + Neo4j - Lucene Index Upgrade + Neo4j Lucene Index Upgrade for legacy index migrations + + + + + org.apache.maven.plugins + maven-dependency-plugin + + + copy + package + + copy + + + + + + org.apache.lucene + lucene-core + ${lucene4.version} + + + org.apache.lucene + lucene-core + ${lucene5.version} + + + org.apache.lucene + lucene-backward-codecs + ${lucene5.version} + + + ${project.build.directory}/lib + + + + + + org.apache.maven.plugins + maven-assembly-plugin + + + standalone + package + + single + + + + src/main/assembly/assembly.xml + + + + + + false + + + + + + + + org.apache.lucene + lucene-core + ${lucene5.version} + + + org.apache.lucene + lucene-backward-codecs + ${lucene5.version} + + + junit + junit + + + org.mockito + mockito-core + + + org.hamcrest + hamcrest-library + + + + diff --git a/community/upgade/src/main/assembly/assembly.xml b/community/upgade/src/main/assembly/assembly.xml new file mode 100644 index 0000000000000..4f76e138ab549 --- /dev/null +++ b/community/upgade/src/main/assembly/assembly.xml @@ -0,0 +1,46 @@ + + + distribution + + jar + + false + + + + ${project.build.outputDirectory} + / + + + ${project.build.directory}/lib + /lib + false + + *.jar + + + + + \ No newline at end of file diff --git a/community/upgade/src/main/java/org/neo4j/upgrade/loader/EmbeddedJarLoader.java b/community/upgade/src/main/java/org/neo4j/upgrade/loader/EmbeddedJarLoader.java new file mode 100644 index 0000000000000..198b5df1da531 --- /dev/null +++ b/community/upgade/src/main/java/org/neo4j/upgrade/loader/EmbeddedJarLoader.java @@ -0,0 +1,170 @@ +/* + * Copyright (c) 2002-2015 "Neo Technology," + * Network Engine for Objects in Lund AB [http://neotechnology.com] + * + * This file is part of Neo4j. + * + * Neo4j is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package org.neo4j.upgrade.loader; + +import java.io.File; +import java.io.IOException; +import java.io.InputStream; +import java.net.JarURLConnection; +import java.net.MalformedURLException; +import java.net.URL; +import java.net.URLClassLoader; +import java.net.URLConnection; +import java.nio.file.Files; +import java.nio.file.StandardCopyOption; +import java.util.ArrayList; +import java.util.Collection; +import java.util.jar.JarEntry; +import java.util.jar.JarFile; + +/** + * Loader that will try to load jar that is embedded into some other jar. + *

+ * In case if we do not want jar file to be visible for classloaders we can embed it into other jar as resource. + * This loaded will allow us to use those jars later on and will provide facility to load classes from them. + *

+ * Please note that this class should not be used as a generic class loader. + * Example use case: ability to ship several versions of artifact for migration purposes (lucene indexes migrator as + * example) + */ +public class EmbeddedJarLoader implements AutoCloseable +{ + private String[] jars; + private Collection extractedFiles; + private URLClassLoader jarsClassLoader; + + EmbeddedJarLoader( String... jars ) + { + this.jars = jars; + } + + /** + * Load class from embedded jar + * @param className fully qualified class name + * @return Loaded class + * @throws ClassNotFoundException in case if specified class not found + * @throws IOException if class cannot be extracted + */ + public Class loadEmbeddedClass( String className ) throws ClassNotFoundException, IOException + { + return getJarsClassLoader().loadClass( className ); + } + + /** + * Get class loaded of embedded jar files + * @return jar files class loader + * @throws IOException if exception occurred during class loader construction + */ + public ClassLoader getJarsClassLoader() throws IOException + { + if ( jarsClassLoader == null ) + { + jarsClassLoader = buildJarClassLoader(); + } + return jarsClassLoader; + } + + /** + * Release class loader that was used for class loading and attempt to delete all extracted jars. + * If deletion will not succeed, they will be deleted on JVM exit. + * @throws IOException + */ + @Override + public void close() throws IOException + { + if ( jarsClassLoader != null ) + { + jarsClassLoader.close(); + } + + if ( extractedFiles != null ) + { + extractedFiles.forEach( File::delete ); + } + } + + private URLClassLoader buildJarClassLoader() throws IOException + { + Collection jarFiles = extractJars(); + URL[] urls = jarFiles.stream().map( EmbeddedJarLoader::getFileURL ).toArray( URL[]::new ); + return new URLClassLoader( urls, null ); + } + + private Collection extractJars() throws IOException + { + extractedFiles = new ArrayList<>(); + for ( String jar : jars ) + { + URL url = getClass().getClassLoader().getResource( jar ); + if ( url == null ) + { + throw new EmbeddedJarNotFoundException( "Jar file '" + jar + "' not found." ); + } + URLConnection connection = url.openConnection(); + if ( connection instanceof JarURLConnection ) + { + JarURLConnection urlConnection = (JarURLConnection) connection; + JarFile jarFile = urlConnection.getJarFile(); + JarEntry jarEntry = urlConnection.getJarEntry(); + if ( jarEntry == null ) + { + throw new EmbeddedJarNotFoundException( "Jar file '" + jar + "' not found." ); + } + File extractedFile = extract( jarFile, jarEntry ); + extractedFiles.add( extractedFile ); + } + else + { + throw new EmbeddedJarNotFoundException( "Jar file '" + jar + "' not found." ); + } + } + return this.extractedFiles; + } + + private File extract( JarFile jarFile, JarEntry jarEntry ) throws IOException + { + File extractedFile = createTempFile( jarEntry ); + try ( InputStream jarInputStream = jarFile.getInputStream( jarEntry ) ) + { + Files.copy( jarInputStream, extractedFile.toPath(), StandardCopyOption.REPLACE_EXISTING ); + } + return extractedFile; + } + + private File createTempFile( JarEntry jarEntry ) throws IOException + { + File tempFile = File.createTempFile( jarEntry.getName(), "jar" ); + tempFile.deleteOnExit(); + return tempFile; + } + + private static URL getFileURL( File file ) + { + try + { + return file.toURI().toURL(); + } + catch ( MalformedURLException e ) + { + throw new RuntimeException( "Can't convert file " + file + " URI into URL.", e ); + } + } + +} diff --git a/community/upgade/src/main/java/org/neo4j/upgrade/loader/EmbeddedJarNotFoundException.java b/community/upgade/src/main/java/org/neo4j/upgrade/loader/EmbeddedJarNotFoundException.java new file mode 100644 index 0000000000000..299d1199c3618 --- /dev/null +++ b/community/upgade/src/main/java/org/neo4j/upgrade/loader/EmbeddedJarNotFoundException.java @@ -0,0 +1,32 @@ +/* + * Copyright (c) 2002-2015 "Neo Technology," + * Network Engine for Objects in Lund AB [http://neotechnology.com] + * + * This file is part of Neo4j. + * + * Neo4j is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package org.neo4j.upgrade.loader; + +/** + * Exception that will be thrown by {@link EmbeddedJarLoader} in case we can't load requested jar. + * @see EmbeddedJarLoader + */ +class EmbeddedJarNotFoundException extends RuntimeException +{ + public EmbeddedJarNotFoundException( String message ) + { + super( message ); + } +} diff --git a/community/upgade/src/main/java/org/neo4j/upgrade/loader/JarLoaderSupplier.java b/community/upgade/src/main/java/org/neo4j/upgrade/loader/JarLoaderSupplier.java new file mode 100644 index 0000000000000..9f209a572710e --- /dev/null +++ b/community/upgade/src/main/java/org/neo4j/upgrade/loader/JarLoaderSupplier.java @@ -0,0 +1,47 @@ +/* + * Copyright (c) 2002-2015 "Neo Technology," + * Network Engine for Objects in Lund AB [http://neotechnology.com] + * + * This file is part of Neo4j. + * + * Neo4j is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package org.neo4j.upgrade.loader; + +import java.util.function.Supplier; + +/** + * Supplier of embedded jar loaders. Will construct jar loader for specified set of jar paths. + * @see EmbeddedJarLoader + */ +public class JarLoaderSupplier implements Supplier +{ + private String[] jarPaths; + + public static JarLoaderSupplier of(String... jarPaths) + { + return new JarLoaderSupplier( jarPaths ); + } + + private JarLoaderSupplier(String... jarPaths) + { + this.jarPaths = jarPaths; + } + + @Override + public EmbeddedJarLoader get() + { + return new EmbeddedJarLoader( jarPaths ); + } +} diff --git a/community/upgade/src/main/java/org/neo4j/upgrade/lucene/IndexUpgraderWrapper.java b/community/upgade/src/main/java/org/neo4j/upgrade/lucene/IndexUpgraderWrapper.java new file mode 100644 index 0000000000000..c3545085cfe7f --- /dev/null +++ b/community/upgade/src/main/java/org/neo4j/upgrade/lucene/IndexUpgraderWrapper.java @@ -0,0 +1,82 @@ +/* + * Copyright (c) 2002-2015 "Neo Technology," + * Network Engine for Objects in Lund AB [http://neotechnology.com] + * + * This file is part of Neo4j. + * + * Neo4j is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package org.neo4j.upgrade.lucene; + +import java.lang.invoke.MethodHandle; +import java.lang.invoke.MethodHandles; +import java.lang.invoke.MethodType; +import java.nio.file.Path; +import java.util.function.Supplier; + +import org.neo4j.upgrade.loader.EmbeddedJarLoader; + +/** + * Index upgrader is a container for loaded lucene native upgrader. + *

+ * During first attempt of index upgrade original migrator will be loaded and will be reused during further + * invocations. + * As soon as upgrade completed, index upgrader should be closed. + * + * @see EmbeddedJarLoader + */ +class IndexUpgraderWrapper implements AutoCloseable +{ + private static final String LUCENE_INDEX_UPGRADER_CLASS_NAME = "org.apache.lucene.index.IndexUpgrader"; + + private EmbeddedJarLoader luceneLoader; + private MethodHandle mainMethod; + private Supplier jarLoaderSupplier; + + public IndexUpgraderWrapper( Supplier jarLoaderSupplier ) + { + this.jarLoaderSupplier = jarLoaderSupplier; + } + + public void upgradeIndex( Path indexPath ) throws Throwable + { + ClassLoader contextClassLoader = Thread.currentThread().getContextClassLoader(); + try + { + if ( mainMethod == null ) + { + luceneLoader = jarLoaderSupplier.get(); + Thread.currentThread().setContextClassLoader( luceneLoader.getJarsClassLoader() ); + Class upgrader = luceneLoader.loadEmbeddedClass( LUCENE_INDEX_UPGRADER_CLASS_NAME ); + MethodHandles.Lookup lookup = MethodHandles.lookup(); + mainMethod = lookup.findStatic( upgrader, "main", + MethodType.methodType( void.class, String[].class ) ); + } + mainMethod.invokeExact( new String[]{indexPath.toString()} ); + } + finally + { + Thread.currentThread().setContextClassLoader( contextClassLoader ); + } + } + + @Override + public void close() throws Exception + { + if ( luceneLoader != null ) + { + luceneLoader.close(); + } + } +} diff --git a/community/upgade/src/main/java/org/neo4j/upgrade/lucene/LegacyIndexMigrationException.java b/community/upgade/src/main/java/org/neo4j/upgrade/lucene/LegacyIndexMigrationException.java new file mode 100644 index 0000000000000..c81113b6c1af4 --- /dev/null +++ b/community/upgade/src/main/java/org/neo4j/upgrade/lucene/LegacyIndexMigrationException.java @@ -0,0 +1,40 @@ +/* + * Copyright (c) 2002-2015 "Neo Technology," + * Network Engine for Objects in Lund AB [http://neotechnology.com] + * + * This file is part of Neo4j. + * + * Neo4j is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package org.neo4j.upgrade.lucene; + +/** + * Exception that will be thrown in case if {@link LuceneLegacyIndexUpgrader} can migrate particular index. + * Name of failed index can be found in {@link LegacyIndexMigrationException#failedIndexName} + */ +public class LegacyIndexMigrationException extends Exception +{ + private final String failedIndexName; + + public LegacyIndexMigrationException( String failedIndexName, String message, Throwable cause ) + { + super( message, cause ); + this.failedIndexName = failedIndexName; + } + + public String getFailedIndexName() + { + return failedIndexName; + } +} diff --git a/community/upgade/src/main/java/org/neo4j/upgrade/lucene/LuceneLegacyIndexUpgrader.java b/community/upgade/src/main/java/org/neo4j/upgrade/lucene/LuceneLegacyIndexUpgrader.java new file mode 100644 index 0000000000000..59f628f3ac97a --- /dev/null +++ b/community/upgade/src/main/java/org/neo4j/upgrade/lucene/LuceneLegacyIndexUpgrader.java @@ -0,0 +1,155 @@ +/* + * Copyright (c) 2002-2015 "Neo Technology," + * Network Engine for Objects in Lund AB [http://neotechnology.com] + * + * This file is part of Neo4j. + * + * Neo4j is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package org.neo4j.upgrade.lucene; + +import java.io.File; +import java.io.IOException; +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.List; +import java.util.function.Predicate; +import java.util.stream.Collectors; +import java.util.stream.Stream; + +import org.neo4j.upgrade.loader.JarLoaderSupplier; + +/** + * Lucene index upgrader that will try to migrate all indexes in specified index root directory. + *

+ * Currently index migration has 2 steps: + *

    + *
  1. Migration to format supported by Lucene 4
  2. + *
  3. Migration to format supported by Lucene 5
  4. + *
+ * Migration performed by using native lucene's IndexUpgraders from corresponding versions. For details see Lucenes + * migration guide. + *

+ * In case if one of the indexes can not be migrated migration is terminated and corresponding exception is thrown. + */ +public class LuceneLegacyIndexUpgrader +{ + private static final String LIBRARY_DIRECTORY = "lib"; + private static final String LUCENE4_CORE_JAR_NAME = "lucene-core-4.10.4.jar"; + private static final String LUCENE5_CORE_JAR_NAME = "lucene-core-5.3.1.jar"; + private static final String LUCENE5_BACKWARD_CODECS_NAME = "lucene-backward-codecs-5.3.1.jar"; + public static final String SEGMENTS_FILE_NAME_PREFIX = "segments"; + + private final Path indexRootPath; + + public LuceneLegacyIndexUpgrader( Path indexRootPath ) + { + if ( Files.exists( indexRootPath ) && !Files.isDirectory( indexRootPath ) ) + { + throw new IllegalArgumentException( "Index path should be a directory" ); + } + this.indexRootPath = indexRootPath; + } + + /** + * Perform index migrations + * @throws LegacyIndexMigrationException in case of exception during index migration + */ + public void upgradeIndexes() throws LegacyIndexMigrationException + { + try + { + if ( !Files.exists( indexRootPath ) ) + { + return; + } + try ( Stream pathStream = Files.walk( indexRootPath ); + IndexUpgraderWrapper lucene4Upgrader = createIndexUpgrader( getLucene4JarPaths() ); + IndexUpgraderWrapper lucene5Upgrader = createIndexUpgrader( getLucene5JarPaths() ) ) + { + List indexPaths = pathStream.filter( getIndexPathFilter() ).collect( Collectors.toList() ); + for (Path indexPath : indexPaths ) + { + try + { + lucene4Upgrader.upgradeIndex( indexPath ); + lucene5Upgrader.upgradeIndex( indexPath ); + } + catch ( Throwable e ) + { + throw new LegacyIndexMigrationException( indexPath.getFileName().toString(), + "Migration of legacy index at path:" + indexPath + " failed.", e ); + } + } + } + } + catch ( LegacyIndexMigrationException ime ) + { + throw ime; + } + catch ( Exception e ) + { + throw new RuntimeException( "Lucene legacy indexes migration failed.", e ); + } + } + + IndexUpgraderWrapper createIndexUpgrader( String[] jars ) + { + return new IndexUpgraderWrapper( JarLoaderSupplier.of( jars ) ); + } + + private static String[] getLucene5JarPaths() + { + return getJarsFullPaths( LUCENE5_CORE_JAR_NAME, LUCENE5_BACKWARD_CODECS_NAME ); + } + + private static String[] getLucene4JarPaths() + { + return getJarsFullPaths( LUCENE4_CORE_JAR_NAME ); + } + + private static String[] getJarsFullPaths( String... jars ) + { + return Stream.of( jars ) + .map( LuceneLegacyIndexUpgrader::getJarPath ).toArray( String[]::new ); + } + + private static String getJarPath( String library ) + { + return LIBRARY_DIRECTORY + File.separator + library; + } + + private static Predicate getIndexPathFilter() + { + return path -> { + try + { + return Files.isDirectory( path ) && isIndexDirectory( path ); + } + catch ( IOException e ) + { + throw new RuntimeException( e ); + } + }; + } + + private static boolean isIndexDirectory( Path path ) throws IOException + { + return Files.list( path ) + .filter( child -> child.getFileName().toString().startsWith( SEGMENTS_FILE_NAME_PREFIX ) ) + .findAny() + .isPresent(); + } + +} diff --git a/community/upgade/src/test/java/org/neo4j/upgrade/lucene/IndexUpgraderWrapperTest.java b/community/upgade/src/test/java/org/neo4j/upgrade/lucene/IndexUpgraderWrapperTest.java new file mode 100644 index 0000000000000..dc46085dd573f --- /dev/null +++ b/community/upgade/src/test/java/org/neo4j/upgrade/lucene/IndexUpgraderWrapperTest.java @@ -0,0 +1,71 @@ +/* + * Copyright (c) 2002-2015 "Neo Technology," + * Network Engine for Objects in Lund AB [http://neotechnology.com] + * + * This file is part of Neo4j. + * + * Neo4j is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package org.neo4j.upgrade.lucene; + +import org.junit.Test; +import org.mockito.Mockito; + +import java.io.IOException; +import java.nio.file.Paths; + +import org.neo4j.upgrade.loader.EmbeddedJarLoader; + +import static org.junit.Assert.assertTrue; +import static org.mockito.Matchers.anyString; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.when; + +public class IndexUpgraderWrapperTest +{ + + @Test + public void indexUpgraderInvokesLuceneMigrator() throws Throwable + { + IndexUpgraderWrapper upgrader = getIndexUpgrader( createJarLoader() ); + + UpgraterStub.resetInvocationMark(); + upgrader.upgradeIndex( Paths.get( "some" ) ); + assertTrue( UpgraterStub.getInvocationMark() ); + } + + @Test + public void indexUpgraderReleaseResourcesOnClose() throws Throwable + { + EmbeddedJarLoader jarLoader = createJarLoader(); + IndexUpgraderWrapper upgrader = getIndexUpgrader( jarLoader ); + + upgrader.upgradeIndex( Paths.get( "some" ) ); + upgrader.close(); + + verify( jarLoader ).close(); + } + + private IndexUpgraderWrapper getIndexUpgrader( EmbeddedJarLoader jarLoader ) throws ClassNotFoundException, IOException + { + return new IndexUpgraderWrapper( () -> jarLoader ); + } + + private EmbeddedJarLoader createJarLoader() throws ClassNotFoundException, IOException + { + EmbeddedJarLoader jarLoader = Mockito.mock( EmbeddedJarLoader.class ); + when( jarLoader.loadEmbeddedClass( anyString() ) ).thenReturn( UpgraterStub.class ); + return jarLoader; + } +} \ No newline at end of file diff --git a/community/upgade/src/test/java/org/neo4j/upgrade/lucene/LuceneLegacyIndexUpgraderTest.java b/community/upgade/src/test/java/org/neo4j/upgrade/lucene/LuceneLegacyIndexUpgraderTest.java new file mode 100644 index 0000000000000..573cbe2f2a45b --- /dev/null +++ b/community/upgade/src/test/java/org/neo4j/upgrade/lucene/LuceneLegacyIndexUpgraderTest.java @@ -0,0 +1,176 @@ +/* + * Copyright (c) 2002-2015 "Neo Technology," + * Network Engine for Objects in Lund AB [http://neotechnology.com] + * + * This file is part of Neo4j. + * + * Neo4j is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package org.neo4j.upgrade.lucene; + +import org.hamcrest.BaseMatcher; +import org.hamcrest.Description; +import org.hamcrest.Matcher; +import org.hamcrest.Matchers; +import org.hamcrest.TypeSafeDiagnosingMatcher; +import org.junit.Rule; +import org.junit.Test; +import org.junit.rules.ExpectedException; + +import java.net.URISyntaxException; +import java.nio.file.Path; +import java.nio.file.Paths; +import java.util.HashSet; +import java.util.Set; +import java.util.function.Supplier; + +import org.neo4j.upgrade.loader.EmbeddedJarLoader; +import org.neo4j.upgrade.loader.JarLoaderSupplier; + +import static org.junit.Assert.assertThat; +import static org.junit.Assert.assertTrue; + +public class LuceneLegacyIndexUpgraderTest +{ + @Rule + public ExpectedException expectedException = ExpectedException.none(); + + @Test + public void failOnFileMigration() throws Exception + { + Path indexFolder = createPathForResource("indexPretender.txt"); + expectedException.expect( IllegalArgumentException.class ); + + new LuceneLegacyIndexUpgrader(indexFolder); + } + + @Test + public void ignoreFoldersWithoutIndexes() throws URISyntaxException, LegacyIndexMigrationException + { + Path indexFolder = createPathForResource("notIndexFolder"); + TrackingLuceneLegacyIndexUpgrader indexUpgrader = new TrackingLuceneLegacyIndexUpgrader( indexFolder ); + indexUpgrader.upgradeIndexes(); + + assertTrue(indexUpgrader.getMigratedIndexes().isEmpty()); + } + + @Test + public void migrateValidIndexes() throws URISyntaxException, LegacyIndexMigrationException + { + Path indexFolder = createPathForResource("indexFolder"); + TrackingLuceneLegacyIndexUpgrader indexUpgrader = new TrackingLuceneLegacyIndexUpgrader( indexFolder ); + indexUpgrader.upgradeIndexes(); + + assertThat(indexUpgrader.getMigratedIndexes(), Matchers.contains("index1", "index2")); + } + + @Test + public void pointIncorrectIndexOnMigrationFailure() throws URISyntaxException, LegacyIndexMigrationException + { + Path indexFolder = createPathForResource("indexFolder"); + TrackingLuceneLegacyIndexUpgrader indexUpgrader = new TrackingLuceneLegacyIndexUpgrader( indexFolder, true ); + + expectedException.expect( LegacyIndexMigrationException.class ); + expectedException.expect( new LegacyIndexMigrationExceptionBaseMatcher("index1") ); + + indexUpgrader.upgradeIndexes(); + } + + private Path createPathForResource(String resourceName) throws URISyntaxException + { + return Paths.get( getClass().getClassLoader().getResource( resourceName ).toURI() ); + } + + private class LegacyIndexMigrationExceptionBaseMatcher extends TypeSafeDiagnosingMatcher + { + + private String failedIndexName; + + public LegacyIndexMigrationExceptionBaseMatcher(String failedIndexName) + { + this.failedIndexName = failedIndexName; + } + + @Override + public void describeTo( Description description ) + { + description.appendText( failedIndexName ).appendText( " should fail during migration. " ); + } + + @Override + protected boolean matchesSafely( LegacyIndexMigrationException item, Description mismatchDescription ) + { + String brokendIndexName = item.getFailedIndexName(); + boolean matched = this.failedIndexName.equals( brokendIndexName ); + if (!matched) + { + mismatchDescription.appendText( "Failed index is: " ).appendText( brokendIndexName ); + } + return matched; + } + } + + private class TrackingLuceneLegacyIndexUpgrader extends LuceneLegacyIndexUpgrader + { + private Set migratedIndexes = new HashSet<>( ); + private boolean failIndexUpgrade; + + public TrackingLuceneLegacyIndexUpgrader( Path indexRootPath) + { + this ( indexRootPath, false ); + } + + public TrackingLuceneLegacyIndexUpgrader( Path indexRootPath, boolean failIndexUpgrade ) + { + super( indexRootPath ); + this.failIndexUpgrade = failIndexUpgrade; + } + + @Override + IndexUpgraderWrapper createIndexUpgrader( String[] jars ) + { + return new IndexUpgraderWrapperStub( JarLoaderSupplier.of( jars ), migratedIndexes, failIndexUpgrade ); + } + + public Set getMigratedIndexes() + { + return migratedIndexes; + } + } + + private class IndexUpgraderWrapperStub extends IndexUpgraderWrapper + { + + private Set migratedIndexes; + private boolean failIndexUpgrade; + + public IndexUpgraderWrapperStub( Supplier jarLoaderSupplier, Set migratedIndexes, + boolean failIndexUpgrade ) + { + super( jarLoaderSupplier ); + this.migratedIndexes = migratedIndexes; + this.failIndexUpgrade = failIndexUpgrade; + } + + @Override + public void upgradeIndex( Path indexPath ) throws Throwable + { + if (failIndexUpgrade) + { + throw new RuntimeException( "Fail index migration: " + indexPath ); + } + migratedIndexes.add( indexPath.getFileName().toString() ); + } + } +} \ No newline at end of file diff --git a/community/upgade/src/test/java/org/neo4j/upgrade/lucene/UpgraterStub.java b/community/upgade/src/test/java/org/neo4j/upgrade/lucene/UpgraterStub.java new file mode 100644 index 0000000000000..c68c92ac7753b --- /dev/null +++ b/community/upgade/src/test/java/org/neo4j/upgrade/lucene/UpgraterStub.java @@ -0,0 +1,41 @@ +/* + * Copyright (c) 2002-2015 "Neo Technology," + * Network Engine for Objects in Lund AB [http://neotechnology.com] + * + * This file is part of Neo4j. + * + * Neo4j is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package org.neo4j.upgrade.lucene; + +class UpgraterStub +{ + private static boolean invocationMark; + + public static void main( String[] args ) + { + invocationMark = true; + } + + public static boolean getInvocationMark() + { + return invocationMark; + } + + public static void resetInvocationMark() + { + invocationMark = false; + } + +} diff --git a/community/upgade/src/test/resources/indexFolder/index1/segments.gen b/community/upgade/src/test/resources/indexFolder/index1/segments.gen new file mode 100644 index 0000000000000..63a7ec9a3ce3e Binary files /dev/null and b/community/upgade/src/test/resources/indexFolder/index1/segments.gen differ diff --git a/community/upgade/src/test/resources/indexFolder/index1/segments_1 b/community/upgade/src/test/resources/indexFolder/index1/segments_1 new file mode 100644 index 0000000000000..587d799f14e13 Binary files /dev/null and b/community/upgade/src/test/resources/indexFolder/index1/segments_1 differ diff --git a/community/upgade/src/test/resources/indexFolder/index2/segments.gen b/community/upgade/src/test/resources/indexFolder/index2/segments.gen new file mode 100644 index 0000000000000..63a7ec9a3ce3e Binary files /dev/null and b/community/upgade/src/test/resources/indexFolder/index2/segments.gen differ diff --git a/community/upgade/src/test/resources/indexFolder/index2/segments_1 b/community/upgade/src/test/resources/indexFolder/index2/segments_1 new file mode 100644 index 0000000000000..587d799f14e13 Binary files /dev/null and b/community/upgade/src/test/resources/indexFolder/index2/segments_1 differ diff --git a/community/upgade/src/test/resources/indexPretender.txt b/community/upgade/src/test/resources/indexPretender.txt new file mode 100644 index 0000000000000..ff3bb63948b4b --- /dev/null +++ b/community/upgade/src/test/resources/indexPretender.txt @@ -0,0 +1 @@ +The quick brown fox jumps over the lazy dog \ No newline at end of file diff --git a/community/upgade/src/test/resources/notIndexFolder/indexPretender.txt b/community/upgade/src/test/resources/notIndexFolder/indexPretender.txt new file mode 100644 index 0000000000000..ff3bb63948b4b --- /dev/null +++ b/community/upgade/src/test/resources/notIndexFolder/indexPretender.txt @@ -0,0 +1 @@ +The quick brown fox jumps over the lazy dog \ No newline at end of file diff --git a/enterprise/backup/LICENSES.txt b/enterprise/backup/LICENSES.txt index 965b18ce9524e..b0e08ef6f602d 100644 --- a/enterprise/backup/LICENSES.txt +++ b/enterprise/backup/LICENSES.txt @@ -4,6 +4,8 @@ libraries. For an overview of the licenses see the NOTICE.txt file. ------------------------------------------------------------------------------ Apache Software License, Version 2.0 Apache Commons Lang + Lucene Core + Lucene Memory The Netty Project ------------------------------------------------------------------------------ diff --git a/enterprise/backup/NOTICE.txt b/enterprise/backup/NOTICE.txt index 5944d0f201676..327f30e73409a 100644 --- a/enterprise/backup/NOTICE.txt +++ b/enterprise/backup/NOTICE.txt @@ -26,5 +26,7 @@ Third-party licenses Apache Software License, Version 2.0 Apache Commons Lang + Lucene Core + Lucene Memory The Netty Project diff --git a/enterprise/cluster/LICENSES.txt b/enterprise/cluster/LICENSES.txt index 965b18ce9524e..b0e08ef6f602d 100644 --- a/enterprise/cluster/LICENSES.txt +++ b/enterprise/cluster/LICENSES.txt @@ -4,6 +4,8 @@ libraries. For an overview of the licenses see the NOTICE.txt file. ------------------------------------------------------------------------------ Apache Software License, Version 2.0 Apache Commons Lang + Lucene Core + Lucene Memory The Netty Project ------------------------------------------------------------------------------ diff --git a/enterprise/cluster/NOTICE.txt b/enterprise/cluster/NOTICE.txt index 5944d0f201676..327f30e73409a 100644 --- a/enterprise/cluster/NOTICE.txt +++ b/enterprise/cluster/NOTICE.txt @@ -26,5 +26,7 @@ Third-party licenses Apache Software License, Version 2.0 Apache Commons Lang + Lucene Core + Lucene Memory The Netty Project diff --git a/enterprise/com/LICENSES.txt b/enterprise/com/LICENSES.txt index 965b18ce9524e..b0e08ef6f602d 100644 --- a/enterprise/com/LICENSES.txt +++ b/enterprise/com/LICENSES.txt @@ -4,6 +4,8 @@ libraries. For an overview of the licenses see the NOTICE.txt file. ------------------------------------------------------------------------------ Apache Software License, Version 2.0 Apache Commons Lang + Lucene Core + Lucene Memory The Netty Project ------------------------------------------------------------------------------ diff --git a/enterprise/com/NOTICE.txt b/enterprise/com/NOTICE.txt index 5944d0f201676..327f30e73409a 100644 --- a/enterprise/com/NOTICE.txt +++ b/enterprise/com/NOTICE.txt @@ -26,5 +26,7 @@ Third-party licenses Apache Software License, Version 2.0 Apache Commons Lang + Lucene Core + Lucene Memory The Netty Project diff --git a/enterprise/enterprise-performance-tests/LICENSES.txt b/enterprise/enterprise-performance-tests/LICENSES.txt index 5972661570d93..4fa48e6a7ab63 100644 --- a/enterprise/enterprise-performance-tests/LICENSES.txt +++ b/enterprise/enterprise-performance-tests/LICENSES.txt @@ -11,6 +11,7 @@ Apache Software License, Version 2.0 Jackson Lucene Common Analyzers Lucene Core + Lucene Memory Lucene QueryParsers Metrics Core Netty/All-in-One diff --git a/enterprise/enterprise-performance-tests/NOTICE.txt b/enterprise/enterprise-performance-tests/NOTICE.txt index 2e43a8d8f6f64..a35783b89c020 100644 --- a/enterprise/enterprise-performance-tests/NOTICE.txt +++ b/enterprise/enterprise-performance-tests/NOTICE.txt @@ -33,6 +33,7 @@ Apache Software License, Version 2.0 Jackson Lucene Common Analyzers Lucene Core + Lucene Memory Lucene QueryParsers Metrics Core Netty/All-in-One diff --git a/enterprise/ha/LICENSES.txt b/enterprise/ha/LICENSES.txt index 2fe3a7e257534..8f0e63038d354 100644 --- a/enterprise/ha/LICENSES.txt +++ b/enterprise/ha/LICENSES.txt @@ -6,6 +6,7 @@ Apache Software License, Version 2.0 Apache Commons Lang Lucene Common Analyzers Lucene Core + Lucene Memory Lucene QueryParsers The Netty Project ------------------------------------------------------------------------------ diff --git a/enterprise/ha/NOTICE.txt b/enterprise/ha/NOTICE.txt index 6c19986494cb9..f23654429bdda 100644 --- a/enterprise/ha/NOTICE.txt +++ b/enterprise/ha/NOTICE.txt @@ -28,6 +28,7 @@ Apache Software License, Version 2.0 Apache Commons Lang Lucene Common Analyzers Lucene Core + Lucene Memory Lucene QueryParsers The Netty Project diff --git a/enterprise/kernel/LICENSES.txt b/enterprise/kernel/LICENSES.txt index fd96cba2c36a7..de7f66b3b397e 100644 --- a/enterprise/kernel/LICENSES.txt +++ b/enterprise/kernel/LICENSES.txt @@ -4,6 +4,8 @@ libraries. For an overview of the licenses see the NOTICE.txt file. ------------------------------------------------------------------------------ Apache Software License, Version 2.0 Apache Commons Lang + Lucene Core + Lucene Memory ------------------------------------------------------------------------------ Apache License diff --git a/enterprise/kernel/NOTICE.txt b/enterprise/kernel/NOTICE.txt index 2e25fef478e2b..8d32d8e0c632b 100644 --- a/enterprise/kernel/NOTICE.txt +++ b/enterprise/kernel/NOTICE.txt @@ -26,4 +26,6 @@ Third-party licenses Apache Software License, Version 2.0 Apache Commons Lang + Lucene Core + Lucene Memory diff --git a/enterprise/neo4j-enterprise/LICENSES.txt b/enterprise/neo4j-enterprise/LICENSES.txt index ee816d90e95f5..2ed8751da404a 100644 --- a/enterprise/neo4j-enterprise/LICENSES.txt +++ b/enterprise/neo4j-enterprise/LICENSES.txt @@ -10,6 +10,7 @@ Apache Software License, Version 2.0 hazelcast-all Lucene Common Analyzers Lucene Core + Lucene Memory Lucene QueryParsers Metrics Core Netty/All-in-One diff --git a/enterprise/neo4j-enterprise/NOTICE.txt b/enterprise/neo4j-enterprise/NOTICE.txt index b684d6754fb6c..553b475710905 100644 --- a/enterprise/neo4j-enterprise/NOTICE.txt +++ b/enterprise/neo4j-enterprise/NOTICE.txt @@ -32,6 +32,7 @@ Apache Software License, Version 2.0 hazelcast-all Lucene Common Analyzers Lucene Core + Lucene Memory Lucene QueryParsers Metrics Core Netty/All-in-One diff --git a/enterprise/query-logging/LICENSES.txt b/enterprise/query-logging/LICENSES.txt index e62824d7096a5..39bcd6a643262 100644 --- a/enterprise/query-logging/LICENSES.txt +++ b/enterprise/query-logging/LICENSES.txt @@ -1,4 +1,213 @@ This file contains the full license text of the included third party libraries. For an overview of the licenses see the NOTICE.txt file. +------------------------------------------------------------------------------ +Apache Software License, Version 2.0 + Lucene Core + Lucene Memory +------------------------------------------------------------------------------ + + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright [yyyy] [name of copyright owner] + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + + diff --git a/enterprise/query-logging/NOTICE.txt b/enterprise/query-logging/NOTICE.txt index edeb6ad15811d..f21c64154d711 100644 --- a/enterprise/query-logging/NOTICE.txt +++ b/enterprise/query-logging/NOTICE.txt @@ -24,3 +24,7 @@ Full license texts are found in LICENSES.txt. Third-party licenses -------------------- +Apache Software License, Version 2.0 + Lucene Core + Lucene Memory + diff --git a/enterprise/server-enterprise/LICENSES.txt b/enterprise/server-enterprise/LICENSES.txt index ef38163f58535..79f15ab36c2a5 100644 --- a/enterprise/server-enterprise/LICENSES.txt +++ b/enterprise/server-enterprise/LICENSES.txt @@ -27,6 +27,7 @@ Apache Software License, Version 2.0 Jetty :: XML utilities Lucene Common Analyzers Lucene Core + Lucene Memory Lucene QueryParsers Metrics Core Netty/All-in-One diff --git a/enterprise/server-enterprise/NOTICE.txt b/enterprise/server-enterprise/NOTICE.txt index 48c9caa665df1..3e8a94af07108 100644 --- a/enterprise/server-enterprise/NOTICE.txt +++ b/enterprise/server-enterprise/NOTICE.txt @@ -49,6 +49,7 @@ Apache Software License, Version 2.0 Jetty :: XML utilities Lucene Common Analyzers Lucene Core + Lucene Memory Lucene QueryParsers Metrics Core Netty/All-in-One diff --git a/packaging/standalone/NOTICE.txt b/packaging/standalone/NOTICE.txt index 137f3ab2f325d..9c70cf4f1e16b 100644 --- a/packaging/standalone/NOTICE.txt +++ b/packaging/standalone/NOTICE.txt @@ -7,16 +7,16 @@ This product includes software ("Software") developed by Neo Technology. The copyright in the bundled Neo4j graph database (including the Software) is owned by Neo Technology. The Software developed and owned -by Neo Technology is licensed under the GNU AFFERO GENERAL PUBLIC -LICENSE Version 3 (http://www.fsf.org/licensing/licenses/agpl-3.0.html) -("AGPL") to all third parties and that license, as required by the AGPL, -is included in the LICENSE.txt file. +by Neo Technology is licensed under the GNU GENERAL PUBLIC LICENSE +Version 3 (http://www.fsf.org/licensing/licenses/gpl-3.0.html) ("GPL") +to all third parties and that license, as required by the GPL, is +included in the LICENSE.txt file. However, if you have executed an End User Software License and Services Agreement or an OEM Software License and Support Services Agreement, or another commercial license agreement with Neo Technology or one of its affiliates (each, a "Commercial Agreement"), the terms of the license in -such Commercial Agreement will supersede the AGPL and you may use the +such Commercial Agreement will supersede the GPL and you may use the software solely pursuant to the terms of the relevant Commercial Agreement. @@ -25,18 +25,11 @@ Full license texts are found in LICENSES.txt. Third-party licenses -------------------- -ASM License - ASM Core - -Academic Free License version 3.0 - Pure Feature Detection Library (has.js) - Apache Software License, Version 2.0 - Apache ServiceMix Bundles: jline-0.9.94 + Apache Commons Configuration + Apache Commons Lang + ColorBrewer Commons BeanUtils - Commons BeanUtils Core - Commons Collections - Commons Configuration Commons Digester Commons IO Commons Lang @@ -44,19 +37,23 @@ Apache Software License, Version 2.0 ConcurrentLinkedHashMap Data Mapper for Jackson ExplorerCanvas - Groovy Jackson - jansi JAX-RS provider for JSON content type - Jettison - Jetty Server - Jetty Utilities + Jetty :: Http Utility + Jetty :: IO Utility + Jetty :: Security + Jetty :: Server Core + Jetty :: Servlet Handling + Jetty :: Utilities + Jetty :: Webapp Application Support + Jetty :: XML utilities Lucene Core + Lucene Memory + Netty/All-in-One + opencsv parboiled-core parboiled-scala RRD4J - Servlet Specification API - StAX API BSD - Scala License Scala Library @@ -65,27 +62,23 @@ BSD - seedrandom.js seedrandom.js BSD License - AntLR Parser Generator - ASM Analysis - ASM Commons - ASM Tree - ASM Util - Blueprints-Core - Blueprints-Neo4jGraph + ASM Core + D3.js Easing Equations (in easing.js) - Gremlin-Groovy: Gremlin for the Groovy Language - Gremlin-Java: Gremlin for the Java Language - Janino - Pipes: A Lazy Data Flow Framework - Pure Feature Detection Library (has.js) + JLine + JQuery Easing Plugin + Scala Compiler Bouncy Castle License - Legion of the Bouncy Castle Java Cryptography APIs + Bouncy Castle PKIX, CMS, EAC, TSP, PKCS, OCSP, CMP, and CRMF APIs + Bouncy Castle Provider Common Development and Distribution License Version 1.1 + Java Servlet API jersey-core jersey-multipart jersey-server + jersey-servlet jsr311-api MIME streaming extension @@ -93,52 +86,73 @@ Creative Commons - Glyphish Glyphish Icons Free Eclipse Public License, Version 1.0 - Jetty Server - Jetty Utilities + Jetty :: Http Utility + Jetty :: IO Utility + Jetty :: Security + Jetty :: Server Core + Jetty :: Servlet Handling + Jetty :: Utilities + Jetty :: Webapp Application Support + Jetty :: XML utilities Logback Access Module Logback Classic Module Logback Core Module -GNU General Public License, version 2 - Color picker - JQuery - jquery.hotkeys.js - jquery.putCursorAtEnd.js - jquery.sortable.js - -GNU General Public License, version 2 or later - Mozilla Rhino - GNU General Public License, version 2 with the Classpath Exception + Java Servlet API jersey-core jersey-multipart jersey-server + jersey-servlet jsr311-api MIME streaming extension GNU Lesser General Public License, Version 2.1 - JAX-RS provider for JSON content type Logback Access Module Logback Classic Module Logback Core Module MIT License + angular-animate + angular-base64 + angular-cookies + angular-local-storage + angular-mocks + angular-resource + angular-sanitize + angular-scenario + angular-ui-bootstrap + angular-ui-codemirror + angular-ui-sortable + AngularJS arbor-graphics.js arbor-tween.js arbor.js + Blob.js + bootstrap + canvg + CodeMirror + CodeMirror CSS Color picker Date Format + es5-shim + FileSaver.js JQuery + jquery-ui jquery.colorhelpers.js jquery.flot.js jquery.hotkeys.js jquery.putCursorAtEnd.js jquery.sortable.js + JSON3 + local-storage-js + Modernizr Pure Feature Detection Library (has.js) SLF4J API Module Underscore.js + UUID.js -Mozilla Public License, Version 1.1 +Mozilla Public License, Version 2.0 Mozilla Rhino Public Domain - JSON2 @@ -150,26 +164,45 @@ Public Domain - RGB Color Public Domain - Tango Tango Icons +SIL OFL 1.1 + Font Awesome + Dependencies with multiple licenses ----------------------------------- -Color picker - GNU General Public License, version 2 - MIT License +Java Servlet API + Common Development and Distribution License Version 1.1 + GNU General Public License, version 2 with the Classpath Exception -JAX-RS provider for JSON content type +Jetty :: Http Utility Apache Software License, Version 2.0 - GNU Lesser General Public License, Version 2.1 + Eclipse Public License, Version 1.0 -JQuery - GNU General Public License, version 2 - MIT License +Jetty :: IO Utility + Apache Software License, Version 2.0 + Eclipse Public License, Version 1.0 + +Jetty :: Security + Apache Software License, Version 2.0 + Eclipse Public License, Version 1.0 + +Jetty :: Server Core + Apache Software License, Version 2.0 + Eclipse Public License, Version 1.0 + +Jetty :: Servlet Handling + Apache Software License, Version 2.0 + Eclipse Public License, Version 1.0 -Jetty Server +Jetty :: Utilities Apache Software License, Version 2.0 Eclipse Public License, Version 1.0 -Jetty Utilities +Jetty :: Webapp Application Support + Apache Software License, Version 2.0 + Eclipse Public License, Version 1.0 + +Jetty :: XML utilities Apache Software License, Version 2.0 Eclipse Public License, Version 1.0 @@ -189,15 +222,6 @@ MIME streaming extension Common Development and Distribution License Version 1.1 GNU General Public License, version 2 with the Classpath Exception -Mozilla Rhino - GNU General Public License, version 2 or later - Mozilla Public License, Version 1.1 - -Pure Feature Detection Library (has.js) - Academic Free License version 3.0 - BSD License - MIT License - jersey-core Common Development and Distribution License Version 1.1 GNU General Public License, version 2 with the Classpath Exception @@ -210,17 +234,9 @@ jersey-server Common Development and Distribution License Version 1.1 GNU General Public License, version 2 with the Classpath Exception -jquery.hotkeys.js - GNU General Public License, version 2 - MIT License - -jquery.putCursorAtEnd.js - GNU General Public License, version 2 - MIT License - -jquery.sortable.js - GNU General Public License, version 2 - MIT License +jersey-servlet + Common Development and Distribution License Version 1.1 + GNU General Public License, version 2 with the Classpath Exception jsr311-api Common Development and Distribution License Version 1.1 diff --git a/packaging/standalone/standalone-advanced/src/main/assemblies/advanced-unix-dist.xml b/packaging/standalone/standalone-advanced/src/main/assemblies/advanced-unix-dist.xml index cc846875f3b2b..77f098c1f6f92 100644 --- a/packaging/standalone/standalone-advanced/src/main/assemblies/advanced-unix-dist.xml +++ b/packaging/standalone/standalone-advanced/src/main/assemblies/advanced-unix-dist.xml @@ -110,6 +110,8 @@ org.neo4j:* jline:jline org.apache.lucene:lucene-core + org.apache.lucene:lucene-analyzers-common + org.apache.lucene:lucene-queryparser org.scala-lang:* org.parboiled:* com.googlecode.concurrentlinkedhashmap:concurrentlinkedhashmap-lru @@ -134,6 +136,8 @@ org.neo4j.doc:* jline:jline org.apache.lucene:lucene-core + org.apache.lucene:lucene-analyzers-common + org.apache.lucene:lucene-queryparser org.scala-lang:* org.parboiled:* org.neo4j.app:windows-service-wrapper:* diff --git a/packaging/standalone/standalone-advanced/src/main/assemblies/advanced-windows-dist.xml b/packaging/standalone/standalone-advanced/src/main/assemblies/advanced-windows-dist.xml index 60f982fe7900e..94d00f0a74105 100644 --- a/packaging/standalone/standalone-advanced/src/main/assemblies/advanced-windows-dist.xml +++ b/packaging/standalone/standalone-advanced/src/main/assemblies/advanced-windows-dist.xml @@ -99,6 +99,8 @@ org.neo4j:* jline:jline org.apache.lucene:lucene-core + org.apache.lucene:lucene-analyzers-common + org.apache.lucene:lucene-queryparser org.scala-lang:* org.parboiled:* com.googlecode.concurrentlinkedhashmap:concurrentlinkedhashmap-lru @@ -121,6 +123,8 @@ org.neo4j.doc:* jline:jline org.apache.lucene:lucene-core + org.apache.lucene:lucene-analyzers-common + org.apache.lucene:lucene-queryparser org.scala-lang:* org.parboiled:* org.neo4j.app:windows-service-wrapper:* diff --git a/packaging/standalone/standalone-advanced/src/main/distribution/text/advanced/LICENSES.txt b/packaging/standalone/standalone-advanced/src/main/distribution/text/advanced/LICENSES.txt index d3b009d0b89c7..ce5a9a70a86ba 100644 --- a/packaging/standalone/standalone-advanced/src/main/distribution/text/advanced/LICENSES.txt +++ b/packaging/standalone/standalone-advanced/src/main/distribution/text/advanced/LICENSES.txt @@ -25,6 +25,7 @@ Apache Software License, Version 2.0 Jetty :: Webapp Application Support Jetty :: XML utilities Lucene Core + Lucene Memory Netty/All-in-One opencsv parboiled-core diff --git a/packaging/standalone/standalone-advanced/src/main/distribution/text/advanced/NOTICE.txt b/packaging/standalone/standalone-advanced/src/main/distribution/text/advanced/NOTICE.txt index 2e8dbae35db72..fdccae8467681 100644 --- a/packaging/standalone/standalone-advanced/src/main/distribution/text/advanced/NOTICE.txt +++ b/packaging/standalone/standalone-advanced/src/main/distribution/text/advanced/NOTICE.txt @@ -47,6 +47,7 @@ Apache Software License, Version 2.0 Jetty :: Webapp Application Support Jetty :: XML utilities Lucene Core + Lucene Memory Netty/All-in-One opencsv parboiled-core diff --git a/packaging/standalone/standalone-community/src/main/assemblies/community-unix-dist.xml b/packaging/standalone/standalone-community/src/main/assemblies/community-unix-dist.xml index 2955f39a4bd75..e7da9cc95a226 100644 --- a/packaging/standalone/standalone-community/src/main/assemblies/community-unix-dist.xml +++ b/packaging/standalone/standalone-community/src/main/assemblies/community-unix-dist.xml @@ -97,6 +97,8 @@ org.neo4j:* jline:jline org.apache.lucene:lucene-core + org.apache.lucene:lucene-analyzers-common + org.apache.lucene:lucene-queryparser org.scala-lang:* org.parboiled:* com.googlecode.concurrentlinkedhashmap:concurrentlinkedhashmap-lru @@ -120,6 +122,8 @@ org.neo4j.doc:* jline:jline org.apache.lucene:lucene-core + org.apache.lucene:lucene-analyzers-common + org.apache.lucene:lucene-queryparser org.scala-lang:* org.parboiled:* org.neo4j.app:windows-service-wrapper:* diff --git a/packaging/standalone/standalone-community/src/main/assemblies/community-windows-dist.xml b/packaging/standalone/standalone-community/src/main/assemblies/community-windows-dist.xml index f24fdd4cdfa7c..e80be06729356 100644 --- a/packaging/standalone/standalone-community/src/main/assemblies/community-windows-dist.xml +++ b/packaging/standalone/standalone-community/src/main/assemblies/community-windows-dist.xml @@ -98,6 +98,8 @@ org.neo4j:* jline:jline org.apache.lucene:lucene-core + org.apache.lucene:lucene-analyzers-common + org.apache.lucene:lucene-queryparser org.scala-lang:* org.parboiled:* com.googlecode.concurrentlinkedhashmap:concurrentlinkedhashmap-lru @@ -119,6 +121,8 @@ org.neo4j.doc:* jline:jline org.apache.lucene:lucene-core + org.apache.lucene:lucene-analyzers-common + org.apache.lucene:lucene-queryparser org.scala-lang:* org.parboiled:* org.neo4j.app:windows-service-wrapper:* diff --git a/packaging/standalone/standalone-community/src/main/distribution/text/community/LICENSES.txt b/packaging/standalone/standalone-community/src/main/distribution/text/community/LICENSES.txt index d3b009d0b89c7..ce5a9a70a86ba 100644 --- a/packaging/standalone/standalone-community/src/main/distribution/text/community/LICENSES.txt +++ b/packaging/standalone/standalone-community/src/main/distribution/text/community/LICENSES.txt @@ -25,6 +25,7 @@ Apache Software License, Version 2.0 Jetty :: Webapp Application Support Jetty :: XML utilities Lucene Core + Lucene Memory Netty/All-in-One opencsv parboiled-core diff --git a/packaging/standalone/standalone-community/src/main/distribution/text/community/NOTICE.txt b/packaging/standalone/standalone-community/src/main/distribution/text/community/NOTICE.txt index 7e02320f8fa4d..e15f734cdd422 100644 --- a/packaging/standalone/standalone-community/src/main/distribution/text/community/NOTICE.txt +++ b/packaging/standalone/standalone-community/src/main/distribution/text/community/NOTICE.txt @@ -48,6 +48,7 @@ Apache Software License, Version 2.0 Jetty :: Webapp Application Support Jetty :: XML utilities Lucene Core + Lucene Memory Netty/All-in-One opencsv parboiled-core diff --git a/packaging/standalone/standalone-enterprise/src/main/assemblies/enterprise-unix-dist.xml b/packaging/standalone/standalone-enterprise/src/main/assemblies/enterprise-unix-dist.xml index 714d09252dbdf..4d7da10c61083 100644 --- a/packaging/standalone/standalone-enterprise/src/main/assemblies/enterprise-unix-dist.xml +++ b/packaging/standalone/standalone-enterprise/src/main/assemblies/enterprise-unix-dist.xml @@ -105,6 +105,8 @@ org.neo4j:* jline:jline org.apache.lucene:lucene-core + org.apache.lucene:lucene-analyzers-common + org.apache.lucene:lucene-queryparser io.netty:netty org.scala-lang:* org.parboiled:* @@ -135,6 +137,8 @@ org.neo4j.doc:* jline:jline org.apache.lucene:lucene-core + org.apache.lucene:lucene-analyzers-common + org.apache.lucene:lucene-queryparser io.netty:netty org.scala-lang:* org.parboiled:* diff --git a/packaging/standalone/standalone-enterprise/src/main/distribution/text/enterprise/LICENSES.txt b/packaging/standalone/standalone-enterprise/src/main/distribution/text/enterprise/LICENSES.txt index 29df548bd2d34..b539c6c476d38 100644 --- a/packaging/standalone/standalone-enterprise/src/main/distribution/text/enterprise/LICENSES.txt +++ b/packaging/standalone/standalone-enterprise/src/main/distribution/text/enterprise/LICENSES.txt @@ -28,6 +28,7 @@ Apache Software License, Version 2.0 Jetty :: Webapp Application Support Jetty :: XML utilities Lucene Core + Lucene Memory Metrics Core Netty/All-in-One opencsv diff --git a/packaging/standalone/standalone-enterprise/src/main/distribution/text/enterprise/NOTICE.txt b/packaging/standalone/standalone-enterprise/src/main/distribution/text/enterprise/NOTICE.txt index d98a1dbbad66b..4b73aafe1c055 100644 --- a/packaging/standalone/standalone-enterprise/src/main/distribution/text/enterprise/NOTICE.txt +++ b/packaging/standalone/standalone-enterprise/src/main/distribution/text/enterprise/NOTICE.txt @@ -50,6 +50,7 @@ Apache Software License, Version 2.0 Jetty :: Webapp Application Support Jetty :: XML utilities Lucene Core + Lucene Memory Metrics Core Netty/All-in-One opencsv