Skip to content

Commit

Permalink
fixing even more backup port conflicts
Browse files Browse the repository at this point in the history
  • Loading branch information
@lassewesth
lassewesth committed Aug 18, 2017
1 parent ca75fae commit 550653d
Show file tree
Hide file tree
Showing 9 changed files with 162 additions and 78 deletions.
17 changes: 16 additions & 1 deletion enterprise/backup/pom.xml
View file
Expand Up @@ -50,9 +50,17 @@
<build>
<plugins>
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-failsafe-plugin</artifactId>
<configuration>
<forkCount>1</forkCount>
<!-- Integration tests spend significant time waiting, so we can run more things in parallel here-->
<forkCount>2C</forkCount>
<!-- Misha says it is not safe to run tests with more than one thread per JVM -->
<!-- He also said he'd fix that! -->
<threadCount>1</threadCount>
<!-- slowest test class takes up to 10 minutes, so let's cater for that, with contingency -->
<parallelTestsTimeoutInSeconds>1200</parallelTestsTimeoutInSeconds>
<parallelTestsTimeoutForcedInSeconds>1800</parallelTestsTimeoutForcedInSeconds>
</configuration>
</plugin>
<plugin>
Expand Down Expand Up @@ -172,5 +180,12 @@
<artifactId>neo4j-kernel</artifactId>
<version>${project.version}</version>
</dependency>
<dependency>
<groupId>org.neo4j</groupId>
<artifactId>neo4j-com</artifactId>
<version>${project.version}</version>
<type>test-jar</type>
<scope>test</scope>
</dependency>
</dependencies>
</project>
106 changes: 64 additions & 42 deletions enterprise/backup/src/test/java/org/neo4j/backup/BackupIT.java
View file

Large diffs are not rendered by default.

3 changes: 2 additions & 1 deletion enterprise/backup/src/test/java/org/neo4j/backup/BackupProtocolIT.java
View file
Expand Up @@ -26,6 +26,7 @@
import org.neo4j.com.Response;
import org.neo4j.com.TargetCaller;
import org.neo4j.com.monitor.RequestMonitor;
import org.neo4j.com.ports.allocation.PortAuthority;
import org.neo4j.com.storecopy.ResponseUnpacker;
import org.neo4j.com.storecopy.StoreWriter;
import org.neo4j.helpers.HostnamePort;
Expand Down Expand Up @@ -76,7 +77,7 @@ private void shouldGatherForensicsInFullBackupRequest( boolean forensics ) throw
Response<Void> response = Response.EMPTY;
StoreId storeId = response.getStoreId();
String host = "localhost";
int port = BackupServer.DEFAULT_PORT;
int port = PortAuthority.allocatePort();
LifeSupport life = new LifeSupport();

LogEntryReader<ReadableClosablePositionAwareChannel> reader = new VersionAwareLogEntryReader<>();
Expand Down
16 changes: 10 additions & 6 deletions enterprise/backup/src/test/java/org/neo4j/backup/BackupServiceIT.java
View file
Expand Up @@ -35,6 +35,7 @@
import java.util.concurrent.Executors;
import java.util.concurrent.TimeUnit;

import org.neo4j.com.ports.allocation.PortAuthority;
import org.neo4j.com.storecopy.StoreCopyServer;
import org.neo4j.com.storecopy.StoreUtil;
import org.neo4j.consistency.checking.full.CheckConsistencyConfig;
Expand All @@ -55,6 +56,7 @@
import org.neo4j.io.pagecache.IOLimiter;
import org.neo4j.io.pagecache.PageCache;
import org.neo4j.kernel.configuration.Config;
import org.neo4j.kernel.configuration.Settings;
import org.neo4j.kernel.impl.factory.DatabaseInfo;
import org.neo4j.kernel.impl.logging.LogService;
import org.neo4j.kernel.impl.spi.SimpleKernelContext;
Expand Down Expand Up @@ -123,7 +125,7 @@ public class BackupServiceIT
private FileSystemAbstraction fileSystem;
private File storeDir;
private File backupDir;
private int backupPort = 8200;
private int backupPort = -1;

private final DefaultFileSystemRule fileSystemRule = new DefaultFileSystemRule();
private final TestDirectory target = TestDirectory.testDirectory();
Expand All @@ -142,7 +144,7 @@ public class BackupServiceIT
public void setup()
{
fileSystem = fileSystemRule.get();
backupPort = backupPort + 1;
backupPort = PortAuthority.allocatePort();
storeDir = dbRule.getStoreDirFile();
backupDir = target.directory( "backup_dir" );
}
Expand Down Expand Up @@ -475,7 +477,9 @@ public void incrementallyBackupDatabaseShouldNotKeepGeneratedIdFiles()
// it should be possible to at this point to start db based on our backup and create couple of properties
// their ids should not clash with already existing
GraphDatabaseService backupBasedDatabase = new TestGraphDatabaseFactory()
.newEmbeddedDatabase( backupDir.getAbsoluteFile() );
.newEmbeddedDatabaseBuilder( backupDir.getAbsoluteFile() )
.setConfig( OnlineBackupSettings.online_backup_enabled, Settings.FALSE )
.newGraphDatabase();
try
{
try ( Transaction transaction = backupBasedDatabase.beginTx() )
Expand All @@ -490,7 +494,7 @@ public void incrementallyBackupDatabaseShouldNotKeepGeneratedIdFiles()
transaction.success();
}

try ( Transaction transaction = backupBasedDatabase.beginTx() )
try ( Transaction ignored = backupBasedDatabase.beginTx() )
{
Node node = findNodeByLabel( (GraphDatabaseAPI) backupBasedDatabase, markerLabel );
// newProperty + 10 defined properties.
Expand Down Expand Up @@ -1068,12 +1072,12 @@ private Node findNodeByLabel( GraphDatabaseAPI graphDatabase, Label label )

private DbRepresentation getBackupDbRepresentation()
{
return DbRepresentation.of( backupDir );
return DbRepresentation.of( backupDir, Config.defaults( OnlineBackupSettings.online_backup_enabled, Settings.FALSE ) );
}

private DbRepresentation getDbRepresentation()
{
return DbRepresentation.of( storeDir );
return DbRepresentation.of( storeDir, Config.defaults( OnlineBackupSettings.online_backup_enabled, Settings.FALSE ) );
}

private static final class StoreSnoopingMonitor extends StoreCopyServer.Monitor.Adapter
Expand Down
35 changes: 29 additions & 6 deletions enterprise/backup/src/test/java/org/neo4j/backup/BackupToolIT.java
View file
Expand Up @@ -29,16 +29,23 @@
import java.io.PrintStream;
import java.nio.file.Path;

import org.neo4j.com.ports.allocation.PortAuthority;
import org.neo4j.graphdb.GraphDatabaseService;
import org.neo4j.graphdb.factory.GraphDatabaseSettings;
import org.neo4j.helpers.HostnamePort;
import org.neo4j.io.fs.DefaultFileSystemAbstraction;
import org.neo4j.io.pagecache.PageCache;
import org.neo4j.io.pagecache.impl.muninn.StandalonePageCacheFactory;
import org.neo4j.kernel.configuration.Config;
import org.neo4j.kernel.configuration.Settings;
import org.neo4j.kernel.impl.store.MetaDataStore;
import org.neo4j.kernel.impl.store.format.standard.StandardV2_3;
import org.neo4j.test.DbRepresentation;
import org.neo4j.test.TestGraphDatabaseFactory;
import org.neo4j.test.rule.EmbeddedDatabaseRule;
import org.neo4j.test.rule.TestDirectory;

import static org.junit.Assert.assertEquals;
import static org.mockito.Mockito.mock;
import static org.neo4j.kernel.impl.store.MetaDataStore.Position.STORE_VERSION;

Expand Down Expand Up @@ -79,14 +86,30 @@ public void oldIncompatibleBackupsThrows() throws Exception
prepareNeoStoreFile( StandardV2_3.STORE_VERSION );

// Start database to backup
dbRule.getGraphDatabaseAPI();
int backupPort = PortAuthority.allocatePort();
GraphDatabaseService db = startGraphDatabase( backupPort );
try
{
expected.expect( BackupTool.ToolFailureException.class );
expected.expectMessage( "Failed to perform backup because existing backup is from a different version." );

expected.expect( BackupTool.ToolFailureException.class );
expected.expectMessage( "Failed to perform backup because existing backup is from a different version." );
// Perform backup
backupTool.executeBackup( new HostnamePort( "localhost", backupPort ), backupDir.toFile(),
ConsistencyCheck.NONE, Config.defaults(), 20L * 60L * 1000L, false );
}
finally
{
db.shutdown();
}
}

// Perform backup
backupTool.executeBackup( new HostnamePort( "localhost", 6362 ), backupDir.toFile(),
ConsistencyCheck.NONE, Config.defaults(), 20L * 60L * 1000L, false );
private GraphDatabaseService startGraphDatabase( int backupPort )
{
return new TestGraphDatabaseFactory().newEmbeddedDatabaseBuilder( testDirectory.directory() )
.setConfig( OnlineBackupSettings.online_backup_enabled, Settings.TRUE )
.setConfig( OnlineBackupSettings.online_backup_server, "127.0.0.1:" + backupPort )
.setConfig( GraphDatabaseSettings.keep_logical_logs, Settings.TRUE )
.newGraphDatabase();
}

private void prepareNeoStoreFile( String storeVersion ) throws Exception
Expand Down
18 changes: 11 additions & 7 deletions enterprise/backup/src/test/java/org/neo4j/backup/IncrementalBackupIT.java
View file
Expand Up @@ -27,6 +27,7 @@

import java.io.File;

import org.neo4j.com.ports.allocation.PortAuthority;
import org.neo4j.graphdb.Direction;
import org.neo4j.graphdb.DynamicRelationshipType;
import org.neo4j.graphdb.GraphDatabaseService;
Expand All @@ -35,6 +36,7 @@
import org.neo4j.graphdb.RelationshipType;
import org.neo4j.graphdb.Transaction;
import org.neo4j.graphdb.factory.GraphDatabaseSettings;
import org.neo4j.kernel.configuration.Config;
import org.neo4j.kernel.configuration.Settings;
import org.neo4j.test.DbRepresentation;
import org.neo4j.test.TestGraphDatabaseFactory;
Expand Down Expand Up @@ -84,17 +86,18 @@ public void shutItDown() throws Exception
public void shouldDoIncrementalBackup() throws Exception
{
DbRepresentation initialDataSetRepresentation = createInitialDataSet( serverPath );
server = startServer( serverPath, "127.0.0.1:6362" );
int port = PortAuthority.allocatePort();
server = startServer( serverPath, "127.0.0.1:" + port );

OnlineBackup backup = OnlineBackup.from( "127.0.0.1" );
OnlineBackup backup = OnlineBackup.from( "127.0.0.1", port );

backup.full( backupPath.getPath() );

assertEquals( initialDataSetRepresentation, getBackupDbRepresentation() );
shutdownServer( server );

DbRepresentation furtherRepresentation = addMoreData2( serverPath );
server = startServer( serverPath, null );
server = startServer( serverPath, "127.0.0.1:" + port );
backup.incremental( backupPath.getPath() );
assertEquals( furtherRepresentation, getBackupDbRepresentation() );
shutdownServer( server );
Expand Down Expand Up @@ -123,17 +126,18 @@ public void shouldNotServeTransactionsWithInvalidHighIds() throws Exception
* Note that this problem can also happen in HA slaves.
*/
DbRepresentation initialDataSetRepresentation = createInitialDataSet( serverPath );
server = startServer( serverPath, "127.0.0.1:6362" );
int port = PortAuthority.allocatePort();
server = startServer( serverPath, "127.0.0.1:" + port );

OnlineBackup backup = OnlineBackup.from( "127.0.0.1" );
OnlineBackup backup = OnlineBackup.from( "127.0.0.1", port);

backup.full( backupPath.getPath() );

assertEquals( initialDataSetRepresentation, getBackupDbRepresentation() );
shutdownServer( server );

DbRepresentation furtherRepresentation = createTransactiongWithWeirdRelationshipGroupRecord( serverPath );
server = startServer( serverPath, null );
server = startServer( serverPath, "127.0.0.1:" + port );
backup.incremental( backupPath.getPath() );
assertEquals( furtherRepresentation, getBackupDbRepresentation() );
shutdownServer( server );
Expand Down Expand Up @@ -235,6 +239,6 @@ private void shutdownServer( ServerInterface server ) throws Exception

private DbRepresentation getBackupDbRepresentation()
{
return DbRepresentation.of( backupPath );
return DbRepresentation.of( backupPath, Config.defaults( OnlineBackupSettings.online_backup_enabled, Settings.FALSE ) );
}
}
6 changes: 4 additions & 2 deletions ...o4j/backup/TestOnlineBackupExtension.java → ...neo4j/backup/OnlineBackupExtensionIT.java
View file
Expand Up @@ -21,12 +21,13 @@

import java.util.Map;

import org.neo4j.com.ports.allocation.PortAuthority;
import org.neo4j.kernel.configuration.Settings;
import org.neo4j.kernel.extension.KernelExtensionFactoryContractTest;

public class TestOnlineBackupExtension extends KernelExtensionFactoryContractTest
public class OnlineBackupExtensionIT extends KernelExtensionFactoryContractTest
{
public TestOnlineBackupExtension()
public OnlineBackupExtensionIT()
{
super( OnlineBackupExtensionFactory.KEY, OnlineBackupExtensionFactory.class );
}
Expand All @@ -38,6 +39,7 @@ protected Map<String, String> configuration( boolean shouldLoad, int instance )
if ( shouldLoad )
{
configuration.put( OnlineBackupSettings.online_backup_enabled.name(), Settings.TRUE );
configuration.put( OnlineBackupSettings.online_backup_server.name(), "127.0.0.1:" + PortAuthority.allocatePort() );
}
return configuration;
}
Expand Down
26 changes: 16 additions & 10 deletions enterprise/backup/src/test/java/org/neo4j/backup/TestConfiguration.java
View file
Expand Up @@ -26,6 +26,7 @@

import java.io.File;

import org.neo4j.com.ports.allocation.PortAuthority;
import org.neo4j.graphdb.GraphDatabaseService;
import org.neo4j.kernel.configuration.Settings;
import org.neo4j.test.TestGraphDatabaseFactory;
Expand All @@ -41,7 +42,6 @@ public class TestConfiguration
private static final File SOURCE_DIR = new File( "target/db" );
private static final String BACKUP_DIR = "target/full-backup";
private static final String HOST_ADDRESS = "127.0.0.1";
private static final int PORT = 6365;

@Before
public void before() throws Exception
Expand All @@ -53,22 +53,26 @@ public void before() throws Exception
@Test
public void testOnByDefault() throws Exception
{
int port = PortAuthority.allocatePort();

GraphDatabaseService db = new TestGraphDatabaseFactory().newEmbeddedDatabaseBuilder( SOURCE_DIR )
.setConfig( OnlineBackupSettings.online_backup_server, "localhost:" + PORT ).newGraphDatabase();
OnlineBackup.from( HOST_ADDRESS, PORT ).full( BACKUP_DIR );
.setConfig( OnlineBackupSettings.online_backup_server, "localhost:" + port ).newGraphDatabase();
OnlineBackup.from( HOST_ADDRESS, port ).full( BACKUP_DIR );
db.shutdown();
}

@Test
public void testOffByConfig() throws Exception
{
int port = PortAuthority.allocatePort();

GraphDatabaseService db = new TestGraphDatabaseFactory().newEmbeddedDatabaseBuilder( SOURCE_DIR )
.setConfig( OnlineBackupSettings.online_backup_enabled, Settings.FALSE )
.setConfig( OnlineBackupSettings.online_backup_server, "localhost:" + PORT )
.setConfig( OnlineBackupSettings.online_backup_server, "localhost:" + port )
.newGraphDatabase();
try
{
OnlineBackup.from( HOST_ADDRESS, PORT ).full( BACKUP_DIR );
OnlineBackup.from( HOST_ADDRESS, port ).full( BACKUP_DIR );
fail( "Shouldn't be possible" );
}
catch ( Exception e )
Expand All @@ -80,33 +84,35 @@ public void testOffByConfig() throws Exception
@Test
public void testEnableDefaultsInConfig() throws Exception
{
int port = PortAuthority.allocatePort();

GraphDatabaseService db = new TestGraphDatabaseFactory().newEmbeddedDatabaseBuilder( SOURCE_DIR )
.setConfig( OnlineBackupSettings.online_backup_enabled, Settings.TRUE )
.setConfig( OnlineBackupSettings.online_backup_server, "localhost:" + PORT )
.setConfig( OnlineBackupSettings.online_backup_server, "localhost:" + port )
.newGraphDatabase();

OnlineBackup.from( HOST_ADDRESS, PORT ).full( BACKUP_DIR );
OnlineBackup.from( HOST_ADDRESS, port ).full( BACKUP_DIR );
db.shutdown();
}

@Test
public void testEnableCustomPortInConfig() throws Exception
{
String customPort = "12345";
int customPort = PortAuthority.allocatePort();
GraphDatabaseService db = new TestGraphDatabaseFactory().newEmbeddedDatabaseBuilder( SOURCE_DIR )
.setConfig( OnlineBackupSettings.online_backup_enabled, Settings.TRUE )
.setConfig( OnlineBackupSettings.online_backup_server, ":" + customPort )
.newGraphDatabase();
try
{
OnlineBackup.from( HOST_ADDRESS, 12346 ).full( BACKUP_DIR );
OnlineBackup.from( HOST_ADDRESS, PortAuthority.allocatePort() ).full( BACKUP_DIR );
fail( "Shouldn't be possible" );
}
catch ( Exception e )
{ // Good
}

OnlineBackup.from( HOST_ADDRESS, Integer.parseInt(customPort) ).full( BACKUP_DIR );
OnlineBackup.from( HOST_ADDRESS, customPort ).full( BACKUP_DIR );
db.shutdown();
}
}

0 comments on commit 550653d

Please sign in to comment.