Skip to content

Commit

Permalink
Converted most of HaBeanIT to unit test [backport from 3.1]
Browse files Browse the repository at this point in the history
  • Loading branch information
@tinwelint @davidegrohmann
tinwelint authored and davidegrohmann committed Sep 21, 2016
1 parent debfe02 commit 02db63a
Show file tree
Hide file tree
Showing 5 changed files with 342 additions and 243 deletions.
2 changes: 1 addition & 1 deletion community/jmx/src/main/java/org/neo4j/jmx/impl/ManagementData.java
View file
Expand Up @@ -30,7 +30,7 @@ public final class ManagementData extends DependencyResolver.Adapter
private final ManagementSupport support; private final ManagementSupport support;
final ManagementBeanProvider provider; final ManagementBeanProvider provider;


ManagementData( ManagementBeanProvider provider, KernelData kernel, ManagementSupport support ) public ManagementData( ManagementBeanProvider provider, KernelData kernel, ManagementSupport support )
{ {
this.provider = provider; this.provider = provider;
this.kernel = kernel; this.kernel = kernel;
Expand Down
11 changes: 5 additions & 6 deletions community/jmx/src/main/java/org/neo4j/jmx/impl/ManagementSupport.java
View file
Expand Up @@ -19,12 +19,9 @@
*/ */
package org.neo4j.jmx.impl; package org.neo4j.jmx.impl;


import static java.lang.management.ManagementFactory.getPlatformMBeanServer;

import java.util.ArrayList; import java.util.ArrayList;
import java.util.Collection; import java.util.Collection;
import java.util.Hashtable; import java.util.Hashtable;

import javax.management.MBeanServer; import javax.management.MBeanServer;
import javax.management.MalformedObjectNameException; import javax.management.MalformedObjectNameException;
import javax.management.ObjectName; import javax.management.ObjectName;
Expand All @@ -34,9 +31,11 @@
import org.neo4j.jmx.ManagementInterface; import org.neo4j.jmx.ManagementInterface;
import org.neo4j.kernel.internal.KernelData; import org.neo4j.kernel.internal.KernelData;


import static java.lang.management.ManagementFactory.getPlatformMBeanServer;

public class ManagementSupport public class ManagementSupport
{ {
final static ManagementSupport load() public static final ManagementSupport load()
{ {
ManagementSupport support = new ManagementSupport(); ManagementSupport support = new ManagementSupport();
for ( ManagementSupport candidate : Service.load( ManagementSupport.class ) ) for ( ManagementSupport candidate : Service.load( ManagementSupport.class ) )
Expand Down Expand Up @@ -67,7 +66,7 @@ protected <T> T makeProxy( KernelBean kernel, ObjectName name, Class<T> beanInte


final <T> Collection<T> getProxiesFor( Class<T> beanInterface, KernelBean kernel ) final <T> Collection<T> getProxiesFor( Class<T> beanInterface, KernelBean kernel )
{ {
Collection<T> result = new ArrayList<T>(); Collection<T> result = new ArrayList<>();
ObjectName query = createObjectNameQuery( kernel.getInstanceId(), beanInterface ); ObjectName query = createObjectNameQuery( kernel.getInstanceId(), beanInterface );
for ( ObjectName name : getMBeanServer().queryNames( query, null ) ) for ( ObjectName name : getMBeanServer().queryNames( query, null ) )
{ {
Expand Down Expand Up @@ -116,7 +115,7 @@ protected String getBeanName( Class<?> beanInterface )


protected ObjectName createObjectName( String instanceId, String beanName, boolean query, String... extraNaming ) protected ObjectName createObjectName( String instanceId, String beanName, boolean query, String... extraNaming )
{ {
Hashtable<String, String> properties = new Hashtable<String, String>(); Hashtable<String, String> properties = new Hashtable<>();
properties.put( "instance", "kernel#" + instanceId ); properties.put( "instance", "kernel#" + instanceId );
properties.put( "name", beanName ); properties.put( "name", beanName );
for ( int i = 0; i < extraNaming.length; i++ ) for ( int i = 0; i < extraNaming.length; i++ )
Expand Down
2 changes: 1 addition & 1 deletion enterprise/ha/src/main/java/org/neo4j/kernel/ha/cluster/member/ClusterMember.java
View file
Expand Up @@ -42,7 +42,7 @@ public ClusterMember( InstanceId instanceId )
this( instanceId, Collections.<String,URI>emptyMap(), StoreId.DEFAULT, true ); this( instanceId, Collections.<String,URI>emptyMap(), StoreId.DEFAULT, true );
} }


ClusterMember( InstanceId instanceId, Map<String,URI> roles, StoreId storeId, boolean alive ) public ClusterMember( InstanceId instanceId, Map<String,URI> roles, StoreId storeId, boolean alive )
{ {
this.instanceId = instanceId; this.instanceId = instanceId;
this.roles = roles; this.roles = roles;
Expand Down
247 changes: 12 additions & 235 deletions enterprise/ha/src/test/java/jmx/HaBeanIT.java
View file
Expand Up @@ -23,30 +23,21 @@
import org.junit.Test; import org.junit.Test;


import java.net.URI; import java.net.URI;
import java.text.DateFormat;
import java.text.SimpleDateFormat;
import java.util.Arrays; import java.util.Arrays;
import java.util.function.Predicate;

import org.neo4j.graphdb.Transaction;
import org.neo4j.graphdb.factory.GraphDatabaseSettings; import org.neo4j.graphdb.factory.GraphDatabaseSettings;
import org.neo4j.helpers.collection.Iterables; import org.neo4j.helpers.collection.Iterables;
import org.neo4j.jmx.Kernel; import org.neo4j.jmx.Kernel;
import org.neo4j.jmx.impl.JmxKernelExtension; import org.neo4j.jmx.impl.JmxKernelExtension;
import org.neo4j.kernel.ha.HaSettings; import org.neo4j.kernel.ha.HaSettings;
import org.neo4j.kernel.ha.HighlyAvailableGraphDatabase; import org.neo4j.kernel.ha.HighlyAvailableGraphDatabase;
import org.neo4j.kernel.ha.cluster.HighAvailabilityMemberState;
import org.neo4j.kernel.ha.cluster.modeswitch.HighAvailabilityModeSwitcher; import org.neo4j.kernel.ha.cluster.modeswitch.HighAvailabilityModeSwitcher;
import org.neo4j.kernel.impl.ha.ClusterManager;
import org.neo4j.kernel.impl.ha.ClusterManager.ManagedCluster; import org.neo4j.kernel.impl.ha.ClusterManager.ManagedCluster;
import org.neo4j.kernel.impl.ha.ClusterManager.RepairKit;
import org.neo4j.management.BranchedStore; import org.neo4j.management.BranchedStore;
import org.neo4j.management.ClusterMemberInfo; import org.neo4j.management.ClusterMemberInfo;
import org.neo4j.management.HighAvailability; import org.neo4j.management.HighAvailability;
import org.neo4j.management.Neo4jManager; import org.neo4j.management.Neo4jManager;
import org.neo4j.test.ha.ClusterRule; import org.neo4j.test.ha.ClusterRule;


import static java.util.concurrent.TimeUnit.SECONDS;
import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertNotNull; import static org.junit.Assert.assertNotNull;
import static org.junit.Assert.assertTrue; import static org.junit.Assert.assertTrue;
Expand All @@ -55,232 +46,42 @@
import static org.neo4j.helpers.collection.Iterables.firstOrNull; import static org.neo4j.helpers.collection.Iterables.firstOrNull;
import static org.neo4j.kernel.configuration.Settings.STRING; import static org.neo4j.kernel.configuration.Settings.STRING;
import static org.neo4j.kernel.configuration.Settings.setting; import static org.neo4j.kernel.configuration.Settings.setting;
import static org.neo4j.kernel.impl.ha.ClusterManager.instanceEvicted;
import static org.neo4j.kernel.impl.ha.ClusterManager.masterAvailable;
import static org.neo4j.kernel.impl.ha.ClusterManager.masterSeesMembers;
import static org.neo4j.kernel.impl.ha.ClusterManager.masterSeesSlavesAsAvailable;
import static org.neo4j.test.ha.ClusterRule.intBase; import static org.neo4j.test.ha.ClusterRule.intBase;
import static org.neo4j.test.ha.ClusterRule.stringWithIntBase; import static org.neo4j.test.ha.ClusterRule.stringWithIntBase;


public class HaBeanIT public class HaBeanIT
{ {
@Rule @Rule
public final ClusterRule clusterRule = new ClusterRule( getClass() ) public final ClusterRule clusterRule = new ClusterRule( HaBeanIT.class )
.withInstanceSetting( setting( "jmx.port", STRING, (String) null ), intBase( 9912 ) ) .withInstanceSetting( setting( "jmx.port", STRING, (String) null ), intBase( 9912 ) )
.withInstanceSetting( HaSettings.ha_server, stringWithIntBase( ":", 1136 ) ) .withInstanceSetting( HaSettings.ha_server, stringWithIntBase( ":", 1136 ) )
.withInstanceSetting( GraphDatabaseSettings.forced_kernel_id, stringWithIntBase( "kernel", 0 ) ); .withInstanceSetting( GraphDatabaseSettings.forced_kernel_id, stringWithIntBase( "kernel", 0 ) );


@Test @Test
public void canGetHaBean() throws Throwable public void shouldAccessHaBeans() throws Throwable
{ {
ManagedCluster cluster = clusterRule.startCluster(); ManagedCluster cluster = clusterRule.startCluster();

// High Availability bean
HighAvailability ha = ha( cluster.getMaster() ); HighAvailability ha = ha( cluster.getMaster() );
assertNotNull( "could not get ha bean", ha ); assertNotNull( "could not get ha bean", ha );
assertMasterInformation( ha ); assertMasterInformation( ha );
} assertMasterAndSlaveInformation( ha.getInstancesInCluster() );

for ( ClusterMemberInfo info : ha.getInstancesInCluster() )
private void assertMasterInformation( HighAvailability ha )
{
assertTrue( "should be available", ha.isAvailable() );
assertEquals( "should be master", HighAvailabilityModeSwitcher.MASTER, ha.getRole() );
}

@Test
public void testLatestTxInfoIsCorrect() throws Throwable
{
ManagedCluster cluster = clusterRule.startCluster();
HighlyAvailableGraphDatabase db = cluster.getMaster();
HighAvailability masterHa = ha( db );
long lastCommitted = masterHa.getLastCommittedTxId();
try ( Transaction tx = db.beginTx() )
{
db.createNode();
tx.success();
}
assertEquals( lastCommitted + 1, masterHa.getLastCommittedTxId() );
}

@Test
public void testUpdatePullWorksAndUpdatesLastUpdateTime() throws Throwable
{
ManagedCluster cluster = clusterRule.startCluster();
HighlyAvailableGraphDatabase master = cluster.getMaster();
HighlyAvailableGraphDatabase slave = cluster.getAnySlave();
try (Transaction tx = master.beginTx())
{
master.createNode();
tx.success();
}
HighAvailability slaveBean = ha( slave );
DateFormat format = new SimpleDateFormat( "yyyy-MM-DD kk:mm:ss.SSSZZZZ" );
// To begin with, no updates
slaveBean.update();
long timeUpdated = format.parse( slaveBean.getLastUpdateTime() ).getTime();
assertTrue( timeUpdated > 0 );
}

@Test
public void testAfterGentleMasterSwitchClusterInfoIsCorrect() throws Throwable
{
ManagedCluster cluster = clusterRule.startCluster();
HighlyAvailableGraphDatabase master = cluster.getMaster();
RepairKit masterShutdown = cluster.shutdown( master );

cluster.await( masterAvailable( master ) );
cluster.await( masterSeesSlavesAsAvailable( 1 ) );

for ( HighlyAvailableGraphDatabase db : cluster.getAllMembers() )
{
assertEquals( 2, ha( db ).getInstancesInCluster().length );
}

masterShutdown.repair();

cluster.await( ClusterManager.allSeesAllAsAvailable() );

for ( HighlyAvailableGraphDatabase db : cluster.getAllMembers() )
{
HighAvailability bean = ha( db );

assertEquals( 3, bean.getInstancesInCluster().length );
for ( ClusterMemberInfo info : bean.getInstancesInCluster() )
{
assertTrue( "every instance should be available", info.isAvailable() );
assertTrue( "every instances should have at least one role", info.getRoles().length > 0 );
if ( HighAvailabilityModeSwitcher.MASTER.equals( info.getRoles()[0] ) )
{
assertEquals( "coordinator should be master",
HighAvailabilityModeSwitcher.MASTER, info.getHaRole() );
}
else
{
assertEquals( "Either master or slave, no other way",
HighAvailabilityModeSwitcher.SLAVE, info.getRoles()[0] );
assertEquals( "instance " + info.getInstanceId() + " is cluster slave but HA master",
HighAvailabilityModeSwitcher.SLAVE, info.getHaRole() );
}
for ( String uri : info.getUris() )
{
assertTrue( "roles should contain URIs",
uri.startsWith( "ha://" ) || uri.startsWith( "backup://" ) );
}
}
}
}

@Test
public void testAfterHardMasterSwitchClusterInfoIsCorrect() throws Throwable
{
ManagedCluster cluster = clusterRule.startCluster();

cluster.await( masterSeesSlavesAsAvailable( 2 ) );

HighlyAvailableGraphDatabase master = cluster.getMaster();
RepairKit masterShutdown = cluster.fail( master );

cluster.await( instanceEvicted( master ) );

for ( HighlyAvailableGraphDatabase db : cluster.getAllMembers() )
{
if ( db.getInstanceState() == HighAvailabilityMemberState.PENDING )
{
continue;
}
// Instance that was hard killed will still be in the cluster
assertEquals( 3, ha( db ).getInstancesInCluster().length );
}

masterShutdown.repair();

cluster.await( ClusterManager.allSeesAllAsAvailable(), 180 );

for ( HighlyAvailableGraphDatabase db : cluster.getAllMembers() )
{ {
int mastersFound = 0; assertTrue( info.isAlive() );
HighAvailability bean = ha( db ); assertTrue( info.isAvailable() );

assertEquals( 3, bean.getInstancesInCluster().length );
for ( ClusterMemberInfo info : bean.getInstancesInCluster() )
{
assertTrue( bean.getInstanceId() + ": every instance should be available: " + info.getInstanceId(),
info.isAvailable() );
for ( String role : info.getRoles() )
{
if ( role.equals( HighAvailabilityModeSwitcher.MASTER ) )
{
mastersFound++;
}
}
}
assertEquals( 1, mastersFound );
} }
}


@Test // Branched data bean
public void canGetBranchedStoreBean() throws Throwable
{
ManagedCluster cluster = clusterRule.startCluster();
BranchedStore bs = beans( cluster.getMaster() ).getBranchedStoreBean(); BranchedStore bs = beans( cluster.getMaster() ).getBranchedStoreBean();
assertNotNull( "could not get branched store bean", bs ); assertNotNull( "could not get branched store bean", bs );
} }


@Test private void assertMasterInformation( HighAvailability ha )
public void joinedInstanceShowsUpAsSlave() throws Throwable
{
ManagedCluster cluster = clusterRule.startCluster();
ClusterMemberInfo[] instancesInCluster = ha( cluster.getMaster() ).getInstancesInCluster();
assertEquals( 3, instancesInCluster.length );
ClusterMemberInfo[] secondInstancesInCluster = ha( cluster.getAnySlave() ).getInstancesInCluster();
assertEquals( 3, secondInstancesInCluster.length );
assertMasterAndSlaveInformation( instancesInCluster );
assertMasterAndSlaveInformation( secondInstancesInCluster );
}

@Test
public void leftInstanceDisappearsFromMemberList() throws Throwable
{
// Start the cluster and make sure it's up.
// Then shut down one of the slaves to see if it disappears from the member list.
ManagedCluster cluster = clusterRule.startCluster();
assertEquals( 3, ha( cluster.getAnySlave() ).getInstancesInCluster().length );
RepairKit repair = cluster.shutdown( cluster.getAnySlave() );

try
{
cluster.await( masterSeesMembers( 2 ) );
HighAvailability haMaster = ha( cluster.getMaster() );
assertEquals( 2, haMaster.getInstancesInCluster().length );
}
finally
{
repair.repair();
}
}

@Test
public void failedMemberIsStillInMemberListAlthoughFailed() throws Throwable
{ {
ManagedCluster cluster = clusterRule.startCluster(); assertTrue( "should be available", ha.isAvailable() );
assertEquals( 3, ha( cluster.getAnySlave() ).getInstancesInCluster().length ); assertEquals( "should be master", HighAvailabilityModeSwitcher.MASTER, ha.getRole() );

// Fail the instance
HighlyAvailableGraphDatabase failedDb = cluster.getAnySlave();
RepairKit dbFailure = cluster.fail( failedDb );
try
{
await( ha( cluster.getMaster() ), dbAlive( false ) );
await( ha( cluster.getAnySlave( failedDb ) ), dbAlive( false ) );
}
finally
{
// Repair the failure and come back
dbFailure.repair();
}
for ( HighlyAvailableGraphDatabase db : cluster.getAllMembers() )
{
await( ha( db ), dbAvailability( true ) );
await( ha( db ), dbAlive( true ) );
}
} }


private Neo4jManager beans( HighlyAvailableGraphDatabase db ) private Neo4jManager beans( HighlyAvailableGraphDatabase db )
Expand Down Expand Up @@ -324,28 +125,4 @@ private ClusterMemberInfo member( ClusterMemberInfo[] members, int instanceId )
members ) ); members ) );
return null; // it will never get here. return null; // it will never get here.
} }

private void await( HighAvailability ha, Predicate<ClusterMemberInfo> predicate ) throws InterruptedException
{
long end = System.currentTimeMillis() + SECONDS.toMillis( 300 );
while ( System.currentTimeMillis() < end )
{
if ( predicate.test( member( ha.getInstancesInCluster(), 2 ) ) )
{
return;
}
Thread.sleep( 500 );
}
fail( "Failed instance didn't show up as such in JMX" );
}

private Predicate<ClusterMemberInfo> dbAvailability( final boolean available )
{
return item -> item.isAvailable() == available;
}

private Predicate<ClusterMemberInfo> dbAlive( final boolean alive )
{
return item -> item.isAlive() == alive;
}
} }

0 comments on commit 02db63a

Please sign in to comment.