From 076371d9b9a905aef753ac3e65c99580333d85fc Mon Sep 17 00:00:00 2001 From: fickludd Date: Fri, 28 Oct 2016 15:38:06 +0200 Subject: [PATCH] Added chaotic stress test for `InternalFlatFileRealm` - broke out common functionality in to `FlatFileStressBase` --- .../neo4j/auth/FlatFileChaoticStressIT.java | 274 ++++++++++++++++++ .../auth/FlatFilePredictableStressIT.java | 184 +----------- .../org/neo4j/auth/FlatFileStressBase.java | 210 ++++++++++++++ 3 files changed, 495 insertions(+), 173 deletions(-) create mode 100644 integrationtests/src/test/java/org/neo4j/auth/FlatFileChaoticStressIT.java create mode 100644 integrationtests/src/test/java/org/neo4j/auth/FlatFileStressBase.java diff --git a/integrationtests/src/test/java/org/neo4j/auth/FlatFileChaoticStressIT.java b/integrationtests/src/test/java/org/neo4j/auth/FlatFileChaoticStressIT.java new file mode 100644 index 0000000000000..6d4e734e67f6e --- /dev/null +++ b/integrationtests/src/test/java/org/neo4j/auth/FlatFileChaoticStressIT.java @@ -0,0 +1,274 @@ +/* + * Copyright (c) 2002-2016 "Neo Technology," + * Network Engine for Objects in Lund AB [http://neotechnology.com] + * + * This file is part of Neo4j. + * + * Neo4j is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ +package org.neo4j.auth; + +import org.junit.Rule; + +import java.io.IOException; +import java.util.LinkedHashSet; +import java.util.Set; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; +import java.util.stream.IntStream; + +import org.neo4j.io.fs.FileSystemAbstraction; +import org.neo4j.kernel.api.exceptions.InvalidArgumentsException; +import org.neo4j.test.rule.fs.EphemeralFileSystemRule; + +public class FlatFileChaoticStressIT extends FlatFileStressBase +{ + @Rule + public EphemeralFileSystemRule fsRule = new EphemeralFileSystemRule(); + + @Override + FileSystemAbstraction getFileSystem() + { + return fsRule.get(); + } + + @Override + ExecutorService setupWorkload( int n ) + { + ExecutorService service = Executors.newFixedThreadPool( n ); + Set usernames = makeWithPrefix( "user", n ); + Set roleNames = makeWithPrefix( "role", n ); + for ( int i = 0; i < n; i++ ) + { + service.submit( new ChaoticAdmin( i, usernames, roleNames ) ); + } + return service; + } + + private Set makeWithPrefix( String prefix, int n ) + { + Set set = new LinkedHashSet<>( ); + IntStream.range( 0, n ).forEach( i -> set.add( prefix + i ) ); + return set; + } + + private class ChaoticAdmin extends IrrationalAdmin + { + private final int number; + private final String[] usernames; + private final String[] roleNames; + + ChaoticAdmin( int number, Set usernames, Set roleNames ) + { + super(); + this.number = number; + this.usernames = usernames.toArray( new String[ usernames.size() ] ); + this.roleNames = roleNames.toArray( new String[ roleNames.size() ] ); + setActions( + this::createUser, + this::deleteUser, + this::changePassword, + this::suspend, + this::activate, + this::createRole, + this::deleteRole, + this::assignRole, + this::unAssignRole + ); + } + + @Override + public String toString() + { + return "ChaoticAdmin " + number; + } + + // __________ ACTIONS ___________ + + private void createUser() + { + String username = randomUser(); + String password = deviousPassword(); + try + { + flatFileRealm.newUser( username, password, false ); + } + catch ( IOException e ) + { + errors.add( e ); + } + catch ( InvalidArgumentsException e ) + { + // ignore + } + } + + private void deleteUser() + { + String username = randomUser(); + try + { + flatFileRealm.deleteUser( username ); + } + catch ( IOException e ) + { + errors.add( e ); + } + catch ( InvalidArgumentsException e ) + { + // ignore + } + } + + private void changePassword() + { + String username = randomUser(); + String password = deviousPassword(); + try + { + flatFileRealm.setUserPassword( username, password, false ); + } + catch ( IOException e ) + { + errors.add( e ); + } + catch ( InvalidArgumentsException e ) + { + // ignore + } + } + + private void suspend() + { + String username = randomUser(); + try + { + flatFileRealm.suspendUser( username ); + } + catch ( IOException e ) + { + errors.add( e ); + } + catch ( InvalidArgumentsException e ) + { + // ignore + } + } + + private void activate() + { + String username = randomUser(); + try + { + flatFileRealm.activateUser( username, false ); + } + catch ( IOException e ) + { + errors.add( e ); + } + catch ( InvalidArgumentsException e ) + { + // ignore + } + } + + private void createRole() + { + String username = randomUser(); + String roleName = randomRole(); + try + { + flatFileRealm.newRole( roleName, username ); + } + catch ( IOException e ) + { + errors.add( e ); + } + catch ( InvalidArgumentsException e ) + { + // ignore + } + } + + private void deleteRole() + { + String roleName = randomRole(); + try + { + flatFileRealm.deleteRole( roleName ); + } + catch ( IOException e ) + { + errors.add( e ); + } + catch ( InvalidArgumentsException e ) + { + // ignore + } + } + + private void assignRole() + { + String username = randomUser(); + String roleName = randomRole(); + try + { + flatFileRealm.addRoleToUser( roleName, username ); + } + catch ( IOException e ) + { + errors.add( e ); + } + catch ( InvalidArgumentsException e ) + { + // ignore + } + } + + private void unAssignRole() + { + String username = randomUser(); + String roleName = randomRole(); + try + { + flatFileRealm.removeRoleFromUser( roleName, username ); + } + catch ( IOException e ) + { + errors.add( e ); + } + catch ( InvalidArgumentsException e ) + { + // ignore + } + } + + // ______________ HELPERS ______________ + + private String deviousPassword() + { + return random.nextBoolean() ? "123" : "321"; + } + + private String randomUser() + { + return usernames[ random.nextInt( usernames.length ) ]; + } + + private String randomRole() + { + return roleNames[ random.nextInt( roleNames.length ) ]; + } + } +} diff --git a/integrationtests/src/test/java/org/neo4j/auth/FlatFilePredictableStressIT.java b/integrationtests/src/test/java/org/neo4j/auth/FlatFilePredictableStressIT.java index c9bea665c07df..1fdff45b12dce 100644 --- a/integrationtests/src/test/java/org/neo4j/auth/FlatFilePredictableStressIT.java +++ b/integrationtests/src/test/java/org/neo4j/auth/FlatFilePredictableStressIT.java @@ -19,155 +19,38 @@ */ package org.neo4j.auth; -import org.junit.After; -import org.junit.Before; import org.junit.Rule; -import org.junit.Test; import java.io.IOException; -import java.time.Clock; -import java.util.Map; -import java.util.Random; -import java.util.Set; -import java.util.concurrent.ConcurrentHashMap; -import java.util.concurrent.Executor; import java.util.concurrent.ExecutorService; import java.util.concurrent.Executors; -import java.util.concurrent.ThreadFactory; -import java.util.concurrent.TimeUnit; -import java.util.stream.Collectors; +import org.neo4j.io.fs.FileSystemAbstraction; import org.neo4j.kernel.api.exceptions.InvalidArgumentsException; -import org.neo4j.kernel.configuration.Config; -import org.neo4j.kernel.impl.util.JobScheduler; -import org.neo4j.kernel.lifecycle.LifecycleAdapter; -import org.neo4j.logging.LogProvider; -import org.neo4j.logging.NullLogProvider; -import org.neo4j.server.security.auth.BasicPasswordPolicy; -import org.neo4j.server.security.auth.CommunitySecurityModule; -import org.neo4j.server.security.auth.ListSnapshot; -import org.neo4j.server.security.auth.RateLimitedAuthenticationStrategy; -import org.neo4j.server.security.auth.User; -import org.neo4j.server.security.auth.UserRepository; -import org.neo4j.server.security.enterprise.auth.EnterpriseSecurityModule; -import org.neo4j.server.security.enterprise.auth.InternalFlatFileRealm; -import org.neo4j.server.security.enterprise.auth.RoleRecord; -import org.neo4j.server.security.enterprise.auth.RoleRepository; import org.neo4j.server.security.enterprise.auth.plugin.api.PredefinedRoles; import org.neo4j.test.rule.fs.EphemeralFileSystemRule; -import static java.util.concurrent.TimeUnit.SECONDS; -import static org.hamcrest.MatcherAssert.assertThat; -import static org.hamcrest.Matchers.empty; -import static org.junit.Assert.assertTrue; - -public class FlatFilePredictableStressIT +public class FlatFilePredictableStressIT extends FlatFileStressBase { @Rule public EphemeralFileSystemRule fsRule = new EphemeralFileSystemRule(); - private InternalFlatFileRealm flatFileRealm; - private UserRepository userRepository; - private RoleRepository roleRepository; - - private volatile boolean keepRunning = true; - private final Set errors = ConcurrentHashMap.newKeySet(); - - @Before - public void setup() throws Throwable - { - Config config = Config.defaults(); - LogProvider logProvider = NullLogProvider.getInstance(); - JobScheduler jobScheduler = new NoopJobScheduler(); - - userRepository = CommunitySecurityModule.getUserRepository( config, logProvider, fsRule.get() ); - roleRepository = EnterpriseSecurityModule.getRoleRepository( config, logProvider, fsRule.get() ); - - flatFileRealm = new InternalFlatFileRealm( - userRepository, - roleRepository, - new BasicPasswordPolicy(), - new RateLimitedAuthenticationStrategy( Clock.systemUTC(), 3 ), - jobScheduler, - CommunitySecurityModule.getInitialUserRepository( config, logProvider, fsRule.get() ) - ); - - flatFileRealm.init(); - flatFileRealm.start(); - } - - @After - public void teardown() throws Throwable + @Override + FileSystemAbstraction getFileSystem() { - flatFileRealm.stop(); - flatFileRealm.shutdown(); + return fsRule.get(); } - @Test - public void shouldMaintainConsistency() throws InterruptedException, IOException + @Override + ExecutorService setupWorkload( int n ) { - final long ONE_SECOND = 1000; - final long TIMEOUT_IN_SECONDS = 10; - final int N = 10; - final int ERROR_LIMIT = 100; - - ExecutorService service = Executors.newFixedThreadPool( 2 * N ); - for ( int i = 0; i < N; i++ ) + ExecutorService service = Executors.newFixedThreadPool( 2 * n ); + for ( int i = 0; i < n; i++ ) { service.submit( new IrrationalUserAdmin( i ) ); service.submit( new IrrationalRoleAdmin( i ) ); } - - for ( int t = 0; t < TIMEOUT_IN_SECONDS; t++ ) - { - Thread.sleep( ONE_SECOND ); - if ( errors.size() > ERROR_LIMIT ) - { - break; - } - } - - keepRunning = false; - service.shutdown(); - service.awaitTermination( 5, SECONDS ); - - // Assert that no errors occured - String msg = String.join( System.lineSeparator(), - errors.stream().map( Throwable::getMessage ).collect( Collectors.toList() ) ); - assertThat( msg, errors, empty() ); - - // Assert that user and role repos are consistent - ListSnapshot users = userRepository.getPersistedSnapshot(); - ListSnapshot roles = roleRepository.getPersistedSnapshot(); - assertTrue( - "User and role repositories are no longer consistent", - RoleRepository.validate( users.values(), roles.values() ) - ); - } - - abstract class IrrationalAdmin implements Runnable - { - final Random random = new Random(); - Runnable[] actions; - - @Override - public void run() - { - while ( keepRunning ) - { - randomAction().run(); - } - } - - private Runnable randomAction() - { - return actions[ random.nextInt( actions.length ) ]; - } - - void setActions( Runnable... actions ) - { - this.actions = actions; - } + return service; } private class IrrationalUserAdmin extends IrrationalAdmin @@ -421,6 +304,7 @@ private void deleteRole() } } } + private void assignRole() { try @@ -476,50 +360,4 @@ private boolean userDoesNotExist( InvalidArgumentsException e ) return e.getMessage().contains( "User '" + username + "' does not exist" ); } } - - private class NoopJobScheduler extends LifecycleAdapter implements JobScheduler - { - @Override - public Executor executor( Group group ) - { - return null; - } - - @Override - public ThreadFactory threadFactory( Group group ) - { - return null; - } - - @Override - public JobHandle schedule( Group group, Runnable job ) - { - return null; - } - - @Override - public JobHandle schedule( Group group, Runnable job, Map metadata ) - { - return null; - } - - @Override - public JobHandle schedule( Group group, Runnable runnable, long initialDelay, TimeUnit timeUnit ) - { - return null; - } - - @Override - public JobHandle scheduleRecurring( Group group, Runnable runnable, long period, TimeUnit timeUnit ) - { - return null; - } - - @Override - public JobHandle scheduleRecurring( Group group, Runnable runnable, long initialDelay, long period, - TimeUnit timeUnit ) - { - return null; - } - } } diff --git a/integrationtests/src/test/java/org/neo4j/auth/FlatFileStressBase.java b/integrationtests/src/test/java/org/neo4j/auth/FlatFileStressBase.java new file mode 100644 index 0000000000000..468677edd9d3a --- /dev/null +++ b/integrationtests/src/test/java/org/neo4j/auth/FlatFileStressBase.java @@ -0,0 +1,210 @@ +/* + * Copyright (c) 2002-2016 "Neo Technology," + * Network Engine for Objects in Lund AB [http://neotechnology.com] + * + * This file is part of Neo4j. + * + * Neo4j is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ +package org.neo4j.auth; + +import org.junit.After; +import org.junit.Before; +import org.junit.Test; + +import java.io.IOException; +import java.time.Clock; +import java.util.Map; +import java.util.Random; +import java.util.Set; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.Executor; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.ThreadFactory; +import java.util.concurrent.TimeUnit; +import java.util.stream.Collectors; + +import org.neo4j.io.fs.FileSystemAbstraction; +import org.neo4j.kernel.configuration.Config; +import org.neo4j.kernel.impl.util.JobScheduler; +import org.neo4j.kernel.lifecycle.LifecycleAdapter; +import org.neo4j.logging.LogProvider; +import org.neo4j.logging.NullLogProvider; +import org.neo4j.server.security.auth.BasicPasswordPolicy; +import org.neo4j.server.security.auth.CommunitySecurityModule; +import org.neo4j.server.security.auth.ListSnapshot; +import org.neo4j.server.security.auth.RateLimitedAuthenticationStrategy; +import org.neo4j.server.security.auth.User; +import org.neo4j.server.security.auth.UserRepository; +import org.neo4j.server.security.enterprise.auth.EnterpriseSecurityModule; +import org.neo4j.server.security.enterprise.auth.InternalFlatFileRealm; +import org.neo4j.server.security.enterprise.auth.RoleRecord; +import org.neo4j.server.security.enterprise.auth.RoleRepository; + +import static java.util.concurrent.TimeUnit.SECONDS; +import static org.hamcrest.MatcherAssert.assertThat; +import static org.hamcrest.Matchers.empty; +import static org.junit.Assert.assertTrue; + +abstract class FlatFileStressBase +{ + private final long ONE_SECOND = 1000; + protected long TIMEOUT_IN_SECONDS = 10; + protected int N = 10; + protected int ERROR_LIMIT = 100; + + InternalFlatFileRealm flatFileRealm; + private UserRepository userRepository; + private RoleRepository roleRepository; + + private volatile boolean keepRunning = true; + final Set errors = ConcurrentHashMap.newKeySet(); + + @Before + public void setup() throws Throwable + { + Config config = Config.defaults(); + LogProvider logProvider = NullLogProvider.getInstance(); + JobScheduler jobScheduler = new NoopJobScheduler(); + + userRepository = CommunitySecurityModule.getUserRepository( config, logProvider, getFileSystem() ); + roleRepository = EnterpriseSecurityModule.getRoleRepository( config, logProvider, getFileSystem() ); + + flatFileRealm = new InternalFlatFileRealm( + userRepository, + roleRepository, + new BasicPasswordPolicy(), + new RateLimitedAuthenticationStrategy( Clock.systemUTC(), 3 ), + jobScheduler, + CommunitySecurityModule.getInitialUserRepository( config, logProvider, getFileSystem() ) + ); + + flatFileRealm.init(); + flatFileRealm.start(); + } + + abstract FileSystemAbstraction getFileSystem(); + + @After + public void teardown() throws Throwable + { + flatFileRealm.stop(); + flatFileRealm.shutdown(); + } + + @Test + public void shouldMaintainConsistency() throws InterruptedException, IOException + { + ExecutorService service = setupWorkload( N ); + + for ( int t = 0; t < TIMEOUT_IN_SECONDS; t++ ) + { + Thread.sleep( ONE_SECOND ); + if ( errors.size() > ERROR_LIMIT ) + { + break; + } + } + + keepRunning = false; + service.shutdown(); + service.awaitTermination( 5, SECONDS ); + + // Assert that no errors occured + String msg = String.join( System.lineSeparator(), + errors.stream().map( Throwable::getMessage ).collect( Collectors.toList() ) ); + assertThat( msg, errors, empty() ); + + // Assert that user and role repos are consistent + ListSnapshot users = userRepository.getPersistedSnapshot(); + ListSnapshot roles = roleRepository.getPersistedSnapshot(); + assertTrue( + "User and role repositories are no longer consistent", + RoleRepository.validate( users.values(), roles.values() ) + ); + } + + abstract ExecutorService setupWorkload( int n ); + + abstract class IrrationalAdmin implements Runnable + { + final Random random = new Random(); + Runnable[] actions; + + @Override + public void run() + { + while ( keepRunning ) + { + randomAction().run(); + } + } + + private Runnable randomAction() + { + return actions[ random.nextInt( actions.length ) ]; + } + + void setActions( Runnable... actions ) + { + this.actions = actions; + } + } + + private class NoopJobScheduler extends LifecycleAdapter implements JobScheduler + { + @Override + public Executor executor( Group group ) + { + return null; + } + + @Override + public ThreadFactory threadFactory( Group group ) + { + return null; + } + + @Override + public JobHandle schedule( Group group, Runnable job ) + { + return null; + } + + @Override + public JobHandle schedule( Group group, Runnable job, Map metadata ) + { + return null; + } + + @Override + public JobHandle schedule( Group group, Runnable runnable, long initialDelay, TimeUnit timeUnit ) + { + return null; + } + + @Override + public JobHandle scheduleRecurring( Group group, Runnable runnable, long period, TimeUnit timeUnit ) + { + return null; + } + + @Override + public JobHandle scheduleRecurring( Group group, Runnable runnable, long initialDelay, long period, + TimeUnit timeUnit ) + { + return null; + } + } +}