From dd89171671e9048c7f7926b048a5b42e5dbaef32 Mon Sep 17 00:00:00 2001 From: Davide Grohmann Date: Mon, 17 Oct 2016 17:03:40 +0200 Subject: [PATCH] Ignore more transient failures in backup stress test I.e., connection reset by peer and store has been closed server side during backup. Also extracts into predicates the various checks on exceptions to determine if they are transient or not. --- .../coreedge/stresstests/BackupLoad.java | 37 +++----------- .../coreedge/stresstests/CatchUpLoad.java | 41 ++------------- .../stresstests/IsChannelClosedException.java | 45 +++++++++++++++++ .../stresstests/IsConnectionException.java | 43 ++++++++++++++++ .../stresstests/IsConnectionRestByPeer.java | 43 ++++++++++++++++ .../coreedge/stresstests/IsStoreClosed.java | 50 +++++++++++++++++++ 6 files changed, 191 insertions(+), 68 deletions(-) create mode 100644 stresstests/src/test/java/org/neo4j/coreedge/stresstests/IsChannelClosedException.java create mode 100644 stresstests/src/test/java/org/neo4j/coreedge/stresstests/IsConnectionException.java create mode 100644 stresstests/src/test/java/org/neo4j/coreedge/stresstests/IsConnectionRestByPeer.java create mode 100644 stresstests/src/test/java/org/neo4j/coreedge/stresstests/IsStoreClosed.java diff --git a/stresstests/src/test/java/org/neo4j/coreedge/stresstests/BackupLoad.java b/stresstests/src/test/java/org/neo4j/coreedge/stresstests/BackupLoad.java index 9a415dc73f27f..40e4d61b7ee6d 100644 --- a/stresstests/src/test/java/org/neo4j/coreedge/stresstests/BackupLoad.java +++ b/stresstests/src/test/java/org/neo4j/coreedge/stresstests/BackupLoad.java @@ -20,18 +20,21 @@ package org.neo4j.coreedge.stresstests; import java.io.File; -import java.net.ConnectException; import java.util.concurrent.locks.LockSupport; import java.util.function.BiFunction; import java.util.function.BooleanSupplier; +import java.util.function.Predicate; import org.neo4j.backup.OnlineBackup; -import org.neo4j.com.ComException; import org.neo4j.coreedge.discovery.Cluster; import org.neo4j.helpers.SocketAddress; class BackupLoad extends RepeatUntilOnSelectedMemberCallable { + private final Predicate isTransientError = + new IsConnectionException().or( new IsConnectionRestByPeer() ).or( new IsChannelClosedException() ) + .or( new IsStoreClosed() ); + private final File baseDirectory; private final BiFunction backupAddress; @@ -56,7 +59,7 @@ protected void doWorkOnMember( boolean isCore, int id ) } catch ( RuntimeException e ) { - if ( isConnectionError( e ) ) + if ( isTransientError.test( e ) ) { // if we could not connect, wait a bit and try again... LockSupport.parkNanos( 10_000_000 ); @@ -70,32 +73,4 @@ protected void doWorkOnMember( boolean isCore, int id ) throw new RuntimeException( "Not consistent backup from " + address ); } } - - private boolean isConnectionError( RuntimeException e ) - { - return isConnectionException( e ) || isChannelClosedException( e ); - } - - private boolean isConnectionException( Throwable e ) - { - if ( e == null ) - { - return false; - } - - return e instanceof ConnectException || isConnectionException( e.getCause() ); - - } - - private boolean isChannelClosedException( Throwable e ) - { - if ( e == null ) - { - return false; - } - - boolean match = e instanceof ComException && e.getMessage() != null && - e.getMessage().startsWith( "Channel has been closed" ); - return match || isChannelClosedException( e.getCause() ); - } } diff --git a/stresstests/src/test/java/org/neo4j/coreedge/stresstests/CatchUpLoad.java b/stresstests/src/test/java/org/neo4j/coreedge/stresstests/CatchUpLoad.java index 90c90173495be..69fb37689c9d6 100644 --- a/stresstests/src/test/java/org/neo4j/coreedge/stresstests/CatchUpLoad.java +++ b/stresstests/src/test/java/org/neo4j/coreedge/stresstests/CatchUpLoad.java @@ -36,7 +36,6 @@ import org.neo4j.coreedge.discovery.EdgeClusterMember; import org.neo4j.coreedge.handlers.ExceptionMonitoringHandler; import org.neo4j.kernel.impl.transaction.log.TransactionIdStore; -import org.neo4j.kernel.impl.util.UnsatisfiedDependencyException; import org.neo4j.kernel.internal.GraphDatabaseAPI; import org.neo4j.kernel.monitoring.Monitors; @@ -44,7 +43,7 @@ class CatchUpLoad extends RepeatUntilCallable { - private static final IllegalStateException databaseShutdownEx = new IllegalStateException( "database is shutdown" ); + private final Predicate isStoreClosed = new IsStoreClosed(); private Cluster cluster; CatchUpLoad( BooleanSupplier keepGoing, Runnable onFailure, Cluster cluster ) @@ -133,7 +132,7 @@ private Supplier startAndRegisterExceptionMonitor( EdgeClusterMember // the database is create when starting the edge... final Monitors monitors = edgeClusterMember.database().getDependencyResolver().resolveDependency( Monitors.class ); - ExceptionMonitor exceptionMonitor = new ExceptionMonitor( new ConnectionResetFilter() ); + ExceptionMonitor exceptionMonitor = new ExceptionMonitor( new IsConnectionRestByPeer() ); monitors.addMonitorListener( exceptionMonitor, CatchUpClient.class.getName() ); return exceptionMonitor; } @@ -143,7 +142,7 @@ private long txId( ClusterMember member, boolean fail ) GraphDatabaseAPI database = member.database(); if ( database == null ) { - return errorValueOrThrow( fail, databaseShutdownEx ); + return errorValueOrThrow( fail, new IllegalStateException( "database is shutdown" ) ); } try @@ -153,7 +152,7 @@ private long txId( ClusterMember member, boolean fail ) } catch ( Throwable ex ) { - return errorValueOrThrow( fail && !isStoreClosed( ex ), ex ); + return errorValueOrThrow( fail && !isStoreClosed.test( ex ), ex ); } } @@ -169,38 +168,6 @@ private long errorValueOrThrow( boolean fail, Throwable error ) } } - private boolean isStoreClosed( Throwable ex ) - { - if ( ex == null ) - { - return false; - } - - if ( ex instanceof UnsatisfiedDependencyException ) - { - return true; - } - - if ( ex instanceof IllegalStateException ) - { - String message = ex.getMessage(); - return message.startsWith( "MetaDataStore for file " ) && message.endsWith( " is closed" ); - } - - return isStoreClosed( ex.getCause() ); - } - - private static class ConnectionResetFilter implements Predicate - { - private static final String MSG = "Connection reset by peer"; - - @Override - public boolean test( Throwable throwable ) - { - return (throwable instanceof IOException) && MSG.equals( throwable.getMessage() ); - } - } - private static class ExceptionMonitor implements ExceptionMonitoringHandler.Monitor, Supplier { private final AtomicReference exception = new AtomicReference<>(); diff --git a/stresstests/src/test/java/org/neo4j/coreedge/stresstests/IsChannelClosedException.java b/stresstests/src/test/java/org/neo4j/coreedge/stresstests/IsChannelClosedException.java new file mode 100644 index 0000000000000..2a611c162d6ce --- /dev/null +++ b/stresstests/src/test/java/org/neo4j/coreedge/stresstests/IsChannelClosedException.java @@ -0,0 +1,45 @@ +/* + * Copyright (c) 2002-2016 "Neo Technology," + * Network Engine for Objects in Lund AB [http://neotechnology.com] + * + * This file is part of Neo4j. + * + * Neo4j is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ +package org.neo4j.coreedge.stresstests; + +import java.util.function.Predicate; + +import org.neo4j.com.ComException; + +class IsChannelClosedException implements Predicate +{ + + @Override + public boolean test( Throwable e ) + { + if ( e == null ) + { + return false; + } + + if ( e instanceof ComException && e.getMessage() != null && + e.getMessage().startsWith( "Channel has been closed" ) ) + { + return true; + } + + return test( e.getCause() ); + } +} diff --git a/stresstests/src/test/java/org/neo4j/coreedge/stresstests/IsConnectionException.java b/stresstests/src/test/java/org/neo4j/coreedge/stresstests/IsConnectionException.java new file mode 100644 index 0000000000000..1c8e475dfb015 --- /dev/null +++ b/stresstests/src/test/java/org/neo4j/coreedge/stresstests/IsConnectionException.java @@ -0,0 +1,43 @@ +/* + * Copyright (c) 2002-2016 "Neo Technology," + * Network Engine for Objects in Lund AB [http://neotechnology.com] + * + * This file is part of Neo4j. + * + * Neo4j is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ +package org.neo4j.coreedge.stresstests; + +import java.net.ConnectException; +import java.util.function.Predicate; + +class IsConnectionException implements Predicate +{ + + @Override + public boolean test( Throwable e ) + { + if ( e == null ) + { + return false; + } + + if ( e instanceof ConnectException ) + { + return true; + } + + return test( e.getCause() ); + } +} diff --git a/stresstests/src/test/java/org/neo4j/coreedge/stresstests/IsConnectionRestByPeer.java b/stresstests/src/test/java/org/neo4j/coreedge/stresstests/IsConnectionRestByPeer.java new file mode 100644 index 0000000000000..886dbdaae4326 --- /dev/null +++ b/stresstests/src/test/java/org/neo4j/coreedge/stresstests/IsConnectionRestByPeer.java @@ -0,0 +1,43 @@ +/* + * Copyright (c) 2002-2016 "Neo Technology," + * Network Engine for Objects in Lund AB [http://neotechnology.com] + * + * This file is part of Neo4j. + * + * Neo4j is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ +package org.neo4j.coreedge.stresstests; + +import java.io.IOException; +import java.util.function.Predicate; + +class IsConnectionRestByPeer implements Predicate +{ + @Override + public boolean test( Throwable e ) + { + if ( e == null ) + { + return false; + } + + if ( e instanceof IOException && e.getMessage() != null && + e.getMessage().startsWith( "Connection reset by peer" ) ) + { + return true; + } + + return test( e.getCause() ); + } +} diff --git a/stresstests/src/test/java/org/neo4j/coreedge/stresstests/IsStoreClosed.java b/stresstests/src/test/java/org/neo4j/coreedge/stresstests/IsStoreClosed.java new file mode 100644 index 0000000000000..3352783042f76 --- /dev/null +++ b/stresstests/src/test/java/org/neo4j/coreedge/stresstests/IsStoreClosed.java @@ -0,0 +1,50 @@ +/* + * Copyright (c) 2002-2016 "Neo Technology," + * Network Engine for Objects in Lund AB [http://neotechnology.com] + * + * This file is part of Neo4j. + * + * Neo4j is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ +package org.neo4j.coreedge.stresstests; + +import java.util.function.Predicate; + +import org.neo4j.kernel.impl.util.UnsatisfiedDependencyException; + +class IsStoreClosed implements Predicate +{ + @Override + public boolean test( Throwable ex ) + { + + if ( ex == null ) + { + return false; + } + + if ( ex instanceof UnsatisfiedDependencyException ) + { + return true; + } + + if ( ex instanceof IllegalStateException ) + { + String message = ex.getMessage(); + return message.startsWith( "MetaDataStore for file " ) && message.endsWith( " is closed" ); + } + + return test( ex.getCause() ); + } +}