Introduce lock acquisition timeout

Introduce property that defines timeout for shared and exclusive lock acquisition in lock clients.
In case if user configure timeout any acquisitions that take more then configured time interval will be terminated and LockAcquisitionTimeoutException will be raised.
Add support of new property to community and forseti lock managers.

Unify clocks usage during database construction/creation: now all components will use exactly the same instance of clock that provided by PlatformModule.

community/common/src/main/java/org/neo4j/kernel/api/exceptions/Status.java

@@ -184,6 +184,8 @@ enum Transaction implements Status
                 "transaction was active. Transaction may succeed if retried." ),
         LockClientStopped( TransientError,
                 "Transaction terminated, no more locks can be acquired." ),
+        LockAcquisitionTimeout( TransientError,
+                "Unable to acquire lock within configured timeout." ),
         Terminated( TransientError,
                 "Explicitly terminated by the user." ),
         Interrupted( TransientError,

community/kernel/src/main/java/org/neo4j/graphdb/factory/GraphDatabaseSettings.java

@@ -186,6 +186,10 @@ public abstract class GraphDatabaseSettings
     @Description("The maximum time interval of a transaction within which it should be completed.")
     public static final Setting<Long> transaction_timeout = setting( "dbms.transaction.timeout", DURATION, String.valueOf( UNSPECIFIED_TIMEOUT ) );
 
+    @Description( "The maximum time interval within which lock should be acquired." )
+    public static final Setting<Long> lock_acquisition_timeout = setting( "dbms.lock.acquisition.timeout", DURATION,
+            String.valueOf( UNSPECIFIED_TIMEOUT ) );
+
     @Description( "The maximum amount of time to wait for running transactions to complete before allowing "
                   + "initiated database shutdown to continue" )
     @Internal

community/kernel/src/main/java/org/neo4j/kernel/impl/factory/CommunityEditionModule.java

@@ -20,6 +20,7 @@
 package org.neo4j.kernel.impl.factory;
 
 import java.io.File;
+import java.time.Clock;
 
 import org.neo4j.graphdb.DependencyResolver;
 import org.neo4j.graphdb.factory.GraphDatabaseSettings;
@@ -90,7 +91,7 @@ public CommunityEditionModule( PlatformModule platformModule )
 
         GraphDatabaseFacade graphDatabaseFacade = platformModule.graphDatabaseFacade;
 
-        lockManager = dependencies.satisfyDependency( createLockManager( config, logging ) );
+        lockManager = dependencies.satisfyDependency( createLockManager( config, platformModule.clock, logging ) );
         statementLocksFactory = createStatementLocksFactory( lockManager, config, logging );
 
         idTypeConfigurationProvider = createIdTypeConfigurationProvider( config );
@@ -199,33 +200,33 @@ protected IdGeneratorFactory createIdGeneratorFactory( FileSystemAbstraction fs,
         return new DefaultIdGeneratorFactory( fs, idTypeConfigurationProvider );
     }
 
-    public static Locks createLockManager( Config config, LogService logging )
+    public static Locks createLockManager( Config config, Clock clock, LogService logging )
     {
         String key = config.get( GraphDatabaseFacadeFactory.Configuration.lock_manager );
         for ( Locks.Factory candidate : Service.load( Locks.Factory.class ) )
         {
             String candidateId = candidate.getKeys().iterator().next();
             if ( candidateId.equals( key ) )
             {
-                return candidate.newInstance( ResourceTypes.values() );
+                return candidate.newInstance( config, clock, ResourceTypes.values() );
             }
             else if ( key.equals( "" ) )
             {
                 logging.getInternalLog( CommunityEditionModule.class )
                         .info( "No locking implementation specified, defaulting to '" + candidateId + "'" );
-                return candidate.newInstance( ResourceTypes.values() );
+                return candidate.newInstance( config, clock, ResourceTypes.values() );
             }
         }
 
         if ( key.equals( "community" ) )
         {
-            return new CommunityLockManger();
+            return new CommunityLockManger( config, clock );
         }
         else if ( key.equals( "" ) )
         {
             logging.getInternalLog( CommunityEditionModule.class )
                     .info( "No locking implementation specified, defaulting to 'community'" );
-            return new CommunityLockManger();
+            return new CommunityLockManger( config, clock );
         }
 
         throw new IllegalArgumentException( "No lock manager found with the name '" + key + "'." );

community/kernel/src/main/java/org/neo4j/kernel/impl/factory/DataSourceModule.java

@@ -160,8 +160,7 @@ public DataSourceModule( final PlatformModule platformModule, EditionModule edit
 
         SchemaWriteGuard schemaWriteGuard = deps.satisfyDependency( editionModule.schemaWriteGuard );
 
-        Clock clock = getClock();
-        guard = createGuard( deps, clock, logging );
+        guard = createGuard( deps, platformModule.clock, logging );
 
         kernelEventHandlers = new KernelEventHandlers( logging.getInternalLog( KernelEventHandlers.class ) );
 
@@ -216,7 +215,7 @@ public DataSourceModule( final PlatformModule platformModule, EditionModule edit
                 platformModule.tracers,
                 procedures,
                 editionModule.ioLimiter,
-                clock, editionModule.accessCapability ) );
+                platformModule.clock, editionModule.accessCapability ) );
 
         dataSourceManager.register( neoStoreDataSource );
 
@@ -236,11 +235,6 @@ public DataSourceModule( final PlatformModule platformModule, EditionModule edit
         this.kernelAPI = neoStoreDataSource::getKernel;
     }
 
-    protected Clock getClock()
-    {
-        return Clock.systemUTC();
-    }
-
     protected RelationshipProxy.RelationshipActions createRelationshipActions(
             final GraphDatabaseService graphDatabaseService,
             final ThreadToStatementContextBridge threadToStatementContextBridge,

community/kernel/src/main/java/org/neo4j/kernel/impl/factory/PlatformModule.java

@@ -21,6 +21,7 @@
 
 import java.io.File;
 import java.io.IOException;
+import java.time.Clock;
 import java.util.HashMap;
 import java.util.List;
 import java.util.Map;
@@ -104,10 +105,13 @@ public class PlatformModule
 
     public final TransactionStats transactionMonitor;
 
+    public final Clock clock;
+
     public PlatformModule( File providedStoreDir, Map<String, String> params, DatabaseInfo databaseInfo,
             GraphDatabaseFacadeFactory.Dependencies externalDependencies, GraphDatabaseFacade graphDatabaseFacade )
     {
         this.databaseInfo = databaseInfo;
+        clock = createClock();
         this.dataSourceManager = new DataSourceManager();
         dependencies = new org.neo4j.kernel.impl.util.Dependencies(
                 new DataSourceManager.DependencyResolverSupplier( dataSourceManager ) );
@@ -168,7 +172,7 @@ public PlatformModule( File providedStoreDir, Map<String, String> params, Databa
         // Anyways please fix this.
         dependencies.satisfyDependency( dataSourceManager );
 
-        availabilityGuard = new AvailabilityGuard( Clocks.systemClock(), logging.getInternalLog(
+        availabilityGuard = new AvailabilityGuard( clock, logging.getInternalLog(
                 AvailabilityGuard.class ) );
 
         transactionMonitor = dependencies.satisfyDependency( createTransactionStats() );
@@ -182,6 +186,11 @@ public PlatformModule( File providedStoreDir, Map<String, String> params, Databa
         publishPlatformInfo( dependencies.resolveDependency( UsageData.class ) );
     }
 
+    protected Clock createClock()
+    {
+        return Clocks.systemClock();
+    }
+
     private void publishPlatformInfo( UsageData sysInfo )
     {
         sysInfo.set( UsageDataKeys.version, Version.getNeo4jVersion() );

community/kernel/src/main/java/org/neo4j/kernel/impl/locking/LockAcquisitionTimeoutException.java

@@ -0,0 +1,40 @@
+/*
+ * Copyright (c) 2002-2016 "Neo Technology,"
+ * Network Engine for Objects in Lund AB [http://neotechnology.com]
+ *
+ * This file is part of Neo4j.
+ *
+ * Neo4j is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program.  If not, see <http://www.gnu.org/licenses/>.
+ */
+package org.neo4j.kernel.impl.locking;
+
+import org.neo4j.graphdb.TransactionTerminatedException;
+import org.neo4j.kernel.api.exceptions.Status;
+import org.neo4j.storageengine.api.lock.ResourceType;
+
+/**
+ * Used in lock clients for cases when we unable to acquire a lock for a time that exceed configured
+ * timeout, if any.
+ *
+ * @see Locks.Client
+ */
+public class LockAcquisitionTimeoutException extends TransactionTerminatedException
+{
+    public LockAcquisitionTimeoutException( ResourceType resourceType, long resourceId, long timeoutMillis )
+    {
+        super( Status.Transaction.LockAcquisitionTimeout,
+                String.format( "Unable to acquire lock for resource: %s with id: %d within %d millis.", resourceType,
+                        resourceId, timeoutMillis ) );
+    }
+}

community/kernel/src/main/java/org/neo4j/kernel/impl/locking/Locks.java

@@ -19,7 +19,10 @@
  */
 package org.neo4j.kernel.impl.locking;
 
+import java.time.Clock;
+
 import org.neo4j.helpers.Service;
+import org.neo4j.kernel.configuration.Config;
 import org.neo4j.storageengine.api.lock.AcquireLockTimeoutException;
 import org.neo4j.storageengine.api.lock.ResourceLocker;
 import org.neo4j.storageengine.api.lock.ResourceType;
@@ -53,7 +56,7 @@ public Factory( String key, String... altKeys )
             super( key, altKeys );
         }
 
-        public abstract Locks newInstance( ResourceType[] resourceTypes );
+        public abstract Locks newInstance( Config config, Clock clocks, ResourceType[] resourceTypes );
     }
 
     /** For introspection and debugging. */

community/kernel/src/main/java/org/neo4j/kernel/impl/locking/ResourceTypes.java

@@ -85,19 +85,6 @@ public static long indexEntryResourceId( long labelId, long propertyKeyId, Strin
         long hob = hash( labelId + hash( propertyKeyId ) );
         hob <<= 32;
         return hob + propertyValue.hashCode();
-
-        // The previous schema index entry hashing method used up until and
-        // including Neo4j 2.1.x looks like the following:
-        //
-        //   long result = labelId;
-        //   result = 31 * result + propertyKeyId;
-        //   result = 31 * result + propertyValue.hashCode();
-        //   return result;
-        //
-        // It was replaced because it was prone to collisions. I left it in
-        // this comment in case we need it for supporting rolling upgrades.
-        // This comment can be deleted once RU from 2.1 to 2.2 is no longer a
-        // concern.
     }
 
     private static int hash( long value )

community/kernel/src/main/java/org/neo4j/kernel/impl/locking/community/CommunityLockManger.java

@@ -19,13 +19,21 @@
  */
 package org.neo4j.kernel.impl.locking.community;
 
+import java.time.Clock;
+
+import org.neo4j.kernel.configuration.Config;
 import org.neo4j.kernel.impl.locking.Locks;
 
 public class CommunityLockManger implements Locks
 {
-    private final LockManagerImpl manager = new LockManagerImpl( new RagManager() );
+    private final LockManagerImpl manager;
     private volatile boolean closed;
 
+    public CommunityLockManger( Config config, Clock clock )
+    {
+        manager = new LockManagerImpl( new RagManager(), config, clock );
+    }
+
     @Override
     public Client newClient()
     {

community/kernel/src/main/java/org/neo4j/kernel/impl/locking/community/LockManagerImpl.java

@@ -19,43 +19,55 @@
  */
 package org.neo4j.kernel.impl.locking.community;
 
+import java.time.Clock;
 import java.util.HashMap;
 import java.util.Map;
 
+import org.neo4j.graphdb.factory.GraphDatabaseSettings;
 import org.neo4j.helpers.collection.Visitor;
 import org.neo4j.kernel.DeadlockDetectedException;
+import org.neo4j.kernel.configuration.Config;
 import org.neo4j.kernel.impl.transaction.IllegalResourceException;
 import org.neo4j.logging.Logger;
 
 public class LockManagerImpl
 {
     private final Map<Object,RWLock> resourceLockMap = new HashMap<>();
     private final RagManager ragManager;
+    private Clock clock;
 
-    public LockManagerImpl( RagManager ragManager )
+    /**
+     * Time within which any particular lock should be acquired.
+     * @see GraphDatabaseSettings#lock_acquisition_timeout
+     */
+    private long lockAcquisitionTimeoutMillis;
+
+    public LockManagerImpl( RagManager ragManager, Config config, Clock clock )
     {
         this.ragManager = ragManager;
+        this.clock = clock;
+        this.lockAcquisitionTimeoutMillis = config.get( GraphDatabaseSettings.lock_acquisition_timeout );
     }
 
-    public boolean getReadLock( Object resource, Object tx )
+    public boolean getReadLock( LockResource resource, Object tx )
             throws DeadlockDetectedException, IllegalResourceException
     {
         return unusedResourceGuard( resource, tx, getRWLockForAcquiring( resource, tx ).acquireReadLock( tx ) );
     }
 
-    public boolean tryReadLock( Object resource, Object tx )
+    public boolean tryReadLock( LockResource resource, Object tx )
             throws IllegalResourceException
     {
         return unusedResourceGuard( resource, tx, getRWLockForAcquiring( resource, tx ).tryAcquireReadLock( tx ) );
     }
 
-    public boolean getWriteLock( Object resource, Object tx )
+    public boolean getWriteLock( LockResource resource, Object tx )
             throws DeadlockDetectedException, IllegalResourceException
     {
         return unusedResourceGuard( resource, tx, getRWLockForAcquiring( resource, tx ).acquireWriteLock( tx ) );
     }
 
-    public boolean tryWriteLock( Object resource, Object tx )
+    public boolean tryWriteLock( LockResource resource, Object tx )
             throws IllegalResourceException
     {
         return unusedResourceGuard( resource, tx, getRWLockForAcquiring( resource, tx ).tryAcquireWriteLock( tx ) );
@@ -138,7 +150,7 @@ private void assertValidArguments( Object resource, Object tx )
         }
     }
 
-    private RWLock getRWLockForAcquiring( Object resource, Object tx )
+    private RWLock getRWLockForAcquiring( LockResource resource, Object tx )
     {
         assertValidArguments( resource, tx );
         synchronized ( resourceLockMap )
@@ -155,9 +167,9 @@ private RWLock getRWLockForAcquiring( Object resource, Object tx )
     }
 
     // visible for testing
-    protected RWLock createLock( Object resource )
+    protected RWLock createLock( LockResource resource )
     {
-        return new RWLock( resource, ragManager );
+        return new RWLock( resource, ragManager, clock, lockAcquisitionTimeoutMillis );
     }
 
     private RWLock getRWLockForReleasing( Object resource, Object tx, int readCountPrerequisite,