New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

deadlock on agent shutdown #52

Closed
ypujante opened this Issue May 5, 2011 · 1 comment

Comments

Projects
None yet
1 participant
@ypujante
Member

ypujante commented May 5, 2011

Here are the stack traces:

"/anet-cloud/i001" prio=3 tid=0x00000000026fe000 nid=0x13 waiting for monitor entry [0xfffffd7fec216000]
   java.lang.Thread.State: BLOCKED (on object monitor)
    at org.linkedin.glu.agent.impl.script.ScriptManagerImpl.findScript(ScriptManagerImpl.groovy)
    - waiting to lock <0xfffffd7ff3624880> (a org.linkedin.glu.agent.impl.script.ScriptManagerImpl)
    at org.linkedin.glu.agent.impl.script.ScriptManagerImpl$findScript.callCurrent(Unknown Source)
    at org.linkedin.glu.agent.impl.script.ScriptManagerImpl.getScript(ScriptManagerImpl.groovy:245)
    at sun.reflect.GeneratedMethodAccessor162.invoke(Unknown Source)
    at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:25)
    at java.lang.reflect.Method.invoke(Method.java:597)
    at org.codehaus.groovy.reflection.CachedMethod.invoke(CachedMethod.java:88)
    at groovy.lang.MetaMethod.doMethodInvoke(MetaMethod.java:233)
    at org.codehaus.groovy.runtime.metaclass.ClosureMetaClass.invokeMethod(ClosureMetaClass.java:362)
    at groovy.lang.MetaClassImpl.invokeMethod(MetaClassImpl.java:886)
    at org.codehaus.groovy.runtime.callsite.PogoMetaClassSite.callCurrent(PogoMetaClassSite.java:66)
    at org.codehaus.groovy.runtime.callsite.AbstractCallSite.callCurrent(AbstractCallSite.java:149)
    at org.linkedin.glu.agent.impl.script.ScriptManagerImpl$_createNode_closure5.doCall(ScriptManagerImpl.groovy:200)
    at sun.reflect.GeneratedMethodAccessor188.invoke(Unknown Source)
    at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:25)
    at java.lang.reflect.Method.invoke(Method.java:597)
    at org.codehaus.groovy.reflection.CachedMethod.invoke(CachedMethod.java:88)
    at groovy.lang.MetaMethod.doMethodInvoke(MetaMethod.java:233)
    at org.codehaus.groovy.runtime.metaclass.ClosureMetaClass.invokeMethod(ClosureMetaClass.java:273)
    at groovy.lang.MetaClassImpl.invokeMethod(MetaClassImpl.java:886)
    at org.codehaus.groovy.runtime.callsite.PogoMetaClassSite.callCurrent(PogoMetaClassSite.java:66)
    at org.codehaus.groovy.runtime.callsite.AbstractCallSite.callCurrent(AbstractCallSite.java:149)
    at org.linkedin.glu.agent.impl.script.ScriptManagerImpl$_createNode_closure5.doCall(ScriptManagerImpl.groovy)
    at sun.reflect.GeneratedMethodAccessor187.invoke(Unknown Source)
    at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:25)
    at java.lang.reflect.Method.invoke(Method.java:597)
    at org.codehaus.groovy.reflection.CachedMethod.invoke(CachedMethod.java:88)
    at groovy.lang.MetaMethod.doMethodInvoke(MetaMethod.java:233)
    at org.codehaus.groovy.runtime.metaclass.ClosureMetaClass.invokeMethod(ClosureMetaClass.java:273)
    at groovy.lang.MetaClassImpl.invokeMethod(MetaClassImpl.java:886)
    at groovy.lang.Closure.call(Closure.java:276)
    at org.codehaus.groovy.runtime.ConvertedMap.invokeCustom(ConvertedMap.java:48)
    at org.codehaus.groovy.runtime.ConversionHandler.invoke(ConversionHandler.java:79)
    at $Proxy5.getState(Unknown Source)
    at sun.reflect.GeneratedMethodAccessor186.invoke(Unknown Source)
    at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:25)
    at java.lang.reflect.Method.invoke(Method.java:597)
    at org.codehaus.groovy.reflection.CachedMethod.invoke(CachedMethod.java:88)
    at groovy.lang.MetaMethod.doMethodInvoke(MetaMethod.java:233)
    at groovy.lang.MetaClassImpl$GetBeanMethodMetaProperty.getProperty(MetaClassImpl.java:3468)
    at org.codehaus.groovy.runtime.callsite.GetEffectivePojoPropertySite.getProperty(GetEffectivePojoPropertySite.java:61)
    at org.codehaus.groovy.runtime.callsite.AbstractCallSite.callGetProperty(AbstractCallSite.java:235)
    at JettyServerWithWarsScript$_closure2.doCall(JettyServerWithWarsScript.groovy:71)
    at sun.reflect.GeneratedMethodAccessor204.invoke(Unknown Source)
    at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:25)
    at java.lang.reflect.Method.invoke(Method.java:597)
    at org.codehaus.groovy.reflection.CachedMethod.invoke(CachedMethod.java:88)
    at groovy.lang.MetaMethod.doMethodInvoke(MetaMethod.java:233)
    at org.codehaus.groovy.runtime.metaclass.ClosureMetaClass.invokeMethod(ClosureMetaClass.java:273)
    at groovy.lang.MetaClassImpl.invokeMethod(MetaClassImpl.java:886)
    at org.codehaus.groovy.runtime.callsite.PogoMetaClassSite.callCurrent(PogoMetaClassSite.java:66)
    at org.codehaus.groovy.runtime.callsite.AbstractCallSite.callCurrent(AbstractCallSite.java:149)
    at JettyServerWithWarsScript$_closure2.doCall(JettyServerWithWarsScript.groovy)
    at sun.reflect.GeneratedMethodAccessor203.invoke(Unknown Source)
    at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:25)
    at java.lang.reflect.Method.invoke(Method.java:597)
    at org.codehaus.groovy.reflection.CachedMethod.invoke(CachedMethod.java:88)
    at groovy.lang.MetaMethod.doMethodInvoke(MetaMethod.java:233)
    at org.codehaus.groovy.runtime.metaclass.ClosureMetaClass.invokeMethod(ClosureMetaClass.java:273)
    at groovy.lang.MetaClassImpl.invokeMethod(MetaClassImpl.java:886)
    at org.codehaus.groovy.runtime.callsite.PogoMetaClassSite.call(PogoMetaClassSite.java:39)
    at org.codehaus.groovy.runtime.callsite.CallSiteArray.defaultCall(CallSiteArray.java:40)
    at org.codehaus.groovy.runtime.callsite.PogoMetaClassSite.call(PogoMetaClassSite.java:54)
    at org.codehaus.groovy.runtime.callsite.AbstractCallSite.call(AbstractCallSite.java:120)
    at org.linkedin.glu.agent.impl.script.TimerExecution.execute(TimerExecution.groovy:32)
    at sun.reflect.GeneratedMethodAccessor175.invoke(Unknown Source)
    at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:25)
    at java.lang.reflect.Method.invoke(Method.java:597)
    at org.codehaus.groovy.reflection.CachedMethod.invoke(CachedMethod.java:88)
    at groovy.lang.MetaMethod.doMethodInvoke(MetaMethod.java:233)
    at org.codehaus.groovy.runtime.metaclass.ClosureMetaClass.invokeMethod(ClosureMetaClass.java:362)
    at groovy.lang.MetaClassImpl.invokeMethod(MetaClassImpl.java:886)
    at org.codehaus.groovy.runtime.callsite.PogoMetaClassSite.callCurrent(PogoMetaClassSite.java:66)
    at org.codehaus.groovy.runtime.callsite.AbstractCallSite.callCurrent(AbstractCallSite.java:145)
    at org.linkedin.glu.agent.impl.script.FutureExecutionImpl$_closure1.doCall(FutureExecutionImpl.groovy:66)
    at sun.reflect.GeneratedMethodAccessor173.invoke(Unknown Source)
    at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:25)
    at java.lang.reflect.Method.invoke(Method.java:597)
    at org.codehaus.groovy.reflection.CachedMethod.invoke(CachedMethod.java:88)
    at groovy.lang.MetaMethod.doMethodInvoke(MetaMethod.java:233)
    at org.codehaus.groovy.runtime.metaclass.ClosureMetaClass.invokeMethod(ClosureMetaClass.java:273)
    at groovy.lang.MetaClassImpl.invokeMethod(MetaClassImpl.java:886)
    at org.codehaus.groovy.runtime.callsite.PogoMetaClassSite.callCurrent(PogoMetaClassSite.java:66)
    at org.codehaus.groovy.runtime.callsite.AbstractCallSite.callCurrent(AbstractCallSite.java:149)
    at org.linkedin.glu.agent.impl.script.FutureExecutionImpl$_closure1.doCall(FutureExecutionImpl.groovy)
    at sun.reflect.GeneratedMethodAccessor172.invoke(Unknown Source)
    at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:25)
    at java.lang.reflect.Method.invoke(Method.java:597)
    at org.codehaus.groovy.reflection.CachedMethod.invoke(CachedMethod.java:88)
    at groovy.lang.MetaMethod.doMethodInvoke(MetaMethod.java:233)
    at org.codehaus.groovy.runtime.metaclass.ClosureMetaClass.invokeMethod(ClosureMetaClass.java:273)
    at groovy.lang.MetaClassImpl.invokeMethod(MetaClassImpl.java:886)
    at groovy.lang.Closure.call(Closure.java:276)
    at org.codehaus.groovy.runtime.ConvertedClosure.invokeCustom(ConvertedClosure.java:51)
    at org.codehaus.groovy.runtime.ConversionHandler.invoke(ConversionHandler.java:79)
    at $Proxy7.call(Unknown Source)
    at java.util.concurrent.FutureTask$Sync.innerRun(FutureTask.java:303)
    at java.util.concurrent.FutureTask.run(FutureTask.java:138)
    at java_util_concurrent_RunnableFuture$run$0.call(Unknown Source)
    at org.linkedin.glu.agent.impl.script.FutureExecutionImpl.run(FutureExecutionImpl.groovy:80)
    at java_util_concurrent_RunnableFuture$run.call(Unknown Source)
    at org.linkedin.glu.agent.impl.script.ScriptExecution$_closure2.doCall(ScriptExecution.groovy:568)
    at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
    at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:39)
    at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:25)
    at java.lang.reflect.Method.invoke(Method.java:597)
    at org.codehaus.groovy.reflection.CachedMethod.invoke(CachedMethod.java:88)
    at groovy.lang.MetaMethod.doMethodInvoke(MetaMethod.java:233)
    at org.codehaus.groovy.runtime.metaclass.ClosureMetaClass.invokeMethod(ClosureMetaClass.java:273)
    at groovy.lang.MetaClassImpl.invokeMethod(MetaClassImpl.java:886)
    at org.codehaus.groovy.runtime.callsite.PogoMetaClassSite.callCurrent(PogoMetaClassSite.java:66)
    at org.codehaus.groovy.runtime.callsite.AbstractCallSite.callCurrent(AbstractCallSite.java:149)
    at org.linkedin.glu.agent.impl.script.ScriptExecution$_closure2.doCall(ScriptExecution.groovy)
    at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
    at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:39)
    at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:25)
    at java.lang.reflect.Method.invoke(Method.java:597)
    at org.codehaus.groovy.reflection.CachedMethod.invoke(CachedMethod.java:88)
    at groovy.lang.MetaMethod.doMethodInvoke(MetaMethod.java:233)
    at org.codehaus.groovy.runtime.metaclass.ClosureMetaClass.invokeMethod(ClosureMetaClass.java:273)
    at groovy.lang.MetaClassImpl.invokeMethod(MetaClassImpl.java:886)
    at groovy.lang.Closure.call(Closure.java:276)
    at groovy.lang.Closure.call(Closure.java:271)
    at groovy.lang.Closure.run(Closure.java:354)
    at java.lang.Thread.run(Thread.java:619)

and

"Thread-11" prio=3 tid=0x0000000002aa2000 nid=0x36e9f in Object.wait() [0xfffffd7fea9ce000]
   java.lang.Thread.State: WAITING (on object monitor)
    at java.lang.Object.wait(Native Method)
    - waiting on <0xfffffd7ff3a6c268> (a java.lang.Thread)
    at java.lang.Thread.join(Thread.java:1143)
    - locked <0xfffffd7ff3a6c268> (a java.lang.Thread)
    at java.lang.Thread.join(Thread.java:1196)
    at sun.reflect.GeneratedMethodAccessor140.invoke(Unknown Source)
    at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:25)
    at java.lang.reflect.Method.invoke(Method.java:597)
    at org.codehaus.groovy.runtime.callsite.PojoMetaMethodSite$PojoCachedMethodSiteNoUnwrapNoCoerce.invoke(PojoMetaMethodSite.java:229)
    at org.codehaus.groovy.runtime.callsite.PojoMetaMethodSite.call(PojoMetaMethodSite.java:52)
    at org.codehaus.groovy.runtime.callsite.AbstractCallSite.call(AbstractCallSite.java:120)
    at org.codehaus.groovy.runtime.callsite.AbstractCallSite.callSafe(AbstractCallSite.java:83)
    at org.linkedin.glu.agent.impl.script.ScriptExecution.waitForShutdown(ScriptExecution.groovy:141)
    at org.linkedin.util.lifecycle.Shutdownable$waitForShutdown$1.call(Unknown Source)
    at org.linkedin.glu.agent.impl.script.ScriptNode.waitForShutdown(ScriptNode.groovy:68)
    at org.linkedin.util.lifecycle.Shutdownable$waitForShutdown$1.call(Unknown Source)
    at org.linkedin.groovy.util.concurrent.GroovyConcurrentUtils$_waitForShutdownMultiple_closure2.doCall(GroovyConcurrentUtils.groovy:74)
    at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
    at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:39)
    at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:25)
    at java.lang.reflect.Method.invoke(Method.java:597)
    at org.codehaus.groovy.reflection.CachedMethod.invoke(CachedMethod.java:88)
    at groovy.lang.MetaMethod.doMethodInvoke(MetaMethod.java:233)
    at org.codehaus.groovy.runtime.metaclass.ClosureMetaClass.invokeMethod(ClosureMetaClass.java:273)
    at groovy.lang.MetaClassImpl.invokeMethod(MetaClassImpl.java:886)
    at groovy.lang.Closure.call(Closure.java:276)
    at groovy.lang.Closure.call(Closure.java:289)
    at org.codehaus.groovy.runtime.DefaultGroovyMethods.each(DefaultGroovyMethods.java:1198)
    at org.codehaus.groovy.runtime.DefaultGroovyMethods.each(DefaultGroovyMethods.java:1174)
    at org.codehaus.groovy.runtime.dgm$109.invoke(Unknown Source)
    at org.codehaus.groovy.runtime.callsite.PojoMetaMethodSite$PojoMetaMethodSiteNoUnwrapNoCoerce.invoke(PojoMetaMethodSite.java:270)
    at org.codehaus.groovy.runtime.callsite.PojoMetaMethodSite.call(PojoMetaMethodSite.java:52)
    at org.codehaus.groovy.runtime.callsite.CallSiteArray.defaultCall(CallSiteArray.java:40)
    at org.codehaus.groovy.runtime.callsite.AbstractCallSite.call(AbstractCallSite.java:116)
    at org.codehaus.groovy.runtime.callsite.AbstractCallSite.call(AbstractCallSite.java:124)
    at org.linkedin.groovy.util.concurrent.GroovyConcurrentUtils.waitForShutdownMultiple(GroovyConcurrentUtils.groovy:74)
    at org.linkedin.groovy.util.concurrent.GroovyConcurrentUtils$waitForShutdownMultiple.call(Unknown Source)
    at org.codehaus.groovy.runtime.callsite.CallSiteArray.defaultCall(CallSiteArray.java:40)
    at org.codehaus.groovy.runtime.callsite.AbstractCallSite.call(AbstractCallSite.java:116)
    at org.codehaus.groovy.runtime.callsite.AbstractCallSite.call(AbstractCallSite.java:132)
    at org.linkedin.glu.agent.impl.script.ScriptManagerImpl.waitForShutdown(ScriptManagerImpl.groovy:487)
    - locked <0xfffffd7ff3624880> (a org.linkedin.glu.agent.impl.script.ScriptManagerImpl)
    at org.linkedin.util.lifecycle.Shutdownable$waitForShutdown.call(Unknown Source)
    at org.codehaus.groovy.runtime.callsite.CallSiteArray.defaultCall(CallSiteArray.java:40)
    at org.codehaus.groovy.runtime.callsite.AbstractCallSite.call(AbstractCallSite.java:116)
    at org.codehaus.groovy.runtime.callsite.AbstractCallSite.call(AbstractCallSite.java:124)
    at org.linkedin.glu.agent.impl.script.StateKeeperScriptManager.waitForShutdown(StateKeeperScriptManager.groovy:219)
    at org.linkedin.util.lifecycle.Shutdownable$waitForShutdown.call(Unknown Source)
    at org.codehaus.groovy.runtime.callsite.CallSiteArray.defaultCall(CallSiteArray.java:40)
    at org.codehaus.groovy.runtime.callsite.AbstractCallSite.call(AbstractCallSite.java:116)
    at org.codehaus.groovy.runtime.callsite.AbstractCallSite.call(AbstractCallSite.java:124)
    at org.linkedin.glu.agent.impl.AgentImpl.waitForShutdown(AgentImpl.groovy:136)
    at org.linkedin.util.lifecycle.Shutdownable$waitForShutdown.callCurrent(Unknown Source)
    at org.codehaus.groovy.runtime.callsite.CallSiteArray.defaultCallCurrent(CallSiteArray.java:44)
    at org.codehaus.groovy.runtime.callsite.AbstractCallSite.callCurrent(AbstractCallSite.java:141)
    at org.codehaus.groovy.runtime.callsite.AbstractCallSite.callCurrent(AbstractCallSite.java:149)
    at org.linkedin.glu.agent.impl.AgentImpl.waitForShutdown(AgentImpl.groovy:141)
    at org.linkedin.util.lifecycle.Shutdownable$waitForShutdown$1.call(Unknown Source)
    at org.codehaus.groovy.runtime.callsite.CallSiteArray.defaultCall(CallSiteArray.java:40)
    at org.codehaus.groovy.runtime.callsite.AbstractCallSite.call(AbstractCallSite.java:116)
    at org.codehaus.groovy.runtime.callsite.AbstractCallSite.call(AbstractCallSite.java:120)
    at org.linkedin.glu.agent.server.AgentMain$_registerTerminationHandler_closure13.doCall(AgentMain.groovy:457)
    at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
    at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:39)
    at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:25)
    at java.lang.reflect.Method.invoke(Method.java:597)
    at org.codehaus.groovy.reflection.CachedMethod.invoke(CachedMethod.java:88)
    at groovy.lang.MetaMethod.doMethodInvoke(MetaMethod.java:233)
    at org.codehaus.groovy.runtime.metaclass.ClosureMetaClass.invokeMethod(ClosureMetaClass.java:273)
    at groovy.lang.MetaClassImpl.invokeMethod(MetaClassImpl.java:886)
    at org.codehaus.groovy.runtime.callsite.PogoMetaClassSite.callCurrent(PogoMetaClassSite.java:66)
    at org.codehaus.groovy.runtime.callsite.CallSiteArray.defaultCallCurrent(CallSiteArray.java:44)
    at org.codehaus.groovy.runtime.callsite.AbstractCallSite.callCurrent(AbstractCallSite.java:141)
    at org.codehaus.groovy.runtime.callsite.AbstractCallSite.callCurrent(AbstractCallSite.java:149)
    at org.linkedin.glu.agent.server.AgentMain$_registerTerminationHandler_closure13.doCall(AgentMain.groovy)
    at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
    at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:39)
    at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:25)
    at java.lang.reflect.Method.invoke(Method.java:597)
    at org.codehaus.groovy.reflection.CachedMethod.invoke(CachedMethod.java:88)
    at groovy.lang.MetaMethod.doMethodInvoke(MetaMethod.java:233)
    at org.codehaus.groovy.runtime.metaclass.ClosureMetaClass.invokeMethod(ClosureMetaClass.java:273)
    at groovy.lang.MetaClassImpl.invokeMethod(MetaClassImpl.java:886)
    at groovy.lang.Closure.call(Closure.java:276)
    at groovy.lang.Closure.call(Closure.java:271)
    at groovy.lang.Closure.run(Closure.java:354)
    at java.lang.Thread.run(Thread.java:619)

@ghost ghost assigned ypujante May 5, 2011

@ypujante

This comment has been minimized.

Member

ypujante commented May 5, 2011

Fixed with 2.2.3

@ypujante ypujante closed this May 5, 2011

Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment