deadlock during cluster failover

SourceForge Submitter: eiman .
We are running JBoss 3.2.5 on jdk 1.4.2 running on
redhat linux.

In our environment we are running a two node cluster
(node A and node B). We restarted node A and then
within a minute restarted node B. Upon doing that we
found a deadlock in node B:

Java stack information for the threads listed above:
===================================================
"MessageDispatcher up processing thread":
at
org.jboss.system.ServiceController.create(ServiceController.java:229)

• waiting to lock <0x56505440> (a
  org.jboss.system.ServiceController)
  at sun.reflect.NativeMethodAccessorImpl.invoke0(Native
  Method)
  at
  sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:39)
  at
  sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:25)
  at java.lang.reflect.Method.invoke(Method.java:324)
  at
  org.jboss.mx.server.ReflectedDispatcher.dispatch(ReflectedDispatcher.java:60)
  at
  org.jboss.mx.server.Invocation.dispatch(Invocation.java:61)
  at
  org.jboss.mx.server.Invocation.dispatch(Invocation.java:53)
  at
  org.jboss.mx.server.Invocation.invoke(Invocation.java:86)
  at
  org.jboss.mx.server.AbstractMBeanInvoker.invoke(AbstractMBeanInvoker.java:185)
  at
  org.jboss.mx.server.MBeanServerImpl.invoke(MBeanServerImpl.java:473)
  at
  org.jboss.mx.util.MBeanProxyExt.invoke(MBeanProxyExt.java:176)
  at $Proxy33.create(Unknown Source)
  at
  org.jboss.web.AbstractWebContainer.create(AbstractWebContainer.java:291)
  at
  org.jboss.deployment.MainDeployer.create(MainDeployer.java:790)
  at
  org.jboss.deployment.MainDeployer.create(MainDeployer.java:782)
  at
  org.jboss.deployment.MainDeployer.create(MainDeployer.java:782)
  at
  org.jboss.deployment.MainDeployer.deploy(MainDeployer.java:644)
  at
  org.jboss.deployment.MainDeployer.deploy(MainDeployer.java:608)
  at
  org.jboss.deployment.MainDeployer.deploy(MainDeployer.java:592)
  at sun.reflect.NativeMethodAccessorImpl.invoke0(Native
  Method)
  at
  sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:39)
  at
  sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:25)
  at java.lang.reflect.Method.invoke(Method.java:324)
  at
  org.jboss.mx.server.ReflectedDispatcher.dispatch(ReflectedDispatcher.java:60)
  at
  org.jboss.mx.server.Invocation.dispatch(Invocation.java:61)
  at
  org.jboss.mx.server.Invocation.dispatch(Invocation.java:53)
  at
  org.jboss.mx.server.Invocation.invoke(Invocation.java:86)
  at
  org.jboss.mx.server.AbstractMBeanInvoker.invoke(AbstractMBeanInvoker.java:185)
  at
  org.jboss.mx.server.MBeanServerImpl.invoke(MBeanServerImpl.java:473)
  at
  org.jboss.ha.singleton.HASingletonController.invokeSingletonMBeanMethod(HASingletonController.java:114)
  at
  org.jboss.ha.singleton.HASingletonController.startSingleton(HASingletonController.java:63)
  at
  org.jboss.ha.singleton.HASingletonSupport.makeThisNodeMaster(HASingletonSupport.java:137)
  at
  org.jboss.ha.singleton.HASingletonSupport.partitionTopologyChanged(HASingletonSupport.java:118)
  at
  org.jboss.ha.jmx.HAServiceMBeanSupport$1.replicantsChanged(HAServiceMBeanSupport.java:197)
  at
  org.jboss.ha.framework.server.DistributedReplicantManagerImpl.notifyKeyListeners(DistributedReplicantManagerImpl.java:701)
• locked <0x56609350> (a java.util.HashMap)
  at
  org.jboss.ha.framework.server.DistributedReplicantManagerImpl._remove(DistributedReplicantManagerImpl.java:512)
  at sun.reflect.NativeMethodAccessorImpl.invoke0(Native
  Method)
  at
  sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:39)
  at
  sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:25)
  at java.lang.reflect.Method.invoke(Method.java:324)
  at
  org.jgroups.blocks.MethodCall.invoke(MethodCall.java:236)
  at
  org.jboss.ha.framework.server.HAPartitionImpl.handle(HAPartitionImpl.java:826)
  at
  org.jgroups.blocks.RequestCorrelator.handleRequest(RequestCorrelator.java:615)
  at
  org.jgroups.blocks.RequestCorrelator.receiveMessage(RequestCorrelator.java:512)
  at
  org.jgroups.blocks.RequestCorrelator.receive(RequestCorrelator.java:326)
  at
  org.jgroups.blocks.MessageDispatcher$ProtocolAdapter.handleUp(MessageDispatcher.java:722)
  at
  org.jgroups.blocks.MessageDispatcher$ProtocolAdapter.access$300(MessageDispatcher.java:554)
  at
  org.jgroups.blocks.MessageDispatcher$1.run(MessageDispatcher.java:691)
  at java.lang.Thread.run(Thread.java:534)
  "main":
  at
  org.jboss.ha.framework.server.DistributedReplicantManagerImpl.registerListener(DistributedReplicantManagerImpl.java:398)
• waiting to lock <0x56609350> (a java.util.HashMap)
  at
  org.jboss.ha.framework.server.HATarget.updateHAPartition(HATarget.java:130)
  at
  org.jboss.ha.framework.server.HATarget.<init>(HATarget.java:58)
  at
  org.jboss.proxy.ejb.ProxyFactoryHA.setupInvokers(ProxyFactoryHA.java:102)
  at
  org.jboss.proxy.ejb.ProxyFactory.start(ProxyFactory.java:186)
  at
  org.jboss.proxy.ejb.ProxyFactoryHA.start(ProxyFactoryHA.java:85)
  at
  org.jboss.ejb.StatelessSessionContainer.startService(StatelessSessionContainer.java:205)
  at
  org.jboss.system.ServiceMBeanSupport.jbossInternalStart(ServiceMBeanSupport.java:271)
  at
  org.jboss.system.ServiceMBeanSupport.jbossInternalLifecycle(ServiceMBeanSupport.java:221)
  at sun.reflect.GeneratedMethodAccessor3.invoke(Unknown
  Source)
  at
  sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:25)
  at java.lang.reflect.Method.invoke(Method.java:324)
  at
  org.jboss.mx.server.ReflectedDispatcher.dispatch(ReflectedDispatcher.java:60)
  at
  org.jboss.mx.server.Invocation.dispatch(Invocation.java:61)
  at
  org.jboss.mx.server.Invocation.dispatch(Invocation.java:53)
  at
  org.jboss.mx.server.Invocation.invoke(Invocation.java:86)
  at
  org.jboss.mx.server.AbstractMBeanInvoker.invoke(AbstractMBeanInvoker.java:185)
  at
  org.jboss.mx.server.MBeanServerImpl.invoke(MBeanServerImpl.java:473)
  at
  org.jboss.system.ServiceController$ServiceProxy.invoke(ServiceController.java:837)
  at $Proxy17.start(Unknown Source)
  at
  org.jboss.system.ServiceController.start(ServiceController.java:367)
• locked <0x56505440> (a
  org.jboss.system.ServiceController)
  at sun.reflect.GeneratedMethodAccessor7.invoke(Unknown
  Source)
  at
  sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:25)
  at java.lang.reflect.Method.invoke(Method.java:324)
  at
  org.jboss.mx.server.ReflectedDispatcher.dispatch(ReflectedDispatcher.java:60)
  at
  org.jboss.mx.server.Invocation.dispatch(Invocation.java:61)
  at
  org.jboss.mx.server.Invocation.dispatch(Invocation.java:53)
  at
  org.jboss.mx.server.Invocation.invoke(Invocation.java:86)
  at
  org.jboss.mx.server.AbstractMBeanInvoker.invoke(AbstractMBeanInvoker.java:185)
  at
  org.jboss.mx.server.MBeanServerImpl.invoke(MBeanServerImpl.java:473)
  at
  org.jboss.mx.util.MBeanProxyExt.invoke(MBeanProxyExt.java:176)
  at $Proxy48.start(Unknown Source)
  at
  org.jboss.ejb.EjbModule.startService(EjbModule.java:367)
  at
  org.jboss.system.ServiceMBeanSupport.jbossInternalStart(ServiceMBeanSupport.java:271)
  at
  org.jboss.system.ServiceMBeanSupport.jbossInternalLifecycle(ServiceMBeanSupport.java:221)
  at sun.reflect.GeneratedMethodAccessor3.invoke(Unknown
  Source)
  at
  sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:25)
  at java.lang.reflect.Method.invoke(Method.java:324)
  at
  org.jboss.mx.server.ReflectedDispatcher.dispatch(ReflectedDispatcher.java:60)
  at
  org.jboss.mx.server.Invocation.dispatch(Invocation.java:61)
  at
  org.jboss.mx.server.Invocation.dispatch(Invocation.java:53)
  at
  org.jboss.mx.server.Invocation.invoke(Invocation.java:86)
  at
  org.jboss.mx.server.AbstractMBeanInvoker.invoke(AbstractMBeanInvoker.java:185)
  at
  org.jboss.mx.server.MBeanServerImpl.invoke(MBeanServerImpl.java:473)
  at
  org.jboss.system.ServiceController$ServiceProxy.invoke(ServiceController.java:837)
  at $Proxy17.start(Unknown Source)
  at
  org.jboss.system.ServiceController.start(ServiceController.java:367)
• locked <0x56505440> (a
  org.jboss.system.ServiceController)
  at sun.reflect.GeneratedMethodAccessor7.invoke(Unknown
  Source)
  at
  sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:25)
  at java.lang.reflect.Method.invoke(Method.java:324)
  at
  org.jboss.mx.server.ReflectedDispatcher.dispatch(ReflectedDispatcher.java:60)
  at
  org.jboss.mx.server.Invocation.dispatch(Invocation.java:61)
  at
  org.jboss.mx.server.Invocation.dispatch(Invocation.java:53)
  at
  org.jboss.mx.server.Invocation.invoke(Invocation.java:86)
  at
  org.jboss.mx.server.AbstractMBeanInvoker.invoke(AbstractMBeanInvoker.java:185)
  at
  org.jboss.mx.server.MBeanServerImpl.invoke(MBeanServerImpl.java:473)
  at
  org.jboss.mx.util.MBeanProxyExt.invoke(MBeanProxyExt.java:176)
  at $Proxy14.start(Unknown Source)
  at org.jboss.ejb.EJBDeployer.start(EJBDeployer.java:570)
• locked <0x56679000> (a org.jboss.ejb.EJBDeployer)
  at
  org.jboss.deployment.MainDeployer.start(MainDeployer.java:836)
  at
  org.jboss.deployment.MainDeployer.start(MainDeployer.java:828)
  at
  org.jboss.deployment.MainDeployer.deploy(MainDeployer.java:645)
  at
  org.jboss.deployment.MainDeployer.deploy(MainDeployer.java:608)
  at
  sun.reflect.GeneratedMethodAccessor36.invoke(Unknown
  Source)
  at
  sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:25)
  at java.lang.reflect.Method.invoke(Method.java:324)
  at
  org.jboss.mx.server.ReflectedDispatcher.dispatch(ReflectedDispatcher.java:60)
  at
  org.jboss.mx.server.Invocation.dispatch(Invocation.java:61)
  at
  org.jboss.mx.server.Invocation.dispatch(Invocation.java:53)
  at
  org.jboss.mx.server.Invocation.invoke(Invocation.java:86)
  at
  org.jboss.mx.server.AbstractMBeanInvoker.invoke(AbstractMBeanInvoker.java:185)
  at
  org.jboss.mx.server.MBeanServerImpl.invoke(MBeanServerImpl.java:473)
  at
  org.jboss.mx.util.MBeanProxyExt.invoke(MBeanProxyExt.java:176)
  at $Proxy7.deploy(Unknown Source)
  at
  org.jboss.deployment.scanner.URLDeploymentScanner.deploy(URLDeploymentScanner.java:304)
  at
  org.jboss.deployment.scanner.URLDeploymentScanner.scan(URLDeploymentScanner.java:478)
• locked <0x56505080> (a
  org.jboss.deployment.scanner.URLDeploymentScanner)
  at
  org.jboss.deployment.scanner.AbstractDeploymentScanner$ScannerThread.doScan(AbstractDeploymentScanner.java:201)
  at
  org.jboss.deployment.scanner.AbstractDeploymentScanner.startService(AbstractDeploymentScanner.java:274)
• locked <0x565044c0> (a
  org.jboss.deployment.scanner.AbstractDeploymentScanner$ScannerThread)
  at
  org.jboss.system.ServiceMBeanSupport.jbossInternalStart(ServiceMBeanSupport.java:271)
  at
  org.jboss.system.ServiceMBeanSupport.jbossInternalLifecycle(ServiceMBeanSupport.java:221)
  at sun.reflect.GeneratedMethodAccessor3.invoke(Unknown
  Source)
  at
  sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:25)
  at java.lang.reflect.Method.invoke(Method.java:324)
  at
  org.jboss.mx.server.ReflectedDispatcher.dispatch(ReflectedDispatcher.java:60)
  at
  org.jboss.mx.server.Invocation.dispatch(Invocation.java:61)
  at
  org.jboss.mx.server.Invocation.dispatch(Invocation.java:53)
  at
  org.jboss.mx.server.Invocation.invoke(Invocation.java:86)
  at
  org.jboss.mx.server.AbstractMBeanInvoker.invoke(AbstractMBeanInvoker.java:185)
  at
  org.jboss.mx.server.MBeanServerImpl.invoke(MBeanServerImpl.java:473)
  at
  org.jboss.system.ServiceController$ServiceProxy.invoke(ServiceController.java:837)
  at $Proxy0.start(Unknown Source)
  at
  org.jboss.system.ServiceController.start(ServiceController.java:367)
• locked <0x56505440> (a
  org.jboss.system.ServiceController)
  at sun.reflect.GeneratedMethodAccessor7.invoke(Unknown
  Source)
  at
  sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:25)
  at java.lang.reflect.Method.invoke(Method.java:324)
  at
  org.jboss.mx.server.ReflectedDispatcher.dispatch(ReflectedDispatcher.java:60)
  at
  org.jboss.mx.server.Invocation.dispatch(Invocation.java:61)
  at
  org.jboss.mx.server.Invocation.dispatch(Invocation.java:53)
  at
  org.jboss.mx.server.Invocation.invoke(Invocation.java:86)
  at
  org.jboss.mx.server.AbstractMBeanInvoker.invoke(AbstractMBeanInvoker.java:185)
  at
  org.jboss.mx.server.MBeanServerImpl.invoke(MBeanServerImpl.java:473)
  at
  org.jboss.mx.util.MBeanProxyExt.invoke(MBeanProxyExt.java:176)
  at $Proxy4.start(Unknown Source)
  at
  org.jboss.deployment.SARDeployer.start(SARDeployer.java:251)
  at
  org.jboss.deployment.MainDeployer.start(MainDeployer.java:836)
  at
  org.jboss.deployment.MainDeployer.deploy(MainDeployer.java:645)
  at
  org.jboss.deployment.MainDeployer.deploy(MainDeployer.java:608)
  at
  org.jboss.deployment.MainDeployer.deploy(MainDeployer.java:592)
  at sun.reflect.NativeMethodAccessorImpl.invoke0(Native
  Method)
  at
  sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:39)
  at
  sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:25)
  at java.lang.reflect.Method.invoke(Method.java:324)
  at
  org.jboss.mx.server.ReflectedDispatcher.dispatch(ReflectedDispatcher.java:60)
  at
  org.jboss.mx.server.Invocation.dispatch(Invocation.java:61)
  at
  org.jboss.mx.server.Invocation.dispatch(Invocation.java:53)
  at
  org.jboss.mx.server.Invocation.invoke(Invocation.java:86)
  at
  org.jboss.mx.server.AbstractMBeanInvoker.invoke(AbstractMBeanInvoker.java:185)
  at
  org.jboss.mx.server.MBeanServerImpl.invoke(MBeanServerImpl.java:473)
  at
  org.jboss.mx.util.MBeanProxyExt.invoke(MBeanProxyExt.java:176)
  at $Proxy5.deploy(Unknown Source)
  at
  org.jboss.system.server.ServerImpl.doStart(ServerImpl.java:407)
  at
  org.jboss.system.server.ServerImpl.start(ServerImpl.java:311)
  at org.jboss.Main.boot(Main.java:145)
  at org.jboss.Main$1.run(Main.java:399)
  at java.lang.Thread.run(Thread.java:534)

Found 1 deadlock.