Full thread dump Java HotSpot(TM) 64-Bit Server VM (1.5.0_13-b05 mixed mode): [[ Thread processing a STATE_REQ message in STATE_TRANSFER that has come back down the protocol stack to lock the barrier. It is waiting for the in_flight_threads to clear. ]] "Incoming" prio=1 tid=0x000000004ed08930 nid=0x207 waiting on condition [0x0000000041a26000..0x0000000041a27b80] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:118) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1841) at org.jgroups.protocols.BARRIER.closeBarrier(BARRIER.java:158) at org.jgroups.protocols.BARRIER.down(BARRIER.java:91) at org.jgroups.protocols.pbcast.NAKACK.down(NAKACK.java:639) at org.jgroups.protocols.UNICAST.down(UNICAST.java:437) at org.jgroups.protocols.pbcast.STABLE.down(STABLE.java:317) at org.jgroups.protocols.FRAG2.down(FRAG2.java:175) at org.jgroups.protocols.VIEW_SYNC.down(VIEW_SYNC.java:173) at org.jgroups.protocols.pbcast.GMS.down(GMS.java:838) at org.jgroups.protocols.FC.down(FC.java:370) at org.jgroups.protocols.pbcast.STATE_TRANSFER.handleStateReq(STATE_TRANSFER.java:371) - locked <0x00002aaab714b780> (a java.util.HashMap) at org.jgroups.protocols.pbcast.STATE_TRANSFER.up(STATE_TRANSFER.java:117) at org.jgroups.protocols.FC.up(FC.java:408) at org.jgroups.protocols.pbcast.GMS.up(GMS.java:767) at org.jgroups.protocols.VIEW_SYNC.up(VIEW_SYNC.java:161) at org.jgroups.protocols.FRAG2.up(FRAG2.java:205) at org.jgroups.protocols.pbcast.STABLE.up(STABLE.java:234) at org.jgroups.protocols.UNICAST.handleDataReceived(UNICAST.java:579) at org.jgroups.protocols.UNICAST.up(UNICAST.java:250) at org.jgroups.protocols.pbcast.NAKACK.up(NAKACK.java:714) at org.jgroups.protocols.BARRIER.up(BARRIER.java:122) at org.jgroups.protocols.VERIFY_SUSPECT.up(VERIFY_SUSPECT.java:167) at org.jgroups.protocols.FD.up(FD.java:322) at org.jgroups.protocols.MERGE3.up(MERGE3.java:126) at org.jgroups.protocols.Discovery.up(Discovery.java:246) at org.jgroups.protocols.TP$IncomingPacket.handleMyMessage(TP.java:1535) at org.jgroups.protocols.TP$IncomingPacket.run(TP.java:1479) at java.util.concurrent.ThreadPoolExecutor$Worker.runTask(ThreadPoolExecutor.java:650) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:675) at java.lang.Thread.run(Thread.java:595) [[ Thread processing another STATE_REQ message in STATE_TRANSFER that is waiting for the above thread to release its lock on the state_requesters HashMap. They are in the in_flight_threads set causing a deadlock with the top thread. ]] "Incoming" prio=1 tid=0x0000000050259e40 nid=0x206 waiting for monitor entry [0x0000000045a67000..0x0000000045a67c00] at org.jgroups.protocols.pbcast.STATE_TRANSFER.handleStateReq(STATE_TRANSFER.java:358) - waiting to lock <0x00002aaab714b780> (a java.util.HashMap) at org.jgroups.protocols.pbcast.STATE_TRANSFER.up(STATE_TRANSFER.java:117) at org.jgroups.protocols.FC.up(FC.java:408) at org.jgroups.protocols.pbcast.GMS.up(GMS.java:767) at org.jgroups.protocols.VIEW_SYNC.up(VIEW_SYNC.java:161) at org.jgroups.protocols.FRAG2.up(FRAG2.java:205) at org.jgroups.protocols.pbcast.STABLE.up(STABLE.java:234) at org.jgroups.protocols.UNICAST.handleDataReceived(UNICAST.java:579) at org.jgroups.protocols.UNICAST.up(UNICAST.java:250) at org.jgroups.protocols.pbcast.NAKACK.up(NAKACK.java:714) at org.jgroups.protocols.BARRIER.up(BARRIER.java:122) at org.jgroups.protocols.VERIFY_SUSPECT.up(VERIFY_SUSPECT.java:167) at org.jgroups.protocols.FD.up(FD.java:322) at org.jgroups.protocols.MERGE3.up(MERGE3.java:126) at org.jgroups.protocols.Discovery.up(Discovery.java:246) at org.jgroups.protocols.TP$IncomingPacket.handleMyMessage(TP.java:1535) at org.jgroups.protocols.TP$IncomingPacket.run(TP.java:1479) at java.util.concurrent.ThreadPoolExecutor$Worker.runTask(ThreadPoolExecutor.java:650) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:675) at java.lang.Thread.run(Thread.java:595) -------------------------------------------------------------------------------------------------------------------------------------- [[ 8 OOB, 2 Incoming, 1 UDP mcast, 1 UDP ucast all waiting for the barrier to open. They are in the in_flight_threads set causing a deadlock with the top thread. ]] "OOB" prio=1 tid=0x000000004f05d100 nid=0x1de waiting on condition [0x000000004626f000..0x000000004626fd80] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:118) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1841) at org.jgroups.protocols.BARRIER.up(BARRIER.java:110) at org.jgroups.protocols.VERIFY_SUSPECT.up(VERIFY_SUSPECT.java:167) at org.jgroups.protocols.FD.up(FD.java:322) at org.jgroups.protocols.MERGE3.up(MERGE3.java:126) at org.jgroups.protocols.Discovery.up(Discovery.java:246) at org.jgroups.protocols.TP$IncomingPacket.handleMyMessage(TP.java:1535) at org.jgroups.protocols.TP$IncomingPacket.run(TP.java:1484) at java.util.concurrent.ThreadPoolExecutor$Worker.runTask(ThreadPoolExecutor.java:650) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:675) at java.lang.Thread.run(Thread.java:595) ----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------