2017-05-23 04:55:17 Full thread dump Java HotSpot(TM) Server VM (25.07-b1 mixed mode): "Attach Listener" #43 daemon prio=9 os_prio=-179 tid=0x004fbc00 nid=58 lwp_id=6993333 waiting on condition [0x00000000] java.lang.Thread.State: RUNNABLE "Thread-4 (ActiveMQ-scheduled-threads-5225388)" #42 prio=5 os_prio=-179 tid=0x00ae8200 nid=57 lwp_id=6993143 waiting on condition [0x34740000] java.lang.Thread.State: WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x404a4348> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1088) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1067) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) at java.lang.Thread.run(Thread.java:745) "Thread-3 (ActiveMQ-scheduled-threads-5225388)" #41 prio=5 os_prio=-179 tid=0x02098400 nid=56 lwp_id=6993142 waiting on condition [0x34800000] java.lang.Thread.State: WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x404a4348> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1088) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1067) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) at java.lang.Thread.run(Thread.java:745) "Thread-2 (ActiveMQ-scheduled-threads-5225388)" #40 prio=5 os_prio=-179 tid=0x00aaf400 nid=55 lwp_id=6993140 waiting on condition [0x348c0000] java.lang.Thread.State: WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x404a4348> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1088) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1067) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) at java.lang.Thread.run(Thread.java:745) "Thread-1 (ActiveMQ-scheduled-threads-5225388)" #34 prio=5 os_prio=-179 tid=0x00aae600 nid=49 lwp_id=6993134 waiting on condition [0x34d40000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x404a4348> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1093) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1067) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) at java.lang.Thread.run(Thread.java:745) "Thread-4 (ActiveMQ-server-org.apache.activemq.artemis.core.server.impl.ActiveMQServerImpl$3@1b4de6a-27365147)" #33 prio=5 os_prio=-179 tid=0x02096800 nid=48 lwp_id=6993133 waiting on condition [0x34e00000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x40404508> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1066) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) at java.lang.Thread.run(Thread.java:745) "Thread-3 (ActiveMQ-server-org.apache.activemq.artemis.core.server.impl.ActiveMQServerImpl$3@1b4de6a-27365147)" #28 prio=5 os_prio=-179 tid=0x0205d600 nid=43 lwp_id=6993128 waiting on condition [0x35340000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x40404508> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1066) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) at java.lang.Thread.run(Thread.java:745) "Thread-0 (org.apache.activemq.artemis.core.remoting.impl.invm.InVMConnector-3817308)" #27 prio=5 os_prio=-179 tid=0x0083fc00 nid=42 lwp_id=6993127 waiting for monitor entry [0x35400000] java.lang.Thread.State: BLOCKED (on object monitor) at org.apache.activemq.artemis.core.replication.ReplicationEndpoint.registerJournal(ReplicationEndpoint.java:143) - waiting to lock <0x404a3ea8> (a org.apache.activemq.artemis.core.replication.ReplicationEndpoint) at org.apache.activemq.artemis.core.replication.ReplicationEndpoint.finishSynchronization(ReplicationEndpoint.java:342) at org.apache.activemq.artemis.core.replication.ReplicationEndpoint.handleStartReplicationSynchronization(ReplicationEndpoint.java:435) at org.apache.activemq.artemis.core.replication.ReplicationEndpoint.handlePacket(ReplicationEndpoint.java:195) at org.apache.activemq.artemis.tests.integration.cluster.util.BackupSyncDelay$ReplicationChannelHandler.handlePacket(BackupSyncDelay.java:183) - locked <0x404a3f18> (a org.apache.activemq.artemis.tests.integration.cluster.util.BackupSyncDelay$ReplicationChannelHandler) at org.apache.activemq.artemis.core.protocol.core.impl.ChannelImpl.handlePacket(ChannelImpl.java:623) at org.apache.activemq.artemis.core.protocol.core.impl.RemotingConnectionImpl.doBufferReceived(RemotingConnectionImpl.java:379) - locked <0x404a3f38> (a java.lang.Object) at org.apache.activemq.artemis.core.protocol.core.impl.RemotingConnectionImpl.bufferReceived(RemotingConnectionImpl.java:362) at org.apache.activemq.artemis.core.client.impl.ClientSessionFactoryImpl$DelegatingBufferHandler.bufferReceived(ClientSessionFactoryImpl.java:1143) at org.apache.activemq.artemis.core.remoting.impl.invm.InVMConnection$1.run(InVMConnection.java:196) at org.apache.activemq.artemis.utils.OrderedExecutorFactory$OrderedExecutor$ExecutorTask.run(OrderedExecutorFactory.java:101) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) at java.lang.Thread.run(Thread.java:745) "threadDeathWatcher-1-1" #25 daemon prio=1 os_prio=-179 tid=0x004ff400 nid=40 lwp_id=6993125 waiting on condition [0x35580000] java.lang.Thread.State: TIMED_WAITING (sleeping) at java.lang.Thread.sleep(Native Method) at io.netty.util.ThreadDeathWatcher$Watcher.run(ThreadDeathWatcher.java:150) at io.netty.util.concurrent.DefaultThreadFactory$DefaultRunnableDecorator.run(DefaultThreadFactory.java:144) at java.lang.Thread.run(Thread.java:745) "Thread-0 (ActiveMQ-scheduled-threads-5225388)" #24 prio=5 os_prio=-179 tid=0x00fb8000 nid=39 lwp_id=6993124 waiting on condition [0x35640000] java.lang.Thread.State: WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x404a4348> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1088) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1067) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) at java.lang.Thread.run(Thread.java:745) "activemq-buffer-timeout" #23 prio=5 os_prio=-179 tid=0x00fb6200 nid=38 lwp_id=6993123 waiting on condition [0x35700000] java.lang.Thread.State: WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x404a4988> (a java.util.concurrent.Semaphore$NonfairSync) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer.parkAndCheckInterrupt(AbstractQueuedSynchronizer.java:836) at java.util.concurrent.locks.AbstractQueuedSynchronizer.doAcquireSharedInterruptibly(AbstractQueuedSynchronizer.java:997) at java.util.concurrent.locks.AbstractQueuedSynchronizer.acquireSharedInterruptibly(AbstractQueuedSynchronizer.java:1304) at java.util.concurrent.Semaphore.acquire(Semaphore.java:312) at org.apache.activemq.artemis.core.io.buffer.TimedBuffer$CheckTimer.run(TimedBuffer.java:390) at java.lang.Thread.run(Thread.java:745) "AMQ119000: Activation for server ActiveMQServerImpl::ReplicatedLargeMessageWithDelayFailoverTest/backupServers" #21 prio=5 os_prio=-179 tid=0x00f3bc00 nid=36 lwp_id=6993121 waiting on condition [0x35880000] java.lang.Thread.State: WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x404ca8b0> (a java.util.concurrent.locks.ReentrantReadWriteLock$NonfairSync) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer.parkAndCheckInterrupt(AbstractQueuedSynchronizer.java:836) at java.util.concurrent.locks.AbstractQueuedSynchronizer.acquireQueued(AbstractQueuedSynchronizer.java:870) at java.util.concurrent.locks.AbstractQueuedSynchronizer.acquire(AbstractQueuedSynchronizer.java:1199) at java.util.concurrent.locks.ReentrantReadWriteLock$WriteLock.lock(ReentrantReadWriteLock.java:943) at org.apache.activemq.artemis.core.journal.impl.JournalImpl.stop(JournalImpl.java:2332) - locked <0x404ca060> (a org.apache.activemq.artemis.core.journal.impl.JournalImpl) at org.apache.activemq.artemis.core.persistence.impl.journal.JournalStorageManager.stop(JournalStorageManager.java:244) - locked <0x404c9c90> (a org.apache.activemq.artemis.core.persistence.impl.journal.JournalStorageManager) at org.apache.activemq.artemis.core.persistence.impl.journal.JournalStorageManager.stop(JournalStorageManager.java:181) at org.apache.activemq.artemis.core.replication.ReplicationEndpoint.stop(ReplicationEndpoint.java:319) - locked <0x404a3ea8> (a org.apache.activemq.artemis.core.replication.ReplicationEndpoint) at org.apache.activemq.artemis.core.server.impl.ActiveMQServerImpl.stopComponent(ActiveMQServerImpl.java:1110) at org.apache.activemq.artemis.core.server.impl.SharedNothingBackupActivation.run(SharedNothingBackupActivation.java:252) at org.apache.activemq.artemis.core.server.impl.ActiveMQServerImpl$ActivationThread.run(ActiveMQServerImpl.java:2491) "Thread-4 (ActiveMQ-scheduled-threads-15512410)" #20 prio=5 os_prio=-179 tid=0x00e85c00 nid=35 lwp_id=6993120 waiting on condition [0x35940000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x404c1d18> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1093) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1067) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) at java.lang.Thread.run(Thread.java:745) "Thread-3 (ActiveMQ-scheduled-threads-15512410)" #19 prio=5 os_prio=-179 tid=0x00d27a00 nid=34 lwp_id=6993119 waiting on condition [0x35a00000] java.lang.Thread.State: WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x404c1d18> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1088) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1067) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) at java.lang.Thread.run(Thread.java:745) "Thread-2 (ActiveMQ-server-org.apache.activemq.artemis.core.server.impl.ActiveMQServerImpl$3@1b4de6a-27365147)" #18 prio=5 os_prio=-179 tid=0x00d23a00 nid=33 lwp_id=6993118 waiting on condition [0x35ac0000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x40404508> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1066) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) at java.lang.Thread.run(Thread.java:745) "Thread-2 (ActiveMQ-scheduled-threads-15512410)" #17 prio=5 os_prio=-179 tid=0x00d20800 nid=32 lwp_id=6993117 waiting on condition [0x35b80000] java.lang.Thread.State: WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x404c1d18> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1088) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1067) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) at java.lang.Thread.run(Thread.java:745) "activemq-failure-check-thread" #16 prio=5 os_prio=-179 tid=0x00a8f800 nid=31 lwp_id=6993116 waiting on condition [0x35c40000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x40563ba8> (a java.util.concurrent.CountDownLatch$Sync) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer.doAcquireSharedNanos(AbstractQueuedSynchronizer.java:1037) at java.util.concurrent.locks.AbstractQueuedSynchronizer.tryAcquireSharedNanos(AbstractQueuedSynchronizer.java:1328) at java.util.concurrent.CountDownLatch.await(CountDownLatch.java:277) at org.apache.activemq.artemis.core.remoting.server.impl.RemotingServiceImpl$FailureCheckAndFlushThread.run(RemotingServiceImpl.java:716) "Thread-1 (ActiveMQ-server-org.apache.activemq.artemis.core.server.impl.ActiveMQServerImpl$3@1b4de6a-27365147)" #15 prio=5 os_prio=-179 tid=0x00a8d600 nid=30 lwp_id=6993115 waiting on condition [0x35d00000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x40567b58> (a java.util.concurrent.CountDownLatch$Sync) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer.doAcquireSharedNanos(AbstractQueuedSynchronizer.java:1037) at java.util.concurrent.locks.AbstractQueuedSynchronizer.tryAcquireSharedNanos(AbstractQueuedSynchronizer.java:1328) at java.util.concurrent.CountDownLatch.await(CountDownLatch.java:277) at org.apache.activemq.artemis.core.client.impl.ServerLocatorImpl$StaticConnector.connect(ServerLocatorImpl.java:1684) at org.apache.activemq.artemis.core.client.impl.ServerLocatorImpl.connect(ServerLocatorImpl.java:618) - locked <0x40567be8> (a org.apache.activemq.artemis.core.client.impl.ServerLocatorImpl) at org.apache.activemq.artemis.core.client.impl.ServerLocatorImpl.connect(ServerLocatorImpl.java:609) at org.apache.activemq.artemis.core.client.impl.ServerLocatorImpl$4.run(ServerLocatorImpl.java:570) at org.apache.activemq.artemis.utils.OrderedExecutorFactory$OrderedExecutor$ExecutorTask.run(OrderedExecutorFactory.java:101) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) at java.lang.Thread.run(Thread.java:745) "Thread-1 (ActiveMQ-scheduled-threads-15512410)" #14 prio=5 os_prio=-179 tid=0x00a11c00 nid=29 lwp_id=6993114 waiting on condition [0x35dc0000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x404c1d18> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1093) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1067) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) at java.lang.Thread.run(Thread.java:745) "Thread-0 (ActiveMQ-scheduled-threads-15512410)" #13 prio=5 os_prio=-179 tid=0x01afbe00 nid=28 lwp_id=6993113 waiting on condition [0x36000000] java.lang.Thread.State: WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x404c1d18> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1088) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1067) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) at java.lang.Thread.run(Thread.java:745) "activemq-buffer-timeout" #12 prio=5 os_prio=-179 tid=0x01af0600 nid=27 lwp_id=6993112 waiting on condition [0x360c0000] java.lang.Thread.State: WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x4056f7d8> (a java.util.concurrent.Semaphore$NonfairSync) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer.parkAndCheckInterrupt(AbstractQueuedSynchronizer.java:836) at java.util.concurrent.locks.AbstractQueuedSynchronizer.doAcquireSharedInterruptibly(AbstractQueuedSynchronizer.java:997) at java.util.concurrent.locks.AbstractQueuedSynchronizer.acquireSharedInterruptibly(AbstractQueuedSynchronizer.java:1304) at java.util.concurrent.Semaphore.acquire(Semaphore.java:312) at org.apache.activemq.artemis.core.io.buffer.TimedBuffer$CheckTimer.run(TimedBuffer.java:390) at java.lang.Thread.run(Thread.java:745) "Thread-0 (ActiveMQ-server-org.apache.activemq.artemis.core.server.impl.ActiveMQServerImpl$3@1b4de6a-27365147)" #11 prio=5 os_prio=-179 tid=0x01a3b200 nid=26 lwp_id=6993111 waiting on condition [0x36d00000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x40573650> (a java.util.concurrent.CountDownLatch$Sync) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer.doAcquireSharedNanos(AbstractQueuedSynchronizer.java:1037) at java.util.concurrent.locks.AbstractQueuedSynchronizer.tryAcquireSharedNanos(AbstractQueuedSynchronizer.java:1328) at java.util.concurrent.CountDownLatch.await(CountDownLatch.java:277) at org.apache.activemq.artemis.core.client.impl.ServerLocatorImpl$StaticConnector.connect(ServerLocatorImpl.java:1684) at org.apache.activemq.artemis.core.client.impl.ServerLocatorImpl.connect(ServerLocatorImpl.java:618) - locked <0x405736e0> (a org.apache.activemq.artemis.core.client.impl.ServerLocatorImpl) at org.apache.activemq.artemis.core.client.impl.ServerLocatorImpl.connect(ServerLocatorImpl.java:609) at org.apache.activemq.artemis.core.server.cluster.ClusterController$ConnectRunnable.run(ClusterController.java:401) at org.apache.activemq.artemis.utils.OrderedExecutorFactory$OrderedExecutor$ExecutorTask.run(OrderedExecutorFactory.java:101) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) at java.lang.Thread.run(Thread.java:745) "Service Thread" #7 daemon prio=9 os_prio=-179 tid=0x009fe400 nid=23 lwp_id=6993104 runnable [0x00000000] java.lang.Thread.State: RUNNABLE "C2 CompilerThread1" #6 daemon prio=9 os_prio=-179 tid=0x009f7a00 nid=21 lwp_id=6993102 waiting on condition [0x00000000] java.lang.Thread.State: RUNNABLE "C2 CompilerThread0" #5 daemon prio=9 os_prio=-179 tid=0x009f4600 nid=20 lwp_id=6993101 waiting on condition [0x00000000] java.lang.Thread.State: RUNNABLE "Signal Dispatcher" #4 daemon prio=9 os_prio=-179 tid=0x009f2600 nid=19 lwp_id=6993100 runnable [0x00000000] java.lang.Thread.State: RUNNABLE "Finalizer" #3 daemon prio=8 os_prio=-179 tid=0x009e1000 nid=18 lwp_id=6993099 in Object.wait() [0x395c0000] java.lang.Thread.State: WAITING (on object monitor) at java.lang.Object.wait(Native Method) - waiting on <0x40577530> (a java.lang.ref.ReferenceQueue$Lock) at java.lang.ref.ReferenceQueue.remove(ReferenceQueue.java:160) - locked <0x40577530> (a java.lang.ref.ReferenceQueue$Lock) at java.lang.ref.ReferenceQueue.remove(ReferenceQueue.java:181) at java.lang.ref.Finalizer$FinalizerThread.run(Finalizer.java:249) "Reference Handler" #2 daemon prio=10 os_prio=-179 tid=0x009cfc00 nid=17 lwp_id=6993098 in Object.wait() [0x39680000] java.lang.Thread.State: WAITING (on object monitor) at java.lang.Object.wait(Native Method) - waiting on <0x4056c150> (a java.lang.ref.Reference$Lock) at java.lang.Object.wait(Object.java:502) at java.lang.ref.Reference.tryHandlePending(Reference.java:228) - locked <0x4056c150> (a java.lang.ref.Reference$Lock) at java.lang.ref.Reference$ReferenceHandler.run(Reference.java:160) "main" #1 prio=5 os_prio=-179 tid=0x0045e400 nid=1 lwp_id=-1 waiting for monitor entry [0x7fffc000] java.lang.Thread.State: BLOCKED (on object monitor) at org.apache.activemq.artemis.core.replication.ReplicationEndpoint.stop(ReplicationEndpoint.java:267) - waiting to lock <0x404a3ea8> (a org.apache.activemq.artemis.core.replication.ReplicationEndpoint) at org.apache.activemq.artemis.core.server.impl.SharedNothingBackupActivation.preStorageClose(SharedNothingBackupActivation.java:376) at org.apache.activemq.artemis.core.server.impl.ActiveMQServerImpl.stop(ActiveMQServerImpl.java:903) at org.apache.activemq.artemis.core.server.impl.ActiveMQServerImpl.stop(ActiveMQServerImpl.java:788) at org.apache.activemq.artemis.core.server.impl.ActiveMQServerImpl.internalStop(ActiveMQServerImpl.java:683) at org.apache.activemq.artemis.core.server.impl.ActiveMQServerImpl.stop(ActiveMQServerImpl.java:676) at org.apache.activemq.artemis.tests.util.ActiveMQTestBase.stopComponentOutputExceptions(ActiveMQTestBase.java:2205) at org.apache.activemq.artemis.tests.util.ActiveMQTestBase.tearDown(ActiveMQTestBase.java:278) - locked <0x40757180> (a java.util.ArrayList) at org.apache.activemq.artemis.tests.integration.cluster.failover.FailoverTestBase.tearDown(FailoverTestBase.java:220) at org.apache.activemq.artemis.tests.integration.cluster.failover.ReplicatedLargeMessageWithDelayFailoverTest.tearDown(ReplicatedLargeMessageWithDelayFailoverTest.java:62) at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.lang.reflect.Method.invoke(Method.java:498) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:47) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:44) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.rules.ExternalResource$1.evaluate(ExternalResource.java:48) at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:55) at org.junit.rules.ExternalResource$1.evaluate(ExternalResource.java:48) at org.junit.rules.ExternalResource$1.evaluate(ExternalResource.java:48) at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:55) at org.junit.rules.RunRules.evaluate(RunRules.java:20) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:271) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:70) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:50) at org.junit.runners.ParentRunner$3.run(ParentRunner.java:238) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:63) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:236) at org.junit.runners.ParentRunner.access$000(ParentRunner.java:53) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:229) at org.junit.runners.ParentRunner.run(ParentRunner.java:309) at org.apache.maven.surefire.junit4.JUnit4Provider.execute(JUnit4Provider.java:283) at org.apache.maven.surefire.junit4.JUnit4Provider.executeWithRerun(JUnit4Provider.java:173) at org.apache.maven.surefire.junit4.JUnit4Provider.executeTestSet(JUnit4Provider.java:153) at org.apache.maven.surefire.junit4.JUnit4Provider.invoke(JUnit4Provider.java:128) at org.apache.maven.surefire.booter.ForkedBooter.invokeProviderInSameClassLoader(ForkedBooter.java:203) at org.apache.maven.surefire.booter.ForkedBooter.runSuitesInProcess(ForkedBooter.java:155) at org.apache.maven.surefire.booter.ForkedBooter.main(ForkedBooter.java:103) "VM Thread" os_prio=-179 tid=0x0074b000 nid=16 lwp_id=6993097 runnable "GC task thread#0 (ParallelGC)" os_prio=-179 tid=0x00486600 nid=3 lwp_id=6993084 runnable "GC task thread#1 (ParallelGC)" os_prio=-179 tid=0x004af000 nid=4 lwp_id=6993085 runnable "GC task thread#2 (ParallelGC)" os_prio=-179 tid=0x004b0e00 nid=5 lwp_id=6993086 runnable "GC task thread#3 (ParallelGC)" os_prio=-179 tid=0x004b2e00 nid=6 lwp_id=6993087 runnable "GC task thread#4 (ParallelGC)" os_prio=-179 tid=0x004b4e00 nid=7 lwp_id=6993088 runnable "GC task thread#5 (ParallelGC)" os_prio=-179 tid=0x004b6c00 nid=8 lwp_id=6993089 runnable "GC task thread#6 (ParallelGC)" os_prio=-179 tid=0x004b8c00 nid=9 lwp_id=6993090 runnable "GC task thread#7 (ParallelGC)" os_prio=-179 tid=0x004bac00 nid=10 lwp_id=6993091 runnable "GC task thread#8 (ParallelGC)" os_prio=-179 tid=0x004bca00 nid=11 lwp_id=6993092 runnable "GC task thread#9 (ParallelGC)" os_prio=-179 tid=0x004bea00 nid=12 lwp_id=6993093 runnable "GC task thread#10 (ParallelGC)" os_prio=-179 tid=0x004c3a00 nid=13 lwp_id=6993094 runnable "GC task thread#11 (ParallelGC)" os_prio=-179 tid=0x004c5a00 nid=14 lwp_id=6993095 runnable "GC task thread#12 (ParallelGC)" os_prio=-179 tid=0x004c7a00 nid=15 lwp_id=6993096 runnable "VM Periodic Task Thread" os_prio=-179 tid=0x009fc600 nid=22 lwp_id=6993103 waiting on condition JNI global references: 222 Found one Java-level deadlock: ============================= "Thread-0 (org.apache.activemq.artemis.core.remoting.impl.invm.InVMConnector-3817308)": waiting to lock monitor 0x00877690 (object 0x404a3ea8, a org.apache.activemq.artemis.core.replication.ReplicationEndpoint), which is held by "AMQ119000: Activation for server ActiveMQServerImpl::ReplicatedLargeMessageWithDelayFailoverTest/backupServers" "AMQ119000: Activation for server ActiveMQServerImpl::ReplicatedLargeMessageWithDelayFailoverTest/backupServers": waiting for ownable synchronizer 0x404ca8b0, (a java.util.concurrent.locks.ReentrantReadWriteLock$NonfairSync), which is held by "Thread-0 (org.apache.activemq.artemis.core.remoting.impl.invm.InVMConnector-3817308)" Java stack information for the threads listed above: =================================================== "Thread-0 (org.apache.activemq.artemis.core.remoting.impl.invm.InVMConnector-3817308)": at org.apache.activemq.artemis.core.replication.ReplicationEndpoint.registerJournal(ReplicationEndpoint.java:143) - waiting to lock <0x404a3ea8> (a org.apache.activemq.artemis.core.replication.ReplicationEndpoint) at org.apache.activemq.artemis.core.replication.ReplicationEndpoint.finishSynchronization(ReplicationEndpoint.java:342) at org.apache.activemq.artemis.core.replication.ReplicationEndpoint.handleStartReplicationSynchronization(ReplicationEndpoint.java:435) at org.apache.activemq.artemis.core.replication.ReplicationEndpoint.handlePacket(ReplicationEndpoint.java:195) at org.apache.activemq.artemis.tests.integration.cluster.util.BackupSyncDelay$ReplicationChannelHandler.handlePacket(BackupSyncDelay.java:183) - locked <0x404a3f18> (a org.apache.activemq.artemis.tests.integration.cluster.util.BackupSyncDelay$ReplicationChannelHandler) at org.apache.activemq.artemis.core.protocol.core.impl.ChannelImpl.handlePacket(ChannelImpl.java:623) at org.apache.activemq.artemis.core.protocol.core.impl.RemotingConnectionImpl.doBufferReceived(RemotingConnectionImpl.java:379) - locked <0x404a3f38> (a java.lang.Object) at org.apache.activemq.artemis.core.protocol.core.impl.RemotingConnectionImpl.bufferReceived(RemotingConnectionImpl.java:362) at org.apache.activemq.artemis.core.client.impl.ClientSessionFactoryImpl$DelegatingBufferHandler.bufferReceived(ClientSessionFactoryImpl.java:1143) at org.apache.activemq.artemis.core.remoting.impl.invm.InVMConnection$1.run(InVMConnection.java:196) at org.apache.activemq.artemis.utils.OrderedExecutorFactory$OrderedExecutor$ExecutorTask.run(OrderedExecutorFactory.java:101) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) at java.lang.Thread.run(Thread.java:745) "AMQ119000: Activation for server ActiveMQServerImpl::ReplicatedLargeMessageWithDelayFailoverTest/backupServers": at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x404ca8b0> (a java.util.concurrent.locks.ReentrantReadWriteLock$NonfairSync) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer.parkAndCheckInterrupt(AbstractQueuedSynchronizer.java:836) at java.util.concurrent.locks.AbstractQueuedSynchronizer.acquireQueued(AbstractQueuedSynchronizer.java:870) at java.util.concurrent.locks.AbstractQueuedSynchronizer.acquire(AbstractQueuedSynchronizer.java:1199) at java.util.concurrent.locks.ReentrantReadWriteLock$WriteLock.lock(ReentrantReadWriteLock.java:943) at org.apache.activemq.artemis.core.journal.impl.JournalImpl.stop(JournalImpl.java:2332) - locked <0x404ca060> (a org.apache.activemq.artemis.core.journal.impl.JournalImpl) at org.apache.activemq.artemis.core.persistence.impl.journal.JournalStorageManager.stop(JournalStorageManager.java:244) - locked <0x404c9c90> (a org.apache.activemq.artemis.core.persistence.impl.journal.JournalStorageManager) at org.apache.activemq.artemis.core.persistence.impl.journal.JournalStorageManager.stop(JournalStorageManager.java:181) at org.apache.activemq.artemis.core.replication.ReplicationEndpoint.stop(ReplicationEndpoint.java:319) - locked <0x404a3ea8> (a org.apache.activemq.artemis.core.replication.ReplicationEndpoint) at org.apache.activemq.artemis.core.server.impl.ActiveMQServerImpl.stopComponent(ActiveMQServerImpl.java:1110) at org.apache.activemq.artemis.core.server.impl.SharedNothingBackupActivation.run(SharedNothingBackupActivation.java:252) at org.apache.activemq.artemis.core.server.impl.ActiveMQServerImpl$ActivationThread.run(ActiveMQServerImpl.java:2491) Found 1 deadlock.