Major Description
Issue occurs with many clients (200-1000) . Each client is assigned a new reply queue.
"ActiveMQ Transport: tcp:///XXX.XXX.XXX.XXX:19093@62021": waiting to lock monitor 0x00007fdfd001d3e8 (object 0x0000000405231968, a org.apache.activemq.broker.region.virtual.VirtualDestinationInterceptor), which is held by "ActiveMQ NIO Worker 7" "ActiveMQ NIO Worker 7" waiting for ownable synchronizer 0x0000000405d1fce0, (a java.util.concurrent.locks.ReentrantReadWriteLock$NonfairSync), which is held by "ActiveMQ NIO Worker 5" "ActiveMQ NIO Worker 5": waiting to lock monitor 0x00007fdfd001d3e8 (object 0x0000000405231968, a org.apache.activemq.broker.region.virtual.VirtualDestinationInterceptor), which is held by "ActiveMQ NIO Worker 7"
Here are the stack traces:
"ActiveMQ Transport: tcp:///XXX.XXX.XXX.XXX:19093@62021": at org.apache.activemq.broker.region.virtual.VirtualDestinationInterceptor.create(VirtualDestinationInterceptor.java:82) - waiting to lock <0x0000000405231968> (a org.apache.activemq.broker.region.virtual.VirtualDestinationInterceptor) at org.apache.activemq.broker.region.CompositeDestinationInterceptor.create(CompositeDestinationInterceptor.java:52) at org.apache.activemq.broker.region.RegionBroker.addConsumer(RegionBroker.java:423) at org.apache.activemq.broker.jmx.ManagedRegionBroker.addConsumer(ManagedRegionBroker.java:243) at org.apache.activemq.broker.BrokerFilter.addConsumer(BrokerFilter.java:103) at org.apache.activemq.advisory.AdvisoryBroker.addConsumer(AdvisoryBroker.java:130) at org.apache.activemq.broker.BrokerFilter.addConsumer(BrokerFilter.java:103) at org.apache.activemq.broker.BrokerFilter.addConsumer(BrokerFilter.java:103) at org.apache.activemq.broker.BrokerFilter.addConsumer(BrokerFilter.java:103) at org.apache.activemq.broker.MutableBrokerFilter.addConsumer(MutableBrokerFilter.java:108) at org.apache.activemq.broker.BrokerFilter.addConsumer(BrokerFilter.java:103) at org.apache.activemq.broker.BrokerFilter.addConsumer(BrokerFilter.java:103) at org.apache.activemq.security.AuthorizationBroker.addConsumer(AuthorizationBroker.java:183) at org.apache.activemq.broker.MutableBrokerFilter.addConsumer(MutableBrokerFilter.java:108) at org.apache.activemq.broker.TransportConnection.processAddConsumer(TransportConnection.java:696) at org.apache.activemq.command.ConsumerInfo.visit(ConsumerInfo.java:352) at org.apache.activemq.broker.TransportConnection.service(TransportConnection.java:326) at org.apache.activemq.broker.TransportConnection$1.onCommand(TransportConnection.java:190) at org.apache.activemq.transport.MutexTransport.onCommand(MutexTransport.java:50) at org.apache.activemq.transport.WireFormatNegotiator.onCommand(WireFormatNegotiator.java:125) at org.apache.activemq.transport.AbstractInactivityMonitor.onCommand(AbstractInactivityMonitor.java:301) at org.apache.activemq.transport.TransportSupport.doConsume(TransportSupport.java:83) at org.apache.activemq.transport.tcp.TcpTransport.doRun(TcpTransport.java:214) at org.apache.activemq.transport.tcp.TcpTransport.run(TcpTransport.java:196) at java.lang.Thread.run(Thread.java:748) "ActiveMQ NIO Worker 7": at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x0000000405d1fce0> (a java.util.concurrent.locks.ReentrantReadWriteLock$NonfairSync) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer.parkAndCheckInterrupt(AbstractQueuedSynchronizer.java:836) at java.util.concurrent.locks.AbstractQueuedSynchronizer.acquireQueued(AbstractQueuedSynchronizer.java:870) at java.util.concurrent.locks.AbstractQueuedSynchronizer.acquire(AbstractQueuedSynchronizer.java:1199) at java.util.concurrent.locks.ReentrantReadWriteLock$WriteLock.lock(ReentrantReadWriteLock.java:943) at org.apache.activemq.broker.region.AbstractRegion.addDestination(AbstractRegion.java:132) at org.apache.activemq.broker.region.RegionBroker.addDestination(RegionBroker.java:348) at org.apache.activemq.broker.region.virtual.VirtualTopic.create(VirtualTopic.java:100) at org.apache.activemq.broker.region.virtual.VirtualDestinationInterceptor.create(VirtualDestinationInterceptor.java:83) - locked <0x0000000405231968> (a org.apache.activemq.broker.region.virtual.VirtualDestinationInterceptor) at org.apache.activemq.broker.region.CompositeDestinationInterceptor.create(CompositeDestinationInterceptor.java:52) at org.apache.activemq.broker.region.RegionBroker.addConsumer(RegionBroker.java:423) at org.apache.activemq.broker.jmx.ManagedRegionBroker.addConsumer(ManagedRegionBroker.java:243) at org.apache.activemq.broker.BrokerFilter.addConsumer(BrokerFilter.java:103) at org.apache.activemq.advisory.AdvisoryBroker.addConsumer(AdvisoryBroker.java:130) at org.apache.activemq.broker.BrokerFilter.addConsumer(BrokerFilter.java:103) at org.apache.activemq.broker.BrokerFilter.addConsumer(BrokerFilter.java:103) at org.apache.activemq.broker.BrokerFilter.addConsumer(BrokerFilter.java:103) at org.apache.activemq.broker.MutableBrokerFilter.addConsumer(MutableBrokerFilter.java:108) at org.apache.activemq.broker.BrokerFilter.addConsumer(BrokerFilter.java:103) at org.apache.activemq.broker.BrokerFilter.addConsumer(BrokerFilter.java:103) at org.apache.activemq.security.AuthorizationBroker.addConsumer(AuthorizationBroker.java:183) at org.apache.activemq.broker.MutableBrokerFilter.addConsumer(MutableBrokerFilter.java:108) at org.apache.activemq.broker.TransportConnection.processAddConsumer(TransportConnection.java:696) at org.apache.activemq.command.ConsumerInfo.visit(ConsumerInfo.java:352) at org.apache.activemq.broker.TransportConnection.service(TransportConnection.java:326) at org.apache.activemq.broker.TransportConnection$1.onCommand(TransportConnection.java:190) at org.apache.activemq.transport.MutexTransport.onCommand(MutexTransport.java:45) at org.apache.activemq.transport.mqtt.MQTTInactivityMonitor.onCommand(MQTTInactivityMonitor.java:162) at org.apache.activemq.transport.mqtt.MQTTTransportFilter.sendToActiveMQ(MQTTTransportFilter.java:106) at org.apache.activemq.transport.mqtt.MQTTProtocolConverter.sendToActiveMQ(MQTTProtocolConverter.java:181) at org.apache.activemq.transport.mqtt.strategy.AbstractMQTTSubscriptionStrategy.doSubscribe(AbstractMQTTSubscriptionStrategy.java:210) at org.apache.activemq.transport.mqtt.strategy.MQTTVirtualTopicSubscriptionStrategy.onSubscribe(MQTTVirtualTopicSubscriptionStrategy.java:118) at org.apache.activemq.transport.mqtt.strategy.AbstractMQTTSubscriptionStrategy.onSubscribe(AbstractMQTTSubscriptionStrategy.java:118) at org.apache.activemq.transport.mqtt.MQTTProtocolConverter.onSubscribe(MQTTProtocolConverter.java:362) at org.apache.activemq.transport.mqtt.MQTTProtocolConverter.onMQTTCommand(MQTTProtocolConverter.java:212) at org.apache.activemq.transport.mqtt.MQTTTransportFilter.onCommand(MQTTTransportFilter.java:94) at org.apache.activemq.transport.TransportSupport.doConsume(TransportSupport.java:83) at org.apache.activemq.transport.nio.NIOSSLTransport.doConsume(NIOSSLTransport.java:440) at org.apache.activemq.transport.mqtt.MQTTCodec$1.onFrame(MQTTCodec.java:65) at org.apache.activemq.transport.mqtt.MQTTCodec.processCommand(MQTTCodec.java:90) at org.apache.activemq.transport.mqtt.MQTTCodec.access$400(MQTTCodec.java:26) at org.apache.activemq.transport.mqtt.MQTTCodec$4.parse(MQTTCodec.java:213) at org.apache.activemq.transport.mqtt.MQTTCodec$3.parse(MQTTCodec.java:179) at org.apache.activemq.transport.mqtt.MQTTCodec$2.parse(MQTTCodec.java:138) at org.apache.activemq.transport.mqtt.MQTTCodec.parse(MQTTCodec.java:76) at org.apache.activemq.transport.mqtt.MQTTNIOSSLTransport.processCommand(MQTTNIOSSLTransport.java:57) at org.apache.activemq.transport.nio.NIOSSLTransport.serviceRead(NIOSSLTransport.java:221) at org.apache.activemq.transport.nio.NIOSSLTransport$2.onSelect(NIOSSLTransport.java:176) at org.apache.activemq.transport.nio.SelectorSelection.onSelect(SelectorSelection.java:98) at org.apache.activemq.transport.nio.SelectorWorker$1.run(SelectorWorker.java:118) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) "ActiveMQ NIO Worker 5": at org.apache.activemq.broker.region.virtual.VirtualDestinationInterceptor.remove(VirtualDestinationInterceptor.java:89) - waiting to lock <0x0000000405231968> (a org.apache.activemq.broker.region.virtual.VirtualDestinationInterceptor) at org.apache.activemq.broker.region.CompositeDestinationInterceptor.remove(CompositeDestinationInterceptor.java:46) at org.apache.activemq.broker.region.AbstractRegion.removeDestination(AbstractRegion.java:230) at org.apache.activemq.broker.jmx.ManagedQueueRegion.removeDestination(ManagedQueueRegion.java:62) at org.apache.activemq.broker.region.RegionBroker.removeDestination(RegionBroker.java:363) at org.apache.activemq.broker.region.RegionBroker.removeDestinationInfo(RegionBroker.java:376) at org.apache.activemq.broker.BrokerFilter.removeDestinationInfo(BrokerFilter.java:268) at org.apache.activemq.advisory.AdvisoryBroker.removeDestinationInfo(AdvisoryBroker.java:322) at org.apache.activemq.broker.BrokerFilter.removeDestinationInfo(BrokerFilter.java:268) at org.apache.activemq.broker.BrokerFilter.removeDestinationInfo(BrokerFilter.java:268) at org.apache.activemq.broker.BrokerFilter.removeDestinationInfo(BrokerFilter.java:268) at org.apache.activemq.broker.MutableBrokerFilter.removeDestinationInfo(MutableBrokerFilter.java:274) at org.apache.activemq.broker.BrokerFilter.removeDestinationInfo(BrokerFilter.java:268) at org.apache.activemq.broker.BrokerFilter.removeDestinationInfo(BrokerFilter.java:268) at org.apache.activemq.security.AuthorizationBroker.removeDestinationInfo(AuthorizationBroker.java:144) at org.apache.activemq.broker.MutableBrokerFilter.removeDestinationInfo(MutableBrokerFilter.java:274) at org.apache.activemq.broker.TransportConnection.processRemoveDestination(TransportConnection.java:611) at org.apache.activemq.command.DestinationInfo.visit(DestinationInfo.java:124) at org.apache.activemq.broker.TransportConnection.service(TransportConnection.java:326) at org.apache.activemq.broker.TransportConnection$1.onCommand(TransportConnection.java:190) at org.apache.activemq.transport.MutexTransport.onCommand(MutexTransport.java:45) at org.apache.activemq.transport.mqtt.MQTTInactivityMonitor.onCommand(MQTTInactivityMonitor.java:162) at org.apache.activemq.transport.mqtt.MQTTTransportFilter.sendToActiveMQ(MQTTTransportFilter.java:106) at org.apache.activemq.transport.mqtt.MQTTProtocolConverter.sendToActiveMQ(MQTTProtocolConverter.java:181) at org.apache.activemq.transport.mqtt.strategy.MQTTVirtualTopicSubscriptionStrategy.onUnSubscribe(MQTTVirtualTopicSubscriptionStrategy.java:159) at org.apache.activemq.transport.mqtt.MQTTProtocolConverter.onUnSubscribe(MQTTProtocolConverter.java:389) at org.apache.activemq.transport.mqtt.MQTTProtocolConverter.onMQTTCommand(MQTTProtocolConverter.java:215) at org.apache.activemq.transport.mqtt.MQTTTransportFilter.onCommand(MQTTTransportFilter.java:94) at org.apache.activemq.transport.TransportSupport.doConsume(TransportSupport.java:83) at org.apache.activemq.transport.nio.NIOSSLTransport.doConsume(NIOSSLTransport.java:440) at org.apache.activemq.transport.mqtt.MQTTCodec$1.onFrame(MQTTCodec.java:65) at org.apache.activemq.transport.mqtt.MQTTCodec.processCommand(MQTTCodec.java:90) at org.apache.activemq.transport.mqtt.MQTTCodec.access$400(MQTTCodec.java:26) at org.apache.activemq.transport.mqtt.MQTTCodec$4.parse(MQTTCodec.java:213) at org.apache.activemq.transport.mqtt.MQTTCodec$3.parse(MQTTCodec.java:179) at org.apache.activemq.transport.mqtt.MQTTCodec$2.parse(MQTTCodec.java:138) at org.apache.activemq.transport.mqtt.MQTTCodec.parse(MQTTCodec.java:76) at org.apache.activemq.transport.mqtt.MQTTNIOSSLTransport.processCommand(MQTTNIOSSLTransport.java:57) at org.apache.activemq.transport.nio.NIOSSLTransport.serviceRead(NIOSSLTransport.java:221) at org.apache.activemq.transport.nio.NIOSSLTransport$2.onSelect(NIOSSLTransport.java:176) at org.apache.activemq.transport.nio.SelectorSelection.onSelect(SelectorSelection.java:98) at org.apache.activemq.transport.nio.SelectorWorker$1.run(SelectorWorker.java:118) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748)
