$ echo $AAC_CP /var/dtests/node_data/clients/java/aac/src/..:/usr/share/java/log4j-1.2.14.jar:/usr/share/java/slf4j-api-amq-1.7.12/slf4j-api-1.7.12.redhat-1.jar:/var/dtests/node_data/clients/java/aac/src/../lib/slf4j-log4j12-1.7.12.jar:/var/dtests/node_data/clients/java/aac/src/../lib/jopt-simple-4.8.jar:/usr/share/java/geronimo-jms_1.1_spec-amq-1.1.1/geronimo-jms_1.1_spec-1.1.1.redhat-1.jar:/usr/share/java/qpid-jms-0.3.0/netty-all-4.0.17.Final-redhat-1.jar:/usr/share/java/qpid-proton-java-0.9.1/proton-j-0.9.1.redhat-1.jar:/root/qpid-jms/qpid-jms-selector/target/qpid-jms-selector-0.1-SNAPSHOT.jar:/root/qpid-jms/qpid-jms-discovery/target/qpid-jms-discovery-0.5.0-SNAPSHOT.jar:/root/qpid-jms-client-0.5.0-SNAPSHOT.jar:/var/dtests/node_data/clients/java/aac/resources/ $ java -cp $AAC_CP com.redhat.mqe.jms.aac1_sender --broker "failover:(amqp://localhost:5672)" --count 10 --log-msgs dict -a test_reconnect_tx_rx --conn-username guest --conn-password guest 10:23:22,939 TRACE Using port 0 10:23:23,084 DEBUG Initiating initial connection attempt task 10:23:23,089 DEBUG Connection attempt:[1] to: amqp://localhost:5672 in-progress 10:23:23,229 TRACE Task create -> JmsConnectionInfo {sendTimeout = -1, requestTimeout = -1, connectTimeout = 15000, closeTimeout = 15000} held until connection recovered: 10:23:23,395 DEBUG Using SLF4J as the default logging framework 10:23:23,395 DEBUG Using SLF4J as the default logging framework 10:23:23,397 DEBUG -Dio.netty.eventLoopThreads: 4 10:23:23,397 DEBUG -Dio.netty.eventLoopThreads: 4 10:23:23,422 DEBUG UID: 0 10:23:23,422 DEBUG UID: 0 10:23:23,423 DEBUG Java version: 7 10:23:23,423 DEBUG Java version: 7 10:23:23,423 DEBUG -Dio.netty.noUnsafe: false 10:23:23,423 DEBUG -Dio.netty.noUnsafe: false 10:23:23,428 DEBUG java.nio.ByteBuffer.cleaner: available 10:23:23,428 DEBUG java.nio.ByteBuffer.cleaner: available 10:23:23,429 DEBUG java.nio.Buffer.address: available 10:23:23,429 DEBUG java.nio.Buffer.address: available 10:23:23,429 DEBUG sun.misc.Unsafe.theUnsafe: available 10:23:23,429 DEBUG sun.misc.Unsafe.theUnsafe: available 10:23:23,430 DEBUG sun.misc.Unsafe.copyMemory: available 10:23:23,430 DEBUG sun.misc.Unsafe.copyMemory: available 10:23:23,430 DEBUG java.nio.Bits.unaligned: true 10:23:23,430 DEBUG java.nio.Bits.unaligned: true 10:23:23,431 DEBUG sun.misc.Unsafe: available 10:23:23,431 DEBUG sun.misc.Unsafe: available 10:23:23,431 DEBUG -Dio.netty.noJavassist: false 10:23:23,431 DEBUG -Dio.netty.noJavassist: false 10:23:23,439 DEBUG Javassist: unavailable 10:23:23,439 DEBUG Javassist: unavailable 10:23:23,439 DEBUG You don't have Javassist in your class path or you don't have enough permission to load dynamically generated classes. Please check the configuration for better performance. 10:23:23,439 DEBUG You don't have Javassist in your class path or you don't have enough permission to load dynamically generated classes. Please check the configuration for better performance. 10:23:23,439 DEBUG -Dio.netty.tmpdir: /tmp (java.io.tmpdir) 10:23:23,439 DEBUG -Dio.netty.tmpdir: /tmp (java.io.tmpdir) 10:23:23,439 DEBUG -Dio.netty.bitMode: 64 (sun.arch.data.model) 10:23:23,439 DEBUG -Dio.netty.bitMode: 64 (sun.arch.data.model) 10:23:23,440 DEBUG -Dio.netty.noPreferDirect: false 10:23:23,440 DEBUG -Dio.netty.noPreferDirect: false 10:23:23,482 DEBUG -Dio.netty.noKeySetOptimization: false 10:23:23,482 DEBUG -Dio.netty.noKeySetOptimization: false 10:23:23,483 DEBUG -Dio.netty.selectorAutoRebuildThreshold: 512 10:23:23,483 DEBUG -Dio.netty.selectorAutoRebuildThreshold: 512 10:23:23,490 TRACE Instrumented an optimized java.util.Set into: sun.nio.ch.EPollSelectorImpl@270e838f 10:23:23,490 TRACE Instrumented an optimized java.util.Set into: sun.nio.ch.EPollSelectorImpl@270e838f 10:23:23,537 DEBUG -Dio.netty.allocator.numHeapArenas: 2 10:23:23,537 DEBUG -Dio.netty.allocator.numHeapArenas: 2 10:23:23,537 DEBUG -Dio.netty.allocator.numDirectArenas: 2 10:23:23,537 DEBUG -Dio.netty.allocator.numDirectArenas: 2 10:23:23,538 DEBUG -Dio.netty.allocator.pageSize: 8192 10:23:23,538 DEBUG -Dio.netty.allocator.pageSize: 8192 10:23:23,538 DEBUG -Dio.netty.allocator.maxOrder: 11 10:23:23,538 DEBUG -Dio.netty.allocator.maxOrder: 11 10:23:23,538 DEBUG -Dio.netty.allocator.chunkSize: 16777216 10:23:23,538 DEBUG -Dio.netty.allocator.chunkSize: 16777216 10:23:23,588 DEBUG -Dio.netty.initialSeedUniquifier: 0x0f1a8a55996703cc 10:23:23,588 DEBUG -Dio.netty.initialSeedUniquifier: 0x0f1a8a55996703cc 10:23:23,600 DEBUG -Dio.netty.threadLocalDirectBufferSize: 65536 10:23:23,600 DEBUG -Dio.netty.threadLocalDirectBufferSize: 65536 10:23:23,635 DEBUG -Dio.netty.allocator.type: unpooled 10:23:23,635 DEBUG -Dio.netty.allocator.type: unpooled 10:23:23,733 TRACE Channel has become active! Channel is [id: 0x064f080e, /127.0.0.1:33816 => localhost/127.0.0.1:5672] 10:23:23,734 DEBUG Executing Failover Task: create -> JmsConnectionInfo {sendTimeout = -1, requestTimeout = -1, connectTimeout = 15000, closeTimeout = 15000} 10:23:23,806 DEBUG -Dio.netty.leakDetectionLevel: simple 10:23:23,806 DEBUG -Dio.netty.leakDetectionLevel: simple 10:23:23,807 TRACE Attempted write of: 8 bytes 10:23:23,871 TRACE New data read: 68 bytes incoming: UnpooledHeapByteBuf(ridx: 0, widx: 68, cap: 65536) 10:23:23,875 TRACE New Proton Event: CONNECTION_INIT 10:23:23,876 TRACE New Proton Event: SESSION_INIT 10:23:23,876 TRACE New Proton Event: CONNECTION_LOCAL_OPEN 10:23:23,886 DEBUG Unknown SASL mechanism: [DIGEST-MD5] 10:23:23,887 INFO Best match for SASL auth was: SASL-CRAM-MD5 10:23:23,890 TRACE Attempted write of: 36 bytes 10:23:23,891 TRACE New data read: 77 bytes incoming: UnpooledHeapByteBuf(ridx: 0, widx: 77, cap: 65536) 10:23:23,976 TRACE Attempted write of: 54 bytes 10:23:23,979 TRACE New data read: 24 bytes incoming: UnpooledHeapByteBuf(ridx: 0, widx: 24, cap: 65536) 10:23:23,994 TRACE Attempted write of: 299 bytes 10:23:23,995 TRACE New data read: 172 bytes incoming: UnpooledHeapByteBuf(ridx: 0, widx: 172, cap: 65536) 10:23:23,997 TRACE New Proton Event: CONNECTION_REMOTE_OPEN 10:23:23,998 TRACE New Proton Event: SESSION_LOCAL_OPEN 10:23:24,003 TRACE Attempted write of: 47 bytes 10:23:24,004 TRACE New data read: 32 bytes incoming: UnpooledHeapByteBuf(ridx: 0, widx: 32, cap: 65536) 10:23:24,004 TRACE New Proton Event: SESSION_REMOTE_OPEN 10:23:24,005 DEBUG AmqpSession { ID:dhcp-75-219.lab.eng.brq.redhat.com-53842-1440491002956-0:1:-1 } is now open: 10:23:24,005 DEBUG AmqpConnection { ID:dhcp-75-219.lab.eng.brq.redhat.com-53842-1440491002956-0:1 } is now open: 10:23:24,005 TRACE IdleTimeoutCheck being initiated, initial delay: 15000 10:23:24,007 TRACE First connection requst has completed: 10:23:24,007 INFO Connection ID:dhcp-75-219.lab.eng.brq.redhat.com-53842-1440491002956-0:1 connected to remote Broker: amqp://localhost:5672 10:23:24,031 DEBUG Executing Failover Task: create -> JmsSessionInfo {} 10:23:24,032 TRACE Attempted write of: 47 bytes 10:23:24,033 TRACE New data read: 32 bytes incoming: UnpooledHeapByteBuf(ridx: 0, widx: 32, cap: 65536) 10:23:24,033 TRACE New Proton Event: SESSION_INIT 10:23:24,033 TRACE New Proton Event: SESSION_LOCAL_OPEN 10:23:24,034 TRACE New Proton Event: SESSION_REMOTE_OPEN 10:23:24,034 DEBUG AmqpSession { ID:dhcp-75-219.lab.eng.brq.redhat.com-53842-1440491002956-0:1:1 } is now open: 10:23:24,049 DEBUG Executing Failover Task: create -> JmsProducerInfo {producerId = ID:dhcp-75-219.lab.eng.brq.redhat.com-53842-1440491002956-0:1:1:1, destination = test_reconnect_tx_rx} 10:23:24,050 DEBUG Creating AmqpFixedProducer for: test_reconnect_tx_rx 10:23:24,066 TRACE Attempted write of: 323 bytes 10:23:24,068 TRACE New data read: 250 bytes incoming: UnpooledHeapByteBuf(ridx: 0, widx: 250, cap: 65536) 10:23:24,069 TRACE New Proton Event: LINK_INIT 10:23:24,069 TRACE New Proton Event: LINK_LOCAL_OPEN 10:23:24,069 TRACE New Proton Event: LINK_REMOTE_OPEN 10:23:24,069 DEBUG AmqpFixedProducer { ID:dhcp-75-219.lab.eng.brq.redhat.com-53842-1440491002956-0:1:1:1 } is now open: 10:23:24,069 TRACE New Proton Event: LINK_FLOW 10:23:24,090 DEBUG Executing Failover Task: send -> JmsOutboundMessageDispatch {dispatchId = ID:dhcp-75-219.lab.eng.brq.redhat.com-53842-1440491002956-0:1:1:1-1} 10:23:24,091 TRACE Producer sending message: JmsOutboundMessageDispatch {dispatchId = ID:dhcp-75-219.lab.eng.brq.redhat.com-53842-1440491002956-0:1:1:1-1} 10:23:24,100 TRACE Attempted write of: 183 bytes 10:23:24,114 TRACE New data read: 42 bytes incoming: UnpooledHeapByteBuf(ridx: 0, widx: 42, cap: 65536) 10:23:24,114 TRACE New Proton Event: LINK_FLOW 10:23:24,879 TRACE New data read: 38 bytes incoming: UnpooledHeapByteBuf(ridx: 0, widx: 38, cap: 65536) 10:23:24,880 TRACE New Proton Event: DELIVERY 10:23:24,881 TRACE Outcome of delivery was accepted: DeliveryImpl [_tag=[48], _link=org.apache.qpid.proton.engine.impl.SenderImpl@4dd629db, _deliveryState=null, _settled=false, _remoteSettled=true, _remoteDeliveryState=Accepted{}, _flags=0, _transportDelivery=org.apache.qpid.proton.engine.impl.TransportDelivery@275610c9, _dataSize=0, _complete=true, _updated=true, _done=true, _offset=0] {'redelivered': False, 'reply_to': None, 'id': 'dhcp-75-219.lab.eng.brq.redhat.com-53842-1440491002956-0:1:1:1-1', 'user_id':None, 'correlation_id': None, 'priority': 4, 'durable': True, 'ttl': 0, 'type': None, 'expiration': 0, 'timestamp': 1440491004085, 'properties': {'JMSXDeliveryCount': 1}, 'content': None} 10:23:24,901 DEBUG Executing Failover Task: send -> JmsOutboundMessageDispatch {dispatchId = ID:dhcp-75-219.lab.eng.brq.redhat.com-53842-1440491002956-0:1:1:1-2} 10:23:24,901 TRACE Producer sending message: JmsOutboundMessageDispatch {dispatchId = ID:dhcp-75-219.lab.eng.brq.redhat.com-53842-1440491002956-0:1:1:1-2} 10:23:24,902 TRACE Attempted write of: 184 bytes 10:23:24,903 TRACE New data read: 42 bytes incoming: UnpooledHeapByteBuf(ridx: 0, widx: 42, cap: 65536) 10:23:24,904 TRACE New Proton Event: LINK_FLOW 10:23:25,449 TRACE Channel has gone inactive! Channel is [id: 0x064f080e, /127.0.0.1:33816 :> localhost/127.0.0.1:5672] 10:23:25,450 TRACE Firing onTransportClosed listener 10:23:25,451 DEBUG Transport connection remotely closed 10:23:25,453 DEBUG Failover: the provider reports failure: Transport connection remotely closed. 10:23:25,453 DEBUG handling Provider failure: Transport connection remotely closed. 10:23:25,455 TRACE stack java.io.IOException: Transport connection remotely closed. at org.apache.qpid.jms.provider.amqp.AmqpProvider$19.run(AmqpProvider.java:752) at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:471) at java.util.concurrent.FutureTask.run(FutureTask.java:262) at java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.access$201(ScheduledThreadPoolExecutor.java:178) at java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.run(ScheduledThreadPoolExecutor.java:292) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1145) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615) at java.lang.Thread.run(Thread.java:745) 10:23:25,459 TRACE Cancelling scheduled IdleTimeoutCheck 10:23:25,461 DEBUG Connection attempt:[1] to: amqp://localhost:5672 in-progress 10:23:25,463 TRACE Instrumented an optimized java.util.Set into: sun.nio.ch.EPollSelectorImpl@38250a08 10:23:25,463 TRACE Instrumented an optimized java.util.Set into: sun.nio.ch.EPollSelectorImpl@38250a08 10:23:25,469 DEBUG Signalling connection recovery: AmqpProvider: localhost:5672 10:23:25,469 DEBUG Connection ID:dhcp-75-219.lab.eng.brq.redhat.com-53842-1440491002956-0:1 is starting recovery. 10:23:25,469 TRACE Channel has become active! Channel is [id: 0xf459651a, /127.0.0.1:33817 => localhost/127.0.0.1:5672] 10:23:25,471 TRACE Attempted write of: 8 bytes 10:23:25,504 TRACE Channel has gone inactive! Channel is [id: 0xf459651a, /127.0.0.1:33817 :> localhost/127.0.0.1:5672] 10:23:25,505 TRACE Firing onTransportClosed listener 10:23:25,505 DEBUG Transport connection remotely closed 10:23:25,505 DEBUG handling Provider failure: Transport connection remotely closed. 10:23:25,505 TRACE stack java.io.IOException: Transport connection remotely closed. at org.apache.qpid.jms.provider.amqp.AmqpProvider$19.run(AmqpProvider.java:752) at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:471) at java.util.concurrent.FutureTask.run(FutureTask.java:262) at java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.access$201(ScheduledThreadPoolExecutor.java:178) at java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.run(ScheduledThreadPoolExecutor.java:292) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1145) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615) at java.lang.Thread.run(Thread.java:745) 10:23:25,507 DEBUG Connection attempt:[2] to: amqp://localhost:5672 in-progress 10:23:25,508 TRACE Instrumented an optimized java.util.Set into: sun.nio.ch.EPollSelectorImpl@50e68513 10:23:25,508 TRACE Instrumented an optimized java.util.Set into: sun.nio.ch.EPollSelectorImpl@50e68513 10:23:25,511 INFO Connection attempt:[2] to: amqp://localhost:5672 failed 10:23:25,512 TRACE Next reconnect attempt will be in 20 milliseconds 10:23:25,532 DEBUG Connection attempt:[3] to: amqp://localhost:5672 in-progress 10:23:25,534 TRACE Instrumented an optimized java.util.Set into: sun.nio.ch.EPollSelectorImpl@ded0b93 10:23:25,534 TRACE Instrumented an optimized java.util.Set into: sun.nio.ch.EPollSelectorImpl@ded0b93 10:23:25,536 INFO Connection attempt:[3] to: amqp://localhost:5672 failed 10:23:25,537 TRACE Next reconnect attempt will be in 40 milliseconds 10:23:25,578 DEBUG Connection attempt:[4] to: amqp://localhost:5672 in-progress 10:23:25,580 TRACE Instrumented an optimized java.util.Set into: sun.nio.ch.EPollSelectorImpl@50344c81 10:23:25,580 TRACE Instrumented an optimized java.util.Set into: sun.nio.ch.EPollSelectorImpl@50344c81 10:23:25,585 INFO Connection attempt:[4] to: amqp://localhost:5672 failed 10:23:25,586 TRACE Next reconnect attempt will be in 80 milliseconds 10:23:25,666 DEBUG Connection attempt:[5] to: amqp://localhost:5672 in-progress 10:23:25,668 TRACE Instrumented an optimized java.util.Set into: sun.nio.ch.EPollSelectorImpl@30ab0028 10:23:25,668 TRACE Instrumented an optimized java.util.Set into: sun.nio.ch.EPollSelectorImpl@30ab0028 10:23:25,674 INFO Connection attempt:[5] to: amqp://localhost:5672 failed 10:23:25,675 TRACE Next reconnect attempt will be in 160 milliseconds 10:23:25,835 DEBUG Connection attempt:[6] to: amqp://localhost:5672 in-progress 10:23:25,837 TRACE Instrumented an optimized java.util.Set into: sun.nio.ch.EPollSelectorImpl@35886ba1 10:23:25,837 TRACE Instrumented an optimized java.util.Set into: sun.nio.ch.EPollSelectorImpl@35886ba1 10:23:25,839 INFO Connection attempt:[6] to: amqp://localhost:5672 failed 10:23:25,840 TRACE Next reconnect attempt will be in 320 milliseconds 10:23:26,161 DEBUG Connection attempt:[7] to: amqp://localhost:5672 in-progress 10:23:26,163 TRACE Instrumented an optimized java.util.Set into: sun.nio.ch.EPollSelectorImpl@2e8d02e8 10:23:26,163 TRACE Instrumented an optimized java.util.Set into: sun.nio.ch.EPollSelectorImpl@2e8d02e8 10:23:26,165 DEBUG Signalling connection recovery: AmqpProvider: localhost:5672 10:23:26,166 TRACE Channel has become active! Channel is [id: 0x8b13956a, /127.0.0.1:33823 => localhost/127.0.0.1:5672] 10:23:26,166 DEBUG Connection ID:dhcp-75-219.lab.eng.brq.redhat.com-53842-1440491002956-0:1 is starting recovery. 10:23:26,168 TRACE Attempted write of: 8 bytes 10:23:26,170 TRACE New data read: 68 bytes incoming: UnpooledHeapByteBuf(ridx: 0, widx: 68, cap: 65536) 10:23:26,170 TRACE New Proton Event: CONNECTION_INIT 10:23:26,170 TRACE New Proton Event: SESSION_INIT 10:23:26,171 TRACE New Proton Event: CONNECTION_LOCAL_OPEN 10:23:26,179 DEBUG Unknown SASL mechanism: [DIGEST-MD5] 10:23:26,179 INFO Best match for SASL auth was: SASL-CRAM-MD5 10:23:26,179 TRACE Attempted write of: 36 bytes 10:23:26,180 TRACE New data read: 78 bytes incoming: UnpooledHeapByteBuf(ridx: 0, widx: 78, cap: 65536) 10:23:26,181 TRACE Attempted write of: 54 bytes 10:23:26,183 TRACE New data read: 24 bytes incoming: UnpooledHeapByteBuf(ridx: 0, widx: 24, cap: 65536) 10:23:26,191 TRACE Attempted write of: 299 bytes 10:23:26,192 TRACE New data read: 172 bytes incoming: UnpooledHeapByteBuf(ridx: 0, widx: 172, cap: 65536) 10:23:26,193 TRACE New Proton Event: CONNECTION_REMOTE_OPEN 10:23:26,193 TRACE New Proton Event: SESSION_LOCAL_OPEN 10:23:26,193 TRACE Attempted write of: 47 bytes 10:23:26,194 TRACE New data read: 32 bytes incoming: UnpooledHeapByteBuf(ridx: 0, widx: 32, cap: 65536) 10:23:26,195 TRACE New Proton Event: SESSION_REMOTE_OPEN 10:23:26,195 DEBUG AmqpSession { ID:dhcp-75-219.lab.eng.brq.redhat.com-53842-1440491002956-0:1:-1 } is now open: 10:23:26,195 DEBUG AmqpConnection { ID:dhcp-75-219.lab.eng.brq.redhat.com-53842-1440491002956-0:1 } is now open: 10:23:26,195 TRACE IdleTimeoutCheck being initiated, initial delay: 15000 10:23:26,196 TRACE Attempted write of: 47 bytes 10:23:26,197 TRACE New data read: 32 bytes incoming: UnpooledHeapByteBuf(ridx: 0, widx: 32, cap: 65536) 10:23:26,197 TRACE New Proton Event: SESSION_INIT 10:23:26,197 TRACE New Proton Event: SESSION_LOCAL_OPEN 10:23:26,198 TRACE New Proton Event: SESSION_REMOTE_OPEN 10:23:26,198 DEBUG AmqpSession { ID:dhcp-75-219.lab.eng.brq.redhat.com-53842-1440491002956-0:1:1 } is now open: 10:23:26,198 DEBUG Creating AmqpFixedProducer for: test_reconnect_tx_rx 10:23:26,199 TRACE Attempted write of: 323 bytes 10:23:26,201 TRACE New data read: 250 bytes incoming: UnpooledHeapByteBuf(ridx: 0, widx: 250, cap: 65536) 10:23:26,201 TRACE New Proton Event: LINK_INIT 10:23:26,201 TRACE New Proton Event: LINK_LOCAL_OPEN 10:23:26,202 TRACE New Proton Event: LINK_REMOTE_OPEN 10:23:26,202 DEBUG AmqpFixedProducer { ID:dhcp-75-219.lab.eng.brq.redhat.com-53842-1440491002956-0:1:1:1 } is now open: 10:23:26,202 TRACE New Proton Event: LINK_FLOW 10:23:26,202 DEBUG Connection ID:dhcp-75-219.lab.eng.brq.redhat.com-53842-1440491002956-0:1 is finalizing recovery. 10:23:26,202 DEBUG Executing Failover Task: send -> JmsOutboundMessageDispatch {dispatchId = ID:dhcp-75-219.lab.eng.brq.redhat.com-53842-1440491002956-0:1:1:1-2} 10:23:26,203 TRACE Producer sending message: JmsOutboundMessageDispatch {dispatchId = ID:dhcp-75-219.lab.eng.brq.redhat.com-53842-1440491002956-0:1:1:1-2} 10:23:26,204 TRACE Attempted write of: 183 bytes 10:23:26,205 TRACE New data read: 42 bytes incoming: UnpooledHeapByteBuf(ridx: 0, widx: 42, cap: 65536) 10:23:26,206 TRACE New Proton Event: LINK_FLOW 10:23:27,147 TRACE New data read: 38 bytes incoming: UnpooledHeapByteBuf(ridx: 0, widx: 38, cap: 65536) 10:23:27,150 TRACE New Proton Event: DELIVERY 10:23:27,151 TRACE Outcome of delivery was accepted: DeliveryImpl [_tag=[48], _link=org.apache.qpid.proton.engine.impl.SenderImpl@611270a0, _deliveryState=null, _settled=false, _remoteSettled=true, _remoteDeliveryState=Accepted{}, _flags=0, _transportDelivery=org.apache.qpid.proton.engine.impl.TransportDelivery@1a04a2a, _dataSize=0, _complete=true, _updated=true, _done=true, _offset=0] {'redelivered': False, 'reply_to': None, 'id': 'dhcp-75-219.lab.eng.brq.redhat.com-53842-1440491002956-0:1:1:1-2', 'user_id':None, 'correlation_id': None, 'priority': 4, 'durable': True, 'ttl': 0, 'type': None, 'expiration': 0, 'timestamp': 1440491004900, 'properties': {'JMSXDeliveryCount': 1}, 'content': None} 10:23:27,152 DEBUG Executing Failover Task: send -> JmsOutboundMessageDispatch {dispatchId = ID:dhcp-75-219.lab.eng.brq.redhat.com-53842-1440491002956-0:1:1:1-3} 10:23:27,152 TRACE Producer sending message: JmsOutboundMessageDispatch {dispatchId = ID:dhcp-75-219.lab.eng.brq.redhat.com-53842-1440491002956-0:1:1:1-3} 10:23:27,153 TRACE Attempted write of: 184 bytes 10:23:27,154 TRACE New data read: 42 bytes incoming: UnpooledHeapByteBuf(ridx: 0, widx: 42, cap: 65536) 10:23:27,155 TRACE New Proton Event: LINK_FLOW 10:23:28,113 TRACE New data read: 38 bytes incoming: UnpooledHeapByteBuf(ridx: 0, widx: 38, cap: 65536) 10:23:28,114 TRACE New Proton Event: DELIVERY 10:23:28,115 TRACE Outcome of delivery was accepted: DeliveryImpl [_tag=[48], _link=org.apache.qpid.proton.engine.impl.SenderImpl@611270a0, _deliveryState=null, _settled=false, _remoteSettled=true, _remoteDeliveryState=Accepted{}, _flags=0, _transportDelivery=org.apache.qpid.proton.engine.impl.TransportDelivery@3c37114f, _dataSize=0, _complete=true, _updated=true, _done=true, _offset=0] {'redelivered': False, 'reply_to': None, 'id': 'dhcp-75-219.lab.eng.brq.redhat.com-53842-1440491002956-0:1:1:1-3', 'user_id':None, 'correlation_id': None, 'priority': 4, 'durable': True, 'ttl': 0, 'type': None, 'expiration': 0, 'timestamp': 1440491007152, 'properties': {'JMSXDeliveryCount': 1}, 'content': None} 10:23:28,116 DEBUG Executing Failover Task: send -> JmsOutboundMessageDispatch {dispatchId = ID:dhcp-75-219.lab.eng.brq.redhat.com-53842-1440491002956-0:1:1:1-4} 10:23:28,116 TRACE Producer sending message: JmsOutboundMessageDispatch {dispatchId = ID:dhcp-75-219.lab.eng.brq.redhat.com-53842-1440491002956-0:1:1:1-4} 10:23:28,117 TRACE Attempted write of: 184 bytes 10:23:28,118 TRACE New data read: 42 bytes incoming: UnpooledHeapByteBuf(ridx: 0, widx: 42, cap: 65536) 10:23:28,119 TRACE New Proton Event: LINK_FLOW 10:23:29,114 TRACE New data read: 38 bytes incoming: UnpooledHeapByteBuf(ridx: 0, widx: 38, cap: 65536) 10:23:29,115 TRACE New Proton Event: DELIVERY 10:23:29,115 TRACE Outcome of delivery was accepted: DeliveryImpl [_tag=[48], _link=org.apache.qpid.proton.engine.impl.SenderImpl@611270a0, _deliveryState=null, _settled=false, _remoteSettled=true, _remoteDeliveryState=Accepted{}, _flags=0, _transportDelivery=org.apache.qpid.proton.engine.impl.TransportDelivery@19b8db54, _dataSize=0, _complete=true, _updated=true, _done=true, _offset=0] {'redelivered': False, 'reply_to': None, 'id': 'dhcp-75-219.lab.eng.brq.redhat.com-53842-1440491002956-0:1:1:1-4', 'user_id':None, 'correlation_id': None, 'priority': 4, 'durable': True, 'ttl': 0, 'type': None, 'expiration': 0, 'timestamp': 1440491008115, 'properties': {'JMSXDeliveryCount': 1}, 'content': None} 10:23:29,116 DEBUG Executing Failover Task: send -> JmsOutboundMessageDispatch {dispatchId = ID:dhcp-75-219.lab.eng.brq.redhat.com-53842-1440491002956-0:1:1:1-5} 10:23:29,116 TRACE Producer sending message: JmsOutboundMessageDispatch {dispatchId = ID:dhcp-75-219.lab.eng.brq.redhat.com-53842-1440491002956-0:1:1:1-5} 10:23:29,117 TRACE Attempted write of: 184 bytes 10:23:29,118 TRACE New data read: 42 bytes incoming: UnpooledHeapByteBuf(ridx: 0, widx: 42, cap: 65536) 10:23:29,119 TRACE New Proton Event: LINK_FLOW 10:23:29,187 TRACE Channel has gone inactive! Channel is [id: 0x8b13956a, /127.0.0.1:33823 :> localhost/127.0.0.1:5672] 10:23:29,194 TRACE Firing onTransportClosed listener 10:23:29,194 DEBUG Transport connection remotely closed 10:23:29,195 DEBUG Failover: the provider reports failure: Transport connection remotely closed. 10:23:29,195 DEBUG handling Provider failure: Transport connection remotely closed. 10:23:29,195 TRACE stack java.io.IOException: Transport connection remotely closed. at org.apache.qpid.jms.provider.amqp.AmqpProvider$19.run(AmqpProvider.java:752) at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:471) at java.util.concurrent.FutureTask.run(FutureTask.java:262) at java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.access$201(ScheduledThreadPoolExecutor.java:178) at java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.run(ScheduledThreadPoolExecutor.java:292) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1145) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615) at java.lang.Thread.run(Thread.java:745) 10:23:29,196 TRACE Cancelling scheduled IdleTimeoutCheck 10:23:29,196 DEBUG Connection attempt:[1] to: amqp://localhost:5672 in-progress 10:23:29,198 TRACE Instrumented an optimized java.util.Set into: sun.nio.ch.EPollSelectorImpl@d709ed1 10:23:29,198 TRACE Instrumented an optimized java.util.Set into: sun.nio.ch.EPollSelectorImpl@d709ed1 10:23:29,202 DEBUG Signalling connection recovery: AmqpProvider: localhost:5672 10:23:29,202 TRACE Channel has become active! Channel is [id: 0x3dd84af6, /127.0.0.1:33824 => localhost/127.0.0.1:5672] 10:23:29,202 DEBUG Connection ID:dhcp-75-219.lab.eng.brq.redhat.com-53842-1440491002956-0:1 is starting recovery. 10:23:29,203 TRACE Attempted write of: 8 bytes 10:23:29,260 TRACE Exception on channel! Channel is [id: 0x3dd84af6, /127.0.0.1:33824 => localhost/127.0.0.1:5672] 10:23:29,261 TRACE Firing onTransportError listener 10:23:29,263 TRACE Channel has gone inactive! Channel is [id: 0x3dd84af6, /127.0.0.1:33824 :> localhost/127.0.0.1:5672] 10:23:29,263 INFO Transport failed: Connection reset by peer 10:23:29,263 DEBUG handling Provider failure: Connection reset by peer 10:23:29,264 TRACE stack java.io.IOException: Connection reset by peer at sun.nio.ch.FileDispatcherImpl.read0(Native Method) at sun.nio.ch.SocketDispatcher.read(SocketDispatcher.java:39) at sun.nio.ch.IOUtil.readIntoNativeBuffer(IOUtil.java:223) at sun.nio.ch.IOUtil.read(IOUtil.java:197) at sun.nio.ch.SocketChannelImpl.read(SocketChannelImpl.java:384) at io.netty.buffer.UnpooledHeapByteBuf.setBytes(UnpooledHeapByteBuf.java:256) at io.netty.buffer.AbstractByteBuf.writeBytes(AbstractByteBuf.java:871) at io.netty.channel.socket.nio.NioSocketChannel.doReadBytes(NioSocketChannel.java:208) at io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:108) at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:494) at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:461) at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:378) at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:350) at io.netty.util.concurrent.SingleThreadEventExecutor$2.run(SingleThreadEventExecutor.java:101) at java.lang.Thread.run(Thread.java:745) 10:23:29,265 DEBUG Failover: the provider reports failure: Connection reset by peer 10:23:29,265 DEBUG handling Provider failure: Connection reset by peer 10:23:29,265 DEBUG Connection attempt:[2] to: amqp://localhost:5672 in-progress 10:23:29,265 TRACE stack java.io.IOException: Connection reset by peer at sun.nio.ch.FileDispatcherImpl.read0(Native Method) at sun.nio.ch.SocketDispatcher.read(SocketDispatcher.java:39) at sun.nio.ch.IOUtil.readIntoNativeBuffer(IOUtil.java:223) at sun.nio.ch.IOUtil.read(IOUtil.java:197) at sun.nio.ch.SocketChannelImpl.read(SocketChannelImpl.java:384) at io.netty.buffer.UnpooledHeapByteBuf.setBytes(UnpooledHeapByteBuf.java:256) at io.netty.buffer.AbstractByteBuf.writeBytes(AbstractByteBuf.java:871) at io.netty.channel.socket.nio.NioSocketChannel.doReadBytes(NioSocketChannel.java:208) at io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:108) at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:494) at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:461) at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:378) at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:350) at io.netty.util.concurrent.SingleThreadEventExecutor$2.run(SingleThreadEventExecutor.java:101) at java.lang.Thread.run(Thread.java:745) 10:23:29,267 TRACE Instrumented an optimized java.util.Set into: sun.nio.ch.EPollSelectorImpl@7a431f85 10:23:29,267 TRACE Instrumented an optimized java.util.Set into: sun.nio.ch.EPollSelectorImpl@7a431f85 10:23:29,270 INFO Connection attempt:[2] to: amqp://localhost:5672 failed 10:23:29,271 TRACE Next reconnect attempt will be in 20 milliseconds 10:23:29,292 DEBUG Connection attempt:[3] to: amqp://localhost:5672 in-progress 10:23:29,293 TRACE Instrumented an optimized java.util.Set into: sun.nio.ch.EPollSelectorImpl@2199ea11 10:23:29,293 TRACE Instrumented an optimized java.util.Set into: sun.nio.ch.EPollSelectorImpl@2199ea11 10:23:29,296 INFO Connection attempt:[3] to: amqp://localhost:5672 failed 10:23:29,297 TRACE Next reconnect attempt will be in 40 milliseconds 10:23:29,337 DEBUG Connection attempt:[4] to: amqp://localhost:5672 in-progress 10:23:29,339 TRACE Instrumented an optimized java.util.Set into: sun.nio.ch.EPollSelectorImpl@ea6ad47 10:23:29,339 TRACE Instrumented an optimized java.util.Set into: sun.nio.ch.EPollSelectorImpl@ea6ad47 10:23:29,344 INFO Connection attempt:[4] to: amqp://localhost:5672 failed 10:23:29,345 TRACE Next reconnect attempt will be in 80 milliseconds 10:23:29,425 DEBUG Connection attempt:[5] to: amqp://localhost:5672 in-progress 10:23:29,427 TRACE Instrumented an optimized java.util.Set into: sun.nio.ch.EPollSelectorImpl@31f4564b 10:23:29,427 TRACE Instrumented an optimized java.util.Set into: sun.nio.ch.EPollSelectorImpl@31f4564b 10:23:29,430 INFO Connection attempt:[5] to: amqp://localhost:5672 failed 10:23:29,431 TRACE Next reconnect attempt will be in 160 milliseconds 10:23:29,591 DEBUG Connection attempt:[6] to: amqp://localhost:5672 in-progress 10:23:29,593 TRACE Instrumented an optimized java.util.Set into: sun.nio.ch.EPollSelectorImpl@16a0b3a5 10:23:29,593 TRACE Instrumented an optimized java.util.Set into: sun.nio.ch.EPollSelectorImpl@16a0b3a5 10:23:29,595 INFO Connection attempt:[6] to: amqp://localhost:5672 failed 10:23:29,596 TRACE Next reconnect attempt will be in 320 milliseconds 10:23:29,917 DEBUG Connection attempt:[7] to: amqp://localhost:5672 in-progress 10:23:29,919 TRACE Instrumented an optimized java.util.Set into: sun.nio.ch.EPollSelectorImpl@472324c2 10:23:29,919 TRACE Instrumented an optimized java.util.Set into: sun.nio.ch.EPollSelectorImpl@472324c2 10:23:29,923 DEBUG Signalling connection recovery: AmqpProvider: localhost:5672 10:23:29,923 TRACE Channel has become active! Channel is [id: 0x93633196, /127.0.0.1:33830 => localhost/127.0.0.1:5672] 10:23:29,923 DEBUG Connection ID:dhcp-75-219.lab.eng.brq.redhat.com-53842-1440491002956-0:1 is starting recovery. 10:23:29,925 TRACE Attempted write of: 8 bytes 10:23:29,927 TRACE New data read: 68 bytes incoming: UnpooledHeapByteBuf(ridx: 0, widx: 68, cap: 65536) 10:23:29,928 TRACE New Proton Event: CONNECTION_INIT 10:23:29,928 TRACE New Proton Event: SESSION_INIT 10:23:29,928 TRACE New Proton Event: CONNECTION_LOCAL_OPEN 10:23:29,937 DEBUG Unknown SASL mechanism: [DIGEST-MD5] 10:23:29,937 INFO Best match for SASL auth was: SASL-CRAM-MD5 10:23:29,937 TRACE Attempted write of: 36 bytes 10:23:29,939 TRACE New data read: 78 bytes incoming: UnpooledHeapByteBuf(ridx: 0, widx: 78, cap: 65536) 10:23:29,940 TRACE Attempted write of: 54 bytes 10:23:29,942 TRACE New data read: 24 bytes incoming: UnpooledHeapByteBuf(ridx: 0, widx: 24, cap: 65536) 10:23:29,944 TRACE Attempted write of: 299 bytes 10:23:29,945 TRACE New data read: 172 bytes incoming: UnpooledHeapByteBuf(ridx: 0, widx: 172, cap: 65536) 10:23:29,946 TRACE New Proton Event: CONNECTION_REMOTE_OPEN 10:23:29,946 TRACE New Proton Event: SESSION_LOCAL_OPEN 10:23:29,946 TRACE Attempted write of: 47 bytes 10:23:29,947 TRACE New data read: 32 bytes incoming: UnpooledHeapByteBuf(ridx: 0, widx: 32, cap: 65536) 10:23:29,948 TRACE New Proton Event: SESSION_REMOTE_OPEN 10:23:29,948 DEBUG AmqpSession { ID:dhcp-75-219.lab.eng.brq.redhat.com-53842-1440491002956-0:1:-1 } is now open: 10:23:29,948 DEBUG AmqpConnection { ID:dhcp-75-219.lab.eng.brq.redhat.com-53842-1440491002956-0:1 } is now open: 10:23:29,948 TRACE IdleTimeoutCheck being initiated, initial delay: 15000 10:23:29,949 TRACE Attempted write of: 47 bytes 10:23:29,950 TRACE New data read: 32 bytes incoming: UnpooledHeapByteBuf(ridx: 0, widx: 32, cap: 65536) 10:23:29,950 TRACE New Proton Event: SESSION_INIT 10:23:29,950 TRACE New Proton Event: SESSION_LOCAL_OPEN 10:23:29,950 TRACE New Proton Event: SESSION_REMOTE_OPEN 10:23:29,950 DEBUG AmqpSession { ID:dhcp-75-219.lab.eng.brq.redhat.com-53842-1440491002956-0:1:1 } is now open: 10:23:29,951 DEBUG Creating AmqpFixedProducer for: test_reconnect_tx_rx 10:23:29,952 TRACE Attempted write of: 323 bytes 10:23:29,953 TRACE New data read: 250 bytes incoming: UnpooledHeapByteBuf(ridx: 0, widx: 250, cap: 65536) 10:23:29,954 TRACE New Proton Event: LINK_INIT 10:23:29,954 TRACE New Proton Event: LINK_LOCAL_OPEN 10:23:29,954 TRACE New Proton Event: LINK_REMOTE_OPEN 10:23:29,954 DEBUG AmqpFixedProducer { ID:dhcp-75-219.lab.eng.brq.redhat.com-53842-1440491002956-0:1:1:1 } is now open: 10:23:29,954 TRACE New Proton Event: LINK_FLOW 10:23:29,954 DEBUG Connection ID:dhcp-75-219.lab.eng.brq.redhat.com-53842-1440491002956-0:1 is finalizing recovery. 10:23:29,955 DEBUG Executing Failover Task: send -> JmsOutboundMessageDispatch {dispatchId = ID:dhcp-75-219.lab.eng.brq.redhat.com-53842-1440491002956-0:1:1:1-5} 10:23:29,955 TRACE Producer sending message: JmsOutboundMessageDispatch {dispatchId = ID:dhcp-75-219.lab.eng.brq.redhat.com-53842-1440491002956-0:1:1:1-5} 10:23:29,957 TRACE Attempted write of: 183 bytes 10:23:29,958 TRACE New data read: 42 bytes incoming: UnpooledHeapByteBuf(ridx: 0, widx: 42, cap: 65536) 10:23:29,958 TRACE New Proton Event: LINK_FLOW 10:23:30,847 TRACE New data read: 38 bytes incoming: UnpooledHeapByteBuf(ridx: 0, widx: 38, cap: 65536) 10:23:30,848 TRACE New Proton Event: DELIVERY 10:23:30,848 TRACE Outcome of delivery was accepted: DeliveryImpl [_tag=[48], _link=org.apache.qpid.proton.engine.impl.SenderImpl@879de1d, _deliveryState=null, _settled=false, _remoteSettled=true, _remoteDeliveryState=Accepted{}, _flags=0, _transportDelivery=org.apache.qpid.proton.engine.impl.TransportDelivery@78e54643, _dataSize=0, _complete=true, _updated=true, _done=true, _offset=0] {'redelivered': False, 'reply_to': None, 'id': 'dhcp-75-219.lab.eng.brq.redhat.com-53842-1440491002956-0:1:1:1-5', 'user_id':None, 'correlation_id': None, 'priority': 4, 'durable': True, 'ttl': 0, 'type': None, 'expiration': 0, 'timestamp': 1440491009116, 'properties': {'JMSXDeliveryCount': 1}, 'content': None} 10:23:30,849 DEBUG Executing Failover Task: send -> JmsOutboundMessageDispatch {dispatchId = ID:dhcp-75-219.lab.eng.brq.redhat.com-53842-1440491002956-0:1:1:1-6} 10:23:30,850 TRACE Producer sending message: JmsOutboundMessageDispatch {dispatchId = ID:dhcp-75-219.lab.eng.brq.redhat.com-53842-1440491002956-0:1:1:1-6} 10:23:30,851 TRACE Attempted write of: 184 bytes 10:23:30,852 TRACE New data read: 42 bytes incoming: UnpooledHeapByteBuf(ridx: 0, widx: 42, cap: 65536) 10:23:30,852 TRACE New Proton Event: LINK_FLOW 10:23:31,840 TRACE New data read: 38 bytes incoming: UnpooledHeapByteBuf(ridx: 0, widx: 38, cap: 65536) 10:23:31,841 TRACE New Proton Event: DELIVERY 10:23:31,841 TRACE Outcome of delivery was accepted: DeliveryImpl [_tag=[48], _link=org.apache.qpid.proton.engine.impl.SenderImpl@879de1d, _deliveryState=null, _settled=false, _remoteSettled=true, _remoteDeliveryState=Accepted{}, _flags=0, _transportDelivery=org.apache.qpid.proton.engine.impl.TransportDelivery@27ba8e87, _dataSize=0, _complete=true, _updated=true, _done=true, _offset=0] {'redelivered': False, 'reply_to': None, 'id': 'dhcp-75-219.lab.eng.brq.redhat.com-53842-1440491002956-0:1:1:1-6', 'user_id':None, 'correlation_id': None, 'priority': 4, 'durable': True, 'ttl': 0, 'type': None, 'expiration': 0, 'timestamp': 1440491010849, 'properties': {'JMSXDeliveryCount': 1}, 'content': None} 10:23:31,843 DEBUG Executing Failover Task: send -> JmsOutboundMessageDispatch {dispatchId = ID:dhcp-75-219.lab.eng.brq.redhat.com-53842-1440491002956-0:1:1:1-7} 10:23:31,843 TRACE Producer sending message: JmsOutboundMessageDispatch {dispatchId = ID:dhcp-75-219.lab.eng.brq.redhat.com-53842-1440491002956-0:1:1:1-7} 10:23:31,844 TRACE Attempted write of: 184 bytes 10:23:31,845 TRACE New data read: 42 bytes incoming: UnpooledHeapByteBuf(ridx: 0, widx: 42, cap: 65536) 10:23:31,846 TRACE New Proton Event: LINK_FLOW 10:23:32,843 TRACE New data read: 38 bytes incoming: UnpooledHeapByteBuf(ridx: 0, widx: 38, cap: 65536) 10:23:32,844 TRACE New Proton Event: DELIVERY 10:23:32,844 TRACE Outcome of delivery was accepted: DeliveryImpl [_tag=[48], _link=org.apache.qpid.proton.engine.impl.SenderImpl@879de1d, _deliveryState=null, _settled=false, _remoteSettled=true, _remoteDeliveryState=Accepted{}, _flags=0, _transportDelivery=org.apache.qpid.proton.engine.impl.TransportDelivery@44fe9319, _dataSize=0, _complete=true, _updated=true, _done=true, _offset=0] {'redelivered': False, 'reply_to': None, 'id': 'dhcp-75-219.lab.eng.brq.redhat.com-53842-1440491002956-0:1:1:1-7', 'user_id':None, 'correlation_id': None, 'priority': 4, 'durable': True, 'ttl': 0, 'type': None, 'expiration': 0, 'timestamp': 1440491011842, 'properties': {'JMSXDeliveryCount': 1}, 'content': None} 10:23:32,846 DEBUG Executing Failover Task: send -> JmsOutboundMessageDispatch {dispatchId = ID:dhcp-75-219.lab.eng.brq.redhat.com-53842-1440491002956-0:1:1:1-8} 10:23:32,846 TRACE Producer sending message: JmsOutboundMessageDispatch {dispatchId = ID:dhcp-75-219.lab.eng.brq.redhat.com-53842-1440491002956-0:1:1:1-8} 10:23:32,847 TRACE Attempted write of: 184 bytes 10:23:32,848 TRACE New data read: 42 bytes incoming: UnpooledHeapByteBuf(ridx: 0, widx: 42, cap: 65536) 10:23:32,848 TRACE New Proton Event: LINK_FLOW 10:23:32,906 TRACE Channel has gone inactive! Channel is [id: 0x93633196, /127.0.0.1:33830 :> localhost/127.0.0.1:5672] 10:23:32,906 TRACE Firing onTransportClosed listener 10:23:32,907 DEBUG Transport connection remotely closed 10:23:32,907 DEBUG Failover: the provider reports failure: Transport connection remotely closed. 10:23:32,907 DEBUG handling Provider failure: Transport connection remotely closed. 10:23:32,907 TRACE stack java.io.IOException: Transport connection remotely closed. at org.apache.qpid.jms.provider.amqp.AmqpProvider$19.run(AmqpProvider.java:752) at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:471) at java.util.concurrent.FutureTask.run(FutureTask.java:262) at java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.access$201(ScheduledThreadPoolExecutor.java:178) at java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.run(ScheduledThreadPoolExecutor.java:292) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1145) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615) at java.lang.Thread.run(Thread.java:745) 10:23:32,909 TRACE Cancelling scheduled IdleTimeoutCheck 10:23:32,909 DEBUG Connection attempt:[1] to: amqp://localhost:5672 in-progress 10:23:32,911 TRACE Instrumented an optimized java.util.Set into: sun.nio.ch.EPollSelectorImpl@74f89c35 10:23:32,911 TRACE Instrumented an optimized java.util.Set into: sun.nio.ch.EPollSelectorImpl@74f89c35 10:23:32,918 DEBUG Signalling connection recovery: AmqpProvider: localhost:5672 10:23:32,918 DEBUG Connection ID:dhcp-75-219.lab.eng.brq.redhat.com-53842-1440491002956-0:1 is starting recovery. 10:23:32,919 TRACE Channel has become active! Channel is [id: 0xc1695041, /127.0.0.1:33831 => localhost/127.0.0.1:5672] 10:23:32,920 TRACE Attempted write of: 8 bytes 10:23:32,925 TRACE Exception on channel! Channel is [id: 0xc1695041, /127.0.0.1:33831 => localhost/127.0.0.1:5672] 10:23:32,925 TRACE Firing onTransportError listener 10:23:32,925 INFO Transport failed: Connection reset by peer 10:23:32,926 DEBUG handling Provider failure: Connection reset by peer 10:23:32,926 TRACE stack java.io.IOException: Connection reset by peer at sun.nio.ch.FileDispatcherImpl.read0(Native Method) at sun.nio.ch.SocketDispatcher.read(SocketDispatcher.java:39) at sun.nio.ch.IOUtil.readIntoNativeBuffer(IOUtil.java:223) at sun.nio.ch.IOUtil.read(IOUtil.java:197) at sun.nio.ch.SocketChannelImpl.read(SocketChannelImpl.java:384) at io.netty.buffer.UnpooledHeapByteBuf.setBytes(UnpooledHeapByteBuf.java:256) at io.netty.buffer.AbstractByteBuf.writeBytes(AbstractByteBuf.java:871) at io.netty.channel.socket.nio.NioSocketChannel.doReadBytes(NioSocketChannel.java:208) at io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:108) at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:494) at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:461) at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:378) at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:350) at io.netty.util.concurrent.SingleThreadEventExecutor$2.run(SingleThreadEventExecutor.java:101) at java.lang.Thread.run(Thread.java:745) 10:23:32,926 TRACE Channel has gone inactive! Channel is [id: 0xc1695041, /127.0.0.1:33831 :> localhost/127.0.0.1:5672] 10:23:32,929 DEBUG Failover: the provider reports failure: Connection reset by peer 10:23:32,929 DEBUG Connection attempt:[2] to: amqp://localhost:5672 in-progress 10:23:32,929 DEBUG handling Provider failure: Connection reset by peer 10:23:32,930 TRACE stack java.io.IOException: Connection reset by peer at sun.nio.ch.FileDispatcherImpl.read0(Native Method) at sun.nio.ch.SocketDispatcher.read(SocketDispatcher.java:39) at sun.nio.ch.IOUtil.readIntoNativeBuffer(IOUtil.java:223) at sun.nio.ch.IOUtil.read(IOUtil.java:197) at sun.nio.ch.SocketChannelImpl.read(SocketChannelImpl.java:384) at io.netty.buffer.UnpooledHeapByteBuf.setBytes(UnpooledHeapByteBuf.java:256) at io.netty.buffer.AbstractByteBuf.writeBytes(AbstractByteBuf.java:871) at io.netty.channel.socket.nio.NioSocketChannel.doReadBytes(NioSocketChannel.java:208) at io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:108) at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:494) at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:461) at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:378) at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:350) at io.netty.util.concurrent.SingleThreadEventExecutor$2.run(SingleThreadEventExecutor.java:101) at java.lang.Thread.run(Thread.java:745) 10:23:32,931 TRACE Instrumented an optimized java.util.Set into: sun.nio.ch.EPollSelectorImpl@3d3d7d31 10:23:32,931 TRACE Instrumented an optimized java.util.Set into: sun.nio.ch.EPollSelectorImpl@3d3d7d31 10:23:32,934 INFO Connection attempt:[2] to: amqp://localhost:5672 failed 10:23:32,937 TRACE Next reconnect attempt will be in 20 milliseconds 10:23:32,958 DEBUG Connection attempt:[3] to: amqp://localhost:5672 in-progress 10:23:32,959 TRACE Instrumented an optimized java.util.Set into: sun.nio.ch.EPollSelectorImpl@481e5c05 10:23:32,959 TRACE Instrumented an optimized java.util.Set into: sun.nio.ch.EPollSelectorImpl@481e5c05 10:23:32,962 INFO Connection attempt:[3] to: amqp://localhost:5672 failed 10:23:32,965 TRACE Next reconnect attempt will be in 40 milliseconds 10:23:33,006 DEBUG Connection attempt:[4] to: amqp://localhost:5672 in-progress 10:23:33,007 TRACE Instrumented an optimized java.util.Set into: sun.nio.ch.EPollSelectorImpl@4ed14172 10:23:33,007 TRACE Instrumented an optimized java.util.Set into: sun.nio.ch.EPollSelectorImpl@4ed14172 10:23:33,010 INFO Connection attempt:[4] to: amqp://localhost:5672 failed 10:23:33,011 TRACE Next reconnect attempt will be in 80 milliseconds 10:23:33,091 DEBUG Connection attempt:[5] to: amqp://localhost:5672 in-progress 10:23:33,093 TRACE Instrumented an optimized java.util.Set into: sun.nio.ch.EPollSelectorImpl@76f9d876 10:23:33,093 TRACE Instrumented an optimized java.util.Set into: sun.nio.ch.EPollSelectorImpl@76f9d876 10:23:33,096 INFO Connection attempt:[5] to: amqp://localhost:5672 failed 10:23:33,097 TRACE Next reconnect attempt will be in 160 milliseconds 10:23:33,258 DEBUG Connection attempt:[6] to: amqp://localhost:5672 in-progress 10:23:33,260 TRACE Instrumented an optimized java.util.Set into: sun.nio.ch.EPollSelectorImpl@5aa33bb4 10:23:33,260 TRACE Instrumented an optimized java.util.Set into: sun.nio.ch.EPollSelectorImpl@5aa33bb4 10:23:33,262 INFO Connection attempt:[6] to: amqp://localhost:5672 failed 10:23:33,263 TRACE Next reconnect attempt will be in 320 milliseconds 10:23:33,584 DEBUG Connection attempt:[7] to: amqp://localhost:5672 in-progress 10:23:33,589 TRACE Instrumented an optimized java.util.Set into: sun.nio.ch.EPollSelectorImpl@121c8b94 10:23:33,589 TRACE Instrumented an optimized java.util.Set into: sun.nio.ch.EPollSelectorImpl@121c8b94 10:23:33,593 DEBUG Signalling connection recovery: AmqpProvider: localhost:5672 10:23:33,594 DEBUG Connection ID:dhcp-75-219.lab.eng.brq.redhat.com-53842-1440491002956-0:1 is starting recovery. 10:23:33,595 TRACE Channel has become active! Channel is [id: 0xade8c92b, /127.0.0.1:33837 => localhost/127.0.0.1:5672] 10:23:33,596 TRACE Attempted write of: 8 bytes 10:23:33,608 TRACE New data read: 68 bytes incoming: UnpooledHeapByteBuf(ridx: 0, widx: 68, cap: 65536) 10:23:33,612 TRACE New Proton Event: CONNECTION_INIT 10:23:33,612 TRACE New Proton Event: SESSION_INIT 10:23:33,612 TRACE New Proton Event: CONNECTION_LOCAL_OPEN 10:23:33,613 DEBUG Unknown SASL mechanism: [DIGEST-MD5] 10:23:33,614 INFO Best match for SASL auth was: SASL-CRAM-MD5 10:23:33,614 TRACE Attempted write of: 36 bytes 10:23:33,615 TRACE New data read: 78 bytes incoming: UnpooledHeapByteBuf(ridx: 0, widx: 78, cap: 65536) 10:23:33,617 TRACE Attempted write of: 54 bytes 10:23:33,619 TRACE New data read: 24 bytes incoming: UnpooledHeapByteBuf(ridx: 0, widx: 24, cap: 65536) 10:23:33,621 TRACE Attempted write of: 299 bytes 10:23:33,622 TRACE New data read: 172 bytes incoming: UnpooledHeapByteBuf(ridx: 0, widx: 172, cap: 65536) 10:23:33,623 TRACE New Proton Event: CONNECTION_REMOTE_OPEN 10:23:33,623 TRACE New Proton Event: SESSION_LOCAL_OPEN 10:23:33,624 TRACE Attempted write of: 47 bytes 10:23:33,625 TRACE New data read: 32 bytes incoming: UnpooledHeapByteBuf(ridx: 0, widx: 32, cap: 65536) 10:23:33,625 TRACE New Proton Event: SESSION_REMOTE_OPEN 10:23:33,625 DEBUG AmqpSession { ID:dhcp-75-219.lab.eng.brq.redhat.com-53842-1440491002956-0:1:-1 } is now open: 10:23:33,625 DEBUG AmqpConnection { ID:dhcp-75-219.lab.eng.brq.redhat.com-53842-1440491002956-0:1 } is now open: 10:23:33,625 TRACE IdleTimeoutCheck being initiated, initial delay: 15000 10:23:33,626 TRACE Attempted write of: 47 bytes 10:23:33,627 TRACE New data read: 32 bytes incoming: UnpooledHeapByteBuf(ridx: 0, widx: 32, cap: 65536) 10:23:33,627 TRACE New Proton Event: SESSION_INIT 10:23:33,627 TRACE New Proton Event: SESSION_LOCAL_OPEN 10:23:33,628 TRACE New Proton Event: SESSION_REMOTE_OPEN 10:23:33,628 DEBUG AmqpSession { ID:dhcp-75-219.lab.eng.brq.redhat.com-53842-1440491002956-0:1:1 } is now open: 10:23:33,628 DEBUG Creating AmqpFixedProducer for: test_reconnect_tx_rx 10:23:33,630 TRACE Attempted write of: 323 bytes 10:23:33,632 TRACE New data read: 250 bytes incoming: UnpooledHeapByteBuf(ridx: 0, widx: 250, cap: 65536) 10:23:33,633 TRACE New Proton Event: LINK_INIT 10:23:33,633 TRACE New Proton Event: LINK_LOCAL_OPEN 10:23:33,633 TRACE New Proton Event: LINK_REMOTE_OPEN 10:23:33,633 DEBUG AmqpFixedProducer { ID:dhcp-75-219.lab.eng.brq.redhat.com-53842-1440491002956-0:1:1:1 } is now open: 10:23:33,633 DEBUG Connection ID:dhcp-75-219.lab.eng.brq.redhat.com-53842-1440491002956-0:1 is finalizing recovery. 10:23:33,634 DEBUG Executing Failover Task: send -> JmsOutboundMessageDispatch {dispatchId = ID:dhcp-75-219.lab.eng.brq.redhat.com-53842-1440491002956-0:1:1:1-8} 10:23:33,634 TRACE New Proton Event: LINK_FLOW 10:23:33,634 TRACE Producer sending message: JmsOutboundMessageDispatch {dispatchId = ID:dhcp-75-219.lab.eng.brq.redhat.com-53842-1440491002956-0:1:1:1-8} 10:23:33,636 TRACE Attempted write of: 183 bytes 10:23:33,637 TRACE New data read: 42 bytes incoming: UnpooledHeapByteBuf(ridx: 0, widx: 42, cap: 65536) 10:23:33,637 TRACE New Proton Event: LINK_FLOW 10:23:34,566 TRACE New data read: 38 bytes incoming: UnpooledHeapByteBuf(ridx: 0, widx: 38, cap: 65536) 10:23:34,566 TRACE New Proton Event: DELIVERY 10:23:34,567 TRACE Outcome of delivery was accepted: DeliveryImpl [_tag=[48], _link=org.apache.qpid.proton.engine.impl.SenderImpl@26e2b304, _deliveryState=null, _settled=false, _remoteSettled=true, _remoteDeliveryState=Accepted{}, _flags=0, _transportDelivery=org.apache.qpid.proton.engine.impl.TransportDelivery@6d56df8d, _dataSize=0, _complete=true, _updated=true, _done=true, _offset=0] {'redelivered': False, 'reply_to': None, 'id': 'dhcp-75-219.lab.eng.brq.redhat.com-53842-1440491002956-0:1:1:1-8', 'user_id':None, 'correlation_id': None, 'priority': 4, 'durable': True, 'ttl': 0, 'type': None, 'expiration': 0, 'timestamp': 1440491012845, 'properties': {'JMSXDeliveryCount': 1}, 'content': None} 10:23:34,568 DEBUG Executing Failover Task: send -> JmsOutboundMessageDispatch {dispatchId = ID:dhcp-75-219.lab.eng.brq.redhat.com-53842-1440491002956-0:1:1:1-9} 10:23:34,568 TRACE Producer sending message: JmsOutboundMessageDispatch {dispatchId = ID:dhcp-75-219.lab.eng.brq.redhat.com-53842-1440491002956-0:1:1:1-9} 10:23:34,569 TRACE Attempted write of: 184 bytes 10:23:34,570 TRACE New data read: 42 bytes incoming: UnpooledHeapByteBuf(ridx: 0, widx: 42, cap: 65536) 10:23:34,571 TRACE New Proton Event: LINK_FLOW 10:23:35,562 TRACE New data read: 38 bytes incoming: UnpooledHeapByteBuf(ridx: 0, widx: 38, cap: 65536) 10:23:35,563 TRACE New Proton Event: DELIVERY 10:23:35,564 TRACE Outcome of delivery was accepted: DeliveryImpl [_tag=[48], _link=org.apache.qpid.proton.engine.impl.SenderImpl@26e2b304, _deliveryState=null, _settled=false, _remoteSettled=true, _remoteDeliveryState=Accepted{}, _flags=0, _transportDelivery=org.apache.qpid.proton.engine.impl.TransportDelivery@35924daa, _dataSize=0, _complete=true, _updated=true, _done=true, _offset=0] {'redelivered': False, 'reply_to': None, 'id': 'dhcp-75-219.lab.eng.brq.redhat.com-53842-1440491002956-0:1:1:1-9', 'user_id':None, 'correlation_id': None, 'priority': 4, 'durable': True, 'ttl': 0, 'type': None, 'expiration': 0, 'timestamp': 1440491014568, 'properties': {'JMSXDeliveryCount': 1}, 'content': None} 10:23:35,565 DEBUG Executing Failover Task: send -> JmsOutboundMessageDispatch {dispatchId = ID:dhcp-75-219.lab.eng.brq.redhat.com-53842-1440491002956-0:1:1:1-10} 10:23:35,566 TRACE Producer sending message: JmsOutboundMessageDispatch {dispatchId = ID:dhcp-75-219.lab.eng.brq.redhat.com-53842-1440491002956-0:1:1:1-10} 10:23:35,567 TRACE Attempted write of: 185 bytes 10:23:35,568 TRACE New data read: 42 bytes incoming: UnpooledHeapByteBuf(ridx: 0, widx: 42, cap: 65536) 10:23:35,568 TRACE New Proton Event: LINK_FLOW 10:23:36,565 TRACE New data read: 38 bytes incoming: UnpooledHeapByteBuf(ridx: 0, widx: 38, cap: 65536) 10:23:36,566 TRACE New Proton Event: DELIVERY 10:23:36,566 TRACE Outcome of delivery was accepted: DeliveryImpl [_tag=[48], _link=org.apache.qpid.proton.engine.impl.SenderImpl@26e2b304, _deliveryState=null, _settled=false, _remoteSettled=true, _remoteDeliveryState=Accepted{}, _flags=0, _transportDelivery=org.apache.qpid.proton.engine.impl.TransportDelivery@277b9404, _dataSize=0, _complete=true, _updated=true, _done=true, _offset=0] {'redelivered': False, 'reply_to': None, 'id': 'dhcp-75-219.lab.eng.brq.redhat.com-53842-1440491002956-0:1:1:1-10', 'user_id':None, 'correlation_id': None, 'priority': 4, 'durable': True, 'ttl': 0, 'type': None, 'expiration': 0, 'timestamp': 1440491015565, 'properties': {'JMSXDeliveryCount': 1}, 'content': None} 10:23:36,571 DEBUG Executing Failover Task: destroy -> JmsSessionInfo {} 10:23:36,576 TRACE Attempted write of: 12 bytes 10:23:36,577 TRACE New data read: 21 bytes incoming: UnpooledHeapByteBuf(ridx: 0, widx: 21, cap: 65536) 10:23:36,578 TRACE New Proton Event: SESSION_LOCAL_CLOSE 10:23:36,578 TRACE New Proton Event: SESSION_REMOTE_CLOSE 10:23:36,578 DEBUG AmqpSession { ID:dhcp-75-219.lab.eng.brq.redhat.com-53842-1440491002956-0:1:1 } is now closed: 10:23:36,578 TRACE New Proton Event: SESSION_LOCAL_CLOSE 10:23:36,579 TRACE New Proton Event: LINK_FINAL 10:23:36,579 TRACE New Proton Event: SESSION_FINAL 10:23:36,579 DEBUG Executing Failover Task: destroy -> JmsConnectionInfo {sendTimeout = -1, requestTimeout = -1, connectTimeout = 15000, closeTimeout = 15000} 10:23:36,579 TRACE Attempted write of: 12 bytes 10:23:36,580 TRACE New data read: 21 bytes incoming: UnpooledHeapByteBuf(ridx: 0, widx: 21, cap: 65536) 10:23:36,581 TRACE New Proton Event: CONNECTION_LOCAL_CLOSE 10:23:36,581 TRACE New Proton Event: TRANSPORT_HEAD_CLOSED 10:23:36,581 TRACE New Proton Event: CONNECTION_REMOTE_CLOSE 10:23:36,581 DEBUG AmqpConnection { ID:dhcp-75-219.lab.eng.brq.redhat.com-53842-1440491002956-0:1 } is now closed: 10:23:36,581 TRACE Channel has gone inactive! Channel is [id: 0xade8c92b, /127.0.0.1:33837 :> localhost/127.0.0.1:5672] 10:23:36,581 TRACE Firing onTransportClosed listener 10:23:36,584 TRACE Shutdown of ExecutorService: java.util.concurrent.ThreadPoolExecutor@4d2c70d2[Running, pool size = 0, active threads = 0, queued tasks = 0, completed tasks = 0] with await termination: 0 millis 10:23:36,586 DEBUG Shutdown of ExecutorService: java.util.concurrent.ThreadPoolExecutor@4d2c70d2[Terminated, pool size = 0, active threads = 0, queued tasks = 0, completed tasks = 0] is shutdown: true and terminated: true took: 0.001 seconds. 10:23:36,587 TRACE New Proton Event: TRANSPORT_TAIL_CLOSED 10:23:36,587 TRACE New Proton Event: TRANSPORT_CLOSED 10:23:36,587 TRACE New Proton Event: CONNECTION_LOCAL_CLOSE 10:23:36,588 DEBUG Transport connection remotely closed 10:23:36,588 TRACE Cancelling scheduled IdleTimeoutCheck 10:23:36,588 TRACE Shutdown of ExecutorService: java.util.concurrent.ScheduledThreadPoolExecutor@70941f0a[Running, pool size = 1, active threads = 0, queued tasks = 0, completed tasks = 22] with await termination: 10000 millis 10:23:36,593 DEBUG Shutdown of ExecutorService: java.util.concurrent.ScheduledThreadPoolExecutor@70941f0a[Terminated, pool size = 0, active threads = 0, queued tasks = 0, completed tasks = 22] is shutdown: true and terminated: true took: 0.004 seconds.