Details
Description
Remoting deadlock caused by shutdown/receive race after VM pause
It is triggering a deadlock where
JsseConnectedSslStreamChannel.doFlush() and
RemoteReadListener.handleEvent() indirectly take two locks in the
opposite order. There would be a very minute chance of this
concurrently, so it's probably why we haven't seen it before.
"main": at org.xnio.channels.FramedMessageChannel.closeAction(FramedMessageChannel.java:334) - waiting to lock <0x00000000e1a7ec70> (a org.xnio.Buffers$4) at org.xnio.channels.TranslatingSuspendableChannel.close(TranslatingSuspendableChannel.java:811) at org.xnio.IoUtils.safeClose(IoUtils.java:137) at org.xnio.channels.TranslatingSuspendableChannel$3.handleEvent(TranslatingSuspendableChannel.java:123) at org.xnio.ChannelListeners.invokeChannelListener(ChannelListeners.java:72) at org.xnio.channels.TranslatingSuspendableChannel.close(TranslatingSuspendableChannel.java:813) at org.xnio.ssl.JsseConnectedSslStreamChannel.handleHandshake(JsseConnectedSslStreamChannel.java:412) - locked <0x00000000e1a72588> (a org.xnio.ByteBufferSlicePool$PooledByteBuffer) at org.xnio.ssl.JsseConnectedSslStreamChannel.doFlush(JsseConnectedSslStreamChannel.java:638) at org.xnio.ssl.JsseConnectedSslStreamChannel.flushAction(JsseConnectedSslStreamChannel.java:613) - locked <0x00000000e1a72628> (a org.xnio.ByteBufferSlicePool$PooledByteBuffer) at org.xnio.channels.TranslatingSuspendableChannel.flush(TranslatingSuspendableChannel.java:604) at org.xnio.channels.FramedMessageChannel.flushAction(FramedMessageChannel.java:279) - locked <0x00000000e1a80cd0> (a org.xnio.Buffers$4) at org.xnio.channels.TranslatingSuspendableChannel.flush(TranslatingSuspendableChannel.java:604) at org.jboss.remoting3.remote.RemoteConnection$RemoteWriteListener.shutdownWrites(RemoteConnection.java:259) - locked <0x00000000e1a8af88> (a java.util.ArrayDeque) at org.jboss.remoting3.remote.RemoteConnection.shutdownWrites(RemoteConnection.java:128) at org.jboss.remoting3.remote.RemoteConnectionHandler.sendCloseRequest(RemoteConnectionHandler.java:235) at org.jboss.remoting3.remote.RemoteConnectionHandler.closeAction(RemoteConnectionHandler.java:378) at org.jboss.remoting3.spi.AbstractHandleableCloseable.closeAsync(AbstractHandleableCloseable.java:359) at org.jboss.remoting3.ConnectionImpl.closeAction(ConnectionImpl.java:52) at org.jboss.remoting3.spi.AbstractHandleableCloseable.close(AbstractHandleableCloseable.java:153) at org.jboss.as.protocol.StreamUtils.safeClose(StreamUtils.java:66) at org.jboss.as.protocol.ProtocolConnectionManager.shutdown(ProtocolConnectionManager.java:119) at org.jboss.as.protocol.mgmt.FutureManagementChannel$Establishing.close(FutureManagementChannel.java:228) at org.jboss.as.protocol.StreamUtils.safeClose(StreamUtils.java:66) at org.jboss.as.cli.impl.CLIModelControllerClient.close(CLIModelControllerClient.java:182) - locked <0x00000000e30005a0> (a java.lang.String) at org.jboss.as.protocol.StreamUtils.safeClose(StreamUtils.java:66) at org.jboss.as.cli.impl.CommandContextImpl.disconnectController(CommandContextImpl.java:983) at org.jboss.as.cli.impl.CommandContextImpl.terminateSession(CommandContextImpl.java:653) at org.jboss.as.cli.impl.CliLauncher.processCommands(CliLauncher.java:306) at org.jboss.as.cli.impl.CliLauncher.main(CliLauncher.java:251) at org.jboss.as.cli.CommandLineMain.main(CommandLineMain.java:34) at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:57) at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.lang.reflect.Method.invoke(Method.java:606) at org.jboss.modules.Module.run(Module.java:312) at org.jboss.modules.Main.main(Main.java:460) "Remoting "cli-client" read-1": at org.xnio.ssl.JsseConnectedSslStreamChannel.read(JsseConnectedSslStreamChannel.java:474) - waiting to lock <0x00000000e1a72588> (a org.xnio.ByteBufferSlicePool$PooledByteBuffer) at org.xnio.ssl.JsseConnectedSslStreamChannel.read(JsseConnectedSslStreamChannel.java:449) at org.xnio.channels.FramedMessageChannel.receive(FramedMessageChannel.java:87) - locked <0x00000000e1a7ec70> (a org.xnio.Buffers$4) at org.jboss.remoting3.remote.RemoteReadListener.handleEvent(RemoteReadListener.java:72) at org.jboss.remoting3.remote.RemoteReadListener.handleEvent(RemoteReadListener.java:45) at org.xnio.ChannelListeners.invokeChannelListener(ChannelListeners.java:72) at org.xnio.channels.TranslatingSuspendableChannel.handleReadable(TranslatingSuspendableChannel.java:189) at org.xnio.channels.TranslatingSuspendableChannel$1.handleEvent(TranslatingSuspendableChannel.java:103) at org.xnio.ChannelListeners.invokeChannelListener(ChannelListeners.java:72) at org.xnio.channels.TranslatingSuspendableChannel.handleReadable(TranslatingSuspendableChannel.java:189) at org.xnio.ssl.JsseConnectedSslStreamChannel.handleReadable(JsseConnectedSslStreamChannel.java:180) at org.xnio.channels.TranslatingSuspendableChannel$1.handleEvent(TranslatingSuspendableChannel.java:103) at org.xnio.ChannelListeners.invokeChannelListener(ChannelListeners.java:72) at org.xnio.nio.NioHandle.run(NioHandle.java:90) at org.xnio.nio.WorkerThread.run(WorkerThread.java:198)