ISPN000136: Execution error org.infinispan.remoting.RpcException: No more valid responses. Received invalid responses from all of [Sender

We have 2 node cluster. If we stop the other node, I get continuous errors in logs with following traces. Restarting the node has resolved errors but I want to understand root cause and find if any configuration can fix the issue.

ERROR 06/20 19:01:40.303 [ComponentInit_2] o.i.i.InvocationContextInterceptor ISPN000136: Execution error
org.infinispan.remoting.RpcException: No more valid responses. Received invalid responses from all of [Sender

{address=tsps-blr-win02-11856, responded=true}

]
at org.infinispan.remoting.transport.jgroups.CommandAwareRpcDispatcher$FutureCollator.getResponseList(CommandAwareRpcDispatcher.java:656)
at org.infinispan.remoting.transport.jgroups.CommandAwareRpcDispatcher.processCalls(CommandAwareRpcDispatcher.java:493)
at org.infinispan.remoting.transport.jgroups.CommandAwareRpcDispatcher.invokeRemoteCommands(CommandAwareRpcDispatcher.java:152)
at org.infinispan.remoting.transport.jgroups.JGroupsTransport.invokeRemotely(JGroupsTransport.java:564)
at org.infinispan.remoting.rpc.RpcManagerImpl.invokeRemotely(RpcManagerImpl.java:287)
at org.infinispan.interceptors.distribution.BaseDistributionInterceptor.invokeClusterGetCommandRemotely(BaseDistributionInterceptor.java:177)
at org.infinispan.interceptors.distribution.BaseDistributionInterceptor.retrieveFromRemoteSource(BaseDistributionInterceptor.java:165)
at org.infinispan.interceptors.distribution.TxDistributionInterceptor.remoteGet(TxDistributionInterceptor.java:363)
at org.infinispan.interceptors.distribution.TxDistributionInterceptor.remoteGetBeforeWrite(TxDistributionInterceptor.java:346)
at org.infinispan.interceptors.distribution.TxDistributionInterceptor.handleTxWriteCommand(TxDistributionInterceptor.java:304)
at org.infinispan.interceptors.distribution.TxDistributionInterceptor.visitPutKeyValueCommand(TxDistributionInterceptor.java:109)
at org.infinispan.commands.write.PutKeyValueCommand.acceptVisitor(PutKeyValueCommand.java:71)
at org.infinispan.interceptors.base.CommandInterceptor.invokeNextInterceptor(CommandInterceptor.java:97)
at org.infinispan.interceptors.EntryWrappingInterceptor.invokeNextAndApplyChanges(EntryWrappingInterceptor.java:388)
at org.infinispan.interceptors.EntryWrappingInterceptor.setSkipRemoteGetsAndInvokeNextForDataCommand(EntryWrappingInterceptor.java:453)
at org.infinispan.interceptors.EntryWrappingInterceptor.visitPutKeyValueCommand(EntryWrappingInterceptor.java:195)
at org.infinispan.commands.write.PutKeyValueCommand.acceptVisitor(PutKeyValueCommand.java:71)
at org.infinispan.interceptors.base.CommandInterceptor.invokeNextInterceptor(CommandInterceptor.java:97)
at org.infinispan.interceptors.locking.OptimisticLockingInterceptor.visitDataWriteCommand(OptimisticLockingInterceptor.java:147)
at org.infinispan.interceptors.locking.AbstractTxLockingInterceptor.visitPutKeyValueCommand(AbstractTxLockingInterceptor.java:68)
at org.infinispan.commands.write.PutKeyValueCommand.acceptVisitor(PutKeyValueCommand.java:71)
at org.infinispan.interceptors.base.CommandInterceptor.invokeNextInterceptor(CommandInterceptor.java:97)
at org.infinispan.interceptors.base.CommandInterceptor.handleDefault(CommandInterceptor.java:111)
at org.infinispan.commands.AbstractVisitor.visitPutKeyValueCommand(AbstractVisitor.java:44)
at org.infinispan.commands.write.PutKeyValueCommand.acceptVisitor(PutKeyValueCommand.java:71)
at org.infinispan.interceptors.base.CommandInterceptor.invokeNextInterceptor(CommandInterceptor.java:97)
at org.infinispan.interceptors.TxInterceptor.enlistWriteAndInvokeNext(TxInterceptor.java:367)
at org.infinispan.interceptors.TxInterceptor.visitPutKeyValueCommand(TxInterceptor.java:263)
at org.infinispan.commands.write.PutKeyValueCommand.acceptVisitor(PutKeyValueCommand.java:71)
at org.infinispan.interceptors.base.CommandInterceptor.invokeNextInterceptor(CommandInterceptor.java:97)
at org.infinispan.interceptors.base.CommandInterceptor.handleDefault(CommandInterceptor.java:111)
at org.infinispan.commands.AbstractVisitor.visitPutKeyValueCommand(AbstractVisitor.java:44)
at org.infinispan.commands.write.PutKeyValueCommand.acceptVisitor(PutKeyValueCommand.java:71)
at org.infinispan.interceptors.base.CommandInterceptor.invokeNextInterceptor(CommandInterceptor.java:97)
at org.infinispan.statetransfer.StateTransferInterceptor.handleTxWriteCommand(StateTransferInterceptor.java:258)
at org.infinispan.statetransfer.StateTransferInterceptor.handleWriteCommand(StateTransferInterceptor.java:247)
at org.infinispan.statetransfer.StateTransferInterceptor.visitPutKeyValueCommand(StateTransferInterceptor.java:108)
at org.infinispan.commands.write.PutKeyValueCommand.acceptVisitor(PutKeyValueCommand.java:71)
at org.infinispan.interceptors.base.CommandInterceptor.invokeNextInterceptor(CommandInterceptor.java:97)
at org.infinispan.interceptors.CacheMgmtInterceptor.updateStoreStatistics(CacheMgmtInterceptor.java:191)
at org.infinispan.interceptors.CacheMgmtInterceptor.visitPutKeyValueCommand(CacheMgmtInterceptor.java:177)
at org.infinispan.commands.write.PutKeyValueCommand.acceptVisitor(PutKeyValueCommand.java:71)
at org.infinispan.interceptors.base.CommandInterceptor.invokeNextInterceptor(CommandInterceptor.java:97)
at org.infinispan.interceptors.base.CommandInterceptor.handleDefault(CommandInterceptor.java:111)
at org.infinispan.commands.AbstractVisitor.visitPutKeyValueCommand(AbstractVisitor.java:44)
at org.infinispan.interceptors.IsMarshallableInterceptor.visitPutKeyValueCommand(IsMarshallableInterceptor.java:107)
at org.infinispan.commands.write.PutKeyValueCommand.acceptVisitor(PutKeyValueCommand.java:71)
at org.infinispan.interceptors.base.CommandInterceptor.invokeNextInterceptor(CommandInterceptor.java:97)
at org.infinispan.interceptors.InvocationContextInterceptor.handleAll(InvocationContextInterceptor.java:102)
at org.infinispan.interceptors.InvocationContextInterceptor.handleDefault(InvocationContextInterceptor.java:71)
at org.infinispan.commands.AbstractVisitor.visitPutKeyValueCommand(AbstractVisitor.java:44)
at org.infinispan.commands.write.PutKeyValueCommand.acceptVisitor(PutKeyValueCommand.java:71)
at org.infinispan.interceptors.base.CommandInterceptor.invokeNextInterceptor(CommandInterceptor.java:97)
at org.infinispan.interceptors.BatchingInterceptor.handleDefault(BatchingInterceptor.java:66)
at org.infinispan.commands.AbstractVisitor.visitPutKeyValueCommand(AbstractVisitor.java:44)
at org.infinispan.commands.write.PutKeyValueCommand.acceptVisitor(PutKeyValueCommand.java:71)
at org.infinispan.interceptors.InterceptorChain.invoke(InterceptorChain.java:336)
at org.infinispan.cache.impl.CacheImpl.executeCommandAndCommitIfNeeded(CacheImpl.java:1617)
at org.infinispan.cache.impl.CacheImpl.putInternal(CacheImpl.java:1097)
at org.infinispan.cache.impl.CacheImpl.put(CacheImpl.java:1089)
at org.infinispan.cache.impl.DecoratedCache.put(DecoratedCache.java:437)
at com.bmc.truesight.api.lookup.ContainerLookupCache.addCiToCiTypeCiCache(ContainerLookupCache.java:468)
at com.bmc.truesight.api.lookup.ContainerLookupCache.populateCiToServiceIfServiceCI(ContainerLookupCache.java:530)
at com.bmc.truesight.platform.etl.CI.SyncTSIMCIs.lambda$null$113(SyncTSIMCIs.java:130)
at java.util.ArrayList$ArrayListSpliterator.forEachRemaining(ArrayList.java:1382)
at java.util.stream.ReferencePipeline$Head.forEach(ReferencePipeline.java:580)
at com.bmc.truesight.platform.etl.CI.SyncTSIMCIs.lambda$processResponseMsgs$114(SyncTSIMCIs.java:125)
at java.util.stream.ForEachOps$ForEachOp$OfRef.accept(ForEachOps.java:184)
at java.util.stream.ReferencePipeline$2$1.accept(ReferencePipeline.java:175)
at java.util.ArrayList$ArrayListSpliterator.forEachRemaining(ArrayList.java:1382)
at java.util.stream.AbstractPipeline.copyInto(AbstractPipeline.java:481)
at java.util.stream.AbstractPipeline.wrapAndCopyInto(AbstractPipeline.java:471)
at java.util.stream.ForEachOps$ForEachOp.evaluateSequential(ForEachOps.java:151)
at java.util.stream.ForEachOps$ForEachOp$OfRef.evaluateSequential(ForEachOps.java:174)
at java.util.stream.AbstractPipeline.evaluate(AbstractPipeline.java:234)
at java.util.stream.ReferencePipeline.forEach(ReferencePipeline.java:418)
at com.bmc.truesight.platform.etl.CI.SyncTSIMCIs.processResponseMsgs(SyncTSIMCIs.java:118)
at com.bmc.truesight.platform.etl.CI.SyncTSIMCIs.sendSyncCIRequest(SyncTSIMCIs.java:97)
at com.bmc.truesight.platform.etl.CI.SyncTSIMCIs.syncCis(SyncTSIMCIs.java:63)
at com.bmc.truesight.platform.message.InitializeTSIMServers.syncDataFromChildServer(InitializeTSIMServers.java:184)
at com.bmc.truesight.platform.message.InitializeTSIMServers.initializeChildBPPMserver(InitializeTSIMServers.java:93)
at com.bmc.truesight.platform.etl.component.tsim.TSIMWSUtility.performTSIMUpdateOrInitilize(TSIMWSUtility.java:274)
at com.bmc.truesight.platform.etl.component.tsim.TSIMWSUtility.initializeTSIM(TSIMWSUtility.java:331)
at com.bmc.truesight.platform.etl.component.management.types.TSIMComponent.initializationNotification(TSIMComponent.java:130)
at com.bmc.truesight.platform.etl.component.management.tasks.ComponentManagementTask.runInitialization(ComponentManagementTask.java:297)
at com.bmc.truesight.platform.etl.component.management.tasks.ComponentManagementTask.process(ComponentManagementTask.java:91)
at com.bmc.truesight.api.qmgr.TSQueue.run(TSQueue.java:85)
at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:511)
at java.util.concurrent.FutureTask.run(FutureTask.java:266)
at java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.access$201(ScheduledThreadPoolExecutor.java:180)
at java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.run(ScheduledThreadPoolExecutor.java:293)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
at java.lang.Thread.run(Thread.java:748)

Our Cache.conf used is as below

<?xml version="1.0" encoding="UTF-8"?>
<infinispan
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
xsi:schemaLocation="urn:infinispan:config:7.0 http://www.infinispan.org/schemas/infinispan-config-7.0.xsd"
xmlns="urn:infinispan:config:7.0">

<jgroups>
<stack-file name="configurationFile" path="cache-replicator.conf"/>
</jgroups>
<cache-container default-cache="defaultCache">
<!--
Enables collection of global JMX statistics, and allows you to customize the JMX domain name under which
MBeans are registered.
-->

<jmx>
<property name="enabled">false</property>
<property name="jmxDomain">org.infinispan</property>
<property name="cacheManagerName">CacheManager</property>
<property name="statistics">false</property>
</jmx>

<!--
If the transport is omitted, there is no way to create distributed or clustered caches.
There is no added cost to defining a transport and not creating a cache that uses one, since the transport
is created and initialized lazily. By default, the JGroupsTransport is used.
Note that the JGroups transport uses sensible defaults if no configuration property is defined.
See:
http://community.jboss.org/wiki/ClusteredConfigurationQuickStart
http://docs.jboss.org/infinispan/5.0/apidocs/org/infinispan/remoting/transport/jgroups/JGroupsTransport.html

-->

<transport cluster="TrueSight-Cache-cluster" stack="configurationFile">
</transport>
<!--
Used to register JVM shutdown hooks.
hookBehavior: DEFAULT, REGISTER, DONT_REGISTER
-->
<!-- <shutdown hookBehavior="DONT_REGISTER"/> -->

<local-cache name="defaultCache" statistics="false">
<!-- <storeAsBinary /> -->

<!--
Tunes the locking and concurrency characteristics of the cache.
See:
http://community.jboss.org/wiki/LockingandConcurrency
-->
<locking
isolation="READ_COMMITTED"
acquire-timeout="20000"
write-skew="false"
concurrency-level="5000"
striping="false"
/>

<!--
Used to register JMX statistics in any available MBean server

<jmxStatistics enabled="false"/>-->

<!--
Used to control whether objects stored in Infinispan nodes are primarily stored as rehydrated Objects or as
raw byte arrays. See:
http://community.jboss.org/wiki/Marshalling#Lazy_Deserialization
-->
<!-- <storeAsBinary enabled="true"/> -->

<!--
Used to enable invocation batching and allow the use of Cache.startBatch()/endBatch() methods.
http://community.jboss.org/wiki/Batching

<invocationBatching enabled="true"/> -->

<transaction mode="BATCH"/>

<!--
This element specifies that the cache is clustered.
modes supported: distribution (d), replication (r) or invalidation .
http://community.jboss.org/wiki/Clusteringmodes
-->

</local-cache>
<replicated-cache name="defaultCache" mode="ASYNC" queue-size="500" queue-flush-interval="10000">

<state-transfer await-initial-transfer="false"
timeout="240000"
enabled="true"
chunk-size="10000"
/>

<!--Default write-skew is false, concurrency-level is 16 and acquire-timeout is 20 seconds in infinispan. Increasing -->
<locking concurrency-level="1000" striping="true" acquire-timeout="240000"
isolation="READ_COMMITTED" write-skew="false"/>
<!-- <sync replTimeout="20000"/> -->

<!-- async replication. -->

</replicated-cache>
</cache-container>
</infinispan>