-
Bug
-
Resolution: Done
-
Critical
-
jboss-fuse-6.1
-
%
-
-
- A user is facing 100% utilization of CPU for every profile related commands. It takes around 20 minutes to execute a command in Fabric which leads to long maintenance windows for them. They are already in latest patch with number of profiles more than > 100.
- This issue is in production environment. gitGcOnLoad is set to true
JBossFuse:admin@prod-Node2> config:proplist --pid io.fabric8.datastore service.pid = io.fabric8.datastore gitGcOnLoad = true gitPullPeriod = 1000 type = caching-git fabric.zookeeper.pid = io.fabric8.datastore JBossFuse:admin@prod-Node2>
- Thread Dumps:
"pool-106-thread-1" prio=6 tid=0x000000000c1fc000 nid=0x56e4 waiting for monitor entry [0x00000000112af000] java.lang.Thread.State: BLOCKED (on object monitor) at io.fabric8.git.internal.GitDataStore.gitOperation(GitDataStore.java:906) - waiting to lock <0x00000000c47dfd90> (a java.lang.Object) at io.fabric8.git.internal.GitDataStore.gitOperation(GitDataStore.java:901) at io.fabric8.git.internal.GitDataStore.gitOperation(GitDataStore.java:891) at io.fabric8.git.internal.GitDataStore.pull(GitDataStore.java:1251) at io.fabric8.git.internal.GitDataStore$1.run(GitDataStore.java:186) at java.util.concurrent.Executors$RunnableAdapter.call(Unknown Source) at java.util.concurrent.FutureTask.runAndReset(Unknown Source) at java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.access$301(Unknown Source) at java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.run(Unknown Source) at java.util.concurrent.ThreadPoolExecutor.runWorker(Unknown Source) at java.util.concurrent.ThreadPoolExecutor$Worker.run(Unknown Source) at java.lang.Thread.run(Unknown Source) Locked ownable synchronizers: - <0x00000000c47df7c8> (a java.util.concurrent.ThreadPoolExecutor$Worker) "pool-104-thread-1" prio=6 tid=0x000000000b920000 nid=0x53d0 runnable [0x0000000010a0d000] java.lang.Thread.State: RUNNABLE at java.io.RandomAccessFile.open(Native Method) at java.io.RandomAccessFile.<init>(Unknown Source) at org.eclipse.jgit.internal.storage.file.PackFile.doOpen(PackFile.java:600) - locked <0x00000000c1608618> (a java.lang.Object) at org.eclipse.jgit.internal.storage.file.PackFile.beginWindowCache(PackFile.java:582) - locked <0x00000000c1608438> (a org.eclipse.jgit.internal.storage.file.PackFile) at org.eclipse.jgit.internal.storage.file.WindowCache.load(WindowCache.java:284) at org.eclipse.jgit.internal.storage.file.WindowCache.getOrLoad(WindowCache.java:368) - locked <0x00000000c170a6e8> (a org.eclipse.jgit.internal.storage.file.WindowCache$Lock) at org.eclipse.jgit.internal.storage.file.WindowCache.get(WindowCache.java:179) at org.eclipse.jgit.internal.storage.file.WindowCursor.pin(WindowCursor.java:357) at org.eclipse.jgit.internal.storage.file.WindowCursor.copy(WindowCursor.java:226) at org.eclipse.jgit.internal.storage.file.PackFile.readFully(PackFile.java:555) at org.eclipse.jgit.internal.storage.file.PackFile.load(PackFile.java:713) at org.eclipse.jgit.internal.storage.file.PackFile.get(PackFile.java:256) at org.eclipse.jgit.internal.storage.file.ObjectDirectory.openObject1(ObjectDirectory.java:371) at org.eclipse.jgit.internal.storage.file.FileObjectDatabase.openObjectImpl1(FileObjectDatabase.java:173) at org.eclipse.jgit.internal.storage.file.FileObjectDatabase.openObject(FileObjectDatabase.java:158) at org.eclipse.jgit.internal.storage.file.WindowCursor.open(WindowCursor.java:145) at org.eclipse.jgit.revwalk.RevWalk.getCachedBytes(RevWalk.java:853) at org.eclipse.jgit.revwalk.RevCommit.parseHeaders(RevCommit.java:143) at org.eclipse.jgit.revwalk.RewriteTreeFilter.include(RewriteTreeFilter.java:109) at org.eclipse.jgit.revwalk.PendingGenerator.next(PendingGenerator.java:141) at org.eclipse.jgit.revwalk.BlockRevQueue.<init>(BlockRevQueue.java:65) at org.eclipse.jgit.revwalk.FIFORevQueue.<init>(FIFORevQueue.java:64) at org.eclipse.jgit.revwalk.StartGenerator.next(StartGenerator.java:157) at org.eclipse.jgit.revwalk.RevWalk.next(RevWalk.java:414) at org.gitective.core.CommitUtils.getLastCommit(CommitUtils.java:418) at io.fabric8.git.internal.CachingGitDataStore.addProfileData(CachingGitDataStore.java:162) at io.fabric8.git.internal.CachingGitDataStore.addProfileData(CachingGitDataStore.java:152) at io.fabric8.git.internal.CachingGitDataStore.addProfileData(CachingGitDataStore.java:152) at io.fabric8.git.internal.CachingGitDataStore.pouplateVersionData(CachingGitDataStore.java:132) at io.fabric8.git.internal.CachingGitDataStore$1$1.call(CachingGitDataStore.java:91) at io.fabric8.git.internal.CachingGitDataStore$1$1.call(CachingGitDataStore.java:88) at io.fabric8.git.internal.GitDataStore.gitOperation(GitDataStore.java:934) - locked <0x00000000c47dfd90> (a java.lang.Object) at io.fabric8.git.internal.GitDataStore.gitOperation(GitDataStore.java:901) at io.fabric8.git.internal.GitDataStore.gitOperation(GitDataStore.java:896) at io.fabric8.git.internal.CachingGitDataStore$1.load(CachingGitDataStore.java:88) at io.fabric8.git.internal.CachingGitDataStore$1.load(CachingGitDataStore.java:85) at com.google.common.cache.LocalCache$LoadingValueReference.loadFuture(LocalCache.java:3522) at com.google.common.cache.LocalCache$Segment.loadSync(LocalCache.java:2315) at com.google.common.cache.LocalCache$Segment.lockedGetOrLoad(LocalCache.java:2278) - locked <0x00000000c4e5a0b8> (a com.google.common.cache.LocalCache$StrongEntry) at com.google.common.cache.LocalCache$Segment.get(LocalCache.java:2193) at com.google.common.cache.LocalCache.get(LocalCache.java:3932) at com.google.common.cache.LocalCache.getOrLoad(LocalCache.java:3936) at com.google.common.cache.LocalCache$LocalLoadingCache.get(LocalCache.java:4806) at io.fabric8.git.internal.CachingGitDataStore.getVersionData(CachingGitDataStore.java:116) at io.fabric8.git.internal.CachingGitDataStore.hasProfile(CachingGitDataStore.java:187) at io.fabric8.internal.VersionImpl.getProfile(VersionImpl.java:100) at io.fabric8.internal.ContainerImpl$ContainerProfile.getParents(ContainerImpl.java:271) at io.fabric8.internal.ProfileOverlayImpl.fillParentProfiles(ProfileOverlayImpl.java:248) at io.fabric8.internal.ProfileOverlayImpl.getInheritedProfiles(ProfileOverlayImpl.java:243) at io.fabric8.internal.ProfileOverlayImpl.getConfigurations(ProfileOverlayImpl.java:346) at io.fabric8.internal.ProfileOverlayImpl.getContainerConfiguration(ProfileOverlayImpl.java:121) at io.fabric8.internal.ProfileImpl.getContainerConfigList(ProfileImpl.java:149) at io.fabric8.internal.ProfileOverlayImpl.getRepositories(ProfileOverlayImpl.java:106) at io.fabric8.features.FabricFeaturesServiceImpl.listInstalledRepositories(FabricFeaturesServiceImpl.java:327) at io.fabric8.features.FabricFeaturesServiceImpl.run(FabricFeaturesServiceImpl.java:93) - locked <0x00000000c47dde28> (a io.fabric8.features.FabricFeaturesServiceImpl) at io.fabric8.service.AbstractDataStore.doRunCallbacks(AbstractDataStore.java:222) at io.fabric8.service.AbstractDataStore$1.run(AbstractDataStore.java:210) at java.util.concurrent.Executors$RunnableAdapter.call(Unknown Source) at java.util.concurrent.FutureTask.run(Unknown Source) at java.util.concurrent.ThreadPoolExecutor.runWorker(Unknown Source) at java.util.concurrent.ThreadPoolExecutor$Worker.run(Unknown Source) at java.lang.Thread.run(Unknown Source) Locked ownable synchronizers: - <0x00000000c4810a88> (a java.util.concurrent.ThreadPoolExecutor$Worker)
- is related to
-
ENTESB-2581 Git repo very slow after adding 100s of profiles
- Closed