Japanese

Announcement

For appeals, questions and feedback about Oracle Forums, please email oracle-forums-moderators_us@oracle.com. Technical questions should be asked in the appropriate category. Thank you!

Weblogic has stuck threads causing datasources down.

1004773Oct 1 2014

My Application is deployed on Weblogic Application server.

For some time, the application processes requests properly but later it shut down the datasource telling too many open files.

Below is the thread dump observed,

===== FULL THREAD DUMP ===============

Wed Oct 1 15:51:30 2014

Oracle JRockit(R) R28.0.0-679-130297-1.6.0_17-20100312-2128-linux-ia32

"Main Thread" id=1 idx=0x4 tid=26393 prio=5 alive, waiting, native_blocked

-- Waiting for notification on: weblogic/t3/srvr/T3Srvr@0xd103f650[fat lock]

at jrockit/vm/Threads.waitForNotifySignal(JLjava/lang/Object;)Z(Native Method)

at java/lang/Object.wait(J)V(Native Method)

at java/lang/Object.wait(Object.java:485)

at weblogic/t3/srvr/T3Srvr.waitForDeath(T3Srvr.java:902)

^-- Lock released while waiting: weblogic/t3/srvr/T3Srvr@0xd103f650[fat lock]

at weblogic/t3/srvr/T3Srvr.run(T3Srvr.java:465)

at weblogic/Server.main(Server.java:67)

at jrockit/vm/RNI.c2java(IIIII)V(Native Method)

-- end of trace

"(OC Main Thread)" id=2 idx=0x8 tid=26394 prio=5 alive, native_waiting, daemon

"(GC Worker Thread 1)" id=? idx=0xc tid=26395 prio=5 alive, daemon

"(GC Worker Thread 2)" id=? idx=0x10 tid=26396 prio=5 alive, daemon

"(Code Generation Thread 1)" id=3 idx=0x14 tid=26397 prio=5 alive, native_waiting, daemon

"(Code Optimization Thread 1)" id=4 idx=0x18 tid=26398 prio=5 alive, native_waiting, daemon

"(VM Periodic Task)" id=5 idx=0x1c tid=26399 prio=10 alive, native_blocked, daemon

"(Attach Listener)" id=6 idx=0x20 tid=26400 prio=5 alive, native_blocked, daemon

"Finalizer" id=7 idx=0x24 tid=26401 prio=8 alive, native_waiting, daemon

at jrockit/memory/Finalizer.waitForFinalizees(J[Ljava/lang/Object;)I(Native Method)

at jrockit/memory/Finalizer.access$700(Finalizer.java:12)

at jrockit/memory/Finalizer$4.run(Finalizer.java:189)

at java/lang/Thread.run(Thread.java:619)

at jrockit/vm/RNI.c2java(IIIII)V(Native Method)

-- end of trace

"Reference Handler" id=8 idx=0x28 tid=26402 prio=10 alive, native_waiting, daemon

at java/lang/ref/Reference.waitForActivatedQueue(J)Ljava/lang/ref/Reference;(Native Method)

at java/lang/ref/Reference.access$100(Reference.java:11)

at java/lang/ref/Reference$ReferenceHandler.run(Reference.java:82)

at jrockit/vm/RNI.c2java(IIIII)V(Native Method)

-- end of trace

"(Sensor Event Thread)" id=9 idx=0x2c tid=26403 prio=5 alive, native_blocked, daemon

"JDWP Transport Listener: dt_socket" id=10 idx=0x30 tid=26404 prio=10 alive, in native, daemon

"JDWP Event Helper Thread" id=11 idx=0x34 tid=26405 prio=10 alive, in native, daemon

"VM JFR Buffer Thread" id=12 idx=0x38 tid=26406 prio=5 alive, in native, daemon

"Timer-0" id=15 idx=0x3c tid=26408 prio=5 alive, waiting, native_blocked, daemon

-- Waiting for notification on: java/util/TaskQueue@0xd04ed3f8[fat lock]

at jrockit/vm/Threads.waitForNotifySignal(JLjava/lang/Object;)Z(Native Method)

at java/lang/Object.wait(J)V(Native Method)

at java/lang/Object.wait(Object.java:485)

at java/util/TimerThread.mainLoop(Timer.java:483)

^-- Lock released while waiting: java/util/TaskQueue@0xd04ed3f8[fat lock]

at java/util/TimerThread.run(Timer.java:462)

at jrockit/vm/RNI.c2java(IIIII)V(Native Method)

-- end of trace

"Timer-1" id=16 idx=0x40 tid=26414 prio=5 alive, waiting, native_blocked, daemon

-- Waiting for notification on: java/util/TaskQueue@0xd04ed460[fat lock]

at jrockit/vm/Threads.waitForNotifySignal(JLjava/lang/Object;)Z(Native Method)

at java/lang/Object.wait(J)V(Native Method)

at java/util/TimerThread.mainLoop(Timer.java:509)

^-- Lock released while waiting: java/util/TaskQueue@0xd04ed460[fat lock]

at java/util/TimerThread.run(Timer.java:462)

at jrockit/vm/RNI.c2java(IIIII)V(Native Method)

-- end of trace

"[ACTIVE] ExecuteThread: '0' for queue: 'weblogic.kernel.Default (self-tuning)'" id=17 idx=0x44 tid=26415 prio=5 alive, daemon

at bea/jmapi/DiagnosticCommandImpl.execute(Ljava/lang/String;Ljava/io/FileDescriptor;)V(Native Method)

at bea/jmapi/DiagnosticCommandImpl.execute(DiagnosticCommandImpl.java:53)

at com/bea/jvm/DiagnosticCommand.execute(DiagnosticCommand.java:245)

at com/bea/jvm/DiagnosticCommand$Command.execute(DiagnosticCommand.java:404)

at bea/jmapi/ThreadSystemImpl.getThreadStackDump(ThreadSystemImpl.java:102)

at weblogic/t3/srvr/JRockitRuntime.getThreadStackDump(JRockitRuntime.java:83)

at jrockit/vm/RNI.c2java(IIIII)V(Native Method)

at jrockit/vm/Reflect.invokeMethod(Ljava/lang/Object;Ljava/lang/Object;[Ljava/lang/Object;)Ljava/lang/Object;(Native Method)

at sun/reflect/NativeMethodAccessorImpl.invoke0(Ljava/lang/reflect/Method;Ljava/lang/Object;[Ljava/lang/Object;)Ljava/lang/Object;(Native Method)

at sun/reflect/NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:39)

at sun/reflect/DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:25)

at java/lang/reflect/Method.invoke(Method.java:597)

at weblogic/management/jmx/modelmbean/WLSModelMBean.getAttribute(WLSModelMBean.java:525)

at com/sun/jmx/interceptor/DefaultMBeanServerInterceptor.getAttribute(DefaultMBeanServerInterceptor.java:666)

at com/sun/jmx/mbeanserver/JmxMBeanServer.getAttribute(JmxMBeanServer.java:638)

at weblogic/management/jmx/mbeanserver/WLSMBeanServerInterceptorBase$12.run(WLSMBeanServerInterceptorBase.java:326)

at jrockit/vm/AccessController.doPrivileged(AccessController.java:254)

at jrockit/vm/AccessController.doPrivileged(AccessController.java:268)

at weblogic/management/jmx/mbeanserver/WLSMBeanServerInterceptorBase.getAttribute(WLSMBeanServerInterceptorBase.java:324)

at weblogic/management/mbeanservers/internal/JMXContextInterceptor.getAttribute(JMXContextInterceptor.java:163)

at weblogic/management/jmx/mbeanserver/WLSMBeanServerInterceptorBase$12.run(WLSMBeanServerInterceptorBase.java:326)

at jrockit/vm/AccessController.doPrivileged(AccessController.java:254)

at jrockit/vm/AccessController.doPrivileged(AccessController.java:268)

at weblogic/management/jmx/mbeanserver/WLSMBeanServerInterceptorBase.getAttribute(WLSMBeanServerInterceptorBase.java:324)

at weblogic/management/mbeanservers/internal/SecurityInterceptor.getAttribute(SecurityInterceptor.java:299)

at weblogic/management/jmx/mbeanserver/WLSMBeanServer.getAttribute(WLSMBeanServer.java:279)

at weblogic/management/mbeanservers/internal/JMXConnectorSubjectForwarder$5$1.run(JMXConnectorSubjectForwarder.java:326)

at jrockit/vm/AccessController.doPrivileged(AccessController.java:254)

at jrockit/vm/AccessController.doPrivileged(AccessController.java:268)

at weblogic/management/mbeanservers/internal/JMXConnectorSubjectForwarder$5.run(JMXConnectorSubjectForwarder.java:324)

at weblogic/security/acl/internal/AuthenticatedSubject.doAs(AuthenticatedSubject.java:363)

at weblogic/management/mbeanservers/internal/JMXConnectorSubjectForwarder.getAttribute(JMXConnectorSubjectForwarder.java:319)

at javax/management/remote/rmi/RMIConnectionImpl.doOperation(RMIConnectionImpl.java:1403)

at javax/management/remote/rmi/RMIConnectionImpl.access$200(RMIConnectionImpl.java:72)

at javax/management/remote/rmi/RMIConnectionImpl$PrivilegedOperation.run(RMIConnectionImpl.java:1264)

at jrockit/vm/AccessController.doPrivileged(AccessController.java:254)

at javax/management/remote/rmi/RMIConnectionImpl.doPrivilegedOperation(RMIConnectionImpl.java:1366)

at javax/management/remote/rmi/RMIConnectionImpl.getAttribute(RMIConnectionImpl.java:600)

at javax/management/remote/rmi/RMIConnectionImpl_WLSkel.invoke(ILweblogic/rmi/spi/InboundRequest;Lweblogic/rmi/spi/OutboundResponse;Ljava/lang/Object;)Lweblogic/rmi/spi/OutboundResponse;(Unknown Source)

at weblogic/rmi/internal/BasicServerRef.invoke(BasicServerRef.java:589)

at weblogic/rmi/internal/BasicServerRef$1.run(BasicServerRef.java:477)

at weblogic/security/acl/internal/AuthenticatedSubject.doAs(AuthenticatedSubject.java:363)

at weblogic/security/service/SecurityManager.runAs(SecurityManager.java:147)

at weblogic/rmi/internal/BasicServerRef.handleRequest(BasicServerRef.java:473)

at weblogic/rmi/internal/wls/WLSExecuteRequest.run(WLSExecuteRequest.java:118)

at weblogic/work/ExecuteThread.execute(ExecuteThread.java:201)

at weblogic/work/ExecuteThread.run(ExecuteThread.java:173)

at jrockit/vm/RNI.c2java(IIIII)V(Native Method)

-- end of trace

"JFR request timer" id=18 idx=0x48 tid=26416 prio=5 alive, waiting, native_blocked, daemon

-- Waiting for notification on: java/util/TaskQueue@0xd04b6268[fat lock]

at jrockit/vm/Threads.waitForNotifySignal(JLjava/lang/Object;)Z(Native Method)

at java/lang/Object.wait(J)V(Native Method)

at java/lang/Object.wait(Object.java:485)

at java/util/TimerThread.mainLoop(Timer.java:483)

^-- Lock released while waiting: java/util/TaskQueue@0xd04b6268[fat lock]

at java/util/TimerThread.run(Timer.java:462)

at jrockit/vm/RNI.c2java(IIIII)V(Native Method)

-- end of trace

"weblogic.time.TimeEventGenerator" id=20 idx=0x4c tid=26417 prio=9 alive, waiting, native_blocked, daemon

-- Waiting for notification on: weblogic/time/common/internal/TimeTable@0xd04f3238[fat lock]

at jrockit/vm/Threads.waitForNotifySignal(JLjava/lang/Object;)Z(Native Method)

at java/lang/Object.wait(J)V(Native Method)

at weblogic/time/common/internal/TimeTable.snooze(TimeTable.java:286)

^-- Lock released while waiting: weblogic/time/common/internal/TimeTable@0xd04f3238[fat lock]

at weblogic/time/common/internal/TimeEventGenerator.run(TimeEventGenerator.java:117)

at java/lang/Thread.run(Thread.java:619)

at jrockit/vm/RNI.c2java(IIIII)V(Native Method)

-- end of trace

"JMAPI event thread" id=21 idx=0x50 tid=26418 prio=5 alive, in native, daemon

"weblogic.timers.TimerThread" id=22 idx=0x54 tid=26419 prio=9 alive, waiting, native_blocked, daemon

-- Waiting for notification on: weblogic/timers/internal/TimerThread@0xd0dd7600[fat lock]

at jrockit/vm/Threads.waitForNotifySignal(JLjava/lang/Object;)Z(Native Method)

at java/lang/Object.wait(J)V(Native Method)

at weblogic/timers/internal/TimerThread$Thread.run(TimerThread.java:262)

^-- Lock released while waiting: weblogic/timers/internal/TimerThread@0xd0dd7600[fat lock]

at jrockit/vm/RNI.c2java(IIIII)V(Native Method)

-- end of trace

"[STANDBY] ExecuteThread: '1' for queue: 'weblogic.kernel.Default (self-tuning)'" id=23 idx=0x58 tid=26420 prio=5 alive, waiting, native_blocked, daemon

-- Waiting for notification on: weblogic/work/ExecuteThread@0xd0d52f50[fat lock]

at jrockit/vm/Threads.waitForNotifySignal(JLjava/lang/Object;)Z(Native Method)

at java/lang/Object.wait(J)V(Native Method)

at java/lang/Object.wait(Object.java:485)

at weblogic/work/ExecuteThread.waitForRequest(ExecuteThread.java:157)

^-- Lock released while waiting: weblogic/work/ExecuteThread@0xd0d52f50[fat lock]

at weblogic/work/ExecuteThread.run(ExecuteThread.java:178)

at jrockit/vm/RNI.c2java(IIIII)V(Native Method)

-- end of trace

"ExecuteThread: '0' for queue: 'weblogic.socket.Muxer'" id=26 idx=0x5c tid=26421 prio=5 alive, blocked, native_blocked, daemon

-- Blocked trying to get lock: java/lang/String@0xd0d78d00[fat lock]

at jrockit/vm/Threads.waitForUnblockSignal()V(Native Method)

at jrockit/vm/Locks.fatLockBlockOrSpin(Locks.java:1402)

at jrockit/vm/Locks.lockFat(Locks.java:1503)

at jrockit/vm/Locks.monitorEnterSecondStageHard(Locks.java:1045)

at jrockit/vm/Locks.monitorEnterSecondStage(Locks.java:996)

at jrockit/vm/Locks.monitorEnter(Locks.java:2170)

at weblogic/socket/EPollSocketMuxer.processSockets(EPollSocketMuxer.java:153)

at weblogic/socket/SocketReaderRequest.run(SocketReaderRequest.java:29)

at weblogic/socket/SocketReaderRequest.execute(SocketReaderRequest.java:42)

at weblogic/kernel/ExecuteThread.execute(ExecuteThread.java:145)

at weblogic/kernel/ExecuteThread.run(ExecuteThread.java:117)

at jrockit/vm/RNI.c2java(IIIII)V(Native Method)

-- end of trace

"ExecuteThread: '1' for queue: 'weblogic.socket.Muxer'" id=27 idx=0x60 tid=26422 prio=5 alive, blocked, native_blocked, daemon

-- Blocked trying to get lock: java/lang/String@0xd0d78d00[fat lock]

at jrockit/vm/Threads.waitForUnblockSignal()V(Native Method)

at jrockit/vm/Locks.fatLockBlockOrSpin(Locks.java:1402)

at jrockit/vm/Locks.fatLockReacquire(Locks.java:1640)

at jrockit/vm/Locks.lockFat(Locks.java:1508)

at jrockit/vm/Locks.monitorEnterSecondStageHard(Locks.java:1045)

at jrockit/vm/Locks.monitorEnterSecondStage(Locks.java:996)

at jrockit/vm/Locks.monitorEnter(Locks.java:2170)

at weblogic/socket/EPollSocketMuxer.processSockets(EPollSocketMuxer.java:153)

at weblogic/socket/SocketReaderRequest.run(SocketReaderRequest.java:29)

at weblogic/socket/SocketReaderRequest.execute(SocketReaderRequest.java:42)

at weblogic/kernel/ExecuteThread.execute(ExecuteThread.java:145)

at weblogic/kernel/ExecuteThread.run(ExecuteThread.java:117)

at jrockit/vm/RNI.c2java(IIIII)V(Native Method)

-- end of trace

"ExecuteThread: '2' for queue: 'weblogic.socket.Muxer'" id=28 idx=0x64 tid=26423 prio=5 alive, in native, daemon

at jrockit/ext/epoll/EPoll.epollWait0(ILjava/nio/ByteBuffer;II)I(Native Method)

at jrockit/ext/epoll/EPoll.epollWait(EPoll.java:115)

at weblogic/socket/EPollSocketMuxer.processSockets(EPollSocketMuxer.java:156)

^-- Holding lock: java/lang/String@0xd0d78d00[fat lock]

at weblogic/socket/SocketReaderRequest.run(SocketReaderRequest.java:29)

at weblogic/socket/SocketReaderRequest.execute(SocketReaderRequest.java:42)

at weblogic/kernel/ExecuteThread.execute(ExecuteThread.java:145)

at weblogic/kernel/ExecuteThread.run(ExecuteThread.java:117)

at jrockit/vm/RNI.c2java(IIIII)V(Native Method)

-- end of trace

"VDE Transaction Processor Thread" id=31 idx=0x68 tid=26436 prio=2 alive, waiting, native_blocked, daemon

-- Waiting for notification on: com/octetstring/vde/backend/standard/TransactionProcessor@0xd0d53228[fat lock]

at jrockit/vm/Threads.waitForNotifySignal(JLjava/lang/Object;)Z(Native Method)

at java/lang/Object.wait(J)V(Native Method)

at java/lang/Object.wait(Object.java:485)

at com/octetstring/vde/backend/standard/TransactionProcessor.waitChange(TransactionProcessor.java:367)

^-- Lock released while waiting: com/octetstring/vde/backend/standard/TransactionProcessor@0xd0d53228[fat lock]

at com/octetstring/vde/backend/standard/TransactionProcessor.run(TransactionProcessor.java:212)

at jrockit/vm/RNI.c2java(IIIII)V(Native Method)

-- end of trace

"DoSManager" id=34 idx=0x70 tid=26438 prio=6 alive, sleeping, native_waiting, daemon

at java/lang/Thread.sleep(J)V(Native Method)

at com/octetstring/vde/DoSManager.run(DoSManager.java:433)

at jrockit/vm/RNI.c2java(IIIII)V(Native Method)

-- end of trace

"[ACTIVE] ExecuteThread: '2' for queue: 'weblogic.kernel.Default (self-tuning)'" id=35 idx=0x74 tid=26440 prio=5 alive, waiting, native_blocked, daemon

-- Waiting for notification on: weblogic/work/ExecuteThread@0xd0d53028[fat lock]

at jrockit/vm/Threads.waitForNotifySignal(JLjava/lang/Object;)Z(Native Method)

at java/lang/Object.wait(J)V(Native Method)

at java/lang/Object.wait(Object.java:485)

at weblogic/work/ExecuteThread.waitForRequest(ExecuteThread.java:157)

^-- Lock released while waiting: weblogic/work/ExecuteThread@0xd0d53028[fat lock]

at weblogic/work/ExecuteThread.run(ExecuteThread.java:178)

at jrockit/vm/RNI.c2java(IIIII)V(Native Method)

-- end of trace

"[STANDBY] ExecuteThread: '3' for queue: 'weblogic.kernel.Default (self-tuning)'" id=36 idx=0x78 tid=26441 prio=5 alive, waiting, native_blocked, daemon

-- Waiting for notification on: weblogic/work/ExecuteThread@0xd0d53100[fat lock]

at jrockit/vm/Threads.waitForNotifySignal(JLjava/lang/Object;)Z(Native Method)

at java/lang/Object.wait(J)V(Native Method)

at java/lang/Object.wait(Object.java:485)

at weblogic/work/ExecuteThread.waitForRequest(ExecuteThread.java:157)

^-- Lock released while waiting: weblogic/work/ExecuteThread@0xd0d53100[fat lock]

at weblogic/work/ExecuteThread.run(ExecuteThread.java:178)

at jrockit/vm/RNI.c2java(IIIII)V(Native Method)

-- end of trace

"Thread-12" id=37 idx=0x7c tid=26442 prio=5 alive, parked, native_blocked, daemon

-- Parking to wait for: java/util/concurrent/locks/AbstractQueuedSynchronizer$ConditionObject@0xd0d54fd0

at jrockit/vm/Locks.park0(J)V(Native Method)

at jrockit/vm/Locks.park(Locks.java:2221)

at sun/misc/Unsafe.park(ZJ)V(Native Method)

at java/util/concurrent/locks/LockSupport.park(LockSupport.java:158)

at java/util/concurrent/locks/AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1925)

at java/util/concurrent/LinkedBlockingQueue.take(LinkedBlockingQueue.java:358)

at weblogic/utils/concurrent/JDK15ConcurrentBlockingQueue.take(JDK15ConcurrentBlockingQueue.java:89)

at weblogic/store/internal/PersistentStoreImpl.getOutstandingWork(PersistentStoreImpl.java:650)

at weblogic/store/internal/PersistentStoreImpl.run(PersistentStoreImpl.java:707)

at weblogic/store/internal/PersistentStoreImpl$2.run(PersistentStoreImpl.java:464)

at jrockit/vm/RNI.c2java(IIIII)V(Native Method)

-- end of trace

"WebSphere MQ Trace Monitor" id=38 idx=0x80 tid=26444 prio=5 alive, sleeping, native_waiting, daemon

at java/lang/Thread.sleep(J)V(Native Method)

at com/ibm/mq/commonservices/internal/utils/TraceMonitor.run(TraceMonitor.java:123)

at jrockit/vm/RNI.c2java(IIIII)V(Native Method)

-- end of trace

"Timer-2" id=39 idx=0x84 tid=26445 prio=5 alive, waiting, native_blocked, daemon

-- Waiting for notification on: java/util/TaskQueue@0xd3372c88[fat lock]

at jrockit/vm/Threads.waitForNotifySignal(JLjava/lang/Object;)Z(Native Method)

at java/lang/Object.wait(J)V(Native Method)

at java/util/TimerThread.mainLoop(Timer.java:509)

^-- Lock released while waiting: java/util/TaskQueue@0xd3372c88[fat lock]

at java/util/TimerThread.run(Timer.java:462)

at jrockit/vm/RNI.c2java(IIIII)V(Native Method)

-- end of trace

"Timer-3" id=40 idx=0x88 tid=26446 prio=5 alive, waiting, native_blocked, daemon

-- Waiting for notification on: java/util/TaskQueue@0xd33d35a0[fat lock]

at jrockit/vm/Threads.waitForNotifySignal(JLjava/lang/Object;)Z(Native Method)

at java/lang/Object.wait(J)V(Native Method)

at java/util/TimerThread.mainLoop(Timer.java:509)

^-- Lock released while waiting: java/util/TaskQueue@0xd33d35a0[fat lock]

at java/util/TimerThread.run(Timer.java:462)

at jrockit/vm/RNI.c2java(IIIII)V(Native Method)

-- end of trace

"Timer-4" id=41 idx=0x8c tid=26447 prio=5 alive, waiting, native_blocked, daemon

-- Waiting for notification on: java/util/TaskQueue@0xd33d3bc8[fat lock]

at jrockit/vm/Threads.waitForNotifySignal(JLjava/lang/Object;)Z(Native Method)

at java/lang/Object.wait(J)V(Native Method)

at java/util/TimerThread.mainLoop(Timer.java:509)

^-- Lock released while waiting: java/util/TaskQueue@0xd33d3bc8[fat lock]

at java/util/TimerThread.run(Timer.java:462)

at jrockit/vm/RNI.c2java(IIIII)V(Native Method)

-- end of trace

"Timer-5" id=42 idx=0x90 tid=26448 prio=5 alive, waiting, native_blocked, daemon

-- Waiting for notification on: java/util/TaskQueue@0xd3319a60[fat lock]

at jrockit/vm/Threads.waitForNotifySignal(JLjava/lang/Object;)Z(Native Method)

at java/lang/Object.wait(J)V(Native Method)

at java/util/TimerThread.mainLoop(Timer.java:509)

^-- Lock released while waiting: java/util/TaskQueue@0xd3319a60[fat lock]

at java/util/TimerThread.run(Timer.java:462)

at jrockit/vm/RNI.c2java(IIIII)V(Native Method)

-- end of trace

"Timer-6" id=43 idx=0x94 tid=26457 prio=5 alive, waiting, native_blocked, daemon

-- Waiting for notification on: java/util/TaskQueue@0xb78cee20[fat lock]

at jrockit/vm/Threads.waitForNotifySignal(JLjava/lang/Object;)Z(Native Method)

at java/lang/Object.wait(J)V(Native Method)

at java/util/TimerThread.mainLoop(Timer.java:509)

^-- Lock released while waiting: java/util/TaskQueue@0xb78cee20[fat lock]

at java/util/TimerThread.run(Timer.java:462)

at jrockit/vm/RNI.c2java(IIIII)V(Native Method)

-- end of trace

"Timer-7" id=44 idx=0x98 tid=26458 prio=5 alive, waiting, native_blocked, daemon

-- Waiting for notification on: java/util/TaskQueue@0xb78cfa28[fat lock]

at jrockit/vm/Threads.waitForNotifySignal(JLjava/lang/Object;)Z(Native Method)

at java/lang/Object.wait(J)V(Native Method)

at java/util/TimerThread.mainLoop(Timer.java:509)

^-- Lock released while waiting: java/util/TaskQueue@0xb78cfa28[fat lock]

at java/util/TimerThread.run(Timer.java:462)

at jrockit/vm/RNI.c2java(IIIII)V(Native Method)

-- end of trace

"Timer-8" id=45 idx=0x9c tid=26459 prio=5 alive, waiting, native_blocked, daemon

-- Waiting for notification on: java/util/TaskQueue@0xb78d0640[fat lock]

at jrockit/vm/Threads.waitForNotifySignal(JLjava/lang/Object;)Z(Native Method)

at java/lang/Object.wait(J)V(Native Method)

at java/util/TimerThread.mainLoop(Timer.java:509)

^-- Lock released while waiting: java/util/TaskQueue@0xb78d0640[fat lock]

at java/util/TimerThread.run(Timer.java:462)

at jrockit/vm/RNI.c2java(IIIII)V(Native Method)

-- end of trace

"Timer-9" id=46 idx=0xa0 tid=26460 prio=5 alive, waiting, native_blocked, daemon

-- Waiting for notification on: java/util/TaskQueue@0xb78e05e8[fat lock]

at jrockit/vm/Threads.waitForNotifySignal(JLjava/lang/Object;)Z(Native Method)

at java/lang/Object.wait(J)V(Native Method)

at java/util/TimerThread.mainLoop(Timer.java:509)

^-- Lock released while waiting: java/util/TaskQueue@0xb78e05e8[fat lock]

at java/util/TimerThread.run(Timer.java:462)

at jrockit/vm/RNI.c2java(IIIII)V(Native Method)

-- end of trace

"pool-2-thread-1" id=47 idx=0xa4 tid=26461 prio=5 alive, parked, native_blocked

at jrockit/vm/Locks.park0(J)V(Native Method)

at jrockit/vm/Locks.park(Locks.java:2221)

at sun/misc/Unsafe.park(ZJ)V(Native Method)

at java/util/concurrent/locks/LockSupport.parkNanos(LockSupport.java:198)

at java/util/concurrent/locks/AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1963)

at java/util/concurrent/DelayQueue.take(DelayQueue.java:164)

at java/util/concurrent/ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:583)

at java/util/concurrent/ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:576)

at java/util/concurrent/ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:947)

at java/util/concurrent/ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:907)

at java/lang/Thread.run(Thread.java:619)

at jrockit/vm/RNI.c2java(IIIII)V(Native Method)

-- end of trace

"Timer-10" id=48 idx=0xa8 tid=26462 prio=5 alive, waiting, native_blocked, daemon

-- Waiting for notification on: java/util/TaskQueue@0xb78e2ab8[fat lock]

at jrockit/vm/Threads.waitForNotifySignal(JLjava/lang/Object;)Z(Native Method)

at java/lang/Object.wait(J)V(Native Method)

at java/util/TimerThread.mainLoop(Timer.java:509)

^-- Lock released while waiting: java/util/TaskQueue@0xb78e2ab8[fat lock]

at java/util/TimerThread.run(Timer.java:462)

at jrockit/vm/RNI.c2java(IIIII)V(Native Method)

-- end of trace

"Timer-11" id=49 idx=0xac tid=26463 prio=5 alive, waiting, native_blocked, daemon

-- Waiting for notification on: java/util/TaskQueue@0xb63fa980[fat lock]

at jrockit/vm/Threads.waitForNotifySignal(JLjava/lang/Object;)Z(Native Method)

at java/lang/Object.wait(J)V(Native Method)

at java/util/TimerThread.mainLoop(Timer.java:509)

^-- Lock released while waiting: java/util/TaskQueue@0xb63fa980[fat lock]

at java/util/TimerThread.run(Timer.java:462)

at jrockit/vm/RNI.c2java(IIIII)V(Native Method)

-- end of trace

"Timer-12" id=50 idx=0xb0 tid=26464 prio=5 alive, waiting, native_blocked, daemon

-- Waiting for notification on: java/util/TaskQueue@0xb63fb560[fat lock]

at jrockit/vm/Threads.waitForNotifySignal(JLjava/lang/Object;)Z(Native Method)

at java/lang/Object.wait(J)V(Native Method)

at java/util/TimerThread.mainLoop(Timer.java:509)

^-- Lock released while waiting: java/util/TaskQueue@0xb63fb560[fat lock]

at java/util/TimerThread.run(Timer.java:462)

at jrockit/vm/RNI.c2java(IIIII)V(Native Method)

-- end of trace

"pool-4-thread-1" id=51 idx=0xb4 tid=26465 prio=5 alive, parked, native_blocked

at jrockit/vm/Locks.park0(J)V(Native Method)

at jrockit/vm/Locks.park(Locks.java:2221)

at sun/misc/Unsafe.park(ZJ)V(Native Method)

at java/util/concurrent/locks/LockSupport.parkNanos(LockSupport.java:198)

at java/util/concurrent/locks/AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1963)

at java/util/concurrent/DelayQueue.take(DelayQueue.java:164)

at java/util/concurrent/ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:583)

at java/util/concurrent/ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:576)

at java/util/concurrent/ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:947)

at java/util/concurrent/ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:907)

at java/lang/Thread.run(Thread.java:619)

at jrockit/vm/RNI.c2java(IIIII)V(Native Method)

-- end of trace

"Thread-25" id=52 idx=0xb8 tid=26466 prio=5 alive, parked, native_blocked, daemon

-- Parking to wait for: java/util/concurrent/locks/AbstractQueuedSynchronizer$ConditionObject@0xb63fdcb8

at jrockit/vm/Locks.park0(J)V(Native Method)

at jrockit/vm/Locks.park(Locks.java:2221)

at sun/misc/Unsafe.park(ZJ)V(Native Method)

at java/util/concurrent/locks/LockSupport.park(LockSupport.java:158)

at java/util/concurrent/locks/AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1925)

at java/util/concurrent/LinkedBlockingQueue.take(LinkedBlockingQueue.java:358)

at weblogic/utils/concurrent/JDK15ConcurrentBlockingQueue.take(JDK15ConcurrentBlockingQueue.java:89)

at weblogic/store/internal/PersistentStoreImpl.getOutstandingWork(PersistentStoreImpl.java:650)

at weblogic/store/internal/PersistentStoreImpl.run(PersistentStoreImpl.java:707)

at weblogic/store/internal/PersistentStoreImpl$2.run(PersistentStoreImpl.java:464)

at jrockit/vm/RNI.c2java(IIIII)V(Native Method)

-- end of trace

"DynamicListenThread[Default]" id=53 idx=0xbc tid=26467 prio=9 alive, in native, daemon

at java/net/PlainSocketImpl.socketAccept(Ljava/net/SocketImpl;)V(Native Method)

at java/net/PlainSocketImpl.accept(PlainSocketImpl.java:390)

^-- Holding lock: java/net/SocksSocketImpl@0xb6401fa0[thin lock]

at java/net/ServerSocket.implAccept(ServerSocket.java:453)

at java/net/ServerSocket.accept(ServerSocket.java:421)

at weblogic/socket/WeblogicServerSocket.accept(WeblogicServerSocket.java:38)

at weblogic/server/channels/DynamicListenThread$SocketAccepter.accept(DynamicListenThread.java:535)

at weblogic/server/channels/DynamicListenThread$SocketAccepter.access$200(DynamicListenThread.java:420)

at weblogic/server/channels/DynamicListenThread.run(DynamicListenThread.java:171)

at java/lang/Thread.run(Thread.java:619)

at jrockit/vm/RNI.c2java(IIIII)V(Native Method)

-- end of trace

"JMSCCThreadPoolMaster" id=54 idx=0xc0 tid=26474 prio=5 alive, waiting, native_blocked, daemon

-- Waiting for notification on: java/lang/Object@0xb64030b8[fat lock]

at jrockit/vm/Threads.waitForNotifySignal(JLjava/lang/Object;)Z(Native Method)

at java/lang/Object.wait(J)V(Native Method)

at java/lang/Object.wait(Object.java:485)

at com/ibm/msg/client/commonservices/j2se/workqueue/WorkQueueManagerImplementation$WorkQueueManagerThread.waitForNotification(WorkQueueManagerImplementation.java:647)

^-- Lock released while waiting: java/lang/Object@0xb64030b8[fat lock]

at com/ibm/msg/client/commonservices/j2se/workqueue/WorkQueueManagerImplementation$WorkQueueManagerThread.waitForNotification(WorkQueueManagerImplementation.java:618)

at com/ibm/msg/client/commonservices/j2se/workqueue/WorkQueueManagerImplementation$WorkQueueManagerThread.run(WorkQueueManagerImplementation.java:882)

at jrockit/vm/RNI.c2java(IIIII)V(Native Method)

-- end of trace

"RcvThread: com.ibm.mq.jmqi.remote.internal.RemoteTCPConnection[qmid=CANNED_DATA,fap=9,peer=/10.180.5.105,localport=11101,ssl=no]" id=55 idx=0xc4 tid=26475 prio=5 alive, in native, daemon

at jrockit/net/SocketNativeIO.readBytesPinned(Ljava/io/FileDescriptor;[BIII)I(Native Method)

at jrockit/net/SocketNativeIO.socketRead(SocketNativeIO.java:32)

at java/net/SocketInputStream.socketRead0(Ljava/io/FileDescriptor;[BIII)I(SocketInputStream.java)

at java/net/SocketInputStream.read(SocketInputStream.java:129)

at com/ibm/mq/jmqi/remote/internal/RemoteTCPConnection.receive(RemoteTCPConnection.java:1355)

at com/ibm/mq/jmqi/remote/internal/RemoteRcvThread.receiveBuffer(RemoteRcvThread.java:617)

at com/ibm/mq/jmqi/remote/internal/RemoteRcvThread.receiveOneTSH(RemoteRcvThread.java:580)

at com/ibm/mq/jmqi/remote/internal/RemoteRcvThread.run(RemoteRcvThread.java:139)

at com/ibm/msg/client/commonservices/workqueue/WorkQueueItem.runTask(WorkQueueItem.java:209)

at com/ibm/msg/client/commonservices/workqueue/SimpleWorkQueueItem.runItem(SimpleWorkQueueItem.java:100)

at com/ibm/msg/client/commonservices/workqueue/WorkQueueItem.run(WorkQueueItem.java:224)

at com/ibm/msg/client/commonservices/workqueue/WorkQueueManager.runWorkQueueItem(WorkQueueManager.java:296)

at com/ibm/msg/client/commonservices/j2se/workqueue/WorkQueueManagerImplementation$ThreadPoolWorker.run(WorkQueueManagerImplementation.java:1190)

at jrockit/vm/RNI.c2java(IIIII)V(Native Method)

-- end of trace

"RcvThread: com.ibm.mq.jmqi.remote.internal.RemoteTCPConnection[qmid=CANNED_DATA,fap=9,peer=/10.180.5.105,localport=37244,ssl=no]" id=56 idx=0xc8 tid=26487 prio=5 alive, in native, daemon