Java HotSpot Virtual Machine

Hi,
Our application is a muti-threaded application which is polling the SNMP servers. We have used the third-party adventnet API for polling mechanism.

But after running for 12 to 24 hours application gets stuck.
On analyzing the thread dump we found more than 2500 threads stuck at "waiting on condition" inside the API called. At the beginning the no. of thread in "waiting on condition" state are around 600 but over a period of time they keep on growing.

Note that there is no condition in the code on which the threads have been made to wait explicitly.
Similar bug has been reported (JVM Bug# 6401751) but it was closed mentioning it as not reproducible. But we are able reproduce this in our production environment (after 12 to 24 hours).

It was mentioned in one of the post (http://forums.sun.com/thread.jspa?messageID=4368296) that "waiting on condition" means that JVM is waiting form something.
Can it be an application issue?

Below is an excerpt from the thread dumps & pstack. We have tried to associate stack in thread dump & pstack. The stacks are for 8 threads (two of each type):
===========================
1)
===========================
----------------- lwp# 6921 / thread# 6921 --------------------
ffffffff7eed3890 lwp_mutex_timedlock (10011be40, 0)
ffffffff7e25b4d8 __1cHMonitorEwait6Mil_i_ (ffffffff7e78e210, 120660740, ffffffff7e78ff94, ffffffff7e7943fc, 9800, 9ba8) + 328
ffffffff7e2b5f08 __1cIVMThreadHexecute6FpnMVM_Operation__v_ (ffffffff7e78e2f0, 0, a000, a2e8, c558, 3fea88) + 228
ffffffff7e30929c __1cODeoptimizationNuncommon_trap6FpnKJavaThread_i_pn0ALUnrollBlock__ (124da4140, fffffffffffffffc, fffffffffffffffc, ff000000, 100, 0) + 874
ffffffff7606c940 ???????? (fffffffffffffffc, 3, fffffffe365cc558, 11, 6, 817fe2)
ffffffff764eea1c ???????? (fffffffe365c17e8, ffffffff724c0848, fffffffe365c1ee8, fffffffe365c1820, ffffffff71c63190, fffffffa910feec0)
ffffffff76425dbc ???????? (1, fffffffe365c1740, fffffffe71c39eb8, ffffffff71c63190, ffffffff71c62ed8, ffffffff72526698)
ffffffff7656cf98 ???????? (ffffffff71c0c6e0, fffffffe365c1740, ffffffff7251d5e8, 0, 1, ffffffffffffffff)
ffffffff764b26c4 ???????? (ffffffff71c63190, fffffffe365c1740, 0, 0, fffffffe365d3608, 0)
ffffffff764b783c ???????? (ffffffff71c63190, fffffffe8bd22110, fffffffe8efdd240, fffffffe365d3608, 1, ff7ffffdf1c72000)
ffffffff764933f4 ???????? (ffffffff71c4ed78, fffffffe365c1740, ffffffff7298ee48, ffffffff71c00698, fffffffe365c2158, ff7ffffdf1c72000)
ffffffff7662574c ???????? (fffffffe8eed2c68, fffffffe8a238e88, ffffffff71c00698, ffffffff7298ee48, fffffffe8bd22110, 1)
ffffffff764b979c ???????? (ffffffff71c63190, 1, fffffffe8ebff7b8, fffffffe06d82648, fffffffe7418eee8, fffffffe7419d368)
ffffffff76528fc0 ???????? (ffffffff71c63190, ff7ffffdf1c72000, fffffffe353d4a50, fffffffe86af0e48, ffffffff71f24998, 0)
ffffffff765a1158 ???????? (dbba0, fffffffea4df17b0, 0, 0, 0, 0)
ffffffff76087868 ???????? (fffffffdff1b23c0, ffffffff71c2e470, fffffffa910ff7b8, ff000000, ffffffff7e1e6908, 0)
ffffffff76005c80 ???????? (fffffffa910ff7b8, ffffffff7e1e9008, 0, ffffffff760175d0, 8, fffffffa910ff610)
ffffffff7600022c ???????? (fffffffa910ff8b0, fffffffa910ffd70, a, ffffffff71c2f298, 8, fffffffa910ff700)
ffffffff7e1e8c50 __1cJJavaCallsLcall_helper6FpnJJavaValue_pnMmethodHandle_pnRJavaCallArguments_pnGThread__v_ (fffffffa910ffd70, ffffffff760001e0, fffffffa910ffbf0, 124da4140, 124da4140, 1) + 180
ffffffff7e1e8d50 __1cJJavaCallsEcall6FpnJJavaValue_nMmethodHandle_pnRJavaCallArguments_pnGThread__v_ (fffffffa910ffd68, 129a501a8, fffffffa910ffbf0, 124da4140, ffffffff7e7a81f0, ffffffff7e7a8318) + 68
ffffffff7e2ba68c __1cJJavaCallsMcall_virtual6FpnJJavaValue_nLKlassHandle_nMsymbolHandle_4pnRJavaCallArguments_pnGThread__v_ (fffffffa910ffd68, 129a50188, ffffffff7e7a81f0, ffffffff7e7a8318, fffffffa910ffbf0, 124da4140) + e0
ffffffff7e2c2c9c __1cJJavaCallsMcall_virtual6FpnJJavaValue_nGHandle_nLKlassHandle_nMsymbolHandle_5pnGThread__v_ (fffffffa910ffd68, 129a50180, 129a50188, ffffffff7e7a81f0, ffffffff7e7a8318, 124da4140) + 44
ffffffff7e2dccfc __1cMthread_entry6FpnKJavaThread_pnGThread__v_ (124da4140, 124da4140, 12315d310, 10b023a50, ffffffff7e2d9414, 1000) + fc
ffffffff7e2d943c __1cKJavaThreadRthread_main_inner6M_v_ (124da4140, 40, ffffffff7eff7f00, 0, ffffffff7eff8f58, fffffffbed5ab200) + 3c
ffffffff7e515e2c java_start (124da4140, 100000, 0, 0, ffffffff7e515d40, 1) + ec
ffffffff7eed25f8 lwpstart (0, 0, 0, 0, 0, 0)

"Device Poller Thread" prio=5 tid=0x0000000124da4140 nid=0x1b09 waiting on condition [0xfffffffa910fe000..0xfffffffa910ff8b0]
at com.adventnet.snmp.snmp2.ASNTypes.encodeLength(ASNTypes.java:105)
at com.adventnet.snmp.snmp2.ASNTypes.encodeOid(ASNTypes.java:284)
at com.adventnet.snmp.snmp2.SnmpOID.encode(SnmpOID.java:65)
at com.adventnet.snmp.snmp2.SnmpVarBind.encode(SnmpVarBind.java:148)
at com.adventnet.snmp.snmp2.ASN1Parser.encodeVarBindList(ASN1Parser.java:443)
at com.adventnet.snmp.snmp2.SnmpPDU.encode(SnmpPDU.java:1040)
at com.adventnet.snmp.snmp2.SnmpSession.send(SnmpSession.java:899)
at com.adventnet.snmp.snmp2.SnmpSession.syncSend(SnmpSession.java:995)
at ISM.ISM_178_1_0.SNMP.SparseBulkHandler.handGetRequestPDU(SparseBulkHandler.java:263)
at ISM.ISM_178_1_0.SNMP.SparseBulkHandler.getSelectedRows(SparseBulkHandler.java:250)
at ISM.ISM_178_1_0.TableInfo.getTableLeavesObjects(TableInfo.java:248)
at ISM.ISM_178_1_0.DevicePoller.pollDevice(DevicePoller.java:379)
at ISM.ISM_178_1_0.DevicePoller.run(DevicePoller.java:168)
at EDU.oswego.cs.dl.util.concurrent.PooledExecutor$Worker.run(PooledExecutor.java:748)
at java.lang.Thread.run(Unknown Source)
=============================
----------------- lwp# 6920 / thread# 6920 --------------------
ffffffff7eed3890 lwp_mutex_timedlock (10011be40, 0)
ffffffff7e25b4d8 __1cHMonitorEwait6Mil_i_ (ffffffff7e78e210, 119b8cf50, ffffffff7e78ff94, ffffffff7e7943fc, 9800, 9ba8) + 328
ffffffff7e2b5f08 __1cIVMThreadHexecute6FpnMVM_Operation__v_ (ffffffff7e78e2f0, 0, a000, a2e8, c558, 3fea88) + 228
ffffffff7e30929c __1cODeoptimizationNuncommon_trap6FpnKJavaThread_i_pn0ALUnrollBlock__ (11f903d40, fffffffffffffffc, fffffffffffffffc, ff000000, 100, 0) + 874
ffffffff7606c940 ???????? (fffffffffffffffc, 3, fffffffe308236d0, 11, 6, 817fe2)
ffffffff764eea1c ???????? (fffffffe30818960, ffffffff724c0848, fffffffe30819060, fffffffe30818998, ffffffff71c63190, fffffffa912feec0)
ffffffff76425dbc ???????? (1, fffffffe308188b8, fffffffe71c39eb8, ffffffff71c63190, ffffffff71c62ed8, ffffffff72526698)
ffffffff7656cf98 ???????? (ffffffff71c0c6e0, fffffffe308188b8, ffffffff7251d5e8, 0, 1, ffffffffffffffff)
ffffffff764b26c4 ???????? (ffffffff71c63190, fffffffe308188b8, 0, 0, fffffffe3082a780, 0)
ffffffff764b783c ???????? (ffffffff71c63190, fffffffe8c416d58, fffffffe8eff4718, fffffffe3082a780, 1, ff7ffffdf1c72000)
ffffffff764933f4 ???????? (ffffffff71c4ed78, fffffffe308188b8, ffffffff7298ee48, ffffffff71c00698, fffffffe308192d0, ff7ffffdf1c72000)
ffffffff7662574c ???????? (fffffffe8eed9040, fffffffe8a6d8c28, ffffffff71c00698, ffffffff7298ee48, fffffffe8c416d58, 1)
ffffffff764b979c ???????? (ffffffff71c63190, 1, fffffffe8ec02110, fffffffe04ea05d0, fffffffe7418eee8, fffffffe7419d368)
ffffffff76528fc0 ???????? (ffffffff71c63190, ff7ffffdf1c72000, fffffffe30fedd78, fffffffe86af0e48, ffffffff71f24998, 0)
ffffffff765a1158 ???????? (dbba0, fffffffea4df1840, 0, 0, 0, 0)
ffffffff76087868 ???????? (fffffffdff19e470, ffffffff71c2e470, fffffffa912ff7b8, ff000000, ffffffff7e1e6908, 0)
ffffffff76005c80 ???????? (fffffffa912ff7b8, ffffffff7e1e9008, 0, ffffffff760175d0, 8, fffffffa912ff610)
ffffffff7600022c ???????? (fffffffa912ff8b0, fffffffa912ffd70, a, ffffffff71c2f298, 8, fffffffa912ff700)
ffffffff7e1e8c50 __1cJJavaCallsLcall_helper6FpnJJavaValue_pnMmethodHandle_pnRJavaCallArguments_pnGThread__v_ (fffffffa912ffd70, ffffffff760001e0, fffffffa912ffbf0, 11f903d40, 11f903d40, 1) + 180
ffffffff7e1e8d50 __1cJJavaCallsEcall6FpnJJavaValue_nMmethodHandle_pnRJavaCallArguments_pnGThread__v_ (fffffffa912ffd68, 1023f4688, fffffffa912ffbf0, 11f903d40, ffffffff7e7a81f0, ffffffff7e7a8318) + 68
ffffffff7e2ba68c __1cJJavaCallsMcall_virtual6FpnJJavaValue_nLKlassHandle_nMsymbolHandle_4pnRJavaCallArguments_pnGThread__v_ (fffffffa912ffd68, 1023f4668, ffffffff7e7a81f0, ffffffff7e7a8318, fffffffa912ffbf0, 11f903d40) + e0
ffffffff7e2c2c9c __1cJJavaCallsMcall_virtual6FpnJJavaValue_nGHandle_nLKlassHandle_nMsymbolHandle_5pnGThread__v_ (fffffffa912ffd68, 1023f4660, 1023f4668, ffffffff7e7a81f0, ffffffff7e7a8318, 11f903d40) + 44
ffffffff7e2dccfc __1cMthread_entry6FpnKJavaThread_pnGThread__v_ (11f903d40, 11f903d40, 11a041160, 11904bbf0, ffffffff7e2d9414, 1000) + fc
ffffffff7e2d943c __1cKJavaThreadRthread_main_inner6M_v_ (11f903d40, 40, ffffffff7eff7f00, 0, ffffffff7eff8f58, fffffffbed5a2a00) + 3c
ffffffff7e515e2c java_start (11f903d40, 100000, 0, 0, ffffffff7e515d40, 1) + ec
ffffffff7eed25f8 lwpstart (0, 0, 0, 0, 0, 0)

"Device Poller Thread" prio=5 tid=0x000000011f903d40 nid=0x1b08 waiting on condition [0xfffffffa912fe000..0xfffffffa912ff8b0]
at com.adventnet.snmp.snmp2.ASNTypes.encodeLength(ASNTypes.java:105)
at com.adventnet.snmp.snmp2.ASNTypes.encodeOid(ASNTypes.java:284)
at com.adventnet.snmp.snmp2.SnmpOID.encode(SnmpOID.java:65)
at com.adventnet.snmp.snmp2.SnmpVarBind.encode(SnmpVarBind.java:148)
at com.adventnet.snmp.snmp2.ASN1Parser.encodeVarBindList(ASN1Parser.java:443)
at com.adventnet.snmp.snmp2.SnmpPDU.encode(SnmpPDU.java:1040)
at com.adventnet.snmp.snmp2.SnmpSession.send(SnmpSession.java:899)
at com.adventnet.snmp.snmp2.SnmpSession.syncSend(SnmpSession.java:995)
at ISM.ISM_178_1_0.SNMP.SparseBulkHandler.handGetRequestPDU(SparseBulkHandler.java:263)
at ISM.ISM_178_1_0.SNMP.SparseBulkHandler.getSelectedRows(SparseBulkHandler.java:250)
at ISM.ISM_178_1_0.TableInfo.getTableLeavesObjects(TableInfo.java:248)
at ISM.ISM_178_1_0.DevicePoller.pollDevice(DevicePoller.java:379)
at ISM.ISM_178_1_0.DevicePoller.run(DevicePoller.java:168)
at EDU.oswego.cs.dl.util.concurrent.PooledExecutor$Worker.run(PooledExecutor.java:748)
at java.lang.Thread.run(Unknown Source)
========================
2
========================
----------------- lwp# 6907 / thread# 6907 --------------------
ffffffff7eed3890 lwp_mutex_timedlock (10011be40, 0)
ffffffff7e25b4d8 __1cHMonitorEwait6Mil_i_ (ffffffff7e78e210, 123149ca0, ffffffff7e78ff94, ffffffff7e7943fc, 9800, 9ba8) + 328
ffffffff7e2b5f08 __1cIVMThreadHexecute6FpnMVM_Operation__v_ (ffffffff7e78e2f0, 0, a000, a2e8, c558, 3fea88) + 228
ffffffff7e30929c __1cODeoptimizationNuncommon_trap6FpnKJavaThread_i_pn0ALUnrollBlock__ (124400160, fffffffffffffffc, fffffffffffffffc, ffffffff71c00698, fffffffe0f57bd50, ff7ffffdf1c72000) + 874
ffffffff7606c940 ???????? (fffffffffffffffc, 124400160, 124400160, ff000000, 100, 0)
ffffffff76479654 ???????? (ffffffff71c63190, ffffffffffffffff, fffffffe0f57c6f0, 10, 6, 817fe2)
ffffffff764ee7b4 ???????? (fffffffe3e550830, ffffffff724c0848, fffffffe3e550868, fffffffe3e5503c0, ffffffff71c63190, fffffffde6dfeec0)
ffffffff76425dbc ???????? (1, fffffffe3e550890, fffffffe71c39eb8, ffffffff71c63190, ffffffff71c62ed8, ffffffff72526698)
ffffffff7656cf98 ???????? (ffffffff71c0c6e0, fffffffe3e550890, ffffffff7251d5e8, 0, 1, ffffffffffffffff)
ffffffff764b26c4 ???????? (ffffffff71c63190, fffffffe3e550890, 0, 0, fffffffe0f583a68, 0)
ffffffff764b783c ???????? (ffffffff71c63190, fffffffe889cf2f0, fffffffe8ecf7158, fffffffe0f583a68, 1, ff7ffffdf1c72000)
ffffffff764933f4 ???????? (ffffffff71c4ed78, fffffffe0f57b338, ffffffff7298ee48, ffffffff71c00698, fffffffe0f57bd50, ff7ffffdf1c72000)
ffffffff7662574c ???????? (fffffffe8ec1ac58, fffffffe8a2300a0, ffffffff71c00698, ffffffff7298ee48, fffffffe889cf2f0, 1)
ffffffff764b979c ???????? (ffffffff71c63190, 1, fffffffe8e95d6d8, fffffffe1f42e3a0, fffffffe73af6140, fffffffe7419d368)
ffffffff76528fc0 ???????? (ffffffff71c63190, ff7ffffdf1c72000, fffffffdff80de60, fffffffe84196e48, ffffffff71f24998, ffffffff71c2e7c0)
ffffffff765a1158 ???????? (dbba0, fffffffea0710bc0, ffffffff71c04f68, 128af17e0, ffffffff7e2d9414, 0)
ffffffff76087868 ???????? (fffffffdf6797860, ffffffff71c2e470, fffffffde6dff7b8, ff000000, ffffffff7e1e6908, 0)
ffffffff76005c80 ???????? (fffffffde6dff7b8, ffffffff7e1e9008, 0, ffffffff760175d0, 8, fffffffde6dff610)
ffffffff7600022c ???????? (fffffffde6dff8b0, fffffffde6dffd70, a, ffffffff71c2f298, 8, fffffffde6dff700)
ffffffff7e1e8c50 __1cJJavaCallsLcall_helper6FpnJJavaValue_pnMmethodHandle_pnRJavaCallArguments_pnGThread__v_ (fffffffde6dffd70, ffffffff760001e0, fffffffde6dffbf0, 124400160, 124400160, 1) + 180
ffffffff7e1e8d50 __1cJJavaCallsEcall6FpnJJavaValue_nMmethodHandle_pnRJavaCallArguments_pnGThread__v_ (fffffffde6dffd68, 106de5158, fffffffde6dffbf0, 124400160, ffffffff7e7a81f0, ffffffff7e7a8318) + 68
ffffffff7e2ba68c __1cJJavaCallsMcall_virtual6FpnJJavaValue_nLKlassHandle_nMsymbolHandle_4pnRJavaCallArguments_pnGThread__v_ (fffffffde6dffd68, 106de5138, ffffffff7e7a81f0, ffffffff7e7a8318, fffffffde6dffbf0, 124400160) + e0
ffffffff7e2c2c9c __1cJJavaCallsMcall_virtual6FpnJJavaValue_nGHandle_nLKlassHandle_nMsymbolHandle_5pnGThread__v_ (fffffffde6dffd68, 106de5130, 106de5138, ffffffff7e7a81f0, ffffffff7e7a8318, 124400160) + 44
ffffffff7e2dccfc __1cMthread_entry6FpnKJavaThread_pnGThread__v_ (124400160, 124400160, 105232b60, 128af17e0, ffffffff7e2d9414, 1000) + fc
ffffffff7e2d943c __1cKJavaThreadRthread_main_inner6M_v_ (124400160, 40, ffffffff7eff7f00, 0, ffffffff7eff8f58, fffffffde7903200) + 3c
ffffffff7e515e2c java_start (124400160, 100000, 0, 0, ffffffff7e515d40, 1) + ec
ffffffff7eed25f8 lwpstart (0, 0, 0, 0, 0, 0)

"Device Poller Thread" prio=5 tid=0x0000000124400160 nid=0x1afb waiting on condition [0xfffffffde6dfe000..0xfffffffde6dff8b0]
at com.adventnet.snmp.snmp2.ASNTypes.encodeOid(ASNTypes.java:273)
at com.adventnet.snmp.snmp2.SnmpOID.encode(SnmpOID.java:65)
at com.adventnet.snmp.snmp2.SnmpVarBind.encode(SnmpVarBind.java:148)
at com.adventnet.snmp.snmp2.ASN1Parser.encodeVarBindList(ASN1Parser.java:443)
at com.adventnet.snmp.snmp2.SnmpPDU.encode(SnmpPDU.java:1040)
at com.adventnet.snmp.snmp2.SnmpSession.send(SnmpSession.java:899)
at com.adventnet.snmp.snmp2.SnmpSession.syncSend(SnmpSession.java:995)
at ISM.ISM_178_1_0.SNMP.SparseBulkHandler.handGetRequestPDU(SparseBulkHandler.java:263)
at ISM.ISM_178_1_0.SNMP.SparseBulkHandler.getSelectedRows(SparseBulkHandler.java:250)
at ISM.ISM_178_1_0.TableInfo.getTableLeavesObjects(TableInfo.java:248)
at ISM.ISM_178_1_0.DevicePoller.pollDevice(DevicePoller.java:379)
at ISM.ISM_178_1_0.DevicePoller.run(DevicePoller.java:168)
at EDU.oswego.cs.dl.util.concurrent.PooledExecutor$Worker.run(PooledExecutor.java:748)
at java.lang.Thread.run(Unknown Source)
===============================
----------------- lwp# 6865 / thread# 6865 --------------------
ffffffff7eed3890 lwp_mutex_timedlock (10011be40, 0)
ffffffff7e25b4d8 __1cHMonitorEwait6Mil_i_ (ffffffff7e78e210, 125c1b2e0, ffffffff7e78ff94, ffffffff7e7943fc, 9800, 9ba8) + 328
ffffffff7e2b5f08 __1cIVMThreadHexecute6FpnMVM_Operation__v_ (ffffffff7e78e2f0, 0, a000, a2e8, c558, 3fea88) + 228
ffffffff7e30929c __1cODeoptimizationNuncommon_trap6FpnKJavaThread_i_pn0ALUnrollBlock__ (11a0cf700, fffffffffffffffc, fffffffffffffffc, 100115700, ffffffff7e1fca90, 0) + 874
ffffffff7606c940 ???????? (fffffffffffffffc, fffffffa93afed78, ffffffff72bba220, ffffffff76012a78, 0, ffffffff7606cf98)
ffffffff76479654 ???????? (ffffffff71c63190, ffffffffffffffff, fffffffef535b858, 10, 6, ffffffff7606cf98)
ffffffff764ee7b4 ???????? (fffffffef45c3a08, ffffffff724c0848, fffffffef45c3a40, fffffffdf5da9638, ffffffff71c63190, ffffffff7606cf98)
ffffffff76425dbc ???????? (1, fffffffef45c3a68, fffffffe71c39eb8, ffffffff71c63190, ffffffff71c62ed8, ffffffff72526698)
ffffffff7656cf98 ???????? (ffffffff71c0c6e0, fffffffef45c3a68, ffffffff7251d5e8, 0, 1, ffffffffffffffff)
ffffffff764b26c4 ???????? (ffffffff71c63190, fffffffef45c3a68, 0, 0, fffffffe00de0288, 0)
ffffffff764b783c ???????? (ffffffff71c63190, fffffffe9146b0d0, fffffffe92ae7440, fffffffe00de0288, 1, ff7ffffdf1c72000)
ffffffff764933f4 ???????? (ffffffff71c4ed78, fffffffe00dce978, ffffffff7298ee48, ffffffff71c00698, fffffffe00dcf398, ff7ffffdf1c72000)
ffffffff7662574c ???????? (fffffffe92a509f0, fffffffe91c00140, ffffffff71c00698, ffffffff7298ee48, fffffffe9146b0d0, 1)
ffffffff764b979c ???????? (ffffffff71c63190, 1, fffffffe929c4db8, fffffffdfed95ee8, fffffffe7418f1f0, fffffffe7419d368)
ffffffff76528fc0 ???????? (ffffffff71c63190, ff7ffffdf1c72000, fffffffe1d211160, fffffffe86af0ed8, ffffffff71f24998, fffffffa93aff440)
ffffffff765a1158 ???????? (dbba0, fffffffe98d6d800, 0, 0, 0, 0)
ffffffff76087868 ???????? (fffffffe112ca820, ffffffff71c2e470, fffffffa93aff7b8, ff000000, ffffffff7e1e6908, 0)
ffffffff76005c80 ???????? (fffffffa93aff7b8, ffffffff7e1e9008, 0, ffffffff760175d0, 8, fffffffa93aff610)
ffffffff7600022c ???????? (fffffffa93aff8b0, fffffffa93affd70, a, ffffffff71c2f298, 8, fffffffa93aff700)
ffffffff7e1e8c50 __1cJJavaCallsLcall_helper6FpnJJavaValue_pnMmethodHandle_pnRJavaCallArguments_pnGThread__v_ (fffffffa93affd70, ffffffff760001e0, fffffffa93affbf0, 11a0cf700, 11a0cf700, 1) + 180
ffffffff7e1e8d50 __1cJJavaCallsEcall6FpnJJavaValue_nMmethodHandle_pnRJavaCallArguments_pnGThread__v_ (fffffffa93affd68, 123362968, fffffffa93affbf0, 11a0cf700, ffffffff7e7a81f0, ffffffff7e7a8318) + 68
ffffffff7e2ba68c __1cJJavaCallsMcall_virtual6FpnJJavaValue_nLKlassHandle_nMsymbolHandle_4pnRJavaCallArguments_pnGThread__v_ (fffffffa93affd68, 123362948, ffffffff7e7a81f0, ffffffff7e7a8318, fffffffa93affbf0, 11a0cf700) + e0
ffffffff7e2c2c9c __1cJJavaCallsMcall_virtual6FpnJJavaValue_nGHandle_nLKlassHandle_nMsymbolHandle_5pnGThread__v_ (fffffffa93affd68, 123362940, 123362948, ffffffff7e7a81f0, ffffffff7e7a8318, 11a0cf700) + 44
ffffffff7e2dccfc __1cMthread_entry6FpnKJavaThread_pnGThread__v_ (11a0cf700, 11a0cf700, 116cfc950, 116cfc190, ffffffff7e2d9414, 1000) + fc
ffffffff7e2d943c __1cKJavaThreadRthread_main_inner6M_v_ (11a0cf700, 40, ffffffff7eff7f00, 0, ffffffff7eff8f58, fffffffbed6c8a00) + 3c
ffffffff7e515e2c java_start (11a0cf700, 100000, 0, 0, ffffffff7e515d40, 1) + ec
ffffffff7eed25f8 lwpstart (0, 0, 0, 0, 0, 0)

"Device Poller Thread" prio=5 tid=0x000000011a0cf700 nid=0x1ad1 waiting on condition [0xfffffffa93afe000..0xfffffffa93aff8b0]
at com.adventnet.snmp.snmp2.ASNTypes.encodeOid(ASNTypes.java:273)
at com.adventnet.snmp.snmp2.SnmpOID.encode(SnmpOID.java:65)
at com.adventnet.snmp.snmp2.SnmpVarBind.encode(SnmpVarBind.java:148)
at com.adventnet.snmp.snmp2.ASN1Parser.encodeVarBindList(ASN1Parser.java:443)
at com.adventnet.snmp.snmp2.SnmpPDU.encode(SnmpPDU.java:1040)
at com.adventnet.snmp.snmp2.SnmpSession.send(SnmpSession.java:899)
at com.adventnet.snmp.snmp2.SnmpSession.syncSend(SnmpSession.java:995)
at ISM.ISM_178_1_0.SNMP.SparseBulkHandler.handGetRequestPDU(SparseBulkHandler.java:263)
at ISM.ISM_178_1_0.SNMP.SparseBulkHandler.getSelectedRows(SparseBulkHandler.java:250)
at ISM.ISM_178_1_0.TableInfo.getTableLeavesObjects(TableInfo.java:248)
at ISM.ISM_178_1_0.DevicePoller.pollDevice(DevicePoller.java:379)
at ISM.ISM_178_1_0.DevicePoller.run(DevicePoller.java:168)
at EDU.oswego.cs.dl.util.concurrent.PooledExecutor$Worker.run(PooledExecutor.java:748)
at java.lang.Thread.run(Unknown Source)
==============================
3)
==============================
----------------- lwp# 6864 / thread# 6864 --------------------
ffffffff7eed3890 lwp_mutex_timedlock (10011be40, 0)
ffffffff7e25b4d8 __1cHMonitorEwait6Mil_i_ (ffffffff7e78e210, 10f06c350, ffffffff7e78ff94, ffffffff7e7943fc, 9800, 9ba8) + 328
ffffffff7e2b5f08 __1cIVMThreadHexecute6FpnMVM_Operation__v_ (ffffffff7e78e2f0, 0, a000, a2e8, c558, 3fea88) + 228
ffffffff7e30929c __1cODeoptimizationNuncommon_trap6FpnKJavaThread_i_pn0ALUnrollBlock__ (1114291b0, fffffffffffffffc, fffffffffffffffc, 33, ffffffff96b14cb2, 0) + 874
ffffffff7606c940 ???????? (fffffffffffffffc, fffffffe99032ad0, b, 9, fffffffffffffe5a, 0)
ffffffff7654ca9c ???????? (fffffffe3e550d00, fffffffe3e550d20, 0, 9, 30, fffffffe2fb47438)
ffffffff76531688 ???????? (fffffffeaeb599e8, fffffffeae8c99b8, ffffffff71c63190, 1, fffffffebde8740c, 13)
ffffffff764b51cc ???????? (ffffffff71c63190, fffffffe7ef07850, fffffffeae8c99b8, 0, fffffffe91ccc6f8, fffffffe2fb47630)
ffffffff764aecf0 ???????? (ffffffff71c4ed78, fffffffeae6eb198, fffffffeb22434e0, fffffffe3e550d80, fffffffe91ccc6f8, 70)
ffffffff764b979c ???????? (ffffffff71c63190, 1, fffffffe91ccc6f8, fffffffe23df7898, fffffffe7418f1f0, fffffffe7419d368)
ffffffff76528fc0 ???????? (ffffffff71c63190, ff7ffffdf1c72000, fffffffdf65f1c28, fffffffe86af0ed8, ffffffff71f24998, fffffffa93cff440)
ffffffff765a1158 ???????? (dbba0, fffffffe98d6d890, 0, 0, 0, 0)
ffffffff76087868 ???????? (fffffffe112c8fc0, ffffffff71c2e470, fffffffa93cff7b8, ff000000, ffffffff7e1e6908, 0)
ffffffff76005c80 ???????? (fffffffa93cff7b8, ffffffff7e1e9008, 0, ffffffff760175d0, 8, fffffffa93cff610)
ffffffff7600022c ???????? (fffffffa93cff8b0, fffffffa93cffd70, a, ffffffff71c2f298, 8, fffffffa93cff700)
ffffffff7e1e8c50 __1cJJavaCallsLcall_helper6FpnJJavaValue_pnMmethodHandle_pnRJavaCallArguments_pnGThread__v_ (fffffffa93cffd70, ffffffff760001e0, fffffffa93cffbf0, 1114291b0, 1114291b0, 1) + 180
ffffffff7e1e8d50 __1cJJavaCallsEcall6FpnJJavaValue_nMmethodHandle_pnRJavaCallArguments_pnGThread__v_ (fffffffa93cffd68, 102398088, fffffffa93cffbf0, 1114291b0, ffffffff7e7a81f0, ffffffff7e7a8318) + 68
ffffffff7e2ba68c __1cJJavaCallsMcall_virtual6FpnJJavaValue_nLKlassHandle_nMsymbolHandle_4pnRJavaCallArguments_pnGThread__v_ (fffffffa93cffd68, 102398068, ffffffff7e7a81f0, ffffffff7e7a8318, fffffffa93cffbf0, 1114291b0) + e0
ffffffff7e2c2c9c __1cJJavaCallsMcall_virtual6FpnJJavaValue_nGHandle_nLKlassHandle_nMsymbolHandle_5pnGThread__v_ (fffffffa93cffd68, 102398060, 102398068, ffffffff7e7a81f0, ffffffff7e7a8318, 1114291b0) + 44
ffffffff7e2dccfc __1cMthread_entry6FpnKJavaThread_pnGThread__v_ (1114291b0, 1114291b0, 1134656c0, 113465800, ffffffff7e2d9414, 1000) + fc
ffffffff7e2d943c __1cKJavaThreadRthread_main_inner6M_v_ (1114291b0, 40, ffffffff7eff7f00, 0, ffffffff7eff8f58, fffffffbed48b200) + 3c
ffffffff7e515e2c java_start (1114291b0, 100000, 0, 0, ffffffff7e515d40, 1) + ec
ffffffff7eed25f8 lwpstart (0, 0, 0, 0, 0, 0)

"Device Poller Thread" prio=5 tid=0x00000001114291b0 nid=0x1ad0 waiting on condition [0xfffffffa93cfe000..0xfffffffa93cff8b0]
at com.adventnet.snmp.snmp2.SnmpDateAndTime.convBytesToString(SnmpDateAndTime.java:228)
at com.adventnet.snmp.snmp2.SnmpDateAndTime.<init>(SnmpDateAndTime.java:177)
at com.adventnet.snmp.snmp2.SnmpString.toString(SnmpString.java:228)
at com.adventnet.snmp.snmp2.SnmpString.toValue(SnmpString.java:216)
at ISM.ISM_178_1_0.OIDVal.<init>(OIDVal.java:42)
at ISM.ISM_178_1_0.TableInfo.getTableLeavesObjects(TableInfo.java:300)
at ISM.ISM_178_1_0.DevicePoller.pollDevice(DevicePoller.java:379)
at ISM.ISM_178_1_0.DevicePoller.run(DevicePoller.java:168)
at EDU.oswego.cs.dl.util.concurrent.PooledExecutor$Worker.run(PooledExecutor.java:748)
at java.lang.Thread.run(Unknown Source)
============
----------------- lwp# 6862 / thread# 6862 --------------------
ffffffff7eed3890 lwp_mutex_timedlock (10011be40, 0)
ffffffff7e25b4d8 __1cHMonitorEwait6Mil_i_ (ffffffff7e78e210, 108bbc940, ffffffff7e78ff94, ffffffff7e7943fc, 9800, 9ba8) + 328
ffffffff7e2b5f08 __1cIVMThreadHexecute6FpnMVM_Operation__v_ (ffffffff7e78e2f0, 0, a000, a2e8, c558, 3fea88) + 228
ffffffff7e30929c __1cODeoptimizationNuncommon_trap6FpnKJavaThread_i_pn0ALUnrollBlock__ (11e3b6200, fffffffffffffffc, fffffffffffffffc, 33, ffffffff66343ad2, 0) + 874
ffffffff7606c940 ???????? (fffffffffffffffc, fffffffe99032a50, b, 9, fffffffffffffe5a, 0)
ffffffff7654ca9c ???????? (fffffffef5815e40, fffffffef5815e60, 0, 20, 30, fffffffdff3761d8)
ffffffff76531688 ???????? (fffffffed6adf238, fffffffed68c1578, ffffffff71c63190, 1, fffffffe8d6b61ac, 13)
ffffffff764b51cc ???????? (ffffffff71c63190, fffffffe7ef07910, fffffffed68c1578, 0, fffffffe929c1bb8, fffffffdff3763d0)
ffffffff766261b8 ???????? (fffffffe92a45f88, fffffffed6777a50, ffffffff71c00698, ffffffff7298ee48, 0, 1)
ffffffff764b979c ???????? (ffffffff71c63190, 1, fffffffe929c1bb8, fffffffe1cd06378, fffffffe7418f1f0, fffffffe7419d368)
ffffffff76528fc0 ???????? (ffffffff71c63190, ff7ffffdf1c72000, fffffffe1c60b4f0, fffffffe86af0ed8, ffffffff71f24998, fffffffa940ff440)
ffffffff765a1158 ???????? (dbba0, fffffffe98d6d9b0, 0, 0, 0, 0)
ffffffff76087868 ???????? (fffffffe112ba980, ffffffff71c2e470, fffffffa940ff7b8, ff000000, ffffffff7e1e6908, 0)
ffffffff76005c80 ???????? (fffffffa940ff7b8, ffffffff7e1e9008, 0, ffffffff760175d0, 8, fffffffa940ff610)
ffffffff7600022c ???????? (fffffffa940ff8b0, fffffffa940ffd70, a, ffffffff71c2f298, 8, fffffffa940ff700)
ffffffff7e1e8c50 __1cJJavaCallsLcall_helper6FpnJJavaValue_pnMmethodHandle_pnRJavaCallArguments_pnGThread__v_ (fffffffa940ffd70, ffffffff760001e0, fffffffa940ffbf0, 11e3b6200, 11e3b6200, 1) + 180
ffffffff7e1e8d50 __1cJJavaCallsEcall6FpnJJavaValue_nMmethodHandle_pnRJavaCallArguments_pnGThread__v_ (fffffffa940ffd68, 11e27ad48, fffffffa940ffbf0, 11e3b6200, ffffffff7e7a81f0, ffffffff7e7a8318) + 68
ffffffff7e2ba68c __1cJJavaCallsMcall_virtual6FpnJJavaValue_nLKlassHandle_nMsymbolHandle_4pnRJavaCallArguments_pnGThread__v_ (fffffffa940ffd68, 11e27ad28, ffffffff7e7a81f0, ffffffff7e7a8318, fffffffa940ffbf0, 11e3b6200) + e0
ffffffff7e2c2c9c __1cJJavaCallsMcall_virtual6FpnJJavaValue_nGHandle_nLKlassHandle_nMsymbolHandle_5pnGThread__v_ (fffffffa940ffd68, 11e27ad20, 11e27ad28, ffffffff7e7a81f0, ffffffff7e7a8318, 11e3b6200) + 44
ffffffff7e2dccfc __1cMthread_entry6FpnKJavaThread_pnGThread__v_ (11e3b6200, 11e3b6200, 107dddfa0, 113463e00, ffffffff7e2d9414, 0) + fc
ffffffff7e2d943c __1cKJavaThreadRthread_main_inner6M_v_ (11e3b6200, 40, ffffffff7eff7f00, 0, ffffffff7eff8f58, fffffffbed676200) + 3c
ffffffff7e515e2c java_start (11e3b6200, 100000, 0, 0, ffffffff7e515d40, 1) + ec
ffffffff7eed25f8 lwpstart (0, 0, 0, 0, 0, 0)

"Device Poller Thread" prio=5 tid=0x000000011e3b6200 nid=0x1ace waiting on condition [0xfffffffa940fe000..0xfffffffa940ff8b0]
at com.adventnet.snmp.snmp2.SnmpDateAndTime.convBytesToString(SnmpDateAndTime.java:228)
at com.adventnet.snmp.snmp2.SnmpDateAndTime.<init>(SnmpDateAndTime.java:177)
at com.adventnet.snmp.snmp2.SnmpString.toString(SnmpString.java:228)
at com.adventnet.snmp.snmp2.SnmpString.toValue(SnmpString.java:216)
at ISM.ISM_178_1_0.OIDVal.<init>(OIDVal.java:42)
at ISM.ISM_178_1_0.TableInfo.getTableLeavesObjects(TableInfo.java:300)
at ISM.ISM_178_1_0.DevicePoller.pollDevice(DevicePoller.java:379)
at ISM.ISM_178_1_0.DevicePoller.run(DevicePoller.java:168)
at EDU.oswego.cs.dl.util.concurrent.PooledExecutor$Worker.run(PooledExecutor.java:748)
at java.lang.Thread.run(Unknown Source)
=============================
4)
=============================
----------------- lwp# 6857 / thread# 6857 --------------------
ffffffff7eed3890 lwp_mutex_timedlock (10011be40, 0)
ffffffff7e25b4d8 __1cHMonitorEwait6Mil_i_ (ffffffff7e78e210, 11c480190, ffffffff7e78ff94, ffffffff7e7943fc, 9800, 9ba8) + 328
ffffffff7e2b5f08 __1cIVMThreadHexecute6FpnMVM_Operation__v_ (ffffffff7e78e2f0, 0, a000, a2e8, c558, 3fea88) + 228
ffffffff7e30929c __1cODeoptimizationNuncommon_trap6FpnKJavaThread_i_pn0ALUnrollBlock__ (115b82a80, fffffffffffffffc, fffffffffffffffc, 0, 100, 0) + 874
ffffffff7606c940 ???????? (fffffffffffffffc, 4, fffffffdf550c648, f, 6, 817fe2)
ffffffff764eea38 ???????? (fffffffef5816130, ffffffff724c0848, fffffffdf550c1b8, fffffffef5816168, ffffffff71c63190, fffffffa94afeec0)
ffffffff76425dbc ???????? (1, fffffffef5816580, fffffffe71c39eb8, ffffffff71c63190, ffffffff71c62ed8, ffffffff72526698)
ffffffff7656cf98 ???????? (ffffffff71c0c6e0, fffffffef5816580, ffffffff7251d5e8, 0, 1, ffffffffffffffff)
ffffffff764b26c4 ???????? (ffffffff71c63190, fffffffef5816580, 0, 0, fffffffdf5513c88, 0)
ffffffff764b783c ???????? (ffffffff71c63190, fffffffe910c9198, fffffffe92b77018, fffffffdf5513c88, 1, ff7ffffdf1c72000)
ffffffff764933f4 ???????? (ffffffff71c4ed78, fffffffdf54e3998, ffffffff7298ee48, ffffffff71c00698, fffffffeef954fb0, ff7ffffdf1c72000)
ffffffff7662574c ???????? (fffffffe92a7c178, fffffffe91bfb8f0, ffffffff71c00698, ffffffff7298ee48, fffffffe910c9198, 1)
ffffffff764b979c ???????? (ffffffff71c63190, 1, fffffffe929d1a68, fffffffe1b815138, fffffffe7418f1f0, fffffffe7419d368)
ffffffff76528fc0 ???????? (ffffffff71c63190, ff7ffffdf1c72000, fffffffe01097948, fffffffe86af0ed8, ffffffff71f24998, fffffffa94aff440)
ffffffff765a1158 ???????? (dbba0, fffffffe98d6dc80, 0, 0, 0, 0)
ffffffff76087868 ???????? (fffffffe11290f20, ffffffff71c2e470, fffffffa94aff7b8, ff000000, ffffffff7e1e6908, 0)
ffffffff76005c80 ???????? (fffffffa94aff7b8, ffffffff7e1e9008, 0, ffffffff760175d0, 8, fffffffa94aff610)
ffffffff7600022c ???????? (fffffffa94aff8b0, fffffffa94affd70, a, ffffffff71c2f298, 8, fffffffa94aff700)
ffffffff7e1e8c50 __1cJJavaCallsLcall_helper6FpnJJavaValue_pnMmethodHandle_pnRJavaCallArguments_pnGThread__v_ (fffffffa94affd70, ffffffff760001e0, fffffffa94affbf0, 115b82a80, 115b82a80, 1) + 180
ffffffff7e1e8d50 __1cJJavaCallsEcall6FpnJJavaValue_nMmethodHandle_pnRJavaCallArguments_pnGThread__v_ (fffffffa94affd68, 11cca8ff8, fffffffa94affbf0, 115b82a80, ffffffff7e7a81f0, ffffffff7e7a8318) + 68
ffffffff7e2ba68c __1cJJavaCallsMcall_virtual6FpnJJavaValue_nLKlassHandle_nMsymbolHandle_4pnRJavaCallArguments_pnGThread__v_ (fffffffa94affd68, 11cca8fd8, ffffffff7e7a81f0, ffffffff7e7a8318, fffffffa94affbf0, 115b82a80) + e0
ffffffff7e2c2c9c __1cJJavaCallsMcall_virtual6FpnJJavaValue_nGHandle_nLKlassHandle_nMsymbolHandle_5pnGThread__v_ (fffffffa94affd68, 11cca8fd0, 11cca8fd8, ffffffff7e7a81f0, ffffffff7e7a8318, 115b82a80) + 44
ffffffff7e2dccfc __1cMthread_entry6FpnKJavaTh