Very Intermittent java.net.ConnectionException (connection refused)
843859Nov 16 2005 — edited Nov 16 2005Hello friends. We've got applications running mysql 4.1.1.2, tomcat-5.5.9, Hibernate 2.1 on Windows. Each application instance has a dedicated mysql database.
Very infrequently, log files have a series of java.net.ConnectionExceptions such as below:
*****************************************************************************
** BEGIN NESTED EXCEPTION **
java.net.SocketException
MESSAGE: java.net.ConnectException: Connection refused: connect
STACKTRACE:
java.net.SocketException: java.net.ConnectException: Connection refused: connect
at com.mysql.jdbc.StandardSocketFactory.connect(StandardSocketFactory.java:151)
at com.mysql.jdbc.MysqlIO.<init>(MysqlIO.java:280)
at com.mysql.jdbc.Connection.createNewIO(Connection.java:1699)
at com.mysql.jdbc.Connection.<init>(Connection.java:405)
at com.mysql.jdbc.NonRegisteringDriver.connect(NonRegisteringDriver.java:268)
at java.sql.DriverManager.getConnection(DriverManager.java:512)
at java.sql.DriverManager.getConnection(DriverManager.java:140)
at org.apache.commons.dbcp.DriverManagerConnectionFactory.createConnection(DriverManagerConnectionFactory.java:51)
at org.apache.commons.dbcp.PoolableConnectionFactory.makeObject(PoolableConnectionFactory.java:290)
at org.apache.commons.pool.impl.GenericObjectPool.borrowObject(GenericObjectPool.java:771)
at org.apache.commons.dbcp.PoolingDataSource.getConnection(PoolingDataSource.java:95)
at net.sf.hibernate.connection.DBCPConnectionProvider.getConnection(DBCPConnectionProvider.java:41)
at net.sf.hibernate.impl.BatcherImpl.openConnection(BatcherImpl.java:289)
at net.sf.hibernate.impl.SessionImpl.connect(SessionImpl.java:3361)
at net.sf.hibernate.impl.SessionImpl.connection(SessionImpl.java:3321)
at net.sf.hibernate.transaction.JDBCTransaction.begin(JDBCTransaction.java:40)
at net.sf.hibernate.transaction.JDBCTransactionFactory.beginTransaction(JDBCTransactionFactory.java:19)
at net.sf.hibernate.impl.SessionImpl.beginTransaction(SessionImpl.java:2251)
at us.fl.ci.tlh.mobile.persistence.CredentialsDao.getCredentials(CredentialsDao.java:81)
at us.fl.ci.tlh.mobile.services.AuthenticationService.isAuthenticated(AuthenticationService.java:127)
at us.fl.ci.tlh.mobile.services.AuthenticationService.<init>(AuthenticationService.java:99)
at us.fl.ci.tlh.struts.action.LoginAction.execute(LoginAction.java:122)
at org.apache.struts.action.RequestProcessor.processActionPerform(RequestProcessor.java:484)
at org.apache.struts.action.RequestProcessor.process(RequestProcessor.java:274)
at org.apache.struts.action.ActionServlet.process(ActionServlet.java:1482)
at org.apache.struts.action.ActionServlet.doGet(ActionServlet.java:507)
at javax.servlet.http.HttpServlet.service(HttpServlet.java:689)
at javax.servlet.http.HttpServlet.service(HttpServlet.java:802)
at org.apache.catalina.core.ApplicationFilterChain.internalDoFilter(ApplicationFilterChain.java:252)
at org.apache.catalina.core.ApplicationFilterChain.doFilter(ApplicationFilterChain.java:173)
at org.apache.catalina.core.StandardWrapperValve.invoke(StandardWrapperValve.java:213)
at org.apache.catalina.core.StandardContextValve.invoke(StandardContextValve.java:178)
at org.apache.catalina.core.StandardHostValve.invoke(StandardHostValve.java:126)
at org.apache.catalina.valves.ErrorReportValve.invoke(ErrorReportValve.java:105)
at org.apache.catalina.core.StandardEngineValve.invoke(StandardEngineValve.java:107)
at org.apache.catalina.connector.CoyoteAdapter.service(CoyoteAdapter.java:148)
at org.apache.coyote.http11.Http11Processor.process(Http11Processor.java:856)
at org.apache.coyote.http11.Http11Protocol$Http11ConnectionHandler.processConnection(Http11Protocol.java:744)
at org.apache.tomcat.util.net.PoolTcpEndpoint.processSocket(PoolTcpEndpoint.java:527)
at org.apache.tomcat.util.net.LeaderFollowerWorkerThread.runIt(LeaderFollowerWorkerThread.java:80)
at org.apache.tomcat.util.threads.ThreadPool$ControlRunnable.run(ThreadPool.java:684)
at java.lang.Thread.run(Thread.java:534)
** END NESTED EXCEPTION **
net.sf.hibernate.exception.GenericJDBCException: Cannot open connection
at net.sf.hibernate.exception.ErrorCodeConverter.handledNonSpecificException(ErrorCodeConverter.java:90)
at net.sf.hibernate.exception.ErrorCodeConverter.convert(ErrorCodeConverter.java:79)
at net.sf.hibernate.exception.JDBCExceptionHelper.convert(JDBCExceptionHelper.java:30)
at net.sf.hibernate.impl.BatcherImpl.convert(BatcherImpl.java:325)
at net.sf.hibernate.impl.BatcherImpl.openConnection(BatcherImpl.java:292)
at net.sf.hibernate.impl.SessionImpl.connect(SessionImpl.java:3361)
at net.sf.hibernate.impl.SessionImpl.connection(SessionImpl.java:3321)
at net.sf.hibernate.transaction.JDBCTransaction.begin(JDBCTransaction.java:40)
at net.sf.hibernate.transaction.JDBCTransactionFactory.beginTransaction(JDBCTransactionFactory.java:19)
at net.sf.hibernate.impl.SessionImpl.beginTransaction(SessionImpl.java:2251)
at us.fl.ci.tlh.mobile.persistence.CredentialsDao.getCredentials(CredentialsDao.java:81)
at us.fl.ci.tlh.mobile.services.AuthenticationService.isAuthenticated(AuthenticationService.java:127)
at us.fl.ci.tlh.mobile.services.AuthenticationService.<init>(AuthenticationService.java:99)
at us.fl.ci.tlh.struts.action.LoginAction.execute(LoginAction.java:122)
at org.apache.struts.action.RequestProcessor.processActionPerform(RequestProcessor.java:484)
at org.apache.struts.action.RequestProcessor.process(RequestProcessor.java:274)
at org.apache.struts.action.ActionServlet.process(ActionServlet.java:1482)
at org.apache.struts.action.ActionServlet.doGet(ActionServlet.java:507)
at javax.servlet.http.HttpServlet.service(HttpServlet.java:689)
at javax.servlet.http.HttpServlet.service(HttpServlet.java:802)
at org.apache.catalina.core.ApplicationFilterChain.internalDoFilter(ApplicationFilterChain.java:252)
at org.apache.catalina.core.ApplicationFilterChain.doFilter(ApplicationFilterChain.java:173)
at org.apache.catalina.core.StandardWrapperValve.invoke(StandardWrapperValve.java:213)
at org.apache.catalina.core.StandardContextValve.invoke(StandardContextValve.java:178)
at org.apache.catalina.core.StandardHostValve.invoke(StandardHostValve.java:126)
at org.apache.catalina.valves.ErrorReportValve.invoke(ErrorReportValve.java:105)
at org.apache.catalina.core.StandardEngineValve.invoke(StandardEngineValve.java:107)
at org.apache.catalina.connector.CoyoteAdapter.service(CoyoteAdapter.java:148)
at org.apache.coyote.http11.Http11Processor.process(Http11Processor.java:856)
at org.apache.coyote.http11.Http11Protocol$Http11ConnectionHandler.processConnection(Http11Protocol.java:744)
at org.apache.tomcat.util.net.PoolTcpEndpoint.processSocket(PoolTcpEndpoint.java:527)
at org.apache.tomcat.util.net.LeaderFollowerWorkerThread.runIt(LeaderFollowerWorkerThread.java:80)
at org.apache.tomcat.util.threads.ThreadPool$ControlRunnable.run(ThreadPool.java:684)
at java.lang.Thread.run(Thread.java:534)
Caused by: com.mysql.jdbc.CommunicationsException: Communications link failure due to underlying exception:
******************************************************************************
Note that the application runs fine 99% of the time, but it appears there are times when the database is not accessible, and we get these errors. After a short time, the application recovers, though in some instances a reboot of the machine was required.
Some posts indicate that these types of errors are either network related, or configuration related; i.e., wrong port, firewall, whatever. In our case, however, the application runs fine most of the time, then suddenly spits out connection errors when trying to communicate to localhost.
Can anyone give me any guidance on troubleshooting this? Thanks!
- cm