Skip to Main Content

New to Java

Announcement

For appeals, questions and feedback about Oracle Forums, please email oracle-forums-moderators_us@oracle.com. Technical questions should be asked in the appropriate category. Thank you!

memory leak

854145Apr 9 2011 — edited Apr 9 2011
Hello, I'm new to Java, but I've selected Java to wrote multi-threaded class to scan a list of domains. Its works fine, but after 6 hours it consumes 1G of RAM, after 10 hours 1.5G and speed is very low, I would say it stops. I even cant connect to the JVM with jconsole.
import org.apache.http.HttpEntity;
import org.apache.http.HttpResponse;
import org.apache.http.client.HttpClient;
import org.apache.http.client.methods.HttpGet;
import org.apache.http.conn.ClientConnectionManager;
import org.apache.http.conn.ConnectTimeoutException;
import org.apache.http.conn.scheme.PlainSocketFactory;
import org.apache.http.conn.scheme.Scheme;
import org.apache.http.conn.scheme.SchemeRegistry;
import org.apache.http.impl.client.DefaultHttpClient;
import org.apache.http.impl.conn.tsccm.ThreadSafeClientConnManager;
import org.apache.http.params.BasicHttpParams;
import org.apache.http.params.HttpConnectionParams;
import org.apache.http.params.HttpParams;
import org.apache.http.protocol.BasicHttpContext;
import org.apache.http.protocol.HttpContext;
import org.apache.http.util.EntityUtils;

import java.net.SocketTimeoutException;
import java.net.UnknownHostException;
import java.sql.*;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.Executors;
import java.util.concurrent.ThreadPoolExecutor;

public class Scan {
    private static Connection localConnection = null;
    private static Statement localStatement = null;
    private static ResultSet localResultSet = null;
    private static int totalRows = 0;

    static class GetThread implements Runnable {

        private final HttpClient httpClient;
        private final HttpContext context;
        private final HttpGet httpget;
        private int ID = 0;
        private int statusCode = 999;
        private String content = null;

        public GetThread(HttpClient httpClient, HttpGet httpget, int ID) {
            this.httpClient = httpClient;
            this.context = new BasicHttpContext();
            this.httpget = httpget;
            this.ID = ID;
        }

        public void run() {
            try {
                HttpResponse response = httpClient.execute(httpget, context);
                HttpEntity entity = response.getEntity();
                statusCode = response.getStatusLine().getStatusCode();
                content = "";
                System.out.println("Check ID: " + ID);

                if (entity != null) {
                    content = EntityUtils.toString(entity).replace("'", "").replace("\"", "");
                }
            } catch (UnknownHostException uhe) {
                // host is off
                content = "";
            } catch (ConnectTimeoutException cte) {
                // host is off
                content = "";
            } catch (SocketTimeoutException ste) {
                // host is off
                content = "";
            } catch (Exception ex) {
                content = "";
            } finally {
                httpget.abort();
            }

            String query = "UPDATE com SET status = '" + statusCode + "', content= '" + content + "' WHERE domainID=" + ID;

            try {
                localConnection.createStatement().execute(query);
            } catch (SQLException localSQLException) {
                System.out.println("SQLException: " + localSQLException);
            }
        }

    }

    public static void main(String[] args) {
        System.out.println("Starting...");

        try {
            Class.forName("com.mysql.jdbc.Driver").newInstance();
        } catch (Exception localException) {
            System.out.println("com.mysql.jdbc.Driver: " + localException);
        }

        try {
            localConnection = DriverManager.getConnection("jdbc:mysql://localhost/scan?user=root&password=0000");
            localStatement = localConnection.createStatement();

            countTotalRows();

            HttpParams httpParams = new BasicHttpParams();
            HttpConnectionParams.setConnectionTimeout(httpParams, 2000);
            HttpConnectionParams.setSoTimeout(httpParams, 2000);

            SchemeRegistry schemeRegistry = new SchemeRegistry();
            schemeRegistry.register(new Scheme("http", PlainSocketFactory.getSocketFactory(), 80));

            ClientConnectionManager cm = new ThreadSafeClientConnManager(httpParams, schemeRegistry);
            HttpClient httpClient = new DefaultHttpClient(cm, httpParams);

            int total = 10000;

            ExecutorService executor = Executors.newFixedThreadPool(500);

            Thread monitor = new Thread(new MyMonitorThread((ThreadPoolExecutor) executor));
            monitor.setDaemon(true);
            monitor.start();

            while (totalRows > 0) {
                localStatement.executeQuery("SELECT domainID, domain FROM com WHERE status = '' LIMIT " + total);
                totalRows -= total;
                localResultSet = localStatement.getResultSet();

                while (localResultSet.next()) {
                    HttpGet httpget = new HttpGet("http://" + localResultSet.getString("domain").toLowerCase());
                    executor.submit(new GetThread(httpClient, httpget, localResultSet.getInt("domainID")));
                }

                Thread.sleep(1000);

                while (((ThreadPoolExecutor) executor).getActiveCount() > 1) {
                    try {
                        Thread.sleep(1000);
                    } catch (InterruptedException e) {
                        e.printStackTrace();
                    }
                }

                localResultSet.close();
                System.out.println("New cycle");
            }

            executor.shutdown();
            monitor.interrupt();

            localStatement.close();
            localConnection.close();
        } catch (SQLException localSQLException) {
            System.out.println("SQLException: " + localSQLException);
        } catch (InterruptedException e) {
            e.printStackTrace();
        }

        System.out.println("DONE!");
    }

    private static void countTotalRows() throws SQLException {
        localStatement.executeQuery("SELECT count(domainID) as total FROM com WHERE status = ''");
        localResultSet = localStatement.getResultSet();
        while (localResultSet.next())
            totalRows = localResultSet.getInt("total");

        localResultSet.close();
    }

}
{code}

and the monitor class
{code}
class MyMonitorThread implements Runnable {
    private ThreadPoolExecutor executor;

    public MyMonitorThread(ThreadPoolExecutor executor) {
        this.executor = executor;
    }

    @Override
    public void run() {
        try {
            do {
                System.out
                        .println(String
                                .format("[monitor] [%d/%d] Active: %d, Completed: %d, Task: %d, isShutdown: %s, isTerminated: %s",
                                        this.executor.getPoolSize(),
                                        this.executor.getCorePoolSize(),
                                        this.executor.getActiveCount(),
                                        this.executor.getCompletedTaskCount(),
                                        this.executor.getTaskCount(),
                                        this.executor.isShutdown(),
                                        this.executor.isTerminated()));
                Thread.sleep(5000);
            } while (!Thread.currentThread().isInterrupted());
        } catch (Exception e) {
        }
    }
}
{code}

I really tried to figure it out by myself with jmap, debugging, YourKit Profiler and failed. I would appreciate any help, thanks.                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                        
Comments
Locked Post
New comments cannot be posted to this locked post.
Post Details
Locked on May 7 2011
Added on Apr 9 2011
4 comments
448 views