Hello, I'm new to Java, but I've selected Java to wrote multi-threaded class to scan a list of domains. Its works fine, but after 6 hours it consumes 1G of RAM, after 10 hours 1.5G and speed is very low, I would say it stops. I even cant connect to the JVM with jconsole.
import org.apache.http.HttpEntity;
import org.apache.http.HttpResponse;
import org.apache.http.client.HttpClient;
import org.apache.http.client.methods.HttpGet;
import org.apache.http.conn.ClientConnectionManager;
import org.apache.http.conn.ConnectTimeoutException;
import org.apache.http.conn.scheme.PlainSocketFactory;
import org.apache.http.conn.scheme.Scheme;
import org.apache.http.conn.scheme.SchemeRegistry;
import org.apache.http.impl.client.DefaultHttpClient;
import org.apache.http.impl.conn.tsccm.ThreadSafeClientConnManager;
import org.apache.http.params.BasicHttpParams;
import org.apache.http.params.HttpConnectionParams;
import org.apache.http.params.HttpParams;
import org.apache.http.protocol.BasicHttpContext;
import org.apache.http.protocol.HttpContext;
import org.apache.http.util.EntityUtils;
import java.net.SocketTimeoutException;
import java.net.UnknownHostException;
import java.sql.*;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.Executors;
import java.util.concurrent.ThreadPoolExecutor;
public class Scan {
private static Connection localConnection = null;
private static Statement localStatement = null;
private static ResultSet localResultSet = null;
private static int totalRows = 0;
static class GetThread implements Runnable {
private final HttpClient httpClient;
private final HttpContext context;
private final HttpGet httpget;
private int ID = 0;
private int statusCode = 999;
private String content = null;
public GetThread(HttpClient httpClient, HttpGet httpget, int ID) {
this.httpClient = httpClient;
this.context = new BasicHttpContext();
this.httpget = httpget;
this.ID = ID;
}
public void run() {
try {
HttpResponse response = httpClient.execute(httpget, context);
HttpEntity entity = response.getEntity();
statusCode = response.getStatusLine().getStatusCode();
content = "";
System.out.println("Check ID: " + ID);
if (entity != null) {
content = EntityUtils.toString(entity).replace("'", "").replace("\"", "");
}
} catch (UnknownHostException uhe) {
// host is off
content = "";
} catch (ConnectTimeoutException cte) {
// host is off
content = "";
} catch (SocketTimeoutException ste) {
// host is off
content = "";
} catch (Exception ex) {
content = "";
} finally {
httpget.abort();
}
String query = "UPDATE com SET status = '" + statusCode + "', content= '" + content + "' WHERE domainID=" + ID;
try {
localConnection.createStatement().execute(query);
} catch (SQLException localSQLException) {
System.out.println("SQLException: " + localSQLException);
}
}
}
public static void main(String[] args) {
System.out.println("Starting...");
try {
Class.forName("com.mysql.jdbc.Driver").newInstance();
} catch (Exception localException) {
System.out.println("com.mysql.jdbc.Driver: " + localException);
}
try {
localConnection = DriverManager.getConnection("jdbc:mysql://localhost/scan?user=root&password=0000");
localStatement = localConnection.createStatement();
countTotalRows();
HttpParams httpParams = new BasicHttpParams();
HttpConnectionParams.setConnectionTimeout(httpParams, 2000);
HttpConnectionParams.setSoTimeout(httpParams, 2000);
SchemeRegistry schemeRegistry = new SchemeRegistry();
schemeRegistry.register(new Scheme("http", PlainSocketFactory.getSocketFactory(), 80));
ClientConnectionManager cm = new ThreadSafeClientConnManager(httpParams, schemeRegistry);
HttpClient httpClient = new DefaultHttpClient(cm, httpParams);
int total = 10000;
ExecutorService executor = Executors.newFixedThreadPool(500);
Thread monitor = new Thread(new MyMonitorThread((ThreadPoolExecutor) executor));
monitor.setDaemon(true);
monitor.start();
while (totalRows > 0) {
localStatement.executeQuery("SELECT domainID, domain FROM com WHERE status = '' LIMIT " + total);
totalRows -= total;
localResultSet = localStatement.getResultSet();
while (localResultSet.next()) {
HttpGet httpget = new HttpGet("http://" + localResultSet.getString("domain").toLowerCase());
executor.submit(new GetThread(httpClient, httpget, localResultSet.getInt("domainID")));
}
Thread.sleep(1000);
while (((ThreadPoolExecutor) executor).getActiveCount() > 1) {
try {
Thread.sleep(1000);
} catch (InterruptedException e) {
e.printStackTrace();
}
}
localResultSet.close();
System.out.println("New cycle");
}
executor.shutdown();
monitor.interrupt();
localStatement.close();
localConnection.close();
} catch (SQLException localSQLException) {
System.out.println("SQLException: " + localSQLException);
} catch (InterruptedException e) {
e.printStackTrace();
}
System.out.println("DONE!");
}
private static void countTotalRows() throws SQLException {
localStatement.executeQuery("SELECT count(domainID) as total FROM com WHERE status = ''");
localResultSet = localStatement.getResultSet();
while (localResultSet.next())
totalRows = localResultSet.getInt("total");
localResultSet.close();
}
}
{code}
and the monitor class
{code}
class MyMonitorThread implements Runnable {
private ThreadPoolExecutor executor;
public MyMonitorThread(ThreadPoolExecutor executor) {
this.executor = executor;
}
@Override
public void run() {
try {
do {
System.out
.println(String
.format("[monitor] [%d/%d] Active: %d, Completed: %d, Task: %d, isShutdown: %s, isTerminated: %s",
this.executor.getPoolSize(),
this.executor.getCorePoolSize(),
this.executor.getActiveCount(),
this.executor.getCompletedTaskCount(),
this.executor.getTaskCount(),
this.executor.isShutdown(),
this.executor.isTerminated()));
Thread.sleep(5000);
} while (!Thread.currentThread().isInterrupted());
} catch (Exception e) {
}
}
}
{code}
I really tried to figure it out by myself with jmap, debugging, YourKit Profiler and failed. I would appreciate any help, thanks.