import java.io.UnsupportedEncodingException; import java.util.ArrayList; import java.util.List; import java.util.Map; import java.util.Set; import java.util.concurrent.atomic.AtomicInteger; import org.apache.http.HttpEntity; import org.apache.http.HttpHost; import org.apache.http.HttpResponse; import org.apache.http.client.HttpClient; import org.apache.http.client.entity.GzipDecompressingEntity; import org.apache.http.client.entity.UrlEncodedFormEntity; import org.apache.http.client.methods.HttpGet; import org.apache.http.client.methods.HttpPost; import org.apache.http.client.methods.HttpRequestBase; import org.apache.http.impl.client.DefaultHttpClient; import org.apache.http.message.BasicNameValuePair; import org.apache.http.params.CoreConnectionPNames; import org.apache.http.params.HttpParams; import org.apache.http.util.EntityUtils; import org.apache.http.conn.params.ConnRoutePNames ; import org.apache.http.NameValuePair; import java.util.concurrent.*; public class InvokeHttpRequest { static int reCount=10; static HttpClient httpClient; static boolean useProxy=true; static List PROXY_HOSTS = new ArrayList(){ /** * */ private static final long serialVersionUID = 1L;}; static{ PROXY_HOSTS.add("192.168.0.98"); PROXY_HOSTS.add("192.168.0.102"); PROXY_HOSTS.add("192.168.0.96"); PROXY_HOSTS.add("192.168.0.101"); PROXY_HOSTS.add("192.168.0.103"); PROXY_HOSTS.add("192.168.0.90"); PROXY_HOSTS.add("192.168.0.93"); PROXY_HOSTS.add("192.168.0.94"); PROXY_HOSTS.add("192.168.0.99"); PROXY_HOSTS.add("192.168.0.100"); } //static int PROXY_USE_COUNT=0; static AtomicInteger PROXY_USE_COUNT=new AtomicInteger(); public static void main(String args[]){ } static HttpHost getProxyHost(){ int useCount=PROXY_USE_COUNT.getAndIncrement(); int index=useCount%PROXY_HOSTS.size(); String host=PROXY_HOSTS.get(index); HttpHost proxy = new HttpHost(host,3234); return proxy; } public InvokeHttpRequest() { } public static HttpClient getHttpClient(){ if(httpClient ==null){ synchronized(InvokeHttpRequest.class){ if(httpClient == null){ httpClient = new DefaultHttpClient(); } } } return httpClient; } public static String get(String url, String encoding, String referer) { try { HttpRequestBase httpRequest = new HttpGet(url); HttpParams httpParams = httpRequest.getParams(); httpRequest.setHeader("Connection", "close"); httpRequest.setHeader("User-Agent", "Mozilla/5.0 (Windows NT 5.1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/28.0.1500.71 Safari/537.36"); httpRequest.setHeader("Accept", "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8"); httpRequest.setHeader("Accept-Encoding", "gzip,deflate,sdch"); httpRequest.setHeader("Accept-Language", "zh-CN,zh;q=0.8"); if(referer != null){ httpRequest.setHeader("referer", referer); } httpParams.setParameter(CoreConnectionPNames.CONNECTION_TIMEOUT, 10 * 1000); httpParams.setParameter(CoreConnectionPNames.SO_TIMEOUT, 60 * 1000); HttpClient httpClient = new DefaultHttpClient(); int count=0; while(count<=reCount){ // httpClient.getParams().setParameter(ConnRoutePNames.DEFAULT_PROXY, // getProxyHost()); try{ HttpResponse response = httpClient.execute(httpRequest); HttpEntity entity = response.getEntity(); if(response.getFirstHeader("Content-Encoding")!=null && "gzip".equals(response.getFirstHeader("Content-Encoding").getValue())){ entity=new GzipDecompressingEntity(entity); } if (entity != null) { byte[] content = null; content = EntityUtils.toByteArray(entity); String contents = new String(content, encoding); if(response.getStatusLine().getStatusCode()!=200){ throw new Exception("http error: "+response.getStatusLine().getStatusCode()); } return contents; } }catch(Exception e){ // httpRequest.abort(); // httpClient.getConnectionManager().shutdown() // println e } count++; } } catch (Exception e) { e.printStackTrace(); } return null; } public static byte[] getData(String url, String referer) { try { HttpRequestBase httpRequest = new HttpGet(url); HttpParams httpParams = httpRequest.getParams(); httpRequest.setHeader("Connection", "close"); httpRequest.setHeader("User-Agent", "Mozilla/5.0 (Windows NT 5.1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/28.0.1500.71 Safari/537.36"); httpRequest.setHeader("Accept", "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8"); httpRequest.setHeader("Accept-Encoding", "gzip,deflate,sdch"); httpRequest.setHeader("Accept-Language", "zh-CN,zh;q=0.8"); if(referer != null){ httpRequest.setHeader("referer", referer); } httpParams.setParameter(CoreConnectionPNames.CONNECTION_TIMEOUT, 10 * 1000); httpParams.setParameter(CoreConnectionPNames.SO_TIMEOUT, 60 * 1000); HttpClient httpClient = new DefaultHttpClient(); int count=0; while(count<=reCount){ httpClient.getParams().setParameter(ConnRoutePNames.DEFAULT_PROXY, getProxyHost()); try{ HttpResponse response = httpClient.execute(httpRequest); HttpEntity entity = response.getEntity(); if(response.getFirstHeader("Content-Encoding")!=null && "gzip".equals(response.getFirstHeader("Content-Encoding").getValue())){ entity=new GzipDecompressingEntity(entity); } if (entity != null) { byte[] content = null; content = EntityUtils.toByteArray(entity); if(response.getStatusLine().getStatusCode()!=200){ throw new Exception("http error: "+response.getStatusLine().getStatusCode()); } return content; } }catch(Exception e){ // httpRequest.abort(); // httpClient.getConnectionManager().shutdown() // println e } count++; } } catch (Exception e) { e.printStackTrace(); } return null; } public static String post(String url, Map params, String encoding, String referer) { try { HttpPost httpRequest = new HttpPost(url); List nvps = new ArrayList (); Set keySet = params.keySet(); for(String key : keySet) { nvps.add(new BasicNameValuePair(key, String.valueOf(params.get(key)))); } try { httpRequest.setEntity(new UrlEncodedFormEntity(nvps, encoding)); } catch (UnsupportedEncodingException e) { e.printStackTrace(); } HttpParams httpParams = httpRequest.getParams(); httpRequest.setHeader("Connection", "close"); httpRequest.setHeader("User-Agent", "Mozilla/5.0 (Windows NT 5.1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/27.0.1453.110"); if(referer != null ){ httpRequest.setHeader("referer", referer); } httpParams.setParameter(CoreConnectionPNames.CONNECTION_TIMEOUT, 20 * 1000); httpParams.setParameter(CoreConnectionPNames.SO_TIMEOUT, 60 * 1000); HttpClient httpClient = new DefaultHttpClient(); int count=0; while(count<=reCount){ httpClient.getParams().setParameter(ConnRoutePNames.DEFAULT_PROXY, getProxyHost()); try{ HttpResponse response = httpClient.execute(httpRequest); HttpEntity entity = response.getEntity(); if(response.getFirstHeader("Content-Encoding")!=null && "gzip".equals(response.getFirstHeader("Content-Encoding").getValue())){ entity=new GzipDecompressingEntity(entity); } if (entity != null) { byte[] content = null; content = EntityUtils.toByteArray(entity); String contents = new String(content, encoding); if(response.getStatusLine().getStatusCode()!=200){ throw new Exception("http error: "+response.getStatusLine().getStatusCode()); } return contents; } }catch(Exception e){ // println e } count++; } } catch (Exception e) { e.printStackTrace(); } return null; } }