266 lines
7.9 KiB
Java
266 lines
7.9 KiB
Java
|
|
import java.io.UnsupportedEncodingException;
|
|
import java.util.ArrayList;
|
|
import java.util.List;
|
|
import java.util.Map;
|
|
import java.util.Set;
|
|
import java.util.concurrent.atomic.AtomicInteger;
|
|
|
|
import org.apache.http.HttpEntity;
|
|
import org.apache.http.HttpHost;
|
|
import org.apache.http.HttpResponse;
|
|
import org.apache.http.client.HttpClient;
|
|
import org.apache.http.client.entity.GzipDecompressingEntity;
|
|
import org.apache.http.client.entity.UrlEncodedFormEntity;
|
|
import org.apache.http.client.methods.HttpGet;
|
|
import org.apache.http.client.methods.HttpPost;
|
|
import org.apache.http.client.methods.HttpRequestBase;
|
|
import org.apache.http.impl.client.DefaultHttpClient;
|
|
import org.apache.http.message.BasicNameValuePair;
|
|
import org.apache.http.params.CoreConnectionPNames;
|
|
import org.apache.http.params.HttpParams;
|
|
import org.apache.http.util.EntityUtils;
|
|
import org.apache.http.conn.params.ConnRoutePNames ;
|
|
import org.apache.http.NameValuePair;
|
|
|
|
import java.util.concurrent.*;
|
|
|
|
public class InvokeHttpRequest {
|
|
|
|
static int reCount=10;
|
|
|
|
static HttpClient httpClient;
|
|
|
|
static boolean useProxy=true;
|
|
|
|
static List<String> PROXY_HOSTS = new ArrayList<String>(){
|
|
|
|
/**
|
|
*
|
|
*/
|
|
private static final long serialVersionUID = 1L;};
|
|
|
|
static{
|
|
PROXY_HOSTS.add("192.168.0.98");
|
|
PROXY_HOSTS.add("192.168.0.102");
|
|
PROXY_HOSTS.add("192.168.0.96");
|
|
PROXY_HOSTS.add("192.168.0.101");
|
|
PROXY_HOSTS.add("192.168.0.103");
|
|
PROXY_HOSTS.add("192.168.0.90");
|
|
PROXY_HOSTS.add("192.168.0.93");
|
|
PROXY_HOSTS.add("192.168.0.94");
|
|
PROXY_HOSTS.add("192.168.0.99");
|
|
PROXY_HOSTS.add("192.168.0.100");
|
|
}
|
|
//static int PROXY_USE_COUNT=0;
|
|
static AtomicInteger PROXY_USE_COUNT=new AtomicInteger();
|
|
|
|
public static void main(String args[]){
|
|
|
|
}
|
|
static HttpHost getProxyHost(){
|
|
int useCount=PROXY_USE_COUNT.getAndIncrement();
|
|
|
|
int index=useCount%PROXY_HOSTS.size();
|
|
|
|
String host=PROXY_HOSTS.get(index);
|
|
|
|
HttpHost proxy = new HttpHost(host,3234);
|
|
|
|
return proxy;
|
|
|
|
}
|
|
public InvokeHttpRequest() {
|
|
|
|
}
|
|
|
|
public static HttpClient getHttpClient(){
|
|
if(httpClient ==null){
|
|
synchronized(InvokeHttpRequest.class){
|
|
if(httpClient == null){
|
|
httpClient = new DefaultHttpClient();
|
|
|
|
}
|
|
}
|
|
}
|
|
return httpClient;
|
|
}
|
|
|
|
public static String get(String url, String encoding, String referer) {
|
|
|
|
try {
|
|
HttpRequestBase httpRequest = new HttpGet(url);
|
|
HttpParams httpParams = httpRequest.getParams();
|
|
httpRequest.setHeader("Connection", "close");
|
|
httpRequest.setHeader("User-Agent", "Mozilla/5.0 (Windows NT 5.1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/28.0.1500.71 Safari/537.36");
|
|
httpRequest.setHeader("Accept", "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8");
|
|
httpRequest.setHeader("Accept-Encoding", "gzip,deflate,sdch");
|
|
httpRequest.setHeader("Accept-Language", "zh-CN,zh;q=0.8");
|
|
|
|
if(referer != null){
|
|
httpRequest.setHeader("referer", referer);
|
|
}
|
|
|
|
httpParams.setParameter(CoreConnectionPNames.CONNECTION_TIMEOUT, 10 * 1000);
|
|
httpParams.setParameter(CoreConnectionPNames.SO_TIMEOUT, 60 * 1000);
|
|
HttpClient httpClient = new DefaultHttpClient();
|
|
|
|
int count=0;
|
|
while(count<=reCount){
|
|
// httpClient.getParams().setParameter(ConnRoutePNames.DEFAULT_PROXY,
|
|
// getProxyHost());
|
|
try{
|
|
HttpResponse response = httpClient.execute(httpRequest);
|
|
|
|
HttpEntity entity = response.getEntity();
|
|
|
|
if(response.getFirstHeader("Content-Encoding")!=null && "gzip".equals(response.getFirstHeader("Content-Encoding").getValue())){
|
|
entity=new GzipDecompressingEntity(entity);
|
|
}
|
|
if (entity != null) {
|
|
byte[] content = null;
|
|
content = EntityUtils.toByteArray(entity);
|
|
String contents = new String(content, encoding);
|
|
|
|
if(response.getStatusLine().getStatusCode()!=200){
|
|
throw new Exception("http error: "+response.getStatusLine().getStatusCode());
|
|
}
|
|
|
|
return contents;
|
|
}
|
|
}catch(Exception e){
|
|
// httpRequest.abort();
|
|
// httpClient.getConnectionManager().shutdown()
|
|
// println e
|
|
}
|
|
count++;
|
|
}
|
|
|
|
} catch (Exception e) {
|
|
e.printStackTrace();
|
|
}
|
|
|
|
return null;
|
|
}
|
|
|
|
public static byte[] getData(String url, String referer) {
|
|
|
|
try {
|
|
HttpRequestBase httpRequest = new HttpGet(url);
|
|
HttpParams httpParams = httpRequest.getParams();
|
|
httpRequest.setHeader("Connection", "close");
|
|
httpRequest.setHeader("User-Agent", "Mozilla/5.0 (Windows NT 5.1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/28.0.1500.71 Safari/537.36");
|
|
httpRequest.setHeader("Accept", "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8");
|
|
httpRequest.setHeader("Accept-Encoding", "gzip,deflate,sdch");
|
|
httpRequest.setHeader("Accept-Language", "zh-CN,zh;q=0.8");
|
|
|
|
if(referer != null){
|
|
httpRequest.setHeader("referer", referer);
|
|
}
|
|
|
|
httpParams.setParameter(CoreConnectionPNames.CONNECTION_TIMEOUT, 10 * 1000);
|
|
httpParams.setParameter(CoreConnectionPNames.SO_TIMEOUT, 60 * 1000);
|
|
HttpClient httpClient = new DefaultHttpClient();
|
|
|
|
int count=0;
|
|
while(count<=reCount){
|
|
httpClient.getParams().setParameter(ConnRoutePNames.DEFAULT_PROXY,
|
|
getProxyHost());
|
|
try{
|
|
HttpResponse response = httpClient.execute(httpRequest);
|
|
|
|
HttpEntity entity = response.getEntity();
|
|
|
|
if(response.getFirstHeader("Content-Encoding")!=null && "gzip".equals(response.getFirstHeader("Content-Encoding").getValue())){
|
|
entity=new GzipDecompressingEntity(entity);
|
|
}
|
|
if (entity != null) {
|
|
byte[] content = null;
|
|
content = EntityUtils.toByteArray(entity);
|
|
|
|
if(response.getStatusLine().getStatusCode()!=200){
|
|
throw new Exception("http error: "+response.getStatusLine().getStatusCode());
|
|
}
|
|
|
|
return content;
|
|
}
|
|
}catch(Exception e){
|
|
// httpRequest.abort();
|
|
// httpClient.getConnectionManager().shutdown()
|
|
// println e
|
|
}
|
|
count++;
|
|
}
|
|
|
|
} catch (Exception e) {
|
|
e.printStackTrace();
|
|
}
|
|
|
|
return null;
|
|
}
|
|
|
|
public static String post(String url, Map<String, String> params, String encoding, String referer) {
|
|
|
|
try {
|
|
HttpPost httpRequest = new HttpPost(url);
|
|
List<NameValuePair> nvps = new ArrayList <NameValuePair>();
|
|
|
|
Set<String> keySet = params.keySet();
|
|
for(String key : keySet) {
|
|
nvps.add(new BasicNameValuePair(key, String.valueOf(params.get(key))));
|
|
}
|
|
|
|
try {
|
|
httpRequest.setEntity(new UrlEncodedFormEntity(nvps, encoding));
|
|
} catch (UnsupportedEncodingException e) {
|
|
e.printStackTrace();
|
|
}
|
|
|
|
HttpParams httpParams = httpRequest.getParams();
|
|
httpRequest.setHeader("Connection", "close");
|
|
httpRequest.setHeader("User-Agent", "Mozilla/5.0 (Windows NT 5.1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/27.0.1453.110");
|
|
if(referer != null ){
|
|
httpRequest.setHeader("referer", referer);
|
|
}
|
|
httpParams.setParameter(CoreConnectionPNames.CONNECTION_TIMEOUT, 20 * 1000);
|
|
httpParams.setParameter(CoreConnectionPNames.SO_TIMEOUT, 60 * 1000);
|
|
|
|
HttpClient httpClient = new DefaultHttpClient();
|
|
|
|
int count=0;
|
|
while(count<=reCount){
|
|
httpClient.getParams().setParameter(ConnRoutePNames.DEFAULT_PROXY,
|
|
getProxyHost());
|
|
try{
|
|
HttpResponse response = httpClient.execute(httpRequest);
|
|
|
|
HttpEntity entity = response.getEntity();
|
|
if(response.getFirstHeader("Content-Encoding")!=null && "gzip".equals(response.getFirstHeader("Content-Encoding").getValue())){
|
|
entity=new GzipDecompressingEntity(entity);
|
|
}
|
|
|
|
if (entity != null) {
|
|
byte[] content = null;
|
|
content = EntityUtils.toByteArray(entity);
|
|
String contents = new String(content, encoding);
|
|
|
|
if(response.getStatusLine().getStatusCode()!=200){
|
|
throw new Exception("http error: "+response.getStatusLine().getStatusCode());
|
|
}
|
|
|
|
return contents;
|
|
}
|
|
}catch(Exception e){
|
|
// println e
|
|
}
|
|
count++;
|
|
}
|
|
|
|
} catch (Exception e) {
|
|
e.printStackTrace();
|
|
}
|
|
|
|
return null;
|
|
}
|
|
}
|