This commit is contained in:
2021-04-20 15:44:20 +08:00
parent b62e27ff26
commit 4ada2b415d
5 changed files with 264 additions and 30 deletions
@@ -0,0 +1,230 @@
package com.fangjia.script.companyCrawl
import com.fangjia.script.crawl.JQueryUtils
import com.fangjia.script.util.ABuYunUtil
import org.apache.http.Header
import org.apache.http.HttpResponse
import org.apache.http.client.HttpClient
import org.apache.http.client.methods.HttpGet
import org.jsoup.Jsoup
import org.jsoup.select.Elements
import org.apache.http.impl.client.HttpClients;
import java.util.concurrent.atomic.AtomicInteger
class GsxtCrawl {
def static final proxyPort = 9020;
def static final proxyServer = "http-dyn.abuyun.com";
def static final proxyUser = "HZU1XU28H3G4Z44D";
def static final proxyPass = "466C86ED77FE43D4";
public final static int AbuyunMaxCount = 10;
static AtomicInteger AbuyunRequestCount = new AtomicInteger(0);
def companyKeys = ["亨鑫","捷东","盈复","隆庆","卓源 ","合万","通广","安盛","东利","瑞科 ","凡迪","捷信","安盛","安盛","东进","源盛","茂禄","泰亚","宏成","光益","华圣","益全","长凯","宏新","锦立","宜佳","万美","正欣","和利","国宏","和裕","新昌","佳升","正德","源美","宝峰","金广","龙奥","宏美","成同","义丰","全洪","国聚","亨信","百益","广宏","仁瑞","亿昌","东旭","聚凯","洪成","兴同","赛迪","亨国","浩长","达宏","寿满","亿信","本盛","盈凯","浩汇","元辉","祥生","皇弘","讯思","新州","国泰","成合","茂浩","信源","润凯","虎光","安瑞","益吉","达兴","顺光","发盛","金优","捷丰","春凯","昌兴","本佳","远盛","盛达","亚飞","东昌","益如","宝东","德信","浩盈","多泰","泰联","升聚","辉丰","茂发","盈德","旭嘉","优源","益同","协盈","锦泰","捷久","本宏","德泰","聚富","全庆","凯安","进恒","全伟","优元","凯安","义丰","丰宇","瑞优","隆泰","诺德","台升","裕大","亨恒","义浩","百亨","欣昌","茂全","同亨","长圣","成谦","达泰","高协","正思","安巨","新耀","利仁","国益","旺洪","汇中","旭昌","弘康","优盛","旭达","晶安","华浩","耀春","圣本","康丰","光复","同凌","宇伯","乐鸿","成泰","宏协","华信","达泰","广丰","泰全","汇恒","杰森","天畅","恒通","亚达","庆合","丰圣","德长","茂百","汇久","欣鼎","谦和","厚辉","耀德","瑞隆","多辉","昌茂","成高","益升","安达","兴飞","百进","鼎宝","祥盛","信和","隆顺","亚欣","厚全","博文","讯诺","泰如","亨瑞","凯祥","捷浩","文舟","金庆","成仁","万协","宇中","宏祥","永多","艺峰","美益","创赢","奥龙","德宇","国福","飞利","久欣","如亨","安广","协佳","升捷","盛圣","富凯","长鑫","元卓","荣亨","亨仁","康鑫","信和","百新","干瑞","润万","富庭","信宇","源天","凯佳","正升","裕荣","厚泽","晶富","广新","正多","鼎高","丰汇","永华","兴升","复腾","隆诺","中发","广源","康胜","同进","仁捷","康凯","义洪","晶盛","正祥","裕祥","宇新","洪祥","润岳","义富","盛安","浩易"]
static main(args) {
def task = new GsxtCrawl();
//task.process()
task.run()
}
def run(){
HttpClient client = HttpClients.createDefault();
HttpGet get = new HttpGet("http://www.gsxt.gov.cn/corp-query-homepage.html");
get.setHeader("Upgrade-Insecure-Requests", "1");
get.setHeader("User-Agent", "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/70.0.3538.110 Safari/537.36");
HttpResponse response = client.execute(get);
println getJsluid(response)
// fristCrawl()
}
static String getJsluid(HttpResponse response) {
Header header = response.getFirstHeader("set-cookie");
String[] split = header.getValue().split(";");
for (String s : split) {
if (s.contains("__jsluid")) {
return s.trim();
}
}
return "";
}
def fristCrawl(){
def responseMap = [:]
try {
//headers信息
Map<String, String> headers = new LinkedHashMap<String, String>()
headers.put("Host","www.gsxt.gov.cn")
headers.put("Connection","keep-alive")
headers.put("Upgrade-Insecure-Requests","1")
headers.put("Cache-Control","max-age=0")
// headers.put("Content-Type","application/x-www-form-urlencoded")
headers.put("User-Agent","Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/88.0.4324.104 Safari/537.36")
headers.put("Accept","text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.9")
// headers.put("Cookie","__jsluid_h=b4ff8b9b0fd86b6ff99edf27e111a9f2")
headers.put("Accept-Language","zh-CN,zh;q=0.9")
// headers.put("Accept-Encoding","gzip, deflate")
//String s = doPost("http://www.gsxt.gov.cn/corp-query-homepage.html", null, headers)
def s = ABuYunUtil.doGet("http://www.gsxt.gov.cn/corp-query-homepage.html",null, headers, "utf-8")
return s
} catch (Exception e) {
e.printStackTrace()
}
return null
}
void process(Object args) {
def url = "http://www.gsxt.gov.cn/corp-query-search-1.html "
for(def key : companyKeys){
// def queryBody = "tab=ent_tab&province=&geetest_challenge=1f31bf4565a8f89b93f4f4b25bb2143a&geetest_validate=9584ff51b3aff476fcac55248ec14459&geetest_seccode=9584ff51b3aff476fcac55248ec14459%7Cjordan&token=115406527&searchword="+ URLEncoder.encode("宏美", "utf-8")
def queryBody = "tab=ent_tab&province=&geetest_challenge=daa40fb7efec814ff072b12ee12bb25a&geetest_validate=009a8748b31392ffdfb626fbbdccb4be&geetest_seccode=009a8748b31392ffdfb626fbbdccb4be%7Cjordan&token=115394764&searchword="+ URLEncoder.encode(key, "utf-8")
def s = crawl(url, queryBody);
if(s){
print s
parser(s);
}
println s
}
}
def parser(content){
def document = Jsoup.parse(content);
if(document){
def detailUrls = []
Elements elements = JQueryUtils.select([".search_list_item"], document)
if(elements && elements.size()>0){
for(def el : elements){
detailUrls << el.attr("href")
}
}
}
}
def static crawl(url, bodyquery){
def responseMap = [:]
try {
//headers信息
Map<String, String> headers = new LinkedHashMap<String, String>()
headers.put("Host","www.gsxt.gov.cn")
headers.put("Connection","keep-alive")
headers.put("Content-Length","226")
headers.put("Cache-Control","max-age=0")
headers.put("Upgrade-Insecure-Requests","1")
headers.put("Origin","http://www.gsxt.gov.cn")
headers.put("Content-Type","application/x-www-form-urlencoded")
headers.put("User-Agent","Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/88.0.4324.104 Safari/537.36")
headers.put("Accept","text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.9")
headers.put("Referer","http://www.gsxt.gov.cn/corp-query-homepage.html")
// headers.put("Accept-Encoding","gzip, deflate")
headers.put("Accept-Language","zh-CN,zh;q=0.9")
// headers.put("Cookie:"," __jsluid_h=4e2326174f7f9d90ad9ac1f92301f2c9; JSESSIONID=8ac4f9ab894733d7c98c9a3d5c26; SECTOKEN=7446451709198207648; __jsl_clearance=1618823989.806|0|jQc7QRg%2B%2FzcIlz44iNwQg0PjZJg%3D; gsxtBrowseHistory1=%0FS%04%06%1D%04%1D%10SNS%24%26%3B%22%3D%3A71%3A%3B01%3A%219FFDDDDFFDFE%40DDDDEFDECDLE%40L%40L%40GSXS%11%1A%00%1A%15%19%11SNS%E5%91%BD%E6%9F%A3%E5%B9%B6%E6%97%84%E8%AE%AE%E7%AF%95%E5%BA%BE%E5%BA%8E%E8%AF%8A%E7%AF%95%E7%91%B2%E6%9D%BD%E9%98%A4%E5%84%98%E5%8E%8CSXS%11%1A%00%00%0D%04%11SNEEMDXS%02%1D%07%1D%00%00%1D%19%11SNEBELLFGCDL%40FG%09; tlb_cookie=S172.16.12.115")
String s = doPost(url, bodyquery, headers)
return s
} catch (Exception e) {
e.printStackTrace()
}
return null
}
static String doPost(String strUrl, String content, Map headers) {
String result = "";
OutputStream outputStream = null;
InputStream inputStream = null;
StringBuilder sb = new StringBuilder();
try {
checkAbuyun(true);
def responseMap = [:]
// 创建代理服务器地址对象
InetSocketAddress addr = new InetSocketAddress(proxyServer, proxyPort);
// 创建HTTP类型代理对象
Proxy proxy = new Proxy(Proxy.Type.HTTP, addr);
URL url = new URL(strUrl);
//通过调用url.openConnection()来获得一个新的URLConnection对象,并且将其结果强制转换为HttpURLConnection.
HttpURLConnection urlConnection = (HttpURLConnection) url.openConnection();
urlConnection.setRequestMethod("POST");
//设置连接的超时值为30000毫秒,超时将抛出SocketTimeoutException异常
urlConnection.setConnectTimeout(30000);
//设置读取的超时值为30000毫秒,超时将抛出SocketTimeoutException异常
urlConnection.setReadTimeout(30000);
//将url连接用于输出,这样才能使用getOutputStream()。getOutputStream()返回的输出流用于传输数据
urlConnection.setDoOutput(true);
urlConnection.setDoInput(true);
//设置通用请求属性为默认浏览器编码类型
// urlConnection.setRequestProperty("content-type", "application/x-www-form-urlencoded");
if (headers != null && headers.keySet().size() > 0) {
for (String head : headers.keySet()) {
urlConnection.setRequestProperty(head, headers.get(head));
}
}
//getOutputStream()返回的输出流,用于写入参数数据。
outputStream = urlConnection.getOutputStream();
if(content){
outputStream.write(content.getBytes());
}
outputStream.flush();
outputStream.close();
//此时将调用接口方法。getInputStream()返回的输入流可以读取返回的数据。
inputStream = urlConnection.getInputStream();
InputStreamReader reader = new InputStreamReader(inputStream, "UTF-8");
char[] buff = new char[1024];
int length = 0;
while ((length = reader.read(buff)) != -1) {
String s = new String(buff, 0, length);
sb.append(s);
}
} catch (IOException e) {
e.printStackTrace();
}finally{
if(outputStream!=null){
outputStream.close();
}
if(inputStream!=null){
inputStream.close();
}
}
return sb.toString();
}
public static checkAbuyun(boolean check) {
if (check) {
while (AbuyunRequestCount.get() >= AbuyunMaxCount) {
try {
Thread.sleep(5000);
} catch (InterruptedException e) {
e.printStackTrace()
}
}
AbuyunRequestCount.incrementAndGet();
} else {
AbuyunRequestCount.decrementAndGet();
}
}
}
@@ -61,10 +61,10 @@ class DealPriceMathBatch extends BaseBatch{
districtMap.putAll(detailMap)
}
}
def startDate = DateUtil.getFirstDayOfMonth(DateUtil.parseDate("2018-11-01","yyyy-MM-dd"))
def startDate = DateUtil.getFirstDayOfMonth(DateUtil.parseDate("2020-01-01","yyyy-MM-dd"))
// def startDate = DateUtil.getFirstDayOfMonth(new Date())
// def endDate = DateUtil.getFirstDayOfMonth(new Date(), 1)
def endDate = DateUtil.getFirstDayOfMonth(DateUtil.parseDate("2019-12-31","yyyy-MM-dd"))
def endDate = DateUtil.getFirstDayOfMonth(DateUtil.parseDate("2021-04-30","yyyy-MM-dd"))
while(flag){
println "开始加载解析数据......"
@@ -596,7 +596,7 @@ class DealPriceMathBatch extends BaseBatch{
}
def getOpenApiMatch(def city, def region, def name){
def jsonObject = OpenApiClient.addressMatch(city, name, "property", false, region, "fuzzy", false)
def jsonObject = OpenApiClient.addressMatch(city, name, "property", false, region, null,"fuzzy", false)
if(jsonObject && jsonObject.credit>0 ){
return [city:jsonObject.city, region:jsonObject.district, name:jsonObject.name, address:jsonObject.address]
}
@@ -15,32 +15,35 @@ class LogStat {
}
def run(){
ExcelUtils.multiThreadingModify("D:\\data\\建行匹配小区2(1).xlsx",{ d ->
def calDate = DateUtil.parseDate(d['M'], "yyyy-MM-dd")
def weekend = DateUtils.getFirstDayOfWeek(calDate)
def dbo = CommonDatautils.queryDistrictPrice(d['N'], d['O'], d['P'], weekend)
if(dbo){
d['E'] = dbo.avg_price
ExcelUtils.multiThreadingModify("D:\\数据\\建行\\小区名单.xlsx",{ d ->
// def calDate = DateUtil.parseDate(d['M'], "yyyy-MM-dd")
// def weekend = DateUtils.getFirstDayOfWeek(calDate)
// def dbo = CommonDatautils.queryDistrictPrice(d['N'], d['O'], d['P'], weekend)
// if(dbo){
// d['E'] = dbo.avg_price
// }
if(d['A']){
def district = CommonDatautils.findDistrict(d['A'], d['B'], d['C'], "district")
if(district){
def startDate = DateUtil.addMonth(new Date(), -6)
def start = 20200901//startDate.replace("-","").toInteger()
def end = 20201201//DateUtil.formatByPattern(calDate,"yyyyMMdd").toInteger()
def alias = []
if(district.alias){
if(!district.alias.contains(district.name)){
district.alias << district.name
}
alias = district.alias
}else{
alias << district.name
}
def count = findHouses(d['A'], d['B'], d['C'], alias, start, end)
d['D'] = count
println(d)
}
}
def district = CommonDatautils.findDistrict(d['N'], d['O'], d['P'], "district")
if(district){
def startDate = DateUtil.addMonth(new Date(), -6)
def start = startDate.replace("-","").toInteger()
def end = DateUtil.formatByPattern(calDate,"yyyyMMdd").toInteger()
def alias = []
if(district.alias){
if(!district.alias.contains(district.name)){
district.alias << district.name
}
alias = district.alias
}else{
alias << district.name
}
def count = findHouses(d['N'], d['O'], d['P'], alias, start, end)
d['G'] = count
println(d)
}
},0)
}
@@ -48,7 +51,7 @@ class LogStat {
def findHouses(def city, def region, def name, def alias, def startDate, def endDate){
def aliasStr = escapeExprSpecialWord(alias.join("','"))
String sql = "select count(*) as c from latest_seawater where cat='sell' and city='${city}' and p_date > ${startDate} and region='${region}' and district_name in ('${aliasStr}')"
String sql = "select count(*) as c from latest_seawater where cat='sell' and city='${city}' and p_date >= ${startDate} and p_date<${endDate} and region='${region}' and district_name in ('${aliasStr}')"
def count = 0
ElasticUtils.executeSqlQuery(sql,{data->
count = data.c
@@ -47,7 +47,7 @@ class YouDaDataParser {
def final AtomicInteger count_ = new AtomicInteger(0)
static def statDate = "2021-02-01"
static def statDate = "2021-03-01"
static main(args) {
def task = new YouDaDataParser();
// task.newHouseparse();//友达一手房存入mongo
@@ -285,7 +285,7 @@ class YouDaDataParser {
def esfParser(){
def files = ["D:\\数据\\友达\\2021年3月\\二手房\\查询1.DBF"]
def files = ["D:\\数据\\友达\\2021年4月\\二手房\\查询1.DBF"]
def datas = []
for(def fileName : files){
def rows = readDBF(fileName);
@@ -425,6 +425,7 @@ class ABuYunUtil {
buffer.append(bf, 0, len);
}
} catch (IOException e) {
e.printStackTrace()
throw new Exception("doGet(String targetUrl, Map<String, String> params, Map<String, String> headers, String charset)异常, 异常信息为:" + e);
} finally {
// println("本次请求响应的connection.getResponseCode()为:" + connection.getResponseCode())