update
This commit is contained in:
+230
@@ -0,0 +1,230 @@
|
||||
package com.fangjia.script.companyCrawl
|
||||
|
||||
import com.fangjia.script.crawl.JQueryUtils
|
||||
import com.fangjia.script.util.ABuYunUtil
|
||||
import org.apache.http.Header
|
||||
import org.apache.http.HttpResponse
|
||||
import org.apache.http.client.HttpClient
|
||||
import org.apache.http.client.methods.HttpGet
|
||||
import org.jsoup.Jsoup
|
||||
import org.jsoup.select.Elements
|
||||
|
||||
import org.apache.http.impl.client.HttpClients;
|
||||
|
||||
|
||||
import java.util.concurrent.atomic.AtomicInteger
|
||||
|
||||
class GsxtCrawl {
|
||||
|
||||
def static final proxyPort = 9020;
|
||||
def static final proxyServer = "http-dyn.abuyun.com";
|
||||
def static final proxyUser = "HZU1XU28H3G4Z44D";
|
||||
def static final proxyPass = "466C86ED77FE43D4";
|
||||
|
||||
public final static int AbuyunMaxCount = 10;
|
||||
static AtomicInteger AbuyunRequestCount = new AtomicInteger(0);
|
||||
|
||||
def companyKeys = ["亨鑫","捷东","盈复","隆庆","卓源 ","合万","通广","安盛","东利","瑞科 ","凡迪","捷信","安盛","安盛","东进","源盛","茂禄","泰亚","宏成","光益","华圣","益全","长凯","宏新","锦立","宜佳","万美","正欣","和利","国宏","和裕","新昌","佳升","正德","源美","宝峰","金广","龙奥","宏美","成同","义丰","全洪","国聚","亨信","百益","广宏","仁瑞","亿昌","东旭","聚凯","洪成","兴同","赛迪","亨国","浩长","达宏","寿满","亿信","本盛","盈凯","浩汇","元辉","祥生","皇弘","讯思","新州","国泰","成合","茂浩","信源","润凯","虎光","安瑞","益吉","达兴","顺光","发盛","金优","捷丰","春凯","昌兴","本佳","远盛","盛达","亚飞","东昌","益如","宝东","德信","浩盈","多泰","泰联","升聚","辉丰","茂发","盈德","旭嘉","优源","益同","协盈","锦泰","捷久","本宏","德泰","聚富","全庆","凯安","进恒","全伟","优元","凯安","义丰","丰宇","瑞优","隆泰","诺德","台升","裕大","亨恒","义浩","百亨","欣昌","茂全","同亨","长圣","成谦","达泰","高协","正思","安巨","新耀","利仁","国益","旺洪","汇中","旭昌","弘康","优盛","旭达","晶安","华浩","耀春","圣本","康丰","光复","同凌","宇伯","乐鸿","成泰","宏协","华信","达泰","广丰","泰全","汇恒","杰森","天畅","恒通","亚达","庆合","丰圣","德长","茂百","汇久","欣鼎","谦和","厚辉","耀德","瑞隆","多辉","昌茂","成高","益升","安达","兴飞","百进","鼎宝","祥盛","信和","隆顺","亚欣","厚全","博文","讯诺","泰如","亨瑞","凯祥","捷浩","文舟","金庆","成仁","万协","宇中","宏祥","永多","艺峰","美益","创赢","奥龙","德宇","国福","飞利","久欣","如亨","安广","协佳","升捷","盛圣","富凯","长鑫","元卓","荣亨","亨仁","康鑫","信和","百新","干瑞","润万","富庭","信宇","源天","凯佳","正升","裕荣","厚泽","晶富","广新","正多","鼎高","丰汇","永华","兴升","复腾","隆诺","中发","广源","康胜","同进","仁捷","康凯","义洪","晶盛","正祥","裕祥","宇新","洪祥","润岳","义富","盛安","浩易"]
|
||||
static main(args) {
|
||||
def task = new GsxtCrawl();
|
||||
//task.process()
|
||||
task.run()
|
||||
}
|
||||
|
||||
def run(){
|
||||
HttpClient client = HttpClients.createDefault();
|
||||
HttpGet get = new HttpGet("http://www.gsxt.gov.cn/corp-query-homepage.html");
|
||||
get.setHeader("Upgrade-Insecure-Requests", "1");
|
||||
get.setHeader("User-Agent", "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/70.0.3538.110 Safari/537.36");
|
||||
|
||||
HttpResponse response = client.execute(get);
|
||||
println getJsluid(response)
|
||||
// fristCrawl()
|
||||
}
|
||||
|
||||
|
||||
static String getJsluid(HttpResponse response) {
|
||||
Header header = response.getFirstHeader("set-cookie");
|
||||
String[] split = header.getValue().split(";");
|
||||
for (String s : split) {
|
||||
if (s.contains("__jsluid")) {
|
||||
return s.trim();
|
||||
}
|
||||
}
|
||||
return "";
|
||||
}
|
||||
|
||||
|
||||
def fristCrawl(){
|
||||
def responseMap = [:]
|
||||
try {
|
||||
//headers信息
|
||||
Map<String, String> headers = new LinkedHashMap<String, String>()
|
||||
headers.put("Host","www.gsxt.gov.cn")
|
||||
headers.put("Connection","keep-alive")
|
||||
headers.put("Upgrade-Insecure-Requests","1")
|
||||
headers.put("Cache-Control","max-age=0")
|
||||
// headers.put("Content-Type","application/x-www-form-urlencoded")
|
||||
headers.put("User-Agent","Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/88.0.4324.104 Safari/537.36")
|
||||
headers.put("Accept","text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.9")
|
||||
// headers.put("Cookie","__jsluid_h=b4ff8b9b0fd86b6ff99edf27e111a9f2")
|
||||
headers.put("Accept-Language","zh-CN,zh;q=0.9")
|
||||
// headers.put("Accept-Encoding","gzip, deflate")
|
||||
|
||||
//String s = doPost("http://www.gsxt.gov.cn/corp-query-homepage.html", null, headers)
|
||||
def s = ABuYunUtil.doGet("http://www.gsxt.gov.cn/corp-query-homepage.html",null, headers, "utf-8")
|
||||
return s
|
||||
} catch (Exception e) {
|
||||
e.printStackTrace()
|
||||
}
|
||||
return null
|
||||
}
|
||||
|
||||
|
||||
void process(Object args) {
|
||||
|
||||
def url = "http://www.gsxt.gov.cn/corp-query-search-1.html "
|
||||
for(def key : companyKeys){
|
||||
// def queryBody = "tab=ent_tab&province=&geetest_challenge=1f31bf4565a8f89b93f4f4b25bb2143a&geetest_validate=9584ff51b3aff476fcac55248ec14459&geetest_seccode=9584ff51b3aff476fcac55248ec14459%7Cjordan&token=115406527&searchword="+ URLEncoder.encode("宏美", "utf-8")
|
||||
def queryBody = "tab=ent_tab&province=&geetest_challenge=daa40fb7efec814ff072b12ee12bb25a&geetest_validate=009a8748b31392ffdfb626fbbdccb4be&geetest_seccode=009a8748b31392ffdfb626fbbdccb4be%7Cjordan&token=115394764&searchword="+ URLEncoder.encode(key, "utf-8")
|
||||
def s = crawl(url, queryBody);
|
||||
if(s){
|
||||
print s
|
||||
parser(s);
|
||||
|
||||
}
|
||||
println s
|
||||
}
|
||||
}
|
||||
|
||||
def parser(content){
|
||||
def document = Jsoup.parse(content);
|
||||
if(document){
|
||||
def detailUrls = []
|
||||
Elements elements = JQueryUtils.select([".search_list_item"], document)
|
||||
if(elements && elements.size()>0){
|
||||
for(def el : elements){
|
||||
detailUrls << el.attr("href")
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
}
|
||||
}
|
||||
|
||||
def static crawl(url, bodyquery){
|
||||
def responseMap = [:]
|
||||
try {
|
||||
//headers信息
|
||||
Map<String, String> headers = new LinkedHashMap<String, String>()
|
||||
headers.put("Host","www.gsxt.gov.cn")
|
||||
headers.put("Connection","keep-alive")
|
||||
headers.put("Content-Length","226")
|
||||
headers.put("Cache-Control","max-age=0")
|
||||
headers.put("Upgrade-Insecure-Requests","1")
|
||||
headers.put("Origin","http://www.gsxt.gov.cn")
|
||||
headers.put("Content-Type","application/x-www-form-urlencoded")
|
||||
headers.put("User-Agent","Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/88.0.4324.104 Safari/537.36")
|
||||
headers.put("Accept","text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.9")
|
||||
headers.put("Referer","http://www.gsxt.gov.cn/corp-query-homepage.html")
|
||||
// headers.put("Accept-Encoding","gzip, deflate")
|
||||
headers.put("Accept-Language","zh-CN,zh;q=0.9")
|
||||
|
||||
// headers.put("Cookie:"," __jsluid_h=4e2326174f7f9d90ad9ac1f92301f2c9; JSESSIONID=8ac4f9ab894733d7c98c9a3d5c26; SECTOKEN=7446451709198207648; __jsl_clearance=1618823989.806|0|jQc7QRg%2B%2FzcIlz44iNwQg0PjZJg%3D; gsxtBrowseHistory1=%0FS%04%06%1D%04%1D%10SNS%24%26%3B%22%3D%3A71%3A%3B01%3A%219FFDDDDFFDFE%40DDDDEFDECDLE%40L%40L%40GSXS%11%1A%00%1A%15%19%11SNS%E5%91%BD%E6%9F%A3%E5%B9%B6%E6%97%84%E8%AE%AE%E7%AF%95%E5%BA%BE%E5%BA%8E%E8%AF%8A%E7%AF%95%E7%91%B2%E6%9D%BD%E9%98%A4%E5%84%98%E5%8E%8CSXS%11%1A%00%00%0D%04%11SNEEMDXS%02%1D%07%1D%00%00%1D%19%11SNEBELLFGCDL%40FG%09; tlb_cookie=S172.16.12.115")
|
||||
|
||||
|
||||
String s = doPost(url, bodyquery, headers)
|
||||
return s
|
||||
} catch (Exception e) {
|
||||
e.printStackTrace()
|
||||
}
|
||||
return null
|
||||
}
|
||||
|
||||
|
||||
static String doPost(String strUrl, String content, Map headers) {
|
||||
String result = "";
|
||||
OutputStream outputStream = null;
|
||||
InputStream inputStream = null;
|
||||
StringBuilder sb = new StringBuilder();
|
||||
try {
|
||||
checkAbuyun(true);
|
||||
|
||||
def responseMap = [:]
|
||||
// 创建代理服务器地址对象
|
||||
InetSocketAddress addr = new InetSocketAddress(proxyServer, proxyPort);
|
||||
// 创建HTTP类型代理对象
|
||||
Proxy proxy = new Proxy(Proxy.Type.HTTP, addr);
|
||||
|
||||
URL url = new URL(strUrl);
|
||||
//通过调用url.openConnection()来获得一个新的URLConnection对象,并且将其结果强制转换为HttpURLConnection.
|
||||
HttpURLConnection urlConnection = (HttpURLConnection) url.openConnection();
|
||||
urlConnection.setRequestMethod("POST");
|
||||
//设置连接的超时值为30000毫秒,超时将抛出SocketTimeoutException异常
|
||||
urlConnection.setConnectTimeout(30000);
|
||||
//设置读取的超时值为30000毫秒,超时将抛出SocketTimeoutException异常
|
||||
urlConnection.setReadTimeout(30000);
|
||||
//将url连接用于输出,这样才能使用getOutputStream()。getOutputStream()返回的输出流用于传输数据
|
||||
urlConnection.setDoOutput(true);
|
||||
urlConnection.setDoInput(true);
|
||||
//设置通用请求属性为默认浏览器编码类型
|
||||
// urlConnection.setRequestProperty("content-type", "application/x-www-form-urlencoded");
|
||||
|
||||
|
||||
if (headers != null && headers.keySet().size() > 0) {
|
||||
for (String head : headers.keySet()) {
|
||||
urlConnection.setRequestProperty(head, headers.get(head));
|
||||
}
|
||||
}
|
||||
//getOutputStream()返回的输出流,用于写入参数数据。
|
||||
outputStream = urlConnection.getOutputStream();
|
||||
if(content){
|
||||
outputStream.write(content.getBytes());
|
||||
}
|
||||
outputStream.flush();
|
||||
outputStream.close();
|
||||
//此时将调用接口方法。getInputStream()返回的输入流可以读取返回的数据。
|
||||
inputStream = urlConnection.getInputStream();
|
||||
|
||||
|
||||
InputStreamReader reader = new InputStreamReader(inputStream, "UTF-8");
|
||||
char[] buff = new char[1024];
|
||||
int length = 0;
|
||||
while ((length = reader.read(buff)) != -1) {
|
||||
String s = new String(buff, 0, length);
|
||||
sb.append(s);
|
||||
}
|
||||
} catch (IOException e) {
|
||||
e.printStackTrace();
|
||||
}finally{
|
||||
if(outputStream!=null){
|
||||
outputStream.close();
|
||||
}
|
||||
if(inputStream!=null){
|
||||
inputStream.close();
|
||||
}
|
||||
}
|
||||
|
||||
return sb.toString();
|
||||
}
|
||||
|
||||
public static checkAbuyun(boolean check) {
|
||||
if (check) {
|
||||
while (AbuyunRequestCount.get() >= AbuyunMaxCount) {
|
||||
try {
|
||||
Thread.sleep(5000);
|
||||
} catch (InterruptedException e) {
|
||||
e.printStackTrace()
|
||||
}
|
||||
}
|
||||
AbuyunRequestCount.incrementAndGet();
|
||||
} else {
|
||||
AbuyunRequestCount.decrementAndGet();
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
}
|
||||
+3
-3
@@ -61,10 +61,10 @@ class DealPriceMathBatch extends BaseBatch{
|
||||
districtMap.putAll(detailMap)
|
||||
}
|
||||
}
|
||||
def startDate = DateUtil.getFirstDayOfMonth(DateUtil.parseDate("2018-11-01","yyyy-MM-dd"))
|
||||
def startDate = DateUtil.getFirstDayOfMonth(DateUtil.parseDate("2020-01-01","yyyy-MM-dd"))
|
||||
// def startDate = DateUtil.getFirstDayOfMonth(new Date())
|
||||
// def endDate = DateUtil.getFirstDayOfMonth(new Date(), 1)
|
||||
def endDate = DateUtil.getFirstDayOfMonth(DateUtil.parseDate("2019-12-31","yyyy-MM-dd"))
|
||||
def endDate = DateUtil.getFirstDayOfMonth(DateUtil.parseDate("2021-04-30","yyyy-MM-dd"))
|
||||
|
||||
while(flag){
|
||||
println "开始加载解析数据......"
|
||||
@@ -596,7 +596,7 @@ class DealPriceMathBatch extends BaseBatch{
|
||||
}
|
||||
|
||||
def getOpenApiMatch(def city, def region, def name){
|
||||
def jsonObject = OpenApiClient.addressMatch(city, name, "property", false, region, "fuzzy", false)
|
||||
def jsonObject = OpenApiClient.addressMatch(city, name, "property", false, region, null,"fuzzy", false)
|
||||
if(jsonObject && jsonObject.credit>0 ){
|
||||
return [city:jsonObject.city, region:jsonObject.district, name:jsonObject.name, address:jsonObject.address]
|
||||
}
|
||||
|
||||
@@ -15,32 +15,35 @@ class LogStat {
|
||||
}
|
||||
|
||||
def run(){
|
||||
ExcelUtils.multiThreadingModify("D:\\data\\建行匹配小区2(1).xlsx",{ d ->
|
||||
def calDate = DateUtil.parseDate(d['M'], "yyyy-MM-dd")
|
||||
def weekend = DateUtils.getFirstDayOfWeek(calDate)
|
||||
def dbo = CommonDatautils.queryDistrictPrice(d['N'], d['O'], d['P'], weekend)
|
||||
if(dbo){
|
||||
d['E'] = dbo.avg_price
|
||||
ExcelUtils.multiThreadingModify("D:\\数据\\建行\\小区名单.xlsx",{ d ->
|
||||
// def calDate = DateUtil.parseDate(d['M'], "yyyy-MM-dd")
|
||||
// def weekend = DateUtils.getFirstDayOfWeek(calDate)
|
||||
// def dbo = CommonDatautils.queryDistrictPrice(d['N'], d['O'], d['P'], weekend)
|
||||
// if(dbo){
|
||||
// d['E'] = dbo.avg_price
|
||||
// }
|
||||
|
||||
if(d['A']){
|
||||
def district = CommonDatautils.findDistrict(d['A'], d['B'], d['C'], "district")
|
||||
if(district){
|
||||
def startDate = DateUtil.addMonth(new Date(), -6)
|
||||
def start = 20200901//startDate.replace("-","").toInteger()
|
||||
def end = 20201201//DateUtil.formatByPattern(calDate,"yyyyMMdd").toInteger()
|
||||
def alias = []
|
||||
if(district.alias){
|
||||
if(!district.alias.contains(district.name)){
|
||||
district.alias << district.name
|
||||
}
|
||||
alias = district.alias
|
||||
}else{
|
||||
alias << district.name
|
||||
}
|
||||
def count = findHouses(d['A'], d['B'], d['C'], alias, start, end)
|
||||
d['D'] = count
|
||||
println(d)
|
||||
}
|
||||
}
|
||||
|
||||
def district = CommonDatautils.findDistrict(d['N'], d['O'], d['P'], "district")
|
||||
if(district){
|
||||
def startDate = DateUtil.addMonth(new Date(), -6)
|
||||
def start = startDate.replace("-","").toInteger()
|
||||
def end = DateUtil.formatByPattern(calDate,"yyyyMMdd").toInteger()
|
||||
def alias = []
|
||||
if(district.alias){
|
||||
if(!district.alias.contains(district.name)){
|
||||
district.alias << district.name
|
||||
}
|
||||
alias = district.alias
|
||||
}else{
|
||||
alias << district.name
|
||||
}
|
||||
def count = findHouses(d['N'], d['O'], d['P'], alias, start, end)
|
||||
d['G'] = count
|
||||
println(d)
|
||||
}
|
||||
},0)
|
||||
}
|
||||
|
||||
@@ -48,7 +51,7 @@ class LogStat {
|
||||
def findHouses(def city, def region, def name, def alias, def startDate, def endDate){
|
||||
|
||||
def aliasStr = escapeExprSpecialWord(alias.join("','"))
|
||||
String sql = "select count(*) as c from latest_seawater where cat='sell' and city='${city}' and p_date > ${startDate} and region='${region}' and district_name in ('${aliasStr}')"
|
||||
String sql = "select count(*) as c from latest_seawater where cat='sell' and city='${city}' and p_date >= ${startDate} and p_date<${endDate} and region='${region}' and district_name in ('${aliasStr}')"
|
||||
def count = 0
|
||||
ElasticUtils.executeSqlQuery(sql,{data->
|
||||
count = data.c
|
||||
|
||||
+2
-2
@@ -47,7 +47,7 @@ class YouDaDataParser {
|
||||
def final AtomicInteger count_ = new AtomicInteger(0)
|
||||
|
||||
|
||||
static def statDate = "2021-02-01"
|
||||
static def statDate = "2021-03-01"
|
||||
static main(args) {
|
||||
def task = new YouDaDataParser();
|
||||
// task.newHouseparse();//友达一手房存入mongo
|
||||
@@ -285,7 +285,7 @@ class YouDaDataParser {
|
||||
|
||||
|
||||
def esfParser(){
|
||||
def files = ["D:\\数据\\友达\\2021年3月\\二手房\\查询1.DBF"]
|
||||
def files = ["D:\\数据\\友达\\2021年4月\\二手房\\查询1.DBF"]
|
||||
def datas = []
|
||||
for(def fileName : files){
|
||||
def rows = readDBF(fileName);
|
||||
|
||||
@@ -425,6 +425,7 @@ class ABuYunUtil {
|
||||
buffer.append(bf, 0, len);
|
||||
}
|
||||
} catch (IOException e) {
|
||||
e.printStackTrace()
|
||||
throw new Exception("doGet(String targetUrl, Map<String, String> params, Map<String, String> headers, String charset)异常, 异常信息为:" + e);
|
||||
} finally {
|
||||
// println("本次请求响应的connection.getResponseCode()为:" + connection.getResponseCode())
|
||||
|
||||
Reference in New Issue
Block a user