diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/companyCrawl/GsxtCrawl.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/companyCrawl/GsxtCrawl.groovy new file mode 100644 index 000000000..2fed8b9c0 --- /dev/null +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/companyCrawl/GsxtCrawl.groovy @@ -0,0 +1,230 @@ +package com.fangjia.script.companyCrawl + +import com.fangjia.script.crawl.JQueryUtils +import com.fangjia.script.util.ABuYunUtil +import org.apache.http.Header +import org.apache.http.HttpResponse +import org.apache.http.client.HttpClient +import org.apache.http.client.methods.HttpGet +import org.jsoup.Jsoup +import org.jsoup.select.Elements + +import org.apache.http.impl.client.HttpClients; + + +import java.util.concurrent.atomic.AtomicInteger + +class GsxtCrawl { + + def static final proxyPort = 9020; + def static final proxyServer = "http-dyn.abuyun.com"; + def static final proxyUser = "HZU1XU28H3G4Z44D"; + def static final proxyPass = "466C86ED77FE43D4"; + + public final static int AbuyunMaxCount = 10; + static AtomicInteger AbuyunRequestCount = new AtomicInteger(0); + + def companyKeys = ["亨鑫","捷东","盈复","隆庆","卓源 ","合万","通广","安盛","东利","瑞科 ","凡迪","捷信","安盛","安盛","东进","源盛","茂禄","泰亚","宏成","光益","华圣","益全","长凯","宏新","锦立","宜佳","万美","正欣","和利","国宏","和裕","新昌","佳升","正德","源美","宝峰","金广","龙奥","宏美","成同","义丰","全洪","国聚","亨信","百益","广宏","仁瑞","亿昌","东旭","聚凯","洪成","兴同","赛迪","亨国","浩长","达宏","寿满","亿信","本盛","盈凯","浩汇","元辉","祥生","皇弘","讯思","新州","国泰","成合","茂浩","信源","润凯","虎光","安瑞","益吉","达兴","顺光","发盛","金优","捷丰","春凯","昌兴","本佳","远盛","盛达","亚飞","东昌","益如","宝东","德信","浩盈","多泰","泰联","升聚","辉丰","茂发","盈德","旭嘉","优源","益同","协盈","锦泰","捷久","本宏","德泰","聚富","全庆","凯安","进恒","全伟","优元","凯安","义丰","丰宇","瑞优","隆泰","诺德","台升","裕大","亨恒","义浩","百亨","欣昌","茂全","同亨","长圣","成谦","达泰","高协","正思","安巨","新耀","利仁","国益","旺洪","汇中","旭昌","弘康","优盛","旭达","晶安","华浩","耀春","圣本","康丰","光复","同凌","宇伯","乐鸿","成泰","宏协","华信","达泰","广丰","泰全","汇恒","杰森","天畅","恒通","亚达","庆合","丰圣","德长","茂百","汇久","欣鼎","谦和","厚辉","耀德","瑞隆","多辉","昌茂","成高","益升","安达","兴飞","百进","鼎宝","祥盛","信和","隆顺","亚欣","厚全","博文","讯诺","泰如","亨瑞","凯祥","捷浩","文舟","金庆","成仁","万协","宇中","宏祥","永多","艺峰","美益","创赢","奥龙","德宇","国福","飞利","久欣","如亨","安广","协佳","升捷","盛圣","富凯","长鑫","元卓","荣亨","亨仁","康鑫","信和","百新","干瑞","润万","富庭","信宇","源天","凯佳","正升","裕荣","厚泽","晶富","广新","正多","鼎高","丰汇","永华","兴升","复腾","隆诺","中发","广源","康胜","同进","仁捷","康凯","义洪","晶盛","正祥","裕祥","宇新","洪祥","润岳","义富","盛安","浩易"] + static main(args) { + def task = new GsxtCrawl(); + //task.process() + task.run() + } + + def run(){ + HttpClient client = HttpClients.createDefault(); + HttpGet get = new HttpGet("http://www.gsxt.gov.cn/corp-query-homepage.html"); + get.setHeader("Upgrade-Insecure-Requests", "1"); + get.setHeader("User-Agent", "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/70.0.3538.110 Safari/537.36"); + + HttpResponse response = client.execute(get); + println getJsluid(response) + // fristCrawl() + } + + + static String getJsluid(HttpResponse response) { + Header header = response.getFirstHeader("set-cookie"); + String[] split = header.getValue().split(";"); + for (String s : split) { + if (s.contains("__jsluid")) { + return s.trim(); + } + } + return ""; + } + + + def fristCrawl(){ + def responseMap = [:] + try { + //headers信息 + Map headers = new LinkedHashMap() + headers.put("Host","www.gsxt.gov.cn") + headers.put("Connection","keep-alive") + headers.put("Upgrade-Insecure-Requests","1") + headers.put("Cache-Control","max-age=0") +// headers.put("Content-Type","application/x-www-form-urlencoded") + headers.put("User-Agent","Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/88.0.4324.104 Safari/537.36") + headers.put("Accept","text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.9") +// headers.put("Cookie","__jsluid_h=b4ff8b9b0fd86b6ff99edf27e111a9f2") + headers.put("Accept-Language","zh-CN,zh;q=0.9") +// headers.put("Accept-Encoding","gzip, deflate") + + //String s = doPost("http://www.gsxt.gov.cn/corp-query-homepage.html", null, headers) + def s = ABuYunUtil.doGet("http://www.gsxt.gov.cn/corp-query-homepage.html",null, headers, "utf-8") + return s + } catch (Exception e) { + e.printStackTrace() + } + return null + } + + + void process(Object args) { + + def url = "http://www.gsxt.gov.cn/corp-query-search-1.html " + for(def key : companyKeys){ + // def queryBody = "tab=ent_tab&province=&geetest_challenge=1f31bf4565a8f89b93f4f4b25bb2143a&geetest_validate=9584ff51b3aff476fcac55248ec14459&geetest_seccode=9584ff51b3aff476fcac55248ec14459%7Cjordan&token=115406527&searchword="+ URLEncoder.encode("宏美", "utf-8") + def queryBody = "tab=ent_tab&province=&geetest_challenge=daa40fb7efec814ff072b12ee12bb25a&geetest_validate=009a8748b31392ffdfb626fbbdccb4be&geetest_seccode=009a8748b31392ffdfb626fbbdccb4be%7Cjordan&token=115394764&searchword="+ URLEncoder.encode(key, "utf-8") + def s = crawl(url, queryBody); + if(s){ + print s + parser(s); + + } + println s + } + } + + def parser(content){ + def document = Jsoup.parse(content); + if(document){ + def detailUrls = [] + Elements elements = JQueryUtils.select([".search_list_item"], document) + if(elements && elements.size()>0){ + for(def el : elements){ + detailUrls << el.attr("href") + } + } + + + } + } + + def static crawl(url, bodyquery){ + def responseMap = [:] + try { + //headers信息 + Map headers = new LinkedHashMap() + headers.put("Host","www.gsxt.gov.cn") + headers.put("Connection","keep-alive") + headers.put("Content-Length","226") + headers.put("Cache-Control","max-age=0") + headers.put("Upgrade-Insecure-Requests","1") + headers.put("Origin","http://www.gsxt.gov.cn") + headers.put("Content-Type","application/x-www-form-urlencoded") + headers.put("User-Agent","Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/88.0.4324.104 Safari/537.36") + headers.put("Accept","text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.9") + headers.put("Referer","http://www.gsxt.gov.cn/corp-query-homepage.html") +// headers.put("Accept-Encoding","gzip, deflate") + headers.put("Accept-Language","zh-CN,zh;q=0.9") + +// headers.put("Cookie:"," __jsluid_h=4e2326174f7f9d90ad9ac1f92301f2c9; JSESSIONID=8ac4f9ab894733d7c98c9a3d5c26; SECTOKEN=7446451709198207648; __jsl_clearance=1618823989.806|0|jQc7QRg%2B%2FzcIlz44iNwQg0PjZJg%3D; gsxtBrowseHistory1=%0FS%04%06%1D%04%1D%10SNS%24%26%3B%22%3D%3A71%3A%3B01%3A%219FFDDDDFFDFE%40DDDDEFDECDLE%40L%40L%40GSXS%11%1A%00%1A%15%19%11SNS%E5%91%BD%E6%9F%A3%E5%B9%B6%E6%97%84%E8%AE%AE%E7%AF%95%E5%BA%BE%E5%BA%8E%E8%AF%8A%E7%AF%95%E7%91%B2%E6%9D%BD%E9%98%A4%E5%84%98%E5%8E%8CSXS%11%1A%00%00%0D%04%11SNEEMDXS%02%1D%07%1D%00%00%1D%19%11SNEBELLFGCDL%40FG%09; tlb_cookie=S172.16.12.115") + + + String s = doPost(url, bodyquery, headers) + return s + } catch (Exception e) { + e.printStackTrace() + } + return null + } + + + static String doPost(String strUrl, String content, Map headers) { + String result = ""; + OutputStream outputStream = null; + InputStream inputStream = null; + StringBuilder sb = new StringBuilder(); + try { + checkAbuyun(true); + + def responseMap = [:] + // 创建代理服务器地址对象 + InetSocketAddress addr = new InetSocketAddress(proxyServer, proxyPort); + // 创建HTTP类型代理对象 + Proxy proxy = new Proxy(Proxy.Type.HTTP, addr); + + URL url = new URL(strUrl); + //通过调用url.openConnection()来获得一个新的URLConnection对象,并且将其结果强制转换为HttpURLConnection. + HttpURLConnection urlConnection = (HttpURLConnection) url.openConnection(); + urlConnection.setRequestMethod("POST"); + //设置连接的超时值为30000毫秒,超时将抛出SocketTimeoutException异常 + urlConnection.setConnectTimeout(30000); + //设置读取的超时值为30000毫秒,超时将抛出SocketTimeoutException异常 + urlConnection.setReadTimeout(30000); + //将url连接用于输出,这样才能使用getOutputStream()。getOutputStream()返回的输出流用于传输数据 + urlConnection.setDoOutput(true); + urlConnection.setDoInput(true); + //设置通用请求属性为默认浏览器编码类型 +// urlConnection.setRequestProperty("content-type", "application/x-www-form-urlencoded"); + + + if (headers != null && headers.keySet().size() > 0) { + for (String head : headers.keySet()) { + urlConnection.setRequestProperty(head, headers.get(head)); + } + } + //getOutputStream()返回的输出流,用于写入参数数据。 + outputStream = urlConnection.getOutputStream(); + if(content){ + outputStream.write(content.getBytes()); + } + outputStream.flush(); + outputStream.close(); + //此时将调用接口方法。getInputStream()返回的输入流可以读取返回的数据。 + inputStream = urlConnection.getInputStream(); + + + InputStreamReader reader = new InputStreamReader(inputStream, "UTF-8"); + char[] buff = new char[1024]; + int length = 0; + while ((length = reader.read(buff)) != -1) { + String s = new String(buff, 0, length); + sb.append(s); + } + } catch (IOException e) { + e.printStackTrace(); + }finally{ + if(outputStream!=null){ + outputStream.close(); + } + if(inputStream!=null){ + inputStream.close(); + } + } + + return sb.toString(); + } + + public static checkAbuyun(boolean check) { + if (check) { + while (AbuyunRequestCount.get() >= AbuyunMaxCount) { + try { + Thread.sleep(5000); + } catch (InterruptedException e) { + e.printStackTrace() + } + } + AbuyunRequestCount.incrementAndGet(); + } else { + AbuyunRequestCount.decrementAndGet(); + } + } + + + + + +} diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/dealPrice/DealPriceMathBatch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/dealPrice/DealPriceMathBatch.groovy index b3be6c694..ca2ee7334 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/dealPrice/DealPriceMathBatch.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/dealPrice/DealPriceMathBatch.groovy @@ -61,10 +61,10 @@ class DealPriceMathBatch extends BaseBatch{ districtMap.putAll(detailMap) } } - def startDate = DateUtil.getFirstDayOfMonth(DateUtil.parseDate("2018-11-01","yyyy-MM-dd")) + def startDate = DateUtil.getFirstDayOfMonth(DateUtil.parseDate("2020-01-01","yyyy-MM-dd")) // def startDate = DateUtil.getFirstDayOfMonth(new Date()) // def endDate = DateUtil.getFirstDayOfMonth(new Date(), 1) - def endDate = DateUtil.getFirstDayOfMonth(DateUtil.parseDate("2019-12-31","yyyy-MM-dd")) + def endDate = DateUtil.getFirstDayOfMonth(DateUtil.parseDate("2021-04-30","yyyy-MM-dd")) while(flag){ println "开始加载解析数据......" @@ -596,7 +596,7 @@ class DealPriceMathBatch extends BaseBatch{ } def getOpenApiMatch(def city, def region, def name){ - def jsonObject = OpenApiClient.addressMatch(city, name, "property", false, region, "fuzzy", false) + def jsonObject = OpenApiClient.addressMatch(city, name, "property", false, region, null,"fuzzy", false) if(jsonObject && jsonObject.credit>0 ){ return [city:jsonObject.city, region:jsonObject.district, name:jsonObject.name, address:jsonObject.address] } diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/LogStat.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/LogStat.groovy index 6305d35b9..1d490eea2 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/LogStat.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/LogStat.groovy @@ -15,32 +15,35 @@ class LogStat { } def run(){ - ExcelUtils.multiThreadingModify("D:\\data\\建行匹配小区2(1).xlsx",{ d -> - def calDate = DateUtil.parseDate(d['M'], "yyyy-MM-dd") - def weekend = DateUtils.getFirstDayOfWeek(calDate) - def dbo = CommonDatautils.queryDistrictPrice(d['N'], d['O'], d['P'], weekend) - if(dbo){ - d['E'] = dbo.avg_price + ExcelUtils.multiThreadingModify("D:\\数据\\建行\\小区名单.xlsx",{ d -> +// def calDate = DateUtil.parseDate(d['M'], "yyyy-MM-dd") +// def weekend = DateUtils.getFirstDayOfWeek(calDate) +// def dbo = CommonDatautils.queryDistrictPrice(d['N'], d['O'], d['P'], weekend) +// if(dbo){ +// d['E'] = dbo.avg_price +// } + + if(d['A']){ + def district = CommonDatautils.findDistrict(d['A'], d['B'], d['C'], "district") + if(district){ + def startDate = DateUtil.addMonth(new Date(), -6) + def start = 20200901//startDate.replace("-","").toInteger() + def end = 20201201//DateUtil.formatByPattern(calDate,"yyyyMMdd").toInteger() + def alias = [] + if(district.alias){ + if(!district.alias.contains(district.name)){ + district.alias << district.name + } + alias = district.alias + }else{ + alias << district.name + } + def count = findHouses(d['A'], d['B'], d['C'], alias, start, end) + d['D'] = count + println(d) + } } - def district = CommonDatautils.findDistrict(d['N'], d['O'], d['P'], "district") - if(district){ - def startDate = DateUtil.addMonth(new Date(), -6) - def start = startDate.replace("-","").toInteger() - def end = DateUtil.formatByPattern(calDate,"yyyyMMdd").toInteger() - def alias = [] - if(district.alias){ - if(!district.alias.contains(district.name)){ - district.alias << district.name - } - alias = district.alias - }else{ - alias << district.name - } - def count = findHouses(d['N'], d['O'], d['P'], alias, start, end) - d['G'] = count - println(d) - } },0) } @@ -48,7 +51,7 @@ class LogStat { def findHouses(def city, def region, def name, def alias, def startDate, def endDate){ def aliasStr = escapeExprSpecialWord(alias.join("','")) - String sql = "select count(*) as c from latest_seawater where cat='sell' and city='${city}' and p_date > ${startDate} and region='${region}' and district_name in ('${aliasStr}')" + String sql = "select count(*) as c from latest_seawater where cat='sell' and city='${city}' and p_date >= ${startDate} and p_date<${endDate} and region='${region}' and district_name in ('${aliasStr}')" def count = 0 ElasticUtils.executeSqlQuery(sql,{data-> count = data.c diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/YouDaDataParser.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/YouDaDataParser.groovy index a3a6e7989..32b251e77 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/YouDaDataParser.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/YouDaDataParser.groovy @@ -47,7 +47,7 @@ class YouDaDataParser { def final AtomicInteger count_ = new AtomicInteger(0) - static def statDate = "2021-02-01" + static def statDate = "2021-03-01" static main(args) { def task = new YouDaDataParser(); // task.newHouseparse();//友达一手房存入mongo @@ -285,7 +285,7 @@ class YouDaDataParser { def esfParser(){ - def files = ["D:\\数据\\友达\\2021年3月\\二手房\\查询1.DBF"] + def files = ["D:\\数据\\友达\\2021年4月\\二手房\\查询1.DBF"] def datas = [] for(def fileName : files){ def rows = readDBF(fileName); diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/util/ABuYunUtil.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/util/ABuYunUtil.groovy index 4bb2eaa12..f983a1c60 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/util/ABuYunUtil.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/util/ABuYunUtil.groovy @@ -425,6 +425,7 @@ class ABuYunUtil { buffer.append(bf, 0, len); } } catch (IOException e) { + e.printStackTrace() throw new Exception("doGet(String targetUrl, Map params, Map headers, String charset)异常, 异常信息为:" + e); } finally { // println("本次请求响应的connection.getResponseCode()为:" + connection.getResponseCode())