From 2a8dfcedd65218a0213899d2a68e851cf1c154f4 Mon Sep 17 00:00:00 2001 From: Jian Shen Date: Mon, 18 Apr 2022 10:00:52 +0800 Subject: [PATCH 1/3] =?UTF-8?q?=E4=BF=AE=E6=94=B9=E9=87=8D=E5=BA=86?= =?UTF-8?q?=E5=86=9C=E5=95=86=E8=A1=8C=E5=BC=80=E5=8F=91=E5=95=86=E6=95=B0?= =?UTF-8?q?=E6=8D=AE=E5=AF=BC=E5=87=BA?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../com/fangjia/script/temp/shenjian/cqrcb/SeaweedExport.groovy | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/cqrcb/SeaweedExport.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/cqrcb/SeaweedExport.groovy index a437ea0c3..9817c00cb 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/cqrcb/SeaweedExport.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/cqrcb/SeaweedExport.groovy @@ -76,7 +76,7 @@ class SeaweedExport { ret[1] = obj.developer ret[2] = SeaweedUtils.formatCubagePercent(obj.cubage_percent) ret[3] = SeaweedUtils.formatVirescencePercent(obj.virescence_percent) - ret[4] = SeaweedUtils.formatAlias(obj.alias) + ret[4] = SeaweedUtils.formatAlias(obj.mapping_alias) ret[5] = SeaweedUtils.formatCarSiteCount(obj.car_site_count) ret[6] = SeaweedUtils.calcCarAndHouseholdRadio(obj.car_site_count, obj.household_count) ret[7] = "" From 90de7d5288d317458c93f142e8da92d74843bade Mon Sep 17 00:00:00 2001 From: gaojuan Date: Mon, 18 Apr 2022 15:08:57 +0800 Subject: [PATCH 2/3] =?UTF-8?q?=E6=88=BF=E6=BA=90=E6=97=B6=E9=97=B4?= =?UTF-8?q?=E4=BF=AE=E6=94=B9?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../com/fangjia/script/ccbevaluation/BigdataForsaleBatch.groovy | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataForsaleBatch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataForsaleBatch.groovy index 2749ed7f4..4978bd962 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataForsaleBatch.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataForsaleBatch.groovy @@ -74,7 +74,7 @@ class BigdataForsaleBatch extends BaseBatch{ // def distIds = getDistrictIds(cityCodeDataMap.get(city)) def distCodeMap = CcbCommonUtils.findAllDistrictCodeValue(city) def districts = CcbCommonUtils.findAllDistrictByCity(city, "district", ["city", "region", "name", "lng","estate_type2","name_cat","address", "alias", "finish_building_date", "house_property", "property_life", "build_type", "build_structure"]); - exportSellCSV(index, city, cityCodeDataMap, distCodeMap, districts, 20220218, 20220319); + exportSellCSV(index, city, cityCodeDataMap, distCodeMap, districts, 20220318, 20220419); // exportSellCSV(index, city, cityCodeDataMap, distCodeMap, districts, 20210801, 20210831); //测试通过匹配接口配房源效果 // exportSellCSV(index, city, cityCodeDataMap, distCodeMap, districts, 20201201, 20211118) From 54c92a53cc69c2c63b7d285af63ebb15fd4e1be9 Mon Sep 17 00:00:00 2001 From: gaojuan Date: Mon, 18 Apr 2022 15:17:15 +0800 Subject: [PATCH 3/3] update --- projects/fangjia-script/pom.xml | 27 + .../ccbevaluation/BigdataDealBatch.groovy | 1 + .../ccbevaluation/GuiDingPriceBatch.groovy | 2 +- .../com/fangjia/script/fuzhou/Test1.groovy | 2 +- .../com/fangjia/script/gj/OpenApiUtils.groovy | 2 +- .../script/gj/paimai/Caa123Crawl.groovy | 63 ++- .../fangjia/script/gj/paimai/DataUtil.groovy | 290 ++++++++++ .../script/gj/paimai/JdPaimaiCrawl.groovy | 528 ++++++++++++++++++ .../script/gj/paimai/PaimaiDataUtil.groovy | 79 ++- .../script/gj/paimai/PaimaiTest.groovy | 53 +- .../script/gj/paimai/RmfysszcCrawl.groovy | 419 +++++++++++--- .../fangjia/script/loudong/CqfdcMatch.groovy | 173 ++++++ .../script/thirdPriceCrawl/DataStat.groovy | 8 +- .../script/thirdPriceCrawl/Esf58Crawl.groovy | 83 +-- .../thirdPriceCrawl/SynchDistrictPrice.groovy | 6 +- 15 files changed, 1564 insertions(+), 172 deletions(-) create mode 100644 projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/paimai/DataUtil.groovy create mode 100644 projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/paimai/JdPaimaiCrawl.groovy create mode 100644 projects/fangjia-script/src/main/groovy/com/fangjia/script/loudong/CqfdcMatch.groovy diff --git a/projects/fangjia-script/pom.xml b/projects/fangjia-script/pom.xml index c71006bb8..9469b6d02 100644 --- a/projects/fangjia-script/pom.xml +++ b/projects/fangjia-script/pom.xml @@ -494,5 +494,32 @@ msgpack 0.6.12 + + + + com.asprise.ocr + java-ocr-api + [15,) + + + + net.java.dev.jna + jna + 4.1.0 + + + + net.sourceforge.tess4j + tess4j + 3.4.0 + + + com.sun.jna + jna + + + + + diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataDealBatch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataDealBatch.groovy index aa12539b9..d34dedcc2 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataDealBatch.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataDealBatch.groovy @@ -508,6 +508,7 @@ class BigdataDealBatch extends BaseBatch{ def q = new BasicDBObject() q.put("contractTime", new BasicDBObject('$gte', startDate).append('$lte', endDate)); q.put("area", new BasicDBObject('$gte', 20)) + q.put("status", 0) def dbcursor = youda_esfhouse_coll.find(q).batchSize(2000) def rets = [] diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/GuiDingPriceBatch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/GuiDingPriceBatch.groovy index 412677ae5..2476de4c3 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/GuiDingPriceBatch.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/GuiDingPriceBatch.groovy @@ -44,7 +44,7 @@ class GuiDingPriceBatch extends BaseBatch{ def distCodeMap = CcbCommonUtils.findAllDistrictCodeValue(city) //添加自己库中的人工指导价 - def prices = CcbCommonUtils.findPersonPrice(city, "2022-03-1", "2022-04-1") + def prices = CcbCommonUtils.findPersonPrice(city, "2022-04-1", "2022-05-1") if(prices){ def distMap = getDistrictMap(city) def rets = createCityDatas(cityCodeDatas, distCodeMap, distMap, prices) diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/fuzhou/Test1.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/fuzhou/Test1.groovy index a8a616b7c..5a6b21aac 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/fuzhou/Test1.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/fuzhou/Test1.groovy @@ -14,7 +14,7 @@ import com.mongodb.DBCollection //福州第三方均价统计,发送钉钉消息 //def datas = ExcelUtils.read("d:\\数据\\福州\\福州4月2.xlsx",0,1) //def datas = CsvUtils.read("d:\\数据\\福州\\FJW_THIRD_PARTY_AVG_PRICE_202105.csv"); -def datas = readTxt("d:\\数据\\福州\\FJW_THIRD_PARTY_AVG_PRICE_202202.csv") +def datas = readTxt("d:\\数据\\福州\\FJW_THIRD_PARTY_AVG_PRICE_202203.csv") def count0 = 0 def count1 = 0 def count2 = 0 diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/OpenApiUtils.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/OpenApiUtils.groovy index 244ae41c0..77239e3ea 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/OpenApiUtils.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/OpenApiUtils.groovy @@ -13,7 +13,7 @@ import org.apache.commons.lang.StringUtils class OpenApiUtils { static def matchDistrict(def city, def address, def suggestDistrict, def preciseLevelflag){ - address = address.replace("(","(").replace(")",")").replace(" ","").replace("#","").replace("<","").replace(">","") + address = address.replace("(","(").replace(")",")").replace(" ","").replace("#","").replace("<","").replace(">","").replace("%","").replace("\"","") def url = "http://open.fangjia.com/address/matchV2?city="+city+"&address="+address+"&suggestDistrict="+suggestDistrict+"&token=F54F52381C49BB9EB4A33EB1B65604AE4B71A28AEE53518A94A2F360408B9056D57553931D15CE6DDE765562DAD286BA38E05A4CDAFC51C3D527A4959BF8E75A3B95DB7108FCEA340DDE61925616DB55118E1851E67D83EAD800460D100D6B667A4ED8EE67C8F7FB&category=district&__refresh_cache=1" if(preciseLevelflag){ url = url +"&preciseLevel=strict" diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/paimai/Caa123Crawl.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/paimai/Caa123Crawl.groovy index 4229926a1..afaae338f 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/paimai/Caa123Crawl.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/paimai/Caa123Crawl.groovy @@ -88,9 +88,10 @@ def getDetailLinks(driver, cityList){ driver.findElement(By.className("next")).click() } } - println city +" "+urls.size() + def rets = batchParse(urls); saveToDB(rets); + println city +" "+urls.size() } } println cityMap.provice +" 完成" @@ -123,7 +124,7 @@ def getCity(driver, proviceMap){ for(def c : cityEles){ def cityid = c.attr("data-cityid") def city = c.text() - if(!["石家庄市","唐山市","秦皇岛市","邯郸市","邢台市","保定市"].contains(city)){ + if(!["石家庄市","太原市","长治市","朔州市","廊坊市","衡水市","唐山市","秦皇岛市","邯郸市","邢台市","保定市","沧州市","张家口市","承德市"].contains(city)){ cityMap.put(city, cityid) } @@ -170,8 +171,7 @@ def batchParse(def urls){ WebDriver driver = new ChromeDriver() def ret = [] for(def map :urls){ - driver.get(map.url); - def doc = Jsoup.parse(driver.getPageSource()); + def doc = getDocument(map.url, driver) def res = parse(map.province, map.city, map.url, doc); if(res){ ret << res @@ -180,6 +180,30 @@ def batchParse(def urls){ return ret; } +def getDocument(url, driver){ + def stop = false + def error = 0 + def doc = null + while(!stop){ + try{ + driver.get(url) + doc = Jsoup.parse(driver.getPageSource()); + if(doc){ + stop = true; + }else{ + error++ + Thread.sleep(5000) + } + }catch(Exception e){ + error++ + } + if(error>=10){ + stop = true; + } + } + return doc +} + def parse(province, city, url,doc){ if(doc){ def result = [:] @@ -205,7 +229,12 @@ def parse(province, city, url,doc){ def finishDateStr = JQueryUtils.selectText(["#sf-countdown .time"], doc) if (finishDateStr) { - result.finishDate = com.fangjia.script.util.DateUtils.parseFromTightByPattern(finishDateStr, "yyyy年MM月dd日 HH:mm"); + try{ + result.finishDate = com.fangjia.script.util.DateUtils.parseFromTightByPattern(finishDateStr, "yyyy年MM月dd日 HH:mm"); + }catch(Exception e){ + + } + } def curPriceStr = JQueryUtils.selectText([".colorred"], doc) @@ -255,7 +284,12 @@ def parse(province, city, url,doc){ startDateStr = startDateStr.toString().substring(startDateStr.indexOf("定于")+2, startDateStr.indexOf("日")+1).trim() } - result.startDate = com.fangjia.script.util.DateUtils.parseString2Date(startDateStr) + try{ + result.startDate = com.fangjia.script.util.DateUtils.parseString2Date(startDateStr.trim()) + }catch(Exception e){ + + } + def imgs = JQueryUtils.select(".sf-pic-slide img", doc) if (imgs) { def imgLinks = [] @@ -315,7 +349,7 @@ def saveToDB(rets){ ret.each{k,v-> obj.put(k,v) } - + obj.source= "中拍网" if(dbo){ obj.m_date = new Date() paimaiResultColl.update(q, new BasicDBObject('$set', obj)) @@ -346,15 +380,18 @@ def getAreaNumber(input, maxArea) { val = val.toString().substring(val.toString().indexOf("室")+1, val.toString().size()) } def v = val.replace("平方米", "").replace("㎡", "").replace("平", "").replace("方米", "").replace(",", "").replace("m2", ""); - if (maxArea) { - if (v.toDouble() < maxArea) { - area = v; + if(v && v.toString().isNumber()){ + if (maxArea) { + if (v.toDouble() < maxArea) { + area = v; + break; + } + } else { + area = v break; } - } else { - area = v - break; } + } if (area) { diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/paimai/DataUtil.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/paimai/DataUtil.groovy new file mode 100644 index 000000000..8cce171c6 --- /dev/null +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/paimai/DataUtil.groovy @@ -0,0 +1,290 @@ +package com.fangjia.script.gj.paimai + +import org.apache.commons.codec.binary.Base64 +import org.apache.commons.lang.StringUtils +import sun.font.Font2D +import sun.font.Font2DHandle +import sun.font.TrueTypeFont +import sun.font.TrueTypeGlyphMapper + +import java.awt.Font +import java.awt.font.FontRenderContext +import java.awt.font.GlyphVector +import java.awt.geom.AffineTransform +import java.lang.reflect.Field +import java.util.regex.Matcher +import java.util.regex.Pattern + +class DataUtil { + static def getAreaNumber(def input, def maxArea) { + Pattern pattern = Pattern.compile('\\d+(.\\d+)?(平方米|㎡|m2|方米|平)') + Matcher matcher = pattern.matcher(input) + + List list = new ArrayList() + while (matcher.find()) { + list.add(matcher.group()) + } + if (list.size() > 0) { + def area = null + + for (def val : list) { + def v = val.replace("平方米", "").replace("㎡", "").replace("平", "").replace("方米", "").replace(",", "").replace("m2", "") + if (maxArea) { + if (v.isDouble()) { + if (v.toDouble() < maxArea) { + area = v + break + } + } + } else { + area = v + break + } + } + + if (area) { + return area + } + } + return null + } + + static def getPaiMaiType(def s) { + if (s) { + if (s.toString().contains("结束")) { + return "已完成" + } else if (s.toString().contains("进行") || s.toString().contains("正在拍卖")) { + return "进行中" + } else if (s.toString().contains("即将开始")) { + return "即将开始" + } else if (s.toString().contains("流拍")) { + return "流拍" + } else if (s.toString().contains("撤回")) { + return "撤回" + } else if (s.toString().contains("暂缓")) { + return "暂缓" + } else if (s.toString().contains("中止")) { + return "中止" + } else if (s.toString().contains("失败")) { + return "失败" + } + } + return s + } + + + static def getDateRange(def s) { + s = s.replace("起", "") + Pattern pattern = Pattern.compile('法院将于(\\d+年\\d{1,2}[月|日]\\d{1,2}日\\d{0,2}时?)至(\\d+年\\d{1,2}月\\d{1,2}日\\d{0,2}时?)') + Matcher matcher = pattern.matcher(s) + + List list = new ArrayList() + while (matcher.find()) { + def str = matcher.group() + list.add(str) +// if(str) { +// def indx1 = str.indexOf("年") +// def indx2 = str.indexOf("月") +// def month = str.substring(indx1+1, indx2); +// if(month.toInteger()<12) { +// list.add(str); +// } +// } + } + if (list.size() >= 1) { + String s1 = list.get(0) + s1 = s1.replace("法院将于", "").replace(" ", "") + def array = s1.split("至") + return array + } + return null + } + + + static def getStartPrice(def s) { + Pattern pattern = Pattern.compile('起拍价:\\d+(.\\d+)?万?元') + Matcher matcher = pattern.matcher(s) + + List list = new ArrayList() + while (matcher.find()) { + list.add(matcher.group()) + } + if (list.size() >= 1) { + String s1 = list.get(0) + if (s1.contains("万元")) { + s1 = s1.toString().replace("起拍价:", "").replace("万元", "").replace(",", "").replace(",", "") + return (int) (s1.toDouble() * 10000) + } else { + s1 = s1.toString().replace("起拍价:", "").replace("元", "").replace(",", "").replace(",", "") + if (s1.isInteger()) { + return s1.toInteger() + } else { + return (int) (s1.toDouble()) + } + } + } + return null + } + + static def getEstateType(def s) { + Pattern pattern = Pattern.compile('房屋类型:.*;') + Matcher matcher = pattern.matcher(s) + + List list = new ArrayList() + while (matcher.find()) { + list.add(matcher.group()) + } + if (list.size() >= 1) { + String s1 = list.get(0) + if (s1) { + s1 = s1.substring(s1.indexOf(":") + 1, s1.indexOf(";")) + return s1.replace(")", "").replace("】", "") + } + } + return null + } + + static def findEstateType(def s) { + if (s) { + def villTypes = ["独栋", "联排", "联列", "叠加", "双拼", "花园面积", "泳池", "别墅", "花园住宅", "带花园", "洋房", "叠拼", "独院", "联体", "微墅", "大平墅"] + for (def type : villTypes) { + if (s.contains(type)) { + return "别墅" + } + } + + def districtTypes = ["住宅", "公寓"] + for (def type : districtTypes) { + if (s.contains(type)) { + return "普通住宅" + } + } + } + + return null + + } + + static def filterChar(def detailMap) { + def newDetailMap = [:] + detailMap.each { def k, v -> + if (k.toString().contains(".")) { + newDetailMap.put(k.toString().replace(".", "\\u002e"), v) + } else { + newDetailMap.put(k, v) + } + } + return newDetailMap + } + + /** + * 匹配符合条件的字符串 + * + * @param str + * @param pattern + * @return + */ + static String match(String str, Pattern pattern) { + if (str == null) { + return null + } + Matcher matcher = pattern.matcher(str) + if (matcher.find()) { + return matcher.group() + } + return null + } + + /** + * font-secret字符串专用解密工具 + * + * @param key 密匙 + * @param encodeString 加密后的字符串 + * @return 解密后的字符串 + */ + static String decodeString(String key, String encodeString) { + if (!key) { + return "" + } + try { + //base64解码,初始化字体 + Base64 base64 = new Base64() + byte[] ss = base64.decodeBase64(key) + InputStream inputStream = new ByteArrayInputStream(ss) + Font dynamicFont = Font.createFont(Font.TRUETYPE_FONT, inputStream) + FontRenderContext fontRenderContext = new FontRenderContext(new AffineTransform(), false, false) + GlyphVector glyphVector = dynamicFont.createGlyphVector(fontRenderContext, "") + + //获取font中字形的映射关系,字段为private,使用反射 + Class clazz = Font.class + Field[] fs = clazz.getDeclaredFields() + Font2DHandle font2DHandle = null + for (int i = 0; i < fs.length; i++) { + fs[i].setAccessible(true)// 将目标属性设置为可以访问 + if (fs[i].getName().equals("font2DHandle")) { + font2DHandle = (Font2DHandle) fs[i].get(dynamicFont) + } + + } + + //得到映射关系 + Font2D font2D = font2DHandle.font2D + TrueTypeFont trueTypeFont = (TrueTypeFont) font2D + TrueTypeGlyphMapper charToGlyphMapper = (TrueTypeGlyphMapper) trueTypeFont.getMapper() + + //开始解密,encodeString为加密后的字符串 + StringBuffer buffer = new StringBuffer() + char[] chars = encodeString.toCharArray() + for (int i = 0; i < chars.length; i++) { + buffer.append(charToGlyphMapper.charToGlyph(chars[i]) - 1) + } + return buffer.toString() + } catch (Exception e) { + e.printStackTrace() + } + return "" + } + + static List SITE_SUFFIXS = [".gov.cn",".com.cn", ".com", ".cn", ".net", ".org", ".cc", ".tm", ".biz", ".info", ".tv"] + + static String urlToSiteName(String url) { + if (StringUtils.isEmpty(url)) { + return null + } + + for (String suffix : SITE_SUFFIXS) { + if (url.indexOf(suffix) > -1) { + String s = StringUtils.substringBefore(url, suffix) + if (s.indexOf(".") > -1) { + s = StringUtils.substringAfterLast(s, ".") + if (StringUtils.isNotBlank(s)) { + return s + suffix + } + } else if (s.indexOf("://") > -1) { + s = StringUtils.substringAfterLast(s, "://") + if (StringUtils.isNotBlank(s)) { + return s + suffix + } + } else if (s.indexOf("://") == -1 && s.indexOf(".") == -1) { + return s + suffix + } + + return null + } + } + return null + } + + static double getRound(int scale, double dSource) { + BigDecimal deSource = new BigDecimal(dSource) + return deSource.setScale(scale, BigDecimal.ROUND_HALF_UP).doubleValue() + } + + static def filterEmpty(def s){ + if(s && s.contains("暂无")){ + return null + } + return s + } + +} diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/paimai/JdPaimaiCrawl.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/paimai/JdPaimaiCrawl.groovy new file mode 100644 index 000000000..aac9c7315 --- /dev/null +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/paimai/JdPaimaiCrawl.groovy @@ -0,0 +1,528 @@ +package com.fangjia.script.gj.paimai + +import com.fangjia.core.RegionData +import com.fangjia.script.crawl.JQueryUtils +import com.fangjia.script.gj.OpenApiUtils +import com.fangjia.script.thirdPriceCrawl.CrawalUtil +import com.fangjia.script.util.ABuYunUtil +import com.fangjia.script.util.DateUtils +import com.fangjia.script.util.MongoUtils +import com.mongodb.BasicDBObject +import com.mongodb.DBCollection +import com.mongodb.DBObject +import net.sf.json.JSONObject +import org.apache.poi.ss.util.RegionUtil +import org.jsoup.Jsoup +import org.jsoup.nodes.Document +import org.openqa.selenium.WebDriver +import org.openqa.selenium.chrome.ChromeDriver + +import javax.sound.midi.MidiSystem +import java.util.regex.Pattern + +class JdPaimaiCrawl { + + DBCollection paimaiResultColl = MongoUtils.getCollectionByAuth20190102("fangjia", "pai_mai") + + static main(args) { + def task = new JdPaimaiCrawl(); + task.run(); + } + def run(){ + def existsIds = findAllIds() + println "已存在:"+existsIds.size() + System.setProperty("webdriver.chrome.driver", "C:\\Users\\gaojuan\\AppData\\Local\\Google\\Chrome\\Application\\chromedriver.exe"); + System.setProperty("http.nonProxyHosts", "localhost"); + WebDriver driver = new ChromeDriver() + def links = getList(driver) + def ids = getDetailLinks(links, driver) + println "共:"+ids.size() + parserList( existsIds,ids, driver) + + + + } + + def getList(driver){ + def list = [] + for(def i=1;i<=32;i++){ + if(i==8){ + list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=560&_=1564047626268" + list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=573&_=1564047626268" + list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=579&_=1564047626268" + list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=584&_=1564047626268" + list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=589&_=1564047626268" + list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=593&_=1564047626268" + list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=598&_=1564047626268" + list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=604&_=1564047626268" + list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=609&_=1564047626268" + list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=613&_=1564047626268" + list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=617&_=1564047626268" + list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=621&_=1564047626268" + list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=632&_=1564047626268" + list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=6858&_=1564047626268" + }else if(i==5){ + list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=142&_=1564047626268" + list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=148&_=1564047626268" + list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=164&_=1564047626268" + list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=199&_=1564047626268" + list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=224&_=1564047626268" + list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=239&_=1564047626268" + list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=248&_=1564047626268" + list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=258&_=1564047626268" + list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=264&_=1564047626268" + list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=274&_=1564047626268" + list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=275&_=1564047626268" + }else if(i==17){ + list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=1432&_=1564047626268" + list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=1441&_=1564047626268" + list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=1458&_=1564047626268" + list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=1466&_=1564047626268" + list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=1475&_=1564047626268" + list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=1477&_=1564047626268" + list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=1479&_=1564047626268" + list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=3154&_=1564047626268" + list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=1381&_=1564047626268" + list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=1387&_=1564047626268" + list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=1396&_=1564047626268" + list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=1405&_=1564047626268" + list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=1413&_=1564047626268" + list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=1421&_=1564047626268" + list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=2922&_=1564047626268" + list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=2980&_=1564047626268" + list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=2983&_=1564047626268" + }else if(i==19){ + list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=1601&_=1564047626268" + list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=1607&_=1564047626268" + list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=1609&_=1564047626268" + list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=1611&_=1564047626268" + list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=1617&_=1564047626268" + list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=1627&_=1564047626268" + list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=1634&_=1564047626268" + list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=1709&_=1564047626268" + list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=1643&_=1564047626268" + list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=1650&_=1564047626268" + list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=1655&_=1564047626268" + list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=1607&_=1564047626268" + list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=1609&_=1564047626268" + list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=1666&_=1564047626268" + list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=1672&_=1564047626268" + list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=1677&_=1564047626268" + list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=1684&_=1564047626268" + list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=1690&_=1564047626268" + list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=1698&_=1564047626268" + list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=1704&_=1564047626268" + list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=1705&_=1564047626268" + }else{ + list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&_=1564047626268" + } + } + def links = [] + for(def url : list){ + driver.get(url) + def doc = Jsoup.parse(driver.getPageSource()); + //def document = ABuYunUtil.doPost(url, null, null, "utf-8", true) + if(doc){ + def json = doc.toString().substring(doc.toString().indexOf("(")+1, doc.toString().lastIndexOf(")")) + JSONObject obj = JSONObject.fromObject(json); + if(obj && obj.total){ + def total = obj.total + def page = total/40 + def p = page.toInteger() + if(page > p){ + p = p +1 + } + + for(def i=1;i<=p;i++){ + links << url+"&page="+i + } +// if(links){ +// break; +// } + } + } + } + return links; + } + + def getDetailLinks(links, driver){ + def rets = [] + def citys = ["北京","上海","天津","重庆"] + for(def link : links){ + driver.get(link) + def doc = Jsoup.parse(driver.getPageSource()); + if(doc) { + def json = doc.toString().substring(doc.toString().indexOf("(") + 1, doc.toString().lastIndexOf(")")) + JSONObject obj = JSONObject.fromObject(json); + if (obj && obj.ls) { + for(def o : obj.ls){ + def map = [:] + if(citys.contains(o.province)){ + map.province = o.province + map.city = o.province + map.region = o.city + }else{ + map.province = o.province + map.city = o.city + } + map.id = o.id.toString() + + rets << map + } + } + } + } + return rets + } + + def parserList(existsIds,list, driver){ + def rets = [] + for(def map : list) { + if(!existsIds.contains(map.id)){ + def url = "https://paimai.jd.com/" + map.id + def doc = getDocument(url, driver) + if (doc) { + def result = parsering(doc, url, map.province, map.city, map.region) + if (result) { + rets << result + } + if (rets.size() % 100 == 0) { + def add= saveData(rets) + println DateUtils.formatDateByPattern(new Date(), "yyyy-MM-dd HH:mm:ss")+" add:"+add + rets = [] + } + } + } + } + if(rets.size()>0){ + saveData(rets) + print "完" + } + + } + + def saveData(def results){ + def add = 0 + for(def ret : results){ + def q = new BasicDBObject() + q.put("url", ret.url) + def dbo = paimaiResultColl.findOne(q); + + DBObject obj = new BasicDBObject(); + ret.each{k,v-> + obj.put(k,v) + } + + if(dbo){ + obj.m_date = new Date() + paimaiResultColl.update(q, new BasicDBObject('$set', obj)) + }else{ + add++ + obj.c_date = new Date() + obj.m_date = new Date() + try{ + paimaiResultColl.save(obj) + }catch(Exception e){ + println obj.url + } + + } + } + return add + } + + def parsering(doc, def url, def province, def city, def region){ + def result = [:] + if(doc) { + result.proince = province + result.city = city + if(region){ + result.region = region + } + def title = JQueryUtils.selectText([".pm-name"], doc) + result.title = title + result.address = JQueryUtils.selectText([".pm-location em"], doc) + if(result.address) { + //调用获取城市 +// if(!city) { +// city = RegionData.formatCity(cityMap.name) +// } + + if(city) { + def rMap = OpenApiUtils.matchDistrict(city, result.title, null, true) + if (rMap) { + result.matchCity = rMap.city + result.matchRegion = rMap.region + result.matchName = rMap.name + result.matchAddress = rMap.address + result.houseNum = rMap.house_num + result.roomNum = rMap.room_num + } + } + + } + + result.status = JQueryUtils.selectText([".pm-summary .state"], doc) + + def enrollCount = JQueryUtils.selectText([".pm-summary .times span:eq(0) em"], doc) + if(enrollCount) { + result.enrollCount = enrollCount.toInteger() + } + + def lookCount = JQueryUtils.selectText([".pm-summary .times span:eq(1) em"], doc) + if(lookCount) { + result.lookCount = lookCount.toInteger() + } + + def callCount = JQueryUtils.selectText([".pm-summary .times span:eq(2) em"], doc) + if(callCount) { + result.callCount = callCount.toInteger() + } + + + def curPriceStr = JQueryUtils.selectText([".mc .current em"], doc) + if(curPriceStr) { + curPriceStr = curPriceStr.replace(",", "") + if(curPriceStr.isInteger()) { + result.curPrice = curPriceStr.toInteger() + }else { + result.curPrice = (int)(curPriceStr.toDouble()) + } + } + + def startPriceStr = JQueryUtils.selectText([".mc .base em"], doc) + if(startPriceStr) { + startPriceStr = startPriceStr.replace(",", "") + + if(startPriceStr.isInteger()) { + result.startPrice = startPriceStr.toInteger() + }else { + result.startPrice = (int)(startPriceStr.toDouble()) + } + + } + + def tradePriceStr = JQueryUtils.selectText([".mc .deal em"], doc) + if(tradePriceStr) { + tradePriceStr = tradePriceStr.replace(",", "") + if(tradePriceStr.isInteger()) { + result.tradePrice = tradePriceStr.toInteger() + }else { + result.tradePrice = (int)(tradePriceStr.toDouble()) + } + + } + + def finishDateStr = JQueryUtils.selectText([".endtime"], doc) + if(finishDateStr && finishDateStr.contains("结束时间")) { + finishDateStr = finishDateStr.substring(5, finishDateStr.indexOf(" ")) + result.finishDate = DateUtils.parseFromTightByPattern(finishDateStr, "yyyy年MM月dd日 HH:mm:ss") + } + + def lis = JQueryUtils.select(".description ul li", doc) + if(lis && lis.size()) { + for(def li : lis) { + def str = li.select("span").text() + if(str.contains("起拍")) { + def v= li.select("em").text().replace("¥", "").replace(",", "") + if(v) { + if(v.isInteger()) { + result.startPrice = v.toDouble().toInteger() + }else { + result.startPrice = (int)(v.toDouble()) + } + + } + }else if(str.contains("评估价")) { + def v = li.select("em").text().replace("¥", "").replace(",", "") + if(v.isInteger()) { + result.evaluationPrice = v.toDouble().toInteger() + }else { + result.evaluationPrice = (int)(v.toDouble()) + } + }else if(str.contains("加价幅度")) { + def v = li.select("em").text().replace("¥", "").replace(",", "") + if(v) { + result.addPrice = v.toDouble().toInteger() + } + }else if(str.contains("保证金")) { + def v = li.select("em").text().replace("¥", "").replace(",", "") + if(v) { + if(v.isInteger()) { + result.depositPrice = v.toDouble().toInteger() + }else { + result.depositPrice = (int)(v.toDouble()) + } + + } + }else if(str.contains("延时周期")) { + + result.delayWeekly = li.select("em").text().replace("¥", "").replace(",", "") + + }else if(str.contains("优先竞买权人")) { + result.preemptivePerson = li.select("em").text().replace("¥", "").replace(",", "") + } + } + } + + def trs = JQueryUtils.select([".floor-content div:eq(2) table tr"], doc) + if(trs) { + def detailMap = [:] + for(def tr : trs) { + def d_tds = tr.select("td") + if(d_tds && d_tds.size()>=2) { + if(d_tds.size()==3) { + detailMap.put(d_tds[1].text(), d_tds[2].text()) + }else { + detailMap.put(d_tds[0].text(), d_tds[1].text()) + } + }else { + if(d_tds[0] && d_tds[0].text().contains("拍品介绍")) { + detailMap.put("拍品介绍", d_tds[0].text().replace("拍品介绍 ", "")) + } + } + } + if(detailMap) { + result.detailMap = DataUtil.filterChar(detailMap) + } + if(!result.address) { + result.address = detailMap.get("拍品名称") + if(!result.address) { + result.address = detailMap.get("标的名称") + } + } + + } + + + def imgs =JQueryUtils.select(".floor-content div:eq(2) a", doc) + if(imgs) { + def imgLinks = [] + for(def img : imgs) { + def href = img.attr("href") + if(href.contains("http")) { + imgLinks << href + }else { + imgLinks << url.substring(0, url.indexOf(":")+1)+href + } + } + result.imgUrls = imgLinks + } + + } + + //解析面积 + def article = JQueryUtils.selectText([".floor-content"], doc) + if(article) { + def area = DataUtil.getAreaNumber(article, 5000) + if(!area) { + article = JQueryUtils.selectText(["body"], doc) + if(article) { + area = DataUtil.getAreaNumber(article, 5000) + } + } + + if(area) { + result.area = area.toDouble() + if(result.tradePrice) { + result.tradeAvgPrice = (int)(result.tradePrice/area.toDouble()) + } + } + } + + def s1 = JQueryUtils.selectText(["li:contains(竞买公告) .floor-content"], doc) + if(s1) { + def start = s1.indexOf("法院将于") + def end = s1.indexOf("止") + if(start>-1 && end>-1 && end>start){ + s1 = s1.substring(s1.indexOf("法院将于"), s1.indexOf("止")) + } + def dateRangs = DataUtil.getDateRange(s1) + if(dateRangs && dateRangs.size()==2) { + + for(def j=0;jind+1) { + date = date +"时" + } + + pattern = "yyyy年MM月dd日HH时" + } + if(j==0) { + try{ + result.startDate = DateUtils.parseFromTightByPattern(date, pattern) + }catch(Exception e){ + + } + + }else { + if(!result.finishDate) { + result.finishDate = DateUtils.parseFromTightByPattern(date, pattern) + } + } + } + } + if(!result.startPrice) { + result.startPrice = DataUtil.getStartPrice(s1) + } + + def estateType = DataUtil.findEstateType(s1) + if(estateType) { + result.matchEstateType = estateType + } + + } + result.content = com.fangjia.core.util.DataUtil.zip(doc.toString()) + result.introduce = JQueryUtils.select([".floor-content div:eq(2) table"], doc).toString() + result.url = url + result.city = city + result.mDate = new Date() + result.source = "京东" + return result; + } + + def getDocument(url, driver){ + def stop = false + def error = 0 + def doc = null + while(!stop){ + try{ + driver.get(url) + doc = Jsoup.parse(driver.getPageSource()); + if(doc){ + stop = true; + }else{ + error++ + Thread.sleep(5000) + } + }catch(Exception e){ + error++ + } + if(error>=10){ + stop = true; + } + } + return doc + } + def findAllIds(){ + def q = new BasicDBObject() + q.put("url", Pattern.compile(".*jd.com.*")) + def dbcursor = paimaiResultColl.find(q) + def ids = [] + for(def d : dbcursor){ + def url = d.url + if(url){ + def id = url.substring(url.toString().lastIndexOf("/")+1, url.toString().length()) + ids << id + } + } + return ids; + } + +} diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/paimai/PaimaiDataUtil.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/paimai/PaimaiDataUtil.groovy index f98faa3ee..90a450a66 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/paimai/PaimaiDataUtil.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/paimai/PaimaiDataUtil.groovy @@ -1,5 +1,6 @@ package com.fangjia.script.gj.paimai +import java.text.SimpleDateFormat import java.util.regex.Matcher import java.util.regex.Pattern import java.util.zip.GZIPInputStream @@ -94,32 +95,32 @@ class PaimaiDataUtil { } - public static def getDateRange(def s) { - s = s.replace("起","") - Pattern pattern = Pattern.compile('法院将于(\\d+年\\d{1,2}月\\d{1,2}日\\d{0,2}时?)至(\\d+年\\d{1,2}月\\d{1,2}日\\d{0,2}时?)'); - Matcher matcher = pattern.matcher(s); - - List list = new ArrayList(); - while (matcher.find()) { - def str = matcher.group() - list.add(str); -// if(str) { -// def indx1 = str.indexOf("年") -// def indx2 = str.indexOf("月") -// def month = str.substring(indx1+1, indx2); -// if(month.toInteger()<12) { -// list.add(str); -// } -// } - } - if(list.size()>=1) { - String s1 = list.get(0) - s1 = s1.replace("法院将于", "").replace(" ","") - def array = s1.split("至"); - return array - } - return null; - } +// public static def getDateRange(def s) { +// s = s.replace("起","") +// Pattern pattern = Pattern.compile('法院将于(\\d+年\\d{1,2}月\\d{1,2}日\\d{0,2}时?)至(\\d+年\\d{1,2}月\\d{1,2}日\\d{0,2}时?)'); +// Matcher matcher = pattern.matcher(s); +// +// List list = new ArrayList(); +// while (matcher.find()) { +// def str = matcher.group() +// list.add(str); +//// if(str) { +//// def indx1 = str.indexOf("年") +//// def indx2 = str.indexOf("月") +//// def month = str.substring(indx1+1, indx2); +//// if(month.toInteger()<12) { +//// list.add(str); +//// } +//// } +// } +// if(list.size()>=1) { +// String s1 = list.get(0) +// s1 = s1.replace("法院将于", "").replace(" ","") +// def array = s1.split("至"); +// return array +// } +// return null; +// } public static def getStartPrice(def s) { @@ -159,4 +160,30 @@ class PaimaiDataUtil { return newDetailMap; } + public static def getDateRange(def s) { + s = s.replace("起","") + Pattern pattern = Pattern.compile('法院将于(\\d+年\\d+月\\d+日\\d+时)至(\\d+年\\d+月\\d+日\\d+时)'); + Matcher matcher = pattern.matcher(s); + + List list = new ArrayList(); + while (matcher.find()) { + list.add(matcher.group()); + } + if(list.size()>=1) { + String s1 = list.get(0) + s1 = s1.replace("法院将于", "") + def array = s1.split("至"); + return array + } + return null; + } + + public static Date parseFromTightByPattern(String s, String pattern){ + if (s == null || s.length() == 0){ + return null + } + SimpleDateFormat format = new SimpleDateFormat(pattern); + return format.parse(s) + } + } diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/paimai/PaimaiTest.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/paimai/PaimaiTest.groovy index e35e9f251..acced0959 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/paimai/PaimaiTest.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/paimai/PaimaiTest.groovy @@ -83,15 +83,19 @@ class PaimaiTest { driver.get("https:"+data.href) def doc = Jsoup.parse(driver.getPageSource()); if(citys.contains(data.proince)){ - def cityMap = [:] - cityMap.proince = data.proince - cityMap.city = data.proince - cityMap.url = data.href - cityList << cityMap + def links = JQueryUtils.select(".condition .s-subtab a", doc) + for(def ele : links) { + def cityMap = [:] + cityMap.proince = data.proince + cityMap.city = data.proince + cityMap.region =ele.text() + cityMap.url = data.href + cityList << cityMap + } }else{ def links = JQueryUtils.select(".condition .s-subtab a", doc) for(def ele : links){ - if(!["合肥市","芜湖市","蚌埠市","淮南市","马鞍山市","淮北市","铜陵市"].contains(ele.text())){ + if(!["乌鲁木齐市","克拉玛依市","合肥市","芜湖市","蚌埠市","淮南市","马鞍山市","淮北市","铜陵市"].contains(ele.text())){ def cityMap = [:] cityMap.proince = data.proince cityMap.city = ele.text() @@ -109,7 +113,7 @@ class PaimaiTest { driver.get(url) def doc = Jsoup.parse(driver.getPageSource()); def datas = [] - def keys = ["上海","天津","重庆","北京","浙江","河北","山西","辽宁","吉林","江苏","福建","江西","山东","河南","湖北","湖南","广东"] + def keys = [ "天津","上海","重庆","北京","浙江","河北","陕西","山西","新疆","辽宁","吉林","江苏","福建","江西","山东","河南","湖北","湖南","西藏","宁夏","甘肃","广东","安徽","广西","海南","四川","贵州","云南"] if(doc){ def eles = JQueryUtils.select(".condition a", doc) for(def e : eles){ @@ -123,11 +127,34 @@ class PaimaiTest { } return datas } + def getDocument(url, driver){ + def stop = false + def error = 0 + def doc = null + while(!stop){ + try{ + driver.get(url) + doc = Jsoup.parse(driver.getPageSource()); + if(doc){ + stop = true; + }else{ + error++ + Thread.sleep(5000) + } + }catch(Exception e){ + error++ + } + if(error>=10){ + stop = true; + } + } + return doc + } def parser(detailDatas, driver, proince, city){ def results = [] for(def m : detailDatas){ - try{ + // try{ driver.get("https:"+m.url) Thread.sleep(3000) def doc = Jsoup.parse(driver.getPageSource()); @@ -137,9 +164,9 @@ class PaimaiTest { obj.city = city results << obj } - }catch(Exception e){ - println m.url+" 解析有问题!" - } +// }catch(Exception e){ +// println m.url+" 解析有问题!" +// } } return results @@ -410,7 +437,7 @@ class PaimaiTest { } } -// result.content = DataUtil.zip(s) + result.content = DataUtil.zip(doc.toString()) def el = JQueryUtils.select([".d-block table:eq(1)"], doc) if(!el) { @@ -549,7 +576,7 @@ class PaimaiTest { ret.each{k,v-> obj.put(k,v) } - + obj.source = "公拍网" if(dbo){ obj.m_date = new Date() paimaiResultColl.update(q, new BasicDBObject('$set', obj)) diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/paimai/RmfysszcCrawl.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/paimai/RmfysszcCrawl.groovy index 244cc0d60..a7c7dccb2 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/paimai/RmfysszcCrawl.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/paimai/RmfysszcCrawl.groovy @@ -1,12 +1,4 @@ -package com.fangjia.script.gj.paimai - -import com.fangjia.core.util.DataUtil -import com.fangjia.script.BaseBatch -import com.fangjia.script.BatchOwner -import com.fangjia.script.Watcher -import com.fangjia.script.crawl.JQueryUtils -import com.fangjia.script.gj.OpenApiUtils -import com.fangjia.script.util.ABuYunUtil +import com.fangjia.script.gj.paimai.PaimaiDataUtil import com.fangjia.script.util.DateUtils import com.fangjia.script.util.MongoUtils import com.mongodb.BasicDBObject @@ -14,66 +6,266 @@ import com.mongodb.DBCollection import com.mongodb.DBObject import org.jsoup.Jsoup import org.openqa.selenium.By -import org.openqa.selenium.JavascriptExecutor import org.openqa.selenium.WebDriver -import org.openqa.selenium.WebElement import org.openqa.selenium.chrome.ChromeDriver +import org.openqa.selenium.support.ui.WebDriverWait import org.openqa.selenium.support.ui.ExpectedCondition import org.openqa.selenium.support.ui.ExpectedConditions -import org.openqa.selenium.support.ui.WebDriverWait +import com.fangjia.script.gj.OpenApiUtils -import java.util.concurrent.TimeUnit import java.util.regex.Matcher import java.util.regex.Pattern - +import com.fangjia.core.util.DataUtil +import com.fangjia.script.crawl.JQueryUtils System.setProperty("webdriver.chrome.driver", "C:\\Users\\gaojuan\\AppData\\Local\\Google\\Chrome\\Application\\chromedriver.exe"); System.setProperty("http.nonProxyHosts", "localhost"); + + def url = "https://www1.rmfysszc.gov.cn/projects.shtml?dh=3&gpstate=1&wsbm_slt=1" WebDriver driver = new ChromeDriver() -driver.get(url); -driver.findElementById("1").click() +run(url, driver) +//driver.get(url); +//driver.findElementById("1").click() +//driver.findElementById("search_sub").click() +// +//def provinces = getAllProvinceValue(driver.getPageSource()) +//for(def m : provinces){ +// def city_setting = driver.findElementById("city") +// //${m.province_id} +// city_setting.findElement(By.xpath("//option[@value='${m.province_id}']")).click() +// +////driver.manage().timeouts().implicitlyWait(30, TimeUnit.SECONDS); +////WebDriverWait wait = new WebDriverWait(driver,20) +////wait.until(ExpectedConditions.visibilityOf(driver.findElement(By.cssSelector("pagecur1")))) +// Thread.sleep(10000) +// def doc = Jsoup.parse(driver.getPageSource()); +// def citys = getAllCity(doc) +// getDetailByCity(m.province, citys, driver) +//} -def city_setting = driver.findElementById("city") -city_setting.findElement(By.xpath("//option[@value='31']")).click() -driver.findElementById("search_sub").click() -//driver.manage().timeouts().implicitlyWait(30, TimeUnit.SECONDS); -//WebDriverWait wait = new WebDriverWait(driver,20) -//wait.until(ExpectedConditions.visibilityOf(driver.findElement(By.cssSelector("pagecur1")))) -Thread.sleep(10000) -def doc = Jsoup.parse(driver.getPageSource()); -def links = getLinks(doc) +def run(url, driver){ + def values = getAllCityRegionValue(url, driver) + def finishCity = ["北京市","天津市","张家口市","邯郸市","保定市","邢台市","石家庄市","唐山市","秦皇岛市"] + for(def m : values){ + if(!finishCity.contains(m.city)){ + driver.get(url); + driver.findElementById("1").click() + Thread.sleep(3000) + + def pro_setting = driver.findElementById("city") + pro_setting.findElement(By.xpath("//option[@value='${m.province_id}']")).click() + Thread.sleep(3000) + + def city_setting = driver.findElementById("city1") + city_setting.findElement(By.xpath("//option[@value='${m.city_id}']")).click() + Thread.sleep(4000) + + def region_setting = driver.findElementById("city2") + region_setting.findElement(By.xpath("//option[@value='${m.region_id}']")).click() + + driver.findElementById("search_sub").click() + Thread.sleep(10000) + getDetailByPage(m.province, m.city, m.region, driver) + } -def eles = driver.findElementsByClassName("pagecur1") -for(def ele : eles){ - ele.click(); - Thread.sleep(10000) - doc = Jsoup.parse(driver.getPageSource()); - links.addAll(getLinks(doc)) -} -def rets = [] -for(def durl : links){ - driver.get(durl); - doc = Jsoup.parse(driver.getPageSource()); - def res = parse(durl, doc); - if(res){ - rets << res } } -saveToDB(rets); + +def getAllCityRegionValue(def url, def driver){ + driver.get(url); + driver.findElementById("1").click() + driver.findElementById("search_sub").click() + + def provinces = getAllProvinceValue(driver.getPageSource()) + def values = [] + for(def m : provinces){ + def city_setting = driver.findElementById("city") + + city_setting.findElement(By.xpath("//option[@value='${m.province_id}']")).click() + + Thread.sleep(10000) + def doc = Jsoup.parse(driver.getPageSource()); + def citys = getAllCity(doc) + def cityRegions = getAllCityByRegion(m, citys, driver) + values.addAll(cityRegions) + //getDetailByCity(m.province, citys, driver) + } + return values +} + +def getDetailByCity(province, citys, driver){ + for(def m : citys){ + def city_setting = driver.findElementById("city1") + city_setting.findElement(By.xpath("//option[@value='${m.city_id}']")).click() + Thread.sleep(10000) + def doc = Jsoup.parse(driver.getPageSource()); + def regions = getAllRegion(doc) + getDetailByRegion(province, m.city, regions, driver) + } +} + +def getAllCityByRegion(provinceMap, citys, driver){ + def datas = [] + for(def m : citys){ + def city_setting = driver.findElementById("city1") + city_setting.findElement(By.xpath("//option[@value='${m.city_id}']")).click() + Thread.sleep(10000) + def doc = Jsoup.parse(driver.getPageSource()); + def regions = getAllRegion(provinceMap, m, doc) + datas.addAll(regions) + } + return datas +} + +def getDetailByRegion(province, city, regions, driver){ + def links = [] + for(def r : regions){ + def city_setting = driver.findElementById("city2") + city_setting.findElement(By.xpath("//option[@value='${r.region_id}']")).click() + driver.findElementById("search_sub").click() + Thread.sleep(10000) + + def stop = false; + while(!stop){ + def doc = Jsoup.parse(driver.getPageSource()); + links.addAll(getLinks(r.region, doc)) + def eles = JQueryUtils.select("a:contains(下一页)", doc) + if(eles){ + driver.findElement(By.xpath("//a[contains(text(), '下一页')]")).click() + Thread.sleep(3000) + }else{ + stop = true; + } + } + + } + def datas = parseDatas(province, city, links, driver) + saveToDB(datas); + println r.region +" "+ links.size() +} + +def getDetailByPage(province, city, region, driver){ + def links = [] + def stop = false; + while(!stop){ + def doc = Jsoup.parse(driver.getPageSource()); + links.addAll(getLinks(doc)) + def eles = JQueryUtils.select("a:contains(下一页)", doc) + if(eles){ + driver.findElement(By.xpath("//a[contains(text(), '下一页')]")).click() + Thread.sleep(3000) + }else{ + stop = true; + } + } + + def datas = parseDatas(province, city, region, links, driver) + println province +" "+city+" "+region +" "+ links.size() + saveToDB(datas); +} +def getAllProvinceValue(doc){ + def optionsEls = JQueryUtils.select("#city option", doc) + def rets = [] + for(def eles : optionsEls){ + def val = eles.attr("value") + if(val!="" && val!="0"){ + def province = eles.text() + rets << ["province":province, "province_id":val] -def parse(url, doc){ + } + + } + return rets +} + +def getAllCity(doc){ + def optionsEls = JQueryUtils.select("#city1 option", doc) + def rets = [] + for(def eles : optionsEls){ + def val = eles.attr("value") + if(val!=""){ + def province = eles.text() + rets << ["city":province, "city_id":val] + } + + } + return rets +} + +def getAllRegion(provinceMap, cityMap,doc){ + def optionsEls = JQueryUtils.select("#city2 option", doc) + def rets = [] + for(def eles : optionsEls){ + def val = eles.attr("value") + if(val!=""){ + def region = eles.text() + def regionMap = [:] + regionMap.province = provinceMap.province + regionMap.province_id = provinceMap.province_id + regionMap.city = cityMap.city + regionMap.city_id = cityMap.city_id + regionMap.region = region + regionMap.region_id = val + rets << regionMap + } + + } + return rets +} + +def parseDatas(province, city, region,links, driver){ + def datas = [] + for(def url: links){ + def doc = getDocument(url, driver) + def result = parse(city, region, url, doc) + if(result){ + result.province = province + result.city = city + result.region = region + datas << result + } + } + return datas +} + +def getDocument(url, driver){ + def stop = false + def error = 0 + def doc = null + while(!stop){ + try{ + driver.get(url) + doc = Jsoup.parse(driver.getPageSource()); + if(doc){ + stop = true; + }else{ + error++ + Thread.sleep(5000) + } + }catch(Exception e){ + error++ + } + if(error>=10){ + stop = true; + } + } + return doc +} + +def parse(city, region, url, doc){ def result=[:] if(doc) { + def title = JQueryUtils.selectText(["#Title"], doc) result.title = title; result.address = title if(result.address){ - def rMap = OpenApiUtils.matchDistrict("上海",result.address) + def rMap = OpenApiUtils.matchDistrict(city, result.address, null, true) if(rMap) { result.matchCity = rMap.city result.matchRegion = rMap.region @@ -99,6 +291,7 @@ def parse(url, doc){ def enrollCount = JQueryUtils.selectText([".bmnumber"], doc) if(enrollCount) { + enrollCount = enrollCount.replace("人报名","") if(enrollCount.isInteger()){ result.enrollCount = enrollCount.toInteger() } @@ -107,6 +300,7 @@ def parse(url, doc){ def callCount = JQueryUtils.selectText([".sc"], doc) if(callCount) { + callCount =callCount.replace("人收藏","") if(callCount.isInteger()){ result.callCount = callCount.toInteger() } @@ -114,6 +308,7 @@ def parse(url, doc){ def lookCount = JQueryUtils.selectText([".hits"], doc) if(lookCount) { + lookCount = lookCount.replace("次围观","") if(lookCount.isInteger()){ result.lookCount = lookCount.toInteger() } @@ -123,8 +318,8 @@ def parse(url, doc){ def curPriceStr = JQueryUtils.selectText(["#price div:eq(0)"], doc) if(curPriceStr && curPriceStr.contains("起拍价")) { curPriceStr = curPriceStr.replace("起拍价", "").replace("¥","") - if(curPriceStr.contains("万元")){ - curPriceStr = curPriceStr.replace("万元","") + if(curPriceStr.contains("万")){ + curPriceStr = curPriceStr.replace("万元","").replace("万","") if(curPriceStr.isInteger()){ result.startPrice = curPriceStr.toInteger()*10000 }else{ @@ -144,6 +339,44 @@ def parse(url, doc){ } } + def tableEles = JQueryUtils.select("#bg1", doc) + if(tableEles){ + def evaluatePriceStr = tableEles.select("td:contains(值:)").text() + if(evaluatePriceStr && evaluatePriceStr.contains(":")){ + def arrs = evaluatePriceStr.split(":") + if(arrs.size()==2){ + evaluatePriceStr = arrs[1] + } + } + + def depositPriceStr = tableEles.select("td:contains(金:)").text() + if(depositPriceStr && depositPriceStr.contains(":")){ + def arrs = depositPriceStr.split(":") + if(arrs.size()==2){ + depositPriceStr = arrs[1] + } + } + if(depositPriceStr){ + depositPriceStr = depositPriceStr.replace("万元","") + if(depositPriceStr.toString().isNumber()){ + result.depositPrice = depositPriceStr.toDouble()*10000 + } + } + + def courtStr = tableEles.select("td:contains(院:)").text() + if(courtStr && courtStr.contains(":")){ + result.court = courtStr.substring(courtStr.indexOf(":")+1, courtStr.size()) + } + def personStr = tableEles.select("td:contains(人:)").text() + if(personStr && personStr.contains(":")){ + result.person = personStr.substring(personStr.indexOf(":")+1, personStr.size()) + } + + def countStr = tableEles.select("td:contains(拍卖阶段:)").text().replace("拍卖阶段:","") + if(countStr && countStr.contains("第") && countStr.contains("次")){ + result.count = countStr.substring(countStr.indexOf("第")+1, countStr.indexOf("次")) + } + } def lis = JQueryUtils.select("#bg1 div:eq(0) table tr", doc) if(lis && lis.size()) { @@ -151,18 +384,23 @@ def parse(url, doc){ def str = li.select("span").text() if(str.contains("评估值")) { def v = li.select("span:eq(1)").text().replace("万元", "") - if(v.isInteger()) { - result.evaluationPrice = v.toInteger()*10000 - }else{ - result.evaluationPrice = (int)(v.toDouble()*10000) + if(v.isNumber()){ + if(v.isInteger()) { + result.evaluationPrice = v.toInteger()*10000 + }else{ + result.evaluationPrice = (int)(v.toDouble()*10000) + } } + }else if(str.contains("保证金")) { def v = li.select("span:eq(1)").text().replace("万元", "") - if(v) { - if(v.isInteger()) { - result.depositPrice = v.toInteger()*10000 - }else { - result.depositPrice = (int)(v.toDouble()*10000) + if(v.isNumber()) { + if (v) { + if (v.isInteger()) { + result.depositPrice = v.toInteger() * 10000 + } else { + result.depositPrice = (int) (v.toDouble() * 10000) + } } } }else if (str.contains("公告日期:")){ @@ -177,13 +415,32 @@ def parse(url, doc){ def trs = tables[1].select("tr") if(trs) { def detailMap = [:] + for(def tr : trs) { def d_tds = tr.select("td") + if(tr.text().contains("调查情况表")){ + continue + } + if(d_tds.size()>1){ + def lastEleStr = d_tds[d_tds.size()-1].toString() + if(lastEleStr.contains('height="42"')){ + d_tds.remove(d_tds[d_tds.size()-1]) + } + } + + if(d_tds.size()>3){ + continue + } if(d_tds && d_tds.size()>=2) { if(d_tds.size()==3) { - detailMap.put(d_tds[1].text(), d_tds[2].text()) + if(d_tds[1].text() && d_tds[2].text()){ + detailMap.put(d_tds[1].text(), d_tds[2].text()) + } + }else { - detailMap.put(d_tds[0].text(), d_tds[1].text()) + if(d_tds[0].text() && d_tds[1].text()){ + detailMap.put(d_tds[0].text(), d_tds[1].text()) + } } }else { if(d_tds[0] && d_tds[0].text().contains("拍品介绍")) { @@ -202,8 +459,6 @@ def parse(url, doc){ result.introduce = tables[1].toString() } - - def imgs =JQueryUtils.select("#bdjs11 img", doc) if(imgs) { def imgLinks = [] @@ -232,6 +487,20 @@ def parse(url, doc){ result.tradeAvgPrice = (int)(result.tradePrice/result.area) } + def s1 = JQueryUtils.selectText(["#pmgg"], doc) + if(s1) { + def start = s1.indexOf("法院将于") + def end = s1.indexOf("止") + if(start>-1 && end>-1 && end>start){ + s1 = s1.substring(s1.indexOf("法院将于"), s1.indexOf("止")) + } + def dateRangs = PaimaiDataUtil.getDateRange(s1) + if(dateRangs && dateRangs.size()==2) { + result.startDate = PaimaiDataUtil.parseFromTightByPattern(dateRangs[0], "yyyy年MM月dd日HH时") + result.finishDate = PaimaiDataUtil.parseFromTightByPattern(dateRangs[1], "yyyy年MM月dd日HH时") + } + } + result.content = DataUtil.zip(doc.toString()) result.url = url @@ -246,7 +515,7 @@ def getLinks(doc){ if(eles){ for(def el :eles){ def href = el.attr("href") - urls << "https://www.rmfysszc.gov.cn/statichtml/rm_obj/"+href.substring(href.indexOf("/")+1,href.length()) + urls <<"https://www.rmfysszc.gov.cn/statichtml/rm_obj/"+href.substring(href.indexOf("/")+1,href.length()) } } } @@ -264,13 +533,19 @@ def saveToDB(rets){ ret.each{k,v-> obj.put(k,v) } - - if(dbo){ - obj.remove("url") - paimaiResultColl.update(q, new BasicDBObject('$set', obj)) - }else{ - paimaiResultColl.save(obj) - } + obj.source= "人民法院诉讼资产网" + try{ + if(dbo){ + obj.m_date = new Date() + paimaiResultColl.update(q, new BasicDBObject('$set', obj)) + }else{ + obj.c_date = new Date() + obj.m_date = new Date() + paimaiResultColl.save(obj) + } + }catch(Exception e){ + println "error:"+obj.url + } } } @@ -287,14 +562,16 @@ def getAreaNumber(input, maxArea) { for (def val : list) { def v = val.replace("平方米", "").replace("㎡", "").replace("平", "").replace("方米", "").replace(",", "").replace("m2", ""); - if (maxArea) { - if (v.toDouble() < maxArea) { - area = v; + if(v && v.isNumber()){ + if (maxArea) { + if (v.toDouble() < maxArea) { + area = v; + break; + } + } else { + area = v break; } - } else { - area = v - break; } } diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/loudong/CqfdcMatch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/loudong/CqfdcMatch.groovy new file mode 100644 index 000000000..1ce1c48c7 --- /dev/null +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/loudong/CqfdcMatch.groovy @@ -0,0 +1,173 @@ +package com.fangjia.script.loudong + +import com.fangjia.script.util.ExcelUtils +import com.fangjia.script.util.MongoUtils +import com.fangjia.script.util.OpenApiClient +import com.mongodb.BasicDBObject +import com.mongodb.DBCollection + +class CqfdcMatch { + + static DBCollection loudongFdcConn = MongoUtils.getCollectionByName("fangjia_crawl", "loudong_cqfdc", "192.168.0.235", 27018) + static DBCollection loudongConn = MongoUtils.getCollectionByName("fangjia_crawl", "loudong", "192.168.0.235", 27018) + static main(args) { + def task = new CqfdcMatch(); +// task.run() +// task.stat() + task.export2() + } + def export(){ + def q = new BasicDBObject() + q.put("match_city", new BasicDBObject('$exists', false)) + + def field = new BasicDBObject() + field.put("city",1) + field.put("region",1) + field.put("district_name",1) + field.put("address",1) + field.put("type",1) + def dbcursor = loudongConn.find(q, field).batchSize(5000) + def keys = [] + def rets = [] + rets << ["城市","区域","名称","地址","类型"] + for(def d : dbcursor){ + def key = d.city +"##"+d.district_name + if(!keys.contains(key)){ + keys << key + rets << [d.city, d.region, d.district_name, d.address, d.type] + } + } + ExcelUtils.write("D:\\数据\\重庆农商行\\重庆楼栋未匹配1.xlsx",rets, "数据") + } + + def run(){ + def datas= findAddress2() + def results = match(datas); + update(results) + } + + + def export2(){ + def q = new BasicDBObject() + q.put("match_city", new BasicDBObject('$exists', false)) + + def field = new BasicDBObject() + field.put("city",1) + field.put("name",1) + field.put("use_type",1) + def dbcursor = loudongFdcConn.find(q, field).batchSize(5000) + def keys = [] + def rets = [] + rets << ["城市","地址","类型"] + for(def d : dbcursor){ + def key = d.city +"##"+d.name + if(!keys.contains(key)){ + keys << key + rets << [d.city, d.name,d.use_type] + } + } + ExcelUtils.write("D:\\数据\\重庆农商行\\重庆楼栋未匹配2.xlsx",rets, "数据") + } + + def update(results){ + for(def m : results){ + def q = new BasicDBObject() + q.put("city", m.old_city) + q.put("district_name", m.old_name) + q.put("region",m.old_region) + + def udata = new BasicDBObject() + udata.put("match_city", m.match_city) + udata.put("match_region", m.match_region) + udata.put("match_name", m.match_name) + loudongConn.update(q, new BasicDBObject('$set',udata),false, true) + } + } + + def match(def datas){ + def rets = [] + for(def d : datas){ + def jsonObject = OpenApiClient.addressMatch(d.city, d.name, "property", false, d.region, null) + if(jsonObject != null && jsonObject.precise!=-1) { + def map = [:] + map.match_city = jsonObject.get("city") + map.match_region = jsonObject.get("district") + map.match_name = jsonObject.get("name") + map.old_name = d.name + map.old_city = d.city + map.old_region = d.region + rets << map + } + } + return rets; + } + + def findAddress(){ + def q = new BasicDBObject() + def dbcursor = loudongConn.find(q) + def rets = [] + def keys = [] + for(def d : dbcursor){ + if(!keys.contains(d.name)){ + + keys << d.name + def map = [:] + map.city = d.city + map.name = d.name + rets << map + } + } + return rets; + } + + def findAddress2(){ + def q = new BasicDBObject() + def dbcursor = loudongConn.find(q).batchSize(5000) + def rets = [] + def keys = [] + for(def d : dbcursor){ + def key = d.city +"##"+d.region+"##"+d.district_name + if(!keys.contains(key)){ + keys << key + def map = [:] + map.city = d.city + map.region = d.region + map.name = d.district_name + rets << map + } + } + return rets; + } + + + def stat(){ + def q = new BasicDBObject() + q.put("match_city", new BasicDBObject('$exists', true)) + + def field = new BasicDBObject() + field.put("match_city",1) + field.put("match_region",1) + field.put("match_name",1) + def dbcursor = loudongConn.find(q, field).batchSize(2000) + def keys = [] + for(def d : dbcursor){ + def key = d.match_city+"##"+d.match_region+"##"+d.match_name + if(!keys.contains(key)){ + keys << key + } + } + + def dbcursor2 = loudongFdcConn.find(q, field).batchSize(5000) + for(def d : dbcursor2){ + def key = d.match_city+"##"+d.match_region+"##"+d.match_name + if(!keys.contains(key)){ + keys << key + } + } + println "小区:"+keys.size() + + } + + + +} diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/thirdPriceCrawl/DataStat.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/thirdPriceCrawl/DataStat.groovy index 97dc2e689..97af5b6dc 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/thirdPriceCrawl/DataStat.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/thirdPriceCrawl/DataStat.groovy @@ -40,11 +40,11 @@ class DataStat { def run(){ - def source = "搜房"// - def citys = findCrawlCity(source, 202112); + def source = "58同城"// + def citys = findCrawlCity(source, 202202); for(def city : citys){ - def count1 = findCount(source, city, 202112); - def count2 = findCount(source, city, 202202); + def count1 = findCount(source, city, 202202); + def count2 = findCount(source, city, 202203); println city +" "+count1+" "+count2 // println count1 // println "--- ----" diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/thirdPriceCrawl/Esf58Crawl.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/thirdPriceCrawl/Esf58Crawl.groovy index 9362389ca..ee0edcfe2 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/thirdPriceCrawl/Esf58Crawl.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/thirdPriceCrawl/Esf58Crawl.groovy @@ -40,19 +40,17 @@ class Esf58Crawl extends BaseBatch{ def static useDistKeys = [] static def totalCount = new AtomicLong() - def cityList = [ + def cityList = ["成都":"cd|102","杭州":"hz|79","重庆":"cq|37","苏州":"su|5","无锡":"wx|93", "深圳":"sz|4","郑州":"zz|342", "长沙":"cs|414","东莞":"dg|413", + "保定":"bd|424","保亭":"baoting|10367","南宁":"nn|845", "扬州":"yz|637","汕头":"st|783","泰州":"taizhou|693","淮安":"ha|968","湛江":"zhanjiang|791","锦州":"jinzhou|2354","昆明":"km|541","柳州":"liuzhou|7133","肇庆":"zq|901","连云港":"lyg|2049","日照":"rizhao|3177","常德":"changde|872", + "沧州":"cangzhou|652", "泉州":"qz|291","清远":"qingyuan|7303", "西宁":"xn|2052","遵义":"zunyi|7620","株洲":"zhuzhou|1086","贵阳":"gy|2015","岳阳":"yy|821","徐州":"xz|471", "哈尔滨":"hrb|202","宜昌":"yc|858", "廊坊":"lf|772","秦皇岛":"qhd|1078","惠州":"huizhou|722","洛阳":"luoyang|556", + "石家庄":"sjz|241","太原":"ty|740","盐城":"yancheng|613","信阳":"xy|8694","南充":"nanchong|2378","南阳":"ny|592","厦门":"xm|606", + "绵阳":"mianyang|1057","济宁":"jining|450","邯郸":"hd|572", + "淮北":"huaibei|9357","滁州":"chuzhou|10266", "福州":"fz|304","蚌埠":"bengbu|3470","淮南":"hn|2319","青岛":"qd|122","广州":"gz|3","天津":"tj|18","南京":"nj|172", "嘉兴":"jx|497","大连":"dl|147","武汉":"wh|158", - "兰州":"lz|952","合肥":"hf|837", "乌鲁木齐":"xj|984","南昌":"nc|669","沈阳":"sy|188","西安":"xa|483","苏州":"su|5","无锡":"wx|93","潍坊":"wf|362", + "兰州":"lz|952","合肥":"hf|837", "乌鲁木齐":"xj|984","南昌":"nc|669","沈阳":"sy|188","西安":"xa|483","潍坊":"wf|362", "芜湖":"wuhu|2045","东莞":"dg|413","常州":"cz|463","佛山":"fs|222", - "南通":"nt|394","东莞":"dg|413","吉林":"jl|700","中山":"zs|771","阜阳":"fy|2325", + "南通":"nt|394","吉林":"jl|700","中山":"zs|771","阜阳":"fy|2325", "呼和浩特":"hu|811", - "济南":"jn|265","温州":"wz|330","台州":"tz|403","东营":"dy|623","咸阳":"xianyang|7453","安庆":"anqing|3251","宿州":"suzhou|3359","六安":"la|2328","唐山":"ts|276","商丘":"sq|1029","大同":"dt|6964","威海":"weihai|518","安阳":"ay|1096", - "临汾":"linfen|5669","临沂":"linyi|505","九江":"jj|2247","淄博":"zb|385","北京":"bj|1","上海":"sh|2", - "三亚":"sanya|2422","烟台":"yt|228","宁波":"nb|135","金华":"jh|531", "绍兴":"sx|355","湖州":"huzhou|831","张家口":"zjk|3328","珠海":"zh|910","海口":"haikou|2053","保定":"bd|424","保亭":"baoting|10367","南宁":"nn|845", "扬州":"yz|637","汕头":"st|783","泰州":"taizhou|693","淮安":"ha|968","湛江":"zhanjiang|791","锦州":"jinzhou|2354","昆明":"km|541","柳州":"liuzhou|7133","肇庆":"zq|901","连云港":"lyg|2049","日照":"rizhao|3177","常德":"changde|872", - "沧州":"cangzhou|652", "泉州":"qz|291","清远":"qingyuan|7303", "西宁":"xn|2052","遵义":"zunyi|7620","株洲":"zhuzhou|1086","贵阳":"gy|2015","岳阳":"yy|821","徐州":"xz|471", "哈尔滨":"hrb|202","宜昌":"yc|858", "廊坊":"lf|772","秦皇岛":"qhd|1078","惠州":"huizhou|722","洛阳":"luoyang|556", - "深圳":"sz|4","石家庄":"sjz|241","太原":"ty|740","成都":"cd|102","杭州":"hz|79","重庆":"cq|37","盐城":"yancheng|613","信阳":"xy|8694","南充":"nanchong|2378","南阳":"ny|592","厦门":"xm|606", - "绵阳":"mianyang|1057","济宁":"jining|450","邯郸":"hd|572", - "淮北":"huaibei|9357","滁州":"chuzhou|10266", "马鞍山":"mas|2039","铜陵":"tongling|10285","宣城":"xuancheng|5633","亳州":"bozhou|2329","黄山":"huangshan|2323", "池州":"chizhou|10260","巢湖":"ch|10229","和县":"hexian|10892","霍邱":"hq|11226","桐城":"tongcheng|11296", "宁国":"ningguo|5645","天长":"tianchang|10273","东至":"dongzhi|10262","无为":"wuweixian|10232", @@ -76,7 +74,7 @@ class Esf58Crawl extends BaseBatch{ "五指山":"wzs|9952","三沙":"sansha|13722","琼海":"qh|10136", "文昌":"wenchang|9984","万宁":"wanning|10022","屯昌":"tunchang|10044","琼中":"qiongzhong|10064","陵水":"lingshui|10184", "东方":"df|10250","定安":"da|10303","澄迈":"cm|10331","白沙":"baish|10380", - "儋州":"danzhou|10394", "郑州":"zz|342","新乡":"xx|1016","许昌":"xc|977", + "儋州":"danzhou|10394", "新乡":"xx|1016","许昌":"xc|977", "平顶山":"pds|1005","焦作":"jiaozuo|3266","开封":"kaifeng|2342", "濮阳":"puyang|2346","周口":"zk|933","驻马店":"zmd|1067","漯河":"luohe|2347", "三门峡":"smx|9317","鹤壁":"hb|2344","济源":"jiyuan|9918","明港":"mg|8541","鄢陵":"yanling|9123", @@ -98,7 +96,7 @@ class Esf58Crawl extends BaseBatch{ "广水":"guangshuishi|9657","赤壁":"chibishi|9623","老河口":"laohekou|895","谷城":"gucheng|899","宜城":"yichengshi|897", "南漳":"nanzhang|898","云梦":"yunmeng|3438","安陆":"anlu|3442","大悟":"dawu|3437","孝昌":"xiaochang|3436", "当阳":"dangyang|865","枝江":"zhijiang|866","嘉鱼":"jiayuxian|9624","随县":"suixia|9660", - "长沙":"cs|414","益阳":"yiyang|10198","衡阳":"hy|914", + "益阳":"yiyang|10198","衡阳":"hy|914", "湘潭":"xiangtan|2047","郴州":"chenzhou|5695","邵阳":"shaoyang|2303","怀化":"hh|5756", "永州":"yongzhou|2307","娄底":"ld|9481","湘西":"xiangxi|10219","张家界":"zjj|6788","醴陵":"liling|1091", "澧县":"lixian|876","桂阳":"czguiyang|5699","资兴":"zixing|5698","永兴":"yongxing|5701","常宁":"changningshi|921", @@ -181,7 +179,10 @@ class Esf58Crawl extends BaseBatch{ "长兴":"changxing|834","嘉善":"jiashanx|14357","海宁":"haining|500","德清":"deqing|835","东阳":"dongyang|536", "安吉":"anji|836","苍南":"cangnanxian|7579","临海":"linhai|407","永康":"yongkang|537","玉环":"yuhuan|409", "平湖":"pinghushi|501","海盐":"haiyan|504","武义县":"wuyix|14528","嵊州":"shengzhou|359","新昌":"xinchang|361", - "江山":"jiangshanshi|6796","平阳":"pingyangxian|7575" + "威海":"weihai|518","安阳":"ay|1096", + "临汾":"linfen|5669","临沂":"linyi|505","九江":"jj|2247","淄博":"zb|385","北京":"bj|1","上海":"sh|2", + "三亚":"sanya|2422","烟台":"yt|228","宁波":"nb|135","金华":"jh|531", "绍兴":"sx|355","湖州":"huzhou|831","张家口":"zjk|3328","珠海":"zh|910","海口":"aihkou|2053", + "江山":"jiangshanshi|6796","平阳":"pingyangxian|7575","济南":"jn|265","温州":"wz|330","台州":"tz|403","东营":"dy|623","咸阳":"xianyang|7453","安庆":"anqing|3251","宿州":"suzhou|3359","六安":"la|2328","唐山":"ts|27 6","商丘":"sq|1029","大同":"dt|6964", ] static main(args) { @@ -191,37 +192,37 @@ class Esf58Crawl extends BaseBatch{ void process(Object args) { //addContent("福州","https://fz.58.com/xiaoqu/", 70); -// def citys = ["新余","松原","楚雄","吕梁","雅安","本溪","鹤壁","拉萨","黔东南","忻州","兴安盟","乌海","贵港","三门峡","仙桃","安康","白城","平凉","庆阳","潮州","白银","吴忠","防城港","延安","丽江","黔西南","喀什","钦州","鸡西","萍乡","石嘴山","鹰潭","保山","哈密","铜仁","武威","普洱","河池","酒泉","朔州","百色","鹤岗","张家界","定西","湘西","济源","昭通","中卫","张掖","克拉玛依","黑河","铜川","辽源","云浮","博尔塔拉","固原","白山","天门","潜江","贺州","临沧","伊春","临夏","石河子","来宾","双鸭山","嘉峪关","海东"] -// for(def city : citys){ -// def str = cityList.get(city) + def citys = ["佛山","潍坊","长春","太原","南通","厦门","呼和浩特","常州","中山","德阳","镇江","泰安","银川","襄阳","江门","荆州","上海","衡阳","滨州","遂宁","鞍山","赣州","阜阳","枣庄","聊城","芜湖","吉林","德州","信阳","新乡","宿迁","沧州","桂林","包头","常德","乐山","遵义","眉山","株洲","驻马店","日照","许昌","成都","衡水","宜宾","泸州","营口","达州","宝鸡","广安","吉安","孝感","鄂尔多斯","十堰","清远","湘潭","上饶","漯河","邢台","平顶山","菏泽","宜春","承德","赤峰","延边","茂名","焦作","内江","韶关","梅州","郴州","黄冈","红河","绥化","娄底","丹东","邵阳","开封","阳江","黄石","周口","牡丹江","长治","咸宁","汉中","自贡","衢州","晋中","永州","北海","抚顺","大庆","呼伦贝尔","晋城","渭南","濮阳","河源","齐齐哈尔","曲靖","舟山","运城","巴中","怀化","铁岭","佳木斯","朝阳","宁德","葫芦岛","通辽","益阳","资阳","荆门","鄂州","大理","辽阳","阳泉","榆林","梧州","乌兰察布","黔南","巴音郭楞","广元","昌吉","玉溪","揭阳","随州","汕尾","北京","玉林","盘锦","六盘水","抚州","景德镇","四平","恩施","安顺","伊犁","毕节","丽水","天水","通化","阜新","攀枝花","阿克苏","重庆"] + for(def city : citys){ + def str = cityList.get(city) + for(def i=0;i<1;i++){ + def districts = CommonDatautils.findAllDistricts(city, "district", ["city", "region", "name"]); + if(districts){ + def arrs = str.split("\\|") + def url = "https://"+arrs[0]+".58.com/xiaoqu/" + distMap.put(city, districts) + addCrawlKeys(city); + addDetailContent(city, "https://"+arrs[0]+".58.com", arrs[1]); + } + } + } + +// cityList.each{city, str-> // for(def i=0;i<1;i++){ -// def districts = CommonDatautils.findAllDistricts(city, "district", ["city", "region", "name"]); -// if(districts){ -// def arrs = str.split("\\|") -// def url = "https://"+arrs[0]+".58.com/xiaoqu/" -// distMap.put(city, districts) -// addCrawlKeys(city); -// addDetailContent(city, "https://"+arrs[0]+".58.com", arrs[1]); +// if(citys.contains(city)){ +// def districts = CommonDatautils.findAllDistricts(city, "district", ["city", "region", "name"]); +// if(districts){ +// def arrs = str.split("\\|") +// def url = "https://"+arrs[0]+".58.com/xiaoqu/" +// distMap.put(city, districts) +// +// addCrawlKeys(city); +// addDetailContent(city, "https://"+arrs[0]+".58.com", arrs[1]); +// } // } // } // } - cityList.each{city, str-> - for(def i=0;i<1;i++){ -// if(citys.contains(city)){ - def districts = CommonDatautils.findAllDistricts(city, "district", ["city", "region", "name"]); - if(districts){ - def arrs = str.split("\\|") - def url = "https://"+arrs[0]+".58.com/xiaoqu/" - distMap.put(city, districts) - - addCrawlKeys(city); - addDetailContent(city, "https://"+arrs[0]+".58.com", arrs[1]); - } -// } - } - } - } @@ -230,6 +231,10 @@ class Esf58Crawl extends BaseBatch{ for(def i=0;i5){ + districtName = districtName.substring(0,3) + } for(def j=1;j<=districtName.length()-1;j++) { // if(j<=3){ def keyword = districtName.toString().substring(0, j); diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/thirdPriceCrawl/SynchDistrictPrice.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/thirdPriceCrawl/SynchDistrictPrice.groovy index 0ca7c1afb..279f5f3c1 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/thirdPriceCrawl/SynchDistrictPrice.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/thirdPriceCrawl/SynchDistrictPrice.groovy @@ -15,10 +15,10 @@ class SynchDistrictPrice { def static final SHORT_DATE = Utils.getLastMonth() def thirdPartyColl = CrawalDBUtil.getThirdPartyCol(); def districtDetailColl = CrawalDBUtil.getDistrictDetailColl(); - def sourceMap = ["fang.com":"搜房","jiwu.com":"吉屋","58.com":"58同城","lianjia.com":"链家","ke.com":"贝壳找房","fangdd.com":"房多多","house365.com":"365淘房", - "anjuke.com":"安居客","cityhouse.cn":"城市房产","zhuge.com":"诸葛找房","5i5j.com":"我爱我家"] +// def sourceMap = ["fang.com":"搜房","58.com":"58同城","lianjia.com":"链家","ke.com":"贝壳找房","fangdd.com":"房多多","house365.com":"365淘房", +// "anjuke.com":"安居客","cityhouse.cn":"城市房产","zhuge.com":"诸葛找房","5i5j.com":"我爱我家","jiwu.com":"吉屋"] //"fang.com":"搜房", -// def sourceMap = ["fang.com":"搜房"] + def sourceMap = ["ke.com":"贝壳找房","fang.com":"搜房","cityhouse.cn":"城市房产"] static main(args) { def task = new SynchDistrictPrice() task.run()