diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/fuzhou/Test1.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/fuzhou/Test1.groovy index cf1e4950f..4e1922326 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/fuzhou/Test1.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/fuzhou/Test1.groovy @@ -2,13 +2,19 @@ package com.fangjia.script.fuzhou import com.fangjia.script.commData.CommonDatautils import com.fangjia.script.common.DataHandleUtils +import com.fangjia.script.gj.ccb.FileUtils import com.fangjia.script.thirdPriceCrawl.CrawalDBUtil +import com.fangjia.script.util.CsvUtils +import com.fangjia.script.util.DateUtils import com.fangjia.script.util.ExcelUtils import com.fangjia.script.util.MongoUtils +import com.fangjia.script.yinjihaun.util.DingDingMessageUtil import com.mongodb.BasicDBObject import com.mongodb.DBCollection - -def datas = ExcelUtils.read("d:\\数据\\福州\\福州4月2.xlsx",0,1) +//福州第三方均价统计,发送钉钉消息 +//def datas = ExcelUtils.read("d:\\数据\\福州\\福州4月2.xlsx",0,1) +//def datas = CsvUtils.read("d:\\数据\\福州\\FJW_THIRD_PARTY_AVG_PRICE_202105.csv"); +def datas = readTxt("d:\\数据\\福州\\FJW_THIRD_PARTY_AVG_PRICE_202105.csv") def count0 = 0 def count1 = 0 def count2 = 0 @@ -18,35 +24,44 @@ def count5 = 0 def count6 = 0 def count7 = 0 def count8 = 0 -def total=4943 +def count9 = 0 +def total = datas.size() +def regions = ["鼓楼","仓山","台江","晋安","马尾","长乐"] +def regionCount = 0 + +def rets = [] +rets << ["网站有对比价数量","有对比价楼盘量","有对比价占比"] for(def d : datas){ def list = [] - if(d['G']){ - list << d['G'] + if(regions.contains(d[3])){ + regionCount ++ } - if(d['H']){ - list << d['H'] + if(d[6]){ + list << d[6] } - if(d['I']){ - list << d['I'] + if(d[7]){ + list << d[7] } - if(d['J']){ - list << d['J'] + if(d[8]){ + list << d[8] } - if(d['K']){ - list << d['K'] + if(d[9]){ + list << d[9] } - if(d['L']){ - list << d['L'] + if(d[10]){ + list << d[10] } - if(d['M']){ - list << d['M'] + if(d[11]){ + list << d[11] } - if(d['N']){ - list << d['N'] + if(d[12]){ + list << d[12] } - if(d['O']){ - list << d['O'] + if(d[13]){ + list << d[13] + } + if(d[14]){ + list << d[14] } if(list.size()==0){ count0++ @@ -76,18 +91,53 @@ for(def d : datas){ if(list.size()==8){ count8++ } + if(list.size()==9){ + count9++ + } } -println("count0:"+ (DataHandleUtils.getDoubleRound(2, count0*100.0/total))) -println("count1:"+ (DataHandleUtils.getDoubleRound(2, count1*100.0/total))) -println("count2:"+ (DataHandleUtils.getDoubleRound(2, count2*100.0/total))) -println("count3:"+ (DataHandleUtils.getDoubleRound(2, count3*100.0/total))) -println("count4:"+ (DataHandleUtils.getDoubleRound(2, count4*100.0/total))) -println("count5:"+ (DataHandleUtils.getDoubleRound(2, count5*100.0/total))) -println("count6:"+ (DataHandleUtils.getDoubleRound(2, count6*100.0/total))) -println("count7:"+ (DataHandleUtils.getDoubleRound(2, count7*100.0/total))) -println("count8:"+ (DataHandleUtils.getDoubleRound(2, count8*100.0/total))) +println("总量:"+ total) +def pre1 = DataHandleUtils.getDoubleRound(2, count1*100.0/total) +println("count1:"+ pre1) +def pre2 = DataHandleUtils.getDoubleRound(2, count2*100.0/total) +println("count2:"+ pre2) +def pre3 = DataHandleUtils.getDoubleRound(2, count3*100.0/total) +println("count3:"+ pre3) +def pre4 = DataHandleUtils.getDoubleRound(2, count4*100.0/total) +println("count4:"+ pre4) + +def pre5 = DataHandleUtils.getDoubleRound(2, count5*100.0/total) +println("count5:"+ pre5) + +def pre6 = DataHandleUtils.getDoubleRound(2, count6*100.0/total) +println("count6:"+ pre6) + +def pre7 = DataHandleUtils.getDoubleRound(2, count7*100.0/total) +println("count7:"+ pre7) + +def pre8 = DataHandleUtils.getDoubleRound(2, count8*100.0/total) +println("count8:"+ pre8) + +def pre9 = DataHandleUtils.getDoubleRound(2, count9*100.0/total) +println("count9:"+ pre9) + +rets << ["1家",count1, pre1+"%"] +rets << ["2家",count2, pre2+"%"] +rets << ["3家",count3, pre3+"%"] +rets << ["4家",count4, pre4+"%"] +rets << ["5家",count5, pre5+"%"] +rets << ["6家",count6, pre6+"%"] +rets << ["7家",count7, pre7+"%"] +rets << ["8家",count8, pre8+"%"] +rets << ["9家",count9, pre9+"%"] + +def curMonth = DateUtils.formatDateByPattern(new Date(), "yyyyMM") +ExcelUtils.write("d:\\数据\\福州\\多方均价对比统计数据"+curMonth+".xlsx",rets,"统计") +def s = "第三方均价总量:${total}\n6区总量:${regionCount}\ncount1: "+ count1+"( "+pre1+"% )\ncount2: "+ count2+"( "+pre2+"% )\ncount3: "+count3+"( "+pre3+"% )\ncount4: "+count4+"( "+pre4+"% )\ncount5: "+count5+"( "+pre5+"% )\ncount6: "+count6+"( "+pre6+"% )\ncount7: "+count7+"( "+pre7+"%)\ncount8: "+count8+"( "+pre8+"% )\ncount9: "+count9+" ("+pre9+"% )" + +def access_token = "065f4a05d5f87eb39a9f9e048eeefb80c1f7560446f8926441de80453c160c60" +DingDingMessageUtil.sendTextMessage(access_token, s) ////def districts = CommonDatautils.findAllDistricts("福州", "district", ["city", "region", "name"]); @@ -128,6 +178,18 @@ println("count8:"+ (DataHandleUtils.getDoubleRound(2, count8*100.0/total))) //} +def readTxt(def fileName){ + File file = new File(fileName); + def rows = []; + + file.eachLine{ + def row = it.split("@\\|@",-1) + rows << row + } + return rows; +} + + diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/LandDetailCrawl.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/LandDetailCrawl.groovy index 6a44c0463..29bd4df02 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/LandDetailCrawl.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/LandDetailCrawl.groovy @@ -65,12 +65,12 @@ class LandDetailCrawl { } def updateData(){ - def datas = ExcelUtils.read("D:\\数据\\土地\\土地成交3月(2).xlsx",0, 1) + def datas = ExcelUtils.read("D:\\数据\\土地\\土地-21-06.xlsx",0, 1) if(datas && datas.size()>0){ for(def data : datas){ def id = data['A'] DBObject upMap = new BasicDBObject() - def area = data['E'] + def area = data['I'] if(area){ if(area){ diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/WuXiAddressTest.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/WuXiAddressTest.groovy new file mode 100644 index 000000000..f8c8c0d5c --- /dev/null +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/WuXiAddressTest.groovy @@ -0,0 +1,90 @@ +package com.fangjia.script.gj + +import com.fangjia.script.util.ExcelUtils +import com.fangjia.script.yinjihaun.amap.GaoDeApiUtils + +class WuXiAddressTest { + static main(args) { + def task = new WuXiAddressTest() + task.run() + } + + def run(){ +// def datas = ExcelUtils.read("D:\\数据\\无锡\\教育机构-标准化(已处理).xlsx",0,1) +// for(def d : datas){ +// def locationStr = getLocationInfo(d['D'], d['Q']) +// def township = getStreetInfo(locationStr) +// def name = getJuWeiHuiInfo(locationStr); +// +// } + + ExcelUtils.multiThreadingModify("D:\\数据\\无锡\\教育机构-标准化(已处理).xlsx",{d-> + if(!d['S']){ + def locationStr = getLocationInfo(d['D'], d['Q']) + def township = getStreetInfo(locationStr) + def name = getJuWeiHuiInfo(locationStr, "村委会", "1000"); + if(!name){ + name = getJuWeiHuiInfo(locationStr,"村委会", "2000"); + if(name){ + d['T'] = 2000 + } + }else{ + d['T'] = 1000 + } + + if(!name){ + name = getJuWeiHuiInfo(locationStr, "村委会", "3000"); + if(name){ + d['T'] = 3000 + } + } + d['R'] = township + d['S'] = name + } + + + },0) + + } + + def getLocationInfo(def city, def address){ + def gloJson = GaoDeApiUtils.geo(city, address) + if(gloJson) { + def glocation = null + if (gloJson && gloJson.get("geocodes")) { + def geocodes = gloJson.get("geocodes") + glocation = geocodes.get(0).location + } + return glocation + } + return null + } + + + def getStreetInfo(def glocation){ + def township = null + if(glocation){ + def jsonObject = GaoDeApiUtils.geocode(glocation) + if(jsonObject && jsonObject.regeocode) { + def addressComponent = jsonObject.regeocode.addressComponent + if(addressComponent){ + township = addressComponent.township + } + } + } + return township; + } + + def getJuWeiHuiInfo(def glocation, def keyword, def radius){ + def json = GaoDeApiUtils.around2(glocation, radius,keyword, null) + def name = null + if(json && json.info=="OK" && json.pois){ + def pois = json.pois + if(pois){ + def poi = pois[0] + name = poi.name + } + } + return name; + } +} diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/YouDaDataParser.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/YouDaDataParser.groovy index 25c631832..0c2be14c8 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/YouDaDataParser.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/YouDaDataParser.groovy @@ -47,7 +47,7 @@ class YouDaDataParser { def final AtomicInteger count_ = new AtomicInteger(0) - static def statDate = "2021-04-01" + static def statDate = "2021-05-01" static main(args) { def task = new YouDaDataParser(); // task.newHouseparse();//友达一手房存入mongo @@ -285,7 +285,7 @@ class YouDaDataParser { def esfParser(){ - def files = ["D:\\数据\\友达\\2021年5月\\二手房\\查询1.DBF"] + def files = ["D:\\数据\\友达\\2021年6月\\二手房\\查询1.DBF"] def datas = [] for(def fileName : files){ def rows = readDBF(fileName); diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/district/FangjiaClientUtils.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/district/FangjiaClientUtils.groovy index a131a414c..8eacf492b 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/district/FangjiaClientUtils.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/district/FangjiaClientUtils.groovy @@ -65,6 +65,22 @@ class FangjiaClientUtils { } + + public static def addressStand(def address){ + def jsonObj = null + try { + def url = "http://192.168.0.19:39080/FangJiaNlpService/ner?address="+URLEncoder.encode(address,"UTF-8") + +// def url = "http://192.168.0.19:39080/FangJiaAddressService/structure?address="+URLEncoder.encode(address,"UTF-8") + def content = new HttpRequest().newGet(url, null, "utf-8") + jsonObj = JSONObject.fromObject(content) + } catch (Exception e) { + e.printStackTrace(); + } + return jsonObj; + } + + public static Map JsonToMap(JSONObject json) { Map columnValMap = new HashMap(); Set jsonKeys = json.keySet(); diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/district/ZsPersonProcess.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/district/ZsPersonProcess.groovy new file mode 100644 index 000000000..abb8e3ae9 --- /dev/null +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/district/ZsPersonProcess.groovy @@ -0,0 +1,1146 @@ +package com.fangjia.script.gj.district + +import com.fangjia.core.RegionData +import com.fangjia.core.util.BaiDuApiUtils +import com.fangjia.core.util.GeocodeUtils +import com.fangjia.core.util.HttpRequest +import com.fangjia.script.commData.CommonDatautils +import com.fangjia.script.util.CosineSimilarAlgorithm +import com.fangjia.script.util.DataUtil +import com.fangjia.script.util.DbUtils +import com.fangjia.script.util.ExcelUtils +import com.fangjia.script.util.GeoHelper +import com.fangjia.script.util.MongoUtils +import com.fangjia.script.util.OpenApiClient +import com.mongodb.BasicDBObject +import com.mongodb.DBCollection +import net.sf.json.JSONObject +import org.apache.commons.lang.StringUtils + +import java.util.regex.Matcher +import java.util.regex.Pattern + +class ZsPersonProcess { + + static final DBCollection FANGJIA_OFFICIAL_AREA_MAP = MongoUtils.getCollectionByName("business", "fangjia_official_area_map", "mongo.db.fangjia.com") + + static final String TOKEN = "F54F52381C49BB9EB4A33EB1B65604AE4B71A28AEE53518A94A2F360408B9056D57553931D15CE6DDE765562DAD286BA38E05A4CDAFC51C3D527A4959BF8E75A3B95DB7108FCEA340DDE61925616DB55118E1851E67D83EAD800460D100D6B667A4ED8EE67C8F7FB"; + + static main(args) { + def task = new ZsPersonProcess() +// task.run() + +// task.parse() + +// def d= task.addressStandardize("成都","半岛城邦2期") +// if(d){ +// println "keywords:"+d.name+" street:"+d.street+" number:"+d.streetNumber +// } + +// def dist = ["city":"重庆", "region":"临海", "name":"伟星·金桥郡",address:""] +// if(dist.name){ +// def arrs = FangjiaClientUtils.getNlpKey(dist.name, 3) +// println "Name: "+arrs +// } +// if(dist.address){ +// def arrs = FangjiaClientUtils.getNlpKey(dist.address, 3) +// println "Address: "+arrs +// } +// def loactionMap = task.matchMap(dist) +// println loactionMap + +// task.matchMap() + +// task.matchDb() +// task.matchDb2() +// task.matchDb3() + + task.saveOrUpdate() + } + + def matchMap(){ +// def datas = ExcelUtils.read("d:\\数据\\浙商\\浙商人工.xlsx", 0,1) +// for(def d : datas){ +// if(d['U']){ +// def dist = ["city":d['U'], "region":d['V'], "name":d['G']] +// def loactionMap = matchMap(dist) +// if(loactionMap){ +// d['W'] = loactionMap.source +// d['E'] = loactionMap.region +// d['F'] = loactionMap.mapName +// d['H'] = loactionMap.address +// d['I'] = loactionMap.lng +// d['J'] = loactionMap.lat +// } +// } +// } + + ExcelUtils.multiThreadingModify("d:\\数据\\浙商\\浙商人工(单-恢复).xlsx", { d -> + if(d['U'] && !d['Y']){ + def name = null + def dist = [] + def loactionMap = null + if(d['W']){ + name = d['W'] + dist = ["city":d['U'], "region":d['V'], "name":name, "address":d['X']] + } + + loactionMap = matchMap(dist) + if(d['X'] && !d['X'].toString().endsWith("路") && !d['X'].toString().endsWith("街") && (!d['W'] || !loactionMap)){ + dist = ["city":d['U'], "region":d['V'], "name":d['X'], "address":null] + loactionMap = matchMap(dist) + } + + if(loactionMap){ + d['Y'] = loactionMap.source + d['Z'] = loactionMap.region + d['AA'] = loactionMap.mapName + d['AB'] = loactionMap.address + d['AC'] = loactionMap.lng + d['AD'] = loactionMap.lat + d['AE'] = loactionMap.score + d['AF'] = loactionMap.type + } + } + },0) + } + + + def parse(){ + def keywords = ["路","街","巷","大道"] +// def datas = ExcelUtils.read("d:\\数据\\浙商\\浙商人工.xlsx", 0,1) + ExcelUtils.modify("d:\\数据\\浙商\\浙商人工(单).xlsx",{d-> + if(!d['Y']){ + def str = d['G'] + + for(def key : keywords){ + if(d['G'].toString().endsWith(key)){ + str = d['H'] + } + } + + if(d['U']){ + def json = addressStandardize(d['U'],str.toString()) + + println "["+d['U']+" "+str +"] "+json + if(json){ + if(json.name){ + d['W'] = json.name + } + + if(json.street && json.streetNumber){ + d['X'] = json.street + json.streetNumber + }else if(json.street){ + d['X'] = json.street + } + } + + if(!d['X'] || (d['X'] && !d['X'].toString().contains("号"))){ + json = addressStandardize(d['U'], d['H'].toString()) + if(json){ + if(json.name){ + if(!d['W']){ + d['W'] = json.name + } + }else if(json.street && json.streetNumber){ + d['X'] = json.street + json.streetNumber + }else if(json.street){ + d['X'] = json.street + } + } + } + + println json + } + } + },0) + + + } + + def run(){ + ExcelUtils.multiThreadingModify("d:\\数据\\浙商\\浙商人工.xlsx",{d-> + + if(!d['S'] && !d['T']){ + def dbo = null + if(d['I']){ + def code = d['I'].toInteger().toString() + if(code.toString().length()==4){ + code = code+"00" + } + println "code:"+code + dbo = findRegionCodeValue(code) + if(dbo){ + d['T'] = dbo.province + d['U'] = dbo.fangjia_city + d['V'] = dbo.fangjia_district + } + } + if(!dbo){ + if(d['D']){ + def citycode = d['E'].toInteger().toString() + if(citycode.toString().length()==4){ + citycode = citycode+"00" + } + println "citycode:"+citycode + dbo = findCityCodeValue(d['D'].toInteger(), citycode) + if(!dbo){ + dbo = findCityCodeValue(d['D'], null) + } + }else{ + if(d['E'] && d['I']){ + def code1 = d['E'].toInteger().toString() + if(code1.toString().length()==4){ + code1 = code1+"00" + } + def code2 = d['I'].toInteger().toString() + if(code2.toString().length()==4){ + code2 = code2+"00" + } + println "code1:"+code1 +" code2:"+code2 + dbo = findCityCodeValue(code1, code2) + } + } + + if(dbo){ + if(dbo.province.contains('省')){ + d['T'] = dbo.province + d['U'] = dbo.fangjia_city + }else{ + d['T'] = dbo.fangjia_city + d['U'] = dbo.fangjia_city + } + } + } + } + + },0) + } + + public static JSONObject addressStandardize(String city, String address) { + if (StringUtils.isBlank(city) || StringUtils.isBlank(address)) { + throw new IllegalArgumentException(); + } + + try { + Map params = new HashMap(); + params.put("city", city); + params.put("address", address); + params.put("token", TOKEN); + String content = new HttpRequest().get("http://open.fangjia.com/address/standard", params, "utf-8") + JSONObject result = JSONObject.fromObject(content) + if (result.getInt("code") == 200) { + return result.getJSONObject("result"); + } else if (result.getInt("code") == 500) { + println("${city} ${address} ${result}") + } + } catch (Exception e) { + println("${city} ${address} ${e.getMessage()}") + e.printStackTrace(); + } + return null + } + + def findCityCodeValue(provinceCode, cityCode){ + def q = new BasicDBObject() + q.put("province_code", provinceCode.toString()) + if(cityCode){ + q.put("city_code", cityCode.toString()) + } + def dbcursor = FANGJIA_OFFICIAL_AREA_MAP.find(q) + def count = dbcursor.count() + if(count >1){ + def data = null + for(def db : dbcursor){ + data = db + break + } + return data; + } + } + + def findRegionCodeValue(value){ + def q = new BasicDBObject() + q.put("district_code", value.toString()) + def dbcursor = FANGJIA_OFFICIAL_AREA_MAP.find(q) + def count = dbcursor.count() + if(count == 1){ + def data = null + for(def db : dbcursor){ + data = db + break + } + return data; + } + return null + } + + def matchMap(def dist){ + def gaodeRest = FangjiaClientUtils.searchMapData(dist.city, dist.name,"住宅", "高德") + if(!gaodeRest && dist.address){ + gaodeRest = FangjiaClientUtils.searchMapData(dist.city, dist.address,"住宅", "高德") + } + def gdlocationMap = null + def bdlocationMap = null + if(gaodeRest){ + gdlocationMap = getMapLocation2(dist, gaodeRest); + if(gdlocationMap){ + gdlocationMap.source = "高德" + } + } + + def baiduRest = FangjiaClientUtils.searchMapData(dist.city, dist.name,"住宅", "百度") + if(baiduRest){ + bdlocationMap = getMapLocation2(dist, baiduRest); + if(bdlocationMap){ + bdlocationMap.source = "百度" + } + } + + if(gdlocationMap && bdlocationMap){ + if(gdlocationMap.score>=bdlocationMap.score){ + return gdlocationMap + }else{ + return bdlocationMap + } + }else if(gdlocationMap){ + return gdlocationMap + }else if(bdlocationMap){ + return bdlocationMap + } + return null + } + + def getMapLocation(def dist, def rets){ + def lat = null + def lng = null + def region = null + def mapName = null + def address = null + def score = 0 + def type = null + def maxScore = 0 + for(def r : rets){ + if(!r.city){ + continue + } + if(!r.city.contains(dist.city)){ + continue + } + if(["公交车站"].contains(r.type)){ + continue + } + def name = nameReplace(r.name, dist.city) + + def distName = nameReplace(dist.name, dist.city) + + if(distName == name){ + lat = r.lat + lng = r.lng + mapName = r.name + region = r.district + address = r.address + type = r.type + + if(dist.region && r.district && r.district.contains(dist.region)){ + score = 10 + }else { + score = 5 + } + + }else{ + def word = null + if (name.length() >= distName.length()) { + word = name.replace(distName, "") + } else if (name.length() < distName.length()) { + word = dist.name.replace(name, "") + } + if (word) { + def words = ["商住楼", "小区","公寓","别墅","花园"] + for (def w : words) { + if (word == w ) { + lat = r.lat + lng = r.lng + region = r.district + mapName = r.name + address = r.address + type = r.type + if(r.district && dist.region && r.district.contains(dist.region)){ + score = 9 + }else { + score = 5 + } + break + } + } + if(!lng){ + Pattern pattern = Pattern.compile("\\d+号楼"); + Matcher matcher = pattern.matcher(word); + if(matcher.find()){ + def newWord = word.replace(matcher.group(),"") + if(!newWord){ + lat = r.lat + lng = r.lng + region = r.district + mapName = r.name + address = r.address + type = r.type + if(r.district && dist.region && r.district.contains(dist.region)){ + score = 8 + }else { + score = 5 + } + break + } + } + } + + if(!lng){ + Pattern pattern = Pattern.compile("[A-Za-z]区"); + Matcher matcher = pattern.matcher(word); + if(matcher.find()){ + def newWord = word.replace(matcher.group(),"") + if(!newWord){ + lat = r.lat + lng = r.lng + region = r.district + mapName = r.name + address = r.address + type = r.type + if(r.district && dist.region && r.district.contains(dist.region)){ + score = 7 + }else { + score = 5 + } + break + } + } + } + } + } + if(!lng && dist.address && (dist.address.contains("号") || dist.address.contains("弄") || dist.address.contains("交汇")) && r.address){ + if((r.address.contains("号") || r.address.contains("弄") || r.address.contains("交汇")) && r.address.contains(dist.address)){ + lat = r.lat + lng = r.lng + region = r.district + mapName = r.name + address = r.address + type = r.type + if(r.district && dist.region && r.district.contains(dist.region)){ + score = 8 + }else { + score = 5 + } + } + } + + if(lng){ + break + } + + if(!lng){ + def score1 = CosineSimilarAlgorithm.getSimilarity(name, distName)/2; + def score2 = 0.0 + def score3 = 0.0 + if(r.district && dist.region && r.district.contains(dist.region)){ + score3 = 0.2 + } + if(dist.address && r.address){ + score2 = CosineSimilarAlgorithm.getSimilarity(dist.address, r.address)/2 + } + score3 = com.fangjia.script.common.DataHandleUtils.getDoubleRound(2, (score1 + score2)*10) + if(score3 >maxScore){ + lat = r.lat + lng = r.lng + region = r.district + mapName = r.name + address = r.address + type = r.type + score = score3 + maxScore = score3 + } + println dist.name +" "+ name+" "+dist.address +" "+r.address + " " +score3 + } + } + if(lng){ + return ["region":region, "lng":lng, "lat":lat, "mapName":mapName, "address":address, "score":score, "type":type] + } + return null + } + + + + def getMapLocation2(def dist, def rets){ + def data1 = findSimilarData(dist, rets, true) + def data2 = findSimilarData(dist, rets, false) + println data1 + println data2 + if(data1 && data1.score>1){ + return data1 + }else if(data2 && data2.score>1){ + return data2 + } + return null + } + + def findSimilarData(def dist, def rets, def regionImpr){ + def lat = null + def lng = null + def region = null + def mapName = null + def address = null + def score = 0 + def type = null + def maxScore = 0 + def standardName = null + for(def r : rets){ + if(!r.city){ + continue + } + if(!r.city.contains(dist.city)){ + continue + } + if(["公交车站"].contains(r.type)){ + continue + } + if(regionImpr && !r.district){ + continue + } + if(regionImpr && r.district && dist.region && !r.district.contains(dist.region) && !dist.region.contains(r.district)){ + continue + } + + def name = nameReplace(r.name, dist.city) + + def distName = nameReplace(dist.name, dist.city) + + def score3 = 0.0 + if(r.district && dist.region && r.district.contains(dist.region)){ + score3 = 0.2 + } + + def score1 = CosineSimilarAlgorithm.getSimilarity(name, distName)/2; + def score2 = 0.0 + if(dist.address && r.address){ + score2 = CosineSimilarAlgorithm.getSimilarity(dist.address, r.address)/2 + } + score3 = score1 + score2 + if(r.type && r.type.contains("住宅")){ + score3 = score3 + 0.2 + } + if(score3 >maxScore){ + lat = r.lat + lng = r.lng + region = r.district + mapName = r.name + standardName = name + address = r.address + type = r.type + score = com.fangjia.script.common.DataHandleUtils.getDoubleRound(2, score3*10) + maxScore = score3 + } + println dist.name +" "+ name+" "+dist.address +" "+r.address + " " +score3 + + } + if(lng){ + return ["region":region, "lng":lng, "lat":lat, "mapName":mapName, "address":address, "score":score, "type":type, "standardName":standardName] + } + return null + } + + def nameReplace(def s, city){ + if(s.contains("(") && s.contains(")")){ + def s1 = s.toString().substring(s.indexOf("(")+1, s.indexOf(")")) + if(s1.contains("路")||s1.contains("街")||s1.contains("大道")){ + s = s.toString().substring(0, s.indexOf("(")) + } + } + def name = s.replace("?","").replace("-","").replace("&","").replace("·","").replace(".","").replace("(","").replace(")","").replace("(","").replace(")","").replace("【","").replace("】","").replace("11期","十一期").replace("12期","十二期").replace("13期","十三期").replace("14期","十四期").replace("15期","十五期").replace("16期","十六期").replace("17期","十七期").replace("18期","十八期").replace("19期","十九期").replace("10期","十期").replace("1期","一期").replace("2期","二期").replace("3期","三期").replace("4期","四期").replace("5期","五期").replace("6期","六期").replace("7期","七期").replace("8期","八期").replace("9期","九期").replace("1区","一区").replace("2区","二区").replace("3区","三区").replace("4区","四区").replace("5区","五区").replace("6区","六区").replace("7区","七区").replace("8区","八区").replace("9区","九区").replace(" ","").replace("建设中","").replace("钢混","").trim() + if(name.startsWith(city)){ + def newName = name.replace(city+"市","").replace(city,"") + if(newName && newName=="花园"){ + return name + }else{ + name = newName + } + } + return name + } + + def matchDb(){ + ExcelUtils.multiThreadingModify("d:\\数据\\浙商\\测试.xlsx",{d-> + if(d['Y'] && d['S'] && d['S'].toString().toDouble().toInteger()==6){ + def results = null + def name = null + if(d['U'] && d['Z'] && d['AA']){ + name = nameReplace(d['AA'], d['U']) + results = OpenApiClient.addressMatch(d['U'], name, "property", false, d['Z'], null) + } + + if((d['AB'] && results && results.precise<0) || !results){ + results = OpenApiClient.addressMatch(d['U'], d['AB'], "property", false, d['Z'], null) + } + + + if(results && results.precise>=0){ + d['AL'] = results.city + d['AM'] = results.district + d['AN'] = results.name + + println "输入:"+d['V']+" "+d['W']+" 地图:"+d['Z']+" "+d['AA']+" 匹配:"+results.district+" "+results.name + def alias = results.alias + if(!alias){ + alias = [] + } + if(!alias.contains(results.name)){ + alias << results.name + } + + if(d['V']!="东莞"){ + def distance = null + if(d['AC'] && d['AD']){ + def maplng = null + def maplat = null + if(d['Y'] && d['Y']=="高德"){ + def baidu = BaiDuApiUtils.geoconv(d['AC'].toDouble(), d['AD'].toDouble(), 3) + maplng = baidu.x + maplat = baidu.y + }else if(d['Y'] && d['Y']=="百度"){ + maplng = d['AC'] + maplat = d['AD'] + } + distance = GeoHelper.GetDistance(maplat, maplng, results.lat, results.lng) + } + + d['AQ'] = distance.toInteger() + + if(d['V']){//原始数据有区域 + if(results.district && d['Z'] && results.district==d['Z']){//结果与地图区域相同 + if(results.district==d['V']){ + if(alias.contains(name)){ + + if(d['W']){ + def oldName = nameReplace(d['W'], d['U']) + if(oldName!=name){ + def group1 = pattern(oldName) + def group2 = pattern(name) + if(group1!=group2){ + results = OpenApiClient.addressMatch(d['U'], oldName, "property", false, d['V'], null) + if(results && results.precise>=0 && results.district == d['V']){ + if(results.name!=oldName){ + if(results.alias && results.alias.contains(oldName)){ + d['AP'] = "已存在,区域相同" + }else{ + d['AP'] = "新增小区【"+oldName+"】" + } + }else{ + d['AP'] = "已存在,区域相同" + } + }else{ + d['AP'] = "新增小区【"+oldName+"】" + } + }else{ + if(!alias.contains(oldName)){ + d['AP'] = "添加别名【"+oldName+"】" + }else{ + d['AP'] = "已存在" + } + } + }else{ + if(results.district==d['V']){ + d['AP'] = "已存在" + } + } + }else{ + d['AP'] = "已存在,区域相同" + } + }else{ + def group = pattern(name) + if(group){ + d['AP'] = "新增小区【"+name+"】" + }else{ + d['AP'] = "添加别名【"+ d['W'] +"】" + } + } + }else{ +// def msg = "" +// def names = [] +// if(d['W'] && !alias.contains(d['W'])){ +// def fq = pattern(d['W']) +// if(fq){ +// msg = "添加小区【"+ d['AA'] +"】" +// }else{ +// names << d['W'] +// msg = "添加别名【"+ d['W'] +"】" +// } +// } + d['AP'] = "输入区域问题 " + } + }else{ + if(d['Z'] && d['V'] && d['Z']==d['V']){//结果与地图区域不同,输入与地图相同 + d['AP'] = "新增小区【"+d['AA']+"】" + }else{ + if(results.address && d['AB'] && results.address.contains("路") && results.address.contains("号")){ + if(d['AB'].contains("路") && d['AB'].contains("号") && results.address.contains(d['AB'])){ + d['AP'] = "库中区域存疑 " + }else{ + d['AP'] = "区域都不同,地址不同" + } + }else{ + d['AP'] = "区域地址存疑 " + } + } + } + }else{ + if(results.district && d['Z'] && results.district==d['Z']) {//结果与地图区域相同 + if(!alias.contains(name)){ + def group1 = pattern(name) + if(group1 && name==d['W']){ + d['AP'] = "新增小区【"+d['AA']+"】" + }else if(group1 && d['X'] && d['AB'] && d['X'].contains("路") && d['X'].contains("号") && d['AB'].contains("路") && d['AB'].contains("号") && d['AB'].contains(d['X'])){ + d['AP'] = "新增小区【"+d['AA']+"】 添加别名【"+d['W']+"】" + } + }else{ + d['AP'] = "区域缺失,名称存在" + } + } + + } + }else{ + d['AP'] = "城市需特殊处理" + } + + + + }else{ + d['AP'] = "新增小区【"+d['AA']+"】" + } + } + },0) + } + + + def matchDb2(){ + ExcelUtils.multiThreadingModify("d:\\数据\\浙商\\浙商人工(单-测试入库-全).xlsx",{d-> + if(d['Y'] && d['S'] && d['S'].toString().toDouble().toInteger()==6){ + def results = null + def name = null + if(d['U'] && d['Z'] && d['AA']){ + name = nameReplace(d['AA'], d['U']) + results = OpenApiClient.addressMatch(d['U'], name, "property", false, d['Z'], null) + } + + if((d['AB'] && results && results.precise<0) || !results){ + results = OpenApiClient.addressMatch(d['U'], d['AB'], "property", false, d['Z'], null) + } + + + if(results && results.precise>=0){ + d['AL'] = results.city + d['AM'] = results.district + d['AN'] = results.name + + println "输入:"+d['V']+" "+d['W']+" 地图:"+d['Z']+" "+d['AA']+" 匹配:"+results.district+" "+results.name + + if(d['V']!="东莞"){ + def alias = results.alias + if(!alias){ + alias = [] + } + if(!alias.contains(results.name)){ + alias << results.name + } + + def distance = null + if(d['AC'] && d['AD']){ + def maplng = null + def maplat = null + if(d['Y'] && d['Y']=="高德"){ + def baidu = BaiDuApiUtils.geoconv(d['AC'].toDouble(), d['AD'].toDouble(), 3) + maplng = baidu.x + maplat = baidu.y + }else if(d['Y'] && d['Y']=="百度"){ + maplng = d['AC'] + maplat = d['AD'] + } + distance = GeoHelper.GetDistance(maplat, maplng, results.lat, results.lng) + } + + d['AQ'] = distance.toInteger() + +// if(d['V']){//原始数据有区域 + if(distance>5000){ + if(results.district && d['Z'] && results.district==d['Z']){ + if(alias.contains(name)){ + + }else{ + d['AP'] = "新增小区【"+d['AA']+"】" + } + }else{ + d['AP'] = "新增小区【"+d['AA']+"】" + } + + }else{ + + } + + }else{ + d['AP'] = "城市需特殊处理" + } + }else{ + d['AP'] = "新增小区【"+d['AA']+"】" + } + } + },0) + } + + + def getOptMsg(def d, def name, def results){ + def msg = "" + if(results.district && d['Z'] && results.district==d['Z']){//结果与地图区域相同 + if(alias.contains(name)){ + + if(d['W']){ + def oldName = nameReplace(d['W'], d['U']) + if(oldName!=name && !alias.contains(oldName)){ + def group1 = pattern(oldName) + def group2 = pattern(name) + if(group1 && group1!=group2){ + results = OpenApiClient.addressMatch(d['U'], oldName, "property", false, d['V'], null) + if(results && results.precise>=0 && results.district == d['V']){ + if(results.name!=oldName){ + if(results.alias && results.alias.contains(oldName)){ + msg = "已存在,区域相同" + }else{ + msg = "新增小区【"+d['W']+"】" + } + }else{ + msg = "已存在,区域相同" + } + }else{ + msg = "新增小区【"+d['W']+"】" + } + }else{ + if(results.district!=d['V']){ + msg = "输入区域问题" + }else{ + msg = "添加别名【"+ d['W'] +"】" + } + + } + }else{ + if(results.district==d['V']){ + msg = "已存在" + }else{ + msg = "输入区域问题" + } + } + }else{ + msg = "已存在,输入无名称" + } + + }else{ + def group = pattern(name) + if(group){ + msg = "新增小区【"+name+"】" + if(d['W']){ + def oldName = nameReplace(d['W'], d['U']) + if(oldName==name){ + if(d['V'] != d['Z']){ + msg = msg +" 输入区域问题" + } + } + } + + }else{ + if(distance>1000){ + if(d['W'] && results.address && d['AB'] && d['AB'].contains(results.address)){ + msg = "添加别名【"+ d['W'] +"】" + }else{ + if(d['X'] && d['AB'] && d['X'].contains("路") && d['X'].contains("号") && d['AB'].contains("路") && d['AB'].contains("号") && d['AB'].contains(d['X'])){ + msg = "新增小区【"+d['AA']+"】" + } + } + }else{ + msg = "添加别名【"+ d['W'] +"】" + } + + } + d['AP'] = msg + } + }else{ + msg = "区域不同,距离相近" + if(!alias.contains(name)){ + def group = pattern(name) + if(group){ + msg = msg +" 新增小区【"+d['AA']+"】" + }else{ + msg = msg + " 添加别名【"+ d['W'] +"】" + } + } + } + return msg; + } + + def matchDb3(){ + ExcelUtils.multiThreadingModify("d:\\数据\\浙商\\浙商人工(单-测试入库-全).xlsx",{d-> + if(d['Y'] && d['S'] && d['S'].toString().toDouble().toInteger()==6){ + def results = null + def name = null + if(d['U'] && d['Z'] && d['AA']){ + name = nameReplace(d['AA'], d['U']) + results = OpenApiClient.addressMatch(d['U'], name, "property", false, d['Z'], null) + } + + if((d['AB'] && results && results.precise<0) || !results){ + results = OpenApiClient.addressMatch(d['U'], d['AB'], "property", false, d['Z'], null) + } + + + if(results && results.precise>=0){ + d['AL'] = results.city + d['AM'] = results.district + d['AN'] = results.name + + println "输入:"+d['V']+" "+d['W']+" 地图:"+d['Z']+" "+d['AA']+" 匹配:"+results.district+" "+results.name + + if(d['V']!="东莞"){ + def alias = getAllAlias(results) + def distance = calcuteDistance(d, results.lat, results.lng) + d['AO'] = distance.toInteger() + + def score1 = com.fangjia.script.common.DataHandleUtils.getDoubleRound(2, CosineSimilarAlgorithm.getSimilarity(name, results.name)*10) + d['AP'] = score1 + if(distance>5000){ + def group = pattern(name) + if(group){ + if(alias.contains(name)){ + d['AQ'] = "已存在[距离远]" + }else{ + d['AQ'] = "新增【"+d['AA']+"】[距离远]" + } + }else{ + if(results.district==d['Z'] && score1>7){ + d['AQ'] = "已存在[距离远]" + if(results.district!=d['V']){ + d['AQ'] = "已存在[距离远][输入区域有误]" + } + }else if(score1<=3){ + d['AQ'] = "匹配可疑[距离远]" + }else{ + d['AQ'] = "其它可疑[距离远]" + } + } + }else{ + + if(results.district==d['Z'] && score1>7){ + d['AQ'] = "已存在[距离近]" + if(results.district!=d['V']){ + d['AQ'] = "已存在[距离近][输入区域有误]" + } + + }else if(score1<=3){ + d['AQ'] = "匹配可疑[距离近]" + }else if(score1>3 && score1<=7){ + if(distance<1000){ + if(alias.contains(name)){ + d['AQ'] = "已存在[距离近]" + if(results.district!=d['V']){ + d['AQ'] = "已存在[距离近][输入区域有误]" + } + }else{ + d['AQ'] = "加别名[距离近]" + if(results.district!=d['V']){ + d['AQ'] = "加别名[距离近][输入区域有误]" + } + def group = pattern(name) + def group1 = pattern(results.name) + if(group && group!=group1){ + d['AQ'] = "加别名[距离近][分期]" + } + } + }else{ + d['AQ'] = "待定" + if(results.address && d['AB'] && d['AB'].contains(results.address)){ + d['AQ'] = "加别名[距离近][路相关]" + } + } + }else if(results.district!=d['Z'] && score1>7){ + d['AQ'] = "[距离近][区域不同]" + def group = pattern(name) + def group1 = pattern(results.name) + if(group && group!=group1){ + d['AQ'] = "[距离近][区域不同][分期]" + } + } + } + }else{ + d['AQ'] = "城市需特殊处理" + } + }else{ + d['AQ'] = "新增小区【"+d['AA']+"】" + } + } + },0) + } + + + def saveOrUpdate(){ + def dists =[] + def rows = [] + ExcelUtils.modify("d:\\数据\\浙商\\浙商人工(单-测试入库-全).xlsx",{d-> + if(d['Y'] && d['S'] && d['S'].toString().toDouble().toInteger()==6) { + if(d['AQ'] && d['AQ'].contains("新增")){ + if(d['U'] && d['Z'] && d['AA']){ + def name = nameReplace(d['AA'], d['U']) + def key = d['U']+"##"+d['Z']+"##"+name + if(!dists.contains(key)){ + dists << key + def address = DataUtil.formateAddress(d['U'], d['AB']) + def lat2 = null + def lng2 = null + def lat = null + def lng = null + if(d['Y']=="高德"){ + lng = (d['AC'].toDouble()*10000000000).longValue() + lat = (d['AD'].toDouble()*10000000000).longValue() + def gmap = GeocodeUtils.getGaodeToBaidu(d['AC'].toDouble(), d['AD'].toDouble()) + if(gmap){ + lng2 = (gmap.get("lng")*10000000000).longValue() + lat2 = (gmap.get("lat")*10000000000).longValue() + } + }else if(d['Y']=="百度"){ + lng2 = (d['AC'].toDouble()*10000000000).longValue() + lat2 = (d['AD'].toDouble()*10000000000).longValue() + def gmap = GeocodeUtils.geoconvBaiduToGoogle(d['AC'].toDouble(), d['AD'].toDouble()) + if(gmap){ + lng = (gmap.get("lng")*10000000000).longValue() + lat = (gmap.get("lat")*10000000000).longValue() + } + } + def estateType = "普通住宅" + if(d['AF']){ + if(d['AF'].contains("别墅")){ + estateType = "别墅" + }else if(d['AF'].contains("商住")){ + estateType = "商住" + } + } + def alias = [] + if(d['AG'] && d['AG']!=name){ + alias << d['AG'] + } + rows << [d['U'], d['Z'], name, address, estateType, lng, lat, lng2, lat2, alias] + } + } + } + } + },0) + + + + for(def row : rows){ + + def isRegion = RegionData.isRegion(row[0], row[1]) + if(isRegion){ + def obj = findData(row[0], row[1], row[2]) + if(!obj){ + obj = CommonDatautils.findDistrictByAlias(row[0], row[1], row[2], "district") + } + if(!obj){ + //saveData(row[0], row[1], row[2], row[3], row[4], row[5], row[6], row[7], row[8], row[9]) + //FangjiaClientUtils.sendDistrictPriceCalculate(row[0], row[1], row[2], 4); + println "【新增】 ${row[0]} ${row[1]} ${row[2]}" + }else{ + println "【已存在】 ${row[0]} ${row[1]} ${row[2]}" + } + }else{ + println "区域不能标准化 -- ${row[0]} ${row[1]} ${row[2]}" + } + + } + + println "【add】"+dists.size() + + } + + def findData(city, region, newName){ + def q = new BasicDBObject() + q.put("city", city) + q.put("region", region) + q.put("name", newName) + q.put("status", 0) + q.put("cat", "district") + return DbUtils.seaweed.findOne(q) + } + + def saveData(city, region, newName, newAddress, estateType, lng2, lat2, lng, lat, alias){ + def d = new BasicDBObject() + d.put("city", city) + d.put("region", region) + d.put("name", newName) + if(alias){ + d.put("alias", alias) + } + + if(newAddress){ + d.put("address", newAddress) + } + + if(estateType){ + d.put("estate_type2", estateType) + } + d.put("status", 0) + d.put("cat", "district") + + d.put("c_date", new Date()) + d.put("m_date", new Date()) + def markMap = [:] + markMap.put("data_source", "机器入库") + d.put("mark_map",markMap) + + d.put("lat", lat) + d.put("lat2", lat2) + d.put("lng", lng) + d.put("lng2", lng2) + DbUtils.seaweed.save(d) + return d; + } + + def getAllAlias(def results){ + def alias = results.alias + if(!alias){ + alias = [] + } + if(!alias.contains(results.name)){ + alias << results.name + } + return alias + } + + def calcuteDistance(d, dbLat, dbLng) { + def distance = null + if(d['AC'] && d['AD']){ + def maplng = null + def maplat = null + if(d['Y'] && d['Y']=="高德"){ + def baidu = BaiDuApiUtils.geoconv(d['AC'].toDouble(), d['AD'].toDouble(), 3) + maplng = baidu.x + maplat = baidu.y + }else if(d['Y'] && d['Y']=="百度"){ + maplng = d['AC'] + maplat = d['AD'] + } + distance = GeoHelper.GetDistance(maplat, maplng, dbLat, dbLng) + } + return distance; + } + def pattern(def s){ + Pattern pattern = Pattern.compile("(东|南|北|西|甲|乙|丙|丁|一|二|三|四|五|六|七|八|九|十|东|南|西|北|\\w)+(期|区|街区)") + def matcher = pattern.matcher(s) + if(matcher.find()){ + return matcher.group(); + } + return null + } +} diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/es/EsMonthHouseStat.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/es/EsMonthHouseStat.groovy index 7860afad9..d8c7e72a3 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/es/EsMonthHouseStat.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/es/EsMonthHouseStat.groovy @@ -18,8 +18,10 @@ class EsMonthHouseStat { static main(args) { def task = new EsMonthHouseStat() // task.run() - task.tradeRun() +// task.tradeRun() // task.houseRun() + + task.statHouseUpdateCount() } def run(){ @@ -233,4 +235,37 @@ class EsMonthHouseStat { rMap.total_new = rets.size() return rMap } + + + def statHouseUpdateCount(){ + def citys = CommonDatautils.findAllCity() + def root = "lianjia.com" + def rets = [] + rets << ["城市", "5月上旬","5月中旬","5月下旬", "6月前10天"] + for(def city : citys){ + def count1 = findHouseCount(city, root, 20210501, 20210510) + def count2 = findHouseCount(city, root, 20210510, 20210520) + def count3 = findHouseCount(city, root, 20210520, 20210530) + def count4 = findHouseCount(city, root, 20210601, 20210610) + rets << [city, count1, count2, count3, count4] + } + ExcelUtils.write("d:\\数据\\房源分析\\城市房源分析.xlsx", rets, "数据") + } + + + def findHouseCount(def city, def root, def startDate, def endDate){ + def aliasStr = null; + def total = 0 + try{ + String sql = "select count(*) as c from latest_seawater where cat='sell' and city='${city}' and p_date >= ${startDate} and p_date<${endDate} and root='${root}'" + + ElasticUtils.executeSqlQuery(sql,{data-> + total = data.c + }) + }catch(Exception e){ + e.printStackTrace() + } + + return total + } } diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/location/LocationTest.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/location/LocationTest.groovy index 55e0943ab..e75f561b2 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/location/LocationTest.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/location/LocationTest.groovy @@ -1,11 +1,14 @@ package com.fangjia.script.gj.location import com.fangjia.core.RegionData +import com.fangjia.core.util.BaiDuApiUtils +import com.fangjia.core.util.GeocodeUtils import com.fangjia.script.commData.CommonDatautils import com.fangjia.script.gj.district.FangjiaClientUtils import com.fangjia.script.looplineinfo.InsidePolygon import com.fangjia.script.looplineinfo.Point import com.fangjia.script.util.DateUtils +import com.fangjia.script.util.DbUtils import com.fangjia.script.util.ExcelUtils import com.fangjia.script.util.GeoHelper import com.fangjia.script.util.MongoUtils @@ -21,9 +24,16 @@ class LocationTest { static DBCollection POLYLINECOLL = MongoUtils.getCollectionByAuth20190102("fangjia_crawl", "region_polyline") static main(args) { - def task = new LocationTest() + Date date=new Date(1622802609); + SimpleDateFormat format = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss"); + String str = format.format(date); + System.out.println(str); + +// def task = new LocationTest() // task.run() - task.run2() +// task.run2() +// task.run3() +// task.run4() } def run(){ @@ -43,11 +53,14 @@ class LocationTest { def loactionMap = matchMap(dist) if(loactionMap){ - ret << "地图" + ret << loactionMap.source ret << loactionMap.region ret << loactionMap.mapName + ret << loactionMap.address def distance = GeoHelper.GetDistance(dist.lat/10000000000.0, dist.lng/10000000000.0, loactionMap.lat, loactionMap.lng) ret << distance + ret << loactionMap.lng + ret << loactionMap.lat if(dist.region==loactionMap.region){ println index+" "+dist.region +" "+dist.name +" 不在区域范围内,与地图相同" }else{ @@ -61,31 +74,188 @@ class LocationTest { } } - ExcelUtils.write("d:\\数据\\福州\\区域经纬度错误3.xlsx", rets, "区域不在行政范围内") + ExcelUtils.write("d:\\数据\\福州\\区域经纬度错误5.xlsx", rets, "区域不在行政范围内") } def run2(){ - ExcelUtils.multiThreadingModify("d:\\数据\\福州\\福州小区.xlsx") + def datas = ExcelUtils.read("d:\\数据\\福州\\福州小区.xlsx", 0,1) + def rets = [] + rets << ["城市", "区域", "名称", "来源","地图区域", "地图名称","距离", "地图地址", "地图经度", "地图纬度"] + def index = 0 + for(def d : datas){ + def count = d['H'].toInteger() + def label = d['G'].toInteger() + if(count && count>=4 && label!=-1){ + index++ + def city = d['C'] + def region = d['D'] + def name = d['B'] + + def dist = CommonDatautils.queryDistrictById(d['A'].toString()) + def ret = [] + ret << city + ret << region + ret << name + + def loactionMap = matchMap(dist) + if(loactionMap){ + ret << loactionMap.source + ret << loactionMap.region + ret << loactionMap.mapName + def distance = null + if(loactionMap.source=="高德"){ + distance = GeoHelper.GetDistance(dist.lat/10000000000.0, dist.lng/10000000000.0, loactionMap.lat, loactionMap.lng) + }else if(loactionMap.source=="百度"){ + distance = GeoHelper.GetDistance(dist.lat2/10000000000.0, dist.lng2/10000000000.0, loactionMap.lat, loactionMap.lng) + } + ret << distance + ret << loactionMap.address + ret << loactionMap.lng + ret << loactionMap.lat + if(dist.region==loactionMap.region){ + println index+" "+dist.region +" "+dist.name +" 不在区域范围内,与地图相同" + }else{ + println index+" "+dist.region +" "+dist.name +" 不在区域范围内 地图:"+ loactionMap.region+" "+distance + } + }else{ + // ret << "" + println index+" "+dist.region +" "+dist.name +" 不在区域范围内,地图没找到" + } + + rets << ret + } + } + + ExcelUtils.write("d:\\数据\\福州\\相同经纬度小区4.xlsx", rets, "数据") + + } + + + + def run3(){ + ExcelUtils.modify("d:\\数据\\福州\\区域经纬度错误5.xlsx", {d-> + def distance = null + if(d['G']){ + distance = d['G'].toDouble() + } + if(d['E'] && d['B'] == d['E'] && distance && distance>=1000){ + def mapName = d['F'].toString().replace("(建设中)","").replace("?","").replace("&","").replace("·","").replace(".","").replace("(","").replace(")","").replace("(","").replace(")","").replace("11期","十一期").replace("12期","十二期").replace("13期","十三期").replace("14期","十四期").replace("15期","十五期").replace("16期","十六期").replace("17期","十七期").replace("18期","十八期").replace("19期","十九期").replace("10期","十期").replace("1期","一期").replace("2期","二期").replace("3期","三期").replace("4期","四期").replace("5期","五期").replace("6期","六期").replace("7期","七期").replace("8期","八期").replace("9期","九期") + if(mapName.contains("-") && mapName.endsWith("楼")){ + mapName = mapName.substring(0, mapName.indexOf("-")) + } + if(d['C']==mapName){ + def q = new BasicDBObject() + q.put("city", d['A']) + q.put("region", d['B']) + q.put("name", d['C']) + q.put("cat", 'district') + q.put("status", 0) + def lng2 = null + def lat2 = null + def lng = null + def lat = null + if(d['D']=="高德"){ + lng = (d['I'].toDouble()*10000000000).longValue() + lat = (d['J'].toDouble()*10000000000).longValue() + def gmap = GeocodeUtils.getGaodeToBaidu(d['I'].toDouble(), d['J'].toDouble()) + if(gmap){ + lng2 = (gmap.get("lng")*10000000000).longValue() + lat2 = (gmap.get("lat")*10000000000).longValue() + } + + }else{ + lng2 = (d['I'].toDouble()*10000000000).longValue() + lat2 = (d['J'].toDouble()*10000000000).longValue() + def gmap = GeocodeUtils.geoconvBaiduToGoogle(d['I'].toDouble(), d['J'].toDouble()) + if(gmap){ + lng = (gmap.get("lng")*10000000000).longValue() + lat = (gmap.get("lat")*10000000000).longValue() + } + } + def udata = new BasicDBObject() + udata.put("lng", lng) + udata.put("lat", lat) + udata.put("lng2", lng2) + udata.put("lat2", lat2) + udata.put("m_date", new Date()) + DbUtils.seaweed.update(q, new BasicDBObject('$set', udata)) + + + def dbo = DbUtils.seaweed.findOne(q) + d['K'] = "修正" + + } + } + },0) + + } + + + def run4(){ + ExcelUtils.modify("d:\\数据\\福州\\福州相同经纬度小区4.xlsx", {d-> + def region = d['B'] + def name = d['C'] + if(!d['K']){ + def dist = ["city":d['A'], "region":region, "name":name] + def loactionMap = matchMap(dist) + if(loactionMap){ + d['D'] << loactionMap.source + d['E'] << loactionMap.region + d['F'] << loactionMap.mapName + def distance = null + if(loactionMap.source=="高德"){ + distance = GeoHelper.GetDistance(dist.lat/10000000000.0, dist.lng/10000000000.0, loactionMap.lat, loactionMap.lng) + }else if(loactionMap.source=="百度"){ + distance = GeoHelper.GetDistance(dist.lat2/10000000000.0, dist.lng2/10000000000.0, loactionMap.lat, loactionMap.lng) + } + d['G'] << distance + d['H'] << loactionMap.address + d['I'] << loactionMap.lng + d['J'] << loactionMap.lat + if(dist.region==loactionMap.region){ + println index+" "+dist.region +" "+dist.name +" 不在区域范围内,与地图相同" + }else{ + println index+" "+dist.region +" "+dist.name +" 不在区域范围内 地图:"+ loactionMap.region+" "+distance + } + }else{ + // ret << "" + println index+" "+dist.region +" "+dist.name +" 不在区域范围内,地图没找到" + } + } + },0) } def matchMap(def dist){ def gaodeRest = FangjiaClientUtils.searchMapData(dist.city, dist.name,"住宅", "高德") - def locationMap = null + def gdlocationMap = null + def bdlocationMap = null if(gaodeRest){ - locationMap = getMapLocation(dist, gaodeRest); - if(locationMap){ - return locationMap + gdlocationMap = getMapLocation(dist, gaodeRest); + if(gdlocationMap){ + gdlocationMap.source = "高德" } } - def baiduRest = null - if (!locationMap) { - baiduRest = FangjiaClientUtils.searchMapData(dist.city, dist.name,"住宅", "百度") - } + def baiduRest = FangjiaClientUtils.searchMapData(dist.city, dist.name,"住宅", "百度") if(baiduRest){ - locationMap = getMapLocation(dist, baiduRest); + bdlocationMap = getMapLocation(dist, baiduRest); + if(bdlocationMap){ + bdlocationMap.source = "百度" + } } - return locationMap + + if(gdlocationMap && bdlocationMap){ + if(gdlocationMap.score>=bdlocationMap.score){ + return gdlocationMap + }else{ + return bdlocationMap + } + }else if(gdlocationMap){ + return gdlocationMap + }else if(bdlocationMap){ + return bdlocationMap + } + return null } def getMapLocation(def dist, def rets){ @@ -93,6 +263,8 @@ class LocationTest { def lng = null def region = null def mapName = null + def address = null + def score = 0 for(def r : rets){ if(!r.city){ continue @@ -100,12 +272,20 @@ class LocationTest { if(!r.city.contains(dist.city)){ continue } - def name = r.name.replace("?","").replace("&","").replace("·","").replace(".","").replace("(","").replace(")","").replace("(","").replace(")","").replace("【","").replace("】","").replace("11期","十一期").replace("12期","十二期").replace("13期","十三期").replace("14期","十四期").replace("15期","十五期").replace("16期","十六期").replace("17期","十七期").replace("18期","十八期").replace("19期","十九期").replace("10期","十期").replace("1期","一期").replace("2期","二期").replace("3期","三期").replace("4期","四期").replace("5期","五期").replace("6期","六期").replace("7期","七期").replace("8期","八期").replace("9期","九期").replace(" ","").trim() + def name = r.name.replace("?","").replace("&","").replace("·","").replace(".","").replace("(","").replace(")","").replace("(","").replace(")","").replace("【","").replace("】","").replace("11期","十一期").replace("12期","十二期").replace("13期","十三期").replace("14期","十四期").replace("15期","十五期").replace("16期","十六期").replace("17期","十七期").replace("18期","十八期").replace("19期","十九期").replace("10期","十期").replace("1期","一期").replace("2期","二期").replace("3期","三期").replace("4期","四期").replace("5期","五期").replace("6期","六期").replace("7期","七期").replace("8期","八期").replace("9期","九期").replace(" ","").replace("建设中","").replace("钢混","").trim() if(dist.name == name){ lat = r.lat lng = r.lng mapName = r.name region = r.district + address = r.address + + if(r.district.contains(dist.region)){ + score = 10 + }else { + score = 5 + } + }else{ def word = null if (name.length() >= dist.name.length()) { @@ -116,11 +296,18 @@ class LocationTest { if (word) { def words = ["商住楼", "小区","公寓","别墅"] for (def w : words) { - if (word == w) { + if (word == w ) { lat = r.lat lng = r.lng region = r.district mapName = r.name + address = r.address + + if(r.district.contains(dist.region)){ + score = 9 + }else { + score = 5 + } break } } @@ -132,6 +319,30 @@ class LocationTest { lng = r.lng region = r.district mapName = r.name + address = r.address + if(r.district.contains(dist.region)){ + score = 8 + }else { + score = 5 + } + break + } + } + + if(!lng){ + Pattern pattern = Pattern.compile("[A-Za-z]区"); + Matcher matcher = pattern.matcher(word); + if(matcher.find()){ + lat = r.lat + lng = r.lng + region = r.district + mapName = r.name + address = r.address + if(r.district.contains(dist.region)){ + score = 7 + }else { + score = 5 + } break } } @@ -142,10 +353,8 @@ class LocationTest { } } if(lng){ - return ["region":region, "lng":lng, "lat":lat, "mapName":mapName] + return ["region":region, "lng":lng, "lat":lat, "mapName":mapName, "address":address, "score":score] } - - return null } diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/test/AddressStandard.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/test/AddressStandard.groovy new file mode 100644 index 000000000..a5cbbabbf --- /dev/null +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/test/AddressStandard.groovy @@ -0,0 +1,301 @@ +package com.fangjia.script.gj.test + +import com.fangjia.script.common.DataHandleUtils +import com.fangjia.script.gj.district.FangjiaClientUtils +import com.fangjia.script.util.ExcelUtils + +import java.util.concurrent.atomic.AtomicInteger + +class AddressStandard { + static main(args) { + def task = new AddressStandard() + task.run()//地址标准化 + task.test()//统计标准化后的结果 + + } + + def run2(){ + ExcelUtils.multiThreadingModify("d://数据//地址标准化//阿里接口拆分数据(1).xlsx",{d-> + def json = FangjiaClientUtils.addressStand(d['A']) + + if(json && json.status!=0){ + println d['A'] +" "+json + } + },0) + } + + def run(){ + // def datas = ExcelUtils.read("d://数据//地址标准化//阿里接口拆分数据(1-test).xlsx",0,1) + + + ExcelUtils.multiThreadingModify("d://数据//地址标准化//阿里接口拆分数据(1).xlsx",{d-> + if(d['A']){ + for(def i=18;i + total.addAndGet(1) + def sameCount = 0 + def list = [] + def a_prov = d['B']?d['B']:"" + if(a_prov && (a_prov.toString().endsWith("省") || a_prov.toString().endsWith("自治区"))){ + a_prov = a_prov.replace("维吾尔自治区","").replace("回族自治区","").replace("壮族自治区","").replace("自治区省","").replace("自治区","").replace("省","") + } + def b_prov = d['S']?d['S']:"" + if(b_prov && (b_prov.toString().endsWith("省") || b_prov.toString().endsWith("自治区"))){ + b_prov = b_prov.replace("维吾尔自治区","").replace("回族自治区","").replace("壮族自治区","").replace("自治区省","").replace("自治区","").replace("省","") + } + if(a_prov == b_prov){ + provCount.addAndGet(1) + sameCount++ + }else{ + list << "prov" + } + + def a_city = d['C']?d['C']:"" + def b_city = d['T']?d['T']:"" + if(a_city && a_city.toString().endsWith("市")){ + a_city = a_city.replace("市","") + } + if(b_city && b_city.toString().endsWith("市")){ + b_city = b_city.replace("市","") + } + if(a_city == b_city){ + cityCount.addAndGet(1) + sameCount++ + }else{ + list << "city" + } + + def a_dist = d['D']?d['D']:"" + def b_dist = d['U']?d['U']:"" + if(a_dist && (a_dist.endsWith("区")||a_dist.endsWith("县"))){ + a_dist = a_dist.replace("新区","").replace("区","").replace("县","") + } + if(b_dist && (b_dist.endsWith("区")||b_dist.endsWith("县"))){ + b_dist = b_dist.replace("新区","").replace("区","").replace("县","") + } + + if(a_dist == b_dist){ + distCount.addAndGet(1) + sameCount++ + }else{ + list << "dist" + } + + def a_town = d['E']?d['E']:"" + def b_town = d['V']?d['V']:"" + if(a_town==b_town){ + townCount.addAndGet(1) + sameCount++ + }else{ + list << "town" + } + + + + def a_road = d['F']?d['F']:"" + def b_road = d['X']?d['X']:"" + if(a_road == b_road){ + roadCount.addAndGet(1) + sameCount++ + }else{ + list << "road" + } + + + def a_roadno = d['H']?d['H']:"" + def b_roadno = d['Y']?d['Y']:"" + if(a_roadno == b_roadno){ + roadnoConnt.addAndGet(1) + sameCount++ + }else{ + list << "roadno" + } + + def a_community = d['I']?d['I']:"" + def b_community = d['W']?d['W']:"" + if(a_community == b_community){ + communityCount.addAndGet(1) + sameCount++ + }else{ + list << "community" + } + + def a_poi = d['J']?d['J']:"" + def b_poi = d['AA']?d['AA']:"" + if(a_poi == b_poi){ + poiCount.addAndGet(1) + sameCount++ + }else{ + list << "poi" + } + + def a_subpoi = d['K']?d['K']:"" + def b_subpoi = d['AB']?d['AB']:"" + if(a_subpoi == b_subpoi){ + subpoiCount.addAndGet(1) + sameCount++ + }else{ + list << "subpoi" + } + + def a_houseno = d['L']?d['L'].replace("号楼","").replace("幢楼","").replace("栋号","").replace("幢","").replace("栋","").replace("座","").replace("号","").replace("楼","").replace("第","").replace("楼",""):"" + def b_houseno = d['AD']?d['AD'].replace("号楼","").replace("幢楼","").replace("栋号","").replace("幢","").replace("栋","").replace("座","").replace("号","").replace("楼","").replace("第","").replace("楼",""):"" + if(a_houseno == b_houseno){ + housenoCount.addAndGet(1) + sameCount++ + }else{ + list << "houseno" + } + + def a_cellno = d['M']?chNumberConver(d['M'].replace("单元","")):"" + def b_cellno = d['AE']?chNumberConver(d['AE'].replace("单元","")):"" + if(a_cellno == b_cellno){ + cellnoCount.addAndGet(1) + sameCount++ + }else{ + list << "cellno" + } + + def a_floorno = d['N']?d['N'].replace("层楼","").replace("层","").replace("楼",""):"" + def b_floorno = d['AF']?d['AF'].replace("层楼","").replace("层","").replace("楼",""):"" + if(a_floorno == b_floorno){ + floornoCount.addAndGet(1) + sameCount++ + }else{ + list << "floorno" + } + + def a_roomno = d['O']?d['O'].replace("室号","").replace("号门","").replace("室","").replace("房","").replace("户","").replace("号","").replace("门",""):"" + def b_roomno = d['AG']?d['AG'].replace("室号","").replace("号门","").replace("室","").replace("房","").replace("户","").replace("号","").replace("门",""):"" + if(a_roomno == b_roomno){ + roomnoCount.addAndGet(1) + sameCount++ + }else{ + list << "roomno" + } + + def a_dev = d['Q']?d['Q']:'' + def b_dev = d['AH']?d['AH']:'' + if(a_dev == b_dev){ + devCount.addAndGet(1) + sameCount++ + }else{ + list << "dev" + } + + d['AI'] = sameCount + d['AJ'] = list.join(",") + + },0) + + def totalCount = total.get() + println totalCount + def prov_pre = DataHandleUtils.getDoubleRound(2, provCount.get()*100.0/totalCount) + def city_pre = DataHandleUtils.getDoubleRound(2, cityCount.get()*100.0/totalCount) + def dist_pre = DataHandleUtils.getDoubleRound(2, distCount.get()*100.0/totalCount) + def town_pre = DataHandleUtils.getDoubleRound(2, townCount.get()*100.0/totalCount) + def community_pre = DataHandleUtils.getDoubleRound(2, communityCount.get()*100.0/totalCount) + def road_pre = DataHandleUtils.getDoubleRound(2, roadCount.get()*100.0/totalCount) + def roadno_pre = DataHandleUtils.getDoubleRound(2, roadnoConnt.get()*100.0/totalCount) + def poi_pre = DataHandleUtils.getDoubleRound(2, poiCount.get()*100.0/totalCount) + def subpoi_pre = DataHandleUtils.getDoubleRound(2, subpoiCount.get()*100.0/totalCount) + def houseno_pre = DataHandleUtils.getDoubleRound(2, housenoCount.get()*100.0/totalCount) + def cellno_pre = DataHandleUtils.getDoubleRound(2, cellnoCount.get()*100.0/totalCount) + def floor_pre = DataHandleUtils.getDoubleRound(2, floornoCount.get()*100.0/totalCount) + def roomno_pre = DataHandleUtils.getDoubleRound(2, roomnoCount.get()*100.0/totalCount) + def datas = [] + datas << ["总量", "prov","city","dist","town","community","road","roadno","poi","subpoi","houseno","cellno","floor","roomno"] + datas << [totalCount,prov_pre,city_pre,dist_pre,town_pre,community_pre,road_pre,roadno_pre,poi_pre,subpoi_pre,houseno_pre,cellno_pre,floor_pre,roomno_pre] + ExcelUtils.write("d://数据//地址标准化//测试结果1.xlsx", datas, "数据") + + } + + def chNumberConver(def s){ + if(s && s.size()==1){ + return s.replace("一","1").replace("二","2").replace("三","3").replace("四","4").replace("五","5").replace("六","6").replace("七","7").replace("八","8").replace("九","9").replace("十","10") + } + return s; + } +} diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/test/CrawlDistrictIdSelect.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/test/CrawlDistrictIdSelect.groovy index b26bd079b..d436aa081 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/test/CrawlDistrictIdSelect.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/test/CrawlDistrictIdSelect.groovy @@ -15,8 +15,8 @@ class CrawlDistrictIdSelect { static main(args) { def task = new CrawlDistrictIdSelect() -// task.run() - task.synch() + task.run() +// task.synch() } def synch(){ @@ -38,25 +38,34 @@ class CrawlDistrictIdSelect { def run(){ def datas = findCrawlData() + def list = [] datas.each {k,v-> if(v){ for(def d : v){ d.c_date = new Date() d.m_date = new Date() - districtseedsColl.save(d) + list << d } } + + if(list.size()==10000){ + districtseedsColl.insert(list) + list = [] + } + } + if(list.size()>0){ + districtseedsColl.insert(list) } } def findCrawlData(){ def q = new BasicDBObject() // q.put("city","福州") - q.put("source", "诸葛找房") - q.put("s_date",202010) + q.put("source", "安居客") + q.put("s_date",202012) def dbcursor = thirdPartyColl.find(q).batchSize(10000).setOptions(Bytes.QUERYOPTION_NOTIMEOUT) def retMap = [:] - def prefix = "xiaoqu.zhuge.com/" + def prefix = "anjuke.com/" def keyList =new HashSet() for(def db : dbcursor){ def url = null @@ -68,11 +77,11 @@ class CrawlDistrictIdSelect { url = db.url } if(url && url.contains(prefix) ){ - def district_id = url.substring(url.indexOf(prefix)+prefix.length(),url.size()-1) + def district_id = db.district_id//url.substring(url.indexOf(prefix)+prefix.length(),url.size()-1) if(district_id=="0"){ continue } - def shortCity = url.substring(url.indexOf("//")+2, url.indexOf(".xiaoqu.zhuge.com")) + def shortCity = url.substring(url.indexOf("//")+2, url.indexOf(".anjuke.com")) def key = shortCity +"#"+district_id def ids = retMap.get(key) @@ -83,7 +92,7 @@ class CrawlDistrictIdSelect { if(!keyList.contains(key)){ def map = new BasicDBObject() - map.root = "zhuge.com" + map.root = "anjuke.com" map.city_short= shortCity map.city = db.city map.region = db.region diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/thirdPriceCrawl/Esf58Crawl.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/thirdPriceCrawl/Esf58Crawl.groovy index 05d0c713e..53823ee85 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/thirdPriceCrawl/Esf58Crawl.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/thirdPriceCrawl/Esf58Crawl.groovy @@ -166,8 +166,8 @@ class Esf58Crawl extends BaseBatch{ def addr = JQueryUtils.selectText(".desc", detailDocument); if(info){ if(info.size()>=4){ - city = info[2].replace("小区","") - region = info[3].replace("小区","") + city = info[2].replace("小区","").trim() + region = info[3].replace("小区","").trim() } } if(!addr){ @@ -212,8 +212,8 @@ class Esf58Crawl extends BaseBatch{ def saveThirdParty(metadata){ DBObject obj = new BasicDBObject(); obj.put("district_id", metadata.url); - obj.put("city", metadata.city); - obj.put("region", metadata.region); + obj.put("city", metadata.city.trim()); + obj.put("region", metadata.region.trim()); obj.put("block", metadata.block); obj.put("name", metadata.name); obj.put("s_date", SHORT_DATE); @@ -221,6 +221,7 @@ class Esf58Crawl extends BaseBatch{ obj.put("addres", metadata.address); obj.put("avg_price", metadata.avg_price.toInteger()); obj.put("c_date", new Date()); + println obj.toString() thirdPartyColl.save(obj); } diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/thirdPriceCrawl/SynchDistrictPrice.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/thirdPriceCrawl/SynchDistrictPrice.groovy index b43b98223..93370a4a3 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/thirdPriceCrawl/SynchDistrictPrice.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/thirdPriceCrawl/SynchDistrictPrice.groovy @@ -4,14 +4,17 @@ import com.fangjia.script.util.CommonUtils import com.mongodb.BasicDBObject import org.apache.solr.common.util.Hash +/** + * 福州农商行 + * 将抓取的小区均价同步到第三方均价表中 + */ class SynchDistrictPrice { def static final SHORT_DATE = Utils.getLastMonth() def thirdPartyColl = CrawalDBUtil.getThirdPartyCol(); def districtDetailColl = CrawalDBUtil.getDistrictDetailColl(); - def sourceMap = ["jiwu.com":"吉屋"] - //"58.com":"58同城","lianjia.com":"链家","ke.com":"贝壳找房","fang.com":"搜房","fangdd.com":"房多多","jiwu.com":"吉屋" - + def sourceMap = ["jiwu.com":"吉屋","58.com":"58同城","lianjia.com":"链家","ke.com":"贝壳找房","fang.com":"搜房","fangdd.com":"房多多","house365.com":"365淘房", + "anjuke.com":"安居客"] static main(args) { def task = new SynchDistrictPrice() task.run() @@ -19,13 +22,14 @@ class SynchDistrictPrice { def run(){ - def citys = ["福州"]//CommonUtils.findAllCity() + def citys = CommonUtils.findAllCity() sourceMap.each { k, v -> for(def city : citys){ def rets = findCrawlDistrictDetails(city, k, v) - synchData(city, rets, v) println city +" "+rets.size() + synchData(city, rets, v) + } } @@ -76,7 +80,7 @@ class SynchDistrictPrice { }catch(Exception e){ println e.getMessage() } - + println "["+source +"-"+city + "] add:"+ rets.size()+" 原抓取:"+count }else{ for(def r : rets){ def flag = findIsExists(r.city, r.region, r.name, SHORT_DATE, source) @@ -85,9 +89,10 @@ class SynchDistrictPrice { thirdPartyColl.save(r) } } + println "["+source +"-"+city + "] add:"+ addIndex+" 原抓取:"+count } - println "["+source +"-"+city + "] add:"+ addIndex+" 原抓取:"+count + } def findIsExists(def city, def region, def name, def sDate, def source){ diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/thirdPriceCrawl/TestCrawlTotalCount.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/thirdPriceCrawl/TestCrawlTotalCount.groovy index 15910f5cb..24b482d1e 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/thirdPriceCrawl/TestCrawlTotalCount.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/thirdPriceCrawl/TestCrawlTotalCount.groovy @@ -14,8 +14,8 @@ class TestCrawlTotalCount { def run(){ def sources = ["房估估","诸葛找房","安居客","搜房","中原地产","赶集","链家","兔博士","城市房产","贝壳找房","Q房","Q房(新)","中原地产(新)","乐有家","乐有家(新)","房探","房多多","小资家","我爱我家","麦田","房途","C21","乐居","房产超市","58同城","365淘房"] for(def source :sources){ - def count1 = findTotalCount(source, 202102); - def count2 = findTotalCount(source, 202103); + def count1 = findTotalCount(source, 202105); + def count2 = findTotalCount(source, 202106); println source +" "+count1 +" "+count2 } } diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/util/ExcelUtils.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/util/ExcelUtils.groovy index 4cad85c53..1cf782236 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/util/ExcelUtils.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/util/ExcelUtils.groovy @@ -1,5 +1,6 @@ package com.fangjia.script.util +import com.fangjia.script.lixue.util.DateFormat import org.apache.commons.lang.StringUtils import org.apache.poi.hssf.record.BoolErrRecord @@ -389,8 +390,8 @@ public class ExcelUtils{ try{ closure.call(row) int count = counter.incrementAndGet() - if(count % 100 == 0){ - println("执行进度:${com.fangjia.core.util.DataUtil.roundDouble((count * 100)/total, 2)}%") + if(count % 1000 == 0){ + println(DateUtils.formatDateByPattern(new Date(), "yyyy-MM-dd HH:mm:ss")+"执行进度:${com.fangjia.core.util.DataUtil.roundDouble((count * 100)/total, 2)}%") } }catch (Exception e){ println(row) diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/util/OpenApiClient.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/util/OpenApiClient.groovy index a23842026..6e5696107 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/util/OpenApiClient.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/util/OpenApiClient.groovy @@ -121,9 +121,9 @@ public class OpenApiClient { params.put("suggestName", suggestName) } - String content = new HttpRequest().get("http://open.fangjia.com/address/match", params, "utf-8") + String content = new HttpRequest().get("http://open.fangjia.com/address/matchV2", params, "utf-8") while (content == null) { - content = new HttpRequest().get("http:///open.fangjia.com/address/match", params, "utf-8") + content = new HttpRequest().get("http:///open.fangjia.com/address/matchV2", params, "utf-8") println("${city} ${input} ${cat}程序调用异常") Thread.sleep(6000) } diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/yinjihaun/amap/GaoDeApiUtils.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/yinjihaun/amap/GaoDeApiUtils.groovy index ad4b4f415..781687012 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/yinjihaun/amap/GaoDeApiUtils.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/yinjihaun/amap/GaoDeApiUtils.groovy @@ -173,6 +173,41 @@ class GaoDeApiUtils { } + def static around2(String location,String radius, String keywords = null, types = null) { + Map params = new HashMap() + params.put("key", GAO_DE_AK) + if(keywords){ + params.put("keywords", keywords) + } + if(types){ + params.put("types", types) + } + if(location){ + params.put("location", location) + } + if(radius){ + params.put("radius", radius) + } + params.put("sortrule", "distance") + String content = null; + try { + content = new HttpRequest().newGet("https://restapi.amap.com/v3/place/around", params, "utf-8") + } catch (Exception e) { + return null; + } + JSONObject jsonObj = JSONObject.fromObject(content) + if(jsonObj && jsonObj.get("infocode").equals("10000")){ + return jsonObj + }else if(jsonObj && jsonObj.get("infocode").equals("20003")){ + return null; + } else { + return around(location, radius, keywords, types) + } + return null; + } + + + def static searchKey(def keyword, def cityCode){ def url = "https://www.amap.com/service/poiInfo?query_type=TQUERY&pagesize=20&pagenum=1&qii=true&cluster_state=5&need_utd=true&utd_sceneid=1000&div=PC1000&addr_poi_merge=true&is_classify=true&zoom=10&city="+cityCode+"&geoobj=113.082026%7C22.516777%7C115.059608%7C22.922141&keywords="+URLEncoder.encode(keyword,"UTF-8") String content