diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/fuzhou/Test1.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/fuzhou/Test1.groovy new file mode 100644 index 000000000..cf1e4950f --- /dev/null +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/fuzhou/Test1.groovy @@ -0,0 +1,133 @@ +package com.fangjia.script.fuzhou + +import com.fangjia.script.commData.CommonDatautils +import com.fangjia.script.common.DataHandleUtils +import com.fangjia.script.thirdPriceCrawl.CrawalDBUtil +import com.fangjia.script.util.ExcelUtils +import com.fangjia.script.util.MongoUtils +import com.mongodb.BasicDBObject +import com.mongodb.DBCollection + +def datas = ExcelUtils.read("d:\\数据\\福州\\福州4月2.xlsx",0,1) +def count0 = 0 +def count1 = 0 +def count2 = 0 +def count3 = 0 +def count4 = 0 +def count5 = 0 +def count6 = 0 +def count7 = 0 +def count8 = 0 +def total=4943 +for(def d : datas){ + def list = [] + if(d['G']){ + list << d['G'] + } + if(d['H']){ + list << d['H'] + } + if(d['I']){ + list << d['I'] + } + if(d['J']){ + list << d['J'] + } + if(d['K']){ + list << d['K'] + } + if(d['L']){ + list << d['L'] + } + if(d['M']){ + list << d['M'] + } + if(d['N']){ + list << d['N'] + } + if(d['O']){ + list << d['O'] + } + if(list.size()==0){ + count0++ + } + if(list.size()==1){ + count1++ + } + if(list.size()==2){ + count2++ + } + if(list.size()==3){ + count3++ + } + if(list.size()==4){ + count4++ + } + if(list.size()==5){ + count5++ + } + + if(list.size()==6){ + count6++ + } + if(list.size()==7){ + count7++ + } + if(list.size()==8){ + count8++ + } +} + +println("count0:"+ (DataHandleUtils.getDoubleRound(2, count0*100.0/total))) +println("count1:"+ (DataHandleUtils.getDoubleRound(2, count1*100.0/total))) +println("count2:"+ (DataHandleUtils.getDoubleRound(2, count2*100.0/total))) +println("count3:"+ (DataHandleUtils.getDoubleRound(2, count3*100.0/total))) +println("count4:"+ (DataHandleUtils.getDoubleRound(2, count4*100.0/total))) +println("count5:"+ (DataHandleUtils.getDoubleRound(2, count5*100.0/total))) +println("count6:"+ (DataHandleUtils.getDoubleRound(2, count6*100.0/total))) +println("count7:"+ (DataHandleUtils.getDoubleRound(2, count7*100.0/total))) +println("count8:"+ (DataHandleUtils.getDoubleRound(2, count8*100.0/total))) + + + +////def districts = CommonDatautils.findAllDistricts("福州", "district", ["city", "region", "name"]); +//def total = 8018 +//def names = ["诸葛找房":"zhugezhaofang_esf_price","安居客":"anjuke_esf_price","搜房":"soufang_esf_price","链家":"lianjia_esf_price","城市房产":"cityhouse_esf_price","房探":"fangtan_esf_price","房多多":"fangdoudou_esf_price","麦田":"maitian_esf_price", "58同城":"58tc_esf_price"] +// +//def rets = [] +//def sDate = 202011 +//rets << ["时间","诸葛找房","安居客","搜房","链家","城市房产","房探","房多多","麦田", "58同城"] +//def ret = [] +//ret << sDate +//names.each{k,v-> +// def crawlCount = crawlCount(k, sDate) +// def count = count(v, sDate) +// println k +" "+DataHandleUtils.getDoubleRound(2, count*100.0/total)+"("+crawlCount+")" +// ret << DataHandleUtils.getDoubleRound(2,count*100.0/total)+"("+crawlCount+")" +//} +//rets.add(ret) +//ExcelUtils.write("d:\\数据\\第三方均价覆盖率1.xlsx",rets,"数据") +// +//def crawlCount(k, sDate){ +// def thirdPartyColl = CrawalDBUtil.getThirdPartyCol(); +// def q = new BasicDBObject() +// q.put("city","福州") +// q.put("s_date", sDate) +// q.put("source",k) +// return thirdPartyColl.count(q) +//} +// +// +//def count(name, sDate){ +// DBCollection AVERAGE_STATISTICS = MongoUtils.getCollectionByName("fangjia", "average_statistics", "192.168.0.43") +// def q = new BasicDBObject() +// q.put("city","福州") +// q.put("s_date", sDate) +// q.put(name, new BasicDBObject('$gt', 0)); +// return AVERAGE_STATISTICS.count(q) +//} + + + + + diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/ccb/CcbCityPriceStat.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/ccb/CcbCityPriceStat.groovy new file mode 100644 index 000000000..c60a811b5 --- /dev/null +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/ccb/CcbCityPriceStat.groovy @@ -0,0 +1,77 @@ +package com.fangjia.script.gj.ccb + +import com.fangjia.script.commData.CommonDatautils +import com.fangjia.script.util.CommonUtils +import com.fangjia.script.util.DateUtils +import com.fangjia.script.util.ExcelUtils +import com.fangjia.script.util.MongoUtils +import com.mongodb.BasicDBObject +import com.mongodb.DBCollection + +class CcbCityPriceStat { + + public static final def lineUrl = "mongodb://developer:G00IDCDEV12385@192.168.0.43:28017"; + public static final DBCollection DEAL_PRICE_COLL = MongoUtils.getCollectionByName2("fangjia", "deal_price", lineUrl) + + static main(args) { + def task = new CcbCityPriceStat() + task.run() + } + def run(){ + def citys = ["北京","上海","广州","深圳","重庆","郑州","长沙","西安","成都","东莞","佛山","杭州","合肥","南京","青岛","沈阳","苏州","天津","武汉","珠海","中山","长春","扬州","烟台","徐州","无锡","温州","太原","台州","石家庄","绍兴","厦门","泉州","宁波","南通","南宁","南昌","兰州","常州","大连","福州","贵阳","哈尔滨","海口","惠州","济南","嘉兴","金华","昆明","遵义","淄博","株洲","镇江","肇庆","漳州","湛江","岳阳","银川","宜昌","盐城","芜湖","乌鲁木齐","唐山","泰州","汕头","三亚","清远","秦皇岛","马鞍山","洛阳","柳州","连云港","廊坊","九江","吉林","淮安","湖州","呼和浩特","沧州","保定","安庆","张家口","西宁","日照","拉萨","锦州","大同","常德","北海","文昌","陵水","保亭"] + def rets = [] + for(def city : citys){ + def districts = CommonDatautils.findAllDistrictByCity(city, "district", ["city","region", "name", "alias"]) + def priceMap = CommonDatautils.queryDistrictPrice(city, DateUtils.parseFromTightYYYYMMDD("2021-05-30")) + def cityRets = [] + for(def dist : districts){ + def tradeAvgPrice = findTrades(dist, "2021-02-01", "2021-06-01") + def weekendPrice = priceMap.get(city +"##"+dist.region+"##"+dist.name) + if(tradeAvgPrice && weekendPrice){ + def row = [city, dist.region, dist.name, weekendPrice, tradeAvgPrice.toInteger()] + cityRets << row + println row.toString() + } + } + print city +" "+ cityRets.size() + rets.addAll(cityRets) + } + + println "总量:"+rets.size() + ExcelUtils.write("d:\\数据\\92个城市价格数据.xlsx", rets, "所有") + } + + + def findTrades(dist, startDate, endDate){ + def q = new BasicDBObject() + q.put("city", dist.city) + q.put("region", dist.region) + + if(dist.alias){ + if(!dist.alias.contains(dist.name)){ + dist.alias << dist.name + } + q.put("district_name", new BasicDBObject('$in', dist.alias)) + }else{ + q.put("district_name", dist.name) + } + + q.put("trade_date", new BasicDBObject('$gte', DateUtils.parseFromTightYYYYMMDD(startDate)).append('$lt',DateUtils.parseFromTightYYYYMMDD(endDate))) + def dbcorsor = DEAL_PRICE_COLL.find(q) + def avg_price = 0 + def total_price = 0 + def count = 0 + + for(def d : dbcorsor){ + if(d.avg_price){ + total_price = total_price + d.avg_price + count++ + } + } + if(count > 3){ + avg_price = total_price/count + } + return avg_price + } + +} diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/chongqingblank/DataMatch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/chongqingblank/DataMatch.groovy new file mode 100644 index 000000000..9f84f6479 --- /dev/null +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/chongqingblank/DataMatch.groovy @@ -0,0 +1,322 @@ +package com.fangjia.script.gj.chongqingblank + +import com.fangjia.core.RegionData +import com.fangjia.script.commData.CommonDatautils +import com.fangjia.script.gj.district.FangjiaClientUtils +import com.fangjia.script.util.CommonUtils +import com.fangjia.script.util.CosineSimilarAlgorithm +import com.fangjia.script.util.ExcelUtils +import com.fangjia.script.util.MongoUtils +import com.fangjia.script.util.OpenApiClient +import com.mongodb.BasicDBObject +import com.mongodb.DBCollection +import net.sf.json.JSONObject +import com.fangjia.script.gj.* + +import java.util.regex.Matcher +import java.util.regex.Pattern + +class DataMatch { + + static final DBCollection FANGJIA_OFFICIAL_AREA_MAP = MongoUtils.getCollectionByName("business", "fangjia_official_area_map", "mongo.db.fangjia.com") + + + + static main(args) { + def task = new DataMatch() +// task.run1() + task.run2() +// task.run3() +// println task.findAddress("昌州街道棠香路") + +// task.run2MatchV2( "D:\\数据\\重庆农商行\\估值1-V2.xlsx") + } + + def run1(){ + def fileName = "D:\\数据\\重庆农商行\\估值2-V5.xlsx" + //matchRegionValue(fileName); + matchDistrict(fileName) +// ExcelUtils.multiThreadingModify(fileName,{d-> +// if(d['O']=="新增"){ +// +// d['L'] = d['P'] +// d['M'] = d['Q'] +// d['N'] = d['R'] +// } +// },0) + + } + + def parseAddress(def fileName){ + ExcelUtils.multiThreadingModify(fileName,{d-> + + },0) + } + + def matchDistrict(fileName){ + ExcelUtils.multiThreadingModify(fileName,{d-> + def region = d['Q'] + def city = d['P'] +// if(!d['I'] && !d['J']){ +// city = d['B'] +// region = d['C'] +// } + if(d['R'] && !d['G']){ + def results = null +// if(d['E'] && isStreet(d['E'])){//加上效果并没有想象中的好 +// results = OpenApiClient.addressMatch(city, d['E'], "property", false, region, d['D']) +// } + if((results && results.precise<0) || !results){ + results = OpenApiClient.addressMatch(city, d['R'], "property", false, region, null) + } +// if((d['E'] && results && results.precise<0) || !results){ +// results = OpenApiClient.addressMatch(city, d['E'], "property", false, region, d['D']) +// } + if(results && results.precise>=0){ + d['K'] = results.city + d['L'] = results.district + d['M'] = results.name + d['N'] = results.address + } + } + },0); + } + + def run2(){ + def fileName = "D:\\数据\\重庆农商行\\估值2-V2.xlsx" +// matchRegionValue(fileName); + matchDistrict2(fileName) + } + + + def matchDistrict2(fileName){ + ExcelUtils.multiThreadingModify(fileName,{d-> + def region = d['I'] + def city = d['J'] + if(!d['I']){ + city = d['J'] + } + if(city && d['E'] && !d['O'] ){ + def addressResult = OpenApiClient.addressStandard(city, d['E']) + if(addressResult){ + if(addressResult.street && addressResult.streetNumber){ + d['H'] = addressResult.street+addressResult.streetNumber + } + } + def results = null + if(d['H']){ + results = OpenApiClient.addressMatch(city, d['H'], "property", false, region, null) + } + + if((d['E'] && results && results.precise<0) || !results){ + results = OpenApiClient.addressMatch(city, d['E'], "property", false, region, null) + } + + if(results && results.precise>=0){ + d['K'] = results.city + d['L'] = results.district + d['M'] = results.name + d['N'] = results.address + } + } + },0); + } + + def run3(){ + def fileName = "D:\\数据\\重庆农商行\\估值3.xlsx" +// matchDistrict3(fileName) + ExcelUtils.multiThreadingModify(fileName,{d-> + if(d['I']){ + d['I'] = d['I'].toString().replace("省","").replace("市","") + } + if(d['J'] && d['I']){ + d['J'] = RegionData.formatRegion(d['I'], d['J']) + } + },0) + } + + + def matchDistrict3(fileName){ + ExcelUtils.multiThreadingModify(fileName,{d-> + def address = d['B'] + if(address) { + def arrs = address.split("->") + def city = null + def region = null + def addr = null + if (arrs && arrs.size() == 4) { + city = arrs[0] + region = arrs[2] + addr = arrs[3] + d['I'] = city + d['J'] = region + } else if (arrs && arrs.size() == 2) { + def parseCity = CommonUtils.parseCityInAddress(arrs[1]) + city = parseCity + d['I'] = parseCity + addr = arrs[1] + } + if (addr && !d['O']) { + def addressResult = OpenApiClient.addressStandard(city, addr) + if (addressResult) { + if (!region) { + region = addressResult.district + d['J'] = region + } + if (addressResult.street && addressResult.streetNumber) { + d['H'] = addressResult.street + addressResult.streetNumber + } + } + def results = null + def name = null + if (d['C']) { + name = d['C'].toString().replace(".", "").replace("(", "").replace(")", "").replace("。", "") + .replace("·", "").replace("-", "").replace("`", "").replace("?", "").replace("‘", "") + .replace("1期", "一期").replace("2期", "二期").replace("3期", "三期") + .replace("4期", "四期").replace("5期", "五期").replace("12区", "十二区").replace("1区", "一区").replace("2区", "二区").replace("3区", "三区") + .replace("4区", "四区").replace("5区", "五区").replace("a区", "A区") + .replace("B区", "B区").replace("c区", "C区").replace("d区", "D区") + } + if (name) { + results = OpenApiClient.addressMatch(city, name, "property", false, region, null) + } + + if ((addr && results && results.precise < 0) || !results) { + results = OpenApiClient.addressMatch(city, addr, "property", false, region, name) + } + if (results && results.precise >= 0) { + d['K'] = results.city + d['L'] = results.district + d['M'] = results.name + d['N'] = results.address + } + } + } + + },0); + } + + + + def matchRegionValue(fileName){ + ExcelUtils.multiThreadingModify(fileName,{d-> + def code = d['C'] + if(code && code.toString().isNumber()){ + if(code.toInteger().toString()=="500226"){ + d['I'] = "荣昌" + d['J'] = "重庆" + }else if(code.toInteger().toString()=="500223"){ + d['I'] = "潼南" + d['J'] = "重庆" + }else{ + def data = findRegionCodeValue(code.toInteger().toString()); + if(data){ + d['I'] = data.fangjia_district + d['J'] = data.fangjia_city + }else{ + data = findCityCodeValue(code.toInteger().toString()) + if(data){ + d['J'] = data.fangjia_city + } + } + } + + } + + },0) + } + + def findCityCodeValue(value){ + def q = new BasicDBObject() + q.put("city_code", value) + def dbcursor = FANGJIA_OFFICIAL_AREA_MAP.find(q) + def count = dbcursor.count() + if(count >1){ + def data = null + for(def db : dbcursor){ + data = db + break + } + return data; + } + } + + def findRegionCodeValue(value){ + def q = new BasicDBObject() + q.put("district_code", value) + def dbcursor = FANGJIA_OFFICIAL_AREA_MAP.find(q) + def count = dbcursor.count() + if(count == 1){ + def data = null + for(def db : dbcursor){ + data = db + break + } + return data; + } + } + + def findAddress(String address) { + //Pattern ROAD_PATTERN = Pattern.compile('.*[路|大道|巷|胡同|街]\\d+号'); + Pattern ROAD_PATTERN = Pattern.compile('.[里,路,道,街,巷,路段,胡同].?\\d+.?\\d+[号,弄]|.[里,路,道,街,巷,路段,胡同].?\\d+.?[号,弄]') + Matcher m = ROAD_PATTERN.matcher(address) + String newAddress + if (m.find()) { + newAddress = m.group() + } + return newAddress + } + + def run2MatchV2(fileName){ + ExcelUtils.multiThreadingModify(fileName, { d -> + def region = d['I'] + def city = d['J'] + if(!d['I'] && !d['J']){ + city = d['B'] + region = d['C'] + } + if(d['D']) { + def name = d['D'].toString().replace(".", "").replace("(", "").replace(")", "").replace("。", "") + .replace("·", "").replace("-", "").replace("`", "").replace("?", "").replace("‘", "") + .replace("1期", "一期").replace("2期", "二期").replace("3期", "三期") + .replace("4期", "四期").replace("5期", "五期").replace("12区", "十二区").replace("1区", "一区").replace("2区", "二区").replace("3区", "三区") + .replace("4区", "四区").replace("5区", "五区").replace("a区", "A区") + .replace("B区", "B区").replace("c区", "C区").replace("d区", "D区") + + def result = OpenApiUtils.matchDetailDistrict(city, name, region, true) + if (!result && d['E']) { + if (findAddress(d['E'])) { + result = OpenApiUtils.matchDetailDistrict(city, d['E'], region, true) + } + } + if(result){ + def matchResult = result.matchResult + if (result.credit > 0.5 && result.matchResult && !result.matchResult.match) { + matchResult = amendMatch(result, d['E']) + } + if (matchResult.match) { + d['AQ'] = result.city + d['AR'] = result.district + d['AS'] = result.name + d['AT'] = result.address + } + } + } + + },0) + } + + def amendMatch(result, address){ + def matchResult = result.matchResult + if(result.credit > 0.65 && matchResult.distance < 2500){ + matchResult.match = true + }else if(result.credit>0.5 && address && result.address && findAddress(result.address)){ + if(result.address.contains(address) || address.contains(result.address)){ + matchResult.match = true + } + }else if(address && address.contains(result.district) && address.contains(result.name)){ + matchResult.match = true + } + return matchResult + } +} diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/chongqingblank/EvaluationPrice.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/chongqingblank/EvaluationPrice.groovy new file mode 100644 index 000000000..90eaa4e13 --- /dev/null +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/chongqingblank/EvaluationPrice.groovy @@ -0,0 +1,47 @@ +package com.fangjia.script.gj.chongqingblank + +import com.fangjia.script.commData.CommonDatautils +import com.fangjia.script.gj.district.FangjiaClientUtils +import com.fangjia.script.util.CosineSimilarAlgorithm +import com.fangjia.script.util.ExcelUtils +import com.fangjia.script.util.OpenApiClient +import net.sf.json.JSONObject + +class EvaluationPrice { + + static main(args) { + def task = new EvaluationPrice() + task.run() + } + + def run(){ + ExcelUtils.multiThreadingModify("D:\\数据\\重庆农商行\\估值3-V3.xlsx",{ d-> + if(d['K'] && d['M']){ + def priceMap = getEvaluationPrice(d['J'], d['K'], d['L'], d['D'].toString()) + if(priceMap){ + d['E'] = priceMap.avgPrice + d['F'] = priceMap.totalPrice + } + } + },0); + } + + + def getEvaluationPrice(city, region, name, area){ + Map params = new HashMap(); + params.put("city", city); + params.put("district", region); + params.put("name", name); + params.put("size", area); + params.put("__refresh_cache","1") + + JSONObject result = OpenApiClient.evaluate(params); + if (result.getInt("code") == 200) { + if (result.containsKey("result")) { + return result.getJSONObject("result") + } + } else if (result.getInt("code") == 500) { + println("小区估价失败:${params.city} ${params.name} ${params.size} ${result}") + } + } +} diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/chongqingblank/TestMatchResult.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/chongqingblank/TestMatchResult.groovy new file mode 100644 index 000000000..9ff1149d5 --- /dev/null +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/chongqingblank/TestMatchResult.groovy @@ -0,0 +1,629 @@ +package com.fangjia.script.gj.chongqingblank + +import com.fangjia.core.RegionData +import com.fangjia.datastore.driver.DSCollection +import com.fangjia.datastore.driver.DataStore +import com.fangjia.script.commData.CommonDatautils +import com.fangjia.script.gj.district.FangjiaClientUtils +import com.fangjia.script.gj.district.NameProcess +import com.fangjia.script.util.CommonUtils +import com.fangjia.script.util.CosineSimilarAlgorithm +import com.fangjia.script.util.ExcelUtils +import com.fangjia.script.util.GeoHelper +import com.fangjia.script.util.OpenApiClient +import com.fangjia.script.yinjihaun.amap.GaoDeApiUtils +import com.fangjia.script.yinjihaun.amap.GaoDeMapApiData +import com.mongodb.BasicDBObject +import com.mongodb.DBObject + +class TestMatchResult { + + private static final DSCollection SEAWEED = new DataStore("seaweed.db.fangjia.com", 27007).getCollection("fangjia.seaweed") + + static main(args) { + def task = new TestMatchResult() +// task.run1("D:\\数据\\重庆农商行\\估值1.xlsx") + task.run2("D:\\数据\\重庆农商行\\估值2-V2.xlsx") +// task.run3("D:\\数据\\重庆农商行\\估值3.xlsx") + +// task.nearDistRun("D:\\数据\\重庆农商行\\估值2-V5.xlsx") +// task.run2V1("D:\\数据\\重庆农商行\\估值2-V2.xlsx") + +// task.similarScore("D:\\数据\\重庆农商行\\估值2-V5.xlsx") + +// task.mapRun("D:\\数据\\重庆农商行\\估值2-V4-map.xlsx"); + + } + + + + def run1(def fileName){ + ExcelUtils.multiThreadingModify(fileName, { d -> + def region = d['I'] + def city = d['J'] + if (!d['I'] && !d['J']) { + city = d['B'] + region = d['C'] + } + def newName = null + if(d['D']=="大川菲尔小城"){ + if (d['D']) { + newName = d['D'].toString().replace(".", "").replace("(", "").replace(")", "").replace("。", "") + .replace("·", "").replace("-", "").replace("`", "").replace("?", "").replace("‘", "") + .replace("1期", "一期").replace("2期", "二期").replace("3期", "三期") + .replace("4期", "四期").replace("5期", "五期").replace("12区", "十二区").replace("1区", "一区").replace("2区", "二区").replace("3区", "三区") + .replace("4区", "四区").replace("5区", "五区").replace("a区", "A区") + .replace("B区", "B区").replace("c区", "C区").replace("d区", "D区") + } + if (!region || (region && region == d['L'])) { + if (d['D'] && !d['O'] && d['M']) { + if (d['M'] == newName) { + d['O'] = "相同" + } + if (!d['O']) { + def dbo = CommonDatautils.findDistrictByAlias(d['K'], d['L'], d['D'], "district") + if (dbo) { + d['O'] = "别名匹配" + } + } +// if (!d['O'] && d['M'] && d['E']) { +// if (d['E'].toString().contains(d['M'])) { +// d['O'] = "名称in地址" +// } +// } + + if (!d['O']) { + def py1 = com.fangjia.core.util.DataUtil.hanyuQuanpin(newName, " ") + def py2 = com.fangjia.core.util.DataUtil.hanyuQuanpin(d['M'], " ") + if (py1 == py2) { + d['O'] = "拼音相同" + } + } + + if (!d['O'] && d['D'] && d['M'] && newName) { + def word = null + if (newName.toString().length() >= d['M'].toString().length()) { + word = newName.toString().replace(d['M'], "") + } else if (newName.toString().length() < d['M'].toString().length()) { + word = d['M'].toString().replace(newName, "") + } + if (word) { + def words = ["重庆", "小区", "(高层)", "花园", "恒都", "(洋房)", "社区"] + for (def w : words) { + if (word == w) { + d['O'] = "去部分词后相同" + break + } + } + + if (!d['O'] && !word.endsWith("区") && !word.endsWith("期") && !word.endsWith("团") && !word.endsWith("东苑") && !word.endsWith("西苑") && !word.endsWith("南苑") && !word.endsWith("北苑")) { + if (d['N'] && d['N'].toString().endsWith("号") && d['E'] && d['E'].toString().contains(d['N'])) { + + def score = CosineSimilarAlgorithm.getSimilarity(d['M'], newName); + if (score >= 0.65) { + println newName + " " + d['M'] + " " + score + d['O'] = "地址相同" + } + } + } + } + } + + if (!d['O'] && newName && !newName.contains("期") && !newName.endsWith("组团") && (!newName.endsWith("区") || newName.endsWith("小区"))) { + + def gaodeRest = FangjiaClientUtils.getMapData(city, region, newName, d['E'], "住宅", "高德") + if (!gaodeRest) { + gaodeRest = FangjiaClientUtils.getMapData(city, region, newName, d['E'], "住宅", "百度") + } + if (gaodeRest && gaodeRest.name) { + def mapName = gaodeRest.name.replace(".", "").replace("(", "").replace(")", "").replace("·", "").replace("`", "").replace("?", "") + if (mapName == d['M'] && gaodeRest.score >= 0.7) { + d['O'] = gaodeRest.source + "匹配" + println newName + " " + d['M'] + " " + gaodeRest.score + } + } + } + } + } + + + if (d['D'] && !d['O']) { + def gaodeRest = FangjiaClientUtils.getMapData(city, region, newName, d['E'], "住宅", "高德") + def mapName = null + if (gaodeRest && gaodeRest.name) { + mapName = gaodeRest.name.replace(".", "").replace("(", "").replace(")", "").replace("·", "").replace("`", "").replace("?", "") + } + if (!gaodeRest || (gaodeRest && mapName != newName)) { + gaodeRest = FangjiaClientUtils.getMapData(city, region, newName, d['E'], "住宅", "百度") + if (gaodeRest && gaodeRest.name) { + mapName = gaodeRest.name.replace(".", "").replace("(", "").replace(")", "").replace("·", "").replace("`", "").replace("?", "") + } + } + if (gaodeRest && mapName == newName) { + d['P'] = gaodeRest.district + d['Q'] = mapName + d['R'] = gaodeRest.address + d['S'] = gaodeRest.lng + d['T'] = gaodeRest.lat + d['U'] = gaodeRest.type + d['v'] = gaodeRest.source + } + } + } + + },0); + } + + def similarScore(def fileName){ + ExcelUtils.multiThreadingModify(fileName, { d -> + if(d['D'] && d['M']){ + def score = CosineSimilarAlgorithm.getSimilarity(d['D'], d['M']); + d['Q'] = score + } + },0) + } + + + def run2(def fileName){ + ExcelUtils.multiThreadingModify(fileName,{ d-> + def region = d['I'] + def city = d['J'] + if(!d['I']){ + city = d['J'] + } + +// if(d['E']=="巴南区花溪镇花溪新街88号2单元3-4#") { +// println "ss" + + + if (!region || (region && region == d['L'])) { + if (d['N'] && !d['O'] && d['E']) { + if (d['N'].toString().endsWith("号") && d['E'] && d['E'].toString().contains(d['N'])) { + d['O'] = "地址匹配" + } + } + } + + if (!d['O'] && d['M'] && d['E']) { + def matchName = d['M'] + if(matchName.toString().endsWith("小区")){ + matchName = matchName.toString().replace("小区","") + } + if(matchName.toString().startsWith(d['L'])){ + matchName = matchName.toString().replace(d['L'],"") + } + def oldName= NameProcess.nameStandard(city,region, d['E']) + if (oldName.toString().contains(matchName)) { + d['O'] = "名称in地址" + } + } + + if (d['E'] && !d['O'] ) { + def gaodeRest = null + + if (!gaodeRest && d['E']) { + gaodeRest = FangjiaClientUtils.getMapData(city, region, d['E'], null, "住宅", "高德") + } + if(!gaodeRest && d['H']){ + gaodeRest = FangjiaClientUtils.getMapData(city, region, d['H'], null, "住宅", "高德") + } + if (!gaodeRest && d['E']) { + gaodeRest = FangjiaClientUtils.getMapData(city, region, d['E'], null, "住宅", "百度") + } + if (!gaodeRest && d['H']) { + gaodeRest = FangjiaClientUtils.getMapData(city, region, d['H'], null, "住宅", "百度") + } + + if (gaodeRest && gaodeRest.score >= 0.5) { + d['P'] = gaodeRest.district + d['Q'] = gaodeRest.name + d['R'] = gaodeRest.address + d['S'] = gaodeRest.lng + d['T'] = gaodeRest.lat + d['U'] = gaodeRest.type + d['v'] = gaodeRest.source + } + } + + if(!d['O']){ + def locationMap = getLocation(city, d['E']) + if(locationMap){ + def mapRets = findNearestByMap(locationMap.lat, locationMap.lng) + if(mapRets){ + def o = mapRets.get(0) + d['W'] = o.cityname + d['X'] = o.adname + d['Y'] = o.name + if(o.address && o.address instanceof String){ + d['Z'] = o.address + } + d['AA'] = o.type + d['AB'] = o.distance + d['AC'] = o.location + } + def dbRets = findNearestDistrict(city, locationMap.lat, locationMap.lng, 1000, 10) + if(dbRets){ + def b = dbRets.get(0) + d['AD'] = b.region + d['AE'] = b.name + d['AF'] = b.address + d['AG'] = b.distance + } + } + } + +// } + + },0); + } + + def distanceRun(def fileName){ + ExcelUtils.multiThreadingModify(fileName, { d -> + def city = d['J'] + if(!d['O'] && d['E'] && d['E'].toString().length()>6){ + def locationMap = getLocation(city, d['E']) + if(locationMap){ + def mapRets = findNearestByMap(locationMap.lat, locationMap.lng) + if(mapRets){ + def o = mapRets.get(0) + d['W'] = o.cityname + d['X'] = o.adname + d['Y'] = o.name + if(o.address && o.address instanceof String){ + d['Z'] = o.address + } + d['AA'] = o.type + d['AB'] = o.distance + d['AC'] = o.location + } + def dbRets = findNearestDistrict(city, locationMap.lat, locationMap.lng, 1000, 10) + if(dbRets){ + def b = dbRets.get(0) + d['AD'] = b.region + d['AE'] = b.name + d['AF'] = b.address + d['AG'] = b.distance + } + } + } + },0); + } + + def nearDistRun(def fileName){ + ExcelUtils.multiThreadingModify(fileName, { d -> + def city = d['B'] +// if(!d['I'] && !d['J']){ +// city = d['B'] +// } + if(d['O'] && d['O']=="周边"){ + def locationMap = null + if(d['B'] && d['E']){ + locationMap = getLocation(city, d['E']) + } +// if(!locationMap && d['C']){ +// locationMap = getLocation(city, d['C']) +// } + + if(locationMap){ + def dbRets = findNearestDistrict(city, locationMap.lat, locationMap.lng, 1000, 10) + if(dbRets){ + def b = dbRets.get(0) + d['K'] = b.city + d['L'] = b.region + d['M'] = b.name + d['N'] = b.address + d['P'] = b.distance + } + } + } + },0); + } + + def run2V1(def fileName){ + ExcelUtils.multiThreadingModify(fileName,{ d-> + def city = d['J'] + if(d['O']){ + d['AH'] = d['L'] + d['AI'] = d['M'] + d['AJ'] = d['N'] + } + if(!d['O']){ + if(d['R'] && d['R'].contains("号")){ + def addr = d['R'].toString().substring(0, d['R'].indexOf("号")) + if((addr && d['E'] && d['E'].contains(addr))){ + def region = RegionData.formatRegion(city, d['P']) + if(d['U'] && d['U'].toString().contains("住宅")){ + d['AH'] = region + d['AI'] = d['Q'] + d['AJ'] = d['R'] + d['AK'] = d['U'] + d['AM'] = d['S']+","+d['T'] + d['O'] ="地图匹配" + }else if(!d['U']){ + d['AH'] = region + d['AI'] = d['Q'] + d['AJ'] = d['R'] + d['AK'] = d['U'] + d['AM'] = d['S']+","+d['T'] + d['O'] ="地图匹配" + } + } + } + } + + if(!d['O']){ + if(d['Y'] && d['AE']){ + def mapName = d['Y'].toString().replace(".", "").replace("(", "").replace(")", "").replace("。", "") + .replace("·", "").replace("-", "").replace("`", "").replace("?", "").replace("‘", "") + .replace("1期", "一期").replace("2期", "二期").replace("3期", "三期") + .replace("4期", "四期").replace("5期", "五期").replace("12区", "十二区").replace("1区", "一区").replace("2区", "二区").replace("3区", "三区") + .replace("4区", "四区").replace("5区", "五区").replace("a区", "A区") + .replace("B区", "B区").replace("c区", "C区").replace("d区", "D区") + if(mapName == d['AE']){ + def region = RegionData.formatRegion(city, d['AD']) + d['AH'] = region + d['AI'] = d['AE'] + d['AJ'] = d['AF'] + d['AK'] = d['DB'] + d['O'] ="名称相同" + }else{ + if(d['AB'].toString().toDouble()>=d['AG'].toString().toDouble()){ + def region = RegionData.formatRegion(city, d['AD']) + d['AH'] = region + d['AI'] = d['AE'] + d['AJ'] = d['AF'] + d['AK'] = "DB库近" + d['AL'] = d['AG'] + d['O'] ="距离比较" + }else{ + def region = RegionData.formatRegion(city, d['X']) + d['AH'] = region + d['AI'] = d['Y'] + d['AJ'] = d['Z'] + d['AK'] = d['AA'] + d['AL'] = d['AB'] + d['AM'] = d['AC'] + d['O'] ="距离比较" + } + } + }else if(d['Y']){ + def region = RegionData.formatRegion(city, d['X']) + d['AH'] = region + d['AI'] = d['Y'] + d['AJ'] = d['Z'] + d['AK'] = d['AA'] + d['AL'] = d['AB'] + d['AM'] = d['AC'] + d['O'] ="采用地图相近" + }else if(d['AE']){ + def region = RegionData.formatRegion(city, d['AD']) + d['AH'] = region + d['AI'] = d['AE'] + d['AJ'] = d['AF'] + d['AK'] = "DB库近" + d['AL'] = d['AG'] + d['O'] ="采用DB相近" + } + } + },0); + } + + + def findNearestByMap(lng, lat){ + def rets = GaoDeMapApiData.doSearchByGaoDe("120000|120203|120300|120301|120302|120303|120304", lng, lat, 1000, 1, 10) + return rets + + } + + public List findNearestDistrict(def city, double lat, double lng, int radiusMeters, int limit) { + DBObject districtFields = new BasicDBObject("city", 1).append("region", 1).append("name", 1).append("address", 1) + districtFields.put("estate_type2", 1) + districtFields.put("lat", 1) + districtFields.put("lng", 1) + + + DBObject sdq = new BasicDBObject(); + sdq.put("sd.point", lat + "," + lng); + sdq.put("sd.sfield", "lat_lng"); + sdq.put("sd.distance", String.valueOf((double) radiusMeters / 1000.0)); + + DBObject q = new BasicDBObject(); + q.put("cat", "district"); + q.put("city", city); + q.put("visible", 0); + q.put('$sd', sdq); + + List list = new ArrayList() + Iterator iter = SEAWEED.find(q, districtFields).limit(limit * 2).iterator() + while(iter.hasNext()) { + def o = iter.next() + o.distance = GeoHelper.GetDistance(o.lat/10000000000.0, o.lng/10000000000.0, lat, lng) + list.add(o) + } + + if (list.size() == 0) { + return Collections.EMPTY_LIST; + } + list.sort{o1,o2-> + return o1.distance <=> o2.distance + } + if (list.size() > limit) { + list = list.subList(0, limit); + } + return list; + } + + def getLocation(city, addr){ + + if(addr){ + def rMap = [:] + def reps = GaoDeApiUtils.geo(city, addr); + if(reps && reps.count.toInteger()>=1){ + def map = reps.geocodes.get(0) + def location = map.location + def level = map.level + if(location){ + def locMap = location.split(",") +// def lat = dist.lat/10000000000.0; +// def lng = dist.lng/10000000000.0; +// def distance = GeoHelper.GetDistance(locMap[1].toDouble(), locMap[0].toDouble(), lat, lng) +// rMap.distance = distance +// rMap.level = level + rMap.lat = locMap[1].toDouble() + rMap.lng = locMap[0].toDouble() +// rMap.keyword = map.formatted_address +// rMap.location = location +// rMap.region = map.district + return rMap; + } + } + } + return null; + } + + + + def run3(def fileName){ + ExcelUtils.multiThreadingModify(fileName,{ d-> + def city = d['I'] + def region = d['J'] + + def newName = null + if(d['C']){ + if(d['C'].toString().length()<2){ + newName = null + }else{ + newName = d['C'].toString().replace(".", "").replace("(", "").replace(")", "").replace("。", "") + .replace("·", "").replace("-", "").replace("`", "").replace("?", "").replace("‘", "") + .replace("1期", "一期").replace("2期", "二期").replace("3期", "三期") + .replace("4期", "四期").replace("5期", "五期").replace("12区", "十二区").replace("a区", "A区") + .replace("B区", "B区").replace("c区", "C区").replace("d区", "D区") + } + + } + def address = null + if(d['H']){ + address = d['H'] + }else{ + def arrs = d['B'].split("->") + if(arrs && arrs.size()==4){ + address = arrs[3] + }else if(arrs && arrs.size()==2){ + address = arrs[1] + } + } + if(!region ||(region && d['L'] && region.contains(d['L']))) { + if (d['B'] && !d['O'] && d['M']) { + if (d['M'] == newName) { + d['O'] = "相同" + } + if (!d['O']) { + def dbo = CommonDatautils.findDistrictByAlias(d['K'], d['L'], newName, "district") + if (dbo) { + d['O'] = "别名匹配" + } + } + + if (!d['O']) { + def py1 = com.fangjia.core.util.DataUtil.hanyuQuanpin(newName, " ") + def py2 = com.fangjia.core.util.DataUtil.hanyuQuanpin(d['M'], " ") + if (py1 == py2) { + d['O'] = "拼音相同" + } + } + + if (!d['O'] && d['M'] && newName) { + def word = null + if (newName.toString().length() >= d['M'].toString().length()) { + word = newName.toString().replace(d['M'], "") + } else if (newName.toString().length() < d['M'].toString().length()) { + word = d['M'].toString().replace(newName, "") + } + if (word) { + def words = ["重庆", "小区", "(高层)", "花园", "恒都", "(洋房)", "社区"] + for (def w : words) { + if (word == w) { + d['O'] = "去部分词后相同" + break + } + } + + if (!d['O'] && !word.endsWith("区") && !word.endsWith("期") && !word.endsWith("团") && !word.endsWith("东苑") && !word.endsWith("西苑") && !word.endsWith("南苑") && !word.endsWith("北苑")) { + if (d['N'] && d['N'].toString().endsWith("号") && address.toString().contains(d['N'])) { + + def score = CosineSimilarAlgorithm.getSimilarity(d['M'], newName); + if (score >= 0.65) { + println newName + " " + d['M'] + " " + score + d['O'] = "地址相同" + } + } + } + } + } + + if (!d['O'] && newName && !newName.contains("期") && !newName.endsWith("组团") && (!newName.endsWith("区") || newName.endsWith("小区"))) { + + def gaodeRest = FangjiaClientUtils.getMapData(city, region, newName, address, "住宅", "高德") + if (!gaodeRest) { + gaodeRest = FangjiaClientUtils.getMapData(city, region, newName, address, "住宅", "百度") + } + if (gaodeRest && gaodeRest.name) { + def mapName = gaodeRest.name.replace(".", "").replace("(", "").replace(")", "").replace("·", "").replace("`", "").replace("?", "") + if (mapName == d['M'] && gaodeRest.score >= 0.7) { + d['O'] = gaodeRest.source + "匹配" + println newName + " " + d['M'] + " " + gaodeRest.score + } + } + } + } + } + + + + if(newName && !d['O']){ + def gaodeRest = FangjiaClientUtils.getMapData(city, region, newName, address, "住宅", "高德") + def mapName = null + if(gaodeRest && gaodeRest.name){ + mapName = gaodeRest.name.replace(".", "").replace("(", "").replace(")", "").replace("·", "").replace("`", "").replace("?", "") + } + if(!gaodeRest || (gaodeRest && mapName != newName)) { + gaodeRest = FangjiaClientUtils.getMapData(city, region, newName, d['E'], "住宅", "百度") + if(gaodeRest && gaodeRest.name){ + mapName = gaodeRest.name.replace(".", "").replace("(", "").replace(")", "").replace("·", "").replace("`", "").replace("?", "") + } + } + if(gaodeRest && mapName== newName){ + d['P'] = gaodeRest.district + d['Q'] = mapName + d['R'] = gaodeRest.address + d['S'] = gaodeRest.lng + d['T'] = gaodeRest.lat + d['U'] = gaodeRest.type + d['v'] = gaodeRest.source + } + } + + },0); + } + + def mapRun(fileName){ + ExcelUtils.multiThreadingModify(fileName, { d -> + def city = d['J'] + def region = d['I'] + if(city && region && d['Q'] && d['O']!="距离比较" && d['O']!="地址匹配"){ + def gaodeRest = FangjiaClientUtils.getMapData(city, region, d['Q'], null, "住宅", "高德") + if(gaodeRest){ + d['V'] = gaodeRest.city + d['W'] = gaodeRest.district + d['X'] = gaodeRest.name + d['Y'] = gaodeRest.address + d['Z'] = gaodeRest.lng + d['AA'] = gaodeRest.lat + d['AB'] = gaodeRest.type + } + + d['AC'] = NameProcess.nameStandard(city,region, d['Q']) + } + },0) + } +} diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/chongqingblank/TestPrice.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/chongqingblank/TestPrice.groovy new file mode 100644 index 000000000..e252654d5 --- /dev/null +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/chongqingblank/TestPrice.groovy @@ -0,0 +1,89 @@ +package com.fangjia.script.gj.chongqingblank + +import com.fangjia.script.commData.CommonDatautils +import com.fangjia.script.util.ExcelUtils +import com.fangjia.script.util.MongoUtils +import com.mongodb.BasicDBObject +import com.mongodb.DBCollection +import com.mongodb.QueryOperators + +class TestPrice { + + def static final DBCollection AVERAGE_STATISTICS = MongoUtils.getCollectionByName("fangjia", "average_statistics", "192.168.0.43") + public static final def lineUrl = "mongodb://developer:G00IDCDEV12385@192.168.0.43:28017"; + public static final DBCollection DEAL_PRICE_COLL = MongoUtils.getCollectionByName2("fangjia", "deal_price", lineUrl) + + static main(args) { + def task = new TestPrice() + task.run() + } + + def run(){ + ExcelUtils.multiThreadingModify("D:\\数据\\重庆农商行\\估值1-V2.xlsx",{ d-> + if(d['K'] && d['M']){ + def dbo = findThirdPrice(d['K'], d['L'], d['M'], 202012, "lianjia_esf_price") + if(dbo){ + d['AU'] = dbo.lianjia_esf_price + } + + def district = CommonDatautils.findDistrict(d['K'], d['L'], d['M'], "district") + def startDate = 20201101 + def endDate = 20210601 + def trades = findTrades(district, startDate, endDate) + if(trades){ + d['AV'] = (int)(trades.sum()/trades.size()) + d['AW'] = trades.size() + } + } + },0); + } + + def findThirdPrice(def city, def region, def name, def s_date,def fieldName){ + def q = new BasicDBObject() + q.put("city", city) + q.put("region", region) + q.put("name", name) + q.put("s_date", new BasicDBObject(QueryOperators.GTE, s_date)) + q.put(fieldName, new BasicDBObject(QueryOperators.EXISTS, true)) + def dbcursor = AVERAGE_STATISTICS.find(q).sort(new BasicDBObject("s_date",-1)) + def obj = null; + for(def d : dbcursor){ + obj = d; + break; + } + return obj; + } + + def findTrades(dist, startDate, endDate){ + def q = new BasicDBObject() + q.put("city", dist.city) + q.put("region", dist.region) +// q.put("source", "链家在线") + + def alias = [] + if(dist.alias){ + if(!dist.alias.contains(dist.name)){ + dist.alias << dist.name + } + alias = dist.alias + }else{ + alias << dist.name + } + + if(alias){ + q.put("district_name", new BasicDBObject('$in', alias)) + } + + q.put("trade_date", new BasicDBObject('$gte', startDate).append('$lt',endDate)) + def dbcorsor = DEAL_PRICE_COLL.find(q) + def prices = [] + for(def d : dbcorsor){ + if(d.avg_price){ + prices << d.avg_price + } + } + + return prices + } + +} diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data/RegionExport.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data/RegionExport.groovy new file mode 100644 index 000000000..6e967d2bc --- /dev/null +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data/RegionExport.groovy @@ -0,0 +1,29 @@ +package com.fangjia.script.gj.data + +import com.fangjia.core.RegionData +import com.fangjia.script.commData.CommonDatautils +import com.fangjia.script.util.CommonUtils +import com.fangjia.script.util.ExcelUtils + +class RegionExport { + + static main(args) { + def task = new RegionExport() + task.run() + } + + def run(){ + ExcelUtils.multiThreadingModify("d:\\数据\\亲邻\\小区城市列表.xls",{d-> + def city = d['B'] + def newCity = RegionData.formatCity(city) + if(newCity){ + def regions = CommonUtils.getAllRegions(newCity, "region") + def regionStr = "" + regions.each{k,v-> + regionStr = regionStr + k+"\n" + } + d['C'] = regionStr + } + },0) + } +} diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/es/EsMonthHouseStat.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/es/EsMonthHouseStat.groovy new file mode 100644 index 000000000..7860afad9 --- /dev/null +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/es/EsMonthHouseStat.groovy @@ -0,0 +1,236 @@ +package com.fangjia.script.gj.es + +import com.fangjia.script.commData.CommonDatautils +import com.fangjia.script.common.DataHandleUtils +import com.fangjia.script.util.DateUtils +import com.fangjia.script.util.ElasticUtils +import com.fangjia.script.util.ExcelUtils +import com.fangjia.script.util.MongoUtils +import com.mongodb.BasicDBObject +import com.mongodb.DBCollection +import org.apache.lucene.queryparser.classic.QueryParser + +class EsMonthHouseStat { + + public static final def lineUrl = "mongodb://developer:G00IDCDEV12385@192.168.0.43:28017"; + public static final DBCollection DEAL_PRICE_COLL = MongoUtils.getCollectionByName2("fangjia", "deal_price", lineUrl) + + static main(args) { + def task = new EsMonthHouseStat() +// task.run() + task.tradeRun() +// task.houseRun() + } + + def run(){ + def citys = ["北京","上海","广州","深圳","重庆","郑州","长沙","西安","成都","东莞","佛山","杭州","合肥","南京","青岛","沈阳","苏州","天津","武汉","珠海","中山","长春","扬州","烟台","徐州","无锡","温州","太原","台州","石家庄","绍兴","厦门","泉州","宁波","南通","南宁","南昌","兰州","常州","大连","福州","贵阳","哈尔滨","海口","惠州","济南","嘉兴","金华","昆明","遵义","淄博","株洲","镇江","肇庆","漳州","湛江","岳阳","银川","宜昌","盐城","芜湖","乌鲁木齐","唐山","泰州","汕头","三亚","清远","秦皇岛","马鞍山","洛阳","柳州","连云港","廊坊","九江","吉林","淮安","湖州","呼和浩特","沧州","保定","安庆","张家口","西宁","日照","拉萨","锦州","大同","常德","北海","文昌","陵水","保亭"] + def rets = [] + def dates = ["2020-05","2020-06","2020-07","2020-08","2020-09","2020-10","2020-11","2020-12","2021-01","2021-02","2021-03","2021-04","2021-05"] + def headers = ["城市"] + headers.addAll(dates) + rets << headers + for(def city : citys){ + def datas = [] + datas << city + for(def d : dates){ + def date = DateUtils.parseFromTightYYYYMMDD(d+"-01") + def startDate = DateUtils.formatDateByPattern(date, "yyyyMMdd") + def endDate = DateUtils.addMonth(date, 1).replace("-","") + def count = findHouseCount(city, startDate.toInteger(), endDate.toInteger()) + datas << count + } + rets << datas + println datas + } + ExcelUtils.write("d:\\数据\\建行92个城市房源统计.xlsx", rets, "统计") + } + + def tradeRun(){ + def citys = ["北京","上海","广州","深圳","重庆","郑州","长沙","西安","成都","东莞","佛山","杭州","合肥","南京","青岛","沈阳","苏州","天津","武汉","珠海","中山","长春","扬州","烟台","徐州","无锡","温州","太原","台州","石家庄","绍兴","厦门","泉州","宁波","南通","南宁","南昌","兰州","常州","大连","福州","贵阳","哈尔滨","海口","惠州","济南","嘉兴","金华","昆明","遵义","淄博","株洲","镇江","肇庆","漳州","湛江","岳阳","银川","宜昌","盐城","芜湖","乌鲁木齐","唐山","泰州","汕头","三亚","清远","秦皇岛","马鞍山","洛阳","柳州","连云港","廊坊","九江","吉林","淮安","湖州","呼和浩特","沧州","保定","安庆","张家口","西宁","日照","拉萨","锦州","大同","常德","北海","文昌","陵水","保亭"] + def rets = [] + def rets2 = [] + //, + //"2021-01","2021-02","2021-03","2021-04","2021-05" + //def dates = ["2020-01","2020-02","2020-03","2020-04","2020-05","2020-06","2020-07","2020-08","2020-09","2020-10","2020-11","2020-12"] + def dates = ["2021-04"] + def headers = ["城市"] + headers.addAll(dates) + rets << headers + + + def headers2 = ["城市"] + headers2.addAll(dates) + rets2 << headers2 + for(def city : citys){ + def datas = [] + datas << city + + def datas2 = [] + datas2 << city + for(def d : dates){ + def startDate = DateUtils.parseFromTightYYYYMMDD(d+"-01") + def endDateStr = DateUtils.addMonth(startDate, 1).replace("-","") + def endDate = DateUtils.parseFromTightYYYYMMDD2(endDateStr) + //def count = findTrades(city, startDate, endDate) + def districts = CommonDatautils.findAllDistricts(city, "district", ["city", "region", "name", "alias"]); + def oldCount = 0 + def newCount = 0 + for(def dist : districts){ + def countMap = findTradesDetail(dist, startDate, endDate) + oldCount = oldCount + countMap.total_old + newCount = newCount + countMap.total_new + println dist.city + " " + dist.name +" "+countMap.total_old+" "+countMap.total_new + } + + datas << oldCount + datas2 << newCount + println city +" "+ oldCount +" "+newCount + } + rets << datas + + rets2 << datas2 + println datas + } + ExcelUtils.write("d:\\数据\\建行92个城市成交统计(去重前匹配).xlsx", rets, "统计") + + ExcelUtils.write("d:\\数据\\建行92个城市成交统计(去重后匹配).xlsx", rets2, "统计") + } + + def houseRun(){ + ExcelUtils.multiThreadingModify("d:\\数据\\小区案例数据覆盖率.xlsx",{d-> + def city = d['B'] + if(city == "湖州"){ + def count = findHouseCount(city, 20210301, 20210601) + d['E'] = count + + def districts = CommonDatautils.findAllDistricts(city, "district", ["city", "region", "name", "alias"]); + def disthouseCount = 0 + for(def dist : districts){ + def alias = [] + if(dist.alias){ + if(!dist.alias.contains(dist.name)){ + dist.alias << dist.name + } + alias = dist.alias + }else{ + alias << dist.name + } + + def distCount = findHouseCount(dist.city, dist.region, alias, 20210301, 20210601) + if(distCount){ + disthouseCount++ + } + println dist.city+" "+dist.region+" "+dist.name+" "+distCount + } + + println "disthouseCount:"+disthouseCount + println districts.size() + def precent = DataHandleUtils.getDoubleRound(2, disthouseCount*100.0/districts.size()) + d['D'] = precent+"%" + } + },0) + } + + + def findHouseCount(def city, def region, def alias, def startDate, def endDate){ + def aliasStr = null; + def total = 0 + try{ + aliasStr = escapeExprSpecialWord(alias.join("','")) + String sql = "select count(*) as c from latest_seawater where cat='sell' and city='${city}' and p_date >= ${startDate} and p_date<${endDate} and region='${region}' and district_name in ('${aliasStr}')" + + ElasticUtils.executeSqlQuery(sql,{data-> + total = data.c + }) + }catch(Exception e){ + e.printStackTrace() + } + + return total + } + + /** + * 转义字符串中的特殊字符 + * 仅过滤Lucene regexps, 其他的勿用 + * @param keyword + * @return + */ + public static String escapeExprSpecialWord(String keyword) { + return QueryParser.escape(keyword); + } + + def findHouseCount(def city, def startDate, def endDate){ + + String sql = "select count(*) as c from latest_seawater where cat='sell' and city='${city}' and p_date >= ${startDate} and p_date<${endDate} " + def total = 0 + ElasticUtils.executeSqlQuery(sql,{ data-> + total = data.c + }) + return total + } + + + + + + def findTrades(city, startDate, endDate){ + def q = new BasicDBObject() + q.put("city", city) +// q.put("region", dist.region) +// q.put("source", "链家在线") + +// def alias = [] +// if(dist.alias){ +// if(!dist.alias.contains(dist.name)){ +// dist.alias << dist.name +// } +// alias = dist.alias +// }else{ +// alias << dist.name +// } + +// if(alias){ +// q.put("district_name", new BasicDBObject('$in', alias)) +// } + + q.put("trade_date", new BasicDBObject('$gte', startDate).append('$lt',endDate)) + def count = DEAL_PRICE_COLL.count(q) + return count + } + + + + def findTradesDetail(dist, startDate, endDate){ + def q = new BasicDBObject() + q.put("city", dist.city) + q.put("region", dist.region) + + def alias = [] + if(dist.alias){ + if(!dist.alias.contains(dist.name)){ + dist.alias << dist.name + } + alias = dist.alias + }else{ + alias << dist.name + } + + if(alias){ + q.put("district_name", new BasicDBObject('$in', alias)) + } + q.put("trade_date", new BasicDBObject('$gte', startDate).append('$lt',endDate)) + def dbcursor = DEAL_PRICE_COLL.find(q) + def total = dbcursor.count() + def rets = new HashSet() + for(def d : dbcursor){ + def key=d.city+"##"+d.region+"##"+d.district_name+"##"+d.area+"##"+d.avg_price+"##"+DateUtils.formatDateByPattern(d.trade_date,"yyyy-MM-dd") + if(!rets.contains(key)){ + rets << key + } + } + def rMap = [:] + rMap.total_old = total + rMap.total_new = rets.size() + return rMap + } +} diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/es/EsSelectTest.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/es/EsSelectTest.groovy new file mode 100644 index 000000000..6f2307343 --- /dev/null +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/es/EsSelectTest.groovy @@ -0,0 +1,66 @@ +package com.fangjia.script.gj.es + +import com.fangjia.core.util.DateUtil +import com.fangjia.script.util.ElasticUtils +import org.elasticsearch.index.query.QueryBuilder +import org.elasticsearch.index.query.QueryBuilders +import org.elasticsearch.xpack.sql.jdbc.EsDataSource + +import java.sql.Connection +import java.sql.ResultSet +import java.sql.Statement + +class EsSelectTest { + EsDataSource dataSource = new EsDataSource(); + String address = "jdbc:es://http://192.168.0.86:9200/timezone=UTC&page.size=250&connect.timeout=120s&network.timeout=120s&page.timeout=120s"; + + static main(args) { + def task = new EsSelectTest() + task.run() + } + + def run(){ + + QueryBuilder queryBuilder = QueryBuilders.boolQuery().must(QueryBuilders.termQuery("request", "/ws/czbank")) + //.must(QueryBuilders.termQuery("request_body", "findSample")) + + + def houses = ElasticUtils.executeSimpleQuery("192.168.0.86", "fangjia-log*", queryBuilder); + if(houses) { + for (int k = 0; k < houses.size(); k++) { + println k + } + } + + + def endDate = new Date() + dataSource.setUrl(address) + Properties connectionProperties = new Properties() + dataSource.setProperties(connectionProperties) + StringBuffer sqlBuffer = new StringBuffer(); + String d = DateUtil.formatByPattern(new Date(), "yyyy.MM.dd") + long startD = DateUtil.getBeforeMinute(new Date(), -7).getTime(); + long endD = DateUtil.getBeforeMinute(endDate, 1).getTime(); + println(DateUtil.formatByPattern(new Date(startD), "yyyy-MM-dd HH:mm:ss")+"===="+DateUtil.formatByPattern(new Date(endD), "yyyy-MM-dd HH:mm:ss")); + + + sqlBuffer.append("select request_time FROM \"fangjia-log*\" WHERE 1=1"); + sqlBuffer.append(" and request='/ws/czbank' and request_body='findSample'") + + ResultSet rs = null + try { + Connection connection = dataSource.getConnection() + Statement statement = connection.createStatement() + rs = statement.executeQuery(sqlBuffer.toString()) + } catch (Exception e) { + e.printStackTrace() + } + StringBuffer msgBuffer = new StringBuffer(); + while (rs.next()) { + + def responsetime = rs.getLong("responsetime"); + def timestamp = DateUtil.formatByPattern(new Date(rs.getLong("time")), "yyyy-MM-dd HH:mm") + println responsetime + } + } +} diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/location/LocationTest.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/location/LocationTest.groovy new file mode 100644 index 000000000..55e0943ab --- /dev/null +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/location/LocationTest.groovy @@ -0,0 +1,171 @@ +package com.fangjia.script.gj.location + +import com.fangjia.core.RegionData +import com.fangjia.script.commData.CommonDatautils +import com.fangjia.script.gj.district.FangjiaClientUtils +import com.fangjia.script.looplineinfo.InsidePolygon +import com.fangjia.script.looplineinfo.Point +import com.fangjia.script.util.DateUtils +import com.fangjia.script.util.ExcelUtils +import com.fangjia.script.util.GeoHelper +import com.fangjia.script.util.MongoUtils +import com.mongodb.BasicDBObject +import com.mongodb.DBCollection +import java.text.SimpleDateFormat +import java.util.regex.Matcher +import java.util.regex.Pattern; + + +class LocationTest { + + static DBCollection POLYLINECOLL = MongoUtils.getCollectionByAuth20190102("fangjia_crawl", "region_polyline") + + static main(args) { + def task = new LocationTest() +// task.run() + task.run2() + } + + def run(){ + def regionPointsMap = findRegionRange() + def districts = CommonDatautils.findAllDistricts("福州", "district", ["city", "region", "name", "address", "lat", "lng"]); + def index = 0 + def rets = [] + for(def dist : districts){ + def flag = isInArea(dist.lng/10000000000.0, dist.lat/10000000000.0, regionPointsMap.get(dist.region)) + if(!flag){ + index++ + + def ret = [] + ret << dist.city + ret << dist.region + ret << dist.name + + def loactionMap = matchMap(dist) + if(loactionMap){ + ret << "地图" + ret << loactionMap.region + ret << loactionMap.mapName + def distance = GeoHelper.GetDistance(dist.lat/10000000000.0, dist.lng/10000000000.0, loactionMap.lat, loactionMap.lng) + ret << distance + if(dist.region==loactionMap.region){ + println index+" "+dist.region +" "+dist.name +" 不在区域范围内,与地图相同" + }else{ + println index+" "+dist.region +" "+dist.name +" 不在区域范围内 地图:"+ loactionMap.region+" "+distance + } + }else{ + ret << "" + println index+" "+dist.region +" "+dist.name +" 不在区域范围内,地图没找到" + } + rets << ret + + } + } + ExcelUtils.write("d:\\数据\\福州\\区域经纬度错误3.xlsx", rets, "区域不在行政范围内") + } + + def run2(){ + ExcelUtils.multiThreadingModify("d:\\数据\\福州\\福州小区.xlsx") + } + + def matchMap(def dist){ + def gaodeRest = FangjiaClientUtils.searchMapData(dist.city, dist.name,"住宅", "高德") + def locationMap = null + if(gaodeRest){ + locationMap = getMapLocation(dist, gaodeRest); + if(locationMap){ + return locationMap + } + } + + def baiduRest = null + if (!locationMap) { + baiduRest = FangjiaClientUtils.searchMapData(dist.city, dist.name,"住宅", "百度") + } + if(baiduRest){ + locationMap = getMapLocation(dist, baiduRest); + } + return locationMap + } + + def getMapLocation(def dist, def rets){ + def lat = null + def lng = null + def region = null + def mapName = null + for(def r : rets){ + if(!r.city){ + continue + } + if(!r.city.contains(dist.city)){ + continue + } + def name = r.name.replace("?","").replace("&","").replace("·","").replace(".","").replace("(","").replace(")","").replace("(","").replace(")","").replace("【","").replace("】","").replace("11期","十一期").replace("12期","十二期").replace("13期","十三期").replace("14期","十四期").replace("15期","十五期").replace("16期","十六期").replace("17期","十七期").replace("18期","十八期").replace("19期","十九期").replace("10期","十期").replace("1期","一期").replace("2期","二期").replace("3期","三期").replace("4期","四期").replace("5期","五期").replace("6期","六期").replace("7期","七期").replace("8期","八期").replace("9期","九期").replace(" ","").trim() + if(dist.name == name){ + lat = r.lat + lng = r.lng + mapName = r.name + region = r.district + }else{ + def word = null + if (name.length() >= dist.name.length()) { + word = name.replace(dist.name, "") + } else if (name.length() < dist.name.length()) { + word = dist.name.replace(name, "") + } + if (word) { + def words = ["商住楼", "小区","公寓","别墅"] + for (def w : words) { + if (word == w) { + lat = r.lat + lng = r.lng + region = r.district + mapName = r.name + break + } + } + if(!lng){ + Pattern pattern = Pattern.compile("\\d+号楼"); + Matcher matcher = pattern.matcher(word); + if(matcher.find()){ + lat = r.lat + lng = r.lng + region = r.district + mapName = r.name + break + } + } + } + } + if(lng){ + break + } + } + if(lng){ + return ["region":region, "lng":lng, "lat":lat, "mapName":mapName] + } + + + return null + } + + def isInArea(lng, lat, points){ + List pointList = InsidePolygon.createPoint2(points); + //Point point = new Point(lng, lat); + //def flag = InsidePolygon.aboutZone(point, pointList) + def flag = InsidePolygon.isPtInPoly(lng, lat, pointList) + return flag; + } + + def findRegionRange(){ + def q = new BasicDBObject() + q.put("city","福州市") + def dbcursor = POLYLINECOLL.find(q) + def regionMap = [:] + for(def d : dbcursor){ + def region = RegionData.formatRegion("福州",d.region) + regionMap.put(region, d.polyline) + } + return regionMap + } +} diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/thirdPriceCrawl/SynchDistrictPrice.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/thirdPriceCrawl/SynchDistrictPrice.groovy new file mode 100644 index 000000000..b43b98223 --- /dev/null +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/thirdPriceCrawl/SynchDistrictPrice.groovy @@ -0,0 +1,106 @@ +package com.fangjia.script.thirdPriceCrawl + +import com.fangjia.script.util.CommonUtils +import com.mongodb.BasicDBObject +import org.apache.solr.common.util.Hash + +class SynchDistrictPrice { + + def static final SHORT_DATE = Utils.getLastMonth() + def thirdPartyColl = CrawalDBUtil.getThirdPartyCol(); + def districtDetailColl = CrawalDBUtil.getDistrictDetailColl(); + def sourceMap = ["jiwu.com":"吉屋"] + //"58.com":"58同城","lianjia.com":"链家","ke.com":"贝壳找房","fang.com":"搜房","fangdd.com":"房多多","jiwu.com":"吉屋" + + static main(args) { + def task = new SynchDistrictPrice() + task.run() + } + + + def run(){ + def citys = ["福州"]//CommonUtils.findAllCity() + + sourceMap.each { k, v -> + for(def city : citys){ + def rets = findCrawlDistrictDetails(city, k, v) + synchData(city, rets, v) + println city +" "+rets.size() + } + } + + } + + def findCrawlDistrictDetails(city, root, source){ + def q = new BasicDBObject() + q.put("city", city) + q.put("root", root) + def keys = new BasicDBObject() + keys.put("city", 1) + keys.put("region", 1) + keys.put("block", 1) + keys.put("name", 1) + keys.put("address", 1) + keys.put("avg_price", 1) + keys.put("root", 1) + keys.put("url", 1) + def rets = [] + def dbcursor = districtDetailColl.find(q, keys) + def names = new HashSet() + for(def d : dbcursor){ + def s = d.city +"#"+d.region+"#"+d.name + if(!names.contains(s)){ + names << s + d.put("district_id", d.url) + d.put("source", source) + d.put("s_date", SHORT_DATE) + d.removeField("root") + d.removeField("url") + d.removeField("_id") + rets << d + } + } + return rets + } + + def synchData(def city, def rets, def source){ + def q = new BasicDBObject() + q.put("source", source) + q.put("s_date", SHORT_DATE) + q.put("city", city) + def count = thirdPartyColl.count(q) + def addIndex = 0 + if(count==0){ + try{ + thirdPartyColl.insert(rets) + }catch(Exception e){ + println e.getMessage() + } + + }else{ + for(def r : rets){ + def flag = findIsExists(r.city, r.region, r.name, SHORT_DATE, source) + if(!flag){ + addIndex++ + thirdPartyColl.save(r) + } + } + } + + println "["+source +"-"+city + "] add:"+ addIndex+" 原抓取:"+count + } + + def findIsExists(def city, def region, def name, def sDate, def source){ + def q = new BasicDBObject() + q.put("source", source) + q.put("s_date", sDate) + q.put("city", city) + q.put("region", region) + q.put("name", name) + def count = thirdPartyColl.count(q) + if(count){ + return true + } + return false + } +}