新增
This commit is contained in:
@@ -0,0 +1,133 @@
|
||||
package com.fangjia.script.fuzhou
|
||||
|
||||
import com.fangjia.script.commData.CommonDatautils
|
||||
import com.fangjia.script.common.DataHandleUtils
|
||||
import com.fangjia.script.thirdPriceCrawl.CrawalDBUtil
|
||||
import com.fangjia.script.util.ExcelUtils
|
||||
import com.fangjia.script.util.MongoUtils
|
||||
import com.mongodb.BasicDBObject
|
||||
import com.mongodb.DBCollection
|
||||
|
||||
def datas = ExcelUtils.read("d:\\数据\\福州\\福州4月2.xlsx",0,1)
|
||||
def count0 = 0
|
||||
def count1 = 0
|
||||
def count2 = 0
|
||||
def count3 = 0
|
||||
def count4 = 0
|
||||
def count5 = 0
|
||||
def count6 = 0
|
||||
def count7 = 0
|
||||
def count8 = 0
|
||||
def total=4943
|
||||
for(def d : datas){
|
||||
def list = []
|
||||
if(d['G']){
|
||||
list << d['G']
|
||||
}
|
||||
if(d['H']){
|
||||
list << d['H']
|
||||
}
|
||||
if(d['I']){
|
||||
list << d['I']
|
||||
}
|
||||
if(d['J']){
|
||||
list << d['J']
|
||||
}
|
||||
if(d['K']){
|
||||
list << d['K']
|
||||
}
|
||||
if(d['L']){
|
||||
list << d['L']
|
||||
}
|
||||
if(d['M']){
|
||||
list << d['M']
|
||||
}
|
||||
if(d['N']){
|
||||
list << d['N']
|
||||
}
|
||||
if(d['O']){
|
||||
list << d['O']
|
||||
}
|
||||
if(list.size()==0){
|
||||
count0++
|
||||
}
|
||||
if(list.size()==1){
|
||||
count1++
|
||||
}
|
||||
if(list.size()==2){
|
||||
count2++
|
||||
}
|
||||
if(list.size()==3){
|
||||
count3++
|
||||
}
|
||||
if(list.size()==4){
|
||||
count4++
|
||||
}
|
||||
if(list.size()==5){
|
||||
count5++
|
||||
}
|
||||
|
||||
if(list.size()==6){
|
||||
count6++
|
||||
}
|
||||
if(list.size()==7){
|
||||
count7++
|
||||
}
|
||||
if(list.size()==8){
|
||||
count8++
|
||||
}
|
||||
}
|
||||
|
||||
println("count0:"+ (DataHandleUtils.getDoubleRound(2, count0*100.0/total)))
|
||||
println("count1:"+ (DataHandleUtils.getDoubleRound(2, count1*100.0/total)))
|
||||
println("count2:"+ (DataHandleUtils.getDoubleRound(2, count2*100.0/total)))
|
||||
println("count3:"+ (DataHandleUtils.getDoubleRound(2, count3*100.0/total)))
|
||||
println("count4:"+ (DataHandleUtils.getDoubleRound(2, count4*100.0/total)))
|
||||
println("count5:"+ (DataHandleUtils.getDoubleRound(2, count5*100.0/total)))
|
||||
println("count6:"+ (DataHandleUtils.getDoubleRound(2, count6*100.0/total)))
|
||||
println("count7:"+ (DataHandleUtils.getDoubleRound(2, count7*100.0/total)))
|
||||
println("count8:"+ (DataHandleUtils.getDoubleRound(2, count8*100.0/total)))
|
||||
|
||||
|
||||
|
||||
////def districts = CommonDatautils.findAllDistricts("福州", "district", ["city", "region", "name"]);
|
||||
//def total = 8018
|
||||
//def names = ["诸葛找房":"zhugezhaofang_esf_price","安居客":"anjuke_esf_price","搜房":"soufang_esf_price","链家":"lianjia_esf_price","城市房产":"cityhouse_esf_price","房探":"fangtan_esf_price","房多多":"fangdoudou_esf_price","麦田":"maitian_esf_price", "58同城":"58tc_esf_price"]
|
||||
//
|
||||
//def rets = []
|
||||
//def sDate = 202011
|
||||
//rets << ["时间","诸葛找房","安居客","搜房","链家","城市房产","房探","房多多","麦田", "58同城"]
|
||||
//def ret = []
|
||||
//ret << sDate
|
||||
//names.each{k,v->
|
||||
// def crawlCount = crawlCount(k, sDate)
|
||||
// def count = count(v, sDate)
|
||||
// println k +" "+DataHandleUtils.getDoubleRound(2, count*100.0/total)+"("+crawlCount+")"
|
||||
// ret << DataHandleUtils.getDoubleRound(2,count*100.0/total)+"("+crawlCount+")"
|
||||
//}
|
||||
//rets.add(ret)
|
||||
//ExcelUtils.write("d:\\数据\\第三方均价覆盖率1.xlsx",rets,"数据")
|
||||
//
|
||||
//def crawlCount(k, sDate){
|
||||
// def thirdPartyColl = CrawalDBUtil.getThirdPartyCol();
|
||||
// def q = new BasicDBObject()
|
||||
// q.put("city","福州")
|
||||
// q.put("s_date", sDate)
|
||||
// q.put("source",k)
|
||||
// return thirdPartyColl.count(q)
|
||||
//}
|
||||
//
|
||||
//
|
||||
//def count(name, sDate){
|
||||
// DBCollection AVERAGE_STATISTICS = MongoUtils.getCollectionByName("fangjia", "average_statistics", "192.168.0.43")
|
||||
// def q = new BasicDBObject()
|
||||
// q.put("city","福州")
|
||||
// q.put("s_date", sDate)
|
||||
// q.put(name, new BasicDBObject('$gt', 0));
|
||||
// return AVERAGE_STATISTICS.count(q)
|
||||
//}
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
+77
@@ -0,0 +1,77 @@
|
||||
package com.fangjia.script.gj.ccb
|
||||
|
||||
import com.fangjia.script.commData.CommonDatautils
|
||||
import com.fangjia.script.util.CommonUtils
|
||||
import com.fangjia.script.util.DateUtils
|
||||
import com.fangjia.script.util.ExcelUtils
|
||||
import com.fangjia.script.util.MongoUtils
|
||||
import com.mongodb.BasicDBObject
|
||||
import com.mongodb.DBCollection
|
||||
|
||||
class CcbCityPriceStat {
|
||||
|
||||
public static final def lineUrl = "mongodb://developer:G00IDCDEV12385@192.168.0.43:28017";
|
||||
public static final DBCollection DEAL_PRICE_COLL = MongoUtils.getCollectionByName2("fangjia", "deal_price", lineUrl)
|
||||
|
||||
static main(args) {
|
||||
def task = new CcbCityPriceStat()
|
||||
task.run()
|
||||
}
|
||||
def run(){
|
||||
def citys = ["北京","上海","广州","深圳","重庆","郑州","长沙","西安","成都","东莞","佛山","杭州","合肥","南京","青岛","沈阳","苏州","天津","武汉","珠海","中山","长春","扬州","烟台","徐州","无锡","温州","太原","台州","石家庄","绍兴","厦门","泉州","宁波","南通","南宁","南昌","兰州","常州","大连","福州","贵阳","哈尔滨","海口","惠州","济南","嘉兴","金华","昆明","遵义","淄博","株洲","镇江","肇庆","漳州","湛江","岳阳","银川","宜昌","盐城","芜湖","乌鲁木齐","唐山","泰州","汕头","三亚","清远","秦皇岛","马鞍山","洛阳","柳州","连云港","廊坊","九江","吉林","淮安","湖州","呼和浩特","沧州","保定","安庆","张家口","西宁","日照","拉萨","锦州","大同","常德","北海","文昌","陵水","保亭"]
|
||||
def rets = []
|
||||
for(def city : citys){
|
||||
def districts = CommonDatautils.findAllDistrictByCity(city, "district", ["city","region", "name", "alias"])
|
||||
def priceMap = CommonDatautils.queryDistrictPrice(city, DateUtils.parseFromTightYYYYMMDD("2021-05-30"))
|
||||
def cityRets = []
|
||||
for(def dist : districts){
|
||||
def tradeAvgPrice = findTrades(dist, "2021-02-01", "2021-06-01")
|
||||
def weekendPrice = priceMap.get(city +"##"+dist.region+"##"+dist.name)
|
||||
if(tradeAvgPrice && weekendPrice){
|
||||
def row = [city, dist.region, dist.name, weekendPrice, tradeAvgPrice.toInteger()]
|
||||
cityRets << row
|
||||
println row.toString()
|
||||
}
|
||||
}
|
||||
print city +" "+ cityRets.size()
|
||||
rets.addAll(cityRets)
|
||||
}
|
||||
|
||||
println "总量:"+rets.size()
|
||||
ExcelUtils.write("d:\\数据\\92个城市价格数据.xlsx", rets, "所有")
|
||||
}
|
||||
|
||||
|
||||
def findTrades(dist, startDate, endDate){
|
||||
def q = new BasicDBObject()
|
||||
q.put("city", dist.city)
|
||||
q.put("region", dist.region)
|
||||
|
||||
if(dist.alias){
|
||||
if(!dist.alias.contains(dist.name)){
|
||||
dist.alias << dist.name
|
||||
}
|
||||
q.put("district_name", new BasicDBObject('$in', dist.alias))
|
||||
}else{
|
||||
q.put("district_name", dist.name)
|
||||
}
|
||||
|
||||
q.put("trade_date", new BasicDBObject('$gte', DateUtils.parseFromTightYYYYMMDD(startDate)).append('$lt',DateUtils.parseFromTightYYYYMMDD(endDate)))
|
||||
def dbcorsor = DEAL_PRICE_COLL.find(q)
|
||||
def avg_price = 0
|
||||
def total_price = 0
|
||||
def count = 0
|
||||
|
||||
for(def d : dbcorsor){
|
||||
if(d.avg_price){
|
||||
total_price = total_price + d.avg_price
|
||||
count++
|
||||
}
|
||||
}
|
||||
if(count > 3){
|
||||
avg_price = total_price/count
|
||||
}
|
||||
return avg_price
|
||||
}
|
||||
|
||||
}
|
||||
+322
@@ -0,0 +1,322 @@
|
||||
package com.fangjia.script.gj.chongqingblank
|
||||
|
||||
import com.fangjia.core.RegionData
|
||||
import com.fangjia.script.commData.CommonDatautils
|
||||
import com.fangjia.script.gj.district.FangjiaClientUtils
|
||||
import com.fangjia.script.util.CommonUtils
|
||||
import com.fangjia.script.util.CosineSimilarAlgorithm
|
||||
import com.fangjia.script.util.ExcelUtils
|
||||
import com.fangjia.script.util.MongoUtils
|
||||
import com.fangjia.script.util.OpenApiClient
|
||||
import com.mongodb.BasicDBObject
|
||||
import com.mongodb.DBCollection
|
||||
import net.sf.json.JSONObject
|
||||
import com.fangjia.script.gj.*
|
||||
|
||||
import java.util.regex.Matcher
|
||||
import java.util.regex.Pattern
|
||||
|
||||
class DataMatch {
|
||||
|
||||
static final DBCollection FANGJIA_OFFICIAL_AREA_MAP = MongoUtils.getCollectionByName("business", "fangjia_official_area_map", "mongo.db.fangjia.com")
|
||||
|
||||
|
||||
|
||||
static main(args) {
|
||||
def task = new DataMatch()
|
||||
// task.run1()
|
||||
task.run2()
|
||||
// task.run3()
|
||||
// println task.findAddress("昌州街道棠香路")
|
||||
|
||||
// task.run2MatchV2( "D:\\数据\\重庆农商行\\估值1-V2.xlsx")
|
||||
}
|
||||
|
||||
def run1(){
|
||||
def fileName = "D:\\数据\\重庆农商行\\估值2-V5.xlsx"
|
||||
//matchRegionValue(fileName);
|
||||
matchDistrict(fileName)
|
||||
// ExcelUtils.multiThreadingModify(fileName,{d->
|
||||
// if(d['O']=="新增"){
|
||||
//
|
||||
// d['L'] = d['P']
|
||||
// d['M'] = d['Q']
|
||||
// d['N'] = d['R']
|
||||
// }
|
||||
// },0)
|
||||
|
||||
}
|
||||
|
||||
def parseAddress(def fileName){
|
||||
ExcelUtils.multiThreadingModify(fileName,{d->
|
||||
|
||||
},0)
|
||||
}
|
||||
|
||||
def matchDistrict(fileName){
|
||||
ExcelUtils.multiThreadingModify(fileName,{d->
|
||||
def region = d['Q']
|
||||
def city = d['P']
|
||||
// if(!d['I'] && !d['J']){
|
||||
// city = d['B']
|
||||
// region = d['C']
|
||||
// }
|
||||
if(d['R'] && !d['G']){
|
||||
def results = null
|
||||
// if(d['E'] && isStreet(d['E'])){//加上效果并没有想象中的好
|
||||
// results = OpenApiClient.addressMatch(city, d['E'], "property", false, region, d['D'])
|
||||
// }
|
||||
if((results && results.precise<0) || !results){
|
||||
results = OpenApiClient.addressMatch(city, d['R'], "property", false, region, null)
|
||||
}
|
||||
// if((d['E'] && results && results.precise<0) || !results){
|
||||
// results = OpenApiClient.addressMatch(city, d['E'], "property", false, region, d['D'])
|
||||
// }
|
||||
if(results && results.precise>=0){
|
||||
d['K'] = results.city
|
||||
d['L'] = results.district
|
||||
d['M'] = results.name
|
||||
d['N'] = results.address
|
||||
}
|
||||
}
|
||||
},0);
|
||||
}
|
||||
|
||||
def run2(){
|
||||
def fileName = "D:\\数据\\重庆农商行\\估值2-V2.xlsx"
|
||||
// matchRegionValue(fileName);
|
||||
matchDistrict2(fileName)
|
||||
}
|
||||
|
||||
|
||||
def matchDistrict2(fileName){
|
||||
ExcelUtils.multiThreadingModify(fileName,{d->
|
||||
def region = d['I']
|
||||
def city = d['J']
|
||||
if(!d['I']){
|
||||
city = d['J']
|
||||
}
|
||||
if(city && d['E'] && !d['O'] ){
|
||||
def addressResult = OpenApiClient.addressStandard(city, d['E'])
|
||||
if(addressResult){
|
||||
if(addressResult.street && addressResult.streetNumber){
|
||||
d['H'] = addressResult.street+addressResult.streetNumber
|
||||
}
|
||||
}
|
||||
def results = null
|
||||
if(d['H']){
|
||||
results = OpenApiClient.addressMatch(city, d['H'], "property", false, region, null)
|
||||
}
|
||||
|
||||
if((d['E'] && results && results.precise<0) || !results){
|
||||
results = OpenApiClient.addressMatch(city, d['E'], "property", false, region, null)
|
||||
}
|
||||
|
||||
if(results && results.precise>=0){
|
||||
d['K'] = results.city
|
||||
d['L'] = results.district
|
||||
d['M'] = results.name
|
||||
d['N'] = results.address
|
||||
}
|
||||
}
|
||||
},0);
|
||||
}
|
||||
|
||||
def run3(){
|
||||
def fileName = "D:\\数据\\重庆农商行\\估值3.xlsx"
|
||||
// matchDistrict3(fileName)
|
||||
ExcelUtils.multiThreadingModify(fileName,{d->
|
||||
if(d['I']){
|
||||
d['I'] = d['I'].toString().replace("省","").replace("市","")
|
||||
}
|
||||
if(d['J'] && d['I']){
|
||||
d['J'] = RegionData.formatRegion(d['I'], d['J'])
|
||||
}
|
||||
},0)
|
||||
}
|
||||
|
||||
|
||||
def matchDistrict3(fileName){
|
||||
ExcelUtils.multiThreadingModify(fileName,{d->
|
||||
def address = d['B']
|
||||
if(address) {
|
||||
def arrs = address.split("->")
|
||||
def city = null
|
||||
def region = null
|
||||
def addr = null
|
||||
if (arrs && arrs.size() == 4) {
|
||||
city = arrs[0]
|
||||
region = arrs[2]
|
||||
addr = arrs[3]
|
||||
d['I'] = city
|
||||
d['J'] = region
|
||||
} else if (arrs && arrs.size() == 2) {
|
||||
def parseCity = CommonUtils.parseCityInAddress(arrs[1])
|
||||
city = parseCity
|
||||
d['I'] = parseCity
|
||||
addr = arrs[1]
|
||||
}
|
||||
if (addr && !d['O']) {
|
||||
def addressResult = OpenApiClient.addressStandard(city, addr)
|
||||
if (addressResult) {
|
||||
if (!region) {
|
||||
region = addressResult.district
|
||||
d['J'] = region
|
||||
}
|
||||
if (addressResult.street && addressResult.streetNumber) {
|
||||
d['H'] = addressResult.street + addressResult.streetNumber
|
||||
}
|
||||
}
|
||||
def results = null
|
||||
def name = null
|
||||
if (d['C']) {
|
||||
name = d['C'].toString().replace(".", "").replace("(", "").replace(")", "").replace("。", "")
|
||||
.replace("·", "").replace("-", "").replace("`", "").replace("?", "").replace("‘", "")
|
||||
.replace("1期", "一期").replace("2期", "二期").replace("3期", "三期")
|
||||
.replace("4期", "四期").replace("5期", "五期").replace("12区", "十二区").replace("1区", "一区").replace("2区", "二区").replace("3区", "三区")
|
||||
.replace("4区", "四区").replace("5区", "五区").replace("a区", "A区")
|
||||
.replace("B区", "B区").replace("c区", "C区").replace("d区", "D区")
|
||||
}
|
||||
if (name) {
|
||||
results = OpenApiClient.addressMatch(city, name, "property", false, region, null)
|
||||
}
|
||||
|
||||
if ((addr && results && results.precise < 0) || !results) {
|
||||
results = OpenApiClient.addressMatch(city, addr, "property", false, region, name)
|
||||
}
|
||||
if (results && results.precise >= 0) {
|
||||
d['K'] = results.city
|
||||
d['L'] = results.district
|
||||
d['M'] = results.name
|
||||
d['N'] = results.address
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
},0);
|
||||
}
|
||||
|
||||
|
||||
|
||||
def matchRegionValue(fileName){
|
||||
ExcelUtils.multiThreadingModify(fileName,{d->
|
||||
def code = d['C']
|
||||
if(code && code.toString().isNumber()){
|
||||
if(code.toInteger().toString()=="500226"){
|
||||
d['I'] = "荣昌"
|
||||
d['J'] = "重庆"
|
||||
}else if(code.toInteger().toString()=="500223"){
|
||||
d['I'] = "潼南"
|
||||
d['J'] = "重庆"
|
||||
}else{
|
||||
def data = findRegionCodeValue(code.toInteger().toString());
|
||||
if(data){
|
||||
d['I'] = data.fangjia_district
|
||||
d['J'] = data.fangjia_city
|
||||
}else{
|
||||
data = findCityCodeValue(code.toInteger().toString())
|
||||
if(data){
|
||||
d['J'] = data.fangjia_city
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
},0)
|
||||
}
|
||||
|
||||
def findCityCodeValue(value){
|
||||
def q = new BasicDBObject()
|
||||
q.put("city_code", value)
|
||||
def dbcursor = FANGJIA_OFFICIAL_AREA_MAP.find(q)
|
||||
def count = dbcursor.count()
|
||||
if(count >1){
|
||||
def data = null
|
||||
for(def db : dbcursor){
|
||||
data = db
|
||||
break
|
||||
}
|
||||
return data;
|
||||
}
|
||||
}
|
||||
|
||||
def findRegionCodeValue(value){
|
||||
def q = new BasicDBObject()
|
||||
q.put("district_code", value)
|
||||
def dbcursor = FANGJIA_OFFICIAL_AREA_MAP.find(q)
|
||||
def count = dbcursor.count()
|
||||
if(count == 1){
|
||||
def data = null
|
||||
for(def db : dbcursor){
|
||||
data = db
|
||||
break
|
||||
}
|
||||
return data;
|
||||
}
|
||||
}
|
||||
|
||||
def findAddress(String address) {
|
||||
//Pattern ROAD_PATTERN = Pattern.compile('.*[路|大道|巷|胡同|街]\\d+号');
|
||||
Pattern ROAD_PATTERN = Pattern.compile('.[里,路,道,街,巷,路段,胡同].?\\d+.?\\d+[号,弄]|.[里,路,道,街,巷,路段,胡同].?\\d+.?[号,弄]')
|
||||
Matcher m = ROAD_PATTERN.matcher(address)
|
||||
String newAddress
|
||||
if (m.find()) {
|
||||
newAddress = m.group()
|
||||
}
|
||||
return newAddress
|
||||
}
|
||||
|
||||
def run2MatchV2(fileName){
|
||||
ExcelUtils.multiThreadingModify(fileName, { d ->
|
||||
def region = d['I']
|
||||
def city = d['J']
|
||||
if(!d['I'] && !d['J']){
|
||||
city = d['B']
|
||||
region = d['C']
|
||||
}
|
||||
if(d['D']) {
|
||||
def name = d['D'].toString().replace(".", "").replace("(", "").replace(")", "").replace("。", "")
|
||||
.replace("·", "").replace("-", "").replace("`", "").replace("?", "").replace("‘", "")
|
||||
.replace("1期", "一期").replace("2期", "二期").replace("3期", "三期")
|
||||
.replace("4期", "四期").replace("5期", "五期").replace("12区", "十二区").replace("1区", "一区").replace("2区", "二区").replace("3区", "三区")
|
||||
.replace("4区", "四区").replace("5区", "五区").replace("a区", "A区")
|
||||
.replace("B区", "B区").replace("c区", "C区").replace("d区", "D区")
|
||||
|
||||
def result = OpenApiUtils.matchDetailDistrict(city, name, region, true)
|
||||
if (!result && d['E']) {
|
||||
if (findAddress(d['E'])) {
|
||||
result = OpenApiUtils.matchDetailDistrict(city, d['E'], region, true)
|
||||
}
|
||||
}
|
||||
if(result){
|
||||
def matchResult = result.matchResult
|
||||
if (result.credit > 0.5 && result.matchResult && !result.matchResult.match) {
|
||||
matchResult = amendMatch(result, d['E'])
|
||||
}
|
||||
if (matchResult.match) {
|
||||
d['AQ'] = result.city
|
||||
d['AR'] = result.district
|
||||
d['AS'] = result.name
|
||||
d['AT'] = result.address
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
},0)
|
||||
}
|
||||
|
||||
def amendMatch(result, address){
|
||||
def matchResult = result.matchResult
|
||||
if(result.credit > 0.65 && matchResult.distance < 2500){
|
||||
matchResult.match = true
|
||||
}else if(result.credit>0.5 && address && result.address && findAddress(result.address)){
|
||||
if(result.address.contains(address) || address.contains(result.address)){
|
||||
matchResult.match = true
|
||||
}
|
||||
}else if(address && address.contains(result.district) && address.contains(result.name)){
|
||||
matchResult.match = true
|
||||
}
|
||||
return matchResult
|
||||
}
|
||||
}
|
||||
+47
@@ -0,0 +1,47 @@
|
||||
package com.fangjia.script.gj.chongqingblank
|
||||
|
||||
import com.fangjia.script.commData.CommonDatautils
|
||||
import com.fangjia.script.gj.district.FangjiaClientUtils
|
||||
import com.fangjia.script.util.CosineSimilarAlgorithm
|
||||
import com.fangjia.script.util.ExcelUtils
|
||||
import com.fangjia.script.util.OpenApiClient
|
||||
import net.sf.json.JSONObject
|
||||
|
||||
class EvaluationPrice {
|
||||
|
||||
static main(args) {
|
||||
def task = new EvaluationPrice()
|
||||
task.run()
|
||||
}
|
||||
|
||||
def run(){
|
||||
ExcelUtils.multiThreadingModify("D:\\数据\\重庆农商行\\估值3-V3.xlsx",{ d->
|
||||
if(d['K'] && d['M']){
|
||||
def priceMap = getEvaluationPrice(d['J'], d['K'], d['L'], d['D'].toString())
|
||||
if(priceMap){
|
||||
d['E'] = priceMap.avgPrice
|
||||
d['F'] = priceMap.totalPrice
|
||||
}
|
||||
}
|
||||
},0);
|
||||
}
|
||||
|
||||
|
||||
def getEvaluationPrice(city, region, name, area){
|
||||
Map<String, String> params = new HashMap<String, String>();
|
||||
params.put("city", city);
|
||||
params.put("district", region);
|
||||
params.put("name", name);
|
||||
params.put("size", area);
|
||||
params.put("__refresh_cache","1")
|
||||
|
||||
JSONObject result = OpenApiClient.evaluate(params);
|
||||
if (result.getInt("code") == 200) {
|
||||
if (result.containsKey("result")) {
|
||||
return result.getJSONObject("result")
|
||||
}
|
||||
} else if (result.getInt("code") == 500) {
|
||||
println("小区估价失败:${params.city} ${params.name} ${params.size} ${result}")
|
||||
}
|
||||
}
|
||||
}
|
||||
+629
@@ -0,0 +1,629 @@
|
||||
package com.fangjia.script.gj.chongqingblank
|
||||
|
||||
import com.fangjia.core.RegionData
|
||||
import com.fangjia.datastore.driver.DSCollection
|
||||
import com.fangjia.datastore.driver.DataStore
|
||||
import com.fangjia.script.commData.CommonDatautils
|
||||
import com.fangjia.script.gj.district.FangjiaClientUtils
|
||||
import com.fangjia.script.gj.district.NameProcess
|
||||
import com.fangjia.script.util.CommonUtils
|
||||
import com.fangjia.script.util.CosineSimilarAlgorithm
|
||||
import com.fangjia.script.util.ExcelUtils
|
||||
import com.fangjia.script.util.GeoHelper
|
||||
import com.fangjia.script.util.OpenApiClient
|
||||
import com.fangjia.script.yinjihaun.amap.GaoDeApiUtils
|
||||
import com.fangjia.script.yinjihaun.amap.GaoDeMapApiData
|
||||
import com.mongodb.BasicDBObject
|
||||
import com.mongodb.DBObject
|
||||
|
||||
class TestMatchResult {
|
||||
|
||||
private static final DSCollection SEAWEED = new DataStore("seaweed.db.fangjia.com", 27007).getCollection("fangjia.seaweed")
|
||||
|
||||
static main(args) {
|
||||
def task = new TestMatchResult()
|
||||
// task.run1("D:\\数据\\重庆农商行\\估值1.xlsx")
|
||||
task.run2("D:\\数据\\重庆农商行\\估值2-V2.xlsx")
|
||||
// task.run3("D:\\数据\\重庆农商行\\估值3.xlsx")
|
||||
|
||||
// task.nearDistRun("D:\\数据\\重庆农商行\\估值2-V5.xlsx")
|
||||
// task.run2V1("D:\\数据\\重庆农商行\\估值2-V2.xlsx")
|
||||
|
||||
// task.similarScore("D:\\数据\\重庆农商行\\估值2-V5.xlsx")
|
||||
|
||||
// task.mapRun("D:\\数据\\重庆农商行\\估值2-V4-map.xlsx");
|
||||
|
||||
}
|
||||
|
||||
|
||||
|
||||
def run1(def fileName){
|
||||
ExcelUtils.multiThreadingModify(fileName, { d ->
|
||||
def region = d['I']
|
||||
def city = d['J']
|
||||
if (!d['I'] && !d['J']) {
|
||||
city = d['B']
|
||||
region = d['C']
|
||||
}
|
||||
def newName = null
|
||||
if(d['D']=="大川菲尔小城"){
|
||||
if (d['D']) {
|
||||
newName = d['D'].toString().replace(".", "").replace("(", "").replace(")", "").replace("。", "")
|
||||
.replace("·", "").replace("-", "").replace("`", "").replace("?", "").replace("‘", "")
|
||||
.replace("1期", "一期").replace("2期", "二期").replace("3期", "三期")
|
||||
.replace("4期", "四期").replace("5期", "五期").replace("12区", "十二区").replace("1区", "一区").replace("2区", "二区").replace("3区", "三区")
|
||||
.replace("4区", "四区").replace("5区", "五区").replace("a区", "A区")
|
||||
.replace("B区", "B区").replace("c区", "C区").replace("d区", "D区")
|
||||
}
|
||||
if (!region || (region && region == d['L'])) {
|
||||
if (d['D'] && !d['O'] && d['M']) {
|
||||
if (d['M'] == newName) {
|
||||
d['O'] = "相同"
|
||||
}
|
||||
if (!d['O']) {
|
||||
def dbo = CommonDatautils.findDistrictByAlias(d['K'], d['L'], d['D'], "district")
|
||||
if (dbo) {
|
||||
d['O'] = "别名匹配"
|
||||
}
|
||||
}
|
||||
// if (!d['O'] && d['M'] && d['E']) {
|
||||
// if (d['E'].toString().contains(d['M'])) {
|
||||
// d['O'] = "名称in地址"
|
||||
// }
|
||||
// }
|
||||
|
||||
if (!d['O']) {
|
||||
def py1 = com.fangjia.core.util.DataUtil.hanyuQuanpin(newName, " ")
|
||||
def py2 = com.fangjia.core.util.DataUtil.hanyuQuanpin(d['M'], " ")
|
||||
if (py1 == py2) {
|
||||
d['O'] = "拼音相同"
|
||||
}
|
||||
}
|
||||
|
||||
if (!d['O'] && d['D'] && d['M'] && newName) {
|
||||
def word = null
|
||||
if (newName.toString().length() >= d['M'].toString().length()) {
|
||||
word = newName.toString().replace(d['M'], "")
|
||||
} else if (newName.toString().length() < d['M'].toString().length()) {
|
||||
word = d['M'].toString().replace(newName, "")
|
||||
}
|
||||
if (word) {
|
||||
def words = ["重庆", "小区", "(高层)", "花园", "恒都", "(洋房)", "社区"]
|
||||
for (def w : words) {
|
||||
if (word == w) {
|
||||
d['O'] = "去部分词后相同"
|
||||
break
|
||||
}
|
||||
}
|
||||
|
||||
if (!d['O'] && !word.endsWith("区") && !word.endsWith("期") && !word.endsWith("团") && !word.endsWith("东苑") && !word.endsWith("西苑") && !word.endsWith("南苑") && !word.endsWith("北苑")) {
|
||||
if (d['N'] && d['N'].toString().endsWith("号") && d['E'] && d['E'].toString().contains(d['N'])) {
|
||||
|
||||
def score = CosineSimilarAlgorithm.getSimilarity(d['M'], newName);
|
||||
if (score >= 0.65) {
|
||||
println newName + " " + d['M'] + " " + score
|
||||
d['O'] = "地址相同"
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if (!d['O'] && newName && !newName.contains("期") && !newName.endsWith("组团") && (!newName.endsWith("区") || newName.endsWith("小区"))) {
|
||||
|
||||
def gaodeRest = FangjiaClientUtils.getMapData(city, region, newName, d['E'], "住宅", "高德")
|
||||
if (!gaodeRest) {
|
||||
gaodeRest = FangjiaClientUtils.getMapData(city, region, newName, d['E'], "住宅", "百度")
|
||||
}
|
||||
if (gaodeRest && gaodeRest.name) {
|
||||
def mapName = gaodeRest.name.replace(".", "").replace("(", "").replace(")", "").replace("·", "").replace("`", "").replace("?", "")
|
||||
if (mapName == d['M'] && gaodeRest.score >= 0.7) {
|
||||
d['O'] = gaodeRest.source + "匹配"
|
||||
println newName + " " + d['M'] + " " + gaodeRest.score
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
if (d['D'] && !d['O']) {
|
||||
def gaodeRest = FangjiaClientUtils.getMapData(city, region, newName, d['E'], "住宅", "高德")
|
||||
def mapName = null
|
||||
if (gaodeRest && gaodeRest.name) {
|
||||
mapName = gaodeRest.name.replace(".", "").replace("(", "").replace(")", "").replace("·", "").replace("`", "").replace("?", "")
|
||||
}
|
||||
if (!gaodeRest || (gaodeRest && mapName != newName)) {
|
||||
gaodeRest = FangjiaClientUtils.getMapData(city, region, newName, d['E'], "住宅", "百度")
|
||||
if (gaodeRest && gaodeRest.name) {
|
||||
mapName = gaodeRest.name.replace(".", "").replace("(", "").replace(")", "").replace("·", "").replace("`", "").replace("?", "")
|
||||
}
|
||||
}
|
||||
if (gaodeRest && mapName == newName) {
|
||||
d['P'] = gaodeRest.district
|
||||
d['Q'] = mapName
|
||||
d['R'] = gaodeRest.address
|
||||
d['S'] = gaodeRest.lng
|
||||
d['T'] = gaodeRest.lat
|
||||
d['U'] = gaodeRest.type
|
||||
d['v'] = gaodeRest.source
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
},0);
|
||||
}
|
||||
|
||||
def similarScore(def fileName){
|
||||
ExcelUtils.multiThreadingModify(fileName, { d ->
|
||||
if(d['D'] && d['M']){
|
||||
def score = CosineSimilarAlgorithm.getSimilarity(d['D'], d['M']);
|
||||
d['Q'] = score
|
||||
}
|
||||
},0)
|
||||
}
|
||||
|
||||
|
||||
def run2(def fileName){
|
||||
ExcelUtils.multiThreadingModify(fileName,{ d->
|
||||
def region = d['I']
|
||||
def city = d['J']
|
||||
if(!d['I']){
|
||||
city = d['J']
|
||||
}
|
||||
|
||||
// if(d['E']=="巴南区花溪镇花溪新街88号2单元3-4#") {
|
||||
// println "ss"
|
||||
|
||||
|
||||
if (!region || (region && region == d['L'])) {
|
||||
if (d['N'] && !d['O'] && d['E']) {
|
||||
if (d['N'].toString().endsWith("号") && d['E'] && d['E'].toString().contains(d['N'])) {
|
||||
d['O'] = "地址匹配"
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if (!d['O'] && d['M'] && d['E']) {
|
||||
def matchName = d['M']
|
||||
if(matchName.toString().endsWith("小区")){
|
||||
matchName = matchName.toString().replace("小区","")
|
||||
}
|
||||
if(matchName.toString().startsWith(d['L'])){
|
||||
matchName = matchName.toString().replace(d['L'],"")
|
||||
}
|
||||
def oldName= NameProcess.nameStandard(city,region, d['E'])
|
||||
if (oldName.toString().contains(matchName)) {
|
||||
d['O'] = "名称in地址"
|
||||
}
|
||||
}
|
||||
|
||||
if (d['E'] && !d['O'] ) {
|
||||
def gaodeRest = null
|
||||
|
||||
if (!gaodeRest && d['E']) {
|
||||
gaodeRest = FangjiaClientUtils.getMapData(city, region, d['E'], null, "住宅", "高德")
|
||||
}
|
||||
if(!gaodeRest && d['H']){
|
||||
gaodeRest = FangjiaClientUtils.getMapData(city, region, d['H'], null, "住宅", "高德")
|
||||
}
|
||||
if (!gaodeRest && d['E']) {
|
||||
gaodeRest = FangjiaClientUtils.getMapData(city, region, d['E'], null, "住宅", "百度")
|
||||
}
|
||||
if (!gaodeRest && d['H']) {
|
||||
gaodeRest = FangjiaClientUtils.getMapData(city, region, d['H'], null, "住宅", "百度")
|
||||
}
|
||||
|
||||
if (gaodeRest && gaodeRest.score >= 0.5) {
|
||||
d['P'] = gaodeRest.district
|
||||
d['Q'] = gaodeRest.name
|
||||
d['R'] = gaodeRest.address
|
||||
d['S'] = gaodeRest.lng
|
||||
d['T'] = gaodeRest.lat
|
||||
d['U'] = gaodeRest.type
|
||||
d['v'] = gaodeRest.source
|
||||
}
|
||||
}
|
||||
|
||||
if(!d['O']){
|
||||
def locationMap = getLocation(city, d['E'])
|
||||
if(locationMap){
|
||||
def mapRets = findNearestByMap(locationMap.lat, locationMap.lng)
|
||||
if(mapRets){
|
||||
def o = mapRets.get(0)
|
||||
d['W'] = o.cityname
|
||||
d['X'] = o.adname
|
||||
d['Y'] = o.name
|
||||
if(o.address && o.address instanceof String){
|
||||
d['Z'] = o.address
|
||||
}
|
||||
d['AA'] = o.type
|
||||
d['AB'] = o.distance
|
||||
d['AC'] = o.location
|
||||
}
|
||||
def dbRets = findNearestDistrict(city, locationMap.lat, locationMap.lng, 1000, 10)
|
||||
if(dbRets){
|
||||
def b = dbRets.get(0)
|
||||
d['AD'] = b.region
|
||||
d['AE'] = b.name
|
||||
d['AF'] = b.address
|
||||
d['AG'] = b.distance
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// }
|
||||
|
||||
},0);
|
||||
}
|
||||
|
||||
def distanceRun(def fileName){
|
||||
ExcelUtils.multiThreadingModify(fileName, { d ->
|
||||
def city = d['J']
|
||||
if(!d['O'] && d['E'] && d['E'].toString().length()>6){
|
||||
def locationMap = getLocation(city, d['E'])
|
||||
if(locationMap){
|
||||
def mapRets = findNearestByMap(locationMap.lat, locationMap.lng)
|
||||
if(mapRets){
|
||||
def o = mapRets.get(0)
|
||||
d['W'] = o.cityname
|
||||
d['X'] = o.adname
|
||||
d['Y'] = o.name
|
||||
if(o.address && o.address instanceof String){
|
||||
d['Z'] = o.address
|
||||
}
|
||||
d['AA'] = o.type
|
||||
d['AB'] = o.distance
|
||||
d['AC'] = o.location
|
||||
}
|
||||
def dbRets = findNearestDistrict(city, locationMap.lat, locationMap.lng, 1000, 10)
|
||||
if(dbRets){
|
||||
def b = dbRets.get(0)
|
||||
d['AD'] = b.region
|
||||
d['AE'] = b.name
|
||||
d['AF'] = b.address
|
||||
d['AG'] = b.distance
|
||||
}
|
||||
}
|
||||
}
|
||||
},0);
|
||||
}
|
||||
|
||||
def nearDistRun(def fileName){
|
||||
ExcelUtils.multiThreadingModify(fileName, { d ->
|
||||
def city = d['B']
|
||||
// if(!d['I'] && !d['J']){
|
||||
// city = d['B']
|
||||
// }
|
||||
if(d['O'] && d['O']=="周边"){
|
||||
def locationMap = null
|
||||
if(d['B'] && d['E']){
|
||||
locationMap = getLocation(city, d['E'])
|
||||
}
|
||||
// if(!locationMap && d['C']){
|
||||
// locationMap = getLocation(city, d['C'])
|
||||
// }
|
||||
|
||||
if(locationMap){
|
||||
def dbRets = findNearestDistrict(city, locationMap.lat, locationMap.lng, 1000, 10)
|
||||
if(dbRets){
|
||||
def b = dbRets.get(0)
|
||||
d['K'] = b.city
|
||||
d['L'] = b.region
|
||||
d['M'] = b.name
|
||||
d['N'] = b.address
|
||||
d['P'] = b.distance
|
||||
}
|
||||
}
|
||||
}
|
||||
},0);
|
||||
}
|
||||
|
||||
def run2V1(def fileName){
|
||||
ExcelUtils.multiThreadingModify(fileName,{ d->
|
||||
def city = d['J']
|
||||
if(d['O']){
|
||||
d['AH'] = d['L']
|
||||
d['AI'] = d['M']
|
||||
d['AJ'] = d['N']
|
||||
}
|
||||
if(!d['O']){
|
||||
if(d['R'] && d['R'].contains("号")){
|
||||
def addr = d['R'].toString().substring(0, d['R'].indexOf("号"))
|
||||
if((addr && d['E'] && d['E'].contains(addr))){
|
||||
def region = RegionData.formatRegion(city, d['P'])
|
||||
if(d['U'] && d['U'].toString().contains("住宅")){
|
||||
d['AH'] = region
|
||||
d['AI'] = d['Q']
|
||||
d['AJ'] = d['R']
|
||||
d['AK'] = d['U']
|
||||
d['AM'] = d['S']+","+d['T']
|
||||
d['O'] ="地图匹配"
|
||||
}else if(!d['U']){
|
||||
d['AH'] = region
|
||||
d['AI'] = d['Q']
|
||||
d['AJ'] = d['R']
|
||||
d['AK'] = d['U']
|
||||
d['AM'] = d['S']+","+d['T']
|
||||
d['O'] ="地图匹配"
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if(!d['O']){
|
||||
if(d['Y'] && d['AE']){
|
||||
def mapName = d['Y'].toString().replace(".", "").replace("(", "").replace(")", "").replace("。", "")
|
||||
.replace("·", "").replace("-", "").replace("`", "").replace("?", "").replace("‘", "")
|
||||
.replace("1期", "一期").replace("2期", "二期").replace("3期", "三期")
|
||||
.replace("4期", "四期").replace("5期", "五期").replace("12区", "十二区").replace("1区", "一区").replace("2区", "二区").replace("3区", "三区")
|
||||
.replace("4区", "四区").replace("5区", "五区").replace("a区", "A区")
|
||||
.replace("B区", "B区").replace("c区", "C区").replace("d区", "D区")
|
||||
if(mapName == d['AE']){
|
||||
def region = RegionData.formatRegion(city, d['AD'])
|
||||
d['AH'] = region
|
||||
d['AI'] = d['AE']
|
||||
d['AJ'] = d['AF']
|
||||
d['AK'] = d['DB']
|
||||
d['O'] ="名称相同"
|
||||
}else{
|
||||
if(d['AB'].toString().toDouble()>=d['AG'].toString().toDouble()){
|
||||
def region = RegionData.formatRegion(city, d['AD'])
|
||||
d['AH'] = region
|
||||
d['AI'] = d['AE']
|
||||
d['AJ'] = d['AF']
|
||||
d['AK'] = "DB库近"
|
||||
d['AL'] = d['AG']
|
||||
d['O'] ="距离比较"
|
||||
}else{
|
||||
def region = RegionData.formatRegion(city, d['X'])
|
||||
d['AH'] = region
|
||||
d['AI'] = d['Y']
|
||||
d['AJ'] = d['Z']
|
||||
d['AK'] = d['AA']
|
||||
d['AL'] = d['AB']
|
||||
d['AM'] = d['AC']
|
||||
d['O'] ="距离比较"
|
||||
}
|
||||
}
|
||||
}else if(d['Y']){
|
||||
def region = RegionData.formatRegion(city, d['X'])
|
||||
d['AH'] = region
|
||||
d['AI'] = d['Y']
|
||||
d['AJ'] = d['Z']
|
||||
d['AK'] = d['AA']
|
||||
d['AL'] = d['AB']
|
||||
d['AM'] = d['AC']
|
||||
d['O'] ="采用地图相近"
|
||||
}else if(d['AE']){
|
||||
def region = RegionData.formatRegion(city, d['AD'])
|
||||
d['AH'] = region
|
||||
d['AI'] = d['AE']
|
||||
d['AJ'] = d['AF']
|
||||
d['AK'] = "DB库近"
|
||||
d['AL'] = d['AG']
|
||||
d['O'] ="采用DB相近"
|
||||
}
|
||||
}
|
||||
},0);
|
||||
}
|
||||
|
||||
|
||||
def findNearestByMap(lng, lat){
|
||||
def rets = GaoDeMapApiData.doSearchByGaoDe("120000|120203|120300|120301|120302|120303|120304", lng, lat, 1000, 1, 10)
|
||||
return rets
|
||||
|
||||
}
|
||||
|
||||
public List<DBObject> findNearestDistrict(def city, double lat, double lng, int radiusMeters, int limit) {
|
||||
DBObject districtFields = new BasicDBObject("city", 1).append("region", 1).append("name", 1).append("address", 1)
|
||||
districtFields.put("estate_type2", 1)
|
||||
districtFields.put("lat", 1)
|
||||
districtFields.put("lng", 1)
|
||||
|
||||
|
||||
DBObject sdq = new BasicDBObject();
|
||||
sdq.put("sd.point", lat + "," + lng);
|
||||
sdq.put("sd.sfield", "lat_lng");
|
||||
sdq.put("sd.distance", String.valueOf((double) radiusMeters / 1000.0));
|
||||
|
||||
DBObject q = new BasicDBObject();
|
||||
q.put("cat", "district");
|
||||
q.put("city", city);
|
||||
q.put("visible", 0);
|
||||
q.put('$sd', sdq);
|
||||
|
||||
List<DBObject> list = new ArrayList<DBObject>()
|
||||
Iterator<DBObject> iter = SEAWEED.find(q, districtFields).limit(limit * 2).iterator()
|
||||
while(iter.hasNext()) {
|
||||
def o = iter.next()
|
||||
o.distance = GeoHelper.GetDistance(o.lat/10000000000.0, o.lng/10000000000.0, lat, lng)
|
||||
list.add(o)
|
||||
}
|
||||
|
||||
if (list.size() == 0) {
|
||||
return Collections.EMPTY_LIST;
|
||||
}
|
||||
list.sort{o1,o2->
|
||||
return o1.distance <=> o2.distance
|
||||
}
|
||||
if (list.size() > limit) {
|
||||
list = list.subList(0, limit);
|
||||
}
|
||||
return list;
|
||||
}
|
||||
|
||||
def getLocation(city, addr){
|
||||
|
||||
if(addr){
|
||||
def rMap = [:]
|
||||
def reps = GaoDeApiUtils.geo(city, addr);
|
||||
if(reps && reps.count.toInteger()>=1){
|
||||
def map = reps.geocodes.get(0)
|
||||
def location = map.location
|
||||
def level = map.level
|
||||
if(location){
|
||||
def locMap = location.split(",")
|
||||
// def lat = dist.lat/10000000000.0;
|
||||
// def lng = dist.lng/10000000000.0;
|
||||
// def distance = GeoHelper.GetDistance(locMap[1].toDouble(), locMap[0].toDouble(), lat, lng)
|
||||
// rMap.distance = distance
|
||||
// rMap.level = level
|
||||
rMap.lat = locMap[1].toDouble()
|
||||
rMap.lng = locMap[0].toDouble()
|
||||
// rMap.keyword = map.formatted_address
|
||||
// rMap.location = location
|
||||
// rMap.region = map.district
|
||||
return rMap;
|
||||
}
|
||||
}
|
||||
}
|
||||
return null;
|
||||
}
|
||||
|
||||
|
||||
|
||||
def run3(def fileName){
|
||||
ExcelUtils.multiThreadingModify(fileName,{ d->
|
||||
def city = d['I']
|
||||
def region = d['J']
|
||||
|
||||
def newName = null
|
||||
if(d['C']){
|
||||
if(d['C'].toString().length()<2){
|
||||
newName = null
|
||||
}else{
|
||||
newName = d['C'].toString().replace(".", "").replace("(", "").replace(")", "").replace("。", "")
|
||||
.replace("·", "").replace("-", "").replace("`", "").replace("?", "").replace("‘", "")
|
||||
.replace("1期", "一期").replace("2期", "二期").replace("3期", "三期")
|
||||
.replace("4期", "四期").replace("5期", "五期").replace("12区", "十二区").replace("a区", "A区")
|
||||
.replace("B区", "B区").replace("c区", "C区").replace("d区", "D区")
|
||||
}
|
||||
|
||||
}
|
||||
def address = null
|
||||
if(d['H']){
|
||||
address = d['H']
|
||||
}else{
|
||||
def arrs = d['B'].split("->")
|
||||
if(arrs && arrs.size()==4){
|
||||
address = arrs[3]
|
||||
}else if(arrs && arrs.size()==2){
|
||||
address = arrs[1]
|
||||
}
|
||||
}
|
||||
if(!region ||(region && d['L'] && region.contains(d['L']))) {
|
||||
if (d['B'] && !d['O'] && d['M']) {
|
||||
if (d['M'] == newName) {
|
||||
d['O'] = "相同"
|
||||
}
|
||||
if (!d['O']) {
|
||||
def dbo = CommonDatautils.findDistrictByAlias(d['K'], d['L'], newName, "district")
|
||||
if (dbo) {
|
||||
d['O'] = "别名匹配"
|
||||
}
|
||||
}
|
||||
|
||||
if (!d['O']) {
|
||||
def py1 = com.fangjia.core.util.DataUtil.hanyuQuanpin(newName, " ")
|
||||
def py2 = com.fangjia.core.util.DataUtil.hanyuQuanpin(d['M'], " ")
|
||||
if (py1 == py2) {
|
||||
d['O'] = "拼音相同"
|
||||
}
|
||||
}
|
||||
|
||||
if (!d['O'] && d['M'] && newName) {
|
||||
def word = null
|
||||
if (newName.toString().length() >= d['M'].toString().length()) {
|
||||
word = newName.toString().replace(d['M'], "")
|
||||
} else if (newName.toString().length() < d['M'].toString().length()) {
|
||||
word = d['M'].toString().replace(newName, "")
|
||||
}
|
||||
if (word) {
|
||||
def words = ["重庆", "小区", "(高层)", "花园", "恒都", "(洋房)", "社区"]
|
||||
for (def w : words) {
|
||||
if (word == w) {
|
||||
d['O'] = "去部分词后相同"
|
||||
break
|
||||
}
|
||||
}
|
||||
|
||||
if (!d['O'] && !word.endsWith("区") && !word.endsWith("期") && !word.endsWith("团") && !word.endsWith("东苑") && !word.endsWith("西苑") && !word.endsWith("南苑") && !word.endsWith("北苑")) {
|
||||
if (d['N'] && d['N'].toString().endsWith("号") && address.toString().contains(d['N'])) {
|
||||
|
||||
def score = CosineSimilarAlgorithm.getSimilarity(d['M'], newName);
|
||||
if (score >= 0.65) {
|
||||
println newName + " " + d['M'] + " " + score
|
||||
d['O'] = "地址相同"
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if (!d['O'] && newName && !newName.contains("期") && !newName.endsWith("组团") && (!newName.endsWith("区") || newName.endsWith("小区"))) {
|
||||
|
||||
def gaodeRest = FangjiaClientUtils.getMapData(city, region, newName, address, "住宅", "高德")
|
||||
if (!gaodeRest) {
|
||||
gaodeRest = FangjiaClientUtils.getMapData(city, region, newName, address, "住宅", "百度")
|
||||
}
|
||||
if (gaodeRest && gaodeRest.name) {
|
||||
def mapName = gaodeRest.name.replace(".", "").replace("(", "").replace(")", "").replace("·", "").replace("`", "").replace("?", "")
|
||||
if (mapName == d['M'] && gaodeRest.score >= 0.7) {
|
||||
d['O'] = gaodeRest.source + "匹配"
|
||||
println newName + " " + d['M'] + " " + gaodeRest.score
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
|
||||
if(newName && !d['O']){
|
||||
def gaodeRest = FangjiaClientUtils.getMapData(city, region, newName, address, "住宅", "高德")
|
||||
def mapName = null
|
||||
if(gaodeRest && gaodeRest.name){
|
||||
mapName = gaodeRest.name.replace(".", "").replace("(", "").replace(")", "").replace("·", "").replace("`", "").replace("?", "")
|
||||
}
|
||||
if(!gaodeRest || (gaodeRest && mapName != newName)) {
|
||||
gaodeRest = FangjiaClientUtils.getMapData(city, region, newName, d['E'], "住宅", "百度")
|
||||
if(gaodeRest && gaodeRest.name){
|
||||
mapName = gaodeRest.name.replace(".", "").replace("(", "").replace(")", "").replace("·", "").replace("`", "").replace("?", "")
|
||||
}
|
||||
}
|
||||
if(gaodeRest && mapName== newName){
|
||||
d['P'] = gaodeRest.district
|
||||
d['Q'] = mapName
|
||||
d['R'] = gaodeRest.address
|
||||
d['S'] = gaodeRest.lng
|
||||
d['T'] = gaodeRest.lat
|
||||
d['U'] = gaodeRest.type
|
||||
d['v'] = gaodeRest.source
|
||||
}
|
||||
}
|
||||
|
||||
},0);
|
||||
}
|
||||
|
||||
def mapRun(fileName){
|
||||
ExcelUtils.multiThreadingModify(fileName, { d ->
|
||||
def city = d['J']
|
||||
def region = d['I']
|
||||
if(city && region && d['Q'] && d['O']!="距离比较" && d['O']!="地址匹配"){
|
||||
def gaodeRest = FangjiaClientUtils.getMapData(city, region, d['Q'], null, "住宅", "高德")
|
||||
if(gaodeRest){
|
||||
d['V'] = gaodeRest.city
|
||||
d['W'] = gaodeRest.district
|
||||
d['X'] = gaodeRest.name
|
||||
d['Y'] = gaodeRest.address
|
||||
d['Z'] = gaodeRest.lng
|
||||
d['AA'] = gaodeRest.lat
|
||||
d['AB'] = gaodeRest.type
|
||||
}
|
||||
|
||||
d['AC'] = NameProcess.nameStandard(city,region, d['Q'])
|
||||
}
|
||||
},0)
|
||||
}
|
||||
}
|
||||
+89
@@ -0,0 +1,89 @@
|
||||
package com.fangjia.script.gj.chongqingblank
|
||||
|
||||
import com.fangjia.script.commData.CommonDatautils
|
||||
import com.fangjia.script.util.ExcelUtils
|
||||
import com.fangjia.script.util.MongoUtils
|
||||
import com.mongodb.BasicDBObject
|
||||
import com.mongodb.DBCollection
|
||||
import com.mongodb.QueryOperators
|
||||
|
||||
class TestPrice {
|
||||
|
||||
def static final DBCollection AVERAGE_STATISTICS = MongoUtils.getCollectionByName("fangjia", "average_statistics", "192.168.0.43")
|
||||
public static final def lineUrl = "mongodb://developer:G00IDCDEV12385@192.168.0.43:28017";
|
||||
public static final DBCollection DEAL_PRICE_COLL = MongoUtils.getCollectionByName2("fangjia", "deal_price", lineUrl)
|
||||
|
||||
static main(args) {
|
||||
def task = new TestPrice()
|
||||
task.run()
|
||||
}
|
||||
|
||||
def run(){
|
||||
ExcelUtils.multiThreadingModify("D:\\数据\\重庆农商行\\估值1-V2.xlsx",{ d->
|
||||
if(d['K'] && d['M']){
|
||||
def dbo = findThirdPrice(d['K'], d['L'], d['M'], 202012, "lianjia_esf_price")
|
||||
if(dbo){
|
||||
d['AU'] = dbo.lianjia_esf_price
|
||||
}
|
||||
|
||||
def district = CommonDatautils.findDistrict(d['K'], d['L'], d['M'], "district")
|
||||
def startDate = 20201101
|
||||
def endDate = 20210601
|
||||
def trades = findTrades(district, startDate, endDate)
|
||||
if(trades){
|
||||
d['AV'] = (int)(trades.sum()/trades.size())
|
||||
d['AW'] = trades.size()
|
||||
}
|
||||
}
|
||||
},0);
|
||||
}
|
||||
|
||||
def findThirdPrice(def city, def region, def name, def s_date,def fieldName){
|
||||
def q = new BasicDBObject()
|
||||
q.put("city", city)
|
||||
q.put("region", region)
|
||||
q.put("name", name)
|
||||
q.put("s_date", new BasicDBObject(QueryOperators.GTE, s_date))
|
||||
q.put(fieldName, new BasicDBObject(QueryOperators.EXISTS, true))
|
||||
def dbcursor = AVERAGE_STATISTICS.find(q).sort(new BasicDBObject("s_date",-1))
|
||||
def obj = null;
|
||||
for(def d : dbcursor){
|
||||
obj = d;
|
||||
break;
|
||||
}
|
||||
return obj;
|
||||
}
|
||||
|
||||
def findTrades(dist, startDate, endDate){
|
||||
def q = new BasicDBObject()
|
||||
q.put("city", dist.city)
|
||||
q.put("region", dist.region)
|
||||
// q.put("source", "链家在线")
|
||||
|
||||
def alias = []
|
||||
if(dist.alias){
|
||||
if(!dist.alias.contains(dist.name)){
|
||||
dist.alias << dist.name
|
||||
}
|
||||
alias = dist.alias
|
||||
}else{
|
||||
alias << dist.name
|
||||
}
|
||||
|
||||
if(alias){
|
||||
q.put("district_name", new BasicDBObject('$in', alias))
|
||||
}
|
||||
|
||||
q.put("trade_date", new BasicDBObject('$gte', startDate).append('$lt',endDate))
|
||||
def dbcorsor = DEAL_PRICE_COLL.find(q)
|
||||
def prices = []
|
||||
for(def d : dbcorsor){
|
||||
if(d.avg_price){
|
||||
prices << d.avg_price
|
||||
}
|
||||
}
|
||||
|
||||
return prices
|
||||
}
|
||||
|
||||
}
|
||||
+29
@@ -0,0 +1,29 @@
|
||||
package com.fangjia.script.gj.data
|
||||
|
||||
import com.fangjia.core.RegionData
|
||||
import com.fangjia.script.commData.CommonDatautils
|
||||
import com.fangjia.script.util.CommonUtils
|
||||
import com.fangjia.script.util.ExcelUtils
|
||||
|
||||
class RegionExport {
|
||||
|
||||
static main(args) {
|
||||
def task = new RegionExport()
|
||||
task.run()
|
||||
}
|
||||
|
||||
def run(){
|
||||
ExcelUtils.multiThreadingModify("d:\\Êý¾Ý\\Ç×ÁÚ\\Ð¡Çø³ÇÊÐÁбí.xls",{d->
|
||||
def city = d['B']
|
||||
def newCity = RegionData.formatCity(city)
|
||||
if(newCity){
|
||||
def regions = CommonUtils.getAllRegions(newCity, "region")
|
||||
def regionStr = ""
|
||||
regions.each{k,v->
|
||||
regionStr = regionStr + k+"\n"
|
||||
}
|
||||
d['C'] = regionStr
|
||||
}
|
||||
},0)
|
||||
}
|
||||
}
|
||||
+236
@@ -0,0 +1,236 @@
|
||||
package com.fangjia.script.gj.es
|
||||
|
||||
import com.fangjia.script.commData.CommonDatautils
|
||||
import com.fangjia.script.common.DataHandleUtils
|
||||
import com.fangjia.script.util.DateUtils
|
||||
import com.fangjia.script.util.ElasticUtils
|
||||
import com.fangjia.script.util.ExcelUtils
|
||||
import com.fangjia.script.util.MongoUtils
|
||||
import com.mongodb.BasicDBObject
|
||||
import com.mongodb.DBCollection
|
||||
import org.apache.lucene.queryparser.classic.QueryParser
|
||||
|
||||
class EsMonthHouseStat {
|
||||
|
||||
public static final def lineUrl = "mongodb://developer:G00IDCDEV12385@192.168.0.43:28017";
|
||||
public static final DBCollection DEAL_PRICE_COLL = MongoUtils.getCollectionByName2("fangjia", "deal_price", lineUrl)
|
||||
|
||||
static main(args) {
|
||||
def task = new EsMonthHouseStat()
|
||||
// task.run()
|
||||
task.tradeRun()
|
||||
// task.houseRun()
|
||||
}
|
||||
|
||||
def run(){
|
||||
def citys = ["北京","上海","广州","深圳","重庆","郑州","长沙","西安","成都","东莞","佛山","杭州","合肥","南京","青岛","沈阳","苏州","天津","武汉","珠海","中山","长春","扬州","烟台","徐州","无锡","温州","太原","台州","石家庄","绍兴","厦门","泉州","宁波","南通","南宁","南昌","兰州","常州","大连","福州","贵阳","哈尔滨","海口","惠州","济南","嘉兴","金华","昆明","遵义","淄博","株洲","镇江","肇庆","漳州","湛江","岳阳","银川","宜昌","盐城","芜湖","乌鲁木齐","唐山","泰州","汕头","三亚","清远","秦皇岛","马鞍山","洛阳","柳州","连云港","廊坊","九江","吉林","淮安","湖州","呼和浩特","沧州","保定","安庆","张家口","西宁","日照","拉萨","锦州","大同","常德","北海","文昌","陵水","保亭"]
|
||||
def rets = []
|
||||
def dates = ["2020-05","2020-06","2020-07","2020-08","2020-09","2020-10","2020-11","2020-12","2021-01","2021-02","2021-03","2021-04","2021-05"]
|
||||
def headers = ["城市"]
|
||||
headers.addAll(dates)
|
||||
rets << headers
|
||||
for(def city : citys){
|
||||
def datas = []
|
||||
datas << city
|
||||
for(def d : dates){
|
||||
def date = DateUtils.parseFromTightYYYYMMDD(d+"-01")
|
||||
def startDate = DateUtils.formatDateByPattern(date, "yyyyMMdd")
|
||||
def endDate = DateUtils.addMonth(date, 1).replace("-","")
|
||||
def count = findHouseCount(city, startDate.toInteger(), endDate.toInteger())
|
||||
datas << count
|
||||
}
|
||||
rets << datas
|
||||
println datas
|
||||
}
|
||||
ExcelUtils.write("d:\\数据\\建行92个城市房源统计.xlsx", rets, "统计")
|
||||
}
|
||||
|
||||
def tradeRun(){
|
||||
def citys = ["北京","上海","广州","深圳","重庆","郑州","长沙","西安","成都","东莞","佛山","杭州","合肥","南京","青岛","沈阳","苏州","天津","武汉","珠海","中山","长春","扬州","烟台","徐州","无锡","温州","太原","台州","石家庄","绍兴","厦门","泉州","宁波","南通","南宁","南昌","兰州","常州","大连","福州","贵阳","哈尔滨","海口","惠州","济南","嘉兴","金华","昆明","遵义","淄博","株洲","镇江","肇庆","漳州","湛江","岳阳","银川","宜昌","盐城","芜湖","乌鲁木齐","唐山","泰州","汕头","三亚","清远","秦皇岛","马鞍山","洛阳","柳州","连云港","廊坊","九江","吉林","淮安","湖州","呼和浩特","沧州","保定","安庆","张家口","西宁","日照","拉萨","锦州","大同","常德","北海","文昌","陵水","保亭"]
|
||||
def rets = []
|
||||
def rets2 = []
|
||||
//,
|
||||
//"2021-01","2021-02","2021-03","2021-04","2021-05"
|
||||
//def dates = ["2020-01","2020-02","2020-03","2020-04","2020-05","2020-06","2020-07","2020-08","2020-09","2020-10","2020-11","2020-12"]
|
||||
def dates = ["2021-04"]
|
||||
def headers = ["城市"]
|
||||
headers.addAll(dates)
|
||||
rets << headers
|
||||
|
||||
|
||||
def headers2 = ["城市"]
|
||||
headers2.addAll(dates)
|
||||
rets2 << headers2
|
||||
for(def city : citys){
|
||||
def datas = []
|
||||
datas << city
|
||||
|
||||
def datas2 = []
|
||||
datas2 << city
|
||||
for(def d : dates){
|
||||
def startDate = DateUtils.parseFromTightYYYYMMDD(d+"-01")
|
||||
def endDateStr = DateUtils.addMonth(startDate, 1).replace("-","")
|
||||
def endDate = DateUtils.parseFromTightYYYYMMDD2(endDateStr)
|
||||
//def count = findTrades(city, startDate, endDate)
|
||||
def districts = CommonDatautils.findAllDistricts(city, "district", ["city", "region", "name", "alias"]);
|
||||
def oldCount = 0
|
||||
def newCount = 0
|
||||
for(def dist : districts){
|
||||
def countMap = findTradesDetail(dist, startDate, endDate)
|
||||
oldCount = oldCount + countMap.total_old
|
||||
newCount = newCount + countMap.total_new
|
||||
println dist.city + " " + dist.name +" "+countMap.total_old+" "+countMap.total_new
|
||||
}
|
||||
|
||||
datas << oldCount
|
||||
datas2 << newCount
|
||||
println city +" "+ oldCount +" "+newCount
|
||||
}
|
||||
rets << datas
|
||||
|
||||
rets2 << datas2
|
||||
println datas
|
||||
}
|
||||
ExcelUtils.write("d:\\数据\\建行92个城市成交统计(去重前匹配).xlsx", rets, "统计")
|
||||
|
||||
ExcelUtils.write("d:\\数据\\建行92个城市成交统计(去重后匹配).xlsx", rets2, "统计")
|
||||
}
|
||||
|
||||
def houseRun(){
|
||||
ExcelUtils.multiThreadingModify("d:\\数据\\小区案例数据覆盖率.xlsx",{d->
|
||||
def city = d['B']
|
||||
if(city == "湖州"){
|
||||
def count = findHouseCount(city, 20210301, 20210601)
|
||||
d['E'] = count
|
||||
|
||||
def districts = CommonDatautils.findAllDistricts(city, "district", ["city", "region", "name", "alias"]);
|
||||
def disthouseCount = 0
|
||||
for(def dist : districts){
|
||||
def alias = []
|
||||
if(dist.alias){
|
||||
if(!dist.alias.contains(dist.name)){
|
||||
dist.alias << dist.name
|
||||
}
|
||||
alias = dist.alias
|
||||
}else{
|
||||
alias << dist.name
|
||||
}
|
||||
|
||||
def distCount = findHouseCount(dist.city, dist.region, alias, 20210301, 20210601)
|
||||
if(distCount){
|
||||
disthouseCount++
|
||||
}
|
||||
println dist.city+" "+dist.region+" "+dist.name+" "+distCount
|
||||
}
|
||||
|
||||
println "disthouseCount:"+disthouseCount
|
||||
println districts.size()
|
||||
def precent = DataHandleUtils.getDoubleRound(2, disthouseCount*100.0/districts.size())
|
||||
d['D'] = precent+"%"
|
||||
}
|
||||
},0)
|
||||
}
|
||||
|
||||
|
||||
def findHouseCount(def city, def region, def alias, def startDate, def endDate){
|
||||
def aliasStr = null;
|
||||
def total = 0
|
||||
try{
|
||||
aliasStr = escapeExprSpecialWord(alias.join("','"))
|
||||
String sql = "select count(*) as c from latest_seawater where cat='sell' and city='${city}' and p_date >= ${startDate} and p_date<${endDate} and region='${region}' and district_name in ('${aliasStr}')"
|
||||
|
||||
ElasticUtils.executeSqlQuery(sql,{data->
|
||||
total = data.c
|
||||
})
|
||||
}catch(Exception e){
|
||||
e.printStackTrace()
|
||||
}
|
||||
|
||||
return total
|
||||
}
|
||||
|
||||
/**
|
||||
* 转义字符串中的特殊字符
|
||||
* 仅过滤Lucene regexps, 其他的勿用
|
||||
* @param keyword
|
||||
* @return
|
||||
*/
|
||||
public static String escapeExprSpecialWord(String keyword) {
|
||||
return QueryParser.escape(keyword);
|
||||
}
|
||||
|
||||
def findHouseCount(def city, def startDate, def endDate){
|
||||
|
||||
String sql = "select count(*) as c from latest_seawater where cat='sell' and city='${city}' and p_date >= ${startDate} and p_date<${endDate} "
|
||||
def total = 0
|
||||
ElasticUtils.executeSqlQuery(sql,{ data->
|
||||
total = data.c
|
||||
})
|
||||
return total
|
||||
}
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
def findTrades(city, startDate, endDate){
|
||||
def q = new BasicDBObject()
|
||||
q.put("city", city)
|
||||
// q.put("region", dist.region)
|
||||
// q.put("source", "链家在线")
|
||||
|
||||
// def alias = []
|
||||
// if(dist.alias){
|
||||
// if(!dist.alias.contains(dist.name)){
|
||||
// dist.alias << dist.name
|
||||
// }
|
||||
// alias = dist.alias
|
||||
// }else{
|
||||
// alias << dist.name
|
||||
// }
|
||||
|
||||
// if(alias){
|
||||
// q.put("district_name", new BasicDBObject('$in', alias))
|
||||
// }
|
||||
|
||||
q.put("trade_date", new BasicDBObject('$gte', startDate).append('$lt',endDate))
|
||||
def count = DEAL_PRICE_COLL.count(q)
|
||||
return count
|
||||
}
|
||||
|
||||
|
||||
|
||||
def findTradesDetail(dist, startDate, endDate){
|
||||
def q = new BasicDBObject()
|
||||
q.put("city", dist.city)
|
||||
q.put("region", dist.region)
|
||||
|
||||
def alias = []
|
||||
if(dist.alias){
|
||||
if(!dist.alias.contains(dist.name)){
|
||||
dist.alias << dist.name
|
||||
}
|
||||
alias = dist.alias
|
||||
}else{
|
||||
alias << dist.name
|
||||
}
|
||||
|
||||
if(alias){
|
||||
q.put("district_name", new BasicDBObject('$in', alias))
|
||||
}
|
||||
q.put("trade_date", new BasicDBObject('$gte', startDate).append('$lt',endDate))
|
||||
def dbcursor = DEAL_PRICE_COLL.find(q)
|
||||
def total = dbcursor.count()
|
||||
def rets = new HashSet()
|
||||
for(def d : dbcursor){
|
||||
def key=d.city+"##"+d.region+"##"+d.district_name+"##"+d.area+"##"+d.avg_price+"##"+DateUtils.formatDateByPattern(d.trade_date,"yyyy-MM-dd")
|
||||
if(!rets.contains(key)){
|
||||
rets << key
|
||||
}
|
||||
}
|
||||
def rMap = [:]
|
||||
rMap.total_old = total
|
||||
rMap.total_new = rets.size()
|
||||
return rMap
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,66 @@
|
||||
package com.fangjia.script.gj.es
|
||||
|
||||
import com.fangjia.core.util.DateUtil
|
||||
import com.fangjia.script.util.ElasticUtils
|
||||
import org.elasticsearch.index.query.QueryBuilder
|
||||
import org.elasticsearch.index.query.QueryBuilders
|
||||
import org.elasticsearch.xpack.sql.jdbc.EsDataSource
|
||||
|
||||
import java.sql.Connection
|
||||
import java.sql.ResultSet
|
||||
import java.sql.Statement
|
||||
|
||||
class EsSelectTest {
|
||||
EsDataSource dataSource = new EsDataSource();
|
||||
String address = "jdbc:es://http://192.168.0.86:9200/timezone=UTC&page.size=250&connect.timeout=120s&network.timeout=120s&page.timeout=120s";
|
||||
|
||||
static main(args) {
|
||||
def task = new EsSelectTest()
|
||||
task.run()
|
||||
}
|
||||
|
||||
def run(){
|
||||
|
||||
QueryBuilder queryBuilder = QueryBuilders.boolQuery().must(QueryBuilders.termQuery("request", "/ws/czbank"))
|
||||
//.must(QueryBuilders.termQuery("request_body", "findSample"))
|
||||
|
||||
|
||||
def houses = ElasticUtils.executeSimpleQuery("192.168.0.86", "fangjia-log*", queryBuilder);
|
||||
if(houses) {
|
||||
for (int k = 0; k < houses.size(); k++) {
|
||||
println k
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
def endDate = new Date()
|
||||
dataSource.setUrl(address)
|
||||
Properties connectionProperties = new Properties()
|
||||
dataSource.setProperties(connectionProperties)
|
||||
StringBuffer sqlBuffer = new StringBuffer();
|
||||
String d = DateUtil.formatByPattern(new Date(), "yyyy.MM.dd")
|
||||
long startD = DateUtil.getBeforeMinute(new Date(), -7).getTime();
|
||||
long endD = DateUtil.getBeforeMinute(endDate, 1).getTime();
|
||||
println(DateUtil.formatByPattern(new Date(startD), "yyyy-MM-dd HH:mm:ss")+"===="+DateUtil.formatByPattern(new Date(endD), "yyyy-MM-dd HH:mm:ss"));
|
||||
|
||||
|
||||
sqlBuffer.append("select request_time FROM \"fangjia-log*\" WHERE 1=1");
|
||||
sqlBuffer.append(" and request='/ws/czbank' and request_body='findSample'")
|
||||
|
||||
ResultSet rs = null
|
||||
try {
|
||||
Connection connection = dataSource.getConnection()
|
||||
Statement statement = connection.createStatement()
|
||||
rs = statement.executeQuery(sqlBuffer.toString())
|
||||
} catch (Exception e) {
|
||||
e.printStackTrace()
|
||||
}
|
||||
StringBuffer msgBuffer = new StringBuffer();
|
||||
while (rs.next()) {
|
||||
|
||||
def responsetime = rs.getLong("responsetime");
|
||||
def timestamp = DateUtil.formatByPattern(new Date(rs.getLong("time")), "yyyy-MM-dd HH:mm")
|
||||
println responsetime
|
||||
}
|
||||
}
|
||||
}
|
||||
+171
@@ -0,0 +1,171 @@
|
||||
package com.fangjia.script.gj.location
|
||||
|
||||
import com.fangjia.core.RegionData
|
||||
import com.fangjia.script.commData.CommonDatautils
|
||||
import com.fangjia.script.gj.district.FangjiaClientUtils
|
||||
import com.fangjia.script.looplineinfo.InsidePolygon
|
||||
import com.fangjia.script.looplineinfo.Point
|
||||
import com.fangjia.script.util.DateUtils
|
||||
import com.fangjia.script.util.ExcelUtils
|
||||
import com.fangjia.script.util.GeoHelper
|
||||
import com.fangjia.script.util.MongoUtils
|
||||
import com.mongodb.BasicDBObject
|
||||
import com.mongodb.DBCollection
|
||||
import java.text.SimpleDateFormat
|
||||
import java.util.regex.Matcher
|
||||
import java.util.regex.Pattern;
|
||||
|
||||
|
||||
class LocationTest {
|
||||
|
||||
static DBCollection POLYLINECOLL = MongoUtils.getCollectionByAuth20190102("fangjia_crawl", "region_polyline")
|
||||
|
||||
static main(args) {
|
||||
def task = new LocationTest()
|
||||
// task.run()
|
||||
task.run2()
|
||||
}
|
||||
|
||||
def run(){
|
||||
def regionPointsMap = findRegionRange()
|
||||
def districts = CommonDatautils.findAllDistricts("福州", "district", ["city", "region", "name", "address", "lat", "lng"]);
|
||||
def index = 0
|
||||
def rets = []
|
||||
for(def dist : districts){
|
||||
def flag = isInArea(dist.lng/10000000000.0, dist.lat/10000000000.0, regionPointsMap.get(dist.region))
|
||||
if(!flag){
|
||||
index++
|
||||
|
||||
def ret = []
|
||||
ret << dist.city
|
||||
ret << dist.region
|
||||
ret << dist.name
|
||||
|
||||
def loactionMap = matchMap(dist)
|
||||
if(loactionMap){
|
||||
ret << "地图"
|
||||
ret << loactionMap.region
|
||||
ret << loactionMap.mapName
|
||||
def distance = GeoHelper.GetDistance(dist.lat/10000000000.0, dist.lng/10000000000.0, loactionMap.lat, loactionMap.lng)
|
||||
ret << distance
|
||||
if(dist.region==loactionMap.region){
|
||||
println index+" "+dist.region +" "+dist.name +" 不在区域范围内,与地图相同"
|
||||
}else{
|
||||
println index+" "+dist.region +" "+dist.name +" 不在区域范围内 地图:"+ loactionMap.region+" "+distance
|
||||
}
|
||||
}else{
|
||||
ret << ""
|
||||
println index+" "+dist.region +" "+dist.name +" 不在区域范围内,地图没找到"
|
||||
}
|
||||
rets << ret
|
||||
|
||||
}
|
||||
}
|
||||
ExcelUtils.write("d:\\数据\\福州\\区域经纬度错误3.xlsx", rets, "区域不在行政范围内")
|
||||
}
|
||||
|
||||
def run2(){
|
||||
ExcelUtils.multiThreadingModify("d:\\数据\\福州\\福州小区.xlsx")
|
||||
}
|
||||
|
||||
def matchMap(def dist){
|
||||
def gaodeRest = FangjiaClientUtils.searchMapData(dist.city, dist.name,"住宅", "高德")
|
||||
def locationMap = null
|
||||
if(gaodeRest){
|
||||
locationMap = getMapLocation(dist, gaodeRest);
|
||||
if(locationMap){
|
||||
return locationMap
|
||||
}
|
||||
}
|
||||
|
||||
def baiduRest = null
|
||||
if (!locationMap) {
|
||||
baiduRest = FangjiaClientUtils.searchMapData(dist.city, dist.name,"住宅", "百度")
|
||||
}
|
||||
if(baiduRest){
|
||||
locationMap = getMapLocation(dist, baiduRest);
|
||||
}
|
||||
return locationMap
|
||||
}
|
||||
|
||||
def getMapLocation(def dist, def rets){
|
||||
def lat = null
|
||||
def lng = null
|
||||
def region = null
|
||||
def mapName = null
|
||||
for(def r : rets){
|
||||
if(!r.city){
|
||||
continue
|
||||
}
|
||||
if(!r.city.contains(dist.city)){
|
||||
continue
|
||||
}
|
||||
def name = r.name.replace("?","").replace("&","").replace("·","").replace(".","").replace("(","").replace(")","").replace("(","").replace(")","").replace("【","").replace("】","").replace("11期","十一期").replace("12期","十二期").replace("13期","十三期").replace("14期","十四期").replace("15期","十五期").replace("16期","十六期").replace("17期","十七期").replace("18期","十八期").replace("19期","十九期").replace("10期","十期").replace("1期","一期").replace("2期","二期").replace("3期","三期").replace("4期","四期").replace("5期","五期").replace("6期","六期").replace("7期","七期").replace("8期","八期").replace("9期","九期").replace(" ","").trim()
|
||||
if(dist.name == name){
|
||||
lat = r.lat
|
||||
lng = r.lng
|
||||
mapName = r.name
|
||||
region = r.district
|
||||
}else{
|
||||
def word = null
|
||||
if (name.length() >= dist.name.length()) {
|
||||
word = name.replace(dist.name, "")
|
||||
} else if (name.length() < dist.name.length()) {
|
||||
word = dist.name.replace(name, "")
|
||||
}
|
||||
if (word) {
|
||||
def words = ["商住楼", "小区","公寓","别墅"]
|
||||
for (def w : words) {
|
||||
if (word == w) {
|
||||
lat = r.lat
|
||||
lng = r.lng
|
||||
region = r.district
|
||||
mapName = r.name
|
||||
break
|
||||
}
|
||||
}
|
||||
if(!lng){
|
||||
Pattern pattern = Pattern.compile("\\d+号楼");
|
||||
Matcher matcher = pattern.matcher(word);
|
||||
if(matcher.find()){
|
||||
lat = r.lat
|
||||
lng = r.lng
|
||||
region = r.district
|
||||
mapName = r.name
|
||||
break
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
if(lng){
|
||||
break
|
||||
}
|
||||
}
|
||||
if(lng){
|
||||
return ["region":region, "lng":lng, "lat":lat, "mapName":mapName]
|
||||
}
|
||||
|
||||
|
||||
return null
|
||||
}
|
||||
|
||||
def isInArea(lng, lat, points){
|
||||
List<Point> pointList = InsidePolygon.createPoint2(points);
|
||||
//Point point = new Point(lng, lat);
|
||||
//def flag = InsidePolygon.aboutZone(point, pointList)
|
||||
def flag = InsidePolygon.isPtInPoly(lng, lat, pointList)
|
||||
return flag;
|
||||
}
|
||||
|
||||
def findRegionRange(){
|
||||
def q = new BasicDBObject()
|
||||
q.put("city","福州市")
|
||||
def dbcursor = POLYLINECOLL.find(q)
|
||||
def regionMap = [:]
|
||||
for(def d : dbcursor){
|
||||
def region = RegionData.formatRegion("福州",d.region)
|
||||
regionMap.put(region, d.polyline)
|
||||
}
|
||||
return regionMap
|
||||
}
|
||||
}
|
||||
+106
@@ -0,0 +1,106 @@
|
||||
package com.fangjia.script.thirdPriceCrawl
|
||||
|
||||
import com.fangjia.script.util.CommonUtils
|
||||
import com.mongodb.BasicDBObject
|
||||
import org.apache.solr.common.util.Hash
|
||||
|
||||
class SynchDistrictPrice {
|
||||
|
||||
def static final SHORT_DATE = Utils.getLastMonth()
|
||||
def thirdPartyColl = CrawalDBUtil.getThirdPartyCol();
|
||||
def districtDetailColl = CrawalDBUtil.getDistrictDetailColl();
|
||||
def sourceMap = ["jiwu.com":"吉屋"]
|
||||
//"58.com":"58同城","lianjia.com":"链家","ke.com":"贝壳找房","fang.com":"搜房","fangdd.com":"房多多","jiwu.com":"吉屋"
|
||||
|
||||
static main(args) {
|
||||
def task = new SynchDistrictPrice()
|
||||
task.run()
|
||||
}
|
||||
|
||||
|
||||
def run(){
|
||||
def citys = ["福州"]//CommonUtils.findAllCity()
|
||||
|
||||
sourceMap.each { k, v ->
|
||||
for(def city : citys){
|
||||
def rets = findCrawlDistrictDetails(city, k, v)
|
||||
synchData(city, rets, v)
|
||||
println city +" "+rets.size()
|
||||
}
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
def findCrawlDistrictDetails(city, root, source){
|
||||
def q = new BasicDBObject()
|
||||
q.put("city", city)
|
||||
q.put("root", root)
|
||||
def keys = new BasicDBObject()
|
||||
keys.put("city", 1)
|
||||
keys.put("region", 1)
|
||||
keys.put("block", 1)
|
||||
keys.put("name", 1)
|
||||
keys.put("address", 1)
|
||||
keys.put("avg_price", 1)
|
||||
keys.put("root", 1)
|
||||
keys.put("url", 1)
|
||||
def rets = []
|
||||
def dbcursor = districtDetailColl.find(q, keys)
|
||||
def names = new HashSet()
|
||||
for(def d : dbcursor){
|
||||
def s = d.city +"#"+d.region+"#"+d.name
|
||||
if(!names.contains(s)){
|
||||
names << s
|
||||
d.put("district_id", d.url)
|
||||
d.put("source", source)
|
||||
d.put("s_date", SHORT_DATE)
|
||||
d.removeField("root")
|
||||
d.removeField("url")
|
||||
d.removeField("_id")
|
||||
rets << d
|
||||
}
|
||||
}
|
||||
return rets
|
||||
}
|
||||
|
||||
def synchData(def city, def rets, def source){
|
||||
def q = new BasicDBObject()
|
||||
q.put("source", source)
|
||||
q.put("s_date", SHORT_DATE)
|
||||
q.put("city", city)
|
||||
def count = thirdPartyColl.count(q)
|
||||
def addIndex = 0
|
||||
if(count==0){
|
||||
try{
|
||||
thirdPartyColl.insert(rets)
|
||||
}catch(Exception e){
|
||||
println e.getMessage()
|
||||
}
|
||||
|
||||
}else{
|
||||
for(def r : rets){
|
||||
def flag = findIsExists(r.city, r.region, r.name, SHORT_DATE, source)
|
||||
if(!flag){
|
||||
addIndex++
|
||||
thirdPartyColl.save(r)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
println "["+source +"-"+city + "] add:"+ addIndex+" 原抓取:"+count
|
||||
}
|
||||
|
||||
def findIsExists(def city, def region, def name, def sDate, def source){
|
||||
def q = new BasicDBObject()
|
||||
q.put("source", source)
|
||||
q.put("s_date", sDate)
|
||||
q.put("city", city)
|
||||
q.put("region", region)
|
||||
q.put("name", name)
|
||||
def count = thirdPartyColl.count(q)
|
||||
if(count){
|
||||
return true
|
||||
}
|
||||
return false
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user