This commit is contained in:
2021-06-04 11:28:23 +08:00
parent 0e354a525d
commit 851221de80
11 changed files with 1905 additions and 0 deletions
@@ -0,0 +1,133 @@
package com.fangjia.script.fuzhou
import com.fangjia.script.commData.CommonDatautils
import com.fangjia.script.common.DataHandleUtils
import com.fangjia.script.thirdPriceCrawl.CrawalDBUtil
import com.fangjia.script.util.ExcelUtils
import com.fangjia.script.util.MongoUtils
import com.mongodb.BasicDBObject
import com.mongodb.DBCollection
def datas = ExcelUtils.read("d:\\数据\\福州\\福州4月2.xlsx",0,1)
def count0 = 0
def count1 = 0
def count2 = 0
def count3 = 0
def count4 = 0
def count5 = 0
def count6 = 0
def count7 = 0
def count8 = 0
def total=4943
for(def d : datas){
def list = []
if(d['G']){
list << d['G']
}
if(d['H']){
list << d['H']
}
if(d['I']){
list << d['I']
}
if(d['J']){
list << d['J']
}
if(d['K']){
list << d['K']
}
if(d['L']){
list << d['L']
}
if(d['M']){
list << d['M']
}
if(d['N']){
list << d['N']
}
if(d['O']){
list << d['O']
}
if(list.size()==0){
count0++
}
if(list.size()==1){
count1++
}
if(list.size()==2){
count2++
}
if(list.size()==3){
count3++
}
if(list.size()==4){
count4++
}
if(list.size()==5){
count5++
}
if(list.size()==6){
count6++
}
if(list.size()==7){
count7++
}
if(list.size()==8){
count8++
}
}
println("count0:"+ (DataHandleUtils.getDoubleRound(2, count0*100.0/total)))
println("count1:"+ (DataHandleUtils.getDoubleRound(2, count1*100.0/total)))
println("count2:"+ (DataHandleUtils.getDoubleRound(2, count2*100.0/total)))
println("count3:"+ (DataHandleUtils.getDoubleRound(2, count3*100.0/total)))
println("count4:"+ (DataHandleUtils.getDoubleRound(2, count4*100.0/total)))
println("count5:"+ (DataHandleUtils.getDoubleRound(2, count5*100.0/total)))
println("count6:"+ (DataHandleUtils.getDoubleRound(2, count6*100.0/total)))
println("count7:"+ (DataHandleUtils.getDoubleRound(2, count7*100.0/total)))
println("count8:"+ (DataHandleUtils.getDoubleRound(2, count8*100.0/total)))
////def districts = CommonDatautils.findAllDistricts("福州", "district", ["city", "region", "name"]);
//def total = 8018
//def names = ["诸葛找房":"zhugezhaofang_esf_price","安居客":"anjuke_esf_price","搜房":"soufang_esf_price","链家":"lianjia_esf_price","城市房产":"cityhouse_esf_price","房探":"fangtan_esf_price","房多多":"fangdoudou_esf_price","麦田":"maitian_esf_price", "58同城":"58tc_esf_price"]
//
//def rets = []
//def sDate = 202011
//rets << ["时间","诸葛找房","安居客","搜房","链家","城市房产","房探","房多多","麦田", "58同城"]
//def ret = []
//ret << sDate
//names.each{k,v->
// def crawlCount = crawlCount(k, sDate)
// def count = count(v, sDate)
// println k +" "+DataHandleUtils.getDoubleRound(2, count*100.0/total)+"("+crawlCount+")"
// ret << DataHandleUtils.getDoubleRound(2,count*100.0/total)+"("+crawlCount+")"
//}
//rets.add(ret)
//ExcelUtils.write("d:\\数据\\第三方均价覆盖率1.xlsx",rets,"数据")
//
//def crawlCount(k, sDate){
// def thirdPartyColl = CrawalDBUtil.getThirdPartyCol();
// def q = new BasicDBObject()
// q.put("city","福州")
// q.put("s_date", sDate)
// q.put("source",k)
// return thirdPartyColl.count(q)
//}
//
//
//def count(name, sDate){
// DBCollection AVERAGE_STATISTICS = MongoUtils.getCollectionByName("fangjia", "average_statistics", "192.168.0.43")
// def q = new BasicDBObject()
// q.put("city","福州")
// q.put("s_date", sDate)
// q.put(name, new BasicDBObject('$gt', 0));
// return AVERAGE_STATISTICS.count(q)
//}
@@ -0,0 +1,77 @@
package com.fangjia.script.gj.ccb
import com.fangjia.script.commData.CommonDatautils
import com.fangjia.script.util.CommonUtils
import com.fangjia.script.util.DateUtils
import com.fangjia.script.util.ExcelUtils
import com.fangjia.script.util.MongoUtils
import com.mongodb.BasicDBObject
import com.mongodb.DBCollection
class CcbCityPriceStat {
public static final def lineUrl = "mongodb://developer:G00IDCDEV12385@192.168.0.43:28017";
public static final DBCollection DEAL_PRICE_COLL = MongoUtils.getCollectionByName2("fangjia", "deal_price", lineUrl)
static main(args) {
def task = new CcbCityPriceStat()
task.run()
}
def run(){
def citys = ["北京","上海","广州","深圳","重庆","郑州","长沙","西安","成都","东莞","佛山","杭州","合肥","南京","青岛","沈阳","苏州","天津","武汉","珠海","中山","长春","扬州","烟台","徐州","无锡","温州","太原","台州","石家庄","绍兴","厦门","泉州","宁波","南通","南宁","南昌","兰州","常州","大连","福州","贵阳","哈尔滨","海口","惠州","济南","嘉兴","金华","昆明","遵义","淄博","株洲","镇江","肇庆","漳州","湛江","岳阳","银川","宜昌","盐城","芜湖","乌鲁木齐","唐山","泰州","汕头","三亚","清远","秦皇岛","马鞍山","洛阳","柳州","连云港","廊坊","九江","吉林","淮安","湖州","呼和浩特","沧州","保定","安庆","张家口","西宁","日照","拉萨","锦州","大同","常德","北海","文昌","陵水","保亭"]
def rets = []
for(def city : citys){
def districts = CommonDatautils.findAllDistrictByCity(city, "district", ["city","region", "name", "alias"])
def priceMap = CommonDatautils.queryDistrictPrice(city, DateUtils.parseFromTightYYYYMMDD("2021-05-30"))
def cityRets = []
for(def dist : districts){
def tradeAvgPrice = findTrades(dist, "2021-02-01", "2021-06-01")
def weekendPrice = priceMap.get(city +"##"+dist.region+"##"+dist.name)
if(tradeAvgPrice && weekendPrice){
def row = [city, dist.region, dist.name, weekendPrice, tradeAvgPrice.toInteger()]
cityRets << row
println row.toString()
}
}
print city +" "+ cityRets.size()
rets.addAll(cityRets)
}
println "总量:"+rets.size()
ExcelUtils.write("d:\\数据\\92个城市价格数据.xlsx", rets, "所有")
}
def findTrades(dist, startDate, endDate){
def q = new BasicDBObject()
q.put("city", dist.city)
q.put("region", dist.region)
if(dist.alias){
if(!dist.alias.contains(dist.name)){
dist.alias << dist.name
}
q.put("district_name", new BasicDBObject('$in', dist.alias))
}else{
q.put("district_name", dist.name)
}
q.put("trade_date", new BasicDBObject('$gte', DateUtils.parseFromTightYYYYMMDD(startDate)).append('$lt',DateUtils.parseFromTightYYYYMMDD(endDate)))
def dbcorsor = DEAL_PRICE_COLL.find(q)
def avg_price = 0
def total_price = 0
def count = 0
for(def d : dbcorsor){
if(d.avg_price){
total_price = total_price + d.avg_price
count++
}
}
if(count > 3){
avg_price = total_price/count
}
return avg_price
}
}
@@ -0,0 +1,322 @@
package com.fangjia.script.gj.chongqingblank
import com.fangjia.core.RegionData
import com.fangjia.script.commData.CommonDatautils
import com.fangjia.script.gj.district.FangjiaClientUtils
import com.fangjia.script.util.CommonUtils
import com.fangjia.script.util.CosineSimilarAlgorithm
import com.fangjia.script.util.ExcelUtils
import com.fangjia.script.util.MongoUtils
import com.fangjia.script.util.OpenApiClient
import com.mongodb.BasicDBObject
import com.mongodb.DBCollection
import net.sf.json.JSONObject
import com.fangjia.script.gj.*
import java.util.regex.Matcher
import java.util.regex.Pattern
class DataMatch {
static final DBCollection FANGJIA_OFFICIAL_AREA_MAP = MongoUtils.getCollectionByName("business", "fangjia_official_area_map", "mongo.db.fangjia.com")
static main(args) {
def task = new DataMatch()
// task.run1()
task.run2()
// task.run3()
// println task.findAddress("昌州街道棠香路")
// task.run2MatchV2( "D:\\数据\\重庆农商行\\估值1-V2.xlsx")
}
def run1(){
def fileName = "D:\\数据\\重庆农商行\\估值2-V5.xlsx"
//matchRegionValue(fileName);
matchDistrict(fileName)
// ExcelUtils.multiThreadingModify(fileName,{d->
// if(d['O']=="新增"){
//
// d['L'] = d['P']
// d['M'] = d['Q']
// d['N'] = d['R']
// }
// },0)
}
def parseAddress(def fileName){
ExcelUtils.multiThreadingModify(fileName,{d->
},0)
}
def matchDistrict(fileName){
ExcelUtils.multiThreadingModify(fileName,{d->
def region = d['Q']
def city = d['P']
// if(!d['I'] && !d['J']){
// city = d['B']
// region = d['C']
// }
if(d['R'] && !d['G']){
def results = null
// if(d['E'] && isStreet(d['E'])){//加上效果并没有想象中的好
// results = OpenApiClient.addressMatch(city, d['E'], "property", false, region, d['D'])
// }
if((results && results.precise<0) || !results){
results = OpenApiClient.addressMatch(city, d['R'], "property", false, region, null)
}
// if((d['E'] && results && results.precise<0) || !results){
// results = OpenApiClient.addressMatch(city, d['E'], "property", false, region, d['D'])
// }
if(results && results.precise>=0){
d['K'] = results.city
d['L'] = results.district
d['M'] = results.name
d['N'] = results.address
}
}
},0);
}
def run2(){
def fileName = "D:\\数据\\重庆农商行\\估值2-V2.xlsx"
// matchRegionValue(fileName);
matchDistrict2(fileName)
}
def matchDistrict2(fileName){
ExcelUtils.multiThreadingModify(fileName,{d->
def region = d['I']
def city = d['J']
if(!d['I']){
city = d['J']
}
if(city && d['E'] && !d['O'] ){
def addressResult = OpenApiClient.addressStandard(city, d['E'])
if(addressResult){
if(addressResult.street && addressResult.streetNumber){
d['H'] = addressResult.street+addressResult.streetNumber
}
}
def results = null
if(d['H']){
results = OpenApiClient.addressMatch(city, d['H'], "property", false, region, null)
}
if((d['E'] && results && results.precise<0) || !results){
results = OpenApiClient.addressMatch(city, d['E'], "property", false, region, null)
}
if(results && results.precise>=0){
d['K'] = results.city
d['L'] = results.district
d['M'] = results.name
d['N'] = results.address
}
}
},0);
}
def run3(){
def fileName = "D:\\数据\\重庆农商行\\估值3.xlsx"
// matchDistrict3(fileName)
ExcelUtils.multiThreadingModify(fileName,{d->
if(d['I']){
d['I'] = d['I'].toString().replace("省","").replace("市","")
}
if(d['J'] && d['I']){
d['J'] = RegionData.formatRegion(d['I'], d['J'])
}
},0)
}
def matchDistrict3(fileName){
ExcelUtils.multiThreadingModify(fileName,{d->
def address = d['B']
if(address) {
def arrs = address.split("->")
def city = null
def region = null
def addr = null
if (arrs && arrs.size() == 4) {
city = arrs[0]
region = arrs[2]
addr = arrs[3]
d['I'] = city
d['J'] = region
} else if (arrs && arrs.size() == 2) {
def parseCity = CommonUtils.parseCityInAddress(arrs[1])
city = parseCity
d['I'] = parseCity
addr = arrs[1]
}
if (addr && !d['O']) {
def addressResult = OpenApiClient.addressStandard(city, addr)
if (addressResult) {
if (!region) {
region = addressResult.district
d['J'] = region
}
if (addressResult.street && addressResult.streetNumber) {
d['H'] = addressResult.street + addressResult.streetNumber
}
}
def results = null
def name = null
if (d['C']) {
name = d['C'].toString().replace(".", "").replace("(", "").replace(")", "").replace("。", "")
.replace("·", "").replace("-", "").replace("`", "").replace("?", "").replace("‘", "")
.replace("1期", "一期").replace("2期", "二期").replace("3期", "三期")
.replace("4期", "四期").replace("5期", "五期").replace("12区", "十二区").replace("1区", "一区").replace("2区", "二区").replace("3区", "三区")
.replace("4区", "四区").replace("5区", "五区").replace("a区", "A区")
.replace("B区", "B区").replace("c区", "C区").replace("d区", "D区")
}
if (name) {
results = OpenApiClient.addressMatch(city, name, "property", false, region, null)
}
if ((addr && results && results.precise < 0) || !results) {
results = OpenApiClient.addressMatch(city, addr, "property", false, region, name)
}
if (results && results.precise >= 0) {
d['K'] = results.city
d['L'] = results.district
d['M'] = results.name
d['N'] = results.address
}
}
}
},0);
}
def matchRegionValue(fileName){
ExcelUtils.multiThreadingModify(fileName,{d->
def code = d['C']
if(code && code.toString().isNumber()){
if(code.toInteger().toString()=="500226"){
d['I'] = "荣昌"
d['J'] = "重庆"
}else if(code.toInteger().toString()=="500223"){
d['I'] = "潼南"
d['J'] = "重庆"
}else{
def data = findRegionCodeValue(code.toInteger().toString());
if(data){
d['I'] = data.fangjia_district
d['J'] = data.fangjia_city
}else{
data = findCityCodeValue(code.toInteger().toString())
if(data){
d['J'] = data.fangjia_city
}
}
}
}
},0)
}
def findCityCodeValue(value){
def q = new BasicDBObject()
q.put("city_code", value)
def dbcursor = FANGJIA_OFFICIAL_AREA_MAP.find(q)
def count = dbcursor.count()
if(count >1){
def data = null
for(def db : dbcursor){
data = db
break
}
return data;
}
}
def findRegionCodeValue(value){
def q = new BasicDBObject()
q.put("district_code", value)
def dbcursor = FANGJIA_OFFICIAL_AREA_MAP.find(q)
def count = dbcursor.count()
if(count == 1){
def data = null
for(def db : dbcursor){
data = db
break
}
return data;
}
}
def findAddress(String address) {
//Pattern ROAD_PATTERN = Pattern.compile('.*[路|大道|巷|胡同|街]\\d+号');
Pattern ROAD_PATTERN = Pattern.compile('.[里,路,道,街,巷,路段,胡同].?\\d+.?\\d+[号,弄]|.[里,路,道,街,巷,路段,胡同].?\\d+.?[号,弄]')
Matcher m = ROAD_PATTERN.matcher(address)
String newAddress
if (m.find()) {
newAddress = m.group()
}
return newAddress
}
def run2MatchV2(fileName){
ExcelUtils.multiThreadingModify(fileName, { d ->
def region = d['I']
def city = d['J']
if(!d['I'] && !d['J']){
city = d['B']
region = d['C']
}
if(d['D']) {
def name = d['D'].toString().replace(".", "").replace("(", "").replace(")", "").replace("。", "")
.replace("·", "").replace("-", "").replace("`", "").replace("?", "").replace("‘", "")
.replace("1期", "一期").replace("2期", "二期").replace("3期", "三期")
.replace("4期", "四期").replace("5期", "五期").replace("12区", "十二区").replace("1区", "一区").replace("2区", "二区").replace("3区", "三区")
.replace("4区", "四区").replace("5区", "五区").replace("a区", "A区")
.replace("B区", "B区").replace("c区", "C区").replace("d区", "D区")
def result = OpenApiUtils.matchDetailDistrict(city, name, region, true)
if (!result && d['E']) {
if (findAddress(d['E'])) {
result = OpenApiUtils.matchDetailDistrict(city, d['E'], region, true)
}
}
if(result){
def matchResult = result.matchResult
if (result.credit > 0.5 && result.matchResult && !result.matchResult.match) {
matchResult = amendMatch(result, d['E'])
}
if (matchResult.match) {
d['AQ'] = result.city
d['AR'] = result.district
d['AS'] = result.name
d['AT'] = result.address
}
}
}
},0)
}
def amendMatch(result, address){
def matchResult = result.matchResult
if(result.credit > 0.65 && matchResult.distance < 2500){
matchResult.match = true
}else if(result.credit>0.5 && address && result.address && findAddress(result.address)){
if(result.address.contains(address) || address.contains(result.address)){
matchResult.match = true
}
}else if(address && address.contains(result.district) && address.contains(result.name)){
matchResult.match = true
}
return matchResult
}
}
@@ -0,0 +1,47 @@
package com.fangjia.script.gj.chongqingblank
import com.fangjia.script.commData.CommonDatautils
import com.fangjia.script.gj.district.FangjiaClientUtils
import com.fangjia.script.util.CosineSimilarAlgorithm
import com.fangjia.script.util.ExcelUtils
import com.fangjia.script.util.OpenApiClient
import net.sf.json.JSONObject
class EvaluationPrice {
static main(args) {
def task = new EvaluationPrice()
task.run()
}
def run(){
ExcelUtils.multiThreadingModify("D:\\数据\\重庆农商行\\估值3-V3.xlsx",{ d->
if(d['K'] && d['M']){
def priceMap = getEvaluationPrice(d['J'], d['K'], d['L'], d['D'].toString())
if(priceMap){
d['E'] = priceMap.avgPrice
d['F'] = priceMap.totalPrice
}
}
},0);
}
def getEvaluationPrice(city, region, name, area){
Map<String, String> params = new HashMap<String, String>();
params.put("city", city);
params.put("district", region);
params.put("name", name);
params.put("size", area);
params.put("__refresh_cache","1")
JSONObject result = OpenApiClient.evaluate(params);
if (result.getInt("code") == 200) {
if (result.containsKey("result")) {
return result.getJSONObject("result")
}
} else if (result.getInt("code") == 500) {
println("小区估价失败:${params.city} ${params.name} ${params.size} ${result}")
}
}
}
@@ -0,0 +1,629 @@
package com.fangjia.script.gj.chongqingblank
import com.fangjia.core.RegionData
import com.fangjia.datastore.driver.DSCollection
import com.fangjia.datastore.driver.DataStore
import com.fangjia.script.commData.CommonDatautils
import com.fangjia.script.gj.district.FangjiaClientUtils
import com.fangjia.script.gj.district.NameProcess
import com.fangjia.script.util.CommonUtils
import com.fangjia.script.util.CosineSimilarAlgorithm
import com.fangjia.script.util.ExcelUtils
import com.fangjia.script.util.GeoHelper
import com.fangjia.script.util.OpenApiClient
import com.fangjia.script.yinjihaun.amap.GaoDeApiUtils
import com.fangjia.script.yinjihaun.amap.GaoDeMapApiData
import com.mongodb.BasicDBObject
import com.mongodb.DBObject
class TestMatchResult {
private static final DSCollection SEAWEED = new DataStore("seaweed.db.fangjia.com", 27007).getCollection("fangjia.seaweed")
static main(args) {
def task = new TestMatchResult()
// task.run1("D:\\数据\\重庆农商行\\估值1.xlsx")
task.run2("D:\\数据\\重庆农商行\\估值2-V2.xlsx")
// task.run3("D:\\数据\\重庆农商行\\估值3.xlsx")
// task.nearDistRun("D:\\数据\\重庆农商行\\估值2-V5.xlsx")
// task.run2V1("D:\\数据\\重庆农商行\\估值2-V2.xlsx")
// task.similarScore("D:\\数据\\重庆农商行\\估值2-V5.xlsx")
// task.mapRun("D:\\数据\\重庆农商行\\估值2-V4-map.xlsx");
}
def run1(def fileName){
ExcelUtils.multiThreadingModify(fileName, { d ->
def region = d['I']
def city = d['J']
if (!d['I'] && !d['J']) {
city = d['B']
region = d['C']
}
def newName = null
if(d['D']=="大川菲尔小城"){
if (d['D']) {
newName = d['D'].toString().replace(".", "").replace("(", "").replace(")", "").replace("。", "")
.replace("·", "").replace("-", "").replace("`", "").replace("?", "").replace("‘", "")
.replace("1期", "一期").replace("2期", "二期").replace("3期", "三期")
.replace("4期", "四期").replace("5期", "五期").replace("12区", "十二区").replace("1区", "一区").replace("2区", "二区").replace("3区", "三区")
.replace("4区", "四区").replace("5区", "五区").replace("a区", "A区")
.replace("B区", "B区").replace("c区", "C区").replace("d区", "D区")
}
if (!region || (region && region == d['L'])) {
if (d['D'] && !d['O'] && d['M']) {
if (d['M'] == newName) {
d['O'] = "相同"
}
if (!d['O']) {
def dbo = CommonDatautils.findDistrictByAlias(d['K'], d['L'], d['D'], "district")
if (dbo) {
d['O'] = "别名匹配"
}
}
// if (!d['O'] && d['M'] && d['E']) {
// if (d['E'].toString().contains(d['M'])) {
// d['O'] = "名称in地址"
// }
// }
if (!d['O']) {
def py1 = com.fangjia.core.util.DataUtil.hanyuQuanpin(newName, " ")
def py2 = com.fangjia.core.util.DataUtil.hanyuQuanpin(d['M'], " ")
if (py1 == py2) {
d['O'] = "拼音相同"
}
}
if (!d['O'] && d['D'] && d['M'] && newName) {
def word = null
if (newName.toString().length() >= d['M'].toString().length()) {
word = newName.toString().replace(d['M'], "")
} else if (newName.toString().length() < d['M'].toString().length()) {
word = d['M'].toString().replace(newName, "")
}
if (word) {
def words = ["重庆", "小区", "(高层)", "花园", "恒都", "(洋房)", "社区"]
for (def w : words) {
if (word == w) {
d['O'] = "去部分词后相同"
break
}
}
if (!d['O'] && !word.endsWith("区") && !word.endsWith("期") && !word.endsWith("团") && !word.endsWith("东苑") && !word.endsWith("西苑") && !word.endsWith("南苑") && !word.endsWith("北苑")) {
if (d['N'] && d['N'].toString().endsWith("号") && d['E'] && d['E'].toString().contains(d['N'])) {
def score = CosineSimilarAlgorithm.getSimilarity(d['M'], newName);
if (score >= 0.65) {
println newName + " " + d['M'] + " " + score
d['O'] = "地址相同"
}
}
}
}
}
if (!d['O'] && newName && !newName.contains("期") && !newName.endsWith("组团") && (!newName.endsWith("区") || newName.endsWith("小区"))) {
def gaodeRest = FangjiaClientUtils.getMapData(city, region, newName, d['E'], "住宅", "高德")
if (!gaodeRest) {
gaodeRest = FangjiaClientUtils.getMapData(city, region, newName, d['E'], "住宅", "百度")
}
if (gaodeRest && gaodeRest.name) {
def mapName = gaodeRest.name.replace(".", "").replace("(", "").replace(")", "").replace("·", "").replace("`", "").replace("?", "")
if (mapName == d['M'] && gaodeRest.score >= 0.7) {
d['O'] = gaodeRest.source + "匹配"
println newName + " " + d['M'] + " " + gaodeRest.score
}
}
}
}
}
if (d['D'] && !d['O']) {
def gaodeRest = FangjiaClientUtils.getMapData(city, region, newName, d['E'], "住宅", "高德")
def mapName = null
if (gaodeRest && gaodeRest.name) {
mapName = gaodeRest.name.replace(".", "").replace("(", "").replace(")", "").replace("·", "").replace("`", "").replace("?", "")
}
if (!gaodeRest || (gaodeRest && mapName != newName)) {
gaodeRest = FangjiaClientUtils.getMapData(city, region, newName, d['E'], "住宅", "百度")
if (gaodeRest && gaodeRest.name) {
mapName = gaodeRest.name.replace(".", "").replace("(", "").replace(")", "").replace("·", "").replace("`", "").replace("?", "")
}
}
if (gaodeRest && mapName == newName) {
d['P'] = gaodeRest.district
d['Q'] = mapName
d['R'] = gaodeRest.address
d['S'] = gaodeRest.lng
d['T'] = gaodeRest.lat
d['U'] = gaodeRest.type
d['v'] = gaodeRest.source
}
}
}
},0);
}
def similarScore(def fileName){
ExcelUtils.multiThreadingModify(fileName, { d ->
if(d['D'] && d['M']){
def score = CosineSimilarAlgorithm.getSimilarity(d['D'], d['M']);
d['Q'] = score
}
},0)
}
def run2(def fileName){
ExcelUtils.multiThreadingModify(fileName,{ d->
def region = d['I']
def city = d['J']
if(!d['I']){
city = d['J']
}
// if(d['E']=="巴南区花溪镇花溪新街88号2单元3-4#") {
// println "ss"
if (!region || (region && region == d['L'])) {
if (d['N'] && !d['O'] && d['E']) {
if (d['N'].toString().endsWith("号") && d['E'] && d['E'].toString().contains(d['N'])) {
d['O'] = "地址匹配"
}
}
}
if (!d['O'] && d['M'] && d['E']) {
def matchName = d['M']
if(matchName.toString().endsWith("小区")){
matchName = matchName.toString().replace("小区","")
}
if(matchName.toString().startsWith(d['L'])){
matchName = matchName.toString().replace(d['L'],"")
}
def oldName= NameProcess.nameStandard(city,region, d['E'])
if (oldName.toString().contains(matchName)) {
d['O'] = "名称in地址"
}
}
if (d['E'] && !d['O'] ) {
def gaodeRest = null
if (!gaodeRest && d['E']) {
gaodeRest = FangjiaClientUtils.getMapData(city, region, d['E'], null, "住宅", "高德")
}
if(!gaodeRest && d['H']){
gaodeRest = FangjiaClientUtils.getMapData(city, region, d['H'], null, "住宅", "高德")
}
if (!gaodeRest && d['E']) {
gaodeRest = FangjiaClientUtils.getMapData(city, region, d['E'], null, "住宅", "百度")
}
if (!gaodeRest && d['H']) {
gaodeRest = FangjiaClientUtils.getMapData(city, region, d['H'], null, "住宅", "百度")
}
if (gaodeRest && gaodeRest.score >= 0.5) {
d['P'] = gaodeRest.district
d['Q'] = gaodeRest.name
d['R'] = gaodeRest.address
d['S'] = gaodeRest.lng
d['T'] = gaodeRest.lat
d['U'] = gaodeRest.type
d['v'] = gaodeRest.source
}
}
if(!d['O']){
def locationMap = getLocation(city, d['E'])
if(locationMap){
def mapRets = findNearestByMap(locationMap.lat, locationMap.lng)
if(mapRets){
def o = mapRets.get(0)
d['W'] = o.cityname
d['X'] = o.adname
d['Y'] = o.name
if(o.address && o.address instanceof String){
d['Z'] = o.address
}
d['AA'] = o.type
d['AB'] = o.distance
d['AC'] = o.location
}
def dbRets = findNearestDistrict(city, locationMap.lat, locationMap.lng, 1000, 10)
if(dbRets){
def b = dbRets.get(0)
d['AD'] = b.region
d['AE'] = b.name
d['AF'] = b.address
d['AG'] = b.distance
}
}
}
// }
},0);
}
def distanceRun(def fileName){
ExcelUtils.multiThreadingModify(fileName, { d ->
def city = d['J']
if(!d['O'] && d['E'] && d['E'].toString().length()>6){
def locationMap = getLocation(city, d['E'])
if(locationMap){
def mapRets = findNearestByMap(locationMap.lat, locationMap.lng)
if(mapRets){
def o = mapRets.get(0)
d['W'] = o.cityname
d['X'] = o.adname
d['Y'] = o.name
if(o.address && o.address instanceof String){
d['Z'] = o.address
}
d['AA'] = o.type
d['AB'] = o.distance
d['AC'] = o.location
}
def dbRets = findNearestDistrict(city, locationMap.lat, locationMap.lng, 1000, 10)
if(dbRets){
def b = dbRets.get(0)
d['AD'] = b.region
d['AE'] = b.name
d['AF'] = b.address
d['AG'] = b.distance
}
}
}
},0);
}
def nearDistRun(def fileName){
ExcelUtils.multiThreadingModify(fileName, { d ->
def city = d['B']
// if(!d['I'] && !d['J']){
// city = d['B']
// }
if(d['O'] && d['O']=="周边"){
def locationMap = null
if(d['B'] && d['E']){
locationMap = getLocation(city, d['E'])
}
// if(!locationMap && d['C']){
// locationMap = getLocation(city, d['C'])
// }
if(locationMap){
def dbRets = findNearestDistrict(city, locationMap.lat, locationMap.lng, 1000, 10)
if(dbRets){
def b = dbRets.get(0)
d['K'] = b.city
d['L'] = b.region
d['M'] = b.name
d['N'] = b.address
d['P'] = b.distance
}
}
}
},0);
}
def run2V1(def fileName){
ExcelUtils.multiThreadingModify(fileName,{ d->
def city = d['J']
if(d['O']){
d['AH'] = d['L']
d['AI'] = d['M']
d['AJ'] = d['N']
}
if(!d['O']){
if(d['R'] && d['R'].contains("号")){
def addr = d['R'].toString().substring(0, d['R'].indexOf("号"))
if((addr && d['E'] && d['E'].contains(addr))){
def region = RegionData.formatRegion(city, d['P'])
if(d['U'] && d['U'].toString().contains("住宅")){
d['AH'] = region
d['AI'] = d['Q']
d['AJ'] = d['R']
d['AK'] = d['U']
d['AM'] = d['S']+","+d['T']
d['O'] ="地图匹配"
}else if(!d['U']){
d['AH'] = region
d['AI'] = d['Q']
d['AJ'] = d['R']
d['AK'] = d['U']
d['AM'] = d['S']+","+d['T']
d['O'] ="地图匹配"
}
}
}
}
if(!d['O']){
if(d['Y'] && d['AE']){
def mapName = d['Y'].toString().replace(".", "").replace("(", "").replace(")", "").replace("。", "")
.replace("·", "").replace("-", "").replace("`", "").replace("?", "").replace("‘", "")
.replace("1期", "一期").replace("2期", "二期").replace("3期", "三期")
.replace("4期", "四期").replace("5期", "五期").replace("12区", "十二区").replace("1区", "一区").replace("2区", "二区").replace("3区", "三区")
.replace("4区", "四区").replace("5区", "五区").replace("a区", "A区")
.replace("B区", "B区").replace("c区", "C区").replace("d区", "D区")
if(mapName == d['AE']){
def region = RegionData.formatRegion(city, d['AD'])
d['AH'] = region
d['AI'] = d['AE']
d['AJ'] = d['AF']
d['AK'] = d['DB']
d['O'] ="名称相同"
}else{
if(d['AB'].toString().toDouble()>=d['AG'].toString().toDouble()){
def region = RegionData.formatRegion(city, d['AD'])
d['AH'] = region
d['AI'] = d['AE']
d['AJ'] = d['AF']
d['AK'] = "DB库近"
d['AL'] = d['AG']
d['O'] ="距离比较"
}else{
def region = RegionData.formatRegion(city, d['X'])
d['AH'] = region
d['AI'] = d['Y']
d['AJ'] = d['Z']
d['AK'] = d['AA']
d['AL'] = d['AB']
d['AM'] = d['AC']
d['O'] ="距离比较"
}
}
}else if(d['Y']){
def region = RegionData.formatRegion(city, d['X'])
d['AH'] = region
d['AI'] = d['Y']
d['AJ'] = d['Z']
d['AK'] = d['AA']
d['AL'] = d['AB']
d['AM'] = d['AC']
d['O'] ="采用地图相近"
}else if(d['AE']){
def region = RegionData.formatRegion(city, d['AD'])
d['AH'] = region
d['AI'] = d['AE']
d['AJ'] = d['AF']
d['AK'] = "DB库近"
d['AL'] = d['AG']
d['O'] ="采用DB相近"
}
}
},0);
}
def findNearestByMap(lng, lat){
def rets = GaoDeMapApiData.doSearchByGaoDe("120000|120203|120300|120301|120302|120303|120304", lng, lat, 1000, 1, 10)
return rets
}
public List<DBObject> findNearestDistrict(def city, double lat, double lng, int radiusMeters, int limit) {
DBObject districtFields = new BasicDBObject("city", 1).append("region", 1).append("name", 1).append("address", 1)
districtFields.put("estate_type2", 1)
districtFields.put("lat", 1)
districtFields.put("lng", 1)
DBObject sdq = new BasicDBObject();
sdq.put("sd.point", lat + "," + lng);
sdq.put("sd.sfield", "lat_lng");
sdq.put("sd.distance", String.valueOf((double) radiusMeters / 1000.0));
DBObject q = new BasicDBObject();
q.put("cat", "district");
q.put("city", city);
q.put("visible", 0);
q.put('$sd', sdq);
List<DBObject> list = new ArrayList<DBObject>()
Iterator<DBObject> iter = SEAWEED.find(q, districtFields).limit(limit * 2).iterator()
while(iter.hasNext()) {
def o = iter.next()
o.distance = GeoHelper.GetDistance(o.lat/10000000000.0, o.lng/10000000000.0, lat, lng)
list.add(o)
}
if (list.size() == 0) {
return Collections.EMPTY_LIST;
}
list.sort{o1,o2->
return o1.distance <=> o2.distance
}
if (list.size() > limit) {
list = list.subList(0, limit);
}
return list;
}
def getLocation(city, addr){
if(addr){
def rMap = [:]
def reps = GaoDeApiUtils.geo(city, addr);
if(reps && reps.count.toInteger()>=1){
def map = reps.geocodes.get(0)
def location = map.location
def level = map.level
if(location){
def locMap = location.split(",")
// def lat = dist.lat/10000000000.0;
// def lng = dist.lng/10000000000.0;
// def distance = GeoHelper.GetDistance(locMap[1].toDouble(), locMap[0].toDouble(), lat, lng)
// rMap.distance = distance
// rMap.level = level
rMap.lat = locMap[1].toDouble()
rMap.lng = locMap[0].toDouble()
// rMap.keyword = map.formatted_address
// rMap.location = location
// rMap.region = map.district
return rMap;
}
}
}
return null;
}
def run3(def fileName){
ExcelUtils.multiThreadingModify(fileName,{ d->
def city = d['I']
def region = d['J']
def newName = null
if(d['C']){
if(d['C'].toString().length()<2){
newName = null
}else{
newName = d['C'].toString().replace(".", "").replace("(", "").replace(")", "").replace("。", "")
.replace("·", "").replace("-", "").replace("`", "").replace("?", "").replace("‘", "")
.replace("1期", "一期").replace("2期", "二期").replace("3期", "三期")
.replace("4期", "四期").replace("5期", "五期").replace("12区", "十二区").replace("a区", "A区")
.replace("B区", "B区").replace("c区", "C区").replace("d区", "D区")
}
}
def address = null
if(d['H']){
address = d['H']
}else{
def arrs = d['B'].split("->")
if(arrs && arrs.size()==4){
address = arrs[3]
}else if(arrs && arrs.size()==2){
address = arrs[1]
}
}
if(!region ||(region && d['L'] && region.contains(d['L']))) {
if (d['B'] && !d['O'] && d['M']) {
if (d['M'] == newName) {
d['O'] = "相同"
}
if (!d['O']) {
def dbo = CommonDatautils.findDistrictByAlias(d['K'], d['L'], newName, "district")
if (dbo) {
d['O'] = "别名匹配"
}
}
if (!d['O']) {
def py1 = com.fangjia.core.util.DataUtil.hanyuQuanpin(newName, " ")
def py2 = com.fangjia.core.util.DataUtil.hanyuQuanpin(d['M'], " ")
if (py1 == py2) {
d['O'] = "拼音相同"
}
}
if (!d['O'] && d['M'] && newName) {
def word = null
if (newName.toString().length() >= d['M'].toString().length()) {
word = newName.toString().replace(d['M'], "")
} else if (newName.toString().length() < d['M'].toString().length()) {
word = d['M'].toString().replace(newName, "")
}
if (word) {
def words = ["重庆", "小区", "(高层)", "花园", "恒都", "(洋房)", "社区"]
for (def w : words) {
if (word == w) {
d['O'] = "去部分词后相同"
break
}
}
if (!d['O'] && !word.endsWith("区") && !word.endsWith("期") && !word.endsWith("团") && !word.endsWith("东苑") && !word.endsWith("西苑") && !word.endsWith("南苑") && !word.endsWith("北苑")) {
if (d['N'] && d['N'].toString().endsWith("号") && address.toString().contains(d['N'])) {
def score = CosineSimilarAlgorithm.getSimilarity(d['M'], newName);
if (score >= 0.65) {
println newName + " " + d['M'] + " " + score
d['O'] = "地址相同"
}
}
}
}
}
if (!d['O'] && newName && !newName.contains("期") && !newName.endsWith("组团") && (!newName.endsWith("区") || newName.endsWith("小区"))) {
def gaodeRest = FangjiaClientUtils.getMapData(city, region, newName, address, "住宅", "高德")
if (!gaodeRest) {
gaodeRest = FangjiaClientUtils.getMapData(city, region, newName, address, "住宅", "百度")
}
if (gaodeRest && gaodeRest.name) {
def mapName = gaodeRest.name.replace(".", "").replace("(", "").replace(")", "").replace("·", "").replace("`", "").replace("?", "")
if (mapName == d['M'] && gaodeRest.score >= 0.7) {
d['O'] = gaodeRest.source + "匹配"
println newName + " " + d['M'] + " " + gaodeRest.score
}
}
}
}
}
if(newName && !d['O']){
def gaodeRest = FangjiaClientUtils.getMapData(city, region, newName, address, "住宅", "高德")
def mapName = null
if(gaodeRest && gaodeRest.name){
mapName = gaodeRest.name.replace(".", "").replace("(", "").replace(")", "").replace("·", "").replace("`", "").replace("?", "")
}
if(!gaodeRest || (gaodeRest && mapName != newName)) {
gaodeRest = FangjiaClientUtils.getMapData(city, region, newName, d['E'], "住宅", "百度")
if(gaodeRest && gaodeRest.name){
mapName = gaodeRest.name.replace(".", "").replace("(", "").replace(")", "").replace("·", "").replace("`", "").replace("?", "")
}
}
if(gaodeRest && mapName== newName){
d['P'] = gaodeRest.district
d['Q'] = mapName
d['R'] = gaodeRest.address
d['S'] = gaodeRest.lng
d['T'] = gaodeRest.lat
d['U'] = gaodeRest.type
d['v'] = gaodeRest.source
}
}
},0);
}
def mapRun(fileName){
ExcelUtils.multiThreadingModify(fileName, { d ->
def city = d['J']
def region = d['I']
if(city && region && d['Q'] && d['O']!="距离比较" && d['O']!="地址匹配"){
def gaodeRest = FangjiaClientUtils.getMapData(city, region, d['Q'], null, "住宅", "高德")
if(gaodeRest){
d['V'] = gaodeRest.city
d['W'] = gaodeRest.district
d['X'] = gaodeRest.name
d['Y'] = gaodeRest.address
d['Z'] = gaodeRest.lng
d['AA'] = gaodeRest.lat
d['AB'] = gaodeRest.type
}
d['AC'] = NameProcess.nameStandard(city,region, d['Q'])
}
},0)
}
}
@@ -0,0 +1,89 @@
package com.fangjia.script.gj.chongqingblank
import com.fangjia.script.commData.CommonDatautils
import com.fangjia.script.util.ExcelUtils
import com.fangjia.script.util.MongoUtils
import com.mongodb.BasicDBObject
import com.mongodb.DBCollection
import com.mongodb.QueryOperators
class TestPrice {
def static final DBCollection AVERAGE_STATISTICS = MongoUtils.getCollectionByName("fangjia", "average_statistics", "192.168.0.43")
public static final def lineUrl = "mongodb://developer:G00IDCDEV12385@192.168.0.43:28017";
public static final DBCollection DEAL_PRICE_COLL = MongoUtils.getCollectionByName2("fangjia", "deal_price", lineUrl)
static main(args) {
def task = new TestPrice()
task.run()
}
def run(){
ExcelUtils.multiThreadingModify("D:\\数据\\重庆农商行\\估值1-V2.xlsx",{ d->
if(d['K'] && d['M']){
def dbo = findThirdPrice(d['K'], d['L'], d['M'], 202012, "lianjia_esf_price")
if(dbo){
d['AU'] = dbo.lianjia_esf_price
}
def district = CommonDatautils.findDistrict(d['K'], d['L'], d['M'], "district")
def startDate = 20201101
def endDate = 20210601
def trades = findTrades(district, startDate, endDate)
if(trades){
d['AV'] = (int)(trades.sum()/trades.size())
d['AW'] = trades.size()
}
}
},0);
}
def findThirdPrice(def city, def region, def name, def s_date,def fieldName){
def q = new BasicDBObject()
q.put("city", city)
q.put("region", region)
q.put("name", name)
q.put("s_date", new BasicDBObject(QueryOperators.GTE, s_date))
q.put(fieldName, new BasicDBObject(QueryOperators.EXISTS, true))
def dbcursor = AVERAGE_STATISTICS.find(q).sort(new BasicDBObject("s_date",-1))
def obj = null;
for(def d : dbcursor){
obj = d;
break;
}
return obj;
}
def findTrades(dist, startDate, endDate){
def q = new BasicDBObject()
q.put("city", dist.city)
q.put("region", dist.region)
// q.put("source", "链家在线")
def alias = []
if(dist.alias){
if(!dist.alias.contains(dist.name)){
dist.alias << dist.name
}
alias = dist.alias
}else{
alias << dist.name
}
if(alias){
q.put("district_name", new BasicDBObject('$in', alias))
}
q.put("trade_date", new BasicDBObject('$gte', startDate).append('$lt',endDate))
def dbcorsor = DEAL_PRICE_COLL.find(q)
def prices = []
for(def d : dbcorsor){
if(d.avg_price){
prices << d.avg_price
}
}
return prices
}
}
@@ -0,0 +1,29 @@
package com.fangjia.script.gj.data
import com.fangjia.core.RegionData
import com.fangjia.script.commData.CommonDatautils
import com.fangjia.script.util.CommonUtils
import com.fangjia.script.util.ExcelUtils
class RegionExport {
static main(args) {
def task = new RegionExport()
task.run()
}
def run(){
ExcelUtils.multiThreadingModify("d:\\Êý¾Ý\\Ç×ÁÚ\\Ð¡Çø³ÇÊÐÁбí.xls",{d->
def city = d['B']
def newCity = RegionData.formatCity(city)
if(newCity){
def regions = CommonUtils.getAllRegions(newCity, "region")
def regionStr = ""
regions.each{k,v->
regionStr = regionStr + k+"\n"
}
d['C'] = regionStr
}
},0)
}
}
@@ -0,0 +1,236 @@
package com.fangjia.script.gj.es
import com.fangjia.script.commData.CommonDatautils
import com.fangjia.script.common.DataHandleUtils
import com.fangjia.script.util.DateUtils
import com.fangjia.script.util.ElasticUtils
import com.fangjia.script.util.ExcelUtils
import com.fangjia.script.util.MongoUtils
import com.mongodb.BasicDBObject
import com.mongodb.DBCollection
import org.apache.lucene.queryparser.classic.QueryParser
class EsMonthHouseStat {
public static final def lineUrl = "mongodb://developer:G00IDCDEV12385@192.168.0.43:28017";
public static final DBCollection DEAL_PRICE_COLL = MongoUtils.getCollectionByName2("fangjia", "deal_price", lineUrl)
static main(args) {
def task = new EsMonthHouseStat()
// task.run()
task.tradeRun()
// task.houseRun()
}
def run(){
def citys = ["北京","上海","广州","深圳","重庆","郑州","长沙","西安","成都","东莞","佛山","杭州","合肥","南京","青岛","沈阳","苏州","天津","武汉","珠海","中山","长春","扬州","烟台","徐州","无锡","温州","太原","台州","石家庄","绍兴","厦门","泉州","宁波","南通","南宁","南昌","兰州","常州","大连","福州","贵阳","哈尔滨","海口","惠州","济南","嘉兴","金华","昆明","遵义","淄博","株洲","镇江","肇庆","漳州","湛江","岳阳","银川","宜昌","盐城","芜湖","乌鲁木齐","唐山","泰州","汕头","三亚","清远","秦皇岛","马鞍山","洛阳","柳州","连云港","廊坊","九江","吉林","淮安","湖州","呼和浩特","沧州","保定","安庆","张家口","西宁","日照","拉萨","锦州","大同","常德","北海","文昌","陵水","保亭"]
def rets = []
def dates = ["2020-05","2020-06","2020-07","2020-08","2020-09","2020-10","2020-11","2020-12","2021-01","2021-02","2021-03","2021-04","2021-05"]
def headers = ["城市"]
headers.addAll(dates)
rets << headers
for(def city : citys){
def datas = []
datas << city
for(def d : dates){
def date = DateUtils.parseFromTightYYYYMMDD(d+"-01")
def startDate = DateUtils.formatDateByPattern(date, "yyyyMMdd")
def endDate = DateUtils.addMonth(date, 1).replace("-","")
def count = findHouseCount(city, startDate.toInteger(), endDate.toInteger())
datas << count
}
rets << datas
println datas
}
ExcelUtils.write("d:\\数据\\建行92个城市房源统计.xlsx", rets, "统计")
}
def tradeRun(){
def citys = ["北京","上海","广州","深圳","重庆","郑州","长沙","西安","成都","东莞","佛山","杭州","合肥","南京","青岛","沈阳","苏州","天津","武汉","珠海","中山","长春","扬州","烟台","徐州","无锡","温州","太原","台州","石家庄","绍兴","厦门","泉州","宁波","南通","南宁","南昌","兰州","常州","大连","福州","贵阳","哈尔滨","海口","惠州","济南","嘉兴","金华","昆明","遵义","淄博","株洲","镇江","肇庆","漳州","湛江","岳阳","银川","宜昌","盐城","芜湖","乌鲁木齐","唐山","泰州","汕头","三亚","清远","秦皇岛","马鞍山","洛阳","柳州","连云港","廊坊","九江","吉林","淮安","湖州","呼和浩特","沧州","保定","安庆","张家口","西宁","日照","拉萨","锦州","大同","常德","北海","文昌","陵水","保亭"]
def rets = []
def rets2 = []
//,
//"2021-01","2021-02","2021-03","2021-04","2021-05"
//def dates = ["2020-01","2020-02","2020-03","2020-04","2020-05","2020-06","2020-07","2020-08","2020-09","2020-10","2020-11","2020-12"]
def dates = ["2021-04"]
def headers = ["城市"]
headers.addAll(dates)
rets << headers
def headers2 = ["城市"]
headers2.addAll(dates)
rets2 << headers2
for(def city : citys){
def datas = []
datas << city
def datas2 = []
datas2 << city
for(def d : dates){
def startDate = DateUtils.parseFromTightYYYYMMDD(d+"-01")
def endDateStr = DateUtils.addMonth(startDate, 1).replace("-","")
def endDate = DateUtils.parseFromTightYYYYMMDD2(endDateStr)
//def count = findTrades(city, startDate, endDate)
def districts = CommonDatautils.findAllDistricts(city, "district", ["city", "region", "name", "alias"]);
def oldCount = 0
def newCount = 0
for(def dist : districts){
def countMap = findTradesDetail(dist, startDate, endDate)
oldCount = oldCount + countMap.total_old
newCount = newCount + countMap.total_new
println dist.city + " " + dist.name +" "+countMap.total_old+" "+countMap.total_new
}
datas << oldCount
datas2 << newCount
println city +" "+ oldCount +" "+newCount
}
rets << datas
rets2 << datas2
println datas
}
ExcelUtils.write("d:\\数据\\建行92个城市成交统计(去重前匹配).xlsx", rets, "统计")
ExcelUtils.write("d:\\数据\\建行92个城市成交统计(去重后匹配).xlsx", rets2, "统计")
}
def houseRun(){
ExcelUtils.multiThreadingModify("d:\\数据\\小区案例数据覆盖率.xlsx",{d->
def city = d['B']
if(city == "湖州"){
def count = findHouseCount(city, 20210301, 20210601)
d['E'] = count
def districts = CommonDatautils.findAllDistricts(city, "district", ["city", "region", "name", "alias"]);
def disthouseCount = 0
for(def dist : districts){
def alias = []
if(dist.alias){
if(!dist.alias.contains(dist.name)){
dist.alias << dist.name
}
alias = dist.alias
}else{
alias << dist.name
}
def distCount = findHouseCount(dist.city, dist.region, alias, 20210301, 20210601)
if(distCount){
disthouseCount++
}
println dist.city+" "+dist.region+" "+dist.name+" "+distCount
}
println "disthouseCount:"+disthouseCount
println districts.size()
def precent = DataHandleUtils.getDoubleRound(2, disthouseCount*100.0/districts.size())
d['D'] = precent+"%"
}
},0)
}
def findHouseCount(def city, def region, def alias, def startDate, def endDate){
def aliasStr = null;
def total = 0
try{
aliasStr = escapeExprSpecialWord(alias.join("','"))
String sql = "select count(*) as c from latest_seawater where cat='sell' and city='${city}' and p_date >= ${startDate} and p_date<${endDate} and region='${region}' and district_name in ('${aliasStr}')"
ElasticUtils.executeSqlQuery(sql,{data->
total = data.c
})
}catch(Exception e){
e.printStackTrace()
}
return total
}
/**
* 转义字符串中的特殊字符
* 仅过滤Lucene regexps, 其他的勿用
* @param keyword
* @return
*/
public static String escapeExprSpecialWord(String keyword) {
return QueryParser.escape(keyword);
}
def findHouseCount(def city, def startDate, def endDate){
String sql = "select count(*) as c from latest_seawater where cat='sell' and city='${city}' and p_date >= ${startDate} and p_date<${endDate} "
def total = 0
ElasticUtils.executeSqlQuery(sql,{ data->
total = data.c
})
return total
}
def findTrades(city, startDate, endDate){
def q = new BasicDBObject()
q.put("city", city)
// q.put("region", dist.region)
// q.put("source", "链家在线")
// def alias = []
// if(dist.alias){
// if(!dist.alias.contains(dist.name)){
// dist.alias << dist.name
// }
// alias = dist.alias
// }else{
// alias << dist.name
// }
// if(alias){
// q.put("district_name", new BasicDBObject('$in', alias))
// }
q.put("trade_date", new BasicDBObject('$gte', startDate).append('$lt',endDate))
def count = DEAL_PRICE_COLL.count(q)
return count
}
def findTradesDetail(dist, startDate, endDate){
def q = new BasicDBObject()
q.put("city", dist.city)
q.put("region", dist.region)
def alias = []
if(dist.alias){
if(!dist.alias.contains(dist.name)){
dist.alias << dist.name
}
alias = dist.alias
}else{
alias << dist.name
}
if(alias){
q.put("district_name", new BasicDBObject('$in', alias))
}
q.put("trade_date", new BasicDBObject('$gte', startDate).append('$lt',endDate))
def dbcursor = DEAL_PRICE_COLL.find(q)
def total = dbcursor.count()
def rets = new HashSet()
for(def d : dbcursor){
def key=d.city+"##"+d.region+"##"+d.district_name+"##"+d.area+"##"+d.avg_price+"##"+DateUtils.formatDateByPattern(d.trade_date,"yyyy-MM-dd")
if(!rets.contains(key)){
rets << key
}
}
def rMap = [:]
rMap.total_old = total
rMap.total_new = rets.size()
return rMap
}
}
@@ -0,0 +1,66 @@
package com.fangjia.script.gj.es
import com.fangjia.core.util.DateUtil
import com.fangjia.script.util.ElasticUtils
import org.elasticsearch.index.query.QueryBuilder
import org.elasticsearch.index.query.QueryBuilders
import org.elasticsearch.xpack.sql.jdbc.EsDataSource
import java.sql.Connection
import java.sql.ResultSet
import java.sql.Statement
class EsSelectTest {
EsDataSource dataSource = new EsDataSource();
String address = "jdbc:es://http://192.168.0.86:9200/timezone=UTC&page.size=250&connect.timeout=120s&network.timeout=120s&page.timeout=120s";
static main(args) {
def task = new EsSelectTest()
task.run()
}
def run(){
QueryBuilder queryBuilder = QueryBuilders.boolQuery().must(QueryBuilders.termQuery("request", "/ws/czbank"))
//.must(QueryBuilders.termQuery("request_body", "findSample"))
def houses = ElasticUtils.executeSimpleQuery("192.168.0.86", "fangjia-log*", queryBuilder);
if(houses) {
for (int k = 0; k < houses.size(); k++) {
println k
}
}
def endDate = new Date()
dataSource.setUrl(address)
Properties connectionProperties = new Properties()
dataSource.setProperties(connectionProperties)
StringBuffer sqlBuffer = new StringBuffer();
String d = DateUtil.formatByPattern(new Date(), "yyyy.MM.dd")
long startD = DateUtil.getBeforeMinute(new Date(), -7).getTime();
long endD = DateUtil.getBeforeMinute(endDate, 1).getTime();
println(DateUtil.formatByPattern(new Date(startD), "yyyy-MM-dd HH:mm:ss")+"===="+DateUtil.formatByPattern(new Date(endD), "yyyy-MM-dd HH:mm:ss"));
sqlBuffer.append("select request_time FROM \"fangjia-log*\" WHERE 1=1");
sqlBuffer.append(" and request='/ws/czbank' and request_body='findSample'")
ResultSet rs = null
try {
Connection connection = dataSource.getConnection()
Statement statement = connection.createStatement()
rs = statement.executeQuery(sqlBuffer.toString())
} catch (Exception e) {
e.printStackTrace()
}
StringBuffer msgBuffer = new StringBuffer();
while (rs.next()) {
def responsetime = rs.getLong("responsetime");
def timestamp = DateUtil.formatByPattern(new Date(rs.getLong("time")), "yyyy-MM-dd HH:mm")
println responsetime
}
}
}
@@ -0,0 +1,171 @@
package com.fangjia.script.gj.location
import com.fangjia.core.RegionData
import com.fangjia.script.commData.CommonDatautils
import com.fangjia.script.gj.district.FangjiaClientUtils
import com.fangjia.script.looplineinfo.InsidePolygon
import com.fangjia.script.looplineinfo.Point
import com.fangjia.script.util.DateUtils
import com.fangjia.script.util.ExcelUtils
import com.fangjia.script.util.GeoHelper
import com.fangjia.script.util.MongoUtils
import com.mongodb.BasicDBObject
import com.mongodb.DBCollection
import java.text.SimpleDateFormat
import java.util.regex.Matcher
import java.util.regex.Pattern;
class LocationTest {
static DBCollection POLYLINECOLL = MongoUtils.getCollectionByAuth20190102("fangjia_crawl", "region_polyline")
static main(args) {
def task = new LocationTest()
// task.run()
task.run2()
}
def run(){
def regionPointsMap = findRegionRange()
def districts = CommonDatautils.findAllDistricts("福州", "district", ["city", "region", "name", "address", "lat", "lng"]);
def index = 0
def rets = []
for(def dist : districts){
def flag = isInArea(dist.lng/10000000000.0, dist.lat/10000000000.0, regionPointsMap.get(dist.region))
if(!flag){
index++
def ret = []
ret << dist.city
ret << dist.region
ret << dist.name
def loactionMap = matchMap(dist)
if(loactionMap){
ret << "地图"
ret << loactionMap.region
ret << loactionMap.mapName
def distance = GeoHelper.GetDistance(dist.lat/10000000000.0, dist.lng/10000000000.0, loactionMap.lat, loactionMap.lng)
ret << distance
if(dist.region==loactionMap.region){
println index+" "+dist.region +" "+dist.name +" 不在区域范围内,与地图相同"
}else{
println index+" "+dist.region +" "+dist.name +" 不在区域范围内 地图:"+ loactionMap.region+" "+distance
}
}else{
ret << ""
println index+" "+dist.region +" "+dist.name +" 不在区域范围内,地图没找到"
}
rets << ret
}
}
ExcelUtils.write("d:\\数据\\福州\\区域经纬度错误3.xlsx", rets, "区域不在行政范围内")
}
def run2(){
ExcelUtils.multiThreadingModify("d:\\数据\\福州\\福州小区.xlsx")
}
def matchMap(def dist){
def gaodeRest = FangjiaClientUtils.searchMapData(dist.city, dist.name,"住宅", "高德")
def locationMap = null
if(gaodeRest){
locationMap = getMapLocation(dist, gaodeRest);
if(locationMap){
return locationMap
}
}
def baiduRest = null
if (!locationMap) {
baiduRest = FangjiaClientUtils.searchMapData(dist.city, dist.name,"住宅", "百度")
}
if(baiduRest){
locationMap = getMapLocation(dist, baiduRest);
}
return locationMap
}
def getMapLocation(def dist, def rets){
def lat = null
def lng = null
def region = null
def mapName = null
for(def r : rets){
if(!r.city){
continue
}
if(!r.city.contains(dist.city)){
continue
}
def name = r.name.replace("?","").replace("&","").replace("·","").replace(".","").replace("(","").replace(")","").replace("(","").replace(")","").replace("【","").replace("】","").replace("11期","十一期").replace("12期","十二期").replace("13期","十三期").replace("14期","十四期").replace("15期","十五期").replace("16期","十六期").replace("17期","十七期").replace("18期","十八期").replace("19期","十九期").replace("10期","十期").replace("1期","一期").replace("2期","二期").replace("3期","三期").replace("4期","四期").replace("5期","五期").replace("6期","六期").replace("7期","七期").replace("8期","八期").replace("9期","九期").replace(" ","").trim()
if(dist.name == name){
lat = r.lat
lng = r.lng
mapName = r.name
region = r.district
}else{
def word = null
if (name.length() >= dist.name.length()) {
word = name.replace(dist.name, "")
} else if (name.length() < dist.name.length()) {
word = dist.name.replace(name, "")
}
if (word) {
def words = ["商住楼", "小区","公寓","别墅"]
for (def w : words) {
if (word == w) {
lat = r.lat
lng = r.lng
region = r.district
mapName = r.name
break
}
}
if(!lng){
Pattern pattern = Pattern.compile("\\d+号楼");
Matcher matcher = pattern.matcher(word);
if(matcher.find()){
lat = r.lat
lng = r.lng
region = r.district
mapName = r.name
break
}
}
}
}
if(lng){
break
}
}
if(lng){
return ["region":region, "lng":lng, "lat":lat, "mapName":mapName]
}
return null
}
def isInArea(lng, lat, points){
List<Point> pointList = InsidePolygon.createPoint2(points);
//Point point = new Point(lng, lat);
//def flag = InsidePolygon.aboutZone(point, pointList)
def flag = InsidePolygon.isPtInPoly(lng, lat, pointList)
return flag;
}
def findRegionRange(){
def q = new BasicDBObject()
q.put("city","福州市")
def dbcursor = POLYLINECOLL.find(q)
def regionMap = [:]
for(def d : dbcursor){
def region = RegionData.formatRegion("福州",d.region)
regionMap.put(region, d.polyline)
}
return regionMap
}
}
@@ -0,0 +1,106 @@
package com.fangjia.script.thirdPriceCrawl
import com.fangjia.script.util.CommonUtils
import com.mongodb.BasicDBObject
import org.apache.solr.common.util.Hash
class SynchDistrictPrice {
def static final SHORT_DATE = Utils.getLastMonth()
def thirdPartyColl = CrawalDBUtil.getThirdPartyCol();
def districtDetailColl = CrawalDBUtil.getDistrictDetailColl();
def sourceMap = ["jiwu.com":"吉屋"]
//"58.com":"58同城","lianjia.com":"链家","ke.com":"贝壳找房","fang.com":"搜房","fangdd.com":"房多多","jiwu.com":"吉屋"
static main(args) {
def task = new SynchDistrictPrice()
task.run()
}
def run(){
def citys = ["福州"]//CommonUtils.findAllCity()
sourceMap.each { k, v ->
for(def city : citys){
def rets = findCrawlDistrictDetails(city, k, v)
synchData(city, rets, v)
println city +" "+rets.size()
}
}
}
def findCrawlDistrictDetails(city, root, source){
def q = new BasicDBObject()
q.put("city", city)
q.put("root", root)
def keys = new BasicDBObject()
keys.put("city", 1)
keys.put("region", 1)
keys.put("block", 1)
keys.put("name", 1)
keys.put("address", 1)
keys.put("avg_price", 1)
keys.put("root", 1)
keys.put("url", 1)
def rets = []
def dbcursor = districtDetailColl.find(q, keys)
def names = new HashSet()
for(def d : dbcursor){
def s = d.city +"#"+d.region+"#"+d.name
if(!names.contains(s)){
names << s
d.put("district_id", d.url)
d.put("source", source)
d.put("s_date", SHORT_DATE)
d.removeField("root")
d.removeField("url")
d.removeField("_id")
rets << d
}
}
return rets
}
def synchData(def city, def rets, def source){
def q = new BasicDBObject()
q.put("source", source)
q.put("s_date", SHORT_DATE)
q.put("city", city)
def count = thirdPartyColl.count(q)
def addIndex = 0
if(count==0){
try{
thirdPartyColl.insert(rets)
}catch(Exception e){
println e.getMessage()
}
}else{
for(def r : rets){
def flag = findIsExists(r.city, r.region, r.name, SHORT_DATE, source)
if(!flag){
addIndex++
thirdPartyColl.save(r)
}
}
}
println "["+source +"-"+city + "] add:"+ addIndex+" 原抓取:"+count
}
def findIsExists(def city, def region, def name, def sDate, def source){
def q = new BasicDBObject()
q.put("source", source)
q.put("s_date", sDate)
q.put("city", city)
q.put("region", region)
q.put("name", name)
def count = thirdPartyColl.count(q)
if(count){
return true
}
return false
}
}