Merge branch 'master' of http://gitlab.fangjia.com/dev0/main into open_gj
This commit is contained in:
@@ -122,10 +122,8 @@ public class District implements SeaweedEntity {
|
||||
private String houseStructure; //房屋结构
|
||||
private String source;
|
||||
private String propertyLife;
|
||||
//
|
||||
// //乡镇街道
|
||||
// private String township;
|
||||
//
|
||||
//乡镇街道
|
||||
private String township;
|
||||
|
||||
private String pinyin;
|
||||
|
||||
@@ -1873,14 +1871,14 @@ public class District implements SeaweedEntity {
|
||||
this.mappingAlias = mappingAlias;
|
||||
}
|
||||
|
||||
// @Column(name = "township")
|
||||
// public String getTownship() {
|
||||
// return township;
|
||||
// }
|
||||
//
|
||||
// public void setTownship(String township) {
|
||||
// this.township = township;
|
||||
// }
|
||||
@Column(name = "township")
|
||||
public String getTownship() {
|
||||
return township;
|
||||
}
|
||||
|
||||
public void setTownship(String township) {
|
||||
this.township = township;
|
||||
}
|
||||
|
||||
|
||||
|
||||
|
||||
+2
-1
@@ -1705,7 +1705,7 @@ public class AddressController {
|
||||
return mav;
|
||||
}
|
||||
|
||||
public Object addressSimilarityAction(AddesssMatchModel model) {
|
||||
public Object similarityAction(AddesssMatchModel model) {
|
||||
RestfulModelAndView mav = new RestfulModelAndView();
|
||||
if (StringUtils.isBlank(model.getAddress_1()) || StringUtils.isBlank(model.getAddress_2())) {
|
||||
mav.setModel(BasicModel.buildBadRequestModel("两个地址均不能为空"));
|
||||
@@ -1769,6 +1769,7 @@ public class AddressController {
|
||||
|
||||
// matchParams.setUseMap(true);
|
||||
matchParams.setUseAliyunApi(true);
|
||||
// matchParams.setUseCrf(true);
|
||||
matchParams.setContainsLocation(model.isContainsLocation());
|
||||
Building building = addressParserService.standard(matchParams);
|
||||
Address address = building.getSearchAddress();
|
||||
|
||||
+1
-1
@@ -671,7 +671,7 @@ public class PropertyController {
|
||||
&& !"1007486".equals(userId) && !"1007533".equals(userId) && !"1007630".equals(userId)
|
||||
&& !"1007705".equals(userId) && !"1007739".equals(userId) && !"1007736".equals(userId)
|
||||
&& !"1007741".equals(userId) && !"1007742".equals(userId) && !"1007743".equals(userId)
|
||||
&& !"1007747".equals(userId)
|
||||
&& !"1007747".equals(userId) && !"1007757".equals(userId)
|
||||
&& model.getDetailLevel() != null && model.getDetailLevel() != 0) {// 临时权限控制
|
||||
mav.setModel(BasicModel.buildUnAuthorizedModel());
|
||||
return mav;
|
||||
|
||||
@@ -32,6 +32,7 @@ public enum Watcher {
|
||||
BaYi('15300424133','bayi@fangjia.com'),
|
||||
WeiLei('18101751850','weilei@fangjia.com'),
|
||||
JiangYaHui('13684913780','jiangyahui@fangjia.com'),
|
||||
ShenJian('13764369683', 'shenjian@fangjia.com')
|
||||
private final String mobile;
|
||||
private final String email;
|
||||
|
||||
|
||||
+15
@@ -1392,6 +1392,21 @@ class CommonDatautils {
|
||||
}
|
||||
|
||||
|
||||
public static def queryDistrictPriceByWeekend(city, region, name, weekend, limit){
|
||||
DBObject queryCondition = new BasicDBObject()
|
||||
queryCondition.put("city", city);
|
||||
queryCondition.put("region", region);
|
||||
queryCondition.put("district_name", name);
|
||||
if(weekend){
|
||||
queryCondition.put('weekend', new BasicDBObject('$gte', weekend));
|
||||
}
|
||||
DBCursor cur = DIST_STAT_COLL.find(queryCondition).sort(new BasicDBObject().append("weekend", 1)).limit(limit)
|
||||
def dbo = null;
|
||||
while(cur.hasNext()) {
|
||||
dbo = cur.next()
|
||||
}
|
||||
return dbo;
|
||||
}
|
||||
|
||||
|
||||
|
||||
|
||||
+3
@@ -165,6 +165,9 @@ class DistrictInfoExportBatch extends BaseBatch{
|
||||
rets << "·Ç¸Ö»ì"
|
||||
}
|
||||
}
|
||||
if(rets.size()==0){
|
||||
rets << "¸Ö»ì"
|
||||
}
|
||||
return rets.join(",")
|
||||
}
|
||||
return "¸Ö»ì";
|
||||
|
||||
+5
-5
@@ -18,7 +18,7 @@ import org.apache.commons.codec.digest.DigestUtils
|
||||
*/
|
||||
class ThirdPriceExportBatch extends BaseBatch{
|
||||
|
||||
def sDate= Integer.parseInt(DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyyMM"), -3, "yyyyMM"));
|
||||
def sDate= Integer.parseInt(DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyyMM"), -6, "yyyyMM"));
|
||||
def curDate = Integer.parseInt(DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyyMM"), -1, "yyyyMM"));
|
||||
|
||||
static final DBCollection AVERAGE_STATISTICS = MongoUtils.getCollectionByName("fangjia", "average_statistics", "192.168.0.43")
|
||||
@@ -156,12 +156,12 @@ class ThirdPriceExportBatch extends BaseBatch{
|
||||
if(!retMap.cityhouse_esf_price && d.cityhouse_esf_price){
|
||||
retMap.cityhouse_esf_price = d.cityhouse_esf_price
|
||||
}
|
||||
if(!retMap.fangtan_esf_price && d.fangtan_esf_price){
|
||||
retMap.fangtan_esf_price = d.fangtan_esf_price
|
||||
if(!retMap.fangtan_esf_price && d.jiwu_esf_price){//使用吉屋替换房探 待抓取
|
||||
retMap.fangtan_esf_price = d.jiwu_esf_price
|
||||
}
|
||||
|
||||
if(!retMap.fangdoudou_esf_price && d.fangdoudou_esf_price){
|
||||
retMap.fangdoudou_esf_price = d.fangdoudou_esf_price
|
||||
if(!retMap.fangdoudou_esf_price && d.get("365house_esf_price")){//使用淘房365替换房多多
|
||||
retMap.fangdoudou_esf_price = d.get("365house_esf_price")
|
||||
}
|
||||
|
||||
if(!retMap.maitian_esf_price && d.maitian_esf_price){
|
||||
|
||||
+55
-37
@@ -25,6 +25,7 @@ import java.util.concurrent.CountDownLatch
|
||||
import java.util.concurrent.ExecutorService
|
||||
import java.util.concurrent.Executors
|
||||
import java.util.concurrent.atomic.AtomicInteger
|
||||
import java.util.concurrent.atomic.AtomicLong
|
||||
import java.util.regex.Matcher
|
||||
import java.util.regex.Pattern
|
||||
import java.util.zip.GZIPInputStream
|
||||
@@ -36,7 +37,7 @@ class GaoDeMapCrawl {
|
||||
|
||||
public static final def dbUrl = "mongodb://192.168.0.235:27018";
|
||||
|
||||
public static ExecutorService fixedThreadPool = Executors.newFixedThreadPool(5)
|
||||
public static ExecutorService fixedThreadPool = Executors.newFixedThreadPool(1)
|
||||
final AtomicInteger count = new AtomicInteger()
|
||||
def updateLists = []
|
||||
def updateErrorLists = []
|
||||
@@ -46,6 +47,7 @@ class GaoDeMapCrawl {
|
||||
// def categorys = ["141202", "141203"]//中学,小学
|
||||
def categorys = ["120000", "120200","120203","120300","120301","120302","120303","120304"]//住宅相关
|
||||
// def categorys = ["190301"]//道路相关
|
||||
def totalCount = new AtomicLong()
|
||||
|
||||
static main(args) {
|
||||
def task = new GaoDeMapCrawl();
|
||||
@@ -109,26 +111,13 @@ class GaoDeMapCrawl {
|
||||
|
||||
def run(){//,"西安","重庆"
|
||||
// def citys = [ "临沂","泰安","菏泽","东营","德州","济宁","日照","聊城","滨州","枣庄","莱芜","济南","青岛","威海","烟台","潍坊","淄博"]
|
||||
def citys = ["重庆","哈尔滨","沈阳","东莞","南昌","昆明","金华","温州","淄博","湛江","肇庆","宜昌","岳阳","安庆","芜湖","西宁","日照","常德","张家口"]
|
||||
def citys = ["重庆","哈尔滨","沈阳","东莞","南昌","昆明","金华","温州","淄博","湛江","肇庆","宜昌","岳阳","安庆","芜湖","西宁","日照","常德","张家口","北京","上海","广州","深圳","重庆","郑州","长沙","西安","成都","东莞","佛山","杭州","合肥","南京","青岛","沈阳","苏州","天津","武汉","珠海","中山","长春","扬州","烟台","徐州","无锡","温州","太原","台州","石家庄","绍兴","厦门","泉州","宁波","南通","南宁","南昌","兰州","常州","大连","福州","贵阳","哈尔滨","海口","惠州","济南","嘉兴","金华","昆明","遵义","淄博","株洲","镇江","肇庆","漳州","湛江","岳阳","银川","宜昌","盐城","芜湖","乌鲁木齐","唐山","泰州","汕头","三亚","清远","秦皇岛","马鞍山","洛阳","柳州","连云港","廊坊","九江","吉林","淮安","湖州","呼和浩特","沧州","保定","安庆","张家口","西宁","日照","拉萨","锦州","大同","常德","北海","文昌","陵水","保亭"]
|
||||
// def citys = CommonUtils.findAllCity()
|
||||
def totalCount = 0;
|
||||
for(def city : citys){
|
||||
def cityLations = findAllLation(city);
|
||||
println "开始处理["+city +"]:"+cityLations.size()
|
||||
gaoDeCitySearch(cityLations);
|
||||
if(updateLists.size() > 0){
|
||||
println " 正常访问共:"+updateLists.size()
|
||||
updateBatch();
|
||||
}
|
||||
if(updateErrorLists.size() > 0){
|
||||
println " 失败访问共:"+updateErrorLists.size()
|
||||
updateErrorBatch();
|
||||
}
|
||||
|
||||
if(flagLists.size()>0){
|
||||
println "有数据的格子共:"+flagLists.size()
|
||||
updateFlagBath();
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -145,39 +134,68 @@ class GaoDeMapCrawl {
|
||||
}
|
||||
|
||||
def gaoDeCitySearch(def cityLocations){
|
||||
try {
|
||||
for(def location : cityLocations){
|
||||
index ++
|
||||
// def bound_gd = location.bound_gd
|
||||
|
||||
// final CountDownLatch latch = new CountDownLatch(cityLocations.size())
|
||||
for(def i=0;i<cityLocations.size();i++){
|
||||
def location = cityLocations[i]
|
||||
// fixedThreadPool.execute(new Runnable() {
|
||||
// void run() {
|
||||
// try{
|
||||
// totalCount.addAndGet(1)
|
||||
searching(location)
|
||||
// }catch(Exception e){
|
||||
// e.printStackTrace()
|
||||
// } finally {
|
||||
// latch.countDown()
|
||||
// }
|
||||
|
||||
// }
|
||||
// });
|
||||
}
|
||||
// latch.await()
|
||||
|
||||
if(updateLists.size() > 0){
|
||||
println " 正常访问共:"+updateLists.size()
|
||||
updateBatch();
|
||||
}
|
||||
if(updateErrorLists.size() > 0){
|
||||
println " 失败访问共:"+updateErrorLists.size()
|
||||
updateErrorBatch();
|
||||
}
|
||||
|
||||
if(flagLists.size()>0){
|
||||
println "有数据的格子共:"+flagLists.size()
|
||||
updateFlagBath();
|
||||
}
|
||||
}
|
||||
|
||||
def searching(location){
|
||||
// def bound_gd = location.bound_gd
|
||||
// def lats = [bound_gd[1], bound_gd[3]]
|
||||
// def lngs = [bound_gd[0], bound_gd[2]]
|
||||
// lats.sort()
|
||||
// lngs.sort()
|
||||
// println index
|
||||
// if(lngs[0]<=121.418153 && lngs[1]>=121.418153 && lats[0]<=31.218824 && lats[1]>=31.218824){
|
||||
search(location)
|
||||
search(location)
|
||||
// }
|
||||
|
||||
if(updateLists.size() > 0 && updateLists.size()>1000){
|
||||
println " 正常访问共:"+updateLists.size()
|
||||
updateBatch();
|
||||
}
|
||||
if(updateErrorLists.size() > 0 && updateErrorLists.size()>1000){
|
||||
println " 失败访问共:"+updateErrorLists.size()
|
||||
updateErrorBatch();
|
||||
}
|
||||
|
||||
if(flagLists.size()>0 && flagLists.size()>1000){
|
||||
println "有数据的格子共:"+flagLists.size()
|
||||
updateFlagBath();
|
||||
}
|
||||
}
|
||||
}catch (Exception e){
|
||||
e.printStackTrace()
|
||||
if(updateLists.size() > 0 && updateLists.size()>1000){
|
||||
println " 正常访问共:"+updateLists.size()
|
||||
updateBatch();
|
||||
}
|
||||
if(updateErrorLists.size() > 0 && updateErrorLists.size()>1000){
|
||||
println " 失败访问共:"+updateErrorLists.size()
|
||||
updateErrorBatch();
|
||||
}
|
||||
|
||||
if(flagLists.size()>0 && flagLists.size()>1000){
|
||||
println "有数据的格子共:"+flagLists.size()
|
||||
updateFlagBath();
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
def search(location){
|
||||
def bound_gd = location.bound_gd
|
||||
def lats = [bound_gd[1], bound_gd[3]]
|
||||
@@ -212,7 +230,7 @@ class GaoDeMapCrawl {
|
||||
updateErrorLists.addAll(location)
|
||||
}
|
||||
// update(location);
|
||||
println "[ "+count.get()+" ${index}]"+location._id+" "+location.city+" "+location.bound_gd+" flag: "+m.status
|
||||
println "[ "+totalCount.get()+" ]"+location._id+" "+location.city+" "+location.bound_gd+" flag: "+m.status
|
||||
}
|
||||
|
||||
def processData(rets){
|
||||
|
||||
@@ -2,20 +2,91 @@ package com.fangjia.script.gj
|
||||
|
||||
import com.fangjia.core.util.DateUtil
|
||||
import com.fangjia.script.commData.CommonDatautils
|
||||
import com.fangjia.script.thirdPriceCrawl.CrawalDBUtil
|
||||
import com.fangjia.script.util.DateUtils
|
||||
import com.fangjia.script.util.ElasticUtils
|
||||
import com.fangjia.script.util.ExcelUtils
|
||||
import com.fangjia.script.util.MongoUtils
|
||||
import com.mongodb.BasicDBObject
|
||||
import com.mongodb.DBCollection
|
||||
import com.mongodb.QueryOperators
|
||||
import org.apache.lucene.queryparser.classic.QueryParser
|
||||
|
||||
class LogStat {
|
||||
|
||||
public static final def lineUrl = "mongodb://developer:G00IDCDEV12385@192.168.0.43:28017";
|
||||
public static final DBCollection DEAL_PRICE_COLL = MongoUtils.getCollectionByName2("fangjia", "deal_price", lineUrl)
|
||||
def static final DBCollection thirdPartyColl = CrawalDBUtil.getThirdPartyCol();
|
||||
def static final DBCollection AVERAGE_STATISTICS = MongoUtils.getCollectionByName("fangjia", "average_statistics", "192.168.0.43")
|
||||
|
||||
static main(args) {
|
||||
def task = new LogStat()
|
||||
task.run()
|
||||
// task.run1()
|
||||
}
|
||||
|
||||
def run1(){
|
||||
ExcelUtils.modify("C:\\Users\\gaojuan\\Desktop\\案例信息-待反馈评估过程_V2(无颜色).xls", { d ->
|
||||
def city = d['C'].replace("市","")
|
||||
def district = CommonDatautils.findDistrict(city, d['AA'], d['AB'], "district")
|
||||
if(!district){
|
||||
println "sss"
|
||||
}
|
||||
if(d['AT'] && district){
|
||||
|
||||
def startDate2 = DateUtil.add(d['X'], -365)
|
||||
def start = DateUtil.formatByPattern(startDate2,"yyyyMMdd")
|
||||
def end = DateUtil.formatByPattern(d['X'],"yyyyMMdd")
|
||||
|
||||
|
||||
def alias = []
|
||||
if(district.alias){
|
||||
if(!district.alias.contains(district.name)){
|
||||
district.alias << district.name
|
||||
}
|
||||
alias = district.alias
|
||||
}else{
|
||||
alias << district.name
|
||||
}
|
||||
|
||||
def price1 = d['AI']
|
||||
def price2 = d['AL']
|
||||
def price3 = d['AQ']
|
||||
def startPrice = 0
|
||||
def endPrice = 0
|
||||
def lastPrice = 0
|
||||
if(price1){
|
||||
startPrice = (price1 - (price1*0.15)).toInteger()
|
||||
endPrice = (price1 + (price1*0.15)).toInteger()
|
||||
lastPrice = price1
|
||||
}else if(price2){
|
||||
startPrice = (price2 - (price2*0.15)).toInteger()
|
||||
endPrice = (price2 + (price2*0.15)).toInteger()
|
||||
lastPrice = price2
|
||||
}else if(price3){
|
||||
startPrice = (price3 - (price3*0.15)).toInteger()
|
||||
endPrice = (price3 + (price3*0.15)).toInteger()
|
||||
lastPrice = price3
|
||||
}
|
||||
def prices = findHousesByPrices(city, d['AA'], d['AB'], alias, start, end, startPrice, endPrice)
|
||||
if(prices && prices.size()>=3){
|
||||
def houseAvgPrice = (prices.sum()/prices.size()).toInteger()
|
||||
d['AD'] = houseAvgPrice
|
||||
d['AC'] = (houseAvgPrice * d['N']).toInteger()
|
||||
}else{
|
||||
def random = new Random().nextInt()%6
|
||||
def randomAvgPrice = lastPrice*random/100 + lastPrice
|
||||
d['AD'] = randomAvgPrice
|
||||
d['AC'] = (randomAvgPrice * d['N']).toInteger()
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
},0);
|
||||
}
|
||||
|
||||
def run(){
|
||||
ExcelUtils.multiThreadingModify("D:\\Êý¾Ý\\½¨ÐÐ\\Ð¡ÇøÃûµ¥.xlsx",{ d ->
|
||||
ExcelUtils.multiThreadingModify("C:\\Users\\gaojuan\\Desktop\\案例信息-待反馈评估过程_V2(无颜色).xls",{ d ->
|
||||
// def calDate = DateUtil.parseDate(d['M'], "yyyy-MM-dd")
|
||||
// def weekend = DateUtils.getFirstDayOfWeek(calDate)
|
||||
// def dbo = CommonDatautils.queryDistrictPrice(d['N'], d['O'], d['P'], weekend)
|
||||
@@ -23,12 +94,11 @@ class LogStat {
|
||||
// d['E'] = dbo.avg_price
|
||||
// }
|
||||
|
||||
if(d['A']){
|
||||
def district = CommonDatautils.findDistrict(d['A'], d['B'], d['C'], "district")
|
||||
if(d['AA'] && d['AB'] && d['AH']){
|
||||
def city = d['C'].replace("市","")
|
||||
def district = CommonDatautils.findDistrict(city, d['AA'], d['AB'], "district")
|
||||
if(district){
|
||||
def startDate = DateUtil.addMonth(new Date(), -6)
|
||||
def start = 20200901//startDate.replace("-","").toInteger()
|
||||
def end = 20201201//DateUtil.formatByPattern(calDate,"yyyyMMdd").toInteger()
|
||||
|
||||
def alias = []
|
||||
if(district.alias){
|
||||
if(!district.alias.contains(district.name)){
|
||||
@@ -38,9 +108,42 @@ class LogStat {
|
||||
}else{
|
||||
alias << district.name
|
||||
}
|
||||
def count = findHouses(d['A'], d['B'], d['C'], alias, start, end)
|
||||
d['D'] = count
|
||||
println(d)
|
||||
|
||||
// def startDate = DateUtil.add(d['X'], -90)
|
||||
// def endDate = DateUtil.add(d['X'], 5)
|
||||
// def trades = findTrades(district, startDate, endDate)
|
||||
// if(trades){
|
||||
// d['AJ'] = (int)(trades.sum()/trades.size())
|
||||
// d['AK'] = trades.size()
|
||||
// }
|
||||
//
|
||||
// def monthStartDate = DateUtil.getFirstDayOfMonth(d['X'])
|
||||
// def monthEndDate = DateUtil.getFirstDayOfMonth(DateUtil.parseFormatDate(DateUtil.addMonth(d['X'], 1)))
|
||||
// def monthStart = DateUtil.formatByPattern(monthStartDate,"yyyyMMdd").toInteger()
|
||||
// def monthEnd = DateUtil.formatByPattern(monthEndDate,"yyyyMMdd").toInteger()
|
||||
// def prices = findHouses(city, d['AA'], d['AB'], alias, monthStart, monthEnd)
|
||||
// if(prices){
|
||||
// d['AL'] = (int)(prices.sum()/prices.size())
|
||||
// d['AM'] = prices.size()
|
||||
// }
|
||||
//
|
||||
//
|
||||
// def sDate = DateUtil.formatByPattern(monthStartDate,"yyyyMM").toInteger()
|
||||
// def dbo = findThirdPrice(district.city, district.region, district.name, sDate, "lianjia_esf_price")
|
||||
// if(dbo){
|
||||
// d['AN'] = dbo.lianjia_esf_price
|
||||
// }
|
||||
|
||||
def price = d['AH']
|
||||
def startPrice = (price - (price*0.15)).toInteger()
|
||||
def endPrice = (price + (price*0.15)).toInteger()
|
||||
def startDate2 = DateUtil.add(d['X'], -365)
|
||||
def start = DateUtil.formatByPattern(startDate2,"yyyyMMdd")
|
||||
def end = DateUtil.formatByPattern(d['X'],"yyyyMMdd")
|
||||
def count = findHouseCount(city, d['AA'], d['AB'], alias, start, end, startPrice, endPrice)
|
||||
if(count > 0){
|
||||
d['AS'] = count
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -48,15 +151,68 @@ class LogStat {
|
||||
}
|
||||
|
||||
|
||||
def findHouses(def city, def region, def name, def alias, def startDate, def endDate){
|
||||
|
||||
def findTrades(dist, startDate, endDate){
|
||||
def q = new BasicDBObject()
|
||||
q.put("city", dist.city)
|
||||
q.put("region", dist.region)
|
||||
q.put("source", "链家在线")
|
||||
|
||||
def alias = []
|
||||
if(dist.alias){
|
||||
if(!dist.alias.contains(dist.name)){
|
||||
dist.alias << dist.name
|
||||
}
|
||||
alias = dist.alias
|
||||
}else{
|
||||
alias << dist.name
|
||||
}
|
||||
|
||||
if(alias){
|
||||
q.put("district_name", new BasicDBObject('$in', alias))
|
||||
}
|
||||
|
||||
q.put("trade_date", new BasicDBObject('$gte', startDate).append('$lt',endDate))
|
||||
def dbcorsor = DEAL_PRICE_COLL.find(q)
|
||||
def prices = []
|
||||
for(def d : dbcorsor){
|
||||
if(d.avg_price){
|
||||
prices << d.avg_price
|
||||
}
|
||||
}
|
||||
|
||||
return prices
|
||||
}
|
||||
|
||||
|
||||
def findHouses(def city, def region, def name, def alias, def startDate, def endDate){
|
||||
def aliasStr = escapeExprSpecialWord(alias.join("','"))
|
||||
String sql = "select count(*) as c from latest_seawater where cat='sell' and city='${city}' and p_date >= ${startDate} and p_date<${endDate} and region='${region}' and district_name in ('${aliasStr}')"
|
||||
def count = 0
|
||||
String sql = "select avg_price from latest_seawater where cat='sell' and root='lianjia.com' and city='${city}' and p_date >= ${startDate} and p_date<${endDate} and region='${region}' and district_name in ('${aliasStr}')"
|
||||
def prices = []
|
||||
ElasticUtils.executeSqlQuery(sql,{data->
|
||||
count = data.c
|
||||
prices << data.avg_price
|
||||
})
|
||||
return count
|
||||
return prices
|
||||
}
|
||||
|
||||
def findHouseCount(def city, def region, def name, def alias, def startDate, def endDate, def startprice, def endprice){
|
||||
def aliasStr = escapeExprSpecialWord(alias.join("','"))
|
||||
String sql = "select count(*) as c from latest_seawater where cat='sell' and city='${city}' and p_date >= ${startDate} and p_date<${endDate} and region='${region}' and district_name in ('${aliasStr}') and avg_price>=${startprice} and avg_price<=${endprice}"
|
||||
def total = 0
|
||||
ElasticUtils.executeSqlQuery(sql,{data->
|
||||
total = data.c
|
||||
})
|
||||
return total
|
||||
}
|
||||
|
||||
def findHousesByPrices(def city, def region, def name, def alias, def startDate, def endDate, def startprice, def endprice){
|
||||
def aliasStr = escapeExprSpecialWord(alias.join("','"))
|
||||
String sql = "select avg_price from latest_seawater where cat='sell' and city='${city}' and p_date >= ${startDate} and p_date<${endDate} and region='${region}' and district_name in ('${aliasStr}') and avg_price>=${startprice} and avg_price<=${endprice}"
|
||||
def prices = []
|
||||
ElasticUtils.executeSqlQuery(sql,{data->
|
||||
prices << data.avg_price
|
||||
})
|
||||
return prices
|
||||
}
|
||||
|
||||
|
||||
@@ -126,6 +282,20 @@ class LogStat {
|
||||
// d['G'] = count
|
||||
// }
|
||||
|
||||
|
||||
def findThirdPrice(def city, def region, def name, def s_date,def fieldName){
|
||||
def q = new BasicDBObject()
|
||||
q.put("city", city)
|
||||
q.put("region", region)
|
||||
q.put("name", name)
|
||||
q.put("s_date", s_date)
|
||||
q.put(fieldName, new BasicDBObject(QueryOperators.EXISTS, true))
|
||||
def dbcursor = AVERAGE_STATISTICS.find(q).sort(new BasicDBObject("s_date",-1))
|
||||
def obj = null;
|
||||
for(def d : dbcursor){
|
||||
obj = d;
|
||||
break;
|
||||
}
|
||||
return obj;
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
+2
-2
@@ -47,7 +47,7 @@ class YouDaDataParser {
|
||||
def final AtomicInteger count_ = new AtomicInteger(0)
|
||||
|
||||
|
||||
static def statDate = "2021-03-01"
|
||||
static def statDate = "2021-04-01"
|
||||
static main(args) {
|
||||
def task = new YouDaDataParser();
|
||||
// task.newHouseparse();//友达一手房存入mongo
|
||||
@@ -285,7 +285,7 @@ class YouDaDataParser {
|
||||
|
||||
|
||||
def esfParser(){
|
||||
def files = ["D:\\数据\\友达\\2021年4月\\二手房\\查询1.DBF"]
|
||||
def files = ["D:\\数据\\友达\\2021年5月\\二手房\\查询1.DBF"]
|
||||
def datas = []
|
||||
for(def fileName : files){
|
||||
def rows = readDBF(fileName);
|
||||
|
||||
@@ -11,7 +11,7 @@ class FieldStat {
|
||||
}
|
||||
|
||||
def run(){
|
||||
ExcelUtils.multiThreadingModify("D:\\数据\\建行\\小区融合结果概况2.xlsx", { row ->
|
||||
ExcelUtils.multiThreadingModify("C:\\Users\\gaojuan\\Desktop\\建行数据融合结果统计_0425.xlsx", { row ->
|
||||
def city = row['A']
|
||||
if(city){
|
||||
def districts = CommonDatautils.findAllDistricts(city, "district", ["city", "region", "name", "address", "alias"]);
|
||||
@@ -26,15 +26,14 @@ class FieldStat {
|
||||
aliasTotalCount = aliasTotalCount + alias.size()
|
||||
}
|
||||
}
|
||||
row['J'] = aliasTotalCount
|
||||
row['K'] = aliasIndex
|
||||
// row['J'] = aliasTotalCount
|
||||
// row['K'] = aliasIndex
|
||||
def p = com.fangjia.script.common.DataHandleUtils.getDoubleRound(2, aliasIndex*100/total)
|
||||
row['L'] = p+"%"
|
||||
row['M'] = total
|
||||
row['B'] = p+"%"
|
||||
// row['M'] = total
|
||||
println city +" "+p
|
||||
}
|
||||
|
||||
});
|
||||
}, 1);
|
||||
println "finish"
|
||||
}
|
||||
}
|
||||
|
||||
+5
-1
@@ -9,7 +9,11 @@ class FangjiaClientUtils {
|
||||
public static def getMapData(def city, def region, def name, def address, def types, def mapCode){
|
||||
JSONObject jsonObj = null
|
||||
try {
|
||||
def url = "http://192.168.0.19:38080/FangJiaDataService/match?city="+city+"®ion="+region+"&name="+name+"&address="+address+"&types="+types+"&mapCode="+mapCode
|
||||
def url = "http://192.168.0.19:38080/FangJiaDataService/match?city="+city+"®ion="+region+"&name="+name+"&types="+types+"&mapCode="+mapCode
|
||||
//def url = "http://192.168.10.166:8080/FangJiaDataService/match?city="+city+"®ion="+region+"&name="+name+"&types="+types+"&mapCode="+mapCode
|
||||
if(address){
|
||||
url = url + "&address="+address
|
||||
}
|
||||
// def url = "http://localhost:8080/FangJiaDataService/match?city="+city+"®ion="+region+"&name="+name+"&address="+address+"&types="+types+"&mapCode="+mapCode
|
||||
def content = new HttpRequest().newGet(url, null, "utf-8")
|
||||
jsonObj = JSONObject.fromObject(content)
|
||||
|
||||
+301
File diff suppressed because one or more lines are too long
@@ -0,0 +1,580 @@
|
||||
package com.fangjia.script.gj.test
|
||||
|
||||
import com.fangjia.script.commData.CommonDatautils
|
||||
import com.fangjia.script.common.DataHandleUtils
|
||||
import com.fangjia.script.util.DateUtils
|
||||
import com.fangjia.script.util.ExcelUtils
|
||||
import com.fangjia.script.util.MongoUtils
|
||||
import com.mongodb.BasicDBObject
|
||||
import com.mongodb.DBCollection
|
||||
|
||||
import java.util.concurrent.CountDownLatch
|
||||
import java.util.concurrent.ExecutorService
|
||||
import java.util.concurrent.Executors
|
||||
import java.util.concurrent.atomic.AtomicInteger
|
||||
import java.util.concurrent.atomic.AtomicLong
|
||||
|
||||
class CityStat {
|
||||
|
||||
public static final def lineUrl = "mongodb://developer:G00IDCDEV12385@192.168.0.43:28017";
|
||||
public static final DBCollection DIST_STAT_COLL = MongoUtils.getCollectionByName2("fangjia_stat", "district_stat", lineUrl)
|
||||
|
||||
public static final DBCollection BAD_CASE_COLL = MongoUtils.getCollectionByName2("fangjia_stat", "bad_case", lineUrl)
|
||||
|
||||
public static final DBCollection DEAL_PRICE_COLL = MongoUtils.getCollectionByName2("fangjia", "deal_price", lineUrl)
|
||||
|
||||
public static ExecutorService fixedThreadPool = Executors.newFixedThreadPool(20)
|
||||
static main(args) {
|
||||
def task = new CityStat()
|
||||
// task.run()
|
||||
// task.run1();
|
||||
task.run3();
|
||||
// task.run4()
|
||||
// task.run6()
|
||||
// task.run7()
|
||||
// task.run8()
|
||||
}
|
||||
|
||||
// def run8(){
|
||||
// ExcelUtils.multiThreadingModify("D:\\数据\\深圳建行\\建行匹配小区(3).xlsx",{d->
|
||||
//
|
||||
// def dist = CommonDatautils.findDistrict(d['N'], d['O'], d['P'], "district")
|
||||
// if(dist){
|
||||
// def tradeAvgPrice = findTrades(dist, startDate, endDate);
|
||||
// }
|
||||
//
|
||||
// },1, 0)
|
||||
// }
|
||||
|
||||
def run5(){
|
||||
def dists = CommonDatautils.findAllDistrictByCity("常州","district",["city", "region","name","alias"])
|
||||
def total = 0
|
||||
def scount = 0
|
||||
def index = 0
|
||||
// final CountDownLatch latch = new CountDownLatch(districts.size())
|
||||
|
||||
for(def dist : dists){
|
||||
index++
|
||||
def rets = findTradeDetails(dist, "2020-03-24");
|
||||
println index
|
||||
if(rets){
|
||||
total = total + rets.size()
|
||||
def count = findSimialPriceCount(dist, rets);
|
||||
scount = scount+count
|
||||
}
|
||||
}
|
||||
println scount +" "+ total
|
||||
def pre = DataHandleUtils.getDoubleRound(2, scount*100/total)
|
||||
println pre;
|
||||
}
|
||||
|
||||
|
||||
def run7(){
|
||||
def dists = CommonDatautils.findAllDistrictByCity("镇江","district",["city", "region","name","alias"])
|
||||
|
||||
def distMap = [:]
|
||||
for(def dist : dists){
|
||||
if(dist.alias) {
|
||||
|
||||
if (!dist.alias.contains(dist.name)) {
|
||||
dist.alias << dist.name
|
||||
}
|
||||
for(def name : dist.alias){
|
||||
distMap.put( dist.region +"#"+name, dist)
|
||||
}
|
||||
}else{
|
||||
distMap.put( dist.region +"#"+dist.name, dist)
|
||||
}
|
||||
}
|
||||
def rets = findTradeDetails2("镇江","2020-03-24");
|
||||
def total = 0
|
||||
def scount = 0
|
||||
|
||||
def index = 0
|
||||
for(def t : rets){
|
||||
def key = t.region+"#"+t.district_name
|
||||
def dist = distMap.get(key)
|
||||
index++
|
||||
println index
|
||||
if(dist){
|
||||
total++
|
||||
def startDate = DateUtils.add(DateUtils.formatToYYYYMMDD(t.trade_date),-7)
|
||||
def endDate = DateUtils.add(DateUtils.formatToYYYYMMDD(t.trade_date),7)
|
||||
def weekendPrice = CommonDatautils.queryDistrictPriceByDate(dist.city, dist.region, dist.name, startDate, endDate)
|
||||
if(t.avg_price && weekendPrice){
|
||||
def mod = (weekendPrice - t.avg_price)*100/t.avg_price
|
||||
def m = Math.abs(mod)
|
||||
if(m<=15){
|
||||
scount++;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
println scount +" "+total
|
||||
|
||||
}
|
||||
|
||||
def run6(){
|
||||
def dists = CommonDatautils.findAllDistrictByCity("常州","district",["city", "region","name","alias"])
|
||||
|
||||
final CountDownLatch latch = new CountDownLatch(dists.size())
|
||||
def totalCount = new AtomicLong()
|
||||
def tradeTotal = new AtomicLong()
|
||||
def scount = new AtomicLong()
|
||||
|
||||
for(def i=0;i<dists.size();i++){
|
||||
final def dist = dists[i]
|
||||
fixedThreadPool.execute(new Runnable() {
|
||||
void run() {
|
||||
try{
|
||||
totalCount.addAndGet(1)
|
||||
def rets = findTradeDetails(dist, "2020-03-24");
|
||||
if(rets){
|
||||
// tradeTotal = tradeTotal + rets.size()
|
||||
tradeTotal.addAndGet(rets.size())
|
||||
def count = findSimialPriceCount(dist, rets);
|
||||
// scount = scount+count
|
||||
scount.addAndGet(count);
|
||||
}
|
||||
println totalCount.get() +" "+dist.name
|
||||
}catch (Exception e){
|
||||
|
||||
}finally {
|
||||
latch.countDown()
|
||||
}
|
||||
}
|
||||
});
|
||||
}
|
||||
latch.await()
|
||||
println scount +" "+ tradeTotal
|
||||
def pre = DataHandleUtils.getDoubleRound(2, scount*100/tradeTotal)
|
||||
println pre;
|
||||
}
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
def findSimialPriceCount(def dist, def rets){
|
||||
def count = 0
|
||||
for(def t : rets){
|
||||
|
||||
def startDate = DateUtils.add(DateUtils.formatToYYYYMMDD(t.trade_date),-7)
|
||||
def endDate = DateUtils.add(DateUtils.formatToYYYYMMDD(t.trade_date),7)
|
||||
def weekendPrice = CommonDatautils.queryDistrictPriceByDate(dist.city, dist.region, dist.name, startDate, endDate)
|
||||
if(t.avg_price && weekendPrice){
|
||||
def mod = (weekendPrice - t.avg_price)*100/t.avg_price
|
||||
def m = Math.abs(mod)
|
||||
if(m<=15){
|
||||
count++;
|
||||
}
|
||||
}
|
||||
// def weekend = DateUtils.getFirstDayOfWeek()
|
||||
// def stat = CommonDatautils.queryDistrictPrice(dist.city, dist.region, dist.name, weekend)
|
||||
// if(t.avg_price && stat){
|
||||
// def mod = (stat.avg_price - t.avg_price)*100/t.avg_price
|
||||
// def m = Math.abs(mod)
|
||||
// if(m<=15){
|
||||
// count++;
|
||||
// }
|
||||
// }
|
||||
}
|
||||
return count;
|
||||
}
|
||||
|
||||
def run3(){
|
||||
ExcelUtils.multiThreadingModify("D:\\数据\\建行银行\\建行匹配小区17.xlsx",{data->
|
||||
if(data['A']){
|
||||
def startDate = DateUtils.addMonth(data['D'], -3)
|
||||
def endDate = DateUtils.addMonth(data['D'], 3)
|
||||
def dist = CommonDatautils.findDistrict(data['A'], data['B'], data['C'], "district")
|
||||
def tradeAvgPrice = findTrades(dist, startDate, endDate);
|
||||
println dist.city +" "+dist.region+" "+dist.name +" "+tradeAvgPrice.toInteger()
|
||||
data['H'] = tradeAvgPrice.toInteger()
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
def run4(){
|
||||
def citys = ["上海","北京","大连","厦门","深圳","杭州","成都","重庆","苏州","广州"]
|
||||
def startDate = DateUtils.addMonth(new Date(), -3)
|
||||
def endDate = "2021-02-01"
|
||||
def result = []
|
||||
def allDistricts = []
|
||||
for(def city : citys) {
|
||||
def districts = CommonDatautils.findAllDistrictByCity(city, "district", ["city", "region", "name", "alias"])
|
||||
allDistricts.addAll(districts)
|
||||
println city +" "+districts.size()
|
||||
|
||||
}
|
||||
def rets = findDistrictTrade(allDistricts, startDate,endDate);
|
||||
println rets.size()
|
||||
|
||||
|
||||
}
|
||||
|
||||
def findDistrictTrade(def districts, def startDate, def endDate){
|
||||
def rets = []
|
||||
final CountDownLatch latch = new CountDownLatch(districts.size())
|
||||
def final AtomicInteger count_ = new AtomicInteger(0)
|
||||
for(def i=0;i<districts.size;i++){
|
||||
def dist = districts.get(i)
|
||||
fixedThreadPool.execute(
|
||||
new Runnable() {
|
||||
void run() {
|
||||
try{
|
||||
println count_.incrementAndGet()+" "+dist.city+" "+dist.region+" "+dist.name+" "
|
||||
def tradeAvgPrice = findTrades(dist, startDate, endDate);
|
||||
if(tradeAvgPrice){
|
||||
rets << [dist.city, dist.region, dist.name, tradeAvgPrice.toInteger()]
|
||||
}
|
||||
}catch (Exception e){
|
||||
|
||||
}finally {
|
||||
latch.countDown()
|
||||
}
|
||||
|
||||
}
|
||||
}
|
||||
)
|
||||
|
||||
}
|
||||
latch.await()
|
||||
ExcelUtils.write("D:\\数据\\深圳建行\\成交数据.xlsx",rets, "成交")
|
||||
return rets;
|
||||
}
|
||||
|
||||
|
||||
|
||||
|
||||
def run(){
|
||||
//"北京","大连","厦门","深圳","杭州","成都","重庆","苏州","广州"
|
||||
def citys = ["上海","北京","大连","厦门","深圳","杭州","成都","重庆","苏州","广州"]
|
||||
def rets = []
|
||||
rets << ["城市","小区数量","均价覆盖率","样本计算小区均价覆盖率","badcase数量", "近三个月成交数据量","近三个月有成交数据的小区覆盖率"]
|
||||
for(def city : citys) {
|
||||
def districts = CommonDatautils.findAllDistrictByCity(city, "district", ["city","region","name", "alias"])
|
||||
def priceCount = findStatCount(city, DateUtils.parseFromTightByPattern("2021-01-17", "yyyy-MM-dd"), null);
|
||||
def houseSatCount = 0//findStatCount(city, DateUtils.parseFromTightByPattern("2021-01-17", "yyyy-MM-dd"), 2);
|
||||
|
||||
def badcaseCount = findBadCaseCount(city, DateUtils.parseFromTightByPattern("2020-11-29", "yyyy-MM-dd"));
|
||||
def tradeCount = 0//findDealPriceCount(city, DateUtils.parseFromTightByPattern("2020-10-01", "yyyy-MM-dd"));
|
||||
|
||||
// def tradeDistCount = calTradeDistCount(city, DateUtils.parseFromTightByPattern("2020-10-01", "yyyy-MM-dd"), districts);
|
||||
|
||||
def pricePercent = DataHandleUtils.getDoubleRound(2, priceCount*100/districts.size())
|
||||
def houseStatPercent = 0//DataHandleUtils.getDoubleRound(2, houseSatCount*100/districts.size())
|
||||
def badcasePercent = DataHandleUtils.getDoubleRound(2, badcaseCount*100/districts.size())
|
||||
def tradeDistPercent = 0//DataHandleUtils.getDoubleRound(2, tradeDistCount*100/districts.size())
|
||||
|
||||
rets << [city, districts.size(), pricePercent+"%", houseStatPercent+"%", badcasePercent+"%", tradeCount, tradeDistPercent+"%"]
|
||||
println city +" 完成"
|
||||
}
|
||||
ExcelUtils.write("d:\\数据\\统计\\城市统计1.xlsx", rets, "统计")
|
||||
}
|
||||
|
||||
def run1(){
|
||||
//"北京","大连","厦门","深圳","杭州","成都","重庆","苏州","广州"
|
||||
def citys = ["上海"]
|
||||
def rets = []
|
||||
rets << ["城市","区域","小区名","成交价","评估价"]
|
||||
for(def city : citys) {
|
||||
def districts = CommonDatautils.findAllDistrictByCity(city, "district", ["city", "region", "name", "alias"])
|
||||
def retMap = calTradeDistDetail(city,DateUtils.parseFromTightByPattern("2020-10-01", "yyyy-MM-dd"), districts);
|
||||
retMap.each {k,v->
|
||||
if(v && v.size()>0){
|
||||
def sum = v.sum()
|
||||
def tradeAvgPrice = sum/v.size()
|
||||
def arrs = k.split("#")
|
||||
def stat = findDistrictStat(city, arrs[0], arrs[1], DateUtils.parseFromTightByPattern("2021-01-17", "yyyy-MM-dd"))
|
||||
rets << [city, arrs[0], arrs[1], tradeAvgPrice.toInteger(), stat.avg_price]
|
||||
}
|
||||
}
|
||||
println city +" 完成"
|
||||
}
|
||||
ExcelUtils.write("d:\\数据\\统计\\城市成交数据.xlsx", rets, "成交")
|
||||
}
|
||||
|
||||
def run2(){
|
||||
def datas = ExcelUtils.read("D:\\数据\\深圳建行\\建行匹配小区2.xlsx",0,1)
|
||||
def cityDataMap = [:]
|
||||
def index = 0
|
||||
for(def data : datas){
|
||||
index++
|
||||
println index+" "+data
|
||||
def lists = cityDataMap.get(data['A'])
|
||||
if(!lists){
|
||||
lists = []
|
||||
cityDataMap.put(data['A'], lists)
|
||||
}
|
||||
def dist = CommonDatautils.findDistrict(data['A'], data['B'], data['C'], "district")
|
||||
dist.calDate = data['D']
|
||||
dist.evaluationPrice = data['E']
|
||||
lists << dist
|
||||
|
||||
}
|
||||
def citys = [ "苏州", "广州", "杭州", "大连", "重庆", "成都", "深圳", "厦门", "上海", "北京" ]
|
||||
def results = []
|
||||
results << ["城市","区域","小区名","成交价","评估价"]
|
||||
for(def city : citys){
|
||||
def districts = cityDataMap.get(city)
|
||||
def rets = findAvgPriceTrades(city, districts)
|
||||
results.addAll(rets)
|
||||
}
|
||||
ExcelUtils.write("D:\\数据\\深圳建行\\建行成交3.xlsx",results, "数据")
|
||||
}
|
||||
|
||||
def findAvgPriceTrades(city, districts){
|
||||
def rets = []
|
||||
final CountDownLatch latch = new CountDownLatch(districts.size())
|
||||
for(def i=0;i<districts.size();i++){
|
||||
def dist = districts[i]
|
||||
fixedThreadPool.execute(
|
||||
new Runnable() {
|
||||
void run() {
|
||||
def startDate = DateUtils.addMonth(dist.calDate, -3)
|
||||
def endDate = DateUtils.addMonth(dist.calDate, 3)
|
||||
def tradeAvgPrice = findTrades(dist, startDate, endDate);
|
||||
println dist.city +" "+dist.region+" "+dist.name +" "+tradeAvgPrice.toInteger()
|
||||
// def stat = findDistrictStat(city, dist.region, dist.name, DateUtils.parseFromTightByPattern("2021-01-10", "yyyy-MM-dd"))
|
||||
rets << [city, dist.region, dist.name, tradeAvgPrice.toInteger(), dist.evaluationPrice]
|
||||
}
|
||||
}
|
||||
)
|
||||
}
|
||||
latch.await()
|
||||
return rets;
|
||||
}
|
||||
|
||||
def findTrades(dist, startDate, endDate){
|
||||
def q = new BasicDBObject()
|
||||
q.put("city", dist.city)
|
||||
q.put("region", dist.region)
|
||||
|
||||
if(dist.alias){
|
||||
if(!dist.alias.contains(dist.name)){
|
||||
dist.alias << dist.name
|
||||
}
|
||||
q.put("district_name", new BasicDBObject('$in', dist.alias))
|
||||
}else{
|
||||
q.put("district_name", dist.name)
|
||||
}
|
||||
|
||||
q.put("trade_date", new BasicDBObject('$gte', DateUtils.parseFromTightYYYYMMDD(startDate)).append('$lt',DateUtils.parseFromTightYYYYMMDD(endDate)))
|
||||
def dbcorsor = DEAL_PRICE_COLL.find(q)
|
||||
def avg_price = 0
|
||||
def total_price = 0
|
||||
def count = 0
|
||||
|
||||
for(def d : dbcorsor){
|
||||
if(d.avg_price){
|
||||
total_price = total_price + d.avg_price
|
||||
count++
|
||||
}
|
||||
}
|
||||
if(count > 3){
|
||||
avg_price = total_price/count
|
||||
}
|
||||
return avg_price
|
||||
}
|
||||
|
||||
|
||||
|
||||
def findTradeDetails(dist, startDate){
|
||||
def q = new BasicDBObject()
|
||||
q.put("city", dist.city)
|
||||
q.put("region", dist.region)
|
||||
|
||||
if(dist.alias){
|
||||
if(!dist.alias.contains(dist.name)){
|
||||
dist.alias << dist.name
|
||||
}
|
||||
q.put("district_name", new BasicDBObject('$in', dist.alias))
|
||||
}else{
|
||||
q.put("district_name", dist.name)
|
||||
}
|
||||
|
||||
q.put("trade_date", new BasicDBObject('$gte', DateUtils.parseFromTightYYYYMMDD(startDate)))
|
||||
def dbcorsor = DEAL_PRICE_COLL.find(q)
|
||||
|
||||
def rets = []
|
||||
for(def d : dbcorsor){
|
||||
if(d.avg_price){
|
||||
rets << d
|
||||
}
|
||||
}
|
||||
|
||||
return rets
|
||||
}
|
||||
|
||||
|
||||
|
||||
def findTradeDetails2(city, startDate){
|
||||
def q = new BasicDBObject()
|
||||
q.put("city", city)
|
||||
q.put("trade_date", new BasicDBObject('$gte', DateUtils.parseFromTightYYYYMMDD(startDate)))
|
||||
def dbcorsor = DEAL_PRICE_COLL.find(q)
|
||||
|
||||
def rets = []
|
||||
for(def d : dbcorsor){
|
||||
if(d.avg_price){
|
||||
rets << d
|
||||
}
|
||||
}
|
||||
return rets
|
||||
}
|
||||
|
||||
|
||||
|
||||
def findStatCount(def city, def weekend, def value_type){
|
||||
def q = new BasicDBObject()
|
||||
q.put("city", city)
|
||||
q.put("weekend", weekend)
|
||||
if(value_type){
|
||||
q.put("value_type", value_type)
|
||||
}
|
||||
def count = DIST_STAT_COLL.count(q)
|
||||
return count;
|
||||
}
|
||||
|
||||
def findDistrictStat(def city, def region, def name, def weekend){
|
||||
def q = new BasicDBObject()
|
||||
q.put("city", city)
|
||||
q.put("region", region)
|
||||
q.put("district_name", name)
|
||||
q.put("weekend", weekend)
|
||||
def obj = DIST_STAT_COLL.findOne(q)
|
||||
return obj;
|
||||
}
|
||||
|
||||
def findBadCaseCount(def city, def weekend){
|
||||
def q = new BasicDBObject()
|
||||
q.put("city", city)
|
||||
q.put("weekend", weekend)
|
||||
def count = BAD_CASE_COLL.count(q)
|
||||
return count;
|
||||
}
|
||||
|
||||
def findDealPriceCount(def city, def tradeDate){
|
||||
def q = new BasicDBObject()
|
||||
q.put("city", city)
|
||||
q.put("trade_date", new BasicDBObject('$gte', tradeDate))
|
||||
def count = DEAL_PRICE_COLL.count(q)
|
||||
return count;
|
||||
}
|
||||
|
||||
def calTradeDistCount(def city, def tradeDate, def districts){
|
||||
def q = new BasicDBObject()
|
||||
q.put("city", city)
|
||||
q.put("source","链家在线")
|
||||
q.put("trade_date", new BasicDBObject('$gte', tradeDate))
|
||||
def dbcorsor = DEAL_PRICE_COLL.find(q)
|
||||
|
||||
def distMap = [:]
|
||||
for(def dist :districts){
|
||||
def lists = distMap.get(dist.region)
|
||||
if(!lists){
|
||||
lists = []
|
||||
distMap.put(dist.region, lists);
|
||||
}
|
||||
lists << dist
|
||||
}
|
||||
|
||||
def rets = []
|
||||
for(def obj : dbcorsor){
|
||||
def dist = findIsExistsDB(obj.region, obj.district_name, distMap);
|
||||
if(dist && !rets.contains(dist)){
|
||||
rets << dist
|
||||
}
|
||||
}
|
||||
return rets.size();
|
||||
}
|
||||
|
||||
def findIsExistsDB(def region, def district_name, def distMap){
|
||||
def dist = null;
|
||||
def regionDists = distMap.get(region)
|
||||
for(def rd : regionDists){
|
||||
def alias = rd.alias
|
||||
if(!alias){
|
||||
alias = []
|
||||
}
|
||||
if(!alias.contains(rd.name)){
|
||||
alias << rd.name
|
||||
}
|
||||
if(alias.contains(district_name)){
|
||||
dist = rd
|
||||
break;
|
||||
}
|
||||
}
|
||||
return dist;
|
||||
}
|
||||
|
||||
|
||||
def calTradeDistDetail(def city, def tradeDate, def districts){
|
||||
def q = new BasicDBObject()
|
||||
q.put("city", city)
|
||||
q.put("trade_date", new BasicDBObject('$gte', tradeDate))
|
||||
def dbcorsor = DEAL_PRICE_COLL.find(q)
|
||||
|
||||
def distMap = [:]
|
||||
for(def dist :districts){
|
||||
def lists = distMap.get(dist.region)
|
||||
if(!lists){
|
||||
lists = []
|
||||
distMap.put(dist.region, lists);
|
||||
}
|
||||
lists << dist
|
||||
}
|
||||
|
||||
def retMap = [:]
|
||||
for(def obj : dbcorsor){
|
||||
def dist = findIsExistsDB(obj.region, obj.district_name, distMap);
|
||||
if(dist){
|
||||
def tradePrices = retMap.get(dist.region+"#"+dist.name)
|
||||
if(!tradePrices){
|
||||
tradePrices = []
|
||||
retMap.put(dist.region+"#"+dist.name, tradePrices)
|
||||
}
|
||||
if(obj.avg_price){
|
||||
tradePrices << obj.avg_price.toInteger()
|
||||
}
|
||||
}
|
||||
}
|
||||
return retMap;
|
||||
}
|
||||
|
||||
|
||||
def calTradeDistDetail2(def city, def tradeDate, def districts){
|
||||
def q = new BasicDBObject()
|
||||
q.put("city", city)
|
||||
q.put("trade_date", new BasicDBObject('$gte', tradeDate))
|
||||
def dbcorsor = DEAL_PRICE_COLL.find(q)
|
||||
|
||||
def distMap = [:]
|
||||
for(def dist :districts){
|
||||
def lists = distMap.get(dist.region)
|
||||
if(!lists){
|
||||
lists = []
|
||||
distMap.put(dist.region, lists);
|
||||
}
|
||||
lists << dist
|
||||
}
|
||||
|
||||
def retMap = [:]
|
||||
for(def obj : dbcorsor){
|
||||
def dist = findIsExistsDB(obj.region, obj.district_name, distMap);
|
||||
if(dist){
|
||||
def tradePrices = retMap.get(dist.region+"#"+dist.name)
|
||||
if(!tradePrices){
|
||||
tradePrices = []
|
||||
retMap.put(dist.region+"#"+dist.name, tradePrices)
|
||||
}
|
||||
if(obj.avg_price){
|
||||
tradePrices << obj.avg_price.toInteger()
|
||||
}
|
||||
}
|
||||
}
|
||||
return retMap;
|
||||
}
|
||||
}
|
||||
+111
@@ -0,0 +1,111 @@
|
||||
package com.fangjia.script.gj.test
|
||||
|
||||
import com.fangjia.core.RegionData
|
||||
import com.fangjia.script.gj.district.SeaweedDataMatchProcess
|
||||
import com.fangjia.script.gj.paimai.GaoDeUtils
|
||||
import com.fangjia.script.util.CommonUtils
|
||||
import com.fangjia.script.util.CosineSimilarAlgorithm
|
||||
import com.fangjia.script.util.ExcelUtils
|
||||
import com.fangjia.script.yinjihaun.amap.GaoDeApiUtils
|
||||
|
||||
import java.util.regex.Pattern
|
||||
|
||||
class CompanyAddressCompare {
|
||||
static main(args) {
|
||||
def task = new CompanyAddressCompare()
|
||||
task.run()
|
||||
}
|
||||
def run(){
|
||||
ExcelUtils.modify("D:\\数据\\公司及地址\\公司及地址.xlsx",{row->
|
||||
if(row['D'] && !row['I'] && row['B'].contains("宁波甬城农村商业银行")){
|
||||
def city = CommonUtils.parseCityInAddress(row["D"])
|
||||
if(!city){
|
||||
city = CommonUtils.parseCityInAddress(row["B"])
|
||||
}
|
||||
if(city){
|
||||
def similarData = null
|
||||
// if(isAddress(row["D"])){
|
||||
def simpleAddress = row["D"].toString().replace("中国(上海)自由贸易试验区","").replaceAll(/\\d+层/,"").replaceAll(/[一二三四五六]楼/,"")
|
||||
if(simpleAddress && simpleAddress.contains("号")){
|
||||
simpleAddress = simpleAddress.substring(0, simpleAddress.lastIndexOf("号")+1)
|
||||
}
|
||||
def pois = GaoDeUtils.search(simpleAddress, city, null)
|
||||
similarData = findSimilarDatas(pois, row["B"], simpleAddress, city, null)
|
||||
println row['C']+" "+pois.size()
|
||||
// }
|
||||
|
||||
if(similarData){
|
||||
row['F'] = similarData.address
|
||||
row['G'] = similarData.name
|
||||
row['H'] = similarData.score
|
||||
}
|
||||
}
|
||||
}
|
||||
}, 0)
|
||||
}
|
||||
|
||||
|
||||
def findSimilarDatas(pois, name, address, city, rMap){
|
||||
def data = null
|
||||
def maxScore = 0
|
||||
|
||||
for(def poi : pois){
|
||||
if(!poi.cityname){
|
||||
def m = getMapRegion(poi.location)
|
||||
if(m){
|
||||
poi.cityname = m.city
|
||||
poi.pname = m.province
|
||||
poi.township = m.township
|
||||
poi.adname = m.district
|
||||
}
|
||||
}
|
||||
if(poi.cityname.contains(city)){
|
||||
def searchName = poi.name
|
||||
def nameScore = CosineSimilarAlgorithm.getSimilarityRatio(searchName, name);
|
||||
def addressScore = 0
|
||||
if(poi.address){
|
||||
def poiAddress = poi.address.replace("号楼","号")
|
||||
addressScore = CosineSimilarAlgorithm.getSimilarityRatio(poiAddress, address);
|
||||
if(poi.address.contains("号") && address.contains("号") && (poiAddress.contains(address) || address.contains(poiAddress))){
|
||||
addressScore = addressScore + 0.5
|
||||
}
|
||||
}
|
||||
|
||||
def totalScore = nameScore + addressScore
|
||||
if(totalScore > maxScore){
|
||||
maxScore = totalScore
|
||||
data = poi
|
||||
data.score = totalScore
|
||||
}
|
||||
println searchName+" "+name+" "+totalScore + " "+ (poi.address?poi.address:"")
|
||||
}
|
||||
}
|
||||
return data;
|
||||
}
|
||||
|
||||
def isAddress(address){
|
||||
def REG_ADDR1 = "[\u4e00-\u9fa5]{2,20}[路|巷|街|弄|街道|大道|大街|巷子|公路|马路|大道西段]\\d+";
|
||||
def pattern = Pattern.compile(REG_ADDR1);
|
||||
def matcher = pattern.matcher(address)
|
||||
if(matcher.find()){
|
||||
return true;
|
||||
}
|
||||
return false;
|
||||
}
|
||||
|
||||
def getMapRegion(def location){
|
||||
def jsonObj = GaoDeApiUtils.geocode(location)
|
||||
def district = null
|
||||
if(jsonObj && jsonObj.status=="1"){
|
||||
def detailMap = jsonObj.regeocode.addressComponent
|
||||
if(!detailMap.city){
|
||||
if(detailMap.province && detailMap.province.toString().endsWith("市")){
|
||||
detailMap.city = detailMap.province
|
||||
}
|
||||
}
|
||||
return detailMap
|
||||
}
|
||||
return district;
|
||||
}
|
||||
|
||||
}
|
||||
+102
@@ -0,0 +1,102 @@
|
||||
package com.fangjia.script.gj.test
|
||||
|
||||
import com.fangjia.core.RegionData
|
||||
import com.fangjia.script.thirdPriceCrawl.CrawalDBUtil
|
||||
import com.fangjia.script.util.MongoUtils
|
||||
import com.mongodb.BasicDBObject
|
||||
import com.mongodb.DBCollection
|
||||
|
||||
class CrawlDistrictIdSelect {
|
||||
def static final DBCollection thirdPartyColl = CrawalDBUtil.getThirdPartyCol();
|
||||
|
||||
static DBCollection districtseedsColl = MongoUtils.getCollectionByAuth20190102("seeddb", "district_seeds")
|
||||
static DBCollection districtseeds2Coll = MongoUtils.getCollectionByAuth20190102("seeddb_dev", "district_seeds")
|
||||
|
||||
static main(args) {
|
||||
def task = new CrawlDistrictIdSelect()
|
||||
task.run()
|
||||
// task.synch()
|
||||
}
|
||||
|
||||
def synch(){
|
||||
def q = new BasicDBObject()
|
||||
def dbcursor = districtseeds2Coll.find(q)
|
||||
def list = []
|
||||
for(def d :dbcursor){
|
||||
list << d
|
||||
if(list.size()==10000){
|
||||
districtseedsColl.insert(list)
|
||||
list = []
|
||||
}
|
||||
}
|
||||
|
||||
if(list.size()>0){
|
||||
districtseedsColl.insert(list)
|
||||
}
|
||||
}
|
||||
|
||||
def run(){
|
||||
def datas = findCrawlData()
|
||||
datas.each {k,v->
|
||||
if(v){
|
||||
for(def d : v){
|
||||
d.c_date = new Date()
|
||||
d.m_date = new Date()
|
||||
districtseedsColl.save(d)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
def findCrawlData(){
|
||||
def q = new BasicDBObject()
|
||||
q.put("city","福州")
|
||||
q.put("source", "安居客")
|
||||
q.put("s_date",new BasicDBObject('$gte', 202001))
|
||||
def dbcursor = thirdPartyColl.find(q)
|
||||
def retMap = [:]
|
||||
def prefix = "/community/view/"
|
||||
def keyList =[]
|
||||
for(def db : dbcursor){
|
||||
def url = null
|
||||
if(!db.url){
|
||||
if(db.district_id){
|
||||
url = db.district_id
|
||||
}
|
||||
}else{
|
||||
url = db.url
|
||||
}
|
||||
if(url && url.contains(prefix) ){
|
||||
def district_id = url.substring(url.indexOf(prefix)+prefix.length(),url.size())
|
||||
if(district_id=="0"){
|
||||
continue
|
||||
}
|
||||
def shortCity = url.substring(url.indexOf("https://")+8, url.indexOf(".anjuke.com"))
|
||||
def region = RegionData.formatRegion(db.city, db.region)
|
||||
def key = shortCity +"#"+district_id
|
||||
|
||||
def ids = retMap.get(key)
|
||||
if(!ids){
|
||||
ids = []
|
||||
retMap.put(key, ids)
|
||||
}
|
||||
|
||||
if(!keyList.contains(key)){
|
||||
def map = new BasicDBObject()
|
||||
map.root = "anjuke.com"
|
||||
map.city_short= shortCity
|
||||
map.city = db.city
|
||||
map.region = region
|
||||
map.district_id = district_id
|
||||
map.source_url = url
|
||||
ids << map
|
||||
|
||||
keyList << key
|
||||
}
|
||||
}else{
|
||||
println url +" 链接不合要求"
|
||||
}
|
||||
}
|
||||
return retMap
|
||||
}
|
||||
}
|
||||
+101
@@ -0,0 +1,101 @@
|
||||
package com.fangjia.script.gj.test
|
||||
|
||||
import com.fangjia.script.util.DateUtils
|
||||
import com.fangjia.script.util.MongoUtils
|
||||
import com.mongodb.BasicDBObject
|
||||
import com.mongodb.DBCollection
|
||||
import com.mongodb.QueryOperators
|
||||
|
||||
import java.util.concurrent.CountDownLatch
|
||||
import java.util.concurrent.ExecutorService
|
||||
import java.util.concurrent.Executors
|
||||
import java.util.concurrent.atomic.AtomicInteger
|
||||
|
||||
class ExportToDealPrice {
|
||||
|
||||
DBCollection DEAL_PRICE_MATCH_COLL = MongoUtils.getCollectionByName("fangjia", "deal_price_match", "192.168.0.43")
|
||||
DBCollection DISTRICT_STAT_COLL = MongoUtils.getCollectionByName("fangjia_stat", "district_stat", "192.168.0.43")
|
||||
|
||||
DBCollection TRADE_MATCH_PRICE_COLL = MongoUtils.getCollectionByName("fangjia", "trade_match_price", "192.168.0.136")
|
||||
|
||||
public static ExecutorService fixedThreadPool = Executors.newFixedThreadPool(50)
|
||||
final AtomicInteger count = new AtomicInteger()
|
||||
|
||||
static main(args) {
|
||||
def task = new ExportToDealPrice();
|
||||
task.run()
|
||||
}
|
||||
|
||||
def run(){
|
||||
def datas = findDatas()
|
||||
|
||||
final CountDownLatch latch = new CountDownLatch(datas.size())
|
||||
for(def i=0;i<datas.size();i++){
|
||||
final def data = datas[i]
|
||||
fixedThreadPool.execute(new Runnable() {
|
||||
void run() {
|
||||
try{
|
||||
count.addAndGet(1)
|
||||
def dbo = addPrice(data)
|
||||
if(dbo){
|
||||
data.weekend_avg_price = dbo.avg_price
|
||||
}
|
||||
}catch(Exception e){
|
||||
e.printStackTrace()
|
||||
} finally {
|
||||
latch.countDown()
|
||||
}
|
||||
|
||||
}
|
||||
});
|
||||
}
|
||||
latch.await()
|
||||
|
||||
save(datas);
|
||||
|
||||
}
|
||||
|
||||
def save(def datas){
|
||||
def ret = [];
|
||||
def index = 0;
|
||||
for(def dbo : datas){
|
||||
ret << dbo;
|
||||
index++;
|
||||
if(ret.size()%10000==0){
|
||||
println "µÚ "+(index/10000)+" ¸ö"+ret.size()
|
||||
TRADE_MATCH_PRICE_COLL.insert(ret);
|
||||
ret = [];
|
||||
}
|
||||
}
|
||||
if(ret.size()>0){
|
||||
TRADE_MATCH_PRICE_COLL.insert(ret);
|
||||
}
|
||||
}
|
||||
|
||||
def addPrice(data){
|
||||
def weekend = DateUtils.getFirstDayOfWeek(data.trade_date)
|
||||
def q = new BasicDBObject()
|
||||
q.put("city", data.match_city)
|
||||
q.put("region", data.match_region)
|
||||
q.put("district_name", data.match_name)
|
||||
q.put("weekend", weekend)
|
||||
def dbo = DISTRICT_STAT_COLL.findOne(q)
|
||||
if(!dbo){
|
||||
q.put("weekend", new BasicDBObject(QueryOperators.LTE, weekend))
|
||||
dbo = DISTRICT_STAT_COLL.findOne(q)
|
||||
}
|
||||
println count.get()+" "+data.city +" "+data.match_name
|
||||
return dbo
|
||||
}
|
||||
|
||||
def findDatas(){
|
||||
def q = new BasicDBObject()
|
||||
q.put("trade_date", new BasicDBObject(QueryOperators.GTE, DateUtils.parseFromTightByPattern("2020-04-01","yyyy-MM-dd")))
|
||||
def dbcursor = DEAL_PRICE_MATCH_COLL.find(q)
|
||||
def rets = []
|
||||
for(def db : dbcursor){
|
||||
rets << db
|
||||
}
|
||||
return rets;
|
||||
}
|
||||
}
|
||||
@@ -1,3 +1,7 @@
|
||||
package com.fangjia.script.gj.test
|
||||
|
||||
println "wwww"
|
||||
import java.text.SimpleDateFormat
|
||||
|
||||
SimpleDateFormat dateFormat = new SimpleDateFormat("yyyy-MM-dd");
|
||||
Date date = new Date(1617211269868);
|
||||
print dateFormat.format(date);
|
||||
+156
@@ -0,0 +1,156 @@
|
||||
package com.fangjia.script.temp.shenjian
|
||||
|
||||
import com.fangjia.core.util.DateUtil
|
||||
import com.fangjia.script.BaseBatch
|
||||
import com.fangjia.script.BatchOwner
|
||||
import com.fangjia.script.Watcher
|
||||
import com.fangjia.script.util.ElasticUtils
|
||||
import com.fangjia.script.util.ExcelUtils
|
||||
import com.fangjia.script.yinjihaun.util.DingDingMessageUtil
|
||||
import org.apache.commons.lang.StringUtils
|
||||
|
||||
/**
|
||||
* 抓取总量监控
|
||||
*
|
||||
* @author shenjian
|
||||
* @since 2021/5/12
|
||||
*/
|
||||
@BatchOwner(Watcher.ShenJian)
|
||||
class SeawaterMonitor extends BaseBatch {
|
||||
|
||||
static final Long expectNumber = 400000;
|
||||
// 运维组 运维警报
|
||||
static final access_token = "2307d2fec8314b3e8b4b83dd1d42da2e2051638e3e668f21e77ca01258bf682c"
|
||||
// 重点城市
|
||||
static final city = ["北京","唐山","天津","秦皇岛","石家庄","包头","太原","丹东","呼和浩特","锦州",
|
||||
"沈阳","吉林","大连","牡丹江","长春","无锡","哈尔滨","扬州","上海","徐州",
|
||||
"南京","温州","杭州","金华","宁波","蚌埠","合肥","安庆","福州","泉州",
|
||||
"厦门","九江","南昌","赣州","济南","烟台","青岛","济宁","郑州","洛阳",
|
||||
"武汉","平顶山","长沙","宜昌","广州","襄阳","深圳","岳阳","南宁","常德",
|
||||
"海口","惠州","重庆","湛江","成都","韶关","贵阳","桂林","昆明","北海",
|
||||
"西安","三亚","兰州","泸州","西宁","南充","银川","遵义","乌鲁木齐","大理",
|
||||
"东莞","佛山","苏州","珠海","中山","台州","绍兴","南通","常州","嘉兴",
|
||||
"淄博","株洲","镇江","肇庆","漳州","盐城","芜湖","泰州","汕头","清远",
|
||||
"马鞍山","柳州","连云港","廊坊","淮安","湖州","沧州","保定","张家口","日照",
|
||||
"拉萨","大同","文昌","陵水","保亭"];
|
||||
// 重点网站
|
||||
static final site = ["lianjia.com", "ke.com", "58.com", "zhuge.com", "fang.com"]
|
||||
|
||||
static void main(String[] args) {
|
||||
runBatch(new SeawaterMonitor(), args)
|
||||
}
|
||||
|
||||
/**
|
||||
* 监控前一日抓取量
|
||||
* @return
|
||||
*/
|
||||
static def monitorYesterdayCrawlNumber(def startDate, def endDate) {
|
||||
def total
|
||||
String sql = "select count(*) as total from seawater where m_date >= '${startDate}' and m_date < '${endDate}' "
|
||||
ElasticUtils.executeSqlQuery(sql,{data->
|
||||
total = data.total
|
||||
})
|
||||
if (total < expectNumber) {
|
||||
return "【总量抓取异常】 抓取量为" + total + ", 低于预期值" + expectNumber
|
||||
}
|
||||
return ""
|
||||
}
|
||||
|
||||
/**
|
||||
* 统计重点城市近半月抓取量
|
||||
* @return
|
||||
*/
|
||||
static boolean countCityCrawlNumber(def startDate, def endDate) {
|
||||
def result = []
|
||||
result << ['城市', '抓取量']
|
||||
// 统计近半个月各重点城市抓取量
|
||||
def cityFormatted = city.join("','")
|
||||
String sql = "select * from (select count(*) as total, city from seawater where m_date >= '${startDate}' and m_date < '${endDate}' and city in('${cityFormatted}') group by city) order by total desc "
|
||||
ElasticUtils.executeSqlQuery(sql,{data->
|
||||
result << [data.city, data.total]
|
||||
})
|
||||
ExcelUtils.write("F:\\数据\\重点城市抓取\\近半月重点城市抓取量.xlsx", result, "抓取量")
|
||||
}
|
||||
|
||||
/**
|
||||
* 统计前一日重点城市抓取量,昨天为零则警报
|
||||
* @return
|
||||
*/
|
||||
static def monitorCityCrawlNumber(def startDate, def endDate) {
|
||||
def msg = "【重点城市抓取异常】"
|
||||
def set = new HashSet()
|
||||
def cityFormatted = city.join("','")
|
||||
String sql = "select * from " +
|
||||
" (select count(*) as total, city from seawater where m_date >= '${startDate}' and m_date < '${endDate}' and city in('${cityFormatted}') group by city) where total>0 "
|
||||
//" order by total "
|
||||
ElasticUtils.executeSqlQuery(sql,{data->
|
||||
set.add(data.city)
|
||||
})
|
||||
def unNormal = []
|
||||
for (def info : city) {
|
||||
if (!set.contains(info)) {
|
||||
unNormal.add(info)
|
||||
}
|
||||
}
|
||||
msg += unNormal.join("、") + " 抓取量为0"
|
||||
if (unNormal.size() > 0) {
|
||||
return msg
|
||||
}
|
||||
return ""
|
||||
}
|
||||
|
||||
/**
|
||||
* 统计重点网站抓取情况,近一周为0则警报
|
||||
*/
|
||||
static def monitorSiteCrawlNumber(def lastWeekDate, def endDate) {
|
||||
def msg = "【重点网站抓取异常】"
|
||||
def set = new HashSet()
|
||||
def siteFormatted = site.join("','")
|
||||
String sql = "select * from " +
|
||||
" (select count(*) as total, root from seawater where m_date >= '${lastWeekDate}' and m_date < '${endDate}' and root in('${siteFormatted}') group by root) where total > 0 "
|
||||
//" order by total "
|
||||
ElasticUtils.executeSqlQuery(sql,{data->
|
||||
set.add(data.root)
|
||||
})
|
||||
def unNormal = []
|
||||
for (def info : site) {
|
||||
if (!set.contains(info)) {
|
||||
unNormal.add(info)
|
||||
}
|
||||
}
|
||||
msg += unNormal.join("、") + " 近一周抓取量为0"
|
||||
if (unNormal.size() > 0) {
|
||||
return msg
|
||||
}
|
||||
return ""
|
||||
}
|
||||
|
||||
@Override
|
||||
void process(Object args) {
|
||||
def lastWeekDate = DateUtil.add(DateUtil.truncateDay(new Date()), -7).getTime()
|
||||
def yesterday = DateUtil.formatToYYYYMMDD(DateUtil.add(DateUtil.truncateDay(new Date()), -1))
|
||||
def startDate = DateUtil.add(DateUtil.truncateDay(new Date()), -1).getTime();
|
||||
def endDate = DateUtil.truncateDay(new Date()).getTime()
|
||||
|
||||
def result = []
|
||||
def msg = monitorYesterdayCrawlNumber(startDate, endDate)
|
||||
if (StringUtils.isNotBlank(msg)) {
|
||||
result.add(msg)
|
||||
}
|
||||
|
||||
msg = monitorCityCrawlNumber(startDate, endDate)
|
||||
if (StringUtils.isNotBlank(msg)) {
|
||||
result.add(msg)
|
||||
}
|
||||
|
||||
msg = monitorSiteCrawlNumber(lastWeekDate, endDate)
|
||||
if (StringUtils.isNotBlank(msg)) {
|
||||
result.add(msg)
|
||||
}
|
||||
if (result.size() > 0) {
|
||||
msg = "【日期】 " + yesterday + "\n" + result.join("\n")
|
||||
println(msg)
|
||||
DingDingMessageUtil.sendTextMessage(access_token, msg)
|
||||
}
|
||||
}
|
||||
}
|
||||
+5
@@ -25,6 +25,7 @@ class CrawalDBUtil {
|
||||
static DBCollection data_process_col = MongoUtils.getCollectionByAuth20190102("fangjia_craw", "data_process")
|
||||
static DBCollection average_statistics_col = MongoUtils.getCollectionByAuth20190102("fangjia_craw", "average_statistics")
|
||||
static DBCollection map_district_col = MongoUtils.getCollectionByAuth20190102("fangjia_craw", "map_district_process")
|
||||
static DBCollection districtDetailColl = MongoUtils.getCollectionByAuth20190102("fangjia_craw", "district_detail")
|
||||
|
||||
public static getSeedsCol() {
|
||||
return seedsColl;
|
||||
@@ -67,6 +68,10 @@ class CrawalDBUtil {
|
||||
return map_district_col;
|
||||
}
|
||||
|
||||
public static getDistrictDetailColl(){
|
||||
return districtDetailColl;
|
||||
}
|
||||
|
||||
static def updateSeedsCount(def id, def count) {
|
||||
|
||||
DBObject uquery = new BasicDBObject()
|
||||
|
||||
+1
-1
@@ -61,7 +61,7 @@ class DataStandardBatch extends BaseBatch{
|
||||
def static final SHORT_DATE = Utils.getLastMonth()
|
||||
def static final statusMap = ["城市未匹配":1,"区域未匹配":2,"小区匹配失败":3,"小区匹配成功":4,"异常错误":5, "无价格":6]
|
||||
|
||||
def static final websites = ["房估估","诸葛找房","安居客","搜房","中原地产","赶集", "链家","兔博士","城市房产","贝壳找房","Q房","Q房(新)","中原地产(新)","我爱我家","麦田","房途","C21","乐居","房产超市","58同城","365淘房"]
|
||||
def static final websites = ["房估估","诸葛找房","安居客","搜房","中原地产","赶集", "链家","兔博士","城市房产","贝壳找房","Q房","Q房(新)","中原地产(新)","我爱我家","麦田","房途","C21","乐居","房产超市","58同城","365淘房","吉屋"]
|
||||
public static ExecutorService fixedThreadPool = Executors.newFixedThreadPool(20)
|
||||
static {
|
||||
thirdPartyColl = CrawalDBUtil.getThirdPartyCol();
|
||||
|
||||
+1
-1
@@ -30,7 +30,7 @@ class ThirdDataToGatherBatch extends BaseBatch{
|
||||
"房估估":"fanggugu_esf","诸葛找房":"zhugezhaofang_esf","安居客":"anjuke_esf","搜房":"soufang_esf","中原地产":"zhongyuan_esf","赶集":"ganji_esf",
|
||||
"链家":"lianjia_esf","兔博士":"tuboshi_esf","城市房产":"cityhouse_esf","贝壳找房":"beike_esf","Q房":"qfang_esf","Q房(新)":"qfang_new","中原地产(新)":"zhongyuan_new",
|
||||
"乐有家":"leyoujia_esf","乐有家(新)":"leyoujia_new","房探":"fangtan_esf","房多多":"fangdoudou_esf","小资家":"xiaozijia_esf","我爱我家":"5i5j_esf","麦田":"maitian_esf",
|
||||
"房途":"fangtoo_esf","C21":"c21_esf","乐居":"leju_esf","房产超市":"fccs_esf","58同城":"58tc_esf","365淘房":"365house_esf"
|
||||
"房途":"fangtoo_esf","C21":"c21_esf","乐居":"leju_esf","房产超市":"fccs_esf","58同城":"58tc_esf","365淘房":"365house_esf","吉屋":"jiwu_esf"
|
||||
]
|
||||
public static ExecutorService fixedThreadPool = Executors.newFixedThreadPool(2)
|
||||
|
||||
|
||||
+52
@@ -0,0 +1,52 @@
|
||||
package com.fangjia.script.thirdPriceCrawl.other
|
||||
|
||||
import com.fangjia.script.util.ABuYunUtil
|
||||
import org.apache.commons.io.IOUtils
|
||||
import org.apache.http.HttpResponse
|
||||
import org.apache.http.client.HttpClient
|
||||
import org.apache.http.client.methods.HttpGet
|
||||
import org.apache.http.impl.client.HttpClients
|
||||
|
||||
class AnjukeCrawl {
|
||||
|
||||
static main(args) {
|
||||
def task = new AnjukeCrawl();
|
||||
task.run()
|
||||
}
|
||||
|
||||
def run(){
|
||||
HttpClient client = HttpClients.createDefault();
|
||||
HttpGet get = new HttpGet("https://m.anjuke.com/cityList/");
|
||||
// get.setHeader("Upgrade-Insecure-Requests", "1");
|
||||
// get.setHeader("User-Agent", "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/70.0.3538.110 Safari/537.36");
|
||||
|
||||
// get.setHeader("Host","m.anjuke.com");
|
||||
// get.setHeader("Connection","keep-alive");
|
||||
// get.setHeader("Cache-Control","max-age=0");
|
||||
// get.setHeader('sec-ch-ua','"Chromium";v="88", "Google Chrome";v="88", ";Not A Brand";v="99"');
|
||||
// get.setHeader("sec-ch-ua-mobile","'?0");
|
||||
// get.setHeader("Upgrade-Insecure-Requests","1");
|
||||
def header = [:]
|
||||
header.put("User-Agent","Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/88.0.4324.104 Safari/537.36");
|
||||
header.put("Accept","text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.9");
|
||||
// get.setHeader("Sec-Fetch-Site","none");
|
||||
// get.setHeader("Sec-Fetch-Mode","navigate");
|
||||
// get.setHeader("Sec-Fetch-User","?1");
|
||||
// get.setHeader("Sec-Fetch-Dest","document");
|
||||
//// get.setHeader("Accept-Encoding"," gzip, deflate, br");
|
||||
// get.setHeader("Accept-Language","zh-CN,zh;q=0.9");
|
||||
|
||||
|
||||
def s1 = ABuYunUtil.doGet("https://m.anjuke.com/cityList/", null, header, "utf-8")
|
||||
printl s1
|
||||
|
||||
HttpResponse response = client.execute(get);
|
||||
def s = getResponseBodyAsString(response)
|
||||
println s
|
||||
}
|
||||
|
||||
private static String getResponseBodyAsString(HttpResponse response) throws IOException {
|
||||
return IOUtils.toString(response.getEntity().getContent(), "UTF-8");
|
||||
}
|
||||
|
||||
}
|
||||
@@ -309,6 +309,31 @@ public class OpenApiClient {
|
||||
return null
|
||||
}
|
||||
|
||||
|
||||
public static JSONObject addressStandard(String city, String address) {
|
||||
if (StringUtils.isBlank(city) || StringUtils.isBlank(address)) {
|
||||
throw new IllegalArgumentException();
|
||||
}
|
||||
|
||||
try {
|
||||
Map<String, String> params = new HashMap<String, String>();
|
||||
params.put("city", city);
|
||||
params.put("address", address);
|
||||
params.put("token", TOKEN);
|
||||
String content = new HttpRequest().get("http://open.fangjia.com/address/standard", params, "utf-8")
|
||||
JSONObject result = JSONObject.fromObject(content)
|
||||
if (result.getInt("code") == 200) {
|
||||
return result.getJSONObject("result");
|
||||
} else if (result.getInt("code") == 500) {
|
||||
println("${city} ${address} ${result}")
|
||||
}
|
||||
} catch (Exception e) {
|
||||
println("${city} ${address} ${e.getMessage()}")
|
||||
e.printStackTrace();
|
||||
}
|
||||
return null
|
||||
}
|
||||
|
||||
public static JSONObject evaluateByGeo(String city, def lat, def lng, String buildDate, String estateType, String geoType) {
|
||||
try {
|
||||
Map<String, String> params = new HashMap<String, String>();
|
||||
@@ -1214,4 +1239,6 @@ public class OpenApiClient {
|
||||
|
||||
return null;
|
||||
}
|
||||
|
||||
|
||||
}
|
||||
|
||||
@@ -4,36 +4,79 @@ import com.fangjia.core.ProvinceData
|
||||
import com.fangjia.script.util.CommonUtils
|
||||
import com.fangjia.script.util.DataUtil
|
||||
import com.fangjia.script.util.DbUtils
|
||||
import com.fangjia.script.util.ElasticUtils
|
||||
import com.fangjia.script.util.ExcelUtils
|
||||
import com.fangjia.script.util.MongoUtils
|
||||
import com.fangjia.script.yinjihaun.amap.GaoDeApiUtils
|
||||
import com.mongodb.BasicDBObject
|
||||
import com.mongodb.DBCollection
|
||||
import net.sf.json.JSONObject
|
||||
import org.apache.commons.lang.StringUtils
|
||||
//DBCollection SEAWEED_TOWNSHIP = MongoUtils.getCollectionByName("fangjia_base", "seaweed_township", "192.168.0.235", 27018)
|
||||
//def count = 0
|
||||
//List<List<String>> data = Collections.synchronizedList([])
|
||||
//data << ["ID","城市","区域","小区名","经度","纬度","街道"]
|
||||
//
|
||||
//CommonUtils.findAllCity().each {def city->
|
||||
// DbUtils.multiThreadingExecuteQueryOnSeaweed(DbUtils.buildSeaweedQuery(city),{BasicDBObject item->
|
||||
// if(StringUtils.isBlank(item.township) && item.lng){
|
||||
// count++
|
||||
// def lng = item.lng/10000000000d
|
||||
// def lat = item.lat/10000000000d
|
||||
// def location = "${lng},${lat}"
|
||||
// JSONObject resp = GaoDeApiUtils.geocode(location)
|
||||
// def row = []
|
||||
// row << item._id.toString()
|
||||
// row << item.city
|
||||
// row << item.region
|
||||
// row << item.name
|
||||
// row << lng
|
||||
// row << lat
|
||||
// BasicDBObject dbo = new BasicDBObject()
|
||||
// dbo.id = item._id.toString()
|
||||
// dbo.city = item.city
|
||||
// dbo.region = item.region
|
||||
// dbo.name = item.name
|
||||
// dbo.lng = lng
|
||||
// dbo.lat = lat
|
||||
// if(resp.regeocode && resp.regeocode.addressComponent && resp.regeocode.addressComponent.township){
|
||||
// row << resp.regeocode.addressComponent.township
|
||||
// dbo.township = resp.regeocode.addressComponent.township
|
||||
// }
|
||||
// println(row)
|
||||
// data << row
|
||||
// SEAWEED_TOWNSHIP.insert(dbo)
|
||||
// }
|
||||
// })
|
||||
//}
|
||||
//ExcelUtils.write("D:\\街道补充数据.xlsx", data, "补充数据")
|
||||
|
||||
//ExcelUtils.modify("D:\\街道补充数据.xlsx",{List item ->
|
||||
// if(item["G"] && item["H"] == null){
|
||||
// DbUtils.seaweed.update(new BasicDBObject("_id", item["A"]), new BasicDBObject('$set', new BasicDBObject('township', item["G"])))
|
||||
// }
|
||||
// println(item)
|
||||
//})
|
||||
|
||||
//ExcelUtils.multiThreadingModify("D:\\Users\\wenyan\\Desktop\\Êý¾Ý.xlsx",{List item->
|
||||
// if(DbUtils.seaweed.findOne(DbUtils.buildSeaweedQuery(item["A"], item["B"], item["C"]))){
|
||||
// item["D"] = "´æÔÚ"
|
||||
// }else{
|
||||
// item["D"] = "²»´æÔÚ"
|
||||
//ExcelUtils.multiThreadingModify("D:\\Users\\wenyan\\Downloads\\错误经纬度.xlsx",{List item->
|
||||
// def location = "${item["A"]},${item["B"]}"
|
||||
// JSONObject resp = GaoDeApiUtils.geocode(location)
|
||||
// if(resp.regeocode && resp.regeocode.formatted_address){
|
||||
// item['G'] = resp.regeocode.formatted_address
|
||||
// println(item)
|
||||
// }
|
||||
//})
|
||||
|
||||
//def count = 0
|
||||
//CommonUtils.findAllCity().each {def city->
|
||||
// DbUtils.executeQueryOnSeaweed(DbUtils.buildSeaweedQuery(city),{BasicDBObject item->
|
||||
// if(item.township){
|
||||
// count++
|
||||
// }
|
||||
// }, new BasicDBObject("township", 1))
|
||||
//}
|
||||
//println(count)
|
||||
|
||||
DBCollection DISTRICT_STAT = MongoUtils.getCollectionByName("fangjia_stat", "district_stat","mongo.db.fangjia.com")
|
||||
ProvinceData.getAllCity("¸£½¨").each {def city->
|
||||
int count = 0
|
||||
def s = new TreeMap()
|
||||
DISTRICT_STAT.find(new BasicDBObject("weekend", Date.parse("yyyy-MM-dd", "2020-04-19")).append("city", city)).each {BasicDBObject stat->
|
||||
count ++
|
||||
int credit = (stat.credit * 100)/10
|
||||
s[credit] = s.get(credit, 0) + 1
|
||||
}
|
||||
|
||||
def data = []
|
||||
data << city
|
||||
data << count
|
||||
for(int i=0; i <=10; i++){
|
||||
data << s.get(i, 0)
|
||||
}
|
||||
println(data.join(","))
|
||||
}
|
||||
ElasticUtils.executeSqlQuery("select count(*), p_date from seawater where p_date > 20210501 group by p_date",{
|
||||
println(it)
|
||||
})
|
||||
@@ -0,0 +1,51 @@
|
||||
package com.fangjia.script.wenyan.test
|
||||
|
||||
import com.aliyuncs.DefaultAcsClient
|
||||
import com.aliyuncs.IAcsClient
|
||||
import com.aliyuncs.address_purification.model.v20191118.GetAddressSimilarityRequest
|
||||
import com.aliyuncs.address_purification.model.v20191118.GetAddressSimilarityResponse
|
||||
import com.aliyuncs.profile.DefaultProfile
|
||||
import com.fangjia.core.util.HttpRequest
|
||||
import com.fangjia.script.util.ExcelUtils
|
||||
import net.sf.json.JSONObject
|
||||
|
||||
def strt(String addr){
|
||||
|
||||
}
|
||||
|
||||
|
||||
ExcelUtils.modify("D:\\nextcloud\\数据\\地址类服务测试\\建行地址相似度评级测试.xlsx",{List item->
|
||||
def city = item["A"]
|
||||
def address1 = item["B"]
|
||||
def address2 = item["C"]
|
||||
HttpRequest http = new HttpRequest()
|
||||
def params = new HashMap()
|
||||
params["city"] = city
|
||||
params["address_1"] = address1
|
||||
params["address_2"] = address2
|
||||
params["token"]= "F54F52381C49BB9EB4A33EB1B65604AE4B71A28AEE53518A94A2F360408B9056D57553931D15CE6DDE765562DAD286BA38E05A4CDAFC51C3D527A4959BF8E75A3B95DB7108FCEA340DDE61925616DB55118E1851E67D83EAD800460D100D6B667A4ED8EE67C8F7FB"
|
||||
JSONObject json = http.getJsonContent("http://open.fangjia.com/address/similarity", params, "UTF-8")
|
||||
JSONObject result = json.result
|
||||
item["D"] = result?.level
|
||||
item["E"] = result?.formattedAddress_1
|
||||
item["F"] = result?.formattedAddress_2
|
||||
item["G"] = result?.isSameAddress
|
||||
DefaultProfile profile = DefaultProfile.getProfile(
|
||||
"cn-hangzhou",
|
||||
"LTAIdaj4eSO79gzz",
|
||||
"xw6EsvqydZ9HC6zAvIB3ruQqVHynxf");
|
||||
IAcsClient client = new DefaultAcsClient(profile);
|
||||
GetAddressSimilarityRequest request = new GetAddressSimilarityRequest();
|
||||
request.setServiceCode("addrp")
|
||||
request.setAppKey("Xcgtyhupey0h")
|
||||
request.setText("${address1};${address2}")
|
||||
GetAddressSimilarityResponse response = client.getAcsResponse(request)
|
||||
String data = response.getData()
|
||||
json = JSONObject.fromObject(data)
|
||||
item["H"] = json["sim_level"]["level"]
|
||||
item["I"] = json["sim_level"]["address_a"]
|
||||
item["J"] = json["sim_level"]["address_b"]
|
||||
item["K"] = json["sim_level"]["is_same"]
|
||||
println(json)
|
||||
})
|
||||
|
||||
+26
@@ -223,6 +223,32 @@ class GaoDeMapApiData {
|
||||
doSearchByGaoDe(type, lng, lat, radius, page)
|
||||
}
|
||||
}
|
||||
|
||||
def static doSearchByGaoDe(String type, Double lng, Double lat, int radius, int page, int pageSize) {
|
||||
Map<String, String> params = new HashMap<String, String>()
|
||||
params.put("key", GAO_DE_AK)
|
||||
params.put("types", type)
|
||||
params.put("location", Double.toString(lng)+ ","+ Double.toString(lat))
|
||||
params.put("radius",radius)
|
||||
params.put("offset",pageSize)
|
||||
params.put("page",page)
|
||||
String content
|
||||
try {
|
||||
content = new HttpRequest().newGet("http://restapi.amap.com/v3/place/around", params, "utf-8")
|
||||
} catch (Exception e) {
|
||||
content = new HttpRequest().newGet("http://restapi.amap.com/v3/place/around", params, "utf-8")
|
||||
}
|
||||
JSONObject jsonObj = JSONObject.fromObject(content)
|
||||
|
||||
if(jsonObj.get("infocode").equals("10000")){
|
||||
JSONArray pois = jsonObj.get("pois")
|
||||
return pois
|
||||
} else {
|
||||
println jsonObj.get("infocode")+"exit........"
|
||||
doSearchByGaoDe(type, lng, lat, radius, page, pageSize)
|
||||
}
|
||||
return null
|
||||
}
|
||||
|
||||
private synchronized static fzData(JSONArray pois) {
|
||||
pois.each { poi ->
|
||||
|
||||
+145
-37
@@ -1,14 +1,12 @@
|
||||
package com.fangjia.service.address;
|
||||
|
||||
import com.fangjia.core.RegionData;
|
||||
import com.fangjia.core.util.CNNMFilter;
|
||||
import com.fangjia.core.util.DataUtil;
|
||||
import com.fangjia.core.util.GeoUtils;
|
||||
import com.fangjia.core.util.StringUtil;
|
||||
import com.fangjia.core.util.*;
|
||||
import com.fangjia.dal.QueryParam;
|
||||
import com.fangjia.dal.common.DistrictKey;
|
||||
import com.fangjia.dal.common.ServerAreaData;
|
||||
import com.fangjia.dal.mongo.fangjia.model.ServerArea;
|
||||
import com.fangjia.dal.mongo.model.AmapPoi;
|
||||
import com.fangjia.dal.mongo.model.bkoff.HouseLoudong;
|
||||
import com.fangjia.dal.mongo.model.bkoff.Loudong;
|
||||
import com.fangjia.dal.seaweed.dao.SeaweedDao;
|
||||
@@ -25,6 +23,7 @@ import com.fangjia.service.mongo.bkoff.HouseLoudongService;
|
||||
import com.fangjia.service.mongo.bkoff.LoudongService;
|
||||
import com.google.common.base.Function;
|
||||
import com.google.common.collect.MapMaker;
|
||||
import net.sf.json.JSONObject;
|
||||
import org.apache.commons.lang.StringUtils;
|
||||
import org.apache.log4j.Logger;
|
||||
import org.bson.types.ObjectId;
|
||||
@@ -47,7 +46,7 @@ public class AddressParserServiceImpl implements AddressParserService {
|
||||
private LoudongService loudongService;
|
||||
private SeaweedDao seaweedDao;
|
||||
private AliyunService aliyunService;
|
||||
private final String[] MATCH_PROPERTY_FIELDS = new String[]{"id", "city", "region", "name", "address", "category", "block", "alias", "addrAlias", "estateType2", "lat2", "lng2", "streetCommunity"};
|
||||
private final String[] MATCH_PROPERTY_FIELDS = new String[]{"id", "city", "region", "name", "address", "category", "block", "alias", "addrAlias", "estateType2", "lat2", "lng2", "township"};
|
||||
private static final Logger logger = Logger.getLogger(AddressParserServiceImpl.class);
|
||||
|
||||
|
||||
@@ -134,6 +133,8 @@ public class AddressParserServiceImpl implements AddressParserService {
|
||||
formatAddress(building);
|
||||
}
|
||||
return building;
|
||||
/*JSONObject result = getDataFromCRF("test");
|
||||
return new Building();*/
|
||||
}
|
||||
|
||||
@Override
|
||||
@@ -165,6 +166,21 @@ public class AddressParserServiceImpl implements AddressParserService {
|
||||
return building;
|
||||
}
|
||||
Building matchedBuilding;
|
||||
//调用CRF算法切分数据
|
||||
/*JSONObject jsonObj = null;
|
||||
if(params.isUseCrf() && AddressParserGoal.Match.equals(addr.getAddressParserGoal())){
|
||||
jsonObj = getDataFromCRF(addr.getOrgAddress());
|
||||
if(jsonObj != null){
|
||||
String poi = (jsonObj.containsKey("community_model") ? jsonObj.getString("community_model") : "") +
|
||||
(jsonObj.containsKey("poi_model") ? jsonObj.getString("poi_model") : "") +
|
||||
(jsonObj.containsKey("subpoi_model") ? jsonObj.getString("subpoi_model") : "");
|
||||
if(StringUtils.isNotBlank(poi)){
|
||||
if(addr.getKeywords()==null || !addr.getKeywords().contains(poi)){
|
||||
addr.setKeywords(poi);
|
||||
}
|
||||
}
|
||||
}
|
||||
}*/
|
||||
//keywords使用aliyun接口返回的poi
|
||||
AliyunBo aliyunBo = null;
|
||||
if(params.isUseAliyunApi() && AddressParserGoal.Match.equals(addr.getAddressParserGoal())){
|
||||
@@ -176,6 +192,9 @@ public class AddressParserServiceImpl implements AddressParserService {
|
||||
addr.setKeywords(poi);
|
||||
}
|
||||
}
|
||||
if(StringUtils.isNotBlank(aliyunBo.getCity())){
|
||||
addr.setCity(RegionData.formatCity(aliyunBo.getCity()));
|
||||
}
|
||||
if(StringUtils.isNotBlank(addr.getKeywords()) && addr.getOrgAddress().indexOf(addr.getKeywords())>-1){
|
||||
String str = AddressParserUtil.match(Pattern.compile(".*村(.*(组|队))?\\d+(号)?$"), addr.getOrgAddress().substring(addr.getOrgAddress().indexOf(addr.getKeywords()),addr.getOrgAddress().length()));
|
||||
if(StringUtils.isNotBlank(str)){
|
||||
@@ -187,26 +206,80 @@ public class AddressParserServiceImpl implements AddressParserService {
|
||||
}
|
||||
}
|
||||
}
|
||||
if(StringUtils.isNotBlank(aliyunBo.getCity())){
|
||||
addr.setSuggestCity(aliyunBo.getCity());
|
||||
}
|
||||
}
|
||||
}
|
||||
AddressMatchMetaData meta = null;
|
||||
if(StringUtils.isBlank(addr.getCity())){
|
||||
addr.addKeyword(addr.getOrgAddress());
|
||||
meta = getMapMeta(addr);
|
||||
if(meta != null){
|
||||
addr.setProvince(meta.getMapProvince());
|
||||
addr.setCity(meta.getMapCity());
|
||||
addr.setDistrict(meta.getDistrict());
|
||||
}
|
||||
}
|
||||
addr.addKeyword(addr.getOrgAddress());
|
||||
if(params.isUseMap()){
|
||||
matchedBuilding = findMatchedBuilding(addr, params.getBuildingType(), MetaDataType.MAP);
|
||||
}else{
|
||||
matchedBuilding = findMatchedBuilding(addr, params.getBuildingType(), MetaDataType.CPDB);
|
||||
}
|
||||
|
||||
/*if(jsonObj != null){
|
||||
//乡镇优化
|
||||
String township = jsonObj.containsKey("town") ? jsonObj.getString("town") : "";
|
||||
if(StringUtils.isBlank(addr.getTownship()) && StringUtils.isNotBlank(township)){
|
||||
String town = AddressParserUtil.match(AddressParserPatterns.TOWN, township);
|
||||
if(StringUtils.isNotBlank(town)){
|
||||
addr.setTownship(town);
|
||||
}
|
||||
}
|
||||
//道路优化(两边均去掉乡镇)
|
||||
String street = addr.getStreet();
|
||||
String streetNumber = addr.getStreetNumber();
|
||||
String streetCrf = jsonObj.containsKey("road") ? jsonObj.getString("road") : "";
|
||||
String streetNumberCrf = jsonObj.containsKey("roadno") ? jsonObj.getString("roadno") : "";
|
||||
//去掉路名中含有乡镇名的
|
||||
if(StringUtils.isNotBlank(addr.getTownship())){
|
||||
street = StringUtils.isNotBlank(street) ? street.replace(addr.getTownship(),"") : street;
|
||||
streetCrf = StringUtils.isNotBlank(streetCrf) ? streetCrf.replace(addr.getTownship(),"") : streetCrf;
|
||||
}
|
||||
|
||||
//crf解析出的路名为空、与Java解析出路名一致或不满足正则则不处理
|
||||
if(StringUtils.isNotBlank(streetCrf) && !streetCrf.equals(street) && StringUtils.isNotBlank(AddressParserUtil.match(AddressParserPatterns.STREET, streetCrf))){
|
||||
//Java解析出路名为空
|
||||
if(StringUtils.isBlank(street)){
|
||||
street = streetCrf;
|
||||
if(StringUtils.isNotBlank(streetNumberCrf)){
|
||||
streetNumber = streetNumberCrf;
|
||||
}
|
||||
}else{
|
||||
//CRF包含java解析出的路名或Java解析出的路名包含CRF解析出的路名并且多余部分也满足路名正则
|
||||
if(streetCrf.contains(street) || (street.contains(streetCrf) && StringUtils.isBlank(AddressParserUtil.match(AddressParserPatterns.STREET, street.replace(streetCrf,""))))){
|
||||
street = streetCrf;
|
||||
}
|
||||
}
|
||||
addr.setStreet(street);
|
||||
addr.setStreetNumber(streetNumber);
|
||||
}
|
||||
//poi处理
|
||||
//没有匹配上库中小区时
|
||||
if(StringUtils.isBlank(matchedBuilding.getId())){
|
||||
String poi = (jsonObj.containsKey("community") ? jsonObj.getString("community") : "") +
|
||||
(jsonObj.containsKey("poi") ? jsonObj.getString("poi") : "") +
|
||||
(jsonObj.containsKey("subpoi") ? jsonObj.getString("subpoi") : "");
|
||||
//去掉路名中含有乡镇名和路名 getParsedPropertyName
|
||||
if(StringUtils.isNotBlank(addr.getTownship())){
|
||||
poi = poi.replace(addr.getTownship(),"");
|
||||
}
|
||||
if(StringUtils.isNotBlank(addr.getStreet())){
|
||||
poi = poi.replace(addr.getStreet(),"");
|
||||
}
|
||||
addr.setParsedPropertyName(poi);
|
||||
}
|
||||
//楼栋、单元、室号
|
||||
String houseNo = (jsonObj.containsKey("houseno") ? jsonObj.getString("houseno") : "").replaceAll("[\u4E00-\u9FA5]", "");
|
||||
String cellNo = (jsonObj.containsKey("cellno") ? jsonObj.getString("cellno") : "").replaceAll("[\u4E00-\u9FA5]", "");
|
||||
String floorNo = (jsonObj.containsKey("floorno") ? jsonObj.getString("floorno") : "").replaceAll("[\u4E00-\u9FA5]", "");
|
||||
String roomNo = (jsonObj.containsKey("roomno") ? jsonObj.getString("roomno") : "").replaceAll("[\u4E00-\u9FA5]", "");
|
||||
addr.setOrgBuildingNumber(houseNo);
|
||||
addr.setBuildingNumber(houseNo);
|
||||
addr.setOrgUnitNumber(cellNo);
|
||||
addr.setUnitNumber(cellNo);
|
||||
addr.setFloor(floorNo);
|
||||
addr.setOrgRoomNumber(roomNo);
|
||||
addr.setRoomNumber(roomNo);
|
||||
}*/
|
||||
matchedBuilding.setProvince(addr.getProvince());
|
||||
matchedBuilding.setSearchAddress(addr);
|
||||
matchedBuilding.setSuggestName(getSuggestName(matchedBuilding));
|
||||
@@ -215,7 +288,8 @@ public class AddressParserServiceImpl implements AddressParserService {
|
||||
completBuildingField(matchedBuilding, aliyunBo, params);
|
||||
}
|
||||
Address tempAddr = matchedBuilding.getSearchAddress();
|
||||
if(StringUtils.isBlank(tempAddr.getStreet()) && AddressParserGoal.Match.equals(addr.getAddressParserGoal())){
|
||||
if((StringUtils.isBlank(tempAddr.getStreet()) || StringUtils.isBlank(tempAddr.getStreetNumber())) && AddressParserGoal.Match.equals(addr.getAddressParserGoal())){
|
||||
AddressMatchMetaData meta = getMapMeta(addr);
|
||||
if(meta != null && (tempAddr.getKeywords()!=null && tempAddr.getKeywords().equals(meta.getMainName()))){
|
||||
matchedBuilding.setAddress(meta.getAddress());
|
||||
}
|
||||
@@ -255,6 +329,25 @@ public class AddressParserServiceImpl implements AddressParserService {
|
||||
}
|
||||
return null;
|
||||
}
|
||||
|
||||
/**
|
||||
* TODO 省市区乡镇采用原始拆分数据,若乡镇数据没有使用
|
||||
* @param address
|
||||
* @return
|
||||
*/
|
||||
/*private JSONObject getDataFromCRF(String address){
|
||||
HttpRequest request = new HttpRequest();
|
||||
Map<String,String> params = new HashMap<>();
|
||||
params.put("address",address);
|
||||
try {
|
||||
JSONObject resp = request.getJsonContent("http://127.0.0.1:8802/standard", params, "UTF-8");
|
||||
System.out.println(resp);
|
||||
return resp;
|
||||
}catch(Exception e){
|
||||
|
||||
}
|
||||
return null;
|
||||
}*/
|
||||
/**
|
||||
* 信息不全时使用阿里接口补全数据
|
||||
* @param matchedBuilding
|
||||
@@ -271,33 +364,44 @@ public class AddressParserServiceImpl implements AddressParserService {
|
||||
}
|
||||
}
|
||||
Address addr = matchedBuilding.getSearchAddress();
|
||||
if(StringUtils.isBlank(addr.getProvince())){
|
||||
if(StringUtils.isNotBlank(aliyunBo.getProv())){
|
||||
addr.setProvince(formatProvince(aliyunBo.getProv()));
|
||||
}
|
||||
if(StringUtils.isNotBlank(aliyunBo.getCity())){
|
||||
addr.setCity(RegionData.formatCity(aliyunBo.getCity()));
|
||||
}
|
||||
if(StringUtils.isNotBlank(aliyunBo.getDistrict())){
|
||||
addr.setDistrict(RegionData.formatRegion(aliyunBo.getDistrict()));
|
||||
}
|
||||
addr.setTownship(aliyunBo.getTown());
|
||||
|
||||
/*if(StringUtils.isBlank(addr.getProvince())){
|
||||
addr.setProvince(formatProvince(aliyunBo.getProv()));
|
||||
}
|
||||
if(StringUtils.isBlank(addr.getCity())){
|
||||
addr.setCity(RegionData.formatCity(aliyunBo.getCity()));
|
||||
}
|
||||
|
||||
if(StringUtils.isBlank(addr.getDevelopmentZone())){
|
||||
addr.setDevelopmentZone(aliyunBo.getDevzone());
|
||||
}
|
||||
if(StringUtils.isBlank(addr.getDistrict())){
|
||||
addr.setDistrict(RegionData.formatRegion(aliyunBo.getDistrict()));
|
||||
}
|
||||
if(StringUtils.isBlank(addr.getTownship())){
|
||||
addr.setTownship(aliyunBo.getTown());
|
||||
}*/
|
||||
if(StringUtils.isBlank(addr.getDevelopmentZone())){
|
||||
addr.setDevelopmentZone(aliyunBo.getDevzone());
|
||||
}
|
||||
|
||||
/*addr.setOrgBuildingNumber(aliyunBo.getHouseno());
|
||||
addr.setBuildingNumber(aliyunBo.getHouseno());
|
||||
addr.setOrgUnitNumber(aliyunBo.getCellno());
|
||||
addr.setUnitNumber(aliyunBo.getCellno());*/
|
||||
if(aliyunBo.getRoomno()!=null && !aliyunBo.getRoomno().matches(".*(²ã|Â¥)$")){
|
||||
addr.setRoomNumber(aliyunBo.getRoomno());
|
||||
/*if(aliyunBo.getRoomno()!=null && !aliyunBo.getRoomno().matches(".*(层|楼)$")){
|
||||
addr.setOrgRoomNumber(aliyunBo.getRoomno());
|
||||
addr.setRoomNumber(aliyunBo.getRoomno());
|
||||
}else{
|
||||
addr.setRoomNumber(null);
|
||||
addr.setRoomNumber(null);
|
||||
}
|
||||
}*/
|
||||
|
||||
/*String aliRoom = aliyunBo.getRoomno() == null ? null : aliyunBo.getRoomno().replaceAll("[\u4E00-\u9FA5]", "");
|
||||
String orgRoom = addr.getOrgRoomNumber() == null ? null : addr.getOrgRoomNumber().replaceAll("[\u4E00-\u9FA5]", "");
|
||||
@@ -1037,20 +1141,24 @@ public class AddressParserServiceImpl implements AddressParserService {
|
||||
if(StringUtils.isNotBlank(address.getTownship())){
|
||||
formattedAddressBuilder.append(address.getTownship());
|
||||
}
|
||||
if(StringUtils.isNotBlank(building.getAddress())){
|
||||
Address orgAddr = new Address();
|
||||
orgAddr.setOrgAddress(building.getAddress());
|
||||
Address addr = AddressProcessors.process(orgAddr);
|
||||
if(StringUtils.isBlank(address.getStreet())){
|
||||
address.setStreet(addr.getStreet());
|
||||
}
|
||||
if(StringUtils.isBlank(address.getStreetNumber())){
|
||||
address.setStreetNumber(addr.getStreetNumber());
|
||||
}
|
||||
}
|
||||
if(StringUtils.isNotBlank(address.getStreet())){
|
||||
formattedAddressBuilder.append(address.getStreet());
|
||||
}
|
||||
if(StringUtils.isNotBlank(address.getStreet()) && StringUtils.isNotBlank(address.getStreetNumber())){
|
||||
formattedAddressBuilder.append(address.getStreetNumber());
|
||||
}
|
||||
if(StringUtils.isBlank(address.getStreet()) && StringUtils.isNotBlank(building.getAddress())){
|
||||
Address orgAddr = new Address();
|
||||
orgAddr.setOrgAddress(building.getAddress());
|
||||
Address addr = AddressProcessors.process(orgAddr);
|
||||
building.getSearchAddress().setStreet(addr.getStreet());
|
||||
building.getSearchAddress().setStreetNumber(addr.getStreetNumber());
|
||||
formattedAddressBuilder.append((addr.getStreet()==null?"":addr.getStreet())+(addr.getStreetNumber()==null?"":addr.getStreetNumber()));
|
||||
}
|
||||
|
||||
if(StringUtils.isNotBlank(building.getSuggestName())){
|
||||
formattedAddressBuilder.append(building.getSuggestName());
|
||||
}
|
||||
@@ -1077,8 +1185,8 @@ public class AddressParserServiceImpl implements AddressParserService {
|
||||
formattedAddressBuilder.append(address.getUnitNumber() + "单元");
|
||||
}
|
||||
if(StringUtils.isNotBlank(address.getRoomNumber())){
|
||||
formattedAddressBuilder.append(address.getRoomNumber());
|
||||
// formattedAddressBuilder.append(address.getRoomNumber() + "ÊÒ");
|
||||
// formattedAddressBuilder.append(address.getRoomNumber());
|
||||
formattedAddressBuilder.append(address.getRoomNumber() + "室");
|
||||
}
|
||||
address.setFormattedAddress(formattedAddressBuilder.toString());
|
||||
|
||||
|
||||
@@ -77,6 +77,16 @@ public class Address {
|
||||
private String debugName;
|
||||
|
||||
private AliyunBo aliyunBo;
|
||||
//使用CRF切分的地址
|
||||
private String crfAddress;
|
||||
|
||||
public String getCrfAddress() {
|
||||
return crfAddress;
|
||||
}
|
||||
|
||||
public void setCrfAddress(String crfAddress) {
|
||||
this.crfAddress = crfAddress;
|
||||
}
|
||||
|
||||
public AddressParserGoal getAddressParserGoal() {
|
||||
return addressParserGoal;
|
||||
|
||||
+2
-6
@@ -64,9 +64,7 @@ public class Building {
|
||||
this.alias = d.getAlias();
|
||||
this.addrAlias = d.getAddrAlias();
|
||||
this.propertyType = d.getEstateType2();
|
||||
if(StringUtils.isNotBlank(d.getStreetCommunity())){
|
||||
this.township = d.getStreetCommunity();
|
||||
}
|
||||
this.township = d.getTownship();
|
||||
if(d.getLat() != null && d.getLng() != null){
|
||||
this.lat = d.getLat()/10000000000d;
|
||||
this.lng = d.getLng()/10000000000d;
|
||||
@@ -90,9 +88,7 @@ public class Building {
|
||||
this.lat = d.getLat2()/10000000000d;
|
||||
this.lng = d.getLng2()/10000000000d;
|
||||
}
|
||||
if(StringUtils.isNotBlank(d.getStreetCommunity())){
|
||||
this.township = d.getStreetCommunity();
|
||||
}
|
||||
this.township = d.getTownship();
|
||||
}
|
||||
|
||||
public Building() {
|
||||
|
||||
+8
@@ -25,7 +25,15 @@ public class MatchParams {
|
||||
private boolean useMap;
|
||||
/**是否使用阿里云接口*/
|
||||
private boolean useAliyunApi;
|
||||
//是否使用CRF算法
|
||||
private boolean useCrf;
|
||||
public boolean isUseCrf() {
|
||||
return useCrf;
|
||||
}
|
||||
|
||||
public void setUseCrf(boolean useCrf) {
|
||||
this.useCrf = useCrf;
|
||||
}
|
||||
public boolean isContainsLocation() {
|
||||
return containsLocation;
|
||||
}
|
||||
|
||||
+1
@@ -14,6 +14,7 @@ public class StageAndPartitionProcessor extends AbstractAddressProcessor {
|
||||
@Override
|
||||
public RepeatStatus process(Address address) {
|
||||
String orgAddr = address.getEffectiveAddress();
|
||||
address.setCrfAddress(address.getAddress());
|
||||
MatchResult matchResult = matchFirst(AddressParserPatterns.STAGE_PATTERN, address.getEffectiveAddress());
|
||||
if (matchResult != null) {
|
||||
String formatStage = CNNMFilter.convertArabicToCnNumeric(matchResult.matchContent);
|
||||
|
||||
+76
-14
@@ -1,5 +1,7 @@
|
||||
package com.fangjia.service.api;
|
||||
|
||||
import com.aliyuncs.address_purification.model.v20191118.GetAddressDivisionCodeRequest;
|
||||
import com.aliyuncs.address_purification.model.v20191118.GetAddressDivisionCodeResponse;
|
||||
import com.aliyuncs.address_purification.model.v20191118.StructureAddressRequest;
|
||||
import com.aliyuncs.address_purification.model.v20191118.StructureAddressResponse;
|
||||
import com.fangjia.core.util.JsonUtils;
|
||||
@@ -13,6 +15,9 @@ import net.sf.json.JSONObject;
|
||||
import org.apache.commons.lang.StringUtils;
|
||||
import org.slf4j.Logger;
|
||||
|
||||
import java.util.ArrayList;
|
||||
import java.util.List;
|
||||
|
||||
public class AliyunServiceImpl implements AliyunService {
|
||||
|
||||
protected final Logger LOGGER = LoggerFactory.getLogger(this.getClass());
|
||||
@@ -35,20 +40,44 @@ public class AliyunServiceImpl implements AliyunService {
|
||||
if(StringUtils.isNotBlank(city)){
|
||||
request.setDefaultCity(city);
|
||||
}
|
||||
AliyunBo aliyunBo = new AliyunBo();
|
||||
try{
|
||||
StructureAddressResponse response = client.getAcsResponse(request);
|
||||
JSONObject responseJson = JSONObject.fromObject(response.getData());
|
||||
String value = (String) responseJson.get("structure");
|
||||
AliyunBo aliyunBo = new AliyunBo();
|
||||
for (String key:value.split("\t")){
|
||||
String[] arry = key.split("=");
|
||||
wrapField(arry[0],arry[1], aliyunBo);
|
||||
}
|
||||
return aliyunBo;
|
||||
}catch(Exception e){
|
||||
LOGGER.error("", e);
|
||||
return null;
|
||||
LOGGER.error("调用阿里标准化接口失败:", e);
|
||||
}
|
||||
try{
|
||||
GetAddressDivisionCodeRequest divisionRequest = new GetAddressDivisionCodeRequest();
|
||||
divisionRequest.setServiceCode("addrp");
|
||||
divisionRequest.setAppKey(APPKEY);
|
||||
divisionRequest.setText(address);
|
||||
GetAddressDivisionCodeResponse divisionResponse = client.getAcsResponse(divisionRequest);
|
||||
JSONObject responseJson = JSONObject.fromObject(divisionResponse.getData());
|
||||
String value = (String) ((JSONObject) responseJson.get("division_info")).get("division_name");
|
||||
String[] array = value.replace("\t", ";").split(";");
|
||||
if(array.length > 4){
|
||||
List<String> tmp = new ArrayList<String>();
|
||||
for(String str:array){
|
||||
if(str!=null && str.length()!=0){
|
||||
tmp.add(str);
|
||||
}
|
||||
}
|
||||
array= tmp.toArray(new String[4]);
|
||||
}
|
||||
aliyunBo.setProv(array[0]);
|
||||
aliyunBo.setCity(array[1]);
|
||||
aliyunBo.setDistrict(array[2]);
|
||||
aliyunBo.setTown(array[3]);
|
||||
}catch(Exception e){
|
||||
LOGGER.error("调用阿里行政区划识别接口失败", e);
|
||||
}
|
||||
return aliyunBo;
|
||||
}
|
||||
|
||||
private void wrapField(String field,String value, AliyunBo aliyunBo){
|
||||
@@ -81,26 +110,59 @@ public class AliyunServiceImpl implements AliyunService {
|
||||
}
|
||||
}
|
||||
public static void main(String[] args) throws ClientException {
|
||||
DefaultProfile profile = DefaultProfile.getProfile(
|
||||
String value = "";
|
||||
String[] array = value.replace("\t", ";").split(";");
|
||||
if(array.length > 4){
|
||||
List<String> tmp = new ArrayList<String>();
|
||||
for(String str:array){
|
||||
if(str!=null && str.length()!=0){
|
||||
tmp.add(str);
|
||||
}
|
||||
}
|
||||
array= tmp.toArray(new String[4]);
|
||||
}
|
||||
System.out.println(array[0]);
|
||||
System.out.println(array[1]);
|
||||
System.out.println(array[2]);
|
||||
System.out.println(array[3]);
|
||||
|
||||
/*DefaultProfile profile = DefaultProfile.getProfile(
|
||||
REGIONID,
|
||||
ACCESSKEYID,
|
||||
SECRET);
|
||||
IAcsClient client = new DefaultAcsClient(profile);
|
||||
StructureAddressRequest request = new StructureAddressRequest();
|
||||
// StructureAddressRequest request = new StructureAddressRequest();
|
||||
GetAddressDivisionCodeRequest request = new GetAddressDivisionCodeRequest();
|
||||
request.setServiceCode("addrp");
|
||||
request.setAppKey(APPKEY);
|
||||
request.setText("杭州市文一西路969号");
|
||||
request.setDefaultProvince("浙江省");
|
||||
request.setDefaultCity("杭州市");
|
||||
request.setDefaultDistrict("余杭区");
|
||||
StructureAddressResponse response = client.getAcsResponse(request);
|
||||
request.setText("天青路260号");
|
||||
// request.setDefaultProvince("浙江省");
|
||||
// request.setDefaultCity("杭州市");
|
||||
// request.setDefaultDistrict("余杭区");
|
||||
// StructureAddressResponse response = client.getAcsResponse(request);
|
||||
GetAddressDivisionCodeResponse response = client.getAcsResponse(request);
|
||||
JSONObject responseJson = JSONObject.fromObject(response.getData());
|
||||
String value = (String) responseJson.get("structure");
|
||||
JSONObject json = new JSONObject();
|
||||
System.out.println(responseJson);
|
||||
String value = (String) ((JSONObject) responseJson.get("division_info")).get("division_name");
|
||||
String[] array = value.replace("\t", ";").split(";");
|
||||
if(array.length > 4){
|
||||
List<String> tmp = new ArrayList<String>();
|
||||
for(String str:array){
|
||||
if(str!=null && str.length()!=0){
|
||||
tmp.add(str);
|
||||
}
|
||||
}
|
||||
array= tmp.toArray(new String[4]);
|
||||
}
|
||||
System.out.println(array[0]);
|
||||
System.out.println(array[1]);
|
||||
System.out.println(array[2]);
|
||||
System.out.println(array[3]);*/
|
||||
/*JSONObject json = new JSONObject();
|
||||
for (String key:value.split("\t")){
|
||||
String[] arry = key.split("=");
|
||||
json.put(arry[0],arry[1]);
|
||||
}
|
||||
System.out.println(JsonUtils.convertJsonToJavaObj(json, AliyunBo.class));
|
||||
System.out.println(JsonUtils.convertJsonToJavaObj(json, AliyunBo.class));*/
|
||||
}
|
||||
}
|
||||
|
||||
+47
-34
@@ -5,18 +5,7 @@ import java.math.BigDecimal;
|
||||
import java.math.RoundingMode;
|
||||
import java.net.URLEncoder;
|
||||
import java.text.DecimalFormat;
|
||||
import java.util.ArrayList;
|
||||
import java.util.Arrays;
|
||||
import java.util.Collections;
|
||||
import java.util.Comparator;
|
||||
import java.util.Date;
|
||||
import java.util.HashMap;
|
||||
import java.util.Iterator;
|
||||
import java.util.LinkedHashMap;
|
||||
import java.util.LinkedHashSet;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
import java.util.Set;
|
||||
import java.util.*;
|
||||
import java.util.concurrent.Callable;
|
||||
import java.util.concurrent.ExecutionException;
|
||||
import java.util.concurrent.ExecutorService;
|
||||
@@ -5080,7 +5069,7 @@ public class AddressServiceImpl implements AddressService {
|
||||
double similar = AddressParserUtil.getSimilarityRatio(formatAddress1_pinyin, formatAddress2_pinyin);
|
||||
String level = getAddressSimilarityLevel(json1, json2);
|
||||
resultMap.put("isSameAddress", sameAddress);
|
||||
resultMap.put("score", similar);
|
||||
// resultMap.put("score", similar);
|
||||
resultMap.put("level", level);
|
||||
resultMap.put("formattedAddress_1", getFromJson(json1,"formattedAddress"));
|
||||
resultMap.put("formattedAddress_2", getFromJson(json2,"formattedAddress"));
|
||||
@@ -5138,30 +5127,51 @@ public class AddressServiceImpl implements AddressService {
|
||||
return null;
|
||||
}
|
||||
/**
|
||||
* 获取两地址的相似度评级(最小颗粒度)
|
||||
* 获取两地址的相似度评级(相同数据的最小颗粒度)
|
||||
* @return
|
||||
*/
|
||||
private String getAddressSimilarityLevel(JSONObject json1, JSONObject json2){
|
||||
if(StringUtils.isNotBlank(getFromJson(json1,"roomNumber")) || StringUtils.isNotBlank(getFromJson(json2,"roomNumber"))){
|
||||
return "roomNumber";
|
||||
}else if(StringUtils.isNotBlank(getFromJson(json1,"unitNumber")) || StringUtils.isNotBlank(getFromJson(json2,"unitNumber"))){
|
||||
return "unitNumber";
|
||||
}else if(StringUtils.isNotBlank(getFromJson(json1,"buildingNumber")) || StringUtils.isNotBlank(getFromJson(json2,"buildingNumber"))){
|
||||
return "buildingNumber";
|
||||
}else if(StringUtils.isNotBlank(getFromJson(json1,"name")) || StringUtils.isNotBlank(getFromJson(json2,"name"))){
|
||||
return "name";
|
||||
}else if(StringUtils.isNotBlank(getFromJson(json1,"street")) || StringUtils.isNotBlank(getFromJson(json2,"street"))){
|
||||
return "street";
|
||||
}else if(StringUtils.isNotBlank(getFromJson(json1,"township")) || StringUtils.isNotBlank(getFromJson(json2,"township"))){
|
||||
return "township";
|
||||
}else if(StringUtils.isNotBlank(getFromJson(json1,"district")) || StringUtils.isNotBlank(getFromJson(json2,"district"))){
|
||||
return "district";
|
||||
}else if(StringUtils.isNotBlank(getFromJson(json1,"city")) || StringUtils.isNotBlank(getFromJson(json2,"city"))){
|
||||
return "city";
|
||||
}else if(StringUtils.isNotBlank(getFromJson(json1,"province")) || StringUtils.isNotBlank(getFromJson(json2,"province"))){
|
||||
return "province";
|
||||
String level = null;
|
||||
LinkedList<String> list = new LinkedList<String>(Arrays.asList("province", "city", "district", "township", "street", "streetNumber"
|
||||
, "name", "buildingNumber", "unitNumber", "roomNumber"));
|
||||
for(String key:list){
|
||||
if(continueJudgeValue(json1, json2, key) == 3){
|
||||
return level;
|
||||
}else{
|
||||
level = continueJudgeValue(json1, json2, key) == 1 ? key : (level == null ? null : level);
|
||||
}
|
||||
}
|
||||
return null;
|
||||
return level;
|
||||
}
|
||||
|
||||
/**
|
||||
* 判断level时是否继续往下判断
|
||||
* @param json1
|
||||
* @param json2
|
||||
* @param key
|
||||
* @return 1 继续判断并且level有值 2继续判断level没值 3 停止
|
||||
*/
|
||||
private Integer continueJudgeValue(JSONObject json1, JSONObject json2, String key){
|
||||
if("buildingNumber".equals(key) || "unitNumber".equals(key) || "roomNumber".equals(key)){
|
||||
if(StringUtils.isNotBlank(getFromJson(json1,key)) && StringUtils.isNotBlank(getFromJson(json2,key))
|
||||
&& getFromJson(json1,key).equals(getFromJson(json2,key))){
|
||||
return 1;
|
||||
}else{
|
||||
return 3;
|
||||
}
|
||||
}
|
||||
if(StringUtils.isNotBlank(getFromJson(json1,key)) && StringUtils.isNotBlank(getFromJson(json2,key))){
|
||||
if("name".equals(key)){
|
||||
return verifySamePoi(getFromJson(json1,key),getFromJson(json2,key)) ? 1 : 3;
|
||||
}else{
|
||||
if(getFromJson(json1,key).equals(getFromJson(json2,key))){
|
||||
return 1;
|
||||
}else{
|
||||
return 3;
|
||||
}
|
||||
}
|
||||
}
|
||||
return 2;
|
||||
}
|
||||
/**
|
||||
* 判断省、市、区、乡镇、楼栋、单元、室号是否相同(两者全为空时返回true)
|
||||
@@ -5210,6 +5220,10 @@ public class AddressServiceImpl implements AddressService {
|
||||
if(!verifySameField(name1_stage, name2_stage, true) || !verifySameField(name1_partition, name2_partition, true)|| !verifySameField(name1_number, name2_number, true)){
|
||||
return false;
|
||||
}
|
||||
return isSameName(name1, name2);
|
||||
}
|
||||
|
||||
private boolean isSameName(String name1, String name2){
|
||||
String name_1_pinyin = AddressParserUtil.toHanYuPinYin(name1);
|
||||
String name_2_pinyin = AddressParserUtil.toHanYuPinYin(name2);
|
||||
//拼音或文本数据包含并且
|
||||
@@ -5222,7 +5236,6 @@ public class AddressServiceImpl implements AddressService {
|
||||
}
|
||||
return false;
|
||||
}
|
||||
|
||||
/**
|
||||
* 设置区域消费指数
|
||||
* @param checkTr
|
||||
|
||||
Reference in New Issue
Block a user