script 修改
This commit is contained in:
+65
-53
@@ -33,7 +33,7 @@ import java.util.regex.Pattern
|
||||
@BatchOwner(Watcher.GaoJuan)
|
||||
class BigdataDealBatch extends BaseBatch{
|
||||
|
||||
def startDate = Integer.parseInt(DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyyMM"), -3, "yyyyMM"));
|
||||
def startDate = Integer.parseInt(DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyyMM"), -24, "yyyyMM"));
|
||||
def endDate = Integer.parseInt(DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyyMM"), -1, "yyyyMM"));
|
||||
|
||||
def curDate = Integer.parseInt(DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyyMMdd"), -1, "yyyyMMdd"));
|
||||
@@ -61,56 +61,58 @@ class BigdataDealBatch extends BaseBatch{
|
||||
public void process(Object args) {
|
||||
//def weekend = DateUtils.parseFromTightByPattern("2021-07-25","yyyy-MM-dd")//DateUtils.getFirstDayOfWeek(new Date())
|
||||
// def citys = CcbCommonUtils.citys;
|
||||
def citys = CcbCommonUtils.getRunningCitys(args)
|
||||
// def citys = CcbCommonUtils.getRunningCitys(args)
|
||||
if(args && args.size()>=2){
|
||||
curDate = args[1].toString().toInteger()
|
||||
}
|
||||
println "curDate:"+curDate
|
||||
// def allCitys = CommonDatautils.findAllCity()
|
||||
// for(def city : allCitys){
|
||||
//// if(!ccbCitys.contains(city)){
|
||||
// addDealPriceMatch(city);
|
||||
//// }
|
||||
def allCitys = ["福州"] //CommonDatautils.findAllCity()
|
||||
for(def city : allCitys){
|
||||
// if(!ccbCitys.contains(city)){
|
||||
addDealPriceMatch(city);
|
||||
// }
|
||||
}
|
||||
|
||||
|
||||
// def rows = []
|
||||
// def headers = []//["城市ID","成交ID","城市名称","行政区ID","行政区名称","街道","门牌号","小区ID","小区名称","楼栋ID","楼栋编号","楼栋名称","单元ID","单元编号","单元名称","户ID","详细地址","建筑年代","权属","权属年限","期数","所在楼层","层类型","总楼层数","建筑类型","建筑结构","建筑面积","套内面积","户型","室","厅","厨","卫","房屋用途","室内结构","朝向","装修情况","评估编号","成交单价","成交总价","申报总价","评估总价","网签合同编号","交易日期","交易类型","出售付款方式","成交周期","是否有电梯","赠送面积(地面花园)","赠送面积(露台)","赠送面积(阳台)","赠送面积(飘窗)","赠送面积(阁楼)","赠送面积(车库)","赠送面积(地下室)","赠送面积(其他)","房源数据来源","中介公司ID","中介公司名称","备注","创建时间","更新时间","来源公司","成交金额","数据时间"]
|
||||
// def cityCodeDataMap = CcbCommonUtils.findCityCodeValue()
|
||||
// def index = 0
|
||||
// for(def city : citys){
|
||||
// index ++
|
||||
// def distCodeMap = CcbCommonUtils.findAllDistrictCodeValue(city)
|
||||
// def ppeDists = [:]
|
||||
//
|
||||
// def statMap = [:]//getDistrictStatByCity(city, weekend)
|
||||
// if(city=="上海"){
|
||||
// def youDataTrade = getYouDaData(statMap)
|
||||
// def youdaDatas = createData(youDataTrade, cityCodeDataMap, distCodeMap, "youda")
|
||||
// rows.addAll(youdaDatas)
|
||||
// println city +" "+youdaDatas.size()
|
||||
// }
|
||||
//
|
||||
// def trades = getCityData(city, statMap, ppeDists)
|
||||
// if(trades){
|
||||
// def rets = createData(trades, cityCodeDataMap, distCodeMap, "other")
|
||||
// rows.addAll(rets)
|
||||
// println "【${index}】"+city +" 成交共:"+rets.size()
|
||||
//
|
||||
// def cityDatas = []
|
||||
// cityDatas << headers
|
||||
// cityDatas.addAll(rows)
|
||||
// }
|
||||
//
|
||||
// }
|
||||
def rows = []
|
||||
def headers = []//["城市ID","成交ID","城市名称","行政区ID","行政区名称","街道","门牌号","小区ID","小区名称","楼栋ID","楼栋编号","楼栋名称","单元ID","单元编号","单元名称","户ID","详细地址","建筑年代","权属","权属年限","期数","所在楼层","层类型","总楼层数","建筑类型","建筑结构","建筑面积","套内面积","户型","室","厅","厨","卫","房屋用途","室内结构","朝向","装修情况","评估编号","成交单价","成交总价","申报总价","评估总价","网签合同编号","交易日期","交易类型","出售付款方式","成交周期","是否有电梯","赠送面积(地面花园)","赠送面积(露台)","赠送面积(阳台)","赠送面积(飘窗)","赠送面积(阁楼)","赠送面积(车库)","赠送面积(地下室)","赠送面积(其他)","房源数据来源","中介公司ID","中介公司名称","备注","创建时间","更新时间","来源公司","成交金额","数据时间"]
|
||||
def cityCodeDataMap = CcbCommonUtils.findCityCodeValue()
|
||||
def index = 0
|
||||
for(def city : citys){
|
||||
index ++
|
||||
def distCodeMap = CcbCommonUtils.findAllDistrictCodeValue(city)
|
||||
def ppeDists = [:]
|
||||
|
||||
def statMap = [:]//getDistrictStatByCity(city, weekend)
|
||||
if(city=="上海"){
|
||||
def youDataTrade = getYouDaData(statMap)
|
||||
def youdaDatas = createData(youDataTrade, cityCodeDataMap, distCodeMap, "youda")
|
||||
rows.addAll(youdaDatas)
|
||||
println city +" "+youdaDatas.size()
|
||||
}
|
||||
|
||||
def trades = getCityData(city, statMap, ppeDists)
|
||||
if(trades){
|
||||
def rets = createData(trades, cityCodeDataMap, distCodeMap, "other")
|
||||
rows.addAll(rets)
|
||||
println "【${index}】"+city +" 成交共:"+rets.size()
|
||||
|
||||
def cityDatas = []
|
||||
cityDatas << headers
|
||||
cityDatas.addAll(rows)
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
def fileName = "/home/goojia/ccb/BIGDATA_DEAL_GJ.dat"
|
||||
if(CommonDatautils.isWindow()){
|
||||
fileName = "d:\\数据\\建行\\房产估值\\数据文件\\txt\\BIGDATA_DEAL_GJ.dat"
|
||||
}
|
||||
|
||||
FileUtils.batchCreateFile(rows, fileName,headers, "utf-8", false);
|
||||
println "txt 生成 " + rows.size()
|
||||
|
||||
saveToPgSql(rows)
|
||||
//
|
||||
// def fileName = "/home/goojia/ccb/BIGDATA_DEAL_GJ.dat"
|
||||
// if(CommonDatautils.isWindow()){
|
||||
// fileName = "d:\\数据\\建行\\房产估值\\数据文件\\txt\\BIGDATA_DEAL_GJ.dat"
|
||||
// }
|
||||
//
|
||||
// FileUtils.batchCreateFile(rows, fileName,headers, "utf-8", false);
|
||||
// println "txt 生成 " + rows.size()
|
||||
//
|
||||
// saveToPgSql(rows)
|
||||
}
|
||||
|
||||
def createData(def rets, cityCodeDatas, distCodeMap, source){
|
||||
@@ -649,16 +651,21 @@ class BigdataDealBatch extends BaseBatch{
|
||||
def final AtomicInteger count_ = new AtomicInteger(0)
|
||||
|
||||
for(def i=0;i<trades.size();i++){
|
||||
def t = trades[i]
|
||||
fixedThreadPool.execute(
|
||||
new Runnable() {
|
||||
void run() {
|
||||
final def t = trades[i]
|
||||
// fixedThreadPool.execute(
|
||||
// new Runnable() {
|
||||
// void run() {
|
||||
count_.addAndGet(1)
|
||||
try{
|
||||
if(!t.match_id){
|
||||
def region = RegionData.formatRegion(t.city, t.region)
|
||||
def rMap = null
|
||||
if(region) {
|
||||
if(t.house_use && t.house_use.contains("商住")){
|
||||
if(!t.name.contains("商住")){
|
||||
t.name = t.name +"商住楼"
|
||||
}
|
||||
}
|
||||
rMap = SeaweedDataMatchProcess.doMatch(t.city, region, t.name, null);
|
||||
if(!rMap && t.address){
|
||||
def flag = t.address.matches('^[\u4e00-\u9fa5]+?(路|大道|街|巷).+[弄号]$')
|
||||
@@ -668,7 +675,12 @@ class BigdataDealBatch extends BaseBatch{
|
||||
}
|
||||
if(rMap && rMap.region==region){
|
||||
CcbCommonUtils.updateDealPrice(t._id, rMap.city, rMap.region, rMap.name, rMap.id)
|
||||
}else {
|
||||
if (rMap) {
|
||||
println "区域不同"
|
||||
}
|
||||
}
|
||||
|
||||
}
|
||||
println count_.get()+" "+t+" "+ rMap
|
||||
}
|
||||
@@ -678,9 +690,9 @@ class BigdataDealBatch extends BaseBatch{
|
||||
latch.countDown()
|
||||
}
|
||||
|
||||
}
|
||||
}
|
||||
);
|
||||
// }
|
||||
// }
|
||||
// );
|
||||
}
|
||||
latch.await()
|
||||
println city +" "+trades.size()+" "+new Date().toString()
|
||||
|
||||
+2
-2
@@ -617,14 +617,14 @@ class CcbCommonUtils {
|
||||
q.put("city", city)
|
||||
q.put("trade_date", new BasicDBObject('$gte', startDate))
|
||||
q.put("match_id", new BasicDBObject('$exists', false))
|
||||
// q.put("district_name","国贸天琴湾")
|
||||
// q.put("district_name","中海寰宇天下")
|
||||
def dbcursor = DEAL_PRICE_COLL.find(q)
|
||||
def total = dbcursor.count()
|
||||
def rets = new HashSet()
|
||||
def lists = []
|
||||
println city +" "+total
|
||||
for(def d : dbcursor){
|
||||
lists << ["_id":d._id,"city":d.city, "region":d.region, "name":d.district_name, "address":d.address, "match_id":d.match_id]
|
||||
lists << ["_id":d._id,"city":d.city, "region":d.region, "name":d.district_name, "address":d.address, "match_id":d.match_id, "house_use":d.house_use,"lift_household":d.lift_household,"lift":d.lift]
|
||||
}
|
||||
return lists
|
||||
}
|
||||
|
||||
+44
-5
@@ -28,11 +28,12 @@ class DealPriceDataProcess {
|
||||
def task = new DealPriceDataProcess()
|
||||
// task.process()//运当成交pe值不好时行,拉升成交价格及删除pe太大数据
|
||||
// task.addLianJiaPrice();
|
||||
// task.deleteThirdPartyData()//删除第三方
|
||||
task.deleteThirdPartyData()//删除第三方
|
||||
// task.delDealByPe()
|
||||
// task.delDealData()
|
||||
|
||||
task.exportThirdParty()
|
||||
// task.exportThirdParty()
|
||||
// task.statPpe()
|
||||
}
|
||||
|
||||
def delDealByPe(){
|
||||
@@ -379,7 +380,7 @@ class DealPriceDataProcess {
|
||||
def results = []
|
||||
results << ['城市id','城市名称','第三方id','区域','小区id','小区名称','计算类型','计算价格','成交价格','PPE']
|
||||
for(def city : citys) {
|
||||
def rets = findPeGteThirdParty3(city, 202305)
|
||||
def rets = findPeGteThirdParty3(city, 202306)
|
||||
for(def map : rets){
|
||||
results << [map.cityid, map.city, map.data_id, map.region, map.comm_id, map.district_name, map.type, map.clc_price, map.price, map.ppe]
|
||||
}
|
||||
@@ -391,12 +392,12 @@ class DealPriceDataProcess {
|
||||
|
||||
def deleteThirdPartyData(){
|
||||
def citys = CcbCommonUtils.citys_all
|
||||
def month = 202304
|
||||
def month = 202305
|
||||
def rows = []
|
||||
def cityCodeDatas = CcbCommonUtils.findCityCodeValue()
|
||||
def set = new HashSet()
|
||||
for(def city : citys) {
|
||||
def rets = findPeGteThirdParty2(city, 202305)
|
||||
def rets = findPeGteThirdParty2(city, 202306)
|
||||
def total = 0
|
||||
println city +" "+rets.size()
|
||||
if(rets){
|
||||
@@ -490,5 +491,43 @@ class DealPriceDataProcess {
|
||||
return count
|
||||
}
|
||||
|
||||
def statPpe(){
|
||||
def path = "C:\\Users\\gaojuan\\Desktop\\估值数据统计.xlsx"
|
||||
ExcelUtils.multiThreadingModify(path, { row ->
|
||||
def dates = [202208,202209,202210,202211,202212,202301,202302,202303,202304,202305,202306]
|
||||
def deal_pes = []
|
||||
def third_pes = []
|
||||
for(def d : dates){
|
||||
def map = findPeGteCity(row['B'], d)
|
||||
if(map.deal_pe){
|
||||
deal_pes << map.deal_pe
|
||||
}
|
||||
if(map.third_pe){
|
||||
third_pes << map.third_pe
|
||||
}
|
||||
}
|
||||
if(deal_pes){
|
||||
row['F'] = deal_pes.sum()/deal_pes.size()
|
||||
}
|
||||
if(third_pes){
|
||||
row['G'] = third_pes.sum()/third_pes.size()
|
||||
}
|
||||
println row['B'] + " "+row['F']+" "+row['G']
|
||||
})
|
||||
|
||||
}
|
||||
|
||||
|
||||
|
||||
def findPeGteCity(cityName, yearmonth){
|
||||
def sql = "SELECT gujiamdltst_ppe15_pct,gujiatrdpttstppe15pct from t_app_p9pha_plda.avm_evaluation_gj_h where urbn_nm='${cityName}' and gujia_bds_yrmo=${yearmonth}"
|
||||
def map = [:]
|
||||
pgsqlInstance.eachRow(sql,{r->
|
||||
map.deal_pe = r.gujiamdltst_ppe15_pct
|
||||
map.third_pe = r.gujiatrdpttstppe15pct
|
||||
});
|
||||
return map
|
||||
}
|
||||
|
||||
|
||||
}
|
||||
|
||||
+2
-2
@@ -52,7 +52,7 @@ class GuiDingPriceBatch extends BaseBatch{
|
||||
def distCodeMap = CcbCommonUtils.findAllDistrictCodeValue(city)
|
||||
|
||||
//添加自己库中的人工指导价
|
||||
def prices = CcbCommonUtils.findPersonPrice(city, "2023-05-27", "2023-06-21")
|
||||
def prices = CcbCommonUtils.findPersonPrice(city, "2023-05-27", "2023-06-26")
|
||||
// def prices = CcbCommonUtils.findPersonPrice(city, "2022-01-01", "2022-11-22")
|
||||
if(prices){
|
||||
def distMap = getDistrictMap(city)
|
||||
@@ -85,7 +85,7 @@ class GuiDingPriceBatch extends BaseBatch{
|
||||
saveToPgSql(rows)
|
||||
//
|
||||
// //更新静态数据表
|
||||
updateVersion(DateUtils.formatDateByPattern(new Date(), "yyyy-MM")+"-28")
|
||||
// updateVersion(DateUtils.formatDateByPattern(new Date(), "yyyy-MM")+"-28")
|
||||
|
||||
}
|
||||
|
||||
|
||||
+10
@@ -17,4 +17,14 @@ class PgsqlCommUtils {
|
||||
});
|
||||
return set
|
||||
}
|
||||
|
||||
|
||||
public static def findAllDistrictCount(def cityid){
|
||||
def sql = "select count(gujia_comm_id) from base.t0831_bigdata_community_gj_h where urbn_id='${cityid}' and gujia_bds_bds_st=0";
|
||||
def count = 0
|
||||
sqlInstance.eachRow(sql,{r->
|
||||
count = r.count
|
||||
});
|
||||
return count
|
||||
}
|
||||
}
|
||||
|
||||
+68
-21
@@ -45,8 +45,8 @@ DBCollection POLYLINECOLL = MongoUtils.getCollectionByAuth20190102("fangjia_craw
|
||||
def offlineUrl = "mongodb://developer:G00IDCDEV12385@192.168.0.136:28017";
|
||||
def DBCollection youda_esfhouse_coll = MongoUtils.getCollectionByName2("business", "res_esfdealprice", offlineUrl)
|
||||
|
||||
//def pgsqlInstance = Sql.newInstance("jdbc:postgresql://192.168.0.236:5432/warehouse",
|
||||
// "postgres", "postgres", "org.postgresql.Driver")
|
||||
def pgsqlInstance = Sql.newInstance("jdbc:postgresql://192.168.0.236:5432/warehouse",
|
||||
"postgres", "postgres", "org.postgresql.Driver")
|
||||
|
||||
//def citys = ["山南","黄南州","甘南","克孜勒苏","嘉峪关","海北州","阿拉善盟","怒江州","和田","日喀则","昌都","海南州","玉树州","吐鲁番","阿勒泰","三沙","塔城","甘孜州","果洛州","林芝","那曲","阿坝州","神农架"]
|
||||
//def cityCodeDataMap = CcbCommonUtils.findCityCodeValue()
|
||||
@@ -1289,28 +1289,28 @@ def findAvgPrice(def commid, def pgsqlInstance, def cityid){
|
||||
|
||||
def fileName = "D:\\数据\\建行\\房产估值\\指导价\\录入国信达价格_0602.xlsx"
|
||||
|
||||
ExcelUtils.multiThreadingModify(fileName, {d->
|
||||
if(d['A']){
|
||||
def region = RegionData.formatRegion(d['B'], d['C'])
|
||||
if(region){
|
||||
def dist = CommonDatautils.findDistrict(d['B'], region, d['E'], "district")
|
||||
if(dist){
|
||||
d['G'] = dist.region
|
||||
d['H'] = dist._id.toString()
|
||||
}
|
||||
}
|
||||
// if(!d['F']){
|
||||
// def dist = CommonDatautils.findDistrictById(d['A'])
|
||||
////
|
||||
//ExcelUtils.multiThreadingModify(fileName, {d->
|
||||
// if(d['A']){
|
||||
// def region = RegionData.formatRegion(d['B'], d['C'])
|
||||
// if(region){
|
||||
// def dist = CommonDatautils.findDistrict(d['B'], region, d['E'], "district")
|
||||
// if(dist){
|
||||
//// d['H'] = dist.region
|
||||
//// d['G'] = dist.name
|
||||
// d['F'] = dist.status
|
||||
// d['G'] = dist.region
|
||||
// d['H'] = dist._id.toString()
|
||||
// }
|
||||
// }
|
||||
|
||||
}
|
||||
},0)
|
||||
//// if(!d['F']){
|
||||
//// def dist = CommonDatautils.findDistrictById(d['A'])
|
||||
//////
|
||||
//// if(dist){
|
||||
////// d['H'] = dist.region
|
||||
////// d['G'] = dist.name
|
||||
//// d['F'] = dist.status
|
||||
//// }
|
||||
//// }
|
||||
//
|
||||
// }
|
||||
//},0)
|
||||
|
||||
|
||||
|
||||
@@ -1364,6 +1364,53 @@ def findGuidedPrice(sqlInstance, id){
|
||||
return count
|
||||
}
|
||||
|
||||
def findPgDealPrice(c, pgsqlInstance, startDate){
|
||||
def sql = "select count(a.gujia_comm_id) from (SELECT distinct gujia_comm_id from base.t0831_bigdata_deal_gj_h where urbn_nm='${c}' and txn_dt>='${startDate}') a"
|
||||
def count = 0
|
||||
pgsqlInstance.eachRow(sql,{r->
|
||||
count = r.count
|
||||
});
|
||||
return count
|
||||
}
|
||||
|
||||
citys = CcbCommonUtils.citys_all
|
||||
def cityCodeDataMap = CcbCommonUtils.findCityCodeValue()
|
||||
//def dates = ["2022-07-01","2022-08-01","2022-09-01","2022-10-01","2022-11-01","2022-12-01","2023-01-01","2023-02-01","2023-03-01","2023-04-01","2023-05-01","2023-06-01"]
|
||||
def dates = [202208,202209,202210,202211,202212,202301,202302,202303,202304,202305,202306]
|
||||
def rows = []
|
||||
def headers = []
|
||||
headers << "城市"
|
||||
headers.addAll(dates)
|
||||
rows << headers
|
||||
for(def c : citys){
|
||||
def row = []
|
||||
row << c
|
||||
def disCount = PgsqlCommUtils.findAllDistrictCount(cityCodeDataMap.get(c))
|
||||
// for(def d : dates){
|
||||
def startDate = "2022-07-01"
|
||||
def count = findPgDealPrice(c, pgsqlInstance, startDate)
|
||||
row << count*100/disCount
|
||||
|
||||
// def map = findSampleRate(pgsqlInstance, c, d)
|
||||
// row << map.third
|
||||
// }
|
||||
rows << row
|
||||
}
|
||||
ExcelUtils.write("D:\\数据\\建行\\数据统计结果\\成交占比统计-年.xlsx", rows, "第三方")
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
def findSampleRate(pgsqlInstance, cityName, yearmonth){
|
||||
def sql = "SELECT urbn_nm,gujiamdltst_ppe15_pct,gujiatrdpttstppe15pct from t_app_p9pha_plda.avm_evaluation_gj_h where urbn_nm='${cityName}' and gujia_bds_yrmo=${yearmonth}"
|
||||
def map = [:]
|
||||
pgsqlInstance.eachRow(sql,{r->
|
||||
map.deal = r.gujiamdltst_ppe15_pct
|
||||
map.third = r.gujiatrdpttstppe15pct
|
||||
});
|
||||
return map
|
||||
}
|
||||
|
||||
|
||||
|
||||
|
||||
+15
-2
@@ -48,7 +48,7 @@ class HailanTest {
|
||||
}
|
||||
|
||||
def run(){
|
||||
def inputfile = "D:\\数据\\微元素\\排查表.xlsx"
|
||||
def inputfile = "D:\\数据\\微元素\\排查表-价格对比-新.xlsx"
|
||||
// addThirdPrice(inputfile)
|
||||
// match(inputfile)
|
||||
// tradeAvgPrice(inputfile)
|
||||
@@ -58,7 +58,8 @@ class HailanTest {
|
||||
// addHouse(inputfile)
|
||||
// stat(inputfile)
|
||||
// hailanStat(inputfile)
|
||||
addFzFjwPrice(inputfile)
|
||||
// addFzFjwPrice(inputfile)
|
||||
addFjwTobPrice(inputfile)
|
||||
}
|
||||
|
||||
def addHouse(inputfile){
|
||||
@@ -748,6 +749,18 @@ class HailanTest {
|
||||
})
|
||||
}
|
||||
|
||||
|
||||
def addFjwTobPrice(path){
|
||||
ExcelUtils.multiThreadingModify(path,{d->
|
||||
if(d['F']){
|
||||
def dbo = CommonDatautils.queryDistrictPrice("福州", d['F'], d['G'], DateUtils.parseFromTightByPattern("2023-07-02","yyyy-MM-dd"))
|
||||
if(dbo){
|
||||
d['Q'] = dbo.avg_price_2b
|
||||
}
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
def addFzFjwPrice(inputfile){
|
||||
ExcelUtils.modify(inputfile, { row ->
|
||||
if(row['B']) {
|
||||
|
||||
+1
-1
@@ -252,7 +252,7 @@ class CommonDatautils {
|
||||
q.put("visible", 0);
|
||||
q.put("cat", cat);
|
||||
q.put("city", city);
|
||||
// q.put("name", "长滩美墅三期")
|
||||
// q.put("name", "燃灯小区")
|
||||
|
||||
DBObject keys = new BasicDBObject();
|
||||
if(fields){
|
||||
|
||||
+18
-3
@@ -250,8 +250,23 @@ class DataHandleUtils {
|
||||
return sum;
|
||||
|
||||
}
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
public static isStandardRoad(def s){
|
||||
def pattern = Pattern.compile("[路|街道|大道|巷|胡同|街|坊]\\d+(-\\d+)?(号|弄)");
|
||||
def matcher = pattern.matcher(s)
|
||||
if(matcher.find()){
|
||||
return true;
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
|
||||
|
||||
|
||||
static main(args) {
|
||||
def a = isStandardRoad("杨桥西路236号")
|
||||
println a
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
+2
-2
@@ -5,7 +5,7 @@ import com.fangjia.script.util.DbUtils
|
||||
import com.fangjia.script.util.ExcelUtils
|
||||
import com.fangjia.script.util.GeoUtils
|
||||
import com.mongodb.BasicDBObject
|
||||
|
||||
//删除别名是另一个小区的主名
|
||||
class MasterNameInAlias {
|
||||
|
||||
static main(args) {
|
||||
@@ -14,7 +14,7 @@ class MasterNameInAlias {
|
||||
}
|
||||
|
||||
def run(){
|
||||
def citys = CommonDatautils.findAllCity()
|
||||
def citys = CommonDatautils.findAllCity().reverse()
|
||||
def allList = []
|
||||
for(def city : citys){
|
||||
def disticts = CommonDatautils.findAllDistricts(city, "district", ["city","region","name","address","alias","estate_type2", "lat", "lng"])
|
||||
|
||||
+616
@@ -0,0 +1,616 @@
|
||||
package com.fangjia.script.fuzhou
|
||||
|
||||
import com.fangjia.core.util.BaiDuApiUtils
|
||||
import com.fangjia.script.commData.CommonDatautils
|
||||
import com.fangjia.script.common.DataHandleUtils
|
||||
import com.fangjia.script.gj.EsUtils
|
||||
import com.fangjia.script.gj.district.FangjiaClientUtils
|
||||
import com.fangjia.script.linxsh.util.AddressUtil
|
||||
import com.fangjia.script.looplineinfo.InsidePolygon
|
||||
import com.fangjia.script.util.CosineSimilarAlgorithm
|
||||
import com.fangjia.script.util.DbUtils
|
||||
import com.fangjia.script.util.ElasticUtils
|
||||
import com.fangjia.script.util.ExcelUtils
|
||||
import com.fangjia.script.util.OpenApiClient
|
||||
import com.fangjia.script.yinjihaun.amap.GaoDeApiUtils
|
||||
import com.mongodb.BasicDBObject
|
||||
import org.elasticsearch.xpack.sql.jdbc.EsDataSource
|
||||
|
||||
import java.sql.ResultSet
|
||||
import java.sql.ResultSetMetaData
|
||||
import java.util.concurrent.CountDownLatch
|
||||
import java.util.concurrent.ExecutorService
|
||||
import java.util.concurrent.Executors
|
||||
import java.util.concurrent.atomic.AtomicLong
|
||||
|
||||
class AmapAddressUpdate {
|
||||
|
||||
public static ExecutorService fixedThreadPool = Executors.newFixedThreadPool(20)
|
||||
public static EsDataSource dataSource = new EsDataSource()
|
||||
private static final String address = "jdbc:es://http://seawater.es.db.fangjia.com:9200?timezone=UTC&page.size=240&connect.timeout=240000&network.timeout=60000&page.timeout=240000&query.timeout=90000"
|
||||
static {
|
||||
dataSource.setUrl(address)
|
||||
Properties connectionProperties = new Properties()
|
||||
connectionProperties.put("user", "developer")
|
||||
connectionProperties.put("password", "5e8edd851d2fdfbd7415232c67367cc3")
|
||||
dataSource.setProperties(connectionProperties)
|
||||
}
|
||||
|
||||
static main(args) {
|
||||
def task = new AmapAddressUpdate()
|
||||
def path = "d:\\数据\\小区\\福州小区地址与高德地图V2.xlsx"
|
||||
def addr = task.formatAddress("福州", "北三环路201号")
|
||||
println addr
|
||||
// task.run(path)
|
||||
// task.similarMatch(path)
|
||||
// task.addressRun(path)
|
||||
// task.addressSource(path)
|
||||
// task.amapRun()
|
||||
|
||||
// task.addrAndLocationDistance(path)//地址经纬度与库中经纬度比较
|
||||
// task.updateData(path)
|
||||
// task.updateLocation(path)//更新经纬度
|
||||
}
|
||||
def run(path){
|
||||
def citys = ["福州"]//CommonDatautils.findAllCity()
|
||||
def allrets = []
|
||||
allrets << ["城市","区域","名称","地址","地址别名","高德区域","高德名称","高德地址","高德地址格式化","高德经度","高德纬度"]
|
||||
for (def city : citys) {
|
||||
def disticts = CommonDatautils.findAllDistricts(city, "district", ["city", "region", "name","address", "addr_alias","lat", "lng"])
|
||||
def rets = findAmapDetail(disticts)
|
||||
allrets.addAll(rets)
|
||||
}
|
||||
ExcelUtils.write(path, allrets, "数据")
|
||||
}
|
||||
|
||||
def amapRun(){
|
||||
def city = "福州市"
|
||||
def datas = findAllAmapByCity(city)
|
||||
// datas = datas[0..10]
|
||||
println "Es总量:"+datas.size()
|
||||
def rets = match(datas)
|
||||
|
||||
ExcelUtils.write("d:\\数据\\小区\\福州高德数据匹配.xlsx", rets, "数据")
|
||||
}
|
||||
|
||||
def match(datas){
|
||||
def rets = Collections.synchronizedList(new ArrayList());
|
||||
def totalCount = new AtomicLong()
|
||||
final CountDownLatch latch = new CountDownLatch(datas.size())
|
||||
rets << ["ID","高德城市","高德区域","高德名称","高德地址","匹配城市","匹配区域","匹配名称","匹配类型","匹配ID"]
|
||||
for(def i=0;i<datas.size();i++){
|
||||
final def d = datas[i]
|
||||
fixedThreadPool.execute(new Runnable() {
|
||||
void run() {
|
||||
try{
|
||||
totalCount.addAndGet(1)
|
||||
def ret = []
|
||||
def jsonObject = OpenApiClient.addressMatch(d.city, d.name, "property", false, d.region, null)
|
||||
ret << d.id
|
||||
ret << d.city
|
||||
ret << d.region
|
||||
ret << d.name
|
||||
ret << d.address
|
||||
if(jsonObject != null && jsonObject.precise!=-1) {
|
||||
ret << jsonObject.get("city")
|
||||
ret << jsonObject.get("district")
|
||||
ret << jsonObject.get("name")
|
||||
ret << jsonObject.get("propertyType")
|
||||
ret << jsonObject.get("id")
|
||||
}
|
||||
println totalCount.get() +' '+ret
|
||||
rets << ret
|
||||
} catch (Exception e) {
|
||||
e.printStackTrace()
|
||||
} finally {
|
||||
latch.countDown()
|
||||
}
|
||||
}
|
||||
});
|
||||
}
|
||||
latch.await()
|
||||
return rets
|
||||
}
|
||||
|
||||
def findAllAmapByCity(city){
|
||||
final StringBuffer sqlBuffer = new StringBuffer()
|
||||
sqlBuffer.append("SELECT id,cityname,adname,name,address,type,location,m_date,typecode FROM amap_poi_detail WHERE 1=1")
|
||||
sqlBuffer.append(" and cityname = '" + city + "'")
|
||||
sqlBuffer.append(" and type like '%住宅%'")
|
||||
ResultSet rs = dataSource.connection.createStatement().executeQuery(sqlBuffer.toString())
|
||||
def list = []
|
||||
while (rs.next()) {
|
||||
def map = [:]
|
||||
map.id = rs.getString("id")
|
||||
map.city = city
|
||||
map.name = rs.getString("name")
|
||||
map.address = rs.getString("address")
|
||||
map.region = rs.getString("adname")
|
||||
// map.type = rs.getString("type")
|
||||
// def location = rs.getObject("location")
|
||||
// if(location){
|
||||
// map.lng = location.x
|
||||
// map.lat = location.y
|
||||
// }
|
||||
list << map
|
||||
}
|
||||
return list
|
||||
}
|
||||
|
||||
def findAmapDetail(districts){
|
||||
def rets = Collections.synchronizedList(new ArrayList());
|
||||
def totalCount = new AtomicLong()
|
||||
final CountDownLatch latch = new CountDownLatch(districts.size())
|
||||
|
||||
for(def i=0;i<districts.size();i++){
|
||||
final def d = districts[i]
|
||||
fixedThreadPool.execute(new Runnable() {
|
||||
void run() {
|
||||
try{
|
||||
totalCount.addAndGet(1)
|
||||
def row = []
|
||||
row << d.city
|
||||
row << d.region
|
||||
row << d.name
|
||||
row << d.address
|
||||
if(d.addr_alias){
|
||||
row << d.addr_alias.join(" ")
|
||||
}else{
|
||||
row << ''
|
||||
}
|
||||
def dbo = getAmapLocation(d.city+"市", d.region, d.name)
|
||||
if(!dbo){
|
||||
def newName = d.name
|
||||
if(d.name.toString().endsWith("别墅") || d.name.endsWith("商住楼") || d.name.endsWith("钢混") || d.name.endsWith("混合")){
|
||||
newName = newName.replace("(非钢混)","").replace("钢混","").replace("(混合)","").replace("混合","").replace("别墅","").replace("商住楼","")
|
||||
}
|
||||
dbo = getAmapLocation(d.city+"市", d.region, newName)
|
||||
}
|
||||
|
||||
// if(!dbo && d.lat){
|
||||
// dbo = getAmapLocation(d.city+"市", d.region, newName, d.lat/10000000000.0, d.lng/10000000000.0)
|
||||
// }
|
||||
if(dbo){
|
||||
row << dbo.region
|
||||
row << dbo.name
|
||||
row << dbo.address
|
||||
row << formatAddress(d.city, dbo.address)
|
||||
row << dbo.lng
|
||||
row << dbo.lat
|
||||
row << dbo.match_type
|
||||
}else{
|
||||
row << ""
|
||||
row << ""
|
||||
row << ""
|
||||
row << ""
|
||||
row << ""
|
||||
row << ""
|
||||
row << ""
|
||||
}
|
||||
println totalCount.get() +' '+row
|
||||
rets << row
|
||||
} catch (Exception e) {
|
||||
e.printStackTrace()
|
||||
} finally {
|
||||
latch.countDown()
|
||||
}
|
||||
}
|
||||
});
|
||||
}
|
||||
latch.await()
|
||||
return rets
|
||||
}
|
||||
|
||||
def formatAddress(def city, def address){
|
||||
def jsonObj = FangjiaClientUtils.addressStructure(city, address)
|
||||
def address_str = null
|
||||
if(jsonObj && jsonObj.result) {
|
||||
def street = ''
|
||||
def street_number = ''
|
||||
if (jsonObj.result.street) {
|
||||
street = jsonObj.result.street
|
||||
}
|
||||
if (jsonObj.result.streetNumber) {
|
||||
street_number = jsonObj.result.streetNumber
|
||||
}
|
||||
if (street || street_number) {
|
||||
address_str = street
|
||||
if(street_number){
|
||||
address_str = address_str + street_number
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if(address_str && address_str.contains("交叉口")){
|
||||
address_str = address_str.substring(0, address_str.indexOf("交叉口")+3)
|
||||
}
|
||||
return address_str;
|
||||
}
|
||||
|
||||
def getAmapLocation(city, region, name){
|
||||
|
||||
final StringBuffer sqlBuffer = new StringBuffer()
|
||||
sqlBuffer.append("SELECT cityname,adname,name,address,type,location,m_date,typecode FROM amap_poi_detail WHERE 1=1")
|
||||
sqlBuffer.append(" and cityname = '" + city + "'")
|
||||
sqlBuffer.append(" and MATCH(name,'${name}')")
|
||||
sqlBuffer.append(" and type like '%住宅%'")
|
||||
ResultSet rs = dataSource.connection.createStatement().executeQuery(sqlBuffer.toString())
|
||||
def map = [:]
|
||||
while (rs.next()) {
|
||||
// println rs.getString("adname")+" "+rs.getString("name") +" "+rs.getString("address") +" "+rs.getString("type")
|
||||
def mapName = rs.getString("name").replace("·","")
|
||||
if(mapName == name && rs.getString("adname").contains(region)){
|
||||
map.name = rs.getString("name")
|
||||
map.address = rs.getString("address")
|
||||
map.formatAddress = AddressUtil.formatAddress(city, map.address)
|
||||
map.region = rs.getString("adname")
|
||||
map.type = rs.getString("type")
|
||||
map.match_type = "名称相同"
|
||||
def location = rs.getObject("location")
|
||||
if(location){
|
||||
map.lng = location.x
|
||||
map.lat = location.y
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// String sql = "select * from amap_poi_detail where cityname='${city}' and name='${name}' and type like '%住宅%'"
|
||||
// def loc = null
|
||||
// ElasticUtils.executeSqlQuery(sql,{ data->
|
||||
// println data.adname+" "+data.name +" "+data.address +" "+data.type
|
||||
// if(name == data.name && data.adname.contains(region) && data.type.contains("type")){
|
||||
// data.formatAddress = AddressUtil.formatAddress(city, data.address)
|
||||
// loc = data
|
||||
// }
|
||||
// })
|
||||
return map;
|
||||
}
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
def addressRun(inputfile){
|
||||
|
||||
ExcelUtils.multiThreadingModify(inputfile, { row ->
|
||||
// if(row['H'] && row['H'].contains("口步行") && row['H'].contains("(")){
|
||||
// row['I'] = row['H'].substring(0, row['H'].indexOf("("))
|
||||
// }
|
||||
if(row['J']){
|
||||
def dist1 = CommonDatautils.findDistrict(row['A'], row['B'], row['C'], "district")
|
||||
def distance = null
|
||||
if(dist1){
|
||||
distance = com.fangjia.core.util.GeoUtils.calculateDistance(dist1.lng/10000000000d, dist1.lat/10000000000d, row['J'].toString().toDouble(), row['K'].toString().toDouble());
|
||||
}
|
||||
row['M'] = distance
|
||||
}
|
||||
|
||||
def gaodeRoadFlag = false
|
||||
if(row['I']){
|
||||
gaodeRoadFlag = DataHandleUtils.isStandardRoad(row['I'])
|
||||
}
|
||||
def dbRoadFlag = false
|
||||
if(row['D']){
|
||||
dbRoadFlag = DataHandleUtils.isStandardRoad(row['D'])
|
||||
}
|
||||
if(!row['N'] && row['D'] && row['I']){
|
||||
if(row['D']==row['I']){
|
||||
row['N'] = 100
|
||||
}else if(row['E'] && gaodeRoadFlag && row['E'].contains(row['I'])){
|
||||
row['N'] = 1
|
||||
}
|
||||
}
|
||||
|
||||
if(!row['N'] && !dbRoadFlag && gaodeRoadFlag){
|
||||
row['N'] = 2
|
||||
}
|
||||
|
||||
if(!row['N'] && dbRoadFlag && gaodeRoadFlag && row['I'].contains(row['D'])){
|
||||
row['N'] = 3
|
||||
}
|
||||
|
||||
if(!row['N'] && dbRoadFlag && gaodeRoadFlag){
|
||||
row['N'] = 4
|
||||
}
|
||||
|
||||
})
|
||||
}
|
||||
|
||||
def addressSource(inputfile) {
|
||||
|
||||
ExcelUtils.multiThreadingModify(inputfile, { row ->
|
||||
if(!row['N'] && row['D'] && !row['O']){
|
||||
def addresses = findAllEsData(row['A'], row['B'], row['C'])
|
||||
if(addresses){
|
||||
def dbRoadFlag = false
|
||||
if(row['D']){
|
||||
dbRoadFlag = DataHandleUtils.isStandardRoad(row['D'])
|
||||
}
|
||||
if(dbRoadFlag){
|
||||
def list = []
|
||||
list << row['D']
|
||||
if(row['E']){
|
||||
def arr = row['E'].split(" ")
|
||||
list.addAll(arr as List)
|
||||
}
|
||||
for(def addr : list){
|
||||
if(addresses.contains(addr)){
|
||||
row['O'] = "诸葛找房"
|
||||
break
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
|
||||
def findAllEsData(city,region, name){
|
||||
def rets = []
|
||||
def sql = "select * from community where city='${city}' and name='${name}' and source='诸葛找房APP'"
|
||||
rets = findDatas(region, sql)
|
||||
if(!rets){
|
||||
rets = findAliasDatas(city,region, name)
|
||||
}
|
||||
return rets;
|
||||
}
|
||||
|
||||
def findDatas(region, sql){
|
||||
def rets = []
|
||||
ElasticUtils.executeSqlQuery(sql, { d->
|
||||
if(d.region && d.region.contains(region) && d.address){
|
||||
rets << d.address
|
||||
}
|
||||
})
|
||||
return rets;
|
||||
}
|
||||
|
||||
|
||||
def findAliasDatas(city, region, name){
|
||||
def must = []
|
||||
def term1 = ["match":["city":city]]
|
||||
must << term1
|
||||
|
||||
def should = []
|
||||
should << ["match":["alias":name]]
|
||||
def params = ["query":["bool":["must":must, "should":should]]]
|
||||
def s = EsUtils.httpSaveData(params, "http://192.168.0.206:9200/community/_doc/_search");
|
||||
def rets = [];
|
||||
if(s) {
|
||||
def doc = new groovy.json.JsonSlurper().parseText(s)
|
||||
def hits = doc.hits
|
||||
if(hits){
|
||||
hits = hits.hits
|
||||
for(def h : hits){
|
||||
def d = h._source
|
||||
if(d.region && d.region.contains(region) && d.address && d.source=="诸葛找房APP"){
|
||||
rets << d.address
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
return rets;
|
||||
}
|
||||
|
||||
def addrAndLocationDistance(inputfile){
|
||||
ExcelUtils.multiThreadingModify(inputfile, { row ->
|
||||
if(!row['N'] && row['D']){
|
||||
println row['D']
|
||||
def dbRoadFlag = false
|
||||
if(row['D']){
|
||||
dbRoadFlag = DataHandleUtils.isStandardRoad(row['D'])
|
||||
}
|
||||
if(dbRoadFlag){
|
||||
def gloJson = GaoDeApiUtils.geo(row['A'], row['D'])
|
||||
if(gloJson) {
|
||||
def glocation = null
|
||||
if (gloJson && gloJson.get("geocodes")) {
|
||||
def geocodes = gloJson.get("geocodes")
|
||||
def map = geocodes.get(0)
|
||||
if(map.formatted_address.contains(row['D'])){
|
||||
def arrs = map.location.split(",")
|
||||
|
||||
def dist1 = CommonDatautils.findDistrict(row['A'], row['B'], row['C'], "district")
|
||||
def distance = null
|
||||
if(dist1){
|
||||
distance = com.fangjia.core.util.GeoUtils.calculateDistance(dist1.lng/10000000000d, dist1.lat/10000000000d, arrs[0].toString().toDouble(), arrs[1].toString().toDouble());
|
||||
}
|
||||
row['P'] = distance
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
def updateData(path){
|
||||
ExcelUtils.multiThreadingModify(path, { row ->
|
||||
if(row['Q'] && row['Q'].contains("修改地址")){
|
||||
def dist1 = CommonDatautils.findDistrict(row['A'], row['B'], row['C'], "district")
|
||||
def udata = new BasicDBObject()
|
||||
if(row['Q'].contains("可替换原地址")){
|
||||
udata.put("address", row['I'])
|
||||
}else if(row['Q'].contains("可新增地址别名")){
|
||||
udata.put("address", row['I'])
|
||||
def addrAlias = dist1.addr_alias
|
||||
if(!addrAlias){
|
||||
addrAlias = []
|
||||
}
|
||||
addrAlias << dist1.address
|
||||
udata.put("addr_alias", addrAlias)
|
||||
}
|
||||
udata.put("m_date", new Date())
|
||||
DbUtils.seaweed.update(new BasicDBObject('_id', dist1.get("_id")), new BasicDBObject('$set', udata))
|
||||
}
|
||||
|
||||
|
||||
|
||||
})
|
||||
}
|
||||
|
||||
def updateLocation(path){
|
||||
ExcelUtils.multiThreadingModify(path, { row ->
|
||||
|
||||
if(row['R'] && row['R']=="是"){
|
||||
def dist1 = CommonDatautils.findDistrict(row['A'], row['B'], row['C'], "district")
|
||||
if(dist1){
|
||||
def udata = new BasicDBObject()
|
||||
def lat = null
|
||||
def lng = null
|
||||
if(row['J']){
|
||||
lng = row['J'].toString().toDouble()
|
||||
lat = row['K'].toString().toDouble()
|
||||
}else{
|
||||
def gloJson = GaoDeApiUtils.geo(row['A'], row['D'])
|
||||
if(gloJson) {
|
||||
if (gloJson && gloJson.get("geocodes")) {
|
||||
def geocodes = gloJson.get("geocodes")
|
||||
def map = geocodes.get(0)
|
||||
if (map.formatted_address.contains(row['D'])) {
|
||||
def arrs = map.location.split(",")
|
||||
lng = arrs[0].toString().toDouble()
|
||||
lat = arrs[1].toString().toDouble()
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// 高德转百度
|
||||
def baidu = BaiDuApiUtils.geoconv(lng, lat, 3)
|
||||
def lng2 = baidu.x
|
||||
def lat2 = baidu.y
|
||||
Long lng_new = lng *10000000000L
|
||||
Long lat_new = lat *10000000000L
|
||||
udata.put("lng", lng_new)
|
||||
udata.put("lat", lat_new)
|
||||
if(lng2){
|
||||
Long lng2_new = lng2 *10000000000L
|
||||
Long lat2_new = lat2 *10000000000L
|
||||
udata.put("lng2", lng2_new)
|
||||
udata.put("lat2", lat2_new)
|
||||
}
|
||||
udata.put("m_date", new Date())
|
||||
DbUtils.seaweed.update(new BasicDBObject('_id', dist1.get("_id")), new BasicDBObject('$set', udata))
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
})
|
||||
}
|
||||
|
||||
def similarMatch(path){
|
||||
ExcelUtils.multiThreadingModify(path, { row ->
|
||||
if(!row['F']){
|
||||
def d = CommonDatautils.findDistrict(row['A'], row['B'], row['C'], "district")
|
||||
if(d && d.lat){
|
||||
def dbo = getSimilarAmapLocation(d.city+"市", d.region, d.name, d.address, d.lat/10000000000.0, d.lng/10000000000.0)
|
||||
if(dbo){
|
||||
row['F'] = dbo.region
|
||||
row['G'] = dbo.name
|
||||
row['H'] = dbo.address
|
||||
row['I'] = formatAddress(d.city, dbo.address)
|
||||
row['J'] = dbo.lng
|
||||
row['K'] = dbo.lat
|
||||
row['L'] = dbo.score
|
||||
row['M'] = dbo.distance
|
||||
row['S'] = dbo.type
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
})
|
||||
}
|
||||
|
||||
|
||||
|
||||
def getSimilarAmapLocation(city, region, name, address, lat, lng){
|
||||
|
||||
final StringBuffer sqlBuffer = new StringBuffer()
|
||||
sqlBuffer.append("SELECT cityname,adname,name,address,type,location,m_date,typecode FROM amap_poi_detail WHERE 1=1")
|
||||
sqlBuffer.append(" and cityname = '" + city + "'")
|
||||
sqlBuffer.append(" and MATCH(name,'${name}')")
|
||||
ResultSet rs = dataSource.connection.createStatement().executeQuery(sqlBuffer.toString())
|
||||
|
||||
def maxSocreObj = null
|
||||
while (rs.next()) {
|
||||
def map = [:]
|
||||
def mapName = rs.getString("name").replace("·","")
|
||||
|
||||
println name +" "+ mapName
|
||||
if(rs.getString("adname").contains(region)){
|
||||
map.name = rs.getString("name")
|
||||
map.address = rs.getString("address")
|
||||
|
||||
map.region = rs.getString("adname")
|
||||
map.type = rs.getString("type")
|
||||
map.match_type = "名称相似"
|
||||
def location = rs.getObject("location")
|
||||
if(location){
|
||||
map.lng = location.x
|
||||
map.lat = location.y
|
||||
}
|
||||
|
||||
double nameScore = 0
|
||||
double addScore = 0
|
||||
double loctionScore = 0
|
||||
def distance = com.fangjia.core.util.GeoUtils.calculateDistance(lng, lat, map.lng.toString().toDouble(), map.lat.toString().toDouble());
|
||||
map.distance = distance
|
||||
if(map.name && name){
|
||||
nameScore = CosineSimilarAlgorithm.getSimilarity(map.name, name);
|
||||
}
|
||||
if(map.address && address){
|
||||
addScore = CosineSimilarAlgorithm.getSimilarity(map.address, address);
|
||||
}
|
||||
|
||||
if(hasColumn(rs, "shape")){
|
||||
def shape = rs.getObject("shape")
|
||||
if(shape){
|
||||
def coordinates = shape.getObject("coordinates")
|
||||
// def pointList = InsidePolygon.createPoint2(points);
|
||||
def flag = InsidePolygon.isInArea(lng, lat, coordinates)
|
||||
if(flag){
|
||||
loctionScore = 0.5
|
||||
}
|
||||
}
|
||||
}else{
|
||||
|
||||
if(distance < 500 && (nameScore>0.5 || addScore>0.5)){
|
||||
loctionScore = 0.3
|
||||
}
|
||||
}
|
||||
|
||||
def score = nameScore*0.3 + addScore*0.2 + loctionScore
|
||||
map.score = score
|
||||
if(!maxSocreObj){
|
||||
if(score>0){
|
||||
map.formatAddress = AddressUtil.formatAddress(city, map.address)
|
||||
maxSocreObj = map
|
||||
}
|
||||
}else{
|
||||
println score +" "+maxSocreObj.score
|
||||
if(maxSocreObj && score>maxSocreObj.score){
|
||||
map.formatAddress = AddressUtil.formatAddress(city, map.address)
|
||||
maxSocreObj = map
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
return maxSocreObj;
|
||||
}
|
||||
|
||||
|
||||
def hasColumn(ResultSet rs, String columnName) {
|
||||
ResultSetMetaData rsmd = rs.getMetaData();
|
||||
int columns = rsmd.getColumnCount();
|
||||
for (int x = 1; x <= columns; x++) {
|
||||
if (columnName.equals(rsmd.getColumnName(x))) {
|
||||
return true;
|
||||
}
|
||||
}
|
||||
return false;
|
||||
}
|
||||
|
||||
|
||||
}
|
||||
+1
-1
@@ -60,7 +60,7 @@ class DistrictSellStatBatch extends BaseBatch {
|
||||
}
|
||||
|
||||
def calcMonthlyPrice(){
|
||||
def citys = CommonUtils.findAllCity()
|
||||
def citys = CommonUtils.findAllCity()
|
||||
def keys = new BasicDBObject("city", 1).append("name", 1).append("region", 1).append("estate_type2", 1)
|
||||
|
||||
def lastMonthWeekend = DateUtils.truncateDay(DateUtils.getLastSundayInWeekOfLastMonth(new Date()))
|
||||
|
||||
@@ -2,6 +2,9 @@ package com.fangjia.script.fuzhou
|
||||
|
||||
import com.fangjia.core.RegionData
|
||||
import com.fangjia.core.util.GeocodeUtils
|
||||
import com.fangjia.core.util.JsonUtils
|
||||
import com.fangjia.core.util.Message
|
||||
import com.fangjia.core.util.MessageInfo
|
||||
import com.fangjia.script.commData.CommonDatautils
|
||||
import com.fangjia.script.gj.EsUtils
|
||||
import com.fangjia.script.util.DateUtils
|
||||
@@ -9,26 +12,71 @@ import com.fangjia.script.util.ExcelUtils
|
||||
import com.fangjia.script.util.MongoUtils
|
||||
import com.mongodb.BasicDBObject
|
||||
import com.mongodb.DBCollection
|
||||
import net.sf.json.JSONObject
|
||||
import org.apache.commons.io.FileUtils
|
||||
import org.bson.types.ObjectId
|
||||
|
||||
DBCollection AVERAGE_STATISTICS = MongoUtils.getCollectionByName("fangjia", "average_statistics", "192.168.0.43")
|
||||
|
||||
import com.fangjia.core.util.HttpRequest
|
||||
|
||||
ExcelUtils.modify("C:\\Users\\gaojuan\\Downloads\\第三方小区区域.xlsx", { d ->
|
||||
def city = RegionData.formatCity("余姚")
|
||||
d['D'] = city
|
||||
if(city){
|
||||
d['E'] = RegionData.formatRegion(city, d['C'])
|
||||
}
|
||||
|
||||
// if(!d['D']){
|
||||
// def citys = CommonDatautils.getCityByRegion(d['A'])
|
||||
// if(citys && citys.size()==1){
|
||||
// d['F'] = citys.get(0)
|
||||
// }
|
||||
//Message message = new Message();
|
||||
//message.setMsgtype("text");
|
||||
//message.setText(new MessageInfo("测试"));
|
||||
//URL url = new URL("https://oapi.dingtalk.com/robot/send?access_token=35587c9e7accc54ff7ddf2c6ebdc36a3ba065b4a2bdac78c0e9928596361b430");
|
||||
//// 建立http连接
|
||||
//HttpURLConnection conn = (HttpURLConnection) url.openConnection();
|
||||
//conn.setDoOutput(true);
|
||||
//conn.setDoInput(true);
|
||||
//conn.setUseCaches(false);
|
||||
//conn.setRequestMethod("POST");
|
||||
//conn.setRequestProperty("Charset", "UTF-8");
|
||||
//conn.setRequestProperty("Content-Type", "application/Json; charset=UTF-8");
|
||||
//conn.connect();
|
||||
//OutputStream out = conn.getOutputStream();
|
||||
//String textMessage = JsonUtils.toJson(message);
|
||||
//byte[] data = textMessage.getBytes("UTF-8");
|
||||
//out.write(data);
|
||||
//out.flush();
|
||||
//out.close();
|
||||
//System.out.println(conn.getResponseCode());
|
||||
//InputStream ins = conn.getInputStream();
|
||||
//byte[] data1 = new byte[ins.available()];
|
||||
//ins.read(data1);
|
||||
//System.out.println(new String(data1));
|
||||
|
||||
//HttpRequest request = new HttpRequest()
|
||||
//def params = [:]
|
||||
//params.put("","")
|
||||
//request.post("https://oapi.dingtalk.com/robot/send?access_token=35587c9e7accc54ff7ddf2c6ebdc36a3ba065b4a2bdac78c0e9928596361b430","", "utf-8")
|
||||
|
||||
|
||||
//sendMarkdownMessage("租金", "测试")
|
||||
//static void sendMarkdownMessage(String title, String msg){
|
||||
// HttpRequest request = new HttpRequest()
|
||||
// JSONObject requestBody = new JSONObject()
|
||||
// JSONObject markdown = new JSONObject()
|
||||
// markdown.put("title", title)
|
||||
// markdown.put("text", msg)
|
||||
// requestBody.put("markdown", markdown)
|
||||
// requestBody.put("msgtype", "markdown")
|
||||
// request.postWithBody("https://oapi.dingtalk.com/robot/send?access_token=35587c9e7accc54ff7ddf2c6ebdc36a3ba065b4a2bdac78c0e9928596361b430", requestBody, null, "UTF-8")
|
||||
//}
|
||||
//ExcelUtils.modify("C:\\Users\\gaojuan\\Downloads\\第三方小区区域.xlsx", { d ->
|
||||
// def city = RegionData.formatCity("余姚")
|
||||
// d['D'] = city
|
||||
// if(city){
|
||||
// d['E'] = RegionData.formatRegion(city, d['C'])
|
||||
// }
|
||||
})
|
||||
//
|
||||
//// if(!d['D']){
|
||||
//// def citys = CommonDatautils.getCityByRegion(d['A'])
|
||||
//// if(citys && citys.size()==1){
|
||||
//// d['F'] = citys.get(0)
|
||||
//// }
|
||||
//// }
|
||||
//})
|
||||
|
||||
//def datas = ExcelUtils.read("D:\\数据\\福州\\202305月福州统计数据\\福州马尾第三方.xlsx", 0, 1)
|
||||
//def m = [:]
|
||||
|
||||
+19
@@ -1,9 +1,14 @@
|
||||
package com.fangjia.script.gj.district
|
||||
|
||||
import com.fangjia.core.util.HttpRequest
|
||||
import com.fangjia.script.statistics.poi.AmapPoi
|
||||
import com.fangjia.script.temp.shenjian.util.GaoDeUtils
|
||||
import com.fangjia.script.temp.shenjian.util.HttpUtils
|
||||
import com.fangjia.script.util.HttpRequestUtil
|
||||
import com.mongodb.DBObject
|
||||
import net.sf.json.JSONArray
|
||||
import net.sf.json.JSONObject
|
||||
import org.apache.commons.collections.CollectionUtils
|
||||
import org.apache.commons.lang.StringUtils
|
||||
|
||||
class FangjiaClientUtils {
|
||||
@@ -153,4 +158,18 @@ class FangjiaClientUtils {
|
||||
return columnValMap;
|
||||
}
|
||||
|
||||
|
||||
public static def saveOrUpdateDistrict(data) {
|
||||
String jsonStr = com.alibaba.fastjson.JSONObject.toJSONString(data)
|
||||
// def json = HttpUtils.postForJson("http://192.168.0.100:38080/FangJiaDataService/saveData", jsonStr)
|
||||
// def json = HttpUtils.postForJson("http://localhost:8080/FangJiaDataService/saveData", jsonStr)
|
||||
|
||||
com.alibaba.fastjson.JSONObject json = HttpUtils.doPost2("http://192.168.0.100:38080/FangJiaDataService/saveData", jsonStr)
|
||||
if (json.status == 0) {
|
||||
return "成功"
|
||||
}else if(json.status == 1){
|
||||
return json.msg
|
||||
}
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
+5
-2
@@ -10,6 +10,9 @@ import com.fangjia.script.util.MongoUtils
|
||||
import com.mongodb.BasicDBObject
|
||||
import com.mongodb.DBCollection
|
||||
|
||||
/**
|
||||
*处理地址在地址别名中有重复
|
||||
*/
|
||||
class SameDistName {
|
||||
|
||||
static DBCollection POLYLINECOLL = MongoUtils.getCollectionByAuth20190102("fangjia_crawl", "region_polyline")
|
||||
@@ -18,13 +21,13 @@ class SameDistName {
|
||||
def task = new SameDistName()
|
||||
// task.run()
|
||||
//地址和地址别名完全一致
|
||||
// task.addressPocess()
|
||||
task.addressPocess()
|
||||
}
|
||||
|
||||
|
||||
|
||||
def addressPocess(){
|
||||
def citys = CommonDatautils.findAllCity();
|
||||
def citys = ["福州"]//CommonDatautils.findAllCity();
|
||||
|
||||
for(def c : citys){
|
||||
def dists = CommonDatautils.findAllDistrictByCity(c, "district", ["city","region", "name", "address","addr_alias"])
|
||||
|
||||
+121
@@ -0,0 +1,121 @@
|
||||
package com.fangjia.script.gj.location
|
||||
|
||||
|
||||
import com.fangjia.script.commData.CommonDatautils
|
||||
import com.fangjia.script.looplineinfo.InsidePolygon
|
||||
import com.fangjia.script.util.DbUtils
|
||||
import com.fangjia.script.util.ExcelUtils
|
||||
import com.fangjia.script.util.MongoUtils
|
||||
import com.mongodb.BasicDBObject
|
||||
import com.mongodb.DBCollection
|
||||
import com.mongodb.DBObject
|
||||
|
||||
import java.util.concurrent.CountDownLatch
|
||||
import java.util.concurrent.ExecutorService
|
||||
import java.util.concurrent.Executors
|
||||
import java.util.concurrent.atomic.AtomicLong
|
||||
|
||||
/**
|
||||
* 判断小区是否在城市范围内
|
||||
*/
|
||||
class DistrictLocationBatch {
|
||||
|
||||
public static ExecutorService fixedThreadPool = Executors.newFixedThreadPool(20)
|
||||
static DBCollection RANGECOLL = MongoUtils.getCollectionByAuth20190102("fangjia_crawl", "city_polyline")
|
||||
|
||||
static main(args) {
|
||||
def task = new DistrictLocationBatch()
|
||||
// task.run()
|
||||
task.deleteDist()
|
||||
}
|
||||
|
||||
def deleteDist(){
|
||||
ExcelUtils.multiThreadingModify("d:\\数据\\小区\\非本城市小区.xlsx",{d->
|
||||
DBObject q = new BasicDBObject()
|
||||
q.put("city", d['A'])
|
||||
q.put("region", d['B'])
|
||||
q.put("name", d['C'])
|
||||
q.put("cat", 'district')
|
||||
q.put("status", 0)
|
||||
|
||||
def udata = new BasicDBObject()
|
||||
udata.put("status", 1)
|
||||
udata.put("m_date", new Date())
|
||||
DbUtils.seaweed.update(q, new BasicDBObject('$set', udata))
|
||||
})
|
||||
}
|
||||
|
||||
def run() {
|
||||
def citys = CommonDatautils.findAllCity()
|
||||
def cityMap = findCityRange()
|
||||
def allrets = []
|
||||
for (def city : citys) {
|
||||
def points = cityMap.get(city)
|
||||
def pointList = InsidePolygon.createPoint3(points);
|
||||
def disticts = CommonDatautils.findAllDistricts(city, "district", ["city", "region", "name","address", "lat", "lng"])
|
||||
def rets = findNotInCityDistricts(city, disticts, pointList)
|
||||
allrets.addAll(rets)
|
||||
}
|
||||
ExcelUtils.write("d:\\数据\\小区\\非本城市小区.xlsx", allrets, "经纬度不在此城市中")
|
||||
}
|
||||
|
||||
def findNotInCityDistricts(city, districts, pointList){
|
||||
def rets = Collections.synchronizedList(new ArrayList());
|
||||
def totalCount = new AtomicLong()
|
||||
final CountDownLatch latch = new CountDownLatch(districts.size())
|
||||
|
||||
for(def i=0;i<districts.size();i++){
|
||||
final def dist = districts[i]
|
||||
|
||||
fixedThreadPool.execute(new Runnable() {
|
||||
void run() {
|
||||
try{
|
||||
totalCount.addAndGet(1)
|
||||
if(dist.lng){
|
||||
def flag = isInArea(dist.lng/10000000000.0, dist.lat/10000000000.0, pointList)
|
||||
if(!flag) {
|
||||
def ret = []
|
||||
ret << dist.city
|
||||
ret << dist.region
|
||||
ret << dist.name
|
||||
ret << dist.address
|
||||
println ret
|
||||
rets << ret
|
||||
}
|
||||
}
|
||||
} catch (Exception e) {
|
||||
e.printStackTrace()
|
||||
} finally {
|
||||
latch.countDown()
|
||||
}
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
latch.await()
|
||||
println city+" "+ rets.size() +" "+districts.size()
|
||||
return rets
|
||||
}
|
||||
|
||||
def findCityRange(){
|
||||
def q = new BasicDBObject()
|
||||
def dbcursor = RANGECOLL.find(q)
|
||||
def cityMap = [:]
|
||||
for(def d : dbcursor){
|
||||
cityMap.put(d.fjw_city, d.polyline)
|
||||
}
|
||||
return cityMap
|
||||
}
|
||||
|
||||
def isInArea(lng, lat, pointList){
|
||||
def flag = false;
|
||||
for(def pointArr : pointList){
|
||||
flag = InsidePolygon.isPtInPoly(lng, lat, pointArr)
|
||||
if(flag){
|
||||
break;
|
||||
}
|
||||
}
|
||||
return flag;
|
||||
}
|
||||
|
||||
}
|
||||
+84
@@ -0,0 +1,84 @@
|
||||
package com.fangjia.script.gj.location
|
||||
|
||||
import com.fangjia.core.RegionData
|
||||
import com.fangjia.core.util.HttpRequest
|
||||
import com.fangjia.script.commData.CommonDatautils
|
||||
import com.fangjia.script.util.MongoUtils
|
||||
import com.mongodb.BasicDBObject
|
||||
import com.mongodb.DBCollection
|
||||
import net.sf.json.JSONObject
|
||||
|
||||
class GaodeCityRange {
|
||||
|
||||
static DBCollection POLYLINECOLL = MongoUtils.getCollectionByAuth20190102("fangjia_crawl", "city_polyline")
|
||||
|
||||
static main(args) {
|
||||
def task = new GaodeCityRange()
|
||||
task.run()
|
||||
}
|
||||
|
||||
def run(){
|
||||
def citys = CommonDatautils.findAllCity()
|
||||
|
||||
def citys1= ["上海","北京","广州","深圳","成都","天津","西安","杭州","沈阳","重庆","武汉","南京","石家庄","福州","宁波","贵阳","苏州","长沙","大连","昆明","厦门","太原","无锡","郑州","呼和浩特","济南","哈尔滨","兰州","青岛","佛山","长春","东莞","合肥","烟台","乌鲁木齐","南昌","泉州","齐齐哈尔","包头","常德","大同","温州","银川","西宁","扬州","绵阳","威海","吉林","中山","邯郸","湛江","潍坊","保定","常州","秦皇岛","嘉兴","鞍山","马鞍山","宜昌","徐州","株洲","大庆","镇江","芜湖","赣州","南通","九江","洛阳","泰州","惠州","台州","湖州","绍兴","江门","海口","珠海","金华","桂林","连云港","马鞍山","淄博","柳州","盐城","汕头","衡阳","揭阳","茂名","牡丹江","平顶山","本溪","安阳","舟山","岳阳","许昌","四平","淮南","淮安","赤峰","佳木斯","南阳","临沂","临汾","榆林","三亚","唐山","咸阳","宜宾","自贡","廊坊","南宁","攀枝花","襄阳","延安","丽江","大理","湘潭","六盘水","清远","衢州","宿迁","北海","莆田","玉林","张家界","肇庆","曲靖","新乡","营口","潮州","河源","黄石","焦作","盘锦","韶关","阳江","梅州","云浮","玉溪","遵义","中卫","武威","铁岭","天水","泰安","黄冈","广元","阜阳","定西","辽源","丽水","乐山","菏泽","锦州","汉中","黄山","白山","安顺","安康","绥化","钦州","鄂尔多斯","梧州","百色","鸡西","七台河","白城","朝阳","沧州","东营","德州","亳州","安庆","六安","宁德","三沙","临沧","拉萨","南充","泸州","资阳","乌海","邵阳","克拉玛依","巴彦淖尔","荆门","来宾","开封","信阳","固原","驻马店","防城港","朔州","铜川","呼伦贝尔","双鸭山","平凉","崇左","白银","河池","吉安","通化","景德镇","鹤岗","海东","黑河","阜新","抚顺","葫芦岛","张家口","衡水","邢台","石嘴山","济宁","日照","聊城","滨州","淮北","宣城","三明","宿州","龙岩","保山","池州","铜仁","乌兰察布","南平","内江","达州","贺州","巴中","贵港","昭通","怀化","鄂州","伊春","毕节","十堰","遂宁","漯河","松原","眉山","鹤壁","忻州","郴州","晋中","益阳","辽阳","宝鸡","酒泉","张掖","随州","承德","鹰潭","新余","孝感","枣庄","周口","濮阳","阳泉","运城","蚌埠","渭南","滁州","铜陵","嘉峪关","漳州","庆阳","汕尾","上饶","普洱","宜春","德阳","广安","雅安","永州","娄底","荆州","咸宁","三门峡","商丘","长治","吕梁","商洛","金昌","陇南","萍乡","抚州","吴忠","通辽","晋城","阿克苏","喀什","昌吉","楚雄","大兴安岭","吐鲁番","哈密","和田","塔城","阿勒泰","博尔塔拉","巴音郭楞","克孜勒苏","伊犁","红河","昌都","日喀则","山南","那曲","阿里","林芝","兴安盟","锡林郭勒盟","阿拉善盟","临夏","甘南","黔西南","黔东南","黔南","恩施","湘西","延边","昆玉","丹东","石河子","阿拉尔","图木舒克","五家渠","济源","仙桃","潜江","天门","神农架","五指山","琼海","儋州","文昌","万宁","东方","定安","屯昌","澄迈","临高","白沙","昌江","乐东","陵水","保亭","琼中","北屯","铁门关","双河","可克达拉"]
|
||||
HttpRequest request = new HttpRequest()
|
||||
def rows = []
|
||||
|
||||
citys.each{c->
|
||||
if(!citys1.contains(c)){
|
||||
c = c.replace("州","")
|
||||
JSONObject resp = request.getJsonContent("http://restapi.amap.com/v3/config/district?key=4ccfe3c898a9489884bd2bcb1ef6dcde&keywords=${c}&subdistrict=0&extensions=all", null, "UTF-8")
|
||||
if(resp.containsKey("districts")){
|
||||
def datas = resp.get("districts")
|
||||
for(def d : datas){
|
||||
println c +" "+ d.name+" "+d.level
|
||||
if(["上海市","北京市","天津市","重庆市"].contains(d.name.toString())){
|
||||
rows << d
|
||||
}else{
|
||||
if(d.level=="city"){
|
||||
rows << d
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
|
||||
// citys.each{c->
|
||||
// println c
|
||||
// JSONObject resp = request.getJsonContent("http://restapi.amap.com/v3/config/district?key=4ccfe3c898a9489884bd2bcb1ef6dcde&keywords=${c}&subdistrict=0&extensions=all", null, "UTF-8")
|
||||
// if(resp.containsKey("districts")){
|
||||
// def datas = resp.get("districts")
|
||||
// for(def d : datas){
|
||||
// println c +" "+ d.name+" "+d.level
|
||||
// if(["上海市","北京市","天津市","重庆市"].contains(d.name.toString())){
|
||||
// rows << d
|
||||
// }else{
|
||||
// if(d.level=="city"){
|
||||
// rows << d
|
||||
// }
|
||||
// }
|
||||
// }
|
||||
// }
|
||||
// }
|
||||
if(rows){
|
||||
rows.each {r->
|
||||
def obj = new BasicDBObject()
|
||||
obj.put("level", r.level)
|
||||
obj.put("center", r.center)
|
||||
obj.put("center", r.center)
|
||||
obj.put("polyline", r.polyline)
|
||||
obj.put("name", r.name)
|
||||
obj.put("adcode", r.adcode)
|
||||
obj.put("citycode", r.citycode)
|
||||
|
||||
def city = RegionData.formatCity(r.name)
|
||||
obj.put("fjw_city", city)
|
||||
POLYLINECOLL.save(obj)
|
||||
}
|
||||
}
|
||||
|
||||
}
|
||||
}
|
||||
+127
-13
@@ -3,6 +3,9 @@ package com.fangjia.script.gj.location
|
||||
import com.fangjia.core.RegionData
|
||||
import com.fangjia.core.util.BaiDuApiUtils
|
||||
import com.fangjia.core.util.GeocodeUtils
|
||||
import com.fangjia.script.BaseBatch
|
||||
import com.fangjia.script.BatchOwner
|
||||
import com.fangjia.script.Watcher
|
||||
import com.fangjia.script.commData.CommonDatautils
|
||||
import com.fangjia.script.common.DataHandleUtils
|
||||
import com.fangjia.script.gj.district.FangjiaClientUtils
|
||||
@@ -25,28 +28,122 @@ import java.util.concurrent.atomic.AtomicLong
|
||||
import java.util.regex.Matcher
|
||||
import java.util.regex.Pattern;
|
||||
|
||||
|
||||
class LocationTest {
|
||||
/**
|
||||
* 根据区域围栏修正区域
|
||||
*/
|
||||
@BatchOwner(Watcher.GaoJuan)
|
||||
class RegionLocationUpdateBatch extends BaseBatch{
|
||||
|
||||
static DBCollection POLYLINECOLL = MongoUtils.getCollectionByAuth20190102("fangjia_crawl", "region_polyline")
|
||||
public static ExecutorService fixedThreadPool = Executors.newFixedThreadPool(20)
|
||||
DBCollection OPERATION_LOG_COLL = MongoUtils.getCollectionByName("fangjia", "operation_log", "mongo.db.fangjia.com")
|
||||
|
||||
|
||||
def sqlInstance = Sql.newInstance("jdbc:oracle:thin:@192.168.0.100:31521:orcl",
|
||||
"GOOJIA", "GOOJIA", "oracle.jdbc.driver.OracleDriver")
|
||||
static main(args) {
|
||||
// Date date=new Date(1622802609);
|
||||
// SimpleDateFormat format = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss");
|
||||
// String str = format.format(date);
|
||||
// System.out.println(str);
|
||||
runBatch(new RegionLocationUpdateBatch(), args);
|
||||
}
|
||||
|
||||
def task = new LocationTest()
|
||||
task.run()
|
||||
@Override
|
||||
public void process(Object args) {
|
||||
regionUpdate()
|
||||
// task.run()
|
||||
// task.run2()
|
||||
// task.run3()
|
||||
// task.run4()
|
||||
// task.run5()
|
||||
}
|
||||
|
||||
def regionUpdate(){
|
||||
def citys = CommonDatautils.findAllCity()
|
||||
def rets = []
|
||||
def index = 0
|
||||
for(def city : citys){
|
||||
index ++
|
||||
def regionPointsMap = findRegionRange(city)
|
||||
def districts = CommonDatautils.findAllDistricts(city, "district", ["city", "region", "name", "address", "lat", "lng"]);
|
||||
def datas = findInRegionDistricts(city, districts, regionPointsMap)
|
||||
println "【${index}】"+city +" 经纬度处理完成"
|
||||
rets.addAll(datas)
|
||||
}
|
||||
ExcelUtils.write("D:\\数据\\小区\\区域纠正.xlsx", rets, "数据")
|
||||
}
|
||||
|
||||
|
||||
def findInRegionDistricts(city, districts, regionPointsMap){
|
||||
def index = 0
|
||||
def rets = Collections.synchronizedList(new ArrayList());
|
||||
def totalCount = new AtomicLong()
|
||||
final CountDownLatch latch = new CountDownLatch(districts.size())
|
||||
def regionMap = [:]
|
||||
regionPointsMap.each{k,v->
|
||||
def pointList = InsidePolygon.createPoint3(v);
|
||||
regionMap.put(k, pointList)
|
||||
}
|
||||
def keys = regionPointsMap.keySet()
|
||||
for(def i=0;i<districts.size();i++){
|
||||
final def dist = districts[i]
|
||||
fixedThreadPool.execute(new Runnable() {
|
||||
void run() {
|
||||
try{
|
||||
if(dist.lng){
|
||||
totalCount.addAndGet(1)
|
||||
def pointList = regionMap.get(dist.region)
|
||||
def flag = InsidePolygon.isInArea(dist.lng/10000000000.0, dist.lat/10000000000.0, pointList)
|
||||
def newRegion = null
|
||||
if(!flag){
|
||||
for(def m : regionPointsMap){
|
||||
if(m.key != dist.region){
|
||||
pointList = regionMap.get(m.key)
|
||||
flag = InsidePolygon.isInArea(dist.lng/10000000000.0, dist.lat/10000000000.0, pointList)
|
||||
if(flag){
|
||||
newRegion = m.key
|
||||
break;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if(flag){
|
||||
if(newRegion){
|
||||
println dist.city +" "+dist.region+" "+dist.name+" 【${newRegion}】"
|
||||
def udata = [:]
|
||||
udata.put("region", newRegion)
|
||||
udata.put("status", "0")
|
||||
udata.put("cat", "district")
|
||||
udata.put("username", "高娟")
|
||||
udata.put("id", dist._id.toString())
|
||||
def s = FangjiaClientUtils.saveOrUpdateDistrict(udata)
|
||||
def opt = "update"
|
||||
if(s && s!='成功'){
|
||||
if(s && s.contains("该区域小区已存在")){
|
||||
def delUdata = new BasicDBObject()
|
||||
delUdata.put("status", 1)
|
||||
delUdata.put("m_date", new Date())
|
||||
DbUtils.seaweed.update(new BasicDBObject('_id',dist._id), new BasicDBObject('$set', delUdata))
|
||||
saveOptLog(dist.city, dist.region, dist.name)
|
||||
opt = "delete"
|
||||
}
|
||||
}
|
||||
|
||||
rets << [dist.city, dist.region, dist.name, newRegion, opt]
|
||||
}
|
||||
}else{
|
||||
println dist.city +" "+dist.region+" "+dist.name+" 没有找到对应的区域"
|
||||
}
|
||||
}
|
||||
} catch (Exception e) {
|
||||
e.printStackTrace()
|
||||
} finally {
|
||||
latch.countDown()
|
||||
}
|
||||
}
|
||||
});
|
||||
}
|
||||
latch.await()
|
||||
|
||||
return rets;
|
||||
}
|
||||
|
||||
def run(){
|
||||
def city = "º¼ÖÝ"
|
||||
def regionPointsMap = findRegionRange(city)
|
||||
@@ -446,12 +543,11 @@ class LocationTest {
|
||||
|
||||
def findRegionRange(city){
|
||||
def q = new BasicDBObject()
|
||||
q.put("city",city+"ÊÐ")
|
||||
q.put("city",city)
|
||||
def dbcursor = POLYLINECOLL.find(q)
|
||||
def regionMap = [:]
|
||||
for(def d : dbcursor){
|
||||
def region = RegionData.formatRegion(city,d.region)
|
||||
regionMap.put(region, d.polyline)
|
||||
regionMap.put(d.fjw_region, d.polyline)
|
||||
}
|
||||
return regionMap
|
||||
}
|
||||
@@ -525,4 +621,22 @@ class LocationTest {
|
||||
return rets
|
||||
}
|
||||
|
||||
def saveOptLog(city, region, name){
|
||||
def obj = new BasicDBObject();
|
||||
obj.put("city",city);
|
||||
obj.put("region",region);
|
||||
obj.put("district_name",name);
|
||||
obj.put("category","district");
|
||||
obj.put("c_date", new Date())
|
||||
obj.put("m_date", new Date())
|
||||
obj.put("type", "remove")
|
||||
obj.put("user_name", "高娟")
|
||||
obj.put("origin", "script")
|
||||
OPERATION_LOG_COLL.save(obj)
|
||||
}
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
}
|
||||
+23
-18
@@ -1,25 +1,19 @@
|
||||
package com.fangjia.script.gj
|
||||
package com.fangjia.script.gj.location
|
||||
|
||||
|
||||
import com.fangjia.core.Constants
|
||||
import com.fangjia.core.RegionData
|
||||
import com.fangjia.core.util.HttpRequest
|
||||
import com.fangjia.core.util.Mongo3Utils
|
||||
import com.fangjia.script.commData.CommonDatautils
|
||||
import com.fangjia.script.util.FangJiaAreaMapUtils
|
||||
import com.fangjia.script.util.MongoUtils
|
||||
import com.mongodb.BasicDBObject
|
||||
import com.mongodb.DBCollection
|
||||
import com.mongodb.client.MongoCollection
|
||||
import net.sf.json.JSONObject
|
||||
import org.apache.commons.collections.CollectionUtils
|
||||
import org.apache.commons.lang.math.RandomUtils
|
||||
import org.apache.commons.lang3.StringUtils
|
||||
import org.bson.Document
|
||||
|
||||
import java.util.regex.Pattern
|
||||
|
||||
/**
|
||||
* 抓取区域边框
|
||||
* 高德抓取区域边框
|
||||
*/
|
||||
class RegionPointCrawl {
|
||||
|
||||
@@ -28,13 +22,19 @@ class RegionPointCrawl {
|
||||
|
||||
static main(args) {
|
||||
def task = new RegionPointCrawl();
|
||||
//河北省、山西省、辽宁省、吉林省、黑龙江省、江苏省、浙江省、
|
||||
// def provinces = ["安徽省","福建省","江西省","山东省","河南省","湖南省","广东省","海南省","四川省","贵州省","云南省","陕西省","甘肃省","青海省","台湾省","内蒙古自治区","广西壮族自治区","西藏自治区","宁夏回族自治区","新疆维吾尔自治区"]
|
||||
//def provinces = ["河北省","山西省","辽宁省","吉林省","黑龙江省","江苏省","浙江省","安徽省","福建省","江西省","山东省","河南省","湖北省","湖南省","广东省","海南省","四川省","贵州省","云南省","陕西省","甘肃省","青海省","台湾省"]
|
||||
def provinces = ["福建省"]//"内蒙古自治区","广西壮族自治区","新疆维吾尔自治区"
|
||||
//23个省 河北省、山西省、吉林省、辽宁省、黑龙江省、陕西省、甘肃省、青海省、山东省、福建省、浙江省、台湾省、河南省、湖北省、湖南省、江西省、江苏省、安徽省、广东省、海南省、四川省、贵州省、云南省
|
||||
//4个直辖市 北京市、上海市、天津市、重庆市
|
||||
//5个自治区 "内蒙古自治区","广西壮族自治区","西藏自治区","宁夏回族自治区","新疆维吾尔自治区"
|
||||
// def provinces = ["河北省", "山西省","吉林省","辽宁省", "黑龙江省", "陕西省", "甘肃省", "青海省", "山东省", "福建省", "浙江省", "台湾省", "河南省", "湖北省", "湖南省", "江西省", "江苏省", "安徽省", "广东省", "海南省", "四川省", "贵州省", "云南省"]
|
||||
def provinces = ["甘肃省"]
|
||||
def index = 0
|
||||
for(def p : provinces){
|
||||
index ++
|
||||
println index +"/"+provinces.size() +" "+p
|
||||
task.crawlCityAndDistrictCode(p);
|
||||
println "城市 区域 code 完成"
|
||||
task.run(p);
|
||||
println "区域围栏 完成"
|
||||
}
|
||||
|
||||
}
|
||||
@@ -43,8 +43,8 @@ class RegionPointCrawl {
|
||||
crawlCity(province)
|
||||
}
|
||||
|
||||
def run(city){
|
||||
def cityCodes = findCityCode(city);
|
||||
def run(pro){
|
||||
def cityCodes = findCityCode(pro);
|
||||
if(cityCodes){
|
||||
for(def cityMap : cityCodes){
|
||||
def codeMap = findAllDistrictCode(cityMap.city_code);
|
||||
@@ -59,7 +59,8 @@ class RegionPointCrawl {
|
||||
if(ret.size()>0){
|
||||
def standardCity = RegionData.formatCity(cityMap.name)
|
||||
if(standardCity){
|
||||
saveData(standardCity, m.name, m.ad_code , ret);
|
||||
def newRegion = RegionData.formatRegion(standardCity, m.name)
|
||||
saveData(pro, standardCity, m.name, m.ad_code , ret, newRegion);
|
||||
}else{
|
||||
println cityMap.name +" ûÓбê×¼»¯"
|
||||
}
|
||||
@@ -102,7 +103,7 @@ class RegionPointCrawl {
|
||||
def findCityCode(city){
|
||||
Pattern pattern = Pattern.compile(city, Pattern.CASE_INSENSITIVE);
|
||||
def q = null;
|
||||
if(city.contains("省") || city.endsWith("自治区")){
|
||||
if(city.contains("省") || city.endsWith("自治区") || ["上海市","天津市","北京市","重庆市"].contains(city)){
|
||||
q = new BasicDBObject("province", pattern);
|
||||
}else{
|
||||
q = new BasicDBObject("name", pattern);
|
||||
@@ -156,18 +157,22 @@ class RegionPointCrawl {
|
||||
}
|
||||
}
|
||||
|
||||
def saveData(city, region, code, ret){
|
||||
def saveData(pro, city, region, code, ret, formatRegion){
|
||||
def q = new BasicDBObject()
|
||||
q.put("code", code)
|
||||
def dbo = POLYLINECOLL.findOne(q)
|
||||
if(!dbo){
|
||||
def o = new BasicDBObject()
|
||||
o.put("province", pro)
|
||||
o.put("city", city)
|
||||
o.put("region", region)
|
||||
o.put("code", code)
|
||||
o.put("center", ret.center.get(0))
|
||||
o.put("level", ret.level.get(0))
|
||||
o.put("polyline", ret.polyline.get(0));
|
||||
if(formatRegion){
|
||||
o.put("fjw_region", formatRegion)
|
||||
}
|
||||
POLYLINECOLL.save(o);
|
||||
}
|
||||
}
|
||||
+13
-7
@@ -33,7 +33,7 @@ public class BeiKeCrawlByChrome {
|
||||
def DBCollection thirdPartyColl = CrawalDBUtil.getThirdPartyCol();
|
||||
def lineUrl = "mongodb://developer:G00IDCDEV12385@192.168.0.43:28017";
|
||||
DBCollection DEAL_PRICE_COLL = MongoUtils.getCollectionByName2("fangjia", "deal_price", lineUrl)
|
||||
def startMonth = DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyy-MM"), -1, "yyyy-MM")
|
||||
def startMonth = DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyy-MM"), -2, "yyyy-MM")
|
||||
|
||||
|
||||
//广州,佛山 重庆
|
||||
@@ -204,8 +204,11 @@ public class BeiKeCrawlByChrome {
|
||||
if(dealDate && dealDate.getTime()>=endDate.getTime()){
|
||||
detailUrls << durl
|
||||
def dbo = findTrade(durl , "贝壳找房")
|
||||
|
||||
if (!dbo) {
|
||||
parse(city, e, region, driver)
|
||||
parse(city, e, region, driver, null)
|
||||
}else if(dbo && !dbo.lift){
|
||||
parse(city, e, region, driver, dbo._id)
|
||||
}
|
||||
}else{
|
||||
stop = true;
|
||||
@@ -224,7 +227,7 @@ public class BeiKeCrawlByChrome {
|
||||
return detailUrls
|
||||
}
|
||||
|
||||
def parse(city,e, region, driver){
|
||||
def parse(city,e, region, driver, id){
|
||||
try {
|
||||
def detailUrl = e.select(".title a").attr("href")
|
||||
|
||||
@@ -329,8 +332,11 @@ public class BeiKeCrawlByChrome {
|
||||
detail.create_date = new Date()
|
||||
detail.m_date = new Date()
|
||||
println detail
|
||||
DEAL_PRICE_COLL.save(detail)
|
||||
|
||||
if(id){
|
||||
DEAL_PRICE_COLL.update(new BasicDBObject("_id",id),new BasicDBObject('$set',detail))
|
||||
}else{
|
||||
DEAL_PRICE_COLL.save(detail)
|
||||
}
|
||||
}
|
||||
}
|
||||
}catch(Exception ex){
|
||||
@@ -459,9 +465,9 @@ public class BeiKeCrawlByChrome {
|
||||
if(cityEles){
|
||||
for(def e : cityEles){
|
||||
def href = "https:"+e.attr("href")
|
||||
// if (citys.contains(e.text())){
|
||||
if (citys.contains(e.text())){
|
||||
list << [e.text(), href]
|
||||
// }
|
||||
}
|
||||
}
|
||||
}
|
||||
return list
|
||||
|
||||
+13
-6
@@ -33,7 +33,7 @@ public class LianjiaCrawlByChrome {
|
||||
def DBCollection thirdPartyColl = CrawalDBUtil.getThirdPartyCol();
|
||||
def lineUrl = "mongodb://developer:G00IDCDEV12385@192.168.0.43:28017";
|
||||
DBCollection DEAL_PRICE_COLL = MongoUtils.getCollectionByName2("fangjia", "deal_price", lineUrl)
|
||||
def startMonth = DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyy-MM"), -5, "yyyy-MM")
|
||||
def startMonth = DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyy-MM"), -2, "yyyy-MM")
|
||||
|
||||
static main(args) {
|
||||
def task = new LianjiaCrawlByChrome();
|
||||
@@ -303,7 +303,9 @@ public class LianjiaCrawlByChrome {
|
||||
detailUrls << durl
|
||||
def dbo = findTrade(durl , "Á´¼ÒÔÚÏß")
|
||||
if (!dbo) {
|
||||
parse(city, e, regionText, driver)
|
||||
parse(city, e, regionText, driver, null)
|
||||
}else if(dbo && !dbo.lift){
|
||||
parse(city, e, regionText, driver, dbo._id)
|
||||
}
|
||||
}else{
|
||||
stop = true;
|
||||
@@ -326,7 +328,7 @@ public class LianjiaCrawlByChrome {
|
||||
return detailUrls
|
||||
}
|
||||
|
||||
def parse(city,e, regionText, driver){
|
||||
def parse(city,e, regionText, driver, id){
|
||||
try {
|
||||
def price = e.select(".totalPrice .number").text()
|
||||
if(price){
|
||||
@@ -433,7 +435,12 @@ public class LianjiaCrawlByChrome {
|
||||
detail.create_date = new Date()
|
||||
detail.m_date = new Date()
|
||||
println detail
|
||||
DEAL_PRICE_COLL.save(detail)
|
||||
if(id){
|
||||
DEAL_PRICE_COLL.update(new BasicDBObject("_id",id),new BasicDBObject('$set',detail))
|
||||
}else{
|
||||
DEAL_PRICE_COLL.save(detail)
|
||||
}
|
||||
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -563,9 +570,9 @@ public class LianjiaCrawlByChrome {
|
||||
if(cityEles){
|
||||
for(def e : cityEles) {
|
||||
def href = e.attr("href")
|
||||
// if (!href.contains("fang.lianjia") && citys.contains(e.text())){
|
||||
if (!href.contains("fang.lianjia") && citys.contains(e.text())){
|
||||
cityList << [e.text(), href]
|
||||
// }
|
||||
}
|
||||
}
|
||||
}
|
||||
return cityList
|
||||
|
||||
+27
-7
@@ -2,6 +2,7 @@ package com.fangjia.script.gj.poc
|
||||
|
||||
import com.fangjia.script.ccbevaluation.CcbCommonUtils
|
||||
import com.fangjia.script.commData.CommonDatautils
|
||||
import com.fangjia.script.util.DateUtils
|
||||
import com.fangjia.script.util.ExcelUtils
|
||||
import com.fangjia.script.util.MongoUtils
|
||||
import com.mongodb.BasicDBObject
|
||||
@@ -21,10 +22,11 @@ class AddPriceTest {
|
||||
task.run();
|
||||
}
|
||||
def run(){
|
||||
def filePath = "C:\\Users\\gaojuan\\Desktop\\深圳政府指导价.xlsx"
|
||||
def filePath = "D:\\数据\\建行\\房产估值\\指导价\\202306测试偏差较大案例统计数据——tob价格.xlsx"
|
||||
addGXDPrice(filePath)
|
||||
// addCcbPrice(filePath)
|
||||
// addThirdPrice(filePath)
|
||||
// addFjwPrice(filePath)
|
||||
|
||||
}
|
||||
|
||||
@@ -105,12 +107,12 @@ class AddPriceTest {
|
||||
def addGXDPrice(filepath){
|
||||
ExcelUtils.multiThreadingModify(filepath, { d ->
|
||||
if (d['B']) {
|
||||
// def dist = CommonDatautils.findDistrictById(d['B'])
|
||||
// if(dist){
|
||||
// def gxdPrice = findGXDPrice(dist.city, dist.region, dist.name)
|
||||
def gxdPrice = findGXDPrice(d['D'], d['E'], d['C'])
|
||||
d['G'] = gxdPrice
|
||||
// }
|
||||
def dist = CommonDatautils.findDistrictById(d['B'])
|
||||
if(dist){
|
||||
def gxdPrice = findGXDPrice(dist.city, dist.region, dist.name)
|
||||
// def gxdPrice = findGXDPrice(d['D'], d['E'], d['C'])
|
||||
d['I'] = gxdPrice
|
||||
}
|
||||
}
|
||||
})
|
||||
}
|
||||
@@ -129,4 +131,22 @@ class AddPriceTest {
|
||||
}
|
||||
return price
|
||||
}
|
||||
|
||||
|
||||
def addFjwPrice(path){
|
||||
ExcelUtils.multiThreadingModify(path,{d->
|
||||
if(!d['K']){
|
||||
def dist = CommonDatautils.findDistrictById(d['B'])
|
||||
if(dist){
|
||||
// d['N'] = dist.region
|
||||
def dbo = CommonDatautils.queryDistrictPrice(dist.city, dist.region, dist.name, DateUtils.parseFromTightByPattern("2023-06-25","yyyy-MM-dd"))
|
||||
if(dbo){
|
||||
// d['K'] = dbo.avg_price
|
||||
d['K'] = dbo.avg_price_2b
|
||||
}
|
||||
}
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
+4
@@ -294,4 +294,8 @@ class AddressUtil {
|
||||
return result
|
||||
}
|
||||
|
||||
public static void main(String[] args){
|
||||
def addr = AddressUtil.formatAddress("福州", "北三环路201号琴亭信和小区")
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
+46
-1
@@ -52,7 +52,40 @@ class InsidePolygon {
|
||||
|
||||
return list;
|
||||
}
|
||||
|
||||
|
||||
|
||||
|
||||
public static List<Point> createPoint3(String points){
|
||||
|
||||
List<Point> alllist = new ArrayList<Point>();
|
||||
if(points){
|
||||
List<String[]> arrs = new ArrayList<String[]>();
|
||||
if(points.contains("|")){
|
||||
String[] array = points.split("\\|");
|
||||
arrs.addAll(array)
|
||||
}else{
|
||||
arrs.add(points)
|
||||
}
|
||||
|
||||
for(def arrStr : arrs){
|
||||
List<Point> list = new ArrayList<Point>();
|
||||
String[] array = arrStr.split(";");
|
||||
for(int i =0 ;i<array.length-1;i++){
|
||||
def pointArrs = array[i].split("[,|\\|]")
|
||||
Point point = new Point();
|
||||
double x = Double.parseDouble(pointArrs[0]);
|
||||
double y = Double.parseDouble(pointArrs[1]);
|
||||
point.setLon(x);
|
||||
point.setLat(y);
|
||||
list.add(point);
|
||||
}
|
||||
alllist << list
|
||||
}
|
||||
}
|
||||
|
||||
return alllist;
|
||||
}
|
||||
|
||||
public static List<Map<String, Object>> getPoints(String points){
|
||||
String[] array = points.split("-");
|
||||
List<Map<String, Object>> list = new ArrayList<Map<String, Object>>();
|
||||
@@ -227,4 +260,16 @@ class InsidePolygon {
|
||||
}
|
||||
return false;
|
||||
}
|
||||
|
||||
|
||||
public static def isInArea(lng, lat, pointList){
|
||||
def flag = false;
|
||||
for(def pointArr : pointList){
|
||||
flag = InsidePolygon.isPtInPoly(lng, lat, pointArr)
|
||||
if(flag){
|
||||
break;
|
||||
}
|
||||
}
|
||||
return flag;
|
||||
}
|
||||
}
|
||||
|
||||
+40
@@ -1,7 +1,9 @@
|
||||
package com.fangjia.script.temp.shenjian.util
|
||||
|
||||
import net.sf.json.JSONObject
|
||||
import org.apache.commons.httpclient.HttpClient
|
||||
import org.apache.commons.httpclient.methods.PostMethod
|
||||
import org.apache.commons.httpclient.methods.RequestEntity
|
||||
import org.apache.commons.httpclient.methods.StringRequestEntity
|
||||
|
||||
/**
|
||||
@@ -26,4 +28,42 @@ class HttpUtils {
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
|
||||
static def doPost2(String url, String jsonString){
|
||||
HttpClient httpClient = new HttpClient()
|
||||
PostMethod postMethod = new PostMethod(url)
|
||||
String responseMsg = null;
|
||||
try{
|
||||
StringRequestEntity requestEntity = new StringRequestEntity(
|
||||
jsonString,
|
||||
"application/json",
|
||||
"UTF-8");
|
||||
postMethod.setRequestEntity(requestEntity);
|
||||
httpClient.executeMethod(postMethod);
|
||||
InputStream is = postMethod.getResponseBodyAsStream();
|
||||
BufferedReader br = new BufferedReader(new InputStreamReader(is, "UTF-8"));
|
||||
StringBuffer sb = new StringBuffer("");
|
||||
String str= "";
|
||||
while((str = br.readLine()) != null){
|
||||
sb.append(str);
|
||||
}
|
||||
responseMsg = sb.toString();
|
||||
} catch (Exception ex) {
|
||||
ex.printStackTrace();
|
||||
} finally {
|
||||
postMethod.releaseConnection();
|
||||
}
|
||||
return JSONObject.fromObject(responseMsg);
|
||||
}
|
||||
|
||||
public String encodeHtml(String html) {
|
||||
String mh=html;
|
||||
try {
|
||||
mh=new String(html.getBytes("iso-8859-1"));
|
||||
} catch (UnsupportedEncodingException e) {
|
||||
e.printStackTrace();
|
||||
}
|
||||
return mh;
|
||||
}
|
||||
}
|
||||
|
||||
+19
-1
@@ -28,7 +28,25 @@ class TradeDealPriceStat {
|
||||
|
||||
static main(args) {
|
||||
def task = new TradeDealPriceStat()
|
||||
task.run()
|
||||
// task.run()
|
||||
task.deleteRepeat()
|
||||
}
|
||||
|
||||
def deleteRepeat(){
|
||||
def q = new BasicDBObject()
|
||||
q.put("source", "ÎÒ°®ÎÒ¼Ò")
|
||||
def cursor = DEAL_PRICE.find(q).addOption(Bytes.QUERYOPTION_NOTIMEOUT)
|
||||
def set = new HashSet()
|
||||
for(def c : cursor){
|
||||
def tradeDate = DateUtils.formatDateByPattern(c.trade_date, "yyyyMMdd")
|
||||
def key = c.city +"#"+c.region+"#"+c.district_name+"#"+c.avg_price+"#"+tradeDate
|
||||
if(set.contains(key)){
|
||||
DEAL_PRICE.remove(new BasicDBObject('_id', c._id))
|
||||
println c._id
|
||||
}else{
|
||||
set << key
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
def run(){
|
||||
|
||||
+3
-3
@@ -11,8 +11,8 @@ import org.bson.Document
|
||||
import static com.mongodb.client.model.Filters.*
|
||||
|
||||
MongoCollection<Document> OPEN_URI_LOG = Mongo3Utils.getDbCollection(Constants.SEAWATER_MONGO_URI, "business", "open_uri_log")
|
||||
def startDate = DateUtil.parseDate("20230301", "yyyyMMdd")
|
||||
def endDate = DateUtil.parseDate("20230401", "yyyyMMdd")
|
||||
def startDate = DateUtil.parseDate("20230101", "yyyyMMdd")
|
||||
def endDate = DateUtil.parseDate("20230201", "yyyyMMdd")
|
||||
Calendar calendar = Calendar.getInstance()
|
||||
calendar.setTime(startDate)
|
||||
def count = 0
|
||||
@@ -38,6 +38,6 @@ while (startDate.before(endDate)) {
|
||||
|
||||
startDate = nextDate
|
||||
}
|
||||
ExcelUtils.write("D:\\数据\\东莞\\东莞银行总行(不去重).xlsx", data, "202303")
|
||||
ExcelUtils.write("D:\\数据\\东莞\\东莞银行总行(不去重).xlsx", data, "202301")
|
||||
|
||||
|
||||
|
||||
+2
-2
@@ -11,8 +11,8 @@ import org.bson.Document
|
||||
import static com.mongodb.client.model.Filters.*
|
||||
|
||||
MongoCollection<Document> OPEN_URI_LOG = Mongo3Utils.getDbCollection(Constants.SEAWATER_MONGO_URI, "business", "open_uri_log")
|
||||
def startDate = DateUtil.parseDate("20220901", "yyyyMMdd")
|
||||
def endDate = DateUtil.parseDate("20221001", "yyyyMMdd")
|
||||
def startDate = DateUtil.parseDate("20230201", "yyyyMMdd")
|
||||
def endDate = DateUtil.parseDate("20230301", "yyyyMMdd")
|
||||
Calendar calendar = Calendar.getInstance()
|
||||
calendar.setTime(startDate)
|
||||
def count = 0
|
||||
|
||||
Reference in New Issue
Block a user