update
This commit is contained in:
+12
-8
@@ -9,7 +9,7 @@ import com.mongodb.DBCollection
|
|||||||
/**
|
/**
|
||||||
* 深圳公众号成交数据录入
|
* 深圳公众号成交数据录入
|
||||||
*/
|
*/
|
||||||
def datas = ExcelUtils.read("D:\\数据\\建行\\房产估值\\成交\\深圳9月份二手房成交数据汇总.xls", 0, 1)
|
def datas = ExcelUtils.read("D:\\数据\\建行\\房产估值\\成交\\深圳2024年3月份二手房成交数据汇总.xls", 0, 1)
|
||||||
def lineUrl = "mongodb://developer:G00IDCDEV12385@192.168.0.43:28017";
|
def lineUrl = "mongodb://developer:G00IDCDEV12385@192.168.0.43:28017";
|
||||||
DBCollection DEAL_PRICE_COLL = MongoUtils.getCollectionByName2("fangjia", "deal_price", lineUrl)
|
DBCollection DEAL_PRICE_COLL = MongoUtils.getCollectionByName2("fangjia", "deal_price", lineUrl)
|
||||||
|
|
||||||
@@ -75,18 +75,22 @@ for(def d : datas){
|
|||||||
map.area = d['F'].toString().toDouble()
|
map.area = d['F'].toString().toDouble()
|
||||||
map.total_price = (int)(d['G'].toString().toDouble().toInteger()*10000)
|
map.total_price = (int)(d['G'].toString().toDouble().toInteger()*10000)
|
||||||
map.avg_price = (int)(d['H'].toString().toDouble())
|
map.avg_price = (int)(d['H'].toString().toDouble())
|
||||||
map.sell_price = (int)(d['I'].toString().toDouble()*10000)
|
if(d['I']){
|
||||||
def trade_date = d['J']
|
map.sell_price = (int)(d['I'].toString().toDouble()*10000)
|
||||||
if(trade_date){
|
|
||||||
map.trade_date = trade_date
|
|
||||||
}else{
|
|
||||||
def trade_date1 = DateUtils.getRandomDate("2023-09-01", "2023-09-30")
|
|
||||||
map.trade_date = DateUtils.parseFromTightByPattern(trade_date1, "yyyy-MM-dd")
|
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// def trade_date = d['J']
|
||||||
|
// if(trade_date){
|
||||||
|
// map.trade_date = DateUtils.parseFromTightByPattern(trade_date,'yyyy/MM/dd')
|
||||||
|
// }else{
|
||||||
|
def trade_date1 = DateUtils.getRandomDate("2024-03-01", "2024-03-31")
|
||||||
|
map.trade_date = DateUtils.parseFromTightByPattern(trade_date1, "yyyy-MM-dd")
|
||||||
|
// }
|
||||||
|
|
||||||
map.source = "深房家族"
|
map.source = "深房家族"
|
||||||
map.create_date = new Date()
|
map.create_date = new Date()
|
||||||
map.m_date = new Date()
|
map.m_date = new Date()
|
||||||
|
println map
|
||||||
DEAL_PRICE_COLL.save(map)
|
DEAL_PRICE_COLL.save(map)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
+45
-45
@@ -69,51 +69,51 @@ class BigdataDealBatch extends BaseBatch{
|
|||||||
addDealPriceMatch(city);
|
addDealPriceMatch(city);
|
||||||
}
|
}
|
||||||
|
|
||||||
def rows = []
|
// def rows = []
|
||||||
def headers = []//["城市ID","成交ID","城市名称","行政区ID","行政区名称","街道","门牌号","小区ID","小区名称","楼栋ID","楼栋编号","楼栋名称","单元ID","单元编号","单元名称","户ID","详细地址","建筑年代","权属","权属年限","期数","所在楼层","层类型","总楼层数","建筑类型","建筑结构","建筑面积","套内面积","户型","室","厅","厨","卫","房屋用途","室内结构","朝向","装修情况","评估编号","成交单价","成交总价","申报总价","评估总价","网签合同编号","交易日期","交易类型","出售付款方式","成交周期","是否有电梯","赠送面积(地面花园)","赠送面积(露台)","赠送面积(阳台)","赠送面积(飘窗)","赠送面积(阁楼)","赠送面积(车库)","赠送面积(地下室)","赠送面积(其他)","房源数据来源","中介公司ID","中介公司名称","备注","创建时间","更新时间","来源公司","成交金额","数据时间"]
|
// def headers = []//["城市ID","成交ID","城市名称","行政区ID","行政区名称","街道","门牌号","小区ID","小区名称","楼栋ID","楼栋编号","楼栋名称","单元ID","单元编号","单元名称","户ID","详细地址","建筑年代","权属","权属年限","期数","所在楼层","层类型","总楼层数","建筑类型","建筑结构","建筑面积","套内面积","户型","室","厅","厨","卫","房屋用途","室内结构","朝向","装修情况","评估编号","成交单价","成交总价","申报总价","评估总价","网签合同编号","交易日期","交易类型","出售付款方式","成交周期","是否有电梯","赠送面积(地面花园)","赠送面积(露台)","赠送面积(阳台)","赠送面积(飘窗)","赠送面积(阁楼)","赠送面积(车库)","赠送面积(地下室)","赠送面积(其他)","房源数据来源","中介公司ID","中介公司名称","备注","创建时间","更新时间","来源公司","成交金额","数据时间"]
|
||||||
def cityCodeDataMap = CcbCommonUtils.findCityCodeValue()
|
// def cityCodeDataMap = CcbCommonUtils.findCityCodeValue()
|
||||||
def index = 0
|
// def index = 0
|
||||||
for(def city : citys){
|
// for(def city : citys){
|
||||||
index ++
|
// index ++
|
||||||
def distCodeMap = CcbCommonUtils.findAllDistrictCodeValue(city)
|
// def distCodeMap = CcbCommonUtils.findAllDistrictCodeValue(city)
|
||||||
def ppeDists = [:]
|
// def ppeDists = [:]
|
||||||
|
//
|
||||||
def dealIds = findHistoryDealIds(CcbCommonUtils.getCcbCity(city))
|
// def dealIds = findHistoryDealIds(CcbCommonUtils.getCcbCity(city))
|
||||||
println "历史id加载量:"+dealIds.size()
|
// println "历史id加载量:"+dealIds.size()
|
||||||
def statMap = [:]//getDistrictStatByCity(city, weekend)
|
// def statMap = [:]//getDistrictStatByCity(city, weekend)
|
||||||
if(city=="上海"){
|
// if(city=="上海"){
|
||||||
def youDataTrade = getYouDaData(statMap)
|
// def youDataTrade = getYouDaData(statMap)
|
||||||
def youdaDatas = createData(city, youDataTrade, cityCodeDataMap, distCodeMap, "youda", dealIds)
|
// def youdaDatas = createData(city, youDataTrade, cityCodeDataMap, distCodeMap, "youda", dealIds)
|
||||||
rows.addAll(youdaDatas)
|
// rows.addAll(youdaDatas)
|
||||||
println city +" "+youdaDatas.size()
|
// println city +" "+youdaDatas.size()
|
||||||
}
|
// }
|
||||||
|
//
|
||||||
def trades = getCityData(city, statMap, ppeDists)
|
// def trades = getCityData(city, statMap, ppeDists)
|
||||||
if(trades){
|
// if(trades){
|
||||||
def rets = createData(city, trades, cityCodeDataMap, distCodeMap, "other", dealIds)
|
// def rets = createData(city, trades, cityCodeDataMap, distCodeMap, "other", dealIds)
|
||||||
rows.addAll(rets)
|
// rows.addAll(rets)
|
||||||
println "【${index}】"+city +" 成交共:"+rets.size()
|
// println "【${index}】"+city +" 成交共:"+rets.size()
|
||||||
|
//
|
||||||
def cityDatas = []
|
// def cityDatas = []
|
||||||
cityDatas << headers
|
// cityDatas << headers
|
||||||
cityDatas.addAll(rows)
|
// cityDatas.addAll(rows)
|
||||||
}
|
// }
|
||||||
|
//
|
||||||
}
|
// }
|
||||||
|
//
|
||||||
def fileName = "/home/goojia/ccb/BIGDATA_DEAL_GJ.dat"
|
// def fileName = "/home/goojia/ccb/BIGDATA_DEAL_GJ.dat"
|
||||||
if(CommonDatautils.isWindow()){
|
// if(CommonDatautils.isWindow()){
|
||||||
fileName = "d:\\数据\\建行\\房产估值\\数据文件\\txt\\BIGDATA_DEAL_GJ.dat"
|
// fileName = "d:\\数据\\建行\\房产估值\\数据文件\\txt\\BIGDATA_DEAL_GJ.dat"
|
||||||
}
|
// }
|
||||||
|
//
|
||||||
FileUtils.batchCreateFile(rows, fileName,headers, "utf-8", false);
|
// FileUtils.batchCreateFile(rows, fileName,headers, "utf-8", false);
|
||||||
println "txt 生成 " + rows.size()
|
// println "txt 生成 " + rows.size()
|
||||||
|
//
|
||||||
// delPgData()
|
//// delPgData()
|
||||||
saveToPgSql(rows)
|
// saveToPgSql(rows)
|
||||||
|
//
|
||||||
// def datas = readTxt("D:\\数据\\建行\\建行成交\\MODEL_000000_000000_BIGDATA_DEAL_GJ_ADD_20230928_0001.dat")
|
//// def datas = readTxt("D:\\数据\\建行\\建行成交\\MODEL_000000_000000_BIGDATA_DEAL_GJ_ADD_20230928_0001.dat")
|
||||||
// saveToPgSql2(datas)
|
//// saveToPgSql2(datas)
|
||||||
}
|
}
|
||||||
|
|
||||||
def readTxt(def fileName){
|
def readTxt(def fileName){
|
||||||
|
|||||||
+10
-8
@@ -388,19 +388,21 @@ class DealPriceDataProcess {
|
|||||||
|
|
||||||
def deleteThirdPartyData(){
|
def deleteThirdPartyData(){
|
||||||
// ,"陵水","塔城","保亭","长沙","苏州","南昌","沧州","文昌","衡水"
|
// ,"陵水","塔城","保亭","长沙","苏州","南昌","沧州","文昌","衡水"
|
||||||
def citys = ["469005"]//CcbCommonUtils.citys_all
|
// def citys = ["310000","440100","500000","410100","430100","330100","340100","320100","370200","320500","120000","442000","320300","350200","360100","320400","210200","530100","469005","469028","469029","230700","610700","533400","650400","511100","131100"]//CcbCommonUtils.citys_all
|
||||||
|
def citys = CcbCommonUtils.citys_all
|
||||||
def month = Integer.parseInt(DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyyMM"), -1, "yyyyMM"));//上个月日期
|
def month = Integer.parseInt(DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyyMM"), -1, "yyyyMM"));//上个月日期
|
||||||
def rows = []
|
def rows = []
|
||||||
def cityCodeDatas = CcbCommonUtils.findCityCodeValue()
|
def cityCodeDatas = CcbCommonUtils.findCityCodeValue()
|
||||||
def set = new HashSet()
|
def set = new HashSet()
|
||||||
for(def cityid : citys) {
|
for(def city : citys) {
|
||||||
|
def cityid = cityCodeDatas.get(city)
|
||||||
def rets = findPeGteThirdParty2(cityid, DateUtils.formatDateByPattern(new Date(),"yyyyMM"))//当前月日期
|
def rets = findPeGteThirdParty2(cityid, DateUtils.formatDateByPattern(new Date(),"yyyyMM"))//当前月日期
|
||||||
println cityid +" "+rets.size()
|
println city +" "+rets.size()
|
||||||
if(rets){
|
if(rets){
|
||||||
set.add(cityid)
|
set.add(city)
|
||||||
}
|
}
|
||||||
for(def m : rets){
|
for(def m : rets){
|
||||||
// def cityId = cityCodeDatas.get(city)
|
|
||||||
// def sql = "delete from BIGDATA_THIRD_PARTY_PRICE_GJ where cityid='${cityId}' and commid='${m.commid}' and price_date=${month} and community_price=${m.price}"
|
// def sql = "delete from BIGDATA_THIRD_PARTY_PRICE_GJ where cityid='${cityId}' and commid='${m.commid}' and price_date=${month} and community_price=${m.price}"
|
||||||
// sqlInstance236.executeUpdate(sql)
|
// sqlInstance236.executeUpdate(sql)
|
||||||
def querySql = "select count(1) as count from base.t0831_bigdata_third_party_price_gj_h where urbn_id='${cityid}' and gujia_bds_id='${m.commid}' and gujiabds_avprc_dt_val=${month} and gujia_bds_comm_avprc=${m.price}"
|
def querySql = "select count(1) as count from base.t0831_bigdata_third_party_price_gj_h where urbn_id='${cityid}' and gujia_bds_id='${m.commid}' and gujiabds_avprc_dt_val=${month} and gujia_bds_comm_avprc=${m.price}"
|
||||||
@@ -409,7 +411,7 @@ class DealPriceDataProcess {
|
|||||||
total = r.count
|
total = r.count
|
||||||
})
|
})
|
||||||
if(total>0){
|
if(total>0){
|
||||||
println cityid +" "+m.commid+" (按价格)删除:"+total
|
println city +" "+m.commid+" (按价格)删除:"+total
|
||||||
def sql = "delete from base.t0831_bigdata_third_party_price_gj_h where urbn_id='${cityid}' and gujia_bds_id='${m.commid}' and gujiabds_avprc_dt_val=${month} and gujia_bds_comm_avprc=${m.price}"
|
def sql = "delete from base.t0831_bigdata_third_party_price_gj_h where urbn_id='${cityid}' and gujia_bds_id='${m.commid}' and gujiabds_avprc_dt_val=${month} and gujia_bds_comm_avprc=${m.price}"
|
||||||
pgsqlInstance.executeUpdate(sql)
|
pgsqlInstance.executeUpdate(sql)
|
||||||
}else{
|
}else{
|
||||||
@@ -419,7 +421,7 @@ class DealPriceDataProcess {
|
|||||||
total = r.count
|
total = r.count
|
||||||
})
|
})
|
||||||
if(total>0){
|
if(total>0){
|
||||||
println cityId +" "+m.commid+" (按小区)删除:"+total
|
println city +" "+m.commid+" (按小区)删除:"+total
|
||||||
def sql = "delete from base.t0831_bigdata_third_party_price_gj_h where urbn_id='${cityid}' and gujia_bds_id='${m.commid}' and gujiabds_avprc_dt_val=${month} "
|
def sql = "delete from base.t0831_bigdata_third_party_price_gj_h where urbn_id='${cityid}' and gujia_bds_id='${m.commid}' and gujiabds_avprc_dt_val=${month} "
|
||||||
pgsqlInstance.executeUpdate(sql)
|
pgsqlInstance.executeUpdate(sql)
|
||||||
}
|
}
|
||||||
@@ -450,7 +452,7 @@ class DealPriceDataProcess {
|
|||||||
}
|
}
|
||||||
|
|
||||||
def findPeGteThirdParty2(cityid, yearmonth){
|
def findPeGteThirdParty2(cityid, yearmonth){
|
||||||
def sql = "SELECT gujia_comm_id,gujiabds_to_mkprc_mny from t_app_p9pha_plda.avm_evaluation_item_gj_h where gujiabds_clc_yrmo_val=${yearmonth} and gujia_data_tp='第三方' and urbn_id='${cityid}' and gujia_mns_rto>20"
|
def sql = "SELECT gujia_comm_id,gujiabds_to_mkprc_mny from t_app_p9pha_plda.avm_evaluation_item_gj_h where gujiabds_clc_yrmo_val=${yearmonth} and gujia_data_tp='第三方' and urbn_id='${cityid}' and gujia_mns_rto>25"
|
||||||
def rets = []
|
def rets = []
|
||||||
pgsqlInstance.eachRow(sql,{r->
|
pgsqlInstance.eachRow(sql,{r->
|
||||||
def map = [:]
|
def map = [:]
|
||||||
|
|||||||
+1
-1
@@ -52,7 +52,7 @@ class GuiDingPriceBatch extends BaseBatch{
|
|||||||
def distCodeMap = CcbCommonUtils.findAllDistrictCodeValue(city)
|
def distCodeMap = CcbCommonUtils.findAllDistrictCodeValue(city)
|
||||||
|
|
||||||
//添加自己库中的人工指导价
|
//添加自己库中的人工指导价
|
||||||
def prices = CcbCommonUtils.findPersonPrice(city, "2024-03-27", "2024-04-10")
|
def prices = CcbCommonUtils.findPersonPrice(city, "2024-03-27", "2024-04-26")
|
||||||
// def prices = CcbCommonUtils.findPersonPrice(city, "2022-01-01", "2022-11-22")
|
// def prices = CcbCommonUtils.findPersonPrice(city, "2022-01-01", "2022-11-22")
|
||||||
if(prices){
|
if(prices){
|
||||||
def distMap = getDistrictMap(city)
|
def distMap = getDistrictMap(city)
|
||||||
|
|||||||
+52
-56
@@ -1289,70 +1289,66 @@ def findAvgPrice(def commid, def pgsqlInstance, def cityid){
|
|||||||
|
|
||||||
|
|
||||||
|
|
||||||
//def fileName = "D:\\数据\\建行\\房产估值\\指导价\\20240127.xlsx"
|
def fileName = "D:\\数据\\建行\\房产估值\\指导价\\20240425--人工指导价.xls"
|
||||||
//
|
|
||||||
//ExcelUtils.multiThreadingModify(fileName, {d->
|
ExcelUtils.multiThreadingModify(fileName, {d->
|
||||||
// if(d['A']){
|
if(d['C']){
|
||||||
//// def region = RegionData.formatRegion(d['B'], d['C'])
|
// def region = RegionData.formatRegion(d['B'], d['C'])
|
||||||
//// if(region){
|
// if(region){
|
||||||
//// def dist = CommonDatautils.findDistrict(d['B'], region, d['E'], "district")
|
// def dist = CommonDatautils.findDistrict(d['B'], region, d['E'], "district")
|
||||||
//// if(dist){
|
|
||||||
//// d['G'] = dist.region
|
|
||||||
//// d['H'] = dist._id.toString()
|
|
||||||
//// }
|
|
||||||
//// }
|
|
||||||
// if(!d['I']){
|
|
||||||
// def dist = CommonDatautils.findDistrictById(d['C'])
|
|
||||||
// if(dist){
|
// if(dist){
|
||||||
// d['F'] = dist.region
|
// d['G'] = dist.region
|
||||||
//// d['H'] = dist.name
|
// d['H'] = dist._id.toString()
|
||||||
// d['G'] = dist.status
|
|
||||||
//// if(d['A']=="天门" || d['A']=="迪庆"){
|
|
||||||
//// def val = d['E'].toString().toDouble().toInteger() + RandomUtil.random(-200,200)
|
|
||||||
//// d['K'] = val.toInteger()
|
|
||||||
//// }
|
|
||||||
// }
|
// }
|
||||||
// }
|
// }
|
||||||
// }
|
if(!d['L']){
|
||||||
//},0)
|
def dist = CommonDatautils.findDistrictById(d['C'])
|
||||||
|
if(dist){
|
||||||
def fileName = "D:\\数据\\建行\\房产估值\\指导价\\2024-04-08修改人工指导价.xlsx"
|
d['L'] = dist.region
|
||||||
ExcelUtils.multiThreadingModify(fileName, { d ->
|
d['M'] = dist.name
|
||||||
if(d['B'] && !d['D']){
|
d['N'] = dist.status
|
||||||
def jsonObject = FangjiaClientUtils.newMatch(d['A'], d['B'], "property", "district")
|
|
||||||
if(jsonObject != null && jsonObject.credit>0) {
|
|
||||||
d['D'] = jsonObject.get("city")
|
|
||||||
d['E'] = jsonObject.get("district")
|
|
||||||
d['F'] = jsonObject.get("name")
|
|
||||||
def dbo = CommonDatautils.findDistrictById(jsonObject.get("id"))
|
|
||||||
if(dbo){
|
|
||||||
d['G'] = dbo.status
|
|
||||||
d['H'] = dbo.name_cat
|
|
||||||
d['I'] = dbo.estate_type2
|
|
||||||
d['J'] = "接口匹配"
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
}
|
||||||
|
},0)
|
||||||
|
|
||||||
// def objs = CommonDatautils.findDistrictList(d['A'],null, d['B'], "district")
|
//def fileName = "D:\\数据\\建行\\房产估值\\指导价\\2024-04-08修改人工指导价.xlsx"
|
||||||
// if(!objs){
|
//ExcelUtils.multiThreadingModify(fileName, { d ->
|
||||||
// objs = CommonDatautils.findDistrictByAlias(d['A'], null, d['B'], "district")
|
// if(d['B'] && !d['D']){
|
||||||
// }
|
// def jsonObject = FangjiaClientUtils.newMatch(d['A'], d['B'], "property", "district")
|
||||||
// if(objs && objs.size()==1){
|
// if(jsonObject != null && jsonObject.credit>0) {
|
||||||
// def o = objs[0]
|
// d['D'] = jsonObject.get("city")
|
||||||
// if(o){
|
// d['E'] = jsonObject.get("district")
|
||||||
// d['D'] = o.city
|
// d['F'] = jsonObject.get("name")
|
||||||
// d['E'] = o.region
|
// def dbo = CommonDatautils.findDistrictById(jsonObject.get("id"))
|
||||||
// d['F'] = o.name
|
// if(dbo){
|
||||||
// d['G'] = o.status
|
// d['G'] = dbo.status
|
||||||
// d['H'] = o.name_cat
|
// d['H'] = dbo.name_cat
|
||||||
// d['I'] = o.estate_type2
|
// d['I'] = dbo.estate_type2
|
||||||
// d['J'] = "别名"
|
// d['J'] = "接口匹配"
|
||||||
// }
|
// }
|
||||||
// }
|
// }
|
||||||
|
//
|
||||||
|
//// def objs = CommonDatautils.findDistrictList(d['A'],null, d['B'], "district")
|
||||||
}
|
//// if(!objs){
|
||||||
})
|
//// objs = CommonDatautils.findDistrictByAlias(d['A'], null, d['B'], "district")
|
||||||
|
//// }
|
||||||
|
//// if(objs && objs.size()==1){
|
||||||
|
//// def o = objs[0]
|
||||||
|
//// if(o){
|
||||||
|
//// d['D'] = o.city
|
||||||
|
//// d['E'] = o.region
|
||||||
|
//// d['F'] = o.name
|
||||||
|
//// d['G'] = o.status
|
||||||
|
//// d['H'] = o.name_cat
|
||||||
|
//// d['I'] = o.estate_type2
|
||||||
|
//// d['J'] = "别名"
|
||||||
|
//// }
|
||||||
|
//// }
|
||||||
|
//
|
||||||
|
//
|
||||||
|
// }
|
||||||
|
//})
|
||||||
|
|
||||||
//def fileName = "D:\\数据\\建行\\建行成交\\咸阳成交.xlsx"
|
//def fileName = "D:\\数据\\建行\\建行成交\\咸阳成交.xlsx"
|
||||||
//ExcelUtils.multiThreadingModify(fileName, { d ->
|
//ExcelUtils.multiThreadingModify(fileName, { d ->
|
||||||
|
|||||||
+31
@@ -1,6 +1,7 @@
|
|||||||
package com.fangjia.script.commData
|
package com.fangjia.script.commData
|
||||||
|
|
||||||
import com.fangjia.core.RegionData
|
import com.fangjia.core.RegionData
|
||||||
|
import com.fangjia.datastore.driver.FacetResult
|
||||||
import com.fangjia.httpclient.UserAgents
|
import com.fangjia.httpclient.UserAgents
|
||||||
import com.fangjia.script.common.DataHandleUtils
|
import com.fangjia.script.common.DataHandleUtils
|
||||||
import com.fangjia.script.looplineinfo.InsidePolygon
|
import com.fangjia.script.looplineinfo.InsidePolygon
|
||||||
@@ -8,11 +9,16 @@ import com.fangjia.script.util.DbUtils
|
|||||||
import com.fangjia.script.util.ElasticUtils
|
import com.fangjia.script.util.ElasticUtils
|
||||||
import com.fangjia.script.util.JacksonUtil
|
import com.fangjia.script.util.JacksonUtil
|
||||||
import com.fangjia.script.yinjihaun.amap.GaoDeApiUtils
|
import com.fangjia.script.yinjihaun.amap.GaoDeApiUtils
|
||||||
|
import com.mongodb.AggregationOutput
|
||||||
|
import com.mongodb.client.MongoCursor
|
||||||
|
import com.mongodb.client.model.Accumulators
|
||||||
|
import com.mongodb.client.model.Aggregates
|
||||||
import groovy.json.JsonSlurper
|
import groovy.json.JsonSlurper
|
||||||
|
|
||||||
import net.sf.json.JSONArray
|
import net.sf.json.JSONArray
|
||||||
import net.sf.json.JSONObject
|
import net.sf.json.JSONObject
|
||||||
import org.apache.commons.lang.StringUtils
|
import org.apache.commons.lang.StringUtils
|
||||||
|
import org.bson.Document
|
||||||
import org.bson.types.ObjectId
|
import org.bson.types.ObjectId
|
||||||
|
|
||||||
import com.fangjia.core.util.HttpRequest;
|
import com.fangjia.core.util.HttpRequest;
|
||||||
@@ -110,6 +116,7 @@ class CommonDatautils {
|
|||||||
FITMENTS_TYPES_MAPS.put("豪装", ["豪华装修", "高档", "豪华", "豪装", "高级装修", "高装", "奢侈"])
|
FITMENTS_TYPES_MAPS.put("豪装", ["豪华装修", "高档", "豪华", "豪装", "高级装修", "高装", "奢侈"])
|
||||||
}
|
}
|
||||||
|
|
||||||
|
public static final DBCollection COMMUNITY_CONN = MongoUtils.getCollectionByName2("fangjia", "community", "mongodb://admin:G00MONGOADMIN@192.168.0.100:30009")
|
||||||
|
|
||||||
public static DSCollection getHouseDataStoreConnection(){
|
public static DSCollection getHouseDataStoreConnection(){
|
||||||
if(house_dataStoreConnection != null) {
|
if(house_dataStoreConnection != null) {
|
||||||
@@ -1459,6 +1466,28 @@ class CommonDatautils {
|
|||||||
return obj;
|
return obj;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
static def findSeaweedListByMappingAlias(def city, def region, def name, def cat){
|
||||||
|
DBObject q = new BasicDBObject();
|
||||||
|
if(region){
|
||||||
|
q.put('region', region)
|
||||||
|
}
|
||||||
|
|
||||||
|
q.put('city',city)
|
||||||
|
q.put('cat', cat)
|
||||||
|
q.put('status',0)
|
||||||
|
|
||||||
|
BasicDBObject query = new BasicDBObject();
|
||||||
|
query.putAt(QueryOperators.IN, [name]);
|
||||||
|
q.put("mapping_alias", query);
|
||||||
|
|
||||||
|
def dbcursor = SEAWEED_COLL.find(q);
|
||||||
|
def rets = []
|
||||||
|
for(def d : dbcursor){
|
||||||
|
rets << d
|
||||||
|
}
|
||||||
|
return rets;
|
||||||
|
}
|
||||||
|
|
||||||
static def findSeaweeds(city, cat, name_cat){
|
static def findSeaweeds(city, cat, name_cat){
|
||||||
DBObject q = new BasicDBObject();
|
DBObject q = new BasicDBObject();
|
||||||
q.put('city',city)
|
q.put('city',city)
|
||||||
@@ -4601,4 +4630,6 @@ class CommonDatautils {
|
|||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -0,0 +1,3 @@
|
|||||||
|
package com.fangjia.script.fuzhou
|
||||||
|
|
||||||
|
|
||||||
+4
-4
@@ -29,11 +29,11 @@ class FzDataResultStat {
|
|||||||
|
|
||||||
static main(args) {
|
static main(args) {
|
||||||
def task = new FzDataResultStat()
|
def task = new FzDataResultStat()
|
||||||
// task.priceMarkupRun();
|
task.priceMarkupRun();
|
||||||
// task.regionPriceTrendRun()
|
// task.regionPriceTrendRun()
|
||||||
//// task.margeExcel()
|
//// task.margeExcel()
|
||||||
// task.saveCsvData()
|
// task.saveCsvData()
|
||||||
task.testNewPrice()
|
// task.testNewPrice()
|
||||||
}
|
}
|
||||||
|
|
||||||
def margeExcel(){
|
def margeExcel(){
|
||||||
@@ -58,7 +58,7 @@ class FzDataResultStat {
|
|||||||
}
|
}
|
||||||
def priceMarkupRun(){
|
def priceMarkupRun(){
|
||||||
def curMonth = Integer.parseInt(DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyyMM"), 0, "yyyyMM")+"15");
|
def curMonth = Integer.parseInt(DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyyMM"), 0, "yyyyMM")+"15");
|
||||||
def lastMonth = Integer.parseInt(DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyyMM"), -2, "yyyyMM"));
|
def lastMonth = Integer.parseInt(DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyyMM"), -1, "yyyyMM"));
|
||||||
|
|
||||||
// List<String> list1= FileUtils.readLines(new File("D:\\数据\\福州\\FJW_DISTRICT_STAT_${curMonth}.csv"))
|
// List<String> list1= FileUtils.readLines(new File("D:\\数据\\福州\\FJW_DISTRICT_STAT_${curMonth}.csv"))
|
||||||
// def dists = CommonDatautils.findAllDistricts("福州", "district", ["status"])
|
// def dists = CommonDatautils.findAllDistricts("福州", "district", ["status"])
|
||||||
@@ -82,7 +82,7 @@ class FzDataResultStat {
|
|||||||
}
|
}
|
||||||
|
|
||||||
// List<String> list2 = FileUtils.readLines(new File("D:\\数据\\福州\\FJW_DISTRICT_STAT_${lastMonth}.csv"))
|
// List<String> list2 = FileUtils.readLines(new File("D:\\数据\\福州\\FJW_DISTRICT_STAT_${lastMonth}.csv"))
|
||||||
def list2 = findFzDistrictStat(lastMonth)
|
def list2 = findFzDistrictStat(Integer.parseInt(DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyyMM"), -2, "yyyyMM")))
|
||||||
def map2 = [:]
|
def map2 = [:]
|
||||||
for(def obj : list2) {
|
for(def obj : list2) {
|
||||||
def key = obj.city + "##" + obj.region + "##" + obj.name
|
def key = obj.city + "##" + obj.region + "##" + obj.name
|
||||||
|
|||||||
@@ -14,7 +14,7 @@ import com.mongodb.DBCollection
|
|||||||
//福州第三方均价统计,发送钉钉消息
|
//福州第三方均价统计,发送钉钉消息
|
||||||
//def datas = ExcelUtils.read("d:\\数据\\福州\\福州4月2.xlsx",0,1)
|
//def datas = ExcelUtils.read("d:\\数据\\福州\\福州4月2.xlsx",0,1)
|
||||||
//def datas = CsvUtils.read("d:\\数据\\福州\\FJW_THIRD_PARTY_AVG_PRICE_202105.csv");
|
//def datas = CsvUtils.read("d:\\数据\\福州\\FJW_THIRD_PARTY_AVG_PRICE_202105.csv");
|
||||||
def datas = readTxt("d:\\数据\\福州\\FJW_THIRD_PARTY_AVG_PRICE_202402.csv")
|
def datas = readTxt("d:\\数据\\福州\\FJW_THIRD_PARTY_AVG_PRICE_202403.csv")
|
||||||
def count0 = 0
|
def count0 = 0
|
||||||
def count1 = 0
|
def count1 = 0
|
||||||
def count2 = 0
|
def count2 = 0
|
||||||
|
|||||||
+53
-25
@@ -6,6 +6,7 @@ import com.fangjia.script.ccbevaluation.PgsqlCommUtils
|
|||||||
import com.fangjia.script.commData.CommonDatautils
|
import com.fangjia.script.commData.CommonDatautils
|
||||||
import com.fangjia.script.gj.EsSearchUtils
|
import com.fangjia.script.gj.EsSearchUtils
|
||||||
import com.fangjia.script.gj.district.FangjiaClientUtils
|
import com.fangjia.script.gj.district.FangjiaClientUtils
|
||||||
|
import com.fangjia.script.linxsh.util.AddressUtil
|
||||||
import com.fangjia.script.util.DbUtils
|
import com.fangjia.script.util.DbUtils
|
||||||
import com.fangjia.script.util.ElasticUtils
|
import com.fangjia.script.util.ElasticUtils
|
||||||
import com.fangjia.script.util.ExcelUtils
|
import com.fangjia.script.util.ExcelUtils
|
||||||
@@ -32,7 +33,9 @@ class GXDdistrictMatch {
|
|||||||
|
|
||||||
// task.updateDistStatus()//恢复name_cat不正常数据
|
// task.updateDistStatus()//恢复name_cat不正常数据
|
||||||
// task.saveToDB() //保存新增小区
|
// task.saveToDB() //保存新增小区
|
||||||
task.restoreDist()//恢复小区status=0
|
// task.restoreDist()//恢复小区status=0
|
||||||
|
|
||||||
|
task.runMatch()
|
||||||
}
|
}
|
||||||
|
|
||||||
def addOldMatch(){
|
def addOldMatch(){
|
||||||
@@ -133,6 +136,22 @@ class GXDdistrictMatch {
|
|||||||
})
|
})
|
||||||
}
|
}
|
||||||
|
|
||||||
|
def runMatch(){
|
||||||
|
def fileName = "D:\\数据\\福州\\福州贝壳数据.xlsx"//11城重新匹配.xlsx
|
||||||
|
def city = "福州"
|
||||||
|
ExcelUtils.modify(fileName, { d ->
|
||||||
|
println d['C']
|
||||||
|
if(d['B'] && d['A']){
|
||||||
|
def region = RegionData.formatRegion(city, d['B'])
|
||||||
|
if (region) {
|
||||||
|
d = match(city, region, d)
|
||||||
|
} else {
|
||||||
|
println d['A'] + "#" + d['E'] + " 区域无法格式化"
|
||||||
|
}
|
||||||
|
}
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
def updateDistStatus(){
|
def updateDistStatus(){
|
||||||
def fileName = "D:\\数据\\国信达\\国信达11城(全).xlsx"
|
def fileName = "D:\\数据\\国信达\\国信达11城(全).xlsx"
|
||||||
def index = 0
|
def index = 0
|
||||||
@@ -187,27 +206,32 @@ class GXDdistrictMatch {
|
|||||||
}
|
}
|
||||||
|
|
||||||
def saveToDB(){
|
def saveToDB(){
|
||||||
def fileName = "D:\\数据\\国信达\\国信达11城未匹配数据.xlsx"
|
// def fileName = "D:\\数据\\国信达\\国信达11城未匹配数据.xlsx"
|
||||||
|
def fileName = "D:\\数据\\数据建设\\小区\\东莞未匹配成交数据.xlsx"
|
||||||
ExcelUtils.modify(fileName, { d ->
|
ExcelUtils.modify(fileName, { d ->
|
||||||
if(d['U'] && d['AG']!='过滤'){
|
if(d['A']){
|
||||||
def udata = new BasicDBObject()
|
def udata = new BasicDBObject()
|
||||||
udata.put("city", d['AD'])
|
udata.put("city", d['A'])
|
||||||
udata.put("region", d['AE'])
|
udata.put("region", d['B'])
|
||||||
def name = d['X'].replaceAll('[·\\(\\)()]|·', '').replace("建设中", "")
|
def name = d['C'].replaceAll('[·\\(\\)()]|·', '').replace("建设中", "")
|
||||||
udata.put("name", name)
|
udata.put("name", name)
|
||||||
|
|
||||||
def alias = []
|
def alias = []
|
||||||
if(name!=d['C']){
|
if(d['G'] && name!=d['G']){
|
||||||
alias << d['C']
|
alias << d['G']
|
||||||
}
|
|
||||||
if(d['AI'] && !alias.contains(d['AI']) && d['AI']!=name){
|
|
||||||
alias << d['AI']
|
|
||||||
}
|
}
|
||||||
|
// if(d['AI'] && !alias.contains(d['AI']) && d['AI']!=name){
|
||||||
|
// alias << d['AI']
|
||||||
|
// }
|
||||||
if(alias){
|
if(alias){
|
||||||
udata.put("alias", alias)
|
udata.put("alias", alias)
|
||||||
}
|
}
|
||||||
|
|
||||||
udata.put("address", d['AF'])
|
def addr = AddressUtil.formatAddress(d['A'], d['D'])
|
||||||
|
if(addr){
|
||||||
|
udata.put("address", addr)
|
||||||
|
}
|
||||||
|
|
||||||
udata.put("status", "0")
|
udata.put("status", "0")
|
||||||
udata.put("name_cat", "0")
|
udata.put("name_cat", "0")
|
||||||
udata.put("cat", "district")
|
udata.put("cat", "district")
|
||||||
@@ -215,11 +239,11 @@ class GXDdistrictMatch {
|
|||||||
|
|
||||||
def estateType2 = "普通住宅"
|
def estateType2 = "普通住宅"
|
||||||
def type = ""
|
def type = ""
|
||||||
if(d['AH']){
|
// if(d['AH']){
|
||||||
type = d['AH']
|
// type = d['AH']
|
||||||
}else{
|
// }else{
|
||||||
type = d['Y']
|
// type = d['Y']
|
||||||
}
|
// }
|
||||||
if(type){
|
if(type){
|
||||||
if(type.contains("别墅")){
|
if(type.contains("别墅")){
|
||||||
estateType2 = "别墅"
|
estateType2 = "别墅"
|
||||||
@@ -228,12 +252,12 @@ class GXDdistrictMatch {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
udata.put("estateType2", estateType2)
|
udata.put("estateType2", estateType2)
|
||||||
if(d['AA']){
|
if(d['F']){
|
||||||
Long lng = d['AA']*10000000000
|
Long lng = d['F']*10000000000
|
||||||
udata.put("lng", lng)
|
udata.put("lng", lng)
|
||||||
}
|
}
|
||||||
if(d['AB']){
|
if(d['E']){
|
||||||
Long lat = d['AB'] * 10000000000
|
Long lat = d['E'] * 10000000000
|
||||||
udata.put("lat", lat)
|
udata.put("lat", lat)
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -265,7 +289,7 @@ class GXDdistrictMatch {
|
|||||||
if(!d['U']){
|
if(!d['U']){
|
||||||
d = matchApi(city, region, d, "property", false)
|
d = matchApi(city, region, d, "property", false)
|
||||||
if(!d['U']){
|
if(!d['U']){
|
||||||
d = matchApi(city, region, d, null, true)
|
d = matchApi(city, region, d, null, false)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -285,14 +309,18 @@ class GXDdistrictMatch {
|
|||||||
}
|
}
|
||||||
|
|
||||||
def matchApi(city, region, d, cat, useMap){
|
def matchApi(city, region, d, cat, useMap){
|
||||||
def jsonObject = FangjiaClientUtils.newMatch(city, d['H'], cat, useMap)
|
def jsonObject = null
|
||||||
if(!jsonObject){
|
if(!jsonObject && d['H']){
|
||||||
|
jsonObject = FangjiaClientUtils.newMatch(city, d['H'], cat, useMap)
|
||||||
|
}
|
||||||
|
|
||||||
|
if(!jsonObject && d['C']){
|
||||||
jsonObject = FangjiaClientUtils.newMatch(city, region+d['C'], cat, useMap)
|
jsonObject = FangjiaClientUtils.newMatch(city, region+d['C'], cat, useMap)
|
||||||
if(!jsonObject && d['D']){
|
if(!jsonObject && d['D']){
|
||||||
jsonObject = FangjiaClientUtils.newMatch(city, region+d['D'], cat, useMap)
|
jsonObject = FangjiaClientUtils.newMatch(city, region+d['D'], cat, useMap)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
if(jsonObject != null && jsonObject.credit>0) {
|
if(jsonObject != null) {
|
||||||
d['U'] = "接口匹配"
|
d['U'] = "接口匹配"
|
||||||
d['V'] = jsonObject.get("city")
|
d['V'] = jsonObject.get("city")
|
||||||
d['W'] = jsonObject.get("district")
|
d['W'] = jsonObject.get("district")
|
||||||
|
|||||||
+78
@@ -0,0 +1,78 @@
|
|||||||
|
package com.fangjia.script.gj.data_handle
|
||||||
|
|
||||||
|
import com.fangjia.core.RegionData
|
||||||
|
import com.fangjia.script.ccbevaluation.CcbCommonUtils
|
||||||
|
import com.fangjia.script.commData.CommonDatautils
|
||||||
|
import com.fangjia.script.gj.district.SeaweedDataMatchProcess
|
||||||
|
import com.fangjia.script.util.ExcelUtils
|
||||||
|
|
||||||
|
import java.util.concurrent.CountDownLatch
|
||||||
|
import java.util.concurrent.ExecutorService
|
||||||
|
import java.util.concurrent.Executors
|
||||||
|
import java.util.concurrent.atomic.AtomicInteger
|
||||||
|
|
||||||
|
class TempExportData {
|
||||||
|
|
||||||
|
ExecutorService fixedThreadPool = Executors.newFixedThreadPool(30)
|
||||||
|
|
||||||
|
static main(args) {
|
||||||
|
def task = new TempExportData()
|
||||||
|
task.run()
|
||||||
|
}
|
||||||
|
|
||||||
|
def run() {
|
||||||
|
def citys = CommonDatautils.findAllCity()
|
||||||
|
def rets = []
|
||||||
|
rets << ["ID", "城市", "区域", "名称", "地址", "别名区域", "别名名称", "别名地址", "别名所在小区id"]
|
||||||
|
for (def city : citys) {
|
||||||
|
def cityRets = getResults(city)
|
||||||
|
rets.addAll(cityRets)
|
||||||
|
}
|
||||||
|
ExcelUtils.write("D:\\数据\\数据建设\\名称在别名中的小区.xlsx", rets,"数据")
|
||||||
|
}
|
||||||
|
|
||||||
|
def getResults(city) {
|
||||||
|
def rets = []
|
||||||
|
def dists = CommonDatautils.findAllDistricts(city, "district", ["city", "region", "name", "address", "mapping_alias"])
|
||||||
|
|
||||||
|
final CountDownLatch latch = new CountDownLatch(dists.size())
|
||||||
|
final AtomicInteger count_ = new AtomicInteger(0)
|
||||||
|
|
||||||
|
for (def i = 0; i < dists.size(); i++) {
|
||||||
|
def d = dists[i]
|
||||||
|
fixedThreadPool.execute(
|
||||||
|
new Runnable() {
|
||||||
|
void run() {
|
||||||
|
count_.addAndGet(1)
|
||||||
|
try {
|
||||||
|
def objs = CommonDatautils.findSeaweedListByMappingAlias(city, d.region, d.name, "district")
|
||||||
|
if (objs) {
|
||||||
|
def ret = []
|
||||||
|
ret << d._id.toString()
|
||||||
|
ret << d.city
|
||||||
|
ret << d.region
|
||||||
|
ret << d.name
|
||||||
|
ret << d.address
|
||||||
|
|
||||||
|
def str = ""
|
||||||
|
for (def o : objs) {
|
||||||
|
str = str + o.region + "-" + o.name + "-"+o.mapping_alias.toString() + "\n"
|
||||||
|
}
|
||||||
|
ret << str
|
||||||
|
ret << objs.size()
|
||||||
|
rets << ret
|
||||||
|
}
|
||||||
|
} catch (Exception e) {
|
||||||
|
e.printStackTrace()
|
||||||
|
} finally {
|
||||||
|
latch.countDown()
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
);
|
||||||
|
}
|
||||||
|
latch.await()
|
||||||
|
println(city +" "+rets.size())
|
||||||
|
return rets;
|
||||||
|
}
|
||||||
|
}
|
||||||
+544
@@ -0,0 +1,544 @@
|
|||||||
|
package com.fangjia.script.gj.data_handle
|
||||||
|
|
||||||
|
import com.fangjia.core.RegionData
|
||||||
|
import com.fangjia.script.commData.CommonDatautils
|
||||||
|
import com.fangjia.script.util.MongoUtils
|
||||||
|
import com.mongodb.AggregationOutput
|
||||||
|
import com.mongodb.BasicDBObject
|
||||||
|
import com.mongodb.DBCollection
|
||||||
|
import com.mongodb.DBObject
|
||||||
|
|
||||||
|
/**
|
||||||
|
* 乡镇街道中间表
|
||||||
|
* 包含市,区,乡镇街道别名,来源 统计局community表
|
||||||
|
*/
|
||||||
|
class TownMiddleProcess {
|
||||||
|
|
||||||
|
|
||||||
|
def MONGO100_URL = "mongodb://admin:G00MONGOADMIN@192.168.0.100:30009";
|
||||||
|
DBCollection COMMUNITY_CONN = MongoUtils.getCollectionByName2("fangjia", "community", MONGO100_URL)
|
||||||
|
DBCollection ADMINISTRATIVE_DIVISION_CONN = MongoUtils.getCollectionByName2("fangjia", "administrative_division_2023", MONGO100_URL)
|
||||||
|
|
||||||
|
DBCollection ADMINISTRATIVE_MAPPING_COLL = MongoUtils.getCollectionByName("business", "administrative_mapping", "mongo.db.fangjia.com")
|
||||||
|
DBCollection SERVER_AREA_COLL = MongoUtils.getCollectionByName("fangjia", "server_area", "mongo.db.fangjia.com")
|
||||||
|
DBCollection CITYPOLYLINE_COLL = MongoUtils.getCollectionByAuth20190102("fangjia_crawl", "city_polyline")
|
||||||
|
DBCollection REGIONPOLYLINE_COLL = MongoUtils.getCollectionByAuth20190102("fangjia_crawl", "region_polyline")
|
||||||
|
def fullCityMap = CommonDatautils.queryFullNameCity()
|
||||||
|
|
||||||
|
static main(args) {
|
||||||
|
def task = new TownMiddleProcess();
|
||||||
|
task.run()
|
||||||
|
}
|
||||||
|
def run(){
|
||||||
|
//创建省份
|
||||||
|
// createProvince()
|
||||||
|
//创建城市
|
||||||
|
// createCity()
|
||||||
|
//创建区域
|
||||||
|
// createRegion()
|
||||||
|
//创建乡镇街道
|
||||||
|
createTown()
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
def createProvince(){
|
||||||
|
def cat = "省"
|
||||||
|
def tjjMaps = findAllProcincesFromTJJ(cat)
|
||||||
|
def mzbMaps = findAllProcincesFromMZB(cat)
|
||||||
|
|
||||||
|
def tjjKeys = tjjMaps.keySet()
|
||||||
|
def mzbKeys = mzbMaps.keySet()
|
||||||
|
|
||||||
|
def diffKeys = (tjjKeys - mzbKeys) + (mzbKeys - tjjKeys)
|
||||||
|
def lastRets = []
|
||||||
|
tjjMaps.each {key, val->
|
||||||
|
def udata = new BasicDBObject()
|
||||||
|
udata.put("province", key)
|
||||||
|
udata.put("tjj_province_code", val)
|
||||||
|
udata.put("mzb_province_code", mzbMaps.get(key))
|
||||||
|
udata.put("tjj_source_year",2023)
|
||||||
|
udata.put("mzb_source_year",2023)
|
||||||
|
lastRets << udata
|
||||||
|
}
|
||||||
|
if(diffKeys){
|
||||||
|
diffKeys.each {k->
|
||||||
|
def udata = new BasicDBObject()
|
||||||
|
udata.put("province", k)
|
||||||
|
if(tjjMaps.get(k)){
|
||||||
|
udata.put("tjj_province_code", tjjMaps.get(k))
|
||||||
|
udata.put("tjj_source_year",2023)
|
||||||
|
}
|
||||||
|
if(mzbMaps.get(k)){
|
||||||
|
udata.put("mzb_province_code", mzbMaps.get(k))
|
||||||
|
udata.put("mzb_source_year",2023)
|
||||||
|
}
|
||||||
|
lastRets << udata
|
||||||
|
}
|
||||||
|
}
|
||||||
|
saveToDB(lastRets, "省份")
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
def createCity(){
|
||||||
|
def cat = "城市"
|
||||||
|
def tjjMaps = findAllProcincesFromTJJ(cat)
|
||||||
|
def mzbMaps = findAllProcincesFromMZB(cat)
|
||||||
|
def tjjKeys = tjjMaps.keySet()
|
||||||
|
def mzbKeys = mzbMaps.keySet()
|
||||||
|
def sameKeys = tjjKeys.intersect(mzbKeys)
|
||||||
|
def diffKeys = (tjjKeys - mzbKeys) + (mzbKeys - tjjKeys)
|
||||||
|
def lastRets = []
|
||||||
|
sameKeys.each {k->
|
||||||
|
def v = tjjMaps.get(k)
|
||||||
|
lastRets << getAdministrativeCodeInfo(k, v, mzbMaps)
|
||||||
|
}
|
||||||
|
|
||||||
|
if(diffKeys){
|
||||||
|
diffKeys.each {k->
|
||||||
|
def v = tjjMaps.get(k)
|
||||||
|
lastRets <<getAdministrativeCodeInfo(k, v, mzbMaps)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
//添加别名
|
||||||
|
lastRets.each {o->
|
||||||
|
def city = RegionData.formatCity(o.city)
|
||||||
|
if(city){
|
||||||
|
def cityObj = CommonDatautils.findAlCityDetail(city)
|
||||||
|
o.put("alias", cityObj.alias)
|
||||||
|
}else{
|
||||||
|
println o.province+" "+o.city +" 房价网无此城市"
|
||||||
|
}
|
||||||
|
}
|
||||||
|
//添加城市围栏
|
||||||
|
def cityPolylineMap = cityPolyline()
|
||||||
|
//添加pid
|
||||||
|
def provPidMap = createPidData("省份")
|
||||||
|
lastRets.each{o->
|
||||||
|
def shapeData = cityPolylineMap.get(o.city)
|
||||||
|
if(shapeData){
|
||||||
|
o.shape = shapeData.shape
|
||||||
|
o.location = shapeData.location
|
||||||
|
}
|
||||||
|
def provId = provPidMap.get(o.province)
|
||||||
|
if(provId){
|
||||||
|
o.pid = provId
|
||||||
|
}
|
||||||
|
}
|
||||||
|
saveToDB(lastRets, "城市")
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
def createRegion(){
|
||||||
|
def cat = "区域"
|
||||||
|
def tjjMaps = findAllProcincesFromTJJ(cat)
|
||||||
|
def mzbMaps = findAllProcincesFromMZB(cat)
|
||||||
|
def tjjKeys = tjjMaps.keySet()
|
||||||
|
def mzbKeys = mzbMaps.keySet()
|
||||||
|
def sameKeys = tjjKeys.intersect(mzbKeys)
|
||||||
|
def diffKeys = (tjjKeys - mzbKeys) + (mzbKeys - tjjKeys)
|
||||||
|
def lastRets = []
|
||||||
|
sameKeys.each {k->
|
||||||
|
def v = tjjMaps.get(k)
|
||||||
|
def ud = getAdministrativeCodeInfo(k, v, mzbMaps)
|
||||||
|
// ud.source = "tjj|mzb"
|
||||||
|
lastRets << ud
|
||||||
|
}
|
||||||
|
|
||||||
|
if(diffKeys){
|
||||||
|
diffKeys.each {k->
|
||||||
|
def v = tjjMaps.get(k)
|
||||||
|
def ud = getAdministrativeCodeInfo(k, v, mzbMaps)
|
||||||
|
// if(v){
|
||||||
|
// ud.source = "tjj"
|
||||||
|
// }else{
|
||||||
|
// ud.source = "mzb"
|
||||||
|
// }
|
||||||
|
lastRets << ud
|
||||||
|
}
|
||||||
|
}
|
||||||
|
//添加别名
|
||||||
|
def regionMap = CommonDatautils.getAllCityRegions()
|
||||||
|
lastRets.each {o->
|
||||||
|
def city = RegionData.formatCity(o.city)
|
||||||
|
def district = RegionData.formatRegion(city, o.district)
|
||||||
|
if(city && district){
|
||||||
|
def cMap = regionMap.get(city)
|
||||||
|
if(cMap){
|
||||||
|
def regions = cMap.get(district)
|
||||||
|
o.put("alias", regions)
|
||||||
|
}
|
||||||
|
}else{
|
||||||
|
println o.province+" "+o.city+" "+ o.district +" 房价网无此城市"
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
//添加pid
|
||||||
|
def provPidMap = createPidData("城市")
|
||||||
|
|
||||||
|
//添加区域围栏
|
||||||
|
def regionPolylineMap = regionPolyline()
|
||||||
|
|
||||||
|
lastRets.each{o->
|
||||||
|
def shapeData = regionPolylineMap.get(o.city+"|@|"+o.district)
|
||||||
|
if(shapeData){
|
||||||
|
o.shape = shapeData.shape
|
||||||
|
o.location = shapeData.location
|
||||||
|
}
|
||||||
|
def provId = provPidMap.get(o.city)
|
||||||
|
if(provId){
|
||||||
|
o.pid = provId
|
||||||
|
}
|
||||||
|
}
|
||||||
|
//是否为县级市
|
||||||
|
saveToDB(lastRets, "区域")
|
||||||
|
}
|
||||||
|
|
||||||
|
def createTown(){
|
||||||
|
def cat = "乡镇街道"
|
||||||
|
def tjjMaps = findAllProcincesFromTJJ(cat)
|
||||||
|
def lastRets = []
|
||||||
|
tjjMaps.each {k,v->
|
||||||
|
def ud = getAdministrativeCodeInfo(k, v, null)
|
||||||
|
lastRets << ud
|
||||||
|
}
|
||||||
|
|
||||||
|
//添加pid
|
||||||
|
def provPidMap = createPidData("区域")
|
||||||
|
|
||||||
|
//添加区域围栏
|
||||||
|
def regionPolylineMap = townPolyline()
|
||||||
|
|
||||||
|
lastRets.each{o->
|
||||||
|
def shapeData = regionPolylineMap.get(o.city+"|@|"+o.district)
|
||||||
|
if(shapeData){
|
||||||
|
o.shape = shapeData.shape
|
||||||
|
o.location = shapeData.location
|
||||||
|
}
|
||||||
|
def provId = provPidMap.get(o.city)
|
||||||
|
if(provId){
|
||||||
|
o.pid = provId
|
||||||
|
}
|
||||||
|
}
|
||||||
|
//是否为县级市
|
||||||
|
saveToDB(lastRets, "区域")
|
||||||
|
}
|
||||||
|
def getAdministrativeCodeInfo(key, val, mzbMaps){
|
||||||
|
def udata = new BasicDBObject()
|
||||||
|
def keys = key.split("\\|@\\|")
|
||||||
|
def vals = null
|
||||||
|
if(val){
|
||||||
|
vals = val.split("\\|@\\|")
|
||||||
|
}
|
||||||
|
|
||||||
|
def mzb_vals = null
|
||||||
|
if(mzbMaps && mzbMaps.containsKey(key)){
|
||||||
|
def mzb_val_str = mzbMaps.get(key)
|
||||||
|
mzb_vals = mzb_val_str.split("\\|@\\|")
|
||||||
|
}
|
||||||
|
if(keys.size()>=1){
|
||||||
|
udata.put("province", keys[0])
|
||||||
|
if(vals && vals.size()>=1){
|
||||||
|
udata.put("tjj_province_code", vals[0])
|
||||||
|
udata.put("tjj_source_year",2023)
|
||||||
|
}
|
||||||
|
if(mzb_vals && mzb_vals.size()>=1){
|
||||||
|
udata.put("mzb_province_code", mzb_vals[0])
|
||||||
|
udata.put("mzb_source_year",2023)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
if(keys.size()>=2){
|
||||||
|
udata.put("city", keys[1])
|
||||||
|
if(vals && vals.size()>=2){
|
||||||
|
udata.put("tjj_city_code", vals[1])
|
||||||
|
}
|
||||||
|
if(mzb_vals && mzb_vals.size()>=2){
|
||||||
|
udata.put("mzb_city_code", mzb_vals[1])
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
if(keys.size()>=3){
|
||||||
|
udata.put("district", keys[2])
|
||||||
|
if(vals && vals.size()>=3) {
|
||||||
|
udata.put("tjj_district_code", vals[2])
|
||||||
|
}
|
||||||
|
if(mzb_vals && mzb_vals.size()>=3){
|
||||||
|
udata.put("mzb_district_code", mzb_vals[2])
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return udata
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
def saveToDB(lastRets, type){
|
||||||
|
def dataList = []
|
||||||
|
for(def ret : lastRets){
|
||||||
|
ret.put("c_date", new Date())
|
||||||
|
ret.put("m_date", new Date())
|
||||||
|
ret.put("type", type)
|
||||||
|
dataList << ret
|
||||||
|
|
||||||
|
if (dataList.size() >= 5000) {
|
||||||
|
ADMINISTRATIVE_MAPPING_COLL.insert(dataList);
|
||||||
|
dataList.clear();
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
if (dataList.size() > 0) {
|
||||||
|
ADMINISTRATIVE_MAPPING_COLL.insert(dataList);
|
||||||
|
dataList.clear();
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
def createPidData(def type){
|
||||||
|
def dbcursor = ADMINISTRATIVE_MAPPING_COLL.find(new BasicDBObject("type", type))
|
||||||
|
def map = [:]
|
||||||
|
def pids = []
|
||||||
|
def step = 2
|
||||||
|
if(type == "城市"){
|
||||||
|
step = 4
|
||||||
|
}else if(type == "区域"){
|
||||||
|
step = 6
|
||||||
|
}
|
||||||
|
for(def d : dbcursor){
|
||||||
|
def pid = null
|
||||||
|
def code = null
|
||||||
|
if(d.mzb_province_code){
|
||||||
|
if(type == "省份") {
|
||||||
|
code = d.mzb_province_code.toString()
|
||||||
|
}else if(type == "城市"){
|
||||||
|
code = d.mzb_city_code.toString()
|
||||||
|
}else if(type == "区域"){
|
||||||
|
code = d.mzb_district_code.toString()
|
||||||
|
}
|
||||||
|
}else if(d.tjj_province_code){
|
||||||
|
if(type == "省份") {
|
||||||
|
code = d.tjj_province_code.toString()
|
||||||
|
}else if(type == "城市"){
|
||||||
|
code = d.tjj_city_code.toString()
|
||||||
|
}else if(ype == "区域"){
|
||||||
|
code = d.tjj_district_code.toString()
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
pid = code.substring(0,step)
|
||||||
|
if(pids.contains(d.city+"-"+pid)){
|
||||||
|
pid = code
|
||||||
|
}
|
||||||
|
if(!pids.contains(d.city+"-"+pid)){
|
||||||
|
pids << d.city+"-"+pid
|
||||||
|
}
|
||||||
|
if(type == "省份"){
|
||||||
|
map.put(d.province, pid)
|
||||||
|
}else if(type == "城市"){
|
||||||
|
map.put(d.city, pid)
|
||||||
|
}else if(type == "区域"){
|
||||||
|
map.put(d.city+"@"+d.district, pid)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return map;
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
def findAllProcincesFromTJJ(cat){
|
||||||
|
// 定义聚合管道,按 city 字段分组,并计算每个城市的数量
|
||||||
|
DBObject groupFields = new BasicDBObject();
|
||||||
|
def match = null
|
||||||
|
if(cat=="省"){
|
||||||
|
groupFields.put("_id", new BasicDBObject("province_name", '$province_name').append("province_code", '$province_code'))
|
||||||
|
}else if(cat=="城市"){
|
||||||
|
groupFields.put("_id", new BasicDBObject("province_name", '$province_name').append("province_code", '$province_code').append("city_name", '$city_name').append("city_code", '$city_code'))
|
||||||
|
}else if(cat=="区域"){
|
||||||
|
groupFields.put("_id", new BasicDBObject("province_name", '$province_name').append("province_code", '$province_code').append("city_name", '$city_name').append("city_code", '$city_code').append("area_name", '$area_name').append("area_code", '$area_code'))
|
||||||
|
}else if(cat=="乡镇街道"){
|
||||||
|
groupFields.put("_id", new BasicDBObject("province_name", '$province_name').append("province_code", '$province_code').append("city_name", '$city_name').append("city_code", '$city_code').append("area_name", '$area_name').append("area_code", '$area_code').append("name", '$name').append("code", '$code'))
|
||||||
|
}
|
||||||
|
groupFields.put("count", new BasicDBObject('$sum', 1));
|
||||||
|
DBObject group = new BasicDBObject('$group', groupFields);
|
||||||
|
|
||||||
|
// 执行聚合查询
|
||||||
|
AggregationOutput output = COMMUNITY_CONN.aggregate(Arrays.asList(group));
|
||||||
|
// AggregationOutput output = COMMUNITY_CONN.aggregate(Arrays.asList(match, group));
|
||||||
|
|
||||||
|
def map = [:]
|
||||||
|
// 遍历结果
|
||||||
|
for (DBObject result : output.results()) {
|
||||||
|
def obj = (DBObject ) result.get("_id");
|
||||||
|
if(obj.get("city_name").toString()!="省直辖县级行政区划"){
|
||||||
|
def key = obj.get("province_name")
|
||||||
|
def value = obj.get("province_code")
|
||||||
|
if(cat=="城市"){
|
||||||
|
key = key +"|@|"+ obj.get("city_name")
|
||||||
|
value = value +"|@|"+ obj.get("city_code")
|
||||||
|
}else if(cat=="区域"){
|
||||||
|
key = key +"|@|"+ obj.get("city_name")+"|@|"+ obj.get("area_name")
|
||||||
|
value = value +"|@|"+ obj.get("city_code")+"|@|"+ obj.get("area_code")
|
||||||
|
}else if(cat=="乡镇街道"){
|
||||||
|
key = key +"|@|"+ obj.get("city_name")+"|@|"+ obj.get("area_name")+"|@|"+ obj.get("name")
|
||||||
|
value = value +"|@|"+ obj.get("city_code")+"|@|"+ obj.get("area_code")+"|@|"+ obj.get("code")
|
||||||
|
}
|
||||||
|
int count = (Integer) result.get("count");
|
||||||
|
map.put(key, value)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return map;
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
def findAllProcincesFromMZB(cat){
|
||||||
|
|
||||||
|
BasicDBObject match = null;
|
||||||
|
// 定义聚合管道,按 city 字段分组,并计算每个城市的数量
|
||||||
|
DBObject groupFields = new BasicDBObject();
|
||||||
|
|
||||||
|
if(cat == "省"){
|
||||||
|
match = new BasicDBObject('$match', new BasicDBObject("level", "省份"));
|
||||||
|
groupFields.put("_id", new BasicDBObject("province", '$province').append("mca_province_code", '$mca_province_code'))
|
||||||
|
}else if(cat == "城市"){
|
||||||
|
match = new BasicDBObject('$match', new BasicDBObject("level", "城市"));
|
||||||
|
groupFields.put("_id", new BasicDBObject("province", '$province').append("mca_province_code", '$mca_province_code').append("city",'$city').append("mca_city_code",'$mca_city_code'))
|
||||||
|
}else if(cat == "区域"){
|
||||||
|
match = new BasicDBObject('$match', new BasicDBObject("level", "区域"));
|
||||||
|
groupFields.put("_id", new BasicDBObject("province", '$province').append("mca_province_code", '$mca_province_code').append("city",'$city').append("mca_city_code",'$mca_city_code').append("district",'$district').append("mca_district_code",'$mca_district_code'))
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
groupFields.put("count", new BasicDBObject('$sum', 1));
|
||||||
|
DBObject group = new BasicDBObject('$group', groupFields);
|
||||||
|
|
||||||
|
// 执行聚合查询
|
||||||
|
AggregationOutput output = ADMINISTRATIVE_DIVISION_CONN.aggregate(Arrays.asList(match, group));
|
||||||
|
def map = [:]
|
||||||
|
// 遍历结果
|
||||||
|
for (DBObject result : output.results()) {
|
||||||
|
def obj = (DBObject ) result.get("_id");
|
||||||
|
if(obj.get("city").toString()!="省直辖县级行政区划"){
|
||||||
|
def key = obj.get("province")
|
||||||
|
def value = obj.get("mca_province_code")
|
||||||
|
if(cat == "城市"){
|
||||||
|
key = key +"|@|"+obj.get("city")
|
||||||
|
value = value + "|@|" + obj.get("mca_city_code")
|
||||||
|
}else if(cat == "区域"){
|
||||||
|
key = key +"|@|"+obj.get("city")+"|@|"+obj.get("district")
|
||||||
|
value = value + "|@|" + obj.get("mca_city_code")+ "|@|" + obj.get("mca_district_code")
|
||||||
|
}
|
||||||
|
int count = (Integer) result.get("count");
|
||||||
|
map.put(key, value)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return map;
|
||||||
|
}
|
||||||
|
|
||||||
|
def cityPolyline(){
|
||||||
|
def dbcursor = CITYPOLYLINE_COLL.find(new BasicDBObject())
|
||||||
|
def map = [:]
|
||||||
|
for(def d : dbcursor){
|
||||||
|
def resultMap = [:]
|
||||||
|
resultMap.type = "polygon"
|
||||||
|
def arrs = d.polyline.split("\\|")
|
||||||
|
def rets = []
|
||||||
|
for(def s : arrs){
|
||||||
|
def points = s.split(";")
|
||||||
|
def pointList = []
|
||||||
|
for(def pointStr : points){
|
||||||
|
def pointArr = pointStr.split(",")
|
||||||
|
pointList << [pointArr[0].toString().toDouble(), pointArr[1].toString().toDouble()]
|
||||||
|
}
|
||||||
|
rets << pointList
|
||||||
|
}
|
||||||
|
resultMap.coordinates = rets
|
||||||
|
if(rets.size()>1) {
|
||||||
|
resultMap.type = "MultiPolygon"
|
||||||
|
}
|
||||||
|
println d.name+" "+rets.size()
|
||||||
|
|
||||||
|
def udata = new BasicDBObject()
|
||||||
|
udata.put("shape", resultMap)
|
||||||
|
def center = d.center
|
||||||
|
if(center){
|
||||||
|
def pointArrs = center.split(",")
|
||||||
|
udata.put("location", ["lon":pointArrs[0].toString().toDouble(), "lat":pointArrs[1].toString().toDouble()])
|
||||||
|
}
|
||||||
|
map.put(d.name, udata)
|
||||||
|
}
|
||||||
|
return map;
|
||||||
|
}
|
||||||
|
|
||||||
|
def regionPolyline(){
|
||||||
|
def dbcursor = REGIONPOLYLINE_COLL.find(new BasicDBObject("level","district"))
|
||||||
|
def map = [:]
|
||||||
|
for(def d : dbcursor){
|
||||||
|
def resultMap = [:]
|
||||||
|
resultMap.type = "polygon"
|
||||||
|
def arrs = d.polyline.split("\\|")
|
||||||
|
def rets = []
|
||||||
|
for(def s : arrs){
|
||||||
|
def points = s.split(";")
|
||||||
|
def pointList = []
|
||||||
|
for(def pointStr : points){
|
||||||
|
def pointArr = pointStr.split(",")
|
||||||
|
pointList << [pointArr[0].toString().toDouble(), pointArr[1].toString().toDouble()]
|
||||||
|
}
|
||||||
|
rets << pointList
|
||||||
|
}
|
||||||
|
resultMap.coordinates = rets
|
||||||
|
if(rets.size()>1) {
|
||||||
|
resultMap.type = "MultiPolygon"
|
||||||
|
}
|
||||||
|
println d.city+" "+d.region+" "+rets.size()
|
||||||
|
|
||||||
|
def udata = new BasicDBObject()
|
||||||
|
udata.put("shape", resultMap)
|
||||||
|
def center = d.center
|
||||||
|
if(center){
|
||||||
|
def pointArrs = center.split(",")
|
||||||
|
udata.put("location", ["lon":pointArrs[0].toString().toDouble(), "lat":pointArrs[1].toString().toDouble()])
|
||||||
|
}
|
||||||
|
|
||||||
|
def fullCity = fullCityMap.get(d.city)
|
||||||
|
map.put(fullCity.city+"|@|"+d.region, udata)
|
||||||
|
}
|
||||||
|
return map;
|
||||||
|
}
|
||||||
|
|
||||||
|
def townPolyline(){
|
||||||
|
def dbcursor = REGIONPOLYLINE_COLL.find(new BasicDBObject("level","district"))
|
||||||
|
def map = [:]
|
||||||
|
for(def d : dbcursor){
|
||||||
|
def resultMap = [:]
|
||||||
|
resultMap.type = "polygon"
|
||||||
|
def arrs = d.polyline.split("\\|")
|
||||||
|
def rets = []
|
||||||
|
for(def s : arrs){
|
||||||
|
def points = s.split(";")
|
||||||
|
def pointList = []
|
||||||
|
for(def pointStr : points){
|
||||||
|
def pointArr = pointStr.split(",")
|
||||||
|
pointList << [pointArr[0].toString().toDouble(), pointArr[1].toString().toDouble()]
|
||||||
|
}
|
||||||
|
rets << pointList
|
||||||
|
}
|
||||||
|
resultMap.coordinates = rets
|
||||||
|
if(rets.size()>1) {
|
||||||
|
resultMap.type = "MultiPolygon"
|
||||||
|
}
|
||||||
|
println d.city+" "+d.region+" "+rets.size()
|
||||||
|
|
||||||
|
def udata = new BasicDBObject()
|
||||||
|
udata.put("shape", resultMap)
|
||||||
|
def center = d.center
|
||||||
|
if(center){
|
||||||
|
def pointArrs = center.split(",")
|
||||||
|
udata.put("location", ["lon":pointArrs[0].toString().toDouble(), "lat":pointArrs[1].toString().toDouble()])
|
||||||
|
}
|
||||||
|
|
||||||
|
def fullCity = fullCityMap.get(d.city)
|
||||||
|
map.put(fullCity.city+"|@|"+d.region, udata)
|
||||||
|
}
|
||||||
|
return map;
|
||||||
|
}
|
||||||
|
}
|
||||||
+22
-2
@@ -135,6 +135,26 @@ class FangjiaClientUtils {
|
|||||||
return jsonObj;
|
return jsonObj;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
public static def format(def city, def address){
|
||||||
|
def jsonObj = null
|
||||||
|
def content = null
|
||||||
|
try {
|
||||||
|
if(address){
|
||||||
|
address = address.replace("{","")
|
||||||
|
def url = "http://192.168.0.19:31080/FangJiaNlpService/format?address="+URLEncoder.encode(address,"UTF-8")
|
||||||
|
if(city){
|
||||||
|
url = url +"&city="+URLEncoder.encode(city,"UTF-8")
|
||||||
|
}
|
||||||
|
content = new HttpRequest().newGet(url, null, "utf-8")
|
||||||
|
jsonObj = JSONObject.fromObject(content)
|
||||||
|
}
|
||||||
|
} catch (Exception e) {
|
||||||
|
println address +" "+content
|
||||||
|
e.printStackTrace();
|
||||||
|
}
|
||||||
|
return jsonObj;
|
||||||
|
}
|
||||||
|
|
||||||
public static def arroundPrice(city,address){
|
public static def arroundPrice(city,address){
|
||||||
def jsonObj = null
|
def jsonObj = null
|
||||||
def content = null
|
def content = null
|
||||||
@@ -228,9 +248,9 @@ class FangjiaClientUtils {
|
|||||||
def url = null;
|
def url = null;
|
||||||
try {
|
try {
|
||||||
address = address.replace("{","").replace("\\","")
|
address = address.replace("{","").replace("\\","")
|
||||||
url = "http://192.168.0.93:8080/FangJiaNlpService/match?address="+URLEncoder.encode(address,"UTF-8")+"&useMap=${useMap}"
|
// url = "http://192.168.0.93:8080/FangJiaNlpService/match?address="+URLEncoder.encode(address,"UTF-8")+"&useMap=${useMap}"
|
||||||
// url = "http://192.168.20.242:5001/match?address="+URLEncoder.encode(address,"UTF-8")+"&useMap=${useMap}"
|
// url = "http://192.168.20.242:5001/match?address="+URLEncoder.encode(address,"UTF-8")+"&useMap=${useMap}"
|
||||||
// url = "http://192.168.0.100:32080/FangJiaNlpService/match?address="+URLEncoder.encode(address,"UTF-8")+"&useMap=${useMap}"
|
url = "http://192.168.0.19:31080/FangJiaNlpService/match?address="+URLEncoder.encode(address,"UTF-8")+"&useMap=${useMap}"
|
||||||
if(city){
|
if(city){
|
||||||
url = url + "&city="+URLEncoder.encode(city,"UTF-8")
|
url = url + "&city="+URLEncoder.encode(city,"UTF-8")
|
||||||
}
|
}
|
||||||
|
|||||||
+18
-18
@@ -33,12 +33,12 @@ class AddPriceTest {
|
|||||||
task.run();
|
task.run();
|
||||||
}
|
}
|
||||||
def run(){
|
def run(){
|
||||||
def filePath = "C:\\Users\\gaojuan\\Desktop\\7月成交案例badcase.xlsx"
|
def filePath = "D:\\数据\\建行\\房产估值\\估家待优化小区 - 副本.xlsx"
|
||||||
// addGXDPrice(filePath)
|
// addGXDPrice(filePath)
|
||||||
// addCcbPrice(filePath)
|
addCcbPrice(filePath)
|
||||||
// addThirdPrice(filePath)
|
// addThirdPrice(filePath)
|
||||||
// addFjwPrice(filePath)
|
// addFjwPrice(filePath)
|
||||||
exportPkPrice()
|
// exportPkPrice()
|
||||||
// addRegion()
|
// addRegion()
|
||||||
|
|
||||||
|
|
||||||
@@ -153,15 +153,15 @@ class AddPriceTest {
|
|||||||
}
|
}
|
||||||
|
|
||||||
def addCcbPrice(filepath){
|
def addCcbPrice(filepath){
|
||||||
def cityCodeDataMap = CcbCommonUtils.findCityCodeValue()
|
// def cityCodeDataMap = CcbCommonUtils.findCityCodeValue()
|
||||||
ExcelUtils.multiThreadingModify(filepath,{ d->
|
ExcelUtils.multiThreadingModify(filepath,{ d->
|
||||||
if(d['M']){
|
if(d['A']){
|
||||||
def cityId = cityCodeDataMap.get(d['N'])
|
// def cityId = cityCodeDataMap.get(d['N'])
|
||||||
def map = findAvgPrice(d['M'], cityId)
|
def map = findAvgPrice(d['G'], d['A'].toString().toDouble().toInteger())
|
||||||
if(map){
|
if(map){
|
||||||
d['AK'] = map.avg_price
|
// d['AK'] = map.avg_price
|
||||||
d['AL'] = map.calc_type
|
d['Q'] = map.calc_type
|
||||||
d['AM'] = map.credit_value
|
// d['AM'] = map.credit_value
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
})
|
})
|
||||||
@@ -227,12 +227,12 @@ class AddPriceTest {
|
|||||||
|
|
||||||
def addGXDPrice(filepath){
|
def addGXDPrice(filepath){
|
||||||
ExcelUtils.multiThreadingModify(filepath, { d ->
|
ExcelUtils.multiThreadingModify(filepath, { d ->
|
||||||
if (d['B']) {
|
if (!d['K']) {
|
||||||
def dist = CommonDatautils.findDistrictById(d['E'])
|
def dist = CommonDatautils.findDistrictById(d['C'])
|
||||||
if(dist){
|
if(dist){
|
||||||
def gxdPrice = findGXDPrice(dist.city, dist.region, dist.name)
|
def gxdPrice = findGXDPrice(dist.city, dist.region, dist.name)
|
||||||
// def gxdPrice = findGXDPrice(d['D'], d['E'], d['C'])
|
// def gxdPrice = findGXDPrice(d['D'], d['E'], d['C'])
|
||||||
d['L'] = gxdPrice
|
d['K'] = gxdPrice
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
})
|
})
|
||||||
@@ -256,14 +256,14 @@ class AddPriceTest {
|
|||||||
|
|
||||||
def addFjwPrice(path){
|
def addFjwPrice(path){
|
||||||
ExcelUtils.multiThreadingModify(path,{d->
|
ExcelUtils.multiThreadingModify(path,{d->
|
||||||
if(!d['K']){
|
if(!d['P']){
|
||||||
def dist = CommonDatautils.findDistrictById(d['E'])
|
def dist = CommonDatautils.findDistrictById(d['G'])
|
||||||
if(dist){
|
if(dist){
|
||||||
// d['N'] = dist.region
|
// d['N'] = dist.region
|
||||||
def dbo = CommonDatautils.queryDistrictPrice(dist.city, dist.region, dist.name, DateUtils.parseFromTightByPattern("2023-07-16","yyyy-MM-dd"))
|
def dbo = CommonDatautils.queryFirstDistrictPrice(dist.city, dist.region, dist.name)
|
||||||
if(dbo){
|
if(dbo){
|
||||||
// d['K'] = dbo.avg_price
|
d['P'] = dbo.avg_price
|
||||||
d['K'] = dbo.avg_price_2b
|
// d['J'] = dbo.avg_price_2b
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
+63
-163
@@ -4,6 +4,7 @@ import com.fangjia.core.util.HttpRequest
|
|||||||
import com.fangjia.script.temp.shenjian.util.HttpUtils
|
import com.fangjia.script.temp.shenjian.util.HttpUtils
|
||||||
import com.fangjia.script.util.ExcelUtils
|
import com.fangjia.script.util.ExcelUtils
|
||||||
import com.fangjia.script.yinjihaun.util.HttpClient
|
import com.fangjia.script.yinjihaun.util.HttpClient
|
||||||
|
import groovy.json.JsonOutput
|
||||||
import net.sf.json.JSONObject
|
import net.sf.json.JSONObject
|
||||||
import okhttp3.*;
|
import okhttp3.*;
|
||||||
import java.io.IOException;
|
import java.io.IOException;
|
||||||
@@ -15,177 +16,76 @@ import java.util.concurrent.TimeUnit;
|
|||||||
import okhttp3.Request.Builder;
|
import okhttp3.Request.Builder;
|
||||||
|
|
||||||
class QiliangTest {
|
class QiliangTest {
|
||||||
|
String proxyHost = 'tunnel2.qg.net';
|
||||||
|
Integer proxyPort = 14186;
|
||||||
|
String proxyUsername = '113BBC06';
|
||||||
|
String proxyPassword = '3B2FBD05B0AF';
|
||||||
|
String targetUrl = "http://146.56.217.44:9010/uams/farmer/all/v3/sign";
|
||||||
|
|
||||||
// static main(args) {
|
static main(args) {
|
||||||
// def task = new QiliangTest()
|
QiliangTest task = new QiliangTest();
|
||||||
//// task.run()
|
// task.run()
|
||||||
// task.getHttpResult("扬州扬子江北路399号11041")
|
|
||||||
// }
|
|
||||||
|
|
||||||
def run(){//地址测试-2024.3.4-结果.xlsx
|
def data = [
|
||||||
ExcelUtils.modify("D:\\数据\\省联社\\对比.xlsx", { row ->
|
'address': '浙江省湖州市德清县莫干山开发区中小工业园区',
|
||||||
if(!row['N']){
|
'type': '2'
|
||||||
def address = null
|
]
|
||||||
if(row['B'] && row['C'] && !row['C'].toString().contains(row['B'])){
|
def s = task.getProxyResult(data);
|
||||||
address = row['B']+row['C']
|
print(s);
|
||||||
}else{
|
|
||||||
address = row['C']
|
|
||||||
}
|
|
||||||
if(address){
|
|
||||||
def result = getHttpResult(address)
|
|
||||||
if(result && result.code==20000){
|
|
||||||
def data = result.data
|
|
||||||
if(data){
|
|
||||||
if(data.province && data.province!='null'){
|
|
||||||
row['N'] = data.province
|
|
||||||
}
|
|
||||||
if(data.city && data.city!='null'){
|
|
||||||
row['O'] = data.city
|
|
||||||
}
|
|
||||||
if(data.district && data.district!='null'){
|
|
||||||
row['P'] = data.district
|
|
||||||
}
|
|
||||||
if(data.town && data.town!='null'){
|
|
||||||
row['Q'] = data.town
|
|
||||||
}
|
|
||||||
if(data.village && data.village!='null'){
|
|
||||||
row['R'] = data.village
|
|
||||||
}
|
|
||||||
|
|
||||||
if(data.areaCode && data.areaCode!='null'){
|
|
||||||
row['S'] = data.areaCode
|
|
||||||
}
|
|
||||||
|
|
||||||
if(data.farmer == "0"){
|
|
||||||
row['T'] = "非农"
|
|
||||||
}else{
|
|
||||||
row['T'] = "农"
|
|
||||||
}
|
|
||||||
if(data.isGovt == "0"){
|
|
||||||
row['U'] = "否"
|
|
||||||
}else{
|
|
||||||
row['U'] = "是"
|
|
||||||
}
|
|
||||||
|
|
||||||
def list = data.indusParkList
|
|
||||||
if(list && list!="null"){
|
|
||||||
def s = ""
|
|
||||||
for(def r : list){
|
|
||||||
s = s + r.indusParkName +"-"+r.indusParkLevel+"\n"
|
|
||||||
}
|
|
||||||
row['V'] = s
|
|
||||||
}
|
|
||||||
if(data.govtAddr && data.govtAddr!='null'){
|
|
||||||
row['W'] = data.govtAddr?data.govtAddr:''
|
|
||||||
}
|
|
||||||
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
})
|
|
||||||
}
|
}
|
||||||
|
|
||||||
def getHttpResult(address){
|
|
||||||
def data = new JSONObject()
|
|
||||||
data.put("address", address)
|
|
||||||
data.put("type","2")
|
|
||||||
|
|
||||||
Map headers = new HashMap();
|
|
||||||
headers.put("Accept", "application/json, text/plain, */*");
|
|
||||||
headers.put("Accept-Encoding", "gzip, deflate");
|
|
||||||
headers.put("Accept-Language","zh-CN,zh;q=0.9")
|
|
||||||
headers.put("Content-Type","application/json;charset=UTF-8")
|
|
||||||
headers.put("Host","146.56.217.44:9010")
|
|
||||||
headers.put("Origin","http://146.56.217.44:8088")
|
|
||||||
headers.put("Proxy-Connection","keep-alive")
|
|
||||||
headers.put("Referer","http://146.56.217.44:8088/")
|
|
||||||
headers.put("User-Agent","Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/122.0.0.0 Safari/537.36")
|
|
||||||
|
|
||||||
HttpRequest request1 = new HttpRequest()
|
|
||||||
try{
|
String getProxyResult(data){
|
||||||
String res = request1.postWithBody("http://146.56.217.44:9010/uams/farmer/all/v3/sign", JSONObject.fromObject(data.toString()), headers, "UTF-8")
|
// 将 JSON 对象转换为字符串
|
||||||
def response = JSONObject.fromObject(res)
|
String jsonData = new JsonOutput().toJson(data);
|
||||||
println response
|
// 创建代理对象
|
||||||
return response
|
Proxy proxy = new Proxy(Proxy.Type.HTTP, new InetSocketAddress(proxyHost, proxyPort));
|
||||||
}catch(Exception e){
|
|
||||||
e.printStackTrace()
|
// 创建 URL 对象
|
||||||
|
URL url = new URL(targetUrl);
|
||||||
|
|
||||||
|
// 创建连接对象
|
||||||
|
URLConnection connection = url.openConnection(proxy);
|
||||||
|
|
||||||
|
// 设置代理认证
|
||||||
|
java.net.Authenticator.setDefault(new java.net.Authenticator() {
|
||||||
|
protected PasswordAuthentication getPasswordAuthentication() {
|
||||||
|
return new PasswordAuthentication(proxyUsername, proxyPassword.toCharArray())
|
||||||
|
}
|
||||||
|
})
|
||||||
|
|
||||||
|
// 设置请求方法为 POST
|
||||||
|
connection.setRequestMethod("POST");
|
||||||
|
|
||||||
|
// 允许输出流
|
||||||
|
connection.setDoOutput(true);
|
||||||
|
|
||||||
|
// 设置请求头
|
||||||
|
connection.setRequestProperty("Content-Type", "application/json");
|
||||||
|
|
||||||
|
// 发送请求参数
|
||||||
|
OutputStream outputStream = connection.getOutputStream();
|
||||||
|
outputStream.write(jsonData.getBytes("UTF-8"));
|
||||||
|
outputStream.flush();
|
||||||
|
outputStream.close();
|
||||||
|
|
||||||
|
// 读取响应
|
||||||
|
int responseCode = connection.getResponseCode()
|
||||||
|
if (responseCode == HttpURLConnection.HTTP_OK) {
|
||||||
|
BufferedReader reader = new BufferedReader(new InputStreamReader(connection.getInputStream()));
|
||||||
|
StringBuilder response = new StringBuilder();
|
||||||
|
String line;
|
||||||
|
while ((line = reader.readLine()) != null) {
|
||||||
|
response.append(line);
|
||||||
|
}
|
||||||
|
reader.close();
|
||||||
|
return response.toString();
|
||||||
|
} else {
|
||||||
|
println "HTTP request failed with response code ${responseCode}";
|
||||||
}
|
}
|
||||||
return null;
|
return null;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
final static String proxyIp = "tunnel2.qg.net";
|
|
||||||
final static Integer proxyPort = 14186;
|
|
||||||
final static String authKey = "113BBC06";
|
|
||||||
final static String password = "3B2FBD05B0AF";
|
|
||||||
public static final MediaType JSON = MediaType.parse("application/json;charset=utf-8");
|
|
||||||
|
|
||||||
// public String request(params) throws IOException {
|
|
||||||
// Proxy proxy = new Proxy(Proxy.Type.HTTP, new InetSocketAddress(proxyIp, proxyPort));
|
|
||||||
// OkHttpClient client = new OkHttpClient.Builder()
|
|
||||||
// .proxy(proxy)
|
|
||||||
// .proxyAuthenticator((route, response){
|
|
||||||
// String credential = Credentials.basic(authKey, password);
|
|
||||||
// return response.request().newBuilder().header("Proxy-Authorization", credential).build();
|
|
||||||
// }).
|
|
||||||
// build();
|
|
||||||
//// Request request = new Request.Builder().url("http://146.56.217.44:9010/uams/farmer/all/v3/sign").get().build();
|
|
||||||
//
|
|
||||||
// RequestBody body = RequestBody.create(JSON, JSON.toJSONString(params));
|
|
||||||
// Request request = new Builder().url("http://146.56.217.44:9010/uams/farmer/all/v3/sign").post(body).build();
|
|
||||||
//
|
|
||||||
// Response response = client.newCall(request).execute();
|
|
||||||
// return response.body().string();
|
|
||||||
//
|
|
||||||
//// return client.newCall(request).execute();
|
|
||||||
// }
|
|
||||||
public static void main(String[] args) {
|
|
||||||
def proxyHost = "tunnel2.qg.net"
|
|
||||||
def proxyPort = 14186
|
|
||||||
def proxyUsername = '113BBC06'
|
|
||||||
def proxyPassword = '3B2FBD05B0AF'
|
|
||||||
|
|
||||||
def targetUrl = 'http://146.56.217.44:9010/uams/farmer/all/v3/sign'
|
|
||||||
def queryParams = ['address': '扬州扬子江北路399号11041', 'type': '2']
|
|
||||||
|
|
||||||
|
|
||||||
// 创建代理服务器的连接
|
|
||||||
def proxy = new URL("http://${proxyUsername}:${proxyPassword}@${proxyHost}:${proxyPort}")
|
|
||||||
|
|
||||||
// 打开代理连接
|
|
||||||
def proxyConnection = proxy.openConnection()
|
|
||||||
|
|
||||||
// 创建目标URL的连接
|
|
||||||
def url = new URL(targetUrl)
|
|
||||||
def connection = url.openConnection(proxy)
|
|
||||||
|
|
||||||
// 设置请求方法
|
|
||||||
connection.setRequestMethod("GET")
|
|
||||||
|
|
||||||
// 添加请求头(可选)
|
|
||||||
connection.setRequestProperty("Content-Type", "application/json")
|
|
||||||
|
|
||||||
// 构建查询参数
|
|
||||||
def queryString = queryParams.collect { k, v -> "${k}=${v}" }.join('&')
|
|
||||||
|
|
||||||
// 将查询参数附加到目标URL
|
|
||||||
def finalUrl = "${targetUrl}?${queryString}"
|
|
||||||
|
|
||||||
// 发送请求
|
|
||||||
connection.connect()
|
|
||||||
|
|
||||||
// 获取响应
|
|
||||||
def responseCode = connection.getResponseCode()
|
|
||||||
def responseMessage = connection.getResponseMessage()
|
|
||||||
|
|
||||||
// 读取响应内容
|
|
||||||
def reader = new BufferedReader(new InputStreamReader(connection.getInputStream()))
|
|
||||||
def responseContent = reader.readLine()
|
|
||||||
reader.close()
|
|
||||||
|
|
||||||
println "Response Code: ${responseCode}"
|
|
||||||
println "Response Message: ${responseMessage}"
|
|
||||||
println "Response Content: ${responseContent}"
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
|
|||||||
+11
-11
@@ -19,17 +19,17 @@ DBCollection COMMUNITY_CONN = MongoUtils.getCollectionByName2("fangjia", "commun
|
|||||||
//def s = getTownshipInfo("118.702988,39.740979")
|
//def s = getTownshipInfo("118.702988,39.740979")
|
||||||
def filePath = "D:\\数据\\涉农\\县政府(最终).xlsx"
|
def filePath = "D:\\数据\\涉农\\县政府(最终).xlsx"
|
||||||
ExcelUtils.modify(filePath, { row ->
|
ExcelUtils.modify(filePath, { row ->
|
||||||
// if(!row['I']){
|
if(!row['I'] && row['I']=='宝应县'){
|
||||||
// def rets = EsSearchUtils.findPois(row['B'], row['C'], row['C']+"人民政府")
|
def rets = EsSearchUtils.findPois(row['B'], row['C'], row['C']+"人民政府")
|
||||||
// if(rets){
|
if(rets){
|
||||||
// def obj = findOneByName(rets, row['C']+"人民政府")
|
def obj = findOneByName(rets, row['C']+"人民政府")
|
||||||
// if(obj){
|
if(obj){
|
||||||
// row['I'] = obj.lng
|
row['I'] = obj.lng
|
||||||
// row['J'] = obj.lat
|
row['J'] = obj.lat
|
||||||
// println row['C'] + " "+obj
|
println row['C'] + " "+obj
|
||||||
// }
|
}
|
||||||
// }
|
}
|
||||||
// }
|
}
|
||||||
|
|
||||||
if(row['D'] && !row['F']){
|
if(row['D'] && !row['F']){
|
||||||
def townMap = findTownRange(row['B'], TOWN_POLYLINECOLL)
|
def townMap = findTownRange(row['B'], TOWN_POLYLINECOLL)
|
||||||
|
|||||||
@@ -0,0 +1,181 @@
|
|||||||
|
package com.fangjia.script.gj.village
|
||||||
|
|
||||||
|
import com.fangjia.core.util.HttpRequest
|
||||||
|
import com.fangjia.script.temp.shenjian.util.HttpUtils
|
||||||
|
import com.fangjia.script.util.ExcelUtils
|
||||||
|
import com.fangjia.script.yinjihaun.util.HttpClient
|
||||||
|
import groovy.json.JsonOutput
|
||||||
|
import net.sf.json.JSONObject
|
||||||
|
import okhttp3.*;
|
||||||
|
import java.io.IOException;
|
||||||
|
import java.net.InetSocketAddress;
|
||||||
|
import java.net.Proxy;
|
||||||
|
import java.util.concurrent.TimeUnit;
|
||||||
|
|
||||||
|
|
||||||
|
import okhttp3.Request.Builder;
|
||||||
|
|
||||||
|
class QiliangTest1 {
|
||||||
|
def proxyHost = 'tunnel2.qg.net'
|
||||||
|
def proxyPort = 14186
|
||||||
|
def proxyUsername = '113BBC06'
|
||||||
|
def proxyPassword = '3B2FBD05B0AF'
|
||||||
|
String targetUrl = "http://146.56.217.44:9010/uams/farmer/all/v3/sign";
|
||||||
|
|
||||||
|
static main(args) {
|
||||||
|
def task = new QiliangTest()
|
||||||
|
// task.run()
|
||||||
|
|
||||||
|
def data = [
|
||||||
|
'address': '江苏省扬州市宝应县安宜镇宝应大道88号',
|
||||||
|
'type': '1'
|
||||||
|
]
|
||||||
|
def s = task.getProxyResult(data)
|
||||||
|
print(s)
|
||||||
|
}
|
||||||
|
|
||||||
|
def run(){//地址测试-2024.3.4-结果.xlsx
|
||||||
|
ExcelUtils.modify("D:\\数据\\省联社\\对比.xlsx", { row ->
|
||||||
|
if(!row['N']){
|
||||||
|
def address = null
|
||||||
|
if(row['B'] && row['C'] && !row['C'].toString().contains(row['B'])){
|
||||||
|
address = row['B']+row['C']
|
||||||
|
}else{
|
||||||
|
address = row['C']
|
||||||
|
}
|
||||||
|
if(address){
|
||||||
|
def result = getHttpResult(address)
|
||||||
|
if(result && result.code==20000){
|
||||||
|
def data = result.data
|
||||||
|
if(data){
|
||||||
|
if(data.province && data.province!='null'){
|
||||||
|
row['N'] = data.province
|
||||||
|
}
|
||||||
|
if(data.city && data.city!='null'){
|
||||||
|
row['O'] = data.city
|
||||||
|
}
|
||||||
|
if(data.district && data.district!='null'){
|
||||||
|
row['P'] = data.district
|
||||||
|
}
|
||||||
|
if(data.town && data.town!='null'){
|
||||||
|
row['Q'] = data.town
|
||||||
|
}
|
||||||
|
if(data.village && data.village!='null'){
|
||||||
|
row['R'] = data.village
|
||||||
|
}
|
||||||
|
|
||||||
|
if(data.areaCode && data.areaCode!='null'){
|
||||||
|
row['S'] = data.areaCode
|
||||||
|
}
|
||||||
|
|
||||||
|
if(data.farmer == "0"){
|
||||||
|
row['T'] = "非农"
|
||||||
|
}else{
|
||||||
|
row['T'] = "农"
|
||||||
|
}
|
||||||
|
if(data.isGovt == "0"){
|
||||||
|
row['U'] = "否"
|
||||||
|
}else{
|
||||||
|
row['U'] = "是"
|
||||||
|
}
|
||||||
|
|
||||||
|
def list = data.indusParkList
|
||||||
|
if(list && list!="null"){
|
||||||
|
def s = ""
|
||||||
|
for(def r : list){
|
||||||
|
s = s + r.indusParkName +"-"+r.indusParkLevel+"\n"
|
||||||
|
}
|
||||||
|
row['V'] = s
|
||||||
|
}
|
||||||
|
if(data.govtAddr && data.govtAddr!='null'){
|
||||||
|
row['W'] = data.govtAddr?data.govtAddr:''
|
||||||
|
}
|
||||||
|
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
|
JSONObject getHttpResult(address){
|
||||||
|
JSONObject data = new JSONObject()
|
||||||
|
data.put("address", address)
|
||||||
|
data.put("type","2")
|
||||||
|
|
||||||
|
Map headers = new HashMap();
|
||||||
|
headers.put("Accept", "application/json, text/plain, */*");
|
||||||
|
headers.put("Accept-Encoding", "gzip, deflate");
|
||||||
|
headers.put("Accept-Language","zh-CN,zh;q=0.9")
|
||||||
|
headers.put("Content-Type","application/json;charset=UTF-8")
|
||||||
|
headers.put("Host","146.56.217.44:9010")
|
||||||
|
headers.put("Origin","http://146.56.217.44:8088")
|
||||||
|
headers.put("Proxy-Connection","keep-alive")
|
||||||
|
headers.put("Referer","http://146.56.217.44:8088/")
|
||||||
|
headers.put("User-Agent","Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/122.0.0.0 Safari/537.36")
|
||||||
|
|
||||||
|
HttpRequest request1 = new HttpRequest()
|
||||||
|
try{
|
||||||
|
String res = request1.postWithBody("http://146.56.217.44:9010/uams/farmer/all/v3/sign", JSONObject.fromObject(data.toString()), headers, "UTF-8")
|
||||||
|
JSONObject response = JSONObject.fromObject(res)
|
||||||
|
println response
|
||||||
|
return response
|
||||||
|
}catch(Exception e){
|
||||||
|
e.printStackTrace()
|
||||||
|
}
|
||||||
|
return null;
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
def getProxyResult(data){
|
||||||
|
// 将 JSON 对象转换为字符串
|
||||||
|
def jsonData = new JsonOutput().toJson(data)
|
||||||
|
// 创建代理对象
|
||||||
|
def proxy = new Proxy(Proxy.Type.HTTP, new InetSocketAddress(proxyHost, proxyPort))
|
||||||
|
|
||||||
|
// 创建 URL 对象
|
||||||
|
def url = new URL(targetUrl)
|
||||||
|
|
||||||
|
// 创建连接对象
|
||||||
|
def connection = url.openConnection(proxy)
|
||||||
|
|
||||||
|
// 设置代理认证
|
||||||
|
java.net.Authenticator.setDefault(new java.net.Authenticator() {
|
||||||
|
protected PasswordAuthentication getPasswordAuthentication() {
|
||||||
|
return new PasswordAuthentication(proxyUsername, proxyPassword.toCharArray())
|
||||||
|
}
|
||||||
|
})
|
||||||
|
|
||||||
|
// 设置请求方法为 POST
|
||||||
|
connection.setRequestMethod("POST")
|
||||||
|
|
||||||
|
// 允许输出流
|
||||||
|
connection.setDoOutput(true)
|
||||||
|
|
||||||
|
// 设置请求头
|
||||||
|
connection.setRequestProperty("Content-Type", "application/json")
|
||||||
|
|
||||||
|
// 发送请求参数
|
||||||
|
def outputStream = connection.getOutputStream()
|
||||||
|
outputStream.write(jsonData.getBytes("UTF-8"))
|
||||||
|
outputStream.flush()
|
||||||
|
outputStream.close()
|
||||||
|
|
||||||
|
// 读取响应
|
||||||
|
def responseCode = connection.getResponseCode()
|
||||||
|
if (responseCode == HttpURLConnection.HTTP_OK) {
|
||||||
|
def reader = new BufferedReader(new InputStreamReader(connection.getInputStream()))
|
||||||
|
def response = new StringBuilder()
|
||||||
|
String line
|
||||||
|
while ((line = reader.readLine()) != null) {
|
||||||
|
response.append(line)
|
||||||
|
}
|
||||||
|
reader.close()
|
||||||
|
return response.toString()
|
||||||
|
} else {
|
||||||
|
println "HTTP request failed with response code ${responseCode}"
|
||||||
|
}
|
||||||
|
return null
|
||||||
|
}
|
||||||
|
|
||||||
|
}
|
||||||
+17
-16
@@ -3,6 +3,7 @@ package com.fangjia.script.gj.xiamen
|
|||||||
import com.fangjia.core.RegionData
|
import com.fangjia.core.RegionData
|
||||||
import com.fangjia.core.util.DataUtil
|
import com.fangjia.core.util.DataUtil
|
||||||
import com.fangjia.core.util.HttpRequest
|
import com.fangjia.core.util.HttpRequest
|
||||||
|
import com.fangjia.script.commData.CommonDatautils
|
||||||
import com.fangjia.script.common.DataHandleUtils
|
import com.fangjia.script.common.DataHandleUtils
|
||||||
import com.fangjia.script.util.MongoUtils
|
import com.fangjia.script.util.MongoUtils
|
||||||
import com.fangjia.script.xingyebank.Data
|
import com.fangjia.script.xingyebank.Data
|
||||||
@@ -34,16 +35,18 @@ class PoiWash {
|
|||||||
public static def regions = []
|
public static def regions = []
|
||||||
public static def nameKeys = null
|
public static def nameKeys = null
|
||||||
|
|
||||||
|
def fullCityMap = CommonDatautils.queryFullNameCity()
|
||||||
|
|
||||||
static main(args) {
|
static main(args) {
|
||||||
def task = new PoiWash()
|
def task = new PoiWash()
|
||||||
// task.updateId()
|
// task.updateId()
|
||||||
// task.run()
|
// task.run()
|
||||||
// task.synch()
|
// task.synch()
|
||||||
// task.check()
|
// task.check()
|
||||||
task.deduplicate()
|
// task.deduplicate()
|
||||||
// task.updateEsMapPoi()
|
// task.updateEsMapPoi()
|
||||||
// task.checkSpecialSymbols()
|
// task.checkSpecialSymbols()
|
||||||
// task.splitRoad()
|
task.splitRoad()
|
||||||
}
|
}
|
||||||
|
|
||||||
def updateId(){
|
def updateId(){
|
||||||
@@ -452,9 +455,10 @@ class PoiWash {
|
|||||||
|
|
||||||
def splitRoad(){
|
def splitRoad(){
|
||||||
//"三明市",
|
//"三明市",
|
||||||
def citys = [ "漳州市","宁德市","泉州市","厦门市","福州市","南平市","龙岩市","莆田市","上海市","深圳市","北京市","广州市","杭州市","苏州市","成都市","重庆市"]
|
def citys = CommonDatautils.findAllCity()
|
||||||
for(def city : citys) {
|
for(def city : citys) {
|
||||||
def datas = findPoiByQuery(city, ['city','address'])
|
def cityObj = fullCityMap.get(city)
|
||||||
|
def datas = findPoiByQuery(cityObj.city, ['city','address'])
|
||||||
splitRoadRun(datas)
|
splitRoadRun(datas)
|
||||||
println city +" 完成"
|
println city +" 完成"
|
||||||
}
|
}
|
||||||
@@ -472,18 +476,15 @@ class PoiWash {
|
|||||||
void run() {
|
void run() {
|
||||||
try{
|
try{
|
||||||
totalCount.addAndGet(1)
|
totalCount.addAndGet(1)
|
||||||
def jsonObject = com.fangjia.script.gj.district.FangjiaClientUtils.structure(map.city, map.address)
|
def jsonObject = com.fangjia.script.gj.district.FangjiaClientUtils.format(map.city, map.address)
|
||||||
if(jsonObject && jsonObject.containsKey("result")){
|
def street = jsonObject.get("street")
|
||||||
def dataMap = jsonObject.get("result")
|
def streetNumber = jsonObject.get("streetNumber")
|
||||||
if(dataMap.get("street")){
|
if(jsonObject && street && streetNumber && street!="null" && streetNumber!='null'){
|
||||||
updateTotalCount.addAndGet(1)
|
updateTotalCount.addAndGet(1)
|
||||||
def udata = new BasicDBObject()
|
def udata = new BasicDBObject()
|
||||||
udata.put("street", dataMap.get("street"))
|
udata.put("address", jsonObject.get("street")+jsonObject.get("streetNumber"))
|
||||||
if(dataMap.get("streetNumber")){
|
udata.put("m_date", new Date())
|
||||||
udata.put("street_number", dataMap.get("streetNumber"))
|
MAPPOI_CONN.update(new BasicDBObject('_id',map._id),new BasicDBObject('$set', udata))
|
||||||
}
|
|
||||||
MAPPOI_CONN.update(new BasicDBObject('_id',map._id),new BasicDBObject('$set', udata))
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
println updateTotalCount.get()+" / "+totalCount.get()+" "+count +" "
|
println updateTotalCount.get()+" / "+totalCount.get()+" "+count +" "
|
||||||
} catch (Exception e) {
|
} catch (Exception e) {
|
||||||
|
|||||||
+31
-22
@@ -26,8 +26,8 @@ class SeaweedDataSynch extends BaseBatch{
|
|||||||
|
|
||||||
def MONGO100_URL = "mongodb://admin:G00MONGOADMIN@192.168.0.100:30009";
|
def MONGO100_URL = "mongodb://admin:G00MONGOADMIN@192.168.0.100:30009";
|
||||||
DBCollection SEAWEED_CONN = MongoUtils.getCollectionByName2("fangjia", "seaweed", MONGO100_URL)
|
DBCollection SEAWEED_CONN = MongoUtils.getCollectionByName2("fangjia", "seaweed", MONGO100_URL)
|
||||||
DBCollection administrative_division_100_CONN = MongoUtils.getCollectionByName2("fangjia", "administrative_division", MONGO100_URL)
|
DBCollection administrative_division_100_CONN = MongoUtils.getCollectionByName2("fangjia", "administrative_division_2023", MONGO100_URL)
|
||||||
|
DBCollection administrative_division_2022_CONN = MongoUtils.getCollectionByName2("fangjia", "administrative_division", MONGO100_URL)
|
||||||
|
|
||||||
DBCollection stat_administrative_division_100_CONN = MongoUtils.getCollectionByName2("fangjia", "stat_administrative_division", MONGO100_URL)
|
DBCollection stat_administrative_division_100_CONN = MongoUtils.getCollectionByName2("fangjia", "stat_administrative_division", MONGO100_URL)
|
||||||
public static final DBCollection stat_administrative_division_206_coll = MongoUtils.getCollectionByName("crawl", "stats_administrative_division", "192.168.0.206", 27020)
|
public static final DBCollection stat_administrative_division_206_coll = MongoUtils.getCollectionByName("crawl", "stats_administrative_division", "192.168.0.206", 27020)
|
||||||
@@ -40,7 +40,7 @@ class SeaweedDataSynch extends BaseBatch{
|
|||||||
static DBCollection DISTRICT_SELL_STAT = MongoUtils.getCollectionByName("fangjia_stat", "district_sell_stat", "mongo.db.fangjia.com")
|
static DBCollection DISTRICT_SELL_STAT = MongoUtils.getCollectionByName("fangjia_stat", "district_sell_stat", "mongo.db.fangjia.com")
|
||||||
|
|
||||||
|
|
||||||
public static final DBCollection mca_administrative_division_coll = MongoUtils.getCollectionByName("crawl", "mca_administrative_division_2022", "192.168.0.206", 27020)
|
public static final DBCollection mca_administrative_division_coll = MongoUtils.getCollectionByName("crawl", "mca_administrative_division_2023", "192.168.0.206", 27020)
|
||||||
|
|
||||||
|
|
||||||
DBCollection community_100_CONN = MongoUtils.getCollectionByName2("fangjia", "community", MONGO100_URL)
|
DBCollection community_100_CONN = MongoUtils.getCollectionByName2("fangjia", "community", MONGO100_URL)
|
||||||
@@ -58,10 +58,11 @@ class SeaweedDataSynch extends BaseBatch{
|
|||||||
public void process(Object args) {
|
public void process(Object args) {
|
||||||
|
|
||||||
|
|
||||||
run()
|
// run()
|
||||||
// println "全国district, shop, office同步完成"
|
// println "全国district, shop, office同步完成"
|
||||||
//民政部编码上传
|
//民政部编码上传
|
||||||
//codeRun()
|
codeRun()
|
||||||
|
|
||||||
//println "民政部省,市,区同步完成"
|
//println "民政部省,市,区同步完成"
|
||||||
//println new ObjectId("622c433afb459ab08fdab460").getDate()
|
//println new ObjectId("622c433afb459ab08fdab460").getDate()
|
||||||
|
|
||||||
@@ -346,24 +347,32 @@ class SeaweedDataSynch extends BaseBatch{
|
|||||||
|
|
||||||
def codeRun(){
|
def codeRun(){
|
||||||
|
|
||||||
|
// def q = new BasicDBObject()
|
||||||
|
// def dbcursor = mca_administrative_division_coll.find(q)
|
||||||
|
// def rets = []
|
||||||
|
// for(def it : dbcursor){
|
||||||
|
// def udata = new BasicDBObject()
|
||||||
|
// udata.put("_id", it._id)
|
||||||
|
// udata.put("province", it.province)
|
||||||
|
// udata.put("city", it.city)
|
||||||
|
// udata.put("district", it.district)
|
||||||
|
// udata.put("mca_province_code", it.province_code)
|
||||||
|
// udata.put("mca_city_code", it.city_code)
|
||||||
|
// udata.put("mca_district_code", it.district_code)
|
||||||
|
// udata.put("level", it.level)
|
||||||
|
// udata.put("c_date", it.c_date)
|
||||||
|
// rets << udata
|
||||||
|
//// administrative_division_100_CONN.save(udata)
|
||||||
|
//// administrative_division_100_CONN.update(new BasicDBObject('_id', it._id), new BasicDBObject('$set', udata), true, false)
|
||||||
|
// }
|
||||||
|
// administrative_division_100_CONN.insert(rets)
|
||||||
|
//
|
||||||
def q = new BasicDBObject()
|
def q = new BasicDBObject()
|
||||||
def dbcursor = mca_administrative_division_coll.find(q)
|
q.put("province","°ÄÃÅÌØ±ðÐÐÕþÇø")
|
||||||
def rets = []
|
def dbcursor = administrative_division_2022_CONN.find(q)
|
||||||
for(def it : dbcursor){
|
for(def d : dbcursor){
|
||||||
def udata = new BasicDBObject()
|
administrative_division_100_CONN.save(d)
|
||||||
udata.put("_id", it._id)
|
|
||||||
udata.put("province", it.province)
|
|
||||||
udata.put("city", it.city)
|
|
||||||
udata.put("district", it.district)
|
|
||||||
udata.put("mca_province_code", it.province_code)
|
|
||||||
udata.put("mca_city_code", it.city_code)
|
|
||||||
udata.put("mca_district_code", it.district_code)
|
|
||||||
udata.put("level", it.level)
|
|
||||||
udata.put("c_date", it.c_date)
|
|
||||||
rets << udata
|
|
||||||
// administrative_division_100_CONN.save(udata)
|
|
||||||
// administrative_division_100_CONN.update(new BasicDBObject('_id', it._id), new BasicDBObject('$set', udata), true, false)
|
|
||||||
}
|
}
|
||||||
administrative_division_100_CONN.insert(rets)
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
+1
-1
@@ -46,7 +46,7 @@ import java.util.regex.Pattern
|
|||||||
* 8:華硊摯華硊梗靡脤梑
|
* 8:華硊摯華硊梗靡脤梑
|
||||||
* 9:華硊摯華硊梗靡(祥樓Е郖癹秶)脤梑
|
* 9:華硊摯華硊梗靡(祥樓Е郖癹秶)脤梑
|
||||||
* 10:⑽秞⑤饜
|
* 10:⑽秞⑤饜
|
||||||
*11:open½Ó¿ÚÆ¥Åä
|
* 11:open½Ó¿ÚÆ¥Åä
|
||||||
* 14:⑽秞⑤饜ㄛ祥樓華硊癹秶
|
* 14:⑽秞⑤饜ㄛ祥樓華硊癹秶
|
||||||
*/
|
*/
|
||||||
@BatchOwner(Watcher.GaoJuan)
|
@BatchOwner(Watcher.GaoJuan)
|
||||||
|
|||||||
Reference in New Issue
Block a user